// SPDX-License-Identifier: GPL-2.0-only/* * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips * * Copyright (C) The Asahi Linux Contributors */ #include <linux/bits.h>#include <linux/bitfield.h>#include <linux/device.h>#include <linux/init.h>#include <linux/module.h>#include <linux/of.h>#include <linux/of_dma.h>#include <linux/platform_device.h>#include <linux/reset.h>#include <linux/spinlock.h>#include <linux/interrupt.h> #include "dmaengine.h" #define NCHANNELS_MAX 64#define IRQ_NOUTPUTS 4 /* * For allocation purposes we split the cache * memory into blocks of fixed size (given in bytes). */#define SRAM_BLOCK 2048 #define RING_WRITE_SLOT GENMASK(1, 0)#define RING_READ_SLOT GENMASK(5, 4)#define RING_FULL BIT(9)#define RING_EMPTY BIT(8)#define RING_ERR BIT(10) #define STATUS_DESC_DONE BIT(0)#define STATUS_ERR BIT(6) #define FLAG_DESC_NOTIFY BIT(16) #define REG_TX_START 0x0000#define REG_TX_STOP 0x0004#define REG_RX_START 0x0008#define REG_RX_STOP 0x000c#define REG_IMPRINT 0x0090#define REG_TX_SRAM_SIZE 0x0094#define REG_RX_SRAM_SIZE 0x0098 #define REG_CHAN_CTL(ch) (0x8000 + (ch) * 0x200)#define REG_CHAN_CTL_RST_RINGS BIT(0) #define REG_DESC_RING(ch) (0x8070 + (ch) * 0x200)#define REG_REPORT_RING(ch) (0x8074 + (ch) * 0x200) #define REG_RESIDUE(ch) (0x8064 + (ch) * 0x200) #define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200) #define BUS_WIDTH_WORD_SIZE GENMASK(3, 0)#define BUS_WIDTH_FRAME_SIZE GENMASK(7, 4)#define BUS_WIDTH_8BIT 0x00#define BUS_WIDTH_16BIT 0x01#define BUS_WIDTH_32BIT 0x02#define BUS_WIDTH_FRAME_2_WORDS 0x10#define BUS_WIDTH_FRAME_4_WORDS 0x20 #define REG_CHAN_SRAM_CARVEOUT(ch) (0x8050 + (ch) * 0x200)#define CHAN_SRAM_CARVEOUT_SIZE GENMASK(31, 16)#define CHAN_SRAM_CARVEOUT_BASE GENMASK(15, 0) #define REG_CHAN_FIFOCTL(ch) (0x8054 + (ch) * 0x200)#define CHAN_FIFOCTL_LIMIT GENMASK(31, 16)#define CHAN_FIFOCTL_THRESHOLD GENMASK(15, 0) #define REG_DESC_WRITE(ch) (0x10000 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000)#define REG_REPORT_READ(ch) (0x10100 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000) #define REG_TX_INTSTATE(idx) (0x0030 + (idx) * 4)#define REG_RX_INTSTATE(idx) (0x0040 + (idx) * 4)#define REG_GLOBAL_INTSTATE(idx) (0x0050 + (idx) * 4)#define REG_CHAN_INTSTATUS(ch, idx) (0x8010 + (ch) * 0x200 + (idx) * 4)#define REG_CHAN_INTMASK(ch, idx) (0x8020 + (ch) * 0x200 + (idx) * 4) struct admac_data;struct admac_tx; struct admac_chan { unsigned int no; struct admac_data *host; struct dma_chan chan; struct tasklet_struct tasklet; u32 carveout; spinlock_t lock; struct admac_tx *current_tx; int nperiod_acks; /* * We maintain a 'submitted' and 'issued' list mainly for interface * correctness. Typical use of the driver (per channel) will be * prepping, submitting and issuing a single cyclic transaction which * will stay current until terminate_all is called. */ struct list_head submitted; struct list_head issued; struct list_head to_free;}; struct admac_sram { u32 size; /* * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than * 64K and a 32-bit bitfield over 2K blocks covers it. */ u32 allocated;}; struct admac_data { struct dma_device dma; struct device *dev; __iomem void *base; struct reset_control *rstc; struct mutex cache_alloc_lock; struct admac_sram txcache, rxcache; int irq; int irq_index; int nchannels; struct admac_chan channels[] __counted_by(nchannels);}; struct admac_tx { struct dma_async_tx_descriptor tx; bool cyclic; dma_addr_t buf_addr; dma_addr_t buf_end; size_t buf_len; size_t period_len; size_t submitted_pos; size_t reclaimed_pos; struct list_head node;}; static int admac_alloc_sram_carveout(struct admac_data *ad, enum dma_transfer_direction dir, u32 *out){ struct admac_sram *sram; int i, ret = 0, nblocks; ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); if (dir == DMA_MEM_TO_DEV) sram = &ad->txcache; else sram = &ad->rxcache; mutex_lock(&ad->cache_alloc_lock); nblocks = sram->size / SRAM_BLOCK; for (i = 0; i < nblocks; i++) if (!(sram->allocated & BIT(i))) break; if (i < nblocks) { *out = FIELD_PREP(CHAN_SRAM_CARVEOUT_BASE, i * SRAM_BLOCK) | FIELD_PREP(CHAN_SRAM_CARVEOUT_SIZE, SRAM_BLOCK); sram->allocated |= BIT(i); } else { ret = -EBUSY; } mutex_unlock(&ad->cache_alloc_lock); return ret;} static void admac_free_sram_carveout(struct admac_data *ad, enum dma_transfer_direction dir, u32 carveout){ struct admac_sram *sram; u32 base = FIELD_GET(CHAN_SRAM_CARVEOUT_BASE, carveout); int i; if (dir == DMA_MEM_TO_DEV) sram = &ad->txcache; else sram = &ad->rxcache; if (WARN_ON(base >= sram->size)) return; mutex_lock(&ad->cache_alloc_lock); i = base / SRAM_BLOCK; sram->allocated &= ~BIT(i); mutex_unlock(&ad->cache_alloc_lock);} static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val){ void __iomem *addr = ad->base + reg; u32 curr = readl_relaxed(addr); writel_relaxed((curr & ~mask) | (val & mask), addr);} static struct admac_chan *to_admac_chan(struct dma_chan *chan){ return container_of(chan, struct admac_chan, chan);} static struct admac_tx *to_admac_tx(struct dma_async_tx_descriptor *tx){ return container_of(tx, struct admac_tx, tx);} static enum dma_transfer_direction admac_chan_direction(int channo){ /* Channel directions are hardwired */ return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;} static dma_cookie_t admac_tx_submit(struct dma_async_tx_descriptor *tx){ struct admac_tx *adtx = to_admac_tx(tx); struct admac_chan *adchan = to_admac_chan(tx->chan); unsigned long flags; dma_cookie_t cookie; spin_lock_irqsave(&adchan->lock, flags); cookie = dma_cookie_assign(tx); list_add_tail(&adtx->node, &adchan->submitted); spin_unlock_irqrestore(&adchan->lock, flags); return cookie;} static int admac_desc_free(struct dma_async_tx_descriptor *tx){ kfree(to_admac_tx(tx)); return 0;} static struct dma_async_tx_descriptor *admac_prep_dma_cyclic( struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, unsigned long flags){ struct admac_chan *adchan = container_of(chan, struct admac_chan, chan); struct admac_tx *adtx; if (direction != admac_chan_direction(adchan->no)) return NULL; adtx = kzalloc_obj(*adtx, GFP_NOWAIT); if (!adtx) return NULL; adtx->cyclic = true; adtx->buf_addr = buf_addr; adtx->buf_len = buf_len; adtx->buf_end = buf_addr + buf_len; adtx->period_len = period_len; adtx->submitted_pos = 0; adtx->reclaimed_pos = 0; dma_async_tx_descriptor_init(&adtx->tx, chan); adtx->tx.tx_submit = admac_tx_submit; adtx->tx.desc_free = admac_desc_free; return &adtx->tx;} /* * Write one hardware descriptor for a dmaengine cyclic transaction. */static void admac_cyclic_write_one_desc(struct admac_data *ad, int channo, struct admac_tx *tx){ dma_addr_t addr; addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len); /* If happens means we have buggy code */ WARN_ON_ONCE(addr + tx->period_len > tx->buf_end); dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n", channo, &addr, tx->period_len, FLAG_DESC_NOTIFY); writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); writel_relaxed(tx->period_len, ad->base + REG_DESC_WRITE(channo)); writel_relaxed(FLAG_DESC_NOTIFY, ad->base + REG_DESC_WRITE(channo)); tx->submitted_pos += tx->period_len; tx->submitted_pos %= 2 * tx->buf_len;} /* * Write all the hardware descriptors for a dmaengine cyclic * transaction there is space for. */static void admac_cyclic_write_desc(struct admac_data *ad, int channo, struct admac_tx *tx){ int i; for (i = 0; i < 4; i++) { if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL) break; admac_cyclic_write_one_desc(ad, channo, tx); }} static int admac_ring_noccupied_slots(int ringval){ int wrslot = FIELD_GET(RING_WRITE_SLOT, ringval); int rdslot = FIELD_GET(RING_READ_SLOT, ringval); if (wrslot != rdslot) { return (wrslot + 4 - rdslot) % 4; } else { WARN_ON((ringval & (RING_FULL | RING_EMPTY)) == 0); if (ringval & RING_FULL) return 4; else return 0; }} /* * Read from hardware the residue of a cyclic dmaengine transaction. */static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo, struct admac_tx *adtx){ u32 ring1, ring2; u32 residue1, residue2; int nreports; size_t pos; ring1 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo)); ring2 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo)); if (residue2 > residue1) { /* * Controller must have loaded next descriptor between * the two residue reads */ nreports = admac_ring_noccupied_slots(ring1) + 1; } else { /* No descriptor load between the two reads, ring2 is safe to use */ nreports = admac_ring_noccupied_slots(ring2); } pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2; return adtx->buf_len - pos % adtx->buf_len;} static enum dma_status admac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate){ struct admac_chan *adchan = to_admac_chan(chan); struct admac_data *ad = adchan->host; struct admac_tx *adtx; enum dma_status ret; size_t residue; unsigned long flags; ret = dma_cookie_status(chan, cookie, txstate); if (ret == DMA_COMPLETE || !txstate) return ret; spin_lock_irqsave(&adchan->lock, flags); adtx = adchan->current_tx; if (adtx && adtx->tx.cookie == cookie) { ret = DMA_IN_PROGRESS; residue = admac_cyclic_read_residue(ad, adchan->no, adtx); } else { ret = DMA_IN_PROGRESS; residue = 0; list_for_each_entry(adtx, &adchan->issued, node) { if (adtx->tx.cookie == cookie) { residue = adtx->buf_len; break; } } } spin_unlock_irqrestore(&adchan->lock, flags); dma_set_residue(txstate, residue); return ret;} static void admac_start_chan(struct admac_chan *adchan){ struct admac_data *ad = adchan->host; u32 startbit = 1 << (adchan->no / 2); writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index)); writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index)); switch (admac_chan_direction(adchan->no)) { case DMA_MEM_TO_DEV: writel_relaxed(startbit, ad->base + REG_TX_START); break; case DMA_DEV_TO_MEM: writel_relaxed(startbit, ad->base + REG_RX_START); break; default: break; } dev_dbg(adchan->host->dev, "ch%d start\n", adchan->no);} static void admac_stop_chan(struct admac_chan *adchan){ struct admac_data *ad = adchan->host; u32 stopbit = 1 << (adchan->no / 2); switch (admac_chan_direction(adchan->no)) { case DMA_MEM_TO_DEV: writel_relaxed(stopbit, ad->base + REG_TX_STOP); break; case DMA_DEV_TO_MEM: writel_relaxed(stopbit, ad->base + REG_RX_STOP); break; default: break; } dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no);} static void admac_reset_rings(struct admac_chan *adchan){ struct admac_data *ad = adchan->host; writel_relaxed(REG_CHAN_CTL_RST_RINGS, ad->base + REG_CHAN_CTL(adchan->no)); writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no));} static void admac_start_current_tx(struct admac_chan *adchan){ struct admac_data *ad = adchan->host; int ch = adchan->no; admac_reset_rings(adchan); writel_relaxed(0, ad->base + REG_CHAN_CTL(ch)); admac_cyclic_write_one_desc(ad, ch, adchan->current_tx); admac_start_chan(adchan); admac_cyclic_write_desc(ad, ch, adchan->current_tx);} static void admac_issue_pending(struct dma_chan *chan){ struct admac_chan *adchan = to_admac_chan(chan); struct admac_tx *tx; unsigned long flags; spin_lock_irqsave(&adchan->lock, flags); list_splice_tail_init(&adchan->submitted, &adchan->issued); if (!list_empty(&adchan->issued) && !adchan->current_tx) { tx = list_first_entry(&adchan->issued, struct admac_tx, node); list_del(&tx->node); adchan->current_tx = tx; adchan->nperiod_acks = 0; admac_start_current_tx(adchan); } spin_unlock_irqrestore(&adchan->lock, flags);} static int admac_pause(struct dma_chan *chan){ struct admac_chan *adchan = to_admac_chan(chan); admac_stop_chan(adchan); return 0;}Lines 1–500 of 957. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips * * Copyright (C) The Asahi Linux Contributors */ #include <linux/bits.h> #include <linux/bitfield.h> #include <linux/device.h> #include <linux/init.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/reset.h> #include <linux/spinlock.h> #include <linux/interrupt.h> #include "dmaengine.h" #define NCHANNELS_MAX 64 #define IRQ_NOUTPUTS 4 /* * For allocation purposes we split the cache * memory into blocks of fixed size (given in bytes). */ #define SRAM_BLOCK 2048 #define RING_WRITE_SLOT GENMASK(1, 0) #define RING_READ_SLOT GENMASK(5, 4) #define RING_FULL BIT(9) #define RING_EMPTY BIT(8) #define RING_ERR BIT(10) #define STATUS_DESC_DONE BIT(0) #define STATUS_ERR BIT(6) #define FLAG_DESC_NOTIFY BIT(16) #define REG_TX_START 0x0000 #define REG_TX_STOP 0x0004 #define REG_RX_START 0x0008 #define REG_RX_STOP 0x000c #define REG_IMPRINT 0x0090 #define REG_TX_SRAM_SIZE 0x0094 #define REG_RX_SRAM_SIZE 0x0098 #define REG_CHAN_CTL(ch) (0x8000 + (ch) * 0x200) #define REG_CHAN_CTL_RST_RINGS BIT(0) #define REG_DESC_RING(ch) (0x8070 + (ch) * 0x200) #define REG_REPORT_RING(ch) (0x8074 + (ch) * 0x200) #define REG_RESIDUE(ch) (0x8064 + (ch) * 0x200) #define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200) #define BUS_WIDTH_WORD_SIZE GENMASK(3, 0) #define BUS_WIDTH_FRAME_SIZE GENMASK(7, 4) #define BUS_WIDTH_8BIT 0x00 #define BUS_WIDTH_16BIT 0x01 #define BUS_WIDTH_32BIT 0x02 #define BUS_WIDTH_FRAME_2_WORDS 0x10 #define BUS_WIDTH_FRAME_4_WORDS 0x20 #define REG_CHAN_SRAM_CARVEOUT(ch) (0x8050 + (ch) * 0x200) #define CHAN_SRAM_CARVEOUT_SIZE GENMASK(31, 16) #define CHAN_SRAM_CARVEOUT_BASE GENMASK(15, 0) #define REG_CHAN_FIFOCTL(ch) (0x8054 + (ch) * 0x200) #define CHAN_FIFOCTL_LIMIT GENMASK(31, 16) #define CHAN_FIFOCTL_THRESHOLD GENMASK(15, 0) #define REG_DESC_WRITE(ch) (0x10000 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000) #define REG_REPORT_READ(ch) (0x10100 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000) #define REG_TX_INTSTATE(idx) (0x0030 + (idx) * 4) #define REG_RX_INTSTATE(idx) (0x0040 + (idx) * 4) #define REG_GLOBAL_INTSTATE(idx) (0x0050 + (idx) * 4) #define REG_CHAN_INTSTATUS(ch, idx) (0x8010 + (ch) * 0x200 + (idx) * 4) #define REG_CHAN_INTMASK(ch, idx) (0x8020 + (ch) * 0x200 + (idx) * 4) struct admac_data; struct admac_tx; struct admac_chan { unsigned int no; struct admac_data *host; struct dma_chan chan; struct tasklet_struct tasklet; u32 carveout; spinlock_t lock; struct admac_tx *current_tx; int nperiod_acks; /* * We maintain a 'submitted' and 'issued' list mainly for interface * correctness. Typical use of the driver (per channel) will be * prepping, submitting and issuing a single cyclic transaction which * will stay current until terminate_all is called. */ struct list_head submitted; struct list_head issued; struct list_head to_free; }; struct admac_sram { u32 size; /* * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than * 64K and a 32-bit bitfield over 2K blocks covers it. */ u32 allocated; }; struct admac_data { struct dma_device dma; struct device *dev; __iomem void *base; struct reset_control *rstc; struct mutex cache_alloc_lock; struct admac_sram txcache, rxcache; int irq; int irq_index; int nchannels; struct admac_chan channels[] __counted_by(nchannels); }; struct admac_tx { struct dma_async_tx_descriptor tx; bool cyclic; dma_addr_t buf_addr; dma_addr_t buf_end; size_t buf_len; size_t period_len; size_t submitted_pos; size_t reclaimed_pos; struct list_head node; }; static int admac_alloc_sram_carveout(struct admac_data *ad, enum dma_transfer_direction dir, u32 *out) { struct admac_sram *sram; int i, ret = 0, nblocks; ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); if (dir == DMA_MEM_TO_DEV) sram = &ad->txcache; else sram = &ad->rxcache; mutex_lock(&ad->cache_alloc_lock); nblocks = sram->size / SRAM_BLOCK; for (i = 0; i < nblocks; i++) if (!(sram->allocated & BIT(i))) break; if (i < nblocks) { *out = FIELD_PREP(CHAN_SRAM_CARVEOUT_BASE, i * SRAM_BLOCK) | FIELD_PREP(CHAN_SRAM_CARVEOUT_SIZE, SRAM_BLOCK); sram->allocated |= BIT(i); } else { ret = -EBUSY; } mutex_unlock(&ad->cache_alloc_lock); return ret; } static void admac_free_sram_carveout(struct admac_data *ad, enum dma_transfer_direction dir, u32 carveout) { struct admac_sram *sram; u32 base = FIELD_GET(CHAN_SRAM_CARVEOUT_BASE, carveout); int i; if (dir == DMA_MEM_TO_DEV) sram = &ad->txcache; else sram = &ad->rxcache; if (WARN_ON(base >= sram->size)) return; mutex_lock(&ad->cache_alloc_lock); i = base / SRAM_BLOCK; sram->allocated &= ~BIT(i); mutex_unlock(&ad->cache_alloc_lock); } static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val) { void __iomem *addr = ad->base + reg; u32 curr = readl_relaxed(addr); writel_relaxed((curr & ~mask) | (val & mask), addr); } static struct admac_chan *to_admac_chan(struct dma_chan *chan) { return container_of(chan, struct admac_chan, chan); } static struct admac_tx *to_admac_tx(struct dma_async_tx_descriptor *tx) { return container_of(tx, struct admac_tx, tx); } static enum dma_transfer_direction admac_chan_direction(int channo) { /* Channel directions are hardwired */ return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; } static dma_cookie_t admac_tx_submit(struct dma_async_tx_descriptor *tx) { struct admac_tx *adtx = to_admac_tx(tx); struct admac_chan *adchan = to_admac_chan(tx->chan); unsigned long flags; dma_cookie_t cookie; spin_lock_irqsave(&adchan->lock, flags); cookie = dma_cookie_assign(tx); list_add_tail(&adtx->node, &adchan->submitted); spin_unlock_irqrestore(&adchan->lock, flags); return cookie; } static int admac_desc_free(struct dma_async_tx_descriptor *tx) { kfree(to_admac_tx(tx)); return 0; } static struct dma_async_tx_descriptor *admac_prep_dma_cyclic( struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, unsigned long flags) { struct admac_chan *adchan = container_of(chan, struct admac_chan, chan); struct admac_tx *adtx; if (direction != admac_chan_direction(adchan->no)) return NULL; adtx = kzalloc_obj(*adtx, GFP_NOWAIT); if (!adtx) return NULL; adtx->cyclic = true; adtx->buf_addr = buf_addr; adtx->buf_len = buf_len; adtx->buf_end = buf_addr + buf_len; adtx->period_len = period_len; adtx->submitted_pos = 0; adtx->reclaimed_pos = 0; dma_async_tx_descriptor_init(&adtx->tx, chan); adtx->tx.tx_submit = admac_tx_submit; adtx->tx.desc_free = admac_desc_free; return &adtx->tx; } /* * Write one hardware descriptor for a dmaengine cyclic transaction. */ static void admac_cyclic_write_one_desc(struct admac_data *ad, int channo, struct admac_tx *tx) { dma_addr_t addr; addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len); /* If happens means we have buggy code */ WARN_ON_ONCE(addr + tx->period_len > tx->buf_end); dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n", channo, &addr, tx->period_len, FLAG_DESC_NOTIFY); writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); writel_relaxed(tx->period_len, ad->base + REG_DESC_WRITE(channo)); writel_relaxed(FLAG_DESC_NOTIFY, ad->base + REG_DESC_WRITE(channo)); tx->submitted_pos += tx->period_len; tx->submitted_pos %= 2 * tx->buf_len; } /* * Write all the hardware descriptors for a dmaengine cyclic * transaction there is space for. */ static void admac_cyclic_write_desc(struct admac_data *ad, int channo, struct admac_tx *tx) { int i; for (i = 0; i < 4; i++) { if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL) break; admac_cyclic_write_one_desc(ad, channo, tx); } } static int admac_ring_noccupied_slots(int ringval) { int wrslot = FIELD_GET(RING_WRITE_SLOT, ringval); int rdslot = FIELD_GET(RING_READ_SLOT, ringval); if (wrslot != rdslot) { return (wrslot + 4 - rdslot) % 4; } else { WARN_ON((ringval & (RING_FULL | RING_EMPTY)) == 0); if (ringval & RING_FULL) return 4; else return 0; } } /* * Read from hardware the residue of a cyclic dmaengine transaction. */ static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo, struct admac_tx *adtx) { u32 ring1, ring2; u32 residue1, residue2; int nreports; size_t pos; ring1 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo)); ring2 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo)); if (residue2 > residue1) { /* * Controller must have loaded next descriptor between * the two residue reads */ nreports = admac_ring_noccupied_slots(ring1) + 1; } else { /* No descriptor load between the two reads, ring2 is safe to use */ nreports = admac_ring_noccupied_slots(ring2); } pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2; return adtx->buf_len - pos % adtx->buf_len; } static enum dma_status admac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct admac_chan *adchan = to_admac_chan(chan); struct admac_data *ad = adchan->host; struct admac_tx *adtx; enum dma_status ret; size_t residue; unsigned long flags; ret = dma_cookie_status(chan, cookie, txstate); if (ret == DMA_COMPLETE || !txstate) return ret; spin_lock_irqsave(&adchan->lock, flags); adtx = adchan->current_tx; if (adtx && adtx->tx.cookie == cookie) { ret = DMA_IN_PROGRESS; residue = admac_cyclic_read_residue(ad, adchan->no, adtx); } else { ret = DMA_IN_PROGRESS; residue = 0; list_for_each_entry(adtx, &adchan->issued, node) { if (adtx->tx.cookie == cookie) { residue = adtx->buf_len; break; } } } spin_unlock_irqrestore(&adchan->lock, flags); dma_set_residue(txstate, residue); return ret; } static void admac_start_chan(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; u32 startbit = 1 << (adchan->no / 2); writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index)); writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index)); switch (admac_chan_direction(adchan->no)) { case DMA_MEM_TO_DEV: writel_relaxed(startbit, ad->base + REG_TX_START); break; case DMA_DEV_TO_MEM: writel_relaxed(startbit, ad->base + REG_RX_START); break; default: break; } dev_dbg(adchan->host->dev, "ch%d start\n", adchan->no); } static void admac_stop_chan(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; u32 stopbit = 1 << (adchan->no / 2); switch (admac_chan_direction(adchan->no)) { case DMA_MEM_TO_DEV: writel_relaxed(stopbit, ad->base + REG_TX_STOP); break; case DMA_DEV_TO_MEM: writel_relaxed(stopbit, ad->base + REG_RX_STOP); break; default: break; } dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no); } static void admac_reset_rings(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; writel_relaxed(REG_CHAN_CTL_RST_RINGS, ad->base + REG_CHAN_CTL(adchan->no)); writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no)); } static void admac_start_current_tx(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; int ch = adchan->no; admac_reset_rings(adchan); writel_relaxed(0, ad->base + REG_CHAN_CTL(ch)); admac_cyclic_write_one_desc(ad, ch, adchan->current_tx); admac_start_chan(adchan); admac_cyclic_write_desc(ad, ch, adchan->current_tx); } static void admac_issue_pending(struct dma_chan *chan) { struct admac_chan *adchan = to_admac_chan(chan); struct admac_tx *tx; unsigned long flags; spin_lock_irqsave(&adchan->lock, flags); list_splice_tail_init(&adchan->submitted, &adchan->issued); if (!list_empty(&adchan->issued) && !adchan->current_tx) { tx = list_first_entry(&adchan->issued, struct admac_tx, node); list_del(&tx->node); adchan->current_tx = tx; adchan->nperiod_acks = 0; admac_start_current_tx(adchan); } spin_unlock_irqrestore(&adchan->lock, flags); } static int admac_pause(struct dma_chan *chan) { struct admac_chan *adchan = to_admac_chan(chan); admac_stop_chan(adchan); return 0; }