// SPDX-License-Identifier: GPL-2.0-or-later/* * talitos - Freescale Integrated Security Engine (SEC) device driver * * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. * * Scatterlist Crypto API glue code copied from files with the following: * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> * * Crypto algorithm registration code copied from hifn driver: * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> * All rights reserved. */ #include <linux/kernel.h>#include <linux/module.h>#include <linux/device.h>#include <linux/interrupt.h>#include <linux/crypto.h>#include <linux/hw_random.h>#include <linux/of.h>#include <linux/of_irq.h>#include <linux/platform_device.h>#include <linux/dma-mapping.h>#include <linux/io.h>#include <linux/spinlock.h>#include <linux/rtnetlink.h>#include <linux/slab.h> #include <crypto/algapi.h>#include <crypto/aes.h>#include <crypto/internal/des.h>#include <crypto/sha1.h>#include <crypto/sha2.h>#include <crypto/md5.h>#include <crypto/internal/aead.h>#include <crypto/authenc.h>#include <crypto/internal/skcipher.h>#include <crypto/hash.h>#include <crypto/internal/hash.h>#include <crypto/scatterwalk.h> #include "talitos.h" static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, unsigned int len, bool is_sec1){ ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); if (is_sec1) { ptr->len1 = cpu_to_be16(len); } else { ptr->len = cpu_to_be16(len); ptr->eptr = upper_32_bits(dma_addr); }} static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, struct talitos_ptr *src_ptr, bool is_sec1){ dst_ptr->ptr = src_ptr->ptr; if (is_sec1) { dst_ptr->len1 = src_ptr->len1; } else { dst_ptr->len = src_ptr->len; dst_ptr->eptr = src_ptr->eptr; }} static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, bool is_sec1){ if (is_sec1) return be16_to_cpu(ptr->len1); else return be16_to_cpu(ptr->len);} static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, bool is_sec1){ if (!is_sec1) ptr->j_extent = val;} static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1){ if (!is_sec1) ptr->j_extent |= val;} /* * map virtual single (contiguous) pointer to h/w descriptor pointer */static void __map_single_talitos_ptr(struct device *dev, struct talitos_ptr *ptr, unsigned int len, void *data, enum dma_data_direction dir, unsigned long attrs){ dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs); struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); to_talitos_ptr(ptr, dma_addr, len, is_sec1);} static void map_single_talitos_ptr(struct device *dev, struct talitos_ptr *ptr, unsigned int len, void *data, enum dma_data_direction dir){ __map_single_talitos_ptr(dev, ptr, len, data, dir, 0);} static void map_single_talitos_ptr_nosync(struct device *dev, struct talitos_ptr *ptr, unsigned int len, void *data, enum dma_data_direction dir){ __map_single_talitos_ptr(dev, ptr, len, data, dir, DMA_ATTR_SKIP_CPU_SYNC);} /* * unmap bus single (contiguous) h/w descriptor pointer */static void unmap_single_talitos_ptr(struct device *dev, struct talitos_ptr *ptr, enum dma_data_direction dir){ struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); dma_unmap_single(dev, be32_to_cpu(ptr->ptr), from_talitos_ptr_len(ptr, is_sec1), dir);} static int reset_channel(struct device *dev, int ch){ struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; bool is_sec1 = has_ftr_sec1(priv); if (is_sec1) { setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS1_CCCR_LO_RESET); while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & TALITOS1_CCCR_LO_RESET) && --timeout) cpu_relax(); } else { setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS2_CCCR_RESET); while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS2_CCCR_RESET) && --timeout) cpu_relax(); } if (timeout == 0) { dev_err(dev, "failed to reset channel %d\n", ch); return -EIO; } /* set 36-bit addressing, done writeback enable and done IRQ enable */ setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); /* enable chaining descriptors */ if (is_sec1) setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_NE); /* and ICCR writeback, if available */ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_IWSE); return 0;} static int reset_device(struct device *dev){ struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; bool is_sec1 = has_ftr_sec1(priv); u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; setbits32(priv->reg + TALITOS_MCR, mcr); while ((in_be32(priv->reg + TALITOS_MCR) & mcr) && --timeout) cpu_relax(); if (priv->irq[1] > 0) { mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; setbits32(priv->reg + TALITOS_MCR, mcr); } if (timeout == 0) { dev_err(dev, "failed to reset device\n"); return -EIO; } return 0;} /* * Reset and initialize the device */static int init_device(struct device *dev){ struct talitos_private *priv = dev_get_drvdata(dev); int ch, err; bool is_sec1 = has_ftr_sec1(priv); /* * Master reset * errata documentation: warning: certain SEC interrupts * are not fully cleared by writing the MCR:SWR bit, * set bit twice to completely reset */ err = reset_device(dev); if (err) return err; err = reset_device(dev); if (err) return err; /* reset channels */ for (ch = 0; ch < priv->num_channels; ch++) { err = reset_channel(dev, ch); if (err) return err; } /* enable channel done and error interrupts */ if (is_sec1) { clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); /* disable parity error check in DEU (erroneous? test vect.) */ setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); } else { setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); } /* disable integrity check error interrupts (use writeback instead) */ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, TALITOS_MDEUICR_LO_ICE); return 0;} static void dma_map_request(struct device *dev, struct talitos_request *request, struct talitos_desc *desc, bool is_sec1){ struct talitos_edesc *edesc = container_of(desc, struct talitos_edesc, desc); dma_addr_t dma_desc, prev_dma_desc; struct talitos_edesc *prev_edesc = NULL; if (is_sec1) { while (edesc) { edesc->desc.hdr1 = edesc->desc.hdr; dma_desc = dma_map_single(dev, &edesc->desc.hdr1, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); if (!prev_edesc) { request->dma_desc = dma_desc; goto next; } /* Chain in any previous descriptors. */ prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); dma_sync_single_for_device(dev, prev_dma_desc, TALITOS_DESC_SIZE, DMA_TO_DEVICE); next: prev_edesc = edesc; prev_dma_desc = dma_desc; edesc = edesc->next_desc; } } else { request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); }} /** * talitos_submit - submits a descriptor to the device for processing * @dev: the SEC device to be used * @ch: the SEC device channel to be used * @desc: the descriptor to be processed by the device * @callback: whom to call when processing is complete * @context: a handle for use by caller (optional) * * desc must contain valid dma-mapped (bus physical) address pointers. * callback must check err and feedback in descriptor header * for device processing status. */static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, void (*callback)(struct device *dev, struct talitos_desc *desc, void *context, int error), void *context){ struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request; unsigned long flags; int head; bool is_sec1 = has_ftr_sec1(priv); spin_lock_irqsave(&priv->chan[ch].head_lock, flags); if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { /* h/w fifo is full */ spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); return -EAGAIN; } head = priv->chan[ch].head; request = &priv->chan[ch].fifo[head]; /* map descriptor and save caller data */ dma_map_request(dev, request, desc, is_sec1); request->callback = callback; request->context = context; /* increment fifo head */ priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); smp_wmb(); request->desc = desc; /* GO! */ wmb(); out_be32(priv->chan[ch].reg + TALITOS_FF, upper_32_bits(request->dma_desc)); out_be32(priv->chan[ch].reg + TALITOS_FF_LO, lower_32_bits(request->dma_desc)); spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); return -EINPROGRESS;} static __be32 get_request_hdr(struct device *dev, struct talitos_request *request, bool is_sec1){ struct talitos_edesc *edesc; dma_addr_t dma_desc; if (!is_sec1) { dma_sync_single_for_cpu(dev, request->dma_desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); return request->desc->hdr; } edesc = container_of(request->desc, struct talitos_edesc, desc); dma_desc = request->dma_desc; while (edesc->next_desc) { dma_desc = be32_to_cpu(edesc->desc.next_desc); edesc = edesc->next_desc; } dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); return edesc->desc.hdr1;} static void dma_unmap_request(struct device *dev, struct talitos_request *request, bool is_sec1){ struct talitos_edesc *edesc; if (is_sec1) { dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); edesc = container_of(request->desc, struct talitos_edesc, desc); while (edesc->next_desc) { dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc), TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); edesc = edesc->next_desc; } } else { dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); }} /* * process what was done, notify callback of error if not */static void flush_channel(struct device *dev, int ch, int error, int reset_ch){ struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request, saved_req; unsigned long flags; int tail, status; bool is_sec1 = has_ftr_sec1(priv); spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); tail = priv->chan[ch].tail; while (priv->chan[ch].fifo[tail].desc) { __be32 hdr; request = &priv->chan[ch].fifo[tail]; /* descriptors with their done bits set don't get the error */ rmb(); hdr = get_request_hdr(dev, request, is_sec1); if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; else if (!error) break; else status = error; dma_unmap_request(dev, request, is_sec1); /* copy entries so we can call callback outside lock */ saved_req.desc = request->desc; saved_req.callback = request->callback; saved_req.context = request->context; /* release request entry in fifo */ smp_wmb(); request->desc = NULL; /* increment fifo tail */ priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); atomic_dec(&priv->chan[ch].submit_count); saved_req.callback(dev, saved_req.desc, saved_req.context, status); /* channel may resume processing in single desc error case */ if (error && !reset_ch && status == error) return; spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); tail = priv->chan[ch].tail; } spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);} /* * process completed requests for channels that have done status */#define DEF_TALITOS1_DONE(name, ch_done_mask) \static void talitos1_done_##name(unsigned long data) \{ \ struct device *dev = (struct device *)data; \ struct talitos_private *priv = dev_get_drvdata(dev); \ unsigned long flags; \ \ if (ch_done_mask & 0x10000000) \ flush_channel(dev, 0, 0, 0); \ if (ch_done_mask & 0x40000000) \ flush_channel(dev, 1, 0, 0); \ if (ch_done_mask & 0x00010000) \ flush_channel(dev, 2, 0, 0); \ if (ch_done_mask & 0x00040000) \ flush_channel(dev, 3, 0, 0); \ \ /* At this point, all completed channels have been processed */ \ /* Unmask done interrupts for channels completed later on. */ \ spin_lock_irqsave(&priv->reg_lock, flags); \ clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ spin_unlock_irqrestore(&priv->reg_lock, flags); \} DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) #define DEF_TALITOS2_DONE(name, ch_done_mask) \static void talitos2_done_##name(unsigned long data) \{ \ struct device *dev = (struct device *)data; \ struct talitos_private *priv = dev_get_drvdata(dev); \ unsigned long flags; \ \ if (ch_done_mask & 1) \ flush_channel(dev, 0, 0, 0); \Lines 1–500 of 3635. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * talitos - Freescale Integrated Security Engine (SEC) device driver * * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. * * Scatterlist Crypto API glue code copied from files with the following: * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> * * Crypto algorithm registration code copied from hifn driver: * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> * All rights reserved. */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/device.h> #include <linux/interrupt.h> #include <linux/crypto.h> #include <linux/hw_random.h> #include <linux/of.h> #include <linux/of_irq.h> #include <linux/platform_device.h> #include <linux/dma-mapping.h> #include <linux/io.h> #include <linux/spinlock.h> #include <linux/rtnetlink.h> #include <linux/slab.h> #include <crypto/algapi.h> #include <crypto/aes.h> #include <crypto/internal/des.h> #include <crypto/sha1.h> #include <crypto/sha2.h> #include <crypto/md5.h> #include <crypto/internal/aead.h> #include <crypto/authenc.h> #include <crypto/internal/skcipher.h> #include <crypto/hash.h> #include <crypto/internal/hash.h> #include <crypto/scatterwalk.h> #include "talitos.h" static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, unsigned int len, bool is_sec1) { ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); if (is_sec1) { ptr->len1 = cpu_to_be16(len); } else { ptr->len = cpu_to_be16(len); ptr->eptr = upper_32_bits(dma_addr); } } static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, struct talitos_ptr *src_ptr, bool is_sec1) { dst_ptr->ptr = src_ptr->ptr; if (is_sec1) { dst_ptr->len1 = src_ptr->len1; } else { dst_ptr->len = src_ptr->len; dst_ptr->eptr = src_ptr->eptr; } } static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, bool is_sec1) { if (is_sec1) return be16_to_cpu(ptr->len1); else return be16_to_cpu(ptr->len); } static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, bool is_sec1) { if (!is_sec1) ptr->j_extent = val; } static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) { if (!is_sec1) ptr->j_extent |= val; } /* * map virtual single (contiguous) pointer to h/w descriptor pointer */ static void __map_single_talitos_ptr(struct device *dev, struct talitos_ptr *ptr, unsigned int len, void *data, enum dma_data_direction dir, unsigned long attrs) { dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs); struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); to_talitos_ptr(ptr, dma_addr, len, is_sec1); } static void map_single_talitos_ptr(struct device *dev, struct talitos_ptr *ptr, unsigned int len, void *data, enum dma_data_direction dir) { __map_single_talitos_ptr(dev, ptr, len, data, dir, 0); } static void map_single_talitos_ptr_nosync(struct device *dev, struct talitos_ptr *ptr, unsigned int len, void *data, enum dma_data_direction dir) { __map_single_talitos_ptr(dev, ptr, len, data, dir, DMA_ATTR_SKIP_CPU_SYNC); } /* * unmap bus single (contiguous) h/w descriptor pointer */ static void unmap_single_talitos_ptr(struct device *dev, struct talitos_ptr *ptr, enum dma_data_direction dir) { struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); dma_unmap_single(dev, be32_to_cpu(ptr->ptr), from_talitos_ptr_len(ptr, is_sec1), dir); } static int reset_channel(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; bool is_sec1 = has_ftr_sec1(priv); if (is_sec1) { setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS1_CCCR_LO_RESET); while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & TALITOS1_CCCR_LO_RESET) && --timeout) cpu_relax(); } else { setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS2_CCCR_RESET); while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS2_CCCR_RESET) && --timeout) cpu_relax(); } if (timeout == 0) { dev_err(dev, "failed to reset channel %d\n", ch); return -EIO; } /* set 36-bit addressing, done writeback enable and done IRQ enable */ setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); /* enable chaining descriptors */ if (is_sec1) setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_NE); /* and ICCR writeback, if available */ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_IWSE); return 0; } static int reset_device(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; bool is_sec1 = has_ftr_sec1(priv); u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; setbits32(priv->reg + TALITOS_MCR, mcr); while ((in_be32(priv->reg + TALITOS_MCR) & mcr) && --timeout) cpu_relax(); if (priv->irq[1] > 0) { mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; setbits32(priv->reg + TALITOS_MCR, mcr); } if (timeout == 0) { dev_err(dev, "failed to reset device\n"); return -EIO; } return 0; } /* * Reset and initialize the device */ static int init_device(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); int ch, err; bool is_sec1 = has_ftr_sec1(priv); /* * Master reset * errata documentation: warning: certain SEC interrupts * are not fully cleared by writing the MCR:SWR bit, * set bit twice to completely reset */ err = reset_device(dev); if (err) return err; err = reset_device(dev); if (err) return err; /* reset channels */ for (ch = 0; ch < priv->num_channels; ch++) { err = reset_channel(dev, ch); if (err) return err; } /* enable channel done and error interrupts */ if (is_sec1) { clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); /* disable parity error check in DEU (erroneous? test vect.) */ setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); } else { setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); } /* disable integrity check error interrupts (use writeback instead) */ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, TALITOS_MDEUICR_LO_ICE); return 0; } static void dma_map_request(struct device *dev, struct talitos_request *request, struct talitos_desc *desc, bool is_sec1) { struct talitos_edesc *edesc = container_of(desc, struct talitos_edesc, desc); dma_addr_t dma_desc, prev_dma_desc; struct talitos_edesc *prev_edesc = NULL; if (is_sec1) { while (edesc) { edesc->desc.hdr1 = edesc->desc.hdr; dma_desc = dma_map_single(dev, &edesc->desc.hdr1, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); if (!prev_edesc) { request->dma_desc = dma_desc; goto next; } /* Chain in any previous descriptors. */ prev_edesc->desc.next_desc = cpu_to_be32(dma_desc); dma_sync_single_for_device(dev, prev_dma_desc, TALITOS_DESC_SIZE, DMA_TO_DEVICE); next: prev_edesc = edesc; prev_dma_desc = dma_desc; edesc = edesc->next_desc; } } else { request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); } } /** * talitos_submit - submits a descriptor to the device for processing * @dev: the SEC device to be used * @ch: the SEC device channel to be used * @desc: the descriptor to be processed by the device * @callback: whom to call when processing is complete * @context: a handle for use by caller (optional) * * desc must contain valid dma-mapped (bus physical) address pointers. * callback must check err and feedback in descriptor header * for device processing status. */ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, void (*callback)(struct device *dev, struct talitos_desc *desc, void *context, int error), void *context) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request; unsigned long flags; int head; bool is_sec1 = has_ftr_sec1(priv); spin_lock_irqsave(&priv->chan[ch].head_lock, flags); if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { /* h/w fifo is full */ spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); return -EAGAIN; } head = priv->chan[ch].head; request = &priv->chan[ch].fifo[head]; /* map descriptor and save caller data */ dma_map_request(dev, request, desc, is_sec1); request->callback = callback; request->context = context; /* increment fifo head */ priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); smp_wmb(); request->desc = desc; /* GO! */ wmb(); out_be32(priv->chan[ch].reg + TALITOS_FF, upper_32_bits(request->dma_desc)); out_be32(priv->chan[ch].reg + TALITOS_FF_LO, lower_32_bits(request->dma_desc)); spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); return -EINPROGRESS; } static __be32 get_request_hdr(struct device *dev, struct talitos_request *request, bool is_sec1) { struct talitos_edesc *edesc; dma_addr_t dma_desc; if (!is_sec1) { dma_sync_single_for_cpu(dev, request->dma_desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); return request->desc->hdr; } edesc = container_of(request->desc, struct talitos_edesc, desc); dma_desc = request->dma_desc; while (edesc->next_desc) { dma_desc = be32_to_cpu(edesc->desc.next_desc); edesc = edesc->next_desc; } dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); return edesc->desc.hdr1; } static void dma_unmap_request(struct device *dev, struct talitos_request *request, bool is_sec1) { struct talitos_edesc *edesc; if (is_sec1) { dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); edesc = container_of(request->desc, struct talitos_edesc, desc); while (edesc->next_desc) { dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc), TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); edesc = edesc->next_desc; } } else { dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); } } /* * process what was done, notify callback of error if not */ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request, saved_req; unsigned long flags; int tail, status; bool is_sec1 = has_ftr_sec1(priv); spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); tail = priv->chan[ch].tail; while (priv->chan[ch].fifo[tail].desc) { __be32 hdr; request = &priv->chan[ch].fifo[tail]; /* descriptors with their done bits set don't get the error */ rmb(); hdr = get_request_hdr(dev, request, is_sec1); if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; else if (!error) break; else status = error; dma_unmap_request(dev, request, is_sec1); /* copy entries so we can call callback outside lock */ saved_req.desc = request->desc; saved_req.callback = request->callback; saved_req.context = request->context; /* release request entry in fifo */ smp_wmb(); request->desc = NULL; /* increment fifo tail */ priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); atomic_dec(&priv->chan[ch].submit_count); saved_req.callback(dev, saved_req.desc, saved_req.context, status); /* channel may resume processing in single desc error case */ if (error && !reset_ch && status == error) return; spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); tail = priv->chan[ch].tail; } spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); } /* * process completed requests for channels that have done status */ #define DEF_TALITOS1_DONE(name, ch_done_mask) \ static void talitos1_done_##name(unsigned long data) \ { \ struct device *dev = (struct device *)data; \ struct talitos_private *priv = dev_get_drvdata(dev); \ unsigned long flags; \ \ if (ch_done_mask & 0x10000000) \ flush_channel(dev, 0, 0, 0); \ if (ch_done_mask & 0x40000000) \ flush_channel(dev, 1, 0, 0); \ if (ch_done_mask & 0x00010000) \ flush_channel(dev, 2, 0, 0); \ if (ch_done_mask & 0x00040000) \ flush_channel(dev, 3, 0, 0); \ \ /* At this point, all completed channels have been processed */ \ /* Unmask done interrupts for channels completed later on. */ \ spin_lock_irqsave(&priv->reg_lock, flags); \ clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ spin_unlock_irqrestore(&priv->reg_lock, flags); \ } DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) #define DEF_TALITOS2_DONE(name, ch_done_mask) \ static void talitos2_done_##name(unsigned long data) \ { \ struct device *dev = (struct device *)data; \ struct talitos_private *priv = dev_get_drvdata(dev); \ unsigned long flags; \ \ if (ch_done_mask & 1) \ flush_channel(dev, 0, 0, 0); \