/* * drivers/dma/fsl_raid.c * * Freescale RAID Engine device driver * * Author: *	Harninder Rai <harninder.rai@freescale.com> *	Naveen Burmi <naveenburmi@freescale.com> * * Rewrite: *	Xuelin Shi <xuelin.shi@freescale.com> * * Copyright (c) 2010-2014 Freescale Semiconductor, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: *     * Redistributions of source code must retain the above copyright *       notice, this list of conditions and the following disclaimer. *     * Redistributions in binary form must reproduce the above copyright *       notice, this list of conditions and the following disclaimer in the *       documentation and/or other materials provided with the distribution. *     * Neither the name of Freescale Semiconductor nor the *       names of its contributors may be used to endorse or promote products *       derived from this software without specific prior written permission. * * ALTERNATIVELY, this software may be distributed under the terms of the * GNU General Public License ("GPL") as published by the Free Software * Foundation, either version 2 of that License or (at your option) any * later version. * * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Theory of operation: * * General capabilities: *	RAID Engine (RE) block is capable of offloading XOR, memcpy and P/Q *	calculations required in RAID5 and RAID6 operations. RE driver *	registers with Linux's ASYNC layer as dma driver. RE hardware *	maintains strict ordering of the requests through chained *	command queueing. * * Data flow: *	Software RAID layer of Linux (MD layer) maintains RAID partitions, *	strips, stripes etc. It sends requests to the underlying ASYNC layer *	which further passes it to RE driver. ASYNC layer decides which request *	goes to which job ring of RE hardware. For every request processed by *	RAID Engine, driver gets an interrupt unless coalescing is set. The *	per job ring interrupt handler checks the status register for errors, *	clears the interrupt and leave the post interrupt processing to the irq *	thread. */#include <linux/interrupt.h>#include <linux/module.h>#include <linux/of.h>#include <linux/of_irq.h>#include <linux/of_platform.h>#include <linux/platform_device.h>#include <linux/dma-mapping.h>#include <linux/dmapool.h>#include <linux/dmaengine.h>#include <linux/io.h>#include <linux/spinlock.h>#include <linux/slab.h> #include "dmaengine.h"#include "fsl_raid.h" #define FSL_RE_MAX_XOR_SRCS	16#define FSL_RE_MAX_PQ_SRCS	16#define FSL_RE_MIN_DESCS	256#define FSL_RE_MAX_DESCS	(4 * FSL_RE_MIN_DESCS)#define FSL_RE_FRAME_FORMAT	0x1#define FSL_RE_MAX_DATA_LEN	(1024*1024) #define to_fsl_re_dma_desc(tx) container_of(tx, struct fsl_re_desc, async_tx) /* Add descriptors into per chan software queue - submit_q */static dma_cookie_t fsl_re_tx_submit(struct dma_async_tx_descriptor *tx){	struct fsl_re_desc *desc;	struct fsl_re_chan *re_chan;	dma_cookie_t cookie;	unsigned long flags; 	desc = to_fsl_re_dma_desc(tx);	re_chan = container_of(tx->chan, struct fsl_re_chan, chan); 	spin_lock_irqsave(&re_chan->desc_lock, flags);	cookie = dma_cookie_assign(tx);	list_add_tail(&desc->node, &re_chan->submit_q);	spin_unlock_irqrestore(&re_chan->desc_lock, flags); 	return cookie;} /* Copy descriptor from per chan software queue into hardware job ring */static void fsl_re_issue_pending(struct dma_chan *chan){	struct fsl_re_chan *re_chan;	int avail;	struct fsl_re_desc *desc, *_desc;	unsigned long flags; 	re_chan = container_of(chan, struct fsl_re_chan, chan); 	spin_lock_irqsave(&re_chan->desc_lock, flags);	avail = FSL_RE_SLOT_AVAIL(		in_be32(&re_chan->jrregs->inbring_slot_avail)); 	list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) {		if (!avail)			break; 		list_move_tail(&desc->node, &re_chan->active_q); 		memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count],		       &desc->hwdesc, sizeof(struct fsl_re_hw_desc)); 		re_chan->inb_count = (re_chan->inb_count + 1) &						FSL_RE_RING_SIZE_MASK;		out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1));		avail--;	}	spin_unlock_irqrestore(&re_chan->desc_lock, flags);} static void fsl_re_desc_done(struct fsl_re_desc *desc){	dma_cookie_complete(&desc->async_tx);	dma_descriptor_unmap(&desc->async_tx);	dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);} static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan){	struct fsl_re_desc *desc, *_desc;	unsigned long flags; 	spin_lock_irqsave(&re_chan->desc_lock, flags);	list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) {		if (async_tx_test_ack(&desc->async_tx))			list_move_tail(&desc->node, &re_chan->free_q);	}	spin_unlock_irqrestore(&re_chan->desc_lock, flags); 	fsl_re_issue_pending(&re_chan->chan);} static void fsl_re_dequeue(struct tasklet_struct *t){	struct fsl_re_chan *re_chan = from_tasklet(re_chan, t, irqtask);	struct fsl_re_desc *desc, *_desc;	struct fsl_re_hw_desc *hwdesc;	unsigned long flags;	unsigned int count, oub_count;	int found; 	fsl_re_cleanup_descs(re_chan); 	spin_lock_irqsave(&re_chan->desc_lock, flags);	count =	FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full));	while (count--) {		found = 0;		hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count];		list_for_each_entry_safe(desc, _desc, &re_chan->active_q,					 node) {			/* compare the hw dma addr to find the completed */			if (desc->hwdesc.lbea32 == hwdesc->lbea32 &&			    desc->hwdesc.addr_low == hwdesc->addr_low) {				found = 1;				break;			}		} 		if (found) {			fsl_re_desc_done(desc);			list_move_tail(&desc->node, &re_chan->ack_q);		} else {			dev_err(re_chan->dev,				"found hwdesc not in sw queue, discard it\n");		} 		oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK;		re_chan->oub_count = oub_count; 		out_be32(&re_chan->jrregs->oubring_job_rmvd,			 FSL_RE_RMVD_JOB(1));	}	spin_unlock_irqrestore(&re_chan->desc_lock, flags);} /* Per Job Ring interrupt handler */static irqreturn_t fsl_re_isr(int irq, void *data){	struct fsl_re_chan *re_chan;	u32 irqstate, status; 	re_chan = dev_get_drvdata((struct device *)data); 	irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status);	if (!irqstate)		return IRQ_NONE; 	/*	 * There's no way in upper layer (read MD layer) to recover from	 * error conditions except restart everything. In long term we	 * need to do something more than just crashing	 */	if (irqstate & FSL_RE_ERROR) {		status = in_be32(&re_chan->jrregs->jr_status);		dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n",			irqstate, status);	} 	/* Clear interrupt */	out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR); 	tasklet_schedule(&re_chan->irqtask); 	return IRQ_HANDLED;} static enum dma_status fsl_re_tx_status(struct dma_chan *chan,					dma_cookie_t cookie,					struct dma_tx_state *txstate){	return dma_cookie_status(chan, cookie, txstate);} static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index,			   size_t length, dma_addr_t addr, bool final){	u32 efrl = length & FSL_RE_CF_LENGTH_MASK; 	efrl |= final << FSL_RE_CF_FINAL_SHIFT;	cf[index].efrl32 = cpu_to_be32(efrl);	cf[index].addr_high = cpu_to_be32(upper_32_bits(addr));	cf[index].addr_low = cpu_to_be32(lower_32_bits(addr));} static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan,					    struct fsl_re_desc *desc,					    void *cf, dma_addr_t paddr){	desc->re_chan = re_chan;	desc->async_tx.tx_submit = fsl_re_tx_submit;	dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan);	INIT_LIST_HEAD(&desc->node); 	desc->hwdesc.fmt32 = cpu_to_be32(FSL_RE_FRAME_FORMAT <<					  FSL_RE_HWDESC_FMT_SHIFT);	desc->hwdesc.lbea32 = cpu_to_be32(upper_32_bits(paddr));	desc->hwdesc.addr_low = cpu_to_be32(lower_32_bits(paddr));	desc->cf_addr = cf;	desc->cf_paddr = paddr; 	desc->cdb_addr = (void *)(cf + FSL_RE_CF_DESC_SIZE);	desc->cdb_paddr = paddr + FSL_RE_CF_DESC_SIZE; 	return desc;} static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan,						  unsigned long flags){	struct fsl_re_desc *desc = NULL;	void *cf;	dma_addr_t paddr;	unsigned long lock_flag; 	fsl_re_cleanup_descs(re_chan); 	spin_lock_irqsave(&re_chan->desc_lock, lock_flag);	if (!list_empty(&re_chan->free_q)) {		/* take one desc from free_q */		desc = list_first_entry(&re_chan->free_q,					struct fsl_re_desc, node);		list_del(&desc->node); 		desc->async_tx.flags = flags;	}	spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag); 	if (!desc) {		desc = kzalloc_obj(*desc, GFP_NOWAIT);		if (!desc)			return NULL; 		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT,				    &paddr);		if (!cf) {			kfree(desc);			return NULL;		} 		desc = fsl_re_init_desc(re_chan, desc, cf, paddr);		desc->async_tx.flags = flags; 		spin_lock_irqsave(&re_chan->desc_lock, lock_flag);		re_chan->alloc_count++;		spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);	} 	return desc;} static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq(		struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,		unsigned int src_cnt, const unsigned char *scf, size_t len,		unsigned long flags){	struct fsl_re_chan *re_chan;	struct fsl_re_desc *desc;	struct fsl_re_xor_cdb *xor;	struct fsl_re_cmpnd_frame *cf;	u32 cdb;	unsigned int i, j;	unsigned int save_src_cnt = src_cnt;	int cont_q = 0; 	re_chan = container_of(chan, struct fsl_re_chan, chan);	if (len > FSL_RE_MAX_DATA_LEN) {		dev_err(re_chan->dev, "genq tx length %zu, max length %d\n",			len, FSL_RE_MAX_DATA_LEN);		return NULL;	} 	desc = fsl_re_chan_alloc_desc(re_chan, flags);	if (desc <= 0)		return NULL; 	if (scf && (flags & DMA_PREP_CONTINUE)) {		cont_q = 1;		src_cnt += 1;	} 	/* Filling xor CDB */	cdb = FSL_RE_XOR_OPCODE << FSL_RE_CDB_OPCODE_SHIFT;	cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT;	cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT;	cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT;	cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT;	xor = desc->cdb_addr;	xor->cdb32 = cdb; 	if (scf) {		/* compute q = src0*coef0^src1*coef1^..., * is GF(8) mult */		for (i = 0; i < save_src_cnt; i++)			xor->gfm[i] = scf[i];		if (cont_q)			xor->gfm[i++] = 1;	} else {		/* compute P, that is XOR all srcs */		for (i = 0; i < src_cnt; i++)			xor->gfm[i] = 1;	} 	/* Filling frame 0 of compound frame descriptor with CDB */	cf = desc->cf_addr;	fill_cfd_frame(cf, 0, sizeof(*xor), desc->cdb_paddr, 0); 	/* Fill CFD's 1st frame with dest buffer */	fill_cfd_frame(cf, 1, len, dest, 0); 	/* Fill CFD's rest of the frames with source buffers */	for (i = 2, j = 0; j < save_src_cnt; i++, j++)		fill_cfd_frame(cf, i, len, src[j], 0); 	/* Fill the last frame and mark it final */	if (cont_q)		fill_cfd_frame(cf, i, len, dest, 1);	else		fill_cfd_frame(cf, i - 1, len, src[j - 1], 1); 	return &desc->async_tx;} /* * Prep function for P parity calculation.In RAID Engine terminology, * XOR calculation is called GenQ calculation done through GenQ command */static struct dma_async_tx_descriptor *fsl_re_prep_dma_xor(		struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,		unsigned int src_cnt, size_t len, unsigned long flags){	/* NULL let genq take all coef as 1 */	return fsl_re_prep_dma_genq(chan, dest, src, src_cnt, NULL, len, flags);} /* * Prep function for P/Q parity calculation.In RAID Engine terminology, * P/Q calculation is called GenQQ done through GenQQ command */static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq(		struct dma_chan *chan, dma_addr_t *dest, dma_addr_t *src,		unsigned int src_cnt, const unsigned char *scf, size_t len,		unsigned long flags){	struct fsl_re_chan *re_chan;	struct fsl_re_desc *desc;	struct fsl_re_pq_cdb *pq;	struct fsl_re_cmpnd_frame *cf;	u32 cdb;	u8 *p;	int gfmq_len, i, j;	unsigned int save_src_cnt = src_cnt; 	re_chan = container_of(chan, struct fsl_re_chan, chan);	if (len > FSL_RE_MAX_DATA_LEN) {		dev_err(re_chan->dev, "pq tx length is %zu, max length is %d\n",			len, FSL_RE_MAX_DATA_LEN);		return NULL;	} 	/*	 * RE requires at least 2 sources, if given only one source, we pass the	 * second source same as the first one.	 * With only one source, generating P is meaningless, only generate Q.	 */	if (src_cnt == 1) {		struct dma_async_tx_descriptor *tx;		dma_addr_t dma_src[2];		unsigned char coef[2]; 		dma_src[0] = *src;		coef[0] = *scf;		dma_src[1] = *src;		coef[1] = 0;		tx = fsl_re_prep_dma_genq(chan, dest[1], dma_src, 2, coef, len,					  flags);		if (tx)			desc = to_fsl_re_dma_desc(tx); 		return tx;	} 	/*	 * During RAID6 array creation, Linux's MD layer gets P and Q	 * calculated separately in two steps. But our RAID Engine has	 * the capability to calculate both P and Q with a single command	 * Hence to merge well with MD layer, we need to provide a hook	 * here and call re_jq_prep_dma_genq() function	 */ 	if (flags & DMA_PREP_PQ_DISABLE_P)		return fsl_re_prep_dma_genq(chan, dest[1], src, src_cnt,				scf, len, flags); 	if (flags & DMA_PREP_CONTINUE)		src_cnt += 3; 	desc = fsl_re_chan_alloc_desc(re_chan, flags);	if (desc <= 0)		return NULL; 	/* Filling GenQQ CDB */	cdb = FSL_RE_PQ_OPCODE << FSL_RE_CDB_OPCODE_SHIFT;	cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT;	cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT;	cdb |= FSL_RE_BUFFER_OUTPUT << FSL_RE_CDB_BUFFER_SHIFT;	cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; 	pq = desc->cdb_addr;	pq->cdb32 = cdb; 	p = pq->gfm_q1;	/* Init gfm_q1[] */	for (i = 0; i < src_cnt; i++)		p[i] = 1; 	/* Align gfm[] to 32bit */	gfmq_len = ALIGN(src_cnt, 4); 	/* Init gfm_q2[] */	p += gfmq_len;	for (i = 0; i < src_cnt; i++)		p[i] = scf[i]; 	/* Filling frame 0 of compound frame descriptor with CDB */	cf = desc->cf_addr;	fill_cfd_frame(cf, 0, sizeof(struct fsl_re_pq_cdb), desc->cdb_paddr, 0); 	/* Fill CFD's 1st & 2nd frame with dest buffers */	for (i = 1, j = 0; i < 3; i++, j++)		fill_cfd_frame(cf, i, len, dest[j], 0); 	/* Fill CFD's rest of the frames with source buffers */	for (i = 3, j = 0; j < save_src_cnt; i++, j++)		fill_cfd_frame(cf, i, len, src[j], 0); 	/* PQ computation continuation */

Lines 1–500 of 892. Load more