// SPDX-License-Identifier: GPL-2.0-only/* * DMA Engine test module * * Copyright (C) 2007 Atmel Corporation * Copyright (C) 2013 Intel Corporation */#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/err.h>#include <linux/delay.h>#include <linux/dma-mapping.h>#include <linux/dmaengine.h>#include <linux/freezer.h>#include <linux/init.h>#include <linux/kthread.h>#include <linux/sched/task.h>#include <linux/module.h>#include <linux/moduleparam.h>#include <linux/random.h>#include <linux/slab.h>#include <linux/wait.h> static bool nobounce;module_param(nobounce, bool, 0644);MODULE_PARM_DESC(nobounce, "Prevent using swiotlb buffer (default: use swiotlb buffer)"); static unsigned int test_buf_size = 16384;module_param(test_buf_size, uint, 0644);MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); static char test_device[32];module_param_string(device, test_device, sizeof(test_device), 0644);MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); static unsigned int threads_per_chan = 1;module_param(threads_per_chan, uint, 0644);MODULE_PARM_DESC(threads_per_chan,		"Number of threads to start per channel (default: 1)"); static unsigned int max_channels;module_param(max_channels, uint, 0644);MODULE_PARM_DESC(max_channels,		"Maximum number of channels to use (default: all)"); static unsigned int iterations;module_param(iterations, uint, 0644);MODULE_PARM_DESC(iterations,		"Iterations before stopping test (default: infinite)"); static unsigned int dmatest;module_param(dmatest, uint, 0644);MODULE_PARM_DESC(dmatest,		"dmatest 0-memcpy 1-memset (default: 0)"); static unsigned int xor_sources = 3;module_param(xor_sources, uint, 0644);MODULE_PARM_DESC(xor_sources,		"Number of xor source buffers (default: 3)"); static unsigned int pq_sources = 3;module_param(pq_sources, uint, 0644);MODULE_PARM_DESC(pq_sources,		"Number of p+q source buffers (default: 3)"); static int timeout = 3000;module_param(timeout, int, 0644);MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "		 "Pass -1 for infinite timeout"); static bool noverify;module_param(noverify, bool, 0644);MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)"); static bool norandom;module_param(norandom, bool, 0644);MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)"); static bool verbose;module_param(verbose, bool, 0644);MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)"); static int alignment = -1;module_param(alignment, int, 0644);MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))"); static unsigned int transfer_size;module_param(transfer_size, uint, 0644);MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))"); static bool polled;module_param(polled, bool, 0644);MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts"); /** * struct dmatest_params - test parameters. * @nobounce:		prevent using swiotlb buffer * @buf_size:		size of the memcpy test buffer * @channel:		bus ID of the channel to test * @device:		bus ID of the DMA Engine to test * @threads_per_chan:	number of threads to start per channel * @max_channels:	maximum number of channels to use * @iterations:		iterations before stopping test * @xor_sources:	number of xor source buffers * @pq_sources:		number of p+q source buffers * @timeout:		transfer timeout in msec, -1 for infinite timeout * @noverify:		disable data verification * @norandom:		disable random offset setup * @alignment:		custom data address alignment taken as 2^alignment * @transfer_size:	custom transfer size in bytes * @polled:		use polling for completion instead of interrupts */struct dmatest_params {	bool		nobounce;	unsigned int	buf_size;	char		channel[20];	char		device[32];	unsigned int	threads_per_chan;	unsigned int	max_channels;	unsigned int	iterations;	unsigned int	xor_sources;	unsigned int	pq_sources;	int		timeout;	bool		noverify;	bool		norandom;	int		alignment;	unsigned int	transfer_size;	bool		polled;}; /** * struct dmatest_info - test information. * @params:		test parameters * @channels:		channels under test * @nr_channels:	number of channels under test * @lock:		access protection to the fields of this structure * @did_init:		module has been initialized completely * @last_error:		test has faced configuration issues */struct dmatest_info {	/* Test parameters */	struct dmatest_params	params; 	/* Internal state */	struct list_head	channels;	unsigned int		nr_channels;	int			last_error;	struct mutex		lock;	bool			did_init;}; static struct dmatest_info test_info = {	.channels = LIST_HEAD_INIT(test_info.channels),	.lock = __MUTEX_INITIALIZER(test_info.lock),}; static int dmatest_run_set(const char *val, const struct kernel_param *kp);static int dmatest_run_get(char *val, const struct kernel_param *kp);static const struct kernel_param_ops run_ops = {	.set = dmatest_run_set,	.get = dmatest_run_get,};static bool dmatest_run;module_param_cb(run, &run_ops, &dmatest_run, 0644);MODULE_PARM_DESC(run, "Run the test (default: false)"); static int dmatest_chan_set(const char *val, const struct kernel_param *kp);static int dmatest_chan_get(char *val, const struct kernel_param *kp);static const struct kernel_param_ops multi_chan_ops = {	.set = dmatest_chan_set,	.get = dmatest_chan_get,}; static char test_channel[20];static struct kparam_string newchan_kps = {	.string = test_channel,	.maxlen = 20,};module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); static int dmatest_test_list_get(char *val, const struct kernel_param *kp);static const struct kernel_param_ops test_list_ops = {	.get = dmatest_test_list_get,};module_param_cb(test_list, &test_list_ops, NULL, 0444);MODULE_PARM_DESC(test_list, "Print current test list"); /* Maximum amount of mismatched bytes in buffer to print */#define MAX_ERROR_COUNT		32 /* * Initialization patterns. All bytes in the source buffer has bit 7 * set, all bytes in the destination buffer has bit 7 cleared. * * Bit 6 is set for all bytes which are to be copied by the DMA * engine. Bit 5 is set for all bytes which are to be overwritten by * the DMA engine. * * The remaining bits are the inverse of a counter which increments by * one for each byte address. */#define PATTERN_SRC		0x80#define PATTERN_DST		0x00#define PATTERN_COPY		0x40#define PATTERN_OVERWRITE	0x20#define PATTERN_COUNT_MASK	0x1f#define PATTERN_MEMSET_IDX	0x01 /* Fixed point arithmetic ops */#define FIXPT_SHIFT		8#define FIXPNT_MASK		0xFF#define FIXPT_TO_INT(a)	((a) >> FIXPT_SHIFT)#define INT_TO_FIXPT(a)	((a) << FIXPT_SHIFT)#define FIXPT_GET_FRAC(a)	((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT) /* poor man's completion - we want to use wait_event_freezable() on it */struct dmatest_done {	bool			done;	wait_queue_head_t	*wait;}; struct dmatest_data {	u8		**raw;	u8		**aligned;	gfp_t		gfp_flags;	unsigned int	cnt;	unsigned int	off;}; struct dmatest_thread {	struct list_head	node;	struct dmatest_info	*info;	struct task_struct	*task;	struct dma_chan		*chan;	struct dmatest_data	src;	struct dmatest_data	dst;	enum dma_transaction_type type;	wait_queue_head_t done_wait;	struct dmatest_done test_done;	bool			done;	bool			pending;}; struct dmatest_chan {	struct list_head	node;	struct dma_chan		*chan;	struct list_head	threads;}; static DECLARE_WAIT_QUEUE_HEAD(thread_wait);static bool wait; static bool is_threaded_test_run(struct dmatest_info *info){	struct dmatest_chan *dtc; 	list_for_each_entry(dtc, &info->channels, node) {		struct dmatest_thread *thread; 		list_for_each_entry(thread, &dtc->threads, node) {			if (!thread->done && !thread->pending)				return true;		}	} 	return false;} static bool is_threaded_test_pending(struct dmatest_info *info){	struct dmatest_chan *dtc; 	list_for_each_entry(dtc, &info->channels, node) {		struct dmatest_thread *thread; 		list_for_each_entry(thread, &dtc->threads, node) {			if (thread->pending)				return true;		}	} 	return false;} static int dmatest_wait_get(char *val, const struct kernel_param *kp){	struct dmatest_info *info = &test_info;	struct dmatest_params *params = &info->params; 	if (params->iterations)		wait_event(thread_wait, !is_threaded_test_run(info));	wait = true;	return param_get_bool(val, kp);} static const struct kernel_param_ops wait_ops = {	.get = dmatest_wait_get,	.set = param_set_bool,};module_param_cb(wait, &wait_ops, &wait, 0444);MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)"); static bool dmatest_match_channel(struct dmatest_params *params,		struct dma_chan *chan){	if (params->channel[0] == '\0')		return true;	return strcmp(dma_chan_name(chan), params->channel) == 0;} static bool dmatest_match_device(struct dmatest_params *params,		struct dma_device *device){	if (params->device[0] == '\0')		return true;	return strcmp(dev_name(device->dev), params->device) == 0;} static unsigned long dmatest_random(void){	unsigned long buf; 	get_random_bytes(&buf, sizeof(buf));	return buf;} static inline u8 gen_inv_idx(u8 index, bool is_memset){	u8 val = is_memset ? PATTERN_MEMSET_IDX : index; 	return ~val & PATTERN_COUNT_MASK;} static inline u8 gen_src_value(u8 index, bool is_memset){	return PATTERN_SRC | gen_inv_idx(index, is_memset);} static inline u8 gen_dst_value(u8 index, bool is_memset){	return PATTERN_DST | gen_inv_idx(index, is_memset);} static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,		unsigned int buf_size, bool is_memset){	unsigned int i;	u8 *buf; 	for (; (buf = *bufs); bufs++) {		for (i = 0; i < start; i++)			buf[i] = gen_src_value(i, is_memset);		for ( ; i < start + len; i++)			buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;		for ( ; i < buf_size; i++)			buf[i] = gen_src_value(i, is_memset);		buf++;	}} static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,		unsigned int buf_size, bool is_memset){	unsigned int i;	u8 *buf; 	for (; (buf = *bufs); bufs++) {		for (i = 0; i < start; i++)			buf[i] = gen_dst_value(i, is_memset);		for ( ; i < start + len; i++)			buf[i] = gen_dst_value(i, is_memset) |						PATTERN_OVERWRITE;		for ( ; i < buf_size; i++)			buf[i] = gen_dst_value(i, is_memset);	}} static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,		unsigned int counter, bool is_srcbuf, bool is_memset){	u8		diff = actual ^ pattern;	u8		expected = pattern | gen_inv_idx(counter, is_memset);	const char	*thread_name = current->comm; 	if (is_srcbuf)		pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",			thread_name, index, expected, actual);	else if ((pattern & PATTERN_COPY)			&& (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))		pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",			thread_name, index, expected, actual);	else if (diff & PATTERN_SRC)		pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",			thread_name, index, expected, actual);	else		pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",			thread_name, index, expected, actual);} static unsigned int dmatest_verify(u8 **bufs, unsigned int start,		unsigned int end, unsigned int counter, u8 pattern,		bool is_srcbuf, bool is_memset){	unsigned int i;	unsigned int error_count = 0;	u8 actual;	u8 expected;	u8 *buf;	unsigned int counter_orig = counter; 	for (; (buf = *bufs); bufs++) {		counter = counter_orig;		for (i = start; i < end; i++) {			actual = buf[i];			expected = pattern | gen_inv_idx(counter, is_memset);			if (actual != expected) {				if (error_count < MAX_ERROR_COUNT)					dmatest_mismatch(actual, pattern, i,							 counter, is_srcbuf,							 is_memset);				error_count++;			}			counter++;		}	} 	if (error_count > MAX_ERROR_COUNT)		pr_warn("%s: %u errors suppressed\n",			current->comm, error_count - MAX_ERROR_COUNT); 	return error_count;}  static void dmatest_callback(void *arg){	struct dmatest_done *done = arg;	struct dmatest_thread *thread =		container_of(done, struct dmatest_thread, test_done);	if (!thread->done) {		done->done = true;		wake_up_all(done->wait);	} else {		/*		 * If thread->done, it means that this callback occurred		 * after the parent thread has cleaned up. This can		 * happen in the case that driver doesn't implement		 * the terminate_all() functionality and a dma operation		 * did not occur within the timeout period		 */		WARN(1, "dmatest: Kernel memory may be corrupted!!\n");	}} static unsigned int min_odd(unsigned int x, unsigned int y){	unsigned int val = min(x, y); 	return val % 2 ? val : val - 1;} static void result(const char *err, unsigned int n, unsigned int src_off,		   unsigned int dst_off, unsigned int len, unsigned long data){	if (IS_ERR_VALUE(data)) {		pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%ld)\n",			current->comm, n, err, src_off, dst_off, len, data);	} else {		pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",			current->comm, n, err, src_off, dst_off, len, data);	}} static void dbg_result(const char *err, unsigned int n, unsigned int src_off,		       unsigned int dst_off, unsigned int len,		       unsigned long data){	pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",		 current->comm, n, err, src_off, dst_off, len, data);} #define verbose_result(err, n, src_off, dst_off, len, data) ({	\	if (verbose)						\		result(err, n, src_off, dst_off, len, data);	\	else							\		dbg_result(err, n, src_off, dst_off, len, data);\}) static unsigned long long dmatest_persec(s64 runtime, unsigned int val){	unsigned long long per_sec = 1000000; 	if (runtime <= 0)		return 0; 	/* drop precision until runtime is 32-bits */	while (runtime > UINT_MAX) {		runtime >>= 1;		per_sec <<= 1;

Lines 1–500 of 1378. Load more