/* * Copyright (C) 2008 Advanced Micro Devices, Inc. * * Author: Joerg Roedel <joerg.roedel@amd.com> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA */ #include <linux/scatterlist.h>#include <linux/dma-mapping.h>#include <linux/stacktrace.h>#include <linux/dma-debug.h>#include <linux/spinlock.h>#include <linux/debugfs.h>#include <linux/uaccess.h>#include <linux/export.h>#include <linux/device.h>#include <linux/types.h>#include <linux/sched.h>#include <linux/ctype.h>#include <linux/list.h>#include <linux/slab.h> #include <asm/sections.h> #define HASH_SIZE       1024ULL#define HASH_FN_SHIFT   13#define HASH_FN_MASK    (HASH_SIZE - 1) enum {	dma_debug_single,	dma_debug_page,	dma_debug_sg,	dma_debug_coherent,}; #define DMA_DEBUG_STACKTRACE_ENTRIES 5 struct dma_debug_entry {	struct list_head list;	struct device    *dev;	int              type;	phys_addr_t      paddr;	u64              dev_addr;	u64              size;	int              direction;	int		 sg_call_ents;	int		 sg_mapped_ents;#ifdef CONFIG_STACKTRACE	struct		 stack_trace stacktrace;	unsigned long	 st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];#endif}; typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *); struct hash_bucket {	struct list_head list;	spinlock_t lock;} ____cacheline_aligned_in_smp; /* Hash list to save the allocated dma addresses */static struct hash_bucket dma_entry_hash[HASH_SIZE];/* List of pre-allocated dma_debug_entry's */static LIST_HEAD(free_entries);/* Lock for the list above */static DEFINE_SPINLOCK(free_entries_lock); /* Global disable flag - will be set in case of an error */static bool global_disable __read_mostly; /* Global error count */static u32 error_count; /* Global error show enable*/static u32 show_all_errors __read_mostly;/* Number of errors to show */static u32 show_num_errors = 1; static u32 num_free_entries;static u32 min_free_entries;static u32 nr_total_entries; /* number of preallocated entries requested by kernel cmdline */static u32 req_entries; /* debugfs dentry's for the stuff above */static struct dentry *dma_debug_dent        __read_mostly;static struct dentry *global_disable_dent   __read_mostly;static struct dentry *error_count_dent      __read_mostly;static struct dentry *show_all_errors_dent  __read_mostly;static struct dentry *show_num_errors_dent  __read_mostly;static struct dentry *num_free_entries_dent __read_mostly;static struct dentry *min_free_entries_dent __read_mostly;static struct dentry *filter_dent           __read_mostly; /* per-driver filter related state */ #define NAME_MAX_LEN	64 static char                  current_driver_name[NAME_MAX_LEN] __read_mostly;static struct device_driver *current_driver                    __read_mostly; static DEFINE_RWLOCK(driver_name_lock); static const char *type2name[4] = { "single", "page",				    "scather-gather", "coherent" }; static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",				   "DMA_FROM_DEVICE", "DMA_NONE" }; /* little merge helper - remove it after the merge window */#ifndef BUS_NOTIFY_UNBOUND_DRIVER#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005#endif /* * The access to some variables in this macro is racy. We can't use atomic_t * here because all these variables are exported to debugfs. Some of them even * writeable. This is also the reason why a lock won't help much. But anyway, * the races are no big deal. Here is why: * *   error_count: the addition is racy, but the worst thing that can happen is *                that we don't count some errors *   show_num_errors: the subtraction is racy. Also no big deal because in *                    worst case this will result in one warning more in the *                    system log than the user configured. This variable is *                    writeable via debugfs. */static inline void dump_entry_trace(struct dma_debug_entry *entry){#ifdef CONFIG_STACKTRACE	if (entry) {		pr_warning("Mapped at:\n");		print_stack_trace(&entry->stacktrace, 0);	}#endif} static bool driver_filter(struct device *dev){	struct device_driver *drv;	unsigned long flags;	bool ret; 	/* driver filter off */	if (likely(!current_driver_name[0]))		return true; 	/* driver filter on and initialized */	if (current_driver && dev && dev->driver == current_driver)		return true; 	/* driver filter on, but we can't filter on a NULL device... */	if (!dev)		return false; 	if (current_driver || !current_driver_name[0])		return false; 	/* driver filter on but not yet initialized */	drv = dev->driver;	if (!drv)		return false; 	/* lock to protect against change of current_driver_name */	read_lock_irqsave(&driver_name_lock, flags); 	ret = false;	if (drv->name &&	    strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {		current_driver = drv;		ret = true;	} 	read_unlock_irqrestore(&driver_name_lock, flags); 	return ret;} #define err_printk(dev, entry, format, arg...) do {			\		error_count += 1;					\		if (driver_filter(dev) &&				\		    (show_all_errors || show_num_errors > 0)) {		\			WARN(1, "%s %s: " format,			\			     dev ? dev_driver_string(dev) : "NULL",	\			     dev ? dev_name(dev) : "NULL", ## arg);	\			dump_entry_trace(entry);			\		}							\		if (!show_all_errors && show_num_errors > 0)		\			show_num_errors -= 1;				\	} while (0); /* * Hash related functions * * Every DMA-API request is saved into a struct dma_debug_entry. To * have quick access to these structs they are stored into a hash. */static int hash_fn(struct dma_debug_entry *entry){	/*	 * Hash function is based on the dma address.	 * We use bits 20-27 here as the index into the hash	 */	return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;} /* * Request exclusive access to a hash bucket for a given dma_debug_entry. */static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,					   unsigned long *flags){	int idx = hash_fn(entry);	unsigned long __flags; 	spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);	*flags = __flags;	return &dma_entry_hash[idx];} /* * Give up exclusive access to the hash bucket */static void put_hash_bucket(struct hash_bucket *bucket,			    unsigned long *flags){	unsigned long __flags = *flags; 	spin_unlock_irqrestore(&bucket->lock, __flags);} static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b){	return ((a->dev_addr == b->dev_addr) &&		(a->dev == b->dev)) ? true : false;} static bool containing_match(struct dma_debug_entry *a,			     struct dma_debug_entry *b){	if (a->dev != b->dev)		return false; 	if ((b->dev_addr <= a->dev_addr) &&	    ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))		return true; 	return false;} /* * Search a given entry in the hash bucket list */static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,						  struct dma_debug_entry *ref,						  match_fn match){	struct dma_debug_entry *entry, *ret = NULL;	int matches = 0, match_lvl, last_lvl = 0; 	list_for_each_entry(entry, &bucket->list, list) {		if (!match(ref, entry))			continue; 		/*		 * Some drivers map the same physical address multiple		 * times. Without a hardware IOMMU this results in the		 * same device addresses being put into the dma-debug		 * hash multiple times too. This can result in false		 * positives being reported. Therefore we implement a		 * best-fit algorithm here which returns the entry from		 * the hash which fits best to the reference value		 * instead of the first-fit.		 */		matches += 1;		match_lvl = 0;		entry->size         == ref->size         ? ++match_lvl : 0;		entry->type         == ref->type         ? ++match_lvl : 0;		entry->direction    == ref->direction    ? ++match_lvl : 0;		entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0; 		if (match_lvl == 4) {			/* perfect-fit - return the result */			return entry;		} else if (match_lvl > last_lvl) {			/*			 * We found an entry that fits better then the			 * previous one			 */			last_lvl = match_lvl;			ret      = entry;		}	} 	/*	 * If we have multiple matches but no perfect-fit, just return	 * NULL.	 */	ret = (matches == 1) ? ret : NULL; 	return ret;} static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,						 struct dma_debug_entry *ref){	return __hash_bucket_find(bucket, ref, exact_match);} static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,						   struct dma_debug_entry *ref,						   unsigned long *flags){ 	unsigned int max_range = dma_get_max_seg_size(ref->dev);	struct dma_debug_entry *entry, index = *ref;	unsigned int range = 0; 	while (range <= max_range) {		entry = __hash_bucket_find(*bucket, &index, containing_match); 		if (entry)			return entry; 		/*		 * Nothing found, go back a hash bucket		 */		put_hash_bucket(*bucket, flags);		range          += (1 << HASH_FN_SHIFT);		index.dev_addr -= (1 << HASH_FN_SHIFT);		*bucket = get_hash_bucket(&index, flags);	} 	return NULL;} /* * Add an entry to a hash bucket */static void hash_bucket_add(struct hash_bucket *bucket,			    struct dma_debug_entry *entry){	list_add_tail(&entry->list, &bucket->list);} /* * Remove entry from a hash bucket list */static void hash_bucket_del(struct dma_debug_entry *entry){	list_del(&entry->list);} /* * Dump mapping entries for debugging purposes */void debug_dma_dump_mappings(struct device *dev){	int idx; 	for (idx = 0; idx < HASH_SIZE; idx++) {		struct hash_bucket *bucket = &dma_entry_hash[idx];		struct dma_debug_entry *entry;		unsigned long flags; 		spin_lock_irqsave(&bucket->lock, flags); 		list_for_each_entry(entry, &bucket->list, list) {			if (!dev || dev == entry->dev) {				dev_info(entry->dev,					 "%s idx %d P=%Lx D=%Lx L=%Lx %s\n",					 type2name[entry->type], idx,					 (unsigned long long)entry->paddr,					 entry->dev_addr, entry->size,					 dir2name[entry->direction]);			}		} 		spin_unlock_irqrestore(&bucket->lock, flags);	}}EXPORT_SYMBOL(debug_dma_dump_mappings); /* * Wrapper function for adding an entry to the hash. * This function takes care of locking itself. */static void add_dma_entry(struct dma_debug_entry *entry){	struct hash_bucket *bucket;	unsigned long flags; 	bucket = get_hash_bucket(entry, &flags);	hash_bucket_add(bucket, entry);	put_hash_bucket(bucket, &flags);} static struct dma_debug_entry *__dma_entry_alloc(void){	struct dma_debug_entry *entry; 	entry = list_entry(free_entries.next, struct dma_debug_entry, list);	list_del(&entry->list);	memset(entry, 0, sizeof(*entry)); 	num_free_entries -= 1;	if (num_free_entries < min_free_entries)		min_free_entries = num_free_entries; 	return entry;} /* struct dma_entry allocator * * The next two functions implement the allocator for * struct dma_debug_entries. */static struct dma_debug_entry *dma_entry_alloc(void){	struct dma_debug_entry *entry = NULL;	unsigned long flags; 	spin_lock_irqsave(&free_entries_lock, flags); 	if (list_empty(&free_entries)) {		pr_err("DMA-API: debugging out of memory - disabling\n");		global_disable = true;		goto out;	} 	entry = __dma_entry_alloc(); #ifdef CONFIG_STACKTRACE	entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;	entry->stacktrace.entries = entry->st_entries;	entry->stacktrace.skip = 2;	save_stack_trace(&entry->stacktrace);#endif out:	spin_unlock_irqrestore(&free_entries_lock, flags); 	return entry;} static void dma_entry_free(struct dma_debug_entry *entry){	unsigned long flags; 	/*	 * add to beginning of the list - this way the entries are	 * more likely cache hot when they are reallocated.	 */	spin_lock_irqsave(&free_entries_lock, flags);	list_add(&entry->list, &free_entries);	num_free_entries += 1;	spin_unlock_irqrestore(&free_entries_lock, flags);} int dma_debug_resize_entries(u32 num_entries){	int i, delta, ret = 0;	unsigned long flags;	struct dma_debug_entry *entry;	LIST_HEAD(tmp); 	spin_lock_irqsave(&free_entries_lock, flags); 	if (nr_total_entries < num_entries) {		delta = num_entries - nr_total_entries; 		spin_unlock_irqrestore(&free_entries_lock, flags); 		for (i = 0; i < delta; i++) {			entry = kzalloc(sizeof(*entry), GFP_KERNEL);			if (!entry)				break; 			list_add_tail(&entry->list, &tmp);		} 		spin_lock_irqsave(&free_entries_lock, flags); 		list_splice(&tmp, &free_entries);		nr_total_entries += i;		num_free_entries += i;	} else {

Lines 1–500 of 1365. Load more