/* * Copyright (C) 2008 Advanced Micro Devices, Inc. * * Author: Joerg Roedel <joerg.roedel@amd.com> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA */ #include <linux/sched/task_stack.h>#include <linux/scatterlist.h>#include <linux/dma-mapping.h>#include <linux/sched/task.h>#include <linux/stacktrace.h>#include <linux/dma-debug.h>#include <linux/spinlock.h>#include <linux/vmalloc.h>#include <linux/debugfs.h>#include <linux/uaccess.h>#include <linux/export.h>#include <linux/device.h>#include <linux/types.h>#include <linux/sched.h>#include <linux/ctype.h>#include <linux/list.h>#include <linux/slab.h> #include <asm/sections.h> #define HASH_SIZE       1024ULL#define HASH_FN_SHIFT   13#define HASH_FN_MASK    (HASH_SIZE - 1) enum {	dma_debug_single,	dma_debug_page,	dma_debug_sg,	dma_debug_coherent,	dma_debug_resource,}; enum map_err_types {	MAP_ERR_CHECK_NOT_APPLICABLE,	MAP_ERR_NOT_CHECKED,	MAP_ERR_CHECKED,}; #define DMA_DEBUG_STACKTRACE_ENTRIES 5 /** * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping * @list: node on pre-allocated free_entries list * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent * @type: single, page, sg, coherent * @pfn: page frame of the start address * @offset: offset of mapping relative to pfn * @size: length of the mapping * @direction: enum dma_data_direction * @sg_call_ents: 'nents' from dma_map_sg * @sg_mapped_ents: 'mapped_ents' from dma_map_sg * @map_err_type: track whether dma_mapping_error() was checked * @stacktrace: support backtraces when a violation is detected */struct dma_debug_entry {	struct list_head list;	struct device    *dev;	int              type;	unsigned long	 pfn;	size_t		 offset;	u64              dev_addr;	u64              size;	int              direction;	int		 sg_call_ents;	int		 sg_mapped_ents;	enum map_err_types  map_err_type;#ifdef CONFIG_STACKTRACE	struct		 stack_trace stacktrace;	unsigned long	 st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];#endif}; typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *); struct hash_bucket {	struct list_head list;	spinlock_t lock;} ____cacheline_aligned_in_smp; /* Hash list to save the allocated dma addresses */static struct hash_bucket dma_entry_hash[HASH_SIZE];/* List of pre-allocated dma_debug_entry's */static LIST_HEAD(free_entries);/* Lock for the list above */static DEFINE_SPINLOCK(free_entries_lock); /* Global disable flag - will be set in case of an error */static bool global_disable __read_mostly; /* Early initialization disable flag, set at the end of dma_debug_init */static bool dma_debug_initialized __read_mostly; static inline bool dma_debug_disabled(void){	return global_disable || !dma_debug_initialized;} /* Global error count */static u32 error_count; /* Global error show enable*/static u32 show_all_errors __read_mostly;/* Number of errors to show */static u32 show_num_errors = 1; static u32 num_free_entries;static u32 min_free_entries;static u32 nr_total_entries; /* number of preallocated entries requested by kernel cmdline */static u32 req_entries; /* debugfs dentry's for the stuff above */static struct dentry *dma_debug_dent        __read_mostly;static struct dentry *global_disable_dent   __read_mostly;static struct dentry *error_count_dent      __read_mostly;static struct dentry *show_all_errors_dent  __read_mostly;static struct dentry *show_num_errors_dent  __read_mostly;static struct dentry *num_free_entries_dent __read_mostly;static struct dentry *min_free_entries_dent __read_mostly;static struct dentry *filter_dent           __read_mostly; /* per-driver filter related state */ #define NAME_MAX_LEN	64 static char                  current_driver_name[NAME_MAX_LEN] __read_mostly;static struct device_driver *current_driver                    __read_mostly; static DEFINE_RWLOCK(driver_name_lock); static const char *const maperr2str[] = {	[MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",	[MAP_ERR_NOT_CHECKED] = "dma map error not checked",	[MAP_ERR_CHECKED] = "dma map error checked",}; static const char *type2name[5] = { "single", "page",				    "scather-gather", "coherent",				    "resource" }; static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",				   "DMA_FROM_DEVICE", "DMA_NONE" }; /* * The access to some variables in this macro is racy. We can't use atomic_t * here because all these variables are exported to debugfs. Some of them even * writeable. This is also the reason why a lock won't help much. But anyway, * the races are no big deal. Here is why: * *   error_count: the addition is racy, but the worst thing that can happen is *                that we don't count some errors *   show_num_errors: the subtraction is racy. Also no big deal because in *                    worst case this will result in one warning more in the *                    system log than the user configured. This variable is *                    writeable via debugfs. */static inline void dump_entry_trace(struct dma_debug_entry *entry){#ifdef CONFIG_STACKTRACE	if (entry) {		pr_warning("Mapped at:\n");		print_stack_trace(&entry->stacktrace, 0);	}#endif} static bool driver_filter(struct device *dev){	struct device_driver *drv;	unsigned long flags;	bool ret; 	/* driver filter off */	if (likely(!current_driver_name[0]))		return true; 	/* driver filter on and initialized */	if (current_driver && dev && dev->driver == current_driver)		return true; 	/* driver filter on, but we can't filter on a NULL device... */	if (!dev)		return false; 	if (current_driver || !current_driver_name[0])		return false; 	/* driver filter on but not yet initialized */	drv = dev->driver;	if (!drv)		return false; 	/* lock to protect against change of current_driver_name */	read_lock_irqsave(&driver_name_lock, flags); 	ret = false;	if (drv->name &&	    strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {		current_driver = drv;		ret = true;	} 	read_unlock_irqrestore(&driver_name_lock, flags); 	return ret;} #define err_printk(dev, entry, format, arg...) do {			\		error_count += 1;					\		if (driver_filter(dev) &&				\		    (show_all_errors || show_num_errors > 0)) {		\			WARN(1, "%s %s: " format,			\			     dev ? dev_driver_string(dev) : "NULL",	\			     dev ? dev_name(dev) : "NULL", ## arg);	\			dump_entry_trace(entry);			\		}							\		if (!show_all_errors && show_num_errors > 0)		\			show_num_errors -= 1;				\	} while (0); /* * Hash related functions * * Every DMA-API request is saved into a struct dma_debug_entry. To * have quick access to these structs they are stored into a hash. */static int hash_fn(struct dma_debug_entry *entry){	/*	 * Hash function is based on the dma address.	 * We use bits 20-27 here as the index into the hash	 */	return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;} /* * Request exclusive access to a hash bucket for a given dma_debug_entry. */static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,					   unsigned long *flags)	__acquires(&dma_entry_hash[idx].lock){	int idx = hash_fn(entry);	unsigned long __flags; 	spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);	*flags = __flags;	return &dma_entry_hash[idx];} /* * Give up exclusive access to the hash bucket */static void put_hash_bucket(struct hash_bucket *bucket,			    unsigned long *flags)	__releases(&bucket->lock){	unsigned long __flags = *flags; 	spin_unlock_irqrestore(&bucket->lock, __flags);} static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b){	return ((a->dev_addr == b->dev_addr) &&		(a->dev == b->dev)) ? true : false;} static bool containing_match(struct dma_debug_entry *a,			     struct dma_debug_entry *b){	if (a->dev != b->dev)		return false; 	if ((b->dev_addr <= a->dev_addr) &&	    ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))		return true; 	return false;} /* * Search a given entry in the hash bucket list */static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,						  struct dma_debug_entry *ref,						  match_fn match){	struct dma_debug_entry *entry, *ret = NULL;	int matches = 0, match_lvl, last_lvl = -1; 	list_for_each_entry(entry, &bucket->list, list) {		if (!match(ref, entry))			continue; 		/*		 * Some drivers map the same physical address multiple		 * times. Without a hardware IOMMU this results in the		 * same device addresses being put into the dma-debug		 * hash multiple times too. This can result in false		 * positives being reported. Therefore we implement a		 * best-fit algorithm here which returns the entry from		 * the hash which fits best to the reference value		 * instead of the first-fit.		 */		matches += 1;		match_lvl = 0;		entry->size         == ref->size         ? ++match_lvl : 0;		entry->type         == ref->type         ? ++match_lvl : 0;		entry->direction    == ref->direction    ? ++match_lvl : 0;		entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0; 		if (match_lvl == 4) {			/* perfect-fit - return the result */			return entry;		} else if (match_lvl > last_lvl) {			/*			 * We found an entry that fits better then the			 * previous one or it is the 1st match.			 */			last_lvl = match_lvl;			ret      = entry;		}	} 	/*	 * If we have multiple matches but no perfect-fit, just return	 * NULL.	 */	ret = (matches == 1) ? ret : NULL; 	return ret;} static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,						 struct dma_debug_entry *ref){	return __hash_bucket_find(bucket, ref, exact_match);} static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,						   struct dma_debug_entry *ref,						   unsigned long *flags){ 	unsigned int max_range = dma_get_max_seg_size(ref->dev);	struct dma_debug_entry *entry, index = *ref;	unsigned int range = 0; 	while (range <= max_range) {		entry = __hash_bucket_find(*bucket, ref, containing_match); 		if (entry)			return entry; 		/*		 * Nothing found, go back a hash bucket		 */		put_hash_bucket(*bucket, flags);		range          += (1 << HASH_FN_SHIFT);		index.dev_addr -= (1 << HASH_FN_SHIFT);		*bucket = get_hash_bucket(&index, flags);	} 	return NULL;} /* * Add an entry to a hash bucket */static void hash_bucket_add(struct hash_bucket *bucket,			    struct dma_debug_entry *entry){	list_add_tail(&entry->list, &bucket->list);} /* * Remove entry from a hash bucket list */static void hash_bucket_del(struct dma_debug_entry *entry){	list_del(&entry->list);} static unsigned long long phys_addr(struct dma_debug_entry *entry){	if (entry->type == dma_debug_resource)		return __pfn_to_phys(entry->pfn) + entry->offset; 	return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;} /* * Dump mapping entries for debugging purposes */void debug_dma_dump_mappings(struct device *dev){	int idx; 	for (idx = 0; idx < HASH_SIZE; idx++) {		struct hash_bucket *bucket = &dma_entry_hash[idx];		struct dma_debug_entry *entry;		unsigned long flags; 		spin_lock_irqsave(&bucket->lock, flags); 		list_for_each_entry(entry, &bucket->list, list) {			if (!dev || dev == entry->dev) {				dev_info(entry->dev,					 "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",					 type2name[entry->type], idx,					 phys_addr(entry), entry->pfn,					 entry->dev_addr, entry->size,					 dir2name[entry->direction],					 maperr2str[entry->map_err_type]);			}		} 		spin_unlock_irqrestore(&bucket->lock, flags);	}}EXPORT_SYMBOL(debug_dma_dump_mappings); /* * For each mapping (initial cacheline in the case of * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a * scatterlist, or the cacheline specified in dma_map_single) insert * into this tree using the cacheline as the key. At * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry.  If * the entry already exists at insertion time add a tag as a reference * count for the overlapping mappings.  For now, the overlap tracking * just ensures that 'unmaps' balance 'maps' before marking the * cacheline idle, but we should also be flagging overlaps as an API * violation. * * Memory usage is mostly constrained by the maximum number of available * dma-debug entries in that we need a free dma_debug_entry before * inserting into the tree.  In the case of dma_map_page and * dma_alloc_coherent there is only one dma_debug_entry and one * dma_active_cacheline entry to track per event.  dma_map_sg(), on the * other hand, consumes a single dma_debug_entry, but inserts 'nents' * entries into the tree. * * At any time debug_dma_assert_idle() can be called to trigger a * warning if any cachelines in the given page are in the active set. */static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);static DEFINE_SPINLOCK(radix_lock);#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT) static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry){	return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +		(entry->offset >> L1_CACHE_SHIFT);} static int active_cacheline_read_overlap(phys_addr_t cln){	int overlap = 0, i; 	for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)		if (radix_tree_tag_get(&dma_active_cacheline, cln, i))			overlap |= 1 << i;	return overlap;} static int active_cacheline_set_overlap(phys_addr_t cln, int overlap){	int i; 	if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)		return overlap; 	for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)		if (overlap & 1 << i)			radix_tree_tag_set(&dma_active_cacheline, cln, i);		else			radix_tree_tag_clear(&dma_active_cacheline, cln, i);

Lines 1–500 of 1736. Load more