active_cacheline_inc_overlap(cln, is_cache_clean);		existing = radix_tree_lookup(&dma_active_cacheline, cln);		/* A lookup failure here after we got -EEXIST is unexpected. */		WARN_ON(!existing);		if (existing)			*overlap_cache_clean =				existing->attrs &				(DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |				 DMA_ATTR_REQUIRE_COHERENT);	}	spin_unlock_irqrestore(&radix_lock, flags); 	return rc;} static void active_cacheline_remove(struct dma_debug_entry *entry){	phys_addr_t cln = to_cacheline_number(entry);	unsigned long flags; 	/* ...mirror the insert case */	if (entry->direction == DMA_TO_DEVICE)		return; 	spin_lock_irqsave(&radix_lock, flags);	/* since we are counting overlaps the final put of the	 * cacheline will occur when the overlap count is 0.	 * active_cacheline_dec_overlap() returns -1 in that case	 */	if (active_cacheline_dec_overlap(cln) < 0)		radix_tree_delete(&dma_active_cacheline, cln);	spin_unlock_irqrestore(&radix_lock, flags);} /* * Dump mappings entries on kernel space for debugging purposes */void debug_dma_dump_mappings(struct device *dev){	int idx;	phys_addr_t cln; 	for (idx = 0; idx < HASH_SIZE; idx++) {		struct hash_bucket *bucket = &dma_entry_hash[idx];		struct dma_debug_entry *entry;		unsigned long flags; 		spin_lock_irqsave(&bucket->lock, flags);		list_for_each_entry(entry, &bucket->list, list) {			if (!dev || dev == entry->dev) {				cln = to_cacheline_number(entry);				dev_info(entry->dev,					 "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n",					 type2name[entry->type], idx,					 &entry->paddr, entry->dev_addr,					 entry->size, &cln,					 dir2name[entry->direction],					 maperr2str[entry->map_err_type],					 entry->attrs);			}		}		spin_unlock_irqrestore(&bucket->lock, flags); 		cond_resched();	}} /* * Dump mappings entries on user space via debugfs */static int dump_show(struct seq_file *seq, void *v){	int idx;	phys_addr_t cln; 	for (idx = 0; idx < HASH_SIZE; idx++) {		struct hash_bucket *bucket = &dma_entry_hash[idx];		struct dma_debug_entry *entry;		unsigned long flags; 		spin_lock_irqsave(&bucket->lock, flags);		list_for_each_entry(entry, &bucket->list, list) {			cln = to_cacheline_number(entry);			seq_printf(seq,				   "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n",				   dev_driver_string(entry->dev),				   dev_name(entry->dev),				   type2name[entry->type], idx,				   &entry->paddr, entry->dev_addr,				   entry->size, &cln,				   dir2name[entry->direction],				   maperr2str[entry->map_err_type],				   entry->attrs);		}		spin_unlock_irqrestore(&bucket->lock, flags);	}	return 0;}DEFINE_SHOW_ATTRIBUTE(dump); /* * Wrapper function for adding an entry to the hash. * This function takes care of locking itself. */static void add_dma_entry(struct dma_debug_entry *entry){	unsigned long attrs = entry->attrs;	bool overlap_cache_clean;	struct hash_bucket *bucket;	unsigned long flags;	int rc; 	bucket = get_hash_bucket(entry, &flags);	hash_bucket_add(bucket, entry);	put_hash_bucket(bucket, flags); 	rc = active_cacheline_insert(entry, &overlap_cache_clean);	if (rc == -ENOMEM) {		pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n");		global_disable = true;	} else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&		   !(attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |			      DMA_ATTR_REQUIRE_COHERENT) &&		     overlap_cache_clean) &&		   dma_get_cache_alignment() >= L1_CACHE_BYTES &&		   !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) &&		     is_swiotlb_active(entry->dev))) {		err_printk(entry->dev, entry,			"cacheline tracking EEXIST, overlapping mappings aren't supported\n");	}} static int dma_debug_create_entries(gfp_t gfp){	struct dma_debug_entry *entry;	int i; 	entry = (void *)get_zeroed_page(gfp);	if (!entry)		return -ENOMEM; 	for (i = 0; i < DMA_DEBUG_DYNAMIC_ENTRIES; i++)		list_add_tail(&entry[i].list, &free_entries); 	num_free_entries += DMA_DEBUG_DYNAMIC_ENTRIES;	nr_total_entries += DMA_DEBUG_DYNAMIC_ENTRIES; 	return 0;} static struct dma_debug_entry *__dma_entry_alloc(void){	struct dma_debug_entry *entry; 	entry = list_entry(free_entries.next, struct dma_debug_entry, list);	list_del(&entry->list);	memset(entry, 0, sizeof(*entry)); 	num_free_entries -= 1;	if (num_free_entries < min_free_entries)		min_free_entries = num_free_entries; 	return entry;} /* * This should be called outside of free_entries_lock scope to avoid potential * deadlocks with serial consoles that use DMA. */static void __dma_entry_alloc_check_leak(u32 nr_entries){	u32 tmp = nr_entries % nr_prealloc_entries; 	/* Shout each time we tick over some multiple of the initial pool */	if (tmp < DMA_DEBUG_DYNAMIC_ENTRIES) {		pr_info("dma_debug_entry pool grown to %u (%u00%%)\n",			nr_entries,			(nr_entries / nr_prealloc_entries));	}} /* struct dma_entry allocator * * The next two functions implement the allocator for * struct dma_debug_entries. */static struct dma_debug_entry *dma_entry_alloc(void){	bool alloc_check_leak = false;	struct dma_debug_entry *entry;	unsigned long flags;	u32 nr_entries; 	spin_lock_irqsave(&free_entries_lock, flags);	if (num_free_entries == 0) {		if (dma_debug_create_entries(GFP_ATOMIC)) {			global_disable = true;			spin_unlock_irqrestore(&free_entries_lock, flags);			pr_err("debugging out of memory - disabling\n");			return NULL;		}		alloc_check_leak = true;		nr_entries = nr_total_entries;	} 	entry = __dma_entry_alloc(); 	spin_unlock_irqrestore(&free_entries_lock, flags); 	if (alloc_check_leak)		__dma_entry_alloc_check_leak(nr_entries); #ifdef CONFIG_STACKTRACE	entry->stack_len = stack_trace_save(entry->stack_entries,					    ARRAY_SIZE(entry->stack_entries),					    1);#endif	return entry;} static void dma_entry_free(struct dma_debug_entry *entry){	unsigned long flags; 	active_cacheline_remove(entry); 	/*	 * add to beginning of the list - this way the entries are	 * more likely cache hot when they are reallocated.	 */	spin_lock_irqsave(&free_entries_lock, flags);	list_add(&entry->list, &free_entries);	num_free_entries += 1;	spin_unlock_irqrestore(&free_entries_lock, flags);} /* * DMA-API debugging init code * * The init code does two things: *   1. Initialize core data structures *   2. Preallocate a given number of dma_debug_entry structs */ static ssize_t filter_read(struct file *file, char __user *user_buf,			   size_t count, loff_t *ppos){	char buf[NAME_MAX_LEN + 1];	unsigned long flags;	int len; 	if (!current_driver_name[0])		return 0; 	/*	 * We can't copy to userspace directly because current_driver_name can	 * only be read under the driver_name_lock with irqs disabled. So	 * create a temporary copy first.	 */	read_lock_irqsave(&driver_name_lock, flags);	len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);	read_unlock_irqrestore(&driver_name_lock, flags); 	return simple_read_from_buffer(user_buf, count, ppos, buf, len);} static ssize_t filter_write(struct file *file, const char __user *userbuf,			    size_t count, loff_t *ppos){	char buf[NAME_MAX_LEN];	unsigned long flags;	size_t len;	int i; 	/*	 * We can't copy from userspace directly. Access to	 * current_driver_name is protected with a write_lock with irqs	 * disabled. Since copy_from_user can fault and may sleep we	 * need to copy to temporary buffer first	 */	len = min(count, (size_t)(NAME_MAX_LEN - 1));	if (copy_from_user(buf, userbuf, len))		return -EFAULT; 	buf[len] = 0; 	write_lock_irqsave(&driver_name_lock, flags); 	/*	 * Now handle the string we got from userspace very carefully.	 * The rules are:	 *         - only use the first token we got	 *         - token delimiter is everything looking like a space	 *           character (' ', '\n', '\t' ...)	 *	 */	if (!isalnum(buf[0])) {		/*		 * If the first character userspace gave us is not		 * alphanumerical then assume the filter should be		 * switched off.		 */		if (current_driver_name[0])			pr_info("switching off dma-debug driver filter\n");		current_driver_name[0] = 0;		current_driver = NULL;		goto out_unlock;	} 	/*	 * Now parse out the first token and use it as the name for the	 * driver to filter for.	 */	for (i = 0; i < NAME_MAX_LEN - 1; ++i) {		current_driver_name[i] = buf[i];		if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)			break;	}	current_driver_name[i] = 0;	current_driver = NULL; 	pr_info("enable driver filter for driver [%s]\n",		current_driver_name); out_unlock:	write_unlock_irqrestore(&driver_name_lock, flags); 	return count;} static const struct file_operations filter_fops = {	.read  = filter_read,	.write = filter_write,	.llseek = default_llseek,}; static int __init dma_debug_fs_init(void){	struct dentry *dentry = debugfs_create_dir("dma-api", NULL); 	debugfs_create_bool("disabled", 0444, dentry, &global_disable);	debugfs_create_u32("error_count", 0444, dentry, &error_count);	debugfs_create_u32("all_errors", 0644, dentry, &show_all_errors);	debugfs_create_u32("num_errors", 0644, dentry, &show_num_errors);	debugfs_create_u32("num_free_entries", 0444, dentry, &num_free_entries);	debugfs_create_u32("min_free_entries", 0444, dentry, &min_free_entries);	debugfs_create_u32("nr_total_entries", 0444, dentry, &nr_total_entries);	debugfs_create_file("driver_filter", 0644, dentry, NULL, &filter_fops);	debugfs_create_file("dump", 0444, dentry, NULL, &dump_fops); 	return 0;}core_initcall_sync(dma_debug_fs_init); static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry){	struct dma_debug_entry *entry;	unsigned long flags;	int count = 0, i; 	for (i = 0; i < HASH_SIZE; ++i) {		spin_lock_irqsave(&dma_entry_hash[i].lock, flags);		list_for_each_entry(entry, &dma_entry_hash[i].list, list) {			if (entry->dev == dev) {				count += 1;				*out_entry = entry;			}		}		spin_unlock_irqrestore(&dma_entry_hash[i].lock, flags);	} 	return count;} static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data){	struct device *dev = data;	struct dma_debug_entry *entry;	int count; 	if (dma_debug_disabled())		return 0; 	switch (action) {	case BUS_NOTIFY_UNBOUND_DRIVER:		count = device_dma_allocations(dev, &entry);		if (count == 0)			break;		err_printk(dev, entry, "device driver has pending "				"DMA allocations while released from device "				"[count=%d]\n"				"One of leaked entries details: "				"[device address=0x%016llx] [size=%llu bytes] "				"[mapped with %s] [mapped as %s]\n",			count, entry->dev_addr, entry->size,			dir2name[entry->direction], type2name[entry->type]);		break;	default:		break;	} 	return 0;} void dma_debug_add_bus(const struct bus_type *bus){	struct notifier_block *nb; 	if (dma_debug_disabled())		return; 	nb = kzalloc_obj(struct notifier_block);	if (nb == NULL) {		pr_err("dma_debug_add_bus: out of memory\n");		return;	} 	nb->notifier_call = dma_debug_device_change; 	bus_register_notifier(bus, nb);} static int dma_debug_init(void){	int i, nr_pages; 	/* Do not use dma_debug_initialized here, since we really want to be	 * called to set dma_debug_initialized	 */	if (global_disable)		return 0; 	for (i = 0; i < HASH_SIZE; ++i) {		INIT_LIST_HEAD(&dma_entry_hash[i].list);		spin_lock_init(&dma_entry_hash[i].lock);	} 	nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES);	for (i = 0; i < nr_pages; ++i)		dma_debug_create_entries(GFP_KERNEL);	if (num_free_entries >= nr_prealloc_entries) {		pr_info("preallocated %d debug entries\n", nr_total_entries);	} else if (num_free_entries > 0) {		pr_warn("%d debug entries requested but only %d allocated\n",			nr_prealloc_entries, nr_total_entries);	} else {		pr_err("debugging out of memory error - disabled\n");		global_disable = true; 		return 0;	}	min_free_entries = num_free_entries; 	dma_debug_initialized = true; 	pr_info("debugging enabled by kernel config\n");	return 0;}core_initcall(dma_debug_init); static __init int dma_debug_cmdline(char *str){	if (!str)		return -EINVAL; 	if (strncmp(str, "off", 3) == 0) {		pr_info("debugging disabled on kernel command line\n");		global_disable = true;	} 	return 1;} static __init int dma_debug_entries_cmdline(char *str){	if (!str)		return -EINVAL;	if (!get_option(&str, &nr_prealloc_entries))		nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;	return 1;} __setup("dma_debug=", dma_debug_cmdline);__setup("dma_debug_entries=", dma_debug_entries_cmdline); static void check_unmap(struct dma_debug_entry *ref){	struct dma_debug_entry *entry;	struct hash_bucket *bucket;	unsigned long flags; 	bucket = get_hash_bucket(ref, &flags);	entry = bucket_find_exact(bucket, ref); 	if (!entry) {		/* must drop lock before calling dma_mapping_error */		put_hash_bucket(bucket, flags); 		if (dma_mapping_error(ref->dev, ref->dev_addr)) {			err_printk(ref->dev, NULL,

Lines 501–1000 of 1670. Previous lines Load more