1610 lines · 41.7 KB · C
RawDownload
		  ACTIVE_CACHELINE_MAX_OVERLAP, &cln);} static int active_cacheline_dec_overlap(phys_addr_t cln){	int overlap = active_cacheline_read_overlap(cln); 	return active_cacheline_set_overlap(cln, --overlap);} static int active_cacheline_insert(struct dma_debug_entry *entry){	phys_addr_t cln = to_cacheline_number(entry);	unsigned long flags;	int rc; 	/* If the device is not writing memory then we don't have any	 * concerns about the cpu consuming stale data.  This mitigates	 * legitimate usages of overlapping mappings.	 */	if (entry->direction == DMA_TO_DEVICE)		return 0; 	spin_lock_irqsave(&radix_lock, flags);	rc = radix_tree_insert(&dma_active_cacheline, cln, entry);	if (rc == -EEXIST)		active_cacheline_inc_overlap(cln);	spin_unlock_irqrestore(&radix_lock, flags); 	return rc;} static void active_cacheline_remove(struct dma_debug_entry *entry){	phys_addr_t cln = to_cacheline_number(entry);	unsigned long flags; 	/* ...mirror the insert case */	if (entry->direction == DMA_TO_DEVICE)		return; 	spin_lock_irqsave(&radix_lock, flags);	/* since we are counting overlaps the final put of the	 * cacheline will occur when the overlap count is 0.	 * active_cacheline_dec_overlap() returns -1 in that case	 */	if (active_cacheline_dec_overlap(cln) < 0)		radix_tree_delete(&dma_active_cacheline, cln);	spin_unlock_irqrestore(&radix_lock, flags);} /* * Wrapper function for adding an entry to the hash. * This function takes care of locking itself. */static void add_dma_entry(struct dma_debug_entry *entry){	struct hash_bucket *bucket;	unsigned long flags;	int rc; 	bucket = get_hash_bucket(entry, &flags);	hash_bucket_add(bucket, entry);	put_hash_bucket(bucket, flags); 	rc = active_cacheline_insert(entry);	if (rc == -ENOMEM) {		pr_err("cacheline tracking ENOMEM, dma-debug disabled\n");		global_disable = true;	} 	/* TODO: report -EEXIST errors here as overlapping mappings are	 * not supported by the DMA API	 */} static int dma_debug_create_entries(gfp_t gfp){	struct dma_debug_entry *entry;	int i; 	entry = (void *)get_zeroed_page(gfp);	if (!entry)		return -ENOMEM; 	for (i = 0; i < DMA_DEBUG_DYNAMIC_ENTRIES; i++)		list_add_tail(&entry[i].list, &free_entries); 	num_free_entries += DMA_DEBUG_DYNAMIC_ENTRIES;	nr_total_entries += DMA_DEBUG_DYNAMIC_ENTRIES; 	return 0;} static struct dma_debug_entry *__dma_entry_alloc(void){	struct dma_debug_entry *entry; 	entry = list_entry(free_entries.next, struct dma_debug_entry, list);	list_del(&entry->list);	memset(entry, 0, sizeof(*entry)); 	num_free_entries -= 1;	if (num_free_entries < min_free_entries)		min_free_entries = num_free_entries; 	return entry;} static void __dma_entry_alloc_check_leak(void){	u32 tmp = nr_total_entries % nr_prealloc_entries; 	/* Shout each time we tick over some multiple of the initial pool */	if (tmp < DMA_DEBUG_DYNAMIC_ENTRIES) {		pr_info("dma_debug_entry pool grown to %u (%u00%%)\n",			nr_total_entries,			(nr_total_entries / nr_prealloc_entries));	}} /* struct dma_entry allocator * * The next two functions implement the allocator for * struct dma_debug_entries. */static struct dma_debug_entry *dma_entry_alloc(void){	struct dma_debug_entry *entry;	unsigned long flags; 	spin_lock_irqsave(&free_entries_lock, flags);	if (num_free_entries == 0) {		if (dma_debug_create_entries(GFP_ATOMIC)) {			global_disable = true;			spin_unlock_irqrestore(&free_entries_lock, flags);			pr_err("debugging out of memory - disabling\n");			return NULL;		}		__dma_entry_alloc_check_leak();	} 	entry = __dma_entry_alloc(); 	spin_unlock_irqrestore(&free_entries_lock, flags); #ifdef CONFIG_STACKTRACE	entry->stack_len = stack_trace_save(entry->stack_entries,					    ARRAY_SIZE(entry->stack_entries),					    1);#endif	return entry;} static void dma_entry_free(struct dma_debug_entry *entry){	unsigned long flags; 	active_cacheline_remove(entry); 	/*	 * add to beginning of the list - this way the entries are	 * more likely cache hot when they are reallocated.	 */	spin_lock_irqsave(&free_entries_lock, flags);	list_add(&entry->list, &free_entries);	num_free_entries += 1;	spin_unlock_irqrestore(&free_entries_lock, flags);} /* * DMA-API debugging init code * * The init code does two things: *   1. Initialize core data structures *   2. Preallocate a given number of dma_debug_entry structs */ static ssize_t filter_read(struct file *file, char __user *user_buf,			   size_t count, loff_t *ppos){	char buf[NAME_MAX_LEN + 1];	unsigned long flags;	int len; 	if (!current_driver_name[0])		return 0; 	/*	 * We can't copy to userspace directly because current_driver_name can	 * only be read under the driver_name_lock with irqs disabled. So	 * create a temporary copy first.	 */	read_lock_irqsave(&driver_name_lock, flags);	len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);	read_unlock_irqrestore(&driver_name_lock, flags); 	return simple_read_from_buffer(user_buf, count, ppos, buf, len);} static ssize_t filter_write(struct file *file, const char __user *userbuf,			    size_t count, loff_t *ppos){	char buf[NAME_MAX_LEN];	unsigned long flags;	size_t len;	int i; 	/*	 * We can't copy from userspace directly. Access to	 * current_driver_name is protected with a write_lock with irqs	 * disabled. Since copy_from_user can fault and may sleep we	 * need to copy to temporary buffer first	 */	len = min(count, (size_t)(NAME_MAX_LEN - 1));	if (copy_from_user(buf, userbuf, len))		return -EFAULT; 	buf[len] = 0; 	write_lock_irqsave(&driver_name_lock, flags); 	/*	 * Now handle the string we got from userspace very carefully.	 * The rules are:	 *         - only use the first token we got	 *         - token delimiter is everything looking like a space	 *           character (' ', '\n', '\t' ...)	 *	 */	if (!isalnum(buf[0])) {		/*		 * If the first character userspace gave us is not		 * alphanumerical then assume the filter should be		 * switched off.		 */		if (current_driver_name[0])			pr_info("switching off dma-debug driver filter\n");		current_driver_name[0] = 0;		current_driver = NULL;		goto out_unlock;	} 	/*	 * Now parse out the first token and use it as the name for the	 * driver to filter for.	 */	for (i = 0; i < NAME_MAX_LEN - 1; ++i) {		current_driver_name[i] = buf[i];		if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)			break;	}	current_driver_name[i] = 0;	current_driver = NULL; 	pr_info("enable driver filter for driver [%s]\n",		current_driver_name); out_unlock:	write_unlock_irqrestore(&driver_name_lock, flags); 	return count;} static const struct file_operations filter_fops = {	.read  = filter_read,	.write = filter_write,	.llseek = default_llseek,}; static int dump_show(struct seq_file *seq, void *v){	int idx; 	for (idx = 0; idx < HASH_SIZE; idx++) {		struct hash_bucket *bucket = &dma_entry_hash[idx];		struct dma_debug_entry *entry;		unsigned long flags; 		spin_lock_irqsave(&bucket->lock, flags);		list_for_each_entry(entry, &bucket->list, list) {			seq_printf(seq,				   "%s %s %s idx %d P=%llx N=%lx D=%llx L=%llx %s %s\n",				   dev_name(entry->dev),				   dev_driver_string(entry->dev),				   type2name[entry->type], idx,				   phys_addr(entry), entry->pfn,				   entry->dev_addr, entry->size,				   dir2name[entry->direction],				   maperr2str[entry->map_err_type]);		}		spin_unlock_irqrestore(&bucket->lock, flags);	}	return 0;}DEFINE_SHOW_ATTRIBUTE(dump); static void dma_debug_fs_init(void){	struct dentry *dentry = debugfs_create_dir("dma-api", NULL); 	debugfs_create_bool("disabled", 0444, dentry, &global_disable);	debugfs_create_u32("error_count", 0444, dentry, &error_count);	debugfs_create_u32("all_errors", 0644, dentry, &show_all_errors);	debugfs_create_u32("num_errors", 0644, dentry, &show_num_errors);	debugfs_create_u32("num_free_entries", 0444, dentry, &num_free_entries);	debugfs_create_u32("min_free_entries", 0444, dentry, &min_free_entries);	debugfs_create_u32("nr_total_entries", 0444, dentry, &nr_total_entries);	debugfs_create_file("driver_filter", 0644, dentry, NULL, &filter_fops);	debugfs_create_file("dump", 0444, dentry, NULL, &dump_fops);} static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry){	struct dma_debug_entry *entry;	unsigned long flags;	int count = 0, i; 	for (i = 0; i < HASH_SIZE; ++i) {		spin_lock_irqsave(&dma_entry_hash[i].lock, flags);		list_for_each_entry(entry, &dma_entry_hash[i].list, list) {			if (entry->dev == dev) {				count += 1;				*out_entry = entry;			}		}		spin_unlock_irqrestore(&dma_entry_hash[i].lock, flags);	} 	return count;} static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data){	struct device *dev = data;	struct dma_debug_entry *entry;	int count; 	if (dma_debug_disabled())		return 0; 	switch (action) {	case BUS_NOTIFY_UNBOUND_DRIVER:		count = device_dma_allocations(dev, &entry);		if (count == 0)			break;		err_printk(dev, entry, "device driver has pending "				"DMA allocations while released from device "				"[count=%d]\n"				"One of leaked entries details: "				"[device address=0x%016llx] [size=%llu bytes] "				"[mapped with %s] [mapped as %s]\n",			count, entry->dev_addr, entry->size,			dir2name[entry->direction], type2name[entry->type]);		break;	default:		break;	} 	return 0;} void dma_debug_add_bus(struct bus_type *bus){	struct notifier_block *nb; 	if (dma_debug_disabled())		return; 	nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);	if (nb == NULL) {		pr_err("dma_debug_add_bus: out of memory\n");		return;	} 	nb->notifier_call = dma_debug_device_change; 	bus_register_notifier(bus, nb);} static int dma_debug_init(void){	int i, nr_pages; 	/* Do not use dma_debug_initialized here, since we really want to be	 * called to set dma_debug_initialized	 */	if (global_disable)		return 0; 	for (i = 0; i < HASH_SIZE; ++i) {		INIT_LIST_HEAD(&dma_entry_hash[i].list);		spin_lock_init(&dma_entry_hash[i].lock);	} 	dma_debug_fs_init(); 	nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES);	for (i = 0; i < nr_pages; ++i)		dma_debug_create_entries(GFP_KERNEL);	if (num_free_entries >= nr_prealloc_entries) {		pr_info("preallocated %d debug entries\n", nr_total_entries);	} else if (num_free_entries > 0) {		pr_warn("%d debug entries requested but only %d allocated\n",			nr_prealloc_entries, nr_total_entries);	} else {		pr_err("debugging out of memory error - disabled\n");		global_disable = true; 		return 0;	}	min_free_entries = num_free_entries; 	dma_debug_initialized = true; 	pr_info("debugging enabled by kernel config\n");	return 0;}core_initcall(dma_debug_init); static __init int dma_debug_cmdline(char *str){	if (!str)		return -EINVAL; 	if (strncmp(str, "off", 3) == 0) {		pr_info("debugging disabled on kernel command line\n");		global_disable = true;	} 	return 0;} static __init int dma_debug_entries_cmdline(char *str){	if (!str)		return -EINVAL;	if (!get_option(&str, &nr_prealloc_entries))		nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;	return 0;} __setup("dma_debug=", dma_debug_cmdline);__setup("dma_debug_entries=", dma_debug_entries_cmdline); static void check_unmap(struct dma_debug_entry *ref){	struct dma_debug_entry *entry;	struct hash_bucket *bucket;	unsigned long flags; 	bucket = get_hash_bucket(ref, &flags);	entry = bucket_find_exact(bucket, ref); 	if (!entry) {		/* must drop lock before calling dma_mapping_error */		put_hash_bucket(bucket, flags); 		if (dma_mapping_error(ref->dev, ref->dev_addr)) {			err_printk(ref->dev, NULL,				   "device driver tries to free an "				   "invalid DMA memory address\n");		} else {			err_printk(ref->dev, NULL,				   "device driver tries to free DMA "				   "memory it has not allocated [device "				   "address=0x%016llx] [size=%llu bytes]\n",				   ref->dev_addr, ref->size);		}		return;	} 	if (ref->size != entry->size) {		err_printk(ref->dev, entry, "device driver frees "			   "DMA memory with different size "			   "[device address=0x%016llx] [map size=%llu bytes] "			   "[unmap size=%llu bytes]\n",			   ref->dev_addr, entry->size, ref->size);	} 	if (ref->type != entry->type) {		err_printk(ref->dev, entry, "device driver frees "			   "DMA memory with wrong function "			   "[device address=0x%016llx] [size=%llu bytes] "			   "[mapped as %s] [unmapped as %s]\n",			   ref->dev_addr, ref->size,			   type2name[entry->type], type2name[ref->type]);	} else if ((entry->type == dma_debug_coherent) &&		   (phys_addr(ref) != phys_addr(entry))) {		err_printk(ref->dev, entry, "device driver frees "			   "DMA memory with different CPU address "			   "[device address=0x%016llx] [size=%llu bytes] "			   "[cpu alloc address=0x%016llx] "			   "[cpu free address=0x%016llx]",			   ref->dev_addr, ref->size,			   phys_addr(entry),			   phys_addr(ref));	} 	if (ref->sg_call_ents && ref->type == dma_debug_sg &&

Lines 501–1000 of 1610. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t