// SPDX-License-Identifier: GPL-2.0-only/* * Dynamic DMA mapping support. * * This implementation is a fallback for platforms that do not support * I/O TLBs (aka DMA address translation hardware). * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com> * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com> * Copyright (C) 2000, 2003 Hewlett-Packard Co *	David Mosberger-Tang <davidm@hpl.hp.com> * * 03/05/07 davidm	Switch from PCI-DMA to generic device DMA API. * 00/12/13 davidm	Rename to swiotlb.c and add mark_clean() to avoid *			unnecessary i-cache flushing. * 04/07/.. ak		Better overflow handling. Assorted fixes. * 05/09/10 linville	Add support for syncing ranges, support syncing for *			DMA_BIDIRECTIONAL mappings, miscellaneous cleanup. * 08/12/11 beckyb	Add highmem support */ #define pr_fmt(fmt) "software IO TLB: " fmt #include <linux/cache.h>#include <linux/cc_platform.h>#include <linux/ctype.h>#include <linux/debugfs.h>#include <linux/dma-direct.h>#include <linux/dma-map-ops.h>#include <linux/export.h>#include <linux/gfp.h>#include <linux/highmem.h>#include <linux/io.h>#include <linux/kmsan-checks.h>#include <linux/iommu-helper.h>#include <linux/init.h>#include <linux/memblock.h>#include <linux/mm.h>#include <linux/pfn.h>#include <linux/rculist.h>#include <linux/scatterlist.h>#include <linux/set_memory.h>#include <linux/spinlock.h>#include <linux/string.h>#include <linux/swiotlb.h>#include <linux/types.h>#ifdef CONFIG_DMA_RESTRICTED_POOL#include <linux/of.h>#include <linux/of_fdt.h>#include <linux/of_reserved_mem.h>#include <linux/slab.h>#endif #define CREATE_TRACE_POINTS#include <trace/events/swiotlb.h> #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) /* * Minimum IO TLB size to bother booting with.  Systems with mainly * 64bit capable cards will only lightly use the swiotlb.  If we can't * allocate a contiguous 1MB, we're probably in trouble anyway. */#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) /** * struct io_tlb_slot - IO TLB slot descriptor * @orig_addr:	The original address corresponding to a mapped entry. * @alloc_size:	Size of the allocated buffer. * @list:	The free list describing the number of free entries available *		from each index. * @pad_slots:	Number of preceding padding slots. Valid only in the first *		allocated non-padding slot. */struct io_tlb_slot {	phys_addr_t orig_addr;	size_t alloc_size;	unsigned short list;	unsigned short pad_slots;}; static bool swiotlb_force_bounce;static bool swiotlb_force_disable; #ifdef CONFIG_SWIOTLB_DYNAMIC static void swiotlb_dyn_alloc(struct work_struct *work); static struct io_tlb_mem io_tlb_default_mem = {	.lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),	.pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),	.dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,					swiotlb_dyn_alloc),}; #else  /* !CONFIG_SWIOTLB_DYNAMIC */ static struct io_tlb_mem io_tlb_default_mem; #endif	/* CONFIG_SWIOTLB_DYNAMIC */ static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;static unsigned long default_nareas; /** * struct io_tlb_area - IO TLB memory area descriptor * * This is a single area with a single lock. * * @used:	The number of used IO TLB block. * @index:	The slot index to start searching in this area for next round. * @lock:	The lock to protect the above data structures in the map and *		unmap calls. */struct io_tlb_area {	unsigned long used;	unsigned int index;	spinlock_t lock;}; /* * Round up number of slabs to the next power of 2. The last area is going * be smaller than the rest if default_nslabs is not power of two. * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE, * otherwise a segment may span two or more areas. It conflicts with free * contiguous slots tracking: free slots are treated contiguous no matter * whether they cross an area boundary. * * Return true if default_nslabs is rounded up. */static bool round_up_default_nslabs(void){	if (!default_nareas)		return false; 	if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)		default_nslabs = IO_TLB_SEGSIZE * default_nareas;	else if (is_power_of_2(default_nslabs))		return false;	default_nslabs = roundup_pow_of_two(default_nslabs);	return true;} /** * swiotlb_adjust_nareas() - adjust the number of areas and slots * @nareas:	Desired number of areas. Zero is treated as 1. * * Adjust the default number of areas in a memory pool. * The default size of the memory pool may also change to meet minimum area * size requirements. */static void swiotlb_adjust_nareas(unsigned int nareas){	if (!nareas)		nareas = 1;	else if (!is_power_of_2(nareas))		nareas = roundup_pow_of_two(nareas); 	default_nareas = nareas; 	pr_info("area num %d.\n", nareas);	if (round_up_default_nslabs())		pr_info("SWIOTLB bounce buffer size roundup to %luMB",			(default_nslabs << IO_TLB_SHIFT) >> 20);} /** * limit_nareas() - get the maximum number of areas for a given memory pool size * @nareas:	Desired number of areas. * @nslots:	Total number of slots in the memory pool. * * Limit the number of areas to the maximum possible number of areas in * a memory pool of the given size. * * Return: Maximum possible number of areas. */static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots){	if (nslots < nareas * IO_TLB_SEGSIZE)		return nslots / IO_TLB_SEGSIZE;	return nareas;} #ifdef CONFIG_DEBUG_FS/* * Track the total used slots with a global atomic value in order to have * correct information to determine the high water mark. */static void inc_used_and_hiwater_real(struct io_tlb_mem *mem,				      unsigned int nslots){	unsigned long old_hiwater, new_used; 	new_used = atomic_long_add_return(nslots, &mem->total_used);	old_hiwater = atomic_long_read(&mem->used_hiwater);	do {		if (new_used <= old_hiwater)			break;	} while (!atomic_long_try_cmpxchg(&mem->used_hiwater,					  &old_hiwater, new_used));} static void dec_used_real(struct io_tlb_mem *mem, unsigned int nslots){	atomic_long_sub(nslots, &mem->total_used);} static void inc_used_and_hiwater_nop(struct io_tlb_mem *mem,				     unsigned int nslots){}static void dec_used_nop(struct io_tlb_mem *mem, unsigned int nslots){} DEFINE_STATIC_CALL(swiotlb_inc_used, inc_used_and_hiwater_nop);DEFINE_STATIC_CALL(swiotlb_dec_used, dec_used_nop); static __always_inline void inc_used_and_hiwater(struct io_tlb_mem *mem,						  unsigned int nslots){	static_call(swiotlb_inc_used)(mem, nslots);} static __always_inline void dec_used(struct io_tlb_mem *mem,				     unsigned int nslots){	static_call(swiotlb_dec_used)(mem, nslots);} static bool track_hiwater_enabled __read_mostly; #else static __always_inline void inc_used_and_hiwater(struct io_tlb_mem *mem,						  unsigned int nslots){} static __always_inline void dec_used(struct io_tlb_mem *mem,				     unsigned int nslots){}#endif /* * The tracking of used slots high watermark can be enabled * by appending "track_hiwater" to the swiotlb= boot parameter. * When disabled the tracking functions are no-ops with near-zero * overhead via static_call. */static int __initsetup_io_tlb_npages(char *str){	if (isdigit(*str)) {		/* avoid tail segment of size < IO_TLB_SEGSIZE */		default_nslabs =			ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);	}	if (*str == ',')		++str;	if (isdigit(*str))		swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));	if (*str == ',')		++str;	if (!strncmp(str, "force", 5)) {		swiotlb_force_bounce = true;		str += 5;	} else if (!strncmp(str, "noforce", 7)) {		swiotlb_force_disable = true;		str += 7;	} #ifdef CONFIG_DEBUG_FS	if (*str == ',')		++str;	if (!strncmp(str, "track_hiwater", 13)) {		track_hiwater_enabled = true;		static_call_update(swiotlb_inc_used,					inc_used_and_hiwater_real);		static_call_update(swiotlb_dec_used, dec_used_real);	}#endif 	return 0;}early_param("swiotlb", setup_io_tlb_npages); unsigned long swiotlb_size_or_default(void){	return default_nslabs << IO_TLB_SHIFT;} void __init swiotlb_adjust_size(unsigned long size){	/*	 * If swiotlb parameter has not been specified, give a chance to	 * architectures such as those supporting memory encryption to	 * adjust/expand SWIOTLB size for their use.	 */	if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)		return; 	size = ALIGN(size, IO_TLB_SIZE);	default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);	if (round_up_default_nslabs())		size = default_nslabs << IO_TLB_SHIFT;	pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);} void swiotlb_print_info(void){	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; 	if (!mem->nslabs) {		pr_warn("No low mem\n");		return;	} 	pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,	       (mem->nslabs << IO_TLB_SHIFT) >> 20);} static inline unsigned long io_tlb_offset(unsigned long val){	return val & (IO_TLB_SEGSIZE - 1);} static inline unsigned long nr_slots(u64 val){	return DIV_ROUND_UP(val, IO_TLB_SIZE);} static void swiotlb_mark_pool_used(struct io_tlb_pool *pool){	unsigned long i; 	for (i = 0; i < pool->nareas; i++) {		pool->areas[i].index = 0;		pool->areas[i].used = pool->area_nslabs;	} 	for (i = 0; i < pool->nslabs; i++) {		pool->slots[i].list = 0;		pool->slots[i].orig_addr = INVALID_PHYS_ADDR;		pool->slots[i].alloc_size = 0;		pool->slots[i].pad_slots = 0;	}} /* * Early SWIOTLB allocation may be too early to allow an architecture to * perform the desired operations.  This function allows the architecture to * call SWIOTLB when the operations are possible.  It needs to be called * before the SWIOTLB memory is used. */void __init swiotlb_update_mem_attributes(void){	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;	unsigned long bytes; 	/*	 * if platform support memory encryption, swiotlb buffers are	 * shared by default.	 */	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))		io_tlb_default_mem.cc_shared = true;	else		io_tlb_default_mem.cc_shared = false; 	if (!mem->nslabs || mem->late_alloc)		return;	bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); 	if (io_tlb_default_mem.cc_shared) {		int ret; 		ret = set_memory_decrypted((unsigned long)mem->vaddr,					   bytes >> PAGE_SHIFT);		if (ret) {			pr_warn("Failed to decrypt default memory pool, disabling it\n");			swiotlb_mark_pool_used(mem);		}	}} static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,		void *vaddr, unsigned long nslabs, bool late_alloc,		unsigned int nareas){	unsigned long bytes = nslabs << IO_TLB_SHIFT, i; 	mem->nslabs = nslabs;	mem->start = start;	mem->end = mem->start + bytes;	mem->late_alloc = late_alloc;	mem->nareas = nareas;	mem->area_nslabs = nslabs / mem->nareas; 	for (i = 0; i < mem->nareas; i++) {		spin_lock_init(&mem->areas[i].lock);		mem->areas[i].index = 0;		mem->areas[i].used = 0;	} 	for (i = 0; i < mem->nslabs; i++) {		mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),					 mem->nslabs - i);		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;		mem->slots[i].alloc_size = 0;		mem->slots[i].pad_slots = 0;	} 	memset(vaddr, 0, bytes);	mem->vaddr = vaddr;	return;} /** * add_mem_pool() - add a memory pool to the allocator * @mem:	Software IO TLB allocator. * @pool:	Memory pool to be added. */static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool){#ifdef CONFIG_SWIOTLB_DYNAMIC	spin_lock(&mem->lock);	list_add_rcu(&pool->node, &mem->pools);	mem->nslabs += pool->nslabs;	spin_unlock(&mem->lock);#else	mem->nslabs = pool->nslabs;#endif} static void __init *swiotlb_memblock_alloc(unsigned long nslabs,		unsigned int flags,		int (*remap)(void *tlb, unsigned long nslabs)){	size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);	void *tlb; 	/*	 * By default allocate the bounce buffer memory from low memory, but	 * allow to pick a location everywhere for hypervisors with guest	 * memory encryption.	 */	if (flags & SWIOTLB_ANY)		tlb = memblock_alloc(bytes, PAGE_SIZE);	else		tlb = memblock_alloc_low(bytes, PAGE_SIZE); 	if (!tlb) {		pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",			__func__, bytes);		return NULL;	} 	if (remap && remap(tlb, nslabs) < 0) {		memblock_free(tlb, PAGE_ALIGN(bytes));		pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);		return NULL;	} 	return tlb;} /* * Statically reserve bounce buffer space and initialize bounce buffer data * structures for the software IO TLB used to implement the DMA API. */void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,		int (*remap)(void *tlb, unsigned long nslabs)){	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;	unsigned long nslabs;	unsigned int nareas;	size_t alloc_size;	void *tlb; 	if (!addressing_limit && !swiotlb_force_bounce)		return;	if (swiotlb_force_disable)		return; 	io_tlb_default_mem.force_bounce = swiotlb_force_bounce; #ifdef CONFIG_SWIOTLB_DYNAMIC	if (!remap)		io_tlb_default_mem.can_grow = true;	if (flags & SWIOTLB_ANY)		io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);	else		io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;#endif 	if (!default_nareas)		swiotlb_adjust_nareas(num_possible_cpus()); 	nslabs = default_nslabs;	nareas = limit_nareas(default_nareas, nslabs);

Lines 1–500 of 2098. Load more