// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2018-2020 Christoph Hellwig. * * DMA operations that map physical memory directly without using an IOMMU. */#include <linux/memblock.h> /* for max_pfn */#include <linux/export.h>#include <linux/mm.h>#include <linux/dma-map-ops.h>#include <linux/scatterlist.h>#include <linux/pfn.h>#include <linux/vmalloc.h>#include <linux/set_memory.h>#include <linux/slab.h>#include <linux/pci-p2pdma.h>#include <linux/cc_platform.h> #include "direct.h" /* * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use * it for entirely different regions. In that case the arch code needs to * override the variable below for dma-direct to work properly. */u64 zone_dma_limit __ro_after_init = DMA_BIT_MASK(24); static inline dma_addr_t phys_to_dma_direct(struct device *dev,		phys_addr_t phys, bool unencrypted){	if (unencrypted)		return phys_to_dma_unencrypted(dev, phys);	return phys_to_dma_encrypted(dev, phys);} static inline struct page *dma_direct_to_page(struct device *dev,		dma_addr_t dma_addr){	return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr)));} u64 dma_direct_get_required_mask(struct device *dev){	bool require_decrypted = force_dma_unencrypted(dev);	phys_addr_t phys = ((phys_addr_t)max_pfn << PAGE_SHIFT) - 1;	u64 max_dma = phys_to_dma_direct(dev, phys, require_decrypted); 	return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;} static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit){	u64 dma_limit = min_not_zero(		dev->coherent_dma_mask,		dev->bus_dma_limit); 	/*	 * Optimistically try the zone that the physical address mask falls	 * into first.  If that returns memory that isn't actually addressable	 * we will fallback to the next lower zone and try again.	 *	 * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding	 * zones.	 */	*phys_limit = dma_to_phys(dev, dma_limit);	if (*phys_limit <= zone_dma_limit)		return GFP_DMA;	if (*phys_limit <= DMA_BIT_MASK(32))		return GFP_DMA32;	return 0;} bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size){	bool require_decrypted = force_dma_unencrypted(dev);	dma_addr_t dma_addr = phys_to_dma_direct(dev, phys, require_decrypted); 	if (dma_addr == DMA_MAPPING_ERROR)		return false;	return dma_addr + size - 1 <=		min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);} static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size){	int ret; 	ret = set_memory_decrypted((unsigned long)vaddr, PFN_UP(size));	if (ret)		pr_warn_ratelimited("leaking DMA memory that can't be decrypted\n");	return ret;} static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size){	int ret; 	ret = set_memory_encrypted((unsigned long)vaddr, PFN_UP(size));	if (ret)		pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");	return ret;} static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size,		unsigned long attrs){	struct page *page = swiotlb_alloc(dev, size, attrs); 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {		swiotlb_free(dev, page, size);		return NULL;	} 	return page;} static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,		gfp_t gfp, bool allow_highmem){	int node = dev_to_node(dev);	struct page *page;	u64 phys_limit; 	WARN_ON_ONCE(!PAGE_ALIGNED(size)); 	gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);	page = dma_alloc_contiguous(dev, size, gfp);	if (page) {		if (dma_coherent_ok(dev, page_to_phys(page), size) &&		    (allow_highmem || !PageHighMem(page)))			return page; 		dma_free_contiguous(dev, page, size);	} 	while ((page = alloc_pages_node(node, gfp, get_order(size)))	       && !dma_coherent_ok(dev, page_to_phys(page), size)) {		__free_pages(page, get_order(size)); 		if (IS_ENABLED(CONFIG_ZONE_DMA32) &&		    phys_limit < DMA_BIT_MASK(64) &&		    !(gfp & (GFP_DMA32 | GFP_DMA)))			gfp |= GFP_DMA32;		else if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA))			gfp = (gfp & ~GFP_DMA32) | GFP_DMA;		else			return NULL;	} 	return page;} /* * Check if a potentially blocking operations needs to dip into the atomic * pools for the given device/gfp. */static bool dma_direct_use_pool(struct device *dev, gfp_t gfp){	return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev);} static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,		dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp,		unsigned long attrs){	struct page *page;	u64 phys_limit; 	if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)))		return NULL; 	gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);	page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, attrs,				   dma_coherent_ok);	if (!page)		return NULL;	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page),					 attrs & __DMA_ATTR_ALLOC_CC_SHARED);	return page;} static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,		dma_addr_t *dma_handle, gfp_t gfp){	struct page *page; 	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);	if (!page)		return NULL; 	/* remove any dirty cache lines on the kernel alias */	if (!PageHighMem(page))		arch_dma_prep_coherent(page, size);	/*	 * return the page pointer as the opaque cookie.	 * Never used for unencrypted allocation	 */	*dma_handle = phys_to_dma_encrypted(dev, page_to_phys(page));	return page;} void *dma_direct_alloc(struct device *dev, size_t size,		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs){	bool remap = false, set_uncached = false;	bool mark_mem_decrypt = false;	bool allow_highmem = true;	struct page *page;	void *cpu_addr; 	if (force_dma_unencrypted(dev))		attrs |= __DMA_ATTR_ALLOC_CC_SHARED; 	if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) {		/*		 * Unencrypted/shared DMA requires a linear-mapped buffer		 * address to look up the PFN and set architecture-required PFN		 * attributes. This is not possible with HighMem. Avoid HighMem		 * allocation.		 */		allow_highmem = false;		mark_mem_decrypt = true;	} 	size = PAGE_ALIGN(size);	if (attrs & DMA_ATTR_NO_WARN)		gfp |= __GFP_NOWARN; 	if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==	     DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))		return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp); 	if (!dev_is_dma_coherent(dev)) {		if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) &&		    !is_swiotlb_for_alloc(dev))			return arch_dma_alloc(dev, size, dma_handle, gfp,					      attrs); 		/*		 * If there is a global pool, always allocate from it for		 * non-coherent devices.		 */		if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL))			return dma_alloc_from_global_coherent(dev, size,					dma_handle); 		/*		 * Otherwise we require the architecture to either be able to		 * mark arbitrary parts of the kernel direct mapping uncached,		 * or remapped it uncached.		 */		set_uncached = IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED);		remap = IS_ENABLED(CONFIG_DMA_DIRECT_REMAP);		if (!set_uncached && !remap) {			pr_warn_once("coherent DMA allocations not supported on this platform.\n");			return NULL;		}	} 	/*	 * Remapping or decrypting memory may block, allocate the memory from	 * the atomic pools instead if we aren't allowed block.	 * FIXME: With CONFIG_DMA_DIRECT_REMAP, the pool is also mapped as	 * DMA-coherent (non-cacheable). We may want to create a separate pool	 * dedicated to CC_SHARED atomic allocations.	 */	if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) &&	    dma_direct_use_pool(dev, gfp)) {		page = dma_direct_alloc_from_pool(dev, size,					dma_handle, &cpu_addr,					gfp, attrs);		return page ? cpu_addr : NULL;	} 	if (is_swiotlb_for_alloc(dev)) {		page = dma_direct_alloc_swiotlb(dev, size, attrs);		if (page) {			/*			 * swiotlb allocations comes from pool already marked			 * decrypted			 */			mark_mem_decrypt = false;			goto setup_page;		}		return NULL;	} 	/* we always manually zero the memory once we are done */	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_highmem);	if (!page)		return NULL; setup_page:	/*	 * dma_alloc_contiguous can return highmem pages depending on a	 * combination the cma= arguments and per-arch setup.  These need to be	 * remapped to return a kernel virtual address.	 */	if (PageHighMem(page)) {		remap = true;		set_uncached = false;	} 	if (mark_mem_decrypt) {		void *lm_addr; 		lm_addr = page_address(page);		if (set_memory_decrypted((unsigned long)lm_addr, PFN_UP(size)))			goto out_leak_pages;	} 	if (remap) {		pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs); 		/* remove any dirty cache lines on the kernel alias */		arch_dma_prep_coherent(page, size); 		/* create a coherent mapping */		cpu_addr = dma_common_contiguous_remap(page, size, prot,					__builtin_return_address(0));		if (!cpu_addr)			goto out_encrypt_pages;	} else {		cpu_addr = page_address(page);	} 	memset(cpu_addr, 0, size); 	if (set_uncached) {		void *uncached_cpu_addr; 		arch_dma_prep_coherent(page, size);		uncached_cpu_addr = arch_dma_set_uncached(cpu_addr, size);		if (IS_ERR(uncached_cpu_addr))			goto out_free_remap_pages;		cpu_addr = uncached_cpu_addr;	} 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page),					 attrs & __DMA_ATTR_ALLOC_CC_SHARED);	return cpu_addr; out_free_remap_pages:	if (remap)		dma_common_free_remap(cpu_addr, size); out_encrypt_pages:	if (mark_mem_decrypt &&	    dma_set_encrypted(dev, page_address(page), size))		goto out_leak_pages; 	if (!swiotlb_free(dev, page, size))		dma_free_contiguous(dev, page, size);	return NULL;out_leak_pages:	return NULL;} void dma_direct_free(struct device *dev, size_t size,		void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs){	phys_addr_t phys;	bool mark_mem_encrypted = false;	struct io_tlb_pool *swiotlb_pool;	unsigned int page_order = get_order(size); 	/*	 * If the allocation used decrypted/shared backing pages, restore	 * the encryption state on free.	 */	if (force_dma_unencrypted(dev))		attrs |= __DMA_ATTR_ALLOC_CC_SHARED; 	if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)		mark_mem_encrypted = true; 	if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==	     DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {		/* cpu_addr is a struct page cookie, not a kernel address */		dma_free_contiguous(dev, cpu_addr, size);		return;	} 	if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) &&	    !dev_is_dma_coherent(dev) &&	    !is_swiotlb_for_alloc(dev)) {		arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);		return;	} 	if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&	    !dev_is_dma_coherent(dev)) {		if (!dma_release_from_global_coherent(page_order, cpu_addr))			WARN_ON_ONCE(1);		return;	} 	/* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&	    dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))		return; 	phys = dma_to_phys(dev, dma_addr);	swiotlb_pool = swiotlb_find_pool(dev, phys);	if (swiotlb_pool)		/* Swiotlb doesn't need a page attribute update on free */		mark_mem_encrypted = false; 	if (is_vmalloc_addr(cpu_addr)) {		vunmap(cpu_addr);	} else {		if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))			arch_dma_clear_uncached(cpu_addr, size);	} 	if (mark_mem_encrypted) {		void *lm_addr; 		lm_addr = phys_to_virt(phys);		if (set_memory_encrypted((unsigned long)lm_addr, PFN_UP(size))) {			pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");			return;		}	} 	if (swiotlb_pool)		swiotlb_free_from_pool(dev, phys, swiotlb_pool);	else		dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);} struct page *dma_direct_alloc_pages(struct device *dev, size_t size,		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp){	unsigned long attrs = 0;	struct page *page;	void *cpu_addr; 	if (force_dma_unencrypted(dev))		attrs |= __DMA_ATTR_ALLOC_CC_SHARED; 	if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))		return dma_direct_alloc_from_pool(dev, size, dma_handle,						  &cpu_addr, gfp, attrs); 	if (is_swiotlb_for_alloc(dev)) {		page = dma_direct_alloc_swiotlb(dev, size, attrs);		if (!page)			return NULL; 		cpu_addr = page_address(page);		goto setup_page;	} 	page = __dma_direct_alloc_pages(dev, size, gfp, false);	if (!page)		return NULL; 	cpu_addr = page_address(page);	if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) &&	    dma_set_decrypted(dev, cpu_addr, size))		goto out_leak_pages;setup_page:	memset(cpu_addr, 0, size);	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page),					 attrs & __DMA_ATTR_ALLOC_CC_SHARED);	return page;out_leak_pages:	return NULL;} void dma_direct_free_pages(struct device *dev, size_t size,		struct page *page, dma_addr_t dma_addr,		enum dma_data_direction dir){	phys_addr_t phys;	void *vaddr = page_address(page);	struct io_tlb_pool *swiotlb_pool;	/*	 * if the device had requested for an unencrypted buffer,	 * convert it to encrypted on free	 */	bool mark_mem_encrypted = force_dma_unencrypted(dev); 	/* If page is not from an atomic pool, dma_free_from_pool_page() fails */	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&	    dma_free_from_pool_page(dev, page, size))		return; 	phys = page_to_phys(page);	swiotlb_pool = swiotlb_find_pool(dev, phys);	if (swiotlb_pool)		mark_mem_encrypted = false; 	if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size))		return; 	if (swiotlb_pool)		swiotlb_free_from_pool(dev, phys, swiotlb_pool);	else

Lines 1–500 of 851. Load more