// SPDX-License-Identifier: GPL-2.0-only OR MIT/* * Copyright © 2024 Intel Corporation * * Authors: *     Matthew Brost <matthew.brost@intel.com> */ #include <linux/dma-mapping.h>#include <linux/export.h>#include <linux/hmm.h>#include <linux/hugetlb_inline.h>#include <linux/memremap.h>#include <linux/mm_types.h>#include <linux/slab.h> #include <drm/drm_device.h>#include <drm/drm_gpusvm.h>#include <drm/drm_pagemap.h>#include <drm/drm_print.h> /** * DOC: Overview * * GPU Shared Virtual Memory (GPU SVM) layer for the Direct Rendering Manager (DRM) * is a component of the DRM framework designed to manage shared virtual memory * between the CPU and GPU. It enables efficient data exchange and processing * for GPU-accelerated applications by allowing memory sharing and * synchronization between the CPU's and GPU's virtual address spaces. * * Key GPU SVM Components: * * - Notifiers: *	Used for tracking memory intervals and notifying the GPU of changes, *	notifiers are sized based on a GPU SVM initialization parameter, with a *	recommendation of 512M or larger. They maintain a Red-BlacK tree and a *	list of ranges that fall within the notifier interval.  Notifiers are *	tracked within a GPU SVM Red-BlacK tree and list and are dynamically *	inserted or removed as ranges within the interval are created or *	destroyed. * - Ranges: *	Represent memory ranges mapped in a DRM device and managed by GPU SVM. *	They are sized based on an array of chunk sizes, which is a GPU SVM *	initialization parameter, and the CPU address space.  Upon GPU fault, *	the largest aligned chunk that fits within the faulting CPU address *	space is chosen for the range size. Ranges are expected to be *	dynamically allocated on GPU fault and removed on an MMU notifier UNMAP *	event. As mentioned above, ranges are tracked in a notifier's Red-Black *	tree. * * - Pages: *	struct drm_gpusvm_pages holds the DMA mapping state for a range of *	CPU virtual addresses: the DMA mapped device addresses, *	the device private pagemap, the IOVA state, the per mapping *	notifier sequence number, and the drm_device that owns the DMA *	mappings. *	A driver embeds one or more struct drm_gpusvm_pages alongside its *	struct drm_gpusvm_range, choosing one of two layouts: * *	1:1 - one drm_gpusvm_pages per range (one drm_device). Simplest *	layout; to mirror a VA range on several devices a driver uses a *	separate range (and notifier) per device, so the HMM fault is taken *	once per device. * *	N:1 - one drm_gpusvm_pages per drm_device, all sharing one range and *	notifier; only the per-device DMA mapping differs. The instances must *	sit in contiguous memory so a single drm_gpusvm_range_set_unmapped() *	can mark them all. A driver can keep one instance inline for the single *	device case and switch to a heap array only when more devices join, *	e.g.: * *	.. code-block:: c * *		struct driver_range { *			struct drm_gpusvm_range base; *			unsigned int num_pages;	// 1: inline_pages, >1: pages[] *			union { *				struct drm_gpusvm_pages inline_pages; *				struct drm_gpusvm_pages *pages; *			}; *		}; * *	In the N:1 case the driver allocates the pages array with a zeroing *	allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with *	drm_gpusvm_init_pages(), and frees each entry with *	drm_gpusvm_free_pages() plus the array itself, from its range free *	callback. Each drm_gpusvm_pages is mapped independently by their own *	drm_device. *	Each drm_gpusvm_pages must be zero-initialised and initialised with *	drm_gpusvm_init_pages(), called once per entry. * * - Operations: *	Define the interface for driver-specific GPU SVM operations such as *	range allocation, notifier allocation, and invalidations. * * - Device Memory Allocations: *	Embedded structure containing enough information for GPU SVM to migrate *	to / from device memory. * * - Device Memory Operations: *	Define the interface for driver-specific device memory operations *	release memory, populate pfns, and copy to / from device memory. * * This layer provides interfaces for allocating, mapping, migrating, and * releasing memory ranges between the CPU and GPU. It handles all core memory * management interactions (DMA mapping, HMM, and migration) and provides * driver-specific virtual functions (vfuncs). This infrastructure is sufficient * to build the expected driver components for an SVM implementation as detailed * below. * * Expected Driver Components: * * - GPU page fault handler: *	Used to create ranges and notifiers based on the fault address, *	optionally migrate the range to device memory, and create GPU bindings. * * - Garbage collector: *	Used to unmap and destroy GPU bindings for ranges.  Ranges are expected *	to be added to the garbage collector upon a MMU_NOTIFY_UNMAP event in *	notifier callback. * * - Notifier callback: *	Used to invalidate and DMA unmap GPU bindings for ranges. */ /** * DOC: Locking * * GPU SVM handles locking for core MM interactions, i.e., it locks/unlocks the * mmap lock as needed. * * GPU SVM introduces a global notifier lock, which safeguards the notifier's * range RB tree and list, as well as the range's DMA mappings and sequence * number. GPU SVM manages all necessary locking and unlocking operations, * except for the recheck range's pages being valid * (drm_gpusvm_pages_valid) when the driver is committing GPU bindings. * This lock corresponds to the ``driver->update`` lock mentioned in * Documentation/mm/hmm.rst. Future revisions may transition from a GPU SVM * global lock to a per-notifier lock if finer-grained locking is deemed * necessary. * * In addition to the locking mentioned above, the driver should implement a * lock to safeguard core GPU SVM function calls that modify state, such as * drm_gpusvm_range_find_or_insert and drm_gpusvm_range_remove. This lock is * denoted as 'driver_svm_lock' in code examples. Finer grained driver side * locking should also be possible for concurrent GPU fault processing within a * single GPU SVM. The 'driver_svm_lock' can be via drm_gpusvm_driver_set_lock * to add annotations to GPU SVM. */ /** * DOC: Partial Unmapping of Ranges * * Partial unmapping of ranges (e.g., 1M out of 2M is unmapped by CPU resulting * in MMU_NOTIFY_UNMAP event) presents several challenges, with the main one * being that a subset of the range still has CPU and GPU mappings. If the * backing store for the range is in device memory, a subset of the backing * store has references. One option would be to split the range and device * memory backing store, but the implementation for this would be quite * complicated. Given that partial unmappings are rare and driver-defined range * sizes are relatively small, GPU SVM does not support splitting of ranges. * * With no support for range splitting, upon partial unmapping of a range, the * driver is expected to invalidate and destroy the entire range. If the range * has device memory as its backing, the driver is also expected to migrate any * remaining pages back to RAM. */ /** * DOC: Examples * * This section provides three examples of how to build the expected driver * components: the GPU page fault handler, the garbage collector, and the * notifier callback. * * The generic code provided does not include logic for complex migration * policies, optimized invalidations, fined grained driver locking, or other * potentially required driver locking (e.g., DMA-resv locks). * * 1) GPU page fault handler * * .. code-block:: c * *	struct driver_range { *		struct drm_gpusvm_range base; *		struct drm_gpusvm_pages pages; *	}; * *	int driver_bind_range(struct drm_gpusvm *gpusvm, struct driver_range *drange) *	{ *		int err = 0; * *		driver_alloc_and_setup_memory_for_bind(gpusvm, drange); * *		drm_gpusvm_notifier_lock(gpusvm); *		if (drm_gpusvm_pages_valid(gpusvm, &drange->pages)) *			driver_commit_bind(gpusvm, drange); *		else *			err = -EAGAIN; *		drm_gpusvm_notifier_unlock(gpusvm); * *		return err; *	} * *	int driver_gpu_fault(struct drm_gpusvm *gpusvm, unsigned long fault_addr, *			     unsigned long gpuva_start, unsigned long gpuva_end) *	{ *		struct drm_gpusvm_ctx ctx = {}; *		struct driver_range *drange; *		struct drm_gpusvm_range *range; *		int err; * *		driver_svm_lock(); *	retry: *		// Always process UNMAPs first so view of GPU SVM ranges is current *		driver_garbage_collector(gpusvm); * *		range = drm_gpusvm_range_find_or_insert(gpusvm, fault_addr, *							gpuva_start, gpuva_end, *						        &ctx); *		if (IS_ERR(range)) { *			err = PTR_ERR(range); *			goto unlock; *		} *		drange = container_of(range, struct driver_range, base); * *		if (driver_migration_policy(range)) { *			err = drm_pagemap_populate_mm(driver_choose_drm_pagemap(), *						      gpuva_start, gpuva_end, gpusvm->mm, *						      ctx->timeslice_ms); *			if (err)	// CPU mappings may have changed *				goto retry; *		} * *		err = drm_gpusvm_get_pages(gpusvm, &drange->pages, *					   gpusvm->mm, &range->notifier->notifier, *					   drm_gpusvm_range_start(range), *					   drm_gpusvm_range_end(range), &ctx); *		if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) {	// CPU mappings changed *			if (err == -EOPNOTSUPP) *				drm_gpusvm_range_evict(gpusvm, range); *			goto retry; *		} else if (err) { *			goto unlock; *		} * *		err = driver_bind_range(gpusvm, drange); *		if (err == -EAGAIN)	// CPU mappings changed *			goto retry * *	unlock: *		driver_svm_unlock(); *		return err; *	} * * 2) Garbage Collector * * .. code-block:: c * *	// The driver owns the drm_gpusvm_pages lifecycle. ops->range_free is *	// the final fallback: drm_gpusvm_free_pages() unmaps any *	// lingering DMA mapping and a no-op if already unmapped and frees the *	// dma_addr array. The normal flow is to DMA unmap before *	// drm_gpusvm_range_remove() (before the range leaves the tree). *	void driver_range_free(struct drm_gpusvm_range *range) *	{ *		struct driver_range *drange = *			container_of(range, struct driver_range, base); * *		drm_gpusvm_free_pages(range->gpusvm, &drange->pages, *				      drm_gpusvm_range_size(range) >> PAGE_SHIFT); *		kfree(drange); *	} * *	void __driver_garbage_collector(struct drm_gpusvm *gpusvm, *					struct drm_gpusvm_range *range) *	{ *		assert_driver_svm_locked(gpusvm); * *		// Partial unmap, migrate any remaining device memory pages back to RAM *		if (range->flags.partial_unmap) *			drm_gpusvm_range_evict(gpusvm, range); * *		driver_unbind_range(range); *		// The pages must be DMA unmapped before drm_gpusvm_range_remove() *		// , so a range is never off the MMU interval tree while still DMA *		// mapped as the original drmsvm design flow. Otherwise a concurrent CPU *		// munmap's notifier could miss this range and free pages still mapped *		// for device DMA. This is the normal unmap point. *		drm_gpusvm_unmap_pages(gpusvm, &drange->pages, *				       drm_gpusvm_range_size(range) >> PAGE_SHIFT, *				       &(struct drm_gpusvm_ctx){ .in_notifier = false }); *		drm_gpusvm_range_remove(gpusvm, range); *	} * *	void driver_garbage_collector(struct drm_gpusvm *gpusvm) *	{ *		assert_driver_svm_locked(gpusvm); * *		for_each_range_in_garbage_collector(gpusvm, range) *			__driver_garbage_collector(gpusvm, range); *	} * * 3) Notifier callback * * .. code-block:: c * *	void driver_invalidation(struct drm_gpusvm *gpusvm, *				 struct drm_gpusvm_notifier *notifier, *				 const struct mmu_notifier_range *mmu_range) *	{ *		struct drm_gpusvm_ctx ctx = { .in_notifier = true, }; *		struct drm_gpusvm_range *range = NULL; *		struct driver_range *drange; * *		driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end); * *		drm_gpusvm_for_each_range(range, notifier, mmu_range->start, *					  mmu_range->end) { *			drange = container_of(range, struct driver_range, base); * *			drm_gpusvm_unmap_pages(gpusvm, &drange->pages, *					       drm_gpusvm_range_size(range) >> PAGE_SHIFT, *					       &ctx); * *			if (mmu_range->event != MMU_NOTIFY_UNMAP) *				continue; * *			drm_gpusvm_range_set_unmapped(range, &drange->pages, 1, mmu_range); *			driver_garbage_collector_add(gpusvm, range); *		} *	} */ /** * npages_in_range() - Calculate the number of pages in a given range * @start: The start address of the range * @end: The end address of the range * * This macro calculates the number of pages in a given memory range, * specified by the start and end addresses. It divides the difference * between the end and start addresses by the page size (PAGE_SIZE) to * determine the number of pages in the range. * * Return: The number of pages in the specified range. */static unsigned longnpages_in_range(unsigned long start, unsigned long end){	return (end - start) >> PAGE_SHIFT;} /** * drm_gpusvm_notifier_find() - Find GPU SVM notifier from GPU SVM * @gpusvm: Pointer to the GPU SVM structure. * @start: Start address of the notifier * @end: End address of the notifier * * Return: A pointer to the drm_gpusvm_notifier if found or NULL */struct drm_gpusvm_notifier *drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,			 unsigned long end){	struct interval_tree_node *itree; 	itree = interval_tree_iter_first(&gpusvm->root, start, end - 1); 	if (itree)		return container_of(itree, struct drm_gpusvm_notifier, itree);	else		return NULL;}EXPORT_SYMBOL_GPL(drm_gpusvm_notifier_find); /** * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier * @notifier: Pointer to the GPU SVM notifier structure. * @start: Start address of the range * @end: End address of the range * * Return: A pointer to the drm_gpusvm_range if found or NULL */struct drm_gpusvm_range *drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,		      unsigned long end){	struct interval_tree_node *itree; 	itree = interval_tree_iter_first(&notifier->root, start, end - 1); 	if (itree)		return container_of(itree, struct drm_gpusvm_range, itree);	else		return NULL;}EXPORT_SYMBOL_GPL(drm_gpusvm_range_find); /** * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier. * @mni: Pointer to the mmu_interval_notifier structure. * @mmu_range: Pointer to the mmu_notifier_range structure. * @cur_seq: Current sequence number. * * This function serves as a generic MMU notifier for GPU SVM. It sets the MMU * notifier sequence number and calls the driver invalidate vfunc under * gpusvm->notifier_lock. * * Return: true if the operation succeeds, false otherwise. */static booldrm_gpusvm_notifier_invalidate(struct mmu_interval_notifier *mni,			       const struct mmu_notifier_range *mmu_range,			       unsigned long cur_seq){	struct drm_gpusvm_notifier *notifier =		container_of(mni, typeof(*notifier), notifier);	struct drm_gpusvm *gpusvm = notifier->gpusvm; 	if (!mmu_notifier_range_blockable(mmu_range))		return false; 	down_write(&gpusvm->notifier_lock);	mmu_interval_set_seq(mni, cur_seq);	gpusvm->ops->invalidate(gpusvm, notifier, mmu_range);	up_write(&gpusvm->notifier_lock); 	return true;} /* * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM */static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {	.invalidate = drm_gpusvm_notifier_invalidate,}; /** * drm_gpusvm_init() - Initialize the GPU SVM. * @gpusvm: Pointer to the GPU SVM structure. * @name: Name of the GPU SVM. * @mm: Pointer to the mm_struct for the address space. * @mm_start: Start address of GPU SVM. * @mm_range: Range of the GPU SVM. * @notifier_size: Size of individual notifiers. * @ops: Pointer to the operations structure for GPU SVM. * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation. *               Entries should be powers of 2 in descending order with last *               entry being SZ_4K. * @num_chunks: Number of chunks. * * This function initializes the GPU SVM. * * Note: If only using the simple drm_gpusvm_pages API (get/unmap/free), * then only @gpusvm and @name are expected. The @drm drm_device for dma * mappings is bound per-pages via drm_gpusvm_init_pages() before the first * drm_gpusvm_get_pages() call. However, the same base * @gpusvm can also be used with both modes together in which case the full * setup is needed, where the core drm_gpusvm_pages API will simply never use * the other fields. * * Return: 0 on success, a negative error code on failure. */int drm_gpusvm_init(struct drm_gpusvm *gpusvm,		    const char *name,		    struct mm_struct *mm,		    unsigned long mm_start, unsigned long mm_range,		    unsigned long notifier_size,		    const struct drm_gpusvm_ops *ops,		    const unsigned long *chunk_sizes, int num_chunks){	if (mm) {		if (!ops->invalidate || !num_chunks)			return -EINVAL;		mmgrab(mm);	} else {		/* No full SVM mode, only core drm_gpusvm_pages API. */		if (ops || num_chunks || mm_range || notifier_size)			return -EINVAL;	} 	gpusvm->name = name;	gpusvm->mm = mm;	gpusvm->mm_start = mm_start;	gpusvm->mm_range = mm_range;	gpusvm->notifier_size = notifier_size;	gpusvm->ops = ops;	gpusvm->chunk_sizes = chunk_sizes;	gpusvm->num_chunks = num_chunks; 	gpusvm->root = RB_ROOT_CACHED;	INIT_LIST_HEAD(&gpusvm->notifier_list); 	init_rwsem(&gpusvm->notifier_lock); 	fs_reclaim_acquire(GFP_KERNEL);	might_lock(&gpusvm->notifier_lock);	fs_reclaim_release(GFP_KERNEL); #ifdef CONFIG_LOCKDEP

Lines 1–500 of 1833. Load more