// SPDX-License-Identifier: MIT/* * Copyright © 2021 Intel Corporation */ #include "xe_device.h" #include <linux/aperture.h>#include <linux/delay.h>#include <linux/fault-inject.h>#include <linux/units.h> #include <drm/drm_client.h>#include <drm/drm_gem_ttm_helper.h>#include <drm/drm_ioctl.h>#include <drm/drm_managed.h>#include <drm/drm_pagemap_util.h>#include <drm/drm_print.h>#include <kunit/static_stub.h>#include <uapi/drm/xe_drm.h> #include "display/xe_display.h"#include "instructions/xe_gpu_commands.h"#include "regs/xe_gt_regs.h"#include "regs/xe_regs.h"#include "xe_bo.h"#include "xe_bo_evict.h"#include "xe_configfs.h"#include "xe_debugfs.h"#include "xe_defaults.h"#include "xe_devcoredump.h"#include "xe_device_sysfs.h"#include "xe_dma_buf.h"#include "xe_drm_client.h"#include "xe_drv.h"#include "xe_exec.h"#include "xe_exec_queue.h"#include "xe_force_wake.h"#include "xe_ggtt.h"#include "xe_gt.h"#include "xe_gt_mcr.h"#include "xe_gt_printk.h"#include "xe_gt_sriov_vf.h"#include "xe_guc.h"#include "xe_guc_pc.h"#include "xe_hw_engine_group.h"#include "xe_hwmon.h"#include "xe_i2c.h"#include "xe_irq.h"#include "xe_late_bind_fw.h"#include "xe_mmio.h"#include "xe_module.h"#include "xe_nvm.h"#include "xe_oa.h"#include "xe_observation.h"#include "xe_pagefault.h"#include "xe_pat.h"#include "xe_pcode.h"#include "xe_pm.h"#include "xe_pmu.h"#include "xe_psmi.h"#include "xe_pxp.h"#include "xe_query.h"#include "xe_ras.h"#include "xe_shrinker.h"#include "xe_soc_remapper.h"#include "xe_survivability_mode.h"#include "xe_sriov.h"#include "xe_svm.h"#include "xe_sysctrl.h"#include "xe_tile.h"#include "xe_ttm_stolen_mgr.h"#include "xe_ttm_sys_mgr.h"#include "xe_vm.h"#include "xe_vm_madvise.h"#include "xe_vram.h"#include "xe_vram_types.h"#include "xe_vsec.h"#include "xe_wait_user_fence.h"#include "xe_wa.h" #include <generated/xe_device_wa_oob.h>#include <generated/xe_wa_oob.h> static int xe_file_open(struct drm_device *dev, struct drm_file *file){	struct xe_device *xe = to_xe_device(dev);	struct xe_drm_client *client;	struct xe_file *xef;	int ret = -ENOMEM;	struct task_struct *task = NULL; 	xef = kzalloc_obj(*xef);	if (!xef)		return ret; 	client = xe_drm_client_alloc();	if (!client) {		kfree(xef);		return ret;	} 	xef->drm = file;	xef->client = client;	xef->xe = xe; 	mutex_init(&xef->vm.lock);	xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); 	mutex_init(&xef->exec_queue.lock);	xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); 	file->driver_priv = xef;	kref_init(&xef->refcount); 	task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);	if (task) {		xef->process_name = kstrdup(task->comm, GFP_KERNEL);		xef->pid = task->pid;		put_task_struct(task);	} 	return 0;} static void xe_file_destroy(struct kref *ref){	struct xe_file *xef = container_of(ref, struct xe_file, refcount); 	xa_destroy(&xef->exec_queue.xa);	mutex_destroy(&xef->exec_queue.lock);	xa_destroy(&xef->vm.xa);	mutex_destroy(&xef->vm.lock); 	xe_drm_client_put(xef->client);	kfree(xef->process_name);	kfree(xef);} /** * xe_file_get() - Take a reference to the xe file object * @xef: Pointer to the xe file * * Anyone with a pointer to xef must take a reference to the xe file * object using this call. * * Return: xe file pointer */struct xe_file *xe_file_get(struct xe_file *xef){	kref_get(&xef->refcount);	return xef;} /** * xe_file_put() - Drop a reference to the xe file object * @xef: Pointer to the xe file * * Used to drop reference to the xef object */void xe_file_put(struct xe_file *xef){	kref_put(&xef->refcount, xe_file_destroy);} static void xe_file_close(struct drm_device *dev, struct drm_file *file){	struct xe_device *xe = to_xe_device(dev);	struct xe_file *xef = file->driver_priv;	struct xe_vm *vm;	struct xe_exec_queue *q;	unsigned long idx; 	guard(xe_pm_runtime)(xe); 	/*	 * No need for exec_queue.lock here as there is no contention for it	 * when FD is closing as IOCTLs presumably can't be modifying the	 * xarray. Taking exec_queue.lock here causes undue dependency on	 * vm->lock taken during xe_exec_queue_kill().	 */	xa_for_each(&xef->exec_queue.xa, idx, q) {		if (q->vm && q->hwe->hw_engine_group)			xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);		xe_exec_queue_kill(q);		xe_exec_queue_put(q);	}	xa_for_each(&xef->vm.xa, idx, vm)		xe_vm_close_and_put(vm); 	xe_file_put(xef);} static const struct drm_ioctl_desc xe_ioctls[] = {	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,			  DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,			  DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,			  DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,			  DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,			  DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,			  DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,			  DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl,			  DRM_RENDER_ALLOW),}; static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg){	struct drm_file *file_priv = file->private_data;	struct xe_device *xe = to_xe_device(file_priv->minor->dev);	long ret; 	if (xe_device_wedged(xe))		return -ECANCELED; 	ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);	ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);	if (ret >= 0)		ret = drm_ioctl(file, cmd, arg); 	return ret;} #ifdef CONFIG_COMPATstatic long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg){	struct drm_file *file_priv = file->private_data;	struct xe_device *xe = to_xe_device(file_priv->minor->dev);	long ret; 	if (xe_device_wedged(xe))		return -ECANCELED; 	ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);	ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);	if (ret >= 0)		ret = drm_compat_ioctl(file, cmd, arg); 	return ret;}#else/* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */#define xe_drm_compat_ioctl NULL#endif static void barrier_open(struct vm_area_struct *vma){	drm_dev_get(vma->vm_private_data);} static void barrier_close(struct vm_area_struct *vma){	drm_dev_put(vma->vm_private_data);} static void barrier_release_dummy_page(struct drm_device *dev, void *res){	struct page *dummy_page = (struct page *)res; 	__free_page(dummy_page);} static vm_fault_t barrier_fault(struct vm_fault *vmf){	struct drm_device *dev = vmf->vma->vm_private_data;	struct vm_area_struct *vma = vmf->vma;	vm_fault_t ret = VM_FAULT_NOPAGE;	pgprot_t prot;	int idx; 	prot = vma_get_page_prot(vma); 	if (drm_dev_enter(dev, &idx)) {		unsigned long pfn; #define LAST_DB_PAGE_OFFSET 0x7ff001		pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +				LAST_DB_PAGE_OFFSET);		ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn,					  pgprot_noncached(prot));		drm_dev_exit(idx);	} else {		struct page *page; 		/* Allocate new dummy page to map all the VA range in this VMA to it*/		page = alloc_page(GFP_KERNEL | __GFP_ZERO);		if (!page)			return VM_FAULT_OOM; 		/* Set the page to be freed using drmm release action */		if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))			return VM_FAULT_OOM; 		ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page),					  prot);	} 	return ret;} static const struct vm_operations_struct vm_ops_barrier = {	.open = barrier_open,	.close = barrier_close,	.fault = barrier_fault,}; static int xe_pci_barrier_mmap(struct file *filp,			       struct vm_area_struct *vma){	struct drm_file *priv = filp->private_data;	struct drm_device *dev = priv->minor->dev;	struct xe_device *xe = to_xe_device(dev); 	if (!IS_DGFX(xe))		return -EINVAL; 	if (vma->vm_end - vma->vm_start > SZ_4K)		return -EINVAL; 	if (vma_is_cow_mapping(vma))		return -EINVAL; 	if (vma->vm_flags & (VM_READ | VM_EXEC))		return -EINVAL; 	vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);	vma->vm_ops = &vm_ops_barrier;	vma->vm_private_data = dev;	drm_dev_get(vma->vm_private_data); 	return 0;} static int xe_mmap(struct file *filp, struct vm_area_struct *vma){	struct drm_file *priv = filp->private_data;	struct drm_device *dev = priv->minor->dev; 	if (drm_dev_is_unplugged(dev))		return -ENODEV; 	switch (vma->vm_pgoff) {	case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:		return xe_pci_barrier_mmap(filp, vma);	} 	return drm_gem_mmap(filp, vma);} static const struct file_operations xe_driver_fops = {	.owner = THIS_MODULE,	.open = drm_open,	.release = drm_release_noglobal,	.unlocked_ioctl = xe_drm_ioctl,	.mmap = xe_mmap,	.poll = drm_poll,	.read = drm_read,	.compat_ioctl = xe_drm_compat_ioctl,	.llseek = noop_llseek,#ifdef CONFIG_PROC_FS	.show_fdinfo = drm_show_fdinfo,#endif	.fop_flags = FOP_UNSIGNED_OFFSET,}; /** * xe_is_xe_file() - Is the file an xe device file? * @file: The file. * * Checks whether the file is opened against * an xe device. * * Return: %true if an xe file, %false if not. */bool xe_is_xe_file(const struct file *file){	return file->f_op == &xe_driver_fops;} static const struct drm_driver regular_driver = {	.driver_features =	    XE_DISPLAY_DRIVER_FEATURES |	    DRIVER_GEM |	    DRIVER_RENDER | DRIVER_SYNCOBJ |	    DRIVER_SYNCOBJ_TIMELINE,	.open = xe_file_open,	.postclose = xe_file_close, 	.gem_prime_import = xe_gem_prime_import, 	.dumb_create = xe_bo_dumb_create,	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,#ifdef CONFIG_PROC_FS	.show_fdinfo = xe_drm_client_fdinfo,#endif	.ioctls = xe_ioctls,	.num_ioctls = ARRAY_SIZE(xe_ioctls),	.fops = &xe_driver_fops,	.name = DRIVER_NAME,	.desc = DRIVER_DESC,	.major = DRIVER_MAJOR,	.minor = DRIVER_MINOR,	.patchlevel = DRIVER_PATCHLEVEL,	XE_DISPLAY_DRIVER_OPS,}; #ifdef CONFIG_PCI_IOVstatic const struct drm_ioctl_desc xe_ioctls_admin_only[] = {	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),	DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),}; static const struct drm_driver admin_only_driver = {	.driver_features =	    DRIVER_GEM | DRIVER_RENDER,	.open = xe_file_open,	.postclose = xe_file_close,	.ioctls = xe_ioctls_admin_only,	.num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only),	.fops = &xe_driver_fops,	.name = DRIVER_NAME,	.desc = DRIVER_DESC,	.major = DRIVER_MAJOR,	.minor = DRIVER_MINOR,	.patchlevel = DRIVER_PATCHLEVEL,}; /** * xe_device_is_admin_only() - Check whether device is admin only or not. * @xe: the &xe_device to check * * Return: true if the device is admin only, false otherwise. */bool xe_device_is_admin_only(const struct xe_device *xe){	KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe);	return xe->drm.driver == &admin_only_driver;}#endif static void xe_device_destroy(struct drm_device *dev, void *dummy){	struct xe_device *xe = to_xe_device(dev); 	xe_bo_dev_fini(&xe->bo_device); 	if (xe->preempt_fence_wq)		destroy_workqueue(xe->preempt_fence_wq); 	if (xe->ordered_wq)		destroy_workqueue(xe->ordered_wq); 	if (xe->unordered_wq)		destroy_workqueue(xe->unordered_wq); 	if (xe->destroy_wq)		destroy_workqueue(xe->destroy_wq); 	ttm_device_fini(&xe->ttm);} /** * xe_device_create() - Create a new &xe_device instance * @pdev: the parent &pci_dev * * Allocate and initialize a device managed Xe device structure. * * Return: pointer to new &xe_device on success, or ERR_PTR on failure. */struct xe_device *xe_device_create(struct pci_dev *pdev){	const struct drm_driver *driver = &regular_driver;	struct xe_device *xe;	int err; #ifdef CONFIG_PCI_IOV	/*	 * Since XE device is not initialized yet, read from configfs	 * directly to decide whether we are in admin-only PF mode or not.	 */	if (xe_configfs_admin_only_pf(pdev))		driver = &admin_only_driver;#endif 	err = aperture_remove_conflicting_pci_devices(pdev, driver->name);

Lines 1–500 of 1563. Load more