// SPDX-License-Identifier: MIT/* * Copyright © 2021 Intel Corporation */ #include "xe_exec_queue.h" #include <linux/nospec.h> #include <drm/drm_device.h>#include <drm/drm_drv.h>#include <drm/drm_file.h>#include <drm/drm_syncobj.h>#include <uapi/drm/xe_drm.h> #include "xe_dep_scheduler.h"#include "xe_device.h"#include "xe_gt.h"#include "xe_gt_sriov_vf.h"#include "xe_hw_engine_class_sysfs.h"#include "xe_hw_engine_group.h"#include "xe_hw_fence.h"#include "xe_irq.h"#include "xe_lrc.h"#include "xe_macros.h"#include "xe_migrate.h"#include "xe_pm.h"#include "xe_ring_ops_types.h"#include "xe_trace.h"#include "xe_vm.h"#include "xe_pxp.h" /** * DOC: Execution Queue * * An Execution queue is an interface for the HW context of execution. * The user creates an execution queue, submits the GPU jobs through those * queues and in the end destroys them. * * Execution queues can also be created by XeKMD itself for driver internal * operations like object migration etc. * * An execution queue is associated with a specified HW engine or a group of * engines (belonging to the same tile and engine class) and any GPU job * submitted on the queue will be run on one of these engines. * * An execution queue is tied to an address space (VM). It holds a reference * of the associated VM and the underlying Logical Ring Context/s (LRC/s) * until the queue is destroyed. * * The execution queue sits on top of the submission backend. It opaquely * handles the GuC and Execlist backends whichever the platform uses, and * the ring operations the different engine classes support. */ enum xe_exec_queue_sched_prop {	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,	XE_EXEC_QUEUE_TIMESLICE = 1,	XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,	XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,}; static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,				      u64 extensions, int ext_number); static void __xe_exec_queue_free(struct xe_exec_queue *q){	int i; 	for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i)		if (q->tlb_inval[i].dep_scheduler)			xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); 	if (xe_exec_queue_uses_pxp(q))		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);	if (q->vm)		xe_vm_put(q->vm); 	if (q->xef)		xe_file_put(q->xef); 	kfree(q);} static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q){	struct xe_tile *tile = gt_to_tile(q->gt);	int i; 	for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) {		struct xe_dep_scheduler *dep_scheduler;		struct xe_gt *gt;		struct workqueue_struct *wq; 		if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT)			gt = tile->primary_gt;		else			gt = tile->media_gt; 		if (!gt)			continue; 		wq = gt->tlb_inval.job_wq; #define MAX_TLB_INVAL_JOBS	16	/* Picking a reasonable value */		dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name,							MAX_TLB_INVAL_JOBS);		if (IS_ERR(dep_scheduler))			return PTR_ERR(dep_scheduler); 		q->tlb_inval[i].dep_scheduler = dep_scheduler;	}#undef MAX_TLB_INVAL_JOBS 	return 0;} static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,						   struct xe_vm *vm,						   u32 logical_mask,						   u16 width, struct xe_hw_engine *hwe,						   u32 flags, u64 extensions){	struct xe_exec_queue *q;	struct xe_gt *gt = hwe->gt;	int err; 	/* only kernel queues can be permanent */	XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); 	q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL);	if (!q)		return ERR_PTR(-ENOMEM); 	kref_init(&q->refcount);	q->flags = flags;	q->hwe = hwe;	q->gt = gt;	q->class = hwe->class;	q->width = width;	q->msix_vec = XE_IRQ_DEFAULT_MSIX;	q->logical_mask = logical_mask;	q->fence_irq = &gt->fence_irq[hwe->class];	q->ring_ops = gt->ring_ops[hwe->class];	q->ops = gt->exec_queue_ops;	INIT_LIST_HEAD(&q->lr.link);	INIT_LIST_HEAD(&q->multi_gt_link);	INIT_LIST_HEAD(&q->hw_engine_group_link);	INIT_LIST_HEAD(&q->pxp.link); 	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;	q->sched_props.preempt_timeout_us =				hwe->eclass->sched_props.preempt_timeout_us;	q->sched_props.job_timeout_ms =				hwe->eclass->sched_props.job_timeout_ms;	if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&	    q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;	else		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; 	if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) {		err = alloc_dep_schedulers(xe, q);		if (err) {			__xe_exec_queue_free(q);			return ERR_PTR(err);		}	} 	if (vm)		q->vm = xe_vm_get(vm); 	if (extensions) {		/*		 * may set q->usm, must come before xe_lrc_create(),		 * may overwrite q->sched_props, must come before q->ops->init()		 */		err = exec_queue_user_extensions(xe, q, extensions, 0);		if (err) {			__xe_exec_queue_free(q);			return ERR_PTR(err);		}	} 	return q;} static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags){	int i, err;	u32 flags = 0; 	/*	 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no	 * other workload can use the EUs at the same time). On MTL this is done	 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there	 * is a dedicated bit for it.	 */	if (xe_exec_queue_uses_pxp(q) &&	    (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) {		if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20)			flags |= XE_LRC_CREATE_PXP;		else			flags |= XE_LRC_CREATE_RUNALONE;	} 	if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL))		flags |= XE_LRC_CREATE_USER_CTX; 	err = q->ops->init(q);	if (err)		return err; 	/*	 * This must occur after q->ops->init to avoid race conditions during VF	 * post-migration recovery, as the fixups for the LRC GGTT addresses	 * depend on the queue being present in the backend tracking structure.	 *	 * In addition to above, we must wait on inflight GGTT changes to avoid	 * writing out stale values here. Such wait provides a solid solution	 * (without a race) only if the function can detect migration instantly	 * from the moment vCPU resumes execution.	 */	for (i = 0; i < q->width; ++i) {		struct xe_lrc *lrc; 		xe_gt_sriov_vf_wait_valid_ggtt(q->gt);		lrc = xe_lrc_create(q->hwe, q->vm, xe_lrc_ring_size(),				    q->msix_vec, flags);		if (IS_ERR(lrc)) {			err = PTR_ERR(lrc);			goto err_lrc;		} 		/* Pairs with READ_ONCE to xe_exec_queue_contexts_hwsp_rebase */		WRITE_ONCE(q->lrc[i], lrc);	} 	return 0; err_lrc:	for (i = i - 1; i >= 0; --i)		xe_lrc_put(q->lrc[i]);	return err;} static void __xe_exec_queue_fini(struct xe_exec_queue *q){	int i; 	q->ops->fini(q); 	for (i = 0; i < q->width; ++i)		xe_lrc_put(q->lrc[i]);} struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,					   u32 logical_mask, u16 width,					   struct xe_hw_engine *hwe, u32 flags,					   u64 extensions){	struct xe_exec_queue *q;	int err; 	/* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */	xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); 	q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags,				  extensions);	if (IS_ERR(q))		return q; 	err = __xe_exec_queue_init(q, flags);	if (err)		goto err_post_alloc; 	/*	 * We can only add the queue to the PXP list after the init is complete,	 * because the PXP termination can call exec_queue_kill and that will	 * go bad if the queue is only half-initialized. This means that we	 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc	 * and we need to do it here instead.	 */	if (xe_exec_queue_uses_pxp(q)) {		err = xe_pxp_exec_queue_add(xe->pxp, q);		if (err)			goto err_post_init;	} 	return q; err_post_init:	__xe_exec_queue_fini(q);err_post_alloc:	__xe_exec_queue_free(q);	return ERR_PTR(err);}ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,						 struct xe_vm *vm,						 enum xe_engine_class class,						 u32 flags, u64 extensions){	struct xe_hw_engine *hwe, *hwe0 = NULL;	enum xe_hw_engine_id id;	u32 logical_mask = 0; 	for_each_hw_engine(hwe, gt, id) {		if (xe_hw_engine_is_reserved(hwe))			continue; 		if (hwe->class == class) {			logical_mask |= BIT(hwe->logical_instance);			if (!hwe0)				hwe0 = hwe;		}	} 	if (!logical_mask)		return ERR_PTR(-ENODEV); 	return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions);} /** * xe_exec_queue_create_bind() - Create bind exec queue. * @xe: Xe device. * @tile: tile which bind exec queue belongs to. * @flags: exec queue creation flags * @user_vm: The user VM which this exec queue belongs to * @extensions: exec queue creation extensions * * Normalize bind exec queue creation. Bind exec queue is tied to migration VM * for access to physical memory required for page table programming. On a * faulting devices the reserved copy engine instance must be used to avoid * deadlocking (user binds cannot get stuck behind faults as kernel binds which * resolve faults depend on user binds). On non-faulting devices any copy engine * can be used. * * Returns exec queue on success, ERR_PTR on failure */struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,						struct xe_tile *tile,						struct xe_vm *user_vm,						u32 flags, u64 extensions){	struct xe_gt *gt = tile->primary_gt;	struct xe_exec_queue *q;	struct xe_vm *migrate_vm; 	migrate_vm = xe_migrate_get_vm(tile->migrate);	if (xe->info.has_usm) {		struct xe_hw_engine *hwe = xe_gt_hw_engine(gt,							   XE_ENGINE_CLASS_COPY,							   gt->usm.reserved_bcs_instance,							   false); 		if (!hwe) {			xe_vm_put(migrate_vm);			return ERR_PTR(-EINVAL);		} 		q = xe_exec_queue_create(xe, migrate_vm,					 BIT(hwe->logical_instance), 1, hwe,					 flags, extensions);	} else {		q = xe_exec_queue_create_class(xe, gt, migrate_vm,					       XE_ENGINE_CLASS_COPY, flags,					       extensions);	}	xe_vm_put(migrate_vm); 	if (!IS_ERR(q)) {		int err = drm_syncobj_create(&q->ufence_syncobj,					     DRM_SYNCOBJ_CREATE_SIGNALED,					     NULL);		if (err) {			xe_exec_queue_put(q);			return ERR_PTR(err);		} 		if (user_vm)			q->user_vm = xe_vm_get(user_vm);	} 	return q;}ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO); void xe_exec_queue_destroy(struct kref *ref){	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);	struct xe_exec_queue *eq, *next;	int i; 	xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0); 	if (q->ufence_syncobj)		drm_syncobj_put(q->ufence_syncobj); 	if (xe_exec_queue_uses_pxp(q))		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); 	xe_exec_queue_last_fence_put_unlocked(q);	for_each_tlb_inval(i)		xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i); 	if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {		list_for_each_entry_safe(eq, next, &q->multi_gt_list,					 multi_gt_link)			xe_exec_queue_put(eq);	} 	if (q->user_vm) {		xe_vm_put(q->user_vm);		q->user_vm = NULL;	} 	q->ops->destroy(q);} void xe_exec_queue_fini(struct xe_exec_queue *q){	/*	 * Before releasing our ref to lrc and xef, accumulate our run ticks	 * and wakeup any waiters.	 */	xe_exec_queue_update_run_ticks(q);	if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal))		wake_up_var(&q->xef->exec_queue.pending_removal); 	__xe_exec_queue_fini(q);	__xe_exec_queue_free(q);} void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance){	switch (q->class) {	case XE_ENGINE_CLASS_RENDER:		snprintf(q->name, sizeof(q->name), "rcs%d", instance);		break;	case XE_ENGINE_CLASS_VIDEO_DECODE:		snprintf(q->name, sizeof(q->name), "vcs%d", instance);		break;	case XE_ENGINE_CLASS_VIDEO_ENHANCE:		snprintf(q->name, sizeof(q->name), "vecs%d", instance);		break;	case XE_ENGINE_CLASS_COPY:		snprintf(q->name, sizeof(q->name), "bcs%d", instance);		break;	case XE_ENGINE_CLASS_COMPUTE:		snprintf(q->name, sizeof(q->name), "ccs%d", instance);		break;	case XE_ENGINE_CLASS_OTHER:		snprintf(q->name, sizeof(q->name), "gsccs%d", instance);		break;	default:		XE_WARN_ON(q->class);	}} struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id){	struct xe_exec_queue *q; 	mutex_lock(&xef->exec_queue.lock);	q = xa_load(&xef->exec_queue.xa, id);	if (q)		xe_exec_queue_get(q);	mutex_unlock(&xef->exec_queue.lock); 	return q;} enum xe_exec_queue_priorityxe_exec_queue_device_get_max_priority(struct xe_device *xe){	return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH :				       XE_EXEC_QUEUE_PRIORITY_NORMAL;} static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q,				   u64 value){	if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH))		return -EINVAL; 	if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe)))		return -EPERM; 	q->sched_props.priority = value;	return 0;} static bool xe_exec_queue_enforce_schedule_limit(void){#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)	return true;#else

Lines 1–500 of 1281. Load more