// SPDX-License-Identifier: MIT/* * Copyright © 2021 Intel Corporation */ #include "xe_exec_queue.h" #include <linux/nospec.h> #include <drm/drm_device.h>#include <drm/drm_drv.h>#include <drm/drm_file.h>#include <drm/drm_syncobj.h>#include <uapi/drm/xe_drm.h> #include "xe_bo.h"#include "xe_dep_scheduler.h"#include "xe_device.h"#include "xe_gt.h"#include "xe_gt_sriov_pf.h"#include "xe_gt_sriov_vf.h"#include "xe_hw_engine_class_sysfs.h"#include "xe_hw_engine_group.h"#include "xe_irq.h"#include "xe_lrc.h"#include "xe_macros.h"#include "xe_migrate.h"#include "xe_pm.h"#include "xe_trace.h"#include "xe_vm.h"#include "xe_pxp.h" /** * DOC: Execution Queue * * An Execution queue is an interface for the HW context of execution. * The user creates an execution queue, submits the GPU jobs through those * queues and in the end destroys them. * * Execution queues can also be created by XeKMD itself for driver internal * operations like object migration etc. * * An execution queue is associated with a specified HW engine or a group of * engines (belonging to the same tile and engine class) and any GPU job * submitted on the queue will be run on one of these engines. * * An execution queue is tied to an address space (VM). It holds a reference * of the associated VM and the underlying Logical Ring Context/s (LRC/s) * until the queue is destroyed. * * The execution queue sits on top of the submission backend. It opaquely * handles the GuC and Execlist backends whichever the platform uses, and * the ring operations the different engine classes support. */ /** * DOC: Multi Queue Group * * Multi Queue Group is another mode of execution supported by the compute * and blitter copy command streamers (CCS and BCS, respectively). It is * an enhancement of the existing hardware architecture and leverages the * same submission model. It enables support for efficient, parallel * execution of multiple queues within a single shared context. The multi * queue group functionality is only supported with GuC submission backend. * All the queues of a group must use the same address space (VM). * * The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property * supports creating a multi queue group and adding queues to a queue group. * * The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with * the queue being created as the primary queue (aka q0) of the group. To add * secondary queues to the group, they need to be created with the above * property with id of the primary queue as the value. The properties of * the primary queue (like priority, time slice) applies to the whole group. * So, these properties can't be set for secondary queues of a group. * * The hardware does not support removing a queue from a multi-queue group. * However, queues can be dynamically added to the group. A group can have * up to 64 queues. To support this, XeKMD holds references to LRCs of the * queues even after the queues are destroyed by the user until the whole * group is destroyed. The secondary queues hold a reference to the primary * queue thus preventing the group from being destroyed when user destroys * the primary queue. Once the primary queue is destroyed, secondary queues * can't be added to the queue group and new job submissions on existing * secondary queues are not allowed. * * The queues of a multi queue group can set their priority within the group * through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property. * This multi queue priority can also be set dynamically through the * XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property * supported by the secondary queues of a multi queue group, other than * DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE. * * When GuC reports an error on any of the queues of a multi queue group, * the queue cleanup mechanism is invoked for all the queues of the group * as hardware cannot make progress on the multi queue context. * * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC * interface. */ enum xe_exec_queue_sched_prop { XE_EXEC_QUEUE_JOB_TIMEOUT = 0, XE_EXEC_QUEUE_TIMESLICE = 1, XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,}; static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, u64 extensions); static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q){ struct xe_exec_queue_group *group = q->multi_queue.group; struct xe_lrc *lrc; unsigned long idx; if (xe_exec_queue_is_multi_queue_secondary(q)) { /* * Put pairs with get from xe_exec_queue_lookup() call * in xe_exec_queue_group_validate(). */ xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q)); return; } if (!group) return; /* Primary queue cleanup */ xa_for_each(&group->xa, idx, lrc) xe_lrc_put(lrc); xa_destroy(&group->xa); mutex_destroy(&group->list_lock); xe_bo_unpin_map_no_vm(group->cgp_bo); kfree(group);} static void __xe_exec_queue_free(struct xe_exec_queue *q){ int i; for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) if (q->tlb_inval[i].dep_scheduler) xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); if (xe_exec_queue_uses_pxp(q)) xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); if (xe_exec_queue_is_multi_queue(q)) xe_exec_queue_group_cleanup(q); if (q->vm) { xe_vm_remove_exec_queue(q->vm, q); xe_vm_put(q->vm); } if (q->xef) xe_file_put(q->xef); kvfree(q->replay_state); kfree(q);} static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q){ struct xe_tile *tile = gt_to_tile(q->gt); int i; for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { struct xe_dep_scheduler *dep_scheduler; struct xe_gt *gt; struct workqueue_struct *wq; if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) gt = tile->primary_gt; else gt = tile->media_gt; if (!gt) continue; wq = gt->tlb_inval.job_wq; #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, MAX_TLB_INVAL_JOBS); if (IS_ERR(dep_scheduler)) return PTR_ERR(dep_scheduler); q->tlb_inval[i].dep_scheduler = dep_scheduler; }#undef MAX_TLB_INVAL_JOBS return 0;} static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, struct xe_vm *vm, u32 logical_mask, u16 width, struct xe_hw_engine *hwe, u32 flags, u64 extensions){ struct xe_exec_queue *q; struct xe_gt *gt = hwe->gt; int err; /* only kernel queues can be permanent */ XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); q = kzalloc_flex(*q, lrc, width); if (!q) return ERR_PTR(-ENOMEM); kref_init(&q->refcount); q->flags = flags; q->hwe = hwe; q->gt = gt; q->class = hwe->class; q->width = width; q->msix_vec = XE_IRQ_DEFAULT_MSIX; q->logical_mask = logical_mask; q->fence_irq = >->fence_irq[hwe->class]; q->ring_ops = gt->ring_ops[hwe->class]; q->ops = gt->exec_queue_ops; INIT_LIST_HEAD(&q->lr.link); INIT_LIST_HEAD(&q->vm_exec_queue_link); INIT_LIST_HEAD(&q->multi_gt_link); INIT_LIST_HEAD(&q->hw_engine_group_link); INIT_LIST_HEAD(&q->pxp.link); spin_lock_init(&q->multi_queue.lock); spin_lock_init(&q->lrc_lookup_lock); q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL; q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; q->sched_props.preempt_timeout_us = hwe->eclass->sched_props.preempt_timeout_us; q->sched_props.job_timeout_ms = hwe->eclass->sched_props.job_timeout_ms; if (q->flags & EXEC_QUEUE_FLAG_KERNEL && q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; else q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { err = alloc_dep_schedulers(xe, q); if (err) { __xe_exec_queue_free(q); return ERR_PTR(err); } } if (vm) q->vm = xe_vm_get(vm); if (extensions) { /* * may set q->usm, must come before xe_lrc_create(), * may overwrite q->sched_props, must come before q->ops->init() */ err = exec_queue_user_extensions(xe, q, extensions); if (err) { __xe_exec_queue_free(q); return ERR_PTR(err); } } return q;} static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u16 idx){ xe_assert(gt_to_xe(q->gt), idx < q->width); scoped_guard(spinlock, &q->lrc_lookup_lock) { q->lrc[idx] = lrc; if (xe_exec_queue_is_multi_queue(q)) q->lrc[idx]->multi_queue.primary_lrc = q->multi_queue.group->primary->lrc[0]; }} /** * xe_exec_queue_get_lrc() - Get the LRC from exec queue. * @q: The exec queue instance. * @idx: Index within multi-LRC array. * * Retrieves LRC of given index for the exec queue under lock * and takes reference. * * Return: Pointer to LRC on success, error on failure, NULL on * lookup failure. */struct xe_lrc *xe_exec_queue_get_lrc(struct xe_exec_queue *q, u16 idx){ struct xe_lrc *lrc; xe_assert(gt_to_xe(q->gt), idx < q->width); scoped_guard(spinlock, &q->lrc_lookup_lock) { lrc = q->lrc[idx]; if (lrc) xe_lrc_get(lrc); } return lrc;} /** * xe_exec_queue_lrc() - Get the LRC from exec queue. * @q: The exec queue instance. * * Retrieves the primary LRC for the exec queue. Note that this function * returns only the first LRC instance, even when multiple parallel LRCs * are configured. This function does not increment reference count, * so the reference can be just forgotten after use. * * Return: Pointer to LRC on success, error on failure */struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q){ return q->lrc[0];} static void __xe_exec_queue_fini(struct xe_exec_queue *q){ int i; q->ops->fini(q); for (i = 0; i < q->width; ++i) xe_lrc_put(q->lrc[i]);} static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags){ int i, err; u32 flags = 0; /* * PXP workloads executing on RCS or CCS must run in isolation (i.e. no * other workload can use the EUs at the same time). On MTL this is done * by setting the RUNALONE bit in the LRC, while starting on Xe2 there * is a dedicated bit for it. */ if (xe_exec_queue_uses_pxp(q) && (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) flags |= XE_LRC_CREATE_PXP; else flags |= XE_LRC_CREATE_RUNALONE; } if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL)) flags |= XE_LRC_CREATE_USER_CTX; if (q->flags & EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX) flags |= XE_LRC_DISABLE_STATE_CACHE_PERF_FIX; err = q->ops->init(q); if (err) return err; /* * This must occur after q->ops->init to avoid race conditions during VF * post-migration recovery, as the fixups for the LRC GGTT addresses * depend on the queue being present in the backend tracking structure. * * In addition to above, we must wait on inflight GGTT changes to avoid * writing out stale values here. Such wait provides a solid solution * (without a race) only if the function can detect migration instantly * from the moment vCPU resumes execution. */ for (i = 0; i < q->width; ++i) { struct xe_lrc *__lrc = NULL; int marker; do { struct xe_lrc *lrc; marker = xe_gt_sriov_vf_wait_valid_ggtt(q->gt); lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state, xe_lrc_ring_size(), q->msix_vec, flags); if (IS_ERR(lrc)) { err = PTR_ERR(lrc); goto err_lrc; } xe_exec_queue_set_lrc(q, lrc, i); if (__lrc) xe_lrc_put(__lrc); __lrc = lrc; } while (marker != xe_vf_migration_fixups_complete_count(q->gt)); } return 0; err_lrc: __xe_exec_queue_fini(q); return err;} /** * xe_exec_queue_create() - Create an exec queue * @xe: Xe device * @vm: VM for the exec queue * @logical_mask: Logical mask of HW engines * @width: Width of the exec queue (number of LRCs) * @hwe: Hardware engine * @flags: Exec queue creation flags * @extensions: Extensions for exec queue creation * * Create an exec queue (allocate and initialize) with the specified parameters * * Return: Pointer to the created exec queue on success, ERR_PTR on failure */struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, u32 logical_mask, u16 width, struct xe_hw_engine *hwe, u32 flags, u64 extensions){ struct xe_exec_queue *q; int err; /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */ xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, extensions); if (IS_ERR(q)) return q; err = __xe_exec_queue_init(q, flags); if (err) goto err_post_alloc; /* * We can only add the queue to the PXP list after the init is complete, * because the PXP termination can call exec_queue_kill and that will * go bad if the queue is only half-initialized. This means that we * can't do it when we handle the PXP extension in __xe_exec_queue_alloc * and we need to do it here instead. */ if (xe_exec_queue_uses_pxp(q)) { err = xe_pxp_exec_queue_add(xe->pxp, q); if (err) goto err_post_init; } return q; err_post_init: __xe_exec_queue_fini(q);err_post_alloc: __xe_exec_queue_free(q); return ERR_PTR(err);}ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); /** * xe_exec_queue_create_class() - Create an exec queue for a specific engine class * @xe: Xe device * @gt: GT for the exec queue * @vm: VM for the exec queue * @class: Engine class * @flags: Exec queue creation flags * @extensions: Extensions for exec queue creation * * Create an exec queue for the specified engine class. * * Return: Pointer to the created exec queue on success, ERR_PTR on failure */struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, struct xe_vm *vm, enum xe_engine_class class, u32 flags, u64 extensions){ struct xe_hw_engine *hwe, *hwe0 = NULL; enum xe_hw_engine_id id; u32 logical_mask = 0; for_each_hw_engine(hwe, gt, id) { if (xe_hw_engine_is_reserved(hwe)) continue; if (hwe->class == class) { logical_mask |= BIT(hwe->logical_instance); if (!hwe0) hwe0 = hwe; } } if (!logical_mask) return ERR_PTR(-ENODEV);Lines 1–500 of 1864. Load more
// SPDX-License-Identifier: MIT /* * Copyright © 2021 Intel Corporation */ #include "xe_exec_queue.h" #include <linux/nospec.h> #include <drm/drm_device.h> #include <drm/drm_drv.h> #include <drm/drm_file.h> #include <drm/drm_syncobj.h> #include <uapi/drm/xe_drm.h> #include "xe_bo.h" #include "xe_dep_scheduler.h" #include "xe_device.h" #include "xe_gt.h" #include "xe_gt_sriov_pf.h" #include "xe_gt_sriov_vf.h" #include "xe_hw_engine_class_sysfs.h" #include "xe_hw_engine_group.h" #include "xe_irq.h" #include "xe_lrc.h" #include "xe_macros.h" #include "xe_migrate.h" #include "xe_pm.h" #include "xe_trace.h" #include "xe_vm.h" #include "xe_pxp.h" /** * DOC: Execution Queue * * An Execution queue is an interface for the HW context of execution. * The user creates an execution queue, submits the GPU jobs through those * queues and in the end destroys them. * * Execution queues can also be created by XeKMD itself for driver internal * operations like object migration etc. * * An execution queue is associated with a specified HW engine or a group of * engines (belonging to the same tile and engine class) and any GPU job * submitted on the queue will be run on one of these engines. * * An execution queue is tied to an address space (VM). It holds a reference * of the associated VM and the underlying Logical Ring Context/s (LRC/s) * until the queue is destroyed. * * The execution queue sits on top of the submission backend. It opaquely * handles the GuC and Execlist backends whichever the platform uses, and * the ring operations the different engine classes support. */ /** * DOC: Multi Queue Group * * Multi Queue Group is another mode of execution supported by the compute * and blitter copy command streamers (CCS and BCS, respectively). It is * an enhancement of the existing hardware architecture and leverages the * same submission model. It enables support for efficient, parallel * execution of multiple queues within a single shared context. The multi * queue group functionality is only supported with GuC submission backend. * All the queues of a group must use the same address space (VM). * * The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property * supports creating a multi queue group and adding queues to a queue group. * * The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with * the queue being created as the primary queue (aka q0) of the group. To add * secondary queues to the group, they need to be created with the above * property with id of the primary queue as the value. The properties of * the primary queue (like priority, time slice) applies to the whole group. * So, these properties can't be set for secondary queues of a group. * * The hardware does not support removing a queue from a multi-queue group. * However, queues can be dynamically added to the group. A group can have * up to 64 queues. To support this, XeKMD holds references to LRCs of the * queues even after the queues are destroyed by the user until the whole * group is destroyed. The secondary queues hold a reference to the primary * queue thus preventing the group from being destroyed when user destroys * the primary queue. Once the primary queue is destroyed, secondary queues * can't be added to the queue group and new job submissions on existing * secondary queues are not allowed. * * The queues of a multi queue group can set their priority within the group * through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property. * This multi queue priority can also be set dynamically through the * XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property * supported by the secondary queues of a multi queue group, other than * DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE. * * When GuC reports an error on any of the queues of a multi queue group, * the queue cleanup mechanism is invoked for all the queues of the group * as hardware cannot make progress on the multi queue context. * * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC * interface. */ enum xe_exec_queue_sched_prop { XE_EXEC_QUEUE_JOB_TIMEOUT = 0, XE_EXEC_QUEUE_TIMESLICE = 1, XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2, XE_EXEC_QUEUE_SCHED_PROP_MAX = 3, }; static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, u64 extensions); static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q) { struct xe_exec_queue_group *group = q->multi_queue.group; struct xe_lrc *lrc; unsigned long idx; if (xe_exec_queue_is_multi_queue_secondary(q)) { /* * Put pairs with get from xe_exec_queue_lookup() call * in xe_exec_queue_group_validate(). */ xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q)); return; } if (!group) return; /* Primary queue cleanup */ xa_for_each(&group->xa, idx, lrc) xe_lrc_put(lrc); xa_destroy(&group->xa); mutex_destroy(&group->list_lock); xe_bo_unpin_map_no_vm(group->cgp_bo); kfree(group); } static void __xe_exec_queue_free(struct xe_exec_queue *q) { int i; for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) if (q->tlb_inval[i].dep_scheduler) xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); if (xe_exec_queue_uses_pxp(q)) xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); if (xe_exec_queue_is_multi_queue(q)) xe_exec_queue_group_cleanup(q); if (q->vm) { xe_vm_remove_exec_queue(q->vm, q); xe_vm_put(q->vm); } if (q->xef) xe_file_put(q->xef); kvfree(q->replay_state); kfree(q); } static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q) { struct xe_tile *tile = gt_to_tile(q->gt); int i; for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) { struct xe_dep_scheduler *dep_scheduler; struct xe_gt *gt; struct workqueue_struct *wq; if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT) gt = tile->primary_gt; else gt = tile->media_gt; if (!gt) continue; wq = gt->tlb_inval.job_wq; #define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */ dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, MAX_TLB_INVAL_JOBS); if (IS_ERR(dep_scheduler)) return PTR_ERR(dep_scheduler); q->tlb_inval[i].dep_scheduler = dep_scheduler; } #undef MAX_TLB_INVAL_JOBS return 0; } static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe, struct xe_vm *vm, u32 logical_mask, u16 width, struct xe_hw_engine *hwe, u32 flags, u64 extensions) { struct xe_exec_queue *q; struct xe_gt *gt = hwe->gt; int err; /* only kernel queues can be permanent */ XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL)); q = kzalloc_flex(*q, lrc, width); if (!q) return ERR_PTR(-ENOMEM); kref_init(&q->refcount); q->flags = flags; q->hwe = hwe; q->gt = gt; q->class = hwe->class; q->width = width; q->msix_vec = XE_IRQ_DEFAULT_MSIX; q->logical_mask = logical_mask; q->fence_irq = >->fence_irq[hwe->class]; q->ring_ops = gt->ring_ops[hwe->class]; q->ops = gt->exec_queue_ops; INIT_LIST_HEAD(&q->lr.link); INIT_LIST_HEAD(&q->vm_exec_queue_link); INIT_LIST_HEAD(&q->multi_gt_link); INIT_LIST_HEAD(&q->hw_engine_group_link); INIT_LIST_HEAD(&q->pxp.link); spin_lock_init(&q->multi_queue.lock); spin_lock_init(&q->lrc_lookup_lock); q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL; q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; q->sched_props.preempt_timeout_us = hwe->eclass->sched_props.preempt_timeout_us; q->sched_props.job_timeout_ms = hwe->eclass->sched_props.job_timeout_ms; if (q->flags & EXEC_QUEUE_FLAG_KERNEL && q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; else q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { err = alloc_dep_schedulers(xe, q); if (err) { __xe_exec_queue_free(q); return ERR_PTR(err); } } if (vm) q->vm = xe_vm_get(vm); if (extensions) { /* * may set q->usm, must come before xe_lrc_create(), * may overwrite q->sched_props, must come before q->ops->init() */ err = exec_queue_user_extensions(xe, q, extensions); if (err) { __xe_exec_queue_free(q); return ERR_PTR(err); } } return q; } static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u16 idx) { xe_assert(gt_to_xe(q->gt), idx < q->width); scoped_guard(spinlock, &q->lrc_lookup_lock) { q->lrc[idx] = lrc; if (xe_exec_queue_is_multi_queue(q)) q->lrc[idx]->multi_queue.primary_lrc = q->multi_queue.group->primary->lrc[0]; } } /** * xe_exec_queue_get_lrc() - Get the LRC from exec queue. * @q: The exec queue instance. * @idx: Index within multi-LRC array. * * Retrieves LRC of given index for the exec queue under lock * and takes reference. * * Return: Pointer to LRC on success, error on failure, NULL on * lookup failure. */ struct xe_lrc *xe_exec_queue_get_lrc(struct xe_exec_queue *q, u16 idx) { struct xe_lrc *lrc; xe_assert(gt_to_xe(q->gt), idx < q->width); scoped_guard(spinlock, &q->lrc_lookup_lock) { lrc = q->lrc[idx]; if (lrc) xe_lrc_get(lrc); } return lrc; } /** * xe_exec_queue_lrc() - Get the LRC from exec queue. * @q: The exec queue instance. * * Retrieves the primary LRC for the exec queue. Note that this function * returns only the first LRC instance, even when multiple parallel LRCs * are configured. This function does not increment reference count, * so the reference can be just forgotten after use. * * Return: Pointer to LRC on success, error on failure */ struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q) { return q->lrc[0]; } static void __xe_exec_queue_fini(struct xe_exec_queue *q) { int i; q->ops->fini(q); for (i = 0; i < q->width; ++i) xe_lrc_put(q->lrc[i]); } static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) { int i, err; u32 flags = 0; /* * PXP workloads executing on RCS or CCS must run in isolation (i.e. no * other workload can use the EUs at the same time). On MTL this is done * by setting the RUNALONE bit in the LRC, while starting on Xe2 there * is a dedicated bit for it. */ if (xe_exec_queue_uses_pxp(q) && (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) flags |= XE_LRC_CREATE_PXP; else flags |= XE_LRC_CREATE_RUNALONE; } if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL)) flags |= XE_LRC_CREATE_USER_CTX; if (q->flags & EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX) flags |= XE_LRC_DISABLE_STATE_CACHE_PERF_FIX; err = q->ops->init(q); if (err) return err; /* * This must occur after q->ops->init to avoid race conditions during VF * post-migration recovery, as the fixups for the LRC GGTT addresses * depend on the queue being present in the backend tracking structure. * * In addition to above, we must wait on inflight GGTT changes to avoid * writing out stale values here. Such wait provides a solid solution * (without a race) only if the function can detect migration instantly * from the moment vCPU resumes execution. */ for (i = 0; i < q->width; ++i) { struct xe_lrc *__lrc = NULL; int marker; do { struct xe_lrc *lrc; marker = xe_gt_sriov_vf_wait_valid_ggtt(q->gt); lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state, xe_lrc_ring_size(), q->msix_vec, flags); if (IS_ERR(lrc)) { err = PTR_ERR(lrc); goto err_lrc; } xe_exec_queue_set_lrc(q, lrc, i); if (__lrc) xe_lrc_put(__lrc); __lrc = lrc; } while (marker != xe_vf_migration_fixups_complete_count(q->gt)); } return 0; err_lrc: __xe_exec_queue_fini(q); return err; } /** * xe_exec_queue_create() - Create an exec queue * @xe: Xe device * @vm: VM for the exec queue * @logical_mask: Logical mask of HW engines * @width: Width of the exec queue (number of LRCs) * @hwe: Hardware engine * @flags: Exec queue creation flags * @extensions: Extensions for exec queue creation * * Create an exec queue (allocate and initialize) with the specified parameters * * Return: Pointer to the created exec queue on success, ERR_PTR on failure */ struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, u32 logical_mask, u16 width, struct xe_hw_engine *hwe, u32 flags, u64 extensions) { struct xe_exec_queue *q; int err; /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */ xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))); q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags, extensions); if (IS_ERR(q)) return q; err = __xe_exec_queue_init(q, flags); if (err) goto err_post_alloc; /* * We can only add the queue to the PXP list after the init is complete, * because the PXP termination can call exec_queue_kill and that will * go bad if the queue is only half-initialized. This means that we * can't do it when we handle the PXP extension in __xe_exec_queue_alloc * and we need to do it here instead. */ if (xe_exec_queue_uses_pxp(q)) { err = xe_pxp_exec_queue_add(xe->pxp, q); if (err) goto err_post_init; } return q; err_post_init: __xe_exec_queue_fini(q); err_post_alloc: __xe_exec_queue_free(q); return ERR_PTR(err); } ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO); /** * xe_exec_queue_create_class() - Create an exec queue for a specific engine class * @xe: Xe device * @gt: GT for the exec queue * @vm: VM for the exec queue * @class: Engine class * @flags: Exec queue creation flags * @extensions: Extensions for exec queue creation * * Create an exec queue for the specified engine class. * * Return: Pointer to the created exec queue on success, ERR_PTR on failure */ struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, struct xe_vm *vm, enum xe_engine_class class, u32 flags, u64 extensions) { struct xe_hw_engine *hwe, *hwe0 = NULL; enum xe_hw_engine_id id; u32 logical_mask = 0; for_each_hw_engine(hwe, gt, id) { if (xe_hw_engine_is_reserved(hwe)) continue; if (hwe->class == class) { logical_mask |= BIT(hwe->logical_instance); if (!hwe0) hwe0 = hwe; } } if (!logical_mask) return ERR_PTR(-ENODEV);