/* SPDX-License-Identifier: MIT *//* * Copyright 2023 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #ifndef AMDGPU_USERQ_H_#define AMDGPU_USERQ_H_#include "amdgpu_eviction_fence.h" #define AMDGPU_MAX_USERQ_COUNT 512 #define to_ev_fence(f) container_of(f, struct amdgpu_eviction_fence, base)#define uq_mgr_to_fpriv(u) container_of(u, struct amdgpu_fpriv, userq_mgr)#define work_to_uq_mgr(w, name) container_of(w, struct amdgpu_userq_mgr, name) enum amdgpu_userq_state {	AMDGPU_USERQ_STATE_UNMAPPED = 0,	AMDGPU_USERQ_STATE_MAPPED,	AMDGPU_USERQ_STATE_PREEMPTED,	AMDGPU_USERQ_STATE_HUNG,	AMDGPU_USERQ_STATE_INVALID_VA,}; struct amdgpu_mqd_prop; struct amdgpu_userq_obj {	void		 *cpu_ptr;	uint64_t	 gpu_addr;	struct amdgpu_bo *obj;}; struct amdgpu_usermode_queue {	int			queue_type;	enum amdgpu_userq_state state;	uint64_t		doorbell_handle;	uint64_t		doorbell_index;	u32			doorbell_offset;	uint64_t		flags;	struct amdgpu_mqd_prop	*userq_prop;	struct amdgpu_userq_mgr *userq_mgr;	struct amdgpu_vm	*vm;	struct amdgpu_userq_obj mqd;	struct amdgpu_userq_obj	db_obj;	struct amdgpu_userq_obj fw_obj;	struct amdgpu_userq_obj wptr_obj; 	/**	 * @fence_drv_lock: Protecting @fence_drv_xa.	 */	struct mutex		fence_drv_lock; 	/**	 * @fence_drv_xa:	 *	 * References to the external fence drivers returned by wait_ioctl.	 * Dropped on the next signaled dma_fence or queue destruction.	 */	struct xarray		fence_drv_xa;	struct amdgpu_userq_fence_driver *fence_drv;	struct dma_fence	*last_fence;	u32			xcp_id;	int			priority;	struct dentry		*debugfs_queue; 	/**	 * @hang_detect_work:	 *	 * Delayed work which runs when userq_fences time out.	 */	struct delayed_work	hang_detect_work;	struct kref		refcount; 	union {		struct {			u64 queue_rb;			u64 wptr;			u64 rptr;			u64 eop;			u64 shadow;			u64 csa;		} va;		u64 va_array[6];	} userq_vas; 	uint32_t	gang_ctx_array_index;}; struct amdgpu_userq_funcs {	int (*mqd_create)(struct amdgpu_usermode_queue *queue,			  struct drm_amdgpu_userq_in *args);	int (*mqd_update)(struct amdgpu_usermode_queue *queue,			  struct drm_amdgpu_userq_in *args);	void (*mqd_destroy)(struct amdgpu_usermode_queue *uq);	int (*unmap)(struct amdgpu_usermode_queue *queue);	int (*map)(struct amdgpu_usermode_queue *queue);	int (*preempt)(struct amdgpu_usermode_queue *queue);	int (*restore)(struct amdgpu_usermode_queue *queue);	int (*reset)(struct amdgpu_usermode_queue *queue);}; /* Usermode queues for gfx */struct amdgpu_userq_mgr {	/**	 * @userq_xa: Per-process user queue map (queue ID → queue)	 * Key: queue_id (unique ID within the process's userq manager)	 * Value: struct amdgpu_usermode_queue	 */	struct xarray			userq_xa;	struct mutex			userq_mutex;	struct amdgpu_device		*adev;	struct delayed_work		resume_work;	struct drm_file			*file;	struct mutex			proc_ctx_lock;	struct amdgpu_userq_obj		proc_ctx_obj; 	bool				proc_ctx_allocated;	uint32_t			proc_ctx_array_index;	/**	 * @reset_work:	 *	 * Reset work which is used when eviction fails.	 */	struct work_struct		reset_work;	atomic_t                        userq_count[AMDGPU_RING_TYPE_MAX];}; struct amdgpu_db_info {	uint64_t doorbell_handle;	uint32_t queue_type;	uint32_t doorbell_offset;	struct amdgpu_userq_obj	*db_obj;}; struct amdgpu_usermode_queue *amdgpu_userq_get(struct amdgpu_userq_mgr *uq_mgr, u32 qid);void amdgpu_userq_put(struct amdgpu_usermode_queue *queue); int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,			  struct amdgpu_device *adev); void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr); void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr); void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr,				  struct amdgpu_eviction_fence_mgr *evf_mgr); u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev);bool amdgpu_userq_enabled(struct drm_device *dev); int amdgpu_userq_suspend(struct amdgpu_device *adev);int amdgpu_userq_resume(struct amdgpu_device *adev); int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,						  u32 idx);int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev,						   u32 idx);void amdgpu_userq_reset_work(struct work_struct *work);void amdgpu_userq_pre_reset(struct amdgpu_device *adev);int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell); /* * CP packs the per-process doorbell_id of the queue in * CTXID0[9:0] on priv-fault (same encoding KFD uses via * KFD_CTXID0_DOORBELL_ID_MASK) */#define AMDGPU_CTXID0_DOORBELL_ID_MASK	0x3ff void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev,				    u32 pasid, u32 doorbell_offset); int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,				   struct amdgpu_usermode_queue *queue,				   u64 addr, u64 expected_size, u64 *va_out); void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,					struct amdgpu_bo_va_mapping *mapping);#endif