Merge tag 'drm-xe-next-2025-12-19' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-next

[airlied: fix guc submit double definition] UAPI Changes: - Multi-Queue support (Niranjana) - Add DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE (Brost) - Add NO_COMPRESSION BO flag and query capability (Sanjay) - Add gt_id to struct drm_xe_oa_unit (Ashutosh) - Expose MERT OA unit (Ashutosh) - Sysfs Survivability refactor (Riana) Cross-subsystem Changes: - VFIO: Add device specific vfio_pci driver variant for Intel graphics (Winiarski) Driver Changes: - MAINTAINERS update (Lucas -> Matt) - Add helper to query compression enable status (Xin) - Xe_VM fixes and updates (Shuicheng, Himal) - Documentation fixes (Winiarski, Swaraj, Niranjana) - Kunit fix (Roper) - Fix potential leaks, uaf, null derref, and oversized allocations (Shuicheng, Sanjay, Mika, Tapani) - Other minor fixes like kbuild duplication and sysfs_emit (Shuicheng, Madhur) - Handle msix vector0 interrupt (Venkata) - Scope-based forcewake and runtime PM (Roper, Raag) - GuC/HuC related fixes and refactors (Lucas, Zhanjun, Brost, Julia, Wajdeczko) - Fix conversion from clock ticks to milliseconds (Harish) - SRIOV PF PF: Add support for MERT (Lukasz) - Enable SR-IOV VF migration and other SRIOV updates (Winiarski, Satya, Brost, Wajdeczko, Piotr, Tomasz, Daniele) - Optimize runtime suspend/resume and other PM improvements (Raag) - Some W/a additions and updates (Bala, Harish, Roper) - Use for_each_tlb_inval() to calculate invalidation fences (Roper) - Fix VFIO link error (Arnd) - Fix ix drm_gpusvm_init() arguments (Arnd) - Other OA refactor (Ashutosh) - Refactor PAT and expose debugfs (Xin) - Enable Indirect Ring State for xe3p_xpc (Niranjana) - MEI interrupt fix (Junxiao) - Add stats for mode switching on hw_engine_group (Francois) - DMA-Buf related changes (Thomas) - Multi Queue feature support (Niranjana) - Enable I2C controller for Crescent Island (Raag) - Enable NVM for Crescent Island (Sasha) - Increase TDF timeout (Jagmeet) - Restore engine registers before restarting schedulers after GT reset (Jan) - Page Reclamation Support for Xe3p Platforms (Brian, Brost, Oak) - Fix performance when pagefaults and 3d/display share resources (Brost) - More OA MERT work (Ashutosh) - Fix return values (Dan) - Some log level and messages improvements (Brost) Signed-off-by: Dave Airlie <airlied@redhat.com>

Dave Airlie committed 10 months ago

Commit 9ec3c8e2 parents c5fb82d 844758b152 files · +5329 −1498Browse files

Unified · Split

152 files

.mailmap +1 −0
Documentation/ABI/testing/sysfs-driver-intel-xe-sriov +1 −1
Documentation/gpu/xe/xe_exec_queue.rst +14 −0
MAINTAINERS +1 −1
drivers/gpu/drm/drm_gpusvm.c +3 −0
drivers/gpu/drm/xe/Makefile +2 −0
drivers/gpu/drm/xe/abi/guc_actions_abi.h +6 −0
drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h +57 −10
drivers/gpu/drm/xe/abi/guc_klvs_abi.h +9 −0
drivers/gpu/drm/xe/abi/guc_lfd_abi.h +171 −0
drivers/gpu/drm/xe/abi/guc_lic_abi.h +77 −0
drivers/gpu/drm/xe/abi/guc_log_abi.h +38 −4
drivers/gpu/drm/xe/display/xe_fb_pin.c +9 −14
drivers/gpu/drm/xe/display/xe_hdcp_gsc.c +9 −22
drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +1 −0
drivers/gpu/drm/xe/regs/xe_gt_regs.h +3 −0
drivers/gpu/drm/xe/regs/xe_gtt_defs.h +1 −0
drivers/gpu/drm/xe/regs/xe_guc_regs.h +3 −0
drivers/gpu/drm/xe/regs/xe_irq_regs.h +1 −0
drivers/gpu/drm/xe/regs/xe_mert_regs.h +21 −0
drivers/gpu/drm/xe/regs/xe_oa_regs.h +17 −0
drivers/gpu/drm/xe/tests/xe_args_test.c +54 −0
drivers/gpu/drm/xe/tests/xe_bo.c +2 −8
drivers/gpu/drm/xe/tests/xe_dma_buf.c +1 −2
drivers/gpu/drm/xe/tests/xe_migrate.c +2 −8
drivers/gpu/drm/xe/tests/xe_mocs.c +8 −19
drivers/gpu/drm/xe/xe_args.h +27 −0
drivers/gpu/drm/xe/xe_bo.c +16 −10
drivers/gpu/drm/xe/xe_bo.h +1 −0
drivers/gpu/drm/xe/xe_debugfs.c +121 −22
drivers/gpu/drm/xe/xe_devcoredump.c +12 −18
drivers/gpu/drm/xe/xe_device.c +77 −23
drivers/gpu/drm/xe/xe_device.h +5 −0
drivers/gpu/drm/xe/xe_device_sysfs.c +13 −20
drivers/gpu/drm/xe/xe_device_types.h +27 −0
drivers/gpu/drm/xe/xe_drm_client.c +31 −36
drivers/gpu/drm/xe/xe_exec.c +7 −2
drivers/gpu/drm/xe/xe_exec_queue.c +441 −14
drivers/gpu/drm/xe/xe_exec_queue.h +68 −0
drivers/gpu/drm/xe/xe_exec_queue_types.h +62 −0
drivers/gpu/drm/xe/xe_execlist.c +1 −1
drivers/gpu/drm/xe/xe_force_wake.c +7 −0
drivers/gpu/drm/xe/xe_force_wake.h +40 −0
drivers/gpu/drm/xe/xe_ggtt.c +1 −2
drivers/gpu/drm/xe/xe_gsc.c +6 −15
drivers/gpu/drm/xe/xe_gsc_debugfs.c +1 −2
drivers/gpu/drm/xe/xe_gsc_proxy.c +7 −10
drivers/gpu/drm/xe/xe_gt.c +87 −87
drivers/gpu/drm/xe/xe_gt.h +2 −0
drivers/gpu/drm/xe/xe_gt_debugfs.c +9 −21
drivers/gpu/drm/xe/xe_gt_freq.c +9 −18
drivers/gpu/drm/xe/xe_gt_idle.c +12 −29
drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +12 −7
drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c +10 −11
drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c +1 −1
drivers/gpu/drm/xe/xe_gt_sriov_vf.c +126 −44
drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c +12 −0
drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h +13 −0
drivers/gpu/drm/xe/xe_gt_stats.c +10 −0
drivers/gpu/drm/xe/xe_gt_stats.h +32 −0
drivers/gpu/drm/xe/xe_gt_stats_types.h +5 −0
drivers/gpu/drm/xe/xe_gt_throttle.c +3 −6
drivers/gpu/drm/xe/xe_gt_types.h +5 −0
drivers/gpu/drm/xe/xe_guc.c +60 −20
drivers/gpu/drm/xe/xe_guc.h +23 −0
drivers/gpu/drm/xe/xe_guc_ads.c +3 −3
drivers/gpu/drm/xe/xe_guc_buf.c +1 −1
drivers/gpu/drm/xe/xe_guc_capture.c +8 −8
drivers/gpu/drm/xe/xe_guc_ct.c +170 −103
drivers/gpu/drm/xe/xe_guc_ct.h +2 −0
drivers/gpu/drm/xe/xe_guc_debugfs.c +9 −6
drivers/gpu/drm/xe/xe_guc_fwif.h +7 −3
drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h +6 −0
drivers/gpu/drm/xe/xe_guc_log.c +406 −101
drivers/gpu/drm/xe/xe_guc_log.h +20 −10
drivers/gpu/drm/xe/xe_guc_pc.c +19 −47
drivers/gpu/drm/xe/xe_guc_submit.c +637 −68
drivers/gpu/drm/xe/xe_guc_submit.h +7 −2
drivers/gpu/drm/xe/xe_guc_submit_types.h +13 −0
drivers/gpu/drm/xe/xe_guc_tlb_inval.c +31 −10
drivers/gpu/drm/xe/xe_huc.c +2 −5
drivers/gpu/drm/xe/xe_huc_debugfs.c +1 −2
drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c +6 −10
drivers/gpu/drm/xe/xe_hw_engine_group.c +71 −8
drivers/gpu/drm/xe/xe_hw_engine_group.h +3 −1
drivers/gpu/drm/xe/xe_hwmon.c +14 −38
drivers/gpu/drm/xe/xe_i2c.c +1 −1
drivers/gpu/drm/xe/xe_irq.c +2 −0
drivers/gpu/drm/xe/xe_lmtt.c +22 −3
drivers/gpu/drm/xe/xe_lrc.c +63 −10
drivers/gpu/drm/xe/xe_lrc.h +6 −1
drivers/gpu/drm/xe/xe_lrc_types.h +3 −0
drivers/gpu/drm/xe/xe_mert.c +82 −0
drivers/gpu/drm/xe/xe_mert.h +32 −0
drivers/gpu/drm/xe/xe_migrate.c +54 −3
drivers/gpu/drm/xe/xe_migrate.h +3 −0
drivers/gpu/drm/xe/xe_mocs.c +6 −12
drivers/gpu/drm/xe/xe_nvm.c +23 −11
drivers/gpu/drm/xe/xe_oa.c +65 −29
drivers/gpu/drm/xe/xe_oa_types.h +1 −0
drivers/gpu/drm/xe/xe_page_reclaim.c +136 −0
drivers/gpu/drm/xe/xe_page_reclaim.h +105 −0
drivers/gpu/drm/xe/xe_pagefault.c +18 −18
drivers/gpu/drm/xe/xe_pat.c +152 −68
drivers/gpu/drm/xe/xe_pat.h +21 −0
drivers/gpu/drm/xe/xe_pci.c +29 −0
drivers/gpu/drm/xe/xe_pci_sriov.c +3 −7
drivers/gpu/drm/xe/xe_pci_types.h +6 −0
drivers/gpu/drm/xe/xe_pcode_api.h +2 −0
drivers/gpu/drm/xe/xe_pm.c +5 −5
drivers/gpu/drm/xe/xe_pmu.c +1 −1
drivers/gpu/drm/xe/xe_pt.c +134 −1
drivers/gpu/drm/xe/xe_pt_types.h +5 −0
drivers/gpu/drm/xe/xe_pxp.c +18 −37
drivers/gpu/drm/xe/xe_query.c +12 −11
drivers/gpu/drm/xe/xe_reg_sr.c +5 −12
drivers/gpu/drm/xe/xe_reg_whitelist.c +59 −22
drivers/gpu/drm/xe/xe_ring_ops.c +34 −34
drivers/gpu/drm/xe/xe_sa.c +66 −1
drivers/gpu/drm/xe/xe_sa.h +18 −2
drivers/gpu/drm/xe/xe_sa_types.h +3 −0
drivers/gpu/drm/xe/xe_sriov_packet.c +1 −1
drivers/gpu/drm/xe/xe_sriov_pf.c +4 −0
drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c +2 −4
drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c +4 −12
drivers/gpu/drm/xe/xe_sriov_vf.c +80 −4
drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +19 −8
drivers/gpu/drm/xe/xe_sriov_vf_ccs.h +1 −0
drivers/gpu/drm/xe/xe_survivability_mode.c +180 −93
drivers/gpu/drm/xe/xe_survivability_mode_types.h +20 −8
drivers/gpu/drm/xe/xe_svm.c +42 −46
drivers/gpu/drm/xe/xe_sync.c +31 −2
drivers/gpu/drm/xe/xe_sync.h +2 −0
drivers/gpu/drm/xe/xe_tile.c +5 −0
drivers/gpu/drm/xe/xe_tile_debugfs.c +11 −6
drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c +1 −2
drivers/gpu/drm/xe/xe_tlb_inval.c +24 −3
drivers/gpu/drm/xe/xe_tlb_inval.h +1 −1
drivers/gpu/drm/xe/xe_tlb_inval_job.c +35 −1
drivers/gpu/drm/xe/xe_tlb_inval_job.h +4 −0
drivers/gpu/drm/xe/xe_tlb_inval_types.h +4 −1
drivers/gpu/drm/xe/xe_trace.h +46 −0
drivers/gpu/drm/xe/xe_uc.c +33 −2
drivers/gpu/drm/xe/xe_uc.h +2 −0
drivers/gpu/drm/xe/xe_uc_fw.c +5 −5
drivers/gpu/drm/xe/xe_vm.c +136 −18
drivers/gpu/drm/xe/xe_vm.h +3 −0
drivers/gpu/drm/xe/xe_vm_types.h +1 −0
drivers/gpu/drm/xe/xe_vram.c +2 −4
drivers/gpu/drm/xe/xe_wa.c +6 −46
drivers/gpu/drm/xe/xe_wa_oob.rules +1 −5
include/uapi/drm/xe_drm.h +78 −3

.mailmap+1−0

@@ -481,6 +481,7 @@ Lorenzo Pieralisi <lpieralisi@kernel.org> <lorenzo.pieralisi@arm.com> · 480 unmodified lines
481	481	Lorenzo Stoakes <lorenzo.stoakes@oracle.com> <lstoakes@gmail.com>
482	482	Luca Ceresoli <luca.ceresoli@bootlin.com> <luca@lucaceresoli.net>
483	483	Luca Weiss <luca@lucaweiss.eu> <luca@z3ntu.xyz>
	484	+Lucas De Marchi <demarchi@kernel.org> <lucas.demarchi@intel.com>
484	485	Lukasz Luba <lukasz.luba@arm.com> <l.luba@partner.samsung.com>
485	486	Luo Jie <quic_luoj@quicinc.com> <luoj@codeaurora.org>
486	487	Lance Yang <lance.yang@linux.dev> <ioworker0@gmail.com>

Documentation/ABI/testing/sysfs-driver-intel-xe-sriov+1−1

@@ -119,7 +119,7 @@ Description: · 118 unmodified lines
119	119	The GT preemption timeout (PT) in [us] to be applied to all functions.
120	120	See sriov_admin/{pf,vf<N>}/profile/preempt_timeout_us for more details.
121	121
122		− sched_priority: (RW/RO) string
	122	+ sched_priority: (WO) string
123	123	The GT scheduling priority to be applied for all functions.
124	124	See sriov_admin/{pf,vf<N>}/profile/sched_priority for more details.
125	125

Documentation/gpu/xe/xe_exec_queue.rst+14−0

@@ -7,6 +7,20 @@ Execution Queue · 6 unmodified lines
7	7	.. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c
8	8	:doc: Execution Queue
9	9
	10	+Multi Queue Group
	11	+=================
	12	+
	13	+.. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c
	14	+ :doc: Multi Queue Group
	15	+
	16	+.. _multi-queue-group-guc-interface:
	17	+
	18	+Multi Queue Group GuC interface
	19	+===============================
	20	+
	21	+.. kernel-doc:: drivers/gpu/drm/xe/xe_guc_submit.c
	22	+ :doc: Multi Queue Group GuC interface
	23	+
10	24	Internal API
11	25	============
12	26

MAINTAINERS+1−1

@@ -12640,7 +12640,7 @@ F: include/drm/intel/ · 12639 unmodified lines
12640	12640	F: include/uapi/drm/i915_drm.h
12641	12641
12642	12642	INTEL DRM XE DRIVER (Lunar Lake and newer)
12643		−M: Lucas De Marchi <lucas.demarchi@intel.com>
	12643	+M: Matthew Brost <matthew.brost@intel.com>
12644	12644	M: Thomas Hellström <thomas.hellstrom@linux.intel.com>
12645	12645	M: Rodrigo Vivi <rodrigo.vivi@intel.com>
12646	12646	L: intel-xe@lists.freedesktop.org

drivers/gpu/drm/drm_gpusvm.c+3−0

@@ -1288,6 +1288,9 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, · 1287 unmodified lines
1288	1288	DMA_BIDIRECTIONAL;
1289	1289
1290	1290	retry:
	1291	+ if (time_after(jiffies, timeout))
	1292	+ return -EBUSY;
	1293	+
1291	1294	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
1292	1295	if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages))
1293	1296	goto set_seqno;

drivers/gpu/drm/xe/Makefile+2−0

@@ -95,6 +95,7 @@ xe-y += xe_bb.o \ · 94 unmodified lines
95	95	xe_oa.o \
96	96	xe_observation.o \
97	97	xe_pagefault.o \
	98	+ xe_page_reclaim.o \
98	99	xe_pat.o \
99	100	xe_pci.o \
100	101	xe_pcode.o \
@@ -173,6 +174,7 @@ xe-$(CONFIG_PCI_IOV) += \ · 72 unmodified lines
173	174	xe_lmtt.o \
174	175	xe_lmtt_2l.o \
175	176	xe_lmtt_ml.o \
	177	+ xe_mert.o \
176	178	xe_pci_sriov.o \
177	179	xe_sriov_packet.o \
178	180	xe_sriov_pf.o \

drivers/gpu/drm/xe/abi/guc_actions_abi.h+6−0

@@ -139,6 +139,10 @@ enum xe_guc_action { · 138 unmodified lines
139	139	XE_GUC_ACTION_DEREGISTER_G2G = 0x4508,
140	140	XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600,
141	141	XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601,
	142	+ XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE = 0x4602,
	143	+ XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC = 0x4603,
	144	+ XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE = 0x4604,
	145	+ XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR = 0x4605,
142	146	XE_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507,
143	147	XE_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A,
144	148	XE_GUC_ACTION_SET_DEVICE_ENGINE_ACTIVITY_BUFFER = 0x550C,
@@ -151,6 +155,8 @@ enum xe_guc_action { · 6 unmodified lines
151	155	XE_GUC_ACTION_TLB_INVALIDATION = 0x7000,
152	156	XE_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001,
153	157	XE_GUC_ACTION_TLB_INVALIDATION_ALL = 0x7002,
	158	+ XE_GUC_ACTION_PAGE_RECLAMATION = 0x7003,
	159	+ XE_GUC_ACTION_PAGE_RECLAMATION_DONE = 0x7004,
154	160	XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002,
155	161	XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003,
156	162	XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004,

drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h+57−10

@@ -502,13 +502,17 @@ · 501 unmodified lines
502	502	#define VF2GUC_VF_RESET_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0
503	503
504	504	/**
505		− * DOC: VF2GUC_NOTIFY_RESFIX_DONE
	505	+ * DOC: VF2GUC_RESFIX_DONE
506	506	*
507		− * This action is used by VF to notify the GuC that the VF KMD has completed
508		− * post-migration recovery steps.
	507	+ * This action is used by VF to inform the GuC that the VF KMD has completed
	508	+ * post-migration recovery steps. From GuC VF compatibility 1.27.0 onwards, it
	509	+ * shall only be sent after posting RESFIX_START and that both @MARKER fields
	510	+ * must match.
509	511	*
510	512	* This message must be sent as `MMIO HXG Message`_.
511	513	*
	514	+ * Updated since GuC VF compatibility 1.27.0.
	515	+ *
512	516	* +---+-------+--------------------------------------------------------------+
513	517	* \| \| Bits \| Description \|
514	518	* +===+=======+==============================================================+
@@ -516,9 +520,11 @@ · 1 unmodified line
516	520	* \| +-------+--------------------------------------------------------------+
517	521	* \| \| 30:28 \| TYPE = GUC_HXG_TYPE_REQUEST_ \|
518	522	* \| +-------+--------------------------------------------------------------+
519		− * \| \| 27:16 \| DATA0 = MBZ \|
	523	+ * \| \| 27:16 \| DATA0 = MARKER = MBZ (only prior 1.27.0) \|
520	524	* \| +-------+--------------------------------------------------------------+
521		− * \| \| 15:0 \| ACTION = _`GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE` = 0x5508 \|
	525	+ * \| \| 27:16 \| DATA0 = MARKER - can't be zero (1.27.0+) \|
	526	+ * \| +-------+--------------------------------------------------------------+
	527	+ * \| \| 15:0 \| ACTION = _`GUC_ACTION_VF2GUC_RESFIX_DONE` = 0x5508 \|
522	528	* +---+-------+--------------------------------------------------------------+
523	529	*
524	530	* +---+-------+--------------------------------------------------------------+
@@ -531,13 +537,13 @@ · 6 unmodified lines
531	537	* \| \| 27:0 \| DATA0 = MBZ \|
532	538	* +---+-------+--------------------------------------------------------------+
533	539	*/
534		−#define GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE 0x5508u
	540	+#define GUC_ACTION_VF2GUC_RESFIX_DONE 0x5508u
535	541
536		−#define VF2GUC_NOTIFY_RESFIX_DONE_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN
537		−#define VF2GUC_NOTIFY_RESFIX_DONE_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0
	542	+#define VF2GUC_RESFIX_DONE_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN
	543	+#define VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER GUC_HXG_REQUEST_MSG_0_DATA0
538	544
539		−#define VF2GUC_NOTIFY_RESFIX_DONE_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN
540		−#define VF2GUC_NOTIFY_RESFIX_DONE_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0
	545	+#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN
	546	+#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0
541	547
542	548	/**
543	549	* DOC: VF2GUC_QUERY_SINGLE_KLV
@@ -656,4 +662,45 @@ · 112 unmodified lines
656	662	#define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN
657	663	#define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_0_USED GUC_HXG_RESPONSE_MSG_0_DATA0
658	664
	665	+/**
	666	+ * DOC: VF2GUC_RESFIX_START
	667	+ *
	668	+ * This action is used by VF to inform the GuC that the VF KMD will be starting
	669	+ * post-migration recovery fixups. The @MARKER sent with this action must match
	670	+ * with the MARKER posted in the VF2GUC_RESFIX_DONE message.
	671	+ *
	672	+ * This message must be sent as `MMIO HXG Message`_.
	673	+ *
	674	+ * Available since GuC VF compatibility 1.27.0.
	675	+ *
	676	+ * +---+-------+--------------------------------------------------------------+
	677	+ * \| \| Bits \| Description \|
	678	+ * +===+=======+==============================================================+
	679	+ * \| 0 \| 31 \| ORIGIN = GUC_HXG_ORIGIN_HOST_ \|
	680	+ * \| +-------+--------------------------------------------------------------+
	681	+ * \| \| 30:28 \| TYPE = GUC_HXG_TYPE_REQUEST_ \|
	682	+ * \| +-------+--------------------------------------------------------------+
	683	+ * \| \| 27:16 \| DATA0 = MARKER - can't be zero \|
	684	+ * \| +-------+--------------------------------------------------------------+
	685	+ * \| \| 15:0 \| ACTION = _`GUC_ACTION_VF2GUC_RESFIX_START` = 0x550F \|
	686	+ * +---+-------+--------------------------------------------------------------+
	687	+ *
	688	+ * +---+-------+--------------------------------------------------------------+
	689	+ * \| \| Bits \| Description \|
	690	+ * +===+=======+==============================================================+
	691	+ * \| 0 \| 31 \| ORIGIN = GUC_HXG_ORIGIN_GUC_ \|
	692	+ * \| +-------+--------------------------------------------------------------+
	693	+ * \| \| 30:28 \| TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ \|
	694	+ * \| +-------+--------------------------------------------------------------+
	695	+ * \| \| 27:0 \| DATA0 = MBZ \|
	696	+ * +---+-------+--------------------------------------------------------------+
	697	+ */
	698	+#define GUC_ACTION_VF2GUC_RESFIX_START 0x550Fu
	699	+
	700	+#define VF2GUC_RESFIX_START_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN
	701	+#define VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER GUC_HXG_REQUEST_MSG_0_DATA0
	702	+
	703	+#define VF2GUC_RESFIX_START_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN
	704	+#define VF2GUC_RESFIX_START_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0
	705	+
659	706	#endif

drivers/gpu/drm/xe/abi/guc_klvs_abi.h+9−0

@@ -352,6 +352,12 @@ enum { · 351 unmodified lines
352	352	* :1: NORMAL = schedule VF always, irrespective of whether it has work or not
353	353	* :2: HIGH = schedule VF in the next time-slice after current active
354	354	* time-slice completes if it has active work
	355	+ *
	356	+ * _`GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT` : 0x8A0D
	357	+ * Given that multi-LRC contexts are incompatible with SRIOV scheduler
	358	+ * groups and cause the latter to be turned off when registered with the
	359	+ * GuC, this config allows the PF to set a threshold for multi-LRC context
	360	+ * registrations by VFs to monitor their behavior.
355	361	*/
356	362
357	363	#define GUC_KLV_VF_CFG_GGTT_START_KEY 0x0001
@@ -410,6 +416,9 @@ enum { · 52 unmodified lines
410	416	#define GUC_SCHED_PRIORITY_NORMAL 1u
411	417	#define GUC_SCHED_PRIORITY_HIGH 2u
412	418
	419	+#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_KEY 0x8a0d
	420	+#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_LEN 1u
	421	+
413	422	/*
414	423	* Workaround keys:
415	424	*/

drivers/gpu/drm/xe/abi/guc_lfd_abi.h+171−0

@@ -0,0 +1,171 @@
	1	+/* SPDX-License-Identifier: MIT */
	2	+/*
	3	+ * Copyright © 2025 Intel Corporation
	4	+ */
	5	+
	6	+#ifndef _ABI_GUC_LFD_ABI_H_
	7	+#define _ABI_GUC_LFD_ABI_H_
	8	+
	9	+#include <linux/types.h>
	10	+
	11	+#include "guc_lic_abi.h"
	12	+
	13	+/* The current major version of GuC-Log-File format. */
	14	+#define GUC_LFD_FORMAT_VERSION_MAJOR 0x0001
	15	+/* The current minor version of GuC-Log-File format. */
	16	+#define GUC_LFD_FORMAT_VERSION_MINOR 0x0000
	17	+
	18	+/** enum guc_lfd_type - Log format descriptor type */
	19	+enum guc_lfd_type {
	20	+ /**
	21	+ * @GUC_LFD_TYPE_FW_REQUIRED_RANGE_START: Start of range for
	22	+ * required LFDs from GuC
	23	+ * @GUC_LFD_TYPE_FW_VERSION: GuC Firmware Version structure.
	24	+ * @GUC_LFD_TYPE_GUC_DEVICE_ID: GuC microcontroller device ID.
	25	+ * @GUC_LFD_TYPE_TSC_FREQUENCY: Frequency of GuC timestamps.
	26	+ * @GUC_LFD_TYPE_GMD_ID: HW GMD ID.
	27	+ * @GUC_LFD_TYPE_BUILD_PLATFORM_ID: GuC build platform ID.
	28	+ * @GUC_LFD_TYPE_FW_REQUIRED_RANGE_END: End of range for
	29	+ * required LFDs from GuC
	30	+ */
	31	+ GUC_LFD_TYPE_FW_REQUIRED_RANGE_START = 0x1,
	32	+ GUC_LFD_TYPE_FW_VERSION = 0x1,
	33	+ GUC_LFD_TYPE_GUC_DEVICE_ID = 0x2,
	34	+ GUC_LFD_TYPE_TSC_FREQUENCY = 0x3,
	35	+ GUC_LFD_TYPE_GMD_ID = 0x4,
	36	+ GUC_LFD_TYPE_BUILD_PLATFORM_ID = 0x5,
	37	+ GUC_LFD_TYPE_FW_REQUIRED_RANGE_END = 0x1FFF,
	38	+
	39	+ /**
	40	+ * @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START: Start of range for
	41	+ * optional LFDs from GuC
	42	+ * @GUC_LFD_TYPE_LOG_EVENTS_BUFFER: Log-event-entries buffer.
	43	+ * @GUC_LFD_TYPE_FW_CRASH_DUMP: GuC generated crash-dump blob.
	44	+ * @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END: End of range for
	45	+ * optional LFDs from GuC
	46	+ */
	47	+ GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START = 0x2000,
	48	+ GUC_LFD_TYPE_LOG_EVENTS_BUFFER = 0x2000,
	49	+ GUC_LFD_TYPE_FW_CRASH_DUMP = 0x2001,
	50	+ GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END = 0x3FFF,
	51	+
	52	+ /**
	53	+ * @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START: Start of range for
	54	+ * required KMD LFDs
	55	+ * @GUC_LFD_TYPE_OS_ID: An identifier for the OS.
	56	+ * @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END: End of this range for
	57	+ * required KMD LFDs
	58	+ */
	59	+ GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START = 0x4000,
	60	+ GUC_LFD_TYPE_OS_ID = 0x4000,
	61	+ GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END = 0x5FFF,
	62	+
	63	+ /**
	64	+ * @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START: Start of range for
	65	+ * optional KMD LFDs
	66	+ * @GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT: Binary representation of
	67	+ * GuC log-events schema.
	68	+ * @GUC_LFD_TYPE_HOST_COMMENT: ASCII string containing comments
	69	+ * from the host/KMD.
	70	+ * @GUC_LFD_TYPE_TIMESTAMP_ANCHOR: A timestamp anchor, to convert
	71	+ * between host and GuC timestamp.
	72	+ * @GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG: Timestamp anchor
	73	+ * configuration, definition of timestamp frequency and bit width.
	74	+ * @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END: End of this range for
	75	+ * optional KMD LFDs
	76	+ */
	77	+ GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START = 0x6000,
	78	+ GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT = 0x6000,
	79	+ GUC_LFD_TYPE_HOST_COMMENT = 0x6001,
	80	+ GUC_LFD_TYPE_TIMESTAMP_ANCHOR = 0x6002,
	81	+ GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG = 0x6003,
	82	+ GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END = 0x7FFF,
	83	+
	84	+ /*
	85	+ * @GUC_LFD_TYPE_RESERVED_RANGE_START: Start of reserved range
	86	+ * @GUC_LFD_TYPE_RESERVED_RANGE_END: End of reserved range
	87	+ */
	88	+ GUC_LFD_TYPE_RESERVED_RANGE_START = 0x8000,
	89	+ GUC_LFD_TYPE_RESERVED_RANGE_END = 0xFFFF,
	90	+};
	91	+
	92	+/** enum guc_lfd_os_type - OS Type LFD-ID */
	93	+enum guc_lfd_os_type {
	94	+ /** @GUC_LFD_OS_TYPE_OSID_WIN: Windows OS */
	95	+ GUC_LFD_OS_TYPE_OSID_WIN = 0x1,
	96	+ /** @GUC_LFD_OS_TYPE_OSID_LIN: Linux OS */
	97	+ GUC_LFD_OS_TYPE_OSID_LIN = 0x2,
	98	+ /** @GUC_LFD_OS_TYPE_OSID_VMW: VMWare OS */
	99	+ GUC_LFD_OS_TYPE_OSID_VMW = 0x3,
	100	+ /** @GUC_LFD_OS_TYPE_OSID_OTHER: Other */
	101	+ GUC_LFD_OS_TYPE_OSID_OTHER = 0x4,
	102	+};
	103	+
	104	+/** struct guc_lfd_data - A generic header structure for all LFD blocks */
	105	+struct guc_lfd_data {
	106	+ /** @header: A 32 bits dword, contains multiple bit fields */
	107	+ u32 header;
	108	+ /* LFD type. See guc_lfd_type */
	109	+#define GUC_LFD_DATA_HEADER_MASK_TYPE GENMASK(31, 16)
	110	+#define GUC_LFD_DATA_HEADER_MASK_MAGIC GENMASK(15, 0)
	111	+
	112	+ /** @data_count: Number of dwords the `data` field contains. */
	113	+ u32 data_count;
	114	+ /** @data: Data defined by GUC_LFD_DATA_HEADER_MASK_TYPE */
	115	+ u32 data[] __counted_by(data_count);
	116	+} __packed;
	117	+
	118	+/**
	119	+ * struct guc_lfd_data_log_events_buf - GuC Log Events Buffer.
	120	+ * This is optional fw LFD data
	121	+ */
	122	+struct guc_lfd_data_log_events_buf {
	123	+ /**
	124	+ * @log_events_format_version: version of GuC log format of buffer
	125	+ */
	126	+ u32 log_events_format_version;
	127	+ /**
	128	+ * @log_event: The log event data.
	129	+ * Size in dwords is LFD block size - 1.
	130	+ */
	131	+ u32 log_event[];
	132	+} __packed;
	133	+
	134	+/** struct guc_lfd_data_os_info - OS Version Information. */
	135	+struct guc_lfd_data_os_info {
	136	+ /**
	137	+ * @os_id: enum values to identify the OS brand.
	138	+ * See guc_lfd_os_type for the range of types
	139	+ */
	140	+ u32 os_id;
	141	+ /**
	142	+ * @build_version: ASCII string containing OS build version
	143	+ * information based on os_id. String is padded with null
	144	+ * characters to ensure its DWORD aligned.
	145	+ * Size in dwords is LFD block size - 1.
	146	+ */
	147	+ char build_version[];
	148	+} __packed;
	149	+
	150	+/**
	151	+ * struct guc_logfile_header - Header of GuC Log Streaming-LFD-File Format.
	152	+ * This structure encapsulates the layout of the guc-log-file format
	153	+ */
	154	+struct guc_lfd_file_header {
	155	+ /**
	156	+ * @magic: A magic number set by producer of a GuC log file to
	157	+ * identify that file is a valid guc-log-file containing a stream
	158	+ * of LFDs.
	159	+ */
	160	+ u64 magic;
	161	+ /** @version: Version of this file format layout */
	162	+ u32 version;
	163	+#define GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR GENMASK(31, 16)
	164	+#define GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR GENMASK(15, 0)
	165	+
	166	+ /** @stream: A stream of one or more guc_lfd_data LFD blocks
	167	+ */
	168	+ u32 stream[];
	169	+} __packed;
	170	+
	171	+#endif

drivers/gpu/drm/xe/abi/guc_lic_abi.h+77−0

@@ -0,0 +1,77 @@
	1	+/* SPDX-License-Identifier: MIT */
	2	+/*
	3	+ * Copyright © 2025 Intel Corporation
	4	+ */
	5	+
	6	+#ifndef _ABI_GUC_LIC_ABI_H_
	7	+#define _ABI_GUC_LIC_ABI_H_
	8	+
	9	+#include <linux/types.h>
	10	+
	11	+/**
	12	+ * enum guc_lic_type - Log Init Config KLV IDs.
	13	+ */
	14	+enum guc_lic_type {
	15	+ /**
	16	+ * @GUC_LIC_TYPE_GUC_SW_VERSION: GuC firmware version. Value
	17	+ * is a 32 bit number represented by guc_sw_version.
	18	+ */
	19	+ GUC_LIC_TYPE_GUC_SW_VERSION = 0x1,
	20	+ /**
	21	+ * @GUC_LIC_TYPE_GUC_DEVICE_ID: GuC device id. Value is a 32
	22	+ * bit.
	23	+ */
	24	+ GUC_LIC_TYPE_GUC_DEVICE_ID = 0x2,
	25	+ /**
	26	+ * @GUC_LIC_TYPE_TSC_FREQUENCY: GuC timestamp counter
	27	+ * frequency. Value is a 32 bit number representing frequency in
	28	+ * kHz. This timestamp is utilized in log entries, timer and
	29	+ * for engine utilization tracking.
	30	+ */
	31	+ GUC_LIC_TYPE_TSC_FREQUENCY = 0x3,
	32	+ /**
	33	+ * @GUC_LIC_TYPE_GMD_ID: HW GMD ID. Value is a 32 bit number
	34	+ * representing graphics, media and display HW architecture IDs.
	35	+ */
	36	+ GUC_LIC_TYPE_GMD_ID = 0x4,
	37	+ /**
	38	+ * @GUC_LIC_TYPE_BUILD_PLATFORM_ID: GuC build platform ID.
	39	+ * Value is 32 bits.
	40	+ */
	41	+ GUC_LIC_TYPE_BUILD_PLATFORM_ID = 0x5,
	42	+};
	43	+
	44	+/**
	45	+ * struct guc_lic - GuC LIC (Log-Init-Config) structure.
	46	+ *
	47	+ * This is populated by the GUC at log init time and is located in the log
	48	+ * buffer memory allocation.
	49	+ */
	50	+struct guc_lic {
	51	+ /**
	52	+ * @magic: A magic number set by GuC to identify that this
	53	+ * structure contains valid information: magic = GUC_LIC_MAGIC.
	54	+ */
	55	+ u32 magic;
	56	+#define GUC_LIC_MAGIC 0x8086900D
	57	+ /**
	58	+ * @version: The version of the this structure.
	59	+ * Major and minor version number are represented as bit fields.
	60	+ */
	61	+ u32 version;
	62	+#define GUC_LIC_VERSION_MASK_MAJOR GENMASK(31, 16)
	63	+#define GUC_LIC_VERSION_MASK_MINOR GENMASK(15, 0)
	64	+
	65	+#define GUC_LIC_VERSION_MAJOR 1u
	66	+#define GUC_LIC_VERSION_MINOR 0u
	67	+
	68	+ /** @data_count: Number of dwords the `data` array contains. */
	69	+ u32 data_count;
	70	+ /**
	71	+ * @data: Array of dwords representing a list of LIC KLVs of
	72	+ * type guc_klv_generic with keys represented by guc_lic_type
	73	+ */
	74	+ u32 data[] __counted_by(data_count);
	75	+} __packed;
	76	+
	77	+#endif

drivers/gpu/drm/xe/abi/guc_log_abi.h+38−4

@@ -8,11 +8,45 @@ · 7 unmodified lines
8	8
9	9	#include <linux/types.h>
10	10
	11	+/**
	12	+ * DOC: GuC Log buffer Layout
	13	+ *
	14	+ * The in-memory log buffer layout is as follows::
	15	+ *
	16	+ * +===============================+ 0000h
	17	+ * \| Crash dump state header \| ^
	18	+ * +-------------------------------+ 32B \|
	19	+ * \| Debug state header \| \|
	20	+ * +-------------------------------+ 64B 4KB
	21	+ * \| Capture state header \| \|
	22	+ * +-------------------------------+ 96B \|
	23	+ * \| \| v
	24	+ * +===============================+ <--- EVENT_DATA_OFFSET
	25	+ * \| Event logs(raw data) \| ^
	26	+ * \| \| \|
	27	+ * \| \| EVENT_DATA_BUFFER_SIZE
	28	+ * \| \| \|
	29	+ * \| \| v
	30	+ * +===============================+ <--- CRASH_DUMP_OFFSET
	31	+ * \| Crash Dump(raw data) \| ^
	32	+ * \| \| \|
	33	+ * \| \| CRASH_DUMP_BUFFER_SIZE
	34	+ * \| \| \|
	35	+ * \| \| v
	36	+ * +===============================+ <--- STATE_CAPTURE_OFFSET
	37	+ * \| Error state capture(raw data) \| ^
	38	+ * \| \| \|
	39	+ * \| \| STATE_CAPTURE_BUFFER_SIZE
	40	+ * \| \| \|
	41	+ * \| \| v
	42	+ * +===============================+ Total: GUC_LOG_SIZE
	43	+ */
	44	+
11	45	/* GuC logging buffer types */
12		−enum guc_log_buffer_type {
13		− GUC_LOG_BUFFER_CRASH_DUMP,
14		− GUC_LOG_BUFFER_DEBUG,
15		− GUC_LOG_BUFFER_CAPTURE,
	46	+enum guc_log_type {
	47	+ GUC_LOG_TYPE_EVENT_DATA,
	48	+ GUC_LOG_TYPE_CRASH_DUMP,
	49	+ GUC_LOG_TYPE_STATE_CAPTURE,
16	50	};
17	51
18	52	#define GUC_LOG_BUFFER_TYPE_MAX 3

drivers/gpu/drm/xe/display/xe_fb_pin.c+9−14

@@ -210,10 +210,11 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, · 209 unmodified lines
210	210	/* TODO: Consider sharing framebuffer mapping?
211	211	* embed i915_vma inside intel_framebuffer
212	212	*/
213		− xe_pm_runtime_get_noresume(xe);
214		− ret = mutex_lock_interruptible(&ggtt->lock);
	213	+ guard(xe_pm_runtime_noresume)(xe);
	214	+ ACQUIRE(mutex_intr, lock)(&ggtt->lock);
	215	+ ret = ACQUIRE_ERR(mutex_intr, &lock);
215	216	if (ret)
216		− goto out;
	217	+ return ret;
217	218
218	219	align = XE_PAGE_SIZE;
219	220	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
@@ -223,15 +224,13 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, · 3 unmodified lines
223	224	vma->node = bo->ggtt_node[tile0->id];
224	225	} else if (view->type == I915_GTT_VIEW_NORMAL) {
225	226	vma->node = xe_ggtt_node_init(ggtt);
226		− if (IS_ERR(vma->node)) {
227		− ret = PTR_ERR(vma->node);
228		− goto out_unlock;
229		− }
	227	+ if (IS_ERR(vma->node))
	228	+ return PTR_ERR(vma->node);
230	229
231	230	ret = xe_ggtt_node_insert_locked(vma->node, xe_bo_size(bo), align, 0);
232	231	if (ret) {
233	232	xe_ggtt_node_fini(vma->node);
234		− goto out_unlock;
	233	+ return ret;
235	234	}
236	235
237	236	xe_ggtt_map_bo(ggtt, vma->node, bo, xe->pat.idx[XE_CACHE_NONE]);
@@ -245,13 +244,13 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, · 7 unmodified lines
245	244	vma->node = xe_ggtt_node_init(ggtt);
246	245	if (IS_ERR(vma->node)) {
247	246	ret = PTR_ERR(vma->node);
248		− goto out_unlock;
	247	+ return ret;
249	248	}
250	249
251	250	ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
252	251	if (ret) {
253	252	xe_ggtt_node_fini(vma->node);
254		− goto out_unlock;
	253	+ return ret;
255	254	}
256	255
257	256	ggtt_ofs = vma->node->base.start;
@@ -265,10 +264,6 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, · 7 unmodified lines
265	264	rot_info->plane[i].dst_stride);
266	265	}
267	266
268		−out_unlock:
269		− mutex_unlock(&ggtt->lock);
270		−out:
271		− xe_pm_runtime_put(xe);
272	267	return ret;
273	268	}
274	269

drivers/gpu/drm/xe/display/xe_hdcp_gsc.c+9−22

@@ -38,8 +38,6 @@ static bool intel_hdcp_gsc_check_status(struct drm_device *drm) · 37 unmodified lines
38	38	struct xe_tile *tile = xe_device_get_root_tile(xe);
39	39	struct xe_gt *gt = tile->media_gt;
40	40	struct xe_gsc *gsc = &gt->uc.gsc;
41		− bool ret = true;
42		− unsigned int fw_ref;
43	41
44	42	if (!gsc \|\| !xe_uc_fw_is_enabled(&gsc->fw)) {
45	43	drm_dbg_kms(&xe->drm,
@@ -47,22 +45,15 @@ static bool intel_hdcp_gsc_check_status(struct drm_device *drm) · 1 unmodified line
47	45	return false;
48	46	}
49	47
50		− xe_pm_runtime_get(xe);
51		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
52		− if (!fw_ref) {
	48	+ guard(xe_pm_runtime)(xe);
	49	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
	50	+ if (!fw_ref.domains) {
53	51	drm_dbg_kms(&xe->drm,
54	52	"failed to get forcewake to check proxy status\n");
55		− ret = false;
56		− goto out;
	53	+ return false;
57	54	}
58	55
59		− if (!xe_gsc_proxy_init_done(gsc))
60		− ret = false;
61		−
62		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
63		−out:
64		− xe_pm_runtime_put(xe);
65		− return ret;
	56	+ return xe_gsc_proxy_init_done(gsc);
66	57	}
67	58
68	59	/This function helps allocate memory for the command that we will send to gsc cs /
@@ -168,17 +159,15 @@ static ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_contex · 99 unmodified lines
168	159	u32 addr_out_off, addr_in_wr_off = 0;
169	160	int ret, tries = 0;
170	161
171		− if (msg_in_len > max_msg_size \|\| msg_out_len > max_msg_size) {
172		− ret = -ENOSPC;
173		− goto out;
174		− }
	162	+ if (msg_in_len > max_msg_size \|\| msg_out_len > max_msg_size)
	163	+ return -ENOSPC;
175	164
176	165	msg_size_in = msg_in_len + HDCP_GSC_HEADER_SIZE;
177	166	msg_size_out = msg_out_len + HDCP_GSC_HEADER_SIZE;
178	167	addr_out_off = PAGE_SIZE;
179	168
180	169	host_session_id = xe_gsc_create_host_session_id();
181		− xe_pm_runtime_get_noresume(xe);
	170	+ guard(xe_pm_runtime_noresume)(xe);
182	171	addr_in_wr_off = xe_gsc_emit_header(xe, &gsc_context->hdcp_bo->vmap,
183	172	addr_in_wr_off, HECI_MEADDRESS_HDCP,
184	173	host_session_id, msg_in_len);
@@ -203,14 +192,12 @@ static ssize_t intel_hdcp_gsc_msg_send(struct intel_hdcp_gsc_context *gsc_contex · 18 unmodified lines
203	192	} while (++tries < 20);
204	193
205	194	if (ret)
206		− goto out;
	195	+ return ret;
207	196
208	197	xe_map_memcpy_from(xe, msg_out, &gsc_context->hdcp_bo->vmap,
209	198	addr_out_off + HDCP_GSC_HEADER_SIZE,
210	199	msg_out_len);
211	200
212		−out:
213		− xe_pm_runtime_put(xe);
214	201	return ret;
215	202	}
216	203

drivers/gpu/drm/xe/instructions/xe_gpu_commands.h+1−0

@@ -47,6 +47,7 @@ · 46 unmodified lines
47	47
48	48	#define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)\|(0x3<<27)\|(0x2<<24)\|((len)-2))
49	49
	50	+#define PIPE_CONTROL0_QUEUE_DRAIN_MODE BIT(12)
50	51	#define PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE BIT(10) /* gen12 */
51	52	#define PIPE_CONTROL0_HDC_PIPELINE_FLUSH BIT(9) /* gen12 */
52	53

drivers/gpu/drm/xe/regs/xe_gt_regs.h+3−0

@@ -227,6 +227,9 @@ · 226 unmodified lines
227	227
228	228	#define MIRROR_FUSE1 XE_REG(0x911c)
229	229
	230	+#define FUSE2 XE_REG(0x9120)
	231	+#define PRODUCTION_HW REG_BIT(2)
	232	+
230	233	#define MIRROR_L3BANK_ENABLE XE_REG(0x9130)
231	234	#define XE3_L3BANK_ENABLE REG_GENMASK(31, 0)
232	235

drivers/gpu/drm/xe/regs/xe_gtt_defs.h+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#define XELPG_GGTT_PTE_PAT0 BIT_ULL(52)
10	10	#define XELPG_GGTT_PTE_PAT1 BIT_ULL(53)
11	11
	12	+#define XE_PTE_ADDR_MASK GENMASK_ULL(51, 12)
12	13	#define GGTT_PTE_VFID GENMASK_ULL(11, 2)
13	14
14	15	#define GUC_GGTT_TOP 0xFEE00000

drivers/gpu/drm/xe/regs/xe_guc_regs.h+3−0

@@ -90,6 +90,9 @@ · 89 unmodified lines
90	90	#define GUC_SEND_INTERRUPT XE_REG(0xc4c8)
91	91	#define GUC_SEND_TRIGGER REG_BIT(0)
92	92
	93	+#define GUC_INTR_CHICKEN XE_REG(0xc50c)
	94	+#define DISABLE_SIGNALING_ENGINES REG_BIT(1)
	95	+
93	96	#define GUC_BCS_RCS_IER XE_REG(0xc550)
94	97	#define GUC_VCS2_VCS1_IER XE_REG(0xc554)
95	98	#define GUC_WD_VECS_IER XE_REG(0xc558)

drivers/gpu/drm/xe/regs/xe_irq_regs.h+1−0

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	20	#define GU_MISC_IRQ REG_BIT(29)
21	21	#define ERROR_IRQ(x) REG_BIT(26 + (x))
22	22	#define DISPLAY_IRQ REG_BIT(16)
	23	+#define SOC_H2DMEMINT_IRQ REG_BIT(13)
23	24	#define I2C_IRQ REG_BIT(12)
24	25	#define GT_DW_IRQ(x) REG_BIT(x)
25	26

drivers/gpu/drm/xe/regs/xe_mert_regs.h+21−0

@@ -0,0 +1,21 @@
	1	+/* SPDX-License-Identifier: MIT */
	2	+/*
	3	+ * Copyright © 2025 Intel Corporation
	4	+ */
	5	+
	6	+#ifndef _XE_MERT_REGS_H_
	7	+#define _XE_MERT_REGS_H_
	8	+
	9	+#include "regs/xe_reg_defs.h"
	10	+
	11	+#define MERT_LMEM_CFG XE_REG(0x1448b0)
	12	+
	13	+#define MERT_TLB_CT_INTR_ERR_ID_PORT XE_REG(0x145190)
	14	+#define MERT_TLB_CT_VFID_MASK REG_GENMASK(16, 9)
	15	+#define MERT_TLB_CT_ERROR_MASK REG_GENMASK(5, 0)
	16	+#define MERT_TLB_CT_LMTT_FAULT 0x05
	17	+
	18	+#define MERT_TLB_INV_DESC_A XE_REG(0x14cf7c)
	19	+#define MERT_TLB_INV_DESC_A_VALID REG_BIT(0)
	20	+
	21	+#endif /* _XE_MERT_REGS_H_ */

drivers/gpu/drm/xe/regs/xe_oa_regs.h+17−0

@@ -100,4 +100,21 @@ · 99 unmodified lines
100	100	#define OAM_COMPRESSION_T3_CONTROL XE_REG(0x1c2e00)
101	101	#define OAM_LAT_MEASURE_ENABLE REG_BIT(4)
102	102
	103	+/* Actual address is MEDIA_GT_GSI_OFFSET + the base addr below */
	104	+#define XE_OAM_SAG_BASE 0x13000
	105	+#define XE_OAM_SCMI_0_BASE 0x14000
	106	+#define XE_OAM_SCMI_1_BASE 0x14800
	107	+#define XE_OAM_SAG_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SAG_BASE)
	108	+#define XE_OAM_SCMI_0_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_0_BASE)
	109	+#define XE_OAM_SCMI_1_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_1_BASE)
	110	+
	111	+#define OAMERT_CONTROL XE_REG(0x1453a0)
	112	+#define OAMERT_DEBUG XE_REG(0x1453a4)
	113	+#define OAMERT_STATUS XE_REG(0x1453a8)
	114	+#define OAMERT_HEAD_POINTER XE_REG(0x1453ac)
	115	+#define OAMERT_TAIL_POINTER XE_REG(0x1453b0)
	116	+#define OAMERT_BUFFER XE_REG(0x1453b4)
	117	+#define OAMERT_CONTEXT_CONTROL XE_REG(0x1453c8)
	118	+#define OAMERT_MMIO_TRG XE_REG(0x1453cc)
	119	+
103	120	#endif

drivers/gpu/drm/xe/tests/xe_args_test.c+54−0

@@ -78,6 +78,24 @@ static void pick_arg_example(struct kunit *test) · 77 unmodified lines
78	78	#undef buz
79	79	}
80	80
	81	+static void if_args_example(struct kunit *test)
	82	+{
	83	+ enum { Z = 1, Q };
	84	+
	85	+#define foo X, Y
	86	+#define bar IF_ARGS(Z, Q, foo)
	87	+#define buz IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo)))
	88	+
	89	+ KUNIT_EXPECT_EQ(test, bar, Z);
	90	+ KUNIT_EXPECT_EQ(test, buz, Q);
	91	+ KUNIT_EXPECT_STREQ(test, __stringify(bar), "Z");
	92	+ KUNIT_EXPECT_STREQ(test, __stringify(buz), "Q");
	93	+
	94	+#undef foo
	95	+#undef bar
	96	+#undef buz
	97	+}
	98	+
81	99	static void sep_comma_example(struct kunit *test)
82	100	{
83	101	#define foo(f) f(X) f(Y) f(Z) f(Q)
@@ -198,6 +216,40 @@ static void last_arg_test(struct kunit *test) · 114 unmodified lines
198	216	KUNIT_EXPECT_STREQ(test, __stringify(LAST_ARG(MAX_ARGS)), "-12");
199	217	}
200	218
	219	+static void if_args_test(struct kunit *test)
	220	+{
	221	+ bool with_args = true;
	222	+ bool no_args = false;
	223	+ enum { X = 100 };
	224	+
	225	+ KUNIT_EXPECT_TRUE(test, IF_ARGS(true, false, FOO_ARGS));
	226	+ KUNIT_EXPECT_FALSE(test, IF_ARGS(true, false, NO_ARGS));
	227	+
	228	+ KUNIT_EXPECT_TRUE(test, CONCATENATE(IF_ARGS(with, no, FOO_ARGS), _args));
	229	+ KUNIT_EXPECT_FALSE(test, CONCATENATE(IF_ARGS(with, no, NO_ARGS), _args));
	230	+
	231	+ KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, FOO_ARGS)), "yes");
	232	+ KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, NO_ARGS)), "no");
	233	+
	234	+ KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, FOO_ARGS), 4);
	235	+ KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, NO_ARGS), -1);
	236	+ KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, FOO_ARGS), 0);
	237	+ KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, NO_ARGS), -1);
	238	+
	239	+ KUNIT_EXPECT_EQ(test,
	240	+ CALL_ARGS(FIRST_ARG,
	241	+ CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), X);
	242	+ KUNIT_EXPECT_EQ(test,
	243	+ CALL_ARGS(FIRST_ARG,
	244	+ CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), -1);
	245	+ KUNIT_EXPECT_EQ(test,
	246	+ CALL_ARGS(COUNT_ARGS,
	247	+ CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), 4);
	248	+ KUNIT_EXPECT_EQ(test,
	249	+ CALL_ARGS(COUNT_ARGS,
	250	+ CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), 12);
	251	+}
	252	+
201	253	static struct kunit_case args_tests[] = {
202	254	KUNIT_CASE(count_args_test),
203	255	KUNIT_CASE(call_args_example),
@@ -209,6 +261,8 @@ static struct kunit_case args_tests[] = { · 5 unmodified lines
209	261	KUNIT_CASE(last_arg_example),
210	262	KUNIT_CASE(last_arg_test),
211	263	KUNIT_CASE(pick_arg_example),
	264	+ KUNIT_CASE(if_args_example),
	265	+ KUNIT_CASE(if_args_test),
212	266	KUNIT_CASE(sep_comma_example),
213	267	{}
214	268	};

drivers/gpu/drm/xe/tests/xe_bo.c+2−8

@@ -185,8 +185,7 @@ static int ccs_test_run_device(struct xe_device *xe) · 184 unmodified lines
185	185	return 0;
186	186	}
187	187
188		− xe_pm_runtime_get(xe);
189		−
	188	+ guard(xe_pm_runtime)(xe);
190	189	for_each_tile(tile, xe, id) {
191	190	/* For igfx run only for primary tile */
192	191	if (!IS_DGFX(xe) && id > 0)
@@ -194,8 +193,6 @@ static int ccs_test_run_device(struct xe_device *xe) · 1 unmodified line
194	193	ccs_test_run_tile(xe, tile, test);
195	194	}
196	195
197		− xe_pm_runtime_put(xe);
198		−
199	196	return 0;
200	197	}
201	198
@@ -356,13 +353,10 @@ static int evict_test_run_device(struct xe_device *xe) · 154 unmodified lines
356	353	return 0;
357	354	}
358	355
359		− xe_pm_runtime_get(xe);
360		−
	356	+ guard(xe_pm_runtime)(xe);
361	357	for_each_tile(tile, xe, id)
362	358	evict_test_run_tile(xe, tile, test);
363	359
364		− xe_pm_runtime_put(xe);
365		−
366	360	return 0;
367	361	}
368	362

drivers/gpu/drm/xe/tests/xe_dma_buf.c+1−2

@@ -266,7 +266,7 @@ static int dma_buf_run_device(struct xe_device *xe) · 265 unmodified lines
266	266	const struct dma_buf_test_params *params;
267	267	struct kunit *test = kunit_get_current_test();
268	268
269		− xe_pm_runtime_get(xe);
	269	+ guard(xe_pm_runtime)(xe);
270	270	for (params = test_params; params->mem_mask; ++params) {
271	271	struct dma_buf_test_params p = *params;
272	272
@@ -274,7 +274,6 @@ static int dma_buf_run_device(struct xe_device *xe) · 1 unmodified line
274	274	test->priv = &p;
275	275	xe_test_dmabuf_import_same_driver(xe);
276	276	}
277		− xe_pm_runtime_put(xe);
278	277
279	278	/* A non-zero return would halt iteration over driver devices */
280	279	return 0;

drivers/gpu/drm/xe/tests/xe_migrate.c+2−8

@@ -344,8 +344,7 @@ static int migrate_test_run_device(struct xe_device *xe) · 343 unmodified lines
344	344	struct xe_tile *tile;
345	345	int id;
346	346
347		− xe_pm_runtime_get(xe);
348		−
	347	+ guard(xe_pm_runtime)(xe);
349	348	for_each_tile(tile, xe, id) {
350	349	struct xe_migrate *m = tile->migrate;
351	350	struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
@@ -356,8 +355,6 @@ static int migrate_test_run_device(struct xe_device *xe) · 4 unmodified lines
356	355	xe_vm_unlock(m->q->vm);
357	356	}
358	357
359		− xe_pm_runtime_put(xe);
360		−
361	358	return 0;
362	359	}
363	360
@@ -759,13 +756,10 @@ static int validate_ccs_test_run_device(struct xe_device *xe) · 395 unmodified lines
759	756	return 0;
760	757	}
761	758
762		− xe_pm_runtime_get(xe);
763		−
	759	+ guard(xe_pm_runtime)(xe);
764	760	for_each_tile(tile, xe, id)
765	761	validate_ccs_test_run_tile(xe, tile, test);
766	762
767		− xe_pm_runtime_put(xe);
768		−
769	763	return 0;
770	764	}
771	765

drivers/gpu/drm/xe/tests/xe_mocs.c+8−19

@@ -43,14 +43,12 @@ static void read_l3cc_table(struct xe_gt *gt, · 42 unmodified lines
43	43	{
44	44	struct kunit *test = kunit_get_current_test();
45	45	u32 l3cc, l3cc_expected;
46		− unsigned int fw_ref, i;
	46	+ unsigned int i;
47	47	u32 reg_val;
48	48
49		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
50		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
51		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
	49	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	50	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
52	51	KUNIT_FAIL_AND_ABORT(test, "Forcewake Failed.\n");
53		− }
54	52
55	53	for (i = 0; i < info->num_mocs_regs; i++) {
56	54	if (!(i & 1)) {
@@ -74,7 +72,6 @@ static void read_l3cc_table(struct xe_gt *gt, · 17 unmodified lines
74	72	KUNIT_EXPECT_EQ_MSG(test, l3cc_expected, l3cc,
75	73	"l3cc idx=%u has incorrect val.\n", i);
76	74	}
77		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
78	75	}
79	76
80	77	static void read_mocs_table(struct xe_gt *gt,
@@ -82,14 +79,14 @@ static void read_mocs_table(struct xe_gt *gt, · 1 unmodified line
82	79	{
83	80	struct kunit *test = kunit_get_current_test();
84	81	u32 mocs, mocs_expected;
85		− unsigned int fw_ref, i;
	82	+ unsigned int i;
86	83	u32 reg_val;
87	84
88	85	KUNIT_EXPECT_TRUE_MSG(test, info->unused_entries_index,
89	86	"Unused entries index should have been defined\n");
90	87
91		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
92		− KUNIT_ASSERT_NE_MSG(test, fw_ref, 0, "Forcewake Failed.\n");
	88	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	89	+ KUNIT_ASSERT_NE_MSG(test, fw_ref.domains, 0, "Forcewake Failed.\n");
93	90
94	91	for (i = 0; i < info->num_mocs_regs; i++) {
95	92	if (regs_are_mcr(gt))
@@ -106,8 +103,6 @@ static void read_mocs_table(struct xe_gt *gt, · 10 unmodified lines
106	103	KUNIT_EXPECT_EQ_MSG(test, mocs_expected, mocs,
107	104	"mocs reg 0x%x has incorrect val.\n", i);
108	105	}
109		−
110		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
111	106	}
112	107
113	108	static int mocs_kernel_test_run_device(struct xe_device *xe)
@@ -120,8 +115,7 @@ static int mocs_kernel_test_run_device(struct xe_device *xe) · 6 unmodified lines
120	115	unsigned int flags;
121	116	int id;
122	117
123		− xe_pm_runtime_get(xe);
124		−
	118	+ guard(xe_pm_runtime)(xe);
125	119	for_each_gt(gt, xe, id) {
126	120	flags = live_mocs_init(&mocs, gt);
127	121	if (flags & HAS_GLOBAL_MOCS)
@@ -130,8 +124,6 @@ static int mocs_kernel_test_run_device(struct xe_device *xe) · 2 unmodified lines
130	124	read_l3cc_table(gt, &mocs.table);
131	125	}
132	126
133		− xe_pm_runtime_put(xe);
134		−
135	127	return 0;
136	128	}
137	129
@@ -155,8 +147,7 @@ static int mocs_reset_test_run_device(struct xe_device *xe) · 17 unmodified lines
155	147	int id;
156	148	struct kunit *test = kunit_get_current_test();
157	149
158		− xe_pm_runtime_get(xe);
159		−
	150	+ guard(xe_pm_runtime)(xe);
160	151	for_each_gt(gt, xe, id) {
161	152	flags = live_mocs_init(&mocs, gt);
162	153	kunit_info(test, "mocs_reset_test before reset\n");
@@ -174,8 +165,6 @@ static int mocs_reset_test_run_device(struct xe_device *xe) · 11 unmodified lines
174	165	read_l3cc_table(gt, &mocs.table);
175	166	}
176	167
177		− xe_pm_runtime_put(xe);
178		−
179	168	return 0;
180	169	}
181	170

drivers/gpu/drm/xe/xe_args.h+27−0

@@ -121,6 +121,33 @@ · 120 unmodified lines
121	121	#define PICK_ARG11(args...) PICK_ARG10(DROP_FIRST_ARG(args))
122	122	#define PICK_ARG12(args...) PICK_ARG11(DROP_FIRST_ARG(args))
123	123
	124	+/**
	125	+ * IF_ARGS() - Make selection based on optional argument list.
	126	+ * @then: token to return if arguments are present
	127	+ * @else: token to return if arguments are empty
	128	+ * @...: arguments to check (optional)
	129	+ *
	130	+ * This macro allows to select a token based on the presence of the argument list.
	131	+ *
	132	+ * Example:
	133	+ *
	134	+ * #define foo X, Y
	135	+ * #define bar IF_ARGS(Z, Q, foo)
	136	+ * #define buz IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo)))
	137	+ *
	138	+ * With above definitions bar expands to Z while buz expands to Q.
	139	+ */
	140	+#if defined(CONFIG_CC_IS_CLANG) \|\| GCC_VERSION >= 100100
	141	+#define IF_ARGS(then, else, ...) FIRST_ARG(__VA_OPT__(then,) else)
	142	+#else
	143	+#define IF_ARGS(then, else, ...) _IF_ARGS(then, else, CALL_ARGS(FIRST_ARG, __VA_ARGS__))
	144	+#define _IF_ARGS(then, else, ...) __IF_ARGS(then, else, CALL_ARGS(COUNT_ARGS, __VA_ARGS__))
	145	+#define __IF_ARGS(then, else, n) ___IF_ARGS(then, else, CALL_ARGS(CONCATENATE, ___IF_ARG, n))
	146	+#define ___IF_ARGS(then, else, if) CALL_ARGS(if, then, else)
	147	+#define ___IF_ARG1(then, else) then
	148	+#define ___IF_ARG0(then, else) else
	149	+#endif
	150	+
124	151	/**
125	152	* ARGS_SEP_COMMA - Definition of a comma character.
126	153	*

drivers/gpu/drm/xe/xe_bo.c+16−10

@@ -516,8 +516,7 @@ static struct ttm_tt xe_ttm_tt_create(struct ttm_buffer_object ttm_bo, · 515 unmodified lines
516	516	* non-coherent and require a CPU:WC mapping.
517	517	*/
518	518	if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) \|\|
519		− (xe->info.graphics_verx100 >= 1270 &&
520		− bo->flags & XE_BO_FLAG_PAGETABLE))
	519	+ (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE))
521	520	caching = ttm_write_combined;
522	521	}
523	522
@@ -2026,13 +2025,9 @@ static int xe_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, · 1502 unmodified lines
2026	2025	struct ttm_buffer_object *ttm_bo = vma->vm_private_data;
2027	2026	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
2028	2027	struct xe_device *xe = xe_bo_device(bo);
2029		− int ret;
2030		−
2031		− xe_pm_runtime_get(xe);
2032		− ret = ttm_bo_vm_access(vma, addr, buf, len, write);
2033		− xe_pm_runtime_put(xe);
2034	2028
2035		− return ret;
	2029	+ guard(xe_pm_runtime)(xe);
	2030	+ return ttm_bo_vm_access(vma, addr, buf, len, write);
2036	2031	}
2037	2032
2038	2033	/**
@@ -3176,7 +3171,8 @@ int xe_gem_create_ioctl(struct drm_device dev, void data, · 1137 unmodified lines
3176	3171	if (XE_IOCTL_DBG(xe, args->flags &
3177	3172	~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING \|
3178	3173	DRM_XE_GEM_CREATE_FLAG_SCANOUT \|
3179		− DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM)))
	3174	+ DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM \|
	3175	+ DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION)))
3180	3176	return -EINVAL;
3181	3177
3182	3178	if (XE_IOCTL_DBG(xe, args->handle))
@@ -3198,6 +3194,12 @@ int xe_gem_create_ioctl(struct drm_device dev, void data, · 15 unmodified lines
3198	3194	if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)
3199	3195	bo_flags \|= XE_BO_FLAG_SCANOUT;
3200	3196
	3197	+ if (args->flags & DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION) {
	3198	+ if (XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20))
	3199	+ return -EOPNOTSUPP;
	3200	+ bo_flags \|= XE_BO_FLAG_NO_COMPRESSION;
	3201	+ }
	3202	+
3201	3203	bo_flags \|= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1);
3202	3204
3203	3205	/* CCS formats need physical placement at a 64K alignment in VRAM. */
@@ -3519,8 +3521,12 @@ bool xe_bo_needs_ccs_pages(struct xe_bo *bo) · 315 unmodified lines
3519	3521	* Compression implies coh_none, therefore we know for sure that WB
3520	3522	* memory can't currently use compression, which is likely one of the
3521	3523	* common cases.
	3524	+ * Additionally, userspace may explicitly request no compression via the
	3525	+ * DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION flag, which should also disable
	3526	+ * CCS usage.
3522	3527	*/
3523		− if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)
	3528	+ if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB \|\|
	3529	+ bo->flags & XE_BO_FLAG_NO_COMPRESSION)
3524	3530	return false;
3525	3531
3526	3532	return true;

drivers/gpu/drm/xe/xe_bo.h+1−0

@@ -50,6 +50,7 @@ · 49 unmodified lines
50	50	#define XE_BO_FLAG_GGTT3 BIT(23)
51	51	#define XE_BO_FLAG_CPU_ADDR_MIRROR BIT(24)
52	52	#define XE_BO_FLAG_FORCE_USER_VRAM BIT(25)
	53	+#define XE_BO_FLAG_NO_COMPRESSION BIT(26)
53	54
54	55	/* this one is trigger internally only */
55	56	#define XE_BO_FLAG_INTERNAL_TEST BIT(30)

drivers/gpu/drm/xe/xe_debugfs.c+121−22

@@ -68,7 +68,7 @@ static int info(struct seq_file m, void data) · 67 unmodified lines
68	68	struct xe_gt *gt;
69	69	u8 id;
70	70
71		− xe_pm_runtime_get(xe);
	71	+ guard(xe_pm_runtime)(xe);
72	72
73	73	drm_printf(&p, "graphics_verx100 %d\n", xe->info.graphics_verx100);
74	74	drm_printf(&p, "media_verx100 %d\n", xe->info.media_verx100);
@@ -93,9 +93,10 @@ static int info(struct seq_file m, void data) · 18 unmodified lines
93	93	xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT));
94	94	drm_printf(&p, "gt%d engine_mask 0x%llx\n", id,
95	95	gt->info.engine_mask);
	96	+ drm_printf(&p, "gt%d multi_queue_engine_class_mask 0x%x\n", id,
	97	+ gt->info.multi_queue_engine_class_mask);
96	98	}
97	99
98		− xe_pm_runtime_put(xe);
99	100	return 0;
100	101	}
101	102
@@ -110,9 +111,8 @@ static int sriov_info(struct seq_file m, void data) · 8 unmodified lines
110	111
111	112	static int workarounds(struct xe_device xe, struct drm_printer p)
112	113	{
113		− xe_pm_runtime_get(xe);
	114	+ guard(xe_pm_runtime)(xe);
114	115	xe_wa_device_dump(xe, p);
115		− xe_pm_runtime_put(xe);
116	116
117	117	return 0;
118	118	}
@@ -134,7 +134,7 @@ static int dgfx_pkg_residencies_show(struct seq_file m, void data) · 15 unmodified lines
134	134
135	135	xe = node_to_xe(m->private);
136	136	p = drm_seq_file_printer(m);
137		− xe_pm_runtime_get(xe);
	137	+ guard(xe_pm_runtime)(xe);
138	138	mmio = xe_root_tile_mmio(xe);
139	139	static const struct {
140	140	u32 offset;
@@ -151,7 +151,6 @@ static int dgfx_pkg_residencies_show(struct seq_file m, void data) · 10 unmodified lines
151	151	for (int i = 0; i < ARRAY_SIZE(residencies); i++)
152	152	read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p);
153	153
154		− xe_pm_runtime_put(xe);
155	154	return 0;
156	155	}
157	156
@@ -163,7 +162,7 @@ static int dgfx_pcie_link_residencies_show(struct seq_file m, void data) · 5 unmodified lines
163	162
164	163	xe = node_to_xe(m->private);
165	164	p = drm_seq_file_printer(m);
166		− xe_pm_runtime_get(xe);
	165	+ guard(xe_pm_runtime)(xe);
167	166	mmio = xe_root_tile_mmio(xe);
168	167
169	168	static const struct {
@@ -178,7 +177,6 @@ static int dgfx_pcie_link_residencies_show(struct seq_file m, void data) · 8 unmodified lines
178	177	for (int i = 0; i < ARRAY_SIZE(residencies); i++)
179	178	read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p);
180	179
181		− xe_pm_runtime_put(xe);
182	180	return 0;
183	181	}
184	182
@@ -277,16 +275,14 @@ static ssize_t wedged_mode_set(struct file f, const char __user ubuf, · 92 unmodified lines
277	275
278	276	xe->wedged.mode = wedged_mode;
279	277
280		− xe_pm_runtime_get(xe);
	278	+ guard(xe_pm_runtime)(xe);
281	279	for_each_gt(gt, xe, id) {
282	280	ret = xe_guc_ads_scheduler_policy_toggle_reset(&gt->uc.guc.ads);
283	281	if (ret) {
284	282	xe_gt_err(gt, "Failed to update GuC ADS scheduler policy. GuC may still cause engine reset even with wedged_mode=2\n");
285		− xe_pm_runtime_put(xe);
286	283	return -EIO;
287	284	}
288	285	}
289		− xe_pm_runtime_put(xe);
290	286
291	287	return size;
292	288	}
@@ -297,6 +293,39 @@ static const struct file_operations wedged_mode_fops = { · 4 unmodified lines
297	293	.write = wedged_mode_set,
298	294	};
299	295
	296	+static ssize_t page_reclaim_hw_assist_show(struct file f, char __user ubuf,
	297	+ size_t size, loff_t *pos)
	298	+{
	299	+ struct xe_device *xe = file_inode(f)->i_private;
	300	+ char buf[8];
	301	+ int len;
	302	+
	303	+ len = scnprintf(buf, sizeof(buf), "%d\n", xe->info.has_page_reclaim_hw_assist);
	304	+ return simple_read_from_buffer(ubuf, size, pos, buf, len);
	305	+}
	306	+
	307	+static ssize_t page_reclaim_hw_assist_set(struct file f, const char __user ubuf,
	308	+ size_t size, loff_t *pos)
	309	+{
	310	+ struct xe_device *xe = file_inode(f)->i_private;
	311	+ bool val;
	312	+ ssize_t ret;
	313	+
	314	+ ret = kstrtobool_from_user(ubuf, size, &val);
	315	+ if (ret)
	316	+ return ret;
	317	+
	318	+ xe->info.has_page_reclaim_hw_assist = val;
	319	+
	320	+ return size;
	321	+}
	322	+
	323	+static const struct file_operations page_reclaim_hw_assist_fops = {
	324	+ .owner = THIS_MODULE,
	325	+ .read = page_reclaim_hw_assist_show,
	326	+ .write = page_reclaim_hw_assist_set,
	327	+};
	328	+
300	329	static ssize_t atomic_svm_timeslice_ms_show(struct file f, char __user ubuf,
301	330	size_t size, loff_t *pos)
302	331	{
@@ -332,6 +361,74 @@ static const struct file_operations atomic_svm_timeslice_ms_fops = { · 29 unmodified lines
332	361	.write = atomic_svm_timeslice_ms_set,
333	362	};
334	363
	364	+static ssize_t min_run_period_lr_ms_show(struct file f, char __user ubuf,
	365	+ size_t size, loff_t *pos)
	366	+{
	367	+ struct xe_device *xe = file_inode(f)->i_private;
	368	+ char buf[32];
	369	+ int len = 0;
	370	+
	371	+ len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_lr_ms);
	372	+
	373	+ return simple_read_from_buffer(ubuf, size, pos, buf, len);
	374	+}
	375	+
	376	+static ssize_t min_run_period_lr_ms_set(struct file f, const char __user ubuf,
	377	+ size_t size, loff_t *pos)
	378	+{
	379	+ struct xe_device *xe = file_inode(f)->i_private;
	380	+ u32 min_run_period_lr_ms;
	381	+ ssize_t ret;
	382	+
	383	+ ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_lr_ms);
	384	+ if (ret)
	385	+ return ret;
	386	+
	387	+ xe->min_run_period_lr_ms = min_run_period_lr_ms;
	388	+
	389	+ return size;
	390	+}
	391	+
	392	+static const struct file_operations min_run_period_lr_ms_fops = {
	393	+ .owner = THIS_MODULE,
	394	+ .read = min_run_period_lr_ms_show,
	395	+ .write = min_run_period_lr_ms_set,
	396	+};
	397	+
	398	+static ssize_t min_run_period_pf_ms_show(struct file f, char __user ubuf,
	399	+ size_t size, loff_t *pos)
	400	+{
	401	+ struct xe_device *xe = file_inode(f)->i_private;
	402	+ char buf[32];
	403	+ int len = 0;
	404	+
	405	+ len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_pf_ms);
	406	+
	407	+ return simple_read_from_buffer(ubuf, size, pos, buf, len);
	408	+}
	409	+
	410	+static ssize_t min_run_period_pf_ms_set(struct file f, const char __user ubuf,
	411	+ size_t size, loff_t *pos)
	412	+{
	413	+ struct xe_device *xe = file_inode(f)->i_private;
	414	+ u32 min_run_period_pf_ms;
	415	+ ssize_t ret;
	416	+
	417	+ ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_pf_ms);
	418	+ if (ret)
	419	+ return ret;
	420	+
	421	+ xe->min_run_period_pf_ms = min_run_period_pf_ms;
	422	+
	423	+ return size;
	424	+}
	425	+
	426	+static const struct file_operations min_run_period_pf_ms_fops = {
	427	+ .owner = THIS_MODULE,
	428	+ .read = min_run_period_pf_ms_show,
	429	+ .write = min_run_period_pf_ms_set,
	430	+};
	431	+
335	432	static ssize_t disable_late_binding_show(struct file f, char __user ubuf,
336	433	size_t size, loff_t *pos)
337	434	{
@@ -375,7 +472,6 @@ void xe_debugfs_register(struct xe_device *xe) · 37 unmodified lines
375	472	struct ttm_resource_manager *man;
376	473	struct xe_tile *tile;
377	474	struct xe_gt *gt;
378		− u32 mem_type;
379	475	u8 tile_id;
380	476	u8 id;
381	477
@@ -400,19 +496,22 @@ void xe_debugfs_register(struct xe_device *xe) · 18 unmodified lines
400	496	debugfs_create_file("atomic_svm_timeslice_ms", 0600, root, xe,
401	497	&atomic_svm_timeslice_ms_fops);
402	498
403		− debugfs_create_file("disable_late_binding", 0600, root, xe,
404		− &disable_late_binding_fops);
	499	+ debugfs_create_file("min_run_period_lr_ms", 0600, root, xe,
	500	+ &min_run_period_lr_ms_fops);
405	501
406		− for (mem_type = XE_PL_VRAM0; mem_type <= XE_PL_VRAM1; ++mem_type) {
407		− man = ttm_manager_type(bdev, mem_type);
	502	+ debugfs_create_file("min_run_period_pf_ms", 0600, root, xe,
	503	+ &min_run_period_pf_ms_fops);
408	504
409		− if (man) {
410		− char name[16];
	505	+ debugfs_create_file("disable_late_binding", 0600, root, xe,
	506	+ &disable_late_binding_fops);
411	507
412		− snprintf(name, sizeof(name), "vram%d_mm", mem_type - XE_PL_VRAM0);
413		− ttm_resource_manager_create_debugfs(man, root, name);
414		− }
415		− }
	508	+ /*
	509	+ * Don't expose page reclaim configuration file if not supported by the
	510	+ * hardware initially.
	511	+ */
	512	+ if (xe->info.has_page_reclaim_hw_assist)
	513	+ debugfs_create_file("page_reclaim_hw_assist", 0600, root, xe,
	514	+ &page_reclaim_hw_assist_fops);
416	515
417	516	man = ttm_manager_type(bdev, XE_PL_TT);
418	517	ttm_resource_manager_create_debugfs(man, root, "gtt_mm");

drivers/gpu/drm/xe/xe_devcoredump.c+12−18

@@ -276,7 +276,6 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) · 275 unmodified lines
276	276	struct xe_devcoredump_snapshot ss = container_of(work, typeof(ss), work);
277	277	struct xe_devcoredump coredump = container_of(ss, typeof(coredump), snapshot);
278	278	struct xe_device *xe = coredump_to_xe(coredump);
279		− unsigned int fw_ref;
280	279
281	280	/*
282	281	* NB: Despite passing a GFP_ flags parameter here, more allocations are done
@@ -287,15 +286,15 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) · 4 unmodified lines
287	286	xe_devcoredump_read, xe_devcoredump_free,
288	287	XE_COREDUMP_TIMEOUT_JIFFIES);
289	288
290		− xe_pm_runtime_get(xe);
	289	+ guard(xe_pm_runtime)(xe);
291	290
292	291	/* keep going if fw fails as we still want to save the memory and SW data */
293		− fw_ref = xe_force_wake_get(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL);
294		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
295		− xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
296		− xe_vm_snapshot_capture_delayed(ss->vm);
297		− xe_guc_exec_queue_snapshot_capture_delayed(ss->ge);
298		− xe_force_wake_put(gt_to_fw(ss->gt), fw_ref);
	292	+ xe_with_force_wake(fw_ref, gt_to_fw(ss->gt), XE_FORCEWAKE_ALL) {
	293	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
	294	+ xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
	295	+ xe_vm_snapshot_capture_delayed(ss->vm);
	296	+ xe_guc_exec_queue_snapshot_capture_delayed(ss->ge);
	297	+ }
299	298
300	299	ss->read.chunk_position = 0;
301	300
@@ -306,7 +305,7 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) · 4 unmodified lines
306	305	ss->read.buffer = kvmalloc(XE_DEVCOREDUMP_CHUNK_MAX,
307	306	GFP_USER);
308	307	if (!ss->read.buffer)
309		− goto put_pm;
	308	+ return;
310	309
311	310	__xe_devcoredump_read(ss->read.buffer,
312	311	XE_DEVCOREDUMP_CHUNK_MAX,
@@ -314,15 +313,12 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work) · 1 unmodified line
314	313	} else {
315	314	ss->read.buffer = kvmalloc(ss->read.size, GFP_USER);
316	315	if (!ss->read.buffer)
317		− goto put_pm;
	316	+ return;
318	317
319	318	__xe_devcoredump_read(ss->read.buffer, ss->read.size, 0,
320	319	coredump);
321	320	xe_devcoredump_snapshot_free(ss);
322	321	}
323		−
324		−put_pm:
325		− xe_pm_runtime_put(xe);
326	322	}
327	323
328	324	static void devcoredump_snapshot(struct xe_devcoredump *coredump,
@@ -332,7 +328,6 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, · 3 unmodified lines
332	328	struct xe_devcoredump_snapshot *ss = &coredump->snapshot;
333	329	struct xe_guc *guc = exec_queue_to_guc(q);
334	330	const char *process_name = "no process";
335		− unsigned int fw_ref;
336	331	bool cookie;
337	332
338	333	ss->snapshot_time = ktime_get_real();
@@ -348,10 +343,10 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, · 9 unmodified lines
348	343	ss->gt = q->gt;
349	344	INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work);
350	345
351		− cookie = dma_fence_begin_signalling();
352		−
353	346	/* keep going if fw fails as we still want to save the memory and SW data */
354		− fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
	347	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
	348	+
	349	+ cookie = dma_fence_begin_signalling();
355	350
356	351	ss->guc.log = xe_guc_log_snapshot_capture(&guc->log, true);
357	352	ss->guc.ct = xe_guc_ct_snapshot_capture(&guc->ct);
@@ -364,7 +359,6 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, · 6 unmodified lines
364	359
365	360	queue_work(system_unbound_wq, &ss->work);
366	361
367		− xe_force_wake_put(gt_to_fw(q->gt), fw_ref);
368	362	dma_fence_end_signalling(cookie);
369	363	}
370	364

drivers/gpu/drm/xe/xe_device.c+77−23

@@ -166,7 +166,7 @@ static void xe_file_close(struct drm_device dev, struct drm_file file) · 165 unmodified lines
166	166	struct xe_exec_queue *q;
167	167	unsigned long idx;
168	168
169		− xe_pm_runtime_get(xe);
	169	+ guard(xe_pm_runtime)(xe);
170	170
171	171	/*
172	172	* No need for exec_queue.lock here as there is no contention for it
@@ -177,15 +177,18 @@ static void xe_file_close(struct drm_device dev, struct drm_file file) · 4 unmodified lines
177	177	xa_for_each(&xef->exec_queue.xa, idx, q) {
178	178	if (q->vm && q->hwe->hw_engine_group)
179	179	xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
180		− xe_exec_queue_kill(q);
	180	+
	181	+ if (xe_exec_queue_is_multi_queue_primary(q))
	182	+ xe_exec_queue_group_kill_put(q->multi_queue.group);
	183	+ else
	184	+ xe_exec_queue_kill(q);
	185	+
181	186	xe_exec_queue_put(q);
182	187	}
183	188	xa_for_each(&xef->vm.xa, idx, vm)
184	189	xe_vm_close_and_put(vm);
185	190
186	191	xe_file_put(xef);
187		−
188		− xe_pm_runtime_put(xe);
189	192	}
190	193
191	194	static const struct drm_ioctl_desc xe_ioctls[] = {
@@ -209,6 +212,8 @@ static const struct drm_ioctl_desc xe_ioctls[] = { · 17 unmodified lines
209	212	DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
210	213	DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
211	214	DRM_RENDER_ALLOW),
	215	+ DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,
	216	+ DRM_RENDER_ALLOW),
212	217	};
213	218
214	219	static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
@@ -220,10 +225,10 @@ static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) · 5 unmodified lines
220	225	if (xe_device_wedged(xe))
221	226	return -ECANCELED;
222	227
223		− ret = xe_pm_runtime_get_ioctl(xe);
	228	+ ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
	229	+ ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
224	230	if (ret >= 0)
225	231	ret = drm_ioctl(file, cmd, arg);
226		− xe_pm_runtime_put(xe);
227	232
228	233	return ret;
229	234	}
@@ -238,10 +243,10 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo · 8 unmodified lines
238	243	if (xe_device_wedged(xe))
239	244	return -ECANCELED;
240	245
241		− ret = xe_pm_runtime_get_ioctl(xe);
	246	+ ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
	247	+ ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
242	248	if (ret >= 0)
243	249	ret = drm_compat_ioctl(file, cmd, arg);
244		− xe_pm_runtime_put(xe);
245	250
246	251	return ret;
247	252	}
@@ -455,6 +460,7 @@ struct xe_device xe_device_create(struct pci_dev pdev, · 207 unmodified lines
455	460	xe->info.revid = pdev->revision;
456	461	xe->info.force_execlist = xe_modparam.force_execlist;
457	462	xe->atomic_svm_timeslice_ms = 5;
	463	+ xe->min_run_period_lr_ms = 5;
458	464
459	465	err = xe_irq_init(xe);
460	466	if (err)
@@ -775,7 +781,6 @@ ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ · 314 unmodified lines
775	781	static int probe_has_flat_ccs(struct xe_device *xe)
776	782	{
777	783	struct xe_gt *gt;
778		− unsigned int fw_ref;
779	784	u32 reg;
780	785
781	786	/* Always enabled/disabled, no runtime check to do */
@@ -786,8 +791,8 @@ static int probe_has_flat_ccs(struct xe_device *xe) · 4 unmodified lines
786	791	if (!gt)
787	792	return 0;
788	793
789		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
790		− if (!fw_ref)
	794	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	795	+ if (!fw_ref.domains)
791	796	return -ETIMEDOUT;
792	797
793	798	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
@@ -797,11 +802,64 @@ static int probe_has_flat_ccs(struct xe_device *xe) · 3 unmodified lines
797	802	drm_dbg(&xe->drm,
798	803	"Flat CCS has been disabled in bios, May lead to performance impact");
799	804
800		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
801		−
802	805	return 0;
803	806	}
804	807
	808	+/*
	809	+ * Detect if the driver is being run on pre-production hardware. We don't
	810	+ * keep workarounds for pre-production hardware long term, so print an
	811	+ * error and add taint if we're being loaded on a pre-production platform
	812	+ * for which the pre-prod workarounds have already been removed.
	813	+ *
	814	+ * The general policy is that we'll remove any workarounds that only apply to
	815	+ * pre-production hardware around the time force_probe restrictions are lifted
	816	+ * for a platform of the next major IP generation (for example, Xe2 pre-prod
	817	+ * workarounds should be removed around the time the first Xe3 platforms have
	818	+ * force_probe lifted).
	819	+ */
	820	+static void detect_preproduction_hw(struct xe_device *xe)
	821	+{
	822	+ struct xe_gt *gt;
	823	+ int id;
	824	+
	825	+ /*
	826	+ * SR-IOV VFs don't have access to the FUSE2 register, so we can't
	827	+ * check pre-production status there. But the host OS will notice
	828	+ * and report the pre-production status, which should be enough to
	829	+ * help us catch mistaken use of pre-production hardware.
	830	+ */
	831	+ if (IS_SRIOV_VF(xe))
	832	+ return;
	833	+
	834	+ /*
	835	+ * The "SW_CAP" fuse contains a bit indicating whether the device is a
	836	+ * production or pre-production device. This fuse is reflected through
	837	+ * the GT "FUSE2" register, even though the contents of the fuse are
	838	+ * not GT-specific. Every GT's reflection of this fuse should show the
	839	+ * same value, so we'll just use the first available GT for lookup.
	840	+ */
	841	+ for_each_gt(gt, xe, id)
	842	+ break;
	843	+
	844	+ if (!gt)
	845	+ return;
	846	+
	847	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	848	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) {
	849	+ xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n");
	850	+ return;
	851	+ }
	852	+
	853	+ if (xe_mmio_read32(&gt->mmio, FUSE2) & PRODUCTION_HW)
	854	+ return;
	855	+
	856	+ xe_info(xe, "Pre-production hardware detected.\n");
	857	+ if (!xe->info.has_pre_prod_wa) {
	858	+ xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n");
	859	+ add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
	860	+ }
	861	+}
	862	+
805	863	int xe_device_probe(struct xe_device *xe)
806	864	{
807	865	struct xe_tile *tile;
@@ -972,6 +1030,8 @@ int xe_device_probe(struct xe_device *xe) · 164 unmodified lines
972	1030	if (err)
973	1031	goto err_unregister_display;
974	1032
	1033	+ detect_preproduction_hw(xe);
	1034	+
975	1035	return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
976	1036
977	1037	err_unregister_display:
@@ -1034,7 +1094,6 @@ void xe_device_wmb(struct xe_device *xe) · 56 unmodified lines
1034	1094	*/
1035	1095	static void tdf_request_sync(struct xe_device *xe)
1036	1096	{
1037		− unsigned int fw_ref;
1038	1097	struct xe_gt *gt;
1039	1098	u8 id;
1040	1099
@@ -1042,8 +1101,8 @@ static void tdf_request_sync(struct xe_device *xe) · 1 unmodified line
1042	1101	if (xe_gt_is_media_type(gt))
1043	1102	continue;
1044	1103
1045		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1046		− if (!fw_ref)
	1104	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	1105	+ if (!fw_ref.domains)
1047	1106	return;
1048	1107
1049	1108	xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
@@ -1058,15 +1117,12 @@ static void tdf_request_sync(struct xe_device *xe) · 8 unmodified lines
1058	1117	if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
1059	1118	300, NULL, false))
1060	1119	xe_gt_err_once(gt, "TD flush timeout\n");
1061		−
1062		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
1063	1120	}
1064	1121	}
1065	1122
1066	1123	void xe_device_l2_flush(struct xe_device *xe)
1067	1124	{
1068	1125	struct xe_gt *gt;
1069		− unsigned int fw_ref;
1070	1126
1071	1127	gt = xe_root_mmio_gt(xe);
1072	1128	if (!gt)
@@ -1075,8 +1131,8 @@ void xe_device_l2_flush(struct xe_device *xe) · 2 unmodified lines
1075	1131	if (!XE_GT_WA(gt, 16023588340))
1076	1132	return;
1077	1133
1078		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1079		− if (!fw_ref)
	1134	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	1135	+ if (!fw_ref.domains)
1080	1136	return;
1081	1137
1082	1138	spin_lock(&gt->global_invl_lock);
@@ -1086,8 +1142,6 @@ void xe_device_l2_flush(struct xe_device *xe) · 3 unmodified lines
1086	1142	xe_gt_err_once(gt, "Global invalidation timeout\n");
1087	1143
1088	1144	spin_unlock(&gt->global_invl_lock);
1089		−
1090		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
1091	1145	}
1092	1146
1093	1147	/**

drivers/gpu/drm/xe/xe_device.h+5−0

@@ -172,6 +172,11 @@ static inline bool xe_device_has_lmtt(struct xe_device *xe) · 171 unmodified lines
172	172	return IS_DGFX(xe);
173	173	}
174	174
	175	+static inline bool xe_device_has_mert(struct xe_device *xe)
	176	+{
	177	+ return xe->info.has_mert;
	178	+}
	179	+
175	180	u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size);
176	181
177	182	void xe_device_snapshot_print(struct xe_device xe, struct drm_printer p);

drivers/gpu/drm/xe/xe_device_sysfs.c+13−20

@@ -57,9 +57,8 @@ vram_d3cold_threshold_store(struct device dev, struct device_attribute attr, · 56 unmodified lines
57	57
58	58	drm_dbg(&xe->drm, "vram_d3cold_threshold: %u\n", vram_d3cold_threshold);
59	59
60		− xe_pm_runtime_get(xe);
	60	+ guard(xe_pm_runtime)(xe);
61	61	ret = xe_pm_set_vram_threshold(xe, vram_d3cold_threshold);
62		− xe_pm_runtime_put(xe);
63	62
64	63	return ret ?: count;
65	64	}
@@ -84,33 +83,31 @@ lb_fan_control_version_show(struct device dev, struct device_attribute attr, c · 18 unmodified lines
84	83	u16 major = 0, minor = 0, hotfix = 0, build = 0;
85	84	int ret;
86	85
87		− xe_pm_runtime_get(xe);
	86	+ guard(xe_pm_runtime)(xe);
88	87
89	88	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0),
90	89	&cap, NULL);
91	90	if (ret)
92		− goto out;
	91	+ return ret;
93	92
94	93	if (REG_FIELD_GET(V1_FAN_PROVISIONED, cap)) {
95	94	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0),
96	95	&ver_low, NULL);
97	96	if (ret)
98		− goto out;
	97	+ return ret;
99	98
100	99	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0),
101	100	&ver_high, NULL);
102	101	if (ret)
103		− goto out;
	102	+ return ret;
104	103
105	104	major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low);
106	105	minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low);
107	106	hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high);
108	107	build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high);
109	108	}
110		−out:
111		− xe_pm_runtime_put(xe);
112	109
113		− return ret ?: sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
	110	+ return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
114	111	}
115	112	static DEVICE_ATTR_ADMIN_RO(lb_fan_control_version);
116	113
@@ -123,33 +120,31 @@ lb_voltage_regulator_version_show(struct device dev, struct device_attribute a · 6 unmodified lines
123	120	u16 major = 0, minor = 0, hotfix = 0, build = 0;
124	121	int ret;
125	122
126		− xe_pm_runtime_get(xe);
	123	+ guard(xe_pm_runtime)(xe);
127	124
128	125	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0),
129	126	&cap, NULL);
130	127	if (ret)
131		− goto out;
	128	+ return ret;
132	129
133	130	if (REG_FIELD_GET(VR_PARAMS_PROVISIONED, cap)) {
134	131	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0),
135	132	&ver_low, NULL);
136	133	if (ret)
137		− goto out;
	134	+ return ret;
138	135
139	136	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0),
140	137	&ver_high, NULL);
141	138	if (ret)
142		− goto out;
	139	+ return ret;
143	140
144	141	major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low);
145	142	minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low);
146	143	hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high);
147	144	build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high);
148	145	}
149		−out:
150		− xe_pm_runtime_put(xe);
151	146
152		− return ret ?: sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
	147	+ return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
153	148	}
154	149	static DEVICE_ATTR_ADMIN_RO(lb_voltage_regulator_version);
155	150
@@ -233,9 +228,8 @@ auto_link_downgrade_capable_show(struct device dev, struct device_attribute at · 77 unmodified lines
233	228	struct xe_device *xe = pdev_to_xe_device(pdev);
234	229	u32 cap, val;
235	230
236		− xe_pm_runtime_get(xe);
	231	+ guard(xe_pm_runtime)(xe);
237	232	val = xe_mmio_read32(xe_root_tile_mmio(xe), BMG_PCIE_CAP);
238		− xe_pm_runtime_put(xe);
239	233
240	234	cap = REG_FIELD_GET(LINK_DOWNGRADE, val);
241	235	return sysfs_emit(buf, "%u\n", cap == DOWNGRADE_CAPABLE);
@@ -251,11 +245,10 @@ auto_link_downgrade_status_show(struct device dev, struct device_attribute att · 9 unmodified lines
251	245	u32 val = 0;
252	246	int ret;
253	247
254		− xe_pm_runtime_get(xe);
	248	+ guard(xe_pm_runtime)(xe);
255	249	ret = xe_pcode_read(xe_device_get_root_tile(xe),
256	250	PCODE_MBOX(DGFX_PCODE_STATUS, DGFX_GET_INIT_STATUS, 0),
257	251	&val, NULL);
258		− xe_pm_runtime_put(xe);
259	252
260	253	return ret ?: sysfs_emit(buf, "%u\n", REG_FIELD_GET(DGFX_LINK_DOWNGRADE_STATUS, val));
261	254	}

drivers/gpu/drm/xe/xe_device_types.h+27−0

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include "xe_late_bind_fw_types.h"
18	18	#include "xe_lmtt_types.h"
19	19	#include "xe_memirq_types.h"
	20	+#include "xe_mert.h"
20	21	#include "xe_oa_types.h"
21	22	#include "xe_pagefault_types.h"
22	23	#include "xe_platform_types.h"
@@ -183,6 +184,13 @@ struct xe_tile { · 160 unmodified lines
183	184	* Media GT shares a pool with its primary GT.
184	185	*/
185	186	struct xe_sa_manager *kernel_bb_pool;
	187	+
	188	+ /**
	189	+ * @mem.reclaim_pool: Pool for PRLs allocated.
	190	+ *
	191	+ * Only main GT has page reclaim list allocations.
	192	+ */
	193	+ struct xe_sa_manager *reclaim_pool;
186	194	} mem;
187	195
188	196	/** @sriov: tile level virtualization data */
@@ -219,6 +227,9 @@ struct xe_tile { · 30 unmodified lines
219	227
220	228	/** @debugfs: debugfs directory associated with this tile */
221	229	struct dentry *debugfs;
	230	+
	231	+ /** @mert: MERT-related data */
	232	+ struct xe_mert mert;
222	233	};
223	234
224	235	/**
@@ -285,6 +296,8 @@ struct xe_device { · 60 unmodified lines
285	296	u8 has_asid:1;
286	297	/** @info.has_atomic_enable_pte_bit: Device has atomic enable PTE bit */
287	298	u8 has_atomic_enable_pte_bit:1;
	299	+ /** @info.has_cached_pt: Supports caching pagetable */
	300	+ u8 has_cached_pt:1;
288	301	/** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */
289	302	u8 has_device_atomics_on_smem:1;
290	303	/** @info.has_fan_control: Device supports fan control */
@@ -297,6 +310,8 @@ struct xe_device { · 6 unmodified lines
297	310	u8 has_heci_cscfi:1;
298	311	/** @info.has_heci_gscfi: device has heci gscfi */
299	312	u8 has_heci_gscfi:1;
	313	+ /** @info.has_i2c: Device has I2C controller */
	314	+ u8 has_i2c:1;
300	315	/** @info.has_late_bind: Device has firmware late binding support */
301	316	u8 has_late_bind:1;
302	317	/** @info.has_llc: Device has a shared CPU+GPU last level cache */
@@ -307,6 +322,12 @@ struct xe_device { · 4 unmodified lines
307	322	u8 has_mbx_power_limits:1;
308	323	/** @info.has_mem_copy_instr: Device supports MEM_COPY instruction */
309	324	u8 has_mem_copy_instr:1;
	325	+ /** @info.has_mert: Device has standalone MERT */
	326	+ u8 has_mert:1;
	327	+ /** @info.has_page_reclaim_hw_assist: Device supports page reclamation feature */
	328	+ u8 has_page_reclaim_hw_assist:1;
	329	+ /** @info.has_pre_prod_wa: Pre-production workarounds still present in driver */
	330	+ u8 has_pre_prod_wa:1;
310	331	/** @info.has_pxp: Device has PXP support */
311	332	u8 has_pxp:1;
312	333	/** @info.has_range_tlb_inval: Has range based TLB invalidations */
@@ -605,6 +626,12 @@ struct xe_device { · 292 unmodified lines
605	626	/** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */
606	627	u32 atomic_svm_timeslice_ms;
607	628
	629	+ /** @min_run_period_lr_ms: LR VM (preempt fence mode) timeslice */
	630	+ u32 min_run_period_lr_ms;
	631	+
	632	+ /** @min_run_period_pf_ms: LR VM (page fault mode) timeslice */
	633	+ u32 min_run_period_pf_ms;
	634	+
608	635	#ifdef TEST_VM_OPS_ERROR
609	636	/**
610	637	* @vm_inject_error_position: inject errors at different places in VM

drivers/gpu/drm/xe/xe_drm_client.c+31−36

@@ -285,32 +285,31 @@ static struct xe_hw_engine any_engine(struct xe_device xe) · 284 unmodified lines
285	285	return NULL;
286	286	}
287	287
288		−static bool force_wake_get_any_engine(struct xe_device *xe,
289		− struct xe_hw_engine **phwe,
290		− unsigned int *pfw_ref)
	288	+/*
	289	+ * Pick any engine and grab its forcewake. On error phwe will be NULL and
	290	+ * the returned forcewake reference will be invalid. Callers should check
	291	+ * phwe against NULL.
	292	+ */
	293	+static struct xe_force_wake_ref force_wake_get_any_engine(struct xe_device *xe,
	294	+ struct xe_hw_engine **phwe)
291	295	{
292	296	enum xe_force_wake_domains domain;
293		− unsigned int fw_ref;
	297	+ struct xe_force_wake_ref fw_ref = {};
294	298	struct xe_hw_engine *hwe;
295		− struct xe_force_wake *fw;
	299	+
	300	+ *phwe = NULL;
296	301
297	302	hwe = any_engine(xe);
298	303	if (!hwe)
299		− return false;
	304	+ return fw_ref; /* will be invalid */
300	305
301	306	domain = xe_hw_engine_to_fw_domain(hwe);
302		− fw = gt_to_fw(hwe->gt);
303		−
304		− fw_ref = xe_force_wake_get(fw, domain);
305		− if (!xe_force_wake_ref_has_domain(fw_ref, domain)) {
306		− xe_force_wake_put(fw, fw_ref);
307		− return false;
308		− }
309	307
310		− *phwe = hwe;
311		− *pfw_ref = fw_ref;
	308	+ fw_ref = xe_force_wake_constructor(gt_to_fw(hwe->gt), domain);
	309	+ if (xe_force_wake_ref_has_domain(fw_ref.domains, domain))
	310	+ phwe = hwe; / valid forcewake */
312	311
313		− return true;
	312	+ return fw_ref;
314	313	}
315	314
316	315	static void show_run_ticks(struct drm_printer p, struct drm_file file)
@@ -322,7 +321,6 @@ static void show_run_ticks(struct drm_printer p, struct drm_file file) · 5 unmodified lines
322	321	struct xe_hw_engine *hwe;
323	322	struct xe_exec_queue *q;
324	323	u64 gpu_timestamp;
325		− unsigned int fw_ref;
326	324
327	325	/*
328	326	* RING_TIMESTAMP registers are inaccessible in VF mode.
@@ -339,29 +337,26 @@ static void show_run_ticks(struct drm_printer p, struct drm_file file) · 10 unmodified lines
339	337	wait_var_event(&xef->exec_queue.pending_removal,
340	338	!atomic_read(&xef->exec_queue.pending_removal));
341	339
342		− xe_pm_runtime_get(xe);
343		− if (!force_wake_get_any_engine(xe, &hwe, &fw_ref)) {
344		− xe_pm_runtime_put(xe);
345		− return;
346		− }
347		−
348		− /* Accumulate all the exec queues from this client */
349		− mutex_lock(&xef->exec_queue.lock);
350		− xa_for_each(&xef->exec_queue.xa, i, q) {
351		− xe_exec_queue_get(q);
352		− mutex_unlock(&xef->exec_queue.lock);
353		−
354		− xe_exec_queue_update_run_ticks(q);
	340	+ scoped_guard(xe_pm_runtime, xe) {
	341	+ CLASS(xe_force_wake_release_only, fw_ref)(force_wake_get_any_engine(xe, &hwe));
	342	+ if (!hwe)
	343	+ return;
355	344
	345	+ /* Accumulate all the exec queues from this client */
356	346	mutex_lock(&xef->exec_queue.lock);
357		− xe_exec_queue_put(q);
358		− }
359		− mutex_unlock(&xef->exec_queue.lock);
	347	+ xa_for_each(&xef->exec_queue.xa, i, q) {
	348	+ xe_exec_queue_get(q);
	349	+ mutex_unlock(&xef->exec_queue.lock);
360	350
361		− gpu_timestamp = xe_hw_engine_read_timestamp(hwe);
	351	+ xe_exec_queue_update_run_ticks(q);
362	352
363		− xe_force_wake_put(gt_to_fw(hwe->gt), fw_ref);
364		− xe_pm_runtime_put(xe);
	353	+ mutex_lock(&xef->exec_queue.lock);
	354	+ xe_exec_queue_put(q);
	355	+ }
	356	+ mutex_unlock(&xef->exec_queue.lock);
	357	+
	358	+ gpu_timestamp = xe_hw_engine_read_timestamp(hwe);
	359	+ }
365	360
366	361	for (class = 0; class < XE_ENGINE_CLASS_MAX; class++) {
367	362	const char *class_name;

drivers/gpu/drm/xe/xe_exec.c+7−2

@@ -121,7 +121,7 @@ int xe_exec_ioctl(struct drm_device dev, void data, struct drm_file *file) · 120 unmodified lines
121	121	u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
122	122	struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
123	123	struct drm_exec *exec = &vm_exec.exec;
124		− u32 i, num_syncs, num_ufence = 0;
	124	+ u32 i, num_syncs, num_in_sync = 0, num_ufence = 0;
125	125	struct xe_validation_ctx ctx;
126	126	struct xe_sched_job *job;
127	127	struct xe_vm *vm;
@@ -183,6 +183,9 @@ int xe_exec_ioctl(struct drm_device dev, void data, struct drm_file *file) · 55 unmodified lines
183	183
184	184	if (xe_sync_is_ufence(&syncs[num_syncs]))
185	185	num_ufence++;
	186	+
	187	+ if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs]))
	188	+ num_in_sync++;
186	189	}
187	190
188	191	if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
@@ -203,7 +206,9 @@ int xe_exec_ioctl(struct drm_device dev, void data, struct drm_file *file) · 14 unmodified lines
203	206	mode = xe_hw_engine_group_find_exec_mode(q);
204	207
205	208	if (mode == EXEC_MODE_DMA_FENCE) {
206		− err = xe_hw_engine_group_get_mode(group, mode, &previous_mode);
	209	+ err = xe_hw_engine_group_get_mode(group, mode, &previous_mode,
	210	+ syncs, num_in_sync ?
	211	+ num_syncs : 0);
207	212	if (err)
208	213	goto err_syncs;
209	214	}

drivers/gpu/drm/xe/xe_exec_queue.c+441−14

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <drm/drm_syncobj.h>
14	14	#include <uapi/drm/xe_drm.h>
15	15
	16	+#include "xe_bo.h"
16	17	#include "xe_dep_scheduler.h"
17	18	#include "xe_device.h"
18	19	#include "xe_gt.h"
@@ -53,6 +54,54 @@ · 34 unmodified lines
53	54	* the ring operations the different engine classes support.
54	55	*/
55	56
	57	+/**
	58	+ * DOC: Multi Queue Group
	59	+ *
	60	+ * Multi Queue Group is another mode of execution supported by the compute
	61	+ * and blitter copy command streamers (CCS and BCS, respectively). It is
	62	+ * an enhancement of the existing hardware architecture and leverages the
	63	+ * same submission model. It enables support for efficient, parallel
	64	+ * execution of multiple queues within a single shared context. The multi
	65	+ * queue group functionality is only supported with GuC submission backend.
	66	+ * All the queues of a group must use the same address space (VM).
	67	+ *
	68	+ * The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property
	69	+ * supports creating a multi queue group and adding queues to a queue group.
	70	+ *
	71	+ * The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field
	72	+ * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with
	73	+ * the queue being created as the primary queue (aka q0) of the group. To add
	74	+ * secondary queues to the group, they need to be created with the above
	75	+ * property with id of the primary queue as the value. The properties of
	76	+ * the primary queue (like priority, time slice) applies to the whole group.
	77	+ * So, these properties can't be set for secondary queues of a group.
	78	+ *
	79	+ * The hardware does not support removing a queue from a multi-queue group.
	80	+ * However, queues can be dynamically added to the group. A group can have
	81	+ * up to 64 queues. To support this, XeKMD holds references to LRCs of the
	82	+ * queues even after the queues are destroyed by the user until the whole
	83	+ * group is destroyed. The secondary queues hold a reference to the primary
	84	+ * queue thus preventing the group from being destroyed when user destroys
	85	+ * the primary queue. Once the primary queue is destroyed, secondary queues
	86	+ * can't be added to the queue group, but they can continue to submit the
	87	+ * jobs if the DRM_XE_MULTI_GROUP_KEEP_ACTIVE flag is set during the multi
	88	+ * queue group creation.
	89	+ *
	90	+ * The queues of a multi queue group can set their priority within the group
	91	+ * through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property.
	92	+ * This multi queue priority can also be set dynamically through the
	93	+ * XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property
	94	+ * supported by the secondary queues of a multi queue group, other than
	95	+ * DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE.
	96	+ *
	97	+ * When GuC reports an error on any of the queues of a multi queue group,
	98	+ * the queue cleanup mechanism is invoked for all the queues of the group
	99	+ * as hardware cannot make progress on the multi queue context.
	100	+ *
	101	+ * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC
	102	+ * interface.
	103	+ */
	104	+
56	105	enum xe_exec_queue_sched_prop {
57	106	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
58	107	XE_EXEC_QUEUE_TIMESLICE = 1,
@@ -61,7 +110,35 @@ enum xe_exec_queue_sched_prop { · 2 unmodified lines
61	110	};
62	111
63	112	static int exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,
64		− u64 extensions, int ext_number);
	113	+ u64 extensions);
	114	+
	115	+static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q)
	116	+{
	117	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	118	+ struct xe_lrc *lrc;
	119	+ unsigned long idx;
	120	+
	121	+ if (xe_exec_queue_is_multi_queue_secondary(q)) {
	122	+ /*
	123	+ * Put pairs with get from xe_exec_queue_lookup() call
	124	+ * in xe_exec_queue_group_validate().
	125	+ */
	126	+ xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q));
	127	+ return;
	128	+ }
	129	+
	130	+ if (!group)
	131	+ return;
	132	+
	133	+ /* Primary queue cleanup */
	134	+ xa_for_each(&group->xa, idx, lrc)
	135	+ xe_lrc_put(lrc);
	136	+
	137	+ xa_destroy(&group->xa);
	138	+ mutex_destroy(&group->list_lock);
	139	+ xe_bo_unpin_map_no_vm(group->cgp_bo);
	140	+ kfree(group);
	141	+}
65	142
66	143	static void __xe_exec_queue_free(struct xe_exec_queue *q)
67	144	{
@@ -73,12 +150,17 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q) · 5 unmodified lines
73	150
74	151	if (xe_exec_queue_uses_pxp(q))
75	152	xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
	153	+
	154	+ if (xe_exec_queue_is_multi_queue(q))
	155	+ xe_exec_queue_group_cleanup(q);
	156	+
76	157	if (q->vm)
77	158	xe_vm_put(q->vm);
78	159
79	160	if (q->xef)
80	161	xe_file_put(q->xef);
81	162
	163	+ kvfree(q->replay_state);
82	164	kfree(q);
83	165	}
84	166
@@ -147,6 +229,7 @@ static struct xe_exec_queue __xe_exec_queue_alloc(struct xe_device xe, · 62 unmodified lines
147	229	INIT_LIST_HEAD(&q->multi_gt_link);
148	230	INIT_LIST_HEAD(&q->hw_engine_group_link);
149	231	INIT_LIST_HEAD(&q->pxp.link);
	232	+ q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL;
150	233
151	234	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
152	235	q->sched_props.preempt_timeout_us =
@@ -175,7 +258,7 @@ static struct xe_exec_queue __xe_exec_queue_alloc(struct xe_device xe, · 22 unmodified lines
175	258	* may set q->usm, must come before xe_lrc_create(),
176	259	* may overwrite q->sched_props, must come before q->ops->init()
177	260	*/
178		− err = exec_queue_user_extensions(xe, q, extensions, 0);
	261	+ err = exec_queue_user_extensions(xe, q, extensions);
179	262	if (err) {
180	263	__xe_exec_queue_free(q);
181	264	return ERR_PTR(err);
@@ -225,8 +308,8 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) · 43 unmodified lines
225	308	struct xe_lrc *lrc;
226	309
227	310	xe_gt_sriov_vf_wait_valid_ggtt(q->gt);
228		− lrc = xe_lrc_create(q->hwe, q->vm, xe_lrc_ring_size(),
229		− q->msix_vec, flags);
	311	+ lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state,
	312	+ xe_lrc_ring_size(), q->msix_vec, flags);
230	313	if (IS_ERR(lrc)) {
231	314	err = PTR_ERR(lrc);
232	315	goto err_lrc;
@@ -383,6 +466,26 @@ struct xe_exec_queue xe_exec_queue_create_bind(struct xe_device xe, · 150 unmodified lines
383	466	}
384	467	ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
385	468
	469	+static void xe_exec_queue_group_kill(struct kref *ref)
	470	+{
	471	+ struct xe_exec_queue_group *group = container_of(ref, struct xe_exec_queue_group,
	472	+ kill_refcount);
	473	+ xe_exec_queue_kill(group->primary);
	474	+}
	475	+
	476	+static inline void xe_exec_queue_group_kill_get(struct xe_exec_queue_group *group)
	477	+{
	478	+ kref_get(&group->kill_refcount);
	479	+}
	480	+
	481	+void xe_exec_queue_group_kill_put(struct xe_exec_queue_group *group)
	482	+{
	483	+ if (!group)
	484	+ return;
	485	+
	486	+ kref_put(&group->kill_refcount, xe_exec_queue_group_kill);
	487	+}
	488	+
386	489	void xe_exec_queue_destroy(struct kref *ref)
387	490	{
388	491	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
@@ -567,6 +670,217 @@ exec_queue_set_pxp_type(struct xe_device xe, struct xe_exec_queue q, u64 value · 178 unmodified lines
567	670	return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM);
568	671	}
569	672
	673	+static int exec_queue_set_hang_replay_state(struct xe_device *xe,
	674	+ struct xe_exec_queue *q,
	675	+ u64 value)
	676	+{
	677	+ size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class);
	678	+ u64 __user *address = u64_to_user_ptr(value);
	679	+ void *ptr;
	680	+
	681	+ ptr = vmemdup_user(address, size);
	682	+ if (XE_IOCTL_DBG(xe, IS_ERR(ptr)))
	683	+ return PTR_ERR(ptr);
	684	+
	685	+ q->replay_state = ptr;
	686	+
	687	+ return 0;
	688	+}
	689	+
	690	+static int xe_exec_queue_group_init(struct xe_device xe, struct xe_exec_queue q)
	691	+{
	692	+ struct xe_tile *tile = gt_to_tile(q->gt);
	693	+ struct xe_exec_queue_group *group;
	694	+ struct xe_bo *bo;
	695	+
	696	+ group = kzalloc(sizeof(*group), GFP_KERNEL);
	697	+ if (!group)
	698	+ return -ENOMEM;
	699	+
	700	+ bo = xe_bo_create_pin_map_novm(xe, tile, SZ_4K, ttm_bo_type_kernel,
	701	+ XE_BO_FLAG_VRAM_IF_DGFX(tile) \|
	702	+ XE_BO_FLAG_PINNED_LATE_RESTORE \|
	703	+ XE_BO_FLAG_FORCE_USER_VRAM \|
	704	+ XE_BO_FLAG_GGTT_INVALIDATE \|
	705	+ XE_BO_FLAG_GGTT, false);
	706	+ if (IS_ERR(bo)) {
	707	+ drm_err(&xe->drm, "CGP bo allocation for queue group failed: %ld\n",
	708	+ PTR_ERR(bo));
	709	+ kfree(group);
	710	+ return PTR_ERR(bo);
	711	+ }
	712	+
	713	+ xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K);
	714	+
	715	+ group->primary = q;
	716	+ group->cgp_bo = bo;
	717	+ INIT_LIST_HEAD(&group->list);
	718	+ kref_init(&group->kill_refcount);
	719	+ xa_init_flags(&group->xa, XA_FLAGS_ALLOC1);
	720	+ mutex_init(&group->list_lock);
	721	+ q->multi_queue.group = group;
	722	+
	723	+ /* group->list_lock is used in submission backend */
	724	+ if (IS_ENABLED(CONFIG_LOCKDEP)) {
	725	+ fs_reclaim_acquire(GFP_KERNEL);
	726	+ might_lock(&group->list_lock);
	727	+ fs_reclaim_release(GFP_KERNEL);
	728	+ }
	729	+
	730	+ return 0;
	731	+}
	732	+
	733	+static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q)
	734	+{
	735	+ return q->gt->info.multi_queue_engine_class_mask & BIT(q->class);
	736	+}
	737	+
	738	+static int xe_exec_queue_group_validate(struct xe_device xe, struct xe_exec_queue q,
	739	+ u32 primary_id)
	740	+{
	741	+ struct xe_exec_queue_group *group;
	742	+ struct xe_exec_queue *primary;
	743	+ int ret;
	744	+
	745	+ /*
	746	+ * Get from below xe_exec_queue_lookup() pairs with put
	747	+ * in xe_exec_queue_group_cleanup().
	748	+ */
	749	+ primary = xe_exec_queue_lookup(q->vm->xef, primary_id);
	750	+ if (XE_IOCTL_DBG(xe, !primary))
	751	+ return -ENOENT;
	752	+
	753	+ if (XE_IOCTL_DBG(xe, !xe_exec_queue_is_multi_queue_primary(primary)) \|\|
	754	+ XE_IOCTL_DBG(xe, q->vm != primary->vm) \|\|
	755	+ XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) {
	756	+ ret = -EINVAL;
	757	+ goto put_primary;
	758	+ }
	759	+
	760	+ group = primary->multi_queue.group;
	761	+ q->multi_queue.valid = true;
	762	+ q->multi_queue.group = group;
	763	+
	764	+ return 0;
	765	+put_primary:
	766	+ xe_exec_queue_put(primary);
	767	+ return ret;
	768	+}
	769	+
	770	+#define XE_MAX_GROUP_SIZE 64
	771	+static int xe_exec_queue_group_add(struct xe_device xe, struct xe_exec_queue q)
	772	+{
	773	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	774	+ u32 pos;
	775	+ int err;
	776	+
	777	+ xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));
	778	+
	779	+ /* Primary queue holds a reference to LRCs of all secondary queues */
	780	+ err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]),
	781	+ XA_LIMIT(1, XE_MAX_GROUP_SIZE - 1), GFP_KERNEL);
	782	+ if (XE_IOCTL_DBG(xe, err)) {
	783	+ xe_lrc_put(q->lrc[0]);
	784	+
	785	+ /* It is invalid if queue group limit is exceeded */
	786	+ if (err == -EBUSY)
	787	+ err = -EINVAL;
	788	+
	789	+ return err;
	790	+ }
	791	+
	792	+ q->multi_queue.pos = pos;
	793	+
	794	+ if (group->primary->multi_queue.keep_active) {
	795	+ xe_exec_queue_group_kill_get(group);
	796	+ q->multi_queue.keep_active = true;
	797	+ }
	798	+
	799	+ return 0;
	800	+}
	801	+
	802	+static void xe_exec_queue_group_delete(struct xe_device xe, struct xe_exec_queue q)
	803	+{
	804	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	805	+ struct xe_lrc *lrc;
	806	+
	807	+ xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));
	808	+
	809	+ lrc = xa_erase(&group->xa, q->multi_queue.pos);
	810	+ xe_assert(xe, lrc);
	811	+ xe_lrc_put(lrc);
	812	+
	813	+ if (q->multi_queue.keep_active) {
	814	+ xe_exec_queue_group_kill_put(group);
	815	+ q->multi_queue.keep_active = false;
	816	+ }
	817	+}
	818	+
	819	+static int exec_queue_set_multi_group(struct xe_device xe, struct xe_exec_queue q,
	820	+ u64 value)
	821	+{
	822	+ if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q)))
	823	+ return -ENODEV;
	824	+
	825	+ if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe)))
	826	+ return -EOPNOTSUPP;
	827	+
	828	+ if (XE_IOCTL_DBG(xe, !q->vm->xef))
	829	+ return -EINVAL;
	830	+
	831	+ if (XE_IOCTL_DBG(xe, xe_exec_queue_is_parallel(q)))
	832	+ return -EINVAL;
	833	+
	834	+ if (XE_IOCTL_DBG(xe, xe_exec_queue_is_multi_queue(q)))
	835	+ return -EINVAL;
	836	+
	837	+ if (value & DRM_XE_MULTI_GROUP_CREATE) {
	838	+ if (XE_IOCTL_DBG(xe, value & ~(DRM_XE_MULTI_GROUP_CREATE \|
	839	+ DRM_XE_MULTI_GROUP_KEEP_ACTIVE)))
	840	+ return -EINVAL;
	841	+
	842	+ /*
	843	+ * KEEP_ACTIVE is not supported in preempt fence mode as in that mode,
	844	+ * VM_DESTROY ioctl expects all exec queues of that VM are already killed.
	845	+ */
	846	+ if (XE_IOCTL_DBG(xe, (value & DRM_XE_MULTI_GROUP_KEEP_ACTIVE) &&
	847	+ xe_vm_in_preempt_fence_mode(q->vm)))
	848	+ return -EINVAL;
	849	+
	850	+ q->multi_queue.valid = true;
	851	+ q->multi_queue.is_primary = true;
	852	+ q->multi_queue.pos = 0;
	853	+ if (value & DRM_XE_MULTI_GROUP_KEEP_ACTIVE)
	854	+ q->multi_queue.keep_active = true;
	855	+
	856	+ return 0;
	857	+ }
	858	+
	859	+ /* While adding secondary queues, the upper 32 bits must be 0 */
	860	+ if (XE_IOCTL_DBG(xe, value & (~0ull << 32)))
	861	+ return -EINVAL;
	862	+
	863	+ return xe_exec_queue_group_validate(xe, q, value);
	864	+}
	865	+
	866	+static int exec_queue_set_multi_queue_priority(struct xe_device xe, struct xe_exec_queue q,
	867	+ u64 value)
	868	+{
	869	+ if (XE_IOCTL_DBG(xe, value > XE_MULTI_QUEUE_PRIORITY_HIGH))
	870	+ return -EINVAL;
	871	+
	872	+ /* For queue creation time (!q->xef) setting, just store the priority value */
	873	+ if (!q->xef) {
	874	+ q->multi_queue.priority = value;
	875	+ return 0;
	876	+ }
	877	+
	878	+ if (!xe_exec_queue_is_multi_queue(q))
	879	+ return -EINVAL;
	880	+
	881	+ return q->ops->set_multi_queue_priority(q, value);
	882	+}
	883	+
570	884	typedef int (xe_exec_queue_set_property_fn)(struct xe_device xe,
571	885	struct xe_exec_queue *q,
572	886	u64 value);
@@ -575,11 +889,76 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { · 2 unmodified lines
575	889	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,
576	890	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,
577	891	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type,
	892	+ [DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state,
	893	+ [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group,
	894	+ [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY] =
	895	+ exec_queue_set_multi_queue_priority,
578	896	};
579	897
	898	+int xe_exec_queue_set_property_ioctl(struct drm_device dev, void data,
	899	+ struct drm_file *file)
	900	+{
	901	+ struct xe_device *xe = to_xe_device(dev);
	902	+ struct xe_file *xef = to_xe_file(file);
	903	+ struct drm_xe_exec_queue_set_property *args = data;
	904	+ struct xe_exec_queue *q;
	905	+ int ret;
	906	+ u32 idx;
	907	+
	908	+ if (XE_IOCTL_DBG(xe, args->reserved[0] \|\| args->reserved[1]))
	909	+ return -EINVAL;
	910	+
	911	+ if (XE_IOCTL_DBG(xe, args->property !=
	912	+ DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
	913	+ return -EINVAL;
	914	+
	915	+ q = xe_exec_queue_lookup(xef, args->exec_queue_id);
	916	+ if (XE_IOCTL_DBG(xe, !q))
	917	+ return -ENOENT;
	918	+
	919	+ idx = array_index_nospec(args->property,
	920	+ ARRAY_SIZE(exec_queue_set_property_funcs));
	921	+ ret = exec_queue_set_property_funcs[idx](xe, q, args->value);
	922	+ if (XE_IOCTL_DBG(xe, ret))
	923	+ goto err_post_lookup;
	924	+
	925	+ xe_exec_queue_put(q);
	926	+ return 0;
	927	+
	928	+ err_post_lookup:
	929	+ xe_exec_queue_put(q);
	930	+ return ret;
	931	+}
	932	+
	933	+static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties)
	934	+{
	935	+ u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) \|
	936	+ BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY);
	937	+
	938	+ /*
	939	+ * Only MULTI_QUEUE_PRIORITY property is valid for secondary queues of a
	940	+ * multi-queue group.
	941	+ */
	942	+ if (xe_exec_queue_is_multi_queue_secondary(q) &&
	943	+ properties & ~secondary_queue_valid_props)
	944	+ return -EINVAL;
	945	+
	946	+ return 0;
	947	+}
	948	+
	949	+static int exec_queue_user_ext_check_final(struct xe_exec_queue *q, u64 properties)
	950	+{
	951	+ /* MULTI_QUEUE_PRIORITY only applies to multi-queue group queues */
	952	+ if ((properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) &&
	953	+ !(properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)))
	954	+ return -EINVAL;
	955	+
	956	+ return 0;
	957	+}
	958	+
580	959	static int exec_queue_user_ext_set_property(struct xe_device *xe,
581	960	struct xe_exec_queue *q,
582		− u64 extension)
	961	+ u64 extension, u64 *properties)
583	962	{
584	963	u64 __user *address = u64_to_user_ptr(extension);
585	964	struct drm_xe_ext_set_property ext;
@@ -595,27 +974,35 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe, · 9 unmodified lines
595	974	XE_IOCTL_DBG(xe, ext.pad) \|\|
596	975	XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&
597	976	ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE &&
598		− ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE))
	977	+ ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE &&
	978	+ ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE &&
	979	+ ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP &&
	980	+ ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
599	981	return -EINVAL;
600	982
601	983	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));
602	984	if (!exec_queue_set_property_funcs[idx])
603	985	return -EINVAL;
604	986
	987	+ *properties \|= BIT_ULL(idx);
	988	+ err = exec_queue_user_ext_check(q, *properties);
	989	+ if (XE_IOCTL_DBG(xe, err))
	990	+ return err;
	991	+
605	992	return exec_queue_set_property_funcs[idx](xe, q, ext.value);
606	993	}
607	994
608	995	typedef int (xe_exec_queue_user_extension_fn)(struct xe_device xe,
609	996	struct xe_exec_queue *q,
610		− u64 extension);
	997	+ u64 extension, u64 *properties);
611	998
612	999	static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {
613	1000	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,
614	1001	};
615	1002
616	1003	#define MAX_USER_EXTENSIONS 16
617		−static int exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,
618		− u64 extensions, int ext_number)
	1004	+static int __exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,
	1005	+ u64 extensions, int ext_number, u64 *properties)
619	1006	{
620	1007	u64 __user *address = u64_to_user_ptr(extensions);
621	1008	struct drm_xe_user_extension ext;
@@ -636,13 +1023,36 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue · 14 unmodified lines
636	1023
637	1024	idx = array_index_nospec(ext.name,
638	1025	ARRAY_SIZE(exec_queue_user_extension_funcs));
639		− err = exec_queue_user_extension_funcs[idx](xe, q, extensions);
	1026	+ err = exec_queue_user_extension_funcs[idx](xe, q, extensions, properties);
640	1027	if (XE_IOCTL_DBG(xe, err))
641	1028	return err;
642	1029
643	1030	if (ext.next_extension)
644		− return exec_queue_user_extensions(xe, q, ext.next_extension,
645		− ++ext_number);
	1031	+ return __exec_queue_user_extensions(xe, q, ext.next_extension,
	1032	+ ++ext_number, properties);
	1033	+
	1034	+ return 0;
	1035	+}
	1036	+
	1037	+static int exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,
	1038	+ u64 extensions)
	1039	+{
	1040	+ u64 properties = 0;
	1041	+ int err;
	1042	+
	1043	+ err = __exec_queue_user_extensions(xe, q, extensions, 0, &properties);
	1044	+ if (XE_IOCTL_DBG(xe, err))
	1045	+ return err;
	1046	+
	1047	+ err = exec_queue_user_ext_check_final(q, properties);
	1048	+ if (XE_IOCTL_DBG(xe, err))
	1049	+ return err;
	1050	+
	1051	+ if (xe_exec_queue_is_multi_queue_primary(q)) {
	1052	+ err = xe_exec_queue_group_init(xe, q);
	1053	+ if (XE_IOCTL_DBG(xe, err))
	1054	+ return err;
	1055	+ }
646	1056
647	1057	return 0;
648	1058	}
@@ -798,12 +1208,18 @@ int xe_exec_queue_create_ioctl(struct drm_device dev, void data, · 149 unmodified lines
798	1208	if (IS_ERR(q))
799	1209	return PTR_ERR(q);
800	1210
	1211	+ if (xe_exec_queue_is_multi_queue_secondary(q)) {
	1212	+ err = xe_exec_queue_group_add(xe, q);
	1213	+ if (XE_IOCTL_DBG(xe, err))
	1214	+ goto put_exec_queue;
	1215	+ }
	1216	+
801	1217	if (xe_vm_in_preempt_fence_mode(vm)) {
802	1218	q->lr.context = dma_fence_context_alloc(1);
803	1219
804	1220	err = xe_vm_add_compute_exec_queue(vm, q);
805	1221	if (XE_IOCTL_DBG(xe, err))
806		− goto put_exec_queue;
	1222	+ goto delete_queue_group;
807	1223	}
808	1224
809	1225	if (q->vm && q->hwe->hw_engine_group) {
@@ -826,6 +1242,9 @@ int xe_exec_queue_create_ioctl(struct drm_device dev, void data, · 16 unmodified lines
826	1242
827	1243	kill_exec_queue:
828	1244	xe_exec_queue_kill(q);
	1245	+delete_queue_group:
	1246	+ if (xe_exec_queue_is_multi_queue_secondary(q))
	1247	+ xe_exec_queue_group_delete(xe, q);
829	1248	put_exec_queue:
830	1249	xe_exec_queue_put(q);
831	1250	return err;
@@ -981,6 +1400,11 @@ void xe_exec_queue_kill(struct xe_exec_queue *q) · 149 unmodified lines
981	1400
982	1401	q->ops->kill(q);
983	1402	xe_vm_remove_compute_exec_queue(q->vm, q);
	1403	+
	1404	+ if (!xe_exec_queue_is_multi_queue_primary(q) && q->multi_queue.keep_active) {
	1405	+ xe_exec_queue_group_kill_put(q->multi_queue.group);
	1406	+ q->multi_queue.keep_active = false;
	1407	+ }
984	1408	}
985	1409
986	1410	int xe_exec_queue_destroy_ioctl(struct drm_device dev, void data,
@@ -1007,7 +1431,10 @@ int xe_exec_queue_destroy_ioctl(struct drm_device dev, void data, · 20 unmodified lines
1007	1431	if (q->vm && q->hwe->hw_engine_group)
1008	1432	xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
1009	1433
1010		− xe_exec_queue_kill(q);
	1434	+ if (xe_exec_queue_is_multi_queue_primary(q))
	1435	+ xe_exec_queue_group_kill_put(q->multi_queue.group);
	1436	+ else
	1437	+ xe_exec_queue_kill(q);
1011	1438
1012	1439	trace_xe_exec_queue_close(q);
1013	1440	xe_exec_queue_put(q);

drivers/gpu/drm/xe/xe_exec_queue.h+68−0

@@ -66,6 +66,55 @@ static inline bool xe_exec_queue_uses_pxp(struct xe_exec_queue *q) · 65 unmodified lines
66	66	return q->pxp.type;
67	67	}
68	68
	69	+/**
	70	+ * xe_exec_queue_is_multi_queue() - Whether an exec_queue is part of a queue group.
	71	+ * @q: The exec_queue
	72	+ *
	73	+ * Return: True if the exec_queue is part of a queue group, false otherwise.
	74	+ */
	75	+static inline bool xe_exec_queue_is_multi_queue(struct xe_exec_queue *q)
	76	+{
	77	+ return q->multi_queue.valid;
	78	+}
	79	+
	80	+/**
	81	+ * xe_exec_queue_is_multi_queue_primary() - Whether an exec_queue is primary queue
	82	+ * of a multi queue group.
	83	+ * @q: The exec_queue
	84	+ *
	85	+ * Return: True if @q is primary queue of a queue group, false otherwise.
	86	+ */
	87	+static inline bool xe_exec_queue_is_multi_queue_primary(struct xe_exec_queue *q)
	88	+{
	89	+ return q->multi_queue.is_primary;
	90	+}
	91	+
	92	+/**
	93	+ * xe_exec_queue_is_multi_queue_secondary() - Whether an exec_queue is secondary queue
	94	+ * of a multi queue group.
	95	+ * @q: The exec_queue
	96	+ *
	97	+ * Return: True if @q is secondary queue of a queue group, false otherwise.
	98	+ */
	99	+static inline bool xe_exec_queue_is_multi_queue_secondary(struct xe_exec_queue *q)
	100	+{
	101	+ return xe_exec_queue_is_multi_queue(q) && !xe_exec_queue_is_multi_queue_primary(q);
	102	+}
	103	+
	104	+/**
	105	+ * xe_exec_queue_multi_queue_primary() - Get multi queue group's primary queue
	106	+ * @q: The exec_queue
	107	+ *
	108	+ * If @q belongs to a multi queue group, then the primary queue of the group will
	109	+ * be returned. Otherwise, @q will be returned.
	110	+ */
	111	+static inline struct xe_exec_queue xe_exec_queue_multi_queue_primary(struct xe_exec_queue q)
	112	+{
	113	+ return xe_exec_queue_is_multi_queue(q) ? q->multi_queue.group->primary : q;
	114	+}
	115	+
	116	+void xe_exec_queue_group_kill_put(struct xe_exec_queue_group *group);
	117	+
69	118	bool xe_exec_queue_is_lr(struct xe_exec_queue *q);
70	119
71	120	bool xe_exec_queue_is_idle(struct xe_exec_queue *q);
@@ -78,6 +127,8 @@ int xe_exec_queue_destroy_ioctl(struct drm_device dev, void data, · 6 unmodified lines
78	127	struct drm_file *file);
79	128	int xe_exec_queue_get_property_ioctl(struct drm_device dev, void data,
80	129	struct drm_file *file);
	130	+int xe_exec_queue_set_property_ioctl(struct drm_device dev, void data,
	131	+ struct drm_file *file);
81	132	enum xe_exec_queue_priority xe_exec_queue_device_get_max_priority(struct xe_device *xe);
82	133
83	134	void xe_exec_queue_last_fence_put(struct xe_exec_queue e, struct xe_vm vm);
@@ -111,4 +162,21 @@ int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue q, void scratch); · 27 unmodified lines
111	162
112	163	struct xe_lrc xe_exec_queue_lrc(struct xe_exec_queue q);
113	164
	165	+/**
	166	+ * xe_exec_queue_idle_skip_suspend() - Can exec queue skip suspend
	167	+ * @q: The exec_queue
	168	+ *
	169	+ * If an exec queue is not parallel and is idle, the suspend steps can be
	170	+ * skipped in the submission backend immediatley signaling the suspend fence.
	171	+ * Parallel queues cannot skip this step due to limitations in the submission
	172	+ * backend.
	173	+ *
	174	+ * Return: True if exec queue is idle and can skip suspend steps, False
	175	+ * otherwise
	176	+ */
	177	+static inline bool xe_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
	178	+{
	179	+ return !xe_exec_queue_is_parallel(q) && xe_exec_queue_is_idle(q);
	180	+}
	181	+
114	182	#endif

drivers/gpu/drm/xe/xe_exec_queue_types.h+62−0

@@ -32,6 +32,44 @@ enum xe_exec_queue_priority { · 31 unmodified lines
32	32	XE_EXEC_QUEUE_PRIORITY_COUNT
33	33	};
34	34
	35	+/**
	36	+ * enum xe_multi_queue_priority - Multi Queue priority values
	37	+ *
	38	+ * The priority values of the queues within the multi queue group.
	39	+ */
	40	+enum xe_multi_queue_priority {
	41	+ /** @XE_MULTI_QUEUE_PRIORITY_LOW: Priority low */
	42	+ XE_MULTI_QUEUE_PRIORITY_LOW = 0,
	43	+ /** @XE_MULTI_QUEUE_PRIORITY_NORMAL: Priority normal */
	44	+ XE_MULTI_QUEUE_PRIORITY_NORMAL,
	45	+ /** @XE_MULTI_QUEUE_PRIORITY_HIGH: Priority high */
	46	+ XE_MULTI_QUEUE_PRIORITY_HIGH,
	47	+};
	48	+
	49	+/**
	50	+ * struct xe_exec_queue_group - Execution multi queue group
	51	+ *
	52	+ * Contains multi queue group information.
	53	+ */
	54	+struct xe_exec_queue_group {
	55	+ /** @primary: Primary queue of this group */
	56	+ struct xe_exec_queue *primary;
	57	+ /** @cgp_bo: BO for the Context Group Page */
	58	+ struct xe_bo *cgp_bo;
	59	+ /** @xa: xarray to store LRCs */
	60	+ struct xarray xa;
	61	+ /** @list: List of all secondary queues in the group */
	62	+ struct list_head list;
	63	+ /** @list_lock: Secondary queue list lock */
	64	+ struct mutex list_lock;
	65	+ /** @kill_refcount: ref count to kill primary queue */
	66	+ struct kref kill_refcount;
	67	+ /** @sync_pending: CGP_SYNC_DONE g2h response pending */
	68	+ bool sync_pending;
	69	+ /** @banned: Group banned */
	70	+ bool banned;
	71	+};
	72	+
35	73	/**
36	74	* struct xe_exec_queue - Execution queue
37	75	*
@@ -111,6 +149,24 @@ struct xe_exec_queue { · 73 unmodified lines
111	149	struct xe_guc_exec_queue *guc;
112	150	};
113	151
	152	+ /** @multi_queue: Multi queue information */
	153	+ struct {
	154	+ /** @multi_queue.group: Queue group information */
	155	+ struct xe_exec_queue_group *group;
	156	+ /** @multi_queue.link: Link into group's secondary queues list */
	157	+ struct list_head link;
	158	+ /** @multi_queue.priority: Queue priority within the multi-queue group */
	159	+ enum xe_multi_queue_priority priority;
	160	+ /** @multi_queue.pos: Position of queue within the multi-queue group */
	161	+ u8 pos;
	162	+ /** @multi_queue.valid: Queue belongs to a multi queue group */
	163	+ u8 valid:1;
	164	+ /** @multi_queue.is_primary: Is primary queue (Q0) of the group */
	165	+ u8 is_primary:1;
	166	+ /** @multi_queue.keep_active: Keep the group active after primary is destroyed */
	167	+ u8 keep_active:1;
	168	+ } multi_queue;
	169	+
114	170	/** @sched_props: scheduling properties */
115	171	struct {
116	172	/** @sched_props.timeslice_us: timeslice period in micro-seconds */
@@ -167,6 +223,9 @@ struct xe_exec_queue { · 50 unmodified lines
167	223	/** @ufence_timeline_value: User fence timeline value */
168	224	u64 ufence_timeline_value;
169	225
	226	+ /** @replay_state: GPU hang replay state */
	227	+ void *replay_state;
	228	+
170	229	/** @ops: submission backend exec queue operations */
171	230	const struct xe_exec_queue_ops *ops;
172	231
@@ -213,6 +272,9 @@ struct xe_exec_queue_ops { · 40 unmodified lines
213	272	int (set_timeslice)(struct xe_exec_queue q, u32 timeslice_us);
214	273	/** @set_preempt_timeout: Set preemption timeout for exec queue */
215	274	int (set_preempt_timeout)(struct xe_exec_queue q, u32 preempt_timeout_us);
	275	+ /** @set_multi_queue_priority: Set multi queue priority */
	276	+ int (set_multi_queue_priority)(struct xe_exec_queue q,
	277	+ enum xe_multi_queue_priority priority);
216	278	/**
217	279	* @suspend: Suspend exec queue from executing, allowed to be called
218	280	* multiple times in a row before resume with the caveat that

drivers/gpu/drm/xe/xe_execlist.c+1−1

@@ -269,7 +269,7 @@ struct xe_execlist_port xe_execlist_port_create(struct xe_device xe, · 268 unmodified lines
269	269
270	270	port->hwe = hwe;
271	271
272		− port->lrc = xe_lrc_create(hwe, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0);
	272	+ port->lrc = xe_lrc_create(hwe, NULL, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0);
273	273	if (IS_ERR(port->lrc)) {
274	274	err = PTR_ERR(port->lrc);
275	275	goto err;

drivers/gpu/drm/xe/xe_force_wake.c+7−0

@@ -166,6 +166,13 @@ static int domain_sleep_wait(struct xe_gt *gt, · 165 unmodified lines
166	166	* xe_force_wake_ref_has_domain() function. Caller must call
167	167	* xe_force_wake_put() function to decrease incremented refcounts.
168	168	*
	169	+ * When possible, scope-based forcewake (through CLASS(xe_force_wake, ...) or
	170	+ * xe_with_force_wake()) should be used instead of direct calls to this
	171	+ * function. Direct usage of get/put should only be used when the function
	172	+ * has goto-based flows that can interfere with scope-based cleanup, or when
	173	+ * the lifetime of the forcewake reference does not match a specific scope
	174	+ * (e.g., forcewake obtained in one function and released in a different one).
	175	+ *
169	176	* Return: opaque reference to woken domains or zero if none of requested
170	177	* domains were awake.
171	178	*/

drivers/gpu/drm/xe/xe_force_wake.h+40−0

@@ -61,4 +61,44 @@ xe_force_wake_ref_has_domain(unsigned int fw_ref, enum xe_force_wake_domains dom · 60 unmodified lines
61	61	return fw_ref & domain;
62	62	}
63	63
	64	+struct xe_force_wake_ref {
	65	+ struct xe_force_wake *fw;
	66	+ unsigned int domains;
	67	+};
	68	+
	69	+static struct xe_force_wake_ref
	70	+xe_force_wake_constructor(struct xe_force_wake *fw, unsigned int domains)
	71	+{
	72	+ struct xe_force_wake_ref fw_ref = { .fw = fw };
	73	+
	74	+ fw_ref.domains = xe_force_wake_get(fw, domains);
	75	+
	76	+ return fw_ref;
	77	+}
	78	+
	79	+DEFINE_CLASS(xe_force_wake, struct xe_force_wake_ref,
	80	+ xe_force_wake_put(_T.fw, _T.domains),
	81	+ xe_force_wake_constructor(fw, domains),
	82	+ struct xe_force_wake *fw, unsigned int domains);
	83	+
	84	+/*
	85	+ * Scoped helper for the forcewake class, using the same trick as scoped_guard()
	86	+ * to bind the lifetime to the next statement/block.
	87	+ */
	88	+#define __xe_with_force_wake(ref, fw, domains, done) \
	89	+ for (CLASS(xe_force_wake, ref)(fw, domains), *(done) = NULL; \
	90	+ !(done); (done) = (void *)1)
	91	+
	92	+#define xe_with_force_wake(ref, fw, domains) \
	93	+ __xe_with_force_wake(ref, fw, domains, __UNIQUE_ID(done))
	94	+
	95	+/*
	96	+ * Used when xe_force_wake_constructor() has already been called by another
	97	+ * function and the current function is responsible for releasing the forcewake
	98	+ * reference in all possible cases and error paths.
	99	+ */
	100	+DEFINE_CLASS(xe_force_wake_release_only, struct xe_force_wake_ref,
	101	+ if (_T.fw) xe_force_wake_put(_T.fw, _T.domains), fw_ref,
	102	+ struct xe_force_wake_ref fw_ref);
	103	+
64	104	#endif

drivers/gpu/drm/xe/xe_ggtt.c+1−2

@@ -396,9 +396,8 @@ static void ggtt_node_remove_work_func(struct work_struct *work) · 395 unmodified lines
396	396	delayed_removal_work);
397	397	struct xe_device *xe = tile_to_xe(node->ggtt->tile);
398	398
399		− xe_pm_runtime_get(xe);
	399	+ guard(xe_pm_runtime)(xe);
400	400	ggtt_node_remove(node);
401		− xe_pm_runtime_put(xe);
402	401	}
403	402
404	403	/**

drivers/gpu/drm/xe/xe_gsc.c+6−15

@@ -352,7 +352,6 @@ static void gsc_work(struct work_struct *work) · 351 unmodified lines
352	352	struct xe_gsc gsc = container_of(work, typeof(gsc), work);
353	353	struct xe_gt *gt = gsc_to_gt(gsc);
354	354	struct xe_device *xe = gt_to_xe(gt);
355		− unsigned int fw_ref;
356	355	u32 actions;
357	356	int ret;
358	357
@@ -361,13 +360,12 @@ static void gsc_work(struct work_struct *work) · 2 unmodified lines
361	360	gsc->work_actions = 0;
362	361	spin_unlock_irq(&gsc->lock);
363	362
364		− xe_pm_runtime_get(xe);
365		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
	363	+ guard(xe_pm_runtime)(xe);
	364	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
366	365
367	366	if (actions & GSC_ACTION_ER_COMPLETE) {
368		− ret = gsc_er_complete(gt);
369		− if (ret)
370		− goto out;
	367	+ if (gsc_er_complete(gt))
	368	+ return;
371	369	}
372	370
373	371	if (actions & GSC_ACTION_FW_LOAD) {
@@ -380,10 +378,6 @@ static void gsc_work(struct work_struct *work) · 6 unmodified lines
380	378
381	379	if (actions & GSC_ACTION_SW_PROXY)
382	380	xe_gsc_proxy_request_handler(gsc);
383		−
384		−out:
385		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
386		− xe_pm_runtime_put(xe);
387	381	}
388	382
389	383	void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec)
@@ -615,7 +609,6 @@ void xe_gsc_print_info(struct xe_gsc gsc, struct drm_printer p) · 225 unmodified lines
615	609	{
616	610	struct xe_gt *gt = gsc_to_gt(gsc);
617	611	struct xe_mmio *mmio = &gt->mmio;
618		− unsigned int fw_ref;
619	612
620	613	xe_uc_fw_print(&gsc->fw, p);
621	614
@@ -624,8 +617,8 @@ void xe_gsc_print_info(struct xe_gsc gsc, struct drm_printer p) · 2 unmodified lines
624	617	if (!xe_uc_fw_is_enabled(&gsc->fw))
625	618	return;
626	619
627		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
628		− if (!fw_ref)
	620	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
	621	+ if (!fw_ref.domains)
629	622	return;
630	623
631	624	drm_printf(p, "\nHECI1 FWSTS: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
@@ -635,6 +628,4 @@ void xe_gsc_print_info(struct xe_gsc gsc, struct drm_printer p) · 3 unmodified lines
635	628	xe_mmio_read32(mmio, HECI_FWSTS4(MTL_GSC_HECI1_BASE)),
636	629	xe_mmio_read32(mmio, HECI_FWSTS5(MTL_GSC_HECI1_BASE)),
637	630	xe_mmio_read32(mmio, HECI_FWSTS6(MTL_GSC_HECI1_BASE)));
638		−
639		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
640	631	}

drivers/gpu/drm/xe/xe_gsc_debugfs.c+1−2

@@ -37,9 +37,8 @@ static int gsc_info(struct seq_file m, void data) · 36 unmodified lines
37	37	struct xe_device *xe = gsc_to_xe(gsc);
38	38	struct drm_printer p = drm_seq_file_printer(m);
39	39
40		− xe_pm_runtime_get(xe);
	40	+ guard(xe_pm_runtime)(xe);
41	41	xe_gsc_print_info(gsc, &p);
42		− xe_pm_runtime_put(xe);
43	42
44	43	return 0;
45	44	}

drivers/gpu/drm/xe/xe_gsc_proxy.c+7−10

@@ -440,22 +440,19 @@ static void xe_gsc_proxy_remove(void *arg) · 439 unmodified lines
440	440	struct xe_gsc *gsc = arg;
441	441	struct xe_gt *gt = gsc_to_gt(gsc);
442	442	struct xe_device *xe = gt_to_xe(gt);
443		− unsigned int fw_ref = 0;
444	443
445	444	if (!gsc->proxy.component_added)
446	445	return;
447	446
448	447	/* disable HECI2 IRQs */
449		− xe_pm_runtime_get(xe);
450		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
451		− if (!fw_ref)
452		− xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n");
	448	+ scoped_guard(xe_pm_runtime, xe) {
	449	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
	450	+ if (!fw_ref.domains)
	451	+ xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n");
453	452
454		− /* try do disable irq even if forcewake failed */
455		− gsc_proxy_irq_toggle(gsc, false);
456		−
457		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
458		− xe_pm_runtime_put(xe);
	453	+ /* try do disable irq even if forcewake failed */
	454	+ gsc_proxy_irq_toggle(gsc, false);
	455	+ }
459	456
460	457	xe_gsc_wait_for_worker_completion(gsc);
461	458

drivers/gpu/drm/xe/xe_gt.c+87−87

@@ -103,14 +103,13 @@ void xe_gt_sanitize(struct xe_gt *gt) · 102 unmodified lines
103	103
104	104	static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
105	105	{
106		− unsigned int fw_ref;
107	106	u32 reg;
108	107
109	108	if (!XE_GT_WA(gt, 16023588340))
110	109	return;
111	110
112		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
113		− if (!fw_ref)
	111	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	112	+ if (!fw_ref.domains)
114	113	return;
115	114
116	115	if (xe_gt_is_main_type(gt)) {
@@ -120,12 +119,10 @@ static void xe_gt_enable_host_l2_vram(struct xe_gt *gt) · 3 unmodified lines
120	119	}
121	120
122	121	xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0xF);
123		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
124	122	}
125	123
126	124	static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
127	125	{
128		− unsigned int fw_ref;
129	126	u32 reg;
130	127
131	128	if (!XE_GT_WA(gt, 16023588340))
@@ -134,15 +131,13 @@ static void xe_gt_disable_host_l2_vram(struct xe_gt *gt) · 2 unmodified lines
134	131	if (xe_gt_is_media_type(gt))
135	132	return;
136	133
137		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
138		− if (!fw_ref)
	134	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	135	+ if (!fw_ref.domains)
139	136	return;
140	137
141	138	reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
142	139	reg &= ~CG_DIS_CNTLBUS;
143	140	xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
144		−
145		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
146	141	}
147	142
148	143	static void gt_reset_worker(struct work_struct *w);
@@ -389,7 +384,6 @@ put_exec_queue: · 240 unmodified lines
389	384
390	385	int xe_gt_init_early(struct xe_gt *gt)
391	386	{
392		− unsigned int fw_ref;
393	387	int err;
394	388
395	389	if (IS_SRIOV_PF(gt_to_xe(gt))) {
@@ -436,13 +430,12 @@ int xe_gt_init_early(struct xe_gt *gt) · 40 unmodified lines
436	430	if (err)
437	431	return err;
438	432
439		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
440		− if (!fw_ref)
	433	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	434	+ if (!fw_ref.domains)
441	435	return -ETIMEDOUT;
442	436
443	437	xe_gt_mcr_init_early(gt);
444	438	xe_pat_init(gt);
445		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
446	439
447	440	return 0;
448	441	}
@@ -460,16 +453,15 @@ static void dump_pat_on_error(struct xe_gt *gt) · 11 unmodified lines
460	453
461	454	static int gt_init_with_gt_forcewake(struct xe_gt *gt)
462	455	{
463		− unsigned int fw_ref;
464	456	int err;
465	457
466		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
467		− if (!fw_ref)
	458	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	459	+ if (!fw_ref.domains)
468	460	return -ETIMEDOUT;
469	461
470	462	err = xe_uc_init(&gt->uc);
471	463	if (err)
472		− goto err_force_wake;
	464	+ return err;
473	465
474	466	xe_gt_topology_init(gt);
475	467	xe_gt_mcr_init(gt);
@@ -478,7 +470,7 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt) · 2 unmodified lines
478	470	if (xe_gt_is_main_type(gt)) {
479	471	err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt);
480	472	if (err)
481		− goto err_force_wake;
	473	+ return err;
482	474	if (IS_SRIOV_PF(gt_to_xe(gt)))
483	475	xe_lmtt_init(&gt_to_tile(gt)->sriov.pf.lmtt);
484	476	}
@@ -492,17 +484,17 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt) · 7 unmodified lines
492	484	err = xe_hw_engines_init_early(gt);
493	485	if (err) {
494	486	dump_pat_on_error(gt);
495		− goto err_force_wake;
	487	+ return err;
496	488	}
497	489
498	490	err = xe_hw_engine_class_sysfs_init(gt);
499	491	if (err)
500		− goto err_force_wake;
	492	+ return err;
501	493
502	494	/* Initialize CCS mode sysfs after early initialization of HW engines */
503	495	err = xe_gt_ccs_mode_sysfs_init(gt);
504	496	if (err)
505		− goto err_force_wake;
	497	+ return err;
506	498
507	499	/*
508	500	* Stash hardware-reported version. Since this register does not exist
@@ -510,25 +502,16 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt) · 1 unmodified line
510	502	*/
511	503	gt->info.gmdid = xe_mmio_read32(&gt->mmio, GMD_ID);
512	504
513		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
514	505	return 0;
515		−
516		−err_force_wake:
517		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
518		−
519		− return err;
520	506	}
521	507
522	508	static int gt_init_with_all_forcewake(struct xe_gt *gt)
523	509	{
524		− unsigned int fw_ref;
525	510	int err;
526	511
527		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
528		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
529		− err = -ETIMEDOUT;
530		− goto err_force_wake;
531		− }
	512	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	513	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
	514	+ return -ETIMEDOUT;
532	515
533	516	xe_gt_mcr_set_implicit_defaults(gt);
534	517	xe_wa_process_gt(gt);
@@ -537,20 +520,20 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) · 2 unmodified lines
537	520
538	521	err = xe_gt_clock_init(gt);
539	522	if (err)
540		− goto err_force_wake;
	523	+ return err;
541	524
542	525	xe_mocs_init(gt);
543	526	err = xe_execlist_init(gt);
544	527	if (err)
545		− goto err_force_wake;
	528	+ return err;
546	529
547	530	err = xe_hw_engines_init(gt);
548	531	if (err)
549		− goto err_force_wake;
	532	+ return err;
550	533
551	534	err = xe_uc_init_post_hwconfig(&gt->uc);
552	535	if (err)
553		− goto err_force_wake;
	536	+ return err;
554	537
555	538	if (xe_gt_is_main_type(gt)) {
556	539	/*
@@ -561,10 +544,8 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) · 4 unmodified lines
561	544
562	545	gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt),
563	546	IS_DGFX(xe) ? SZ_1M : SZ_512K, 16);
564		− if (IS_ERR(gt->usm.bb_pool)) {
565		− err = PTR_ERR(gt->usm.bb_pool);
566		− goto err_force_wake;
567		− }
	547	+ if (IS_ERR(gt->usm.bb_pool))
	548	+ return PTR_ERR(gt->usm.bb_pool);
568	549	}
569	550	}
570	551
@@ -573,12 +554,12 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) · 2 unmodified lines
573	554
574	555	err = xe_migrate_init(tile->migrate);
575	556	if (err)
576		− goto err_force_wake;
	557	+ return err;
577	558	}
578	559
579	560	err = xe_uc_load_hw(&gt->uc);
580	561	if (err)
581		− goto err_force_wake;
	562	+ return err;
582	563
583	564	/* Configure default CCS mode of 1 engine with all resources */
584	565	if (xe_gt_ccs_mode_enabled(gt)) {
@@ -592,14 +573,7 @@ static int gt_init_with_all_forcewake(struct xe_gt *gt) · 7 unmodified lines
592	573	if (IS_SRIOV_PF(gt_to_xe(gt)))
593	574	xe_gt_sriov_pf_init_hw(gt);
594	575
595		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
596		−
597	576	return 0;
598		−
599		−err_force_wake:
600		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
601		−
602		− return err;
603	577	}
604	578
605	579	static void xe_gt_fini(void *arg)
@@ -902,56 +876,42 @@ void xe_gt_reset_async(struct xe_gt *gt) · 296 unmodified lines
902	876
903	877	void xe_gt_suspend_prepare(struct xe_gt *gt)
904	878	{
905		− unsigned int fw_ref;
906		−
907		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
908		−
	879	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
909	880	xe_uc_suspend_prepare(&gt->uc);
910		−
911		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
912	881	}
913	882
914	883	int xe_gt_suspend(struct xe_gt *gt)
915	884	{
916		− unsigned int fw_ref;
917	885	int err;
918	886
919	887	xe_gt_dbg(gt, "suspending\n");
920	888	xe_gt_sanitize(gt);
921	889
922		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
923		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
924		− goto err_msg;
	890	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	891	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
	892	+ xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
	893	+ return -ETIMEDOUT;
	894	+ }
925	895
926	896	err = xe_uc_suspend(&gt->uc);
927		− if (err)
928		− goto err_force_wake;
	897	+ if (err) {
	898	+ xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
	899	+ return err;
	900	+ }
929	901
930	902	xe_gt_idle_disable_pg(gt);
931	903
932	904	xe_gt_disable_host_l2_vram(gt);
933	905
934		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
935	906	xe_gt_dbg(gt, "suspended\n");
936	907
937	908	return 0;
938		−
939		−err_msg:
940		− err = -ETIMEDOUT;
941		−err_force_wake:
942		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
943		− xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
944		−
945		− return err;
946	909	}
947	910
948	911	void xe_gt_shutdown(struct xe_gt *gt)
949	912	{
950		− unsigned int fw_ref;
951		−
952		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	913	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
953	914	do_gt_reset(gt);
954		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
955	915	}
956	916
957	917	/**
@@ -976,32 +936,72 @@ int xe_gt_sanitize_freq(struct xe_gt *gt) · 18 unmodified lines
976	936
977	937	int xe_gt_resume(struct xe_gt *gt)
978	938	{
979		− unsigned int fw_ref;
980	939	int err;
981	940
982	941	xe_gt_dbg(gt, "resuming\n");
983		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
984		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
985		− goto err_msg;
	942	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	943	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
	944	+ xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
	945	+ return -ETIMEDOUT;
	946	+ }
986	947
987	948	err = do_gt_restart(gt);
988	949	if (err)
989		− goto err_force_wake;
	950	+ return err;
990	951
991	952	xe_gt_idle_enable_pg(gt);
992	953
993		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
994	954	xe_gt_dbg(gt, "resumed\n");
995	955
996	956	return 0;
	957	+}
997	958
998		−err_msg:
999		− err = -ETIMEDOUT;
1000		−err_force_wake:
1001		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
1002		− xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));
	959	+/**
	960	+ * xe_gt_runtime_suspend() - GT runtime suspend
	961	+ * @gt: the GT object
	962	+ *
	963	+ * Return: 0 on success, negative error code otherwise.
	964	+ */
	965	+int xe_gt_runtime_suspend(struct xe_gt *gt)
	966	+{
	967	+ xe_gt_dbg(gt, "runtime suspending\n");
1003	968
1004		− return err;
	969	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	970	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
	971	+ xe_gt_err(gt, "runtime suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
	972	+ return -ETIMEDOUT;
	973	+ }
	974	+
	975	+ xe_uc_runtime_suspend(&gt->uc);
	976	+ xe_gt_disable_host_l2_vram(gt);
	977	+
	978	+ xe_gt_dbg(gt, "runtime suspended\n");
	979	+
	980	+ return 0;
	981	+}
	982	+
	983	+/**
	984	+ * xe_gt_runtime_resume() - GT runtime resume
	985	+ * @gt: the GT object
	986	+ *
	987	+ * Return: 0 on success, negative error code otherwise.
	988	+ */
	989	+int xe_gt_runtime_resume(struct xe_gt *gt)
	990	+{
	991	+ xe_gt_dbg(gt, "runtime resuming\n");
	992	+
	993	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	994	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
	995	+ xe_gt_err(gt, "runtime resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
	996	+ return -ETIMEDOUT;
	997	+ }
	998	+
	999	+ xe_gt_enable_host_l2_vram(gt);
	1000	+ xe_uc_runtime_resume(&gt->uc);
	1001	+
	1002	+ xe_gt_dbg(gt, "runtime resumed\n");
	1003	+
	1004	+ return 0;
1005	1005	}
1006	1006
1007	1007	struct xe_hw_engine xe_gt_hw_engine(struct xe_gt gt,

drivers/gpu/drm/xe/xe_gt.h+2−0

@@ -58,6 +58,8 @@ int xe_gt_suspend(struct xe_gt *gt); · 57 unmodified lines
58	58	void xe_gt_shutdown(struct xe_gt *gt);
59	59	int xe_gt_resume(struct xe_gt *gt);
60	60	void xe_gt_reset_async(struct xe_gt *gt);
	61	+int xe_gt_runtime_resume(struct xe_gt *gt);
	62	+int xe_gt_runtime_suspend(struct xe_gt *gt);
61	63	void xe_gt_sanitize(struct xe_gt *gt);
62	64	int xe_gt_sanitize_freq(struct xe_gt *gt);
63	65

drivers/gpu/drm/xe/xe_gt_debugfs.c+9−21

@@ -105,35 +105,24 @@ int xe_gt_debugfs_show_with_rpm(struct seq_file m, void data) · 104 unmodified lines
105	105	struct drm_info_node *node = m->private;
106	106	struct xe_gt *gt = node_to_gt(node);
107	107	struct xe_device *xe = gt_to_xe(gt);
108		− int ret;
109		−
110		− xe_pm_runtime_get(xe);
111		− ret = xe_gt_debugfs_simple_show(m, data);
112		− xe_pm_runtime_put(xe);
113	108
114		− return ret;
	109	+ guard(xe_pm_runtime)(xe);
	110	+ return xe_gt_debugfs_simple_show(m, data);
115	111	}
116	112
117	113	static int hw_engines(struct xe_gt gt, struct drm_printer p)
118	114	{
119	115	struct xe_hw_engine *hwe;
120	116	enum xe_hw_engine_id id;
121		− unsigned int fw_ref;
122		− int ret = 0;
123	117
124		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
125		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
126		− ret = -ETIMEDOUT;
127		− goto fw_put;
128		− }
	118	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	119	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
	120	+ return -ETIMEDOUT;
129	121
130	122	for_each_hw_engine(hwe, gt, id)
131	123	xe_hw_engine_print(hwe, p);
132	124
133		−fw_put:
134		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
135		−
136		− return ret;
	125	+ return 0;
137	126	}
138	127
139	128	static int steering(struct xe_gt gt, struct drm_printer p)
@@ -220,6 +209,7 @@ static const struct drm_info_list vf_safe_debugfs_list[] = { · 80 unmodified lines
220	209	{ "default_lrc_vcs", .show = xe_gt_debugfs_show_with_rpm, .data = vcs_default_lrc },
221	210	{ "default_lrc_vecs", .show = xe_gt_debugfs_show_with_rpm, .data = vecs_default_lrc },
222	211	{ "hwconfig", .show = xe_gt_debugfs_show_with_rpm, .data = hwconfig },
	212	+ { "pat_sw_config", .show = xe_gt_debugfs_simple_show, .data = xe_pat_dump_sw_config },
223	213	};
224	214
225	215	/* everything else should be added here */
@@ -269,9 +259,8 @@ static void force_reset(struct xe_gt *gt) · 43 unmodified lines
269	259	{
270	260	struct xe_device *xe = gt_to_xe(gt);
271	261
272		− xe_pm_runtime_get(xe);
	262	+ guard(xe_pm_runtime)(xe);
273	263	xe_gt_reset_async(gt);
274		− xe_pm_runtime_put(xe);
275	264	}
276	265
277	266	static ssize_t force_reset_write(struct file *file,
@@ -297,9 +286,8 @@ static void force_reset_sync(struct xe_gt *gt) · 19 unmodified lines
297	286	{
298	287	struct xe_device *xe = gt_to_xe(gt);
299	288
300		− xe_pm_runtime_get(xe);
	289	+ guard(xe_pm_runtime)(xe);
301	290	xe_gt_reset(gt);
302		− xe_pm_runtime_put(xe);
303	291	}
304	292
305	293	static ssize_t force_reset_sync_write(struct file *file,

drivers/gpu/drm/xe/xe_gt_freq.c+9−18

@@ -70,9 +70,8 @@ static ssize_t act_freq_show(struct kobject *kobj, · 69 unmodified lines
70	70	struct xe_guc_pc *pc = dev_to_pc(dev);
71	71	u32 freq;
72	72
73		− xe_pm_runtime_get(dev_to_xe(dev));
	73	+ guard(xe_pm_runtime)(dev_to_xe(dev));
74	74	freq = xe_guc_pc_get_act_freq(pc);
75		− xe_pm_runtime_put(dev_to_xe(dev));
76	75
77	76	return sysfs_emit(buf, "%d\n", freq);
78	77	}
@@ -86,9 +85,8 @@ static ssize_t cur_freq_show(struct kobject *kobj, · 7 unmodified lines
86	85	u32 freq;
87	86	ssize_t ret;
88	87
89		− xe_pm_runtime_get(dev_to_xe(dev));
	88	+ guard(xe_pm_runtime)(dev_to_xe(dev));
90	89	ret = xe_guc_pc_get_cur_freq(pc, &freq);
91		− xe_pm_runtime_put(dev_to_xe(dev));
92	90	if (ret)
93	91	return ret;
94	92
@@ -113,9 +111,8 @@ static ssize_t rpe_freq_show(struct kobject *kobj, · 18 unmodified lines
113	111	struct xe_guc_pc *pc = dev_to_pc(dev);
114	112	u32 freq;
115	113
116		− xe_pm_runtime_get(dev_to_xe(dev));
	114	+ guard(xe_pm_runtime)(dev_to_xe(dev));
117	115	freq = xe_guc_pc_get_rpe_freq(pc);
118		− xe_pm_runtime_put(dev_to_xe(dev));
119	116
120	117	return sysfs_emit(buf, "%d\n", freq);
121	118	}
@@ -128,9 +125,8 @@ static ssize_t rpa_freq_show(struct kobject *kobj, · 6 unmodified lines
128	125	struct xe_guc_pc *pc = dev_to_pc(dev);
129	126	u32 freq;
130	127
131		− xe_pm_runtime_get(dev_to_xe(dev));
	128	+ guard(xe_pm_runtime)(dev_to_xe(dev));
132	129	freq = xe_guc_pc_get_rpa_freq(pc);
133		− xe_pm_runtime_put(dev_to_xe(dev));
134	130
135	131	return sysfs_emit(buf, "%d\n", freq);
136	132	}
@@ -154,9 +150,8 @@ static ssize_t min_freq_show(struct kobject *kobj, · 17 unmodified lines
154	150	u32 freq;
155	151	ssize_t ret;
156	152
157		− xe_pm_runtime_get(dev_to_xe(dev));
	153	+ guard(xe_pm_runtime)(dev_to_xe(dev));
158	154	ret = xe_guc_pc_get_min_freq(pc, &freq);
159		− xe_pm_runtime_put(dev_to_xe(dev));
160	155	if (ret)
161	156	return ret;
162	157
@@ -175,9 +170,8 @@ static ssize_t min_freq_store(struct kobject *kobj, · 12 unmodified lines
175	170	if (ret)
176	171	return ret;
177	172
178		− xe_pm_runtime_get(dev_to_xe(dev));
	173	+ guard(xe_pm_runtime)(dev_to_xe(dev));
179	174	ret = xe_guc_pc_set_min_freq(pc, freq);
180		− xe_pm_runtime_put(dev_to_xe(dev));
181	175	if (ret)
182	176	return ret;
183	177
@@ -193,9 +187,8 @@ static ssize_t max_freq_show(struct kobject *kobj, · 9 unmodified lines
193	187	u32 freq;
194	188	ssize_t ret;
195	189
196		− xe_pm_runtime_get(dev_to_xe(dev));
	190	+ guard(xe_pm_runtime)(dev_to_xe(dev));
197	191	ret = xe_guc_pc_get_max_freq(pc, &freq);
198		− xe_pm_runtime_put(dev_to_xe(dev));
199	192	if (ret)
200	193	return ret;
201	194
@@ -214,9 +207,8 @@ static ssize_t max_freq_store(struct kobject *kobj, · 12 unmodified lines
214	207	if (ret)
215	208	return ret;
216	209
217		− xe_pm_runtime_get(dev_to_xe(dev));
	210	+ guard(xe_pm_runtime)(dev_to_xe(dev));
218	211	ret = xe_guc_pc_set_max_freq(pc, freq);
219		− xe_pm_runtime_put(dev_to_xe(dev));
220	212	if (ret)
221	213	return ret;
222	214
@@ -243,9 +235,8 @@ static ssize_t power_profile_store(struct kobject *kobj, · 20 unmodified lines
243	235	struct xe_guc_pc *pc = dev_to_pc(dev);
244	236	int err;
245	237
246		− xe_pm_runtime_get(dev_to_xe(dev));
	238	+ guard(xe_pm_runtime)(dev_to_xe(dev));
247	239	err = xe_guc_pc_set_power_profile(pc, buff);
248		− xe_pm_runtime_put(dev_to_xe(dev));
249	240
250	241	return err ?: count;
251	242	}

drivers/gpu/drm/xe/xe_gt_idle.c+12−29

@@ -105,7 +105,6 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt) · 104 unmodified lines
105	105	struct xe_gt_idle *gtidle = &gt->gtidle;
106	106	struct xe_mmio *mmio = &gt->mmio;
107	107	u32 vcs_mask, vecs_mask;
108		− unsigned int fw_ref;
109	108	int i, j;
110	109
111	110	if (IS_SRIOV_VF(xe))
@@ -137,7 +136,7 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt) · 25 unmodified lines
137	136	}
138	137	}
139	138
140		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
	139	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
141	140	if (xe->info.skip_guc_pc) {
142	141	/*
143	142	* GuC sets the hysteresis value when GuC PC is enabled
@@ -154,13 +153,11 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt) · 10 unmodified lines
154	153	VDN_MFXVDENC_POWERGATE_ENABLE(2));
155	154
156	155	xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
157		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
158	156	}
159	157
160	158	void xe_gt_idle_disable_pg(struct xe_gt *gt)
161	159	{
162	160	struct xe_gt_idle *gtidle = &gt->gtidle;
163		− unsigned int fw_ref;
164	161
165	162	if (IS_SRIOV_VF(gt_to_xe(gt)))
166	163	return;
@@ -168,9 +165,8 @@ void xe_gt_idle_disable_pg(struct xe_gt *gt) · 1 unmodified line
168	165	xe_device_assert_mem_access(gt_to_xe(gt));
169	166	gtidle->powergate_enable = 0;
170	167
171		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
	168	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
172	169	xe_mmio_write32(&gt->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
173		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
174	170	}
175	171
176	172	/**
@@ -189,7 +185,6 @@ int xe_gt_idle_pg_print(struct xe_gt gt, struct drm_printer p) · 12 unmodified lines
189	185	enum xe_gt_idle_state state;
190	186	u32 pg_enabled, pg_status = 0;
191	187	u32 vcs_mask, vecs_mask;
192		− unsigned int fw_ref;
193	188	int n;
194	189	/*
195	190	* Media Slices
@@ -226,14 +221,12 @@ int xe_gt_idle_pg_print(struct xe_gt gt, struct drm_printer p) · 30 unmodified lines
226	221
227	222	/* Do not wake the GT to read powergating status */
228	223	if (state != GT_IDLE_C6) {
229		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
230		− if (!fw_ref)
	224	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	225	+ if (!fw_ref.domains)
231	226	return -ETIMEDOUT;
232	227
233	228	pg_enabled = xe_mmio_read32(&gt->mmio, POWERGATE_ENABLE);
234	229	pg_status = xe_mmio_read32(&gt->mmio, POWERGATE_DOMAIN_STATUS);
235		−
236		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
237	230	}
238	231
239	232	if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) {
@@ -271,13 +264,9 @@ static ssize_t name_show(struct kobject *kobj, · 31 unmodified lines
271	264	struct device *dev = kobj_to_dev(kobj);
272	265	struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
273	266	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
274		− ssize_t ret;
275		−
276		− xe_pm_runtime_get(pc_to_xe(pc));
277		− ret = sysfs_emit(buff, "%s\n", gtidle->name);
278		− xe_pm_runtime_put(pc_to_xe(pc));
279	267
280		− return ret;
	268	+ guard(xe_pm_runtime)(pc_to_xe(pc));
	269	+ return sysfs_emit(buff, "%s\n", gtidle->name);
281	270	}
282	271	static struct kobj_attribute name_attr = __ATTR_RO(name);
283	272
@@ -289,9 +278,8 @@ static ssize_t idle_status_show(struct kobject *kobj, · 5 unmodified lines
289	278	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
290	279	enum xe_gt_idle_state state;
291	280
292		− xe_pm_runtime_get(pc_to_xe(pc));
293		− state = gtidle->idle_status(pc);
294		− xe_pm_runtime_put(pc_to_xe(pc));
	281	+ scoped_guard(xe_pm_runtime, pc_to_xe(pc))
	282	+ state = gtidle->idle_status(pc);
295	283
296	284	return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
297	285	}
@@ -319,9 +307,8 @@ static ssize_t idle_residency_ms_show(struct kobject *kobj, · 21 unmodified lines
319	307	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
320	308	u64 residency;
321	309
322		− xe_pm_runtime_get(pc_to_xe(pc));
323		− residency = xe_gt_idle_residency_msec(gtidle);
324		− xe_pm_runtime_put(pc_to_xe(pc));
	310	+ scoped_guard(xe_pm_runtime, pc_to_xe(pc))
	311	+ residency = xe_gt_idle_residency_msec(gtidle);
325	312
326	313	return sysfs_emit(buff, "%llu\n", residency);
327	314	}
@@ -404,21 +391,17 @@ void xe_gt_idle_enable_c6(struct xe_gt *gt) · 76 unmodified lines
404	391
405	392	int xe_gt_idle_disable_c6(struct xe_gt *gt)
406	393	{
407		− unsigned int fw_ref;
408		−
409	394	xe_device_assert_mem_access(gt_to_xe(gt));
410	395
411	396	if (IS_SRIOV_VF(gt_to_xe(gt)))
412	397	return 0;
413	398
414		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
415		− if (!fw_ref)
	399	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	400	+ if (!fw_ref.domains)
416	401	return -ETIMEDOUT;
417	402
418	403	xe_mmio_write32(&gt->mmio, RC_CONTROL, 0);
419	404	xe_mmio_write32(&gt->mmio, RC_STATE, 0);
420	405
421		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
422		−
423	406	return 0;
424	407	}

drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c+12−7

@@ -269,7 +269,8 @@ static u32 encode_config_ggtt(u32 cfg, const struct xe_gt_sriov_config config, · 268 unmodified lines
269	269	}
270	270
271	271	/* Return: number of configuration dwords written */
272		−static u32 encode_config(u32 cfg, const struct xe_gt_sriov_config config, bool details)
	272	+static u32 encode_config(struct xe_gt gt, u32 cfg, const struct xe_gt_sriov_config *config,
	273	+ bool details)
273	274	{
274	275	u32 n = 0;
275	276
@@ -303,9 +304,11 @@ static u32 encode_config(u32 cfg, const struct xe_gt_sriov_config config, bool · 27 unmodified lines
303	304	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT);
304	305	cfg[n++] = config->preempt_timeout;
305	306
306		−#define encode_threshold_config(TAG, ...) ({ \
307		− cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG); \
308		− cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)]; \
	307	+#define encode_threshold_config(TAG, NAME, VER...) ({ \
	308	+ if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), true, VER)) { \
	309	+ cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG); \
	310	+ cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)]; \
	311	+ } \
309	312	});
310	313
311	314	MAKE_XE_GUC_KLV_THRESHOLDS_SET(encode_threshold_config);
@@ -328,7 +331,7 @@ static int pf_push_full_vf_config(struct xe_gt *gt, unsigned int vfid) · 16 unmodified lines
328	331	return -ENOBUFS;
329	332
330	333	cfg = xe_guc_buf_cpu_ptr(buf);
331		− num_dwords = encode_config(cfg, config, true);
	334	+ num_dwords = encode_config(gt, cfg, config, true);
332	335	xe_gt_assert(gt, num_dwords <= max_cfg_dwords);
333	336
334	337	if (xe_gt_is_media_type(gt)) {
@@ -2518,7 +2521,7 @@ ssize_t xe_gt_sriov_pf_config_save(struct xe_gt gt, unsigned int vfid, void bu · 2183 unmodified lines
2518	2521	ret = -ENOBUFS;
2519	2522	} else {
2520	2523	config = pf_pick_vf_config(gt, vfid);
2521		− ret = encode_config(buf, config, false) * sizeof(u32);
	2524	+ ret = encode_config(gt, buf, config, false) * sizeof(u32);
2522	2525	}
2523	2526	}
2524	2527	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
@@ -2551,11 +2554,13 @@ static int pf_restore_vf_config_klv(struct xe_gt *gt, unsigned int vfid, · 26 unmodified lines
2551	2554	return pf_provision_preempt_timeout(gt, vfid, value[0]);
2552	2555
2553	2556	/* auto-generate case statements */
2554		−#define define_threshold_key_to_provision_case(TAG, ...) \
	2557	+#define define_threshold_key_to_provision_case(TAG, NAME, VER...) \
2555	2558	case MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG): \
2556	2559	BUILD_BUG_ON(MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG) != 1u); \
2557	2560	if (len != MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG)) \
2558	2561	return -EBADMSG; \
	2562	+ if (IF_ARGS(!GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), false, VER)) \
	2563	+ return -EKEYREJECTED; \
2559	2564	return pf_provision_threshold(gt, vfid, \
2560	2565	MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG), \
2561	2566	value[0]);

drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c+10−11

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#include "xe_gt_sriov_pf_monitor.h"
22	22	#include "xe_gt_sriov_pf_policy.h"
23	23	#include "xe_gt_sriov_pf_service.h"
	24	+#include "xe_guc.h"
24	25	#include "xe_pm.h"
25	26	#include "xe_sriov_pf.h"
26	27	#include "xe_sriov_pf_provision.h"
@@ -123,11 +124,10 @@ static int POLICY##_set(void *data, u64 val) \ · 96 unmodified lines
123	124	if (val > (TYPE)~0ull) \
124	125	return -EOVERFLOW; \
125	126	\
126		− xe_pm_runtime_get(xe); \
	127	+ guard(xe_pm_runtime)(xe); \
127	128	err = xe_gt_sriov_pf_policy_set_##POLICY(gt, val); \
128	129	if (!err) \
129	130	xe_sriov_pf_provision_set_custom_mode(xe); \
130		− xe_pm_runtime_put(xe); \
131	131	\
132	132	return err; \
133	133	} \
@@ -189,12 +189,11 @@ static int CONFIG##_set(void *data, u64 val) \ · 55 unmodified lines
189	189	if (val > (TYPE)~0ull) \
190	190	return -EOVERFLOW; \
191	191	\
192		− xe_pm_runtime_get(xe); \
	192	+ guard(xe_pm_runtime)(xe); \
193	193	err = xe_sriov_pf_wait_ready(xe) ?: \
194	194	xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \
195	195	if (!err) \
196	196	xe_sriov_pf_provision_set_custom_mode(xe); \
197		− xe_pm_runtime_put(xe); \
198	197	\
199	198	return err; \
200	199	} \
@@ -249,11 +248,10 @@ static int set_threshold(void *data, u64 val, enum xe_guc_klv_threshold_index in · 48 unmodified lines
249	248	if (val > (u32)~0ull)
250	249	return -EOVERFLOW;
251	250
252		− xe_pm_runtime_get(xe);
	251	+ guard(xe_pm_runtime)(xe);
253	252	err = xe_gt_sriov_pf_config_set_threshold(gt, vfid, index, val);
254	253	if (!err)
255	254	xe_sriov_pf_provision_set_custom_mode(xe);
256		− xe_pm_runtime_put(xe);
257	255
258	256	return err;
259	257	}
@@ -304,9 +302,11 @@ static void pf_add_config_attrs(struct xe_gt gt, struct dentry parent, unsigne · 44 unmodified lines
304	302	&sched_priority_fops);
305	303
306	304	/* register all threshold attributes */
307		−#define register_threshold_attribute(TAG, NAME, ...) \
308		− debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent, \
309		− &NAME##_fops);
	305	+#define register_threshold_attribute(TAG, NAME, VER...) ({ \
	306	+ if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), true, VER)) \
	307	+ debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent, \
	308	+ &NAME##_fops); \
	309	+});
310	310	MAKE_XE_GUC_KLV_THRESHOLDS_SET(register_threshold_attribute)
311	311	#undef register_threshold_attribute
312	312	}
@@ -358,9 +358,8 @@ static ssize_t control_write(struct file file, const char __user buf, size_t c · 45 unmodified lines
358	358	xe_gt_assert(gt, sizeof(cmd) > strlen(control_cmds[n].cmd));
359	359
360	360	if (sysfs_streq(cmd, control_cmds[n].cmd)) {
361		− xe_pm_runtime_get(xe);
	361	+ guard(xe_pm_runtime)(xe);
362	362	ret = control_cmds[n].fn ? (*control_cmds[n].fn)(gt, vfid) : 0;
363		− xe_pm_runtime_put(xe);
364	363	break;
365	364	}
366	365	}

drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c+1−1

@@ -1026,7 +1026,7 @@ static void action_ring_cleanup(void *arg) · 1025 unmodified lines
1026	1026
1027	1027	static void pf_gt_migration_check_support(struct xe_gt *gt)
1028	1028	{
1029		− if (GUC_FIRMWARE_VER(&gt->uc.guc) < MAKE_GUC_VER(70, 54, 0))
	1029	+ if (!GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 54))
1030	1030	xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0");
1031	1031	}
1032	1032

drivers/gpu/drm/xe/xe_gt_sriov_vf.c+126−44

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5
6	6	#include <linux/bitfield.h>
7	7	#include <linux/bsearch.h>
	8	+#include <linux/delay.h>
8	9
9	10	#include <drm/drm_managed.h>
10	11	#include <drm/drm_print.h>
@@ -41,6 +42,37 @@ · 30 unmodified lines
41	42
42	43	#define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 \| (u32)(lo)))
43	44
	45	+#ifdef CONFIG_DRM_XE_DEBUG
	46	+enum VF_MIGRATION_WAIT_POINTS {
	47	+ VF_MIGRATION_WAIT_RESFIX_START = BIT(0),
	48	+ VF_MIGRATION_WAIT_FIXUPS = BIT(1),
	49	+ VF_MIGRATION_WAIT_RESTART_JOBS = BIT(2),
	50	+ VF_MIGRATION_WAIT_RESFIX_DONE = BIT(3),
	51	+};
	52	+
	53	+#define VF_MIGRATION_WAIT_DELAY_IN_MS 1000
	54	+static void vf_post_migration_inject_wait(struct xe_gt *gt,
	55	+ enum VF_MIGRATION_WAIT_POINTS wait)
	56	+{
	57	+ while (gt->sriov.vf.migration.debug.resfix_stoppers & wait) {
	58	+ xe_gt_dbg(gt,
	59	+ "TESTING injecting %u ms delay due to resfix_stoppers=%#x, to continue clear %#x\n",
	60	+ VF_MIGRATION_WAIT_DELAY_IN_MS,
	61	+ gt->sriov.vf.migration.debug.resfix_stoppers, wait);
	62	+
	63	+ msleep(VF_MIGRATION_WAIT_DELAY_IN_MS);
	64	+ }
	65	+}
	66	+
	67	+#define VF_MIGRATION_INJECT_WAIT(gt, _POS) ({ \
	68	+ struct xe_gt *__gt = (gt); \
	69	+ vf_post_migration_inject_wait(__gt, VF_MIGRATION_WAIT_##_POS); \
	70	+ })
	71	+
	72	+#else
	73	+#define VF_MIGRATION_INJECT_WAIT(_gt, ...) typecheck(struct xe_gt *, (_gt))
	74	+#endif
	75	+
44	76	static int guc_action_vf_reset(struct xe_guc *guc)
45	77	{
46	78	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
@@ -299,12 +331,13 @@ void xe_gt_sriov_vf_guc_versions(struct xe_gt *gt, · 252 unmodified lines
299	331	*found = gt->sriov.vf.guc_version;
300	332	}
301	333
302		−static int guc_action_vf_notify_resfix_done(struct xe_guc *guc)
	334	+static int guc_action_vf_resfix_start(struct xe_guc *guc, u16 marker)
303	335	{
304	336	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
305	337	FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) \|
306	338	FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) \|
307		− FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE),
	339	+ FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_START) \|
	340	+ FIELD_PREP(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER, marker),
308	341	};
309	342	int ret;
310	343
@@ -313,28 +346,43 @@ static int guc_action_vf_notify_resfix_done(struct xe_guc *guc) · 2 unmodified lines
313	346	return ret > 0 ? -EPROTO : ret;
314	347	}
315	348
316		−/**
317		− * vf_notify_resfix_done - Notify GuC about resource fixups apply completed.
318		− * @gt: the &xe_gt struct instance linked to target GuC
319		− *
320		− * Returns: 0 if the operation completed successfully, or a negative error
321		− * code otherwise.
322		− */
323		−static int vf_notify_resfix_done(struct xe_gt *gt)
	349	+static int vf_resfix_start(struct xe_gt *gt, u16 marker)
324	350	{
325	351	struct xe_guc *guc = &gt->uc.guc;
326		− int err;
327	352
328	353	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
329	354
330		− err = guc_action_vf_notify_resfix_done(guc);
331		− if (unlikely(err))
332		− xe_gt_sriov_err(gt, "Failed to notify GuC about resource fixup done (%pe)\n",
333		− ERR_PTR(err));
334		− else
335		− xe_gt_sriov_dbg_verbose(gt, "sent GuC resource fixup done\n");
	355	+ VF_MIGRATION_INJECT_WAIT(gt, RESFIX_START);
336	356
337		− return err;
	357	+ xe_gt_sriov_dbg_verbose(gt, "Sending resfix start marker %u\n", marker);
	358	+
	359	+ return guc_action_vf_resfix_start(guc, marker);
	360	+}
	361	+
	362	+static int guc_action_vf_resfix_done(struct xe_guc *guc, u16 marker)
	363	+{
	364	+ u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
	365	+ FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) \|
	366	+ FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) \|
	367	+ FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_DONE) \|
	368	+ FIELD_PREP(VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER, marker),
	369	+ };
	370	+ int ret;
	371	+
	372	+ ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request));
	373	+
	374	+ return ret > 0 ? -EPROTO : ret;
	375	+}
	376	+
	377	+static int vf_resfix_done(struct xe_gt *gt, u16 marker)
	378	+{
	379	+ struct xe_guc *guc = &gt->uc.guc;
	380	+
	381	+ xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
	382	+
	383	+ xe_gt_sriov_dbg_verbose(gt, "Sending resfix done marker %u\n", marker);
	384	+
	385	+ return guc_action_vf_resfix_done(guc, marker);
338	386	}
339	387
340	388	static int guc_action_query_single_klv(struct xe_guc *guc, u32 key,
@@ -1123,12 +1171,8 @@ static bool vf_post_migration_shutdown(struct xe_gt *gt) · 782 unmodified lines
1123	1171	return true;
1124	1172	}
1125	1173
1126		− spin_lock_irq(&gt->sriov.vf.migration.lock);
1127		− gt->sriov.vf.migration.recovery_queued = false;
1128		− spin_unlock_irq(&gt->sriov.vf.migration.lock);
1129		−
1130	1174	xe_guc_ct_flush_and_stop(&gt->uc.guc.ct);
1131		− xe_guc_submit_pause(&gt->uc.guc);
	1175	+ xe_guc_submit_pause_vf(&gt->uc.guc);
1132	1176	xe_tlb_inval_reset(&gt->tlb_inval);
1133	1177
1134	1178	return false;
@@ -1144,6 +1188,8 @@ static int vf_post_migration_fixups(struct xe_gt *gt) · 9 unmodified lines
1144	1188	void *buf = gt->sriov.vf.migration.scratch;
1145	1189	int err;
1146	1190
	1191	+ VF_MIGRATION_INJECT_WAIT(gt, FIXUPS);
	1192	+
1147	1193	/* xe_gt_sriov_vf_query_config will fixup the GGTT addresses */
1148	1194	err = xe_gt_sriov_vf_query_config(gt);
1149	1195	if (err)
@@ -1162,13 +1208,22 @@ static int vf_post_migration_fixups(struct xe_gt *gt) · 12 unmodified lines
1162	1208
1163	1209	static void vf_post_migration_rearm(struct xe_gt *gt)
1164	1210	{
	1211	+ VF_MIGRATION_INJECT_WAIT(gt, RESTART_JOBS);
	1212	+
	1213	+ /*
	1214	+ * Make sure interrupts on the new HW are properly set. The GuC IRQ
	1215	+ * must be working at this point, since the recovery did started,
	1216	+ * but the rest was not enabled using the procedure from spec.
	1217	+ */
	1218	+ xe_irq_resume(gt_to_xe(gt));
	1219	+
1165	1220	xe_guc_ct_restart(&gt->uc.guc.ct);
1166		− xe_guc_submit_unpause_prepare(&gt->uc.guc);
	1221	+ xe_guc_submit_unpause_prepare_vf(&gt->uc.guc);
1167	1222	}
1168	1223
1169	1224	static void vf_post_migration_kickstart(struct xe_gt *gt)
1170	1225	{
1171		− xe_guc_submit_unpause(&gt->uc.guc);
	1226	+ xe_guc_submit_unpause_vf(&gt->uc.guc);
1172	1227	}
1173	1228
1174	1229	static void vf_post_migration_abort(struct xe_gt *gt)
@@ -1183,37 +1238,49 @@ static void vf_post_migration_abort(struct xe_gt *gt) · 8 unmodified lines
1183	1238	xe_guc_submit_pause_abort(&gt->uc.guc);
1184	1239	}
1185	1240
1186		−static int vf_post_migration_notify_resfix_done(struct xe_gt *gt)
	1241	+static int vf_post_migration_resfix_done(struct xe_gt *gt, u16 marker)
1187	1242	{
1188		− bool skip_resfix = false;
	1243	+ VF_MIGRATION_INJECT_WAIT(gt, RESFIX_DONE);
1189	1244
1190	1245	spin_lock_irq(&gt->sriov.vf.migration.lock);
1191		− if (gt->sriov.vf.migration.recovery_queued) {
1192		− skip_resfix = true;
1193		− xe_gt_sriov_dbg(gt, "another recovery imminent, resfix skipped\n");
1194		− } else {
	1246	+ if (gt->sriov.vf.migration.recovery_queued)
	1247	+ xe_gt_sriov_dbg(gt, "another recovery imminent\n");
	1248	+ else
1195	1249	WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, false);
1196		− }
1197	1250	spin_unlock_irq(&gt->sriov.vf.migration.lock);
1198	1251
1199		− if (skip_resfix)
1200		− return -EAGAIN;
	1252	+ return vf_resfix_done(gt, marker);
	1253	+}
1201	1254
1202		− /*
1203		− * Make sure interrupts on the new HW are properly set. The GuC IRQ
1204		− * must be working at this point, since the recovery did started,
1205		− * but the rest was not enabled using the procedure from spec.
1206		− */
1207		− xe_irq_resume(gt_to_xe(gt));
	1255	+static int vf_post_migration_resfix_start(struct xe_gt *gt, u16 marker)
	1256	+{
	1257	+ int err;
	1258	+
	1259	+ err = vf_resfix_start(gt, marker);
1208	1260
1209		− return vf_notify_resfix_done(gt);
	1261	+ guard(spinlock_irq) (&gt->sriov.vf.migration.lock);
	1262	+ gt->sriov.vf.migration.recovery_queued = false;
	1263	+
	1264	+ return err;
	1265	+}
	1266	+
	1267	+static u16 vf_post_migration_next_resfix_marker(struct xe_gt *gt)
	1268	+{
	1269	+ xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
	1270	+
	1271	+ BUILD_BUG_ON(1 + ((typeof(gt->sriov.vf.migration.resfix_marker))~0) >
	1272	+ FIELD_MAX(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER));
	1273	+
	1274	+ /* add 1 to avoid zero-marker */
	1275	+ return 1 + gt->sriov.vf.migration.resfix_marker++;
1210	1276	}
1211	1277
1212	1278	static void vf_post_migration_recovery(struct xe_gt *gt)
1213	1279	{
1214	1280	struct xe_device *xe = gt_to_xe(gt);
1215		− int err;
	1281	+ u16 marker;
1216	1282	bool retry;
	1283	+ int err;
1217	1284
1218	1285	xe_gt_sriov_dbg(gt, "migration recovery in progress\n");
1219	1286
@@ -1227,15 +1294,30 @@ static void vf_post_migration_recovery(struct xe_gt *gt) · 7 unmodified lines
1227	1294	goto fail;
1228	1295	}
1229	1296
	1297	+ marker = vf_post_migration_next_resfix_marker(gt);
	1298	+
	1299	+ err = vf_post_migration_resfix_start(gt, marker);
	1300	+ if (unlikely(err)) {
	1301	+ xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_START step (%pe)\n",
	1302	+ ERR_PTR(err));
	1303	+ goto fail;
	1304	+ }
	1305	+
1230	1306	err = vf_post_migration_fixups(gt);
1231	1307	if (err)
1232	1308	goto fail;
1233	1309
1234	1310	vf_post_migration_rearm(gt);
1235	1311
1236		− err = vf_post_migration_notify_resfix_done(gt);
1237		− if (err && err != -EAGAIN)
	1312	+ err = vf_post_migration_resfix_done(gt, marker);
	1313	+ if (err) {
	1314	+ if (err == -EREMCHG)
	1315	+ goto queue;
	1316	+
	1317	+ xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_DONE step (%pe)\n",
	1318	+ ERR_PTR(err));
1238	1319	goto fail;
	1320	+ }
1239	1321
1240	1322	vf_post_migration_kickstart(gt);
1241	1323

drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c+12−0

@@ -69,4 +69,16 @@ void xe_gt_sriov_vf_debugfs_register(struct xe_gt gt, struct dentry root) · 68 unmodified lines
69	69	vfdentry->d_inode->i_private = gt;
70	70
71	71	drm_debugfs_create_files(vf_info, ARRAY_SIZE(vf_info), vfdentry, minor);
	72	+
	73	+ /*
	74	+ * /sys/kernel/debug/dri/BDF/
	75	+ * ├── tile0
	76	+ * ├── gt0
	77	+ * ├── vf
	78	+ * ├── resfix_stoppers
	79	+ */
	80	+ if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
	81	+ debugfs_create_x8("resfix_stoppers", 0600, vfdentry,
	82	+ &gt->sriov.vf.migration.debug.resfix_stoppers);
	83	+ }
72	84	}

drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h+13−0

@@ -52,6 +52,19 @@ struct xe_gt_sriov_vf_migration { · 51 unmodified lines
52	52	wait_queue_head_t wq;
53	53	/** @scratch: Scratch memory for VF recovery */
54	54	void *scratch;
	55	+ /** @debug: Debug hooks for delaying migration */
	56	+ struct {
	57	+ /**
	58	+ * @debug.resfix_stoppers: Stop and wait at different stages
	59	+ * during post migration recovery
	60	+ */
	61	+ u8 resfix_stoppers;
	62	+ } debug;
	63	+ /**
	64	+ * @resfix_marker: Marker sent on start and on end of post-migration
	65	+ * steps.
	66	+ */
	67	+ u8 resfix_marker;
55	68	/** @recovery_teardown: VF post migration recovery is being torn down */
56	69	bool recovery_teardown;
57	70	/** @recovery_queued: VF post migration recovery in queued */

drivers/gpu/drm/xe/xe_gt_stats.c+10−0

@@ -66,6 +66,16 @@ static const char *const stat_description[__XE_GT_STATS_NUM_IDS] = { · 65 unmodified lines
66	66	DEF_STAT_STR(SVM_4K_BIND_US, "svm_4K_bind_us"),
67	67	DEF_STAT_STR(SVM_64K_BIND_US, "svm_64K_bind_us"),
68	68	DEF_STAT_STR(SVM_2M_BIND_US, "svm_2M_bind_us"),
	69	+ DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT,
	70	+ "hw_engine_group_suspend_lr_queue_count"),
	71	+ DEF_STAT_STR(HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT,
	72	+ "hw_engine_group_skip_lr_queue_count"),
	73	+ DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT,
	74	+ "hw_engine_group_wait_dma_queue_count"),
	75	+ DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
	76	+ "hw_engine_group_suspend_lr_queue_us"),
	77	+ DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
	78	+ "hw_engine_group_wait_dma_queue_us"),
69	79	};
70	80
71	81	/**

drivers/gpu/drm/xe/xe_gt_stats.h+32−0

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	6	#ifndef _XE_GT_STATS_H_
7	7	#define _XE_GT_STATS_H_
8	8
	9	+#include <linux/ktime.h>
	10	+
9	11	#include "xe_gt_stats_types.h"
10	12
11	13	struct xe_gt;
@@ -23,4 +25,34 @@ xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, · 11 unmodified lines
23	25	}
24	26
25	27	#endif
	28	+
	29	+/**
	30	+ * xe_gt_stats_ktime_us_delta() - Get delta in microseconds between now and a
	31	+ * start time
	32	+ * @start: Start time
	33	+ *
	34	+ * Helper for GT stats to get delta in microseconds between now and a start
	35	+ * time, compiles out if GT stats are disabled.
	36	+ *
	37	+ * Return: Delta in microseconds between now and a start time
	38	+ */
	39	+static inline s64 xe_gt_stats_ktime_us_delta(ktime_t start)
	40	+{
	41	+ return IS_ENABLED(CONFIG_DEBUG_FS) ?
	42	+ ktime_us_delta(ktime_get(), start) : 0;
	43	+}
	44	+
	45	+/**
	46	+ * xe_gt_stats_ktime_get() - Get current ktime
	47	+ *
	48	+ * Helper for GT stats to get current ktime, compiles out if GT stats are
	49	+ * disabled.
	50	+ *
	51	+ * Return: Get current ktime
	52	+ */
	53	+static inline ktime_t xe_gt_stats_ktime_get(void)
	54	+{
	55	+ return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0;
	56	+}
	57	+
26	58	#endif

drivers/gpu/drm/xe/xe_gt_stats_types.h+5−0

@@ -44,6 +44,11 @@ enum xe_gt_stats_id { · 43 unmodified lines
44	44	XE_GT_STATS_ID_SVM_4K_BIND_US,
45	45	XE_GT_STATS_ID_SVM_64K_BIND_US,
46	46	XE_GT_STATS_ID_SVM_2M_BIND_US,
	47	+ XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT,
	48	+ XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT,
	49	+ XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT,
	50	+ XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
	51	+ XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
47	52	/* must be the last entry */
48	53	__XE_GT_STATS_NUM_IDS,
49	54	};

drivers/gpu/drm/xe/xe_gt_throttle.c+3−6

@@ -85,7 +85,7 @@ u32 xe_gt_throttle_get_limit_reasons(struct xe_gt *gt) · 84 unmodified lines
85	85	{
86	86	struct xe_device *xe = gt_to_xe(gt);
87	87	struct xe_reg reg;
88		− u32 val, mask;
	88	+ u32 mask;
89	89
90	90	if (xe_gt_is_media_type(gt))
91	91	reg = MTL_MEDIA_PERF_LIMIT_REASONS;
@@ -97,11 +97,8 @@ u32 xe_gt_throttle_get_limit_reasons(struct xe_gt *gt) · 5 unmodified lines
97	97	else
98	98	mask = GT0_PERF_LIMIT_REASONS_MASK;
99	99
100		− xe_pm_runtime_get(xe);
101		− val = xe_mmio_read32(&gt->mmio, reg) & mask;
102		− xe_pm_runtime_put(xe);
103		−
104		− return val;
	100	+ guard(xe_pm_runtime)(xe);
	101	+ return xe_mmio_read32(&gt->mmio, reg) & mask;
105	102	}
106	103
107	104	static bool is_throttled_by(struct xe_gt *gt, u32 mask)

drivers/gpu/drm/xe/xe_gt_types.h+5−0

@@ -140,6 +140,11 @@ struct xe_gt { · 139 unmodified lines
140	140	u64 engine_mask;
141	141	/** @info.gmdid: raw GMD_ID value from hardware */
142	142	u32 gmdid;
	143	+ /**
	144	+ * @multi_queue_engine_class_mask: Bitmask of engine classes with
	145	+ * multi queue support enabled.
	146	+ */
	147	+ u16 multi_queue_engine_class_mask;
143	148	/** @info.id: Unique ID of this GT within the PCI Device */
144	149	u8 id;
145	150	/** @info.has_indirect_ring_state: GT has indirect ring state support */

drivers/gpu/drm/xe/xe_guc.c+60−20

@@ -104,7 +104,7 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc) · 103 unmodified lines
104	104	u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT;
105	105	u32 flags;
106	106
107		− #if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
	107	+ #if (((XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE) % SZ_1M) == 0)
108	108	#define LOG_UNIT SZ_1M
109	109	#define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
110	110	#else
@@ -112,7 +112,7 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc) · 1 unmodified line
112	112	#define LOG_FLAG 0
113	113	#endif
114	114
115		− #if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
	115	+ #if (((XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
116	116	#define CAPTURE_UNIT SZ_1M
117	117	#define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
118	118	#else
@@ -120,20 +120,21 @@ static u32 guc_ctl_log_params_flags(struct xe_guc *guc) · 1 unmodified line
120	120	#define CAPTURE_FLAG 0
121	121	#endif
122	122
123		− BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
124		− BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
125		− BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
126		− BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
127		− BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
128		− BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
	123	+ BUILD_BUG_ON(!XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE);
	124	+ BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE, LOG_UNIT));
	125	+ BUILD_BUG_ON(!XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE);
	126	+ BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE, LOG_UNIT));
	127	+ BUILD_BUG_ON(!XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE);
	128	+ BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
129	129
130	130	flags = GUC_LOG_VALID \|
131	131	GUC_LOG_NOTIFY_ON_HALF_FULL \|
132	132	CAPTURE_FLAG \|
133	133	LOG_FLAG \|
134		− FIELD_PREP(GUC_LOG_CRASH, CRASH_BUFFER_SIZE / LOG_UNIT - 1) \|
135		− FIELD_PREP(GUC_LOG_DEBUG, DEBUG_BUFFER_SIZE / LOG_UNIT - 1) \|
136		− FIELD_PREP(GUC_LOG_CAPTURE, CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) \|
	134	+ FIELD_PREP(GUC_LOG_CRASH_DUMP, XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE / LOG_UNIT - 1) \|
	135	+ FIELD_PREP(GUC_LOG_EVENT_DATA, XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE / LOG_UNIT - 1) \|
	136	+ FIELD_PREP(GUC_LOG_STATE_CAPTURE, XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE /
	137	+ CAPTURE_UNIT - 1) \|
137	138	FIELD_PREP(GUC_LOG_BUF_ADDR, offset);
138	139
139	140	#undef LOG_UNIT
@@ -660,11 +661,9 @@ static void guc_fini_hw(void *arg) · 520 unmodified lines
660	661	{
661	662	struct xe_guc *guc = arg;
662	663	struct xe_gt *gt = guc_to_gt(guc);
663		− unsigned int fw_ref;
664	664
665		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
666		− xe_uc_sanitize_reset(&guc_to_gt(guc)->uc);
667		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
	665	+ xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL)
	666	+ xe_uc_sanitize_reset(&guc_to_gt(guc)->uc);
668	667
669	668	guc_g2g_fini(guc);
670	669	}
@@ -768,6 +767,10 @@ int xe_guc_init(struct xe_guc *guc) · 97 unmodified lines
768	767	if (!xe_uc_fw_is_enabled(&guc->fw))
769	768	return 0;
770	769
	770	+ /* Disable page reclaim if GuC FW does not support */
	771	+ if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 14, 0))
	772	+ xe->info.has_page_reclaim_hw_assist = false;
	773	+
771	774	if (IS_SRIOV_VF(xe)) {
772	775	ret = xe_guc_ct_init(&guc->ct);
773	776	if (ret)
@@ -1485,6 +1488,12 @@ timeout: · 711 unmodified lines
1485	1488	u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
1486	1489	u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
1487	1490
	1491	+ if (unlikely(error == XE_GUC_RESPONSE_VF_MIGRATED)) {
	1492	+ xe_gt_dbg(gt, "GuC mmio request %#x rejected due to MIGRATION (hint %#x)\n",
	1493	+ request[0], hint);
	1494	+ return -EREMCHG;
	1495	+ }
	1496	+
1488	1497	xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
1489	1498	request[0], error, hint);
1490	1499	return -ENXIO;
@@ -1618,18 +1627,51 @@ int xe_guc_start(struct xe_guc *guc) · 127 unmodified lines
1618	1627	return xe_guc_submit_start(guc);
1619	1628	}
1620	1629
	1630	+/**
	1631	+ * xe_guc_runtime_suspend() - GuC runtime suspend
	1632	+ * @guc: The GuC object
	1633	+ *
	1634	+ * Stop further runs of submission tasks on given GuC and runtime suspend
	1635	+ * GuC CT.
	1636	+ */
	1637	+void xe_guc_runtime_suspend(struct xe_guc *guc)
	1638	+{
	1639	+ xe_guc_submit_pause(guc);
	1640	+ xe_guc_submit_disable(guc);
	1641	+ xe_guc_ct_runtime_suspend(&guc->ct);
	1642	+}
	1643	+
	1644	+/**
	1645	+ * xe_guc_runtime_resume() - GuC runtime resume
	1646	+ * @guc: The GuC object
	1647	+ *
	1648	+ * Runtime resume GuC CT and allow further runs of submission tasks on
	1649	+ * given GuC.
	1650	+ */
	1651	+void xe_guc_runtime_resume(struct xe_guc *guc)
	1652	+{
	1653	+ /*
	1654	+ * Runtime PM flows are not applicable for VFs, so it's safe to
	1655	+ * directly enable IRQ.
	1656	+ */
	1657	+ guc_enable_irq(guc);
	1658	+
	1659	+ xe_guc_ct_runtime_resume(&guc->ct);
	1660	+ xe_guc_submit_enable(guc);
	1661	+ xe_guc_submit_unpause(guc);
	1662	+}
	1663	+
1621	1664	void xe_guc_print_info(struct xe_guc guc, struct drm_printer p)
1622	1665	{
1623	1666	struct xe_gt *gt = guc_to_gt(guc);
1624		− unsigned int fw_ref;
1625	1667	u32 status;
1626	1668	int i;
1627	1669
1628	1670	xe_uc_fw_print(&guc->fw, p);
1629	1671
1630	1672	if (!IS_SRIOV_VF(gt_to_xe(gt))) {
1631		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1632		− if (!fw_ref)
	1673	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	1674	+ if (!fw_ref.domains)
1633	1675	return;
1634	1676
1635	1677	status = xe_mmio_read32(&gt->mmio, GUC_STATUS);
@@ -1649,8 +1691,6 @@ void xe_guc_print_info(struct xe_guc guc, struct drm_printer p) · 13 unmodified lines
1649	1691	drm_printf(p, "\t%2d: \t0x%x\n",
1650	1692	i, xe_mmio_read32(&gt->mmio, SOFT_SCRATCH(i)));
1651	1693	}
1652		−
1653		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
1654	1694	}
1655	1695
1656	1696	drm_puts(p, "\n");

drivers/gpu/drm/xe/xe_guc.h+23−0

@@ -18,10 +18,16 @@ · 17 unmodified lines
18	18	*/
19	19	#define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) \| ((min) << 8) \| (pat))
20	20	#define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
	21	+#define MAKE_GUC_VER_ARGS(ver...) \
	22	+ (BUILD_BUG_ON_ZERO(COUNT_ARGS(ver) < 2 \|\| COUNT_ARGS(ver) > 3) + \
	23	+ MAKE_GUC_VER(PICK_ARG1(ver), PICK_ARG2(ver), IF_ARGS(PICK_ARG3(ver), 0, PICK_ARG3(ver))))
	24	+
21	25	#define GUC_SUBMIT_VER(guc) \
22	26	MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY])
23	27	#define GUC_FIRMWARE_VER(guc) \
24	28	MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_RELEASE])
	29	+#define GUC_FIRMWARE_VER_AT_LEAST(guc, ver...) \
	30	+ xe_guc_fw_version_at_least((guc), MAKE_GUC_VER_ARGS(ver))
25	31
26	32	struct drm_printer;
27	33
@@ -35,6 +41,8 @@ int xe_guc_upload(struct xe_guc *guc); · 7 unmodified lines
35	41	int xe_guc_min_load_for_hwconfig(struct xe_guc *guc);
36	42	int xe_guc_enable_communication(struct xe_guc *guc);
37	43	int xe_guc_opt_in_features_enable(struct xe_guc *guc);
	44	+void xe_guc_runtime_suspend(struct xe_guc *guc);
	45	+void xe_guc_runtime_resume(struct xe_guc *guc);
38	46	int xe_guc_suspend(struct xe_guc *guc);
39	47	void xe_guc_notify(struct xe_guc *guc);
40	48	int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);
@@ -94,4 +102,19 @@ static inline struct drm_device guc_to_drm(struct xe_guc guc) · 53 unmodified lines
94	102	return &guc_to_xe(guc)->drm;
95	103	}
96	104
	105	+/**
	106	+ * xe_guc_fw_version_at_least() - Check if GuC is at least of given version.
	107	+ * @guc: the &xe_guc
	108	+ * @ver: the version to check
	109	+ *
	110	+ * The @ver should be prepared using MAKE_GUC_VER(major, minor, patch).
	111	+ *
	112	+ * Return: true if loaded GuC firmware is at least of given version,
	113	+ * false otherwise.
	114	+ */
	115	+static inline bool xe_guc_fw_version_at_least(const struct xe_guc *guc, u32 ver)
	116	+{
	117	+ return GUC_FIRMWARE_VER(guc) >= ver;
	118	+}
	119	+
97	120	#endif

drivers/gpu/drm/xe/xe_guc_ads.c+3−3

@@ -317,7 +317,7 @@ static void guc_waklv_init(struct xe_guc_ads *ads) · 316 unmodified lines
317	317	offset = guc_ads_waklv_offset(ads);
318	318	remain = guc_ads_waklv_size(ads);
319	319
320		− if (XE_GT_WA(gt, 14019882105) \|\| XE_GT_WA(gt, 16021333562))
	320	+ if (XE_GT_WA(gt, 16021333562))
321	321	guc_waklv_enable(ads, NULL, 0, &offset, &remain,
322	322	GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED);
323	323	if (XE_GT_WA(gt, 18024947630))
@@ -347,10 +347,10 @@ static void guc_waklv_init(struct xe_guc_ads *ads) · 23 unmodified lines
347	347	guc_waklv_enable(ads, NULL, 0, &offset, &remain,
348	348	GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET);
349	349
350		− if (GUC_FIRMWARE_VER(&gt->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_GT_WA(gt, 16026508708))
	350	+ if (GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 44) && XE_GT_WA(gt, 16026508708))
351	351	guc_waklv_enable(ads, NULL, 0, &offset, &remain,
352	352	GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH);
353		− if (GUC_FIRMWARE_VER(&gt->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_GT_WA(gt, 16026007364)) {
	353	+ if (GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 47) && XE_GT_WA(gt, 16026007364)) {
354	354	u32 data[] = {
355	355	0x0,
356	356	0xF,

drivers/gpu/drm/xe/xe_guc_buf.c+1−1

@@ -30,7 +30,7 @@ static int guc_buf_cache_init(struct xe_guc_buf_cache *cache, u32 size) · 29 unmodified lines
30	30	struct xe_gt *gt = cache_to_gt(cache);
31	31	struct xe_sa_manager *sam;
32	32
33		− sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32));
	33	+ sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32), 0);
34	34	if (IS_ERR(sam))
35	35	return PTR_ERR(sam);
36	36	cache->sam = sam;

drivers/gpu/drm/xe/xe_guc_capture.c+8−8

@@ -843,7 +843,7 @@ static void check_guc_capture_size(struct xe_guc *guc) · 842 unmodified lines
843	843	{
844	844	int capture_size = guc_capture_output_size_est(guc);
845	845	int spare_size = capture_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER;
846		− u32 buffer_size = xe_guc_log_section_size_capture(&guc->log);
	846	+ u32 buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE;
847	847
848	848	/*
849	849	* NOTE: capture_size is much smaller than the capture region
@@ -949,7 +949,7 @@ guc_capture_init_node(struct xe_guc guc, struct __guc_capture_parsed_output no · 99 unmodified lines
949	949	* ADS module also calls separately for PF vs VF.
950	950	*
951	951	* --> alloc B: GuC output capture buf (registered via guc_init_params(log_param))
952		− * Size = #define CAPTURE_BUFFER_SIZE (warns if on too-small)
	952	+ * Size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE (warns if on too-small)
953	953	* Note2: 'x 3' to hold multiple capture groups
954	954	*
955	955	* GUC Runtime notify capture:
@@ -1367,7 +1367,7 @@ static int __guc_capture_flushlog_complete(struct xe_guc *guc) · 411 unmodified lines
1367	1367	{
1368	1368	u32 action[] = {
1369	1369	XE_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE,
1370		− GUC_LOG_BUFFER_CAPTURE
	1370	+ GUC_LOG_TYPE_STATE_CAPTURE
1371	1371	};
1372	1372
1373	1373	return xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));
@@ -1384,8 +1384,8 @@ static void __guc_capture_process_output(struct xe_guc *guc) · 10 unmodified lines
1384	1384	u32 log_buf_state_offset;
1385	1385	u32 src_data_offset;
1386	1386
1387		− log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_BUFFER_CAPTURE;
1388		− src_data_offset = xe_guc_get_log_buffer_offset(&guc->log, GUC_LOG_BUFFER_CAPTURE);
	1387	+ log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_TYPE_STATE_CAPTURE;
	1388	+ src_data_offset = XE_GUC_LOG_STATE_CAPTURE_OFFSET;
1389	1389
1390	1390	/*
1391	1391	* Make a copy of the state structure, inside GuC log buffer
@@ -1395,15 +1395,15 @@ static void __guc_capture_process_output(struct xe_guc *guc) · 3 unmodified lines
1395	1395	xe_map_memcpy_from(guc_to_xe(guc), &log_buf_state_local, &guc->log.bo->vmap,
1396	1396	log_buf_state_offset, sizeof(struct guc_log_buffer_state));
1397	1397
1398		− buffer_size = xe_guc_get_log_buffer_size(&guc->log, GUC_LOG_BUFFER_CAPTURE);
	1398	+ buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE;
1399	1399	read_offset = log_buf_state_local.read_ptr;
1400	1400	write_offset = log_buf_state_local.sampled_write_ptr;
1401	1401	full_count = FIELD_GET(GUC_LOG_BUFFER_STATE_BUFFER_FULL_CNT, log_buf_state_local.flags);
1402	1402
1403	1403	/* Bookkeeping stuff */
1404	1404	tmp = FIELD_GET(GUC_LOG_BUFFER_STATE_FLUSH_TO_FILE, log_buf_state_local.flags);
1405		− guc->log.stats[GUC_LOG_BUFFER_CAPTURE].flush += tmp;
1406		− new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_BUFFER_CAPTURE,
	1405	+ guc->log.stats[GUC_LOG_TYPE_STATE_CAPTURE].flush += tmp;
	1406	+ new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_TYPE_STATE_CAPTURE,
1407	1407	full_count);
1408	1408
1409	1409	/* Now copy the actual logs. */

drivers/gpu/drm/xe/xe_guc_ct.c+170−103

@@ -42,6 +42,21 @@ static void ct_exit_safe_mode(struct xe_guc_ct *ct); · 41 unmodified lines
42	42	static void guc_ct_change_state(struct xe_guc_ct *ct,
43	43	enum xe_guc_ct_state state);
44	44
	45	+static struct xe_guc ct_to_guc(struct xe_guc_ct ct)
	46	+{
	47	+ return container_of(ct, struct xe_guc, ct);
	48	+}
	49	+
	50	+static struct xe_gt ct_to_gt(struct xe_guc_ct ct)
	51	+{
	52	+ return container_of(ct, struct xe_gt, uc.guc.ct);
	53	+}
	54	+
	55	+static struct xe_device ct_to_xe(struct xe_guc_ct ct)
	56	+{
	57	+ return gt_to_xe(ct_to_gt(ct));
	58	+}
	59	+
45	60	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
46	61	enum {
47	62	/* Internal states, not error conditions */
@@ -68,14 +83,101 @@ enum { · 20 unmodified lines
68	83	static void ct_dead_worker_func(struct work_struct *w);
69	84	static void ct_dead_capture(struct xe_guc_ct ct, struct guc_ctb ctb, u32 reason_code);
70	85
71		−#define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
	86	+static void ct_dead_fini(struct xe_guc_ct *ct)
	87	+{
	88	+ cancel_work_sync(&ct->dead.worker);
	89	+}
	90	+
	91	+static void ct_dead_init(struct xe_guc_ct *ct)
	92	+{
	93	+ spin_lock_init(&ct->dead.lock);
	94	+ INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
	95	+
	96	+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
	97	+ stack_depot_init();
	98	+#endif
	99	+}
	100	+
	101	+static void fast_req_stack_save(struct xe_guc_ct *ct, unsigned int slot)
	102	+{
	103	+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
	104	+ unsigned long entries[SZ_32];
	105	+ unsigned int n;
	106	+
	107	+ n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
	108	+ /* May be called under spinlock, so avoid sleeping */
	109	+ ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT);
	110	+#endif
	111	+}
	112	+
	113	+static void fast_req_dump(struct xe_guc_ct *ct, u16 fence, unsigned int slot)
	114	+{
	115	+ struct xe_gt *gt = ct_to_gt(ct);
	116	+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
	117	+ char *buf __cleanup(kfree) = kmalloc(SZ_4K, GFP_NOWAIT);
	118	+
	119	+ if (buf && stack_depot_snprint(ct->fast_req[slot].stack, buf, SZ_4K, 0))
	120	+ xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s\n",
	121	+ fence, ct->fast_req[slot].action, buf);
	122	+ else
	123	+ xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n",
	124	+ fence, ct->fast_req[slot].action);
	125	+#else
	126	+ xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n",
	127	+ fence, ct->fast_req[slot].action);
	128	+#endif
	129	+}
	130	+
	131	+static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
	132	+{
	133	+ u16 fence_min = U16_MAX, fence_max = 0;
	134	+ struct xe_gt *gt = ct_to_gt(ct);
	135	+ unsigned int n;
	136	+
	137	+ lockdep_assert_held(&ct->lock);
	138	+
	139	+ for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) {
	140	+ if (ct->fast_req[n].fence < fence_min)
	141	+ fence_min = ct->fast_req[n].fence;
	142	+ if (ct->fast_req[n].fence > fence_max)
	143	+ fence_max = ct->fast_req[n].fence;
	144	+
	145	+ if (ct->fast_req[n].fence != fence)
	146	+ continue;
	147	+
	148	+ return fast_req_dump(ct, fence, n);
	149	+ }
	150	+
	151	+ xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n",
	152	+ fence, fence_min, fence_max, ct->fence_seqno);
	153	+}
	154	+
	155	+static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
	156	+{
	157	+ unsigned int slot = fence % ARRAY_SIZE(ct->fast_req);
	158	+
	159	+ fast_req_stack_save(ct, slot);
	160	+ ct->fast_req[slot].fence = fence;
	161	+ ct->fast_req[slot].action = action;
	162	+}
	163	+
	164	+#define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
	165	+
72	166	#else
	167	+
	168	+static void ct_dead_fini(struct xe_guc_ct *ct) { }
	169	+static void ct_dead_init(struct xe_guc_ct *ct) { }
	170	+
	171	+static void fast_req_report(struct xe_guc_ct *ct, u16 fence) { }
	172	+static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { }
	173	+
73	174	#define CT_DEAD(ct, ctb, reason) \
74	175	do { \
75	176	struct guc_ctb *_ctb = (ctb); \
76	177	if (_ctb) \
77	178	_ctb->info.broken = true; \
78	179	} while (0)
	180	+
79	181	#endif
80	182
81	183	/* Used when a CT send wants to block and / or receive data */
@@ -112,24 +214,6 @@ static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) · 30 unmodified lines
112	214	return g2h_fence->seqno == ~0x0;
113	215	}
114	216
115		−static struct xe_guc *
116		−ct_to_guc(struct xe_guc_ct *ct)
117		−{
118		− return container_of(ct, struct xe_guc, ct);
119		−}
120		−
121		−static struct xe_gt *
122		−ct_to_gt(struct xe_guc_ct *ct)
123		−{
124		− return container_of(ct, struct xe_gt, uc.guc.ct);
125		−}
126		−
127		−static struct xe_device *
128		−ct_to_xe(struct xe_guc_ct *ct)
129		−{
130		− return gt_to_xe(ct_to_gt(ct));
131		−}
132		−
133	217	/**
134	218	* DOC: GuC CTB Blob
135	219	*
@@ -169,8 +253,11 @@ ct_to_xe(struct xe_guc_ct *ct) · 33 unmodified lines
169	253	#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
170	254	#define CTB_H2G_BUFFER_OFFSET (CTB_DESC_SIZE * 2)
171	255	#define CTB_H2G_BUFFER_SIZE (SZ_4K)
	256	+#define CTB_H2G_BUFFER_DWORDS (CTB_H2G_BUFFER_SIZE / sizeof(u32))
172	257	#define CTB_G2H_BUFFER_SIZE (SZ_128K)
	258	+#define CTB_G2H_BUFFER_DWORDS (CTB_G2H_BUFFER_SIZE / sizeof(u32))
173	259	#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2)
	260	+#define G2H_ROOM_BUFFER_DWORDS (CTB_G2H_BUFFER_DWORDS / 2)
174	261
175	262	/**
176	263	* xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full
@@ -199,9 +286,7 @@ static void guc_ct_fini(struct drm_device drm, void arg) · 22 unmodified lines
199	286	{
200	287	struct xe_guc_ct *ct = arg;
201	288
202		−#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
203		− cancel_work_sync(&ct->dead.worker);
204		−#endif
	289	+ ct_dead_fini(ct);
205	290	ct_exit_safe_mode(ct);
206	291	destroy_workqueue(ct->g2h_wq);
207	292	xa_destroy(&ct->fence_lookup);
@@ -239,13 +324,8 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) · 31 unmodified lines
239	324	xa_init(&ct->fence_lookup);
240	325	INIT_WORK(&ct->g2h_worker, g2h_worker_func);
241	326	INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);
242		−#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
243		− spin_lock_init(&ct->dead.lock);
244		− INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
245		−#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
246		− stack_depot_init();
247		−#endif
248		−#endif
	327	+
	328	+ ct_dead_init(ct);
249	329	init_waitqueue_head(&ct->wq);
250	330	init_waitqueue_head(&ct->g2h_fence_wq);
251	331
@@ -326,7 +406,7 @@ int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct) · 74 unmodified lines
326	406	static void guc_ct_ctb_h2g_init(struct xe_device xe, struct guc_ctb h2g,
327	407	struct iosys_map *map)
328	408	{
329		− h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);
	409	+ h2g->info.size = CTB_H2G_BUFFER_DWORDS;
330	410	h2g->info.resv_space = 0;
331	411	h2g->info.tail = 0;
332	412	h2g->info.head = 0;
@@ -344,8 +424,8 @@ static void guc_ct_ctb_h2g_init(struct xe_device xe, struct guc_ctb h2g, · 11 unmodified lines
344	424	static void guc_ct_ctb_g2h_init(struct xe_device xe, struct guc_ctb g2h,
345	425	struct iosys_map *map)
346	426	{
347		− g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);
348		− g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);
	427	+ g2h->info.size = CTB_G2H_BUFFER_DWORDS;
	428	+ g2h->info.resv_space = G2H_ROOM_BUFFER_DWORDS;
349	429	g2h->info.head = 0;
350	430	g2h->info.tail = 0;
351	431	g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
@@ -640,6 +720,39 @@ void xe_guc_ct_stop(struct xe_guc_ct *ct) · 288 unmodified lines
640	720	stop_g2h_handler(ct);
641	721	}
642	722
	723	+/**
	724	+ * xe_guc_ct_runtime_suspend() - GuC CT runtime suspend
	725	+ * @ct: the &xe_guc_ct
	726	+ *
	727	+ * Set GuC CT to disabled state.
	728	+ */
	729	+void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct)
	730	+{
	731	+ struct guc_ctb *g2h = &ct->ctbs.g2h;
	732	+ u32 credits = CIRC_SPACE(0, 0, CTB_G2H_BUFFER_DWORDS) - G2H_ROOM_BUFFER_DWORDS;
	733	+
	734	+ /* We should be back to guc_ct_ctb_g2h_init() values */
	735	+ xe_gt_assert(ct_to_gt(ct), g2h->info.space == credits);
	736	+
	737	+ /*
	738	+ * Since we're already in runtime suspend path, we shouldn't have pending
	739	+ * messages. But if there happen to be any, we'd probably want them to be
	740	+ * thrown as errors for further investigation.
	741	+ */
	742	+ xe_guc_ct_disable(ct);
	743	+}
	744	+
	745	+/**
	746	+ * xe_guc_ct_runtime_resume() - GuC CT runtime resume
	747	+ * @ct: the &xe_guc_ct
	748	+ *
	749	+ * Restart GuC CT and set it to enabled state.
	750	+ */
	751	+void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct)
	752	+{
	753	+ xe_guc_ct_restart(ct);
	754	+}
	755	+
643	756	static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
644	757	{
645	758	struct guc_ctb *h2g = &ct->ctbs.h2g;
@@ -747,28 +860,6 @@ static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) · 101 unmodified lines
747	860	spin_unlock_irq(&ct->fast_lock);
748	861	}
749	862
750		−#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
751		−static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
752		−{
753		− unsigned int slot = fence % ARRAY_SIZE(ct->fast_req);
754		−#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
755		− unsigned long entries[SZ_32];
756		− unsigned int n;
757		−
758		− n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
759		−
760		− /* May be called under spinlock, so avoid sleeping */
761		− ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT);
762		−#endif
763		− ct->fast_req[slot].fence = fence;
764		− ct->fast_req[slot].action = action;
765		−}
766		−#else
767		−static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
768		−{
769		−}
770		−#endif
771		−
772	863	/*
773	864	* The CT protocol accepts a 16 bits fence. This field is fully owned by the
774	865	* driver, the GuC will just copy it to the reply message. Since we need to
@@ -1310,10 +1401,12 @@ static int parse_g2h_event(struct xe_guc_ct ct, u32 msg, u32 len) · 535 unmodified lines
1310	1401	lockdep_assert_held(&ct->lock);
1311	1402
1312	1403	switch (action) {
	1404	+ case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE:
1313	1405	case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1314	1406	case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1315	1407	case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1316	1408	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
	1409	+ case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
1317	1410	g2h_release_space(ct, len);
1318	1411	}
1319	1412
@@ -1338,55 +1431,6 @@ static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) · 18 unmodified lines
1338	1431	return 0;
1339	1432	}
1340	1433
1341		−#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
1342		−static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
1343		−{
1344		− u16 fence_min = U16_MAX, fence_max = 0;
1345		− struct xe_gt *gt = ct_to_gt(ct);
1346		− bool found = false;
1347		− unsigned int n;
1348		−#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
1349		− char *buf;
1350		−#endif
1351		−
1352		− lockdep_assert_held(&ct->lock);
1353		−
1354		− for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) {
1355		− if (ct->fast_req[n].fence < fence_min)
1356		− fence_min = ct->fast_req[n].fence;
1357		− if (ct->fast_req[n].fence > fence_max)
1358		− fence_max = ct->fast_req[n].fence;
1359		−
1360		− if (ct->fast_req[n].fence != fence)
1361		− continue;
1362		− found = true;
1363		−
1364		−#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
1365		− buf = kmalloc(SZ_4K, GFP_NOWAIT);
1366		− if (buf && stack_depot_snprint(ct->fast_req[n].stack, buf, SZ_4K, 0))
1367		− xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s",
1368		− fence, ct->fast_req[n].action, buf);
1369		− else
1370		− xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n",
1371		− fence, ct->fast_req[n].action);
1372		− kfree(buf);
1373		−#else
1374		− xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n",
1375		− fence, ct->fast_req[n].action);
1376		−#endif
1377		− break;
1378		− }
1379		−
1380		− if (!found)
1381		− xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n",
1382		− fence, fence_min, fence_max, ct->fence_seqno);
1383		−}
1384		−#else
1385		−static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
1386		−{
1387		−}
1388		−#endif
1389		−
1390	1434	static int parse_g2h_response(struct xe_guc_ct ct, u32 msg, u32 len)
1391	1435	{
1392	1436	struct xe_gt *gt = ct_to_gt(ct);
@@ -1549,6 +1593,15 @@ static int process_g2h_msg(struct xe_guc_ct ct, u32 msg, u32 len) · 156 unmodified lines
1549	1593	ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1550	1594	break;
1551	1595	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
	1596	+ case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
	1597	+ /*
	1598	+ * Page reclamation is an extension of TLB invalidation. Both
	1599	+ * operations share the same seqno and fence. When either
	1600	+ * action completes, we need to signal the corresponding
	1601	+ * fence. Since the handling logic (lookup fence by seqno,
	1602	+ * fence signalling) is identical, we use the same handler
	1603	+ * for both G2H events.
	1604	+ */
1552	1605	ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
1553	1606	break;
1554	1607	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
@@ -1572,6 +1625,13 @@ static int process_g2h_msg(struct xe_guc_ct ct, u32 msg, u32 len) · 17 unmodified lines
1572	1625	ret = xe_guc_g2g_test_notification(guc, payload, adj_len);
1573	1626	break;
1574	1627	#endif
	1628	+ case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE:
	1629	+ ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len);
	1630	+ break;
	1631	+ case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR:
	1632	+ ret = xe_guc_exec_queue_cgp_context_error_handler(guc, payload,
	1633	+ adj_len);
	1634	+ break;
1575	1635	default:
1576	1636	xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
1577	1637	}
@@ -1714,6 +1774,7 @@ static int g2h_read(struct xe_guc_ct ct, u32 msg, bool fast_path) · 136 unmodified lines
1714	1774	switch (action) {
1715	1775	case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1716	1776	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
	1777	+ case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
1717	1778	break; /* Process these in fast-path */
1718	1779	default:
1719	1780	return 0;
@@ -1750,6 +1811,12 @@ static void g2h_fast_path(struct xe_guc_ct ct, u32 msg, u32 len) · 30 unmodified lines
1750	1811	ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1751	1812	break;
1752	1813	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
	1814	+ case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
	1815	+ /*
	1816	+ * Seqno and fence handling of page reclamation and TLB
	1817	+ * invalidation is identical, so we can use the same handler
	1818	+ * for both actions.
	1819	+ */
1753	1820	__g2h_release_space(ct, len);
1754	1821	ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
1755	1822	break;

drivers/gpu/drm/xe/xe_guc_ct.h+2−0

@@ -17,6 +17,8 @@ int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct); · 16 unmodified lines
17	17	int xe_guc_ct_enable(struct xe_guc_ct *ct);
18	18	int xe_guc_ct_restart(struct xe_guc_ct *ct);
19	19	void xe_guc_ct_disable(struct xe_guc_ct *ct);
	20	+void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct);
	21	+void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct);
20	22	void xe_guc_ct_stop(struct xe_guc_ct *ct);
21	23	void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct);
22	24	void xe_guc_ct_fast_path(struct xe_guc_ct *ct);

drivers/gpu/drm/xe/xe_guc_debugfs.c+9−6

@@ -70,13 +70,9 @@ static int guc_debugfs_show(struct seq_file m, void data) · 69 unmodified lines
70	70	struct xe_gt *gt = grandparent->d_inode->i_private;
71	71	struct xe_device *xe = gt_to_xe(gt);
72	72	int (print)(struct xe_guc , struct drm_printer *) = node->info_ent->data;
73		− int ret;
74	73
75		− xe_pm_runtime_get(xe);
76		− ret = print(&gt->uc.guc, &p);
77		− xe_pm_runtime_put(xe);
78		−
79		− return ret;
	74	+ guard(xe_pm_runtime)(xe);
	75	+ return print(&gt->uc.guc, &p);
80	76	}
81	77
82	78	static int guc_log(struct xe_guc guc, struct drm_printer p)
@@ -85,6 +81,12 @@ static int guc_log(struct xe_guc guc, struct drm_printer p) · 2 unmodified lines
85	81	return 0;
86	82	}
87	83
	84	+static int guc_log_lfd(struct xe_guc guc, struct drm_printer p)
	85	+{
	86	+ xe_guc_log_print_lfd(&guc->log, p);
	87	+ return 0;
	88	+}
	89	+
88	90	static int guc_log_dmesg(struct xe_guc guc, struct drm_printer p)
89	91	{
90	92	xe_guc_log_print_dmesg(&guc->log);
@@ -121,6 +123,7 @@ static const struct drm_info_list slpc_debugfs_list[] = { · 30 unmodified lines
121	123	/* everything else should be added here */
122	124	static const struct drm_info_list pf_only_debugfs_list[] = {
123	125	{ "guc_log", .show = guc_debugfs_show, .data = guc_log },
	126	+ { "guc_log_lfd", .show = guc_debugfs_show, .data = guc_log_lfd },
124	127	{ "guc_log_dmesg", .show = guc_debugfs_show, .data = guc_log_dmesg },
125	128	};
126	129

drivers/gpu/drm/xe/xe_guc_fwif.h+7−3

@@ -16,6 +16,8 @@ · 15 unmodified lines
16	16	#define G2H_LEN_DW_DEREGISTER_CONTEXT 3
17	17	#define G2H_LEN_DW_TLB_INVALIDATE 3
18	18	#define G2H_LEN_DW_G2G_NOTIFY_MIN 3
	19	+#define G2H_LEN_DW_MULTI_QUEUE_CONTEXT 3
	20	+#define G2H_LEN_DW_PAGE_RECLAMATION 3
19	21
20	22	#define GUC_ID_MAX 65535
21	23	#define GUC_ID_UNKNOWN 0xffffffff
@@ -62,6 +64,8 @@ struct guc_ctxt_registration_info { · 40 unmodified lines
62	64	u32 wq_base_lo;
63	65	u32 wq_base_hi;
64	66	u32 wq_size;
	67	+ u32 cgp_lo;
	68	+ u32 cgp_hi;
65	69	u32 hwlrca_lo;
66	70	u32 hwlrca_hi;
67	71	};
@@ -91,9 +95,9 @@ struct guc_update_exec_queue_policy { · 23 unmodified lines
91	95	#define GUC_LOG_NOTIFY_ON_HALF_FULL BIT(1)
92	96	#define GUC_LOG_CAPTURE_ALLOC_UNITS BIT(2)
93	97	#define GUC_LOG_LOG_ALLOC_UNITS BIT(3)
94		−#define GUC_LOG_CRASH REG_GENMASK(5, 4)
95		−#define GUC_LOG_DEBUG REG_GENMASK(9, 6)
96		−#define GUC_LOG_CAPTURE REG_GENMASK(11, 10)
	98	+#define GUC_LOG_CRASH_DUMP REG_GENMASK(5, 4)
	99	+#define GUC_LOG_EVENT_DATA REG_GENMASK(9, 6)
	100	+#define GUC_LOG_STATE_CAPTURE REG_GENMASK(11, 10)
97	101	#define GUC_LOG_BUF_ADDR REG_GENMASK(31, 12)
98	102
99	103	#define GUC_CTL_WA 1

drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h+6−0

@@ -24,6 +24,11 @@ · 23 unmodified lines
24	24	* ABI and the associated &NAME, that may be used in code or debugfs/sysfs::
25	25	*
26	26	* define(TAG, NAME)
	27	+ *
	28	+ * If required, KLVs can be labeled with GuC firmware version that added them::
	29	+ *
	30	+ * define(TAG, NAME, MAJOR, MINOR)
	31	+ * define(TAG, NAME, MAJOR, MINOR, PATCH)
27	32	*/
28	33	#define MAKE_XE_GUC_KLV_THRESHOLDS_SET(define) \
29	34	define(CAT_ERR, cat_error_count) \
@@ -32,6 +37,7 @@ · 2 unmodified lines
32	37	define(H2G_STORM, guc_time_us) \
33	38	define(IRQ_STORM, irq_time_us) \
34	39	define(DOORBELL_STORM, doorbell_time_us) \
	40	+ define(MULTI_LRC_COUNT, multi_lrc_count, 70, 53)\
35	41	/* end */
36	42
37	43	/**

drivers/gpu/drm/xe/xe_guc_log.c+406−101

@@ -7,8 +7,10 @@ · 6 unmodified lines
7	7
8	8	#include <linux/fault-inject.h>
9	9
	10	+#include <linux/utsname.h>
10	11	#include <drm/drm_managed.h>
11	12
	13	+#include "abi/guc_lfd_abi.h"
12	14	#include "regs/xe_guc_regs.h"
13	15	#include "xe_bo.h"
14	16	#include "xe_devcoredump.h"
@@ -19,6 +21,77 @@ · 4 unmodified lines
19	21	#include "xe_mmio.h"
20	22	#include "xe_module.h"
21	23
	24	+#define GUC_LOG_CHUNK_SIZE SZ_2M
	25	+
	26	+/* Magic keys define */
	27	+#define GUC_LFD_DRIVER_KEY_STREAMING 0x8086AAAA474C5346
	28	+#define GUC_LFD_LOG_BUFFER_MARKER_2 0xDEADFEED
	29	+#define GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2 0x8086DEAD
	30	+#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2 0xBEEFFEED
	31	+#define GUC_LFD_LOG_BUFFER_MARKER_1V2 0xCABBA9E6
	32	+#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2 0xCABBA9F7
	33	+#define GUC_LFD_DATA_HEADER_MAGIC 0x8086
	34	+
	35	+/* LFD supported LIC type range */
	36	+#define GUC_LIC_TYPE_FIRST GUC_LIC_TYPE_GUC_SW_VERSION
	37	+#define GUC_LIC_TYPE_LAST GUC_LIC_TYPE_BUILD_PLATFORM_ID
	38	+#define GUC_LFD_TYPE_FW_RANGE_FIRST GUC_LFD_TYPE_FW_VERSION
	39	+#define GUC_LFD_TYPE_FW_RANGE_LAST GUC_LFD_TYPE_BUILD_PLATFORM_ID
	40	+
	41	+#define GUC_LOG_BUFFER_STATE_HEADER_LENGTH 4096
	42	+#define GUC_LOG_BUFFER_INIT_CONFIG 3
	43	+
	44	+struct guc_log_buffer_entry_list {
	45	+ u32 offset;
	46	+ u32 rd_ptr;
	47	+ u32 wr_ptr;
	48	+ u32 wrap_offset;
	49	+ u32 buf_size;
	50	+};
	51	+
	52	+struct guc_lic_save {
	53	+ u32 version;
	54	+ /*
	55	+ * Array of init config KLV values.
	56	+ * Range from GUC_LOG_LIC_TYPE_FIRST to GUC_LOG_LIC_TYPE_LAST
	57	+ */
	58	+ u32 values[GUC_LIC_TYPE_LAST - GUC_LIC_TYPE_FIRST + 1];
	59	+ struct guc_log_buffer_entry_list entry[GUC_LOG_BUFFER_INIT_CONFIG];
	60	+};
	61	+
	62	+static struct guc_log_buffer_entry_markers {
	63	+ u32 key[2];
	64	+} const entry_markers[GUC_LOG_BUFFER_INIT_CONFIG + 1] = {
	65	+ {{
	66	+ GUC_LFD_LOG_BUFFER_MARKER_1V2,
	67	+ GUC_LFD_LOG_BUFFER_MARKER_2
	68	+ }},
	69	+ {{
	70	+ GUC_LFD_LOG_BUFFER_MARKER_1V2,
	71	+ GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2
	72	+ }},
	73	+ {{
	74	+ GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2,
	75	+ GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2
	76	+ }},
	77	+ {{
	78	+ GUC_LIC_MAGIC,
	79	+ (FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MAJOR, GUC_LIC_VERSION_MAJOR) \|
	80	+ FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MINOR, GUC_LIC_VERSION_MINOR))
	81	+ }}
	82	+};
	83	+
	84	+static struct guc_log_lic_lfd_map {
	85	+ u32 lic;
	86	+ u32 lfd;
	87	+} const lic_lfd_type_map[] = {
	88	+ {GUC_LIC_TYPE_GUC_SW_VERSION, GUC_LFD_TYPE_FW_VERSION},
	89	+ {GUC_LIC_TYPE_GUC_DEVICE_ID, GUC_LFD_TYPE_GUC_DEVICE_ID},
	90	+ {GUC_LIC_TYPE_TSC_FREQUENCY, GUC_LFD_TYPE_TSC_FREQUENCY},
	91	+ {GUC_LIC_TYPE_GMD_ID, GUC_LFD_TYPE_GMD_ID},
	92	+ {GUC_LIC_TYPE_BUILD_PLATFORM_ID, GUC_LFD_TYPE_BUILD_PLATFORM_ID}
	93	+};
	94	+
22	95	static struct xe_guc *
23	96	log_to_guc(struct xe_guc_log *log)
24	97	{
@@ -37,33 +110,6 @@ log_to_xe(struct xe_guc_log *log) · 12 unmodified lines
37	110	return gt_to_xe(log_to_gt(log));
38	111	}
39	112
40		−static size_t guc_log_size(void)
41		−{
42		− /*
43		− * GuC Log buffer Layout
44		− *
45		− * +===============================+ 00B
46		− * \| Crash dump state header \|
47		− * +-------------------------------+ 32B
48		− * \| Debug state header \|
49		− * +-------------------------------+ 64B
50		− * \| Capture state header \|
51		− * +-------------------------------+ 96B
52		− * \| \|
53		− * +===============================+ PAGE_SIZE (4KB)
54		− * \| Crash Dump logs \|
55		− * +===============================+ + CRASH_SIZE
56		− * \| Debug logs \|
57		− * +===============================+ + DEBUG_SIZE
58		− * \| Capture logs \|
59		− * +===============================+ + CAPTURE_SIZE
60		− */
61		− return PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE +
62		− CAPTURE_BUFFER_SIZE;
63		−}
64		−
65		−#define GUC_LOG_CHUNK_SIZE SZ_2M
66		−
67	113	static struct xe_guc_log_snapshot xe_guc_log_snapshot_alloc(struct xe_guc_log log, bool atomic)
68	114	{
69	115	struct xe_guc_log_snapshot *snapshot;
@@ -145,7 +191,6 @@ struct xe_guc_log_snapshot xe_guc_log_snapshot_capture(struct xe_guc_log log, · 75 unmodified lines
145	191	struct xe_device *xe = log_to_xe(log);
146	192	struct xe_guc *guc = log_to_guc(log);
147	193	struct xe_gt *gt = log_to_gt(log);
148		− unsigned int fw_ref;
149	194	size_t remain;
150	195	int i;
151	196
@@ -165,13 +210,12 @@ struct xe_guc_log_snapshot xe_guc_log_snapshot_capture(struct xe_guc_log log, · 13 unmodified lines
165	210	remain -= size;
166	211	}
167	212
168		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
169		− if (!fw_ref) {
	213	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	214	+ if (!fw_ref.domains)
170	215	snapshot->stamp = ~0ULL;
171		− } else {
	216	+ else
172	217	snapshot->stamp = xe_mmio_read64_2x32(&gt->mmio, GUC_PMTIMESTAMP_LO);
173		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
174		− }
	218	+
175	219	snapshot->ktime = ktime_get_boottime_ns();
176	220	snapshot->level = log->level;
177	221	snapshot->ver_found = guc->fw.versions.found[XE_UC_FW_VER_RELEASE];
@@ -216,6 +260,318 @@ void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_ · 38 unmodified lines
216	260	}
217	261	}
218	262
	263	+static inline void lfd_output_binary(struct drm_printer p, char buf, int buf_size)
	264	+{
	265	+ seq_write(p->arg, buf, buf_size);
	266	+}
	267	+
	268	+static inline int xe_guc_log_add_lfd_header(struct guc_lfd_data *lfd)
	269	+{
	270	+ lfd->header = FIELD_PREP_CONST(GUC_LFD_DATA_HEADER_MASK_MAGIC, GUC_LFD_DATA_HEADER_MAGIC);
	271	+ return offsetof(struct guc_lfd_data, data);
	272	+}
	273	+
	274	+static int xe_guc_log_add_typed_payload(struct drm_printer *p, u32 type,
	275	+ u32 data_len, void *data)
	276	+{
	277	+ struct guc_lfd_data lfd;
	278	+ int len;
	279	+
	280	+ len = xe_guc_log_add_lfd_header(&lfd);
	281	+ lfd.header \|= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, type);
	282	+ /* make length DW aligned */
	283	+ lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32));
	284	+ lfd_output_binary(p, (char *)&lfd, len);
	285	+
	286	+ lfd_output_binary(p, data, data_len);
	287	+ len += lfd.data_count * sizeof(u32);
	288	+
	289	+ return len;
	290	+}
	291	+
	292	+static inline int lic_type_to_index(u32 lic_type)
	293	+{
	294	+ XE_WARN_ON(lic_type < GUC_LIC_TYPE_FIRST \|\| lic_type > GUC_LIC_TYPE_LAST);
	295	+
	296	+ return lic_type - GUC_LIC_TYPE_FIRST;
	297	+}
	298	+
	299	+static inline int lfd_type_to_index(u32 lfd_type)
	300	+{
	301	+ int i, lic_type = 0;
	302	+
	303	+ XE_WARN_ON(lfd_type < GUC_LFD_TYPE_FW_RANGE_FIRST \|\| lfd_type > GUC_LFD_TYPE_FW_RANGE_LAST);
	304	+
	305	+ for (i = 0; i < ARRAY_SIZE(lic_lfd_type_map); i++)
	306	+ if (lic_lfd_type_map[i].lfd == lfd_type)
	307	+ lic_type = lic_lfd_type_map[i].lic;
	308	+
	309	+ /* If not found, lic_type_to_index will warning invalid type */
	310	+ return lic_type_to_index(lic_type);
	311	+}
	312	+
	313	+static int xe_guc_log_add_klv(struct drm_printer *p, u32 lfd_type,
	314	+ struct guc_lic_save *config)
	315	+{
	316	+ int klv_index = lfd_type_to_index(lfd_type);
	317	+
	318	+ return xe_guc_log_add_typed_payload(p, lfd_type, sizeof(u32), &config->values[klv_index]);
	319	+}
	320	+
	321	+static int xe_guc_log_add_os_id(struct drm_printer *p, u32 id)
	322	+{
	323	+ struct guc_lfd_data_os_info os_id;
	324	+ struct guc_lfd_data lfd;
	325	+ int len, info_len, section_len;
	326	+ char *version;
	327	+ u32 blank = 0;
	328	+
	329	+ len = xe_guc_log_add_lfd_header(&lfd);
	330	+ lfd.header \|= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_OS_ID);
	331	+
	332	+ os_id.os_id = id;
	333	+ section_len = offsetof(struct guc_lfd_data_os_info, build_version);
	334	+
	335	+ version = init_utsname()->release;
	336	+ info_len = strlen(version);
	337	+
	338	+ /* make length DW aligned */
	339	+ lfd.data_count = DIV_ROUND_UP(section_len + info_len, sizeof(u32));
	340	+ lfd_output_binary(p, (char *)&lfd, len);
	341	+ lfd_output_binary(p, (char *)&os_id, section_len);
	342	+ lfd_output_binary(p, version, info_len);
	343	+
	344	+ /* Padding with 0 */
	345	+ section_len = lfd.data_count * sizeof(u32) - section_len - info_len;
	346	+ if (section_len)
	347	+ lfd_output_binary(p, (char *)&blank, section_len);
	348	+
	349	+ len += lfd.data_count * sizeof(u32);
	350	+ return len;
	351	+}
	352	+
	353	+static void xe_guc_log_loop_log_init(struct guc_lic init, struct guc_lic_save config)
	354	+{
	355	+ struct guc_klv_generic_dw_t p = (void )init->data;
	356	+ int i;
	357	+
	358	+ for (i = 0; i < init->data_count;) {
	359	+ int klv_len = FIELD_GET(GUC_KLV_0_LEN, p->kl) + 1;
	360	+ int key = FIELD_GET(GUC_KLV_0_KEY, p->kl);
	361	+
	362	+ if (key < GUC_LIC_TYPE_FIRST \|\| key > GUC_LIC_TYPE_LAST) {
	363	+ XE_WARN_ON(key < GUC_LIC_TYPE_FIRST \|\| key > GUC_LIC_TYPE_LAST);
	364	+ break;
	365	+ }
	366	+ config->values[lic_type_to_index(key)] = p->value;
	367	+ i += klv_len + 1; /* Whole KLV structure length in dwords */
	368	+ p = (void )((u32 )p + klv_len);
	369	+ }
	370	+}
	371	+
	372	+static int find_marker(u32 mark0, u32 mark1)
	373	+{
	374	+ int i;
	375	+
	376	+ for (i = 0; i < ARRAY_SIZE(entry_markers); i++)
	377	+ if (mark0 == entry_markers[i].key[0] && mark1 == entry_markers[i].key[1])
	378	+ return i;
	379	+
	380	+ return ARRAY_SIZE(entry_markers);
	381	+}
	382	+
	383	+static void xe_guc_log_load_lic(void guc_log, struct guc_lic_save config)
	384	+{
	385	+ u32 offset = GUC_LOG_BUFFER_STATE_HEADER_LENGTH;
	386	+ struct guc_log_buffer_state *p = guc_log;
	387	+
	388	+ config->version = p->version;
	389	+ while (p->marker[0]) {
	390	+ int index;
	391	+
	392	+ index = find_marker(p->marker[0], p->marker[1]);
	393	+
	394	+ if (index < ARRAY_SIZE(entry_markers)) {
	395	+ if (index == GUC_LOG_BUFFER_INIT_CONFIG) {
	396	+ /* Load log init config */
	397	+ xe_guc_log_loop_log_init((void *)p, config);
	398	+
	399	+ /* LIC structure is the last */
	400	+ return;
	401	+ }
	402	+ config->entry[index].offset = offset;
	403	+ config->entry[index].rd_ptr = p->read_ptr;
	404	+ config->entry[index].wr_ptr = p->write_ptr;
	405	+ config->entry[index].wrap_offset = p->wrap_offset;
	406	+ config->entry[index].buf_size = p->size;
	407	+ }
	408	+ offset += p->size;
	409	+ p++;
	410	+ }
	411	+}
	412	+
	413	+static int
	414	+xe_guc_log_output_lfd_init(struct drm_printer p, struct xe_guc_log_snapshot snapshot,
	415	+ struct guc_lic_save *config)
	416	+{
	417	+ int type, len;
	418	+ size_t size = 0;
	419	+
	420	+ /* FW required types */
	421	+ for (type = GUC_LFD_TYPE_FW_RANGE_FIRST; type <= GUC_LFD_TYPE_FW_RANGE_LAST; type++)
	422	+ size += xe_guc_log_add_klv(p, type, config);
	423	+
	424	+ /* KMD required type(s) */
	425	+ len = xe_guc_log_add_os_id(p, GUC_LFD_OS_TYPE_OSID_LIN);
	426	+ size += len;
	427	+
	428	+ return size;
	429	+}
	430	+
	431	+static void
	432	+xe_guc_log_print_chunks(struct drm_printer p, struct xe_guc_log_snapshot snapshot,
	433	+ u32 from, u32 to)
	434	+{
	435	+ int chunk_from = from % GUC_LOG_CHUNK_SIZE;
	436	+ int chunk_id = from / GUC_LOG_CHUNK_SIZE;
	437	+ int to_chunk_id = to / GUC_LOG_CHUNK_SIZE;
	438	+ int chunk_to = to % GUC_LOG_CHUNK_SIZE;
	439	+ int pos = from;
	440	+
	441	+ do {
	442	+ size_t size = (to_chunk_id == chunk_id ? chunk_to : GUC_LOG_CHUNK_SIZE) -
	443	+ chunk_from;
	444	+
	445	+ lfd_output_binary(p, snapshot->copy[chunk_id] + chunk_from, size);
	446	+ pos += size;
	447	+ chunk_id++;
	448	+ chunk_from = 0;
	449	+ } while (pos < to);
	450	+}
	451	+
	452	+static inline int
	453	+xe_guc_log_add_log_event(struct drm_printer p, struct xe_guc_log_snapshot snapshot,
	454	+ struct guc_lic_save *config)
	455	+{
	456	+ size_t size;
	457	+ u32 data_len, section_len;
	458	+ struct guc_lfd_data lfd;
	459	+ struct guc_log_buffer_entry_list *entry;
	460	+ struct guc_lfd_data_log_events_buf events_buf;
	461	+
	462	+ entry = &config->entry[GUC_LOG_TYPE_EVENT_DATA];
	463	+
	464	+ /* Skip empty log */
	465	+ if (entry->rd_ptr == entry->wr_ptr)
	466	+ return 0;
	467	+
	468	+ size = xe_guc_log_add_lfd_header(&lfd);
	469	+ lfd.header \|= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_LOG_EVENTS_BUFFER);
	470	+ events_buf.log_events_format_version = config->version;
	471	+
	472	+ /* Adjust to log_format_buf */
	473	+ section_len = offsetof(struct guc_lfd_data_log_events_buf, log_event);
	474	+ data_len = section_len;
	475	+
	476	+ /* Calculate data length */
	477	+ data_len += entry->rd_ptr < entry->wr_ptr ? (entry->wr_ptr - entry->rd_ptr) :
	478	+ (entry->wr_ptr + entry->wrap_offset - entry->rd_ptr);
	479	+ /* make length u32 aligned */
	480	+ lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32));
	481	+
	482	+ /* Output GUC_LFD_TYPE_LOG_EVENTS_BUFFER header */
	483	+ lfd_output_binary(p, (char *)&lfd, size);
	484	+ lfd_output_binary(p, (char *)&events_buf, section_len);
	485	+
	486	+ /* Output data from guc log chunks directly */
	487	+ if (entry->rd_ptr < entry->wr_ptr) {
	488	+ xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr,
	489	+ entry->offset + entry->wr_ptr);
	490	+ } else {
	491	+ /* 1st, print from rd to wrap offset */
	492	+ xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr,
	493	+ entry->offset + entry->wrap_offset);
	494	+
	495	+ /* 2nd, print from buf start to wr */
	496	+ xe_guc_log_print_chunks(p, snapshot, entry->offset, entry->offset + entry->wr_ptr);
	497	+ }
	498	+ return size;
	499	+}
	500	+
	501	+static int
	502	+xe_guc_log_add_crash_dump(struct drm_printer p, struct xe_guc_log_snapshot snapshot,
	503	+ struct guc_lic_save *config)
	504	+{
	505	+ struct guc_log_buffer_entry_list *entry;
	506	+ int chunk_from, chunk_id;
	507	+ int from, to, i;
	508	+ size_t size = 0;
	509	+ u32 *buf32;
	510	+
	511	+ entry = &config->entry[GUC_LOG_TYPE_CRASH_DUMP];
	512	+
	513	+ /* Skip zero sized crash dump */
	514	+ if (!entry->buf_size)
	515	+ return 0;
	516	+
	517	+ /* Check if crash dump section are all zero */
	518	+ from = entry->offset;
	519	+ to = entry->offset + entry->buf_size;
	520	+ chunk_from = from % GUC_LOG_CHUNK_SIZE;
	521	+ chunk_id = from / GUC_LOG_CHUNK_SIZE;
	522	+ buf32 = snapshot->copy[chunk_id] + chunk_from;
	523	+
	524	+ for (i = 0; i < entry->buf_size / sizeof(u32); i++)
	525	+ if (buf32[i])
	526	+ break;
	527	+
	528	+ /* Buffer has non-zero data? */
	529	+ if (i < entry->buf_size / sizeof(u32)) {
	530	+ struct guc_lfd_data lfd;
	531	+
	532	+ size = xe_guc_log_add_lfd_header(&lfd);
	533	+ lfd.header \|= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_FW_CRASH_DUMP);
	534	+ /* Calculate data length */
	535	+ lfd.data_count = DIV_ROUND_UP(entry->buf_size, sizeof(u32));
	536	+ /* Output GUC_LFD_TYPE_FW_CRASH_DUMP header */
	537	+ lfd_output_binary(p, (char *)&lfd, size);
	538	+
	539	+ /* rd/wr ptr is not used for crash dump */
	540	+ xe_guc_log_print_chunks(p, snapshot, from, to);
	541	+ }
	542	+ return size;
	543	+}
	544	+
	545	+static void
	546	+xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot snapshot, struct drm_printer p)
	547	+{
	548	+ struct guc_lfd_file_header header;
	549	+ struct guc_lic_save config;
	550	+ size_t size;
	551	+
	552	+ if (!snapshot \|\| !snapshot->size)
	553	+ return;
	554	+
	555	+ header.magic = GUC_LFD_DRIVER_KEY_STREAMING;
	556	+ header.version = FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR,
	557	+ GUC_LFD_FORMAT_VERSION_MINOR) \|
	558	+ FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR,
	559	+ GUC_LFD_FORMAT_VERSION_MAJOR);
	560	+
	561	+ /* Output LFD file header */
	562	+ lfd_output_binary(p, (char *)&header,
	563	+ offsetof(struct guc_lfd_file_header, stream));
	564	+
	565	+ /* Output LFD stream */
	566	+ xe_guc_log_load_lic(snapshot->copy[0], &config);
	567	+ size = xe_guc_log_output_lfd_init(p, snapshot, &config);
	568	+ if (!size)
	569	+ return;
	570	+
	571	+ xe_guc_log_add_log_event(p, snapshot, &config);
	572	+ xe_guc_log_add_crash_dump(p, snapshot, &config);
	573	+}
	574	+
219	575	/**
220	576	* xe_guc_log_print_dmesg - dump a copy of the GuC log to dmesg
221	577	* @log: GuC log structure
@@ -251,13 +607,27 @@ void xe_guc_log_print(struct xe_guc_log log, struct drm_printer p) · 29 unmodified lines
251	607	xe_guc_log_snapshot_free(snapshot);
252	608	}
253	609
	610	+/**
	611	+ * xe_guc_log_print_lfd - dump a copy of the GuC log in LFD format
	612	+ * @log: GuC log structure
	613	+ * @p: the printer object to output to
	614	+ */
	615	+void xe_guc_log_print_lfd(struct xe_guc_log log, struct drm_printer p)
	616	+{
	617	+ struct xe_guc_log_snapshot *snapshot;
	618	+
	619	+ snapshot = xe_guc_log_snapshot_capture(log, false);
	620	+ xe_guc_log_snapshot_print_lfd(snapshot, p);
	621	+ xe_guc_log_snapshot_free(snapshot);
	622	+}
	623	+
254	624	int xe_guc_log_init(struct xe_guc_log *log)
255	625	{
256	626	struct xe_device *xe = log_to_xe(log);
257	627	struct xe_tile *tile = gt_to_tile(log_to_gt(log));
258	628	struct xe_bo *bo;
259	629
260		− bo = xe_managed_bo_create_pin_map(xe, tile, guc_log_size(),
	630	+ bo = xe_managed_bo_create_pin_map(xe, tile, GUC_LOG_SIZE,
261	631	XE_BO_FLAG_SYSTEM \|
262	632	XE_BO_FLAG_GGTT \|
263	633	XE_BO_FLAG_GGTT_INVALIDATE \|
@@ -265,7 +635,7 @@ int xe_guc_log_init(struct xe_guc_log *log) · 1 unmodified line
265	635	if (IS_ERR(bo))
266	636	return PTR_ERR(bo);
267	637
268		− xe_map_memset(xe, &bo->vmap, 0, 0, guc_log_size());
	638	+ xe_map_memset(xe, &bo->vmap, 0, 0, xe_bo_size(bo));
269	639	log->bo = bo;
270	640	log->level = xe_modparam.guc_log_level;
271	641
@@ -274,71 +644,6 @@ int xe_guc_log_init(struct xe_guc_log *log) · 2 unmodified lines
274	644
275	645	ALLOW_ERROR_INJECTION(xe_guc_log_init, ERRNO); /* See xe_pci_probe() */
276	646
277		−static u32 xe_guc_log_section_size_crash(struct xe_guc_log *log)
278		−{
279		− return CRASH_BUFFER_SIZE;
280		−}
281		−
282		−static u32 xe_guc_log_section_size_debug(struct xe_guc_log *log)
283		−{
284		− return DEBUG_BUFFER_SIZE;
285		−}
286		−
287		−/**
288		− * xe_guc_log_section_size_capture - Get capture buffer size within log sections.
289		− * @log: The log object.
290		− *
291		− * This function will return the capture buffer size within log sections.
292		− *
293		− * Return: capture buffer size.
294		− */
295		−u32 xe_guc_log_section_size_capture(struct xe_guc_log *log)
296		−{
297		− return CAPTURE_BUFFER_SIZE;
298		−}
299		−
300		−/**
301		− * xe_guc_get_log_buffer_size - Get log buffer size for a type.
302		− * @log: The log object.
303		− * @type: The log buffer type
304		− *
305		− * Return: buffer size.
306		− */
307		−u32 xe_guc_get_log_buffer_size(struct xe_guc_log *log, enum guc_log_buffer_type type)
308		−{
309		− switch (type) {
310		− case GUC_LOG_BUFFER_CRASH_DUMP:
311		− return xe_guc_log_section_size_crash(log);
312		− case GUC_LOG_BUFFER_DEBUG:
313		− return xe_guc_log_section_size_debug(log);
314		− case GUC_LOG_BUFFER_CAPTURE:
315		− return xe_guc_log_section_size_capture(log);
316		− }
317		− return 0;
318		−}
319		−
320		−/**
321		− * xe_guc_get_log_buffer_offset - Get offset in log buffer for a type.
322		− * @log: The log object.
323		− * @type: The log buffer type
324		− *
325		− * This function will return the offset in the log buffer for a type.
326		− * Return: buffer offset.
327		− */
328		−u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_type type)
329		−{
330		− enum guc_log_buffer_type i;
331		− u32 offset = PAGE_SIZE;/* for the log_buffer_states */
332		−
333		− for (i = GUC_LOG_BUFFER_CRASH_DUMP; i < GUC_LOG_BUFFER_TYPE_MAX; ++i) {
334		− if (i == type)
335		− break;
336		− offset += xe_guc_get_log_buffer_size(log, i);
337		− }
338		−
339		− return offset;
340		−}
341		−
342	647	/**
343	648	* xe_guc_check_log_buf_overflow - Check if log buffer overflowed
344	649	* @log: The log object.
@@ -352,7 +657,7 @@ u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_typ · 7 unmodified lines
352	657	*
353	658	* Return: True if overflowed.
354	659	*/
355		−bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_buffer_type type,
	660	+bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_type type,
356	661	unsigned int full_cnt)
357	662	{
358	663	unsigned int prev_full_cnt = log->stats[type].sampled_overflow;

drivers/gpu/drm/xe/xe_guc_log.h+20−10

@@ -13,14 +13,26 @@ struct drm_printer; · 12 unmodified lines
13	13	struct xe_device;
14	14
15	15	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
16		−#define CRASH_BUFFER_SIZE SZ_1M
17		−#define DEBUG_BUFFER_SIZE SZ_8M
18		−#define CAPTURE_BUFFER_SIZE SZ_2M
	16	+#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE SZ_8M
	17	+#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE SZ_1M
	18	+#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE SZ_2M
19	19	#else
20		−#define CRASH_BUFFER_SIZE SZ_16K
21		−#define DEBUG_BUFFER_SIZE SZ_64K
22		−#define CAPTURE_BUFFER_SIZE SZ_1M
	20	+#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE SZ_64K
	21	+#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE SZ_16K
	22	+#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE SZ_1M
23	23	#endif
	24	+
	25	+#define GUC_LOG_SIZE (SZ_4K + \
	26	+ XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE + \
	27	+ XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE + \
	28	+ XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE)
	29	+
	30	+#define XE_GUC_LOG_EVENT_DATA_OFFSET SZ_4K
	31	+#define XE_GUC_LOG_CRASH_DUMP_OFFSET (XE_GUC_LOG_EVENT_DATA_OFFSET + \
	32	+ XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE)
	33	+#define XE_GUC_LOG_STATE_CAPTURE_OFFSET (XE_GUC_LOG_CRASH_DUMP_OFFSET + \
	34	+ XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE)
	35	+
24	36	/*
25	37	* While we're using plain log level in i915, GuC controls are much more...
26	38	* "elaborate"? We have a couple of bits for verbosity, separate bit for actual
@@ -40,6 +52,7 @@ struct xe_device; · 13 unmodified lines
40	52
41	53	int xe_guc_log_init(struct xe_guc_log *log);
42	54	void xe_guc_log_print(struct xe_guc_log log, struct drm_printer p);
	55	+void xe_guc_log_print_lfd(struct xe_guc_log log, struct drm_printer p);
43	56	void xe_guc_log_print_dmesg(struct xe_guc_log *log);
44	57	struct xe_guc_log_snapshot xe_guc_log_snapshot_capture(struct xe_guc_log log, bool atomic);
45	58	void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot snapshot, struct drm_printer p);
@@ -51,11 +64,8 @@ xe_guc_log_get_level(struct xe_guc_log *log) · 5 unmodified lines
51	64	return log->level;
52	65	}
53	66
54		−u32 xe_guc_log_section_size_capture(struct xe_guc_log *log);
55		−u32 xe_guc_get_log_buffer_size(struct xe_guc_log *log, enum guc_log_buffer_type type);
56		−u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_type type);
57	67	bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log,
58		− enum guc_log_buffer_type type,
	68	+ enum guc_log_type type,
59	69	unsigned int full_cnt);
60	70
61	71	#endif

drivers/gpu/drm/xe/xe_guc_pc.c+19−47

@@ -76,7 +76,7 @@ · 75 unmodified lines
76	76	* exposes a programming interface to the host for the control of SLPC.
77	77	*
78	78	* Frequency management:
79		− * =====================
	79	+ * ---------------------
80	80	*
81	81	* Xe driver enables SLPC with all of its defaults features and frequency
82	82	* selection, which varies per platform.
@@ -87,7 +87,7 @@ · 4 unmodified lines
87	87	* for any workload.
88	88	*
89	89	* Render-C States:
90		− * ================
	90	+ * ----------------
91	91	*
92	92	* Render-C states is also a GuC PC feature that is now enabled in Xe for
93	93	* all platforms.
@@ -499,21 +499,17 @@ u32 xe_guc_pc_get_cur_freq_fw(struct xe_guc_pc *pc) · 405 unmodified lines
499	499	int xe_guc_pc_get_cur_freq(struct xe_guc_pc pc, u32 freq)
500	500	{
501	501	struct xe_gt *gt = pc_to_gt(pc);
502		− unsigned int fw_ref;
503	502
504	503	/*
505	504	* GuC SLPC plays with cur freq request when GuCRC is enabled
506	505	* Block RC6 for a more reliable read.
507	506	*/
508		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
509		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {
510		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
	507	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	508	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
511	509	return -ETIMEDOUT;
512		− }
513	510
514	511	*freq = get_cur_freq(gt);
515	512
516		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
517	513	return 0;
518	514	}
519	515
@@ -1087,13 +1083,8 @@ int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc) · 567 unmodified lines
1087	1083	*/
1088	1084	int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode)
1089	1085	{
1090		− int ret;
1091		−
1092		− xe_pm_runtime_get(pc_to_xe(pc));
1093		− ret = pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
1094		− xe_pm_runtime_put(pc_to_xe(pc));
1095		−
1096		− return ret;
	1086	+ guard(xe_pm_runtime)(pc_to_xe(pc));
	1087	+ return pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
1097	1088	}
1098	1089
1099	1090	/**
@@ -1104,13 +1095,8 @@ int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mod · 4 unmodified lines
1104	1095	*/
1105	1096	int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc)
1106	1097	{
1107		− int ret;
1108		−
1109		− xe_pm_runtime_get(pc_to_xe(pc));
1110		− ret = pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
1111		− xe_pm_runtime_put(pc_to_xe(pc));
1112		−
1113		− return ret;
	1098	+ guard(xe_pm_runtime)(pc_to_xe(pc));
	1099	+ return pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
1114	1100	}
1115	1101
1116	1102	static void pc_init_pcode_freq(struct xe_guc_pc *pc)
@@ -1198,7 +1184,7 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc pc, const char buf) · 81 unmodified lines
1198	1184	return -EINVAL;
1199	1185
1200	1186	guard(mutex)(&pc->freq_lock);
1201		− xe_pm_runtime_get_noresume(pc_to_xe(pc));
	1187	+ guard(xe_pm_runtime_noresume)(pc_to_xe(pc));
1202	1188
1203	1189	ret = pc_action_set_param(pc,
1204	1190	SLPC_PARAM_POWER_PROFILE,
@@ -1209,8 +1195,6 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc pc, const char buf) · 4 unmodified lines
1209	1195	else
1210	1196	pc->power_profile = val;
1211	1197
1212		− xe_pm_runtime_put(pc_to_xe(pc));
1213		−
1214	1198	return ret;
1215	1199	}
1216	1200
@@ -1223,17 +1207,14 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 6 unmodified lines
1223	1207	struct xe_device *xe = pc_to_xe(pc);
1224	1208	struct xe_gt *gt = pc_to_gt(pc);
1225	1209	u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));
1226		− unsigned int fw_ref;
1227	1210	ktime_t earlier;
1228	1211	int ret;
1229	1212
1230	1213	xe_gt_assert(gt, xe_device_uc_enabled(xe));
1231	1214
1232		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1233		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {
1234		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
	1215	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	1216	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
1235	1217	return -ETIMEDOUT;
1236		− }
1237	1218
1238	1219	if (xe->info.skip_guc_pc) {
1239	1220	if (xe->info.platform != XE_PVC)
@@ -1241,9 +1222,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 1 unmodified line
1241	1222
1242	1223	/* Request max possible since dynamic freq mgmt is not enabled */
1243	1224	pc_set_cur_freq(pc, UINT_MAX);
1244		−
1245		− ret = 0;
1246		− goto out;
	1225	+ return 0;
1247	1226	}
1248	1227
1249	1228	xe_map_memset(xe, &pc->bo->vmap, 0, 0, size);
@@ -1252,7 +1231,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 2 unmodified lines
1252	1231	earlier = ktime_get();
1253	1232	ret = pc_action_reset(pc);
1254	1233	if (ret)
1255		− goto out;
	1234	+ return ret;
1256	1235
1257	1236	if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
1258	1237	SLPC_RESET_TIMEOUT_MS)) {
@@ -1263,8 +1242,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 4 unmodified lines
1263	1242	if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
1264	1243	SLPC_RESET_EXTENDED_TIMEOUT_MS)) {
1265	1244	xe_gt_err(gt, "GuC PC Start failed: Dynamic GT frequency control and GT sleep states are now disabled.\n");
1266		− ret = -EIO;
1267		− goto out;
	1245	+ return -EIO;
1268	1246	}
1269	1247
1270	1248	xe_gt_warn(gt, "GuC PC excessive start time: %lldms",
@@ -1273,21 +1251,20 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 2 unmodified lines
1273	1251
1274	1252	ret = pc_init_freqs(pc);
1275	1253	if (ret)
1276		− goto out;
	1254	+ return ret;
1277	1255
1278	1256	ret = pc_set_mert_freq_cap(pc);
1279	1257	if (ret)
1280		− goto out;
	1258	+ return ret;
1281	1259
1282	1260	if (xe->info.platform == XE_PVC) {
1283	1261	xe_guc_pc_gucrc_disable(pc);
1284		− ret = 0;
1285		− goto out;
	1262	+ return 0;
1286	1263	}
1287	1264
1288	1265	ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL);
1289	1266	if (ret)
1290		− goto out;
	1267	+ return ret;
1291	1268
1292	1269	/* Enable SLPC Optimized Strategy for compute */
1293	1270	ret = pc_action_set_strategy(pc, SLPC_OPTIMIZED_STRATEGY_COMPUTE);
@@ -1297,8 +1274,6 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 3 unmodified lines
1297	1274	if (unlikely(ret))
1298	1275	xe_gt_err(gt, "Failed to set SLPC power profile: %pe\n", ERR_PTR(ret));
1299	1276
1300		−out:
1301		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
1302	1277	return ret;
1303	1278	}
1304	1279
@@ -1330,19 +1305,16 @@ static void xe_guc_pc_fini_hw(void *arg) · 25 unmodified lines
1330	1305	{
1331	1306	struct xe_guc_pc *pc = arg;
1332	1307	struct xe_device *xe = pc_to_xe(pc);
1333		− unsigned int fw_ref;
1334	1308
1335	1309	if (xe_device_wedged(xe))
1336	1310	return;
1337	1311
1338		− fw_ref = xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
	1312	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
1339	1313	xe_guc_pc_gucrc_disable(pc);
1340	1314	XE_WARN_ON(xe_guc_pc_stop(pc));
1341	1315
1342	1316	/* Bind requested freq to mert_freq_cap before unload */
1343	1317	pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc)));
1344		−
1345		− xe_force_wake_put(gt_to_fw(pc_to_gt(pc)), fw_ref);
1346	1318	}
1347	1319
1348	1320	/**

drivers/gpu/drm/xe/xe_guc_submit.c+637−68

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	#include "abi/guc_klvs_abi.h"
20	20	#include "regs/xe_lrc_layout.h"
21	21	#include "xe_assert.h"
	22	+#include "xe_bo.h"
22	23	#include "xe_devcoredump.h"
23	24	#include "xe_device.h"
24	25	#include "xe_exec_queue.h"
@@ -47,6 +48,8 @@ · 22 unmodified lines
47	48	#include "xe_uc_fw.h"
48	49	#include "xe_vm.h"
49	50
	51	+#define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN 6
	52	+
50	53	static struct xe_guc *
51	54	exec_queue_to_guc(struct xe_exec_queue *q)
52	55	{
@@ -72,6 +75,7 @@ exec_queue_to_guc(struct xe_exec_queue *q) · 19 unmodified lines
72	75	#define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11)
73	76	#define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 12)
74	77	#define EXEC_QUEUE_STATE_PENDING_TDR_EXIT (1 << 13)
	78	+#define EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND (1 << 14)
75	79
76	80	static bool exec_queue_registered(struct xe_exec_queue *q)
77	81	{
@@ -263,6 +267,21 @@ static void clear_exec_queue_pending_tdr_exit(struct xe_exec_queue *q) · 185 unmodified lines
263	267	atomic_and(~EXEC_QUEUE_STATE_PENDING_TDR_EXIT, &q->guc->state);
264	268	}
265	269
	270	+static bool exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
	271	+{
	272	+ return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND;
	273	+}
	274	+
	275	+static void set_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
	276	+{
	277	+ atomic_or(EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state);
	278	+}
	279	+
	280	+static void clear_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
	281	+{
	282	+ atomic_and(~EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state);
	283	+}
	284	+
266	285	static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)
267	286	{
268	287	return (atomic_read(&q->guc->state) &
@@ -541,7 +560,8 @@ static void init_policies(struct xe_guc guc, struct xe_exec_queue q) · 272 unmodified lines
541	560	u32 slpc_exec_queue_freq_req = 0;
542	561	u32 preempt_timeout_us = q->sched_props.preempt_timeout_us;
543	562
544		− xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
	563	+ xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) &&
	564	+ !xe_exec_queue_is_multi_queue_secondary(q));
545	565
546	566	if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY)
547	567	slpc_exec_queue_freq_req \|= SLPC_CTX_FREQ_REQ_IS_COMPUTE;
@@ -561,6 +581,8 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue · 13 unmodified lines
561	581	{
562	582	struct exec_queue_policy policy;
563	583
	584	+ xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q));
	585	+
564	586	__guc_exec_queue_policy_start_klv(&policy, q->guc->id);
565	587	__guc_exec_queue_policy_add_preemption_timeout(&policy, 1);
566	588
@@ -568,6 +590,89 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue · 1 unmodified line
568	590	__guc_exec_queue_policy_action_size(&policy), 0, 0);
569	591	}
570	592
	593	+static bool vf_recovery(struct xe_guc *guc)
	594	+{
	595	+ return xe_gt_recovery_pending(guc_to_gt(guc));
	596	+}
	597	+
	598	+static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
	599	+{
	600	+ struct xe_guc *guc = exec_queue_to_guc(q);
	601	+ struct xe_device *xe = guc_to_xe(guc);
	602	+
	603	+ /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
	604	+ wake_up_all(&xe->ufence_wq);
	605	+
	606	+ if (xe_exec_queue_is_lr(q))
	607	+ queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);
	608	+ else
	609	+ xe_sched_tdr_queue_imm(&q->guc->sched);
	610	+}
	611	+
	612	+static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q)
	613	+{
	614	+ struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
	615	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	616	+ struct xe_exec_queue *eq;
	617	+
	618	+ xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
	619	+ xe_exec_queue_is_multi_queue(q));
	620	+
	621	+ /* Group banned, skip timeout check in TDR */
	622	+ WRITE_ONCE(group->banned, true);
	623	+ xe_guc_exec_queue_trigger_cleanup(primary);
	624	+
	625	+ mutex_lock(&group->list_lock);
	626	+ list_for_each_entry(eq, &group->list, multi_queue.link)
	627	+ xe_guc_exec_queue_trigger_cleanup(eq);
	628	+ mutex_unlock(&group->list_lock);
	629	+}
	630	+
	631	+static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q)
	632	+{
	633	+ if (xe_exec_queue_is_multi_queue(q)) {
	634	+ struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
	635	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	636	+ struct xe_exec_queue *eq;
	637	+
	638	+ /* Group banned, skip timeout check in TDR */
	639	+ WRITE_ONCE(group->banned, true);
	640	+
	641	+ set_exec_queue_reset(primary);
	642	+ if (!exec_queue_banned(primary) && !exec_queue_check_timeout(primary))
	643	+ xe_guc_exec_queue_trigger_cleanup(primary);
	644	+
	645	+ mutex_lock(&group->list_lock);
	646	+ list_for_each_entry(eq, &group->list, multi_queue.link) {
	647	+ set_exec_queue_reset(eq);
	648	+ if (!exec_queue_banned(eq) && !exec_queue_check_timeout(eq))
	649	+ xe_guc_exec_queue_trigger_cleanup(eq);
	650	+ }
	651	+ mutex_unlock(&group->list_lock);
	652	+ } else {
	653	+ set_exec_queue_reset(q);
	654	+ if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
	655	+ xe_guc_exec_queue_trigger_cleanup(q);
	656	+ }
	657	+}
	658	+
	659	+static void set_exec_queue_group_banned(struct xe_exec_queue *q)
	660	+{
	661	+ struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
	662	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	663	+ struct xe_exec_queue *eq;
	664	+
	665	+ /* Ban all queues of the multi-queue group */
	666	+ xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
	667	+ xe_exec_queue_is_multi_queue(q));
	668	+ set_exec_queue_banned(primary);
	669	+
	670	+ mutex_lock(&group->list_lock);
	671	+ list_for_each_entry(eq, &group->list, multi_queue.link)
	672	+ set_exec_queue_banned(eq);
	673	+ mutex_unlock(&group->list_lock);
	674	+}
	675	+
571	676	#define parallel_read(xe_, map_, field_) \
572	677	xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
573	678	field_)
@@ -575,6 +680,181 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue · 1 unmodified line
575	680	xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
576	681	field_, val_)
577	682
	683	+/**
	684	+ * DOC: Multi Queue Group GuC interface
	685	+ *
	686	+ * The multi queue group coordination between KMD and GuC is through a software
	687	+ * construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page
	688	+ * allocated in the global GTT.
	689	+ *
	690	+ * CGP format:
	691	+ *
	692	+ * +-----------+---------------------------+---------------------------------------------+
	693	+ * \| DWORD \| Name \| Description \|
	694	+ * +-----------+---------------------------+---------------------------------------------+
	695	+ * \| 0 \| Version \| Bits [15:8]=Major ver, [7:0]=Minor ver \|
	696	+ * +-----------+---------------------------+---------------------------------------------+
	697	+ * \| 1..15 \| RESERVED \| MBZ \|
	698	+ * +-----------+---------------------------+---------------------------------------------+
	699	+ * \| 16 \| KMD_QUEUE_UPDATE_MASK_DW0 \| KMD queue mask for queues 31..0 \|
	700	+ * +-----------+---------------------------+---------------------------------------------+
	701	+ * \| 17 \| KMD_QUEUE_UPDATE_MASK_DW1 \| KMD queue mask for queues 63..32 \|
	702	+ * +-----------+---------------------------+---------------------------------------------+
	703	+ * \| 18..31 \| RESERVED \| MBZ \|
	704	+ * +-----------+---------------------------+---------------------------------------------+
	705	+ * \| 32 \| Q0CD_DW0 \| Queue 0 context LRC descriptor lower DWORD \|
	706	+ * +-----------+---------------------------+---------------------------------------------+
	707	+ * \| 33 \| Q0ContextIndex \| Context ID for Queue 0 \|
	708	+ * +-----------+---------------------------+---------------------------------------------+
	709	+ * \| 34 \| Q1CD_DW0 \| Queue 1 context LRC descriptor lower DWORD \|
	710	+ * +-----------+---------------------------+---------------------------------------------+
	711	+ * \| 35 \| Q1ContextIndex \| Context ID for Queue 1 \|
	712	+ * +-----------+---------------------------+---------------------------------------------+
	713	+ * \| ... \|... \| ... \|
	714	+ * +-----------+---------------------------+---------------------------------------------+
	715	+ * \| 158 \| Q63CD_DW0 \| Queue 63 context LRC descriptor lower DWORD \|
	716	+ * +-----------+---------------------------+---------------------------------------------+
	717	+ * \| 159 \| Q63ContextIndex \| Context ID for Queue 63 \|
	718	+ * +-----------+---------------------------+---------------------------------------------+
	719	+ * \| 160..1024 \| RESERVED \| MBZ \|
	720	+ * +-----------+---------------------------+---------------------------------------------+
	721	+ *
	722	+ * While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified
	723	+ * through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies
	724	+ * the CGP address. When the secondary queues are added to the group, the CGP is
	725	+ * updated with entry for that queue and GuC is notified through the H2G interface
	726	+ * XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages
	727	+ * with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also
	728	+ * sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any
	729	+ * error in the CGP. Only one of these CGP update messages can be outstanding
	730	+ * (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW*
	731	+ * fields indicate which queue entry is being updated in the CGP.
	732	+ *
	733	+ * The primary queue (Q0) represents the multi queue group context in GuC and
	734	+ * submission on any queue of the group must be through Q0 GuC interface only.
	735	+ *
	736	+ * As it is not required to register secondary queues with GuC, the secondary queue
	737	+ * context ids in the CGP are populated with Q0 context id.
	738	+ */
	739	+
	740	+#define CGP_VERSION_MAJOR_SHIFT 8
	741	+
	742	+static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe,
	743	+ struct xe_exec_queue *q)
	744	+{
	745	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	746	+ u32 guc_id = group->primary->guc->id;
	747	+
	748	+ /* Currently implementing CGP version 1.0 */
	749	+ xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32,
	750	+ 1 << CGP_VERSION_MAJOR_SHIFT);
	751	+
	752	+ xe_map_wr(xe, &group->cgp_bo->vmap,
	753	+ (32 + q->multi_queue.pos * 2) * sizeof(u32),
	754	+ u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0])));
	755	+
	756	+ xe_map_wr(xe, &group->cgp_bo->vmap,
	757	+ (33 + q->multi_queue.pos * 2) * sizeof(u32),
	758	+ u32, guc_id);
	759	+
	760	+ if (q->multi_queue.pos / 32) {
	761	+ xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32),
	762	+ u32, BIT(q->multi_queue.pos % 32));
	763	+ xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0);
	764	+ } else {
	765	+ xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32),
	766	+ u32, BIT(q->multi_queue.pos));
	767	+ xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0);
	768	+ }
	769	+}
	770	+
	771	+static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
	772	+ struct xe_exec_queue *q,
	773	+ const u32 *action, u32 len)
	774	+{
	775	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	776	+ struct xe_device *xe = guc_to_xe(guc);
	777	+ long ret;
	778	+
	779	+ /*
	780	+ * As all queues of a multi queue group use single drm scheduler
	781	+ * submit workqueue, CGP synchronization with GuC are serialized.
	782	+ * Hence, no locking is required here.
	783	+ * Wait for any pending CGP_SYNC_DONE response before updating the
	784	+ * CGP page and sending CGP_SYNC message.
	785	+ *
	786	+ * FIXME: Support VF migration
	787	+ */
	788	+ ret = wait_event_timeout(guc->ct.wq,
	789	+ !READ_ONCE(group->sync_pending) \|\|
	790	+ xe_guc_read_stopped(guc), HZ);
	791	+ if (!ret \|\| xe_guc_read_stopped(guc)) {
	792	+ /* CGP_SYNC failed. Reset gt, cleanup the group */
	793	+ xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n");
	794	+ set_exec_queue_group_banned(q);
	795	+ xe_gt_reset_async(q->gt);
	796	+ xe_guc_exec_queue_group_trigger_cleanup(q);
	797	+ return;
	798	+ }
	799	+
	800	+ xe_lrc_set_multi_queue_priority(q->lrc[0], q->multi_queue.priority);
	801	+ xe_guc_exec_queue_group_cgp_update(xe, q);
	802	+
	803	+ WRITE_ONCE(group->sync_pending, true);
	804	+ xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1);
	805	+}
	806	+
	807	+static void __register_exec_queue_group(struct xe_guc *guc,
	808	+ struct xe_exec_queue *q,
	809	+ struct guc_ctxt_registration_info *info)
	810	+{
	811	+#define MAX_MULTI_QUEUE_REG_SIZE (8)
	812	+ u32 action[MAX_MULTI_QUEUE_REG_SIZE];
	813	+ int len = 0;
	814	+
	815	+ action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE;
	816	+ action[len++] = info->flags;
	817	+ action[len++] = info->context_idx;
	818	+ action[len++] = info->engine_class;
	819	+ action[len++] = info->engine_submit_mask;
	820	+ action[len++] = 0; /* Reserved */
	821	+ action[len++] = info->cgp_lo;
	822	+ action[len++] = info->cgp_hi;
	823	+
	824	+ xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE);
	825	+#undef MAX_MULTI_QUEUE_REG_SIZE
	826	+
	827	+ /*
	828	+ * The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a
	829	+ * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
	830	+ * from guc.
	831	+ */
	832	+ xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
	833	+}
	834	+
	835	+static void xe_guc_exec_queue_group_add(struct xe_guc *guc,
	836	+ struct xe_exec_queue *q)
	837	+{
	838	+#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2)
	839	+ u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
	840	+ int len = 0;
	841	+
	842	+ xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q));
	843	+
	844	+ action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
	845	+ action[len++] = q->multi_queue.group->primary->guc->id;
	846	+
	847	+ xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
	848	+#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
	849	+
	850	+ /*
	851	+ * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a
	852	+ * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
	853	+ * from guc.
	854	+ */
	855	+ xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
	856	+}
	857	+
578	858	static void __register_mlrc_exec_queue(struct xe_guc *guc,
579	859	struct xe_exec_queue *q,
580	860	struct guc_ctxt_registration_info *info)
@@ -670,6 +950,13 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) · 89 unmodified lines
670	950	info.flags = CONTEXT_REGISTRATION_FLAG_KMD \|
671	951	FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type);
672	952
	953	+ if (xe_exec_queue_is_multi_queue(q)) {
	954	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	955	+
	956	+ info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo);
	957	+ info.cgp_hi = 0;
	958	+ }
	959	+
673	960	if (xe_exec_queue_is_parallel(q)) {
674	961	u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);
675	962	struct iosys_map map = xe_lrc_parallel_map(lrc);
@@ -700,11 +987,18 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) · 24 unmodified lines
700	987
701	988	set_exec_queue_registered(q);
702	989	trace_xe_exec_queue_register(q);
703		− if (xe_exec_queue_is_parallel(q))
	990	+ if (xe_exec_queue_is_multi_queue_primary(q))
	991	+ __register_exec_queue_group(guc, q, &info);
	992	+ else if (xe_exec_queue_is_parallel(q))
704	993	__register_mlrc_exec_queue(guc, q, &info);
705		− else
	994	+ else if (!xe_exec_queue_is_multi_queue_secondary(q))
706	995	__register_exec_queue(guc, &info);
707		− init_policies(guc, q);
	996	+
	997	+ if (!xe_exec_queue_is_multi_queue_secondary(q))
	998	+ init_policies(guc, q);
	999	+
	1000	+ if (xe_exec_queue_is_multi_queue_secondary(q))
	1001	+ xe_guc_exec_queue_group_add(guc, q);
708	1002	}
709	1003
710	1004	static u32 wq_space_until_wrap(struct xe_exec_queue *q)
@@ -712,11 +1006,6 @@ static u32 wq_space_until_wrap(struct xe_exec_queue *q) · 1 unmodified line
712	1006	return (WQ_SIZE - q->guc->wqi_tail);
713	1007	}
714	1008
715		−static bool vf_recovery(struct xe_guc *guc)
716		−{
717		− return xe_gt_recovery_pending(guc_to_gt(guc));
718		−}
719		−
720	1009	static inline void relaxed_ms_sleep(unsigned int delay_ms)
721	1010	{
722	1011	unsigned long min_us, max_us;
@@ -845,7 +1134,7 @@ static void submit_exec_queue(struct xe_exec_queue q, struct xe_sched_job job) · 122 unmodified lines
845	1134	if (!job->restore_replay \|\| job->last_replay) {
846	1135	if (xe_exec_queue_is_parallel(q))
847	1136	wq_item_append(q);
848		− else
	1137	+ else if (!exec_queue_idle_skip_suspend(q))
849	1138	xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
850	1139	job->last_replay = false;
851	1140	}
@@ -853,6 +1142,12 @@ static void submit_exec_queue(struct xe_exec_queue q, struct xe_sched_job job) · 1 unmodified line
853	1142	if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q))
854	1143	return;
855	1144
	1145	+ /*
	1146	+ * All queues in a multi-queue group will use the primary queue
	1147	+ * of the group to interface with GuC.
	1148	+ */
	1149	+ q = xe_exec_queue_multi_queue_primary(q);
	1150	+
856	1151	if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) {
857	1152	action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
858	1153	action[len++] = q->guc->id;
@@ -899,6 +1194,18 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) · 40 unmodified lines
899	1194	trace_xe_sched_job_run(job);
900	1195
901	1196	if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) {
	1197	+ if (xe_exec_queue_is_multi_queue_secondary(q)) {
	1198	+ struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
	1199	+
	1200	+ if (exec_queue_killed_or_banned_or_wedged(primary)) {
	1201	+ killed_or_banned_or_wedged = true;
	1202	+ goto run_job_out;
	1203	+ }
	1204	+
	1205	+ if (!exec_queue_registered(primary))
	1206	+ register_exec_queue(primary, GUC_CONTEXT_NORMAL);
	1207	+ }
	1208	+
902	1209	if (!exec_queue_registered(q))
903	1210	register_exec_queue(q, GUC_CONTEXT_NORMAL);
904	1211	if (!job->restore_replay)
@@ -907,6 +1214,7 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) · 2 unmodified lines
907	1214	job->restore_replay = false;
908	1215	}
909	1216
	1217	+run_job_out:
910	1218	/*
911	1219	* We don't care about job-fence ordering in LR VMs because these fences
912	1220	* are never exported; they are used solely to keep jobs on the pending
@@ -932,6 +1240,11 @@ int xe_guc_read_stopped(struct xe_guc *guc) · 19 unmodified lines
932	1240	return atomic_read(&guc->submission_state.stopped);
933	1241	}
934	1242
	1243	+static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
	1244	+ struct xe_exec_queue *q,
	1245	+ u32 runnable_state);
	1246	+static void handle_deregister_done(struct xe_guc guc, struct xe_exec_queue q);
	1247	+
935	1248	#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \
936	1249	u32 action[] = { \
937	1250	XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \
@@ -945,7 +1258,9 @@ static void disable_scheduling_deregister(struct xe_guc *guc, · 7 unmodified lines
945	1258	MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
946	1259	int ret;
947	1260
948		− set_min_preemption_timeout(guc, q);
	1261	+ if (!xe_exec_queue_is_multi_queue_secondary(q))
	1262	+ set_min_preemption_timeout(guc, q);
	1263	+
949	1264	smp_rmb();
950	1265	ret = wait_event_timeout(guc->ct.wq,
951	1266	(!exec_queue_pending_enable(q) &&
@@ -973,23 +1288,12 @@ static void disable_scheduling_deregister(struct xe_guc *guc, · 21 unmodified lines
973	1288	* Reserve space for both G2H here as the 2nd G2H is sent from a G2H
974	1289	* handler and we are not allowed to reserved G2H space in handlers.
975	1290	*/
976		− xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
977		− G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +
978		− G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
979		−}
980		−
981		−static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
982		−{
983		− struct xe_guc *guc = exec_queue_to_guc(q);
984		− struct xe_device *xe = guc_to_xe(guc);
985		−
986		− /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
987		− wake_up_all(&xe->ufence_wq);
988		−
989		− if (xe_exec_queue_is_lr(q))
990		− queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);
	1291	+ if (xe_exec_queue_is_multi_queue_secondary(q))
	1292	+ handle_multi_queue_secondary_sched_done(guc, q, 0);
991	1293	else
992		− xe_sched_tdr_queue_imm(&q->guc->sched);
	1294	+ xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
	1295	+ G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +
	1296	+ G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
993	1297	}
994	1298
995	1299	/**
@@ -1181,8 +1485,11 @@ static void enable_scheduling(struct xe_exec_queue *q) · 185 unmodified lines
1181	1485	set_exec_queue_enabled(q);
1182	1486	trace_xe_exec_queue_scheduling_enable(q);
1183	1487
1184		− xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1185		− G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
	1488	+ if (xe_exec_queue_is_multi_queue_secondary(q))
	1489	+ handle_multi_queue_secondary_sched_done(guc, q, 1);
	1490	+ else
	1491	+ xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
	1492	+ G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1186	1493
1187	1494	ret = wait_event_timeout(guc->ct.wq,
1188	1495	!exec_queue_pending_enable(q) \|\|
@@ -1206,14 +1513,17 @@ static void disable_scheduling(struct xe_exec_queue *q, bool immediate) · 17 unmodified lines
1206	1513	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
1207	1514	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
1208	1515
1209		− if (immediate)
	1516	+ if (immediate && !xe_exec_queue_is_multi_queue_secondary(q))
1210	1517	set_min_preemption_timeout(guc, q);
1211	1518	clear_exec_queue_enabled(q);
1212	1519	set_exec_queue_pending_disable(q);
1213	1520	trace_xe_exec_queue_scheduling_disable(q);
1214	1521
1215		− xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1216		− G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
	1522	+ if (xe_exec_queue_is_multi_queue_secondary(q))
	1523	+ handle_multi_queue_secondary_sched_done(guc, q, 0);
	1524	+ else
	1525	+ xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
	1526	+ G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1217	1527	}
1218	1528
1219	1529	static void __deregister_exec_queue(struct xe_guc guc, struct xe_exec_queue q)
@@ -1231,8 +1541,11 @@ static void __deregister_exec_queue(struct xe_guc guc, struct xe_exec_queue q) · 11 unmodified lines
1231	1541	set_exec_queue_destroyed(q);
1232	1542	trace_xe_exec_queue_deregister(q);
1233	1543
1234		− xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1235		− G2H_LEN_DW_DEREGISTER_CONTEXT, 1);
	1544	+ if (xe_exec_queue_is_multi_queue_secondary(q))
	1545	+ handle_deregister_done(guc, q);
	1546	+ else
	1547	+ xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
	1548	+ G2H_LEN_DW_DEREGISTER_CONTEXT, 1);
1236	1549	}
1237	1550
1238	1551	static enum drm_gpu_sched_stat
@@ -1245,7 +1558,6 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) · 6 unmodified lines
1245	1558	struct xe_guc *guc = exec_queue_to_guc(q);
1246	1559	const char *process_name = "no process";
1247	1560	struct xe_device *xe = guc_to_xe(guc);
1248		− unsigned int fw_ref;
1249	1561	int err = -ETIME;
1250	1562	pid_t pid = -1;
1251	1563	int i = 0;
@@ -1271,6 +1583,19 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) · 19 unmodified lines
1271	1583	exec_queue_killed_or_banned_or_wedged(q) \|\|
1272	1584	exec_queue_destroyed(q);
1273	1585
	1586	+ /* Skip timeout check if multi-queue group is banned */
	1587	+ if (xe_exec_queue_is_multi_queue(q) &&
	1588	+ READ_ONCE(q->multi_queue.group->banned))
	1589	+ skip_timeout_check = true;
	1590	+
	1591	+ /*
	1592	+ * FIXME: In multi-queue scenario, the TDR must ensure that the whole
	1593	+ * multi-queue group is off the HW before signaling the fences to avoid
	1594	+ * possible memory corruptions. This means disabling scheduling on the
	1595	+ * primary queue before or during the secondary queue's TDR. Need to
	1596	+ * implement this in least obtrusive way.
	1597	+ */
	1598	+
1274	1599	/*
1275	1600	* If devcoredump not captured and GuC capture for the job is not ready
1276	1601	* do manual capture first and decide later if we need to use it
@@ -1278,13 +1603,11 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) · 1 unmodified line
1278	1603	if (!exec_queue_killed(q) && !xe->devcoredump.captured &&
1279	1604	!xe_guc_capture_get_matching_and_lock(q)) {
1280	1605	/* take force wake before engine register manual capture */
1281		− fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
1282		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
	1606	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
	1607	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
1283	1608	xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n");
1284	1609
1285	1610	xe_engine_snapshot_capture_for_queue(q);
1286		−
1287		− xe_force_wake_put(gt_to_fw(q->gt), fw_ref);
1288	1611	}
1289	1612
1290	1613	/*
@@ -1425,7 +1748,10 @@ trigger_reset: · 134 unmodified lines
1425	1748	xe_sched_add_pending_job(sched, job);
1426	1749	xe_sched_submission_start(sched);
1427	1750
1428		− xe_guc_exec_queue_trigger_cleanup(q);
	1751	+ if (xe_exec_queue_is_multi_queue(q))
	1752	+ xe_guc_exec_queue_group_trigger_cleanup(q);
	1753	+ else
	1754	+ xe_guc_exec_queue_trigger_cleanup(q);
1429	1755
1430	1756	/* Mark all outstanding jobs as bad, thus completing them */
1431	1757	spin_lock(&sched->base.job_list_lock);
@@ -1475,17 +1801,23 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w) · 43 unmodified lines
1475	1801	struct xe_exec_queue *q = ge->q;
1476	1802	struct xe_guc *guc = exec_queue_to_guc(q);
1477	1803
1478		− xe_pm_runtime_get(guc_to_xe(guc));
	1804	+ guard(xe_pm_runtime)(guc_to_xe(guc));
1479	1805	trace_xe_exec_queue_destroy(q);
1480	1806
	1807	+ if (xe_exec_queue_is_multi_queue_secondary(q)) {
	1808	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	1809	+
	1810	+ mutex_lock(&group->list_lock);
	1811	+ list_del(&q->multi_queue.link);
	1812	+ mutex_unlock(&group->list_lock);
	1813	+ }
	1814	+
1481	1815	if (xe_exec_queue_is_lr(q))
1482	1816	cancel_work_sync(&ge->lr_tdr);
1483	1817	/* Confirm no work left behind accessing device structures */
1484	1818	cancel_delayed_work_sync(&ge->sched.base.work_tdr);
1485	1819
1486	1820	xe_exec_queue_fini(q);
1487		−
1488		− xe_pm_runtime_put(guc_to_xe(guc));
1489	1821	}
1490	1822
1491	1823	static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)
@@ -1590,9 +1922,10 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) · 98 unmodified lines
1590	1922	{
1591	1923	struct xe_exec_queue *q = msg->private_data;
1592	1924	struct xe_guc *guc = exec_queue_to_guc(q);
	1925	+ bool idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q);
1593	1926
1594		− if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) &&
1595		− exec_queue_enabled(q)) {
	1927	+ if (!idle_skip_suspend && guc_exec_queue_allowed_to_change_state(q) &&
	1928	+ !exec_queue_suspended(q) && exec_queue_enabled(q)) {
1596	1929	wait_event(guc->ct.wq, vf_recovery(guc) \|\|
1597	1930	((q->guc->resume_time != RESUME_PENDING \|\|
1598	1931	xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q)));
@@ -1611,11 +1944,33 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) · 12 unmodified lines
1611	1944	disable_scheduling(q, false);
1612	1945	}
1613	1946	} else if (q->guc->suspend_pending) {
	1947	+ if (idle_skip_suspend)
	1948	+ set_exec_queue_idle_skip_suspend(q);
1614	1949	set_exec_queue_suspended(q);
1615	1950	suspend_fence_signal(q);
1616	1951	}
1617	1952	}
1618	1953
	1954	+static void sched_context(struct xe_exec_queue *q)
	1955	+{
	1956	+ struct xe_guc *guc = exec_queue_to_guc(q);
	1957	+ struct xe_lrc *lrc = q->lrc[0];
	1958	+ u32 action[] = {
	1959	+ XE_GUC_ACTION_SCHED_CONTEXT,
	1960	+ q->guc->id,
	1961	+ };
	1962	+
	1963	+ xe_gt_assert(guc_to_gt(guc), !xe_exec_queue_is_parallel(q));
	1964	+ xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
	1965	+ xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
	1966	+ xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
	1967	+
	1968	+ trace_xe_exec_queue_submit(q);
	1969	+
	1970	+ xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
	1971	+ xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
	1972	+}
	1973	+
1619	1974	static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)
1620	1975	{
1621	1976	struct xe_exec_queue *q = msg->private_data;
@@ -1623,19 +1978,53 @@ static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) · 1 unmodified line
1623	1978	if (guc_exec_queue_allowed_to_change_state(q)) {
1624	1979	clear_exec_queue_suspended(q);
1625	1980	if (!exec_queue_enabled(q)) {
	1981	+ if (exec_queue_idle_skip_suspend(q)) {
	1982	+ struct xe_lrc *lrc = q->lrc[0];
	1983	+
	1984	+ clear_exec_queue_idle_skip_suspend(q);
	1985	+ xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
	1986	+ }
1626	1987	q->guc->resume_time = RESUME_PENDING;
1627	1988	set_exec_queue_pending_resume(q);
1628	1989	enable_scheduling(q);
	1990	+ } else if (exec_queue_idle_skip_suspend(q)) {
	1991	+ clear_exec_queue_idle_skip_suspend(q);
	1992	+ sched_context(q);
1629	1993	}
1630	1994	} else {
1631	1995	clear_exec_queue_suspended(q);
	1996	+ clear_exec_queue_idle_skip_suspend(q);
	1997	+ }
	1998	+}
	1999	+
	2000	+static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg)
	2001	+{
	2002	+ struct xe_exec_queue *q = msg->private_data;
	2003	+
	2004	+ if (guc_exec_queue_allowed_to_change_state(q)) {
	2005	+#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2)
	2006	+ struct xe_guc *guc = exec_queue_to_guc(q);
	2007	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	2008	+ u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
	2009	+ int len = 0;
	2010	+
	2011	+ action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
	2012	+ action[len++] = group->primary->guc->id;
	2013	+
	2014	+ xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
	2015	+#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
	2016	+
	2017	+ xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
1632	2018	}
	2019	+
	2020	+ kfree(msg);
1633	2021	}
1634	2022
1635		−#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */
1636		−#define SET_SCHED_PROPS 2
1637		−#define SUSPEND 3
1638		−#define RESUME 4
	2023	+#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */
	2024	+#define SET_SCHED_PROPS 2
	2025	+#define SUSPEND 3
	2026	+#define RESUME 4
	2027	+#define SET_MULTI_QUEUE_PRIORITY 5
1639	2028	#define OPCODE_MASK 0xf
1640	2029	#define MSG_LOCKED BIT(8)
1641	2030	#define MSG_HEAD BIT(9)
@@ -1659,6 +2048,9 @@ static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) · 17 unmodified lines
1659	2048	case RESUME:
1660	2049	__guc_exec_queue_process_msg_resume(msg);
1661	2050	break;
	2051	+ case SET_MULTI_QUEUE_PRIORITY:
	2052	+ __guc_exec_queue_process_msg_set_multi_queue_priority(msg);
	2053	+ break;
1662	2054	default:
1663	2055	XE_WARN_ON("Unknown message type");
1664	2056	}
@@ -1680,6 +2072,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) · 15 unmodified lines
1680	2072	{
1681	2073	struct xe_gpu_scheduler *sched;
1682	2074	struct xe_guc *guc = exec_queue_to_guc(q);
	2075	+ struct workqueue_struct *submit_wq = NULL;
1683	2076	struct xe_guc_exec_queue *ge;
1684	2077	long timeout;
1685	2078	int err, i;
@@ -1700,8 +2093,20 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) · 14 unmodified lines
1700	2093
1701	2094	timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT :
1702	2095	msecs_to_jiffies(q->sched_props.job_timeout_ms);
	2096	+
	2097	+ /*
	2098	+ * Use primary queue's submit_wq for all secondary queues of a
	2099	+ * multi queue group. This serialization avoids any locking around
	2100	+ * CGP synchronization with GuC.
	2101	+ */
	2102	+ if (xe_exec_queue_is_multi_queue_secondary(q)) {
	2103	+ struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
	2104	+
	2105	+ submit_wq = primary->guc->sched.base.submit_wq;
	2106	+ }
	2107	+
1703	2108	err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
1704		− NULL, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
	2109	+ submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
1705	2110	timeout, guc_to_gt(guc)->ordered_wq, NULL,
1706	2111	q->name, gt_to_xe(q->gt)->drm.dev);
1707	2112	if (err)
@@ -1730,7 +2135,23 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) · 22 unmodified lines
1730	2135
1731	2136	xe_exec_queue_assign_name(q, q->guc->id);
1732	2137
1733		− trace_xe_exec_queue_create(q);
	2138	+ /*
	2139	+ * Maintain secondary queues of the multi queue group in a list
	2140	+ * for handling dependencies across the queues in the group.
	2141	+ */
	2142	+ if (xe_exec_queue_is_multi_queue_secondary(q)) {
	2143	+ struct xe_exec_queue_group *group = q->multi_queue.group;
	2144	+
	2145	+ INIT_LIST_HEAD(&q->multi_queue.link);
	2146	+ mutex_lock(&group->list_lock);
	2147	+ list_add_tail(&q->multi_queue.link, &group->list);
	2148	+ mutex_unlock(&group->list_lock);
	2149	+ }
	2150	+
	2151	+ if (xe_exec_queue_is_multi_queue(q))
	2152	+ trace_xe_exec_queue_create_multi_queue(q);
	2153	+ else
	2154	+ trace_xe_exec_queue_create(q);
1734	2155
1735	2156	return 0;
1736	2157
@@ -1862,6 +2283,27 @@ static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, · 125 unmodified lines
1862	2283	return 0;
1863	2284	}
1864	2285
	2286	+static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q,
	2287	+ enum xe_multi_queue_priority priority)
	2288	+{
	2289	+ struct xe_sched_msg *msg;
	2290	+
	2291	+ xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q));
	2292	+
	2293	+ if (q->multi_queue.priority == priority \|\|
	2294	+ exec_queue_killed_or_banned_or_wedged(q))
	2295	+ return 0;
	2296	+
	2297	+ msg = kmalloc(sizeof(*msg), GFP_KERNEL);
	2298	+ if (!msg)
	2299	+ return -ENOMEM;
	2300	+
	2301	+ q->multi_queue.priority = priority;
	2302	+ guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY);
	2303	+
	2304	+ return 0;
	2305	+}
	2306	+
1865	2307	static int guc_exec_queue_suspend(struct xe_exec_queue *q)
1866	2308	{
1867	2309	struct xe_gpu_scheduler *sched = &q->guc->sched;
@@ -1936,6 +2378,10 @@ static void guc_exec_queue_resume(struct xe_exec_queue *q) · 68 unmodified lines
1936	2378
1937	2379	static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
1938	2380	{
	2381	+ if (xe_exec_queue_is_multi_queue_secondary(q) &&
	2382	+ guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q)))
	2383	+ return true;
	2384	+
1939	2385	return exec_queue_reset(q) \|\| exec_queue_killed_or_banned_or_wedged(q);
1940	2386	}
1941	2387
@@ -1953,6 +2399,7 @@ static const struct xe_exec_queue_ops guc_exec_queue_ops = { · 11 unmodified lines
1953	2399	.set_priority = guc_exec_queue_set_priority,
1954	2400	.set_timeslice = guc_exec_queue_set_timeslice,
1955	2401	.set_preempt_timeout = guc_exec_queue_set_preempt_timeout,
	2402	+ .set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority,
1956	2403	.suspend = guc_exec_queue_suspend,
1957	2404	.suspend_wait = guc_exec_queue_suspend_wait,
1958	2405	.resume = guc_exec_queue_resume,
@@ -2202,6 +2649,22 @@ void xe_guc_submit_pause(struct xe_guc *guc) · 243 unmodified lines
2202	2649	struct xe_exec_queue *q;
2203	2650	unsigned long index;
2204	2651
	2652	+ mutex_lock(&guc->submission_state.lock);
	2653	+ xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
	2654	+ xe_sched_submission_stop(&q->guc->sched);
	2655	+ mutex_unlock(&guc->submission_state.lock);
	2656	+}
	2657	+
	2658	+/**
	2659	+ * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF.
	2660	+ * @guc: the &xe_guc struct instance whose scheduler is to be disabled
	2661	+ */
	2662	+void xe_guc_submit_pause_vf(struct xe_guc *guc)
	2663	+{
	2664	+ struct xe_exec_queue *q;
	2665	+ unsigned long index;
	2666	+
	2667	+ xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
2205	2668	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
2206	2669
2207	2670	mutex_lock(&guc->submission_state.lock);
@@ -2293,14 +2756,15 @@ static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, · 85 unmodified lines
2293	2756	}
2294	2757
2295	2758	/**
2296		− * xe_guc_submit_unpause_prepare - Prepare unpause submission tasks on given GuC.
	2759	+ * xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF.
2297	2760	* @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause
2298	2761	*/
2299		−void xe_guc_submit_unpause_prepare(struct xe_guc *guc)
	2762	+void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc)
2300	2763	{
2301	2764	struct xe_exec_queue *q;
2302	2765	unsigned long index;
2303	2766
	2767	+ xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
2304	2768	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
2305	2769
2306	2770	mutex_lock(&guc->submission_state.lock);
@@ -2376,6 +2840,23 @@ void xe_guc_submit_unpause(struct xe_guc *guc) · 69 unmodified lines
2376	2840	struct xe_exec_queue *q;
2377	2841	unsigned long index;
2378	2842
	2843	+ mutex_lock(&guc->submission_state.lock);
	2844	+ xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
	2845	+ xe_sched_submission_start(&q->guc->sched);
	2846	+ mutex_unlock(&guc->submission_state.lock);
	2847	+}
	2848	+
	2849	+/**
	2850	+ * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF.
	2851	+ * @guc: the &xe_guc struct instance whose scheduler is to be enabled
	2852	+ */
	2853	+void xe_guc_submit_unpause_vf(struct xe_guc *guc)
	2854	+{
	2855	+ struct xe_exec_queue *q;
	2856	+ unsigned long index;
	2857	+
	2858	+ xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
	2859	+
2379	2860	mutex_lock(&guc->submission_state.lock);
2380	2861	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
2381	2862	/*
@@ -2452,7 +2933,11 @@ static void deregister_exec_queue(struct xe_guc guc, struct xe_exec_queue q) · 70 unmodified lines
2452	2933
2453	2934	trace_xe_exec_queue_deregister(q);
2454	2935
2455		− xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));
	2936	+ if (xe_exec_queue_is_multi_queue_secondary(q))
	2937	+ handle_deregister_done(guc, q);
	2938	+ else
	2939	+ xe_guc_ct_send_g2h_handler(&guc->ct, action,
	2940	+ ARRAY_SIZE(action));
2456	2941	}
2457	2942
2458	2943	static void handle_sched_done(struct xe_guc guc, struct xe_exec_queue q,
@@ -2502,6 +2987,16 @@ static void handle_sched_done(struct xe_guc guc, struct xe_exec_queue q, · 43 unmodified lines
2502	2987	}
2503	2988	}
2504	2989
	2990	+static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
	2991	+ struct xe_exec_queue *q,
	2992	+ u32 runnable_state)
	2993	+{
	2994	+ /* Take CT lock here as handle_sched_done() do send a h2g message */
	2995	+ mutex_lock(&guc->ct.lock);
	2996	+ handle_sched_done(guc, q, runnable_state);
	2997	+ mutex_unlock(&guc->ct.lock);
	2998	+}
	2999	+
2505	3000	int xe_guc_sched_done_handler(struct xe_guc guc, u32 msg, u32 len)
2506	3001	{
2507	3002	struct xe_exec_queue *q;
@@ -2585,8 +3080,9 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc guc, u32 msg, u32 len) · 77 unmodified lines
2585	3080	if (unlikely(!q))
2586	3081	return -EPROTO;
2587	3082
2588		− xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d",
2589		− xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
	3083	+ xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x",
	3084	+ xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id,
	3085	+ atomic_read(&q->guc->state));
2590	3086
2591	3087	trace_xe_exec_queue_reset(q);
2592	3088
@@ -2596,9 +3092,7 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc guc, u32 msg, u32 len) · 3 unmodified lines
2596	3092	* jobs by setting timeout of the job to the minimum value kicking
2597	3093	* guc_exec_queue_timedout_job.
2598	3094	*/
2599		− set_exec_queue_reset(q);
2600		− if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
2601		− xe_guc_exec_queue_trigger_cleanup(q);
	3095	+ xe_guc_exec_queue_reset_trigger_cleanup(q);
2602	3096
2603	3097	return 0;
2604	3098	}
@@ -2666,20 +3160,18 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc guc, u32 msg, · 61 unmodified lines
2666	3160	* See bspec 54047 and 72187 for details.
2667	3161	*/
2668	3162	if (type != XE_GUC_CAT_ERR_TYPE_INVALID)
2669		− xe_gt_dbg(gt,
2670		− "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d",
2671		− type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
	3163	+ xe_gt_info(gt,
	3164	+ "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d",
	3165	+ type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
2672	3166	else
2673		− xe_gt_dbg(gt,
2674		− "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d",
2675		− xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
	3167	+ xe_gt_info(gt,
	3168	+ "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d",
	3169	+ xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
2676	3170
2677	3171	trace_xe_exec_queue_memory_cat_error(q);
2678	3172
2679	3173	/* Treat the same as engine reset */
2680		− set_exec_queue_reset(q);
2681		− if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
2682		− xe_guc_exec_queue_trigger_cleanup(q);
	3174	+ xe_guc_exec_queue_reset_trigger_cleanup(q);
2683	3175
2684	3176	return 0;
2685	3177	}
@@ -2706,6 +3198,73 @@ int xe_guc_exec_queue_reset_failure_handler(struct xe_guc guc, u32 msg, u32 le · 20 unmodified lines
2706	3198	return 0;
2707	3199	}
2708	3200
	3201	+int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc guc, u32 msg,
	3202	+ u32 len)
	3203	+{
	3204	+ struct xe_gt *gt = guc_to_gt(guc);
	3205	+ struct xe_device *xe = guc_to_xe(guc);
	3206	+ struct xe_exec_queue *q;
	3207	+ u32 guc_id = msg[2];
	3208	+
	3209	+ if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) {
	3210	+ drm_err(&xe->drm, "Invalid length %u", len);
	3211	+ return -EPROTO;
	3212	+ }
	3213	+
	3214	+ q = g2h_exec_queue_lookup(guc, guc_id);
	3215	+ if (unlikely(!q))
	3216	+ return -EPROTO;
	3217	+
	3218	+ xe_gt_dbg(gt,
	3219	+ "CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x",
	3220	+ msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]);
	3221	+
	3222	+ trace_xe_exec_queue_cgp_context_error(q);
	3223	+
	3224	+ /* Treat the same as engine reset */
	3225	+ xe_guc_exec_queue_reset_trigger_cleanup(q);
	3226	+
	3227	+ return 0;
	3228	+}
	3229	+
	3230	+/**
	3231	+ * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler
	3232	+ * @guc: guc
	3233	+ * @msg: message indicating CGP sync done
	3234	+ * @len: length of message
	3235	+ *
	3236	+ * Set multi queue group's sync_pending flag to false and wakeup anyone waiting
	3237	+ * for CGP synchronization to complete.
	3238	+ *
	3239	+ * Return: 0 on success, -EPROTO for malformed messages.
	3240	+ */
	3241	+int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc guc, u32 msg, u32 len)
	3242	+{
	3243	+ struct xe_device *xe = guc_to_xe(guc);
	3244	+ struct xe_exec_queue *q;
	3245	+ u32 guc_id = msg[0];
	3246	+
	3247	+ if (unlikely(len < 1)) {
	3248	+ drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len);
	3249	+ return -EPROTO;
	3250	+ }
	3251	+
	3252	+ q = g2h_exec_queue_lookup(guc, guc_id);
	3253	+ if (unlikely(!q))
	3254	+ return -EPROTO;
	3255	+
	3256	+ if (!xe_exec_queue_is_multi_queue_primary(q)) {
	3257	+ drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response");
	3258	+ return -EPROTO;
	3259	+ }
	3260	+
	3261	+ /* Wakeup the serialized cgp update wait */
	3262	+ WRITE_ONCE(q->multi_queue.group->sync_pending, false);
	3263	+ xe_guc_ct_wake_waiters(&guc->ct);
	3264	+
	3265	+ return 0;
	3266	+}
	3267	+
2709	3268	static void
2710	3269	guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q,
2711	3270	struct xe_guc_submit_exec_queue_snapshot *snapshot)
@@ -2805,6 +3364,11 @@ xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) · 93 unmodified lines
2805	3364	if (snapshot->parallel_execution)
2806	3365	guc_exec_queue_wq_snapshot_capture(q, snapshot);
2807	3366
	3367	+ if (xe_exec_queue_is_multi_queue(q)) {
	3368	+ snapshot->multi_queue.valid = true;
	3369	+ snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id;
	3370	+ snapshot->multi_queue.pos = q->multi_queue.pos;
	3371	+ }
2808	3372	spin_lock(&sched->base.job_list_lock);
2809	3373	snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list);
2810	3374	snapshot->pending_list = kmalloc_array(snapshot->pending_list_size,
@@ -2887,6 +3451,11 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps · 76 unmodified lines
2887	3451	if (snapshot->parallel_execution)
2888	3452	guc_exec_queue_wq_snapshot_print(snapshot, p);
2889	3453
	3454	+ if (snapshot->multi_queue.valid) {
	3455	+ drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary);
	3456	+ drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos);
	3457	+ }
	3458	+
2890	3459	for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size;
2891	3460	i++)
2892	3461	drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n",

drivers/gpu/drm/xe/xe_guc_submit.h+7−2

@@ -21,9 +21,11 @@ void xe_guc_submit_reset_wait(struct xe_guc *guc); · 20 unmodified lines
21	21	void xe_guc_submit_stop(struct xe_guc *guc);
22	22	int xe_guc_submit_start(struct xe_guc *guc);
23	23	void xe_guc_submit_pause(struct xe_guc *guc);
24		−void xe_guc_submit_unpause(struct xe_guc *guc);
25		−void xe_guc_submit_unpause_prepare(struct xe_guc *guc);
26	24	void xe_guc_submit_pause_abort(struct xe_guc *guc);
	25	+void xe_guc_submit_pause_vf(struct xe_guc *guc);
	26	+void xe_guc_submit_unpause(struct xe_guc *guc);
	27	+void xe_guc_submit_unpause_vf(struct xe_guc *guc);
	28	+void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc);
27	29	void xe_guc_submit_wedge(struct xe_guc *guc);
28	30
29	31	int xe_guc_read_stopped(struct xe_guc *guc);
@@ -34,6 +36,9 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc guc, u32 msg, · 4 unmodified lines
34	36	u32 len);
35	37	int xe_guc_exec_queue_reset_failure_handler(struct xe_guc guc, u32 msg, u32 len);
36	38	int xe_guc_error_capture_handler(struct xe_guc guc, u32 msg, u32 len);
	39	+int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc guc, u32 msg, u32 len);
	40	+int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc guc, u32 msg,
	41	+ u32 len);
37	42
38	43	struct xe_guc_submit_exec_queue_snapshot *
39	44	xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q);

drivers/gpu/drm/xe/xe_guc_submit_types.h+13−0

@@ -135,6 +135,19 @@ struct xe_guc_submit_exec_queue_snapshot { · 134 unmodified lines
135	135	u32 wq[WQ_SIZE / sizeof(u32)];
136	136	} parallel;
137	137
	138	+ /** @multi_queue: snapshot of the multi queue information */
	139	+ struct {
	140	+ /**
	141	+ * @multi_queue.primary: GuC id of the primary exec queue
	142	+ * of the multi queue group.
	143	+ */
	144	+ u32 primary;
	145	+ /** @multi_queue.pos: Position of the exec queue within the multi queue group */
	146	+ u8 pos;
	147	+ /** @valid: The exec queue is part of a multi queue group */
	148	+ bool valid;
	149	+ } multi_queue;
	150	+
138	151	/** @pending_list_size: Size of the pending list snapshot array */
139	152	int pending_list_size;
140	153	/** @pending_list: snapshot of the pending list info */

drivers/gpu/drm/xe/xe_guc_tlb_inval.c+31−10

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include "xe_guc_tlb_inval.h"
14	14	#include "xe_force_wake.h"
15	15	#include "xe_mmio.h"
	16	+#include "xe_sa.h"
16	17	#include "xe_tlb_inval.h"
17	18
18	19	#include "regs/xe_guc_regs.h"
@@ -34,9 +35,12 @@ static int send_tlb_inval(struct xe_guc guc, const u32 action, int len) · 15 unmodified lines
34	35	G2H_LEN_DW_TLB_INVALIDATE, 1);
35	36	}
36	37
37		−#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) \| \
	38	+#define MAKE_INVAL_OP_FLUSH(type, flush_cache) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) \| \
38	39	XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT \| \
39		− XE_GUC_TLB_INVAL_FLUSH_CACHE)
	40	+ (flush_cache ? \
	41	+ XE_GUC_TLB_INVAL_FLUSH_CACHE : 0))
	42	+
	43	+#define MAKE_INVAL_OP(type) MAKE_INVAL_OP_FLUSH(type, true)
40	44
41	45	static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno)
42	46	{
@@ -71,12 +75,11 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) · 28 unmodified lines
71	75	return send_tlb_inval(guc, action, ARRAY_SIZE(action));
72	76	} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
73	77	struct xe_mmio *mmio = &gt->mmio;
74		− unsigned int fw_ref;
75	78
76	79	if (IS_SRIOV_VF(xe))
77	80	return -ECANCELED;
78	81
79		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
	82	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
80	83	if (xe->info.platform == XE_PVC \|\| GRAPHICS_VER(xe) >= 20) {
81	84	xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1,
82	85	PVC_GUC_TLB_INV_DESC1_INVALIDATE);
@@ -86,12 +89,25 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) · 3 unmodified lines
86	89	xe_mmio_write32(mmio, GUC_TLB_INV_CR,
87	90	GUC_TLB_INV_CR_INVALIDATE);
88	91	}
89		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
90	92	}
91	93
92	94	return -ECANCELED;
93	95	}
94	96
	97	+static int send_page_reclaim(struct xe_guc *guc, u32 seqno,
	98	+ u64 gpu_addr)
	99	+{
	100	+ u32 action[] = {
	101	+ XE_GUC_ACTION_PAGE_RECLAMATION,
	102	+ seqno,
	103	+ lower_32_bits(gpu_addr),
	104	+ upper_32_bits(gpu_addr),
	105	+ };
	106	+
	107	+ return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
	108	+ G2H_LEN_DW_PAGE_RECLAMATION, 1);
	109	+}
	110	+
95	111	/*
96	112	* Ensure that roundup_pow_of_two(length) doesn't overflow.
97	113	* Note that roundup_pow_of_two() operates on unsigned long,
@@ -100,20 +116,21 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno) · 2 unmodified lines
100	116	#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))
101	117
102	118	static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
103		− u64 start, u64 end, u32 asid)
	119	+ u64 start, u64 end, u32 asid,
	120	+ struct drm_suballoc *prl_sa)
104	121	{
105	122	#define MAX_TLB_INVALIDATION_LEN 7
106	123	struct xe_guc *guc = tlb_inval->private;
107	124	struct xe_gt *gt = guc_to_gt(guc);
108	125	u32 action[MAX_TLB_INVALIDATION_LEN];
109	126	u64 length = end - start;
110		− int len = 0;
	127	+ int len = 0, err;
111	128
112	129	if (guc_to_xe(guc)->info.force_execlist)
113	130	return -ECANCELED;
114	131
115	132	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
116		− action[len++] = seqno;
	133	+ action[len++] = !prl_sa ? seqno : TLB_INVALIDATION_SEQNO_INVALID;
117	134	if (!gt_to_xe(gt)->info.has_range_tlb_inval \|\|
118	135	length > MAX_RANGE_TLB_INVALIDATION_LENGTH) {
119	136	action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
@@ -154,7 +171,8 @@ static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, · 34 unmodified lines
154	171	ilog2(SZ_2M) + 1)));
155	172	xe_gt_assert(gt, IS_ALIGNED(start, length));
156	173
157		− action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
	174	+ /* Flush on NULL case, Media is not required to modify flush due to no PPC so NOP */
	175	+ action[len++] = MAKE_INVAL_OP_FLUSH(XE_GUC_TLB_INVAL_PAGE_SELECTIVE, !prl_sa);
158	176	action[len++] = asid;
159	177	action[len++] = lower_32_bits(start);
160	178	action[len++] = upper_32_bits(start);
@@ -163,7 +181,10 @@ static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno, · 2 unmodified lines
163	181
164	182	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
165	183
166		− return send_tlb_inval(guc, action, len);
	184	+ err = send_tlb_inval(guc, action, len);
	185	+ if (!err && prl_sa)
	186	+ err = send_page_reclaim(guc, seqno, xe_sa_bo_gpu_addr(prl_sa));
	187	+ return err;
167	188	}
168	189
169	190	static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval)

drivers/gpu/drm/xe/xe_huc.c+2−5

@@ -300,19 +300,16 @@ void xe_huc_sanitize(struct xe_huc *huc) · 299 unmodified lines
300	300	void xe_huc_print_info(struct xe_huc huc, struct drm_printer p)
301	301	{
302	302	struct xe_gt *gt = huc_to_gt(huc);
303		− unsigned int fw_ref;
304	303
305	304	xe_uc_fw_print(&huc->fw, p);
306	305
307	306	if (!xe_uc_fw_is_enabled(&huc->fw))
308	307	return;
309	308
310		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
311		− if (!fw_ref)
	309	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	310	+ if (!fw_ref.domains)
312	311	return;
313	312
314	313	drm_printf(p, "\nHuC status: 0x%08x\n",
315	314	xe_mmio_read32(&gt->mmio, HUC_KERNEL_LOAD_INFO));
316		−
317		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
318	315	}

drivers/gpu/drm/xe/xe_huc_debugfs.c+1−2

@@ -37,9 +37,8 @@ static int huc_info(struct seq_file m, void data) · 36 unmodified lines
37	37	struct xe_device *xe = huc_to_xe(huc);
38	38	struct drm_printer p = drm_seq_file_printer(m);
39	39
40		− xe_pm_runtime_get(xe);
	40	+ guard(xe_pm_runtime)(xe);
41	41	xe_huc_print_info(huc, &p);
42		− xe_pm_runtime_put(xe);
43	42
44	43	return 0;
45	44	}

drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c+6−10

@@ -43,16 +43,14 @@ static ssize_t xe_hw_engine_class_sysfs_attr_show(struct kobject *kobj, · 42 unmodified lines
43	43	{
44	44	struct xe_device *xe = kobj_to_xe(kobj);
45	45	struct kobj_attribute *kattr;
46		− ssize_t ret = -EIO;
47	46
48	47	kattr = container_of(attr, struct kobj_attribute, attr);
49	48	if (kattr->show) {
50		− xe_pm_runtime_get(xe);
51		− ret = kattr->show(kobj, kattr, buf);
52		− xe_pm_runtime_put(xe);
	49	+ guard(xe_pm_runtime)(xe);
	50	+ return kattr->show(kobj, kattr, buf);
53	51	}
54	52
55		− return ret;
	53	+ return -EIO;
56	54	}
57	55
58	56	static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj,
@@ -62,16 +60,14 @@ static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj, · 3 unmodified lines
62	60	{
63	61	struct xe_device *xe = kobj_to_xe(kobj);
64	62	struct kobj_attribute *kattr;
65		− ssize_t ret = -EIO;
66	63
67	64	kattr = container_of(attr, struct kobj_attribute, attr);
68	65	if (kattr->store) {
69		− xe_pm_runtime_get(xe);
70		− ret = kattr->store(kobj, kattr, buf, count);
71		− xe_pm_runtime_put(xe);
	66	+ guard(xe_pm_runtime)(xe);
	67	+ return kattr->store(kobj, kattr, buf, count);
72	68	}
73	69
74		− return ret;
	70	+ return -EIO;
75	71	}
76	72
77	73	static const struct sysfs_ops xe_hw_engine_class_sysfs_ops = {

drivers/gpu/drm/xe/xe_hw_engine_group.c+71−8

@@ -9,7 +9,9 @@ · 8 unmodified lines
9	9	#include "xe_device.h"
10	10	#include "xe_exec_queue.h"
11	11	#include "xe_gt.h"
	12	+#include "xe_gt_stats.h"
12	13	#include "xe_hw_engine_group.h"
	14	+#include "xe_sync.h"
13	15	#include "xe_vm.h"
14	16
15	17	static void
@@ -20,7 +22,8 @@ hw_engine_group_resume_lr_jobs_func(struct work_struct *w) · 4 unmodified lines
20	22	int err;
21	23	enum xe_hw_engine_group_execution_mode previous_mode;
22	24
23		− err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode);
	25	+ err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode,
	26	+ NULL, 0);
24	27	if (err)
25	28	return;
26	29
@@ -188,23 +191,39 @@ void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group · 161 unmodified lines
188	191	/**
189	192	* xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group
190	193	* @group: The hw engine group
	194	+ * @has_deps: dma-fence job triggering suspend has dependencies
191	195	*
192	196	* Return: 0 on success, negative error code on error.
193	197	*/
194		−static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group)
	198	+static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group,
	199	+ bool has_deps)
195	200	{
196	201	int err;
197	202	struct xe_exec_queue *q;
	203	+ struct xe_gt *gt = NULL;
198	204	bool need_resume = false;
	205	+ ktime_t start = xe_gt_stats_ktime_get();
199	206
200	207	lockdep_assert_held_write(&group->mode_sem);
201	208
202	209	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
	210	+ bool idle_skip_suspend;
	211	+
203	212	if (!xe_vm_in_fault_mode(q->vm))
204	213	continue;
205	214
206		− need_resume = true;
	215	+ idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q);
	216	+ if (!idle_skip_suspend && has_deps)
	217	+ return -EAGAIN;
	218	+
	219	+ xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1);
	220	+ if (idle_skip_suspend)
	221	+ xe_gt_stats_incr(q->gt,
	222	+ XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, 1);
	223	+
	224	+ need_resume \|= !idle_skip_suspend;
207	225	q->ops->suspend(q);
	226	+ gt = q->gt;
208	227	}
209	228
210	229	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
@@ -216,6 +235,12 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group · 5 unmodified lines
216	235	return err;
217	236	}
218	237
	238	+ if (gt) {
	239	+ xe_gt_stats_incr(gt,
	240	+ XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
	241	+ xe_gt_stats_ktime_us_delta(start));
	242	+ }
	243	+
219	244	if (need_resume)
220	245	xe_hw_engine_group_resume_faulting_lr_jobs(group);
221	246
@@ -236,7 +261,9 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group · 14 unmodified lines
236	261	{
237	262	long timeout;
238	263	struct xe_exec_queue *q;
	264	+ struct xe_gt *gt = NULL;
239	265	struct dma_fence *fence;
	266	+ ktime_t start = xe_gt_stats_ktime_get();
240	267
241	268	lockdep_assert_held_write(&group->mode_sem);
242	269
@@ -244,18 +271,26 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group · 1 unmodified line
244	271	if (xe_vm_in_lr_mode(q->vm))
245	272	continue;
246	273
	274	+ xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1);
247	275	fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);
248	276	timeout = dma_fence_wait(fence, false);
249	277	dma_fence_put(fence);
	278	+ gt = q->gt;
250	279
251	280	if (timeout < 0)
252	281	return -ETIME;
253	282	}
254	283
	284	+ if (gt) {
	285	+ xe_gt_stats_incr(gt,
	286	+ XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
	287	+ xe_gt_stats_ktime_us_delta(start));
	288	+ }
	289	+
255	290	return 0;
256	291	}
257	292
258		−static int switch_mode(struct xe_hw_engine_group *group)
	293	+static int switch_mode(struct xe_hw_engine_group *group, bool has_deps)
259	294	{
260	295	int err = 0;
261	296	enum xe_hw_engine_group_execution_mode new_mode;
@@ -265,7 +300,8 @@ static int switch_mode(struct xe_hw_engine_group *group) · 3 unmodified lines
265	300	switch (group->cur_mode) {
266	301	case EXEC_MODE_LR:
267	302	new_mode = EXEC_MODE_DMA_FENCE;
268		− err = xe_hw_engine_group_suspend_faulting_lr_jobs(group);
	303	+ err = xe_hw_engine_group_suspend_faulting_lr_jobs(group,
	304	+ has_deps);
269	305	break;
270	306	case EXEC_MODE_DMA_FENCE:
271	307	new_mode = EXEC_MODE_LR;
@@ -281,19 +317,36 @@ static int switch_mode(struct xe_hw_engine_group *group) · 9 unmodified lines
281	317	return 0;
282	318	}
283	319
	320	+static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs)
	321	+{
	322	+ int err, i;
	323	+
	324	+ for (i = 0; i < num_syncs; ++i) {
	325	+ err = xe_sync_entry_wait(syncs + i);
	326	+ if (err)
	327	+ return err;
	328	+ }
	329	+
	330	+ return 0;
	331	+}
	332	+
284	333	/**
285	334	* xe_hw_engine_group_get_mode() - Get the group to execute in the new mode
286	335	* @group: The hw engine group
287	336	* @new_mode: The new execution mode
288	337	* @previous_mode: Pointer to the previous mode provided for use by caller
	338	+ * @syncs: Syncs from exec IOCTL
	339	+ * @num_syncs: Number of syncs from exec IOCTL
289	340	*
290	341	* Return: 0 if successful, -EINTR if locking failed.
291	342	*/
292	343	int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
293	344	enum xe_hw_engine_group_execution_mode new_mode,
294		− enum xe_hw_engine_group_execution_mode *previous_mode)
	345	+ enum xe_hw_engine_group_execution_mode *previous_mode,
	346	+ struct xe_sync_entry *syncs, int num_syncs)
295	347	__acquires(&group->mode_sem)
296	348	{
	349	+ bool has_deps = !!num_syncs;
297	350	int err = down_read_interruptible(&group->mode_sem);
298	351
299	352	if (err)
@@ -303,15 +356,25 @@ __acquires(&group->mode_sem) · 3 unmodified lines
303	356
304	357	if (new_mode != group->cur_mode) {
305	358	up_read(&group->mode_sem);
	359	+retry:
306	360	err = down_write_killable(&group->mode_sem);
307	361	if (err)
308	362	return err;
309	363
310	364	if (new_mode != group->cur_mode) {
311		− err = switch_mode(group);
	365	+ err = switch_mode(group, has_deps);
312	366	if (err) {
313	367	up_write(&group->mode_sem);
314		− return err;
	368	+
	369	+ if (err != -EAGAIN)
	370	+ return err;
	371	+
	372	+ err = wait_syncs(syncs, num_syncs);
	373	+ if (err)
	374	+ return err;
	375	+
	376	+ has_deps = false;
	377	+ goto retry;
315	378	}
316	379	}
317	380	downgrade_write(&group->mode_sem);

drivers/gpu/drm/xe/xe_hw_engine_group.h+3−1

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	struct drm_device;
12	12	struct xe_exec_queue;
13	13	struct xe_gt;
	14	+struct xe_sync_entry;
14	15
15	16	int xe_hw_engine_setup_groups(struct xe_gt *gt);
16	17
@@ -19,7 +20,8 @@ void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct · 2 unmodified lines
19	20
20	21	int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
21	22	enum xe_hw_engine_group_execution_mode new_mode,
22		− enum xe_hw_engine_group_execution_mode *previous_mode);
	23	+ enum xe_hw_engine_group_execution_mode *previous_mode,
	24	+ struct xe_sync_entry *syncs, int num_syncs);
23	25	void xe_hw_engine_group_put(struct xe_hw_engine_group *group);
24	26
25	27	enum xe_hw_engine_group_execution_mode

drivers/gpu/drm/xe/xe_hwmon.c+14−38

@@ -502,7 +502,7 @@ xe_hwmon_power_max_interval_show(struct device dev, struct device_attribute at · 501 unmodified lines
502	502
503	503	int ret = 0;
504	504
505		− xe_pm_runtime_get(hwmon->xe);
	505	+ guard(xe_pm_runtime)(hwmon->xe);
506	506
507	507	mutex_lock(&hwmon->hwmon_lock);
508	508
@@ -521,8 +521,6 @@ xe_hwmon_power_max_interval_show(struct device dev, struct device_attribute at · 12 unmodified lines
521	521
522	522	mutex_unlock(&hwmon->hwmon_lock);
523	523
524		− xe_pm_runtime_put(hwmon->xe);
525		−
526	524	x = REG_FIELD_GET(PWR_LIM_TIME_X, reg_val);
527	525	y = REG_FIELD_GET(PWR_LIM_TIME_Y, reg_val);
528	526
@@ -604,7 +602,7 @@ xe_hwmon_power_max_interval_store(struct device dev, struct device_attribute a · 75 unmodified lines
604	602	rxy = REG_FIELD_PREP(PWR_LIM_TIME_X, x) \|
605	603	REG_FIELD_PREP(PWR_LIM_TIME_Y, y);
606	604
607		− xe_pm_runtime_get(hwmon->xe);
	605	+ guard(xe_pm_runtime)(hwmon->xe);
608	606
609	607	mutex_lock(&hwmon->hwmon_lock);
610	608
@@ -616,8 +614,6 @@ xe_hwmon_power_max_interval_store(struct device dev, struct device_attribute a · 5 unmodified lines
616	614
617	615	mutex_unlock(&hwmon->hwmon_lock);
618	616
619		− xe_pm_runtime_put(hwmon->xe);
620		−
621	617	return count;
622	618	}
623	619
@@ -1124,37 +1120,25 @@ xe_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, · 500 unmodified lines
1124	1120	int channel, long *val)
1125	1121	{
1126	1122	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
1127		− int ret;
1128	1123
1129		− xe_pm_runtime_get(hwmon->xe);
	1124	+ guard(xe_pm_runtime)(hwmon->xe);
1130	1125
1131	1126	switch (type) {
1132	1127	case hwmon_temp:
1133		− ret = xe_hwmon_temp_read(hwmon, attr, channel, val);
1134		− break;
	1128	+ return xe_hwmon_temp_read(hwmon, attr, channel, val);
1135	1129	case hwmon_power:
1136		− ret = xe_hwmon_power_read(hwmon, attr, channel, val);
1137		− break;
	1130	+ return xe_hwmon_power_read(hwmon, attr, channel, val);
1138	1131	case hwmon_curr:
1139		− ret = xe_hwmon_curr_read(hwmon, attr, channel, val);
1140		− break;
	1132	+ return xe_hwmon_curr_read(hwmon, attr, channel, val);
1141	1133	case hwmon_in:
1142		− ret = xe_hwmon_in_read(hwmon, attr, channel, val);
1143		− break;
	1134	+ return xe_hwmon_in_read(hwmon, attr, channel, val);
1144	1135	case hwmon_energy:
1145		− ret = xe_hwmon_energy_read(hwmon, attr, channel, val);
1146		− break;
	1136	+ return xe_hwmon_energy_read(hwmon, attr, channel, val);
1147	1137	case hwmon_fan:
1148		− ret = xe_hwmon_fan_read(hwmon, attr, channel, val);
1149		− break;
	1138	+ return xe_hwmon_fan_read(hwmon, attr, channel, val);
1150	1139	default:
1151		− ret = -EOPNOTSUPP;
1152		− break;
	1140	+ return -EOPNOTSUPP;
1153	1141	}
1154		−
1155		− xe_pm_runtime_put(hwmon->xe);
1156		−
1157		− return ret;
1158	1142	}
1159	1143
1160	1144	static int
@@ -1162,25 +1146,17 @@ xe_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, · 1 unmodified line
1162	1146	int channel, long val)
1163	1147	{
1164	1148	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
1165		− int ret;
1166	1149
1167		− xe_pm_runtime_get(hwmon->xe);
	1150	+ guard(xe_pm_runtime)(hwmon->xe);
1168	1151
1169	1152	switch (type) {
1170	1153	case hwmon_power:
1171		− ret = xe_hwmon_power_write(hwmon, attr, channel, val);
1172		− break;
	1154	+ return xe_hwmon_power_write(hwmon, attr, channel, val);
1173	1155	case hwmon_curr:
1174		− ret = xe_hwmon_curr_write(hwmon, attr, channel, val);
1175		− break;
	1156	+ return xe_hwmon_curr_write(hwmon, attr, channel, val);
1176	1157	default:
1177		− ret = -EOPNOTSUPP;
1178		− break;
	1158	+ return -EOPNOTSUPP;
1179	1159	}
1180		−
1181		− xe_pm_runtime_put(hwmon->xe);
1182		−
1183		− return ret;
1184	1160	}
1185	1161
1186	1162	static int xe_hwmon_read_label(struct device *dev,

drivers/gpu/drm/xe/xe_i2c.c+1−1

@@ -319,7 +319,7 @@ int xe_i2c_probe(struct xe_device *xe) · 318 unmodified lines
319	319	struct xe_i2c *i2c;
320	320	int ret;
321	321
322		− if (xe->info.platform != XE_BATTLEMAGE)
	322	+ if (!xe->info.has_i2c)
323	323	return 0;
324	324
325	325	if (IS_SRIOV_VF(xe))

drivers/gpu/drm/xe/xe_irq.c+2−0

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#include "xe_hw_error.h"
22	22	#include "xe_i2c.h"
23	23	#include "xe_memirq.h"
	24	+#include "xe_mert.h"
24	25	#include "xe_mmio.h"
25	26	#include "xe_pxp.h"
26	27	#include "xe_sriov.h"
@@ -525,6 +526,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) · 498 unmodified lines
525	526	xe_heci_csc_irq_handler(xe, master_ctl);
526	527	xe_display_irq_handler(xe, master_ctl);
527	528	xe_i2c_irq_handler(xe, master_ctl);
	529	+ xe_mert_irq_handler(xe, master_ctl);
528	530	gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
529	531	}
530	532	}

drivers/gpu/drm/xe/xe_lmtt.c+22−3

@@ -8,15 +8,18 @@ · 7 unmodified lines
8	8	#include <drm/drm_managed.h>
9	9
10	10	#include "regs/xe_gt_regs.h"
	11	+#include "regs/xe_mert_regs.h"
11	12
12	13	#include "xe_assert.h"
13	14	#include "xe_bo.h"
14	15	#include "xe_tlb_inval.h"
15	16	#include "xe_lmtt.h"
16	17	#include "xe_map.h"
	18	+#include "xe_mert.h"
17	19	#include "xe_mmio.h"
18	20	#include "xe_res_cursor.h"
19	21	#include "xe_sriov.h"
	22	+#include "xe_tile.h"
20	23	#include "xe_tile_sriov_printk.h"
21	24
22	25	/**
@@ -196,16 +199,22 @@ static void lmtt_setup_dir_ptr(struct xe_lmtt *lmtt) · 173 unmodified lines
196	199	struct xe_device *xe = tile_to_xe(tile);
197	200	dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE);
198	201	struct xe_gt *gt;
	202	+ u32 config;
199	203	u8 id;
200	204
201	205	lmtt_debug(lmtt, "DIR offset %pad\n", &offset);
202	206	lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo));
203	207	lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K));
204	208
	209	+ config = LMEM_EN \| REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K);
	210	+
205	211	for_each_gt_on_tile(gt, tile, id)
206	212	xe_mmio_write32(&gt->mmio,
207	213	GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG,
208		− LMEM_EN \| REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K));
	214	+ config);
	215	+
	216	+ if (xe_device_has_mert(xe) && xe_tile_is_root(tile))
	217	+ xe_mmio_write32(&tile->mmio, MERT_LMEM_CFG, config);
209	218	}
210	219
211	220	/**
@@ -262,19 +271,29 @@ static int lmtt_invalidate_hw(struct xe_lmtt *lmtt) · 50 unmodified lines
262	271	* @lmtt: the &xe_lmtt to invalidate
263	272	*
264	273	* Send requests to all GuCs on this tile to invalidate all TLBs.
	274	+ * If the platform has a standalone MERT, also invalidate MERT's TLB.
265	275	*
266	276	* This function should be called only when running as a PF driver.
267	277	*/
268	278	void xe_lmtt_invalidate_hw(struct xe_lmtt *lmtt)
269	279	{
	280	+ struct xe_tile *tile = lmtt_to_tile(lmtt);
	281	+ struct xe_device *xe = lmtt_to_xe(lmtt);
270	282	int err;
271	283
272		− lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt)));
	284	+ lmtt_assert(lmtt, IS_SRIOV_PF(xe));
273	285
274	286	err = lmtt_invalidate_hw(lmtt);
275	287	if (err)
276		− xe_tile_sriov_err(lmtt_to_tile(lmtt), "LMTT invalidation failed (%pe)",
	288	+ xe_tile_sriov_err(tile, "LMTT invalidation failed (%pe)",
277	289	ERR_PTR(err));
	290	+
	291	+ if (xe_device_has_mert(xe) && xe_tile_is_root(tile)) {
	292	+ err = xe_mert_invalidate_lmtt(tile);
	293	+ if (err)
	294	+ xe_tile_sriov_err(tile, "MERT LMTT invalidation failed (%pe)",
	295	+ ERR_PTR(err));
	296	+ }
278	297	}
279	298
280	299	static void lmtt_write_pte(struct xe_lmtt lmtt, struct xe_lmtt_pt pt,

drivers/gpu/drm/xe/xe_lrc.c+63−10

@@ -44,6 +44,11 @@ · 43 unmodified lines
44	44	#define LRC_INDIRECT_CTX_BO_SIZE SZ_4K
45	45	#define LRC_INDIRECT_RING_STATE_SIZE SZ_4K
46	46
	47	+#define LRC_PRIORITY GENMASK_ULL(10, 9)
	48	+#define LRC_PRIORITY_LOW 0
	49	+#define LRC_PRIORITY_NORMAL 1
	50	+#define LRC_PRIORITY_HIGH 2
	51	+
47	52	/*
48	53	* Layout of the LRC and associated data allocated as
49	54	* lrc->bo:
@@ -91,13 +96,19 @@ gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class) · 41 unmodified lines
91	96	return false;
92	97	}
93	98
94		−size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class)
	99	+/**
	100	+ * xe_gt_lrc_hang_replay_size() - Hang replay size
	101	+ * @gt: The GT
	102	+ * @class: Hardware engine class
	103	+ *
	104	+ * Determine size of GPU hang replay state for a GT and hardware engine class.
	105	+ *
	106	+ * Return: Size of GPU hang replay size
	107	+ */
	108	+size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class)
95	109	{
96	110	struct xe_device *xe = gt_to_xe(gt);
97		− size_t size;
98		−
99		− /* Per-process HW status page (PPHWSP) */
100		− size = LRC_PPHWSP_SIZE;
	111	+ size_t size = 0;
101	112
102	113	/* Engine context image */
103	114	switch (class) {
@@ -123,11 +134,18 @@ size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class) · 19 unmodified lines
123	134	size += 1 * SZ_4K;
124	135	}
125	136
	137	+ return size;
	138	+}
	139	+
	140	+size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class)
	141	+{
	142	+ size_t size = xe_gt_lrc_hang_replay_size(gt, class);
	143	+
126	144	/* Add indirect ring state page */
127	145	if (xe_gt_has_indirect_ring_state(gt))
128	146	size += LRC_INDIRECT_RING_STATE_SIZE;
129	147
130		− return size;
	148	+ return size + LRC_PPHWSP_SIZE;
131	149	}
132	150
133	151	/*
@@ -1386,8 +1404,33 @@ setup_indirect_ctx(struct xe_lrc lrc, struct xe_hw_engine hwe) · 1252 unmodified lines
1386	1404	return 0;
1387	1405	}
1388	1406
	1407	+static u8 xe_multi_queue_prio_to_lrc(struct xe_lrc *lrc, enum xe_multi_queue_priority priority)
	1408	+{
	1409	+ struct xe_device *xe = gt_to_xe(lrc->gt);
	1410	+
	1411	+ xe_assert(xe, (priority >= XE_MULTI_QUEUE_PRIORITY_LOW &&
	1412	+ priority <= XE_MULTI_QUEUE_PRIORITY_HIGH));
	1413	+
	1414	+ /* xe_multi_queue_priority is directly mapped to LRC priority values */
	1415	+ return priority;
	1416	+}
	1417	+
	1418	+/**
	1419	+ * xe_lrc_set_multi_queue_priority() - Set multi queue priority in LRC
	1420	+ * @lrc: Logical Ring Context
	1421	+ * @priority: Multi queue priority of the exec queue
	1422	+ *
	1423	+ * Convert @priority to LRC multi queue priority and update the @lrc descriptor
	1424	+ */
	1425	+void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority)
	1426	+{
	1427	+ lrc->desc &= ~LRC_PRIORITY;
	1428	+ lrc->desc \|= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority));
	1429	+}
	1430	+
1389	1431	static int xe_lrc_init(struct xe_lrc lrc, struct xe_hw_engine hwe,
1390		− struct xe_vm *vm, u32 ring_size, u16 msix_vec,
	1432	+ struct xe_vm vm, void replay_state, u32 ring_size,
	1433	+ u16 msix_vec,
1391	1434	u32 init_flags)
1392	1435	{
1393	1436	struct xe_gt *gt = hwe->gt;
@@ -1402,6 +1445,7 @@ static int xe_lrc_init(struct xe_lrc lrc, struct xe_hw_engine hwe, · 8 unmodified lines
1402	1445
1403	1446	kref_init(&lrc->refcount);
1404	1447	lrc->gt = gt;
	1448	+ lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class);
1405	1449	lrc->size = lrc_size;
1406	1450	lrc->flags = 0;
1407	1451	lrc->ring.size = ring_size;
@@ -1438,11 +1482,14 @@ static int xe_lrc_init(struct xe_lrc lrc, struct xe_hw_engine hwe, · 30 unmodified lines
1438	1482	* scratch.
1439	1483	*/
1440	1484	map = __xe_lrc_pphwsp_map(lrc);
1441		− if (gt->default_lrc[hwe->class]) {
	1485	+ if (gt->default_lrc[hwe->class] \|\| replay_state) {
1442	1486	xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */
1443	1487	xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
1444	1488	gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE,
1445	1489	lrc_size - LRC_PPHWSP_SIZE);
	1490	+ if (replay_state)
	1491	+ xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
	1492	+ replay_state, lrc->replay_size);
1446	1493	} else {
1447	1494	void *init_data = empty_lrc_data(hwe);
1448	1495
@@ -1550,6 +1597,7 @@ err_lrc_finish: · 101 unmodified lines
1550	1597	* xe_lrc_create - Create a LRC
1551	1598	* @hwe: Hardware Engine
1552	1599	* @vm: The VM (address space)
	1600	+ * @replay_state: GPU hang replay state
1553	1601	* @ring_size: LRC ring size
1554	1602	* @msix_vec: MSI-X interrupt vector (for platforms that support it)
1555	1603	* @flags: LRC initialization flags
@@ -1560,7 +1608,7 @@ err_lrc_finish: · 4 unmodified lines
1560	1608	* upon failure.
1561	1609	*/
1562	1610	struct xe_lrc xe_lrc_create(struct xe_hw_engine hwe, struct xe_vm *vm,
1563		− u32 ring_size, u16 msix_vec, u32 flags)
	1611	+ void *replay_state, u32 ring_size, u16 msix_vec, u32 flags)
1564	1612	{
1565	1613	struct xe_lrc *lrc;
1566	1614	int err;
@@ -1569,7 +1617,7 @@ struct xe_lrc xe_lrc_create(struct xe_hw_engine hwe, struct xe_vm *vm, · 2 unmodified lines
1569	1617	if (!lrc)
1570	1618	return ERR_PTR(-ENOMEM);
1571	1619
1572		− err = xe_lrc_init(lrc, hwe, vm, ring_size, msix_vec, flags);
	1620	+ err = xe_lrc_init(lrc, hwe, vm, replay_state, ring_size, msix_vec, flags);
1573	1621	if (err) {
1574	1622	kfree(lrc);
1575	1623	return ERR_PTR(err);
@@ -2235,6 +2283,8 @@ struct xe_lrc_snapshot xe_lrc_snapshot_capture(struct xe_lrc lrc) · 659 unmodified lines
2235	2283	snapshot->lrc_bo = xe_bo_get(lrc->bo);
2236	2284	snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc);
2237	2285	snapshot->lrc_size = lrc->size;
	2286	+ snapshot->replay_offset = 0;
	2287	+ snapshot->replay_size = lrc->replay_size;
2238	2288	snapshot->lrc_snapshot = NULL;
2239	2289	snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc));
2240	2290	snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc);
@@ -2305,6 +2355,9 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer · 64 unmodified lines
2305	2355	}
2306	2356
2307	2357	drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE);
	2358	+ drm_printf(p, "\n\t[HWCTX].replay_offset: 0x%lx\n", snapshot->replay_offset);
	2359	+ drm_printf(p, "\n\t[HWCTX].replay_length: 0x%lx\n", snapshot->replay_size);
	2360	+
2308	2361	drm_puts(p, "\t[HWCTX].data: ");
2309	2362	for (; i < snapshot->lrc_size; i += sizeof(u32)) {
2310	2363	u32 *val = snapshot->lrc_snapshot + i;

drivers/gpu/drm/xe/xe_lrc.h+6−1

@@ -13,6 +13,7 @@ struct drm_printer; · 12 unmodified lines
13	13	struct xe_bb;
14	14	struct xe_device;
15	15	struct xe_exec_queue;
	16	+enum xe_multi_queue_priority;
16	17	enum xe_engine_class;
17	18	struct xe_gt;
18	19	struct xe_hw_engine;
@@ -23,6 +24,7 @@ struct xe_lrc_snapshot { · 4 unmodified lines
23	24	struct xe_bo *lrc_bo;
24	25	void *lrc_snapshot;
25	26	unsigned long lrc_size, lrc_offset;
	27	+ unsigned long replay_size, replay_offset;
26	28
27	29	u32 context_desc;
28	30	u32 ring_addr;
@@ -49,7 +51,7 @@ struct xe_lrc_snapshot { · 20 unmodified lines
49	51	#define XE_LRC_CREATE_USER_CTX BIT(2)
50	52
51	53	struct xe_lrc xe_lrc_create(struct xe_hw_engine hwe, struct xe_vm *vm,
52		− u32 ring_size, u16 msix_vec, u32 flags);
	54	+ void *replay_state, u32 ring_size, u16 msix_vec, u32 flags);
53	55	void xe_lrc_destroy(struct kref *ref);
54	56
55	57	/**
@@ -86,6 +88,7 @@ static inline size_t xe_lrc_ring_size(void) · 30 unmodified lines
86	88	return SZ_16K;
87	89	}
88	90
	91	+size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class);
89	92	size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class);
90	93	u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc);
91	94	u32 xe_lrc_regs_offset(struct xe_lrc *lrc);
@@ -133,6 +136,8 @@ void xe_lrc_dump_default(struct drm_printer *p, · 41 unmodified lines
133	136
134	137	u32 xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue q, u32 *cs);
135	138
	139	+void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority);
	140	+
136	141	struct xe_lrc_snapshot xe_lrc_snapshot_capture(struct xe_lrc lrc);
137	142	void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot);
138	143	void xe_lrc_snapshot_print(struct xe_lrc_snapshot snapshot, struct drm_printer p);

drivers/gpu/drm/xe/xe_lrc_types.h+3−0

@@ -25,6 +25,9 @@ struct xe_lrc { · 24 unmodified lines
25	25	/** @size: size of the lrc and optional indirect ring state */
26	26	u32 size;
27	27
	28	+ /** @replay_size: Size LRC needed for replaying a hang */
	29	+ u32 replay_size;
	30	+
28	31	/** @gt: gt which this LRC belongs to */
29	32	struct xe_gt *gt;
30	33

drivers/gpu/drm/xe/xe_mert.c+82−0

@@ -0,0 +1,82 @@
	1	+// SPDX-License-Identifier: MIT
	2	+/*
	3	+ * Copyright(c) 2025, Intel Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#include "regs/xe_irq_regs.h"
	7	+#include "regs/xe_mert_regs.h"
	8	+
	9	+#include "xe_device.h"
	10	+#include "xe_mert.h"
	11	+#include "xe_mmio.h"
	12	+#include "xe_tile.h"
	13	+
	14	+/**
	15	+ * xe_mert_invalidate_lmtt - Invalidate MERT LMTT
	16	+ * @tile: the &xe_tile
	17	+ *
	18	+ * Trigger invalidation of the MERT LMTT and wait for completion.
	19	+ *
	20	+ * Return: 0 on success or -ETIMEDOUT in case of a timeout.
	21	+ */
	22	+int xe_mert_invalidate_lmtt(struct xe_tile *tile)
	23	+{
	24	+ struct xe_device *xe = tile_to_xe(tile);
	25	+ struct xe_mert *mert = &tile->mert;
	26	+ const long timeout = HZ / 4;
	27	+ unsigned long flags;
	28	+
	29	+ xe_assert(xe, xe_device_has_mert(xe));
	30	+ xe_assert(xe, xe_tile_is_root(tile));
	31	+
	32	+ spin_lock_irqsave(&mert->lock, flags);
	33	+ if (!mert->tlb_inv_triggered) {
	34	+ mert->tlb_inv_triggered = true;
	35	+ reinit_completion(&mert->tlb_inv_done);
	36	+ xe_mmio_write32(&tile->mmio, MERT_TLB_INV_DESC_A, MERT_TLB_INV_DESC_A_VALID);
	37	+ }
	38	+ spin_unlock_irqrestore(&mert->lock, flags);
	39	+
	40	+ if (!wait_for_completion_timeout(&mert->tlb_inv_done, timeout))
	41	+ return -ETIMEDOUT;
	42	+
	43	+ return 0;
	44	+}
	45	+
	46	+/**
	47	+ * xe_mert_irq_handler - Handler for MERT interrupts
	48	+ * @xe: the &xe_device
	49	+ * @master_ctl: interrupt register
	50	+ *
	51	+ * Handle interrupts generated by MERT.
	52	+ */
	53	+void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl)
	54	+{
	55	+ struct xe_tile *tile = xe_device_get_root_tile(xe);
	56	+ unsigned long flags;
	57	+ u32 reg_val;
	58	+ u8 err;
	59	+
	60	+ if (!(master_ctl & SOC_H2DMEMINT_IRQ))
	61	+ return;
	62	+
	63	+ reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT);
	64	+ xe_mmio_write32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT, 0);
	65	+
	66	+ err = FIELD_GET(MERT_TLB_CT_ERROR_MASK, reg_val);
	67	+ if (err == MERT_TLB_CT_LMTT_FAULT)
	68	+ drm_dbg(&xe->drm, "MERT catastrophic error: LMTT fault (VF%u)\n",
	69	+ FIELD_GET(MERT_TLB_CT_VFID_MASK, reg_val));
	70	+ else if (err)
	71	+ drm_dbg(&xe->drm, "MERT catastrophic error: Unexpected fault (0x%x)\n", err);
	72	+
	73	+ spin_lock_irqsave(&tile->mert.lock, flags);
	74	+ if (tile->mert.tlb_inv_triggered) {
	75	+ reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_INV_DESC_A);
	76	+ if (!(reg_val & MERT_TLB_INV_DESC_A_VALID)) {
	77	+ tile->mert.tlb_inv_triggered = false;
	78	+ complete_all(&tile->mert.tlb_inv_done);
	79	+ }
	80	+ }
	81	+ spin_unlock_irqrestore(&tile->mert.lock, flags);
	82	+}

drivers/gpu/drm/xe/xe_mert.h+32−0

@@ -0,0 +1,32 @@
	1	+/* SPDX-License-Identifier: MIT */
	2	+/*
	3	+ * Copyright(c) 2025, Intel Corporation. All rights reserved.
	4	+ */
	5	+
	6	+#ifndef __XE_MERT_H__
	7	+#define __XE_MERT_H__
	8	+
	9	+#include <linux/completion.h>
	10	+#include <linux/spinlock.h>
	11	+#include <linux/types.h>
	12	+
	13	+struct xe_device;
	14	+struct xe_tile;
	15	+
	16	+struct xe_mert {
	17	+ /** @lock: protects the TLB invalidation status */
	18	+ spinlock_t lock;
	19	+ /** @tlb_inv_triggered: indicates if TLB invalidation was triggered */
	20	+ bool tlb_inv_triggered;
	21	+ /** @mert.tlb_inv_done: completion of TLB invalidation */
	22	+ struct completion tlb_inv_done;
	23	+};
	24	+
	25	+#ifdef CONFIG_PCI_IOV
	26	+int xe_mert_invalidate_lmtt(struct xe_tile *tile);
	27	+void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl);
	28	+#else
	29	+static inline void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl) { }
	30	+#endif
	31	+
	32	+#endif /* __XE_MERT_H__ */

drivers/gpu/drm/xe/xe_migrate.c+54−3

@@ -34,6 +34,7 @@ · 33 unmodified lines
34	34	#include "xe_res_cursor.h"
35	35	#include "xe_sa.h"
36	36	#include "xe_sched_job.h"
	37	+#include "xe_sriov_vf_ccs.h"
37	38	#include "xe_sync.h"
38	39	#include "xe_trace_bo.h"
39	40	#include "xe_validation.h"
@@ -1103,12 +1104,16 @@ int xe_migrate_ccs_rw_copy(struct xe_tile tile, struct xe_exec_queue q, · 1063 unmodified lines
1103	1104	u32 batch_size, batch_size_allocated;
1104	1105	struct xe_device *xe = gt_to_xe(gt);
1105	1106	struct xe_res_cursor src_it, ccs_it;
	1107	+ struct xe_sriov_vf_ccs_ctx *ctx;
	1108	+ struct xe_sa_manager *bb_pool;
1106	1109	u64 size = xe_bo_size(src_bo);
1107	1110	struct xe_bb *bb = NULL;
1108	1111	u64 src_L0, src_L0_ofs;
1109	1112	u32 src_L0_pt;
1110	1113	int err;
1111	1114
	1115	+ ctx = &xe->sriov.vf.ccs.contexts[read_write];
	1116	+
1112	1117	xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
1113	1118
1114	1119	xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),
@@ -1141,11 +1146,15 @@ int xe_migrate_ccs_rw_copy(struct xe_tile tile, struct xe_exec_queue q, · 26 unmodified lines
1141	1146	size -= src_L0;
1142	1147	}
1143	1148
	1149	+ bb_pool = ctx->mem.ccs_bb_pool;
	1150	+ guard(mutex) (xe_sa_bo_swap_guard(bb_pool));
	1151	+ xe_sa_bo_swap_shadow(bb_pool);
	1152	+
1144	1153	bb = xe_bb_ccs_new(gt, batch_size, read_write);
1145	1154	if (IS_ERR(bb)) {
1146	1155	drm_err(&xe->drm, "BB allocation failed.\n");
1147	1156	err = PTR_ERR(bb);
1148		− goto err_ret;
	1157	+ return err;
1149	1158	}
1150	1159
1151	1160	batch_size_allocated = batch_size;
@@ -1194,10 +1203,52 @@ int xe_migrate_ccs_rw_copy(struct xe_tile tile, struct xe_exec_queue q, · 42 unmodified lines
1194	1203	xe_assert(xe, (batch_size_allocated == bb->len));
1195	1204	src_bo->bb_ccs[read_write] = bb;
1196	1205
	1206	+ xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
	1207	+ xe_sa_bo_sync_shadow(bb->bo);
1197	1208	return 0;
	1209	+}
1198	1210
1199		−err_ret:
1200		− return err;
	1211	+/**
	1212	+ * xe_migrate_ccs_rw_copy_clear() - Clear the CCS read/write batch buffer
	1213	+ * content.
	1214	+ * @src_bo: The buffer object @src is currently bound to.
	1215	+ * @read_write : Creates BB commands for CCS read/write.
	1216	+ *
	1217	+ * Directly clearing the BB lacks atomicity and can lead to undefined
	1218	+ * behavior if the vCPU is halted mid-operation during the clearing
	1219	+ * process. To avoid this issue, we use a shadow buffer object approach.
	1220	+ *
	1221	+ * First swap the SA BO address with the shadow BO, perform the clearing
	1222	+ * operation on the BB, update the shadow BO in the ring buffer, then
	1223	+ * sync the shadow and the actual buffer to maintain consistency.
	1224	+ *
	1225	+ * Returns: None.
	1226	+ */
	1227	+void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
	1228	+ enum xe_sriov_vf_ccs_rw_ctxs read_write)
	1229	+{
	1230	+ struct xe_bb *bb = src_bo->bb_ccs[read_write];
	1231	+ struct xe_device *xe = xe_bo_device(src_bo);
	1232	+ struct xe_sriov_vf_ccs_ctx *ctx;
	1233	+ struct xe_sa_manager *bb_pool;
	1234	+ u32 *cs;
	1235	+
	1236	+ xe_assert(xe, IS_SRIOV_VF(xe));
	1237	+
	1238	+ ctx = &xe->sriov.vf.ccs.contexts[read_write];
	1239	+ bb_pool = ctx->mem.ccs_bb_pool;
	1240	+
	1241	+ guard(mutex) (xe_sa_bo_swap_guard(bb_pool));
	1242	+ xe_sa_bo_swap_shadow(bb_pool);
	1243	+
	1244	+ cs = xe_sa_bo_cpu_addr(bb->bo);
	1245	+ memset(cs, MI_NOOP, bb->len * sizeof(u32));
	1246	+ xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
	1247	+
	1248	+ xe_sa_bo_sync_shadow(bb->bo);
	1249	+
	1250	+ xe_bb_free(bb, NULL);
	1251	+ src_bo->bb_ccs[read_write] = NULL;
1201	1252	}
1202	1253
1203	1254	/**

drivers/gpu/drm/xe/xe_migrate.h+3−0

@@ -134,6 +134,9 @@ int xe_migrate_ccs_rw_copy(struct xe_tile tile, struct xe_exec_queue q, · 133 unmodified lines
134	134	struct xe_bo *src_bo,
135	135	enum xe_sriov_vf_ccs_rw_ctxs read_write);
136	136
	137	+void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
	138	+ enum xe_sriov_vf_ccs_rw_ctxs read_write);
	139	+
137	140	struct xe_lrc xe_migrate_lrc(struct xe_migrate migrate);
138	141	struct xe_exec_queue xe_migrate_exec_queue(struct xe_migrate migrate);
139	142	struct dma_fence xe_migrate_vram_copy_chunk(struct xe_bo vram_bo, u64 vram_offset,

drivers/gpu/drm/xe/xe_mocs.c+6−12

@@ -811,26 +811,20 @@ int xe_mocs_dump(struct xe_gt gt, struct drm_printer p) · 810 unmodified lines
811	811	struct xe_device *xe = gt_to_xe(gt);
812	812	enum xe_force_wake_domains domain;
813	813	struct xe_mocs_info table;
814		− unsigned int fw_ref, flags;
815		− int err = 0;
	814	+ unsigned int flags;
816	815
817	816	flags = get_mocs_settings(xe, &table);
818	817
819	818	domain = flags & HAS_LNCF_MOCS ? XE_FORCEWAKE_ALL : XE_FW_GT;
820		− xe_pm_runtime_get_noresume(xe);
821		− fw_ref = xe_force_wake_get(gt_to_fw(gt), domain);
822	819
823		− if (!xe_force_wake_ref_has_domain(fw_ref, domain)) {
824		− err = -ETIMEDOUT;
825		− goto err_fw;
826		− }
	820	+ guard(xe_pm_runtime_noresume)(xe);
	821	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), domain);
	822	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, domain))
	823	+ return -ETIMEDOUT;
827	824
828	825	table.ops->dump(&table, flags, gt, p);
829	826
830		−err_fw:
831		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
832		− xe_pm_runtime_put(xe);
833		− return err;
	827	+ return 0;
834	828	}
835	829
836	830	#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)

drivers/gpu/drm/xe/xe_nvm.c+23−11

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include "xe_device_types.h"
11	11	#include "xe_mmio.h"
12	12	#include "xe_nvm.h"
	13	+#include "xe_pcode_api.h"
13	14	#include "regs/xe_gsc_regs.h"
14	15	#include "xe_sriov.h"
15	16
@@ -45,39 +46,50 @@ static bool xe_nvm_non_posted_erase(struct xe_device *xe) · 29 unmodified lines
45	46	{
46	47	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
47	48
48		− if (xe->info.platform != XE_BATTLEMAGE)
	49	+ switch (xe->info.platform) {
	50	+ case XE_CRESCENTISLAND:
	51	+ case XE_BATTLEMAGE:
	52	+ return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) &
	53	+ NVM_NON_POSTED_ERASE_CHICKEN_BIT);
	54	+ default:
49	55	return false;
50		− return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) &
51		− NVM_NON_POSTED_ERASE_CHICKEN_BIT);
	56	+ }
52	57	}
53	58
54	59	static bool xe_nvm_writable_override(struct xe_device *xe)
55	60	{
56	61	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
57	62	bool writable_override;
58		− resource_size_t base;
	63	+ struct xe_reg reg;
	64	+ u32 test_bit;
59	65
60	66	switch (xe->info.platform) {
	67	+ case XE_CRESCENTISLAND:
	68	+ reg = PCODE_SCRATCH(0);
	69	+ test_bit = FDO_MODE;
	70	+ break;
61	71	case XE_BATTLEMAGE:
62		− base = DG2_GSC_HECI2_BASE;
	72	+ reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE);
	73	+ test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
63	74	break;
64	75	case XE_PVC:
65		− base = PVC_GSC_HECI2_BASE;
	76	+ reg = HECI_FWSTS2(PVC_GSC_HECI2_BASE);
	77	+ test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
66	78	break;
67	79	case XE_DG2:
68		− base = DG2_GSC_HECI2_BASE;
	80	+ reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE);
	81	+ test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
69	82	break;
70	83	case XE_DG1:
71		− base = DG1_GSC_HECI2_BASE;
	84	+ reg = HECI_FWSTS2(DG1_GSC_HECI2_BASE);
	85	+ test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
72	86	break;
73	87	default:
74	88	drm_err(&xe->drm, "Unknown platform\n");
75	89	return true;
76	90	}
77	91
78		− writable_override =
79		− !(xe_mmio_read32(mmio, HECI_FWSTS2(base)) &
80		− HECI_FW_STATUS_2_NVM_ACCESS_MODE);
	92	+ writable_override = !(xe_mmio_read32(mmio, reg) & test_bit);
81	93	if (writable_override)
82	94	drm_info(&xe->drm, "NVM access overridden by jumper\n");
83	95	return writable_override;

drivers/gpu/drm/xe/xe_oa.c+65−29

@@ -1941,6 +1941,7 @@ static bool oa_unit_supports_oa_format(struct xe_oa_open_param *param, int type) · 1940 unmodified lines
1941	1941	type == DRM_XE_OA_FMT_TYPE_OAC \|\| type == DRM_XE_OA_FMT_TYPE_PEC;
1942	1942	case DRM_XE_OA_UNIT_TYPE_OAM:
1943	1943	case DRM_XE_OA_UNIT_TYPE_OAM_SAG:
	1944	+ case DRM_XE_OA_UNIT_TYPE_MERT:
1944	1945	return type == DRM_XE_OA_FMT_TYPE_OAM \|\| type == DRM_XE_OA_FMT_TYPE_OAM_MPEC;
1945	1946	default:
1946	1947	return false;
@@ -1966,10 +1967,6 @@ static int xe_oa_assign_hwe(struct xe_oa oa, struct xe_oa_open_param param) · 19 unmodified lines
1966	1967	enum xe_hw_engine_id id;
1967	1968	int ret = 0;
1968	1969
1969		− /* If not provided, OA unit defaults to OA unit 0 as per uapi */
1970		− if (!param->oa_unit)
1971		− param->oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0];
1972		−
1973	1970	/* When we have an exec_q, get hwe from the exec_q */
1974	1971	if (param->exec_q) {
1975	1972	param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class,
@@ -2035,7 +2032,15 @@ int xe_oa_stream_open_ioctl(struct drm_device dev, u64 data, struct drm_file f · 59 unmodified lines
2035	2032	if (ret)
2036	2033	return ret;
2037	2034
	2035	+ /* If not provided, OA unit defaults to OA unit 0 as per uapi */
	2036	+ if (!param.oa_unit)
	2037	+ param.oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0];
	2038	+
2038	2039	if (param.exec_queue_id > 0) {
	2040	+ /* An exec_queue is only needed for OAR/OAC functionality on OAG */
	2041	+ if (XE_IOCTL_DBG(oa->xe, param.oa_unit->type != DRM_XE_OA_UNIT_TYPE_OAG))
	2042	+ return -EINVAL;
	2043	+
2039	2044	param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id);
2040	2045	if (XE_IOCTL_DBG(oa->xe, !param.exec_q))
2041	2046	return -ENOENT;
@@ -2224,6 +2229,8 @@ static const struct xe_mmio_range xe2_oa_mux_regs[] = { · 182 unmodified lines
2224	2229	{ .start = 0xE18C, .end = 0xE18C }, /* SAMPLER_MODE */
2225	2230	{ .start = 0xE590, .end = 0xE590 }, /* TDL_LSC_LAT_MEASURE_TDL_GFX */
2226	2231	{ .start = 0x13000, .end = 0x137FC }, /* PES_0_PESL0 - PES_63_UPPER_PESL3 */
	2232	+ { .start = 0x145194, .end = 0x145194 }, /* SYS_MEM_LAT_MEASURE */
	2233	+ { .start = 0x145340, .end = 0x14537C }, /* MERTSS_PES_0 - MERTSS_PES_7 */
2227	2234	{},
2228	2235	};
2229	2236
@@ -2515,7 +2522,12 @@ int xe_oa_register(struct xe_device *xe) · 285 unmodified lines
2515	2522	static u32 num_oa_units_per_gt(struct xe_gt *gt)
2516	2523	{
2517	2524	if (xe_gt_is_main_type(gt) \|\| GRAPHICS_VER(gt_to_xe(gt)) < 20)
2518		− return 1;
	2525	+ /*
	2526	+ * Mert OA unit belongs to the SoC, not a gt, so should be accessed using
	2527	+ * xe_root_tile_mmio(). However, for all known platforms this is the same as
	2528	+ * accessing via xe_root_mmio_gt()->mmio.
	2529	+ */
	2530	+ return xe_device_has_mert(gt_to_xe(gt)) ? 2 : 1;
2519	2531	else if (!IS_DGFX(gt_to_xe(gt)))
2520	2532	return XE_OAM_UNIT_SCMI_0 + 1; /* SAG + SCMI_0 */
2521	2533	else
@@ -2570,40 +2582,57 @@ static u32 __hwe_oa_unit(struct xe_hw_engine *hwe) · 48 unmodified lines
2570	2582	static struct xe_oa_regs __oam_regs(u32 base)
2571	2583	{
2572	2584	return (struct xe_oa_regs) {
2573		− base,
2574		− OAM_HEAD_POINTER(base),
2575		− OAM_TAIL_POINTER(base),
2576		− OAM_BUFFER(base),
2577		− OAM_CONTEXT_CONTROL(base),
2578		− OAM_CONTROL(base),
2579		− OAM_DEBUG(base),
2580		− OAM_STATUS(base),
2581		− OAM_CONTROL_COUNTER_SEL_MASK,
	2585	+ .base = base,
	2586	+ .oa_head_ptr = OAM_HEAD_POINTER(base),
	2587	+ .oa_tail_ptr = OAM_TAIL_POINTER(base),
	2588	+ .oa_buffer = OAM_BUFFER(base),
	2589	+ .oa_ctx_ctrl = OAM_CONTEXT_CONTROL(base),
	2590	+ .oa_ctrl = OAM_CONTROL(base),
	2591	+ .oa_debug = OAM_DEBUG(base),
	2592	+ .oa_status = OAM_STATUS(base),
	2593	+ .oa_mmio_trg = OAM_MMIO_TRG(base),
	2594	+ .oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK,
2582	2595	};
2583	2596	}
2584	2597
2585	2598	static struct xe_oa_regs __oag_regs(void)
2586	2599	{
2587	2600	return (struct xe_oa_regs) {
2588		− 0,
2589		− OAG_OAHEADPTR,
2590		− OAG_OATAILPTR,
2591		− OAG_OABUFFER,
2592		− OAG_OAGLBCTXCTRL,
2593		− OAG_OACONTROL,
2594		− OAG_OA_DEBUG,
2595		− OAG_OASTATUS,
2596		− OAG_OACONTROL_OA_COUNTER_SEL_MASK,
	2601	+ .base = 0,
	2602	+ .oa_head_ptr = OAG_OAHEADPTR,
	2603	+ .oa_tail_ptr = OAG_OATAILPTR,
	2604	+ .oa_buffer = OAG_OABUFFER,
	2605	+ .oa_ctx_ctrl = OAG_OAGLBCTXCTRL,
	2606	+ .oa_ctrl = OAG_OACONTROL,
	2607	+ .oa_debug = OAG_OA_DEBUG,
	2608	+ .oa_status = OAG_OASTATUS,
	2609	+ .oa_mmio_trg = OAG_MMIOTRIGGER,
	2610	+ .oa_ctrl_counter_select_mask = OAG_OACONTROL_OA_COUNTER_SEL_MASK,
	2611	+ };
	2612	+}
	2613	+
	2614	+static struct xe_oa_regs __oamert_regs(void)
	2615	+{
	2616	+ return (struct xe_oa_regs) {
	2617	+ .base = 0,
	2618	+ .oa_head_ptr = OAMERT_HEAD_POINTER,
	2619	+ .oa_tail_ptr = OAMERT_TAIL_POINTER,
	2620	+ .oa_buffer = OAMERT_BUFFER,
	2621	+ .oa_ctx_ctrl = OAMERT_CONTEXT_CONTROL,
	2622	+ .oa_ctrl = OAMERT_CONTROL,
	2623	+ .oa_debug = OAMERT_DEBUG,
	2624	+ .oa_status = OAMERT_STATUS,
	2625	+ .oa_mmio_trg = OAMERT_MMIO_TRG,
	2626	+ .oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK,
2597	2627	};
2598	2628	}
2599	2629
2600	2630	static void __xe_oa_init_oa_units(struct xe_gt *gt)
2601	2631	{
2602		− /* Actual address is MEDIA_GT_GSI_OFFSET + oam_base_addr[i] */
2603	2632	const u32 oam_base_addr[] = {
2604		− [XE_OAM_UNIT_SAG] = 0x13000,
2605		− [XE_OAM_UNIT_SCMI_0] = 0x14000,
2606		− [XE_OAM_UNIT_SCMI_1] = 0x14800,
	2633	+ [XE_OAM_UNIT_SAG] = XE_OAM_SAG_BASE,
	2634	+ [XE_OAM_UNIT_SCMI_0] = XE_OAM_SCMI_0_BASE,
	2635	+ [XE_OAM_UNIT_SCMI_1] = XE_OAM_SCMI_1_BASE,
2607	2636	};
2608	2637	int i, num_units = gt->oa.num_oa_units;
2609	2638
@@ -2611,8 +2640,15 @@ static void __xe_oa_init_oa_units(struct xe_gt *gt) · 1 unmodified line
2611	2640	struct xe_oa_unit *u = &gt->oa.oa_unit[i];
2612	2641
2613	2642	if (xe_gt_is_main_type(gt)) {
2614		− u->regs = __oag_regs();
2615		− u->type = DRM_XE_OA_UNIT_TYPE_OAG;
	2643	+ if (!i) {
	2644	+ u->regs = __oag_regs();
	2645	+ u->type = DRM_XE_OA_UNIT_TYPE_OAG;
	2646	+ } else {
	2647	+ xe_gt_assert(gt, xe_device_has_mert(gt_to_xe(gt)));
	2648	+ xe_gt_assert(gt, gt == xe_root_mmio_gt(gt_to_xe(gt)));
	2649	+ u->regs = __oamert_regs();
	2650	+ u->type = DRM_XE_OA_UNIT_TYPE_MERT;
	2651	+ }
2616	2652	} else {
2617	2653	xe_gt_assert(gt, GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270);
2618	2654	u->regs = __oam_regs(oam_base_addr[i]);

drivers/gpu/drm/xe/xe_oa_types.h+1−0

@@ -87,6 +87,7 @@ struct xe_oa_regs { · 86 unmodified lines
87	87	struct xe_reg oa_ctrl;
88	88	struct xe_reg oa_debug;
89	89	struct xe_reg oa_status;
	90	+ struct xe_reg oa_mmio_trg;
90	91	u32 oa_ctrl_counter_select_mask;
91	92	};
92	93

drivers/gpu/drm/xe/xe_page_reclaim.c+136−0

@@ -0,0 +1,136 @@
	1	+// SPDX-License-Identifier: MIT
	2	+/*
	3	+ * Copyright © 2025 Intel Corporation
	4	+ */
	5	+
	6	+#include <linux/bitfield.h>
	7	+#include <linux/kref.h>
	8	+#include <linux/mm.h>
	9	+#include <linux/slab.h>
	10	+
	11	+#include "xe_page_reclaim.h"
	12	+
	13	+#include "regs/xe_gt_regs.h"
	14	+#include "xe_assert.h"
	15	+#include "xe_macros.h"
	16	+#include "xe_mmio.h"
	17	+#include "xe_pat.h"
	18	+#include "xe_sa.h"
	19	+#include "xe_tlb_inval_types.h"
	20	+#include "xe_vm.h"
	21	+
	22	+/**
	23	+ * xe_page_reclaim_skip() - Decide whether PRL should be skipped for a VMA
	24	+ * @tile: Tile owning the VMA
	25	+ * @vma: VMA under consideration
	26	+ *
	27	+ * PPC flushing may be handled by HW for specific PAT encodings.
	28	+ * Skip PPC flushing/Page Reclaim for scenarios below due to redundant
	29	+ * flushes.
	30	+ * - pat_index is transient display (1)
	31	+ *
	32	+ * Return: true when page reclamation is unnecessary, false otherwise.
	33	+ */
	34	+bool xe_page_reclaim_skip(struct xe_tile tile, struct xe_vma vma)
	35	+{
	36	+ u8 l3_policy;
	37	+
	38	+ l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index);
	39	+
	40	+ /*
	41	+ * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC
	42	+ * Transient display flushes is taken care by HW, l3_policy = 1.
	43	+ *
	44	+ * HW will sequence these transient flushes at various sync points so
	45	+ * any event of page reclamation will hit these sync points before
	46	+ * page reclamation could execute.
	47	+ */
	48	+ return (l3_policy == XE_L3_POLICY_XD);
	49	+}
	50	+
	51	+/**
	52	+ * xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO
	53	+ * @tlb_inval: TLB invalidation frontend associated with the request
	54	+ * @prl: page reclaim list data that bo will copy from
	55	+ * @fence: tlb invalidation fence that page reclaim action is paired to
	56	+ *
	57	+ * Suballocates a 4K BO out of the tile reclaim pool, copies the PRL CPU
	58	+ * copy into the BO and queues the buffer for release when @fence signals.
	59	+ *
	60	+ * Return: struct drm_suballoc pointer on success or ERR_PTR on failure.
	61	+ */
	62	+struct drm_suballoc xe_page_reclaim_create_prl_bo(struct xe_tlb_inval tlb_inval,
	63	+ struct xe_page_reclaim_list *prl,
	64	+ struct xe_tlb_inval_fence *fence)
	65	+{
	66	+ struct xe_gt *gt = container_of(tlb_inval, struct xe_gt, tlb_inval);
	67	+ struct xe_tile *tile = gt_to_tile(gt);
	68	+ /* (+1) for NULL page_reclaim_entry to indicate end of list */
	69	+ int prl_size = min(prl->num_entries + 1, XE_PAGE_RECLAIM_MAX_ENTRIES) *
	70	+ sizeof(struct xe_guc_page_reclaim_entry);
	71	+ struct drm_suballoc *prl_sa;
	72	+
	73	+ /* Maximum size of PRL is 1 4K-page */
	74	+ prl_sa = __xe_sa_bo_new(tile->mem.reclaim_pool,
	75	+ prl_size, GFP_ATOMIC);
	76	+ if (IS_ERR(prl_sa))
	77	+ return prl_sa;
	78	+
	79	+ memcpy(xe_sa_bo_cpu_addr(prl_sa), prl->entries,
	80	+ prl_size);
	81	+ xe_sa_bo_flush_write(prl_sa);
	82	+ /* Queue up sa_bo_free on tlb invalidation fence signal */
	83	+ xe_sa_bo_free(prl_sa, &fence->base);
	84	+
	85	+ return prl_sa;
	86	+}
	87	+
	88	+/**
	89	+ * xe_page_reclaim_list_invalidate() - Mark a PRL as invalid
	90	+ * @prl: Page reclaim list to reset
	91	+ *
	92	+ * Clears the entries pointer and marks the list as invalid so
	93	+ * future use knows PRL is unusable. It is expected that the entries
	94	+ * have already been released.
	95	+ */
	96	+void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl)
	97	+{
	98	+ xe_page_reclaim_entries_put(prl->entries);
	99	+ prl->entries = NULL;
	100	+ prl->num_entries = XE_PAGE_RECLAIM_INVALID_LIST;
	101	+}
	102	+
	103	+/**
	104	+ * xe_page_reclaim_list_init() - Initialize a page reclaim list
	105	+ * @prl: Page reclaim list to initialize
	106	+ *
	107	+ * NULLs both values in list to prepare on initalization.
	108	+ */
	109	+void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl)
	110	+{
	111	+ // xe_page_reclaim_list_invalidate(prl);
	112	+ prl->entries = NULL;
	113	+ prl->num_entries = 0;
	114	+}
	115	+
	116	+/**
	117	+ * xe_page_reclaim_list_alloc_entries() - Allocate page reclaim list entries
	118	+ * @prl: Page reclaim list to allocate entries for
	119	+ *
	120	+ * Allocate one 4K page for the PRL entries, otherwise assign prl->entries to NULL.
	121	+ */
	122	+int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl)
	123	+{
	124	+ struct page *page;
	125	+
	126	+ if (XE_WARN_ON(prl->entries))
	127	+ return 0;
	128	+
	129	+ page = alloc_page(GFP_KERNEL \| __GFP_ZERO);
	130	+ if (page) {
	131	+ prl->entries = page_address(page);
	132	+ prl->num_entries = 0;
	133	+ }
	134	+
	135	+ return page ? 0 : -ENOMEM;
	136	+}

drivers/gpu/drm/xe/xe_page_reclaim.h+105−0

@@ -0,0 +1,105 @@
	1	+/* SPDX-License-Identifier: MIT */
	2	+/*
	3	+ * Copyright © 2025 Intel Corporation
	4	+ */
	5	+
	6	+#ifndef _XE_PAGE_RECLAIM_H_
	7	+#define _XE_PAGE_RECLAIM_H_
	8	+
	9	+#include <linux/kref.h>
	10	+#include <linux/mm.h>
	11	+#include <linux/slab.h>
	12	+#include <linux/types.h>
	13	+#include <linux/workqueue.h>
	14	+#include <linux/bits.h>
	15	+
	16	+#define XE_PAGE_RECLAIM_MAX_ENTRIES 512
	17	+#define XE_PAGE_RECLAIM_LIST_MAX_SIZE SZ_4K
	18	+
	19	+struct xe_tlb_inval;
	20	+struct xe_tlb_inval_fence;
	21	+struct xe_tile;
	22	+struct xe_vma;
	23	+
	24	+struct xe_guc_page_reclaim_entry {
	25	+ u64 qw;
	26	+/* valid reclaim entry bit */
	27	+#define XE_PAGE_RECLAIM_VALID BIT_ULL(0)
	28	+/*
	29	+ * offset order of page size to be reclaimed
	30	+ * page_size = 1 << (XE_PTE_SHIFT + reclamation_size)
	31	+ */
	32	+#define XE_PAGE_RECLAIM_SIZE GENMASK_ULL(6, 1)
	33	+#define XE_PAGE_RECLAIM_RSVD_0 GENMASK_ULL(11, 7)
	34	+/* lower 20 bits of the physical address */
	35	+#define XE_PAGE_RECLAIM_ADDR_LO GENMASK_ULL(31, 12)
	36	+/* upper 20 bits of the physical address */
	37	+#define XE_PAGE_RECLAIM_ADDR_HI GENMASK_ULL(51, 32)
	38	+#define XE_PAGE_RECLAIM_RSVD_1 GENMASK_ULL(63, 52)
	39	+} __packed;
	40	+
	41	+struct xe_page_reclaim_list {
	42	+ /** @entries: array of page reclaim entries, page allocated */
	43	+ struct xe_guc_page_reclaim_entry *entries;
	44	+ /** @num_entries: number of entries */
	45	+ int num_entries;
	46	+#define XE_PAGE_RECLAIM_INVALID_LIST -1
	47	+};
	48	+
	49	+/**
	50	+ * xe_page_reclaim_list_is_new() - Check if PRL is new allocation
	51	+ * @prl: Pointer to page reclaim list
	52	+ *
	53	+ * PRL indicates it hasn't been allocated through both values being NULL
	54	+ */
	55	+static inline bool xe_page_reclaim_list_is_new(struct xe_page_reclaim_list *prl)
	56	+{
	57	+ return !prl->entries && prl->num_entries == 0;
	58	+}
	59	+
	60	+/**
	61	+ * xe_page_reclaim_list_valid() - Check if the page reclaim list is valid
	62	+ * @prl: Pointer to page reclaim list
	63	+ *
	64	+ * PRL uses the XE_PAGE_RECLAIM_INVALID_LIST to indicate that a PRL
	65	+ * is unusable.
	66	+ */
	67	+static inline bool xe_page_reclaim_list_valid(struct xe_page_reclaim_list *prl)
	68	+{
	69	+ return !xe_page_reclaim_list_is_new(prl) &&
	70	+ prl->num_entries != XE_PAGE_RECLAIM_INVALID_LIST;
	71	+}
	72	+
	73	+bool xe_page_reclaim_skip(struct xe_tile tile, struct xe_vma vma);
	74	+struct drm_suballoc xe_page_reclaim_create_prl_bo(struct xe_tlb_inval tlb_inval,
	75	+ struct xe_page_reclaim_list *prl,
	76	+ struct xe_tlb_inval_fence *fence);
	77	+void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl);
	78	+void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl);
	79	+int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl);
	80	+/**
	81	+ * xe_page_reclaim_entries_get() - Increment the reference count of page reclaim entries.
	82	+ * @entries: Pointer to the array of page reclaim entries.
	83	+ *
	84	+ * This function increments the reference count of the backing page.
	85	+ */
	86	+static inline void xe_page_reclaim_entries_get(struct xe_guc_page_reclaim_entry *entries)
	87	+{
	88	+ if (entries)
	89	+ get_page(virt_to_page(entries));
	90	+}
	91	+
	92	+/**
	93	+ * xe_page_reclaim_entries_put() - Decrement the reference count of page reclaim entries.
	94	+ * @entries: Pointer to the array of page reclaim entries.
	95	+ *
	96	+ * This function decrements the reference count of the backing page
	97	+ * and frees it if the count reaches zero.
	98	+ */
	99	+static inline void xe_page_reclaim_entries_put(struct xe_guc_page_reclaim_entry *entries)
	100	+{
	101	+ if (entries)
	102	+ put_page(virt_to_page(entries));
	103	+}
	104	+
	105	+#endif /* _XE_PAGE_RECLAIM_H_ */

drivers/gpu/drm/xe/xe_pagefault.c+18−18

@@ -223,22 +223,22 @@ static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue, · 222 unmodified lines
223	223
224	224	static void xe_pagefault_print(struct xe_pagefault *pf)
225	225	{
226		− xe_gt_dbg(pf->gt, "\n\tASID: %d\n"
227		− "\tFaulted Address: 0x%08x%08x\n"
228		− "\tFaultType: %d\n"
229		− "\tAccessType: %d\n"
230		− "\tFaultLevel: %d\n"
231		− "\tEngineClass: %d %s\n"
232		− "\tEngineInstance: %d\n",
233		− pf->consumer.asid,
234		− upper_32_bits(pf->consumer.page_addr),
235		− lower_32_bits(pf->consumer.page_addr),
236		− pf->consumer.fault_type,
237		− pf->consumer.access_type,
238		− pf->consumer.fault_level,
239		− pf->consumer.engine_class,
240		− xe_hw_engine_class_to_str(pf->consumer.engine_class),
241		− pf->consumer.engine_instance);
	226	+ xe_gt_info(pf->gt, "\n\tASID: %d\n"
	227	+ "\tFaulted Address: 0x%08x%08x\n"
	228	+ "\tFaultType: %d\n"
	229	+ "\tAccessType: %d\n"
	230	+ "\tFaultLevel: %d\n"
	231	+ "\tEngineClass: %d %s\n"
	232	+ "\tEngineInstance: %d\n",
	233	+ pf->consumer.asid,
	234	+ upper_32_bits(pf->consumer.page_addr),
	235	+ lower_32_bits(pf->consumer.page_addr),
	236	+ pf->consumer.fault_type,
	237	+ pf->consumer.access_type,
	238	+ pf->consumer.fault_level,
	239	+ pf->consumer.engine_class,
	240	+ xe_hw_engine_class_to_str(pf->consumer.engine_class),
	241	+ pf->consumer.engine_instance);
242	242	}
243	243
244	244	static void xe_pagefault_queue_work(struct work_struct *w)
@@ -260,8 +260,8 @@ static void xe_pagefault_queue_work(struct work_struct *w) · 15 unmodified lines
260	260	err = xe_pagefault_service(&pf);
261	261	if (err) {
262	262	xe_pagefault_print(&pf);
263		− xe_gt_dbg(pf.gt, "Fault response: Unsuccessful %pe\n",
264		− ERR_PTR(err));
	263	+ xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n",
	264	+ ERR_PTR(err));
265	265	}
266	266
267	267	pf.producer.ops->ack_fault(&pf, err);

drivers/gpu/drm/xe/xe_pat.c+152−68

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9
10	10	#include <generated/xe_wa_oob.h>
11	11
	12	+#include "regs/xe_gt_regs.h"
12	13	#include "regs/xe_reg_defs.h"
13	14	#include "xe_assert.h"
14	15	#include "xe_device.h"
@@ -50,8 +51,37 @@ · 35 unmodified lines
50	51	#define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1)
51	52	#define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0)
52	53
	54	+#define PAT_LABEL_LEN 20
	55	+
53	56	static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" };
54	57
	58	+static void xe_pat_index_label(char *label, size_t len, int index)
	59	+{
	60	+ snprintf(label, len, "PAT[%2d] ", index);
	61	+}
	62	+
	63	+static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
	64	+{
	65	+ u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
	66	+
	67	+ drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index,
	68	+ XELP_MEM_TYPE_STR_MAP[mem_type], pat);
	69	+}
	70	+
	71	+static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
	72	+{
	73	+ drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index,
	74	+ REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
	75	+ REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
	76	+}
	77	+
	78	+static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
	79	+{
	80	+ drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index,
	81	+ REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
	82	+ REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
	83	+}
	84	+
55	85	struct xe_pat_ops {
56	86	void (program_graphics)(struct xe_gt gt, const struct xe_pat_table_entry table[],
57	87	int n_entries);
@@ -196,6 +226,19 @@ u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) · 138 unmodified lines
196	226	return xe->pat.table[pat_index].coh_mode;
197	227	}
198	228
	229	+bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index)
	230	+{
	231	+ WARN_ON(pat_index >= xe->pat.n_entries);
	232	+ return !!(xe->pat.table[pat_index].value & XE2_COMP_EN);
	233	+}
	234	+
	235	+u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index)
	236	+{
	237	+ WARN_ON(pat_index >= xe->pat.n_entries);
	238	+
	239	+ return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value);
	240	+}
	241	+
199	242	static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
200	243	int n_entries)
201	244	{
@@ -233,24 +276,20 @@ static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry ta · 31 unmodified lines
233	276	static int xelp_dump(struct xe_gt gt, struct drm_printer p)
234	277	{
235	278	struct xe_device *xe = gt_to_xe(gt);
236		− unsigned int fw_ref;
237	279	int i;
238	280
239		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
240		− if (!fw_ref)
	281	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	282	+ if (!fw_ref.domains)
241	283	return -ETIMEDOUT;
242	284
243	285	drm_printf(p, "PAT table:\n");
244	286
245	287	for (i = 0; i < xe->pat.n_entries; i++) {
246	288	u32 pat = xe_mmio_read32(&gt->mmio, XE_REG(_PAT_INDEX(i)));
247		− u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
248	289
249		− drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
250		− XELP_MEM_TYPE_STR_MAP[mem_type], pat);
	290	+ xelp_pat_entry_dump(p, i, pat);
251	291	}
252	292
253		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
254	293	return 0;
255	294	}
256	295
@@ -262,26 +301,20 @@ static const struct xe_pat_ops xelp_pat_ops = { · 5 unmodified lines
262	301	static int xehp_dump(struct xe_gt gt, struct drm_printer p)
263	302	{
264	303	struct xe_device *xe = gt_to_xe(gt);
265		− unsigned int fw_ref;
266	304	int i;
267	305
268		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
269		− if (!fw_ref)
	306	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	307	+ if (!fw_ref.domains)
270	308	return -ETIMEDOUT;
271	309
272	310	drm_printf(p, "PAT table:\n");
273	311
274	312	for (i = 0; i < xe->pat.n_entries; i++) {
275	313	u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
276		− u8 mem_type;
277		−
278		− mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
279	314
280		− drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
281		− XELP_MEM_TYPE_STR_MAP[mem_type], pat);
	315	+ xelp_pat_entry_dump(p, i, pat);
282	316	}
283	317
284		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
285	318	return 0;
286	319	}
287	320
@@ -293,11 +326,10 @@ static const struct xe_pat_ops xehp_pat_ops = { · 5 unmodified lines
293	326	static int xehpc_dump(struct xe_gt gt, struct drm_printer p)
294	327	{
295	328	struct xe_device *xe = gt_to_xe(gt);
296		− unsigned int fw_ref;
297	329	int i;
298	330
299		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
300		− if (!fw_ref)
	331	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	332	+ if (!fw_ref.domains)
301	333	return -ETIMEDOUT;
302	334
303	335	drm_printf(p, "PAT table:\n");
@@ -305,12 +337,9 @@ static int xehpc_dump(struct xe_gt gt, struct drm_printer p) · 1 unmodified line
305	337	for (i = 0; i < xe->pat.n_entries; i++) {
306	338	u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
307	339
308		− drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
309		− REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
310		− REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
	340	+ xehpc_pat_entry_dump(p, i, pat);
311	341	}
312	342
313		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
314	343	return 0;
315	344	}
316	345
@@ -322,11 +351,10 @@ static const struct xe_pat_ops xehpc_pat_ops = { · 5 unmodified lines
322	351	static int xelpg_dump(struct xe_gt gt, struct drm_printer p)
323	352	{
324	353	struct xe_device *xe = gt_to_xe(gt);
325		− unsigned int fw_ref;
326	354	int i;
327	355
328		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
329		− if (!fw_ref)
	356	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	357	+ if (!fw_ref.domains)
330	358	return -ETIMEDOUT;
331	359
332	360	drm_printf(p, "PAT table:\n");
@@ -339,12 +367,9 @@ static int xelpg_dump(struct xe_gt gt, struct drm_printer p) · 6 unmodified lines
339	367	else
340	368	pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
341	369
342		− drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
343		− REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
344		− REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
	370	+ xelpg_pat_entry_dump(p, i, pat);
345	371	}
346	372
347		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
348	373	return 0;
349	374	}
350	375
@@ -358,15 +383,38 @@ static const struct xe_pat_ops xelpg_pat_ops = { · 7 unmodified lines
358	383	.dump = xelpg_dump,
359	384	};
360	385
	386	+static void xe2_pat_entry_dump(struct drm_printer p, const char label, u32 pat, bool rsvd)
	387	+{
	388	+ drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", label,
	389	+ !!(pat & XE2_NO_PROMOTE),
	390	+ !!(pat & XE2_COMP_EN),
	391	+ REG_FIELD_GET(XE2_L3_CLOS, pat),
	392	+ REG_FIELD_GET(XE2_L3_POLICY, pat),
	393	+ REG_FIELD_GET(XE2_L4_POLICY, pat),
	394	+ REG_FIELD_GET(XE2_COH_MODE, pat),
	395	+ pat, rsvd ? " *" : "");
	396	+}
	397	+
	398	+static void xe3p_xpc_pat_entry_dump(struct drm_printer p, const char label, u32 pat, bool rsvd)
	399	+{
	400	+ drm_printf(p, "%s= [ %u, %u, %u, %u, %u ] (%#8x)%s\n", label,
	401	+ !!(pat & XE2_NO_PROMOTE),
	402	+ REG_FIELD_GET(XE2_L3_CLOS, pat),
	403	+ REG_FIELD_GET(XE2_L3_POLICY, pat),
	404	+ REG_FIELD_GET(XE2_L4_POLICY, pat),
	405	+ REG_FIELD_GET(XE2_COH_MODE, pat),
	406	+ pat, rsvd ? " *" : "");
	407	+}
	408	+
361	409	static int xe2_dump(struct xe_gt gt, struct drm_printer p)
362	410	{
363	411	struct xe_device *xe = gt_to_xe(gt);
364		− unsigned int fw_ref;
365	412	u32 pat;
366	413	int i;
	414	+ char label[PAT_LABEL_LEN];
367	415
368		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
369		− if (!fw_ref)
	416	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	417	+ if (!fw_ref.domains)
370	418	return -ETIMEDOUT;
371	419
372	420	drm_printf(p, "PAT table: (* = reserved entry)\n");
@@ -377,14 +425,8 @@ static int xe2_dump(struct xe_gt gt, struct drm_printer p) · 4 unmodified lines
377	425	else
378	426	pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
379	427
380		− drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", i,
381		− !!(pat & XE2_NO_PROMOTE),
382		− !!(pat & XE2_COMP_EN),
383		− REG_FIELD_GET(XE2_L3_CLOS, pat),
384		− REG_FIELD_GET(XE2_L3_POLICY, pat),
385		− REG_FIELD_GET(XE2_L4_POLICY, pat),
386		− REG_FIELD_GET(XE2_COH_MODE, pat),
387		− pat, xe->pat.table[i].valid ? "" : " *");
	428	+ xe_pat_index_label(label, sizeof(label), i);
	429	+ xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
388	430	}
389	431
390	432	/*
@@ -397,16 +439,8 @@ static int xe2_dump(struct xe_gt gt, struct drm_printer p) · 6 unmodified lines
397	439	pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
398	440
399	441	drm_printf(p, "Page Table Access:\n");
400		− drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ] (%#8x)\n",
401		− !!(pat & XE2_NO_PROMOTE),
402		− !!(pat & XE2_COMP_EN),
403		− REG_FIELD_GET(XE2_L3_CLOS, pat),
404		− REG_FIELD_GET(XE2_L3_POLICY, pat),
405		− REG_FIELD_GET(XE2_L4_POLICY, pat),
406		− REG_FIELD_GET(XE2_COH_MODE, pat),
407		− pat);
	442	+ xe2_pat_entry_dump(p, "PTA_MODE", pat, false);
408	443
409		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
410	444	return 0;
411	445	}
412	446
@@ -419,12 +453,12 @@ static const struct xe_pat_ops xe2_pat_ops = { · 6 unmodified lines
419	453	static int xe3p_xpc_dump(struct xe_gt gt, struct drm_printer p)
420	454	{
421	455	struct xe_device *xe = gt_to_xe(gt);
422		− unsigned int fw_ref;
423	456	u32 pat;
424	457	int i;
	458	+ char label[PAT_LABEL_LEN];
425	459
426		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
427		− if (!fw_ref)
	460	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	461	+ if (!fw_ref.domains)
428	462	return -ETIMEDOUT;
429	463
430	464	drm_printf(p, "PAT table: (* = reserved entry)\n");
@@ -432,13 +466,8 @@ static int xe3p_xpc_dump(struct xe_gt gt, struct drm_printer p) · 1 unmodified line
432	466	for (i = 0; i < xe->pat.n_entries; i++) {
433	467	pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
434	468
435		− drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u ] (%#8x)%s\n", i,
436		− !!(pat & XE2_NO_PROMOTE),
437		− REG_FIELD_GET(XE2_L3_CLOS, pat),
438		− REG_FIELD_GET(XE2_L3_POLICY, pat),
439		− REG_FIELD_GET(XE2_L4_POLICY, pat),
440		− REG_FIELD_GET(XE2_COH_MODE, pat),
441		− pat, xe->pat.table[i].valid ? "" : " *");
	469	+ xe_pat_index_label(label, sizeof(label), i);
	470	+ xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
442	471	}
443	472
444	473	/*
@@ -448,15 +477,8 @@ static int xe3p_xpc_dump(struct xe_gt gt, struct drm_printer p) · 3 unmodified lines
448	477	pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
449	478
450	479	drm_printf(p, "Page Table Access:\n");
451		− drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u ] (%#8x)\n",
452		− !!(pat & XE2_NO_PROMOTE),
453		− REG_FIELD_GET(XE2_L3_CLOS, pat),
454		− REG_FIELD_GET(XE2_L3_POLICY, pat),
455		− REG_FIELD_GET(XE2_L4_POLICY, pat),
456		− REG_FIELD_GET(XE2_COH_MODE, pat),
457		− pat);
	480	+ xe3p_xpc_pat_entry_dump(p, "PTA_MODE", pat, false);
458	481
459		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
460	482	return 0;
461	483	}
462	484
@@ -578,3 +600,65 @@ int xe_pat_dump(struct xe_gt gt, struct drm_printer p) · 115 unmodified lines
578	600
579	601	return xe->pat.ops->dump(gt, p);
580	602	}
	603	+
	604	+/**
	605	+ * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer.
	606	+ * @gt: the &xe_gt
	607	+ * @p: the &drm_printer
	608	+ *
	609	+ * Return: 0 on success or a negative error code on failure.
	610	+ */
	611	+int xe_pat_dump_sw_config(struct xe_gt gt, struct drm_printer p)
	612	+{
	613	+ struct xe_device *xe = gt_to_xe(gt);
	614	+ char label[PAT_LABEL_LEN];
	615	+
	616	+ if (!xe->pat.table \|\| !xe->pat.n_entries)
	617	+ return -EOPNOTSUPP;
	618	+
	619	+ drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : "");
	620	+ for (u32 i = 0; i < xe->pat.n_entries; i++) {
	621	+ u32 pat = xe->pat.table[i].value;
	622	+
	623	+ if (GRAPHICS_VERx100(xe) == 3511) {
	624	+ xe_pat_index_label(label, sizeof(label), i);
	625	+ xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
	626	+ } else if (GRAPHICS_VER(xe) == 30 \|\| GRAPHICS_VER(xe) == 20) {
	627	+ xe_pat_index_label(label, sizeof(label), i);
	628	+ xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
	629	+ } else if (xe->info.platform == XE_METEORLAKE) {
	630	+ xelpg_pat_entry_dump(p, i, pat);
	631	+ } else if (xe->info.platform == XE_PVC) {
	632	+ xehpc_pat_entry_dump(p, i, pat);
	633	+ } else if (xe->info.platform == XE_DG2 \|\| GRAPHICS_VERx100(xe) <= 1210) {
	634	+ xelp_pat_entry_dump(p, i, pat);
	635	+ } else {
	636	+ return -EOPNOTSUPP;
	637	+ }
	638	+ }
	639	+
	640	+ if (xe->pat.pat_pta) {
	641	+ u32 pat = xe->pat.pat_pta->value;
	642	+
	643	+ drm_printf(p, "Page Table Access:\n");
	644	+ xe2_pat_entry_dump(p, "PTA_MODE", pat, false);
	645	+ }
	646	+
	647	+ if (xe->pat.pat_ats) {
	648	+ u32 pat = xe->pat.pat_ats->value;
	649	+
	650	+ drm_printf(p, "PCIe ATS/PASID:\n");
	651	+ xe2_pat_entry_dump(p, "PAT_ATS ", pat, false);
	652	+ }
	653	+
	654	+ drm_printf(p, "Cache Level:\n");
	655	+ drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]);
	656	+ drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]);
	657	+ drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]);
	658	+ if (GRAPHICS_VER(xe) >= 20) {
	659	+ drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n",
	660	+ xe->pat.idx[XE_CACHE_NONE_COMPRESSION]);
	661	+ }
	662	+
	663	+ return 0;
	664	+}

drivers/gpu/drm/xe/xe_pat.h+21−0

@@ -49,6 +49,7 @@ void xe_pat_init_early(struct xe_device *xe); · 48 unmodified lines
49	49	void xe_pat_init(struct xe_gt *gt);
50	50
51	51	int xe_pat_dump(struct xe_gt gt, struct drm_printer p);
	52	+int xe_pat_dump_sw_config(struct xe_gt gt, struct drm_printer p);
52	53
53	54	/**
54	55	* xe_pat_index_get_coh_mode - Extract the coherency mode for the given
@@ -58,4 +59,24 @@ int xe_pat_dump(struct xe_gt gt, struct drm_printer p); · 3 unmodified lines
58	59	*/
59	60	u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index);
60	61
	62	+/**
	63	+ * xe_pat_index_get_comp_en - Extract the compression enable flag for
	64	+ * the given pat_index.
	65	+ * @xe: xe device
	66	+ * @pat_index: The pat_index to query
	67	+ *
	68	+ * Return: true if compression is enabled for this pat_index, false otherwise.
	69	+ */
	70	+bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index);
	71	+
	72	+#define XE_L3_POLICY_WB 0 /* Write-back */
	73	+#define XE_L3_POLICY_XD 1 /* WB - Transient Display */
	74	+#define XE_L3_POLICY_UC 3 /* Uncached */
	75	+/**
	76	+ * xe_pat_index_get_l3_policy - Extract the L3 policy for the given pat_index.
	77	+ * @xe: xe device
	78	+ * @pat_index: The pat_index to query
	79	+ */
	80	+u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index);
	81	+
61	82	#endif

drivers/gpu/drm/xe/xe_pci.c+29−0

@@ -108,6 +108,7 @@ static const struct xe_graphics_desc graphics_xe2 = { · 107 unmodified lines
108	108
109	109	static const struct xe_graphics_desc graphics_xe3p_xpc = {
110	110	XE2_GFX_FEATURES,
	111	+ .has_indirect_ring_state = 1,
111	112	.hw_engine_mask =
112	113	GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) \|
113	114	GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0),
@@ -168,6 +169,7 @@ static const struct xe_device_desc tgl_desc = { · 54 unmodified lines
168	169	.pre_gmdid_media_ip = &media_ip_xem,
169	170	PLATFORM(TIGERLAKE),
170	171	.dma_mask_size = 39,
	172	+ .has_cached_pt = true,
171	173	.has_display = true,
172	174	.has_llc = true,
173	175	.has_sriov = true,
@@ -182,6 +184,7 @@ static const struct xe_device_desc rkl_desc = { · 8 unmodified lines
182	184	.pre_gmdid_media_ip = &media_ip_xem,
183	185	PLATFORM(ROCKETLAKE),
184	186	.dma_mask_size = 39,
	187	+ .has_cached_pt = true,
185	188	.has_display = true,
186	189	.has_llc = true,
187	190	.max_gt_per_tile = 1,
@@ -197,6 +200,7 @@ static const struct xe_device_desc adl_s_desc = { · 9 unmodified lines
197	200	.pre_gmdid_media_ip = &media_ip_xem,
198	201	PLATFORM(ALDERLAKE_S),
199	202	.dma_mask_size = 39,
	203	+ .has_cached_pt = true,
200	204	.has_display = true,
201	205	.has_llc = true,
202	206	.has_sriov = true,
@@ -217,6 +221,7 @@ static const struct xe_device_desc adl_p_desc = { · 14 unmodified lines
217	221	.pre_gmdid_media_ip = &media_ip_xem,
218	222	PLATFORM(ALDERLAKE_P),
219	223	.dma_mask_size = 39,
	224	+ .has_cached_pt = true,
220	225	.has_display = true,
221	226	.has_llc = true,
222	227	.has_sriov = true,
@@ -235,6 +240,7 @@ static const struct xe_device_desc adl_n_desc = { · 12 unmodified lines
235	240	.pre_gmdid_media_ip = &media_ip_xem,
236	241	PLATFORM(ALDERLAKE_N),
237	242	.dma_mask_size = 39,
	243	+ .has_cached_pt = true,
238	244	.has_display = true,
239	245	.has_llc = true,
240	246	.has_sriov = true,
@@ -361,7 +367,9 @@ static const struct xe_device_desc bmg_desc = { · 120 unmodified lines
361	367	.has_mbx_power_limits = true,
362	368	.has_gsc_nvm = 1,
363	369	.has_heci_cscfi = 1,
	370	+ .has_i2c = true,
364	371	.has_late_bind = true,
	372	+ .has_pre_prod_wa = 1,
365	373	.has_sriov = true,
366	374	.has_mem_copy_instr = true,
367	375	.max_gt_per_tile = 2,
@@ -381,6 +389,7 @@ static const struct xe_device_desc ptl_desc = { · 13 unmodified lines
381	389	.has_flat_ccs = 1,
382	390	.has_sriov = true,
383	391	.has_mem_copy_instr = true,
	392	+ .has_pre_prod_wa = 1,
384	393	.max_gt_per_tile = 2,
385	394	.needs_scratch = true,
386	395	.needs_shared_vf_gt_wq = true,
@@ -394,6 +403,7 @@ static const struct xe_device_desc nvls_desc = { · 7 unmodified lines
394	403	.has_display = true,
395	404	.has_flat_ccs = 1,
396	405	.has_mem_copy_instr = true,
	406	+ .has_pre_prod_wa = 1,
397	407	.max_gt_per_tile = 2,
398	408	.require_force_probe = true,
399	409	.va_bits = 48,
@@ -406,7 +416,11 @@ static const struct xe_device_desc cri_desc = { · 6 unmodified lines
406	416	.dma_mask_size = 52,
407	417	.has_display = false,
408	418	.has_flat_ccs = false,
	419	+ .has_gsc_nvm = 1,
	420	+ .has_i2c = true,
409	421	.has_mbx_power_limits = true,
	422	+ .has_mert = true,
	423	+ .has_pre_prod_wa = 1,
410	424	.has_sriov = true,
411	425	.max_gt_per_tile = 2,
412	426	.require_force_probe = true,
@@ -663,6 +677,7 @@ static int xe_info_init_early(struct xe_device *xe, · 250 unmodified lines
663	677	xe->info.vram_flags = desc->vram_flags;
664	678
665	679	xe->info.is_dgfx = desc->is_dgfx;
	680	+ xe->info.has_cached_pt = desc->has_cached_pt;
666	681	xe->info.has_fan_control = desc->has_fan_control;
667	682	/* runtime fusing may force flat_ccs to disabled later */
668	683	xe->info.has_flat_ccs = desc->has_flat_ccs;
@@ -670,8 +685,12 @@ static int xe_info_init_early(struct xe_device *xe, · 1 unmodified line
670	685	xe->info.has_gsc_nvm = desc->has_gsc_nvm;
671	686	xe->info.has_heci_gscfi = desc->has_heci_gscfi;
672	687	xe->info.has_heci_cscfi = desc->has_heci_cscfi;
	688	+ xe->info.has_i2c = desc->has_i2c;
673	689	xe->info.has_late_bind = desc->has_late_bind;
674	690	xe->info.has_llc = desc->has_llc;
	691	+ xe->info.has_mert = desc->has_mert;
	692	+ xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist;
	693	+ xe->info.has_pre_prod_wa = desc->has_pre_prod_wa;
675	694	xe->info.has_pxp = desc->has_pxp;
676	695	xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) &&
677	696	desc->has_sriov;
@@ -755,6 +774,7 @@ static struct xe_gt alloc_primary_gt(struct xe_tile tile, · 77 unmodified lines
755	774	gt->info.type = XE_GT_TYPE_MAIN;
756	775	gt->info.id = tile->id * xe->info.max_gt_per_tile;
757	776	gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state;
	777	+ gt->info.multi_queue_engine_class_mask = graphics_desc->multi_queue_engine_class_mask;
758	778	gt->info.engine_mask = graphics_desc->hw_engine_mask;
759	779
760	780	/*
@@ -1153,6 +1173,15 @@ static int xe_pci_runtime_suspend(struct device *dev) · 392 unmodified lines
1153	1173	struct xe_device *xe = pdev_to_xe_device(pdev);
1154	1174	int err;
1155	1175
	1176	+ /*
	1177	+ * We hold an additional reference to the runtime PM to keep PF in D0
	1178	+ * during VFs lifetime, as our VFs do not implement the PM capability.
	1179	+ * This means we should never be runtime suspending as long as VFs are
	1180	+ * enabled.
	1181	+ */
	1182	+ xe_assert(xe, !IS_SRIOV_VF(xe));
	1183	+ xe_assert(xe, !pci_num_vf(pdev));
	1184	+
1156	1185	err = xe_pm_runtime_suspend(xe);
1157	1186	if (err)
1158	1187	return err;

drivers/gpu/drm/xe/xe_pci_sriov.c+3−7

@@ -219,7 +219,6 @@ static int pf_disable_vfs(struct xe_device *xe) · 218 unmodified lines
219	219	int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
220	220	{
221	221	struct xe_device *xe = pdev_to_xe_device(pdev);
222		− int ret;
223	222
224	223	if (!IS_SRIOV_PF(xe))
225	224	return -ENODEV;
@@ -233,14 +232,11 @@ int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) · 7 unmodified lines
233	232	if (num_vfs && pci_num_vf(pdev))
234	233	return -EBUSY;
235	234
236		− xe_pm_runtime_get(xe);
	235	+ guard(xe_pm_runtime)(xe);
237	236	if (num_vfs > 0)
238		− ret = pf_enable_vfs(xe, num_vfs);
	237	+ return pf_enable_vfs(xe, num_vfs);
239	238	else
240		− ret = pf_disable_vfs(xe);
241		− xe_pm_runtime_put(xe);
242		−
243		− return ret;
	239	+ return pf_disable_vfs(xe);
244	240	}
245	241
246	242	/**

drivers/gpu/drm/xe/xe_pci_types.h+6−0

@@ -37,16 +37,21 @@ struct xe_device_desc { · 36 unmodified lines
37	37	u8 require_force_probe:1;
38	38	u8 is_dgfx:1;
39	39
	40	+ u8 has_cached_pt:1;
40	41	u8 has_display:1;
41	42	u8 has_fan_control:1;
42	43	u8 has_flat_ccs:1;
43	44	u8 has_gsc_nvm:1;
44	45	u8 has_heci_gscfi:1;
45	46	u8 has_heci_cscfi:1;
	47	+ u8 has_i2c:1;
46	48	u8 has_late_bind:1;
47	49	u8 has_llc:1;
48	50	u8 has_mbx_power_limits:1;
49	51	u8 has_mem_copy_instr:1;
	52	+ u8 has_mert:1;
	53	+ u8 has_pre_prod_wa:1;
	54	+ u8 has_page_reclaim_hw_assist:1;
50	55	u8 has_pxp:1;
51	56	u8 has_sriov:1;
52	57	u8 needs_scratch:1;
@@ -58,6 +63,7 @@ struct xe_device_desc { · 5 unmodified lines
58	63
59	64	struct xe_graphics_desc {
60	65	u64 hw_engine_mask; /* hardware engines provided by graphics IP */
	66	+ u16 multi_queue_engine_class_mask; /* bitmask of engine classes which support multi queue */
61	67
62	68	u8 has_asid:1;
63	69	u8 has_atomic_enable_pte_bit:1;

drivers/gpu/drm/xe/xe_pcode_api.h+2−0

@@ -77,11 +77,13 @@ · 76 unmodified lines
77	77
78	78	#define PCODE_SCRATCH(x) XE_REG(0x138320 + ((x) * 4))
79	79	/* PCODE_SCRATCH0 */
	80	+#define BREADCRUMB_VERSION REG_GENMASK(31, 29)
80	81	#define AUXINFO_REG_OFFSET REG_GENMASK(17, 15)
81	82	#define OVERFLOW_REG_OFFSET REG_GENMASK(14, 12)
82	83	#define HISTORY_TRACKING REG_BIT(11)
83	84	#define OVERFLOW_SUPPORT REG_BIT(10)
84	85	#define AUXINFO_SUPPORT REG_BIT(9)
	86	+#define FDO_MODE REG_BIT(4)
85	87	#define BOOT_STATUS REG_GENMASK(3, 1)
86	88	#define CRITICAL_FAILURE 4
87	89	#define NON_CRITICAL_FAILURE 7

drivers/gpu/drm/xe/xe_pm.c+5−5

@@ -591,7 +591,7 @@ int xe_pm_runtime_suspend(struct xe_device *xe) · 590 unmodified lines
591	591	}
592	592
593	593	for_each_gt(gt, xe, id) {
594		− err = xe_gt_suspend(gt);
	594	+ err = xe->d3cold.allowed ? xe_gt_suspend(gt) : xe_gt_runtime_suspend(gt);
595	595	if (err)
596	596	goto out_resume;
597	597	}
@@ -633,10 +633,10 @@ int xe_pm_runtime_resume(struct xe_device *xe) · 35 unmodified lines
633	633
634	634	xe_rpm_lockmap_acquire(xe);
635	635
636		− for_each_gt(gt, xe, id)
637		− xe_gt_idle_disable_c6(gt);
638		−
639	636	if (xe->d3cold.allowed) {
	637	+ for_each_gt(gt, xe, id)
	638	+ xe_gt_idle_disable_c6(gt);
	639	+
640	640	err = xe_pcode_ready(xe, true);
641	641	if (err)
642	642	goto out;
@@ -657,7 +657,7 @@ int xe_pm_runtime_resume(struct xe_device *xe) · 14 unmodified lines
657	657	xe_irq_resume(xe);
658	658
659	659	for_each_gt(gt, xe, id)
660		− xe_gt_resume(gt);
	660	+ xe->d3cold.allowed ? xe_gt_resume(gt) : xe_gt_runtime_resume(gt);
661	661
662	662	xe_display_pm_runtime_resume(xe);
663	663

drivers/gpu/drm/xe/xe_pmu.c+1−1

@@ -425,7 +425,7 @@ static ssize_t event_attr_show(struct device *dev, · 424 unmodified lines
425	425	struct perf_pmu_events_attr *pmu_attr =
426	426	container_of(attr, struct perf_pmu_events_attr, attr);
427	427
428		− return sprintf(buf, "event=%#04llx\n", pmu_attr->id);
	428	+ return sysfs_emit(buf, "event=%#04llx\n", pmu_attr->id);
429	429	}
430	430
431	431	#define XE_EVENT_ATTR(name_, v_, id_) \

drivers/gpu/drm/xe/xe_pt.c+134−1

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include "xe_exec_queue.h"
13	13	#include "xe_gt.h"
14	14	#include "xe_migrate.h"
	15	+#include "xe_page_reclaim.h"
15	16	#include "xe_pt_types.h"
16	17	#include "xe_pt_walk.h"
17	18	#include "xe_res_cursor.h"
@@ -1535,6 +1536,9 @@ struct xe_pt_stage_unbind_walk { · 1517 unmodified lines
1535	1536	/** @modified_end: Walk range start, modified like @modified_start. */
1536	1537	u64 modified_end;
1537	1538
	1539	+ /** @prl: Backing pointer to page reclaim list in pt_update_ops */
	1540	+ struct xe_page_reclaim_list *prl;
	1541	+
1538	1542	/* Output */
1539	1543	/* @wupd: Structure to track the page-table updates we're building */
1540	1544	struct xe_walk_update wupd;
@@ -1572,6 +1576,68 @@ static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, · 31 unmodified lines
1572	1576	return false;
1573	1577	}
1574	1578
	1579	+/* Huge 2MB leaf lives directly in a level-1 table and has no children */
	1580	+static bool is_2m_pte(struct xe_pt *pte)
	1581	+{
	1582	+ return pte->level == 1 && !pte->base.children;
	1583	+}
	1584	+
	1585	+/* page_size = 2^(reclamation_size + XE_PTE_SHIFT) */
	1586	+#define COMPUTE_RECLAIM_ADDRESS_MASK(page_size) \
	1587	+({ \
	1588	+ BUILD_BUG_ON(!__builtin_constant_p(page_size)); \
	1589	+ ilog2(page_size) - XE_PTE_SHIFT; \
	1590	+})
	1591	+
	1592	+static int generate_reclaim_entry(struct xe_tile *tile,
	1593	+ struct xe_page_reclaim_list *prl,
	1594	+ u64 pte, struct xe_pt *xe_child)
	1595	+{
	1596	+ struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries;
	1597	+ u64 phys_page = (pte & XE_PTE_ADDR_MASK) >> XE_PTE_SHIFT;
	1598	+ int num_entries = prl->num_entries;
	1599	+ u32 reclamation_size;
	1600	+
	1601	+ xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL);
	1602	+ xe_tile_assert(tile, reclaim_entries);
	1603	+ xe_tile_assert(tile, num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1);
	1604	+
	1605	+ if (!xe_page_reclaim_list_valid(prl))
	1606	+ return -EINVAL;
	1607	+
	1608	+ /**
	1609	+ * reclamation_size indicates the size of the page to be
	1610	+ * invalidated and flushed from non-coherent cache.
	1611	+ * Page size is computed as 2^(reclamation_size + XE_PTE_SHIFT) bytes.
	1612	+ * Only 4K, 64K (level 0), and 2M pages are supported by hardware for page reclaim
	1613	+ */
	1614	+ if (xe_child->level == 0 && !(pte & XE_PTE_PS64)) {
	1615	+ reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_4K); /* reclamation_size = 0 */
	1616	+ } else if (xe_child->level == 0) {
	1617	+ reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_64K); /* reclamation_size = 4 */
	1618	+ } else if (is_2m_pte(xe_child)) {
	1619	+ reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_2M); /* reclamation_size = 9 */
	1620	+ } else {
	1621	+ xe_page_reclaim_list_invalidate(prl);
	1622	+ vm_dbg(&tile_to_xe(tile)->drm,
	1623	+ "PRL invalidate: unsupported PTE level=%u pte=%#llx\n",
	1624	+ xe_child->level, pte);
	1625	+ return -EINVAL;
	1626	+ }
	1627	+
	1628	+ reclaim_entries[num_entries].qw =
	1629	+ FIELD_PREP(XE_PAGE_RECLAIM_VALID, 1) \|
	1630	+ FIELD_PREP(XE_PAGE_RECLAIM_SIZE, reclamation_size) \|
	1631	+ FIELD_PREP(XE_PAGE_RECLAIM_ADDR_LO, phys_page) \|
	1632	+ FIELD_PREP(XE_PAGE_RECLAIM_ADDR_HI, phys_page >> 20);
	1633	+ prl->num_entries++;
	1634	+ vm_dbg(&tile_to_xe(tile)->drm,
	1635	+ "PRL add entry: level=%u pte=%#llx reclamation_size=%u prl_idx=%d\n",
	1636	+ xe_child->level, pte, reclamation_size, num_entries);
	1637	+
	1638	+ return 0;
	1639	+}
	1640	+
1575	1641	static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
1576	1642	unsigned int level, u64 addr, u64 next,
1577	1643	struct xe_ptw **child,
@@ -1579,11 +1645,48 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, · 1 unmodified line
1579	1645	struct xe_pt_walk *walk)
1580	1646	{
1581	1647	struct xe_pt xe_child = container_of(child, typeof(*xe_child), base);
	1648	+ struct xe_pt_stage_unbind_walk *xe_walk =
	1649	+ container_of(walk, typeof(*xe_walk), base);
	1650	+ struct xe_device *xe = tile_to_xe(xe_walk->tile);
1582	1651
1583	1652	XE_WARN_ON(!*child);
1584	1653	XE_WARN_ON(!level);
	1654	+ /* Check for leaf node */
	1655	+ if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) &&
	1656	+ !xe_child->base.children) {
	1657	+ struct iosys_map *leaf_map = &xe_child->bo->vmap;
	1658	+ pgoff_t first = xe_pt_offset(addr, 0, walk);
	1659	+ pgoff_t count = xe_pt_num_entries(addr, next, 0, walk);
	1660	+
	1661	+ for (pgoff_t i = 0; i < count; i++) {
	1662	+ u64 pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
	1663	+ int ret;
	1664	+
	1665	+ /* Account for NULL terminated entry on end (-1) */
	1666	+ if (xe_walk->prl->num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1) {
	1667	+ ret = generate_reclaim_entry(xe_walk->tile, xe_walk->prl,
	1668	+ pte, xe_child);
	1669	+ if (ret)
	1670	+ break;
	1671	+ } else {
	1672	+ /* overflow, mark as invalid */
	1673	+ xe_page_reclaim_list_invalidate(xe_walk->prl);
	1674	+ vm_dbg(&xe->drm,
	1675	+ "PRL invalidate: overflow while adding pte=%#llx",
	1676	+ pte);
	1677	+ break;
	1678	+ }
	1679	+ }
	1680	+ }
1585	1681
1586		− xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
	1682	+ /* If aborting page walk early, invalidate PRL since PTE may be dropped from this abort */
	1683	+ if (xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk) &&
	1684	+ xe_walk->prl && level > 1 && xe_child->base.children && xe_child->num_live != 0) {
	1685	+ xe_page_reclaim_list_invalidate(xe_walk->prl);
	1686	+ vm_dbg(&xe->drm,
	1687	+ "PRL invalidate: kill at level=%u addr=%#llx next=%#llx num_live=%u\n",
	1688	+ level, addr, next, xe_child->num_live);
	1689	+ }
1587	1690
1588	1691	return 0;
1589	1692	}
@@ -1654,6 +1757,8 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, · 64 unmodified lines
1654	1757	{
1655	1758	u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma);
1656	1759	u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma);
	1760	+ struct xe_vm_pgtable_update_op *pt_update_op =
	1761	+ container_of(entries, struct xe_vm_pgtable_update_op, entries[0]);
1657	1762	struct xe_pt_stage_unbind_walk xe_walk = {
1658	1763	.base = {
1659	1764	.ops = &xe_pt_stage_unbind_ops,
@@ -1665,6 +1770,7 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, · 5 unmodified lines
1665	1770	.modified_start = start,
1666	1771	.modified_end = end,
1667	1772	.wupd.entries = entries,
	1773	+ .prl = pt_update_op->prl,
1668	1774	};
1669	1775	struct xe_pt *pt = vm->pt_root[tile->id];
1670	1776
@@ -1897,6 +2003,7 @@ static int unbind_op_prepare(struct xe_tile *tile, · 226 unmodified lines
1897	2003	struct xe_vm_pgtable_update_ops *pt_update_ops,
1898	2004	struct xe_vma *vma)
1899	2005	{
	2006	+ struct xe_device *xe = tile_to_xe(tile);
1900	2007	u32 current_op = pt_update_ops->current_op;
1901	2008	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1902	2009	int err;
@@ -1914,6 +2021,17 @@ static int unbind_op_prepare(struct xe_tile *tile, · 11 unmodified lines
1914	2021	pt_op->vma = vma;
1915	2022	pt_op->bind = false;
1916	2023	pt_op->rebind = false;
	2024	+ /*
	2025	+ * Maintain one PRL located in pt_update_ops that all others in unbind op reference.
	2026	+ * Ensure that PRL is allocated only once, and if invalidated, remains an invalidated PRL.
	2027	+ */
	2028	+ if (xe->info.has_page_reclaim_hw_assist &&
	2029	+ xe_page_reclaim_list_is_new(&pt_update_ops->prl))
	2030	+ xe_page_reclaim_list_alloc_entries(&pt_update_ops->prl);
	2031	+
	2032	+ /* Page reclaim may not be needed due to other features, so skip the corresponding VMA */
	2033	+ pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl) &&
	2034	+ !xe_page_reclaim_skip(tile, vma)) ? &pt_update_ops->prl : NULL;
1917	2035
1918	2036	err = vma_reserve_fences(tile_to_xe(tile), vma);
1919	2037	if (err)
@@ -1979,6 +2097,7 @@ static int unbind_range_prepare(struct xe_vm *vm, · 59 unmodified lines
1979	2097	pt_op->vma = XE_INVALID_VMA;
1980	2098	pt_op->bind = false;
1981	2099	pt_op->rebind = false;
	2100	+ pt_op->prl = NULL;
1982	2101
1983	2102	pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,
1984	2103	pt_op->entries);
@@ -2096,6 +2215,7 @@ xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) · 111 unmodified lines
2096	2215	init_llist_head(&pt_update_ops->deferred);
2097	2216	pt_update_ops->start = ~0x0ull;
2098	2217	pt_update_ops->last = 0x0ull;
	2218	+ xe_page_reclaim_list_init(&pt_update_ops->prl);
2099	2219	}
2100	2220
2101	2221	/**
@@ -2393,6 +2513,17 @@ xe_pt_update_ops_run(struct xe_tile tile, struct xe_vma_ops vops) · 291 unmodified lines
2393	2513	goto kill_vm_tile1;
2394	2514	}
2395	2515	update.ijob = ijob;
	2516	+ /*
	2517	+ * Only add page reclaim for the primary GT. Media GT does not have
	2518	+ * any PPC to flush, so enabling the PPC flush bit for media is
	2519	+ * effectively a NOP and provides no performance benefit nor
	2520	+ * interfere with primary GT.
	2521	+ */
	2522	+ if (xe_page_reclaim_list_valid(&pt_update_ops->prl)) {
	2523	+ xe_tlb_inval_job_add_page_reclaim(ijob, &pt_update_ops->prl);
	2524	+ /* Release ref from alloc, job will now handle it */
	2525	+ xe_page_reclaim_list_invalidate(&pt_update_ops->prl);
	2526	+ }
2396	2527
2397	2528	if (tile->media_gt) {
2398	2529	dep_scheduler = to_dep_scheduler(q, tile->media_gt);
@@ -2518,6 +2649,8 @@ void xe_pt_update_ops_fini(struct xe_tile tile, struct xe_vma_ops vops) · 119 unmodified lines
2518	2649	&vops->pt_update_ops[tile->id];
2519	2650	int i;
2520	2651
	2652	+ xe_page_reclaim_entries_put(pt_update_ops->prl.entries);
	2653	+
2521	2654	lockdep_assert_held(&vops->vm->lock);
2522	2655	xe_vm_assert_held(vops->vm);
2523	2656

drivers/gpu/drm/xe/xe_pt_types.h+5−0

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8
9	9	#include <linux/types.h>
10	10
	11	+#include "xe_page_reclaim.h"
11	12	#include "xe_pt_walk.h"
12	13
13	14	struct xe_bo;
@@ -79,6 +80,8 @@ struct xe_vm_pgtable_update_op { · 65 unmodified lines
79	80	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
80	81	/** @vma: VMA for operation, operation not valid if NULL */
81	82	struct xe_vma *vma;
	83	+ /** @prl: Backing pointer to page reclaim list of pt_update_ops */
	84	+ struct xe_page_reclaim_list *prl;
82	85	/** @num_entries: number of entries for this update operation */
83	86	u32 num_entries;
84	87	/** @bind: is a bind */
@@ -95,6 +98,8 @@ struct xe_vm_pgtable_update_ops { · 10 unmodified lines
95	98	struct llist_head deferred;
96	99	/** @q: exec queue for PT operations */
97	100	struct xe_exec_queue *q;
	101	+ /** @prl: embedded page reclaim list */
	102	+ struct xe_page_reclaim_list prl;
98	103	/** @start: start address of ops */
99	104	u64 start;
100	105	/** @last: last address of ops */

drivers/gpu/drm/xe/xe_pxp.c+18−37

@@ -58,10 +58,9 @@ bool xe_pxp_is_enabled(const struct xe_pxp *pxp) · 57 unmodified lines
58	58	static bool pxp_prerequisites_done(const struct xe_pxp *pxp)
59	59	{
60	60	struct xe_gt *gt = pxp->gt;
61		− unsigned int fw_ref;
62	61	bool ready;
63	62
64		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	63	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
65	64
66	65	/*
67	66	* If force_wake fails we could falsely report the prerequisites as not
@@ -71,14 +70,12 @@ static bool pxp_prerequisites_done(const struct xe_pxp *pxp) · 3 unmodified lines
71	70	* PXP. Therefore, we can just log the force_wake error and not escalate
72	71	* it.
73	72	*/
74		− XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL));
	73	+ XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL));
75	74
76	75	/* PXP requires both HuC authentication via GSC and GSC proxy initialized */
77	76	ready = xe_huc_is_authenticated(&gt->uc.huc, XE_HUC_AUTH_VIA_GSC) &&
78	77	xe_gsc_proxy_init_done(&gt->uc.gsc);
79	78
80		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
81		−
82	79	return ready;
83	80	}
84	81
@@ -104,13 +101,12 @@ int xe_pxp_get_readiness_status(struct xe_pxp *pxp) · 19 unmodified lines
104	101	xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status))
105	102	return -EIO;
106	103
107		− xe_pm_runtime_get(pxp->xe);
	104	+ guard(xe_pm_runtime)(pxp->xe);
108	105
109	106	/* PXP requires both HuC loaded and GSC proxy initialized */
110	107	if (pxp_prerequisites_done(pxp))
111	108	ret = 1;
112	109
113		− xe_pm_runtime_put(pxp->xe);
114	110	return ret;
115	111	}
116	112
@@ -135,35 +131,28 @@ static void pxp_invalidate_queues(struct xe_pxp *pxp); · 18 unmodified lines
135	131	static int pxp_terminate_hw(struct xe_pxp *pxp)
136	132	{
137	133	struct xe_gt *gt = pxp->gt;
138		− unsigned int fw_ref;
139	134	int ret = 0;
140	135
141	136	drm_dbg(&pxp->xe->drm, "Terminating PXP\n");
142	137
143		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
144		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {
145		− ret = -EIO;
146		− goto out;
147		− }
	138	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	139	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
	140	+ return -EIO;
148	141
149	142	/* terminate the hw session */
150	143	ret = xe_pxp_submit_session_termination(pxp, ARB_SESSION);
151	144	if (ret)
152		− goto out;
	145	+ return ret;
153	146
154	147	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, false);
155	148	if (ret)
156		− goto out;
	149	+ return ret;
157	150
158	151	/* Trigger full HW cleanup */
159	152	xe_mmio_write32(&gt->mmio, KCR_GLOBAL_TERMINATE, 1);
160	153
161	154	/* now we can tell the GSC to clean up its own state */
162		− ret = xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION);
163		−
164		−out:
165		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
166		− return ret;
	155	+ return xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION);
167	156	}
168	157
169	158	static void mark_termination_in_progress(struct xe_pxp *pxp)
@@ -326,14 +315,12 @@ static int kcr_pxp_set_status(const struct xe_pxp *pxp, bool enable) · 156 unmodified lines
326	315	{
327	316	u32 val = enable ? _MASKED_BIT_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :
328	317	_MASKED_BIT_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);
329		− unsigned int fw_ref;
330	318
331		− fw_ref = xe_force_wake_get(gt_to_fw(pxp->gt), XE_FW_GT);
332		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT))
	319	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
	320	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
333	321	return -EIO;
334	322
335	323	xe_mmio_write32(&pxp->gt->mmio, KCR_INIT, val);
336		− xe_force_wake_put(gt_to_fw(pxp->gt), fw_ref);
337	324
338	325	return 0;
339	326	}
@@ -453,34 +440,28 @@ out: · 113 unmodified lines
453	440	static int __pxp_start_arb_session(struct xe_pxp *pxp)
454	441	{
455	442	int ret;
456		− unsigned int fw_ref;
457	443
458		− fw_ref = xe_force_wake_get(gt_to_fw(pxp->gt), XE_FW_GT);
459		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT))
	444	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
	445	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
460	446	return -EIO;
461	447
462		− if (pxp_session_is_in_play(pxp, ARB_SESSION)) {
463		− ret = -EEXIST;
464		− goto out_force_wake;
465		− }
	448	+ if (pxp_session_is_in_play(pxp, ARB_SESSION))
	449	+ return -EEXIST;
466	450
467	451	ret = xe_pxp_submit_session_init(&pxp->gsc_res, ARB_SESSION);
468	452	if (ret) {
469	453	drm_err(&pxp->xe->drm, "Failed to init PXP arb session: %pe\n", ERR_PTR(ret));
470		− goto out_force_wake;
	454	+ return ret;
471	455	}
472	456
473	457	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true);
474	458	if (ret) {
475	459	drm_err(&pxp->xe->drm, "PXP ARB session failed to go in play%pe\n", ERR_PTR(ret));
476		− goto out_force_wake;
	460	+ return ret;
477	461	}
478	462
479	463	drm_dbg(&pxp->xe->drm, "PXP ARB session is active\n");
480		−
481		−out_force_wake:
482		− xe_force_wake_put(gt_to_fw(pxp->gt), fw_ref);
483		− return ret;
	464	+ return 0;
484	465	}
485	466
486	467	/**

drivers/gpu/drm/xe/xe_query.c+12−11

@@ -122,7 +122,6 @@ query_engine_cycles(struct xe_device *xe, · 121 unmodified lines
122	122	__ktime_func_t cpu_clock;
123	123	struct xe_hw_engine *hwe;
124	124	struct xe_gt *gt;
125		− unsigned int fw_ref;
126	125
127	126	if (IS_SRIOV_VF(xe))
128	127	return -EOPNOTSUPP;
@@ -158,16 +157,13 @@ query_engine_cycles(struct xe_device *xe, · 29 unmodified lines
158	157	if (!hwe)
159	158	return -EINVAL;
160	159
161		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
162		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
163		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
164		− return -EIO;
165		− }
166		−
167		− hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp,
168		− &resp.cpu_delta, cpu_clock);
	160	+ xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL) {
	161	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
	162	+ return -EIO;
169	163
170		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
	164	+ hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp,
	165	+ &resp.cpu_delta, cpu_clock);
	166	+ }
171	167
172	168	if (GRAPHICS_VER(xe) >= 20)
173	169	resp.width = 64;
@@ -342,6 +338,9 @@ static int query_config(struct xe_device xe, struct drm_xe_device_query query) · 168 unmodified lines
342	338	if (xe->info.has_usm && IS_ENABLED(CONFIG_DRM_XE_GPUSVM))
343	339	config->info[DRM_XE_QUERY_CONFIG_FLAGS] \|=
344	340	DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR;
	341	+ if (GRAPHICS_VER(xe) >= 20)
	342	+ config->info[DRM_XE_QUERY_CONFIG_FLAGS] \|=
	343	+ DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT;
345	344	config->info[DRM_XE_QUERY_CONFIG_FLAGS] \|=
346	345	DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY;
347	346	config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] =
@@ -686,7 +685,9 @@ static int query_oa_units(struct xe_device *xe, · 338 unmodified lines
686	685	du->capabilities = DRM_XE_OA_CAPS_BASE \| DRM_XE_OA_CAPS_SYNCS \|
687	686	DRM_XE_OA_CAPS_OA_BUFFER_SIZE \|
688	687	DRM_XE_OA_CAPS_WAIT_NUM_REPORTS \|
689		− DRM_XE_OA_CAPS_OAM;
	688	+ DRM_XE_OA_CAPS_OAM \|
	689	+ DRM_XE_OA_CAPS_OA_UNIT_GT_ID;
	690	+ du->gt_id = u->gt->info.id;
690	691	j = 0;
691	692	for_each_hw_engine(hwe, gt, hwe_id) {
692	693	if (!xe_hw_engine_is_reserved(hwe) &&

drivers/gpu/drm/xe/xe_reg_sr.c+5−12

@@ -168,7 +168,6 @@ void xe_reg_sr_apply_mmio(struct xe_reg_sr sr, struct xe_gt gt) · 167 unmodified lines
168	168	{
169	169	struct xe_reg_sr_entry *entry;
170	170	unsigned long reg;
171		− unsigned int fw_ref;
172	171
173	172	if (xa_empty(&sr->xa))
174	173	return;
@@ -178,20 +177,14 @@ void xe_reg_sr_apply_mmio(struct xe_reg_sr sr, struct xe_gt gt) · 3 unmodified lines
178	177
179	178	xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
180	179
181		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
182		− if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
183		− goto err_force_wake;
	180	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
	181	+ if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
	182	+ xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n");
	183	+ return;
	184	+ }
184	185
185	186	xa_for_each(&sr->xa, reg, entry)
186	187	apply_one_mmio(gt, entry);
187		−
188		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
189		−
190		− return;
191		−
192		−err_force_wake:
193		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
194		− xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n");
195	188	}
196	189
197	190	/**

drivers/gpu/drm/xe/xe_reg_whitelist.c+59−22

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#include "regs/xe_gt_regs.h"
10	10	#include "regs/xe_oa_regs.h"
11	11	#include "regs/xe_regs.h"
	12	+#include "xe_device.h"
12	13	#include "xe_gt_types.h"
13	14	#include "xe_gt_printk.h"
14	15	#include "xe_platform_types.h"
@@ -26,6 +27,13 @@ static bool match_not_render(const struct xe_device *xe, · 11 unmodified lines
26	27	return hwe->class != XE_ENGINE_CLASS_RENDER;
27	28	}
28	29
	30	+static bool match_has_mert(const struct xe_device *xe,
	31	+ const struct xe_gt *gt,
	32	+ const struct xe_hw_engine *hwe)
	33	+{
	34	+ return xe_device_has_mert((struct xe_device *)xe);
	35	+}
	36	+
29	37	static const struct xe_rtp_entry_sr register_whitelist[] = {
30	38	{ XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"),
31	39	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),
@@ -67,28 +75,6 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { · 35 unmodified lines
67	75	ENGINE_CLASS(RENDER)),
68	76	XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0))
69	77	},
70		− { XE_RTP_NAME("oa_reg_render"),
71		− XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
72		− ENGINE_CLASS(RENDER)),
73		− XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER,
74		− RING_FORCE_TO_NONPRIV_ACCESS_RW),
75		− WHITELIST(OAG_OASTATUS,
76		− RING_FORCE_TO_NONPRIV_ACCESS_RD),
77		− WHITELIST(OAG_OAHEADPTR,
78		− RING_FORCE_TO_NONPRIV_ACCESS_RD \|
79		− RING_FORCE_TO_NONPRIV_RANGE_4))
80		− },
81		− { XE_RTP_NAME("oa_reg_compute"),
82		− XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
83		− ENGINE_CLASS(COMPUTE)),
84		− XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER,
85		− RING_FORCE_TO_NONPRIV_ACCESS_RW),
86		− WHITELIST(OAG_OASTATUS,
87		− RING_FORCE_TO_NONPRIV_ACCESS_RD),
88		− WHITELIST(OAG_OAHEADPTR,
89		− RING_FORCE_TO_NONPRIV_ACCESS_RD \|
90		− RING_FORCE_TO_NONPRIV_RANGE_4))
91		− },
92	78	{ XE_RTP_NAME("14024997852"),
93	79	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),
94	80	XE_RTP_ACTIONS(WHITELIST(FF_MODE,
@@ -96,6 +82,57 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { · 1 unmodified line
96	82	WHITELIST(VFLSKPD,
97	83	RING_FORCE_TO_NONPRIV_ACCESS_RW))
98	84	},
	85	+
	86	+#define WHITELIST_OA_MMIO_TRG(trg, status, head) \
	87	+ WHITELIST(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \
	88	+ WHITELIST(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \
	89	+ WHITELIST(head, RING_FORCE_TO_NONPRIV_ACCESS_RD \| RING_FORCE_TO_NONPRIV_RANGE_4)
	90	+
	91	+#define WHITELIST_OAG_MMIO_TRG \
	92	+ WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR)
	93	+
	94	+#define WHITELIST_OAM_MMIO_TRG \
	95	+ WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SAG_BASE_ADJ), \
	96	+ OAM_STATUS(XE_OAM_SAG_BASE_ADJ), \
	97	+ OAM_HEAD_POINTER(XE_OAM_SAG_BASE_ADJ)), \
	98	+ WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_0_BASE_ADJ), \
	99	+ OAM_STATUS(XE_OAM_SCMI_0_BASE_ADJ), \
	100	+ OAM_HEAD_POINTER(XE_OAM_SCMI_0_BASE_ADJ)), \
	101	+ WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_1_BASE_ADJ), \
	102	+ OAM_STATUS(XE_OAM_SCMI_1_BASE_ADJ), \
	103	+ OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ))
	104	+
	105	+#define WHITELIST_OA_MERT_MMIO_TRG \
	106	+ WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_HEAD_POINTER)
	107	+
	108	+ { XE_RTP_NAME("oag_mmio_trg_rcs"),
	109	+ XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
	110	+ ENGINE_CLASS(RENDER)),
	111	+ XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG)
	112	+ },
	113	+ { XE_RTP_NAME("oag_mmio_trg_ccs"),
	114	+ XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
	115	+ ENGINE_CLASS(COMPUTE)),
	116	+ XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG)
	117	+ },
	118	+ { XE_RTP_NAME("oam_mmio_trg_vcs"),
	119	+ XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED),
	120	+ ENGINE_CLASS(VIDEO_DECODE)),
	121	+ XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG)
	122	+ },
	123	+ { XE_RTP_NAME("oam_mmio_trg_vecs"),
	124	+ XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED),
	125	+ ENGINE_CLASS(VIDEO_ENHANCE)),
	126	+ XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG)
	127	+ },
	128	+ { XE_RTP_NAME("oa_mert_mmio_trg_ccs"),
	129	+ XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COMPUTE)),
	130	+ XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG)
	131	+ },
	132	+ { XE_RTP_NAME("oa_mert_mmio_trg_bcs"),
	133	+ XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COPY)),
	134	+ XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG)
	135	+ },
99	136	};
100	137
101	138	static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe)

drivers/gpu/drm/xe/xe_ring_ops.c+34−34

@@ -12,7 +12,7 @@ · 11 unmodified lines
12	12	#include "regs/xe_engine_regs.h"
13	13	#include "regs/xe_gt_regs.h"
14	14	#include "regs/xe_lrc_layout.h"
15		−#include "xe_exec_queue_types.h"
	15	+#include "xe_exec_queue.h"
16	16	#include "xe_gt.h"
17	17	#include "xe_lrc.h"
18	18	#include "xe_macros.h"
@@ -135,12 +135,11 @@ emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, · 116 unmodified lines
135	135	return i;
136	136	}
137	137
138		−static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
139		− int i)
	138	+static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags,
	139	+ bool invalidate_tlb, u32 *dw, int i)
140	140	{
141	141	u32 flags0 = 0;
142		− u32 flags1 = PIPE_CONTROL_CS_STALL \|
143		− PIPE_CONTROL_COMMAND_CACHE_INVALIDATE \|
	142	+ u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE \|
144	143	PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE \|
145	144	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|
146	145	PIPE_CONTROL_VF_CACHE_INVALIDATE \|
@@ -152,6 +151,11 @@ static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw, · 5 unmodified lines
152	151	if (invalidate_tlb)
153	152	flags1 \|= PIPE_CONTROL_TLB_INVALIDATE;
154	153
	154	+ if (xe_exec_queue_is_multi_queue(q))
	155	+ flags0 \|= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
	156	+ else
	157	+ flags1 \|= PIPE_CONTROL_CS_STALL;
	158	+
155	159	flags1 &= ~mask_flags;
156	160
157	161	if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)
@@ -175,54 +179,52 @@ static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, · 17 unmodified lines
175	179
176	180	static int emit_render_cache_flush(struct xe_sched_job job, u32 dw, int i)
177	181	{
178		− struct xe_gt *gt = job->q->gt;
	182	+ struct xe_exec_queue *q = job->q;
	183	+ struct xe_gt *gt = q->gt;
179	184	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
180		− u32 flags;
	185	+ u32 flags0, flags1;
181	186
182	187	if (XE_GT_WA(gt, 14016712196))
183	188	i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
184	189	LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
185	190
186		− flags = (PIPE_CONTROL_CS_STALL \|
187		− PIPE_CONTROL_TILE_CACHE_FLUSH \|
	191	+ flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH;
	192	+ flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH \|
188	193	PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH \|
189	194	PIPE_CONTROL_DEPTH_CACHE_FLUSH \|
190	195	PIPE_CONTROL_DC_FLUSH_ENABLE \|
191	196	PIPE_CONTROL_FLUSH_ENABLE);
192	197
193	198	if (XE_GT_WA(gt, 1409600907))
194		− flags \|= PIPE_CONTROL_DEPTH_STALL;
	199	+ flags1 \|= PIPE_CONTROL_DEPTH_STALL;
195	200
196	201	if (lacks_render)
197		− flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
	202	+ flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
198	203	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
199		− flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
	204	+ flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
200	205
201		− return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
202		−}
203		−
204		−static int emit_pipe_control_to_ring_end(struct xe_hw_engine hwe, u32 dw, int i)
205		−{
206		− if (hwe->class != XE_ENGINE_CLASS_RENDER)
207		− return i;
	206	+ if (xe_exec_queue_is_multi_queue(q))
	207	+ flags0 \|= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
	208	+ else
	209	+ flags1 \|= PIPE_CONTROL_CS_STALL;
208	210
209		− if (XE_GT_WA(hwe->gt, 16020292621))
210		− i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
211		− RING_NOPID(hwe->mmio_base).addr, 0);
212		−
213		− return i;
	211	+ return emit_pipe_control(dw, i, flags0, flags1, 0, 0);
214	212	}
215	213
216		−static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
217		− int i)
	214	+static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value,
	215	+ bool stall_only, u32 *dw, int i)
218	216	{
219		− u32 flags = PIPE_CONTROL_CS_STALL \| PIPE_CONTROL_GLOBAL_GTT_IVB \|
220		− PIPE_CONTROL_QW_WRITE;
	217	+ u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB \| PIPE_CONTROL_QW_WRITE;
221	218
222	219	if (!stall_only)
223		− flags \|= PIPE_CONTROL_FLUSH_ENABLE;
	220	+ flags1 \|= PIPE_CONTROL_FLUSH_ENABLE;
224	221
225		− return emit_pipe_control(dw, i, 0, flags, addr, value);
	222	+ if (xe_exec_queue_is_multi_queue(q))
	223	+ flags0 \|= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
	224	+ else
	225	+ flags1 \|= PIPE_CONTROL_CS_STALL;
	226	+
	227	+ return emit_pipe_control(dw, i, flags0, flags1, addr, value);
226	228	}
227	229
228	230	static u32 get_ppgtt_flag(struct xe_sched_job *job)
@@ -371,7 +373,7 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, · 142 unmodified lines
371	373	mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
372	374
373	375	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
374		− i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
	376	+ i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i);
375	377
376	378	/* hsdes: 1809175790 */
377	379	if (has_aux_ccs(xe))
@@ -391,12 +393,10 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, · 13 unmodified lines
391	393	job->user_fence.value,
392	394	dw, i);
393	395
394		− i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
	396	+ i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
395	397
396	398	i = emit_user_interrupt(dw, i);
397	399
398		− i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
399		−
400	400	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
401	401
402	402	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));

drivers/gpu/drm/xe/xe_sa.c+66−1

@@ -29,6 +29,7 @@ static void xe_sa_bo_manager_fini(struct drm_device drm, void arg) · 28 unmodified lines
29	29	kvfree(sa_manager->cpu_ptr);
30	30
31	31	sa_manager->bo = NULL;
	32	+ sa_manager->shadow = NULL;
32	33	}
33	34
34	35	/**
@@ -37,12 +38,14 @@ static void xe_sa_bo_manager_fini(struct drm_device drm, void arg) · 2 unmodified lines
37	38	* @size: number of bytes to allocate
38	39	* @guard: number of bytes to exclude from suballocations
39	40	* @align: alignment for each suballocated chunk
	41	+ * @flags: flags for suballocator
40	42	*
41	43	* Prepares the suballocation manager for suballocations.
42	44	*
43	45	* Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure.
44	46	*/
45		−struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u32 guard, u32 align)
	47	+struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size,
	48	+ u32 guard, u32 align, u32 flags)
46	49	{
47	50	struct xe_device *xe = tile_to_xe(tile);
48	51	struct xe_sa_manager *sa_manager;
@@ -79,6 +82,26 @@ struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u3 · 30 unmodified lines
79	82	memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size);
80	83	}
81	84
	85	+ if (flags & XE_SA_BO_MANAGER_FLAG_SHADOW) {
	86	+ struct xe_bo *shadow;
	87	+
	88	+ ret = drmm_mutex_init(&xe->drm, &sa_manager->swap_guard);
	89	+ if (ret)
	90	+ return ERR_PTR(ret);
	91	+
	92	+ shadow = xe_managed_bo_create_pin_map(xe, tile, size,
	93	+ XE_BO_FLAG_VRAM_IF_DGFX(tile) \|
	94	+ XE_BO_FLAG_GGTT \|
	95	+ XE_BO_FLAG_GGTT_INVALIDATE \|
	96	+ XE_BO_FLAG_PINNED_NORESTORE);
	97	+ if (IS_ERR(shadow)) {
	98	+ drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n",
	99	+ size / SZ_1K, shadow);
	100	+ return ERR_CAST(shadow);
	101	+ }
	102	+ sa_manager->shadow = shadow;
	103	+ }
	104	+
82	105	drm_suballoc_manager_init(&sa_manager->base, managed_size, align);
83	106	ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini,
84	107	sa_manager);
@@ -88,6 +111,48 @@ struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u3 · 3 unmodified lines
88	111	return sa_manager;
89	112	}
90	113
	114	+/**
	115	+ * xe_sa_bo_swap_shadow() - Swap the SA BO with shadow BO.
	116	+ * @sa_manager: the XE sub allocator manager
	117	+ *
	118	+ * Swaps the sub-allocator primary buffer object with shadow buffer object.
	119	+ *
	120	+ * Return: None.
	121	+ */
	122	+void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager)
	123	+{
	124	+ struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
	125	+
	126	+ xe_assert(xe, sa_manager->shadow);
	127	+ lockdep_assert_held(&sa_manager->swap_guard);
	128	+
	129	+ swap(sa_manager->bo, sa_manager->shadow);
	130	+ if (!sa_manager->bo->vmap.is_iomem)
	131	+ sa_manager->cpu_ptr = sa_manager->bo->vmap.vaddr;
	132	+}
	133	+
	134	+/**
	135	+ * xe_sa_bo_sync_shadow() - Sync the SA Shadow BO with primary BO.
	136	+ * @sa_bo: the sub-allocator buffer object.
	137	+ *
	138	+ * Synchronize sub-allocator shadow buffer object with primary buffer object.
	139	+ *
	140	+ * Return: None.
	141	+ */
	142	+void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo)
	143	+{
	144	+ struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager);
	145	+ struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
	146	+
	147	+ xe_assert(xe, sa_manager->shadow);
	148	+ lockdep_assert_held(&sa_manager->swap_guard);
	149	+
	150	+ xe_map_memcpy_to(xe, &sa_manager->shadow->vmap,
	151	+ drm_suballoc_soffset(sa_bo),
	152	+ xe_sa_bo_cpu_addr(sa_bo),
	153	+ drm_suballoc_size(sa_bo));
	154	+}
	155	+
91	156	/**
92	157	* __xe_sa_bo_new() - Make a suballocation but use custom gfp flags.
93	158	* @sa_manager: the &xe_sa_manager

drivers/gpu/drm/xe/xe_sa.h+18−2

@@ -14,12 +14,14 @@ · 13 unmodified lines
14	14	struct dma_fence;
15	15	struct xe_tile;
16	16
17		−struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u32 guard, u32 align);
	17	+#define XE_SA_BO_MANAGER_FLAG_SHADOW BIT(0)
	18	+struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size,
	19	+ u32 guard, u32 align, u32 flags);
18	20	struct drm_suballoc __xe_sa_bo_new(struct xe_sa_manager sa_manager, u32 size, gfp_t gfp);
19	21
20	22	static inline struct xe_sa_manager xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u32 align)
21	23	{
22		− return __xe_sa_bo_manager_init(tile, size, SZ_4K, align);
	24	+ return __xe_sa_bo_manager_init(tile, size, SZ_4K, align, 0);
23	25	}
24	26
25	27	/**
@@ -69,4 +71,18 @@ static inline void xe_sa_bo_cpu_addr(struct drm_suballoc sa) · 43 unmodified lines
69	71	drm_suballoc_soffset(sa);
70	72	}
71	73
	74	+void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager);
	75	+void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo);
	76	+
	77	+/**
	78	+ * xe_sa_bo_swap_guard() - Retrieve the SA BO swap guard within sub-allocator.
	79	+ * @sa_manager: the &xe_sa_manager
	80	+ *
	81	+ * Return: Sub alloctor swap guard mutex.
	82	+ */
	83	+static inline struct mutex xe_sa_bo_swap_guard(struct xe_sa_manager sa_manager)
	84	+{
	85	+ return &sa_manager->swap_guard;
	86	+}
	87	+
72	88	#endif

drivers/gpu/drm/xe/xe_sa_types.h+3−0

@@ -12,6 +12,9 @@ struct xe_bo; · 11 unmodified lines
12	12	struct xe_sa_manager {
13	13	struct drm_suballoc_manager base;
14	14	struct xe_bo *bo;
	15	+ struct xe_bo *shadow;
	16	+ /** @swap_guard: Timeline guard updating @bo and @shadow */
	17	+ struct mutex swap_guard;
15	18	void *cpu_ptr;
16	19	bool is_iomem;
17	20	};

drivers/gpu/drm/xe/xe_sriov_packet.c+1−1

@@ -358,7 +358,7 @@ ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, · 357 unmodified lines
358	358
359	359	#define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \
360	360	GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN)
361		−static size_t pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
	361	+static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
362	362	{
363	363	struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid);
364	364	struct xe_sriov_packet *data;

drivers/gpu/drm/xe/xe_sriov_pf.c+4−0

@@ -90,6 +90,7 @@ bool xe_sriov_pf_readiness(struct xe_device *xe) · 89 unmodified lines
90	90	*/
91	91	int xe_sriov_pf_init_early(struct xe_device *xe)
92	92	{
	93	+ struct xe_mert *mert = &xe_device_get_root_tile(xe)->mert;
93	94	int err;
94	95
95	96	xe_assert(xe, IS_SRIOV_PF(xe));
@@ -111,6 +112,9 @@ int xe_sriov_pf_init_early(struct xe_device *xe) · 15 unmodified lines
111	112
112	113	xe_sriov_pf_service_init(xe);
113	114
	115	+ spin_lock_init(&mert->lock);
	116	+ init_completion(&mert->tlb_inv_done);
	117	+
114	118	return 0;
115	119	}
116	120

drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c+2−4

@@ -70,9 +70,8 @@ static ssize_t from_file_write_to_xe_call(struct file file, const char __user · 69 unmodified lines
70	70	if (ret < 0)
71	71	return ret;
72	72	if (yes) {
73		− xe_pm_runtime_get(xe);
	73	+ guard(xe_pm_runtime)(xe);
74	74	ret = call(xe);
75		− xe_pm_runtime_put(xe);
76	75	}
77	76	if (ret < 0)
78	77	return ret;
@@ -209,9 +208,8 @@ static ssize_t from_file_write_to_vf_call(struct file file, const char __user · 130 unmodified lines
209	208	if (ret < 0)
210	209	return ret;
211	210	if (yes) {
212		− xe_pm_runtime_get(xe);
	211	+ guard(xe_pm_runtime)(xe);
213	212	ret = call(xe, vfid);
214		− xe_pm_runtime_put(xe);
215	213	}
216	214	if (ret < 0)
217	215	return ret;

drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c+4−12

@@ -389,16 +389,12 @@ static ssize_t xe_sriov_dev_attr_store(struct kobject kobj, struct attribute a · 388 unmodified lines
389	389	struct xe_sriov_dev_attr *vattr = to_xe_sriov_dev_attr(attr);
390	390	struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj);
391	391	struct xe_device *xe = vkobj->xe;
392		− ssize_t ret;
393	392
394	393	if (!vattr->store)
395	394	return -EPERM;
396	395
397		− xe_pm_runtime_get(xe);
398		− ret = xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count);
399		− xe_pm_runtime_put(xe);
400		−
401		− return ret;
	396	+ guard(xe_pm_runtime)(xe);
	397	+ return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count);
402	398	}
403	399
404	400	static ssize_t xe_sriov_vf_attr_show(struct kobject kobj, struct attribute attr, char *buf)
@@ -423,18 +419,14 @@ static ssize_t xe_sriov_vf_attr_store(struct kobject kobj, struct attribute at · 18 unmodified lines
423	419	struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj);
424	420	struct xe_device *xe = vkobj->xe;
425	421	unsigned int vfid = vkobj->vfid;
426		− ssize_t ret;
427	422
428	423	xe_sriov_pf_assert_vfid(xe, vfid);
429	424
430	425	if (!vattr->store)
431	426	return -EPERM;
432	427
433		− xe_pm_runtime_get(xe);
434		− ret = xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count);
435		− xe_pm_runtime_get(xe);
436		−
437		− return ret;
	428	+ guard(xe_pm_runtime)(xe);
	429	+ return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count);
438	430	}
439	431
440	432	static const struct sysfs_ops xe_sriov_dev_sysfs_ops = {

drivers/gpu/drm/xe/xe_sriov_vf.c+80−4

@@ -49,11 +49,13 @@ · 48 unmodified lines
49	49	*
50	50	* As soon as Virtual GPU of the VM starts, the VF driver within receives
51	51	* the MIGRATED interrupt and schedules post-migration recovery worker.
52		− * That worker queries GuC for new provisioning (using MMIO communication),
	52	+ * That worker sends `VF2GUC_RESFIX_START` action along with non-zero
	53	+ * marker, queries GuC for new provisioning (using MMIO communication),
53	54	* and applies fixups to any non-virtualized resources used by the VF.
54	55	*
55	56	* When the VF driver is ready to continue operation on the newly connected
56		− * hardware, it sends `VF2GUC_NOTIFY_RESFIX_DONE` which causes it to
	57	+ * hardware, it sends `VF2GUC_RESFIX_DONE` action along with the same
	58	+ * marker which was sent with `VF2GUC_RESFIX_START` which causes it to
57	59	* enter the long awaited `VF_RUNNING` state, and therefore start handling
58	60	* CTB messages and scheduling workloads from the VF::
59	61	*
@@ -102,12 +104,17 @@ · 42 unmodified lines
102	104	* \| [ ] new VF provisioning [ ]
103	105	* \| [ ]---------------------------> [ ]
104	106	* \| \| [ ]
	107	+ * \| \| VF2GUC_RESFIX_START [ ]
	108	+ * \| [ ] <---------------------------[ ]
	109	+ * \| [ ] [ ]
	110	+ * \| [ ] success [ ]
	111	+ * \| [ ]---------------------------> [ ]
105	112	* \| \| VF driver applies post [ ]
106	113	* \| \| migration fixups -------[ ]
107	114	* \| \| \| [ ]
108	115	* \| \| -----> [ ]
109	116	* \| \| [ ]
110		− * \| \| VF2GUC_NOTIFY_RESFIX_DONE [ ]
	117	+ * \| \| VF2GUC_RESFIX_DONE [ ]
111	118	* \| [ ] <---------------------------[ ]
112	119	* \| [ ] [ ]
113	120	* \| [ ] GuC sets new VF state to [ ]
@@ -118,6 +125,55 @@ · 4 unmodified lines
118	125	* \| [ ]---------------------------> [ ]
119	126	* \| \| \|
120	127	* \| \| \|
	128	+ *
	129	+ * Handling of VF double migration flow is shown below::
	130	+ *
	131	+ * GuC1 VF
	132	+ * \| \|
	133	+ * \| [ ]<--- start fixups
	134	+ * \| VF2GUC_RESFIX_START(marker) [ ]
	135	+ * [ ] <-------------------------------------------[ ]
	136	+ * [ ] [ ]
	137	+ * [ ]---\ [ ]
	138	+ * [ ] store marker [ ]
	139	+ * [ ]<--/ [ ]
	140	+ * [ ] [ ]
	141	+ * [ ] success [ ]
	142	+ * [ ] ------------------------------------------> [ ]
	143	+ * \| [ ]
	144	+ * \| [ ]---\
	145	+ * \| [ ] do fixups
	146	+ * \| [ ]<--/
	147	+ * \| [ ]
	148	+ * -------------- VF paused / saved ----------------
	149	+ * :
	150	+ *
	151	+ * GuC2
	152	+ * \|
	153	+ * ----------------- VF restored ------------------
	154	+ * \|
	155	+ * [ ]
	156	+ * [ ]---\
	157	+ * [ ] reset marker
	158	+ * [ ]<--/
	159	+ * [ ]
	160	+ * ----------------- VF resumed ------------------
	161	+ * \| [ ]
	162	+ * \| [ ]
	163	+ * \| VF2GUC_RESFIX_DONE(marker) [ ]
	164	+ * [ ] <-------------------------------------------[ ]
	165	+ * [ ] [ ]
	166	+ * [ ]---\ [ ]
	167	+ * [ ] check marker [ ]
	168	+ * [ ] (mismatch) [ ]
	169	+ * [ ]<--/ [ ]
	170	+ * [ ] [ ]
	171	+ * [ ] RESPONSE_VF_MIGRATED [ ]
	172	+ * [ ] ------------------------------------------> [ ]
	173	+ * \| [ ]---\
	174	+ * \| [ ] reschedule fixups
	175	+ * \| [ ]<--/
	176	+ * \| \|
121	177	*/
122	178
123	179	/**
@@ -170,6 +226,26 @@ void xe_sriov_vf_init_early(struct xe_device *xe) · 46 unmodified lines
170	226	vf_migration_init_early(xe);
171	227	}
172	228
	229	+static int vf_migration_init_late(struct xe_device *xe)
	230	+{
	231	+ struct xe_gt *gt = xe_root_mmio_gt(xe);
	232	+ struct xe_uc_fw_version guc_version;
	233	+
	234	+ if (!xe_sriov_vf_migration_supported(xe))
	235	+ return 0;
	236	+
	237	+ xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version);
	238	+ if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 27, 0)) {
	239	+ xe_sriov_vf_migration_disable(xe,
	240	+ "requires GuC ABI >= 1.27.0, but only %u.%u.%u found",
	241	+ guc_version.major, guc_version.minor,
	242	+ guc_version.patch);
	243	+ return 0;
	244	+ }
	245	+
	246	+ return xe_sriov_vf_ccs_init(xe);
	247	+}
	248	+
173	249	/**
174	250	* xe_sriov_vf_init_late() - SR-IOV VF late initialization functions.
175	251	* @xe: the &xe_device to initialize
@@ -180,7 +256,7 @@ void xe_sriov_vf_init_early(struct xe_device *xe) · 4 unmodified lines
180	256	*/
181	257	int xe_sriov_vf_init_late(struct xe_device *xe)
182	258	{
183		− return xe_sriov_vf_ccs_init(xe);
	259	+ return vf_migration_init_late(xe);
184	260	}
185	261
186	262	static int sa_info_vf_ccs(struct seq_file m, void data)

drivers/gpu/drm/xe/xe_sriov_vf_ccs.c+19−8

@@ -150,7 +150,8 @@ static int alloc_bb_pool(struct xe_tile tile, struct xe_sriov_vf_ccs_ctx ctx) · 149 unmodified lines
150	150	xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n",
151	151	ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M);
152	152
153		− sa_manager = xe_sa_bo_manager_init(tile, bb_pool_size, SZ_16);
	153	+ sa_manager = __xe_sa_bo_manager_init(tile, bb_pool_size, SZ_4K, SZ_16,
	154	+ XE_SA_BO_MANAGER_FLAG_SHADOW);
154	155
155	156	if (IS_ERR(sa_manager)) {
156	157	xe_sriov_err(xe, "Suballocator init failed with error: %pe\n",
@@ -162,9 +163,12 @@ static int alloc_bb_pool(struct xe_tile tile, struct xe_sriov_vf_ccs_ctx ctx) · 5 unmodified lines
162	163	offset = 0;
163	164	xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP,
164	165	bb_pool_size);
	166	+ xe_map_memset(xe, &sa_manager->shadow->vmap, offset, MI_NOOP,
	167	+ bb_pool_size);
165	168
166	169	offset = bb_pool_size - sizeof(u32);
167	170	xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END);
	171	+ xe_map_wr(xe, &sa_manager->shadow->vmap, offset, u32, MI_BATCH_BUFFER_END);
168	172
169	173	ctx->mem.ccs_bb_pool = sa_manager;
170	174
@@ -381,6 +385,18 @@ err_ret: · 210 unmodified lines
381	385	return err;
382	386	}
383	387
	388	+#define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET (2 * sizeof(u32))
	389	+void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx)
	390	+{
	391	+ u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool);
	392	+ struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
	393	+ struct xe_device *xe = gt_to_xe(ctx->mig_q->gt);
	394	+
	395	+ xe_device_wmb(xe);
	396	+ xe_map_wr(xe, &lrc->bo->vmap, XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET, u32, addr);
	397	+ xe_device_wmb(xe);
	398	+}
	399	+
384	400	/**
385	401	* xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO.
386	402	* @bo: the &buffer object to which batch buffer commands will be added.
@@ -441,9 +457,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo) · 54 unmodified lines
441	457	if (!bb)
442	458	continue;
443	459
444		− memset(bb->cs, MI_NOOP, bb->len * sizeof(u32));
445		− xe_bb_free(bb, NULL);
446		− bo->bb_ccs[ctx_id] = NULL;
	460	+ xe_migrate_ccs_rw_copy_clear(bo, ctx_id);
447	461	}
448	462	return 0;
449	463	}
@@ -463,8 +477,7 @@ void xe_sriov_vf_ccs_print(struct xe_device xe, struct drm_printer p) · 13 unmodified lines
463	477	if (!IS_VF_CCS_READY(xe))
464	478	return;
465	479
466		− xe_pm_runtime_get(xe);
467		−
	480	+ guard(xe_pm_runtime)(xe);
468	481	for_each_ccs_rw_ctx(ctx_id) {
469	482	bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool;
470	483	if (!bb_pool)
@@ -475,6 +488,4 @@ void xe_sriov_vf_ccs_print(struct xe_device xe, struct drm_printer p) · 4 unmodified lines
475	488	drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool));
476	489	drm_puts(p, "\n");
477	490	}
478		−
479		− xe_pm_runtime_put(xe);
480	491	}

drivers/gpu/drm/xe/xe_sriov_vf_ccs.h+1−0

@@ -20,6 +20,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo); · 19 unmodified lines
20	20	int xe_sriov_vf_ccs_register_context(struct xe_device *xe);
21	21	void xe_sriov_vf_ccs_rebase(struct xe_device *xe);
22	22	void xe_sriov_vf_ccs_print(struct xe_device xe, struct drm_printer p);
	23	+void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx);
23	24
24	25	static inline bool xe_sriov_vf_ccs_ready(struct xe_device *xe)
25	26	{

drivers/gpu/drm/xe/xe_survivability_mode.c+180−93

@@ -16,11 +16,10 @@ · 15 unmodified lines
16	16	#include "xe_heci_gsc.h"
17	17	#include "xe_i2c.h"
18	18	#include "xe_mmio.h"
	19	+#include "xe_nvm.h"
19	20	#include "xe_pcode_api.h"
20	21	#include "xe_vsec.h"
21	22
22		−#define MAX_SCRATCH_MMIO 8
23		−
24	23	/**
25	24	* DOC: Survivability Mode
26	25	*
@@ -48,19 +47,43 @@ · 21 unmodified lines
48	47	*
49	48	* Refer :ref:`xe_configfs` for more details on how to use configfs
50	49	*
51		− * Survivability mode is indicated by the below admin-only readable sysfs which provides additional
52		− * debug information::
	50	+ * Survivability mode is indicated by the below admin-only readable sysfs entry. It
	51	+ * provides information about the type of survivability mode (Boot/Runtime).
	52	+ *
	53	+ * .. code-block:: shell
	54	+ *
	55	+ * # cat /sys/bus/pci/devices/<device>/survivability_mode
	56	+ * Boot
	57	+ *
	58	+ *
	59	+ * Any additional debug information if present will be visible under the directory
	60	+ * ``survivability_info``::
	61	+ *
	62	+ * /sys/bus/pci/devices/<device>/survivability_info/
	63	+ * ├── aux_info0
	64	+ * ├── aux_info1
	65	+ * ├── aux_info2
	66	+ * ├── aux_info3
	67	+ * ├── aux_info4
	68	+ * ├── capability_info
	69	+ * ├── fdo_mode
	70	+ * ├── postcode_trace
	71	+ * └── postcode_trace_overflow
	72	+ *
	73	+ * This directory has the following attributes
	74	+ *
	75	+ * - ``capability_info`` : Indicates Boot status and support for additional information
53	76	*
54		− * /sys/bus/pci/devices/<device>/survivability_mode
	77	+ * - ``postcode_trace``, ``postcode_trace_overflow`` : Each postcode is a 8bit value and
	78	+ * represents a boot failure event. When a new failure event is logged by PCODE the
	79	+ * existing postcodes are shifted left. These entries provide a history of 8 postcodes.
55	80	*
56		− * Capability Information:
57		− * Provides boot status
58		− * Postcode Information:
59		− * Provides information about the failure
60		− * Overflow Information
61		− * Provides history of previous failures
62		− * Auxiliary Information
63		− * Certain failures may have information in addition to postcode information
	81	+ * - ``aux_info<n>`` : Some failures have additional debug information
	82	+ *
	83	+ * - ``fdo_mode`` : To allow recovery in scenarios where MEI itself fails, a new SPI Flash
	84	+ * Descriptor Override (FDO) mode is added in v2 survivability breadcrumbs. This mode is enabled
	85	+ * by PCODE and provides the ability to directly update the firmware via SPI Driver without
	86	+ * any dependency on MEI. Xe KMD initializes the nvm aux driver if FDO mode is enabled.
64	87	*
65	88	* Runtime Survivability
66	89	* =====================
@@ -68,61 +91,77 @@ · 1 unmodified line
68	91	* Certain runtime firmware errors can cause the device to enter a wedged state
69	92	* (:ref:`xe-device-wedging`) requiring a firmware flash to restore normal operation.
70	93	* Runtime Survivability Mode indicates that a firmware flash is necessary to recover the device and
71		− * is indicated by the presence of survivability mode sysfs::
	94	+ * is indicated by the presence of survivability mode sysfs.
	95	+ * Survivability mode sysfs provides information about the type of survivability mode.
72	96	*
73		− * /sys/bus/pci/devices/<device>/survivability_mode
	97	+ * .. code-block:: shell
74	98	*
75		− * Survivability mode sysfs provides information about the type of survivability mode.
	99	+ * # cat /sys/bus/pci/devices/<device>/survivability_mode
	100	+ * Runtime
76	101	*
77	102	* When such errors occur, userspace is notified with the drm device wedged uevent and runtime
78	103	* survivability mode. User can then initiate a firmware flash using userspace tools like fwupd
79	104	* to restore device to normal operation.
80	105	*/
81	106
82		−static u32 aux_history_offset(u32 reg_value)
	107	+static const char * const reg_map[] = {
	108	+ [CAPABILITY_INFO] = "Capability Info",
	109	+ [POSTCODE_TRACE] = "Postcode trace",
	110	+ [POSTCODE_TRACE_OVERFLOW] = "Postcode trace overflow",
	111	+ [AUX_INFO0] = "Auxiliary Info 0",
	112	+ [AUX_INFO1] = "Auxiliary Info 1",
	113	+ [AUX_INFO2] = "Auxiliary Info 2",
	114	+ [AUX_INFO3] = "Auxiliary Info 3",
	115	+ [AUX_INFO4] = "Auxiliary Info 4",
	116	+};
	117	+
	118	+#define FDO_INFO (MAX_SCRATCH_REG + 1)
	119	+
	120	+struct xe_survivability_attribute {
	121	+ struct device_attribute attr;
	122	+ u8 index;
	123	+};
	124	+
	125	+static struct
	126	+xe_survivability_attribute dev_attr_to_survivability_attr(struct device_attribute attr)
83	127	{
84		− return REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, reg_value);
	128	+ return container_of(attr, struct xe_survivability_attribute, attr);
85	129	}
86	130
87		−static void set_survivability_info(struct xe_mmio mmio, struct xe_survivability_info info,
88		− int id, char *name)
	131	+static void set_survivability_info(struct xe_mmio mmio, u32 info, int id)
89	132	{
90		− strscpy(info[id].name, name, sizeof(info[id].name));
91		− info[id].reg = PCODE_SCRATCH(id).raw;
92		− info[id].value = xe_mmio_read32(mmio, PCODE_SCRATCH(id));
	133	+ info[id] = xe_mmio_read32(mmio, PCODE_SCRATCH(id));
93	134	}
94	135
95	136	static void populate_survivability_info(struct xe_device *xe)
96	137	{
97	138	struct xe_survivability *survivability = &xe->survivability;
98		− struct xe_survivability_info *info = survivability->info;
	139	+ u32 *info = survivability->info;
99	140	struct xe_mmio *mmio;
100	141	u32 id = 0, reg_value;
101		− char name[NAME_MAX];
102		− int index;
103	142
104	143	mmio = xe_root_tile_mmio(xe);
105		− set_survivability_info(mmio, info, id, "Capability Info");
106		− reg_value = info[id].value;
	144	+ set_survivability_info(mmio, info, CAPABILITY_INFO);
	145	+ reg_value = info[CAPABILITY_INFO];
	146	+
	147	+ survivability->version = REG_FIELD_GET(BREADCRUMB_VERSION, reg_value);
	148	+ /* FDO mode is exposed only from version 2 */
	149	+ if (survivability->version >= 2)
	150	+ survivability->fdo_mode = REG_FIELD_GET(FDO_MODE, reg_value);
107	151
108	152	if (reg_value & HISTORY_TRACKING) {
109		− id++;
110		− set_survivability_info(mmio, info, id, "Postcode Info");
	153	+ set_survivability_info(mmio, info, POSTCODE_TRACE);
111	154
112		− if (reg_value & OVERFLOW_SUPPORT) {
113		− id = REG_FIELD_GET(OVERFLOW_REG_OFFSET, reg_value);
114		− set_survivability_info(mmio, info, id, "Overflow Info");
115		− }
	155	+ if (reg_value & OVERFLOW_SUPPORT)
	156	+ set_survivability_info(mmio, info, POSTCODE_TRACE_OVERFLOW);
116	157	}
117	158
	159	+ /* Traverse the linked list of aux info registers */
118	160	if (reg_value & AUXINFO_SUPPORT) {
119		− id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value);
120		−
121		− for (index = 0; id && reg_value; index++, reg_value = info[id].value,
122		− id = aux_history_offset(reg_value)) {
123		− snprintf(name, NAME_MAX, "Auxiliary Info %d", index);
124		− set_survivability_info(mmio, info, id, name);
125		− }
	161	+ for (id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value);
	162	+ id >= AUX_INFO0 && id < MAX_SCRATCH_REG;
	163	+ id = REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, info[id]))
	164	+ set_survivability_info(mmio, info, id);
126	165	}
127	166	}
128	167
@@ -130,15 +169,14 @@ static void log_survivability_info(struct pci_dev *pdev) · 1 unmodified line
130	169	{
131	170	struct xe_device *xe = pdev_to_xe_device(pdev);
132	171	struct xe_survivability *survivability = &xe->survivability;
133		− struct xe_survivability_info *info = survivability->info;
	172	+ u32 *info = survivability->info;
134	173	int id;
135	174
136	175	dev_info(&pdev->dev, "Survivability Boot Status : Critical Failure (%d)\n",
137	176	survivability->boot_status);
138		− for (id = 0; id < MAX_SCRATCH_MMIO; id++) {
139		− if (info[id].reg)
140		− dev_info(&pdev->dev, "%s: 0x%x - 0x%x\n", info[id].name,
141		− info[id].reg, info[id].value);
	177	+ for (id = 0; id < MAX_SCRATCH_REG; id++) {
	178	+ if (info[id])
	179	+ dev_info(&pdev->dev, "%s: 0x%x\n", reg_map[id], info[id]);
142	180	}
143	181	}
144	182
@@ -156,43 +194,103 @@ static ssize_t survivability_mode_show(struct device *dev, · 11 unmodified lines
156	194	struct pci_dev *pdev = to_pci_dev(dev);
157	195	struct xe_device *xe = pdev_to_xe_device(pdev);
158	196	struct xe_survivability *survivability = &xe->survivability;
159		− struct xe_survivability_info *info = survivability->info;
160		− int index = 0, count = 0;
161	197
162		− count += sysfs_emit_at(buff, count, "Survivability mode type: %s\n",
163		− survivability->type ? "Runtime" : "Boot");
	198	+ return sysfs_emit(buff, "%s\n", survivability->type ? "Runtime" : "Boot");
	199	+}
	200	+
	201	+static DEVICE_ATTR_ADMIN_RO(survivability_mode);
164	202
165		− if (!check_boot_failure(xe))
166		− return count;
	203	+static ssize_t survivability_info_show(struct device *dev,
	204	+ struct device_attribute attr, char buff)
	205	+{
	206	+ struct xe_survivability_attribute *sa = dev_attr_to_survivability_attr(attr);
	207	+ struct pci_dev *pdev = to_pci_dev(dev);
	208	+ struct xe_device *xe = pdev_to_xe_device(pdev);
	209	+ struct xe_survivability *survivability = &xe->survivability;
	210	+ u32 *info = survivability->info;
167	211
168		− for (index = 0; index < MAX_SCRATCH_MMIO; index++) {
169		− if (info[index].reg)
170		− count += sysfs_emit_at(buff, count, "%s: 0x%x - 0x%x\n", info[index].name,
171		− info[index].reg, info[index].value);
172		− }
	212	+ if (sa->index == FDO_INFO)
	213	+ return sysfs_emit(buff, "%s\n", str_enabled_disabled(survivability->fdo_mode));
173	214
174		− return count;
	215	+ return sysfs_emit(buff, "0x%x\n", info[sa->index]);
175	216	}
176	217
177		−static DEVICE_ATTR_ADMIN_RO(survivability_mode);
	218	+#define SURVIVABILITY_ATTR_RO(name, _index) \
	219	+ struct xe_survivability_attribute attr_##name = { \
	220	+ .attr = __ATTR(name, 0400, survivability_info_show, NULL), \
	221	+ .index = _index, \
	222	+ }
	223	+
	224	+static SURVIVABILITY_ATTR_RO(capability_info, CAPABILITY_INFO);
	225	+static SURVIVABILITY_ATTR_RO(postcode_trace, POSTCODE_TRACE);
	226	+static SURVIVABILITY_ATTR_RO(postcode_trace_overflow, POSTCODE_TRACE_OVERFLOW);
	227	+static SURVIVABILITY_ATTR_RO(aux_info0, AUX_INFO0);
	228	+static SURVIVABILITY_ATTR_RO(aux_info1, AUX_INFO1);
	229	+static SURVIVABILITY_ATTR_RO(aux_info2, AUX_INFO2);
	230	+static SURVIVABILITY_ATTR_RO(aux_info3, AUX_INFO3);
	231	+static SURVIVABILITY_ATTR_RO(aux_info4, AUX_INFO4);
	232	+static SURVIVABILITY_ATTR_RO(fdo_mode, FDO_INFO);
178	233
179	234	static void xe_survivability_mode_fini(void *arg)
180	235	{
181	236	struct xe_device *xe = arg;
	237	+ struct xe_survivability *survivability = &xe->survivability;
182	238	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
183	239	struct device *dev = &pdev->dev;
184	240
185		− sysfs_remove_file(&dev->kobj, &dev_attr_survivability_mode.attr);
	241	+ if (survivability->fdo_mode)
	242	+ xe_nvm_fini(xe);
	243	+
	244	+ device_remove_file(dev, &dev_attr_survivability_mode);
	245	+}
	246	+
	247	+static umode_t survivability_info_attrs_visible(struct kobject kobj, struct attribute attr,
	248	+ int idx)
	249	+{
	250	+ struct xe_device *xe = kdev_to_xe_device(kobj_to_dev(kobj));
	251	+ struct xe_survivability *survivability = &xe->survivability;
	252	+ u32 *info = survivability->info;
	253	+
	254	+ /*
	255	+ * Last index in survivability_info_attrs is fdo mode and is applicable only in
	256	+ * version 2 of survivability mode
	257	+ */
	258	+ if (idx == MAX_SCRATCH_REG && survivability->version >= 2)
	259	+ return 0400;
	260	+
	261	+ if (idx < MAX_SCRATCH_REG && info[idx])
	262	+ return 0400;
	263	+
	264	+ return 0;
186	265	}
187	266
	267	+/* Attributes are ordered according to enum scratch_reg */
	268	+static struct attribute *survivability_info_attrs[] = {
	269	+ &attr_capability_info.attr.attr,
	270	+ &attr_postcode_trace.attr.attr,
	271	+ &attr_postcode_trace_overflow.attr.attr,
	272	+ &attr_aux_info0.attr.attr,
	273	+ &attr_aux_info1.attr.attr,
	274	+ &attr_aux_info2.attr.attr,
	275	+ &attr_aux_info3.attr.attr,
	276	+ &attr_aux_info4.attr.attr,
	277	+ &attr_fdo_mode.attr.attr,
	278	+ NULL,
	279	+};
	280	+
	281	+static const struct attribute_group survivability_info_group = {
	282	+ .name = "survivability_info",
	283	+ .attrs = survivability_info_attrs,
	284	+ .is_visible = survivability_info_attrs_visible,
	285	+};
	286	+
188	287	static int create_survivability_sysfs(struct pci_dev *pdev)
189	288	{
190	289	struct device *dev = &pdev->dev;
191	290	struct xe_device *xe = pdev_to_xe_device(pdev);
192	291	int ret;
193	292
194		− /* create survivability mode sysfs */
195		− ret = sysfs_create_file(&dev->kobj, &dev_attr_survivability_mode.attr);
	293	+ ret = device_create_file(dev, &dev_attr_survivability_mode);
196	294	if (ret) {
197	295	dev_warn(dev, "Failed to create survivability sysfs files\n");
198	296	return ret;
@@ -203,6 +301,12 @@ static int create_survivability_sysfs(struct pci_dev *pdev) · 4 unmodified lines
203	301	if (ret)
204	302	return ret;
205	303
	304	+ if (check_boot_failure(xe)) {
	305	+ ret = devm_device_add_group(dev, &survivability_info_group);
	306	+ if (ret)
	307	+ return ret;
	308	+ }
	309	+
206	310	return 0;
207	311	}
208	312
@@ -220,12 +324,16 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev) · 11 unmodified lines
220	324	/* Make sure xe_heci_gsc_init() knows about survivability mode */
221	325	survivability->mode = true;
222	326
223		− ret = xe_heci_gsc_init(xe);
224		− if (ret)
225		− goto err;
	327	+ xe_heci_gsc_init(xe);
226	328
227	329	xe_vsec_init(xe);
228	330
	331	+ if (survivability->fdo_mode) {
	332	+ ret = xe_nvm_init(xe);
	333	+ if (ret)
	334	+ goto err;
	335	+ }
	336	+
229	337	ret = xe_i2c_probe(xe);
230	338	if (ret)
231	339	goto err;
@@ -235,29 +343,11 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev) · 3 unmodified lines
235	343	return 0;
236	344
237	345	err:
	346	+ dev_err(dev, "Failed to enable Survivability Mode\n");
238	347	survivability->mode = false;
239	348	return ret;
240	349	}
241	350
242		−static int init_survivability_mode(struct xe_device *xe)
243		−{
244		− struct xe_survivability *survivability = &xe->survivability;
245		− struct xe_survivability_info *info;
246		−
247		− survivability->size = MAX_SCRATCH_MMIO;
248		−
249		− info = devm_kcalloc(xe->drm.dev, survivability->size, sizeof(*info),
250		− GFP_KERNEL);
251		− if (!info)
252		− return -ENOMEM;
253		−
254		− survivability->info = info;
255		−
256		− populate_survivability_info(xe);
257		−
258		− return 0;
259		−}
260		−
261	351	/**
262	352	* xe_survivability_mode_is_boot_enabled- check if boot survivability mode is enabled
263	353	* @xe: xe device instance
@@ -325,9 +415,7 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe) · 61 unmodified lines
325	415	return -EINVAL;
326	416	}
327	417
328		− ret = init_survivability_mode(xe);
329		− if (ret)
330		− return ret;
	418	+ populate_survivability_info(xe);
331	419
332	420	ret = create_survivability_sysfs(pdev);
333	421	if (ret)
@@ -356,17 +444,16 @@ int xe_survivability_mode_boot_enable(struct xe_device *xe) · 22 unmodified lines
356	444	{
357	445	struct xe_survivability *survivability = &xe->survivability;
358	446	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
359		− int ret;
360	447
361	448	if (!xe_survivability_mode_is_requested(xe))
362	449	return 0;
363	450
364		− ret = init_survivability_mode(xe);
365		− if (ret)
366		− return ret;
	451	+ populate_survivability_info(xe);
367	452
368		− /* Log breadcrumbs but do not enter survivability mode for Critical boot errors */
369		− if (survivability->boot_status == CRITICAL_FAILURE) {
	453	+ /*
	454	+ * v2 supports survivability mode for critical errors
	455	+ */
	456	+ if (survivability->version < 2 && survivability->boot_status == CRITICAL_FAILURE) {
370	457	log_survivability_info(pdev);
371	458	return -ENXIO;
372	459	}

drivers/gpu/drm/xe/xe_survivability_mode_types.h+20−8

@@ -9,23 +9,29 @@ · 8 unmodified lines
9	9	#include <linux/limits.h>
10	10	#include <linux/types.h>
11	11
	12	+enum scratch_reg {
	13	+ CAPABILITY_INFO,
	14	+ POSTCODE_TRACE,
	15	+ POSTCODE_TRACE_OVERFLOW,
	16	+ AUX_INFO0,
	17	+ AUX_INFO1,
	18	+ AUX_INFO2,
	19	+ AUX_INFO3,
	20	+ AUX_INFO4,
	21	+ MAX_SCRATCH_REG,
	22	+};
	23	+
12	24	enum xe_survivability_type {
13	25	XE_SURVIVABILITY_TYPE_BOOT,
14	26	XE_SURVIVABILITY_TYPE_RUNTIME,
15	27	};
16	28
17		−struct xe_survivability_info {
18		− char name[NAME_MAX];
19		− u32 reg;
20		− u32 value;
21		−};
22		−
23	29	/**
24	30	* struct xe_survivability: Contains survivability mode information
25	31	*/
26	32	struct xe_survivability {
27		− /** @info: struct that holds survivability info from scratch registers */
28		− struct xe_survivability_info *info;
	33	+ /** @info: survivability debug info */
	34	+ u32 info[MAX_SCRATCH_REG];
29	35
30	36	/** @size: number of scratch registers */
31	37	u32 size;
@@ -38,6 +44,12 @@ struct xe_survivability { · 6 unmodified lines
38	44
39	45	/** @type: survivability type */
40	46	enum xe_survivability_type type;
	47	+
	48	+ /** @fdo_mode: indicates if FDO mode is enabled */
	49	+ bool fdo_mode;
	50	+
	51	+ /** @version: breadcrumb version of survivability mode */
	52	+ u8 version;
41	53	};
42	54
43	55	#endif /* _XE_SURVIVABILITY_MODE_TYPES_H_ */

drivers/gpu/drm/xe/xe_svm.c+42−46

@@ -176,24 +176,13 @@ xe_svm_range_notifier_event_end(struct xe_vm vm, struct drm_gpusvm_range r, · 175 unmodified lines
176	176	mmu_range);
177	177	}
178	178
179		−static s64 xe_svm_stats_ktime_us_delta(ktime_t start)
180		−{
181		− return IS_ENABLED(CONFIG_DEBUG_FS) ?
182		− ktime_us_delta(ktime_get(), start) : 0;
183		−}
184		−
185	179	static void xe_svm_tlb_inval_us_stats_incr(struct xe_gt *gt, ktime_t start)
186	180	{
187		− s64 us_delta = xe_svm_stats_ktime_us_delta(start);
	181	+ s64 us_delta = xe_gt_stats_ktime_us_delta(start);
188	182
189	183	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_US, us_delta);
190	184	}
191	185
192		−static ktime_t xe_svm_stats_ktime_get(void)
193		−{
194		− return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0;
195		−}
196		−
197	186	static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,
198	187	struct drm_gpusvm_notifier *notifier,
199	188	const struct mmu_notifier_range *mmu_range)
@@ -202,7 +191,7 @@ static void xe_svm_invalidate(struct drm_gpusvm *gpusvm, · 2 unmodified lines
202	191	struct xe_device *xe = vm->xe;
203	192	struct drm_gpusvm_range r, first;
204	193	struct xe_tile *tile;
205		− ktime_t start = xe_svm_stats_ktime_get();
	194	+ ktime_t start = xe_gt_stats_ktime_get();
206	195	u64 adj_start = mmu_range->start, adj_end = mmu_range->end;
207	196	u8 tile_mask = 0, id;
208	197	long err;
@@ -285,19 +274,21 @@ static int __xe_svm_garbage_collector(struct xe_vm *vm, · 76 unmodified lines
285	274	return 0;
286	275	}
287	276
288		−static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64 range_end)
	277	+static void xe_vma_set_default_attributes(struct xe_vma *vma)
	278	+{
	279	+ vma->attr.preferred_loc.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE;
	280	+ vma->attr.preferred_loc.migration_policy = DRM_XE_MIGRATE_ALL_PAGES;
	281	+ vma->attr.pat_index = vma->attr.default_pat_index;
	282	+ vma->attr.atomic_access = DRM_XE_ATOMIC_UNDEFINED;
	283	+}
	284	+
	285	+static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end)
289	286	{
290	287	struct xe_vma *vma;
291		− struct xe_vma_mem_attr default_attr = {
292		− .preferred_loc = {
293		− .devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE,
294		− .migration_policy = DRM_XE_MIGRATE_ALL_PAGES,
295		− },
296		− .atomic_access = DRM_XE_ATOMIC_UNDEFINED,
297		− };
298		− int err = 0;
	288	+ bool has_default_attr;
	289	+ int err;
299	290
300		− vma = xe_vm_find_vma_by_addr(vm, range_start);
	291	+ vma = xe_vm_find_vma_by_addr(vm, start);
301	292	if (!vma)
302	293	return -EINVAL;
303	294
@@ -306,25 +297,30 @@ static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64 · 2 unmodified lines
306	297	return 0;
307	298	}
308	299
309		− if (xe_vma_has_default_mem_attrs(vma))
310		− return 0;
311		−
312	300	vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx",
313	301	xe_vma_start(vma), xe_vma_end(vma));
314	302
315		− if (xe_vma_start(vma) == range_start && xe_vma_end(vma) == range_end) {
316		− default_attr.pat_index = vma->attr.default_pat_index;
317		− default_attr.default_pat_index = vma->attr.default_pat_index;
318		− vma->attr = default_attr;
319		− } else {
320		− vm_dbg(&vm->xe->drm, "Split VMA start=0x%016llx, vma_end=0x%016llx",
321		− range_start, range_end);
322		− err = xe_vm_alloc_cpu_addr_mirror_vma(vm, range_start, range_end - range_start);
323		− if (err) {
324		− drm_warn(&vm->xe->drm, "VMA SPLIT failed: %pe\n", ERR_PTR(err));
325		− xe_vm_kill(vm, true);
326		− return err;
327		− }
	303	+ has_default_attr = xe_vma_has_default_mem_attrs(vma);
	304	+
	305	+ if (has_default_attr) {
	306	+ start = xe_vma_start(vma);
	307	+ end = xe_vma_end(vma);
	308	+ } else if (xe_vma_start(vma) == start && xe_vma_end(vma) == end) {
	309	+ xe_vma_set_default_attributes(vma);
	310	+ }
	311	+
	312	+ xe_vm_find_cpu_addr_mirror_vma_range(vm, &start, &end);
	313	+
	314	+ if (xe_vma_start(vma) == start && xe_vma_end(vma) == end && has_default_attr)
	315	+ return 0;
	316	+
	317	+ vm_dbg(&vm->xe->drm, "New VMA start=0x%016llx, vma_end=0x%016llx", start, end);
	318	+
	319	+ err = xe_vm_alloc_cpu_addr_mirror_vma(vm, start, end - start);
	320	+ if (err) {
	321	+ drm_warn(&vm->xe->drm, "New VMA MAP failed: %pe\n", ERR_PTR(err));
	322	+ xe_vm_kill(vm, true);
	323	+ return err;
328	324	}
329	325
330	326	/*
@@ -435,7 +431,7 @@ static void xe_svm_copy_us_stats_incr(struct xe_gt *gt, · 104 unmodified lines
435	431	unsigned long npages,
436	432	ktime_t start)
437	433	{
438		− s64 us_delta = xe_svm_stats_ktime_us_delta(start);
	434	+ s64 us_delta = xe_gt_stats_ktime_us_delta(start);
439	435
440	436	if (dir == XE_SVM_COPY_TO_VRAM) {
441	437	switch (npages) {
@@ -487,7 +483,7 @@ static int xe_svm_copy(struct page **pages, · 45 unmodified lines
487	483	u64 vram_addr = XE_VRAM_ADDR_INVALID;
488	484	int err = 0, pos = 0;
489	485	bool sram = dir == XE_SVM_COPY_TO_SRAM;
490		− ktime_t start = xe_svm_stats_ktime_get();
	486	+ ktime_t start = xe_gt_stats_ktime_get();
491	487
492	488	/*
493	489	* This flow is complex: it locates physically contiguous device pages,
@@ -979,7 +975,7 @@ static void xe_svm_range_##elem##_us_stats_incr(struct xe_gt *gt, \ · 485 unmodified lines
979	975	struct xe_svm_range *range, \
980	976	ktime_t start) \
981	977	{ \
982		− s64 us_delta = xe_svm_stats_ktime_us_delta(start); \
	978	+ s64 us_delta = xe_gt_stats_ktime_us_delta(start); \
983	979	\
984	980	switch (xe_svm_range_size(range)) { \
985	981	case SZ_4K: \
@@ -1024,7 +1020,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm vm, struct xe_vma vma, · 38 unmodified lines
1024	1020	struct drm_pagemap *dpagemap;
1025	1021	struct xe_tile *tile = gt_to_tile(gt);
1026	1022	int migrate_try_count = ctx.devmem_only ? 3 : 1;
1027		− ktime_t start = xe_svm_stats_ktime_get(), bind_start, get_pages_start;
	1023	+ ktime_t start = xe_gt_stats_ktime_get(), bind_start, get_pages_start;
1028	1024	int err;
1029	1025
1030	1026	lockdep_assert_held_write(&vm->lock);
@@ -1063,7 +1059,7 @@ retry: · 32 unmodified lines
1063	1059
1064	1060	if (--migrate_try_count >= 0 &&
1065	1061	xe_svm_range_needs_migrate_to_vram(range, vma, !!dpagemap \|\| ctx.devmem_only)) {
1066		− ktime_t migrate_start = xe_svm_stats_ktime_get();
	1062	+ ktime_t migrate_start = xe_gt_stats_ktime_get();
1067	1063
1068	1064	/* TODO : For multi-device dpagemap will be used to find the
1069	1065	* remote tile and remote device. Will need to modify
@@ -1100,7 +1096,7 @@ retry: · 30 unmodified lines
1100	1096	}
1101	1097
1102	1098	get_pages:
1103		− get_pages_start = xe_svm_stats_ktime_get();
	1099	+ get_pages_start = xe_gt_stats_ktime_get();
1104	1100
1105	1101	range_debug(range, "GET PAGES");
1106	1102	err = xe_svm_range_get_pages(vm, range, &ctx);
@@ -1127,7 +1123,7 @@ get_pages: · 20 unmodified lines
1127	1123	xe_svm_range_get_pages_us_stats_incr(gt, range, get_pages_start);
1128	1124	range_debug(range, "PAGE FAULT - BIND");
1129	1125
1130		− bind_start = xe_svm_stats_ktime_get();
	1126	+ bind_start = xe_gt_stats_ktime_get();
1131	1127	xe_validation_guard(&vctx, &vm->xe->val, &exec, (struct xe_val_flags) {}, err) {
1132	1128	err = xe_vm_drm_exec_lock(vm, &exec);
1133	1129	drm_exec_retry_on_contention(&exec);

drivers/gpu/drm/xe/xe_sync.c+31−2

@@ -228,6 +228,32 @@ int xe_sync_entry_add_deps(struct xe_sync_entry sync, struct xe_sched_job job) · 227 unmodified lines
228	228	return 0;
229	229	}
230	230
	231	+/**
	232	+ * xe_sync_entry_wait() - Wait on in-sync
	233	+ * @sync: Sync object
	234	+ *
	235	+ * If the sync is in an in-sync, wait on the sync to signal.
	236	+ *
	237	+ * Return: 0 on success, -ERESTARTSYS on failure (interruption)
	238	+ */
	239	+int xe_sync_entry_wait(struct xe_sync_entry *sync)
	240	+{
	241	+ return xe_sync_needs_wait(sync) ?
	242	+ dma_fence_wait(sync->fence, true) : 0;
	243	+}
	244	+
	245	+/**
	246	+ * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled)
	247	+ * @sync: Sync object
	248	+ *
	249	+ * Return: True if sync needs a wait, False otherwise
	250	+ */
	251	+bool xe_sync_needs_wait(struct xe_sync_entry *sync)
	252	+{
	253	+ return sync->fence &&
	254	+ !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags);
	255	+}
	256	+
231	257	void xe_sync_entry_signal(struct xe_sync_entry sync, struct dma_fence fence)
232	258	{
233	259	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
@@ -311,8 +337,11 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, · 77 unmodified lines
311	337	struct xe_tile *tile;
312	338	u8 id;
313	339
314		− for_each_tile(tile, vm->xe, id)
315		− num_fence += (1 + XE_MAX_GT_PER_TILE);
	340	+ for_each_tile(tile, vm->xe, id) {
	341	+ num_fence++;
	342	+ for_each_tlb_inval(i)
	343	+ num_fence++;
	344	+ }
316	345
317	346	fences = kmalloc_array(num_fence, sizeof(*fences),
318	347	GFP_KERNEL);

drivers/gpu/drm/xe/xe_sync.h+2−0

@@ -29,6 +29,8 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync, · 28 unmodified lines
29	29	struct xe_sched_job *job);
30	30	void xe_sync_entry_signal(struct xe_sync_entry *sync,
31	31	struct dma_fence *fence);
	32	+int xe_sync_entry_wait(struct xe_sync_entry *sync);
	33	+bool xe_sync_needs_wait(struct xe_sync_entry *sync);
32	34	void xe_sync_entry_cleanup(struct xe_sync_entry *sync);
33	35	struct dma_fence *
34	36	xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,

drivers/gpu/drm/xe/xe_tile.c+5−0

@@ -209,6 +209,11 @@ int xe_tile_init(struct xe_tile *tile) · 208 unmodified lines
209	209	if (IS_ERR(tile->mem.kernel_bb_pool))
210	210	return PTR_ERR(tile->mem.kernel_bb_pool);
211	211
	212	+ /* Optimistically anticipate at most 256 TLB fences with PRL */
	213	+ tile->mem.reclaim_pool = xe_sa_bo_manager_init(tile, SZ_1M, XE_PAGE_RECLAIM_LIST_MAX_SIZE);
	214	+ if (IS_ERR(tile->mem.reclaim_pool))
	215	+ return PTR_ERR(tile->mem.reclaim_pool);
	216	+
212	217	return 0;
213	218	}
214	219	void xe_tile_migrate_wait(struct xe_tile *tile)

drivers/gpu/drm/xe/xe_tile_debugfs.c+11−6

@@ -82,13 +82,9 @@ int xe_tile_debugfs_show_with_rpm(struct seq_file m, void data) · 81 unmodified lines
82	82	struct drm_info_node *node = m->private;
83	83	struct xe_tile *tile = node_to_tile(node);
84	84	struct xe_device *xe = tile_to_xe(tile);
85		− int ret;
86	85
87		− xe_pm_runtime_get(xe);
88		− ret = xe_tile_debugfs_simple_show(m, data);
89		− xe_pm_runtime_put(xe);
90		−
91		− return ret;
	86	+ guard(xe_pm_runtime)(xe);
	87	+ return xe_tile_debugfs_simple_show(m, data);
92	88	}
93	89
94	90	static int ggtt(struct xe_tile tile, struct drm_printer p)
@@ -110,6 +106,13 @@ static const struct drm_info_list vf_safe_debugfs_list[] = { · 15 unmodified lines
110	106	{ "sa_info", .show = xe_tile_debugfs_show_with_rpm, .data = sa_info },
111	107	};
112	108
	109	+static void tile_debugfs_create_vram_mm(struct xe_tile *tile)
	110	+{
	111	+ if (tile->mem.vram)
	112	+ ttm_resource_manager_create_debugfs(&tile->mem.vram->ttm.manager, tile->debugfs,
	113	+ "vram_mm");
	114	+}
	115	+
113	116	/**
114	117	* xe_tile_debugfs_register - Register tile's debugfs attributes
115	118	* @tile: the &xe_tile to register
@@ -139,4 +142,6 @@ void xe_tile_debugfs_register(struct xe_tile *tile) · 23 unmodified lines
139	142	drm_debugfs_create_files(vf_safe_debugfs_list,
140	143	ARRAY_SIZE(vf_safe_debugfs_list),
141	144	tile->debugfs, minor);
	145	+
	146	+ tile_debugfs_create_vram_mm(tile);
142	147	}

drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c+1−2

@@ -141,12 +141,11 @@ static int NAME##_set(void *data, u64 val) \ · 140 unmodified lines
141	141	if (val > (TYPE)~0ull) \
142	142	return -EOVERFLOW; \
143	143	\
144		− xe_pm_runtime_get(xe); \
	144	+ guard(xe_pm_runtime)(xe); \
145	145	err = xe_sriov_pf_wait_ready(xe) ?: \
146	146	xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \
147	147	if (!err) \
148	148	xe_sriov_pf_provision_set_custom_mode(xe); \
149		− xe_pm_runtime_put(xe); \
150	149	\
151	150	return err; \
152	151	} \

drivers/gpu/drm/xe/xe_tlb_inval.c+24−3

@@ -199,6 +199,20 @@ void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval) · 198 unmodified lines
199	199	mutex_unlock(&tlb_inval->seqno_lock);
200	200	}
201	201
	202	+/**
	203	+ * xe_tlb_inval_reset_timeout() - Reset TLB inval fence timeout
	204	+ * @tlb_inval: TLB invalidation client
	205	+ *
	206	+ * Reset the TLB invalidation timeout timer.
	207	+ */
	208	+static void xe_tlb_inval_reset_timeout(struct xe_tlb_inval *tlb_inval)
	209	+{
	210	+ lockdep_assert_held(&tlb_inval->pending_lock);
	211	+
	212	+ mod_delayed_work(system_wq, &tlb_inval->fence_tdr,
	213	+ tlb_inval->ops->timeout_delay(tlb_inval));
	214	+}
	215	+
202	216	static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno)
203	217	{
204	218	int seqno_recv = READ_ONCE(tlb_inval->seqno_recv);
@@ -299,6 +313,7 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) · 94 unmodified lines
299	313	* @start: start address
300	314	* @end: end address
301	315	* @asid: address space id
	316	+ * @prl_sa: suballocation of page reclaim list if used, NULL indicates PPC flush
302	317	*
303	318	* Issue a range based TLB invalidation if supported, if not fallback to a full
304	319	* TLB invalidation. Completion of TLB is asynchronous and caller can use
@@ -308,10 +323,10 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval) · 3 unmodified lines
308	323	*/
309	324	int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
310	325	struct xe_tlb_inval_fence *fence, u64 start, u64 end,
311		− u32 asid)
	326	+ u32 asid, struct drm_suballoc *prl_sa)
312	327	{
313	328	return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt,
314		− start, end, asid);
	329	+ start, end, asid, prl_sa);
315	330	}
316	331
317	332	/**
@@ -327,7 +342,7 @@ void xe_tlb_inval_vm(struct xe_tlb_inval tlb_inval, struct xe_vm vm) · 9 unmodified lines
327	342	u64 range = 1ull << vm->xe->info.va_bits;
328	343
329	344	xe_tlb_inval_fence_init(tlb_inval, &fence, true);
330		− xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid);
	345	+ xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid, NULL);
331	346	xe_tlb_inval_fence_wait(&fence);
332	347	}
333	348
@@ -360,6 +375,12 @@ void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno) · 26 unmodified lines
360	375	* process_g2h_msg().
361	376	*/
362	377	spin_lock_irqsave(&tlb_inval->pending_lock, flags);
	378	+ if (seqno == TLB_INVALIDATION_SEQNO_INVALID) {
	379	+ xe_tlb_inval_reset_timeout(tlb_inval);
	380	+ spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
	381	+ return;
	382	+ }
	383	+
363	384	if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) {
364	385	spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
365	386	return;

drivers/gpu/drm/xe/xe_tlb_inval.h+1−1

@@ -23,7 +23,7 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval); · 22 unmodified lines
23	23	void xe_tlb_inval_vm(struct xe_tlb_inval tlb_inval, struct xe_vm vm);
24	24	int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
25	25	struct xe_tlb_inval_fence *fence,
26		− u64 start, u64 end, u32 asid);
	26	+ u64 start, u64 end, u32 asid, struct drm_suballoc *prl_sa);
27	27
28	28	void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
29	29	struct xe_tlb_inval_fence *fence,

drivers/gpu/drm/xe/xe_tlb_inval_job.c+35−1

@@ -7,7 +7,9 @@ · 6 unmodified lines
7	7	#include "xe_dep_job_types.h"
8	8	#include "xe_dep_scheduler.h"
9	9	#include "xe_exec_queue.h"
	10	+#include "xe_gt_printk.h"
10	11	#include "xe_gt_types.h"
	12	+#include "xe_page_reclaim.h"
11	13	#include "xe_tlb_inval.h"
12	14	#include "xe_tlb_inval_job.h"
13	15	#include "xe_migrate.h"
@@ -24,6 +26,8 @@ struct xe_tlb_inval_job { · 10 unmodified lines
24	26	struct xe_exec_queue *q;
25	27	/** @vm: VM which TLB invalidation is being issued for */
26	28	struct xe_vm *vm;
	29	+ /** @prl: Embedded copy of page reclaim list */
	30	+ struct xe_page_reclaim_list prl;
27	31	/** @refcount: ref count of this job */
28	32	struct kref refcount;
29	33	/**
@@ -47,9 +51,16 @@ static struct dma_fence xe_tlb_inval_job_run(struct xe_dep_job dep_job) · 17 unmodified lines
47	51	container_of(dep_job, typeof(*job), dep);
48	52	struct xe_tlb_inval_fence *ifence =
49	53	container_of(job->fence, typeof(*ifence), base);
	54	+ struct drm_suballoc *prl_sa = NULL;
	55	+
	56	+ if (xe_page_reclaim_list_valid(&job->prl)) {
	57	+ prl_sa = xe_page_reclaim_create_prl_bo(job->tlb_inval, &job->prl, ifence);
	58	+ if (IS_ERR(prl_sa))
	59	+ prl_sa = NULL; /* Indicate fall back PPC flush with NULL */
	60	+ }
50	61
51	62	xe_tlb_inval_range(job->tlb_inval, ifence, job->start,
52		− job->end, job->vm->usm.asid);
	63	+ job->end, job->vm->usm.asid, prl_sa);
53	64
54	65	return job->fence;
55	66	}
@@ -107,6 +118,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue q, struct xe_tlb_inval tlb_inval, · 51 unmodified lines
107	118	job->start = start;
108	119	job->end = end;
109	120	job->fence_armed = false;
	121	+ xe_page_reclaim_list_init(&job->prl);
110	122	job->dep.ops = &dep_job_ops;
111	123	job->type = type;
112	124	kref_init(&job->refcount);
@@ -140,6 +152,25 @@ err_job: · 27 unmodified lines
140	152	return ERR_PTR(err);
141	153	}
142	154
	155	+/**
	156	+ * xe_tlb_inval_job_add_page_reclaim() - Embed PRL into a TLB job
	157	+ * @job: TLB invalidation job that may trigger reclamation
	158	+ * @prl: Page reclaim list populated during unbind
	159	+ *
	160	+ * Copies @prl into the job and takes an extra reference to the entry page so
	161	+ * ownership can transfer to the TLB fence when the job is pushed.
	162	+ */
	163	+void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
	164	+ struct xe_page_reclaim_list *prl)
	165	+{
	166	+ struct xe_device *xe = gt_to_xe(job->q->gt);
	167	+
	168	+ xe_gt_WARN_ON(job->q->gt, !xe->info.has_page_reclaim_hw_assist);
	169	+ job->prl = *prl;
	170	+ /* Pair with put in job_destroy */
	171	+ xe_page_reclaim_entries_get(job->prl.entries);
	172	+}
	173	+
143	174	static void xe_tlb_inval_job_destroy(struct kref *ref)
144	175	{
145	176	struct xe_tlb_inval_job job = container_of(ref, typeof(job),
@@ -150,6 +181,9 @@ static void xe_tlb_inval_job_destroy(struct kref *ref) · 4 unmodified lines
150	181	struct xe_device *xe = gt_to_xe(q->gt);
151	182	struct xe_vm *vm = job->vm;
152	183
	184	+ /* BO creation retains a copy (if used), so no longer needed */
	185	+ xe_page_reclaim_entries_put(job->prl.entries);
	186	+
153	187	if (!job->fence_armed)
154	188	kfree(ifence);
155	189	else

drivers/gpu/drm/xe/xe_tlb_inval_job.h+4−0

@@ -12,6 +12,7 @@ struct dma_fence; · 11 unmodified lines
12	12	struct xe_dep_scheduler;
13	13	struct xe_exec_queue;
14	14	struct xe_migrate;
	15	+struct xe_page_reclaim_list;
15	16	struct xe_tlb_inval;
16	17	struct xe_tlb_inval_job;
17	18	struct xe_vm;
@@ -21,6 +22,9 @@ xe_tlb_inval_job_create(struct xe_exec_queue q, struct xe_tlb_inval tlb_inval, · 3 unmodified lines
21	22	struct xe_dep_scheduler *dep_scheduler,
22	23	struct xe_vm *vm, u64 start, u64 end, int type);
23	24
	25	+void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
	26	+ struct xe_page_reclaim_list *prl);
	27	+
24	28	int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job);
25	29
26	30	struct dma_fence xe_tlb_inval_job_push(struct xe_tlb_inval_job job,

drivers/gpu/drm/xe/xe_tlb_inval_types.h+4−1

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#include <linux/workqueue.h>
10	10	#include <linux/dma-fence.h>
11	11
	12	+struct drm_suballoc;
12	13	struct xe_tlb_inval;
13	14
14	15	/** struct xe_tlb_inval_ops - TLB invalidation ops (backend) */
@@ -40,12 +41,13 @@ struct xe_tlb_inval_ops { · 25 unmodified lines
40	41	* @start: Start address
41	42	* @end: End address
42	43	* @asid: Address space ID
	44	+ * @prl_sa: Suballocation for page reclaim list
43	45	*
44	46	* Return 0 on success, -ECANCELED if backend is mid-reset, error on
45	47	* failure
46	48	*/
47	49	int (ppgtt)(struct xe_tlb_inval tlb_inval, u32 seqno, u64 start,
48		− u64 end, u32 asid);
	50	+ u64 end, u32 asid, struct drm_suballoc *prl_sa);
49	51
50	52	/**
51	53	* @initialized: Backend is initialized
@@ -80,6 +82,7 @@ struct xe_tlb_inval { · 28 unmodified lines
80	82	const struct xe_tlb_inval_ops *ops;
81	83	/** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */
82	84	#define TLB_INVALIDATION_SEQNO_MAX 0x100000
	85	+#define TLB_INVALIDATION_SEQNO_INVALID TLB_INVALIDATION_SEQNO_MAX
83	86	int seqno;
84	87	/** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */
85	88	struct mutex seqno_lock;

drivers/gpu/drm/xe/xe_trace.h+46−0

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <linux/types.h>
14	14
15	15	#include "xe_exec_queue_types.h"
	16	+#include "xe_exec_queue.h"
16	17	#include "xe_gpu_scheduler_types.h"
17	18	#include "xe_gt_types.h"
18	19	#include "xe_guc_exec_queue_types.h"
@@ -97,11 +98,51 @@ DECLARE_EVENT_CLASS(xe_exec_queue, · 78 unmodified lines
97	98	__entry->guc_state, __entry->flags)
98	99	);
99	100
	101	+DECLARE_EVENT_CLASS(xe_exec_queue_multi_queue,
	102	+ TP_PROTO(struct xe_exec_queue *q),
	103	+ TP_ARGS(q),
	104	+
	105	+ TP_STRUCT__entry(
	106	+ __string(dev, __dev_name_eq(q))
	107	+ __field(enum xe_engine_class, class)
	108	+ __field(u32, logical_mask)
	109	+ __field(u8, gt_id)
	110	+ __field(u16, width)
	111	+ __field(u32, guc_id)
	112	+ __field(u32, guc_state)
	113	+ __field(u32, flags)
	114	+ __field(u32, primary)
	115	+ ),
	116	+
	117	+ TP_fast_assign(
	118	+ __assign_str(dev);
	119	+ __entry->class = q->class;
	120	+ __entry->logical_mask = q->logical_mask;
	121	+ __entry->gt_id = q->gt->info.id;
	122	+ __entry->width = q->width;
	123	+ __entry->guc_id = q->guc->id;
	124	+ __entry->guc_state = atomic_read(&q->guc->state);
	125	+ __entry->flags = q->flags;
	126	+ __entry->primary = xe_exec_queue_multi_queue_primary(q)->guc->id;
	127	+ ),
	128	+
	129	+ TP_printk("dev=%s, %d:0x%x, gt=%d, width=%d guc_id=%d, guc_state=0x%x, flags=0x%x, primary=%d",
	130	+ __get_str(dev), __entry->class, __entry->logical_mask,
	131	+ __entry->gt_id, __entry->width, __entry->guc_id,
	132	+ __entry->guc_state, __entry->flags,
	133	+ __entry->primary)
	134	+);
	135	+
100	136	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_create,
101	137	TP_PROTO(struct xe_exec_queue *q),
102	138	TP_ARGS(q)
103	139	);
104	140
	141	+DEFINE_EVENT(xe_exec_queue_multi_queue, xe_exec_queue_create_multi_queue,
	142	+ TP_PROTO(struct xe_exec_queue *q),
	143	+ TP_ARGS(q)
	144	+);
	145	+
105	146	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_supress_resume,
106	147	TP_PROTO(struct xe_exec_queue *q),
107	148	TP_ARGS(q)
@@ -172,6 +213,11 @@ DEFINE_EVENT(xe_exec_queue, xe_exec_queue_memory_cat_error, · 64 unmodified lines
172	213	TP_ARGS(q)
173	214	);
174	215
	216	+DEFINE_EVENT(xe_exec_queue, xe_exec_queue_cgp_context_error,
	217	+ TP_PROTO(struct xe_exec_queue *q),
	218	+ TP_ARGS(q)
	219	+);
	220	+
175	221	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_stop,
176	222	TP_PROTO(struct xe_exec_queue *q),
177	223	TP_ARGS(q)

drivers/gpu/drm/xe/xe_uc.c+33−2

@@ -218,9 +218,12 @@ int xe_uc_load_hw(struct xe_uc *uc) · 217 unmodified lines
218	218
219	219	xe_guc_engine_activity_enable_stats(&uc->guc);
220	220
221		− /* We don't fail the driver load if HuC fails to auth, but let's warn */
	221	+ /* We don't fail the driver load if HuC fails to auth */
222	222	ret = xe_huc_auth(&uc->huc, XE_HUC_AUTH_VIA_GUC);
223		− xe_gt_assert(uc_to_gt(uc), !ret);
	223	+ if (ret)
	224	+ xe_gt_err(uc_to_gt(uc),
	225	+ "HuC authentication failed (%pe), continuing with no HuC\n",
	226	+ ERR_PTR(ret));
224	227
225	228	/* GSC load is async */
226	229	xe_gsc_load_start(&uc->gsc);
@@ -301,6 +304,34 @@ int xe_uc_suspend(struct xe_uc *uc) · 74 unmodified lines
301	304	return xe_guc_suspend(&uc->guc);
302	305	}
303	306
	307	+/**
	308	+ * xe_uc_runtime_suspend() - UC runtime suspend
	309	+ * @uc: the UC object
	310	+ *
	311	+ * Runtime suspend all UCs.
	312	+ */
	313	+void xe_uc_runtime_suspend(struct xe_uc *uc)
	314	+{
	315	+ if (!xe_device_uc_enabled(uc_to_xe(uc)))
	316	+ return;
	317	+
	318	+ xe_guc_runtime_suspend(&uc->guc);
	319	+}
	320	+
	321	+/**
	322	+ * xe_uc_runtime_resume() - UC runtime resume
	323	+ * @uc: the UC object
	324	+ *
	325	+ * Runtime resume all UCs.
	326	+ */
	327	+void xe_uc_runtime_resume(struct xe_uc *uc)
	328	+{
	329	+ if (!xe_device_uc_enabled(uc_to_xe(uc)))
	330	+ return;
	331	+
	332	+ xe_guc_runtime_resume(&uc->guc);
	333	+}
	334	+
304	335	/**
305	336	* xe_uc_declare_wedged() - Declare UC wedged
306	337	* @uc: the UC object

drivers/gpu/drm/xe/xe_uc.h+2−0

@@ -14,6 +14,8 @@ int xe_uc_init_post_hwconfig(struct xe_uc *uc); · 13 unmodified lines
14	14	int xe_uc_load_hw(struct xe_uc *uc);
15	15	void xe_uc_gucrc_disable(struct xe_uc *uc);
16	16	int xe_uc_reset_prepare(struct xe_uc *uc);
	17	+void xe_uc_runtime_resume(struct xe_uc *uc);
	18	+void xe_uc_runtime_suspend(struct xe_uc *uc);
17	19	void xe_uc_stop_prepare(struct xe_uc *uc);
18	20	void xe_uc_stop(struct xe_uc *uc);
19	21	int xe_uc_start(struct xe_uc *uc);

drivers/gpu/drm/xe/xe_uc_fw.c+5−5

@@ -115,11 +115,11 @@ struct fw_blobs_by_type { · 114 unmodified lines
115	115	#define XE_GT_TYPE_ANY XE_GT_TYPE_UNINITIALIZED
116	116
117	117	#define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver) \
118		− fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 49, 4)) \
119		− fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 49, 4)) \
120		− fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 45, 2)) \
121		− fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, guc, mtl, 70, 44, 1)) \
122		− fw_def(DG2, GT_TYPE_ANY, major_ver(i915, guc, dg2, 70, 45, 2)) \
	118	+ fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 54, 0)) \
	119	+ fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 54, 0)) \
	120	+ fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 53, 0)) \
	121	+ fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, guc, mtl, 70, 53, 0)) \
	122	+ fw_def(DG2, GT_TYPE_ANY, major_ver(i915, guc, dg2, 70, 53, 0)) \
123	123	fw_def(DG1, GT_TYPE_ANY, major_ver(i915, guc, dg1, 70, 44, 1)) \
124	124	fw_def(ALDERLAKE_N, GT_TYPE_ANY, major_ver(i915, guc, tgl, 70, 44, 1)) \
125	125	fw_def(ALDERLAKE_P, GT_TYPE_ANY, major_ver(i915, guc, adlp, 70, 44, 1)) \

drivers/gpu/drm/xe/xe_vm.c+136−18

@@ -1509,9 +1509,9 @@ struct xe_vm xe_vm_create(struct xe_device xe, u32 flags, struct xe_file *xef) · 1508 unmodified lines
1509	1509
1510	1510	INIT_LIST_HEAD(&vm->preempt.exec_queues);
1511	1511	if (flags & XE_VM_FLAG_FAULT_MODE)
1512		− vm->preempt.min_run_period_ms = 0;
	1512	+ vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms;
1513	1513	else
1514		− vm->preempt.min_run_period_ms = 5;
	1514	+ vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms;
1515	1515
1516	1516	for_each_tile(tile, xe, id)
1517	1517	xe_range_fence_tree_init(&vm->rftree[id]);
@@ -2236,6 +2236,7 @@ vm_bind_ioctl_ops_create(struct xe_vm vm, struct xe_vma_ops vops, · 718 unmodified lines
2236	2236	struct drm_gpuva_ops *ops;
2237	2237	struct drm_gpuva_op *__op;
2238	2238	struct drm_gpuvm_bo *vm_bo;
	2239	+ u64 range_start = addr;
2239	2240	u64 range_end = addr + range;
2240	2241	int err;
2241	2242
@@ -2248,10 +2249,16 @@ vm_bind_ioctl_ops_create(struct xe_vm vm, struct xe_vma_ops vops, · 6 unmodified lines
2248	2249
2249	2250	switch (operation) {
2250	2251	case DRM_XE_VM_BIND_OP_MAP:
	2252	+ if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) {
	2253	+ xe_vm_find_cpu_addr_mirror_vma_range(vm, &range_start, &range_end);
	2254	+ vops->flags \|= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP;
	2255	+ }
	2256	+
	2257	+ fallthrough;
2251	2258	case DRM_XE_VM_BIND_OP_MAP_USERPTR: {
2252	2259	struct drm_gpuvm_map_req map_req = {
2253		− .map.va.addr = addr,
2254		− .map.va.range = range,
	2260	+ .map.va.addr = range_start,
	2261	+ .map.va.range = range_end - range_start,
2255	2262	.map.gem.obj = obj,
2256	2263	.map.gem.offset = bo_offset_or_userptr,
2257	2264	};
@@ -2451,8 +2458,17 @@ static struct xe_vma new_vma(struct xe_vm vm, struct drm_gpuva_op_map *op, · 193 unmodified lines
2451	2458	if (IS_ERR(vma))
2452	2459	return vma;
2453	2460
2454		− if (xe_vma_is_userptr(vma))
	2461	+ if (xe_vma_is_userptr(vma)) {
2455	2462	err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
	2463	+ /*
	2464	+ * -EBUSY has dedicated meaning that a user fence
	2465	+ * attached to the VMA is busy, in practice
	2466	+ * xe_vma_userptr_pin_pages can only fail with -EBUSY if
	2467	+ * we are low on memory so convert this to -ENOMEM.
	2468	+ */
	2469	+ if (err == -EBUSY)
	2470	+ err = -ENOMEM;
	2471	+ }
2456	2472	}
2457	2473	if (err) {
2458	2474	prep_vma_destroy(vm, vma, false);
@@ -2727,7 +2743,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct drm_gpuva_ops ops, · 268 unmodified lines
2727	2743
2728	2744	if (xe_vma_is_cpu_addr_mirror(vma) &&
2729	2745	xe_svm_has_mapping(vm, xe_vma_start(vma),
2730		− xe_vma_end(vma)))
	2746	+ xe_vma_end(vma)) &&
	2747	+ !(vops->flags & XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP))
2731	2748	return -EBUSY;
2732	2749
2733	2750	if (!xe_vma_is_cpu_addr_mirror(vma))
@@ -3107,19 +3124,19 @@ static struct dma_fence ops_execute(struct xe_vm vm, · 373 unmodified lines
3107	3124	struct dma_fence *fence = NULL;
3108	3125	struct dma_fence **fences = NULL;
3109	3126	struct dma_fence_array *cf = NULL;
3110		− int number_tiles = 0, current_fence = 0, n_fence = 0, err;
	3127	+ int number_tiles = 0, current_fence = 0, n_fence = 0, err, i;
3111	3128	u8 id;
3112	3129
3113	3130	number_tiles = vm_ops_setup_tile_args(vm, vops);
3114	3131	if (number_tiles == 0)
3115	3132	return ERR_PTR(-ENODATA);
3116	3133
3117		− if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) {
3118		− for_each_tile(tile, vm->xe, id)
3119		− ++n_fence;
3120		− } else {
3121		− for_each_tile(tile, vm->xe, id)
3122		− n_fence += (1 + XE_MAX_GT_PER_TILE);
	3134	+ for_each_tile(tile, vm->xe, id) {
	3135	+ ++n_fence;
	3136	+
	3137	+ if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT))
	3138	+ for_each_tlb_inval(i)
	3139	+ ++n_fence;
3123	3140	}
3124	3141
3125	3142	fences = kmalloc_array(n_fence, sizeof(*fences), GFP_KERNEL);
@@ -3149,7 +3166,6 @@ static struct dma_fence ops_execute(struct xe_vm vm, · 23 unmodified lines
3149	3166
3150	3167	for_each_tile(tile, vm->xe, id) {
3151	3168	struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q;
3152		− int i;
3153	3169
3154	3170	fence = NULL;
3155	3171	if (!vops->pt_update_ops[id].num_ops)
@@ -3214,7 +3230,8 @@ static void op_add_ufence(struct xe_vm vm, struct xe_vma_op op, · 58 unmodified lines
3214	3230	{
3215	3231	switch (op->base.op) {
3216	3232	case DRM_GPUVA_OP_MAP:
3217		− vma_add_ufence(op->map.vma, ufence);
	3233	+ if (!xe_vma_is_cpu_addr_mirror(op->map.vma))
	3234	+ vma_add_ufence(op->map.vma, ufence);
3218	3235	break;
3219	3236	case DRM_GPUVA_OP_REMAP:
3220	3237	if (op->remap.prev)
@@ -3490,6 +3507,10 @@ static int xe_vm_bind_ioctl_validate_bo(struct xe_device xe, struct xe_bo bo, · 269 unmodified lines
3490	3507	{
3491	3508	u16 coh_mode;
3492	3509
	3510	+ if (XE_IOCTL_DBG(xe, (bo->flags & XE_BO_FLAG_NO_COMPRESSION) &&
	3511	+ xe_pat_index_get_comp_en(xe, pat_index)))
	3512	+ return -EINVAL;
	3513	+
3493	3514	if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) \|\|
3494	3515	XE_IOCTL_DBG(xe, obj_offset >
3495	3516	xe_bo_size(bo) - range)) {
@@ -3913,7 +3934,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, · 417 unmodified lines
3913	3934
3914	3935	err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
3915	3936	&fence[fence_id], start, end,
3916		− vm->usm.asid);
	3937	+ vm->usm.asid, NULL);
3917	3938	if (err)
3918	3939	goto wait;
3919	3940	++fence_id;
@@ -3926,7 +3947,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, · 6 unmodified lines
3926	3947
3927	3948	err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
3928	3949	&fence[fence_id], start, end,
3929		− vm->usm.asid);
	3950	+ vm->usm.asid, NULL);
3930	3951	if (err)
3931	3952	goto wait;
3932	3953	++fence_id;
@@ -4032,10 +4053,18 @@ int xe_vm_validate_protected(struct xe_vm *vm) · 99 unmodified lines
4032	4053	}
4033	4054
4034	4055	struct xe_vm_snapshot {
	4056	+ int uapi_flags;
4035	4057	unsigned long num_snaps;
4036	4058	struct {
4037	4059	u64 ofs, bo_ofs;
4038	4060	unsigned long len;
	4061	+#define XE_VM_SNAP_FLAG_USERPTR BIT(0)
	4062	+#define XE_VM_SNAP_FLAG_READ_ONLY BIT(1)
	4063	+#define XE_VM_SNAP_FLAG_IS_NULL BIT(2)
	4064	+ unsigned long flags;
	4065	+ int uapi_mem_region;
	4066	+ int pat_index;
	4067	+ int cpu_caching;
4039	4068	struct xe_bo *bo;
4040	4069	void *data;
4041	4070	struct mm_struct *mm;
@@ -4064,6 +4093,13 @@ struct xe_vm_snapshot xe_vm_snapshot_capture(struct xe_vm vm) · 22 unmodified lines
4064	4093	goto out_unlock;
4065	4094	}
4066	4095
	4096	+ if (vm->flags & XE_VM_FLAG_FAULT_MODE)
	4097	+ snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_FAULT_MODE;
	4098	+ if (vm->flags & XE_VM_FLAG_LR_MODE)
	4099	+ snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_LR_MODE;
	4100	+ if (vm->flags & XE_VM_FLAG_SCRATCH_PAGE)
	4101	+ snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE;
	4102	+
4067	4103	snap->num_snaps = num_snaps;
4068	4104	i = 0;
4069	4105	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
@@ -4076,9 +4112,25 @@ struct xe_vm_snapshot xe_vm_snapshot_capture(struct xe_vm vm) · 6 unmodified lines
4076	4112
4077	4113	snap->snap[i].ofs = xe_vma_start(vma);
4078	4114	snap->snap[i].len = xe_vma_size(vma);
	4115	+ snap->snap[i].flags = xe_vma_read_only(vma) ?
	4116	+ XE_VM_SNAP_FLAG_READ_ONLY : 0;
	4117	+ snap->snap[i].pat_index = vma->attr.pat_index;
4079	4118	if (bo) {
	4119	+ snap->snap[i].cpu_caching = bo->cpu_caching;
4080	4120	snap->snap[i].bo = xe_bo_get(bo);
4081	4121	snap->snap[i].bo_ofs = xe_vma_bo_offset(vma);
	4122	+ switch (bo->ttm.resource->mem_type) {
	4123	+ case XE_PL_SYSTEM:
	4124	+ case XE_PL_TT:
	4125	+ snap->snap[i].uapi_mem_region = 0;
	4126	+ break;
	4127	+ case XE_PL_VRAM0:
	4128	+ snap->snap[i].uapi_mem_region = 1;
	4129	+ break;
	4130	+ case XE_PL_VRAM1:
	4131	+ snap->snap[i].uapi_mem_region = 2;
	4132	+ break;
	4133	+ }
4082	4134	} else if (xe_vma_is_userptr(vma)) {
4083	4135	struct mm_struct *mm =
4084	4136	to_userptr_vma(vma)->userptr.notifier.mm;
@@ -4089,8 +4141,14 @@ struct xe_vm_snapshot xe_vm_snapshot_capture(struct xe_vm vm) · 4 unmodified lines
4089	4141	snap->snap[i].data = ERR_PTR(-EFAULT);
4090	4142
4091	4143	snap->snap[i].bo_ofs = xe_vma_userptr(vma);
	4144	+ snap->snap[i].flags \|= XE_VM_SNAP_FLAG_USERPTR;
	4145	+ snap->snap[i].uapi_mem_region = 0;
	4146	+ } else if (xe_vma_is_null(vma)) {
	4147	+ snap->snap[i].flags \|= XE_VM_SNAP_FLAG_IS_NULL;
	4148	+ snap->snap[i].uapi_mem_region = -1;
4092	4149	} else {
4093	4150	snap->snap[i].data = ERR_PTR(-ENOENT);
	4151	+ snap->snap[i].uapi_mem_region = -1;
4094	4152	}
4095	4153	i++;
4096	4154	}
@@ -4109,7 +4167,8 @@ void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap) · 12 unmodified lines
4109	4167	struct xe_bo *bo = snap->snap[i].bo;
4110	4168	int err;
4111	4169
4112		− if (IS_ERR(snap->snap[i].data))
	4170	+ if (IS_ERR(snap->snap[i].data) \|\|
	4171	+ snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL)
4113	4172	continue;
4114	4173
4115	4174	snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER);
@@ -4155,15 +4214,32 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot snap, struct drm_printer p) · 39 unmodified lines
4155	4214	return;
4156	4215	}
4157	4216
	4217	+ drm_printf(p, "VM.uapi_flags: 0x%x\n", snap->uapi_flags);
4158	4218	for (i = 0; i < snap->num_snaps; i++) {
4159	4219	drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len);
4160	4220
	4221	+ drm_printf(p, "[%llx].properties: %s\|%s\|mem_region=0x%lx\|pat_index=%d\|cpu_caching=%d\n",
	4222	+ snap->snap[i].ofs,
	4223	+ snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ?
	4224	+ "read_only" : "read_write",
	4225	+ snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL ?
	4226	+ "null_sparse" :
	4227	+ snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ?
	4228	+ "userptr" : "bo",
	4229	+ snap->snap[i].uapi_mem_region == -1 ? 0 :
	4230	+ BIT(snap->snap[i].uapi_mem_region),
	4231	+ snap->snap[i].pat_index,
	4232	+ snap->snap[i].cpu_caching);
	4233	+
4161	4234	if (IS_ERR(snap->snap[i].data)) {
4162	4235	drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs,
4163	4236	PTR_ERR(snap->snap[i].data));
4164	4237	continue;
4165	4238	}
4166	4239
	4240	+ if (snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL)
	4241	+ continue;
	4242	+
4167	4243	drm_printf(p, "[%llx].data: ", snap->snap[i].ofs);
4168	4244
4169	4245	for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) {
@@ -4317,6 +4393,8 @@ static int xe_vm_alloc_vma(struct xe_vm *vm, · 147 unmodified lines
4317	4393
4318	4394	if (is_madvise)
4319	4395	vops.flags \|= XE_VMA_OPS_FLAG_MADVISE;
	4396	+ else
	4397	+ vops.flags \|= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP;
4320	4398
4321	4399	err = vm_bind_ioctl_ops_parse(vm, ops, &vops);
4322	4400	if (err)
@@ -4390,6 +4468,46 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) · 67 unmodified lines
4390	4468	return xe_vm_alloc_vma(vm, &map_req, true);
4391	4469	}
4392	4470
	4471	+static bool is_cpu_addr_vma_with_default_attr(struct xe_vma *vma)
	4472	+{
	4473	+ return vma && xe_vma_is_cpu_addr_mirror(vma) &&
	4474	+ xe_vma_has_default_mem_attrs(vma);
	4475	+}
	4476	+
	4477	+/**
	4478	+ * xe_vm_find_cpu_addr_mirror_vma_range - Extend a VMA range to include adjacent CPU-mirrored VMAs
	4479	+ * @vm: VM to search within
	4480	+ * @start: Input/output pointer to the starting address of the range
	4481	+ * @end: Input/output pointer to the end address of the range
	4482	+ *
	4483	+ * Given a range defined by @start and @range, this function checks the VMAs
	4484	+ * immediately before and after the range. If those neighboring VMAs are
	4485	+ * CPU-address-mirrored and have default memory attributes, the function
	4486	+ * updates @start and @range to include them. This extended range can then
	4487	+ * be used for merging or other operations that require a unified VMA.
	4488	+ *
	4489	+ * The function does not perform the merge itself; it only computes the
	4490	+ * mergeable boundaries.
	4491	+ */
	4492	+void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm vm, u64 start, u64 *end)
	4493	+{
	4494	+ struct xe_vma prev, next;
	4495	+
	4496	+ lockdep_assert_held(&vm->lock);
	4497	+
	4498	+ if (*start >= SZ_4K) {
	4499	+ prev = xe_vm_find_vma_by_addr(vm, *start - SZ_4K);
	4500	+ if (is_cpu_addr_vma_with_default_attr(prev))
	4501	+ *start = xe_vma_start(prev);
	4502	+ }
	4503	+
	4504	+ if (*end < vm->size) {
	4505	+ next = xe_vm_find_vma_by_addr(vm, *end + 1);
	4506	+ if (is_cpu_addr_vma_with_default_attr(next))
	4507	+ *end = xe_vma_end(next);
	4508	+ }
	4509	+}
	4510	+
4393	4511	/**
4394	4512	* xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma
4395	4513	* @vm: Pointer to the xe_vm structure

drivers/gpu/drm/xe/xe_vm.h+3−0

@@ -68,6 +68,9 @@ xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range); · 67 unmodified lines
68	68
69	69	bool xe_vma_has_default_mem_attrs(struct xe_vma *vma);
70	70
	71	+void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm,
	72	+ u64 *start,
	73	+ u64 *end);
71	74	/**
72	75	* xe_vm_has_scratch() - Whether the vm is configured for scratch PTEs
73	76	* @vm: The vm

drivers/gpu/drm/xe/xe_vm_types.h+1−0

@@ -467,6 +467,7 @@ struct xe_vma_ops { · 466 unmodified lines
467	467	#define XE_VMA_OPS_FLAG_MADVISE BIT(1)
468	468	#define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2)
469	469	#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3)
	470	+#define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP BIT(4)
470	471	u32 flags;
471	472	#ifdef TEST_VM_OPS_ERROR
472	473	/** @inject_error: inject error to test error handling */

drivers/gpu/drm/xe/xe_vram.c+2−4

@@ -156,12 +156,11 @@ static int determine_lmem_bar_size(struct xe_device xe, struct xe_vram_region · 155 unmodified lines
156	156	static int get_flat_ccs_offset(struct xe_gt gt, u64 tile_size, u64 poffset)
157	157	{
158	158	struct xe_device *xe = gt_to_xe(gt);
159		− unsigned int fw_ref;
160	159	u64 offset;
161	160	u32 reg;
162	161
163		− fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
164		− if (!fw_ref)
	162	+ CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
	163	+ if (!fw_ref.domains)
165	164	return -ETIMEDOUT;
166	165
167	166	if (GRAPHICS_VER(xe) >= 20) {
@@ -193,7 +192,6 @@ static int get_flat_ccs_offset(struct xe_gt gt, u64 tile_size, u64 poffset) · 25 unmodified lines
193	192	offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K;
194	193	}
195	194
196		− xe_force_wake_put(gt_to_fw(gt), fw_ref);
197	195	*poffset = offset;
198	196
199	197	return 0;

drivers/gpu/drm/xe/xe_wa.c+6−46

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15
16	16	#include "regs/xe_engine_regs.h"
17	17	#include "regs/xe_gt_regs.h"
	18	+#include "regs/xe_guc_regs.h"
18	19	#include "regs/xe_regs.h"
19	20	#include "xe_device_types.h"
20	21	#include "xe_force_wake.h"
@@ -216,20 +217,6 @@ static const struct xe_rtp_entry_sr gt_was[] = { · 195 unmodified lines
216	217	XE_RTP_ACTIONS(SET(XELPMP_SQCNT1, ENFORCE_RAR))
217	218	},
218	219
219		− /* Xe2_LPG */
220		−
221		− { XE_RTP_NAME("16020975621"),
222		− XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)),
223		− XE_RTP_ACTIONS(SET(XEHP_SLICE_UNIT_LEVEL_CLKGATE, SBEUNIT_CLKGATE_DIS))
224		− },
225		− { XE_RTP_NAME("14018157293"),
226		− XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)),
227		− XE_RTP_ACTIONS(SET(XEHPC_L3CLOS_MASK(0), ~0),
228		− SET(XEHPC_L3CLOS_MASK(1), ~0),
229		− SET(XEHPC_L3CLOS_MASK(2), ~0),
230		− SET(XEHPC_L3CLOS_MASK(3), ~0))
231		− },
232		−
233	220	/* Xe2_LPM */
234	221
235	222	{ XE_RTP_NAME("14017421178"),
@@ -315,6 +302,10 @@ static const struct xe_rtp_entry_sr gt_was[] = { · 79 unmodified lines
315	302	XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)),
316	303	XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
317	304	},
	305	+ { XE_RTP_NAME("16028005424"),
	306	+ XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005)),
	307	+ XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES))
	308	+ },
318	309	};
319	310
320	311	static const struct xe_rtp_entry_sr engine_was[] = {
@@ -504,11 +495,6 @@ static const struct xe_rtp_entry_sr engine_was[] = { · 183 unmodified lines
504	495	XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
505	496	XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))
506	497	},
507		− { XE_RTP_NAME("14018957109"),
508		− XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
509		− FUNC(xe_rtp_match_first_render_or_compute)),
510		− XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN5, DISABLE_SAMPLE_G_PERFORMANCE))
511		− },
512	498	{ XE_RTP_NAME("14020338487"),
513	499	XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
514	500	XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))
@@ -518,11 +504,6 @@ static const struct xe_rtp_entry_sr engine_was[] = { · 3 unmodified lines
518	504	FUNC(xe_rtp_match_first_render_or_compute)),
519	505	XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH))
520	506	},
521		− { XE_RTP_NAME("14019322943"),
522		− XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
523		− FUNC(xe_rtp_match_first_render_or_compute)),
524		− XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, TGM_WRITE_EOM_FORCE))
525		− },
526	507	{ XE_RTP_NAME("14018471104"),
527	508	XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
528	509	XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))
@@ -693,7 +674,7 @@ static const struct xe_rtp_entry_sr engine_was[] = { · 164 unmodified lines
693	674	XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
694	675	},
695	676	{ XE_RTP_NAME("18041344222"),
696		− XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001),
	677	+ XE_RTP_RULES(GRAPHICS_VERSION(3000),
697	678	FUNC(xe_rtp_match_first_render_or_compute),
698	679	FUNC(xe_rtp_match_not_sriov_vf),
699	680	FUNC(xe_rtp_match_gt_has_discontiguous_dss_groups)),
@@ -799,17 +780,6 @@ static const struct xe_rtp_entry_sr lrc_was[] = { · 99 unmodified lines
799	780
800	781	/* Xe2_LPG */
801	782
802		− { XE_RTP_NAME("16020518922"),
803		− XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
804		− ENGINE_CLASS(RENDER)),
805		− XE_RTP_ACTIONS(SET(FF_MODE,
806		− DIS_TE_AUTOSTRIP \|
807		− DIS_MESH_PARTIAL_AUTOSTRIP \|
808		− DIS_MESH_AUTOSTRIP),
809		− SET(VFLSKPD,
810		− DIS_PARTIAL_AUTOSTRIP \|
811		− DIS_AUTOSTRIP))
812		− },
813	783	{ XE_RTP_NAME("14019386621"),
814	784	XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
815	785	XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))
@@ -818,20 +788,10 @@ static const struct xe_rtp_entry_sr lrc_was[] = { · 2 unmodified lines
818	788	XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
819	789	XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
820	790	},
821		− { XE_RTP_NAME("14020013138"),
822		− XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
823		− ENGINE_CLASS(RENDER)),
824		− XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))
825		− },
826	791	{ XE_RTP_NAME("14019988906"),
827	792	XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
828	793	XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))
829	794	},
830		− { XE_RTP_NAME("16020183090"),
831		− XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
832		− ENGINE_CLASS(RENDER)),
833		− XE_RTP_ACTIONS(SET(INSTPM(RENDER_RING_BASE), ENABLE_SEMAPHORE_POLL_BIT))
834		− },
835	795	{ XE_RTP_NAME("18033852989"),
836	796	XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
837	797	XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))

drivers/gpu/drm/xe/xe_wa_oob.rules+1−5

@@ -16,15 +16,11 @@ · 15 unmodified lines
16	16	16017236439 PLATFORM(PVC)
17	17	14019821291 MEDIA_VERSION_RANGE(1300, 2000)
18	18	14015076503 MEDIA_VERSION(1300)
19		−16020292621 GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
20		−14018913170 GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
21		− MEDIA_VERSION(2000), GRAPHICS_STEP(A0, A1)
22		− GRAPHICS_VERSION_RANGE(1270, 1274)
	19	+14018913170 GRAPHICS_VERSION_RANGE(1270, 1274)
23	20	MEDIA_VERSION(1300)
24	21	PLATFORM(DG2)
25	22	14018094691 GRAPHICS_VERSION_RANGE(2001, 2002)
26	23	GRAPHICS_VERSION(2004)
27		−14019882105 GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
28	24	18024947630 GRAPHICS_VERSION(2001)
29	25	GRAPHICS_VERSION(2004)
30	26	MEDIA_VERSION(2000)

include/uapi/drm/xe_drm.h+78−3

@@ -106,6 +106,7 @@ extern "C" { · 105 unmodified lines
106	106	#define DRM_XE_OBSERVATION 0x0b
107	107	#define DRM_XE_MADVISE 0x0c
108	108	#define DRM_XE_VM_QUERY_MEM_RANGE_ATTRS 0x0d
	109	+#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x0e
109	110
110	111	/* Must be kept compact -- no holes */
111	112
@@ -123,6 +124,7 @@ extern "C" { · 11 unmodified lines
123	124	#define DRM_IOCTL_XE_OBSERVATION DRM_IOW(DRM_COMMAND_BASE + DRM_XE_OBSERVATION, struct drm_xe_observation_param)
124	125	#define DRM_IOCTL_XE_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_MADVISE, struct drm_xe_madvise)
125	126	#define DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_QUERY_MEM_RANGE_ATTRS, struct drm_xe_vm_query_mem_range_attr)
	127	+#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property)
126	128
127	129	/**
128	130	* DOC: Xe IOCTL Extensions
@@ -210,8 +212,12 @@ struct drm_xe_ext_set_property { · 81 unmodified lines
210	212	/** @pad: MBZ */
211	213	__u32 pad;
212	214
213		− /** @value: property value */
214		− __u64 value;
	215	+ union {
	216	+ /** @value: property value */
	217	+ __u64 value;
	218	+ /** @ptr: pointer to user value */
	219	+ __u64 ptr;
	220	+ };
215	221
216	222	/** @reserved: Reserved */
217	223	__u64 reserved[2];
@@ -403,6 +409,9 @@ struct drm_xe_query_mem_regions { · 185 unmodified lines
403	409	* has low latency hint support
404	410	* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the
405	411	* device has CPU address mirroring support
	412	+ * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT - Flag is set if the
	413	+ * device supports the userspace hint %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION.
	414	+ * This is exposed only on Xe2+.
406	415	* - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment
407	416	* required by this device, typically SZ_4K or SZ_64K
408	417	* - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address
@@ -421,6 +430,7 @@ struct drm_xe_query_config { · 12 unmodified lines
421	430	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0)
422	431	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1)
423	432	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)
	433	+ #define DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT (1 << 3)
424	434	#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2
425	435	#define DRM_XE_QUERY_CONFIG_VA_BITS 3
426	436	#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4
@@ -791,6 +801,17 @@ struct drm_xe_device_query { · 364 unmodified lines
791	801	* need to use VRAM for display surfaces, therefore the kernel requires
792	802	* setting this flag for such objects, otherwise an error is thrown on
793	803	* small-bar systems.
	804	+ * - %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION - Allows userspace to
	805	+ * hint that compression (CCS) should be disabled for the buffer being
	806	+ * created. This can avoid unnecessary memory operations and CCS state
	807	+ * management.
	808	+ * On pre-Xe2 platforms, this flag is currently rejected as compression
	809	+ * control is not supported via PAT index. On Xe2+ platforms, compression
	810	+ * is controlled via PAT entries. If this flag is set, the driver will reject
	811	+ * any VM bind that requests a PAT index enabling compression for this BO.
	812	+ * Note: On dGPU platforms, there is currently no change in behavior with
	813	+ * this flag, but future improvements may leverage it. The current benefit is
	814	+ * primarily applicable to iGPU platforms.
794	815	*
795	816	* @cpu_caching supports the following values:
796	817	* - %DRM_XE_GEM_CPU_CACHING_WB - Allocate the pages with write-back
@@ -837,6 +858,7 @@ struct drm_xe_gem_create { · 40 unmodified lines
837	858	#define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING (1 << 0)
838	859	#define DRM_XE_GEM_CREATE_FLAG_SCANOUT (1 << 1)
839	860	#define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM (1 << 2)
	861	+#define DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION (1 << 3)
840	862	/**
841	863	* @flags: Flags, currently a mask of memory instances of where BO can
842	864	* be placed
@@ -1252,6 +1274,20 @@ struct drm_xe_vm_bind { · 409 unmodified lines
1252	1274	* Given that going into a power-saving state kills PXP HWDRM sessions,
1253	1275	* runtime PM will be blocked while queues of this type are alive.
1254	1276	* All PXP queues will be killed if a PXP invalidation event occurs.
	1277	+ * - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP - Create a multi-queue group
	1278	+ * or add secondary queues to a multi-queue group.
	1279	+ * If the extension's 'value' field has %DRM_XE_MULTI_GROUP_CREATE flag set,
	1280	+ * then a new multi-queue group is created with this queue as the primary queue
	1281	+ * (Q0). Otherwise, the queue gets added to the multi-queue group whose primary
	1282	+ * queue's exec_queue_id is specified in the lower 32 bits of the 'value' field.
	1283	+ * If the extension's 'value' field has %DRM_XE_MULTI_GROUP_KEEP_ACTIVE flag
	1284	+ * set, then the multi-queue group is kept active after the primary queue is
	1285	+ * destroyed.
	1286	+ * All the other non-relevant bits of extension's 'value' field while adding the
	1287	+ * primary or the secondary queues of the group must be set to 0.
	1288	+ * - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY - Set the queue
	1289	+ * priority within the multi-queue group. Current valid priority values are 0–2
	1290	+ * (default is 1), with higher values indicating higher priority.
1255	1291	*
1256	1292	* The example below shows how to use @drm_xe_exec_queue_create to create
1257	1293	* a simple exec_queue (no parallel submission) of class
@@ -1292,6 +1328,11 @@ struct drm_xe_exec_queue_create { · 34 unmodified lines
1292	1328	#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0
1293	1329	#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1
1294	1330	#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE 2
	1331	+#define DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE 3
	1332	+#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP 4
	1333	+#define DRM_XE_MULTI_GROUP_CREATE (1ull << 63)
	1334	+#define DRM_XE_MULTI_GROUP_KEEP_ACTIVE (1ull << 62)
	1335	+#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY 5
1295	1336	/** @extensions: Pointer to the first extension struct, if any */
1296	1337	__u64 extensions;
1297	1338
@@ -1655,6 +1696,9 @@ enum drm_xe_oa_unit_type { · 357 unmodified lines
1655	1696
1656	1697	/** @DRM_XE_OA_UNIT_TYPE_OAM_SAG: OAM_SAG OA unit */
1657	1698	DRM_XE_OA_UNIT_TYPE_OAM_SAG,
	1699	+
	1700	+ /** @DRM_XE_OA_UNIT_TYPE_MERT: MERT OA unit */
	1701	+ DRM_XE_OA_UNIT_TYPE_MERT,
1658	1702	};
1659	1703
1660	1704	/**
@@ -1677,12 +1721,19 @@ struct drm_xe_oa_unit { · 16 unmodified lines
1677	1721	#define DRM_XE_OA_CAPS_OA_BUFFER_SIZE (1 << 2)
1678	1722	#define DRM_XE_OA_CAPS_WAIT_NUM_REPORTS (1 << 3)
1679	1723	#define DRM_XE_OA_CAPS_OAM (1 << 4)
	1724	+#define DRM_XE_OA_CAPS_OA_UNIT_GT_ID (1 << 5)
1680	1725
1681	1726	/** @oa_timestamp_freq: OA timestamp freq */
1682	1727	__u64 oa_timestamp_freq;
1683	1728
	1729	+ /** @gt_id: gt id for this OA unit */
	1730	+ __u16 gt_id;
	1731	+
	1732	+ /** @reserved1: MBZ */
	1733	+ __u16 reserved1[3];
	1734	+
1684	1735	/** @reserved: MBZ */
1685		− __u64 reserved[4];
	1736	+ __u64 reserved[3];
1686	1737
1687	1738	/** @num_engines: number of engines in @eci array */
1688	1739	__u64 num_engines;
@@ -2274,6 +2325,30 @@ struct drm_xe_vm_query_mem_range_attr { · 585 unmodified lines
2274	2325
2275	2326	};
2276	2327
	2328	+/**
	2329	+ * struct drm_xe_exec_queue_set_property - exec queue set property
	2330	+ *
	2331	+ * Sets execution queue properties dynamically.
	2332	+ * Currently only %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY
	2333	+ * property can be dynamically set.
	2334	+ */
	2335	+struct drm_xe_exec_queue_set_property {
	2336	+ /** @extensions: Pointer to the first extension struct, if any */
	2337	+ __u64 extensions;
	2338	+
	2339	+ /** @exec_queue_id: Exec queue ID */
	2340	+ __u32 exec_queue_id;
	2341	+
	2342	+ /** @property: property to set */
	2343	+ __u32 property;
	2344	+
	2345	+ /** @value: property value */
	2346	+ __u64 value;
	2347	+
	2348	+ /** @reserved: Reserved */
	2349	+ __u64 reserved[2];
	2350	+};
	2351	+
2277	2352	#if defined(__cplusplus)
2278	2353	}
2279	2354	#endif