lael-labs/linux

.mailmap+1−0

@@ -481,6 +481,7 @@ Lorenzo Pieralisi <lpieralisi@kernel.org> <lorenzo.pieralisi@arm.com> · 480 unmodified lines
481	Lorenzo Stoakes <lorenzo.stoakes@oracle.com> <lstoakes@gmail.com>	481	Lorenzo Stoakes <lorenzo.stoakes@oracle.com> <lstoakes@gmail.com>
482	Luca Ceresoli <luca.ceresoli@bootlin.com> <luca@lucaceresoli.net>	482	Luca Ceresoli <luca.ceresoli@bootlin.com> <luca@lucaceresoli.net>
483	Luca Weiss <luca@lucaweiss.eu> <luca@z3ntu.xyz>	483	Luca Weiss <luca@lucaweiss.eu> <luca@z3ntu.xyz>
		484	Lucas De Marchi <demarchi@kernel.org> <lucas.demarchi@intel.com>
484	Lukasz Luba <lukasz.luba@arm.com> <l.luba@partner.samsung.com>	485	Lukasz Luba <lukasz.luba@arm.com> <l.luba@partner.samsung.com>
485	Luo Jie <quic_luoj@quicinc.com> <luoj@codeaurora.org>	486	Luo Jie <quic_luoj@quicinc.com> <luoj@codeaurora.org>
486	Lance Yang <lance.yang@linux.dev> <ioworker0@gmail.com>	487	Lance Yang <lance.yang@linux.dev> <ioworker0@gmail.com>

Documentation/ABI/testing/sysfs-driver-intel-xe-sriov+1−1

@@ -119,7 +119,7 @@ Description: · 118 unmodified lines
119	The GT preemption timeout (PT) in [us] to be applied to all functions.	119	The GT preemption timeout (PT) in [us] to be applied to all functions.
120	See sriov_admin/{pf,vf<N>}/profile/preempt_timeout_us for more details.	120	See sriov_admin/{pf,vf<N>}/profile/preempt_timeout_us for more details.
121		121
122	sched_priority: (RW/RO) string	122	sched_priority: (WO) string
123	The GT scheduling priority to be applied for all functions.	123	The GT scheduling priority to be applied for all functions.
124	See sriov_admin/{pf,vf<N>}/profile/sched_priority for more details.	124	See sriov_admin/{pf,vf<N>}/profile/sched_priority for more details.
125		125

Documentation/gpu/xe/xe_exec_queue.rst+14−0

@@ -7,6 +7,20 @@ Execution Queue · 6 unmodified lines
7	.. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c	7	.. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c
8	:doc: Execution Queue	8	:doc: Execution Queue
9		9
		10	Multi Queue Group
		11	=================
		12
		13	.. kernel-doc:: drivers/gpu/drm/xe/xe_exec_queue.c
		14	:doc: Multi Queue Group
		15
		16	.. _multi-queue-group-guc-interface:
		17
		18	Multi Queue Group GuC interface
		19	===============================
		20
		21	.. kernel-doc:: drivers/gpu/drm/xe/xe_guc_submit.c
		22	:doc: Multi Queue Group GuC interface
		23
10	Internal API	24	Internal API
11	============	25	============
12		26

MAINTAINERS+1−1

@@ -12640,7 +12640,7 @@ F: include/drm/intel/ · 12639 unmodified lines
12640	F: include/uapi/drm/i915_drm.h	12640	F: include/uapi/drm/i915_drm.h
12641		12641
12642	INTEL DRM XE DRIVER (Lunar Lake and newer)	12642	INTEL DRM XE DRIVER (Lunar Lake and newer)
12643	M: Lucas De Marchi <lucas.demarchi@intel.com>	12643	M: Matthew Brost <matthew.brost@intel.com>
12644	M: Thomas Hellström <thomas.hellstrom@linux.intel.com>	12644	M: Thomas Hellström <thomas.hellstrom@linux.intel.com>
12645	M: Rodrigo Vivi <rodrigo.vivi@intel.com>	12645	M: Rodrigo Vivi <rodrigo.vivi@intel.com>
12646	L: intel-xe@lists.freedesktop.org	12646	L: intel-xe@lists.freedesktop.org

drivers/gpu/drm/drm_gpusvm.c+3−0

@@ -1288,6 +1288,9 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, · 1287 unmodified lines
1288	DMA_BIDIRECTIONAL;	1288	DMA_BIDIRECTIONAL;
1289		1289
1290	retry:	1290	retry:
		1291	if (time_after(jiffies, timeout))
		1292	return -EBUSY;
		1293
1291	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);	1294	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
1292	if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages))	1295	if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages))
1293	goto set_seqno;	1296	goto set_seqno;

drivers/gpu/drm/xe/Makefile+2−0

 	xe_oa.o \
 	xe_observation.o \
 	xe_pagefault.o \
+	xe_page_reclaim.o \
 	xe_pat.o \
 	xe_pci.o \
 	xe_pcode.o \
 	xe_lmtt.o \
 	xe_lmtt_2l.o \
 	xe_lmtt_ml.o \
+	xe_mert.o \
 	xe_pci_sriov.o \
 	xe_sriov_packet.o \
 	xe_sriov_pf.o \

drivers/gpu/drm/xe/abi/guc_actions_abi.h+6−0

 	XE_GUC_ACTION_DEREGISTER_G2G = 0x4508,
 	XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600,
 	XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601,
+	XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE = 0x4602,
+	XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC = 0x4603,
+	XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE = 0x4604,
+	XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR = 0x4605,
 	XE_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507,
 	XE_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A,
 	XE_GUC_ACTION_SET_DEVICE_ENGINE_ACTIVITY_BUFFER = 0x550C,
 	XE_GUC_ACTION_TLB_INVALIDATION = 0x7000,
 	XE_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001,
 	XE_GUC_ACTION_TLB_INVALIDATION_ALL = 0x7002,
+	XE_GUC_ACTION_PAGE_RECLAMATION = 0x7003,
+	XE_GUC_ACTION_PAGE_RECLAMATION_DONE = 0x7004,
 	XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002,
 	XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003,
 	XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004,

drivers/gpu/drm/xe/abi/guc_actions_sriov_abi.h+57−10

@@ -502,13 +502,17 @@ · 501 unmodified lines
502	#define VF2GUC_VF_RESET_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0	502	#define VF2GUC_VF_RESET_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0
503		503
504	/**	504	/**
505	* DOC: VF2GUC_NOTIFY_RESFIX_DONE	505	* DOC: VF2GUC_RESFIX_DONE
506	*	506	*
507	* This action is used by VF to notify the GuC that the VF KMD has completed	507	* This action is used by VF to inform the GuC that the VF KMD has completed
508	* post-migration recovery steps.	508	* post-migration recovery steps. From GuC VF compatibility 1.27.0 onwards, it
		509	* shall only be sent after posting RESFIX_START and that both @MARKER fields
		510	* must match.
509	*	511	*
510	* This message must be sent as `MMIO HXG Message`_.	512	* This message must be sent as `MMIO HXG Message`_.
511	*	513	*
		514	* Updated since GuC VF compatibility 1.27.0.
		515	*
512	* +---+-------+--------------------------------------------------------------+	516	* +---+-------+--------------------------------------------------------------+
513	* \| \| Bits \| Description \|	517	* \| \| Bits \| Description \|
514	* +===+=======+==============================================================+	518	* +===+=======+==============================================================+
@@ -516,9 +520,11 @@ · 1 unmodified line
516	* \| +-------+--------------------------------------------------------------+	520	* \| +-------+--------------------------------------------------------------+
517	* \| \| 30:28 \| TYPE = GUC_HXG_TYPE_REQUEST_ \|	521	* \| \| 30:28 \| TYPE = GUC_HXG_TYPE_REQUEST_ \|
518	* \| +-------+--------------------------------------------------------------+	522	* \| +-------+--------------------------------------------------------------+
519	* \| \| 27:16 \| DATA0 = MBZ \|	523	* \| \| 27:16 \| DATA0 = MARKER = MBZ (only prior 1.27.0) \|
520	* \| +-------+--------------------------------------------------------------+	524	* \| +-------+--------------------------------------------------------------+
521	* \| \| 15:0 \| ACTION = _`GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE` = 0x5508 \|	525	* \| \| 27:16 \| DATA0 = MARKER - can't be zero (1.27.0+) \|
		526	* \| +-------+--------------------------------------------------------------+
		527	* \| \| 15:0 \| ACTION = _`GUC_ACTION_VF2GUC_RESFIX_DONE` = 0x5508 \|
522	* +---+-------+--------------------------------------------------------------+	528	* +---+-------+--------------------------------------------------------------+
523	*	529	*
524	* +---+-------+--------------------------------------------------------------+	530	* +---+-------+--------------------------------------------------------------+
@@ -531,13 +537,13 @@ · 6 unmodified lines
531	* \| \| 27:0 \| DATA0 = MBZ \|	537	* \| \| 27:0 \| DATA0 = MBZ \|
532	* +---+-------+--------------------------------------------------------------+	538	* +---+-------+--------------------------------------------------------------+
533	*/	539	*/
534	#define GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE 0x5508u	540	#define GUC_ACTION_VF2GUC_RESFIX_DONE 0x5508u
535		541
536	#define VF2GUC_NOTIFY_RESFIX_DONE_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN	542	#define VF2GUC_RESFIX_DONE_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN
537	#define VF2GUC_NOTIFY_RESFIX_DONE_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0	543	#define VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER GUC_HXG_REQUEST_MSG_0_DATA0
538		544
539	#define VF2GUC_NOTIFY_RESFIX_DONE_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN	545	#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN
540	#define VF2GUC_NOTIFY_RESFIX_DONE_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0	546	#define VF2GUC_RESFIX_DONE_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0
541		547
542	/**	548	/**
543	* DOC: VF2GUC_QUERY_SINGLE_KLV	549	* DOC: VF2GUC_QUERY_SINGLE_KLV
@@ -656,4 +662,45 @@ · 112 unmodified lines
656	#define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN	662	#define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN
657	#define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_0_USED GUC_HXG_RESPONSE_MSG_0_DATA0	663	#define PF2GUC_SAVE_RESTORE_VF_RESPONSE_MSG_0_USED GUC_HXG_RESPONSE_MSG_0_DATA0
658		664
		665	/**
		666	* DOC: VF2GUC_RESFIX_START
		667	*
		668	* This action is used by VF to inform the GuC that the VF KMD will be starting
		669	* post-migration recovery fixups. The @MARKER sent with this action must match
		670	* with the MARKER posted in the VF2GUC_RESFIX_DONE message.
		671	*
		672	* This message must be sent as `MMIO HXG Message`_.
		673	*
		674	* Available since GuC VF compatibility 1.27.0.
		675	*
		676	* +---+-------+--------------------------------------------------------------+
		677	* \| \| Bits \| Description \|
		678	* +===+=======+==============================================================+
		679	* \| 0 \| 31 \| ORIGIN = GUC_HXG_ORIGIN_HOST_ \|
		680	* \| +-------+--------------------------------------------------------------+
		681	* \| \| 30:28 \| TYPE = GUC_HXG_TYPE_REQUEST_ \|
		682	* \| +-------+--------------------------------------------------------------+
		683	* \| \| 27:16 \| DATA0 = MARKER - can't be zero \|
		684	* \| +-------+--------------------------------------------------------------+
		685	* \| \| 15:0 \| ACTION = _`GUC_ACTION_VF2GUC_RESFIX_START` = 0x550F \|
		686	* +---+-------+--------------------------------------------------------------+
		687	*
		688	* +---+-------+--------------------------------------------------------------+
		689	* \| \| Bits \| Description \|
		690	* +===+=======+==============================================================+
		691	* \| 0 \| 31 \| ORIGIN = GUC_HXG_ORIGIN_GUC_ \|
		692	* \| +-------+--------------------------------------------------------------+
		693	* \| \| 30:28 \| TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ \|
		694	* \| +-------+--------------------------------------------------------------+
		695	* \| \| 27:0 \| DATA0 = MBZ \|
		696	* +---+-------+--------------------------------------------------------------+
		697	*/
		698	#define GUC_ACTION_VF2GUC_RESFIX_START 0x550Fu
		699
		700	#define VF2GUC_RESFIX_START_REQUEST_MSG_LEN GUC_HXG_REQUEST_MSG_MIN_LEN
		701	#define VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER GUC_HXG_REQUEST_MSG_0_DATA0
		702
		703	#define VF2GUC_RESFIX_START_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN
		704	#define VF2GUC_RESFIX_START_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0
		705
659	#endif	706	#endif

drivers/gpu/drm/xe/abi/guc_klvs_abi.h+9−0

  *      :1: NORMAL = schedule VF always, irrespective of whether it has work or not
  *      :2: HIGH = schedule VF in the next time-slice after current active
  *          time-slice completes if it has active work
+ *
+ * _`GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT` : 0x8A0D
+ *      Given that multi-LRC contexts are incompatible with SRIOV scheduler
+ *      groups and cause the latter to be turned off when registered with the
+ *      GuC, this config allows the PF to set a threshold for multi-LRC context
+ *      registrations by VFs to monitor their behavior.
  */
 #define GUC_KLV_VF_CFG_GGTT_START_KEY		0x0001
 #define   GUC_SCHED_PRIORITY_NORMAL		1u
 #define   GUC_SCHED_PRIORITY_HIGH		2u
+#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_KEY	0x8a0d
+#define GUC_KLV_VF_CFG_THRESHOLD_MULTI_LRC_COUNT_LEN	1u
 /*
  * Workaround keys:
  */

drivers/gpu/drm/xe/abi/guc_lfd_abi.h+171−0

@@ -0,0 +1,171 @@
	1	/* SPDX-License-Identifier: MIT */
	2	/*
	3	* Copyright © 2025 Intel Corporation
	4	*/
	5
	6	#ifndef _ABI_GUC_LFD_ABI_H_
	7	#define _ABI_GUC_LFD_ABI_H_
	8
	9	#include <linux/types.h>
	10
	11	#include "guc_lic_abi.h"
	12
	13	/* The current major version of GuC-Log-File format. */
	14	#define GUC_LFD_FORMAT_VERSION_MAJOR 0x0001
	15	/* The current minor version of GuC-Log-File format. */
	16	#define GUC_LFD_FORMAT_VERSION_MINOR 0x0000
	17
	18	/** enum guc_lfd_type - Log format descriptor type */
	19	enum guc_lfd_type {
	20	/**
	21	* @GUC_LFD_TYPE_FW_REQUIRED_RANGE_START: Start of range for
	22	* required LFDs from GuC
	23	* @GUC_LFD_TYPE_FW_VERSION: GuC Firmware Version structure.
	24	* @GUC_LFD_TYPE_GUC_DEVICE_ID: GuC microcontroller device ID.
	25	* @GUC_LFD_TYPE_TSC_FREQUENCY: Frequency of GuC timestamps.
	26	* @GUC_LFD_TYPE_GMD_ID: HW GMD ID.
	27	* @GUC_LFD_TYPE_BUILD_PLATFORM_ID: GuC build platform ID.
	28	* @GUC_LFD_TYPE_FW_REQUIRED_RANGE_END: End of range for
	29	* required LFDs from GuC
	30	*/
	31	GUC_LFD_TYPE_FW_REQUIRED_RANGE_START = 0x1,
	32	GUC_LFD_TYPE_FW_VERSION = 0x1,
	33	GUC_LFD_TYPE_GUC_DEVICE_ID = 0x2,
	34	GUC_LFD_TYPE_TSC_FREQUENCY = 0x3,
	35	GUC_LFD_TYPE_GMD_ID = 0x4,
	36	GUC_LFD_TYPE_BUILD_PLATFORM_ID = 0x5,
	37	GUC_LFD_TYPE_FW_REQUIRED_RANGE_END = 0x1FFF,
	38
	39	/**
	40	* @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START: Start of range for
	41	* optional LFDs from GuC
	42	* @GUC_LFD_TYPE_LOG_EVENTS_BUFFER: Log-event-entries buffer.
	43	* @GUC_LFD_TYPE_FW_CRASH_DUMP: GuC generated crash-dump blob.
	44	* @GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END: End of range for
	45	* optional LFDs from GuC
	46	*/
	47	GUC_LFD_TYPE_FW_OPTIONAL_RANGE_START = 0x2000,
	48	GUC_LFD_TYPE_LOG_EVENTS_BUFFER = 0x2000,
	49	GUC_LFD_TYPE_FW_CRASH_DUMP = 0x2001,
	50	GUC_LFD_TYPE_FW_OPTIONAL_RANGE_END = 0x3FFF,
	51
	52	/**
	53	* @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START: Start of range for
	54	* required KMD LFDs
	55	* @GUC_LFD_TYPE_OS_ID: An identifier for the OS.
	56	* @GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END: End of this range for
	57	* required KMD LFDs
	58	*/
	59	GUC_LFD_TYPE_KMD_REQUIRED_RANGE_START = 0x4000,
	60	GUC_LFD_TYPE_OS_ID = 0x4000,
	61	GUC_LFD_TYPE_KMD_REQUIRED_RANGE_END = 0x5FFF,
	62
	63	/**
	64	* @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START: Start of range for
	65	* optional KMD LFDs
	66	* @GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT: Binary representation of
	67	* GuC log-events schema.
	68	* @GUC_LFD_TYPE_HOST_COMMENT: ASCII string containing comments
	69	* from the host/KMD.
	70	* @GUC_LFD_TYPE_TIMESTAMP_ANCHOR: A timestamp anchor, to convert
	71	* between host and GuC timestamp.
	72	* @GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG: Timestamp anchor
	73	* configuration, definition of timestamp frequency and bit width.
	74	* @GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END: End of this range for
	75	* optional KMD LFDs
	76	*/
	77	GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_START = 0x6000,
	78	GUC_LFD_TYPE_BINARY_SCHEMA_FORMAT = 0x6000,
	79	GUC_LFD_TYPE_HOST_COMMENT = 0x6001,
	80	GUC_LFD_TYPE_TIMESTAMP_ANCHOR = 0x6002,
	81	GUC_LFD_TYPE_TIMESTAMP_ANCHOR_CONFIG = 0x6003,
	82	GUC_LFD_TYPE_KMD_OPTIONAL_RANGE_END = 0x7FFF,
	83
	84	/*
	85	* @GUC_LFD_TYPE_RESERVED_RANGE_START: Start of reserved range
	86	* @GUC_LFD_TYPE_RESERVED_RANGE_END: End of reserved range
	87	*/
	88	GUC_LFD_TYPE_RESERVED_RANGE_START = 0x8000,
	89	GUC_LFD_TYPE_RESERVED_RANGE_END = 0xFFFF,
	90	};
	91
	92	/** enum guc_lfd_os_type - OS Type LFD-ID */
	93	enum guc_lfd_os_type {
	94	/** @GUC_LFD_OS_TYPE_OSID_WIN: Windows OS */
	95	GUC_LFD_OS_TYPE_OSID_WIN = 0x1,
	96	/** @GUC_LFD_OS_TYPE_OSID_LIN: Linux OS */
	97	GUC_LFD_OS_TYPE_OSID_LIN = 0x2,
	98	/** @GUC_LFD_OS_TYPE_OSID_VMW: VMWare OS */
	99	GUC_LFD_OS_TYPE_OSID_VMW = 0x3,
	100	/** @GUC_LFD_OS_TYPE_OSID_OTHER: Other */
	101	GUC_LFD_OS_TYPE_OSID_OTHER = 0x4,
	102	};
	103
	104	/** struct guc_lfd_data - A generic header structure for all LFD blocks */
	105	struct guc_lfd_data {
	106	/** @header: A 32 bits dword, contains multiple bit fields */
	107	u32 header;
	108	/* LFD type. See guc_lfd_type */
	109	#define GUC_LFD_DATA_HEADER_MASK_TYPE GENMASK(31, 16)
	110	#define GUC_LFD_DATA_HEADER_MASK_MAGIC GENMASK(15, 0)
	111
	112	/** @data_count: Number of dwords the `data` field contains. */
	113	u32 data_count;
	114	/** @data: Data defined by GUC_LFD_DATA_HEADER_MASK_TYPE */
	115	u32 data[] __counted_by(data_count);
	116	} __packed;
	117
	118	/**
	119	* struct guc_lfd_data_log_events_buf - GuC Log Events Buffer.
	120	* This is optional fw LFD data
	121	*/
	122	struct guc_lfd_data_log_events_buf {
	123	/**
	124	* @log_events_format_version: version of GuC log format of buffer
	125	*/
	126	u32 log_events_format_version;
	127	/**
	128	* @log_event: The log event data.
	129	* Size in dwords is LFD block size - 1.
	130	*/
	131	u32 log_event[];
	132	} __packed;
	133
	134	/** struct guc_lfd_data_os_info - OS Version Information. */
	135	struct guc_lfd_data_os_info {
	136	/**
	137	* @os_id: enum values to identify the OS brand.
	138	* See guc_lfd_os_type for the range of types
	139	*/
	140	u32 os_id;
	141	/**
	142	* @build_version: ASCII string containing OS build version
	143	* information based on os_id. String is padded with null
	144	* characters to ensure its DWORD aligned.
	145	* Size in dwords is LFD block size - 1.
	146	*/
	147	char build_version[];
	148	} __packed;
	149
	150	/**
	151	* struct guc_logfile_header - Header of GuC Log Streaming-LFD-File Format.
	152	* This structure encapsulates the layout of the guc-log-file format
	153	*/
	154	struct guc_lfd_file_header {
	155	/**
	156	* @magic: A magic number set by producer of a GuC log file to
	157	* identify that file is a valid guc-log-file containing a stream
	158	* of LFDs.
	159	*/
	160	u64 magic;
	161	/** @version: Version of this file format layout */
	162	u32 version;
	163	#define GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR GENMASK(31, 16)
	164	#define GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR GENMASK(15, 0)
	165
	166	/** @stream: A stream of one or more guc_lfd_data LFD blocks
	167	*/
	168	u32 stream[];
	169	} __packed;
	170
	171	#endif

drivers/gpu/drm/xe/abi/guc_lic_abi.h+77−0

@@ -0,0 +1,77 @@
	1	/* SPDX-License-Identifier: MIT */
	2	/*
	3	* Copyright © 2025 Intel Corporation
	4	*/
	5
	6	#ifndef _ABI_GUC_LIC_ABI_H_
	7	#define _ABI_GUC_LIC_ABI_H_
	8
	9	#include <linux/types.h>
	10
	11	/**
	12	* enum guc_lic_type - Log Init Config KLV IDs.
	13	*/
	14	enum guc_lic_type {
	15	/**
	16	* @GUC_LIC_TYPE_GUC_SW_VERSION: GuC firmware version. Value
	17	* is a 32 bit number represented by guc_sw_version.
	18	*/
	19	GUC_LIC_TYPE_GUC_SW_VERSION = 0x1,
	20	/**
	21	* @GUC_LIC_TYPE_GUC_DEVICE_ID: GuC device id. Value is a 32
	22	* bit.
	23	*/
	24	GUC_LIC_TYPE_GUC_DEVICE_ID = 0x2,
	25	/**
	26	* @GUC_LIC_TYPE_TSC_FREQUENCY: GuC timestamp counter
	27	* frequency. Value is a 32 bit number representing frequency in
	28	* kHz. This timestamp is utilized in log entries, timer and
	29	* for engine utilization tracking.
	30	*/
	31	GUC_LIC_TYPE_TSC_FREQUENCY = 0x3,
	32	/**
	33	* @GUC_LIC_TYPE_GMD_ID: HW GMD ID. Value is a 32 bit number
	34	* representing graphics, media and display HW architecture IDs.
	35	*/
	36	GUC_LIC_TYPE_GMD_ID = 0x4,
	37	/**
	38	* @GUC_LIC_TYPE_BUILD_PLATFORM_ID: GuC build platform ID.
	39	* Value is 32 bits.
	40	*/
	41	GUC_LIC_TYPE_BUILD_PLATFORM_ID = 0x5,
	42	};
	43
	44	/**
	45	* struct guc_lic - GuC LIC (Log-Init-Config) structure.
	46	*
	47	* This is populated by the GUC at log init time and is located in the log
	48	* buffer memory allocation.
	49	*/
	50	struct guc_lic {
	51	/**
	52	* @magic: A magic number set by GuC to identify that this
	53	* structure contains valid information: magic = GUC_LIC_MAGIC.
	54	*/
	55	u32 magic;
	56	#define GUC_LIC_MAGIC 0x8086900D
	57	/**
	58	* @version: The version of the this structure.
	59	* Major and minor version number are represented as bit fields.
	60	*/
	61	u32 version;
	62	#define GUC_LIC_VERSION_MASK_MAJOR GENMASK(31, 16)
	63	#define GUC_LIC_VERSION_MASK_MINOR GENMASK(15, 0)
	64
	65	#define GUC_LIC_VERSION_MAJOR 1u
	66	#define GUC_LIC_VERSION_MINOR 0u
	67
	68	/** @data_count: Number of dwords the `data` array contains. */
	69	u32 data_count;
	70	/**
	71	* @data: Array of dwords representing a list of LIC KLVs of
	72	* type guc_klv_generic with keys represented by guc_lic_type
	73	*/
	74	u32 data[] __counted_by(data_count);
	75	} __packed;
	76
	77	#endif

drivers/gpu/drm/xe/abi/guc_log_abi.h+38−4

@@ -8,11 +8,45 @@ · 7 unmodified lines
8		8
9	#include <linux/types.h>	9	#include <linux/types.h>
10		10
		11	/**
		12	* DOC: GuC Log buffer Layout
		13	*
		14	* The in-memory log buffer layout is as follows::
		15	*
		16	* +===============================+ 0000h
		17	* \| Crash dump state header \| ^
		18	* +-------------------------------+ 32B \|
		19	* \| Debug state header \| \|
		20	* +-------------------------------+ 64B 4KB
		21	* \| Capture state header \| \|
		22	* +-------------------------------+ 96B \|
		23	* \| \| v
		24	* +===============================+ <--- EVENT_DATA_OFFSET
		25	* \| Event logs(raw data) \| ^
		26	* \| \| \|
		27	* \| \| EVENT_DATA_BUFFER_SIZE
		28	* \| \| \|
		29	* \| \| v
		30	* +===============================+ <--- CRASH_DUMP_OFFSET
		31	* \| Crash Dump(raw data) \| ^
		32	* \| \| \|
		33	* \| \| CRASH_DUMP_BUFFER_SIZE
		34	* \| \| \|
		35	* \| \| v
		36	* +===============================+ <--- STATE_CAPTURE_OFFSET
		37	* \| Error state capture(raw data) \| ^
		38	* \| \| \|
		39	* \| \| STATE_CAPTURE_BUFFER_SIZE
		40	* \| \| \|
		41	* \| \| v
		42	* +===============================+ Total: GUC_LOG_SIZE
		43	*/
		44
11	/* GuC logging buffer types */	45	/* GuC logging buffer types */
12	enum guc_log_buffer_type {	46	enum guc_log_type {
13	GUC_LOG_BUFFER_CRASH_DUMP,	47	GUC_LOG_TYPE_EVENT_DATA,
14	GUC_LOG_BUFFER_DEBUG,	48	GUC_LOG_TYPE_CRASH_DUMP,
15	GUC_LOG_BUFFER_CAPTURE,	49	GUC_LOG_TYPE_STATE_CAPTURE,
16	};	50	};
17		51
18	#define GUC_LOG_BUFFER_TYPE_MAX 3	52	#define GUC_LOG_BUFFER_TYPE_MAX 3

drivers/gpu/drm/xe/display/xe_fb_pin.c+9−14

 	/* TODO: Consider sharing framebuffer mapping?
 	 * embed i915_vma inside intel_framebuffer
 	 */
-	xe_pm_runtime_get_noresume(xe);
+	guard(xe_pm_runtime_noresume)(xe);
-	ret = mutex_lock_interruptible(&ggtt->lock);
+	ACQUIRE(mutex_intr, lock)(&ggtt->lock);
+	ret = ACQUIRE_ERR(mutex_intr, &lock);
 	if (ret)
-		goto out;
+		return ret;
 	align = XE_PAGE_SIZE;
 	if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
 		vma->node = bo->ggtt_node[tile0->id];
 	} else if (view->type == I915_GTT_VIEW_NORMAL) {
 		vma->node = xe_ggtt_node_init(ggtt);
-		if (IS_ERR(vma->node)) {
+		if (IS_ERR(vma->node))
-			ret = PTR_ERR(vma->node);
+			return PTR_ERR(vma->node);
-			goto out_unlock;
-		}
 		ret = xe_ggtt_node_insert_locked(vma->node, xe_bo_size(bo), align, 0);
 		if (ret) {
 			xe_ggtt_node_fini(vma->node);
-			goto out_unlock;
+			return ret;
 		}
 		xe_ggtt_map_bo(ggtt, vma->node, bo, xe->pat.idx[XE_CACHE_NONE]);
 		vma->node = xe_ggtt_node_init(ggtt);
 		if (IS_ERR(vma->node)) {
 			ret = PTR_ERR(vma->node);
-			goto out_unlock;
+			return ret;
 		}
 		ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
 		if (ret) {
 			xe_ggtt_node_fini(vma->node);
-			goto out_unlock;
+			return ret;
 		}
 		ggtt_ofs = vma->node->base.start;
 					   rot_info->plane[i].dst_stride);
 	}
-out_unlock:
-	mutex_unlock(&ggtt->lock);
-out:
-	xe_pm_runtime_put(xe);
 	return ret;
 }

drivers/gpu/drm/xe/display/xe_hdcp_gsc.c+9−22

 	struct xe_tile *tile = xe_device_get_root_tile(xe);
 	struct xe_gt *gt = tile->media_gt;
 	struct xe_gsc *gsc = &gt->uc.gsc;
-	bool ret = true;
-	unsigned int fw_ref;
 	if (!gsc || !xe_uc_fw_is_enabled(&gsc->fw)) {
 		drm_dbg_kms(&xe->drm,
 		return false;
 	}
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
-	if (!fw_ref) {
+	if (!fw_ref.domains) {
 		drm_dbg_kms(&xe->drm,
 			    "failed to get forcewake to check proxy status\n");
-		ret = false;
+		return false;
-		goto out;
 	}
-	if (!xe_gsc_proxy_init_done(gsc))
+	return xe_gsc_proxy_init_done(gsc);
-		ret = false;
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-out:
-	xe_pm_runtime_put(xe);
-	return ret;
 }
 /*This function helps allocate memory for the command that we will send to gsc cs */
 	u32 addr_out_off, addr_in_wr_off = 0;
 	int ret, tries = 0;
-	if (msg_in_len > max_msg_size || msg_out_len > max_msg_size) {
+	if (msg_in_len > max_msg_size || msg_out_len > max_msg_size)
-		ret = -ENOSPC;
+		return -ENOSPC;
-		goto out;
-	}
 	msg_size_in = msg_in_len + HDCP_GSC_HEADER_SIZE;
 	msg_size_out = msg_out_len + HDCP_GSC_HEADER_SIZE;
 	addr_out_off = PAGE_SIZE;
 	host_session_id = xe_gsc_create_host_session_id();
-	xe_pm_runtime_get_noresume(xe);
+	guard(xe_pm_runtime_noresume)(xe);
 	addr_in_wr_off = xe_gsc_emit_header(xe, &gsc_context->hdcp_bo->vmap,
 					    addr_in_wr_off, HECI_MEADDRESS_HDCP,
 					    host_session_id, msg_in_len);
 	} while (++tries < 20);
 	if (ret)
-		goto out;
+		return ret;
 	xe_map_memcpy_from(xe, msg_out, &gsc_context->hdcp_bo->vmap,
 			   addr_out_off + HDCP_GSC_HEADER_SIZE,
 			   msg_out_len);
-out:
-	xe_pm_runtime_put(xe);
 	return ret;
 }

drivers/gpu/drm/xe/instructions/xe_gpu_commands.h+1−0

@@ -47,6 +47,7 @@ · 46 unmodified lines
47		47
48	#define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)\|(0x3<<27)\|(0x2<<24)\|((len)-2))	48	#define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)\|(0x3<<27)\|(0x2<<24)\|((len)-2))
49		49
		50	#define PIPE_CONTROL0_QUEUE_DRAIN_MODE BIT(12)
50	#define PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE BIT(10) /* gen12 */	51	#define PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE BIT(10) /* gen12 */
51	#define PIPE_CONTROL0_HDC_PIPELINE_FLUSH BIT(9) /* gen12 */	52	#define PIPE_CONTROL0_HDC_PIPELINE_FLUSH BIT(9) /* gen12 */
52		53

drivers/gpu/drm/xe/regs/xe_gt_regs.h+3−0

@@ -227,6 +227,9 @@ · 226 unmodified lines
227		227
228	#define MIRROR_FUSE1 XE_REG(0x911c)	228	#define MIRROR_FUSE1 XE_REG(0x911c)
229		229
		230	#define FUSE2 XE_REG(0x9120)
		231	#define PRODUCTION_HW REG_BIT(2)
		232
230	#define MIRROR_L3BANK_ENABLE XE_REG(0x9130)	233	#define MIRROR_L3BANK_ENABLE XE_REG(0x9130)
231	#define XE3_L3BANK_ENABLE REG_GENMASK(31, 0)	234	#define XE3_L3BANK_ENABLE REG_GENMASK(31, 0)
232		235

drivers/gpu/drm/xe/regs/xe_gtt_defs.h+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	#define XELPG_GGTT_PTE_PAT0 BIT_ULL(52)	9	#define XELPG_GGTT_PTE_PAT0 BIT_ULL(52)
10	#define XELPG_GGTT_PTE_PAT1 BIT_ULL(53)	10	#define XELPG_GGTT_PTE_PAT1 BIT_ULL(53)
11		11
		12	#define XE_PTE_ADDR_MASK GENMASK_ULL(51, 12)
12	#define GGTT_PTE_VFID GENMASK_ULL(11, 2)	13	#define GGTT_PTE_VFID GENMASK_ULL(11, 2)
13		14
14	#define GUC_GGTT_TOP 0xFEE00000	15	#define GUC_GGTT_TOP 0xFEE00000

drivers/gpu/drm/xe/regs/xe_guc_regs.h+3−0

@@ -90,6 +90,9 @@ · 89 unmodified lines
90	#define GUC_SEND_INTERRUPT XE_REG(0xc4c8)	90	#define GUC_SEND_INTERRUPT XE_REG(0xc4c8)
91	#define GUC_SEND_TRIGGER REG_BIT(0)	91	#define GUC_SEND_TRIGGER REG_BIT(0)
92		92
		93	#define GUC_INTR_CHICKEN XE_REG(0xc50c)
		94	#define DISABLE_SIGNALING_ENGINES REG_BIT(1)
		95
93	#define GUC_BCS_RCS_IER XE_REG(0xc550)	96	#define GUC_BCS_RCS_IER XE_REG(0xc550)
94	#define GUC_VCS2_VCS1_IER XE_REG(0xc554)	97	#define GUC_VCS2_VCS1_IER XE_REG(0xc554)
95	#define GUC_WD_VECS_IER XE_REG(0xc558)	98	#define GUC_WD_VECS_IER XE_REG(0xc558)

drivers/gpu/drm/xe/regs/xe_irq_regs.h+1−0

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	#define GU_MISC_IRQ REG_BIT(29)	20	#define GU_MISC_IRQ REG_BIT(29)
21	#define ERROR_IRQ(x) REG_BIT(26 + (x))	21	#define ERROR_IRQ(x) REG_BIT(26 + (x))
22	#define DISPLAY_IRQ REG_BIT(16)	22	#define DISPLAY_IRQ REG_BIT(16)
		23	#define SOC_H2DMEMINT_IRQ REG_BIT(13)
23	#define I2C_IRQ REG_BIT(12)	24	#define I2C_IRQ REG_BIT(12)
24	#define GT_DW_IRQ(x) REG_BIT(x)	25	#define GT_DW_IRQ(x) REG_BIT(x)
25		26

drivers/gpu/drm/xe/regs/xe_mert_regs.h+21−0

@@ -0,0 +1,21 @@
	1	/* SPDX-License-Identifier: MIT */
	2	/*
	3	* Copyright © 2025 Intel Corporation
	4	*/
	5
	6	#ifndef _XE_MERT_REGS_H_
	7	#define _XE_MERT_REGS_H_
	8
	9	#include "regs/xe_reg_defs.h"
	10
	11	#define MERT_LMEM_CFG XE_REG(0x1448b0)
	12
	13	#define MERT_TLB_CT_INTR_ERR_ID_PORT XE_REG(0x145190)
	14	#define MERT_TLB_CT_VFID_MASK REG_GENMASK(16, 9)
	15	#define MERT_TLB_CT_ERROR_MASK REG_GENMASK(5, 0)
	16	#define MERT_TLB_CT_LMTT_FAULT 0x05
	17
	18	#define MERT_TLB_INV_DESC_A XE_REG(0x14cf7c)
	19	#define MERT_TLB_INV_DESC_A_VALID REG_BIT(0)
	20
	21	#endif /* _XE_MERT_REGS_H_ */

drivers/gpu/drm/xe/regs/xe_oa_regs.h+17−0

@@ -100,4 +100,21 @@ · 99 unmodified lines
100	#define OAM_COMPRESSION_T3_CONTROL XE_REG(0x1c2e00)	100	#define OAM_COMPRESSION_T3_CONTROL XE_REG(0x1c2e00)
101	#define OAM_LAT_MEASURE_ENABLE REG_BIT(4)	101	#define OAM_LAT_MEASURE_ENABLE REG_BIT(4)
102		102
		103	/* Actual address is MEDIA_GT_GSI_OFFSET + the base addr below */
		104	#define XE_OAM_SAG_BASE 0x13000
		105	#define XE_OAM_SCMI_0_BASE 0x14000
		106	#define XE_OAM_SCMI_1_BASE 0x14800
		107	#define XE_OAM_SAG_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SAG_BASE)
		108	#define XE_OAM_SCMI_0_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_0_BASE)
		109	#define XE_OAM_SCMI_1_BASE_ADJ (MEDIA_GT_GSI_OFFSET + XE_OAM_SCMI_1_BASE)
		110
		111	#define OAMERT_CONTROL XE_REG(0x1453a0)
		112	#define OAMERT_DEBUG XE_REG(0x1453a4)
		113	#define OAMERT_STATUS XE_REG(0x1453a8)
		114	#define OAMERT_HEAD_POINTER XE_REG(0x1453ac)
		115	#define OAMERT_TAIL_POINTER XE_REG(0x1453b0)
		116	#define OAMERT_BUFFER XE_REG(0x1453b4)
		117	#define OAMERT_CONTEXT_CONTROL XE_REG(0x1453c8)
		118	#define OAMERT_MMIO_TRG XE_REG(0x1453cc)
		119
103	#endif	120	#endif

drivers/gpu/drm/xe/tests/xe_args_test.c+54−0

@@ -78,6 +78,24 @@ static void pick_arg_example(struct kunit *test) · 77 unmodified lines
78	#undef buz	78	#undef buz
79	}	79	}
80		80
		81	static void if_args_example(struct kunit *test)
		82	{
		83	enum { Z = 1, Q };
		84
		85	#define foo X, Y
		86	#define bar IF_ARGS(Z, Q, foo)
		87	#define buz IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo)))
		88
		89	KUNIT_EXPECT_EQ(test, bar, Z);
		90	KUNIT_EXPECT_EQ(test, buz, Q);
		91	KUNIT_EXPECT_STREQ(test, __stringify(bar), "Z");
		92	KUNIT_EXPECT_STREQ(test, __stringify(buz), "Q");
		93
		94	#undef foo
		95	#undef bar
		96	#undef buz
		97	}
		98
81	static void sep_comma_example(struct kunit *test)	99	static void sep_comma_example(struct kunit *test)
82	{	100	{
83	#define foo(f) f(X) f(Y) f(Z) f(Q)	101	#define foo(f) f(X) f(Y) f(Z) f(Q)
@@ -198,6 +216,40 @@ static void last_arg_test(struct kunit *test) · 114 unmodified lines
198	KUNIT_EXPECT_STREQ(test, __stringify(LAST_ARG(MAX_ARGS)), "-12");	216	KUNIT_EXPECT_STREQ(test, __stringify(LAST_ARG(MAX_ARGS)), "-12");
199	}	217	}
200		218
		219	static void if_args_test(struct kunit *test)
		220	{
		221	bool with_args = true;
		222	bool no_args = false;
		223	enum { X = 100 };
		224
		225	KUNIT_EXPECT_TRUE(test, IF_ARGS(true, false, FOO_ARGS));
		226	KUNIT_EXPECT_FALSE(test, IF_ARGS(true, false, NO_ARGS));
		227
		228	KUNIT_EXPECT_TRUE(test, CONCATENATE(IF_ARGS(with, no, FOO_ARGS), _args));
		229	KUNIT_EXPECT_FALSE(test, CONCATENATE(IF_ARGS(with, no, NO_ARGS), _args));
		230
		231	KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, FOO_ARGS)), "yes");
		232	KUNIT_EXPECT_STREQ(test, __stringify(IF_ARGS(yes, no, NO_ARGS)), "no");
		233
		234	KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, FOO_ARGS), 4);
		235	KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, FOO_ARGS), -1, NO_ARGS), -1);
		236	KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, FOO_ARGS), 0);
		237	KUNIT_EXPECT_EQ(test, IF_ARGS(CALL_ARGS(COUNT_ARGS, NO_ARGS), -1, NO_ARGS), -1);
		238
		239	KUNIT_EXPECT_EQ(test,
		240	CALL_ARGS(FIRST_ARG,
		241	CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), X);
		242	KUNIT_EXPECT_EQ(test,
		243	CALL_ARGS(FIRST_ARG,
		244	CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), -1);
		245	KUNIT_EXPECT_EQ(test,
		246	CALL_ARGS(COUNT_ARGS,
		247	CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, FOO_ARGS), _ARGS)), 4);
		248	KUNIT_EXPECT_EQ(test,
		249	CALL_ARGS(COUNT_ARGS,
		250	CALL_ARGS(CONCATENATE, IF_ARGS(FOO, MAX, NO_ARGS), _ARGS)), 12);
		251	}
		252
201	static struct kunit_case args_tests[] = {	253	static struct kunit_case args_tests[] = {
202	KUNIT_CASE(count_args_test),	254	KUNIT_CASE(count_args_test),
203	KUNIT_CASE(call_args_example),	255	KUNIT_CASE(call_args_example),
@@ -209,6 +261,8 @@ static struct kunit_case args_tests[] = { · 5 unmodified lines
209	KUNIT_CASE(last_arg_example),	261	KUNIT_CASE(last_arg_example),
210	KUNIT_CASE(last_arg_test),	262	KUNIT_CASE(last_arg_test),
211	KUNIT_CASE(pick_arg_example),	263	KUNIT_CASE(pick_arg_example),
		264	KUNIT_CASE(if_args_example),
		265	KUNIT_CASE(if_args_test),
212	KUNIT_CASE(sep_comma_example),	266	KUNIT_CASE(sep_comma_example),
213	{}	267	{}
214	};	268	};

drivers/gpu/drm/xe/tests/xe_bo.c+2−8

 		return 0;
 	}
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_tile(tile, xe, id) {
 		/* For igfx run only for primary tile */
 		if (!IS_DGFX(xe) && id > 0)
 		ccs_test_run_tile(xe, tile, test);
 	}
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 		return 0;
 	}
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_tile(tile, xe, id)
 		evict_test_run_tile(xe, tile, test);
-	xe_pm_runtime_put(xe);
 	return 0;
 }

drivers/gpu/drm/xe/tests/xe_dma_buf.c+1−2

 	const struct dma_buf_test_params *params;
 	struct kunit *test = kunit_get_current_test();
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for (params = test_params; params->mem_mask; ++params) {
 		struct dma_buf_test_params p = *params;
 		test->priv = &p;
 		xe_test_dmabuf_import_same_driver(xe);
 	}
-	xe_pm_runtime_put(xe);
 	/* A non-zero return would halt iteration over driver devices */
 	return 0;

drivers/gpu/drm/xe/tests/xe_migrate.c+2−8

 	struct xe_tile *tile;
 	int id;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_tile(tile, xe, id) {
 		struct xe_migrate *m = tile->migrate;
 		struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
 		xe_vm_unlock(m->q->vm);
 	}
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 		return 0;
 	}
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_tile(tile, xe, id)
 		validate_ccs_test_run_tile(xe, tile, test);
-	xe_pm_runtime_put(xe);
 	return 0;
 }

drivers/gpu/drm/xe/tests/xe_mocs.c+8−19

 {
 	struct kunit *test = kunit_get_current_test();
 	u32 l3cc, l3cc_expected;
-	unsigned int fw_ref, i;
+	unsigned int i;
 	u32 reg_val;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 		KUNIT_FAIL_AND_ABORT(test, "Forcewake Failed.\n");
-	}
 	for (i = 0; i < info->num_mocs_regs; i++) {
 		if (!(i & 1)) {
 		KUNIT_EXPECT_EQ_MSG(test, l3cc_expected, l3cc,
 				    "l3cc idx=%u has incorrect val.\n", i);
 	}
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 static void read_mocs_table(struct xe_gt *gt,
 {
 	struct kunit *test = kunit_get_current_test();
 	u32 mocs, mocs_expected;
-	unsigned int fw_ref, i;
+	unsigned int i;
 	u32 reg_val;
 	KUNIT_EXPECT_TRUE_MSG(test, info->unused_entries_index,
 			      "Unused entries index should have been defined\n");
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	KUNIT_ASSERT_NE_MSG(test, fw_ref, 0, "Forcewake Failed.\n");
+	KUNIT_ASSERT_NE_MSG(test, fw_ref.domains, 0, "Forcewake Failed.\n");
 	for (i = 0; i < info->num_mocs_regs; i++) {
 		if (regs_are_mcr(gt))
 		KUNIT_EXPECT_EQ_MSG(test, mocs_expected, mocs,
 				    "mocs reg 0x%x has incorrect val.\n", i);
 	}
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 static int mocs_kernel_test_run_device(struct xe_device *xe)
 	unsigned int flags;
 	int id;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_gt(gt, xe, id) {
 		flags = live_mocs_init(&mocs, gt);
 		if (flags & HAS_GLOBAL_MOCS)
 			read_l3cc_table(gt, &mocs.table);
 	}
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 	int id;
 	struct kunit *test = kunit_get_current_test();
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_gt(gt, xe, id) {
 		flags = live_mocs_init(&mocs, gt);
 		kunit_info(test, "mocs_reset_test before reset\n");
 			read_l3cc_table(gt, &mocs.table);
 	}
-	xe_pm_runtime_put(xe);
 	return 0;
 }

drivers/gpu/drm/xe/xe_args.h+27−0

@@ -121,6 +121,33 @@ · 120 unmodified lines
121	#define PICK_ARG11(args...) PICK_ARG10(DROP_FIRST_ARG(args))	121	#define PICK_ARG11(args...) PICK_ARG10(DROP_FIRST_ARG(args))
122	#define PICK_ARG12(args...) PICK_ARG11(DROP_FIRST_ARG(args))	122	#define PICK_ARG12(args...) PICK_ARG11(DROP_FIRST_ARG(args))
123		123
		124	/**
		125	* IF_ARGS() - Make selection based on optional argument list.
		126	* @then: token to return if arguments are present
		127	* @else: token to return if arguments are empty
		128	* @...: arguments to check (optional)
		129	*
		130	* This macro allows to select a token based on the presence of the argument list.
		131	*
		132	* Example:
		133	*
		134	* #define foo X, Y
		135	* #define bar IF_ARGS(Z, Q, foo)
		136	* #define buz IF_ARGS(Z, Q, DROP_FIRST_ARG(FIRST_ARG(foo)))
		137	*
		138	* With above definitions bar expands to Z while buz expands to Q.
		139	*/
		140	#if defined(CONFIG_CC_IS_CLANG) \|\| GCC_VERSION >= 100100
		141	#define IF_ARGS(then, else, ...) FIRST_ARG(__VA_OPT__(then,) else)
		142	#else
		143	#define IF_ARGS(then, else, ...) _IF_ARGS(then, else, CALL_ARGS(FIRST_ARG, __VA_ARGS__))
		144	#define _IF_ARGS(then, else, ...) __IF_ARGS(then, else, CALL_ARGS(COUNT_ARGS, __VA_ARGS__))
		145	#define __IF_ARGS(then, else, n) ___IF_ARGS(then, else, CALL_ARGS(CONCATENATE, ___IF_ARG, n))
		146	#define ___IF_ARGS(then, else, if) CALL_ARGS(if, then, else)
		147	#define ___IF_ARG1(then, else) then
		148	#define ___IF_ARG0(then, else) else
		149	#endif
		150
124	/**	151	/**
125	* ARGS_SEP_COMMA - Definition of a comma character.	152	* ARGS_SEP_COMMA - Definition of a comma character.
126	*	153	*

drivers/gpu/drm/xe/xe_bo.c+16−10

 		 * non-coherent and require a CPU:WC mapping.
 		 */
 		if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) ||
-		    (xe->info.graphics_verx100 >= 1270 &&
+		     (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE))
-		     bo->flags & XE_BO_FLAG_PAGETABLE))
 			caching = ttm_write_combined;
 	}
 	struct ttm_buffer_object *ttm_bo = vma->vm_private_data;
 	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
 	struct xe_device *xe = xe_bo_device(bo);
-	int ret;
-	xe_pm_runtime_get(xe);
-	ret = ttm_bo_vm_access(vma, addr, buf, len, write);
-	xe_pm_runtime_put(xe);
-	return ret;
+	guard(xe_pm_runtime)(xe);
+	return ttm_bo_vm_access(vma, addr, buf, len, write);
 }
 /**
 	if (XE_IOCTL_DBG(xe, args->flags &
 			 ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING |
 			   DRM_XE_GEM_CREATE_FLAG_SCANOUT |
-			   DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM)))
+			   DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM |
+			   DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION)))
 		return -EINVAL;
 	if (XE_IOCTL_DBG(xe, args->handle))
 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)
 		bo_flags |= XE_BO_FLAG_SCANOUT;
+	if (args->flags & DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION) {
+		if (XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20))
+			return -EOPNOTSUPP;
+		bo_flags |= XE_BO_FLAG_NO_COMPRESSION;
+	}
 	bo_flags |= args->placement << (ffs(XE_BO_FLAG_SYSTEM) - 1);
 	/* CCS formats need physical placement at a 64K alignment in VRAM. */
 	 * Compression implies coh_none, therefore we know for sure that WB
 	 * memory can't currently use compression, which is likely one of the
 	 * common cases.
+	 * Additionally, userspace may explicitly request no compression via the
+	 * DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION flag, which should also disable
+	 * CCS usage.
 	 */
-	if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)
+	if (bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB ||
+	    bo->flags & XE_BO_FLAG_NO_COMPRESSION)
 		return false;
 	return true;

drivers/gpu/drm/xe/xe_bo.h+1−0

@@ -50,6 +50,7 @@ · 49 unmodified lines
50	#define XE_BO_FLAG_GGTT3 BIT(23)	50	#define XE_BO_FLAG_GGTT3 BIT(23)
51	#define XE_BO_FLAG_CPU_ADDR_MIRROR BIT(24)	51	#define XE_BO_FLAG_CPU_ADDR_MIRROR BIT(24)
52	#define XE_BO_FLAG_FORCE_USER_VRAM BIT(25)	52	#define XE_BO_FLAG_FORCE_USER_VRAM BIT(25)
		53	#define XE_BO_FLAG_NO_COMPRESSION BIT(26)
53		54
54	/* this one is trigger internally only */	55	/* this one is trigger internally only */
55	#define XE_BO_FLAG_INTERNAL_TEST BIT(30)	56	#define XE_BO_FLAG_INTERNAL_TEST BIT(30)

drivers/gpu/drm/xe/xe_debugfs.c+121−22

 	struct xe_gt *gt;
 	u8 id;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	drm_printf(&p, "graphics_verx100 %d\n", xe->info.graphics_verx100);
 	drm_printf(&p, "media_verx100 %d\n", xe->info.media_verx100);
 			   xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT));
 		drm_printf(&p, "gt%d engine_mask 0x%llx\n", id,
 			   gt->info.engine_mask);
+		drm_printf(&p, "gt%d multi_queue_engine_class_mask 0x%x\n", id,
+			   gt->info.multi_queue_engine_class_mask);
 	}
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 static int workarounds(struct xe_device *xe, struct drm_printer *p)
 {
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	xe_wa_device_dump(xe, p);
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 	xe = node_to_xe(m->private);
 	p = drm_seq_file_printer(m);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	mmio = xe_root_tile_mmio(xe);
 	static const struct {
 		u32 offset;
 	for (int i = 0; i < ARRAY_SIZE(residencies); i++)
 		read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p);
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 	xe = node_to_xe(m->private);
 	p = drm_seq_file_printer(m);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	mmio = xe_root_tile_mmio(xe);
 	static const struct {
 	for (int i = 0; i < ARRAY_SIZE(residencies); i++)
 		read_residency_counter(xe, mmio, residencies[i].offset, residencies[i].name, &p);
-	xe_pm_runtime_put(xe);
 	return 0;
 }
 	xe->wedged.mode = wedged_mode;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_gt(gt, xe, id) {
 		ret = xe_guc_ads_scheduler_policy_toggle_reset(&gt->uc.guc.ads);
 		if (ret) {
 			xe_gt_err(gt, "Failed to update GuC ADS scheduler policy. GuC may still cause engine reset even with wedged_mode=2\n");
-			xe_pm_runtime_put(xe);
 			return -EIO;
 		}
 	}
-	xe_pm_runtime_put(xe);
 	return size;
 }
 	.write = wedged_mode_set,
 };
+static ssize_t page_reclaim_hw_assist_show(struct file *f, char __user *ubuf,
+					   size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	char buf[8];
+	int len;
+	len = scnprintf(buf, sizeof(buf), "%d\n", xe->info.has_page_reclaim_hw_assist);
+	return simple_read_from_buffer(ubuf, size, pos, buf, len);
+}
+static ssize_t page_reclaim_hw_assist_set(struct file *f, const char __user *ubuf,
+					  size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	bool val;
+	ssize_t ret;
+	ret = kstrtobool_from_user(ubuf, size, &val);
+	if (ret)
+		return ret;
+	xe->info.has_page_reclaim_hw_assist = val;
+	return size;
+}
+static const struct file_operations page_reclaim_hw_assist_fops = {
+	.owner = THIS_MODULE,
+	.read = page_reclaim_hw_assist_show,
+	.write = page_reclaim_hw_assist_set,
+};
 static ssize_t atomic_svm_timeslice_ms_show(struct file *f, char __user *ubuf,
 					    size_t size, loff_t *pos)
 {
 	.write = atomic_svm_timeslice_ms_set,
 };
+static ssize_t min_run_period_lr_ms_show(struct file *f, char __user *ubuf,
+					 size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	char buf[32];
+	int len = 0;
+	len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_lr_ms);
+	return simple_read_from_buffer(ubuf, size, pos, buf, len);
+}
+static ssize_t min_run_period_lr_ms_set(struct file *f, const char __user *ubuf,
+					size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	u32 min_run_period_lr_ms;
+	ssize_t ret;
+	ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_lr_ms);
+	if (ret)
+		return ret;
+	xe->min_run_period_lr_ms = min_run_period_lr_ms;
+	return size;
+}
+static const struct file_operations min_run_period_lr_ms_fops = {
+	.owner = THIS_MODULE,
+	.read = min_run_period_lr_ms_show,
+	.write = min_run_period_lr_ms_set,
+};
+static ssize_t min_run_period_pf_ms_show(struct file *f, char __user *ubuf,
+					 size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	char buf[32];
+	int len = 0;
+	len = scnprintf(buf, sizeof(buf), "%d\n", xe->min_run_period_pf_ms);
+	return simple_read_from_buffer(ubuf, size, pos, buf, len);
+}
+static ssize_t min_run_period_pf_ms_set(struct file *f, const char __user *ubuf,
+					size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	u32 min_run_period_pf_ms;
+	ssize_t ret;
+	ret = kstrtouint_from_user(ubuf, size, 0, &min_run_period_pf_ms);
+	if (ret)
+		return ret;
+	xe->min_run_period_pf_ms = min_run_period_pf_ms;
+	return size;
+}
+static const struct file_operations min_run_period_pf_ms_fops = {
+	.owner = THIS_MODULE,
+	.read = min_run_period_pf_ms_show,
+	.write = min_run_period_pf_ms_set,
+};
 static ssize_t disable_late_binding_show(struct file *f, char __user *ubuf,
 					 size_t size, loff_t *pos)
 {
 	struct ttm_resource_manager *man;
 	struct xe_tile *tile;
 	struct xe_gt *gt;
-	u32 mem_type;
 	u8 tile_id;
 	u8 id;
 	debugfs_create_file("atomic_svm_timeslice_ms", 0600, root, xe,
 			    &atomic_svm_timeslice_ms_fops);
-	debugfs_create_file("disable_late_binding", 0600, root, xe,
+	debugfs_create_file("min_run_period_lr_ms", 0600, root, xe,
-			    &disable_late_binding_fops);
+			    &min_run_period_lr_ms_fops);
-	for (mem_type = XE_PL_VRAM0; mem_type <= XE_PL_VRAM1; ++mem_type) {
+	debugfs_create_file("min_run_period_pf_ms", 0600, root, xe,
-		man = ttm_manager_type(bdev, mem_type);
+			    &min_run_period_pf_ms_fops);
-		if (man) {
+	debugfs_create_file("disable_late_binding", 0600, root, xe,
-			char name[16];
+			    &disable_late_binding_fops);
-			snprintf(name, sizeof(name), "vram%d_mm", mem_type - XE_PL_VRAM0);
+	/*
-			ttm_resource_manager_create_debugfs(man, root, name);
+	 * Don't expose page reclaim configuration file if not supported by the
-		}
+	 * hardware initially.
-	}
+	 */
+	if (xe->info.has_page_reclaim_hw_assist)
+		debugfs_create_file("page_reclaim_hw_assist", 0600, root, xe,
+				    &page_reclaim_hw_assist_fops);
 	man = ttm_manager_type(bdev, XE_PL_TT);
 	ttm_resource_manager_create_debugfs(man, root, "gtt_mm");

drivers/gpu/drm/xe/xe_devcoredump.c+12−18

 	struct xe_devcoredump_snapshot *ss = container_of(work, typeof(*ss), work);
 	struct xe_devcoredump *coredump = container_of(ss, typeof(*coredump), snapshot);
 	struct xe_device *xe = coredump_to_xe(coredump);
-	unsigned int fw_ref;
 	/*
 	 * NB: Despite passing a GFP_ flags parameter here, more allocations are done
 			      xe_devcoredump_read, xe_devcoredump_free,
 			      XE_COREDUMP_TIMEOUT_JIFFIES);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	/* keep going if fw fails as we still want to save the memory and SW data */
-	fw_ref = xe_force_wake_get(gt_to_fw(ss->gt), XE_FORCEWAKE_ALL);
+	xe_with_force_wake(fw_ref, gt_to_fw(ss->gt), XE_FORCEWAKE_ALL) {
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
+		if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
-		xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
+			xe_gt_info(ss->gt, "failed to get forcewake for coredump capture\n");
-	xe_vm_snapshot_capture_delayed(ss->vm);
+		xe_vm_snapshot_capture_delayed(ss->vm);
-	xe_guc_exec_queue_snapshot_capture_delayed(ss->ge);
+		xe_guc_exec_queue_snapshot_capture_delayed(ss->ge);
-	xe_force_wake_put(gt_to_fw(ss->gt), fw_ref);
+	}
 	ss->read.chunk_position = 0;
 		ss->read.buffer = kvmalloc(XE_DEVCOREDUMP_CHUNK_MAX,
 					   GFP_USER);
 		if (!ss->read.buffer)
-			goto put_pm;
+			return;
 		__xe_devcoredump_read(ss->read.buffer,
 				      XE_DEVCOREDUMP_CHUNK_MAX,
 	} else {
 		ss->read.buffer = kvmalloc(ss->read.size, GFP_USER);
 		if (!ss->read.buffer)
-			goto put_pm;
+			return;
 		__xe_devcoredump_read(ss->read.buffer, ss->read.size, 0,
 				      coredump);
 		xe_devcoredump_snapshot_free(ss);
 	}
-put_pm:
-	xe_pm_runtime_put(xe);
 }
 static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 	struct xe_devcoredump_snapshot *ss = &coredump->snapshot;
 	struct xe_guc *guc = exec_queue_to_guc(q);
 	const char *process_name = "no process";
-	unsigned int fw_ref;
 	bool cookie;
 	ss->snapshot_time = ktime_get_real();
 	ss->gt = q->gt;
 	INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work);
-	cookie = dma_fence_begin_signalling();
 	/* keep going if fw fails as we still want to save the memory and SW data */
-	fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
+	cookie = dma_fence_begin_signalling();
 	ss->guc.log = xe_guc_log_snapshot_capture(&guc->log, true);
 	ss->guc.ct = xe_guc_ct_snapshot_capture(&guc->ct);
 	queue_work(system_unbound_wq, &ss->work);
-	xe_force_wake_put(gt_to_fw(q->gt), fw_ref);
 	dma_fence_end_signalling(cookie);
 }

drivers/gpu/drm/xe/xe_device.c+77−23

@@ -166,7 +166,7 @@ static void xe_file_close(struct drm_device dev, struct drm_file file) · 165 unmodified lines
166	struct xe_exec_queue *q;	166	struct xe_exec_queue *q;
167	unsigned long idx;	167	unsigned long idx;
168		168
169	xe_pm_runtime_get(xe);	169	guard(xe_pm_runtime)(xe);
170		170
171	/*	171	/*
172	* No need for exec_queue.lock here as there is no contention for it	172	* No need for exec_queue.lock here as there is no contention for it
@@ -177,15 +177,18 @@ static void xe_file_close(struct drm_device dev, struct drm_file file) · 4 unmodified lines
177	xa_for_each(&xef->exec_queue.xa, idx, q) {	177	xa_for_each(&xef->exec_queue.xa, idx, q) {
178	if (q->vm && q->hwe->hw_engine_group)	178	if (q->vm && q->hwe->hw_engine_group)
179	xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);	179	xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
180	xe_exec_queue_kill(q);	180
		181	if (xe_exec_queue_is_multi_queue_primary(q))
		182	xe_exec_queue_group_kill_put(q->multi_queue.group);
		183	else
		184	xe_exec_queue_kill(q);
		185
181	xe_exec_queue_put(q);	186	xe_exec_queue_put(q);
182	}	187	}
183	xa_for_each(&xef->vm.xa, idx, vm)	188	xa_for_each(&xef->vm.xa, idx, vm)
184	xe_vm_close_and_put(vm);	189	xe_vm_close_and_put(vm);
185		190
186	xe_file_put(xef);	191	xe_file_put(xef);
187
188	xe_pm_runtime_put(xe);
189	}	192	}
190		193
191	static const struct drm_ioctl_desc xe_ioctls[] = {	194	static const struct drm_ioctl_desc xe_ioctls[] = {
@@ -209,6 +212,8 @@ static const struct drm_ioctl_desc xe_ioctls[] = { · 17 unmodified lines
209	DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),	212	DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
210	DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,	213	DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
211	DRM_RENDER_ALLOW),	214	DRM_RENDER_ALLOW),
		215	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,
		216	DRM_RENDER_ALLOW),
212	};	217	};
213		218
214	static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)	219	static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
@@ -220,10 +225,10 @@ static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) · 5 unmodified lines
220	if (xe_device_wedged(xe))	225	if (xe_device_wedged(xe))
221	return -ECANCELED;	226	return -ECANCELED;
222		227
223	ret = xe_pm_runtime_get_ioctl(xe);	228	ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
		229	ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
224	if (ret >= 0)	230	if (ret >= 0)
225	ret = drm_ioctl(file, cmd, arg);	231	ret = drm_ioctl(file, cmd, arg);
226	xe_pm_runtime_put(xe);
227		232
228	return ret;	233	return ret;
229	}	234	}
@@ -238,10 +243,10 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo · 8 unmodified lines
238	if (xe_device_wedged(xe))	243	if (xe_device_wedged(xe))
239	return -ECANCELED;	244	return -ECANCELED;
240		245
241	ret = xe_pm_runtime_get_ioctl(xe);	246	ACQUIRE(xe_pm_runtime_ioctl, pm)(xe);
		247	ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm);
242	if (ret >= 0)	248	if (ret >= 0)
243	ret = drm_compat_ioctl(file, cmd, arg);	249	ret = drm_compat_ioctl(file, cmd, arg);
244	xe_pm_runtime_put(xe);
245		250
246	return ret;	251	return ret;
247	}	252	}
@@ -455,6 +460,7 @@ struct xe_device xe_device_create(struct pci_dev pdev, · 207 unmodified lines
455	xe->info.revid = pdev->revision;	460	xe->info.revid = pdev->revision;
456	xe->info.force_execlist = xe_modparam.force_execlist;	461	xe->info.force_execlist = xe_modparam.force_execlist;
457	xe->atomic_svm_timeslice_ms = 5;	462	xe->atomic_svm_timeslice_ms = 5;
		463	xe->min_run_period_lr_ms = 5;
458		464
459	err = xe_irq_init(xe);	465	err = xe_irq_init(xe);
460	if (err)	466	if (err)
@@ -775,7 +781,6 @@ ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ · 314 unmodified lines
775	static int probe_has_flat_ccs(struct xe_device *xe)	781	static int probe_has_flat_ccs(struct xe_device *xe)
776	{	782	{
777	struct xe_gt *gt;	783	struct xe_gt *gt;
778	unsigned int fw_ref;
779	u32 reg;	784	u32 reg;
780		785
781	/* Always enabled/disabled, no runtime check to do */	786	/* Always enabled/disabled, no runtime check to do */
@@ -786,8 +791,8 @@ static int probe_has_flat_ccs(struct xe_device *xe) · 4 unmodified lines
786	if (!gt)	791	if (!gt)
787	return 0;	792	return 0;
788		793
789	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);	794	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
790	if (!fw_ref)	795	if (!fw_ref.domains)
791	return -ETIMEDOUT;	796	return -ETIMEDOUT;
792		797
793	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);	798	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
@@ -797,11 +802,64 @@ static int probe_has_flat_ccs(struct xe_device *xe) · 3 unmodified lines
797	drm_dbg(&xe->drm,	802	drm_dbg(&xe->drm,
798	"Flat CCS has been disabled in bios, May lead to performance impact");	803	"Flat CCS has been disabled in bios, May lead to performance impact");
799		804
800	xe_force_wake_put(gt_to_fw(gt), fw_ref);
801
802	return 0;	805	return 0;
803	}	806	}
804		807
		808	/*
		809	* Detect if the driver is being run on pre-production hardware. We don't
		810	* keep workarounds for pre-production hardware long term, so print an
		811	* error and add taint if we're being loaded on a pre-production platform
		812	* for which the pre-prod workarounds have already been removed.
		813	*
		814	* The general policy is that we'll remove any workarounds that only apply to
		815	* pre-production hardware around the time force_probe restrictions are lifted
		816	* for a platform of the next major IP generation (for example, Xe2 pre-prod
		817	* workarounds should be removed around the time the first Xe3 platforms have
		818	* force_probe lifted).
		819	*/
		820	static void detect_preproduction_hw(struct xe_device *xe)
		821	{
		822	struct xe_gt *gt;
		823	int id;
		824
		825	/*
		826	* SR-IOV VFs don't have access to the FUSE2 register, so we can't
		827	* check pre-production status there. But the host OS will notice
		828	* and report the pre-production status, which should be enough to
		829	* help us catch mistaken use of pre-production hardware.
		830	*/
		831	if (IS_SRIOV_VF(xe))
		832	return;
		833
		834	/*
		835	* The "SW_CAP" fuse contains a bit indicating whether the device is a
		836	* production or pre-production device. This fuse is reflected through
		837	* the GT "FUSE2" register, even though the contents of the fuse are
		838	* not GT-specific. Every GT's reflection of this fuse should show the
		839	* same value, so we'll just use the first available GT for lookup.
		840	*/
		841	for_each_gt(gt, xe, id)
		842	break;
		843
		844	if (!gt)
		845	return;
		846
		847	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
		848	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) {
		849	xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n");
		850	return;
		851	}
		852
		853	if (xe_mmio_read32(&gt->mmio, FUSE2) & PRODUCTION_HW)
		854	return;
		855
		856	xe_info(xe, "Pre-production hardware detected.\n");
		857	if (!xe->info.has_pre_prod_wa) {
		858	xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n");
		859	add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK);
		860	}
		861	}
		862
805	int xe_device_probe(struct xe_device *xe)	863	int xe_device_probe(struct xe_device *xe)
806	{	864	{
807	struct xe_tile *tile;	865	struct xe_tile *tile;
@@ -972,6 +1030,8 @@ int xe_device_probe(struct xe_device *xe) · 164 unmodified lines
972	if (err)	1030	if (err)
973	goto err_unregister_display;	1031	goto err_unregister_display;
974		1032
		1033	detect_preproduction_hw(xe);
		1034
975	return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);	1035	return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
976		1036
977	err_unregister_display:	1037	err_unregister_display:
@@ -1034,7 +1094,6 @@ void xe_device_wmb(struct xe_device *xe) · 56 unmodified lines
1034	*/	1094	*/
1035	static void tdf_request_sync(struct xe_device *xe)	1095	static void tdf_request_sync(struct xe_device *xe)
1036	{	1096	{
1037	unsigned int fw_ref;
1038	struct xe_gt *gt;	1097	struct xe_gt *gt;
1039	u8 id;	1098	u8 id;
1040		1099
@@ -1042,8 +1101,8 @@ static void tdf_request_sync(struct xe_device *xe) · 1 unmodified line
1042	if (xe_gt_is_media_type(gt))	1101	if (xe_gt_is_media_type(gt))
1043	continue;	1102	continue;
1044		1103
1045	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);	1104	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
1046	if (!fw_ref)	1105	if (!fw_ref.domains)
1047	return;	1106	return;
1048		1107
1049	xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);	1108	xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
@@ -1058,15 +1117,12 @@ static void tdf_request_sync(struct xe_device *xe) · 8 unmodified lines
1058	if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,	1117	if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
1059	300, NULL, false))	1118	300, NULL, false))
1060	xe_gt_err_once(gt, "TD flush timeout\n");	1119	xe_gt_err_once(gt, "TD flush timeout\n");
1061
1062	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1063	}	1120	}
1064	}	1121	}
1065		1122
1066	void xe_device_l2_flush(struct xe_device *xe)	1123	void xe_device_l2_flush(struct xe_device *xe)
1067	{	1124	{
1068	struct xe_gt *gt;	1125	struct xe_gt *gt;
1069	unsigned int fw_ref;
1070		1126
1071	gt = xe_root_mmio_gt(xe);	1127	gt = xe_root_mmio_gt(xe);
1072	if (!gt)	1128	if (!gt)
@@ -1075,8 +1131,8 @@ void xe_device_l2_flush(struct xe_device *xe) · 2 unmodified lines
1075	if (!XE_GT_WA(gt, 16023588340))	1131	if (!XE_GT_WA(gt, 16023588340))
1076	return;	1132	return;
1077		1133
1078	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);	1134	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
1079	if (!fw_ref)	1135	if (!fw_ref.domains)
1080	return;	1136	return;
1081		1137
1082	spin_lock(&gt->global_invl_lock);	1138	spin_lock(&gt->global_invl_lock);
@@ -1086,8 +1142,6 @@ void xe_device_l2_flush(struct xe_device *xe) · 3 unmodified lines
1086	xe_gt_err_once(gt, "Global invalidation timeout\n");	1142	xe_gt_err_once(gt, "Global invalidation timeout\n");
1087		1143
1088	spin_unlock(&gt->global_invl_lock);	1144	spin_unlock(&gt->global_invl_lock);
1089
1090	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1091	}	1145	}
1092		1146
1093	/**	1147	/**

drivers/gpu/drm/xe/xe_device.h+5−0

 	return IS_DGFX(xe);
 }
+static inline bool xe_device_has_mert(struct xe_device *xe)
+{
+	return xe->info.has_mert;
+}
 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size);
 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p);

drivers/gpu/drm/xe/xe_device_sysfs.c+13−20

 	drm_dbg(&xe->drm, "vram_d3cold_threshold: %u\n", vram_d3cold_threshold);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	ret = xe_pm_set_vram_threshold(xe, vram_d3cold_threshold);
-	xe_pm_runtime_put(xe);
 	return ret ?: count;
 }
 	u16 major = 0, minor = 0, hotfix = 0, build = 0;
 	int ret;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0),
 			    &cap, NULL);
 	if (ret)
-		goto out;
+		return ret;
 	if (REG_FIELD_GET(V1_FAN_PROVISIONED, cap)) {
 		ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0),
 				    &ver_low, NULL);
 		if (ret)
-			goto out;
+			return ret;
 		ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0),
 				    &ver_high, NULL);
 		if (ret)
-			goto out;
+			return ret;
 		major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low);
 		minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low);
 		hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high);
 		build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high);
 	}
-out:
-	xe_pm_runtime_put(xe);
-	return ret ?: sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
+	return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
 }
 static DEVICE_ATTR_ADMIN_RO(lb_fan_control_version);
 	u16 major = 0, minor = 0, hotfix = 0, build = 0;
 	int ret;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_CAPABILITY_STATUS, 0),
 			    &cap, NULL);
 	if (ret)
-		goto out;
+		return ret;
 	if (REG_FIELD_GET(VR_PARAMS_PROVISIONED, cap)) {
 		ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_LOW, 0),
 				    &ver_low, NULL);
 		if (ret)
-			goto out;
+			return ret;
 		ret = xe_pcode_read(root, PCODE_MBOX(PCODE_LATE_BINDING, GET_VERSION_HIGH, 0),
 				    &ver_high, NULL);
 		if (ret)
-			goto out;
+			return ret;
 		major = REG_FIELD_GET(MAJOR_VERSION_MASK, ver_low);
 		minor = REG_FIELD_GET(MINOR_VERSION_MASK, ver_low);
 		hotfix = REG_FIELD_GET(HOTFIX_VERSION_MASK, ver_high);
 		build = REG_FIELD_GET(BUILD_VERSION_MASK, ver_high);
 	}
-out:
-	xe_pm_runtime_put(xe);
-	return ret ?: sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
+	return sysfs_emit(buf, "%u.%u.%u.%u\n", major, minor, hotfix, build);
 }
 static DEVICE_ATTR_ADMIN_RO(lb_voltage_regulator_version);
 	struct xe_device *xe = pdev_to_xe_device(pdev);
 	u32 cap, val;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	val = xe_mmio_read32(xe_root_tile_mmio(xe), BMG_PCIE_CAP);
-	xe_pm_runtime_put(xe);
 	cap = REG_FIELD_GET(LINK_DOWNGRADE, val);
 	return sysfs_emit(buf, "%u\n", cap == DOWNGRADE_CAPABLE);
 	u32 val = 0;
 	int ret;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	ret = xe_pcode_read(xe_device_get_root_tile(xe),
 			    PCODE_MBOX(DGFX_PCODE_STATUS, DGFX_GET_INIT_STATUS, 0),
 			    &val, NULL);
-	xe_pm_runtime_put(xe);
 	return ret ?: sysfs_emit(buf, "%u\n", REG_FIELD_GET(DGFX_LINK_DOWNGRADE_STATUS, val));
 }

drivers/gpu/drm/xe/xe_device_types.h+27−0

 #include "xe_late_bind_fw_types.h"
 #include "xe_lmtt_types.h"
 #include "xe_memirq_types.h"
+#include "xe_mert.h"
 #include "xe_oa_types.h"
 #include "xe_pagefault_types.h"
 #include "xe_platform_types.h"
 		 * Media GT shares a pool with its primary GT.
 		 */
 		struct xe_sa_manager *kernel_bb_pool;
+		/**
+		 * @mem.reclaim_pool: Pool for PRLs allocated.
+		 *
+		 * Only main GT has page reclaim list allocations.
+		 */
+		struct xe_sa_manager *reclaim_pool;
 	} mem;
 	/** @sriov: tile level virtualization data */
 	/** @debugfs: debugfs directory associated with this tile */
 	struct dentry *debugfs;
+	/** @mert: MERT-related data */
+	struct xe_mert mert;
 };
 /**
 		u8 has_asid:1;
 		/** @info.has_atomic_enable_pte_bit: Device has atomic enable PTE bit */
 		u8 has_atomic_enable_pte_bit:1;
+		/** @info.has_cached_pt: Supports caching pagetable */
+		u8 has_cached_pt:1;
 		/** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */
 		u8 has_device_atomics_on_smem:1;
 		/** @info.has_fan_control: Device supports fan control */
 		u8 has_heci_cscfi:1;
 		/** @info.has_heci_gscfi: device has heci gscfi */
 		u8 has_heci_gscfi:1;
+		/** @info.has_i2c: Device has I2C controller */
+		u8 has_i2c:1;
 		/** @info.has_late_bind: Device has firmware late binding support */
 		u8 has_late_bind:1;
 		/** @info.has_llc: Device has a shared CPU+GPU last level cache */
 		u8 has_mbx_power_limits:1;
 		/** @info.has_mem_copy_instr: Device supports MEM_COPY instruction */
 		u8 has_mem_copy_instr:1;
+		/** @info.has_mert: Device has standalone MERT */
+		u8 has_mert:1;
+		/** @info.has_page_reclaim_hw_assist: Device supports page reclamation feature */
+		u8 has_page_reclaim_hw_assist:1;
+		/** @info.has_pre_prod_wa: Pre-production workarounds still present in driver */
+		u8 has_pre_prod_wa:1;
 		/** @info.has_pxp: Device has PXP support */
 		u8 has_pxp:1;
 		/** @info.has_range_tlb_inval: Has range based TLB invalidations */
 	/** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */
 	u32 atomic_svm_timeslice_ms;
+	/** @min_run_period_lr_ms: LR VM (preempt fence mode) timeslice */
+	u32 min_run_period_lr_ms;
+	/** @min_run_period_pf_ms: LR VM (page fault mode) timeslice */
+	u32 min_run_period_pf_ms;
 #ifdef TEST_VM_OPS_ERROR
 	/**
 	 * @vm_inject_error_position: inject errors at different places in VM

drivers/gpu/drm/xe/xe_drm_client.c+31−36

@@ -285,32 +285,31 @@ static struct xe_hw_engine any_engine(struct xe_device xe) · 284 unmodified lines
285	return NULL;	285	return NULL;
286	}	286	}
287		287
288	static bool force_wake_get_any_engine(struct xe_device *xe,	288	/*
289	struct xe_hw_engine **phwe,	289	* Pick any engine and grab its forcewake. On error phwe will be NULL and
290	unsigned int *pfw_ref)	290	* the returned forcewake reference will be invalid. Callers should check
		291	* phwe against NULL.
		292	*/
		293	static struct xe_force_wake_ref force_wake_get_any_engine(struct xe_device *xe,
		294	struct xe_hw_engine **phwe)
291	{	295	{
292	enum xe_force_wake_domains domain;	296	enum xe_force_wake_domains domain;
293	unsigned int fw_ref;	297	struct xe_force_wake_ref fw_ref = {};
294	struct xe_hw_engine *hwe;	298	struct xe_hw_engine *hwe;
295	struct xe_force_wake *fw;	299
		300	*phwe = NULL;
296		301
297	hwe = any_engine(xe);	302	hwe = any_engine(xe);
298	if (!hwe)	303	if (!hwe)
299	return false;	304	return fw_ref; /* will be invalid */
300		305
301	domain = xe_hw_engine_to_fw_domain(hwe);	306	domain = xe_hw_engine_to_fw_domain(hwe);
302	fw = gt_to_fw(hwe->gt);
303
304	fw_ref = xe_force_wake_get(fw, domain);
305	if (!xe_force_wake_ref_has_domain(fw_ref, domain)) {
306	xe_force_wake_put(fw, fw_ref);
307	return false;
308	}
309		307
310	*phwe = hwe;	308	fw_ref = xe_force_wake_constructor(gt_to_fw(hwe->gt), domain);
311	*pfw_ref = fw_ref;	309	if (xe_force_wake_ref_has_domain(fw_ref.domains, domain))
		310	phwe = hwe; / valid forcewake */
312		311
313	return true;	312	return fw_ref;
314	}	313	}
315		314
316	static void show_run_ticks(struct drm_printer p, struct drm_file file)	315	static void show_run_ticks(struct drm_printer p, struct drm_file file)
@@ -322,7 +321,6 @@ static void show_run_ticks(struct drm_printer p, struct drm_file file) · 5 unmodified lines
322	struct xe_hw_engine *hwe;	321	struct xe_hw_engine *hwe;
323	struct xe_exec_queue *q;	322	struct xe_exec_queue *q;
324	u64 gpu_timestamp;	323	u64 gpu_timestamp;
325	unsigned int fw_ref;
326		324
327	/*	325	/*
328	* RING_TIMESTAMP registers are inaccessible in VF mode.	326	* RING_TIMESTAMP registers are inaccessible in VF mode.
@@ -339,29 +337,26 @@ static void show_run_ticks(struct drm_printer p, struct drm_file file) · 10 unmodified lines
339	wait_var_event(&xef->exec_queue.pending_removal,	337	wait_var_event(&xef->exec_queue.pending_removal,
340	!atomic_read(&xef->exec_queue.pending_removal));	338	!atomic_read(&xef->exec_queue.pending_removal));
341		339
342	xe_pm_runtime_get(xe);	340	scoped_guard(xe_pm_runtime, xe) {
343	if (!force_wake_get_any_engine(xe, &hwe, &fw_ref)) {	341	CLASS(xe_force_wake_release_only, fw_ref)(force_wake_get_any_engine(xe, &hwe));
344	xe_pm_runtime_put(xe);	342	if (!hwe)
345	return;	343	return;
346	}
347
348	/* Accumulate all the exec queues from this client */
349	mutex_lock(&xef->exec_queue.lock);
350	xa_for_each(&xef->exec_queue.xa, i, q) {
351	xe_exec_queue_get(q);
352	mutex_unlock(&xef->exec_queue.lock);
353
354	xe_exec_queue_update_run_ticks(q);
355		344
		345	/* Accumulate all the exec queues from this client */
356	mutex_lock(&xef->exec_queue.lock);	346	mutex_lock(&xef->exec_queue.lock);
357	xe_exec_queue_put(q);	347	xa_for_each(&xef->exec_queue.xa, i, q) {
358	}	348	xe_exec_queue_get(q);
359	mutex_unlock(&xef->exec_queue.lock);	349	mutex_unlock(&xef->exec_queue.lock);
360		350
361	gpu_timestamp = xe_hw_engine_read_timestamp(hwe);	351	xe_exec_queue_update_run_ticks(q);
362		352
363	xe_force_wake_put(gt_to_fw(hwe->gt), fw_ref);	353	mutex_lock(&xef->exec_queue.lock);
364	xe_pm_runtime_put(xe);	354	xe_exec_queue_put(q);
		355	}
		356	mutex_unlock(&xef->exec_queue.lock);
		357
		358	gpu_timestamp = xe_hw_engine_read_timestamp(hwe);
		359	}
365		360
366	for (class = 0; class < XE_ENGINE_CLASS_MAX; class++) {	361	for (class = 0; class < XE_ENGINE_CLASS_MAX; class++) {
367	const char *class_name;	362	const char *class_name;

drivers/gpu/drm/xe/xe_exec.c+7−2

 	u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
 	struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
 	struct drm_exec *exec = &vm_exec.exec;
-	u32 i, num_syncs, num_ufence = 0;
+	u32 i, num_syncs, num_in_sync = 0, num_ufence = 0;
 	struct xe_validation_ctx ctx;
 	struct xe_sched_job *job;
 	struct xe_vm *vm;
 		if (xe_sync_is_ufence(&syncs[num_syncs]))
 			num_ufence++;
+		if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs]))
+			num_in_sync++;
 	}
 	if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
 	mode = xe_hw_engine_group_find_exec_mode(q);
 	if (mode == EXEC_MODE_DMA_FENCE) {
-		err = xe_hw_engine_group_get_mode(group, mode, &previous_mode);
+		err = xe_hw_engine_group_get_mode(group, mode, &previous_mode,
+						  syncs, num_in_sync ?
+						  num_syncs : 0);
 		if (err)
 			goto err_syncs;
 	}

drivers/gpu/drm/xe/xe_exec_queue.c+441−14

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	#include <drm/drm_syncobj.h>	13	#include <drm/drm_syncobj.h>
14	#include <uapi/drm/xe_drm.h>	14	#include <uapi/drm/xe_drm.h>
15		15
		16	#include "xe_bo.h"
16	#include "xe_dep_scheduler.h"	17	#include "xe_dep_scheduler.h"
17	#include "xe_device.h"	18	#include "xe_device.h"
18	#include "xe_gt.h"	19	#include "xe_gt.h"
@@ -53,6 +54,54 @@ · 34 unmodified lines
53	* the ring operations the different engine classes support.	54	* the ring operations the different engine classes support.
54	*/	55	*/
55		56
		57	/**
		58	* DOC: Multi Queue Group
		59	*
		60	* Multi Queue Group is another mode of execution supported by the compute
		61	* and blitter copy command streamers (CCS and BCS, respectively). It is
		62	* an enhancement of the existing hardware architecture and leverages the
		63	* same submission model. It enables support for efficient, parallel
		64	* execution of multiple queues within a single shared context. The multi
		65	* queue group functionality is only supported with GuC submission backend.
		66	* All the queues of a group must use the same address space (VM).
		67	*
		68	* The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property
		69	* supports creating a multi queue group and adding queues to a queue group.
		70	*
		71	* The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field
		72	* set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with
		73	* the queue being created as the primary queue (aka q0) of the group. To add
		74	* secondary queues to the group, they need to be created with the above
		75	* property with id of the primary queue as the value. The properties of
		76	* the primary queue (like priority, time slice) applies to the whole group.
		77	* So, these properties can't be set for secondary queues of a group.
		78	*
		79	* The hardware does not support removing a queue from a multi-queue group.
		80	* However, queues can be dynamically added to the group. A group can have
		81	* up to 64 queues. To support this, XeKMD holds references to LRCs of the
		82	* queues even after the queues are destroyed by the user until the whole
		83	* group is destroyed. The secondary queues hold a reference to the primary
		84	* queue thus preventing the group from being destroyed when user destroys
		85	* the primary queue. Once the primary queue is destroyed, secondary queues
		86	* can't be added to the queue group, but they can continue to submit the
		87	* jobs if the DRM_XE_MULTI_GROUP_KEEP_ACTIVE flag is set during the multi
		88	* queue group creation.
		89	*
		90	* The queues of a multi queue group can set their priority within the group
		91	* through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property.
		92	* This multi queue priority can also be set dynamically through the
		93	* XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property
		94	* supported by the secondary queues of a multi queue group, other than
		95	* DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE.
		96	*
		97	* When GuC reports an error on any of the queues of a multi queue group,
		98	* the queue cleanup mechanism is invoked for all the queues of the group
		99	* as hardware cannot make progress on the multi queue context.
		100	*
		101	* Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC
		102	* interface.
		103	*/
		104
56	enum xe_exec_queue_sched_prop {	105	enum xe_exec_queue_sched_prop {
57	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,	106	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
58	XE_EXEC_QUEUE_TIMESLICE = 1,	107	XE_EXEC_QUEUE_TIMESLICE = 1,
@@ -61,7 +110,35 @@ enum xe_exec_queue_sched_prop { · 2 unmodified lines
61	};	110	};
62		111
63	static int exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,	112	static int exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,
64	u64 extensions, int ext_number);	113	u64 extensions);
		114
		115	static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q)
		116	{
		117	struct xe_exec_queue_group *group = q->multi_queue.group;
		118	struct xe_lrc *lrc;
		119	unsigned long idx;
		120
		121	if (xe_exec_queue_is_multi_queue_secondary(q)) {
		122	/*
		123	* Put pairs with get from xe_exec_queue_lookup() call
		124	* in xe_exec_queue_group_validate().
		125	*/
		126	xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q));
		127	return;
		128	}
		129
		130	if (!group)
		131	return;
		132
		133	/* Primary queue cleanup */
		134	xa_for_each(&group->xa, idx, lrc)
		135	xe_lrc_put(lrc);
		136
		137	xa_destroy(&group->xa);
		138	mutex_destroy(&group->list_lock);
		139	xe_bo_unpin_map_no_vm(group->cgp_bo);
		140	kfree(group);
		141	}
65		142
66	static void __xe_exec_queue_free(struct xe_exec_queue *q)	143	static void __xe_exec_queue_free(struct xe_exec_queue *q)
67	{	144	{
@@ -73,12 +150,17 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q) · 5 unmodified lines
73		150
74	if (xe_exec_queue_uses_pxp(q))	151	if (xe_exec_queue_uses_pxp(q))
75	xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);	152	xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
		153
		154	if (xe_exec_queue_is_multi_queue(q))
		155	xe_exec_queue_group_cleanup(q);
		156
76	if (q->vm)	157	if (q->vm)
77	xe_vm_put(q->vm);	158	xe_vm_put(q->vm);
78		159
79	if (q->xef)	160	if (q->xef)
80	xe_file_put(q->xef);	161	xe_file_put(q->xef);
81		162
		163	kvfree(q->replay_state);
82	kfree(q);	164	kfree(q);
83	}	165	}
84		166
@@ -147,6 +229,7 @@ static struct xe_exec_queue __xe_exec_queue_alloc(struct xe_device xe, · 62 unmodified lines
147	INIT_LIST_HEAD(&q->multi_gt_link);	229	INIT_LIST_HEAD(&q->multi_gt_link);
148	INIT_LIST_HEAD(&q->hw_engine_group_link);	230	INIT_LIST_HEAD(&q->hw_engine_group_link);
149	INIT_LIST_HEAD(&q->pxp.link);	231	INIT_LIST_HEAD(&q->pxp.link);
		232	q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL;
150		233
151	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;	234	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
152	q->sched_props.preempt_timeout_us =	235	q->sched_props.preempt_timeout_us =
@@ -175,7 +258,7 @@ static struct xe_exec_queue __xe_exec_queue_alloc(struct xe_device xe, · 22 unmodified lines
175	* may set q->usm, must come before xe_lrc_create(),	258	* may set q->usm, must come before xe_lrc_create(),
176	* may overwrite q->sched_props, must come before q->ops->init()	259	* may overwrite q->sched_props, must come before q->ops->init()
177	*/	260	*/
178	err = exec_queue_user_extensions(xe, q, extensions, 0);	261	err = exec_queue_user_extensions(xe, q, extensions);
179	if (err) {	262	if (err) {
180	__xe_exec_queue_free(q);	263	__xe_exec_queue_free(q);
181	return ERR_PTR(err);	264	return ERR_PTR(err);
@@ -225,8 +308,8 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags) · 43 unmodified lines
225	struct xe_lrc *lrc;	308	struct xe_lrc *lrc;
226		309
227	xe_gt_sriov_vf_wait_valid_ggtt(q->gt);	310	xe_gt_sriov_vf_wait_valid_ggtt(q->gt);
228	lrc = xe_lrc_create(q->hwe, q->vm, xe_lrc_ring_size(),	311	lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state,
229	q->msix_vec, flags);	312	xe_lrc_ring_size(), q->msix_vec, flags);
230	if (IS_ERR(lrc)) {	313	if (IS_ERR(lrc)) {
231	err = PTR_ERR(lrc);	314	err = PTR_ERR(lrc);
232	goto err_lrc;	315	goto err_lrc;
@@ -383,6 +466,26 @@ struct xe_exec_queue xe_exec_queue_create_bind(struct xe_device xe, · 150 unmodified lines
383	}	466	}
384	ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);	467	ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
385		468
		469	static void xe_exec_queue_group_kill(struct kref *ref)
		470	{
		471	struct xe_exec_queue_group *group = container_of(ref, struct xe_exec_queue_group,
		472	kill_refcount);
		473	xe_exec_queue_kill(group->primary);
		474	}
		475
		476	static inline void xe_exec_queue_group_kill_get(struct xe_exec_queue_group *group)
		477	{
		478	kref_get(&group->kill_refcount);
		479	}
		480
		481	void xe_exec_queue_group_kill_put(struct xe_exec_queue_group *group)
		482	{
		483	if (!group)
		484	return;
		485
		486	kref_put(&group->kill_refcount, xe_exec_queue_group_kill);
		487	}
		488
386	void xe_exec_queue_destroy(struct kref *ref)	489	void xe_exec_queue_destroy(struct kref *ref)
387	{	490	{
388	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);	491	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
@@ -567,6 +670,217 @@ exec_queue_set_pxp_type(struct xe_device xe, struct xe_exec_queue q, u64 value · 178 unmodified lines
567	return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM);	670	return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM);
568	}	671	}
569		672
		673	static int exec_queue_set_hang_replay_state(struct xe_device *xe,
		674	struct xe_exec_queue *q,
		675	u64 value)
		676	{
		677	size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class);
		678	u64 __user *address = u64_to_user_ptr(value);
		679	void *ptr;
		680
		681	ptr = vmemdup_user(address, size);
		682	if (XE_IOCTL_DBG(xe, IS_ERR(ptr)))
		683	return PTR_ERR(ptr);
		684
		685	q->replay_state = ptr;
		686
		687	return 0;
		688	}
		689
		690	static int xe_exec_queue_group_init(struct xe_device xe, struct xe_exec_queue q)
		691	{
		692	struct xe_tile *tile = gt_to_tile(q->gt);
		693	struct xe_exec_queue_group *group;
		694	struct xe_bo *bo;
		695
		696	group = kzalloc(sizeof(*group), GFP_KERNEL);
		697	if (!group)
		698	return -ENOMEM;
		699
		700	bo = xe_bo_create_pin_map_novm(xe, tile, SZ_4K, ttm_bo_type_kernel,
		701	XE_BO_FLAG_VRAM_IF_DGFX(tile) \|
		702	XE_BO_FLAG_PINNED_LATE_RESTORE \|
		703	XE_BO_FLAG_FORCE_USER_VRAM \|
		704	XE_BO_FLAG_GGTT_INVALIDATE \|
		705	XE_BO_FLAG_GGTT, false);
		706	if (IS_ERR(bo)) {
		707	drm_err(&xe->drm, "CGP bo allocation for queue group failed: %ld\n",
		708	PTR_ERR(bo));
		709	kfree(group);
		710	return PTR_ERR(bo);
		711	}
		712
		713	xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K);
		714
		715	group->primary = q;
		716	group->cgp_bo = bo;
		717	INIT_LIST_HEAD(&group->list);
		718	kref_init(&group->kill_refcount);
		719	xa_init_flags(&group->xa, XA_FLAGS_ALLOC1);
		720	mutex_init(&group->list_lock);
		721	q->multi_queue.group = group;
		722
		723	/* group->list_lock is used in submission backend */
		724	if (IS_ENABLED(CONFIG_LOCKDEP)) {
		725	fs_reclaim_acquire(GFP_KERNEL);
		726	might_lock(&group->list_lock);
		727	fs_reclaim_release(GFP_KERNEL);
		728	}
		729
		730	return 0;
		731	}
		732
		733	static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q)
		734	{
		735	return q->gt->info.multi_queue_engine_class_mask & BIT(q->class);
		736	}
		737
		738	static int xe_exec_queue_group_validate(struct xe_device xe, struct xe_exec_queue q,
		739	u32 primary_id)
		740	{
		741	struct xe_exec_queue_group *group;
		742	struct xe_exec_queue *primary;
		743	int ret;
		744
		745	/*
		746	* Get from below xe_exec_queue_lookup() pairs with put
		747	* in xe_exec_queue_group_cleanup().
		748	*/
		749	primary = xe_exec_queue_lookup(q->vm->xef, primary_id);
		750	if (XE_IOCTL_DBG(xe, !primary))
		751	return -ENOENT;
		752
		753	if (XE_IOCTL_DBG(xe, !xe_exec_queue_is_multi_queue_primary(primary)) \|\|
		754	XE_IOCTL_DBG(xe, q->vm != primary->vm) \|\|
		755	XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) {
		756	ret = -EINVAL;
		757	goto put_primary;
		758	}
		759
		760	group = primary->multi_queue.group;
		761	q->multi_queue.valid = true;
		762	q->multi_queue.group = group;
		763
		764	return 0;
		765	put_primary:
		766	xe_exec_queue_put(primary);
		767	return ret;
		768	}
		769
		770	#define XE_MAX_GROUP_SIZE 64
		771	static int xe_exec_queue_group_add(struct xe_device xe, struct xe_exec_queue q)
		772	{
		773	struct xe_exec_queue_group *group = q->multi_queue.group;
		774	u32 pos;
		775	int err;
		776
		777	xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));
		778
		779	/* Primary queue holds a reference to LRCs of all secondary queues */
		780	err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]),
		781	XA_LIMIT(1, XE_MAX_GROUP_SIZE - 1), GFP_KERNEL);
		782	if (XE_IOCTL_DBG(xe, err)) {
		783	xe_lrc_put(q->lrc[0]);
		784
		785	/* It is invalid if queue group limit is exceeded */
		786	if (err == -EBUSY)
		787	err = -EINVAL;
		788
		789	return err;
		790	}
		791
		792	q->multi_queue.pos = pos;
		793
		794	if (group->primary->multi_queue.keep_active) {
		795	xe_exec_queue_group_kill_get(group);
		796	q->multi_queue.keep_active = true;
		797	}
		798
		799	return 0;
		800	}
		801
		802	static void xe_exec_queue_group_delete(struct xe_device xe, struct xe_exec_queue q)
		803	{
		804	struct xe_exec_queue_group *group = q->multi_queue.group;
		805	struct xe_lrc *lrc;
		806
		807	xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));
		808
		809	lrc = xa_erase(&group->xa, q->multi_queue.pos);
		810	xe_assert(xe, lrc);
		811	xe_lrc_put(lrc);
		812
		813	if (q->multi_queue.keep_active) {
		814	xe_exec_queue_group_kill_put(group);
		815	q->multi_queue.keep_active = false;
		816	}
		817	}
		818
		819	static int exec_queue_set_multi_group(struct xe_device xe, struct xe_exec_queue q,
		820	u64 value)
		821	{
		822	if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q)))
		823	return -ENODEV;
		824
		825	if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe)))
		826	return -EOPNOTSUPP;
		827
		828	if (XE_IOCTL_DBG(xe, !q->vm->xef))
		829	return -EINVAL;
		830
		831	if (XE_IOCTL_DBG(xe, xe_exec_queue_is_parallel(q)))
		832	return -EINVAL;
		833
		834	if (XE_IOCTL_DBG(xe, xe_exec_queue_is_multi_queue(q)))
		835	return -EINVAL;
		836
		837	if (value & DRM_XE_MULTI_GROUP_CREATE) {
		838	if (XE_IOCTL_DBG(xe, value & ~(DRM_XE_MULTI_GROUP_CREATE \|
		839	DRM_XE_MULTI_GROUP_KEEP_ACTIVE)))
		840	return -EINVAL;
		841
		842	/*
		843	* KEEP_ACTIVE is not supported in preempt fence mode as in that mode,
		844	* VM_DESTROY ioctl expects all exec queues of that VM are already killed.
		845	*/
		846	if (XE_IOCTL_DBG(xe, (value & DRM_XE_MULTI_GROUP_KEEP_ACTIVE) &&
		847	xe_vm_in_preempt_fence_mode(q->vm)))
		848	return -EINVAL;
		849
		850	q->multi_queue.valid = true;
		851	q->multi_queue.is_primary = true;
		852	q->multi_queue.pos = 0;
		853	if (value & DRM_XE_MULTI_GROUP_KEEP_ACTIVE)
		854	q->multi_queue.keep_active = true;
		855
		856	return 0;
		857	}
		858
		859	/* While adding secondary queues, the upper 32 bits must be 0 */
		860	if (XE_IOCTL_DBG(xe, value & (~0ull << 32)))
		861	return -EINVAL;
		862
		863	return xe_exec_queue_group_validate(xe, q, value);
		864	}
		865
		866	static int exec_queue_set_multi_queue_priority(struct xe_device xe, struct xe_exec_queue q,
		867	u64 value)
		868	{
		869	if (XE_IOCTL_DBG(xe, value > XE_MULTI_QUEUE_PRIORITY_HIGH))
		870	return -EINVAL;
		871
		872	/* For queue creation time (!q->xef) setting, just store the priority value */
		873	if (!q->xef) {
		874	q->multi_queue.priority = value;
		875	return 0;
		876	}
		877
		878	if (!xe_exec_queue_is_multi_queue(q))
		879	return -EINVAL;
		880
		881	return q->ops->set_multi_queue_priority(q, value);
		882	}
		883
570	typedef int (xe_exec_queue_set_property_fn)(struct xe_device xe,	884	typedef int (xe_exec_queue_set_property_fn)(struct xe_device xe,
571	struct xe_exec_queue *q,	885	struct xe_exec_queue *q,
572	u64 value);	886	u64 value);
@@ -575,11 +889,76 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { · 2 unmodified lines
575	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,	889	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,
576	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,	890	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,
577	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type,	891	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type,
		892	[DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state,
		893	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group,
		894	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY] =
		895	exec_queue_set_multi_queue_priority,
578	};	896	};
579		897
		898	int xe_exec_queue_set_property_ioctl(struct drm_device dev, void data,
		899	struct drm_file *file)
		900	{
		901	struct xe_device *xe = to_xe_device(dev);
		902	struct xe_file *xef = to_xe_file(file);
		903	struct drm_xe_exec_queue_set_property *args = data;
		904	struct xe_exec_queue *q;
		905	int ret;
		906	u32 idx;
		907
		908	if (XE_IOCTL_DBG(xe, args->reserved[0] \|\| args->reserved[1]))
		909	return -EINVAL;
		910
		911	if (XE_IOCTL_DBG(xe, args->property !=
		912	DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
		913	return -EINVAL;
		914
		915	q = xe_exec_queue_lookup(xef, args->exec_queue_id);
		916	if (XE_IOCTL_DBG(xe, !q))
		917	return -ENOENT;
		918
		919	idx = array_index_nospec(args->property,
		920	ARRAY_SIZE(exec_queue_set_property_funcs));
		921	ret = exec_queue_set_property_funcs[idx](xe, q, args->value);
		922	if (XE_IOCTL_DBG(xe, ret))
		923	goto err_post_lookup;
		924
		925	xe_exec_queue_put(q);
		926	return 0;
		927
		928	err_post_lookup:
		929	xe_exec_queue_put(q);
		930	return ret;
		931	}
		932
		933	static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties)
		934	{
		935	u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) \|
		936	BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY);
		937
		938	/*
		939	* Only MULTI_QUEUE_PRIORITY property is valid for secondary queues of a
		940	* multi-queue group.
		941	*/
		942	if (xe_exec_queue_is_multi_queue_secondary(q) &&
		943	properties & ~secondary_queue_valid_props)
		944	return -EINVAL;
		945
		946	return 0;
		947	}
		948
		949	static int exec_queue_user_ext_check_final(struct xe_exec_queue *q, u64 properties)
		950	{
		951	/* MULTI_QUEUE_PRIORITY only applies to multi-queue group queues */
		952	if ((properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) &&
		953	!(properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)))
		954	return -EINVAL;
		955
		956	return 0;
		957	}
		958
580	static int exec_queue_user_ext_set_property(struct xe_device *xe,	959	static int exec_queue_user_ext_set_property(struct xe_device *xe,
581	struct xe_exec_queue *q,	960	struct xe_exec_queue *q,
582	u64 extension)	961	u64 extension, u64 *properties)
583	{	962	{
584	u64 __user *address = u64_to_user_ptr(extension);	963	u64 __user *address = u64_to_user_ptr(extension);
585	struct drm_xe_ext_set_property ext;	964	struct drm_xe_ext_set_property ext;
@@ -595,27 +974,35 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe, · 9 unmodified lines
595	XE_IOCTL_DBG(xe, ext.pad) \|\|	974	XE_IOCTL_DBG(xe, ext.pad) \|\|
596	XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&	975	XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&
597	ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE &&	976	ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE &&
598	ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE))	977	ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE &&
		978	ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE &&
		979	ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP &&
		980	ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
599	return -EINVAL;	981	return -EINVAL;
600		982
601	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));	983	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));
602	if (!exec_queue_set_property_funcs[idx])	984	if (!exec_queue_set_property_funcs[idx])
603	return -EINVAL;	985	return -EINVAL;
604		986
		987	*properties \|= BIT_ULL(idx);
		988	err = exec_queue_user_ext_check(q, *properties);
		989	if (XE_IOCTL_DBG(xe, err))
		990	return err;
		991
605	return exec_queue_set_property_funcs[idx](xe, q, ext.value);	992	return exec_queue_set_property_funcs[idx](xe, q, ext.value);
606	}	993	}
607		994
608	typedef int (xe_exec_queue_user_extension_fn)(struct xe_device xe,	995	typedef int (xe_exec_queue_user_extension_fn)(struct xe_device xe,
609	struct xe_exec_queue *q,	996	struct xe_exec_queue *q,
610	u64 extension);	997	u64 extension, u64 *properties);
611		998
612	static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {	999	static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {
613	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,	1000	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,
614	};	1001	};
615		1002
616	#define MAX_USER_EXTENSIONS 16	1003	#define MAX_USER_EXTENSIONS 16
617	static int exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,	1004	static int __exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,
618	u64 extensions, int ext_number)	1005	u64 extensions, int ext_number, u64 *properties)
619	{	1006	{
620	u64 __user *address = u64_to_user_ptr(extensions);	1007	u64 __user *address = u64_to_user_ptr(extensions);
621	struct drm_xe_user_extension ext;	1008	struct drm_xe_user_extension ext;
@@ -636,13 +1023,36 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue · 14 unmodified lines
636		1023
637	idx = array_index_nospec(ext.name,	1024	idx = array_index_nospec(ext.name,
638	ARRAY_SIZE(exec_queue_user_extension_funcs));	1025	ARRAY_SIZE(exec_queue_user_extension_funcs));
639	err = exec_queue_user_extension_funcs[idx](xe, q, extensions);	1026	err = exec_queue_user_extension_funcs[idx](xe, q, extensions, properties);
640	if (XE_IOCTL_DBG(xe, err))	1027	if (XE_IOCTL_DBG(xe, err))
641	return err;	1028	return err;
642		1029
643	if (ext.next_extension)	1030	if (ext.next_extension)
644	return exec_queue_user_extensions(xe, q, ext.next_extension,	1031	return __exec_queue_user_extensions(xe, q, ext.next_extension,
645	++ext_number);	1032	++ext_number, properties);
		1033
		1034	return 0;
		1035	}
		1036
		1037	static int exec_queue_user_extensions(struct xe_device xe, struct xe_exec_queue q,
		1038	u64 extensions)
		1039	{
		1040	u64 properties = 0;
		1041	int err;
		1042
		1043	err = __exec_queue_user_extensions(xe, q, extensions, 0, &properties);
		1044	if (XE_IOCTL_DBG(xe, err))
		1045	return err;
		1046
		1047	err = exec_queue_user_ext_check_final(q, properties);
		1048	if (XE_IOCTL_DBG(xe, err))
		1049	return err;
		1050
		1051	if (xe_exec_queue_is_multi_queue_primary(q)) {
		1052	err = xe_exec_queue_group_init(xe, q);
		1053	if (XE_IOCTL_DBG(xe, err))
		1054	return err;
		1055	}
646		1056
647	return 0;	1057	return 0;
648	}	1058	}
@@ -798,12 +1208,18 @@ int xe_exec_queue_create_ioctl(struct drm_device dev, void data, · 149 unmodified lines
798	if (IS_ERR(q))	1208	if (IS_ERR(q))
799	return PTR_ERR(q);	1209	return PTR_ERR(q);
800		1210
		1211	if (xe_exec_queue_is_multi_queue_secondary(q)) {
		1212	err = xe_exec_queue_group_add(xe, q);
		1213	if (XE_IOCTL_DBG(xe, err))
		1214	goto put_exec_queue;
		1215	}
		1216
801	if (xe_vm_in_preempt_fence_mode(vm)) {	1217	if (xe_vm_in_preempt_fence_mode(vm)) {
802	q->lr.context = dma_fence_context_alloc(1);	1218	q->lr.context = dma_fence_context_alloc(1);
803		1219
804	err = xe_vm_add_compute_exec_queue(vm, q);	1220	err = xe_vm_add_compute_exec_queue(vm, q);
805	if (XE_IOCTL_DBG(xe, err))	1221	if (XE_IOCTL_DBG(xe, err))
806	goto put_exec_queue;	1222	goto delete_queue_group;
807	}	1223	}
808		1224
809	if (q->vm && q->hwe->hw_engine_group) {	1225	if (q->vm && q->hwe->hw_engine_group) {
@@ -826,6 +1242,9 @@ int xe_exec_queue_create_ioctl(struct drm_device dev, void data, · 16 unmodified lines
826		1242
827	kill_exec_queue:	1243	kill_exec_queue:
828	xe_exec_queue_kill(q);	1244	xe_exec_queue_kill(q);
		1245	delete_queue_group:
		1246	if (xe_exec_queue_is_multi_queue_secondary(q))
		1247	xe_exec_queue_group_delete(xe, q);
829	put_exec_queue:	1248	put_exec_queue:
830	xe_exec_queue_put(q);	1249	xe_exec_queue_put(q);
831	return err;	1250	return err;
@@ -981,6 +1400,11 @@ void xe_exec_queue_kill(struct xe_exec_queue *q) · 149 unmodified lines
981		1400
982	q->ops->kill(q);	1401	q->ops->kill(q);
983	xe_vm_remove_compute_exec_queue(q->vm, q);	1402	xe_vm_remove_compute_exec_queue(q->vm, q);
		1403
		1404	if (!xe_exec_queue_is_multi_queue_primary(q) && q->multi_queue.keep_active) {
		1405	xe_exec_queue_group_kill_put(q->multi_queue.group);
		1406	q->multi_queue.keep_active = false;
		1407	}
984	}	1408	}
985		1409
986	int xe_exec_queue_destroy_ioctl(struct drm_device dev, void data,	1410	int xe_exec_queue_destroy_ioctl(struct drm_device dev, void data,
@@ -1007,7 +1431,10 @@ int xe_exec_queue_destroy_ioctl(struct drm_device dev, void data, · 20 unmodified lines
1007	if (q->vm && q->hwe->hw_engine_group)	1431	if (q->vm && q->hwe->hw_engine_group)
1008	xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);	1432	xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
1009		1433
1010	xe_exec_queue_kill(q);	1434	if (xe_exec_queue_is_multi_queue_primary(q))
		1435	xe_exec_queue_group_kill_put(q->multi_queue.group);
		1436	else
		1437	xe_exec_queue_kill(q);
1011		1438
1012	trace_xe_exec_queue_close(q);	1439	trace_xe_exec_queue_close(q);
1013	xe_exec_queue_put(q);	1440	xe_exec_queue_put(q);

drivers/gpu/drm/xe/xe_exec_queue.h+68−0

@@ -66,6 +66,55 @@ static inline bool xe_exec_queue_uses_pxp(struct xe_exec_queue *q) · 65 unmodified lines
66	return q->pxp.type;	66	return q->pxp.type;
67	}	67	}
68		68
		69	/**
		70	* xe_exec_queue_is_multi_queue() - Whether an exec_queue is part of a queue group.
		71	* @q: The exec_queue
		72	*
		73	* Return: True if the exec_queue is part of a queue group, false otherwise.
		74	*/
		75	static inline bool xe_exec_queue_is_multi_queue(struct xe_exec_queue *q)
		76	{
		77	return q->multi_queue.valid;
		78	}
		79
		80	/**
		81	* xe_exec_queue_is_multi_queue_primary() - Whether an exec_queue is primary queue
		82	* of a multi queue group.
		83	* @q: The exec_queue
		84	*
		85	* Return: True if @q is primary queue of a queue group, false otherwise.
		86	*/
		87	static inline bool xe_exec_queue_is_multi_queue_primary(struct xe_exec_queue *q)
		88	{
		89	return q->multi_queue.is_primary;
		90	}
		91
		92	/**
		93	* xe_exec_queue_is_multi_queue_secondary() - Whether an exec_queue is secondary queue
		94	* of a multi queue group.
		95	* @q: The exec_queue
		96	*
		97	* Return: True if @q is secondary queue of a queue group, false otherwise.
		98	*/
		99	static inline bool xe_exec_queue_is_multi_queue_secondary(struct xe_exec_queue *q)
		100	{
		101	return xe_exec_queue_is_multi_queue(q) && !xe_exec_queue_is_multi_queue_primary(q);
		102	}
		103
		104	/**
		105	* xe_exec_queue_multi_queue_primary() - Get multi queue group's primary queue
		106	* @q: The exec_queue
		107	*
		108	* If @q belongs to a multi queue group, then the primary queue of the group will
		109	* be returned. Otherwise, @q will be returned.
		110	*/
		111	static inline struct xe_exec_queue xe_exec_queue_multi_queue_primary(struct xe_exec_queue q)
		112	{
		113	return xe_exec_queue_is_multi_queue(q) ? q->multi_queue.group->primary : q;
		114	}
		115
		116	void xe_exec_queue_group_kill_put(struct xe_exec_queue_group *group);
		117
69	bool xe_exec_queue_is_lr(struct xe_exec_queue *q);	118	bool xe_exec_queue_is_lr(struct xe_exec_queue *q);
70		119
71	bool xe_exec_queue_is_idle(struct xe_exec_queue *q);	120	bool xe_exec_queue_is_idle(struct xe_exec_queue *q);
@@ -78,6 +127,8 @@ int xe_exec_queue_destroy_ioctl(struct drm_device dev, void data, · 6 unmodified lines
78	struct drm_file *file);	127	struct drm_file *file);
79	int xe_exec_queue_get_property_ioctl(struct drm_device dev, void data,	128	int xe_exec_queue_get_property_ioctl(struct drm_device dev, void data,
80	struct drm_file *file);	129	struct drm_file *file);
		130	int xe_exec_queue_set_property_ioctl(struct drm_device dev, void data,
		131	struct drm_file *file);
81	enum xe_exec_queue_priority xe_exec_queue_device_get_max_priority(struct xe_device *xe);	132	enum xe_exec_queue_priority xe_exec_queue_device_get_max_priority(struct xe_device *xe);
82		133
83	void xe_exec_queue_last_fence_put(struct xe_exec_queue e, struct xe_vm vm);	134	void xe_exec_queue_last_fence_put(struct xe_exec_queue e, struct xe_vm vm);
@@ -111,4 +162,21 @@ int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue q, void scratch); · 27 unmodified lines
111		162
112	struct xe_lrc xe_exec_queue_lrc(struct xe_exec_queue q);	163	struct xe_lrc xe_exec_queue_lrc(struct xe_exec_queue q);
113		164
		165	/**
		166	* xe_exec_queue_idle_skip_suspend() - Can exec queue skip suspend
		167	* @q: The exec_queue
		168	*
		169	* If an exec queue is not parallel and is idle, the suspend steps can be
		170	* skipped in the submission backend immediatley signaling the suspend fence.
		171	* Parallel queues cannot skip this step due to limitations in the submission
		172	* backend.
		173	*
		174	* Return: True if exec queue is idle and can skip suspend steps, False
		175	* otherwise
		176	*/
		177	static inline bool xe_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
		178	{
		179	return !xe_exec_queue_is_parallel(q) && xe_exec_queue_is_idle(q);
		180	}
		181
114	#endif	182	#endif

drivers/gpu/drm/xe/xe_exec_queue_types.h+62−0

@@ -32,6 +32,44 @@ enum xe_exec_queue_priority { · 31 unmodified lines
32	XE_EXEC_QUEUE_PRIORITY_COUNT	32	XE_EXEC_QUEUE_PRIORITY_COUNT
33	};	33	};
34		34
		35	/**
		36	* enum xe_multi_queue_priority - Multi Queue priority values
		37	*
		38	* The priority values of the queues within the multi queue group.
		39	*/
		40	enum xe_multi_queue_priority {
		41	/** @XE_MULTI_QUEUE_PRIORITY_LOW: Priority low */
		42	XE_MULTI_QUEUE_PRIORITY_LOW = 0,
		43	/** @XE_MULTI_QUEUE_PRIORITY_NORMAL: Priority normal */
		44	XE_MULTI_QUEUE_PRIORITY_NORMAL,
		45	/** @XE_MULTI_QUEUE_PRIORITY_HIGH: Priority high */
		46	XE_MULTI_QUEUE_PRIORITY_HIGH,
		47	};
		48
		49	/**
		50	* struct xe_exec_queue_group - Execution multi queue group
		51	*
		52	* Contains multi queue group information.
		53	*/
		54	struct xe_exec_queue_group {
		55	/** @primary: Primary queue of this group */
		56	struct xe_exec_queue *primary;
		57	/** @cgp_bo: BO for the Context Group Page */
		58	struct xe_bo *cgp_bo;
		59	/** @xa: xarray to store LRCs */
		60	struct xarray xa;
		61	/** @list: List of all secondary queues in the group */
		62	struct list_head list;
		63	/** @list_lock: Secondary queue list lock */
		64	struct mutex list_lock;
		65	/** @kill_refcount: ref count to kill primary queue */
		66	struct kref kill_refcount;
		67	/** @sync_pending: CGP_SYNC_DONE g2h response pending */
		68	bool sync_pending;
		69	/** @banned: Group banned */
		70	bool banned;
		71	};
		72
35	/**	73	/**
36	* struct xe_exec_queue - Execution queue	74	* struct xe_exec_queue - Execution queue
37	*	75	*
@@ -111,6 +149,24 @@ struct xe_exec_queue { · 73 unmodified lines
111	struct xe_guc_exec_queue *guc;	149	struct xe_guc_exec_queue *guc;
112	};	150	};
113		151
		152	/** @multi_queue: Multi queue information */
		153	struct {
		154	/** @multi_queue.group: Queue group information */
		155	struct xe_exec_queue_group *group;
		156	/** @multi_queue.link: Link into group's secondary queues list */
		157	struct list_head link;
		158	/** @multi_queue.priority: Queue priority within the multi-queue group */
		159	enum xe_multi_queue_priority priority;
		160	/** @multi_queue.pos: Position of queue within the multi-queue group */
		161	u8 pos;
		162	/** @multi_queue.valid: Queue belongs to a multi queue group */
		163	u8 valid:1;
		164	/** @multi_queue.is_primary: Is primary queue (Q0) of the group */
		165	u8 is_primary:1;
		166	/** @multi_queue.keep_active: Keep the group active after primary is destroyed */
		167	u8 keep_active:1;
		168	} multi_queue;
		169
114	/** @sched_props: scheduling properties */	170	/** @sched_props: scheduling properties */
115	struct {	171	struct {
116	/** @sched_props.timeslice_us: timeslice period in micro-seconds */	172	/** @sched_props.timeslice_us: timeslice period in micro-seconds */
@@ -167,6 +223,9 @@ struct xe_exec_queue { · 50 unmodified lines
167	/** @ufence_timeline_value: User fence timeline value */	223	/** @ufence_timeline_value: User fence timeline value */
168	u64 ufence_timeline_value;	224	u64 ufence_timeline_value;
169		225
		226	/** @replay_state: GPU hang replay state */
		227	void *replay_state;
		228
170	/** @ops: submission backend exec queue operations */	229	/** @ops: submission backend exec queue operations */
171	const struct xe_exec_queue_ops *ops;	230	const struct xe_exec_queue_ops *ops;
172		231
@@ -213,6 +272,9 @@ struct xe_exec_queue_ops { · 40 unmodified lines
213	int (set_timeslice)(struct xe_exec_queue q, u32 timeslice_us);	272	int (set_timeslice)(struct xe_exec_queue q, u32 timeslice_us);
214	/** @set_preempt_timeout: Set preemption timeout for exec queue */	273	/** @set_preempt_timeout: Set preemption timeout for exec queue */
215	int (set_preempt_timeout)(struct xe_exec_queue q, u32 preempt_timeout_us);	274	int (set_preempt_timeout)(struct xe_exec_queue q, u32 preempt_timeout_us);
		275	/** @set_multi_queue_priority: Set multi queue priority */
		276	int (set_multi_queue_priority)(struct xe_exec_queue q,
		277	enum xe_multi_queue_priority priority);
216	/**	278	/**
217	* @suspend: Suspend exec queue from executing, allowed to be called	279	* @suspend: Suspend exec queue from executing, allowed to be called
218	* multiple times in a row before resume with the caveat that	280	* multiple times in a row before resume with the caveat that

drivers/gpu/drm/xe/xe_execlist.c+1−1

@@ -269,7 +269,7 @@ struct xe_execlist_port xe_execlist_port_create(struct xe_device xe, · 268 unmodified lines
269		269
270	port->hwe = hwe;	270	port->hwe = hwe;
271		271
272	port->lrc = xe_lrc_create(hwe, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0);	272	port->lrc = xe_lrc_create(hwe, NULL, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0);
273	if (IS_ERR(port->lrc)) {	273	if (IS_ERR(port->lrc)) {
274	err = PTR_ERR(port->lrc);	274	err = PTR_ERR(port->lrc);
275	goto err;	275	goto err;

drivers/gpu/drm/xe/xe_force_wake.c+7−0

@@ -166,6 +166,13 @@ static int domain_sleep_wait(struct xe_gt *gt, · 165 unmodified lines
166	* xe_force_wake_ref_has_domain() function. Caller must call	166	* xe_force_wake_ref_has_domain() function. Caller must call
167	* xe_force_wake_put() function to decrease incremented refcounts.	167	* xe_force_wake_put() function to decrease incremented refcounts.
168	*	168	*
		169	* When possible, scope-based forcewake (through CLASS(xe_force_wake, ...) or
		170	* xe_with_force_wake()) should be used instead of direct calls to this
		171	* function. Direct usage of get/put should only be used when the function
		172	* has goto-based flows that can interfere with scope-based cleanup, or when
		173	* the lifetime of the forcewake reference does not match a specific scope
		174	* (e.g., forcewake obtained in one function and released in a different one).
		175	*
169	* Return: opaque reference to woken domains or zero if none of requested	176	* Return: opaque reference to woken domains or zero if none of requested
170	* domains were awake.	177	* domains were awake.
171	*/	178	*/

drivers/gpu/drm/xe/xe_force_wake.h+40−0

@@ -61,4 +61,44 @@ xe_force_wake_ref_has_domain(unsigned int fw_ref, enum xe_force_wake_domains dom · 60 unmodified lines
61	return fw_ref & domain;	61	return fw_ref & domain;
62	}	62	}
63		63
		64	struct xe_force_wake_ref {
		65	struct xe_force_wake *fw;
		66	unsigned int domains;
		67	};
		68
		69	static struct xe_force_wake_ref
		70	xe_force_wake_constructor(struct xe_force_wake *fw, unsigned int domains)
		71	{
		72	struct xe_force_wake_ref fw_ref = { .fw = fw };
		73
		74	fw_ref.domains = xe_force_wake_get(fw, domains);
		75
		76	return fw_ref;
		77	}
		78
		79	DEFINE_CLASS(xe_force_wake, struct xe_force_wake_ref,
		80	xe_force_wake_put(_T.fw, _T.domains),
		81	xe_force_wake_constructor(fw, domains),
		82	struct xe_force_wake *fw, unsigned int domains);
		83
		84	/*
		85	* Scoped helper for the forcewake class, using the same trick as scoped_guard()
		86	* to bind the lifetime to the next statement/block.
		87	*/
		88	#define __xe_with_force_wake(ref, fw, domains, done) \
		89	for (CLASS(xe_force_wake, ref)(fw, domains), *(done) = NULL; \
		90	!(done); (done) = (void *)1)
		91
		92	#define xe_with_force_wake(ref, fw, domains) \
		93	__xe_with_force_wake(ref, fw, domains, __UNIQUE_ID(done))
		94
		95	/*
		96	* Used when xe_force_wake_constructor() has already been called by another
		97	* function and the current function is responsible for releasing the forcewake
		98	* reference in all possible cases and error paths.
		99	*/
		100	DEFINE_CLASS(xe_force_wake_release_only, struct xe_force_wake_ref,
		101	if (_T.fw) xe_force_wake_put(_T.fw, _T.domains), fw_ref,
		102	struct xe_force_wake_ref fw_ref);
		103
64	#endif	104	#endif

drivers/gpu/drm/xe/xe_ggtt.c+1−2

@@ -396,9 +396,8 @@ static void ggtt_node_remove_work_func(struct work_struct *work) · 395 unmodified lines
396	delayed_removal_work);	396	delayed_removal_work);
397	struct xe_device *xe = tile_to_xe(node->ggtt->tile);	397	struct xe_device *xe = tile_to_xe(node->ggtt->tile);
398		398
399	xe_pm_runtime_get(xe);	399	guard(xe_pm_runtime)(xe);
400	ggtt_node_remove(node);	400	ggtt_node_remove(node);
401	xe_pm_runtime_put(xe);
402	}	401	}
403		402
404	/**	403	/**

drivers/gpu/drm/xe/xe_gsc.c+6−15

 	struct xe_gsc *gsc = container_of(work, typeof(*gsc), work);
 	struct xe_gt *gt = gsc_to_gt(gsc);
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u32 actions;
 	int ret;
 	gsc->work_actions = 0;
 	spin_unlock_irq(&gsc->lock);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
 	if (actions & GSC_ACTION_ER_COMPLETE) {
-		ret = gsc_er_complete(gt);
+		if (gsc_er_complete(gt))
-		if (ret)
+			return;
-			goto out;
 	}
 	if (actions & GSC_ACTION_FW_LOAD) {
 	if (actions & GSC_ACTION_SW_PROXY)
 		xe_gsc_proxy_request_handler(gsc);
-out:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_pm_runtime_put(xe);
 }
 void xe_gsc_hwe_irq_handler(struct xe_hw_engine *hwe, u16 intr_vec)
 {
 	struct xe_gt *gt = gsc_to_gt(gsc);
 	struct xe_mmio *mmio = &gt->mmio;
-	unsigned int fw_ref;
 	xe_uc_fw_print(&gsc->fw, p);
 	if (!xe_uc_fw_is_enabled(&gsc->fw))
 		return;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return;
 	drm_printf(p, "\nHECI1 FWSTS: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
 			xe_mmio_read32(mmio, HECI_FWSTS4(MTL_GSC_HECI1_BASE)),
 			xe_mmio_read32(mmio, HECI_FWSTS5(MTL_GSC_HECI1_BASE)),
 			xe_mmio_read32(mmio, HECI_FWSTS6(MTL_GSC_HECI1_BASE)));
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }

drivers/gpu/drm/xe/xe_gsc_debugfs.c+1−2

@@ -37,9 +37,8 @@ static int gsc_info(struct seq_file m, void data) · 36 unmodified lines
37	struct xe_device *xe = gsc_to_xe(gsc);	37	struct xe_device *xe = gsc_to_xe(gsc);
38	struct drm_printer p = drm_seq_file_printer(m);	38	struct drm_printer p = drm_seq_file_printer(m);
39		39
40	xe_pm_runtime_get(xe);	40	guard(xe_pm_runtime)(xe);
41	xe_gsc_print_info(gsc, &p);	41	xe_gsc_print_info(gsc, &p);
42	xe_pm_runtime_put(xe);
43		42
44	return 0;	43	return 0;
45	}	44	}

drivers/gpu/drm/xe/xe_gsc_proxy.c+7−10

@@ -440,22 +440,19 @@ static void xe_gsc_proxy_remove(void *arg) · 439 unmodified lines
440	struct xe_gsc *gsc = arg;	440	struct xe_gsc *gsc = arg;
441	struct xe_gt *gt = gsc_to_gt(gsc);	441	struct xe_gt *gt = gsc_to_gt(gsc);
442	struct xe_device *xe = gt_to_xe(gt);	442	struct xe_device *xe = gt_to_xe(gt);
443	unsigned int fw_ref = 0;
444		443
445	if (!gsc->proxy.component_added)	444	if (!gsc->proxy.component_added)
446	return;	445	return;
447		446
448	/* disable HECI2 IRQs */	447	/* disable HECI2 IRQs */
449	xe_pm_runtime_get(xe);	448	scoped_guard(xe_pm_runtime, xe) {
450	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GSC);	449	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GSC);
451	if (!fw_ref)	450	if (!fw_ref.domains)
452	xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n");	451	xe_gt_err(gt, "failed to get forcewake to disable GSC interrupts\n");
453		452
454	/* try do disable irq even if forcewake failed */	453	/* try do disable irq even if forcewake failed */
455	gsc_proxy_irq_toggle(gsc, false);	454	gsc_proxy_irq_toggle(gsc, false);
456		455	}
457	xe_force_wake_put(gt_to_fw(gt), fw_ref);
458	xe_pm_runtime_put(xe);
459		456
460	xe_gsc_wait_for_worker_completion(gsc);	457	xe_gsc_wait_for_worker_completion(gsc);
461		458

drivers/gpu/drm/xe/xe_gt.c+87−87

 static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	u32 reg;
 	if (!XE_GT_WA(gt, 16023588340))
 		return;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return;
 	if (xe_gt_is_main_type(gt)) {
 	}
 	xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0xF);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	u32 reg;
 	if (!XE_GT_WA(gt, 16023588340))
 	if (xe_gt_is_media_type(gt))
 		return;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return;
 	reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
 	reg &= ~CG_DIS_CNTLBUS;
 	xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 static void gt_reset_worker(struct work_struct *w);
 int xe_gt_init_early(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
 	if (IS_SRIOV_PF(gt_to_xe(gt))) {
 	if (err)
 		return err;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	xe_gt_mcr_init_early(gt);
 	xe_pat_init(gt);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 static int gt_init_with_gt_forcewake(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	err = xe_uc_init(&gt->uc);
 	if (err)
-		goto err_force_wake;
+		return err;
 	xe_gt_topology_init(gt);
 	xe_gt_mcr_init(gt);
 	if (xe_gt_is_main_type(gt)) {
 		err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt);
 		if (err)
-			goto err_force_wake;
+			return err;
 		if (IS_SRIOV_PF(gt_to_xe(gt)))
 			xe_lmtt_init(&gt_to_tile(gt)->sriov.pf.lmtt);
 	}
 	err = xe_hw_engines_init_early(gt);
 	if (err) {
 		dump_pat_on_error(gt);
-		goto err_force_wake;
+		return err;
 	}
 	err = xe_hw_engine_class_sysfs_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 	/* Initialize CCS mode sysfs after early initialization of HW engines */
 	err = xe_gt_ccs_mode_sysfs_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 	/*
 	 * Stash hardware-reported version.  Since this register does not exist
 	 */
 	gt->info.gmdid = xe_mmio_read32(&gt->mmio, GMD_ID);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	return err;
 }
 static int gt_init_with_all_forcewake(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
-		err = -ETIMEDOUT;
+		return -ETIMEDOUT;
-		goto err_force_wake;
-	}
 	xe_gt_mcr_set_implicit_defaults(gt);
 	xe_wa_process_gt(gt);
 	err = xe_gt_clock_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 	xe_mocs_init(gt);
 	err = xe_execlist_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 	err = xe_hw_engines_init(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 	err = xe_uc_init_post_hwconfig(&gt->uc);
 	if (err)
-		goto err_force_wake;
+		return err;
 	if (xe_gt_is_main_type(gt)) {
 		/*
 			gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt),
 								IS_DGFX(xe) ? SZ_1M : SZ_512K, 16);
-			if (IS_ERR(gt->usm.bb_pool)) {
+			if (IS_ERR(gt->usm.bb_pool))
-				err = PTR_ERR(gt->usm.bb_pool);
+				return PTR_ERR(gt->usm.bb_pool);
-				goto err_force_wake;
-			}
 		}
 	}
 		err = xe_migrate_init(tile->migrate);
 		if (err)
-			goto err_force_wake;
+			return err;
 	}
 	err = xe_uc_load_hw(&gt->uc);
 	if (err)
-		goto err_force_wake;
+		return err;
 	/* Configure default CCS mode of 1 engine with all resources */
 	if (xe_gt_ccs_mode_enabled(gt)) {
 	if (IS_SRIOV_PF(gt_to_xe(gt)))
 		xe_gt_sriov_pf_init_hw(gt);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	return err;
 }
 static void xe_gt_fini(void *arg)
 void xe_gt_suspend_prepare(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	xe_uc_suspend_prepare(&gt->uc);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 int xe_gt_suspend(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
 	xe_gt_dbg(gt, "suspending\n");
 	xe_gt_sanitize(gt);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
-		goto err_msg;
+		xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
+		return -ETIMEDOUT;
+	}
 	err = xe_uc_suspend(&gt->uc);
-	if (err)
+	if (err) {
-		goto err_force_wake;
+		xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
+		return err;
+	}
 	xe_gt_idle_disable_pg(gt);
 	xe_gt_disable_host_l2_vram(gt);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	xe_gt_dbg(gt, "suspended\n");
 	return 0;
-err_msg:
-	err = -ETIMEDOUT;
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
-	return err;
 }
 void xe_gt_shutdown(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	do_gt_reset(gt);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 /**
 int xe_gt_resume(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	int err;
 	xe_gt_dbg(gt, "resuming\n");
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
-		goto err_msg;
+		xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
+		return -ETIMEDOUT;
+	}
 	err = do_gt_restart(gt);
 	if (err)
-		goto err_force_wake;
+		return err;
 	xe_gt_idle_enable_pg(gt);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	xe_gt_dbg(gt, "resumed\n");
 	return 0;
+}
-err_msg:
+/**
-	err = -ETIMEDOUT;
+ * xe_gt_runtime_suspend() - GT runtime suspend
-err_force_wake:
+ * @gt: the GT object
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
+ *
-	xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));
+ * Return: 0 on success, negative error code otherwise.
+ */
+int xe_gt_runtime_suspend(struct xe_gt *gt)
+{
+	xe_gt_dbg(gt, "runtime suspending\n");
-	return err;
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
+		xe_gt_err(gt, "runtime suspend failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
+		return -ETIMEDOUT;
+	}
+	xe_uc_runtime_suspend(&gt->uc);
+	xe_gt_disable_host_l2_vram(gt);
+	xe_gt_dbg(gt, "runtime suspended\n");
+	return 0;
+}
+/**
+ * xe_gt_runtime_resume() - GT runtime resume
+ * @gt: the GT object
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int xe_gt_runtime_resume(struct xe_gt *gt)
+{
+	xe_gt_dbg(gt, "runtime resuming\n");
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
+		xe_gt_err(gt, "runtime resume failed (%pe)\n", ERR_PTR(-ETIMEDOUT));
+		return -ETIMEDOUT;
+	}
+	xe_gt_enable_host_l2_vram(gt);
+	xe_uc_runtime_resume(&gt->uc);
+	xe_gt_dbg(gt, "runtime resumed\n");
+	return 0;
 }
 struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt,

drivers/gpu/drm/xe/xe_gt.h+2−0

@@ -58,6 +58,8 @@ int xe_gt_suspend(struct xe_gt *gt); · 57 unmodified lines
58	void xe_gt_shutdown(struct xe_gt *gt);	58	void xe_gt_shutdown(struct xe_gt *gt);
59	int xe_gt_resume(struct xe_gt *gt);	59	int xe_gt_resume(struct xe_gt *gt);
60	void xe_gt_reset_async(struct xe_gt *gt);	60	void xe_gt_reset_async(struct xe_gt *gt);
		61	int xe_gt_runtime_resume(struct xe_gt *gt);
		62	int xe_gt_runtime_suspend(struct xe_gt *gt);
61	void xe_gt_sanitize(struct xe_gt *gt);	63	void xe_gt_sanitize(struct xe_gt *gt);
62	int xe_gt_sanitize_freq(struct xe_gt *gt);	64	int xe_gt_sanitize_freq(struct xe_gt *gt);
63		65

drivers/gpu/drm/xe/xe_gt_debugfs.c+9−21

 	struct drm_info_node *node = m->private;
 	struct xe_gt *gt = node_to_gt(node);
 	struct xe_device *xe = gt_to_xe(gt);
-	int ret;
-	xe_pm_runtime_get(xe);
-	ret = xe_gt_debugfs_simple_show(m, data);
-	xe_pm_runtime_put(xe);
-	return ret;
+	guard(xe_pm_runtime)(xe);
+	return xe_gt_debugfs_simple_show(m, data);
 }
 static int hw_engines(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_hw_engine *hwe;
 	enum xe_hw_engine_id id;
-	unsigned int fw_ref;
-	int ret = 0;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
-		ret = -ETIMEDOUT;
+		return -ETIMEDOUT;
-		goto fw_put;
-	}
 	for_each_hw_engine(hwe, gt, id)
 		xe_hw_engine_print(hwe, p);
-fw_put:
+	return 0;
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	return ret;
 }
 static int steering(struct xe_gt *gt, struct drm_printer *p)
 	{ "default_lrc_vcs", .show = xe_gt_debugfs_show_with_rpm, .data = vcs_default_lrc },
 	{ "default_lrc_vecs", .show = xe_gt_debugfs_show_with_rpm, .data = vecs_default_lrc },
 	{ "hwconfig", .show = xe_gt_debugfs_show_with_rpm, .data = hwconfig },
+	{ "pat_sw_config", .show = xe_gt_debugfs_simple_show, .data = xe_pat_dump_sw_config },
 };
 /* everything else should be added here */
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	xe_gt_reset_async(gt);
-	xe_pm_runtime_put(xe);
 }
 static ssize_t force_reset_write(struct file *file,
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	xe_gt_reset(gt);
-	xe_pm_runtime_put(xe);
 }
 static ssize_t force_reset_sync_write(struct file *file,

drivers/gpu/drm/xe/xe_gt_freq.c+9−18

 	struct xe_guc_pc *pc = dev_to_pc(dev);
 	u32 freq;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	freq = xe_guc_pc_get_act_freq(pc);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	return sysfs_emit(buf, "%d\n", freq);
 }
 	u32 freq;
 	ssize_t ret;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_get_cur_freq(pc, &freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 	struct xe_guc_pc *pc = dev_to_pc(dev);
 	u32 freq;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	freq = xe_guc_pc_get_rpe_freq(pc);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	return sysfs_emit(buf, "%d\n", freq);
 }
 	struct xe_guc_pc *pc = dev_to_pc(dev);
 	u32 freq;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	freq = xe_guc_pc_get_rpa_freq(pc);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	return sysfs_emit(buf, "%d\n", freq);
 }
 	u32 freq;
 	ssize_t ret;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_get_min_freq(pc, &freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 	if (ret)
 		return ret;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_set_min_freq(pc, freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 	u32 freq;
 	ssize_t ret;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_get_max_freq(pc, &freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 	if (ret)
 		return ret;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	ret = xe_guc_pc_set_max_freq(pc, freq);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	if (ret)
 		return ret;
 	struct xe_guc_pc *pc = dev_to_pc(dev);
 	int err;
-	xe_pm_runtime_get(dev_to_xe(dev));
+	guard(xe_pm_runtime)(dev_to_xe(dev));
 	err = xe_guc_pc_set_power_profile(pc, buff);
-	xe_pm_runtime_put(dev_to_xe(dev));
 	return err ?: count;
 }

drivers/gpu/drm/xe/xe_gt_idle.c+12−29

 	struct xe_gt_idle *gtidle = &gt->gtidle;
 	struct xe_mmio *mmio = &gt->mmio;
 	u32 vcs_mask, vecs_mask;
-	unsigned int fw_ref;
 	int i, j;
 	if (IS_SRIOV_VF(xe))
 		}
 	}
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 	if (xe->info.skip_guc_pc) {
 		/*
 		 * GuC sets the hysteresis value when GuC PC is enabled
 					      VDN_MFXVDENC_POWERGATE_ENABLE(2));
 	xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 void xe_gt_idle_disable_pg(struct xe_gt *gt)
 {
 	struct xe_gt_idle *gtidle = &gt->gtidle;
-	unsigned int fw_ref;
 	if (IS_SRIOV_VF(gt_to_xe(gt)))
 		return;
 	xe_device_assert_mem_access(gt_to_xe(gt));
 	gtidle->powergate_enable = 0;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 	xe_mmio_write32(&gt->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }
 /**
 	enum xe_gt_idle_state state;
 	u32 pg_enabled, pg_status = 0;
 	u32 vcs_mask, vecs_mask;
-	unsigned int fw_ref;
 	int n;
 	/*
 	 * Media Slices
 	/* Do not wake the GT to read powergating status */
 	if (state != GT_IDLE_C6) {
-		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-		if (!fw_ref)
+		if (!fw_ref.domains)
 			return -ETIMEDOUT;
 		pg_enabled = xe_mmio_read32(&gt->mmio, POWERGATE_ENABLE);
 		pg_status = xe_mmio_read32(&gt->mmio, POWERGATE_DOMAIN_STATUS);
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	}
 	if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) {
 	struct device *dev = kobj_to_dev(kobj);
 	struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
 	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
-	ssize_t ret;
-	xe_pm_runtime_get(pc_to_xe(pc));
-	ret = sysfs_emit(buff, "%s\n", gtidle->name);
-	xe_pm_runtime_put(pc_to_xe(pc));
-	return ret;
+	guard(xe_pm_runtime)(pc_to_xe(pc));
+	return sysfs_emit(buff, "%s\n", gtidle->name);
 }
 static struct kobj_attribute name_attr = __ATTR_RO(name);
 	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
 	enum xe_gt_idle_state state;
-	xe_pm_runtime_get(pc_to_xe(pc));
+	scoped_guard(xe_pm_runtime, pc_to_xe(pc))
-	state = gtidle->idle_status(pc);
+		state = gtidle->idle_status(pc);
-	xe_pm_runtime_put(pc_to_xe(pc));
 	return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
 }
 	struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
 	u64 residency;
-	xe_pm_runtime_get(pc_to_xe(pc));
+	scoped_guard(xe_pm_runtime, pc_to_xe(pc))
-	residency = xe_gt_idle_residency_msec(gtidle);
+		residency = xe_gt_idle_residency_msec(gtidle);
-	xe_pm_runtime_put(pc_to_xe(pc));
 	return sysfs_emit(buff, "%llu\n", residency);
 }
 int xe_gt_idle_disable_c6(struct xe_gt *gt)
 {
-	unsigned int fw_ref;
 	xe_device_assert_mem_access(gt_to_xe(gt));
 	if (IS_SRIOV_VF(gt_to_xe(gt)))
 		return 0;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	xe_mmio_write32(&gt->mmio, RC_CONTROL, 0);
 	xe_mmio_write32(&gt->mmio, RC_STATE, 0);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }

drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c+12−7

 }
 /* Return: number of configuration dwords written */
-static u32 encode_config(u32 *cfg, const struct xe_gt_sriov_config *config, bool details)
+static u32 encode_config(struct xe_gt *gt, u32 *cfg, const struct xe_gt_sriov_config *config,
+			 bool details)
 {
 	u32 n = 0;
 	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_PREEMPT_TIMEOUT);
 	cfg[n++] = config->preempt_timeout;
-#define encode_threshold_config(TAG, ...) ({					\
+#define encode_threshold_config(TAG, NAME, VER...) ({					\
-	cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG);			\
+	if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), true, VER)) {		\
-	cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)];	\
+		cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_THRESHOLD_##TAG);			\
+		cfg[n++] = config->thresholds[MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG)];	\
+	}										\
 });
 	MAKE_XE_GUC_KLV_THRESHOLDS_SET(encode_threshold_config);
 		return -ENOBUFS;
 	cfg = xe_guc_buf_cpu_ptr(buf);
-	num_dwords = encode_config(cfg, config, true);
+	num_dwords = encode_config(gt, cfg, config, true);
 	xe_gt_assert(gt, num_dwords <= max_cfg_dwords);
 	if (xe_gt_is_media_type(gt)) {
 			ret = -ENOBUFS;
 		} else {
 			config = pf_pick_vf_config(gt, vfid);
-			ret = encode_config(buf, config, false) * sizeof(u32);
+			ret = encode_config(gt, buf, config, false) * sizeof(u32);
 		}
 	}
 	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
 		return pf_provision_preempt_timeout(gt, vfid, value[0]);
 	/* auto-generate case statements */
-#define define_threshold_key_to_provision_case(TAG, ...)				\
+#define define_threshold_key_to_provision_case(TAG, NAME, VER...)			\
 	case MAKE_GUC_KLV_VF_CFG_THRESHOLD_KEY(TAG):					\
 		BUILD_BUG_ON(MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG) != 1u);		\
 		if (len != MAKE_GUC_KLV_VF_CFG_THRESHOLD_LEN(TAG))			\
 			return -EBADMSG;						\
+		if (IF_ARGS(!GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), false, VER))	\
+			return -EKEYREJECTED;						\
 		return pf_provision_threshold(gt, vfid,					\
 					      MAKE_XE_GUC_KLV_THRESHOLD_INDEX(TAG),	\
 					      value[0]);

drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c+10−11

 #include "xe_gt_sriov_pf_monitor.h"
 #include "xe_gt_sriov_pf_policy.h"
 #include "xe_gt_sriov_pf_service.h"
+#include "xe_guc.h"
 #include "xe_pm.h"
 #include "xe_sriov_pf.h"
 #include "xe_sriov_pf_provision.h"
 	if (val > (TYPE)~0ull)							\
 		return -EOVERFLOW;						\
 										\
-	xe_pm_runtime_get(xe);							\
+	guard(xe_pm_runtime)(xe);							\
 	err = xe_gt_sriov_pf_policy_set_##POLICY(gt, val);			\
 	if (!err)								\
 		xe_sriov_pf_provision_set_custom_mode(xe);			\
-	xe_pm_runtime_put(xe);							\
 										\
 	return err;								\
 }										\
 	if (val > (TYPE)~0ull)							\
 		return -EOVERFLOW;						\
 										\
-	xe_pm_runtime_get(xe);							\
+	guard(xe_pm_runtime)(xe);							\
 	err = xe_sriov_pf_wait_ready(xe) ?:					\
 	      xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val);		\
 	if (!err)								\
 		xe_sriov_pf_provision_set_custom_mode(xe);			\
-	xe_pm_runtime_put(xe);							\
 										\
 	return err;								\
 }										\
 	if (val > (u32)~0ull)
 		return -EOVERFLOW;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	err = xe_gt_sriov_pf_config_set_threshold(gt, vfid, index, val);
 	if (!err)
 		xe_sriov_pf_provision_set_custom_mode(xe);
-	xe_pm_runtime_put(xe);
 	return err;
 }
 				   &sched_priority_fops);
 	/* register all threshold attributes */
-#define register_threshold_attribute(TAG, NAME, ...) \
+#define register_threshold_attribute(TAG, NAME, VER...) ({				\
-	debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent, \
+	if (IF_ARGS(GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, VER), true, VER))		\
-				   &NAME##_fops);
+		debugfs_create_file_unsafe("threshold_" #NAME, 0644, parent, parent,	\
+					   &NAME##_fops);				\
+});
 	MAKE_XE_GUC_KLV_THRESHOLDS_SET(register_threshold_attribute)
 #undef register_threshold_attribute
 }
 		xe_gt_assert(gt, sizeof(cmd) > strlen(control_cmds[n].cmd));
 		if (sysfs_streq(cmd, control_cmds[n].cmd)) {
-			xe_pm_runtime_get(xe);
+			guard(xe_pm_runtime)(xe);
 			ret = control_cmds[n].fn ? (*control_cmds[n].fn)(gt, vfid) : 0;
-			xe_pm_runtime_put(xe);
 			break;
 		}
 	}

drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c+1−1

@@ -1026,7 +1026,7 @@ static void action_ring_cleanup(void *arg) · 1025 unmodified lines
1026		1026
1027	static void pf_gt_migration_check_support(struct xe_gt *gt)	1027	static void pf_gt_migration_check_support(struct xe_gt *gt)
1028	{	1028	{
1029	if (GUC_FIRMWARE_VER(&gt->uc.guc) < MAKE_GUC_VER(70, 54, 0))	1029	if (!GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 54))
1030	xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0");	1030	xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0");
1031	}	1031	}
1032		1032

drivers/gpu/drm/xe/xe_gt_sriov_vf.c+126−44

@@ -5,6 +5,7 @@ · 4 unmodified lines
5		5
6	#include <linux/bitfield.h>	6	#include <linux/bitfield.h>
7	#include <linux/bsearch.h>	7	#include <linux/bsearch.h>
		8	#include <linux/delay.h>
8		9
9	#include <drm/drm_managed.h>	10	#include <drm/drm_managed.h>
10	#include <drm/drm_print.h>	11	#include <drm/drm_print.h>
@@ -41,6 +42,37 @@ · 30 unmodified lines
41		42
42	#define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 \| (u32)(lo)))	43	#define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 \| (u32)(lo)))
43		44
		45	#ifdef CONFIG_DRM_XE_DEBUG
		46	enum VF_MIGRATION_WAIT_POINTS {
		47	VF_MIGRATION_WAIT_RESFIX_START = BIT(0),
		48	VF_MIGRATION_WAIT_FIXUPS = BIT(1),
		49	VF_MIGRATION_WAIT_RESTART_JOBS = BIT(2),
		50	VF_MIGRATION_WAIT_RESFIX_DONE = BIT(3),
		51	};
		52
		53	#define VF_MIGRATION_WAIT_DELAY_IN_MS 1000
		54	static void vf_post_migration_inject_wait(struct xe_gt *gt,
		55	enum VF_MIGRATION_WAIT_POINTS wait)
		56	{
		57	while (gt->sriov.vf.migration.debug.resfix_stoppers & wait) {
		58	xe_gt_dbg(gt,
		59	"TESTING injecting %u ms delay due to resfix_stoppers=%#x, to continue clear %#x\n",
		60	VF_MIGRATION_WAIT_DELAY_IN_MS,
		61	gt->sriov.vf.migration.debug.resfix_stoppers, wait);
		62
		63	msleep(VF_MIGRATION_WAIT_DELAY_IN_MS);
		64	}
		65	}
		66
		67	#define VF_MIGRATION_INJECT_WAIT(gt, _POS) ({ \
		68	struct xe_gt *__gt = (gt); \
		69	vf_post_migration_inject_wait(__gt, VF_MIGRATION_WAIT_##_POS); \
		70	})
		71
		72	#else
		73	#define VF_MIGRATION_INJECT_WAIT(_gt, ...) typecheck(struct xe_gt *, (_gt))
		74	#endif
		75
44	static int guc_action_vf_reset(struct xe_guc *guc)	76	static int guc_action_vf_reset(struct xe_guc *guc)
45	{	77	{
46	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {	78	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
@@ -299,12 +331,13 @@ void xe_gt_sriov_vf_guc_versions(struct xe_gt *gt, · 252 unmodified lines
299	*found = gt->sriov.vf.guc_version;	331	*found = gt->sriov.vf.guc_version;
300	}	332	}
301		333
302	static int guc_action_vf_notify_resfix_done(struct xe_guc *guc)	334	static int guc_action_vf_resfix_start(struct xe_guc *guc, u16 marker)
303	{	335	{
304	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {	336	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
305	FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) \|	337	FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) \|
306	FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) \|	338	FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) \|
307	FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_NOTIFY_RESFIX_DONE),	339	FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_START) \|
		340	FIELD_PREP(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER, marker),
308	};	341	};
309	int ret;	342	int ret;
310		343
@@ -313,28 +346,43 @@ static int guc_action_vf_notify_resfix_done(struct xe_guc *guc) · 2 unmodified lines
313	return ret > 0 ? -EPROTO : ret;	346	return ret > 0 ? -EPROTO : ret;
314	}	347	}
315		348
316	/**	349	static int vf_resfix_start(struct xe_gt *gt, u16 marker)
317	* vf_notify_resfix_done - Notify GuC about resource fixups apply completed.
318	* @gt: the &xe_gt struct instance linked to target GuC
319	*
320	* Returns: 0 if the operation completed successfully, or a negative error
321	* code otherwise.
322	*/
323	static int vf_notify_resfix_done(struct xe_gt *gt)
324	{	350	{
325	struct xe_guc *guc = &gt->uc.guc;	351	struct xe_guc *guc = &gt->uc.guc;
326	int err;
327		352
328	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));	353	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
329		354
330	err = guc_action_vf_notify_resfix_done(guc);	355	VF_MIGRATION_INJECT_WAIT(gt, RESFIX_START);
331	if (unlikely(err))
332	xe_gt_sriov_err(gt, "Failed to notify GuC about resource fixup done (%pe)\n",
333	ERR_PTR(err));
334	else
335	xe_gt_sriov_dbg_verbose(gt, "sent GuC resource fixup done\n");
336		356
337	return err;	357	xe_gt_sriov_dbg_verbose(gt, "Sending resfix start marker %u\n", marker);
		358
		359	return guc_action_vf_resfix_start(guc, marker);
		360	}
		361
		362	static int guc_action_vf_resfix_done(struct xe_guc *guc, u16 marker)
		363	{
		364	u32 request[GUC_HXG_REQUEST_MSG_MIN_LEN] = {
		365	FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) \|
		366	FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) \|
		367	FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_VF2GUC_RESFIX_DONE) \|
		368	FIELD_PREP(VF2GUC_RESFIX_DONE_REQUEST_MSG_0_MARKER, marker),
		369	};
		370	int ret;
		371
		372	ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request));
		373
		374	return ret > 0 ? -EPROTO : ret;
		375	}
		376
		377	static int vf_resfix_done(struct xe_gt *gt, u16 marker)
		378	{
		379	struct xe_guc *guc = &gt->uc.guc;
		380
		381	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
		382
		383	xe_gt_sriov_dbg_verbose(gt, "Sending resfix done marker %u\n", marker);
		384
		385	return guc_action_vf_resfix_done(guc, marker);
338	}	386	}
339		387
340	static int guc_action_query_single_klv(struct xe_guc *guc, u32 key,	388	static int guc_action_query_single_klv(struct xe_guc *guc, u32 key,
@@ -1123,12 +1171,8 @@ static bool vf_post_migration_shutdown(struct xe_gt *gt) · 782 unmodified lines
1123	return true;	1171	return true;
1124	}	1172	}
1125		1173
1126	spin_lock_irq(&gt->sriov.vf.migration.lock);
1127	gt->sriov.vf.migration.recovery_queued = false;
1128	spin_unlock_irq(&gt->sriov.vf.migration.lock);
1129
1130	xe_guc_ct_flush_and_stop(&gt->uc.guc.ct);	1174	xe_guc_ct_flush_and_stop(&gt->uc.guc.ct);
1131	xe_guc_submit_pause(&gt->uc.guc);	1175	xe_guc_submit_pause_vf(&gt->uc.guc);
1132	xe_tlb_inval_reset(&gt->tlb_inval);	1176	xe_tlb_inval_reset(&gt->tlb_inval);
1133		1177
1134	return false;	1178	return false;
@@ -1144,6 +1188,8 @@ static int vf_post_migration_fixups(struct xe_gt *gt) · 9 unmodified lines
1144	void *buf = gt->sriov.vf.migration.scratch;	1188	void *buf = gt->sriov.vf.migration.scratch;
1145	int err;	1189	int err;
1146		1190
		1191	VF_MIGRATION_INJECT_WAIT(gt, FIXUPS);
		1192
1147	/* xe_gt_sriov_vf_query_config will fixup the GGTT addresses */	1193	/* xe_gt_sriov_vf_query_config will fixup the GGTT addresses */
1148	err = xe_gt_sriov_vf_query_config(gt);	1194	err = xe_gt_sriov_vf_query_config(gt);
1149	if (err)	1195	if (err)
@@ -1162,13 +1208,22 @@ static int vf_post_migration_fixups(struct xe_gt *gt) · 12 unmodified lines
1162		1208
1163	static void vf_post_migration_rearm(struct xe_gt *gt)	1209	static void vf_post_migration_rearm(struct xe_gt *gt)
1164	{	1210	{
		1211	VF_MIGRATION_INJECT_WAIT(gt, RESTART_JOBS);
		1212
		1213	/*
		1214	* Make sure interrupts on the new HW are properly set. The GuC IRQ
		1215	* must be working at this point, since the recovery did started,
		1216	* but the rest was not enabled using the procedure from spec.
		1217	*/
		1218	xe_irq_resume(gt_to_xe(gt));
		1219
1165	xe_guc_ct_restart(&gt->uc.guc.ct);	1220	xe_guc_ct_restart(&gt->uc.guc.ct);
1166	xe_guc_submit_unpause_prepare(&gt->uc.guc);	1221	xe_guc_submit_unpause_prepare_vf(&gt->uc.guc);
1167	}	1222	}
1168		1223
1169	static void vf_post_migration_kickstart(struct xe_gt *gt)	1224	static void vf_post_migration_kickstart(struct xe_gt *gt)
1170	{	1225	{
1171	xe_guc_submit_unpause(&gt->uc.guc);	1226	xe_guc_submit_unpause_vf(&gt->uc.guc);
1172	}	1227	}
1173		1228
1174	static void vf_post_migration_abort(struct xe_gt *gt)	1229	static void vf_post_migration_abort(struct xe_gt *gt)
@@ -1183,37 +1238,49 @@ static void vf_post_migration_abort(struct xe_gt *gt) · 8 unmodified lines
1183	xe_guc_submit_pause_abort(&gt->uc.guc);	1238	xe_guc_submit_pause_abort(&gt->uc.guc);
1184	}	1239	}
1185		1240
1186	static int vf_post_migration_notify_resfix_done(struct xe_gt *gt)	1241	static int vf_post_migration_resfix_done(struct xe_gt *gt, u16 marker)
1187	{	1242	{
1188	bool skip_resfix = false;	1243	VF_MIGRATION_INJECT_WAIT(gt, RESFIX_DONE);
1189		1244
1190	spin_lock_irq(&gt->sriov.vf.migration.lock);	1245	spin_lock_irq(&gt->sriov.vf.migration.lock);
1191	if (gt->sriov.vf.migration.recovery_queued) {	1246	if (gt->sriov.vf.migration.recovery_queued)
1192	skip_resfix = true;	1247	xe_gt_sriov_dbg(gt, "another recovery imminent\n");
1193	xe_gt_sriov_dbg(gt, "another recovery imminent, resfix skipped\n");	1248	else
1194	} else {
1195	WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, false);	1249	WRITE_ONCE(gt->sriov.vf.migration.recovery_inprogress, false);
1196	}
1197	spin_unlock_irq(&gt->sriov.vf.migration.lock);	1250	spin_unlock_irq(&gt->sriov.vf.migration.lock);
1198		1251
1199	if (skip_resfix)	1252	return vf_resfix_done(gt, marker);
1200	return -EAGAIN;	1253	}
1201		1254
1202	/*	1255	static int vf_post_migration_resfix_start(struct xe_gt *gt, u16 marker)
1203	* Make sure interrupts on the new HW are properly set. The GuC IRQ	1256	{
1204	* must be working at this point, since the recovery did started,	1257	int err;
1205	* but the rest was not enabled using the procedure from spec.	1258
1206	*/	1259	err = vf_resfix_start(gt, marker);
1207	xe_irq_resume(gt_to_xe(gt));
1208		1260
1209	return vf_notify_resfix_done(gt);	1261	guard(spinlock_irq) (&gt->sriov.vf.migration.lock);
		1262	gt->sriov.vf.migration.recovery_queued = false;
		1263
		1264	return err;
		1265	}
		1266
		1267	static u16 vf_post_migration_next_resfix_marker(struct xe_gt *gt)
		1268	{
		1269	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
		1270
		1271	BUILD_BUG_ON(1 + ((typeof(gt->sriov.vf.migration.resfix_marker))~0) >
		1272	FIELD_MAX(VF2GUC_RESFIX_START_REQUEST_MSG_0_MARKER));
		1273
		1274	/* add 1 to avoid zero-marker */
		1275	return 1 + gt->sriov.vf.migration.resfix_marker++;
1210	}	1276	}
1211		1277
1212	static void vf_post_migration_recovery(struct xe_gt *gt)	1278	static void vf_post_migration_recovery(struct xe_gt *gt)
1213	{	1279	{
1214	struct xe_device *xe = gt_to_xe(gt);	1280	struct xe_device *xe = gt_to_xe(gt);
1215	int err;	1281	u16 marker;
1216	bool retry;	1282	bool retry;
		1283	int err;
1217		1284
1218	xe_gt_sriov_dbg(gt, "migration recovery in progress\n");	1285	xe_gt_sriov_dbg(gt, "migration recovery in progress\n");
1219		1286
@@ -1227,15 +1294,30 @@ static void vf_post_migration_recovery(struct xe_gt *gt) · 7 unmodified lines
1227	goto fail;	1294	goto fail;
1228	}	1295	}
1229		1296
		1297	marker = vf_post_migration_next_resfix_marker(gt);
		1298
		1299	err = vf_post_migration_resfix_start(gt, marker);
		1300	if (unlikely(err)) {
		1301	xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_START step (%pe)\n",
		1302	ERR_PTR(err));
		1303	goto fail;
		1304	}
		1305
1230	err = vf_post_migration_fixups(gt);	1306	err = vf_post_migration_fixups(gt);
1231	if (err)	1307	if (err)
1232	goto fail;	1308	goto fail;
1233		1309
1234	vf_post_migration_rearm(gt);	1310	vf_post_migration_rearm(gt);
1235		1311
1236	err = vf_post_migration_notify_resfix_done(gt);	1312	err = vf_post_migration_resfix_done(gt, marker);
1237	if (err && err != -EAGAIN)	1313	if (err) {
		1314	if (err == -EREMCHG)
		1315	goto queue;
		1316
		1317	xe_gt_sriov_err(gt, "Recovery failed at GuC RESFIX_DONE step (%pe)\n",
		1318	ERR_PTR(err));
1238	goto fail;	1319	goto fail;
		1320	}
1239		1321
1240	vf_post_migration_kickstart(gt);	1322	vf_post_migration_kickstart(gt);
1241		1323

drivers/gpu/drm/xe/xe_gt_sriov_vf_debugfs.c+12−0

@@ -69,4 +69,16 @@ void xe_gt_sriov_vf_debugfs_register(struct xe_gt gt, struct dentry root) · 68 unmodified lines
69	vfdentry->d_inode->i_private = gt;	69	vfdentry->d_inode->i_private = gt;
70		70
71	drm_debugfs_create_files(vf_info, ARRAY_SIZE(vf_info), vfdentry, minor);	71	drm_debugfs_create_files(vf_info, ARRAY_SIZE(vf_info), vfdentry, minor);
		72
		73	/*
		74	* /sys/kernel/debug/dri/BDF/
		75	* ├── tile0
		76	* ├── gt0
		77	* ├── vf
		78	* ├── resfix_stoppers
		79	*/
		80	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
		81	debugfs_create_x8("resfix_stoppers", 0600, vfdentry,
		82	&gt->sriov.vf.migration.debug.resfix_stoppers);
		83	}
72	}	84	}

drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h+13−0

@@ -52,6 +52,19 @@ struct xe_gt_sriov_vf_migration { · 51 unmodified lines
52	wait_queue_head_t wq;	52	wait_queue_head_t wq;
53	/** @scratch: Scratch memory for VF recovery */	53	/** @scratch: Scratch memory for VF recovery */
54	void *scratch;	54	void *scratch;
		55	/** @debug: Debug hooks for delaying migration */
		56	struct {
		57	/**
		58	* @debug.resfix_stoppers: Stop and wait at different stages
		59	* during post migration recovery
		60	*/
		61	u8 resfix_stoppers;
		62	} debug;
		63	/**
		64	* @resfix_marker: Marker sent on start and on end of post-migration
		65	* steps.
		66	*/
		67	u8 resfix_marker;
55	/** @recovery_teardown: VF post migration recovery is being torn down */	68	/** @recovery_teardown: VF post migration recovery is being torn down */
56	bool recovery_teardown;	69	bool recovery_teardown;
57	/** @recovery_queued: VF post migration recovery in queued */	70	/** @recovery_queued: VF post migration recovery in queued */

drivers/gpu/drm/xe/xe_gt_stats.c+10−0

@@ -66,6 +66,16 @@ static const char *const stat_description[__XE_GT_STATS_NUM_IDS] = { · 65 unmodified lines
66	DEF_STAT_STR(SVM_4K_BIND_US, "svm_4K_bind_us"),	66	DEF_STAT_STR(SVM_4K_BIND_US, "svm_4K_bind_us"),
67	DEF_STAT_STR(SVM_64K_BIND_US, "svm_64K_bind_us"),	67	DEF_STAT_STR(SVM_64K_BIND_US, "svm_64K_bind_us"),
68	DEF_STAT_STR(SVM_2M_BIND_US, "svm_2M_bind_us"),	68	DEF_STAT_STR(SVM_2M_BIND_US, "svm_2M_bind_us"),
		69	DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT,
		70	"hw_engine_group_suspend_lr_queue_count"),
		71	DEF_STAT_STR(HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT,
		72	"hw_engine_group_skip_lr_queue_count"),
		73	DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT,
		74	"hw_engine_group_wait_dma_queue_count"),
		75	DEF_STAT_STR(HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
		76	"hw_engine_group_suspend_lr_queue_us"),
		77	DEF_STAT_STR(HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
		78	"hw_engine_group_wait_dma_queue_us"),
69	};	79	};
70		80
71	/**	81	/**

drivers/gpu/drm/xe/xe_gt_stats.h+32−0

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	#ifndef _XE_GT_STATS_H_	6	#ifndef _XE_GT_STATS_H_
7	#define _XE_GT_STATS_H_	7	#define _XE_GT_STATS_H_
8		8
		9	#include <linux/ktime.h>
		10
9	#include "xe_gt_stats_types.h"	11	#include "xe_gt_stats_types.h"
10		12
11	struct xe_gt;	13	struct xe_gt;
@@ -23,4 +25,34 @@ xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, · 11 unmodified lines
23	}	25	}
24		26
25	#endif	27	#endif
		28
		29	/**
		30	* xe_gt_stats_ktime_us_delta() - Get delta in microseconds between now and a
		31	* start time
		32	* @start: Start time
		33	*
		34	* Helper for GT stats to get delta in microseconds between now and a start
		35	* time, compiles out if GT stats are disabled.
		36	*
		37	* Return: Delta in microseconds between now and a start time
		38	*/
		39	static inline s64 xe_gt_stats_ktime_us_delta(ktime_t start)
		40	{
		41	return IS_ENABLED(CONFIG_DEBUG_FS) ?
		42	ktime_us_delta(ktime_get(), start) : 0;
		43	}
		44
		45	/**
		46	* xe_gt_stats_ktime_get() - Get current ktime
		47	*
		48	* Helper for GT stats to get current ktime, compiles out if GT stats are
		49	* disabled.
		50	*
		51	* Return: Get current ktime
		52	*/
		53	static inline ktime_t xe_gt_stats_ktime_get(void)
		54	{
		55	return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0;
		56	}
		57
26	#endif	58	#endif

drivers/gpu/drm/xe/xe_gt_stats_types.h+5−0

 	XE_GT_STATS_ID_SVM_4K_BIND_US,
 	XE_GT_STATS_ID_SVM_64K_BIND_US,
 	XE_GT_STATS_ID_SVM_2M_BIND_US,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
+	XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
 	/* must be the last entry */
 	__XE_GT_STATS_NUM_IDS,
 };

drivers/gpu/drm/xe/xe_gt_throttle.c+3−6

 {
 	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_reg reg;
-	u32 val, mask;
+	u32 mask;
 	if (xe_gt_is_media_type(gt))
 		reg = MTL_MEDIA_PERF_LIMIT_REASONS;
 	else
 		mask = GT0_PERF_LIMIT_REASONS_MASK;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
-	val = xe_mmio_read32(&gt->mmio, reg) & mask;
+	return xe_mmio_read32(&gt->mmio, reg) & mask;
-	xe_pm_runtime_put(xe);
-	return val;
 }
 static bool is_throttled_by(struct xe_gt *gt, u32 mask)

drivers/gpu/drm/xe/xe_gt_types.h+5−0

 		u64 engine_mask;
 		/** @info.gmdid: raw GMD_ID value from hardware */
 		u32 gmdid;
+		/**
+		 * @multi_queue_engine_class_mask: Bitmask of engine classes with
+		 * multi queue support enabled.
+		 */
+		u16 multi_queue_engine_class_mask;
 		/** @info.id: Unique ID of this GT within the PCI Device */
 		u8 id;
 		/** @info.has_indirect_ring_state: GT has indirect ring state support */

drivers/gpu/drm/xe/xe_guc.c+60−20

 	u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT;
 	u32 flags;
-	#if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
+	#if (((XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE) % SZ_1M) == 0)
 	#define LOG_UNIT SZ_1M
 	#define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
 	#else
 	#define LOG_FLAG 0
 	#endif
-	#if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
+	#if (((XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
 	#define CAPTURE_UNIT SZ_1M
 	#define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
 	#else
 	#define CAPTURE_FLAG 0
 	#endif
-	BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
+	BUILD_BUG_ON(!XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
+	BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE, LOG_UNIT));
-	BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
+	BUILD_BUG_ON(!XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
+	BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE, LOG_UNIT));
-	BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
+	BUILD_BUG_ON(!XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
+	BUILD_BUG_ON(!IS_ALIGNED(XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
 	flags = GUC_LOG_VALID |
 		GUC_LOG_NOTIFY_ON_HALF_FULL |
 		CAPTURE_FLAG |
 		LOG_FLAG |
-		FIELD_PREP(GUC_LOG_CRASH, CRASH_BUFFER_SIZE / LOG_UNIT - 1) |
+		FIELD_PREP(GUC_LOG_CRASH_DUMP, XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE / LOG_UNIT - 1) |
-		FIELD_PREP(GUC_LOG_DEBUG, DEBUG_BUFFER_SIZE / LOG_UNIT - 1) |
+		FIELD_PREP(GUC_LOG_EVENT_DATA, XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE / LOG_UNIT - 1) |
-		FIELD_PREP(GUC_LOG_CAPTURE, CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) |
+		FIELD_PREP(GUC_LOG_STATE_CAPTURE, XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE /
+			   CAPTURE_UNIT - 1) |
 		FIELD_PREP(GUC_LOG_BUF_ADDR, offset);
 	#undef LOG_UNIT
 {
 	struct xe_guc *guc = arg;
 	struct xe_gt *gt = guc_to_gt(guc);
-	unsigned int fw_ref;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL)
-	xe_uc_sanitize_reset(&guc_to_gt(guc)->uc);
+		xe_uc_sanitize_reset(&guc_to_gt(guc)->uc);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	guc_g2g_fini(guc);
 }
 	if (!xe_uc_fw_is_enabled(&guc->fw))
 		return 0;
+	/* Disable page reclaim if GuC FW does not support */
+	if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 14, 0))
+		xe->info.has_page_reclaim_hw_assist = false;
 	if (IS_SRIOV_VF(xe)) {
 		ret = xe_guc_ct_init(&guc->ct);
 		if (ret)
 		u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header);
 		u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header);
+		if (unlikely(error == XE_GUC_RESPONSE_VF_MIGRATED)) {
+			xe_gt_dbg(gt, "GuC mmio request %#x rejected due to MIGRATION (hint %#x)\n",
+				  request[0], hint);
+			return -EREMCHG;
+		}
 		xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
 			  request[0], error, hint);
 		return -ENXIO;
 	return xe_guc_submit_start(guc);
 }
+/**
+ * xe_guc_runtime_suspend() - GuC runtime suspend
+ * @guc: The GuC object
+ *
+ * Stop further runs of submission tasks on given GuC and runtime suspend
+ * GuC CT.
+ */
+void xe_guc_runtime_suspend(struct xe_guc *guc)
+{
+	xe_guc_submit_pause(guc);
+	xe_guc_submit_disable(guc);
+	xe_guc_ct_runtime_suspend(&guc->ct);
+}
+/**
+ * xe_guc_runtime_resume() - GuC runtime resume
+ * @guc: The GuC object
+ *
+ * Runtime resume GuC CT and allow further runs of submission tasks on
+ * given GuC.
+ */
+void xe_guc_runtime_resume(struct xe_guc *guc)
+{
+	/*
+	 * Runtime PM flows are not applicable for VFs, so it's safe to
+	 * directly enable IRQ.
+	 */
+	guc_enable_irq(guc);
+	xe_guc_ct_runtime_resume(&guc->ct);
+	xe_guc_submit_enable(guc);
+	xe_guc_submit_unpause(guc);
+}
 void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p)
 {
 	struct xe_gt *gt = guc_to_gt(guc);
-	unsigned int fw_ref;
 	u32 status;
 	int i;
 	xe_uc_fw_print(&guc->fw, p);
 	if (!IS_SRIOV_VF(gt_to_xe(gt))) {
-		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-		if (!fw_ref)
+		if (!fw_ref.domains)
 			return;
 		status = xe_mmio_read32(&gt->mmio, GUC_STATUS);
 			drm_printf(p, "\t%2d: \t0x%x\n",
 				   i, xe_mmio_read32(&gt->mmio, SOFT_SCRATCH(i)));
 		}
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	}
 	drm_puts(p, "\n");

drivers/gpu/drm/xe/xe_guc.h+23−0

@@ -18,10 +18,16 @@ · 17 unmodified lines
18	*/	18	*/
19	#define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) \| ((min) << 8) \| (pat))	19	#define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) \| ((min) << 8) \| (pat))
20	#define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)	20	#define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
		21	#define MAKE_GUC_VER_ARGS(ver...) \
		22	(BUILD_BUG_ON_ZERO(COUNT_ARGS(ver) < 2 \|\| COUNT_ARGS(ver) > 3) + \
		23	MAKE_GUC_VER(PICK_ARG1(ver), PICK_ARG2(ver), IF_ARGS(PICK_ARG3(ver), 0, PICK_ARG3(ver))))
		24
21	#define GUC_SUBMIT_VER(guc) \	25	#define GUC_SUBMIT_VER(guc) \
22	MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY])	26	MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY])
23	#define GUC_FIRMWARE_VER(guc) \	27	#define GUC_FIRMWARE_VER(guc) \
24	MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_RELEASE])	28	MAKE_GUC_VER_STRUCT((guc)->fw.versions.found[XE_UC_FW_VER_RELEASE])
		29	#define GUC_FIRMWARE_VER_AT_LEAST(guc, ver...) \
		30	xe_guc_fw_version_at_least((guc), MAKE_GUC_VER_ARGS(ver))
25		31
26	struct drm_printer;	32	struct drm_printer;
27		33
@@ -35,6 +41,8 @@ int xe_guc_upload(struct xe_guc *guc); · 7 unmodified lines
35	int xe_guc_min_load_for_hwconfig(struct xe_guc *guc);	41	int xe_guc_min_load_for_hwconfig(struct xe_guc *guc);
36	int xe_guc_enable_communication(struct xe_guc *guc);	42	int xe_guc_enable_communication(struct xe_guc *guc);
37	int xe_guc_opt_in_features_enable(struct xe_guc *guc);	43	int xe_guc_opt_in_features_enable(struct xe_guc *guc);
		44	void xe_guc_runtime_suspend(struct xe_guc *guc);
		45	void xe_guc_runtime_resume(struct xe_guc *guc);
38	int xe_guc_suspend(struct xe_guc *guc);	46	int xe_guc_suspend(struct xe_guc *guc);
39	void xe_guc_notify(struct xe_guc *guc);	47	void xe_guc_notify(struct xe_guc *guc);
40	int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);	48	int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);
@@ -94,4 +102,19 @@ static inline struct drm_device guc_to_drm(struct xe_guc guc) · 53 unmodified lines
94	return &guc_to_xe(guc)->drm;	102	return &guc_to_xe(guc)->drm;
95	}	103	}
96		104
		105	/**
		106	* xe_guc_fw_version_at_least() - Check if GuC is at least of given version.
		107	* @guc: the &xe_guc
		108	* @ver: the version to check
		109	*
		110	* The @ver should be prepared using MAKE_GUC_VER(major, minor, patch).
		111	*
		112	* Return: true if loaded GuC firmware is at least of given version,
		113	* false otherwise.
		114	*/
		115	static inline bool xe_guc_fw_version_at_least(const struct xe_guc *guc, u32 ver)
		116	{
		117	return GUC_FIRMWARE_VER(guc) >= ver;
		118	}
		119
97	#endif	120	#endif

drivers/gpu/drm/xe/xe_guc_ads.c+3−3

drivers/gpu/drm/xe/xe_guc_buf.c+1−1

@@ -30,7 +30,7 @@ static int guc_buf_cache_init(struct xe_guc_buf_cache *cache, u32 size) · 29 unmodified lines
30	struct xe_gt *gt = cache_to_gt(cache);	30	struct xe_gt *gt = cache_to_gt(cache);
31	struct xe_sa_manager *sam;	31	struct xe_sa_manager *sam;
32		32
33	sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32));	33	sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32), 0);
34	if (IS_ERR(sam))	34	if (IS_ERR(sam))
35	return PTR_ERR(sam);	35	return PTR_ERR(sam);
36	cache->sam = sam;	36	cache->sam = sam;

drivers/gpu/drm/xe/xe_guc_capture.c+8−8

@@ -843,7 +843,7 @@ static void check_guc_capture_size(struct xe_guc *guc) · 842 unmodified lines
843	{	843	{
844	int capture_size = guc_capture_output_size_est(guc);	844	int capture_size = guc_capture_output_size_est(guc);
845	int spare_size = capture_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER;	845	int spare_size = capture_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER;
846	u32 buffer_size = xe_guc_log_section_size_capture(&guc->log);	846	u32 buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE;
847		847
848	/*	848	/*
849	* NOTE: capture_size is much smaller than the capture region	849	* NOTE: capture_size is much smaller than the capture region
@@ -949,7 +949,7 @@ guc_capture_init_node(struct xe_guc guc, struct __guc_capture_parsed_output no · 99 unmodified lines
949	* ADS module also calls separately for PF vs VF.	949	* ADS module also calls separately for PF vs VF.
950	*	950	*
951	* --> alloc B: GuC output capture buf (registered via guc_init_params(log_param))	951	* --> alloc B: GuC output capture buf (registered via guc_init_params(log_param))
952	* Size = #define CAPTURE_BUFFER_SIZE (warns if on too-small)	952	* Size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE (warns if on too-small)
953	* Note2: 'x 3' to hold multiple capture groups	953	* Note2: 'x 3' to hold multiple capture groups
954	*	954	*
955	* GUC Runtime notify capture:	955	* GUC Runtime notify capture:
@@ -1367,7 +1367,7 @@ static int __guc_capture_flushlog_complete(struct xe_guc *guc) · 411 unmodified lines
1367	{	1367	{
1368	u32 action[] = {	1368	u32 action[] = {
1369	XE_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE,	1369	XE_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE,
1370	GUC_LOG_BUFFER_CAPTURE	1370	GUC_LOG_TYPE_STATE_CAPTURE
1371	};	1371	};
1372		1372
1373	return xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));	1373	return xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));
@@ -1384,8 +1384,8 @@ static void __guc_capture_process_output(struct xe_guc *guc) · 10 unmodified lines
1384	u32 log_buf_state_offset;	1384	u32 log_buf_state_offset;
1385	u32 src_data_offset;	1385	u32 src_data_offset;
1386		1386
1387	log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_BUFFER_CAPTURE;	1387	log_buf_state_offset = sizeof(struct guc_log_buffer_state) * GUC_LOG_TYPE_STATE_CAPTURE;
1388	src_data_offset = xe_guc_get_log_buffer_offset(&guc->log, GUC_LOG_BUFFER_CAPTURE);	1388	src_data_offset = XE_GUC_LOG_STATE_CAPTURE_OFFSET;
1389		1389
1390	/*	1390	/*
1391	* Make a copy of the state structure, inside GuC log buffer	1391	* Make a copy of the state structure, inside GuC log buffer
@@ -1395,15 +1395,15 @@ static void __guc_capture_process_output(struct xe_guc *guc) · 3 unmodified lines
1395	xe_map_memcpy_from(guc_to_xe(guc), &log_buf_state_local, &guc->log.bo->vmap,	1395	xe_map_memcpy_from(guc_to_xe(guc), &log_buf_state_local, &guc->log.bo->vmap,
1396	log_buf_state_offset, sizeof(struct guc_log_buffer_state));	1396	log_buf_state_offset, sizeof(struct guc_log_buffer_state));
1397		1397
1398	buffer_size = xe_guc_get_log_buffer_size(&guc->log, GUC_LOG_BUFFER_CAPTURE);	1398	buffer_size = XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE;
1399	read_offset = log_buf_state_local.read_ptr;	1399	read_offset = log_buf_state_local.read_ptr;
1400	write_offset = log_buf_state_local.sampled_write_ptr;	1400	write_offset = log_buf_state_local.sampled_write_ptr;
1401	full_count = FIELD_GET(GUC_LOG_BUFFER_STATE_BUFFER_FULL_CNT, log_buf_state_local.flags);	1401	full_count = FIELD_GET(GUC_LOG_BUFFER_STATE_BUFFER_FULL_CNT, log_buf_state_local.flags);
1402		1402
1403	/* Bookkeeping stuff */	1403	/* Bookkeeping stuff */
1404	tmp = FIELD_GET(GUC_LOG_BUFFER_STATE_FLUSH_TO_FILE, log_buf_state_local.flags);	1404	tmp = FIELD_GET(GUC_LOG_BUFFER_STATE_FLUSH_TO_FILE, log_buf_state_local.flags);
1405	guc->log.stats[GUC_LOG_BUFFER_CAPTURE].flush += tmp;	1405	guc->log.stats[GUC_LOG_TYPE_STATE_CAPTURE].flush += tmp;
1406	new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_BUFFER_CAPTURE,	1406	new_overflow = xe_guc_check_log_buf_overflow(&guc->log, GUC_LOG_TYPE_STATE_CAPTURE,
1407	full_count);	1407	full_count);
1408		1408
1409	/* Now copy the actual logs. */	1409	/* Now copy the actual logs. */

drivers/gpu/drm/xe/xe_guc_ct.c+170−103

@@ -42,6 +42,21 @@ static void ct_exit_safe_mode(struct xe_guc_ct *ct); · 41 unmodified lines
42	static void guc_ct_change_state(struct xe_guc_ct *ct,	42	static void guc_ct_change_state(struct xe_guc_ct *ct,
43	enum xe_guc_ct_state state);	43	enum xe_guc_ct_state state);
44		44
		45	static struct xe_guc ct_to_guc(struct xe_guc_ct ct)
		46	{
		47	return container_of(ct, struct xe_guc, ct);
		48	}
		49
		50	static struct xe_gt ct_to_gt(struct xe_guc_ct ct)
		51	{
		52	return container_of(ct, struct xe_gt, uc.guc.ct);
		53	}
		54
		55	static struct xe_device ct_to_xe(struct xe_guc_ct ct)
		56	{
		57	return gt_to_xe(ct_to_gt(ct));
		58	}
		59
45	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)	60	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
46	enum {	61	enum {
47	/* Internal states, not error conditions */	62	/* Internal states, not error conditions */
@@ -68,14 +83,101 @@ enum { · 20 unmodified lines
68	static void ct_dead_worker_func(struct work_struct *w);	83	static void ct_dead_worker_func(struct work_struct *w);
69	static void ct_dead_capture(struct xe_guc_ct ct, struct guc_ctb ctb, u32 reason_code);	84	static void ct_dead_capture(struct xe_guc_ct ct, struct guc_ctb ctb, u32 reason_code);
70		85
71	#define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)	86	static void ct_dead_fini(struct xe_guc_ct *ct)
		87	{
		88	cancel_work_sync(&ct->dead.worker);
		89	}
		90
		91	static void ct_dead_init(struct xe_guc_ct *ct)
		92	{
		93	spin_lock_init(&ct->dead.lock);
		94	INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
		95
		96	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
		97	stack_depot_init();
		98	#endif
		99	}
		100
		101	static void fast_req_stack_save(struct xe_guc_ct *ct, unsigned int slot)
		102	{
		103	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
		104	unsigned long entries[SZ_32];
		105	unsigned int n;
		106
		107	n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
		108	/* May be called under spinlock, so avoid sleeping */
		109	ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT);
		110	#endif
		111	}
		112
		113	static void fast_req_dump(struct xe_guc_ct *ct, u16 fence, unsigned int slot)
		114	{
		115	struct xe_gt *gt = ct_to_gt(ct);
		116	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
		117	char *buf __cleanup(kfree) = kmalloc(SZ_4K, GFP_NOWAIT);
		118
		119	if (buf && stack_depot_snprint(ct->fast_req[slot].stack, buf, SZ_4K, 0))
		120	xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s\n",
		121	fence, ct->fast_req[slot].action, buf);
		122	else
		123	xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n",
		124	fence, ct->fast_req[slot].action);
		125	#else
		126	xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n",
		127	fence, ct->fast_req[slot].action);
		128	#endif
		129	}
		130
		131	static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
		132	{
		133	u16 fence_min = U16_MAX, fence_max = 0;
		134	struct xe_gt *gt = ct_to_gt(ct);
		135	unsigned int n;
		136
		137	lockdep_assert_held(&ct->lock);
		138
		139	for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) {
		140	if (ct->fast_req[n].fence < fence_min)
		141	fence_min = ct->fast_req[n].fence;
		142	if (ct->fast_req[n].fence > fence_max)
		143	fence_max = ct->fast_req[n].fence;
		144
		145	if (ct->fast_req[n].fence != fence)
		146	continue;
		147
		148	return fast_req_dump(ct, fence, n);
		149	}
		150
		151	xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n",
		152	fence, fence_min, fence_max, ct->fence_seqno);
		153	}
		154
		155	static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
		156	{
		157	unsigned int slot = fence % ARRAY_SIZE(ct->fast_req);
		158
		159	fast_req_stack_save(ct, slot);
		160	ct->fast_req[slot].fence = fence;
		161	ct->fast_req[slot].action = action;
		162	}
		163
		164	#define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code)
		165
72	#else	166	#else
		167
		168	static void ct_dead_fini(struct xe_guc_ct *ct) { }
		169	static void ct_dead_init(struct xe_guc_ct *ct) { }
		170
		171	static void fast_req_report(struct xe_guc_ct *ct, u16 fence) { }
		172	static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { }
		173
73	#define CT_DEAD(ct, ctb, reason) \	174	#define CT_DEAD(ct, ctb, reason) \
74	do { \	175	do { \
75	struct guc_ctb *_ctb = (ctb); \	176	struct guc_ctb *_ctb = (ctb); \
76	if (_ctb) \	177	if (_ctb) \
77	_ctb->info.broken = true; \	178	_ctb->info.broken = true; \
78	} while (0)	179	} while (0)
		180
79	#endif	181	#endif
80		182
81	/* Used when a CT send wants to block and / or receive data */	183	/* Used when a CT send wants to block and / or receive data */
@@ -112,24 +214,6 @@ static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) · 30 unmodified lines
112	return g2h_fence->seqno == ~0x0;	214	return g2h_fence->seqno == ~0x0;
113	}	215	}
114		216
115	static struct xe_guc *
116	ct_to_guc(struct xe_guc_ct *ct)
117	{
118	return container_of(ct, struct xe_guc, ct);
119	}
120
121	static struct xe_gt *
122	ct_to_gt(struct xe_guc_ct *ct)
123	{
124	return container_of(ct, struct xe_gt, uc.guc.ct);
125	}
126
127	static struct xe_device *
128	ct_to_xe(struct xe_guc_ct *ct)
129	{
130	return gt_to_xe(ct_to_gt(ct));
131	}
132
133	/**	217	/**
134	* DOC: GuC CTB Blob	218	* DOC: GuC CTB Blob
135	*	219	*
@@ -169,8 +253,11 @@ ct_to_xe(struct xe_guc_ct *ct) · 33 unmodified lines
169	#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)	253	#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
170	#define CTB_H2G_BUFFER_OFFSET (CTB_DESC_SIZE * 2)	254	#define CTB_H2G_BUFFER_OFFSET (CTB_DESC_SIZE * 2)
171	#define CTB_H2G_BUFFER_SIZE (SZ_4K)	255	#define CTB_H2G_BUFFER_SIZE (SZ_4K)
		256	#define CTB_H2G_BUFFER_DWORDS (CTB_H2G_BUFFER_SIZE / sizeof(u32))
172	#define CTB_G2H_BUFFER_SIZE (SZ_128K)	257	#define CTB_G2H_BUFFER_SIZE (SZ_128K)
		258	#define CTB_G2H_BUFFER_DWORDS (CTB_G2H_BUFFER_SIZE / sizeof(u32))
173	#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2)	259	#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2)
		260	#define G2H_ROOM_BUFFER_DWORDS (CTB_G2H_BUFFER_DWORDS / 2)
174		261
175	/**	262	/**
176	* xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full	263	* xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full
@@ -199,9 +286,7 @@ static void guc_ct_fini(struct drm_device drm, void arg) · 22 unmodified lines
199	{	286	{
200	struct xe_guc_ct *ct = arg;	287	struct xe_guc_ct *ct = arg;
201		288
202	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)	289	ct_dead_fini(ct);
203	cancel_work_sync(&ct->dead.worker);
204	#endif
205	ct_exit_safe_mode(ct);	290	ct_exit_safe_mode(ct);
206	destroy_workqueue(ct->g2h_wq);	291	destroy_workqueue(ct->g2h_wq);
207	xa_destroy(&ct->fence_lookup);	292	xa_destroy(&ct->fence_lookup);
@@ -239,13 +324,8 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) · 31 unmodified lines
239	xa_init(&ct->fence_lookup);	324	xa_init(&ct->fence_lookup);
240	INIT_WORK(&ct->g2h_worker, g2h_worker_func);	325	INIT_WORK(&ct->g2h_worker, g2h_worker_func);
241	INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);	326	INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func);
242	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)	327
243	spin_lock_init(&ct->dead.lock);	328	ct_dead_init(ct);
244	INIT_WORK(&ct->dead.worker, ct_dead_worker_func);
245	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
246	stack_depot_init();
247	#endif
248	#endif
249	init_waitqueue_head(&ct->wq);	329	init_waitqueue_head(&ct->wq);
250	init_waitqueue_head(&ct->g2h_fence_wq);	330	init_waitqueue_head(&ct->g2h_fence_wq);
251		331
@@ -326,7 +406,7 @@ int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct) · 74 unmodified lines
326	static void guc_ct_ctb_h2g_init(struct xe_device xe, struct guc_ctb h2g,	406	static void guc_ct_ctb_h2g_init(struct xe_device xe, struct guc_ctb h2g,
327	struct iosys_map *map)	407	struct iosys_map *map)
328	{	408	{
329	h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);	409	h2g->info.size = CTB_H2G_BUFFER_DWORDS;
330	h2g->info.resv_space = 0;	410	h2g->info.resv_space = 0;
331	h2g->info.tail = 0;	411	h2g->info.tail = 0;
332	h2g->info.head = 0;	412	h2g->info.head = 0;
@@ -344,8 +424,8 @@ static void guc_ct_ctb_h2g_init(struct xe_device xe, struct guc_ctb h2g, · 11 unmodified lines
344	static void guc_ct_ctb_g2h_init(struct xe_device xe, struct guc_ctb g2h,	424	static void guc_ct_ctb_g2h_init(struct xe_device xe, struct guc_ctb g2h,
345	struct iosys_map *map)	425	struct iosys_map *map)
346	{	426	{
347	g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);	427	g2h->info.size = CTB_G2H_BUFFER_DWORDS;
348	g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);	428	g2h->info.resv_space = G2H_ROOM_BUFFER_DWORDS;
349	g2h->info.head = 0;	429	g2h->info.head = 0;
350	g2h->info.tail = 0;	430	g2h->info.tail = 0;
351	g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,	431	g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
@@ -640,6 +720,39 @@ void xe_guc_ct_stop(struct xe_guc_ct *ct) · 288 unmodified lines
640	stop_g2h_handler(ct);	720	stop_g2h_handler(ct);
641	}	721	}
642		722
		723	/**
		724	* xe_guc_ct_runtime_suspend() - GuC CT runtime suspend
		725	* @ct: the &xe_guc_ct
		726	*
		727	* Set GuC CT to disabled state.
		728	*/
		729	void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct)
		730	{
		731	struct guc_ctb *g2h = &ct->ctbs.g2h;
		732	u32 credits = CIRC_SPACE(0, 0, CTB_G2H_BUFFER_DWORDS) - G2H_ROOM_BUFFER_DWORDS;
		733
		734	/* We should be back to guc_ct_ctb_g2h_init() values */
		735	xe_gt_assert(ct_to_gt(ct), g2h->info.space == credits);
		736
		737	/*
		738	* Since we're already in runtime suspend path, we shouldn't have pending
		739	* messages. But if there happen to be any, we'd probably want them to be
		740	* thrown as errors for further investigation.
		741	*/
		742	xe_guc_ct_disable(ct);
		743	}
		744
		745	/**
		746	* xe_guc_ct_runtime_resume() - GuC CT runtime resume
		747	* @ct: the &xe_guc_ct
		748	*
		749	* Restart GuC CT and set it to enabled state.
		750	*/
		751	void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct)
		752	{
		753	xe_guc_ct_restart(ct);
		754	}
		755
643	static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)	756	static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
644	{	757	{
645	struct guc_ctb *h2g = &ct->ctbs.h2g;	758	struct guc_ctb *h2g = &ct->ctbs.h2g;
@@ -747,28 +860,6 @@ static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) · 101 unmodified lines
747	spin_unlock_irq(&ct->fast_lock);	860	spin_unlock_irq(&ct->fast_lock);
748	}	861	}
749		862
750	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
751	static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
752	{
753	unsigned int slot = fence % ARRAY_SIZE(ct->fast_req);
754	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
755	unsigned long entries[SZ_32];
756	unsigned int n;
757
758	n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
759
760	/* May be called under spinlock, so avoid sleeping */
761	ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT);
762	#endif
763	ct->fast_req[slot].fence = fence;
764	ct->fast_req[slot].action = action;
765	}
766	#else
767	static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action)
768	{
769	}
770	#endif
771
772	/*	863	/*
773	* The CT protocol accepts a 16 bits fence. This field is fully owned by the	864	* The CT protocol accepts a 16 bits fence. This field is fully owned by the
774	* driver, the GuC will just copy it to the reply message. Since we need to	865	* driver, the GuC will just copy it to the reply message. Since we need to
@@ -1310,10 +1401,12 @@ static int parse_g2h_event(struct xe_guc_ct ct, u32 msg, u32 len) · 535 unmodified lines
1310	lockdep_assert_held(&ct->lock);	1401	lockdep_assert_held(&ct->lock);
1311		1402
1312	switch (action) {	1403	switch (action) {
		1404	case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE:
1313	case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:	1405	case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1314	case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:	1406	case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1315	case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:	1407	case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
1316	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:	1408	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
		1409	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
1317	g2h_release_space(ct, len);	1410	g2h_release_space(ct, len);
1318	}	1411	}
1319		1412
@@ -1338,55 +1431,6 @@ static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) · 18 unmodified lines
1338	return 0;	1431	return 0;
1339	}	1432	}
1340		1433
1341	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
1342	static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
1343	{
1344	u16 fence_min = U16_MAX, fence_max = 0;
1345	struct xe_gt *gt = ct_to_gt(ct);
1346	bool found = false;
1347	unsigned int n;
1348	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
1349	char *buf;
1350	#endif
1351
1352	lockdep_assert_held(&ct->lock);
1353
1354	for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) {
1355	if (ct->fast_req[n].fence < fence_min)
1356	fence_min = ct->fast_req[n].fence;
1357	if (ct->fast_req[n].fence > fence_max)
1358	fence_max = ct->fast_req[n].fence;
1359
1360	if (ct->fast_req[n].fence != fence)
1361	continue;
1362	found = true;
1363
1364	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
1365	buf = kmalloc(SZ_4K, GFP_NOWAIT);
1366	if (buf && stack_depot_snprint(ct->fast_req[n].stack, buf, SZ_4K, 0))
1367	xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s",
1368	fence, ct->fast_req[n].action, buf);
1369	else
1370	xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n",
1371	fence, ct->fast_req[n].action);
1372	kfree(buf);
1373	#else
1374	xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n",
1375	fence, ct->fast_req[n].action);
1376	#endif
1377	break;
1378	}
1379
1380	if (!found)
1381	xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n",
1382	fence, fence_min, fence_max, ct->fence_seqno);
1383	}
1384	#else
1385	static void fast_req_report(struct xe_guc_ct *ct, u16 fence)
1386	{
1387	}
1388	#endif
1389
1390	static int parse_g2h_response(struct xe_guc_ct ct, u32 msg, u32 len)	1434	static int parse_g2h_response(struct xe_guc_ct ct, u32 msg, u32 len)
1391	{	1435	{
1392	struct xe_gt *gt = ct_to_gt(ct);	1436	struct xe_gt *gt = ct_to_gt(ct);
@@ -1549,6 +1593,15 @@ static int process_g2h_msg(struct xe_guc_ct ct, u32 msg, u32 len) · 156 unmodified lines
1549	ret = xe_guc_pagefault_handler(guc, payload, adj_len);	1593	ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1550	break;	1594	break;
1551	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:	1595	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
		1596	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
		1597	/*
		1598	* Page reclamation is an extension of TLB invalidation. Both
		1599	* operations share the same seqno and fence. When either
		1600	* action completes, we need to signal the corresponding
		1601	* fence. Since the handling logic (lookup fence by seqno,
		1602	* fence signalling) is identical, we use the same handler
		1603	* for both G2H events.
		1604	*/
1552	ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);	1605	ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
1553	break;	1606	break;
1554	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:	1607	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
@@ -1572,6 +1625,13 @@ static int process_g2h_msg(struct xe_guc_ct ct, u32 msg, u32 len) · 17 unmodified lines
1572	ret = xe_guc_g2g_test_notification(guc, payload, adj_len);	1625	ret = xe_guc_g2g_test_notification(guc, payload, adj_len);
1573	break;	1626	break;
1574	#endif	1627	#endif
		1628	case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE:
		1629	ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len);
		1630	break;
		1631	case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR:
		1632	ret = xe_guc_exec_queue_cgp_context_error_handler(guc, payload,
		1633	adj_len);
		1634	break;
1575	default:	1635	default:
1576	xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);	1636	xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action);
1577	}	1637	}
@@ -1714,6 +1774,7 @@ static int g2h_read(struct xe_guc_ct ct, u32 msg, bool fast_path) · 136 unmodified lines
1714	switch (action) {	1774	switch (action) {
1715	case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:	1775	case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1716	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:	1776	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
		1777	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
1717	break; /* Process these in fast-path */	1778	break; /* Process these in fast-path */
1718	default:	1779	default:
1719	return 0;	1780	return 0;
@@ -1750,6 +1811,12 @@ static void g2h_fast_path(struct xe_guc_ct ct, u32 msg, u32 len) · 30 unmodified lines
1750	ret = xe_guc_pagefault_handler(guc, payload, adj_len);	1811	ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1751	break;	1812	break;
1752	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:	1813	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
		1814	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
		1815	/*
		1816	* Seqno and fence handling of page reclamation and TLB
		1817	* invalidation is identical, so we can use the same handler
		1818	* for both actions.
		1819	*/
1753	__g2h_release_space(ct, len);	1820	__g2h_release_space(ct, len);
1754	ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);	1821	ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
1755	break;	1822	break;

drivers/gpu/drm/xe/xe_guc_ct.h+2−0

 int xe_guc_ct_enable(struct xe_guc_ct *ct);
 int xe_guc_ct_restart(struct xe_guc_ct *ct);
 void xe_guc_ct_disable(struct xe_guc_ct *ct);
+void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct);
+void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct);
 void xe_guc_ct_stop(struct xe_guc_ct *ct);
 void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct);
 void xe_guc_ct_fast_path(struct xe_guc_ct *ct);

drivers/gpu/drm/xe/xe_guc_debugfs.c+9−6

 	struct xe_gt *gt = grandparent->d_inode->i_private;
 	struct xe_device *xe = gt_to_xe(gt);
 	int (*print)(struct xe_guc *, struct drm_printer *) = node->info_ent->data;
-	int ret;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
-	ret = print(&gt->uc.guc, &p);
+	return print(&gt->uc.guc, &p);
-	xe_pm_runtime_put(xe);
-	return ret;
 }
 static int guc_log(struct xe_guc *guc, struct drm_printer *p)
 	return 0;
 }
+static int guc_log_lfd(struct xe_guc *guc, struct drm_printer *p)
+{
+	xe_guc_log_print_lfd(&guc->log, p);
+	return 0;
+}
 static int guc_log_dmesg(struct xe_guc *guc, struct drm_printer *p)
 {
 	xe_guc_log_print_dmesg(&guc->log);
 /* everything else should be added here */
 static const struct drm_info_list pf_only_debugfs_list[] = {
 	{ "guc_log", .show = guc_debugfs_show, .data = guc_log },
+	{ "guc_log_lfd", .show = guc_debugfs_show, .data = guc_log_lfd },
 	{ "guc_log_dmesg", .show = guc_debugfs_show, .data = guc_log_dmesg },
 };

drivers/gpu/drm/xe/xe_guc_fwif.h+7−3

 #define G2H_LEN_DW_DEREGISTER_CONTEXT		3
 #define G2H_LEN_DW_TLB_INVALIDATE		3
 #define G2H_LEN_DW_G2G_NOTIFY_MIN		3
+#define G2H_LEN_DW_MULTI_QUEUE_CONTEXT		3
+#define G2H_LEN_DW_PAGE_RECLAMATION		3
 #define GUC_ID_MAX			65535
 #define GUC_ID_UNKNOWN			0xffffffff
 	u32 wq_base_lo;
 	u32 wq_base_hi;
 	u32 wq_size;
+	u32 cgp_lo;
+	u32 cgp_hi;
 	u32 hwlrca_lo;
 	u32 hwlrca_hi;
 };
 #define   GUC_LOG_NOTIFY_ON_HALF_FULL	BIT(1)
 #define   GUC_LOG_CAPTURE_ALLOC_UNITS	BIT(2)
 #define   GUC_LOG_LOG_ALLOC_UNITS	BIT(3)
-#define   GUC_LOG_CRASH			REG_GENMASK(5, 4)
+#define   GUC_LOG_CRASH_DUMP		REG_GENMASK(5, 4)
-#define   GUC_LOG_DEBUG			REG_GENMASK(9, 6)
+#define   GUC_LOG_EVENT_DATA		REG_GENMASK(9, 6)
-#define   GUC_LOG_CAPTURE		REG_GENMASK(11, 10)
+#define   GUC_LOG_STATE_CAPTURE		REG_GENMASK(11, 10)
 #define   GUC_LOG_BUF_ADDR		REG_GENMASK(31, 12)
 #define GUC_CTL_WA			1

drivers/gpu/drm/xe/xe_guc_klv_thresholds_set_types.h+6−0

  * ABI and the associated &NAME, that may be used in code or debugfs/sysfs::
  *
  *	define(TAG, NAME)
+ *
+ * If required, KLVs can be labeled with GuC firmware version that added them::
+ *
+ *	define(TAG, NAME, MAJOR, MINOR)
+ *	define(TAG, NAME, MAJOR, MINOR, PATCH)
  */
 #define MAKE_XE_GUC_KLV_THRESHOLDS_SET(define)		\
 	define(CAT_ERR, cat_error_count)		\
 	define(H2G_STORM, guc_time_us)			\
 	define(IRQ_STORM, irq_time_us)			\
 	define(DOORBELL_STORM, doorbell_time_us)	\
+	define(MULTI_LRC_COUNT, multi_lrc_count, 70, 53)\
 	/* end */
 /**

drivers/gpu/drm/xe/xe_guc_log.c+406−101

@@ -7,8 +7,10 @@ · 6 unmodified lines
7		7
8	#include <linux/fault-inject.h>	8	#include <linux/fault-inject.h>
9		9
		10	#include <linux/utsname.h>
10	#include <drm/drm_managed.h>	11	#include <drm/drm_managed.h>
11		12
		13	#include "abi/guc_lfd_abi.h"
12	#include "regs/xe_guc_regs.h"	14	#include "regs/xe_guc_regs.h"
13	#include "xe_bo.h"	15	#include "xe_bo.h"
14	#include "xe_devcoredump.h"	16	#include "xe_devcoredump.h"
@@ -19,6 +21,77 @@ · 4 unmodified lines
19	#include "xe_mmio.h"	21	#include "xe_mmio.h"
20	#include "xe_module.h"	22	#include "xe_module.h"
21		23
		24	#define GUC_LOG_CHUNK_SIZE SZ_2M
		25
		26	/* Magic keys define */
		27	#define GUC_LFD_DRIVER_KEY_STREAMING 0x8086AAAA474C5346
		28	#define GUC_LFD_LOG_BUFFER_MARKER_2 0xDEADFEED
		29	#define GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2 0x8086DEAD
		30	#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2 0xBEEFFEED
		31	#define GUC_LFD_LOG_BUFFER_MARKER_1V2 0xCABBA9E6
		32	#define GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2 0xCABBA9F7
		33	#define GUC_LFD_DATA_HEADER_MAGIC 0x8086
		34
		35	/* LFD supported LIC type range */
		36	#define GUC_LIC_TYPE_FIRST GUC_LIC_TYPE_GUC_SW_VERSION
		37	#define GUC_LIC_TYPE_LAST GUC_LIC_TYPE_BUILD_PLATFORM_ID
		38	#define GUC_LFD_TYPE_FW_RANGE_FIRST GUC_LFD_TYPE_FW_VERSION
		39	#define GUC_LFD_TYPE_FW_RANGE_LAST GUC_LFD_TYPE_BUILD_PLATFORM_ID
		40
		41	#define GUC_LOG_BUFFER_STATE_HEADER_LENGTH 4096
		42	#define GUC_LOG_BUFFER_INIT_CONFIG 3
		43
		44	struct guc_log_buffer_entry_list {
		45	u32 offset;
		46	u32 rd_ptr;
		47	u32 wr_ptr;
		48	u32 wrap_offset;
		49	u32 buf_size;
		50	};
		51
		52	struct guc_lic_save {
		53	u32 version;
		54	/*
		55	* Array of init config KLV values.
		56	* Range from GUC_LOG_LIC_TYPE_FIRST to GUC_LOG_LIC_TYPE_LAST
		57	*/
		58	u32 values[GUC_LIC_TYPE_LAST - GUC_LIC_TYPE_FIRST + 1];
		59	struct guc_log_buffer_entry_list entry[GUC_LOG_BUFFER_INIT_CONFIG];
		60	};
		61
		62	static struct guc_log_buffer_entry_markers {
		63	u32 key[2];
		64	} const entry_markers[GUC_LOG_BUFFER_INIT_CONFIG + 1] = {
		65	{{
		66	GUC_LFD_LOG_BUFFER_MARKER_1V2,
		67	GUC_LFD_LOG_BUFFER_MARKER_2
		68	}},
		69	{{
		70	GUC_LFD_LOG_BUFFER_MARKER_1V2,
		71	GUC_LFD_CRASH_DUMP_BUFFER_MARKER_2
		72	}},
		73	{{
		74	GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_1V2,
		75	GUC_LFD_STATE_CAPTURE_BUFFER_MARKER_2
		76	}},
		77	{{
		78	GUC_LIC_MAGIC,
		79	(FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MAJOR, GUC_LIC_VERSION_MAJOR) \|
		80	FIELD_PREP_CONST(GUC_LIC_VERSION_MASK_MINOR, GUC_LIC_VERSION_MINOR))
		81	}}
		82	};
		83
		84	static struct guc_log_lic_lfd_map {
		85	u32 lic;
		86	u32 lfd;
		87	} const lic_lfd_type_map[] = {
		88	{GUC_LIC_TYPE_GUC_SW_VERSION, GUC_LFD_TYPE_FW_VERSION},
		89	{GUC_LIC_TYPE_GUC_DEVICE_ID, GUC_LFD_TYPE_GUC_DEVICE_ID},
		90	{GUC_LIC_TYPE_TSC_FREQUENCY, GUC_LFD_TYPE_TSC_FREQUENCY},
		91	{GUC_LIC_TYPE_GMD_ID, GUC_LFD_TYPE_GMD_ID},
		92	{GUC_LIC_TYPE_BUILD_PLATFORM_ID, GUC_LFD_TYPE_BUILD_PLATFORM_ID}
		93	};
		94
22	static struct xe_guc *	95	static struct xe_guc *
23	log_to_guc(struct xe_guc_log *log)	96	log_to_guc(struct xe_guc_log *log)
24	{	97	{
@@ -37,33 +110,6 @@ log_to_xe(struct xe_guc_log *log) · 12 unmodified lines
37	return gt_to_xe(log_to_gt(log));	110	return gt_to_xe(log_to_gt(log));
38	}	111	}
39		112
40	static size_t guc_log_size(void)
41	{
42	/*
43	* GuC Log buffer Layout
44	*
45	* +===============================+ 00B
46	* \| Crash dump state header \|
47	* +-------------------------------+ 32B
48	* \| Debug state header \|
49	* +-------------------------------+ 64B
50	* \| Capture state header \|
51	* +-------------------------------+ 96B
52	* \| \|
53	* +===============================+ PAGE_SIZE (4KB)
54	* \| Crash Dump logs \|
55	* +===============================+ + CRASH_SIZE
56	* \| Debug logs \|
57	* +===============================+ + DEBUG_SIZE
58	* \| Capture logs \|
59	* +===============================+ + CAPTURE_SIZE
60	*/
61	return PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE +
62	CAPTURE_BUFFER_SIZE;
63	}
64
65	#define GUC_LOG_CHUNK_SIZE SZ_2M
66
67	static struct xe_guc_log_snapshot xe_guc_log_snapshot_alloc(struct xe_guc_log log, bool atomic)	113	static struct xe_guc_log_snapshot xe_guc_log_snapshot_alloc(struct xe_guc_log log, bool atomic)
68	{	114	{
69	struct xe_guc_log_snapshot *snapshot;	115	struct xe_guc_log_snapshot *snapshot;
@@ -145,7 +191,6 @@ struct xe_guc_log_snapshot xe_guc_log_snapshot_capture(struct xe_guc_log log, · 75 unmodified lines
145	struct xe_device *xe = log_to_xe(log);	191	struct xe_device *xe = log_to_xe(log);
146	struct xe_guc *guc = log_to_guc(log);	192	struct xe_guc *guc = log_to_guc(log);
147	struct xe_gt *gt = log_to_gt(log);	193	struct xe_gt *gt = log_to_gt(log);
148	unsigned int fw_ref;
149	size_t remain;	194	size_t remain;
150	int i;	195	int i;
151		196
@@ -165,13 +210,12 @@ struct xe_guc_log_snapshot xe_guc_log_snapshot_capture(struct xe_guc_log log, · 13 unmodified lines
165	remain -= size;	210	remain -= size;
166	}	211	}
167		212
168	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);	213	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
169	if (!fw_ref) {	214	if (!fw_ref.domains)
170	snapshot->stamp = ~0ULL;	215	snapshot->stamp = ~0ULL;
171	} else {	216	else
172	snapshot->stamp = xe_mmio_read64_2x32(&gt->mmio, GUC_PMTIMESTAMP_LO);	217	snapshot->stamp = xe_mmio_read64_2x32(&gt->mmio, GUC_PMTIMESTAMP_LO);
173	xe_force_wake_put(gt_to_fw(gt), fw_ref);	218
174	}
175	snapshot->ktime = ktime_get_boottime_ns();	219	snapshot->ktime = ktime_get_boottime_ns();
176	snapshot->level = log->level;	220	snapshot->level = log->level;
177	snapshot->ver_found = guc->fw.versions.found[XE_UC_FW_VER_RELEASE];	221	snapshot->ver_found = guc->fw.versions.found[XE_UC_FW_VER_RELEASE];
@@ -216,6 +260,318 @@ void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_ · 38 unmodified lines
216	}	260	}
217	}	261	}
218		262
		263	static inline void lfd_output_binary(struct drm_printer p, char buf, int buf_size)
		264	{
		265	seq_write(p->arg, buf, buf_size);
		266	}
		267
		268	static inline int xe_guc_log_add_lfd_header(struct guc_lfd_data *lfd)
		269	{
		270	lfd->header = FIELD_PREP_CONST(GUC_LFD_DATA_HEADER_MASK_MAGIC, GUC_LFD_DATA_HEADER_MAGIC);
		271	return offsetof(struct guc_lfd_data, data);
		272	}
		273
		274	static int xe_guc_log_add_typed_payload(struct drm_printer *p, u32 type,
		275	u32 data_len, void *data)
		276	{
		277	struct guc_lfd_data lfd;
		278	int len;
		279
		280	len = xe_guc_log_add_lfd_header(&lfd);
		281	lfd.header \|= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, type);
		282	/* make length DW aligned */
		283	lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32));
		284	lfd_output_binary(p, (char *)&lfd, len);
		285
		286	lfd_output_binary(p, data, data_len);
		287	len += lfd.data_count * sizeof(u32);
		288
		289	return len;
		290	}
		291
		292	static inline int lic_type_to_index(u32 lic_type)
		293	{
		294	XE_WARN_ON(lic_type < GUC_LIC_TYPE_FIRST \|\| lic_type > GUC_LIC_TYPE_LAST);
		295
		296	return lic_type - GUC_LIC_TYPE_FIRST;
		297	}
		298
		299	static inline int lfd_type_to_index(u32 lfd_type)
		300	{
		301	int i, lic_type = 0;
		302
		303	XE_WARN_ON(lfd_type < GUC_LFD_TYPE_FW_RANGE_FIRST \|\| lfd_type > GUC_LFD_TYPE_FW_RANGE_LAST);
		304
		305	for (i = 0; i < ARRAY_SIZE(lic_lfd_type_map); i++)
		306	if (lic_lfd_type_map[i].lfd == lfd_type)
		307	lic_type = lic_lfd_type_map[i].lic;
		308
		309	/* If not found, lic_type_to_index will warning invalid type */
		310	return lic_type_to_index(lic_type);
		311	}
		312
		313	static int xe_guc_log_add_klv(struct drm_printer *p, u32 lfd_type,
		314	struct guc_lic_save *config)
		315	{
		316	int klv_index = lfd_type_to_index(lfd_type);
		317
		318	return xe_guc_log_add_typed_payload(p, lfd_type, sizeof(u32), &config->values[klv_index]);
		319	}
		320
		321	static int xe_guc_log_add_os_id(struct drm_printer *p, u32 id)
		322	{
		323	struct guc_lfd_data_os_info os_id;
		324	struct guc_lfd_data lfd;
		325	int len, info_len, section_len;
		326	char *version;
		327	u32 blank = 0;
		328
		329	len = xe_guc_log_add_lfd_header(&lfd);
		330	lfd.header \|= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_OS_ID);
		331
		332	os_id.os_id = id;
		333	section_len = offsetof(struct guc_lfd_data_os_info, build_version);
		334
		335	version = init_utsname()->release;
		336	info_len = strlen(version);
		337
		338	/* make length DW aligned */
		339	lfd.data_count = DIV_ROUND_UP(section_len + info_len, sizeof(u32));
		340	lfd_output_binary(p, (char *)&lfd, len);
		341	lfd_output_binary(p, (char *)&os_id, section_len);
		342	lfd_output_binary(p, version, info_len);
		343
		344	/* Padding with 0 */
		345	section_len = lfd.data_count * sizeof(u32) - section_len - info_len;
		346	if (section_len)
		347	lfd_output_binary(p, (char *)&blank, section_len);
		348
		349	len += lfd.data_count * sizeof(u32);
		350	return len;
		351	}
		352
		353	static void xe_guc_log_loop_log_init(struct guc_lic init, struct guc_lic_save config)
		354	{
		355	struct guc_klv_generic_dw_t p = (void )init->data;
		356	int i;
		357
		358	for (i = 0; i < init->data_count;) {
		359	int klv_len = FIELD_GET(GUC_KLV_0_LEN, p->kl) + 1;
		360	int key = FIELD_GET(GUC_KLV_0_KEY, p->kl);
		361
		362	if (key < GUC_LIC_TYPE_FIRST \|\| key > GUC_LIC_TYPE_LAST) {
		363	XE_WARN_ON(key < GUC_LIC_TYPE_FIRST \|\| key > GUC_LIC_TYPE_LAST);
		364	break;
		365	}
		366	config->values[lic_type_to_index(key)] = p->value;
		367	i += klv_len + 1; /* Whole KLV structure length in dwords */
		368	p = (void )((u32 )p + klv_len);
		369	}
		370	}
		371
		372	static int find_marker(u32 mark0, u32 mark1)
		373	{
		374	int i;
		375
		376	for (i = 0; i < ARRAY_SIZE(entry_markers); i++)
		377	if (mark0 == entry_markers[i].key[0] && mark1 == entry_markers[i].key[1])
		378	return i;
		379
		380	return ARRAY_SIZE(entry_markers);
		381	}
		382
		383	static void xe_guc_log_load_lic(void guc_log, struct guc_lic_save config)
		384	{
		385	u32 offset = GUC_LOG_BUFFER_STATE_HEADER_LENGTH;
		386	struct guc_log_buffer_state *p = guc_log;
		387
		388	config->version = p->version;
		389	while (p->marker[0]) {
		390	int index;
		391
		392	index = find_marker(p->marker[0], p->marker[1]);
		393
		394	if (index < ARRAY_SIZE(entry_markers)) {
		395	if (index == GUC_LOG_BUFFER_INIT_CONFIG) {
		396	/* Load log init config */
		397	xe_guc_log_loop_log_init((void *)p, config);
		398
		399	/* LIC structure is the last */
		400	return;
		401	}
		402	config->entry[index].offset = offset;
		403	config->entry[index].rd_ptr = p->read_ptr;
		404	config->entry[index].wr_ptr = p->write_ptr;
		405	config->entry[index].wrap_offset = p->wrap_offset;
		406	config->entry[index].buf_size = p->size;
		407	}
		408	offset += p->size;
		409	p++;
		410	}
		411	}
		412
		413	static int
		414	xe_guc_log_output_lfd_init(struct drm_printer p, struct xe_guc_log_snapshot snapshot,
		415	struct guc_lic_save *config)
		416	{
		417	int type, len;
		418	size_t size = 0;
		419
		420	/* FW required types */
		421	for (type = GUC_LFD_TYPE_FW_RANGE_FIRST; type <= GUC_LFD_TYPE_FW_RANGE_LAST; type++)
		422	size += xe_guc_log_add_klv(p, type, config);
		423
		424	/* KMD required type(s) */
		425	len = xe_guc_log_add_os_id(p, GUC_LFD_OS_TYPE_OSID_LIN);
		426	size += len;
		427
		428	return size;
		429	}
		430
		431	static void
		432	xe_guc_log_print_chunks(struct drm_printer p, struct xe_guc_log_snapshot snapshot,
		433	u32 from, u32 to)
		434	{
		435	int chunk_from = from % GUC_LOG_CHUNK_SIZE;
		436	int chunk_id = from / GUC_LOG_CHUNK_SIZE;
		437	int to_chunk_id = to / GUC_LOG_CHUNK_SIZE;
		438	int chunk_to = to % GUC_LOG_CHUNK_SIZE;
		439	int pos = from;
		440
		441	do {
		442	size_t size = (to_chunk_id == chunk_id ? chunk_to : GUC_LOG_CHUNK_SIZE) -
		443	chunk_from;
		444
		445	lfd_output_binary(p, snapshot->copy[chunk_id] + chunk_from, size);
		446	pos += size;
		447	chunk_id++;
		448	chunk_from = 0;
		449	} while (pos < to);
		450	}
		451
		452	static inline int
		453	xe_guc_log_add_log_event(struct drm_printer p, struct xe_guc_log_snapshot snapshot,
		454	struct guc_lic_save *config)
		455	{
		456	size_t size;
		457	u32 data_len, section_len;
		458	struct guc_lfd_data lfd;
		459	struct guc_log_buffer_entry_list *entry;
		460	struct guc_lfd_data_log_events_buf events_buf;
		461
		462	entry = &config->entry[GUC_LOG_TYPE_EVENT_DATA];
		463
		464	/* Skip empty log */
		465	if (entry->rd_ptr == entry->wr_ptr)
		466	return 0;
		467
		468	size = xe_guc_log_add_lfd_header(&lfd);
		469	lfd.header \|= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_LOG_EVENTS_BUFFER);
		470	events_buf.log_events_format_version = config->version;
		471
		472	/* Adjust to log_format_buf */
		473	section_len = offsetof(struct guc_lfd_data_log_events_buf, log_event);
		474	data_len = section_len;
		475
		476	/* Calculate data length */
		477	data_len += entry->rd_ptr < entry->wr_ptr ? (entry->wr_ptr - entry->rd_ptr) :
		478	(entry->wr_ptr + entry->wrap_offset - entry->rd_ptr);
		479	/* make length u32 aligned */
		480	lfd.data_count = DIV_ROUND_UP(data_len, sizeof(u32));
		481
		482	/* Output GUC_LFD_TYPE_LOG_EVENTS_BUFFER header */
		483	lfd_output_binary(p, (char *)&lfd, size);
		484	lfd_output_binary(p, (char *)&events_buf, section_len);
		485
		486	/* Output data from guc log chunks directly */
		487	if (entry->rd_ptr < entry->wr_ptr) {
		488	xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr,
		489	entry->offset + entry->wr_ptr);
		490	} else {
		491	/* 1st, print from rd to wrap offset */
		492	xe_guc_log_print_chunks(p, snapshot, entry->offset + entry->rd_ptr,
		493	entry->offset + entry->wrap_offset);
		494
		495	/* 2nd, print from buf start to wr */
		496	xe_guc_log_print_chunks(p, snapshot, entry->offset, entry->offset + entry->wr_ptr);
		497	}
		498	return size;
		499	}
		500
		501	static int
		502	xe_guc_log_add_crash_dump(struct drm_printer p, struct xe_guc_log_snapshot snapshot,
		503	struct guc_lic_save *config)
		504	{
		505	struct guc_log_buffer_entry_list *entry;
		506	int chunk_from, chunk_id;
		507	int from, to, i;
		508	size_t size = 0;
		509	u32 *buf32;
		510
		511	entry = &config->entry[GUC_LOG_TYPE_CRASH_DUMP];
		512
		513	/* Skip zero sized crash dump */
		514	if (!entry->buf_size)
		515	return 0;
		516
		517	/* Check if crash dump section are all zero */
		518	from = entry->offset;
		519	to = entry->offset + entry->buf_size;
		520	chunk_from = from % GUC_LOG_CHUNK_SIZE;
		521	chunk_id = from / GUC_LOG_CHUNK_SIZE;
		522	buf32 = snapshot->copy[chunk_id] + chunk_from;
		523
		524	for (i = 0; i < entry->buf_size / sizeof(u32); i++)
		525	if (buf32[i])
		526	break;
		527
		528	/* Buffer has non-zero data? */
		529	if (i < entry->buf_size / sizeof(u32)) {
		530	struct guc_lfd_data lfd;
		531
		532	size = xe_guc_log_add_lfd_header(&lfd);
		533	lfd.header \|= FIELD_PREP(GUC_LFD_DATA_HEADER_MASK_TYPE, GUC_LFD_TYPE_FW_CRASH_DUMP);
		534	/* Calculate data length */
		535	lfd.data_count = DIV_ROUND_UP(entry->buf_size, sizeof(u32));
		536	/* Output GUC_LFD_TYPE_FW_CRASH_DUMP header */
		537	lfd_output_binary(p, (char *)&lfd, size);
		538
		539	/* rd/wr ptr is not used for crash dump */
		540	xe_guc_log_print_chunks(p, snapshot, from, to);
		541	}
		542	return size;
		543	}
		544
		545	static void
		546	xe_guc_log_snapshot_print_lfd(struct xe_guc_log_snapshot snapshot, struct drm_printer p)
		547	{
		548	struct guc_lfd_file_header header;
		549	struct guc_lic_save config;
		550	size_t size;
		551
		552	if (!snapshot \|\| !snapshot->size)
		553	return;
		554
		555	header.magic = GUC_LFD_DRIVER_KEY_STREAMING;
		556	header.version = FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MINOR,
		557	GUC_LFD_FORMAT_VERSION_MINOR) \|
		558	FIELD_PREP_CONST(GUC_LFD_FILE_HEADER_VERSION_MASK_MAJOR,
		559	GUC_LFD_FORMAT_VERSION_MAJOR);
		560
		561	/* Output LFD file header */
		562	lfd_output_binary(p, (char *)&header,
		563	offsetof(struct guc_lfd_file_header, stream));
		564
		565	/* Output LFD stream */
		566	xe_guc_log_load_lic(snapshot->copy[0], &config);
		567	size = xe_guc_log_output_lfd_init(p, snapshot, &config);
		568	if (!size)
		569	return;
		570
		571	xe_guc_log_add_log_event(p, snapshot, &config);
		572	xe_guc_log_add_crash_dump(p, snapshot, &config);
		573	}
		574
219	/**	575	/**
220	* xe_guc_log_print_dmesg - dump a copy of the GuC log to dmesg	576	* xe_guc_log_print_dmesg - dump a copy of the GuC log to dmesg
221	* @log: GuC log structure	577	* @log: GuC log structure
@@ -251,13 +607,27 @@ void xe_guc_log_print(struct xe_guc_log log, struct drm_printer p) · 29 unmodified lines
251	xe_guc_log_snapshot_free(snapshot);	607	xe_guc_log_snapshot_free(snapshot);
252	}	608	}
253		609
		610	/**
		611	* xe_guc_log_print_lfd - dump a copy of the GuC log in LFD format
		612	* @log: GuC log structure
		613	* @p: the printer object to output to
		614	*/
		615	void xe_guc_log_print_lfd(struct xe_guc_log log, struct drm_printer p)
		616	{
		617	struct xe_guc_log_snapshot *snapshot;
		618
		619	snapshot = xe_guc_log_snapshot_capture(log, false);
		620	xe_guc_log_snapshot_print_lfd(snapshot, p);
		621	xe_guc_log_snapshot_free(snapshot);
		622	}
		623
254	int xe_guc_log_init(struct xe_guc_log *log)	624	int xe_guc_log_init(struct xe_guc_log *log)
255	{	625	{
256	struct xe_device *xe = log_to_xe(log);	626	struct xe_device *xe = log_to_xe(log);
257	struct xe_tile *tile = gt_to_tile(log_to_gt(log));	627	struct xe_tile *tile = gt_to_tile(log_to_gt(log));
258	struct xe_bo *bo;	628	struct xe_bo *bo;
259		629
260	bo = xe_managed_bo_create_pin_map(xe, tile, guc_log_size(),	630	bo = xe_managed_bo_create_pin_map(xe, tile, GUC_LOG_SIZE,
261	XE_BO_FLAG_SYSTEM \|	631	XE_BO_FLAG_SYSTEM \|
262	XE_BO_FLAG_GGTT \|	632	XE_BO_FLAG_GGTT \|
263	XE_BO_FLAG_GGTT_INVALIDATE \|	633	XE_BO_FLAG_GGTT_INVALIDATE \|
@@ -265,7 +635,7 @@ int xe_guc_log_init(struct xe_guc_log *log) · 1 unmodified line
265	if (IS_ERR(bo))	635	if (IS_ERR(bo))
266	return PTR_ERR(bo);	636	return PTR_ERR(bo);
267		637
268	xe_map_memset(xe, &bo->vmap, 0, 0, guc_log_size());	638	xe_map_memset(xe, &bo->vmap, 0, 0, xe_bo_size(bo));
269	log->bo = bo;	639	log->bo = bo;
270	log->level = xe_modparam.guc_log_level;	640	log->level = xe_modparam.guc_log_level;
271		641
@@ -274,71 +644,6 @@ int xe_guc_log_init(struct xe_guc_log *log) · 2 unmodified lines
274		644
275	ALLOW_ERROR_INJECTION(xe_guc_log_init, ERRNO); /* See xe_pci_probe() */	645	ALLOW_ERROR_INJECTION(xe_guc_log_init, ERRNO); /* See xe_pci_probe() */
276		646
277	static u32 xe_guc_log_section_size_crash(struct xe_guc_log *log)
278	{
279	return CRASH_BUFFER_SIZE;
280	}
281
282	static u32 xe_guc_log_section_size_debug(struct xe_guc_log *log)
283	{
284	return DEBUG_BUFFER_SIZE;
285	}
286
287	/**
288	* xe_guc_log_section_size_capture - Get capture buffer size within log sections.
289	* @log: The log object.
290	*
291	* This function will return the capture buffer size within log sections.
292	*
293	* Return: capture buffer size.
294	*/
295	u32 xe_guc_log_section_size_capture(struct xe_guc_log *log)
296	{
297	return CAPTURE_BUFFER_SIZE;
298	}
299
300	/**
301	* xe_guc_get_log_buffer_size - Get log buffer size for a type.
302	* @log: The log object.
303	* @type: The log buffer type
304	*
305	* Return: buffer size.
306	*/
307	u32 xe_guc_get_log_buffer_size(struct xe_guc_log *log, enum guc_log_buffer_type type)
308	{
309	switch (type) {
310	case GUC_LOG_BUFFER_CRASH_DUMP:
311	return xe_guc_log_section_size_crash(log);
312	case GUC_LOG_BUFFER_DEBUG:
313	return xe_guc_log_section_size_debug(log);
314	case GUC_LOG_BUFFER_CAPTURE:
315	return xe_guc_log_section_size_capture(log);
316	}
317	return 0;
318	}
319
320	/**
321	* xe_guc_get_log_buffer_offset - Get offset in log buffer for a type.
322	* @log: The log object.
323	* @type: The log buffer type
324	*
325	* This function will return the offset in the log buffer for a type.
326	* Return: buffer offset.
327	*/
328	u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_type type)
329	{
330	enum guc_log_buffer_type i;
331	u32 offset = PAGE_SIZE;/* for the log_buffer_states */
332
333	for (i = GUC_LOG_BUFFER_CRASH_DUMP; i < GUC_LOG_BUFFER_TYPE_MAX; ++i) {
334	if (i == type)
335	break;
336	offset += xe_guc_get_log_buffer_size(log, i);
337	}
338
339	return offset;
340	}
341
342	/**	647	/**
343	* xe_guc_check_log_buf_overflow - Check if log buffer overflowed	648	* xe_guc_check_log_buf_overflow - Check if log buffer overflowed
344	* @log: The log object.	649	* @log: The log object.
@@ -352,7 +657,7 @@ u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_typ · 7 unmodified lines
352	*	657	*
353	* Return: True if overflowed.	658	* Return: True if overflowed.
354	*/	659	*/
355	bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_buffer_type type,	660	bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log, enum guc_log_type type,
356	unsigned int full_cnt)	661	unsigned int full_cnt)
357	{	662	{
358	unsigned int prev_full_cnt = log->stats[type].sampled_overflow;	663	unsigned int prev_full_cnt = log->stats[type].sampled_overflow;

drivers/gpu/drm/xe/xe_guc_log.h+20−10

 struct xe_device;
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
-#define CRASH_BUFFER_SIZE       SZ_1M
+#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE	SZ_8M
-#define DEBUG_BUFFER_SIZE       SZ_8M
+#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE	SZ_1M
-#define CAPTURE_BUFFER_SIZE     SZ_2M
+#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE	SZ_2M
 #else
-#define CRASH_BUFFER_SIZE	SZ_16K
+#define XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE	SZ_64K
-#define DEBUG_BUFFER_SIZE	SZ_64K
+#define XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE	SZ_16K
-#define CAPTURE_BUFFER_SIZE	SZ_1M
+#define XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE	SZ_1M
 #endif
+#define GUC_LOG_SIZE (SZ_4K + \
+		      XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE + \
+		      XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE + \
+		      XE_GUC_LOG_STATE_CAPTURE_BUFFER_SIZE)
+#define XE_GUC_LOG_EVENT_DATA_OFFSET	SZ_4K
+#define XE_GUC_LOG_CRASH_DUMP_OFFSET	(XE_GUC_LOG_EVENT_DATA_OFFSET + \
+					 XE_GUC_LOG_EVENT_DATA_BUFFER_SIZE)
+#define XE_GUC_LOG_STATE_CAPTURE_OFFSET	(XE_GUC_LOG_CRASH_DUMP_OFFSET + \
+					 XE_GUC_LOG_CRASH_DUMP_BUFFER_SIZE)
 /*
  * While we're using plain log level in i915, GuC controls are much more...
  * "elaborate"? We have a couple of bits for verbosity, separate bit for actual
 int xe_guc_log_init(struct xe_guc_log *log);
 void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p);
+void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p);
 void xe_guc_log_print_dmesg(struct xe_guc_log *log);
 struct xe_guc_log_snapshot *xe_guc_log_snapshot_capture(struct xe_guc_log *log, bool atomic);
 void xe_guc_log_snapshot_print(struct xe_guc_log_snapshot *snapshot, struct drm_printer *p);
 	return log->level;
 }
-u32 xe_guc_log_section_size_capture(struct xe_guc_log *log);
-u32 xe_guc_get_log_buffer_size(struct xe_guc_log *log, enum guc_log_buffer_type type);
-u32 xe_guc_get_log_buffer_offset(struct xe_guc_log *log, enum guc_log_buffer_type type);
 bool xe_guc_check_log_buf_overflow(struct xe_guc_log *log,
-				   enum guc_log_buffer_type type,
+				   enum guc_log_type type,
 				   unsigned int full_cnt);
 #endif

drivers/gpu/drm/xe/xe_guc_pc.c+19−47

@@ -76,7 +76,7 @@ · 75 unmodified lines
76	* exposes a programming interface to the host for the control of SLPC.	76	* exposes a programming interface to the host for the control of SLPC.
77	*	77	*
78	* Frequency management:	78	* Frequency management:
79	* =====================	79	* ---------------------
80	*	80	*
81	* Xe driver enables SLPC with all of its defaults features and frequency	81	* Xe driver enables SLPC with all of its defaults features and frequency
82	* selection, which varies per platform.	82	* selection, which varies per platform.
@@ -87,7 +87,7 @@ · 4 unmodified lines
87	* for any workload.	87	* for any workload.
88	*	88	*
89	* Render-C States:	89	* Render-C States:
90	* ================	90	* ----------------
91	*	91	*
92	* Render-C states is also a GuC PC feature that is now enabled in Xe for	92	* Render-C states is also a GuC PC feature that is now enabled in Xe for
93	* all platforms.	93	* all platforms.
@@ -499,21 +499,17 @@ u32 xe_guc_pc_get_cur_freq_fw(struct xe_guc_pc *pc) · 405 unmodified lines
499	int xe_guc_pc_get_cur_freq(struct xe_guc_pc pc, u32 freq)	499	int xe_guc_pc_get_cur_freq(struct xe_guc_pc pc, u32 freq)
500	{	500	{
501	struct xe_gt *gt = pc_to_gt(pc);	501	struct xe_gt *gt = pc_to_gt(pc);
502	unsigned int fw_ref;
503		502
504	/*	503	/*
505	* GuC SLPC plays with cur freq request when GuCRC is enabled	504	* GuC SLPC plays with cur freq request when GuCRC is enabled
506	* Block RC6 for a more reliable read.	505	* Block RC6 for a more reliable read.
507	*/	506	*/
508	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);	507	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
509	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {	508	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
510	xe_force_wake_put(gt_to_fw(gt), fw_ref);
511	return -ETIMEDOUT;	509	return -ETIMEDOUT;
512	}
513		510
514	*freq = get_cur_freq(gt);	511	*freq = get_cur_freq(gt);
515		512
516	xe_force_wake_put(gt_to_fw(gt), fw_ref);
517	return 0;	513	return 0;
518	}	514	}
519		515
@@ -1087,13 +1083,8 @@ int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc) · 567 unmodified lines
1087	*/	1083	*/
1088	int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode)	1084	int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode)
1089	{	1085	{
1090	int ret;	1086	guard(xe_pm_runtime)(pc_to_xe(pc));
1091		1087	return pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
1092	xe_pm_runtime_get(pc_to_xe(pc));
1093	ret = pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
1094	xe_pm_runtime_put(pc_to_xe(pc));
1095
1096	return ret;
1097	}	1088	}
1098		1089
1099	/**	1090	/**
@@ -1104,13 +1095,8 @@ int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mod · 4 unmodified lines
1104	*/	1095	*/
1105	int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc)	1096	int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc)
1106	{	1097	{
1107	int ret;	1098	guard(xe_pm_runtime)(pc_to_xe(pc));
1108		1099	return pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
1109	xe_pm_runtime_get(pc_to_xe(pc));
1110	ret = pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
1111	xe_pm_runtime_put(pc_to_xe(pc));
1112
1113	return ret;
1114	}	1100	}
1115		1101
1116	static void pc_init_pcode_freq(struct xe_guc_pc *pc)	1102	static void pc_init_pcode_freq(struct xe_guc_pc *pc)
@@ -1198,7 +1184,7 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc pc, const char buf) · 81 unmodified lines
1198	return -EINVAL;	1184	return -EINVAL;
1199		1185
1200	guard(mutex)(&pc->freq_lock);	1186	guard(mutex)(&pc->freq_lock);
1201	xe_pm_runtime_get_noresume(pc_to_xe(pc));	1187	guard(xe_pm_runtime_noresume)(pc_to_xe(pc));
1202		1188
1203	ret = pc_action_set_param(pc,	1189	ret = pc_action_set_param(pc,
1204	SLPC_PARAM_POWER_PROFILE,	1190	SLPC_PARAM_POWER_PROFILE,
@@ -1209,8 +1195,6 @@ int xe_guc_pc_set_power_profile(struct xe_guc_pc pc, const char buf) · 4 unmodified lines
1209	else	1195	else
1210	pc->power_profile = val;	1196	pc->power_profile = val;
1211		1197
1212	xe_pm_runtime_put(pc_to_xe(pc));
1213
1214	return ret;	1198	return ret;
1215	}	1199	}
1216		1200
@@ -1223,17 +1207,14 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 6 unmodified lines
1223	struct xe_device *xe = pc_to_xe(pc);	1207	struct xe_device *xe = pc_to_xe(pc);
1224	struct xe_gt *gt = pc_to_gt(pc);	1208	struct xe_gt *gt = pc_to_gt(pc);
1225	u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));	1209	u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));
1226	unsigned int fw_ref;
1227	ktime_t earlier;	1210	ktime_t earlier;
1228	int ret;	1211	int ret;
1229		1212
1230	xe_gt_assert(gt, xe_device_uc_enabled(xe));	1213	xe_gt_assert(gt, xe_device_uc_enabled(xe));
1231		1214
1232	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);	1215	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
1233	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {	1216	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
1234	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1235	return -ETIMEDOUT;	1217	return -ETIMEDOUT;
1236	}
1237		1218
1238	if (xe->info.skip_guc_pc) {	1219	if (xe->info.skip_guc_pc) {
1239	if (xe->info.platform != XE_PVC)	1220	if (xe->info.platform != XE_PVC)
@@ -1241,9 +1222,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 1 unmodified line
1241		1222
1242	/* Request max possible since dynamic freq mgmt is not enabled */	1223	/* Request max possible since dynamic freq mgmt is not enabled */
1243	pc_set_cur_freq(pc, UINT_MAX);	1224	pc_set_cur_freq(pc, UINT_MAX);
1244		1225	return 0;
1245	ret = 0;
1246	goto out;
1247	}	1226	}
1248		1227
1249	xe_map_memset(xe, &pc->bo->vmap, 0, 0, size);	1228	xe_map_memset(xe, &pc->bo->vmap, 0, 0, size);
@@ -1252,7 +1231,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 2 unmodified lines
1252	earlier = ktime_get();	1231	earlier = ktime_get();
1253	ret = pc_action_reset(pc);	1232	ret = pc_action_reset(pc);
1254	if (ret)	1233	if (ret)
1255	goto out;	1234	return ret;
1256		1235
1257	if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,	1236	if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
1258	SLPC_RESET_TIMEOUT_MS)) {	1237	SLPC_RESET_TIMEOUT_MS)) {
@@ -1263,8 +1242,7 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 4 unmodified lines
1263	if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,	1242	if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
1264	SLPC_RESET_EXTENDED_TIMEOUT_MS)) {	1243	SLPC_RESET_EXTENDED_TIMEOUT_MS)) {
1265	xe_gt_err(gt, "GuC PC Start failed: Dynamic GT frequency control and GT sleep states are now disabled.\n");	1244	xe_gt_err(gt, "GuC PC Start failed: Dynamic GT frequency control and GT sleep states are now disabled.\n");
1266	ret = -EIO;	1245	return -EIO;
1267	goto out;
1268	}	1246	}
1269		1247
1270	xe_gt_warn(gt, "GuC PC excessive start time: %lldms",	1248	xe_gt_warn(gt, "GuC PC excessive start time: %lldms",
@@ -1273,21 +1251,20 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 2 unmodified lines
1273		1251
1274	ret = pc_init_freqs(pc);	1252	ret = pc_init_freqs(pc);
1275	if (ret)	1253	if (ret)
1276	goto out;	1254	return ret;
1277		1255
1278	ret = pc_set_mert_freq_cap(pc);	1256	ret = pc_set_mert_freq_cap(pc);
1279	if (ret)	1257	if (ret)
1280	goto out;	1258	return ret;
1281		1259
1282	if (xe->info.platform == XE_PVC) {	1260	if (xe->info.platform == XE_PVC) {
1283	xe_guc_pc_gucrc_disable(pc);	1261	xe_guc_pc_gucrc_disable(pc);
1284	ret = 0;	1262	return 0;
1285	goto out;
1286	}	1263	}
1287		1264
1288	ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL);	1265	ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL);
1289	if (ret)	1266	if (ret)
1290	goto out;	1267	return ret;
1291		1268
1292	/* Enable SLPC Optimized Strategy for compute */	1269	/* Enable SLPC Optimized Strategy for compute */
1293	ret = pc_action_set_strategy(pc, SLPC_OPTIMIZED_STRATEGY_COMPUTE);	1270	ret = pc_action_set_strategy(pc, SLPC_OPTIMIZED_STRATEGY_COMPUTE);
@@ -1297,8 +1274,6 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) · 3 unmodified lines
1297	if (unlikely(ret))	1274	if (unlikely(ret))
1298	xe_gt_err(gt, "Failed to set SLPC power profile: %pe\n", ERR_PTR(ret));	1275	xe_gt_err(gt, "Failed to set SLPC power profile: %pe\n", ERR_PTR(ret));
1299		1276
1300	out:
1301	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1302	return ret;	1277	return ret;
1303	}	1278	}
1304		1279
@@ -1330,19 +1305,16 @@ static void xe_guc_pc_fini_hw(void *arg) · 25 unmodified lines
1330	{	1305	{
1331	struct xe_guc_pc *pc = arg;	1306	struct xe_guc_pc *pc = arg;
1332	struct xe_device *xe = pc_to_xe(pc);	1307	struct xe_device *xe = pc_to_xe(pc);
1333	unsigned int fw_ref;
1334		1308
1335	if (xe_device_wedged(xe))	1309	if (xe_device_wedged(xe))
1336	return;	1310	return;
1337		1311
1338	fw_ref = xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);	1312	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
1339	xe_guc_pc_gucrc_disable(pc);	1313	xe_guc_pc_gucrc_disable(pc);
1340	XE_WARN_ON(xe_guc_pc_stop(pc));	1314	XE_WARN_ON(xe_guc_pc_stop(pc));
1341		1315
1342	/* Bind requested freq to mert_freq_cap before unload */	1316	/* Bind requested freq to mert_freq_cap before unload */
1343	pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc)));	1317	pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc)));
1344
1345	xe_force_wake_put(gt_to_fw(pc_to_gt(pc)), fw_ref);
1346	}	1318	}
1347		1319
1348	/**	1320	/**

drivers/gpu/drm/xe/xe_guc_submit.c+637−68

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	#include "abi/guc_klvs_abi.h"	19	#include "abi/guc_klvs_abi.h"
20	#include "regs/xe_lrc_layout.h"	20	#include "regs/xe_lrc_layout.h"
21	#include "xe_assert.h"	21	#include "xe_assert.h"
		22	#include "xe_bo.h"
22	#include "xe_devcoredump.h"	23	#include "xe_devcoredump.h"
23	#include "xe_device.h"	24	#include "xe_device.h"
24	#include "xe_exec_queue.h"	25	#include "xe_exec_queue.h"
@@ -47,6 +48,8 @@ · 22 unmodified lines
47	#include "xe_uc_fw.h"	48	#include "xe_uc_fw.h"
48	#include "xe_vm.h"	49	#include "xe_vm.h"
49		50
		51	#define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN 6
		52
50	static struct xe_guc *	53	static struct xe_guc *
51	exec_queue_to_guc(struct xe_exec_queue *q)	54	exec_queue_to_guc(struct xe_exec_queue *q)
52	{	55	{
@@ -72,6 +75,7 @@ exec_queue_to_guc(struct xe_exec_queue *q) · 19 unmodified lines
72	#define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11)	75	#define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11)
73	#define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 12)	76	#define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 12)
74	#define EXEC_QUEUE_STATE_PENDING_TDR_EXIT (1 << 13)	77	#define EXEC_QUEUE_STATE_PENDING_TDR_EXIT (1 << 13)
		78	#define EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND (1 << 14)
75		79
76	static bool exec_queue_registered(struct xe_exec_queue *q)	80	static bool exec_queue_registered(struct xe_exec_queue *q)
77	{	81	{
@@ -263,6 +267,21 @@ static void clear_exec_queue_pending_tdr_exit(struct xe_exec_queue *q) · 185 unmodified lines
263	atomic_and(~EXEC_QUEUE_STATE_PENDING_TDR_EXIT, &q->guc->state);	267	atomic_and(~EXEC_QUEUE_STATE_PENDING_TDR_EXIT, &q->guc->state);
264	}	268	}
265		269
		270	static bool exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
		271	{
		272	return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND;
		273	}
		274
		275	static void set_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
		276	{
		277	atomic_or(EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state);
		278	}
		279
		280	static void clear_exec_queue_idle_skip_suspend(struct xe_exec_queue *q)
		281	{
		282	atomic_and(~EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state);
		283	}
		284
266	static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)	285	static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)
267	{	286	{
268	return (atomic_read(&q->guc->state) &	287	return (atomic_read(&q->guc->state) &
@@ -541,7 +560,8 @@ static void init_policies(struct xe_guc guc, struct xe_exec_queue q) · 272 unmodified lines
541	u32 slpc_exec_queue_freq_req = 0;	560	u32 slpc_exec_queue_freq_req = 0;
542	u32 preempt_timeout_us = q->sched_props.preempt_timeout_us;	561	u32 preempt_timeout_us = q->sched_props.preempt_timeout_us;
543		562
544	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));	563	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) &&
		564	!xe_exec_queue_is_multi_queue_secondary(q));
545		565
546	if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY)	566	if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY)
547	slpc_exec_queue_freq_req \|= SLPC_CTX_FREQ_REQ_IS_COMPUTE;	567	slpc_exec_queue_freq_req \|= SLPC_CTX_FREQ_REQ_IS_COMPUTE;
@@ -561,6 +581,8 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue · 13 unmodified lines
561	{	581	{
562	struct exec_queue_policy policy;	582	struct exec_queue_policy policy;
563		583
		584	xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q));
		585
564	__guc_exec_queue_policy_start_klv(&policy, q->guc->id);	586	__guc_exec_queue_policy_start_klv(&policy, q->guc->id);
565	__guc_exec_queue_policy_add_preemption_timeout(&policy, 1);	587	__guc_exec_queue_policy_add_preemption_timeout(&policy, 1);
566		588
@@ -568,6 +590,89 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue · 1 unmodified line
568	__guc_exec_queue_policy_action_size(&policy), 0, 0);	590	__guc_exec_queue_policy_action_size(&policy), 0, 0);
569	}	591	}
570		592
		593	static bool vf_recovery(struct xe_guc *guc)
		594	{
		595	return xe_gt_recovery_pending(guc_to_gt(guc));
		596	}
		597
		598	static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
		599	{
		600	struct xe_guc *guc = exec_queue_to_guc(q);
		601	struct xe_device *xe = guc_to_xe(guc);
		602
		603	/** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
		604	wake_up_all(&xe->ufence_wq);
		605
		606	if (xe_exec_queue_is_lr(q))
		607	queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);
		608	else
		609	xe_sched_tdr_queue_imm(&q->guc->sched);
		610	}
		611
		612	static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q)
		613	{
		614	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
		615	struct xe_exec_queue_group *group = q->multi_queue.group;
		616	struct xe_exec_queue *eq;
		617
		618	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
		619	xe_exec_queue_is_multi_queue(q));
		620
		621	/* Group banned, skip timeout check in TDR */
		622	WRITE_ONCE(group->banned, true);
		623	xe_guc_exec_queue_trigger_cleanup(primary);
		624
		625	mutex_lock(&group->list_lock);
		626	list_for_each_entry(eq, &group->list, multi_queue.link)
		627	xe_guc_exec_queue_trigger_cleanup(eq);
		628	mutex_unlock(&group->list_lock);
		629	}
		630
		631	static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q)
		632	{
		633	if (xe_exec_queue_is_multi_queue(q)) {
		634	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
		635	struct xe_exec_queue_group *group = q->multi_queue.group;
		636	struct xe_exec_queue *eq;
		637
		638	/* Group banned, skip timeout check in TDR */
		639	WRITE_ONCE(group->banned, true);
		640
		641	set_exec_queue_reset(primary);
		642	if (!exec_queue_banned(primary) && !exec_queue_check_timeout(primary))
		643	xe_guc_exec_queue_trigger_cleanup(primary);
		644
		645	mutex_lock(&group->list_lock);
		646	list_for_each_entry(eq, &group->list, multi_queue.link) {
		647	set_exec_queue_reset(eq);
		648	if (!exec_queue_banned(eq) && !exec_queue_check_timeout(eq))
		649	xe_guc_exec_queue_trigger_cleanup(eq);
		650	}
		651	mutex_unlock(&group->list_lock);
		652	} else {
		653	set_exec_queue_reset(q);
		654	if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
		655	xe_guc_exec_queue_trigger_cleanup(q);
		656	}
		657	}
		658
		659	static void set_exec_queue_group_banned(struct xe_exec_queue *q)
		660	{
		661	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
		662	struct xe_exec_queue_group *group = q->multi_queue.group;
		663	struct xe_exec_queue *eq;
		664
		665	/* Ban all queues of the multi-queue group */
		666	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)),
		667	xe_exec_queue_is_multi_queue(q));
		668	set_exec_queue_banned(primary);
		669
		670	mutex_lock(&group->list_lock);
		671	list_for_each_entry(eq, &group->list, multi_queue.link)
		672	set_exec_queue_banned(eq);
		673	mutex_unlock(&group->list_lock);
		674	}
		675
571	#define parallel_read(xe_, map_, field_) \	676	#define parallel_read(xe_, map_, field_) \
572	xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \	677	xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
573	field_)	678	field_)
@@ -575,6 +680,181 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue · 1 unmodified line
575	xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \	680	xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \
576	field_, val_)	681	field_, val_)
577		682
		683	/**
		684	* DOC: Multi Queue Group GuC interface
		685	*
		686	* The multi queue group coordination between KMD and GuC is through a software
		687	* construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page
		688	* allocated in the global GTT.
		689	*
		690	* CGP format:
		691	*
		692	* +-----------+---------------------------+---------------------------------------------+
		693	* \| DWORD \| Name \| Description \|
		694	* +-----------+---------------------------+---------------------------------------------+
		695	* \| 0 \| Version \| Bits [15:8]=Major ver, [7:0]=Minor ver \|
		696	* +-----------+---------------------------+---------------------------------------------+
		697	* \| 1..15 \| RESERVED \| MBZ \|
		698	* +-----------+---------------------------+---------------------------------------------+
		699	* \| 16 \| KMD_QUEUE_UPDATE_MASK_DW0 \| KMD queue mask for queues 31..0 \|
		700	* +-----------+---------------------------+---------------------------------------------+
		701	* \| 17 \| KMD_QUEUE_UPDATE_MASK_DW1 \| KMD queue mask for queues 63..32 \|
		702	* +-----------+---------------------------+---------------------------------------------+
		703	* \| 18..31 \| RESERVED \| MBZ \|
		704	* +-----------+---------------------------+---------------------------------------------+
		705	* \| 32 \| Q0CD_DW0 \| Queue 0 context LRC descriptor lower DWORD \|
		706	* +-----------+---------------------------+---------------------------------------------+
		707	* \| 33 \| Q0ContextIndex \| Context ID for Queue 0 \|
		708	* +-----------+---------------------------+---------------------------------------------+
		709	* \| 34 \| Q1CD_DW0 \| Queue 1 context LRC descriptor lower DWORD \|
		710	* +-----------+---------------------------+---------------------------------------------+
		711	* \| 35 \| Q1ContextIndex \| Context ID for Queue 1 \|
		712	* +-----------+---------------------------+---------------------------------------------+
		713	* \| ... \|... \| ... \|
		714	* +-----------+---------------------------+---------------------------------------------+
		715	* \| 158 \| Q63CD_DW0 \| Queue 63 context LRC descriptor lower DWORD \|
		716	* +-----------+---------------------------+---------------------------------------------+
		717	* \| 159 \| Q63ContextIndex \| Context ID for Queue 63 \|
		718	* +-----------+---------------------------+---------------------------------------------+
		719	* \| 160..1024 \| RESERVED \| MBZ \|
		720	* +-----------+---------------------------+---------------------------------------------+
		721	*
		722	* While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified
		723	* through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies
		724	* the CGP address. When the secondary queues are added to the group, the CGP is
		725	* updated with entry for that queue and GuC is notified through the H2G interface
		726	* XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages
		727	* with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also
		728	* sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any
		729	* error in the CGP. Only one of these CGP update messages can be outstanding
		730	* (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW*
		731	* fields indicate which queue entry is being updated in the CGP.
		732	*
		733	* The primary queue (Q0) represents the multi queue group context in GuC and
		734	* submission on any queue of the group must be through Q0 GuC interface only.
		735	*
		736	* As it is not required to register secondary queues with GuC, the secondary queue
		737	* context ids in the CGP are populated with Q0 context id.
		738	*/
		739
		740	#define CGP_VERSION_MAJOR_SHIFT 8
		741
		742	static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe,
		743	struct xe_exec_queue *q)
		744	{
		745	struct xe_exec_queue_group *group = q->multi_queue.group;
		746	u32 guc_id = group->primary->guc->id;
		747
		748	/* Currently implementing CGP version 1.0 */
		749	xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32,
		750	1 << CGP_VERSION_MAJOR_SHIFT);
		751
		752	xe_map_wr(xe, &group->cgp_bo->vmap,
		753	(32 + q->multi_queue.pos * 2) * sizeof(u32),
		754	u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0])));
		755
		756	xe_map_wr(xe, &group->cgp_bo->vmap,
		757	(33 + q->multi_queue.pos * 2) * sizeof(u32),
		758	u32, guc_id);
		759
		760	if (q->multi_queue.pos / 32) {
		761	xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32),
		762	u32, BIT(q->multi_queue.pos % 32));
		763	xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0);
		764	} else {
		765	xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32),
		766	u32, BIT(q->multi_queue.pos));
		767	xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0);
		768	}
		769	}
		770
		771	static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
		772	struct xe_exec_queue *q,
		773	const u32 *action, u32 len)
		774	{
		775	struct xe_exec_queue_group *group = q->multi_queue.group;
		776	struct xe_device *xe = guc_to_xe(guc);
		777	long ret;
		778
		779	/*
		780	* As all queues of a multi queue group use single drm scheduler
		781	* submit workqueue, CGP synchronization with GuC are serialized.
		782	* Hence, no locking is required here.
		783	* Wait for any pending CGP_SYNC_DONE response before updating the
		784	* CGP page and sending CGP_SYNC message.
		785	*
		786	* FIXME: Support VF migration
		787	*/
		788	ret = wait_event_timeout(guc->ct.wq,
		789	!READ_ONCE(group->sync_pending) \|\|
		790	xe_guc_read_stopped(guc), HZ);
		791	if (!ret \|\| xe_guc_read_stopped(guc)) {
		792	/* CGP_SYNC failed. Reset gt, cleanup the group */
		793	xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n");
		794	set_exec_queue_group_banned(q);
		795	xe_gt_reset_async(q->gt);
		796	xe_guc_exec_queue_group_trigger_cleanup(q);
		797	return;
		798	}
		799
		800	xe_lrc_set_multi_queue_priority(q->lrc[0], q->multi_queue.priority);
		801	xe_guc_exec_queue_group_cgp_update(xe, q);
		802
		803	WRITE_ONCE(group->sync_pending, true);
		804	xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1);
		805	}
		806
		807	static void __register_exec_queue_group(struct xe_guc *guc,
		808	struct xe_exec_queue *q,
		809	struct guc_ctxt_registration_info *info)
		810	{
		811	#define MAX_MULTI_QUEUE_REG_SIZE (8)
		812	u32 action[MAX_MULTI_QUEUE_REG_SIZE];
		813	int len = 0;
		814
		815	action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE;
		816	action[len++] = info->flags;
		817	action[len++] = info->context_idx;
		818	action[len++] = info->engine_class;
		819	action[len++] = info->engine_submit_mask;
		820	action[len++] = 0; /* Reserved */
		821	action[len++] = info->cgp_lo;
		822	action[len++] = info->cgp_hi;
		823
		824	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE);
		825	#undef MAX_MULTI_QUEUE_REG_SIZE
		826
		827	/*
		828	* The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a
		829	* XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
		830	* from guc.
		831	*/
		832	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
		833	}
		834
		835	static void xe_guc_exec_queue_group_add(struct xe_guc *guc,
		836	struct xe_exec_queue *q)
		837	{
		838	#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2)
		839	u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
		840	int len = 0;
		841
		842	xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q));
		843
		844	action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
		845	action[len++] = q->multi_queue.group->primary->guc->id;
		846
		847	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
		848	#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
		849
		850	/*
		851	* The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a
		852	* XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
		853	* from guc.
		854	*/
		855	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
		856	}
		857
578	static void __register_mlrc_exec_queue(struct xe_guc *guc,	858	static void __register_mlrc_exec_queue(struct xe_guc *guc,
579	struct xe_exec_queue *q,	859	struct xe_exec_queue *q,
580	struct guc_ctxt_registration_info *info)	860	struct guc_ctxt_registration_info *info)
@@ -670,6 +950,13 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) · 89 unmodified lines
670	info.flags = CONTEXT_REGISTRATION_FLAG_KMD \|	950	info.flags = CONTEXT_REGISTRATION_FLAG_KMD \|
671	FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type);	951	FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type);
672		952
		953	if (xe_exec_queue_is_multi_queue(q)) {
		954	struct xe_exec_queue_group *group = q->multi_queue.group;
		955
		956	info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo);
		957	info.cgp_hi = 0;
		958	}
		959
673	if (xe_exec_queue_is_parallel(q)) {	960	if (xe_exec_queue_is_parallel(q)) {
674	u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);	961	u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc);
675	struct iosys_map map = xe_lrc_parallel_map(lrc);	962	struct iosys_map map = xe_lrc_parallel_map(lrc);
@@ -700,11 +987,18 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) · 24 unmodified lines
700		987
701	set_exec_queue_registered(q);	988	set_exec_queue_registered(q);
702	trace_xe_exec_queue_register(q);	989	trace_xe_exec_queue_register(q);
703	if (xe_exec_queue_is_parallel(q))	990	if (xe_exec_queue_is_multi_queue_primary(q))
		991	__register_exec_queue_group(guc, q, &info);
		992	else if (xe_exec_queue_is_parallel(q))
704	__register_mlrc_exec_queue(guc, q, &info);	993	__register_mlrc_exec_queue(guc, q, &info);
705	else	994	else if (!xe_exec_queue_is_multi_queue_secondary(q))
706	__register_exec_queue(guc, &info);	995	__register_exec_queue(guc, &info);
707	init_policies(guc, q);	996
		997	if (!xe_exec_queue_is_multi_queue_secondary(q))
		998	init_policies(guc, q);
		999
		1000	if (xe_exec_queue_is_multi_queue_secondary(q))
		1001	xe_guc_exec_queue_group_add(guc, q);
708	}	1002	}
709		1003
710	static u32 wq_space_until_wrap(struct xe_exec_queue *q)	1004	static u32 wq_space_until_wrap(struct xe_exec_queue *q)
@@ -712,11 +1006,6 @@ static u32 wq_space_until_wrap(struct xe_exec_queue *q) · 1 unmodified line
712	return (WQ_SIZE - q->guc->wqi_tail);	1006	return (WQ_SIZE - q->guc->wqi_tail);
713	}	1007	}
714		1008
715	static bool vf_recovery(struct xe_guc *guc)
716	{
717	return xe_gt_recovery_pending(guc_to_gt(guc));
718	}
719
720	static inline void relaxed_ms_sleep(unsigned int delay_ms)	1009	static inline void relaxed_ms_sleep(unsigned int delay_ms)
721	{	1010	{
722	unsigned long min_us, max_us;	1011	unsigned long min_us, max_us;
@@ -845,7 +1134,7 @@ static void submit_exec_queue(struct xe_exec_queue q, struct xe_sched_job job) · 122 unmodified lines
845	if (!job->restore_replay \|\| job->last_replay) {	1134	if (!job->restore_replay \|\| job->last_replay) {
846	if (xe_exec_queue_is_parallel(q))	1135	if (xe_exec_queue_is_parallel(q))
847	wq_item_append(q);	1136	wq_item_append(q);
848	else	1137	else if (!exec_queue_idle_skip_suspend(q))
849	xe_lrc_set_ring_tail(lrc, lrc->ring.tail);	1138	xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
850	job->last_replay = false;	1139	job->last_replay = false;
851	}	1140	}
@@ -853,6 +1142,12 @@ static void submit_exec_queue(struct xe_exec_queue q, struct xe_sched_job job) · 1 unmodified line
853	if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q))	1142	if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q))
854	return;	1143	return;
855		1144
		1145	/*
		1146	* All queues in a multi-queue group will use the primary queue
		1147	* of the group to interface with GuC.
		1148	*/
		1149	q = xe_exec_queue_multi_queue_primary(q);
		1150
856	if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) {	1151	if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) {
857	action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;	1152	action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
858	action[len++] = q->guc->id;	1153	action[len++] = q->guc->id;
@@ -899,6 +1194,18 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) · 40 unmodified lines
899	trace_xe_sched_job_run(job);	1194	trace_xe_sched_job_run(job);
900		1195
901	if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) {	1196	if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) {
		1197	if (xe_exec_queue_is_multi_queue_secondary(q)) {
		1198	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
		1199
		1200	if (exec_queue_killed_or_banned_or_wedged(primary)) {
		1201	killed_or_banned_or_wedged = true;
		1202	goto run_job_out;
		1203	}
		1204
		1205	if (!exec_queue_registered(primary))
		1206	register_exec_queue(primary, GUC_CONTEXT_NORMAL);
		1207	}
		1208
902	if (!exec_queue_registered(q))	1209	if (!exec_queue_registered(q))
903	register_exec_queue(q, GUC_CONTEXT_NORMAL);	1210	register_exec_queue(q, GUC_CONTEXT_NORMAL);
904	if (!job->restore_replay)	1211	if (!job->restore_replay)
@@ -907,6 +1214,7 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) · 2 unmodified lines
907	job->restore_replay = false;	1214	job->restore_replay = false;
908	}	1215	}
909		1216
		1217	run_job_out:
910	/*	1218	/*
911	* We don't care about job-fence ordering in LR VMs because these fences	1219	* We don't care about job-fence ordering in LR VMs because these fences
912	* are never exported; they are used solely to keep jobs on the pending	1220	* are never exported; they are used solely to keep jobs on the pending
@@ -932,6 +1240,11 @@ int xe_guc_read_stopped(struct xe_guc *guc) · 19 unmodified lines
932	return atomic_read(&guc->submission_state.stopped);	1240	return atomic_read(&guc->submission_state.stopped);
933	}	1241	}
934		1242
		1243	static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
		1244	struct xe_exec_queue *q,
		1245	u32 runnable_state);
		1246	static void handle_deregister_done(struct xe_guc guc, struct xe_exec_queue q);
		1247
935	#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \	1248	#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \
936	u32 action[] = { \	1249	u32 action[] = { \
937	XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \	1250	XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \
@@ -945,7 +1258,9 @@ static void disable_scheduling_deregister(struct xe_guc *guc, · 7 unmodified lines
945	MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);	1258	MAKE_SCHED_CONTEXT_ACTION(q, DISABLE);
946	int ret;	1259	int ret;
947		1260
948	set_min_preemption_timeout(guc, q);	1261	if (!xe_exec_queue_is_multi_queue_secondary(q))
		1262	set_min_preemption_timeout(guc, q);
		1263
949	smp_rmb();	1264	smp_rmb();
950	ret = wait_event_timeout(guc->ct.wq,	1265	ret = wait_event_timeout(guc->ct.wq,
951	(!exec_queue_pending_enable(q) &&	1266	(!exec_queue_pending_enable(q) &&
@@ -973,23 +1288,12 @@ static void disable_scheduling_deregister(struct xe_guc *guc, · 21 unmodified lines
973	* Reserve space for both G2H here as the 2nd G2H is sent from a G2H	1288	* Reserve space for both G2H here as the 2nd G2H is sent from a G2H
974	* handler and we are not allowed to reserved G2H space in handlers.	1289	* handler and we are not allowed to reserved G2H space in handlers.
975	*/	1290	*/
976	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),	1291	if (xe_exec_queue_is_multi_queue_secondary(q))
977	G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +	1292	handle_multi_queue_secondary_sched_done(guc, q, 0);
978	G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
979	}
980
981	static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q)
982	{
983	struct xe_guc *guc = exec_queue_to_guc(q);
984	struct xe_device *xe = guc_to_xe(guc);
985
986	/** to wakeup xe_wait_user_fence ioctl if exec queue is reset */
987	wake_up_all(&xe->ufence_wq);
988
989	if (xe_exec_queue_is_lr(q))
990	queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);
991	else	1293	else
992	xe_sched_tdr_queue_imm(&q->guc->sched);	1294	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
		1295	G2H_LEN_DW_SCHED_CONTEXT_MODE_SET +
		1296	G2H_LEN_DW_DEREGISTER_CONTEXT, 2);
993	}	1297	}
994		1298
995	/**	1299	/**
@@ -1181,8 +1485,11 @@ static void enable_scheduling(struct xe_exec_queue *q) · 185 unmodified lines
1181	set_exec_queue_enabled(q);	1485	set_exec_queue_enabled(q);
1182	trace_xe_exec_queue_scheduling_enable(q);	1486	trace_xe_exec_queue_scheduling_enable(q);
1183		1487
1184	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),	1488	if (xe_exec_queue_is_multi_queue_secondary(q))
1185	G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);	1489	handle_multi_queue_secondary_sched_done(guc, q, 1);
		1490	else
		1491	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
		1492	G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1186		1493
1187	ret = wait_event_timeout(guc->ct.wq,	1494	ret = wait_event_timeout(guc->ct.wq,
1188	!exec_queue_pending_enable(q) \|\|	1495	!exec_queue_pending_enable(q) \|\|
@@ -1206,14 +1513,17 @@ static void disable_scheduling(struct xe_exec_queue *q, bool immediate) · 17 unmodified lines
1206	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));	1513	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
1207	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));	1514	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
1208		1515
1209	if (immediate)	1516	if (immediate && !xe_exec_queue_is_multi_queue_secondary(q))
1210	set_min_preemption_timeout(guc, q);	1517	set_min_preemption_timeout(guc, q);
1211	clear_exec_queue_enabled(q);	1518	clear_exec_queue_enabled(q);
1212	set_exec_queue_pending_disable(q);	1519	set_exec_queue_pending_disable(q);
1213	trace_xe_exec_queue_scheduling_disable(q);	1520	trace_xe_exec_queue_scheduling_disable(q);
1214		1521
1215	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),	1522	if (xe_exec_queue_is_multi_queue_secondary(q))
1216	G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);	1523	handle_multi_queue_secondary_sched_done(guc, q, 0);
		1524	else
		1525	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
		1526	G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1217	}	1527	}
1218		1528
1219	static void __deregister_exec_queue(struct xe_guc guc, struct xe_exec_queue q)	1529	static void __deregister_exec_queue(struct xe_guc guc, struct xe_exec_queue q)
@@ -1231,8 +1541,11 @@ static void __deregister_exec_queue(struct xe_guc guc, struct xe_exec_queue q) · 11 unmodified lines
1231	set_exec_queue_destroyed(q);	1541	set_exec_queue_destroyed(q);
1232	trace_xe_exec_queue_deregister(q);	1542	trace_xe_exec_queue_deregister(q);
1233		1543
1234	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),	1544	if (xe_exec_queue_is_multi_queue_secondary(q))
1235	G2H_LEN_DW_DEREGISTER_CONTEXT, 1);	1545	handle_deregister_done(guc, q);
		1546	else
		1547	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
		1548	G2H_LEN_DW_DEREGISTER_CONTEXT, 1);
1236	}	1549	}
1237		1550
1238	static enum drm_gpu_sched_stat	1551	static enum drm_gpu_sched_stat
@@ -1245,7 +1558,6 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) · 6 unmodified lines
1245	struct xe_guc *guc = exec_queue_to_guc(q);	1558	struct xe_guc *guc = exec_queue_to_guc(q);
1246	const char *process_name = "no process";	1559	const char *process_name = "no process";
1247	struct xe_device *xe = guc_to_xe(guc);	1560	struct xe_device *xe = guc_to_xe(guc);
1248	unsigned int fw_ref;
1249	int err = -ETIME;	1561	int err = -ETIME;
1250	pid_t pid = -1;	1562	pid_t pid = -1;
1251	int i = 0;	1563	int i = 0;
@@ -1271,6 +1583,19 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) · 19 unmodified lines
1271	exec_queue_killed_or_banned_or_wedged(q) \|\|	1583	exec_queue_killed_or_banned_or_wedged(q) \|\|
1272	exec_queue_destroyed(q);	1584	exec_queue_destroyed(q);
1273		1585
		1586	/* Skip timeout check if multi-queue group is banned */
		1587	if (xe_exec_queue_is_multi_queue(q) &&
		1588	READ_ONCE(q->multi_queue.group->banned))
		1589	skip_timeout_check = true;
		1590
		1591	/*
		1592	* FIXME: In multi-queue scenario, the TDR must ensure that the whole
		1593	* multi-queue group is off the HW before signaling the fences to avoid
		1594	* possible memory corruptions. This means disabling scheduling on the
		1595	* primary queue before or during the secondary queue's TDR. Need to
		1596	* implement this in least obtrusive way.
		1597	*/
		1598
1274	/*	1599	/*
1275	* If devcoredump not captured and GuC capture for the job is not ready	1600	* If devcoredump not captured and GuC capture for the job is not ready
1276	* do manual capture first and decide later if we need to use it	1601	* do manual capture first and decide later if we need to use it
@@ -1278,13 +1603,11 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) · 1 unmodified line
1278	if (!exec_queue_killed(q) && !xe->devcoredump.captured &&	1603	if (!exec_queue_killed(q) && !xe->devcoredump.captured &&
1279	!xe_guc_capture_get_matching_and_lock(q)) {	1604	!xe_guc_capture_get_matching_and_lock(q)) {
1280	/* take force wake before engine register manual capture */	1605	/* take force wake before engine register manual capture */
1281	fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);	1606	CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
1282	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))	1607	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
1283	xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n");	1608	xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n");
1284		1609
1285	xe_engine_snapshot_capture_for_queue(q);	1610	xe_engine_snapshot_capture_for_queue(q);
1286
1287	xe_force_wake_put(gt_to_fw(q->gt), fw_ref);
1288	}	1611	}
1289		1612
1290	/*	1613	/*
@@ -1425,7 +1748,10 @@ trigger_reset: · 134 unmodified lines
1425	xe_sched_add_pending_job(sched, job);	1748	xe_sched_add_pending_job(sched, job);
1426	xe_sched_submission_start(sched);	1749	xe_sched_submission_start(sched);
1427		1750
1428	xe_guc_exec_queue_trigger_cleanup(q);	1751	if (xe_exec_queue_is_multi_queue(q))
		1752	xe_guc_exec_queue_group_trigger_cleanup(q);
		1753	else
		1754	xe_guc_exec_queue_trigger_cleanup(q);
1429		1755
1430	/* Mark all outstanding jobs as bad, thus completing them */	1756	/* Mark all outstanding jobs as bad, thus completing them */
1431	spin_lock(&sched->base.job_list_lock);	1757	spin_lock(&sched->base.job_list_lock);
@@ -1475,17 +1801,23 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w) · 43 unmodified lines
1475	struct xe_exec_queue *q = ge->q;	1801	struct xe_exec_queue *q = ge->q;
1476	struct xe_guc *guc = exec_queue_to_guc(q);	1802	struct xe_guc *guc = exec_queue_to_guc(q);
1477		1803
1478	xe_pm_runtime_get(guc_to_xe(guc));	1804	guard(xe_pm_runtime)(guc_to_xe(guc));
1479	trace_xe_exec_queue_destroy(q);	1805	trace_xe_exec_queue_destroy(q);
1480		1806
		1807	if (xe_exec_queue_is_multi_queue_secondary(q)) {
		1808	struct xe_exec_queue_group *group = q->multi_queue.group;
		1809
		1810	mutex_lock(&group->list_lock);
		1811	list_del(&q->multi_queue.link);
		1812	mutex_unlock(&group->list_lock);
		1813	}
		1814
1481	if (xe_exec_queue_is_lr(q))	1815	if (xe_exec_queue_is_lr(q))
1482	cancel_work_sync(&ge->lr_tdr);	1816	cancel_work_sync(&ge->lr_tdr);
1483	/* Confirm no work left behind accessing device structures */	1817	/* Confirm no work left behind accessing device structures */
1484	cancel_delayed_work_sync(&ge->sched.base.work_tdr);	1818	cancel_delayed_work_sync(&ge->sched.base.work_tdr);
1485		1819
1486	xe_exec_queue_fini(q);	1820	xe_exec_queue_fini(q);
1487
1488	xe_pm_runtime_put(guc_to_xe(guc));
1489	}	1821	}
1490		1822
1491	static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)	1823	static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)
@@ -1590,9 +1922,10 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) · 98 unmodified lines
1590	{	1922	{
1591	struct xe_exec_queue *q = msg->private_data;	1923	struct xe_exec_queue *q = msg->private_data;
1592	struct xe_guc *guc = exec_queue_to_guc(q);	1924	struct xe_guc *guc = exec_queue_to_guc(q);
		1925	bool idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q);
1593		1926
1594	if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) &&	1927	if (!idle_skip_suspend && guc_exec_queue_allowed_to_change_state(q) &&
1595	exec_queue_enabled(q)) {	1928	!exec_queue_suspended(q) && exec_queue_enabled(q)) {
1596	wait_event(guc->ct.wq, vf_recovery(guc) \|\|	1929	wait_event(guc->ct.wq, vf_recovery(guc) \|\|
1597	((q->guc->resume_time != RESUME_PENDING \|\|	1930	((q->guc->resume_time != RESUME_PENDING \|\|
1598	xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q)));	1931	xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q)));
@@ -1611,11 +1944,33 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) · 12 unmodified lines
1611	disable_scheduling(q, false);	1944	disable_scheduling(q, false);
1612	}	1945	}
1613	} else if (q->guc->suspend_pending) {	1946	} else if (q->guc->suspend_pending) {
		1947	if (idle_skip_suspend)
		1948	set_exec_queue_idle_skip_suspend(q);
1614	set_exec_queue_suspended(q);	1949	set_exec_queue_suspended(q);
1615	suspend_fence_signal(q);	1950	suspend_fence_signal(q);
1616	}	1951	}
1617	}	1952	}
1618		1953
		1954	static void sched_context(struct xe_exec_queue *q)
		1955	{
		1956	struct xe_guc *guc = exec_queue_to_guc(q);
		1957	struct xe_lrc *lrc = q->lrc[0];
		1958	u32 action[] = {
		1959	XE_GUC_ACTION_SCHED_CONTEXT,
		1960	q->guc->id,
		1961	};
		1962
		1963	xe_gt_assert(guc_to_gt(guc), !xe_exec_queue_is_parallel(q));
		1964	xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q));
		1965	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
		1966	xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q));
		1967
		1968	trace_xe_exec_queue_submit(q);
		1969
		1970	xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
		1971	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
		1972	}
		1973
1619	static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)	1974	static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)
1620	{	1975	{
1621	struct xe_exec_queue *q = msg->private_data;	1976	struct xe_exec_queue *q = msg->private_data;
@@ -1623,19 +1978,53 @@ static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) · 1 unmodified line
1623	if (guc_exec_queue_allowed_to_change_state(q)) {	1978	if (guc_exec_queue_allowed_to_change_state(q)) {
1624	clear_exec_queue_suspended(q);	1979	clear_exec_queue_suspended(q);
1625	if (!exec_queue_enabled(q)) {	1980	if (!exec_queue_enabled(q)) {
		1981	if (exec_queue_idle_skip_suspend(q)) {
		1982	struct xe_lrc *lrc = q->lrc[0];
		1983
		1984	clear_exec_queue_idle_skip_suspend(q);
		1985	xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
		1986	}
1626	q->guc->resume_time = RESUME_PENDING;	1987	q->guc->resume_time = RESUME_PENDING;
1627	set_exec_queue_pending_resume(q);	1988	set_exec_queue_pending_resume(q);
1628	enable_scheduling(q);	1989	enable_scheduling(q);
		1990	} else if (exec_queue_idle_skip_suspend(q)) {
		1991	clear_exec_queue_idle_skip_suspend(q);
		1992	sched_context(q);
1629	}	1993	}
1630	} else {	1994	} else {
1631	clear_exec_queue_suspended(q);	1995	clear_exec_queue_suspended(q);
		1996	clear_exec_queue_idle_skip_suspend(q);
		1997	}
		1998	}
		1999
		2000	static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg)
		2001	{
		2002	struct xe_exec_queue *q = msg->private_data;
		2003
		2004	if (guc_exec_queue_allowed_to_change_state(q)) {
		2005	#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2)
		2006	struct xe_guc *guc = exec_queue_to_guc(q);
		2007	struct xe_exec_queue_group *group = q->multi_queue.group;
		2008	u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
		2009	int len = 0;
		2010
		2011	action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
		2012	action[len++] = group->primary->guc->id;
		2013
		2014	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
		2015	#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
		2016
		2017	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
1632	}	2018	}
		2019
		2020	kfree(msg);
1633	}	2021	}
1634		2022
1635	#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */	2023	#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */
1636	#define SET_SCHED_PROPS 2	2024	#define SET_SCHED_PROPS 2
1637	#define SUSPEND 3	2025	#define SUSPEND 3
1638	#define RESUME 4	2026	#define RESUME 4
		2027	#define SET_MULTI_QUEUE_PRIORITY 5
1639	#define OPCODE_MASK 0xf	2028	#define OPCODE_MASK 0xf
1640	#define MSG_LOCKED BIT(8)	2029	#define MSG_LOCKED BIT(8)
1641	#define MSG_HEAD BIT(9)	2030	#define MSG_HEAD BIT(9)
@@ -1659,6 +2048,9 @@ static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) · 17 unmodified lines
1659	case RESUME:	2048	case RESUME:
1660	__guc_exec_queue_process_msg_resume(msg);	2049	__guc_exec_queue_process_msg_resume(msg);
1661	break;	2050	break;
		2051	case SET_MULTI_QUEUE_PRIORITY:
		2052	__guc_exec_queue_process_msg_set_multi_queue_priority(msg);
		2053	break;
1662	default:	2054	default:
1663	XE_WARN_ON("Unknown message type");	2055	XE_WARN_ON("Unknown message type");
1664	}	2056	}
@@ -1680,6 +2072,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) · 15 unmodified lines
1680	{	2072	{
1681	struct xe_gpu_scheduler *sched;	2073	struct xe_gpu_scheduler *sched;
1682	struct xe_guc *guc = exec_queue_to_guc(q);	2074	struct xe_guc *guc = exec_queue_to_guc(q);
		2075	struct workqueue_struct *submit_wq = NULL;
1683	struct xe_guc_exec_queue *ge;	2076	struct xe_guc_exec_queue *ge;
1684	long timeout;	2077	long timeout;
1685	int err, i;	2078	int err, i;
@@ -1700,8 +2093,20 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) · 14 unmodified lines
1700		2093
1701	timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT :	2094	timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT :
1702	msecs_to_jiffies(q->sched_props.job_timeout_ms);	2095	msecs_to_jiffies(q->sched_props.job_timeout_ms);
		2096
		2097	/*
		2098	* Use primary queue's submit_wq for all secondary queues of a
		2099	* multi queue group. This serialization avoids any locking around
		2100	* CGP synchronization with GuC.
		2101	*/
		2102	if (xe_exec_queue_is_multi_queue_secondary(q)) {
		2103	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
		2104
		2105	submit_wq = primary->guc->sched.base.submit_wq;
		2106	}
		2107
1703	err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,	2108	err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
1704	NULL, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,	2109	submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
1705	timeout, guc_to_gt(guc)->ordered_wq, NULL,	2110	timeout, guc_to_gt(guc)->ordered_wq, NULL,
1706	q->name, gt_to_xe(q->gt)->drm.dev);	2111	q->name, gt_to_xe(q->gt)->drm.dev);
1707	if (err)	2112	if (err)
@@ -1730,7 +2135,23 @@ static int guc_exec_queue_init(struct xe_exec_queue *q) · 22 unmodified lines
1730		2135
1731	xe_exec_queue_assign_name(q, q->guc->id);	2136	xe_exec_queue_assign_name(q, q->guc->id);
1732		2137
1733	trace_xe_exec_queue_create(q);	2138	/*
		2139	* Maintain secondary queues of the multi queue group in a list
		2140	* for handling dependencies across the queues in the group.
		2141	*/
		2142	if (xe_exec_queue_is_multi_queue_secondary(q)) {
		2143	struct xe_exec_queue_group *group = q->multi_queue.group;
		2144
		2145	INIT_LIST_HEAD(&q->multi_queue.link);
		2146	mutex_lock(&group->list_lock);
		2147	list_add_tail(&q->multi_queue.link, &group->list);
		2148	mutex_unlock(&group->list_lock);
		2149	}
		2150
		2151	if (xe_exec_queue_is_multi_queue(q))
		2152	trace_xe_exec_queue_create_multi_queue(q);
		2153	else
		2154	trace_xe_exec_queue_create(q);
1734		2155
1735	return 0;	2156	return 0;
1736		2157
@@ -1862,6 +2283,27 @@ static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, · 125 unmodified lines
1862	return 0;	2283	return 0;
1863	}	2284	}
1864		2285
		2286	static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q,
		2287	enum xe_multi_queue_priority priority)
		2288	{
		2289	struct xe_sched_msg *msg;
		2290
		2291	xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q));
		2292
		2293	if (q->multi_queue.priority == priority \|\|
		2294	exec_queue_killed_or_banned_or_wedged(q))
		2295	return 0;
		2296
		2297	msg = kmalloc(sizeof(*msg), GFP_KERNEL);
		2298	if (!msg)
		2299	return -ENOMEM;
		2300
		2301	q->multi_queue.priority = priority;
		2302	guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY);
		2303
		2304	return 0;
		2305	}
		2306
1865	static int guc_exec_queue_suspend(struct xe_exec_queue *q)	2307	static int guc_exec_queue_suspend(struct xe_exec_queue *q)
1866	{	2308	{
1867	struct xe_gpu_scheduler *sched = &q->guc->sched;	2309	struct xe_gpu_scheduler *sched = &q->guc->sched;
@@ -1936,6 +2378,10 @@ static void guc_exec_queue_resume(struct xe_exec_queue *q) · 68 unmodified lines
1936		2378
1937	static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)	2379	static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
1938	{	2380	{
		2381	if (xe_exec_queue_is_multi_queue_secondary(q) &&
		2382	guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q)))
		2383	return true;
		2384
1939	return exec_queue_reset(q) \|\| exec_queue_killed_or_banned_or_wedged(q);	2385	return exec_queue_reset(q) \|\| exec_queue_killed_or_banned_or_wedged(q);
1940	}	2386	}
1941		2387
@@ -1953,6 +2399,7 @@ static const struct xe_exec_queue_ops guc_exec_queue_ops = { · 11 unmodified lines
1953	.set_priority = guc_exec_queue_set_priority,	2399	.set_priority = guc_exec_queue_set_priority,
1954	.set_timeslice = guc_exec_queue_set_timeslice,	2400	.set_timeslice = guc_exec_queue_set_timeslice,
1955	.set_preempt_timeout = guc_exec_queue_set_preempt_timeout,	2401	.set_preempt_timeout = guc_exec_queue_set_preempt_timeout,
		2402	.set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority,
1956	.suspend = guc_exec_queue_suspend,	2403	.suspend = guc_exec_queue_suspend,
1957	.suspend_wait = guc_exec_queue_suspend_wait,	2404	.suspend_wait = guc_exec_queue_suspend_wait,
1958	.resume = guc_exec_queue_resume,	2405	.resume = guc_exec_queue_resume,
@@ -2202,6 +2649,22 @@ void xe_guc_submit_pause(struct xe_guc *guc) · 243 unmodified lines
2202	struct xe_exec_queue *q;	2649	struct xe_exec_queue *q;
2203	unsigned long index;	2650	unsigned long index;
2204		2651
		2652	mutex_lock(&guc->submission_state.lock);
		2653	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
		2654	xe_sched_submission_stop(&q->guc->sched);
		2655	mutex_unlock(&guc->submission_state.lock);
		2656	}
		2657
		2658	/**
		2659	* xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF.
		2660	* @guc: the &xe_guc struct instance whose scheduler is to be disabled
		2661	*/
		2662	void xe_guc_submit_pause_vf(struct xe_guc *guc)
		2663	{
		2664	struct xe_exec_queue *q;
		2665	unsigned long index;
		2666
		2667	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
2205	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));	2668	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
2206		2669
2207	mutex_lock(&guc->submission_state.lock);	2670	mutex_lock(&guc->submission_state.lock);
@@ -2293,14 +2756,15 @@ static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, · 85 unmodified lines
2293	}	2756	}
2294		2757
2295	/**	2758	/**
2296	* xe_guc_submit_unpause_prepare - Prepare unpause submission tasks on given GuC.	2759	* xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF.
2297	* @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause	2760	* @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause
2298	*/	2761	*/
2299	void xe_guc_submit_unpause_prepare(struct xe_guc *guc)	2762	void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc)
2300	{	2763	{
2301	struct xe_exec_queue *q;	2764	struct xe_exec_queue *q;
2302	unsigned long index;	2765	unsigned long index;
2303		2766
		2767	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
2304	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));	2768	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
2305		2769
2306	mutex_lock(&guc->submission_state.lock);	2770	mutex_lock(&guc->submission_state.lock);
@@ -2376,6 +2840,23 @@ void xe_guc_submit_unpause(struct xe_guc *guc) · 69 unmodified lines
2376	struct xe_exec_queue *q;	2840	struct xe_exec_queue *q;
2377	unsigned long index;	2841	unsigned long index;
2378		2842
		2843	mutex_lock(&guc->submission_state.lock);
		2844	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
		2845	xe_sched_submission_start(&q->guc->sched);
		2846	mutex_unlock(&guc->submission_state.lock);
		2847	}
		2848
		2849	/**
		2850	* xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF.
		2851	* @guc: the &xe_guc struct instance whose scheduler is to be enabled
		2852	*/
		2853	void xe_guc_submit_unpause_vf(struct xe_guc *guc)
		2854	{
		2855	struct xe_exec_queue *q;
		2856	unsigned long index;
		2857
		2858	xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc)));
		2859
2379	mutex_lock(&guc->submission_state.lock);	2860	mutex_lock(&guc->submission_state.lock);
2380	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {	2861	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
2381	/*	2862	/*
@@ -2452,7 +2933,11 @@ static void deregister_exec_queue(struct xe_guc guc, struct xe_exec_queue q) · 70 unmodified lines
2452		2933
2453	trace_xe_exec_queue_deregister(q);	2934	trace_xe_exec_queue_deregister(q);
2454		2935
2455	xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));	2936	if (xe_exec_queue_is_multi_queue_secondary(q))
		2937	handle_deregister_done(guc, q);
		2938	else
		2939	xe_guc_ct_send_g2h_handler(&guc->ct, action,
		2940	ARRAY_SIZE(action));
2456	}	2941	}
2457		2942
2458	static void handle_sched_done(struct xe_guc guc, struct xe_exec_queue q,	2943	static void handle_sched_done(struct xe_guc guc, struct xe_exec_queue q,
@@ -2502,6 +2987,16 @@ static void handle_sched_done(struct xe_guc guc, struct xe_exec_queue q, · 43 unmodified lines
2502	}	2987	}
2503	}	2988	}
2504		2989
		2990	static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc,
		2991	struct xe_exec_queue *q,
		2992	u32 runnable_state)
		2993	{
		2994	/* Take CT lock here as handle_sched_done() do send a h2g message */
		2995	mutex_lock(&guc->ct.lock);
		2996	handle_sched_done(guc, q, runnable_state);
		2997	mutex_unlock(&guc->ct.lock);
		2998	}
		2999
2505	int xe_guc_sched_done_handler(struct xe_guc guc, u32 msg, u32 len)	3000	int xe_guc_sched_done_handler(struct xe_guc guc, u32 msg, u32 len)
2506	{	3001	{
2507	struct xe_exec_queue *q;	3002	struct xe_exec_queue *q;
@@ -2585,8 +3080,9 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc guc, u32 msg, u32 len) · 77 unmodified lines
2585	if (unlikely(!q))	3080	if (unlikely(!q))
2586	return -EPROTO;	3081	return -EPROTO;
2587		3082
2588	xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d",	3083	xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x",
2589	xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);	3084	xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id,
		3085	atomic_read(&q->guc->state));
2590		3086
2591	trace_xe_exec_queue_reset(q);	3087	trace_xe_exec_queue_reset(q);
2592		3088
@@ -2596,9 +3092,7 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc guc, u32 msg, u32 len) · 3 unmodified lines
2596	* jobs by setting timeout of the job to the minimum value kicking	3092	* jobs by setting timeout of the job to the minimum value kicking
2597	* guc_exec_queue_timedout_job.	3093	* guc_exec_queue_timedout_job.
2598	*/	3094	*/
2599	set_exec_queue_reset(q);	3095	xe_guc_exec_queue_reset_trigger_cleanup(q);
2600	if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
2601	xe_guc_exec_queue_trigger_cleanup(q);
2602		3096
2603	return 0;	3097	return 0;
2604	}	3098	}
@@ -2666,20 +3160,18 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc guc, u32 msg, · 61 unmodified lines
2666	* See bspec 54047 and 72187 for details.	3160	* See bspec 54047 and 72187 for details.
2667	*/	3161	*/
2668	if (type != XE_GUC_CAT_ERR_TYPE_INVALID)	3162	if (type != XE_GUC_CAT_ERR_TYPE_INVALID)
2669	xe_gt_dbg(gt,	3163	xe_gt_info(gt,
2670	"Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d",	3164	"Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d",
2671	type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);	3165	type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
2672	else	3166	else
2673	xe_gt_dbg(gt,	3167	xe_gt_info(gt,
2674	"Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d",	3168	"Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d",
2675	xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);	3169	xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id);
2676		3170
2677	trace_xe_exec_queue_memory_cat_error(q);	3171	trace_xe_exec_queue_memory_cat_error(q);
2678		3172
2679	/* Treat the same as engine reset */	3173	/* Treat the same as engine reset */
2680	set_exec_queue_reset(q);	3174	xe_guc_exec_queue_reset_trigger_cleanup(q);
2681	if (!exec_queue_banned(q) && !exec_queue_check_timeout(q))
2682	xe_guc_exec_queue_trigger_cleanup(q);
2683		3175
2684	return 0;	3176	return 0;
2685	}	3177	}
@@ -2706,6 +3198,73 @@ int xe_guc_exec_queue_reset_failure_handler(struct xe_guc guc, u32 msg, u32 le · 20 unmodified lines
2706	return 0;	3198	return 0;
2707	}	3199	}
2708		3200
		3201	int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc guc, u32 msg,
		3202	u32 len)
		3203	{
		3204	struct xe_gt *gt = guc_to_gt(guc);
		3205	struct xe_device *xe = guc_to_xe(guc);
		3206	struct xe_exec_queue *q;
		3207	u32 guc_id = msg[2];
		3208
		3209	if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) {
		3210	drm_err(&xe->drm, "Invalid length %u", len);
		3211	return -EPROTO;
		3212	}
		3213
		3214	q = g2h_exec_queue_lookup(guc, guc_id);
		3215	if (unlikely(!q))
		3216	return -EPROTO;
		3217
		3218	xe_gt_dbg(gt,
		3219	"CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x",
		3220	msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]);
		3221
		3222	trace_xe_exec_queue_cgp_context_error(q);
		3223
		3224	/* Treat the same as engine reset */
		3225	xe_guc_exec_queue_reset_trigger_cleanup(q);
		3226
		3227	return 0;
		3228	}
		3229
		3230	/**
		3231	* xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler
		3232	* @guc: guc
		3233	* @msg: message indicating CGP sync done
		3234	* @len: length of message
		3235	*
		3236	* Set multi queue group's sync_pending flag to false and wakeup anyone waiting
		3237	* for CGP synchronization to complete.
		3238	*
		3239	* Return: 0 on success, -EPROTO for malformed messages.
		3240	*/
		3241	int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc guc, u32 msg, u32 len)
		3242	{
		3243	struct xe_device *xe = guc_to_xe(guc);
		3244	struct xe_exec_queue *q;
		3245	u32 guc_id = msg[0];
		3246
		3247	if (unlikely(len < 1)) {
		3248	drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len);
		3249	return -EPROTO;
		3250	}
		3251
		3252	q = g2h_exec_queue_lookup(guc, guc_id);
		3253	if (unlikely(!q))
		3254	return -EPROTO;
		3255
		3256	if (!xe_exec_queue_is_multi_queue_primary(q)) {
		3257	drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response");
		3258	return -EPROTO;
		3259	}
		3260
		3261	/* Wakeup the serialized cgp update wait */
		3262	WRITE_ONCE(q->multi_queue.group->sync_pending, false);
		3263	xe_guc_ct_wake_waiters(&guc->ct);
		3264
		3265	return 0;
		3266	}
		3267
2709	static void	3268	static void
2710	guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q,	3269	guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q,
2711	struct xe_guc_submit_exec_queue_snapshot *snapshot)	3270	struct xe_guc_submit_exec_queue_snapshot *snapshot)
@@ -2805,6 +3364,11 @@ xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) · 93 unmodified lines
2805	if (snapshot->parallel_execution)	3364	if (snapshot->parallel_execution)
2806	guc_exec_queue_wq_snapshot_capture(q, snapshot);	3365	guc_exec_queue_wq_snapshot_capture(q, snapshot);
2807		3366
		3367	if (xe_exec_queue_is_multi_queue(q)) {
		3368	snapshot->multi_queue.valid = true;
		3369	snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id;
		3370	snapshot->multi_queue.pos = q->multi_queue.pos;
		3371	}
2808	spin_lock(&sched->base.job_list_lock);	3372	spin_lock(&sched->base.job_list_lock);
2809	snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list);	3373	snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list);
2810	snapshot->pending_list = kmalloc_array(snapshot->pending_list_size,	3374	snapshot->pending_list = kmalloc_array(snapshot->pending_list_size,
@@ -2887,6 +3451,11 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps · 76 unmodified lines
2887	if (snapshot->parallel_execution)	3451	if (snapshot->parallel_execution)
2888	guc_exec_queue_wq_snapshot_print(snapshot, p);	3452	guc_exec_queue_wq_snapshot_print(snapshot, p);
2889		3453
		3454	if (snapshot->multi_queue.valid) {
		3455	drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary);
		3456	drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos);
		3457	}
		3458
2890	for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size;	3459	for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size;
2891	i++)	3460	i++)
2892	drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n",	3461	drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n",

drivers/gpu/drm/xe/xe_guc_submit.h+7−2

 void xe_guc_submit_stop(struct xe_guc *guc);
 int xe_guc_submit_start(struct xe_guc *guc);
 void xe_guc_submit_pause(struct xe_guc *guc);
-void xe_guc_submit_unpause(struct xe_guc *guc);
-void xe_guc_submit_unpause_prepare(struct xe_guc *guc);
 void xe_guc_submit_pause_abort(struct xe_guc *guc);
+void xe_guc_submit_pause_vf(struct xe_guc *guc);
+void xe_guc_submit_unpause(struct xe_guc *guc);
+void xe_guc_submit_unpause_vf(struct xe_guc *guc);
+void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc);
 void xe_guc_submit_wedge(struct xe_guc *guc);
 int xe_guc_read_stopped(struct xe_guc *guc);
 					       u32 len);
 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len);
 int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len);
+int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
+int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg,
+						u32 len);
 struct xe_guc_submit_exec_queue_snapshot *
 xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q);

drivers/gpu/drm/xe/xe_guc_submit_types.h+13−0

@@ -135,6 +135,19 @@ struct xe_guc_submit_exec_queue_snapshot { · 134 unmodified lines
135	u32 wq[WQ_SIZE / sizeof(u32)];	135	u32 wq[WQ_SIZE / sizeof(u32)];
136	} parallel;	136	} parallel;
137		137
		138	/** @multi_queue: snapshot of the multi queue information */
		139	struct {
		140	/**
		141	* @multi_queue.primary: GuC id of the primary exec queue
		142	* of the multi queue group.
		143	*/
		144	u32 primary;
		145	/** @multi_queue.pos: Position of the exec queue within the multi queue group */
		146	u8 pos;
		147	/** @valid: The exec queue is part of a multi queue group */
		148	bool valid;
		149	} multi_queue;
		150
138	/** @pending_list_size: Size of the pending list snapshot array */	151	/** @pending_list_size: Size of the pending list snapshot array */
139	int pending_list_size;	152	int pending_list_size;
140	/** @pending_list: snapshot of the pending list info */	153	/** @pending_list: snapshot of the pending list info */

drivers/gpu/drm/xe/xe_guc_tlb_inval.c+31−10

 #include "xe_guc_tlb_inval.h"
 #include "xe_force_wake.h"
 #include "xe_mmio.h"
+#include "xe_sa.h"
 #include "xe_tlb_inval.h"
 #include "regs/xe_guc_regs.h"
 			      G2H_LEN_DW_TLB_INVALIDATE, 1);
 }
-#define MAKE_INVAL_OP(type)	((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
+#define MAKE_INVAL_OP_FLUSH(type, flush_cache)	((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
 		XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
-		XE_GUC_TLB_INVAL_FLUSH_CACHE)
+		(flush_cache ? \
+		XE_GUC_TLB_INVAL_FLUSH_CACHE : 0))
+#define MAKE_INVAL_OP(type)	MAKE_INVAL_OP_FLUSH(type, true)
 static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno)
 {
 		return send_tlb_inval(guc, action, ARRAY_SIZE(action));
 	} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
 		struct xe_mmio *mmio = &gt->mmio;
-		unsigned int fw_ref;
 		if (IS_SRIOV_VF(xe))
 			return -ECANCELED;
-		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
 		if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
 			xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1,
 					PVC_GUC_TLB_INV_DESC1_INVALIDATE);
 			xe_mmio_write32(mmio, GUC_TLB_INV_CR,
 					GUC_TLB_INV_CR_INVALIDATE);
 		}
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	}
 	return -ECANCELED;
 }
+static int send_page_reclaim(struct xe_guc *guc, u32 seqno,
+			     u64 gpu_addr)
+{
+	u32 action[] = {
+		XE_GUC_ACTION_PAGE_RECLAMATION,
+		seqno,
+		lower_32_bits(gpu_addr),
+		upper_32_bits(gpu_addr),
+	};
+	return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
+			      G2H_LEN_DW_PAGE_RECLAMATION, 1);
+}
 /*
  * Ensure that roundup_pow_of_two(length) doesn't overflow.
  * Note that roundup_pow_of_two() operates on unsigned long,
 #define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))
 static int send_tlb_inval_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
-				u64 start, u64 end, u32 asid)
+				u64 start, u64 end, u32 asid,
+				struct drm_suballoc *prl_sa)
 {
 #define MAX_TLB_INVALIDATION_LEN	7
 	struct xe_guc *guc = tlb_inval->private;
 	struct xe_gt *gt = guc_to_gt(guc);
 	u32 action[MAX_TLB_INVALIDATION_LEN];
 	u64 length = end - start;
-	int len = 0;
+	int len = 0, err;
 	if (guc_to_xe(guc)->info.force_execlist)
 		return -ECANCELED;
 	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
-	action[len++] = seqno;
+	action[len++] = !prl_sa ? seqno : TLB_INVALIDATION_SEQNO_INVALID;
 	if (!gt_to_xe(gt)->info.has_range_tlb_inval ||
 	    length > MAX_RANGE_TLB_INVALIDATION_LENGTH) {
 		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
 						    ilog2(SZ_2M) + 1)));
 		xe_gt_assert(gt, IS_ALIGNED(start, length));
-		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
+		/* Flush on NULL case, Media is not required to modify flush due to no PPC so NOP */
+		action[len++] = MAKE_INVAL_OP_FLUSH(XE_GUC_TLB_INVAL_PAGE_SELECTIVE, !prl_sa);
 		action[len++] = asid;
 		action[len++] = lower_32_bits(start);
 		action[len++] = upper_32_bits(start);
 	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
-	return send_tlb_inval(guc, action, len);
+	err = send_tlb_inval(guc, action, len);
+	if (!err && prl_sa)
+		err = send_page_reclaim(guc, seqno, xe_sa_bo_gpu_addr(prl_sa));
+	return err;
 }
 static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval)

drivers/gpu/drm/xe/xe_huc.c+2−5

 void xe_huc_print_info(struct xe_huc *huc, struct drm_printer *p)
 {
 	struct xe_gt *gt = huc_to_gt(huc);
-	unsigned int fw_ref;
 	xe_uc_fw_print(&huc->fw, p);
 	if (!xe_uc_fw_is_enabled(&huc->fw))
 		return;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return;
 	drm_printf(p, "\nHuC status: 0x%08x\n",
 		   xe_mmio_read32(&gt->mmio, HUC_KERNEL_LOAD_INFO));
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 }

drivers/gpu/drm/xe/xe_huc_debugfs.c+1−2

@@ -37,9 +37,8 @@ static int huc_info(struct seq_file m, void data) · 36 unmodified lines
37	struct xe_device *xe = huc_to_xe(huc);	37	struct xe_device *xe = huc_to_xe(huc);
38	struct drm_printer p = drm_seq_file_printer(m);	38	struct drm_printer p = drm_seq_file_printer(m);
39		39
40	xe_pm_runtime_get(xe);	40	guard(xe_pm_runtime)(xe);
41	xe_huc_print_info(huc, &p);	41	xe_huc_print_info(huc, &p);
42	xe_pm_runtime_put(xe);
43		42
44	return 0;	43	return 0;
45	}	44	}

drivers/gpu/drm/xe/xe_hw_engine_class_sysfs.c+6−10

 {
 	struct xe_device *xe = kobj_to_xe(kobj);
 	struct kobj_attribute *kattr;
-	ssize_t ret = -EIO;
 	kattr = container_of(attr, struct kobj_attribute, attr);
 	if (kattr->show) {
-		xe_pm_runtime_get(xe);
+		guard(xe_pm_runtime)(xe);
-		ret = kattr->show(kobj, kattr, buf);
+		return kattr->show(kobj, kattr, buf);
-		xe_pm_runtime_put(xe);
 	}
-	return ret;
+	return -EIO;
 }
 static ssize_t xe_hw_engine_class_sysfs_attr_store(struct kobject *kobj,
 {
 	struct xe_device *xe = kobj_to_xe(kobj);
 	struct kobj_attribute *kattr;
-	ssize_t ret = -EIO;
 	kattr = container_of(attr, struct kobj_attribute, attr);
 	if (kattr->store) {
-		xe_pm_runtime_get(xe);
+		guard(xe_pm_runtime)(xe);
-		ret = kattr->store(kobj, kattr, buf, count);
+		return kattr->store(kobj, kattr, buf, count);
-		xe_pm_runtime_put(xe);
 	}
-	return ret;
+	return -EIO;
 }
 static const struct sysfs_ops xe_hw_engine_class_sysfs_ops = {

drivers/gpu/drm/xe/xe_hw_engine_group.c+71−8

@@ -9,7 +9,9 @@ · 8 unmodified lines
9	#include "xe_device.h"	9	#include "xe_device.h"
10	#include "xe_exec_queue.h"	10	#include "xe_exec_queue.h"
11	#include "xe_gt.h"	11	#include "xe_gt.h"
		12	#include "xe_gt_stats.h"
12	#include "xe_hw_engine_group.h"	13	#include "xe_hw_engine_group.h"
		14	#include "xe_sync.h"
13	#include "xe_vm.h"	15	#include "xe_vm.h"
14		16
15	static void	17	static void
@@ -20,7 +22,8 @@ hw_engine_group_resume_lr_jobs_func(struct work_struct *w) · 4 unmodified lines
20	int err;	22	int err;
21	enum xe_hw_engine_group_execution_mode previous_mode;	23	enum xe_hw_engine_group_execution_mode previous_mode;
22		24
23	err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode);	25	err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode,
		26	NULL, 0);
24	if (err)	27	if (err)
25	return;	28	return;
26		29
@@ -188,23 +191,39 @@ void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group · 161 unmodified lines
188	/**	191	/**
189	* xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group	192	* xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group
190	* @group: The hw engine group	193	* @group: The hw engine group
		194	* @has_deps: dma-fence job triggering suspend has dependencies
191	*	195	*
192	* Return: 0 on success, negative error code on error.	196	* Return: 0 on success, negative error code on error.
193	*/	197	*/
194	static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group)	198	static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group,
		199	bool has_deps)
195	{	200	{
196	int err;	201	int err;
197	struct xe_exec_queue *q;	202	struct xe_exec_queue *q;
		203	struct xe_gt *gt = NULL;
198	bool need_resume = false;	204	bool need_resume = false;
		205	ktime_t start = xe_gt_stats_ktime_get();
199		206
200	lockdep_assert_held_write(&group->mode_sem);	207	lockdep_assert_held_write(&group->mode_sem);
201		208
202	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {	209	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
		210	bool idle_skip_suspend;
		211
203	if (!xe_vm_in_fault_mode(q->vm))	212	if (!xe_vm_in_fault_mode(q->vm))
204	continue;	213	continue;
205		214
206	need_resume = true;	215	idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q);
		216	if (!idle_skip_suspend && has_deps)
		217	return -EAGAIN;
		218
		219	xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1);
		220	if (idle_skip_suspend)
		221	xe_gt_stats_incr(q->gt,
		222	XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, 1);
		223
		224	need_resume \|= !idle_skip_suspend;
207	q->ops->suspend(q);	225	q->ops->suspend(q);
		226	gt = q->gt;
208	}	227	}
209		228
210	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {	229	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
@@ -216,6 +235,12 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group · 5 unmodified lines
216	return err;	235	return err;
217	}	236	}
218		237
		238	if (gt) {
		239	xe_gt_stats_incr(gt,
		240	XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
		241	xe_gt_stats_ktime_us_delta(start));
		242	}
		243
219	if (need_resume)	244	if (need_resume)
220	xe_hw_engine_group_resume_faulting_lr_jobs(group);	245	xe_hw_engine_group_resume_faulting_lr_jobs(group);
221		246
@@ -236,7 +261,9 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group · 14 unmodified lines
236	{	261	{
237	long timeout;	262	long timeout;
238	struct xe_exec_queue *q;	263	struct xe_exec_queue *q;
		264	struct xe_gt *gt = NULL;
239	struct dma_fence *fence;	265	struct dma_fence *fence;
		266	ktime_t start = xe_gt_stats_ktime_get();
240		267
241	lockdep_assert_held_write(&group->mode_sem);	268	lockdep_assert_held_write(&group->mode_sem);
242		269
@@ -244,18 +271,26 @@ static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group · 1 unmodified line
244	if (xe_vm_in_lr_mode(q->vm))	271	if (xe_vm_in_lr_mode(q->vm))
245	continue;	272	continue;
246		273
		274	xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1);
247	fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);	275	fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);
248	timeout = dma_fence_wait(fence, false);	276	timeout = dma_fence_wait(fence, false);
249	dma_fence_put(fence);	277	dma_fence_put(fence);
		278	gt = q->gt;
250		279
251	if (timeout < 0)	280	if (timeout < 0)
252	return -ETIME;	281	return -ETIME;
253	}	282	}
254		283
		284	if (gt) {
		285	xe_gt_stats_incr(gt,
		286	XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
		287	xe_gt_stats_ktime_us_delta(start));
		288	}
		289
255	return 0;	290	return 0;
256	}	291	}
257		292
258	static int switch_mode(struct xe_hw_engine_group *group)	293	static int switch_mode(struct xe_hw_engine_group *group, bool has_deps)
259	{	294	{
260	int err = 0;	295	int err = 0;
261	enum xe_hw_engine_group_execution_mode new_mode;	296	enum xe_hw_engine_group_execution_mode new_mode;
@@ -265,7 +300,8 @@ static int switch_mode(struct xe_hw_engine_group *group) · 3 unmodified lines
265	switch (group->cur_mode) {	300	switch (group->cur_mode) {
266	case EXEC_MODE_LR:	301	case EXEC_MODE_LR:
267	new_mode = EXEC_MODE_DMA_FENCE;	302	new_mode = EXEC_MODE_DMA_FENCE;
268	err = xe_hw_engine_group_suspend_faulting_lr_jobs(group);	303	err = xe_hw_engine_group_suspend_faulting_lr_jobs(group,
		304	has_deps);
269	break;	305	break;
270	case EXEC_MODE_DMA_FENCE:	306	case EXEC_MODE_DMA_FENCE:
271	new_mode = EXEC_MODE_LR;	307	new_mode = EXEC_MODE_LR;
@@ -281,19 +317,36 @@ static int switch_mode(struct xe_hw_engine_group *group) · 9 unmodified lines
281	return 0;	317	return 0;
282	}	318	}
283		319
		320	static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs)
		321	{
		322	int err, i;
		323
		324	for (i = 0; i < num_syncs; ++i) {
		325	err = xe_sync_entry_wait(syncs + i);
		326	if (err)
		327	return err;
		328	}
		329
		330	return 0;
		331	}
		332
284	/**	333	/**
285	* xe_hw_engine_group_get_mode() - Get the group to execute in the new mode	334	* xe_hw_engine_group_get_mode() - Get the group to execute in the new mode
286	* @group: The hw engine group	335	* @group: The hw engine group
287	* @new_mode: The new execution mode	336	* @new_mode: The new execution mode
288	* @previous_mode: Pointer to the previous mode provided for use by caller	337	* @previous_mode: Pointer to the previous mode provided for use by caller
		338	* @syncs: Syncs from exec IOCTL
		339	* @num_syncs: Number of syncs from exec IOCTL
289	*	340	*
290	* Return: 0 if successful, -EINTR if locking failed.	341	* Return: 0 if successful, -EINTR if locking failed.
291	*/	342	*/
292	int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,	343	int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
293	enum xe_hw_engine_group_execution_mode new_mode,	344	enum xe_hw_engine_group_execution_mode new_mode,
294	enum xe_hw_engine_group_execution_mode *previous_mode)	345	enum xe_hw_engine_group_execution_mode *previous_mode,
		346	struct xe_sync_entry *syncs, int num_syncs)
295	__acquires(&group->mode_sem)	347	__acquires(&group->mode_sem)
296	{	348	{
		349	bool has_deps = !!num_syncs;
297	int err = down_read_interruptible(&group->mode_sem);	350	int err = down_read_interruptible(&group->mode_sem);
298		351
299	if (err)	352	if (err)
@@ -303,15 +356,25 @@ __acquires(&group->mode_sem) · 3 unmodified lines
303		356
304	if (new_mode != group->cur_mode) {	357	if (new_mode != group->cur_mode) {
305	up_read(&group->mode_sem);	358	up_read(&group->mode_sem);
		359	retry:
306	err = down_write_killable(&group->mode_sem);	360	err = down_write_killable(&group->mode_sem);
307	if (err)	361	if (err)
308	return err;	362	return err;
309		363
310	if (new_mode != group->cur_mode) {	364	if (new_mode != group->cur_mode) {
311	err = switch_mode(group);	365	err = switch_mode(group, has_deps);
312	if (err) {	366	if (err) {
313	up_write(&group->mode_sem);	367	up_write(&group->mode_sem);
314	return err;	368
		369	if (err != -EAGAIN)
		370	return err;
		371
		372	err = wait_syncs(syncs, num_syncs);
		373	if (err)
		374	return err;
		375
		376	has_deps = false;
		377	goto retry;
315	}	378	}
316	}	379	}
317	downgrade_write(&group->mode_sem);	380	downgrade_write(&group->mode_sem);

drivers/gpu/drm/xe/xe_hw_engine_group.h+3−1

 struct drm_device;
 struct xe_exec_queue;
 struct xe_gt;
+struct xe_sync_entry;
 int xe_hw_engine_setup_groups(struct xe_gt *gt);
 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
 				enum xe_hw_engine_group_execution_mode new_mode,
-				enum xe_hw_engine_group_execution_mode *previous_mode);
+				enum xe_hw_engine_group_execution_mode *previous_mode,
+				struct xe_sync_entry *syncs, int num_syncs);
 void xe_hw_engine_group_put(struct xe_hw_engine_group *group);
 enum xe_hw_engine_group_execution_mode

drivers/gpu/drm/xe/xe_hwmon.c+14−38

 	int ret = 0;
-	xe_pm_runtime_get(hwmon->xe);
+	guard(xe_pm_runtime)(hwmon->xe);
 	mutex_lock(&hwmon->hwmon_lock);
 	mutex_unlock(&hwmon->hwmon_lock);
-	xe_pm_runtime_put(hwmon->xe);
 	x = REG_FIELD_GET(PWR_LIM_TIME_X, reg_val);
 	y = REG_FIELD_GET(PWR_LIM_TIME_Y, reg_val);
 	rxy = REG_FIELD_PREP(PWR_LIM_TIME_X, x) |
 			       REG_FIELD_PREP(PWR_LIM_TIME_Y, y);
-	xe_pm_runtime_get(hwmon->xe);
+	guard(xe_pm_runtime)(hwmon->xe);
 	mutex_lock(&hwmon->hwmon_lock);
 	mutex_unlock(&hwmon->hwmon_lock);
-	xe_pm_runtime_put(hwmon->xe);
 	return count;
 }
 	      int channel, long *val)
 {
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
-	int ret;
-	xe_pm_runtime_get(hwmon->xe);
+	guard(xe_pm_runtime)(hwmon->xe);
 	switch (type) {
 	case hwmon_temp:
-		ret = xe_hwmon_temp_read(hwmon, attr, channel, val);
+		return xe_hwmon_temp_read(hwmon, attr, channel, val);
-		break;
 	case hwmon_power:
-		ret = xe_hwmon_power_read(hwmon, attr, channel, val);
+		return xe_hwmon_power_read(hwmon, attr, channel, val);
-		break;
 	case hwmon_curr:
-		ret = xe_hwmon_curr_read(hwmon, attr, channel, val);
+		return xe_hwmon_curr_read(hwmon, attr, channel, val);
-		break;
 	case hwmon_in:
-		ret = xe_hwmon_in_read(hwmon, attr, channel, val);
+		return xe_hwmon_in_read(hwmon, attr, channel, val);
-		break;
 	case hwmon_energy:
-		ret = xe_hwmon_energy_read(hwmon, attr, channel, val);
+		return xe_hwmon_energy_read(hwmon, attr, channel, val);
-		break;
 	case hwmon_fan:
-		ret = xe_hwmon_fan_read(hwmon, attr, channel, val);
+		return xe_hwmon_fan_read(hwmon, attr, channel, val);
-		break;
 	default:
-		ret = -EOPNOTSUPP;
+		return -EOPNOTSUPP;
-		break;
 	}
-	xe_pm_runtime_put(hwmon->xe);
-	return ret;
 }
 static int
 	       int channel, long val)
 {
 	struct xe_hwmon *hwmon = dev_get_drvdata(dev);
-	int ret;
-	xe_pm_runtime_get(hwmon->xe);
+	guard(xe_pm_runtime)(hwmon->xe);
 	switch (type) {
 	case hwmon_power:
-		ret = xe_hwmon_power_write(hwmon, attr, channel, val);
+		return xe_hwmon_power_write(hwmon, attr, channel, val);
-		break;
 	case hwmon_curr:
-		ret = xe_hwmon_curr_write(hwmon, attr, channel, val);
+		return xe_hwmon_curr_write(hwmon, attr, channel, val);
-		break;
 	default:
-		ret = -EOPNOTSUPP;
+		return -EOPNOTSUPP;
-		break;
 	}
-	xe_pm_runtime_put(hwmon->xe);
-	return ret;
 }
 static int xe_hwmon_read_label(struct device *dev,

drivers/gpu/drm/xe/xe_i2c.c+1−1

@@ -319,7 +319,7 @@ int xe_i2c_probe(struct xe_device *xe) · 318 unmodified lines
319	struct xe_i2c *i2c;	319	struct xe_i2c *i2c;
320	int ret;	320	int ret;
321		321
322	if (xe->info.platform != XE_BATTLEMAGE)	322	if (!xe->info.has_i2c)
323	return 0;	323	return 0;
324		324
325	if (IS_SRIOV_VF(xe))	325	if (IS_SRIOV_VF(xe))

drivers/gpu/drm/xe/xe_irq.c+2−0

 #include "xe_hw_error.h"
 #include "xe_i2c.h"
 #include "xe_memirq.h"
+#include "xe_mert.h"
 #include "xe_mmio.h"
 #include "xe_pxp.h"
 #include "xe_sriov.h"
 				xe_heci_csc_irq_handler(xe, master_ctl);
 			xe_display_irq_handler(xe, master_ctl);
 			xe_i2c_irq_handler(xe, master_ctl);
+			xe_mert_irq_handler(xe, master_ctl);
 			gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
 		}
 	}

drivers/gpu/drm/xe/xe_lmtt.c+22−3

 #include <drm/drm_managed.h>
 #include "regs/xe_gt_regs.h"
+#include "regs/xe_mert_regs.h"
 #include "xe_assert.h"
 #include "xe_bo.h"
 #include "xe_tlb_inval.h"
 #include "xe_lmtt.h"
 #include "xe_map.h"
+#include "xe_mert.h"
 #include "xe_mmio.h"
 #include "xe_res_cursor.h"
 #include "xe_sriov.h"
+#include "xe_tile.h"
 #include "xe_tile_sriov_printk.h"
 /**
 	struct xe_device *xe = tile_to_xe(tile);
 	dma_addr_t offset = xe_bo_main_addr(lmtt->pd->bo, XE_PAGE_SIZE);
 	struct xe_gt *gt;
+	u32 config;
 	u8 id;
 	lmtt_debug(lmtt, "DIR offset %pad\n", &offset);
 	lmtt_assert(lmtt, xe_bo_is_vram(lmtt->pd->bo));
 	lmtt_assert(lmtt, IS_ALIGNED(offset, SZ_64K));
+	config = LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K);
 	for_each_gt_on_tile(gt, tile, id)
 		xe_mmio_write32(&gt->mmio,
 				GRAPHICS_VER(xe) >= 20 ? XE2_LMEM_CFG : LMEM_CFG,
-				LMEM_EN | REG_FIELD_PREP(LMTT_DIR_PTR, offset / SZ_64K));
+				config);
+	if (xe_device_has_mert(xe) && xe_tile_is_root(tile))
+		xe_mmio_write32(&tile->mmio, MERT_LMEM_CFG, config);
 }
 /**
  * @lmtt: the &xe_lmtt to invalidate
  *
  * Send requests to all GuCs on this tile to invalidate all TLBs.
+ * If the platform has a standalone MERT, also invalidate MERT's TLB.
  *
  * This function should be called only when running as a PF driver.
  */
 void xe_lmtt_invalidate_hw(struct xe_lmtt *lmtt)
 {
+	struct xe_tile *tile = lmtt_to_tile(lmtt);
+	struct xe_device *xe = lmtt_to_xe(lmtt);
 	int err;
-	lmtt_assert(lmtt, IS_SRIOV_PF(lmtt_to_xe(lmtt)));
+	lmtt_assert(lmtt, IS_SRIOV_PF(xe));
 	err = lmtt_invalidate_hw(lmtt);
 	if (err)
-		xe_tile_sriov_err(lmtt_to_tile(lmtt), "LMTT invalidation failed (%pe)",
+		xe_tile_sriov_err(tile, "LMTT invalidation failed (%pe)",
 				  ERR_PTR(err));
+	if (xe_device_has_mert(xe) && xe_tile_is_root(tile)) {
+		err = xe_mert_invalidate_lmtt(tile);
+		if (err)
+			xe_tile_sriov_err(tile, "MERT LMTT invalidation failed (%pe)",
+					  ERR_PTR(err));
+	}
 }
 static void lmtt_write_pte(struct xe_lmtt *lmtt, struct xe_lmtt_pt *pt,

drivers/gpu/drm/xe/xe_lrc.c+63−10

 #define LRC_INDIRECT_CTX_BO_SIZE		SZ_4K
 #define LRC_INDIRECT_RING_STATE_SIZE		SZ_4K
+#define LRC_PRIORITY				GENMASK_ULL(10, 9)
+#define LRC_PRIORITY_LOW			0
+#define LRC_PRIORITY_NORMAL			1
+#define LRC_PRIORITY_HIGH			2
 /*
  * Layout of the LRC and associated data allocated as
  * lrc->bo:
 	return false;
 }
-size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class)
+/**
+ * xe_gt_lrc_hang_replay_size() - Hang replay size
+ * @gt: The GT
+ * @class: Hardware engine class
+ *
+ * Determine size of GPU hang replay state for a GT and hardware engine class.
+ *
+ * Return: Size of GPU hang replay size
+ */
+size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	size_t size;
+	size_t size = 0;
-	/* Per-process HW status page (PPHWSP) */
-	size = LRC_PPHWSP_SIZE;
 	/* Engine context image */
 	switch (class) {
 		size += 1 * SZ_4K;
 	}
+	return size;
+}
+size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class)
+{
+	size_t size = xe_gt_lrc_hang_replay_size(gt, class);
 	/* Add indirect ring state page */
 	if (xe_gt_has_indirect_ring_state(gt))
 		size += LRC_INDIRECT_RING_STATE_SIZE;
-	return size;
+	return size + LRC_PPHWSP_SIZE;
 }
 /*
 	return 0;
 }
+static u8 xe_multi_queue_prio_to_lrc(struct xe_lrc *lrc, enum xe_multi_queue_priority priority)
+{
+	struct xe_device *xe = gt_to_xe(lrc->gt);
+	xe_assert(xe, (priority >= XE_MULTI_QUEUE_PRIORITY_LOW &&
+		       priority <= XE_MULTI_QUEUE_PRIORITY_HIGH));
+	/* xe_multi_queue_priority is directly mapped to LRC priority values */
+	return priority;
+}
+/**
+ * xe_lrc_set_multi_queue_priority() - Set multi queue priority in LRC
+ * @lrc: Logical Ring Context
+ * @priority: Multi queue priority of the exec queue
+ *
+ * Convert @priority to LRC multi queue priority and update the @lrc descriptor
+ */
+void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority)
+{
+	lrc->desc &= ~LRC_PRIORITY;
+	lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority));
+}
 static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
-		       struct xe_vm *vm, u32 ring_size, u16 msix_vec,
+		       struct xe_vm *vm, void *replay_state, u32 ring_size,
+		       u16 msix_vec,
 		       u32 init_flags)
 {
 	struct xe_gt *gt = hwe->gt;
 	kref_init(&lrc->refcount);
 	lrc->gt = gt;
+	lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class);
 	lrc->size = lrc_size;
 	lrc->flags = 0;
 	lrc->ring.size = ring_size;
 	 * scratch.
 	 */
 	map = __xe_lrc_pphwsp_map(lrc);
-	if (gt->default_lrc[hwe->class]) {
+	if (gt->default_lrc[hwe->class] || replay_state) {
 		xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE);	/* PPHWSP */
 		xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
 				 gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE,
 				 lrc_size - LRC_PPHWSP_SIZE);
+		if (replay_state)
+			xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
+					 replay_state, lrc->replay_size);
 	} else {
 		void *init_data = empty_lrc_data(hwe);
  * xe_lrc_create - Create a LRC
  * @hwe: Hardware Engine
  * @vm: The VM (address space)
+ * @replay_state: GPU hang replay state
  * @ring_size: LRC ring size
  * @msix_vec: MSI-X interrupt vector (for platforms that support it)
  * @flags: LRC initialization flags
  * upon failure.
  */
 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm,
-			     u32 ring_size, u16 msix_vec, u32 flags)
+			     void *replay_state, u32 ring_size, u16 msix_vec, u32 flags)
 {
 	struct xe_lrc *lrc;
 	int err;
 	if (!lrc)
 		return ERR_PTR(-ENOMEM);
-	err = xe_lrc_init(lrc, hwe, vm, ring_size, msix_vec, flags);
+	err = xe_lrc_init(lrc, hwe, vm, replay_state, ring_size, msix_vec, flags);
 	if (err) {
 		kfree(lrc);
 		return ERR_PTR(err);
 	snapshot->lrc_bo = xe_bo_get(lrc->bo);
 	snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc);
 	snapshot->lrc_size = lrc->size;
+	snapshot->replay_offset = 0;
+	snapshot->replay_size = lrc->replay_size;
 	snapshot->lrc_snapshot = NULL;
 	snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc));
 	snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc);
 	}
 	drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE);
+	drm_printf(p, "\n\t[HWCTX].replay_offset: 0x%lx\n", snapshot->replay_offset);
+	drm_printf(p, "\n\t[HWCTX].replay_length: 0x%lx\n", snapshot->replay_size);
 	drm_puts(p, "\t[HWCTX].data: ");
 	for (; i < snapshot->lrc_size; i += sizeof(u32)) {
 		u32 *val = snapshot->lrc_snapshot + i;

drivers/gpu/drm/xe/xe_lrc.h+6−1

 struct xe_bb;
 struct xe_device;
 struct xe_exec_queue;
+enum xe_multi_queue_priority;
 enum xe_engine_class;
 struct xe_gt;
 struct xe_hw_engine;
 	struct xe_bo *lrc_bo;
 	void *lrc_snapshot;
 	unsigned long lrc_size, lrc_offset;
+	unsigned long replay_size, replay_offset;
 	u32 context_desc;
 	u32 ring_addr;
 #define XE_LRC_CREATE_USER_CTX		BIT(2)
 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm,
-			     u32 ring_size, u16 msix_vec, u32 flags);
+			     void *replay_state, u32 ring_size, u16 msix_vec, u32 flags);
 void xe_lrc_destroy(struct kref *ref);
 /**
 	return SZ_16K;
 }
+size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class);
 size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class);
 u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc);
 u32 xe_lrc_regs_offset(struct xe_lrc *lrc);
 u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs);
+void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority);
 struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc);
 void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot);
 void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p);

drivers/gpu/drm/xe/xe_lrc_types.h+3−0

@@ -25,6 +25,9 @@ struct xe_lrc { · 24 unmodified lines
25	/** @size: size of the lrc and optional indirect ring state */	25	/** @size: size of the lrc and optional indirect ring state */
26	u32 size;	26	u32 size;
27		27
		28	/** @replay_size: Size LRC needed for replaying a hang */
		29	u32 replay_size;
		30
28	/** @gt: gt which this LRC belongs to */	31	/** @gt: gt which this LRC belongs to */
29	struct xe_gt *gt;	32	struct xe_gt *gt;
30		33

drivers/gpu/drm/xe/xe_mert.c+82−0

@@ -0,0 +1,82 @@
	1	// SPDX-License-Identifier: MIT
	2	/*
	3	* Copyright(c) 2025, Intel Corporation. All rights reserved.
	4	*/
	5
	6	#include "regs/xe_irq_regs.h"
	7	#include "regs/xe_mert_regs.h"
	8
	9	#include "xe_device.h"
	10	#include "xe_mert.h"
	11	#include "xe_mmio.h"
	12	#include "xe_tile.h"
	13
	14	/**
	15	* xe_mert_invalidate_lmtt - Invalidate MERT LMTT
	16	* @tile: the &xe_tile
	17	*
	18	* Trigger invalidation of the MERT LMTT and wait for completion.
	19	*
	20	* Return: 0 on success or -ETIMEDOUT in case of a timeout.
	21	*/
	22	int xe_mert_invalidate_lmtt(struct xe_tile *tile)
	23	{
	24	struct xe_device *xe = tile_to_xe(tile);
	25	struct xe_mert *mert = &tile->mert;
	26	const long timeout = HZ / 4;
	27	unsigned long flags;
	28
	29	xe_assert(xe, xe_device_has_mert(xe));
	30	xe_assert(xe, xe_tile_is_root(tile));
	31
	32	spin_lock_irqsave(&mert->lock, flags);
	33	if (!mert->tlb_inv_triggered) {
	34	mert->tlb_inv_triggered = true;
	35	reinit_completion(&mert->tlb_inv_done);
	36	xe_mmio_write32(&tile->mmio, MERT_TLB_INV_DESC_A, MERT_TLB_INV_DESC_A_VALID);
	37	}
	38	spin_unlock_irqrestore(&mert->lock, flags);
	39
	40	if (!wait_for_completion_timeout(&mert->tlb_inv_done, timeout))
	41	return -ETIMEDOUT;
	42
	43	return 0;
	44	}
	45
	46	/**
	47	* xe_mert_irq_handler - Handler for MERT interrupts
	48	* @xe: the &xe_device
	49	* @master_ctl: interrupt register
	50	*
	51	* Handle interrupts generated by MERT.
	52	*/
	53	void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl)
	54	{
	55	struct xe_tile *tile = xe_device_get_root_tile(xe);
	56	unsigned long flags;
	57	u32 reg_val;
	58	u8 err;
	59
	60	if (!(master_ctl & SOC_H2DMEMINT_IRQ))
	61	return;
	62
	63	reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT);
	64	xe_mmio_write32(&tile->mmio, MERT_TLB_CT_INTR_ERR_ID_PORT, 0);
	65
	66	err = FIELD_GET(MERT_TLB_CT_ERROR_MASK, reg_val);
	67	if (err == MERT_TLB_CT_LMTT_FAULT)
	68	drm_dbg(&xe->drm, "MERT catastrophic error: LMTT fault (VF%u)\n",
	69	FIELD_GET(MERT_TLB_CT_VFID_MASK, reg_val));
	70	else if (err)
	71	drm_dbg(&xe->drm, "MERT catastrophic error: Unexpected fault (0x%x)\n", err);
	72
	73	spin_lock_irqsave(&tile->mert.lock, flags);
	74	if (tile->mert.tlb_inv_triggered) {
	75	reg_val = xe_mmio_read32(&tile->mmio, MERT_TLB_INV_DESC_A);
	76	if (!(reg_val & MERT_TLB_INV_DESC_A_VALID)) {
	77	tile->mert.tlb_inv_triggered = false;
	78	complete_all(&tile->mert.tlb_inv_done);
	79	}
	80	}
	81	spin_unlock_irqrestore(&tile->mert.lock, flags);
	82	}

drivers/gpu/drm/xe/xe_mert.h+32−0

@@ -0,0 +1,32 @@
	1	/* SPDX-License-Identifier: MIT */
	2	/*
	3	* Copyright(c) 2025, Intel Corporation. All rights reserved.
	4	*/
	5
	6	#ifndef __XE_MERT_H__
	7	#define __XE_MERT_H__
	8
	9	#include <linux/completion.h>
	10	#include <linux/spinlock.h>
	11	#include <linux/types.h>
	12
	13	struct xe_device;
	14	struct xe_tile;
	15
	16	struct xe_mert {
	17	/** @lock: protects the TLB invalidation status */
	18	spinlock_t lock;
	19	/** @tlb_inv_triggered: indicates if TLB invalidation was triggered */
	20	bool tlb_inv_triggered;
	21	/** @mert.tlb_inv_done: completion of TLB invalidation */
	22	struct completion tlb_inv_done;
	23	};
	24
	25	#ifdef CONFIG_PCI_IOV
	26	int xe_mert_invalidate_lmtt(struct xe_tile *tile);
	27	void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl);
	28	#else
	29	static inline void xe_mert_irq_handler(struct xe_device *xe, u32 master_ctl) { }
	30	#endif
	31
	32	#endif /* __XE_MERT_H__ */

drivers/gpu/drm/xe/xe_migrate.c+54−3

@@ -34,6 +34,7 @@ · 33 unmodified lines
34	#include "xe_res_cursor.h"	34	#include "xe_res_cursor.h"
35	#include "xe_sa.h"	35	#include "xe_sa.h"
36	#include "xe_sched_job.h"	36	#include "xe_sched_job.h"
		37	#include "xe_sriov_vf_ccs.h"
37	#include "xe_sync.h"	38	#include "xe_sync.h"
38	#include "xe_trace_bo.h"	39	#include "xe_trace_bo.h"
39	#include "xe_validation.h"	40	#include "xe_validation.h"
@@ -1103,12 +1104,16 @@ int xe_migrate_ccs_rw_copy(struct xe_tile tile, struct xe_exec_queue q, · 1063 unmodified lines
1103	u32 batch_size, batch_size_allocated;	1104	u32 batch_size, batch_size_allocated;
1104	struct xe_device *xe = gt_to_xe(gt);	1105	struct xe_device *xe = gt_to_xe(gt);
1105	struct xe_res_cursor src_it, ccs_it;	1106	struct xe_res_cursor src_it, ccs_it;
		1107	struct xe_sriov_vf_ccs_ctx *ctx;
		1108	struct xe_sa_manager *bb_pool;
1106	u64 size = xe_bo_size(src_bo);	1109	u64 size = xe_bo_size(src_bo);
1107	struct xe_bb *bb = NULL;	1110	struct xe_bb *bb = NULL;
1108	u64 src_L0, src_L0_ofs;	1111	u64 src_L0, src_L0_ofs;
1109	u32 src_L0_pt;	1112	u32 src_L0_pt;
1110	int err;	1113	int err;
1111		1114
		1115	ctx = &xe->sriov.vf.ccs.contexts[read_write];
		1116
1112	xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);	1117	xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
1113		1118
1114	xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),	1119	xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),
@@ -1141,11 +1146,15 @@ int xe_migrate_ccs_rw_copy(struct xe_tile tile, struct xe_exec_queue q, · 26 unmodified lines
1141	size -= src_L0;	1146	size -= src_L0;
1142	}	1147	}
1143		1148
		1149	bb_pool = ctx->mem.ccs_bb_pool;
		1150	guard(mutex) (xe_sa_bo_swap_guard(bb_pool));
		1151	xe_sa_bo_swap_shadow(bb_pool);
		1152
1144	bb = xe_bb_ccs_new(gt, batch_size, read_write);	1153	bb = xe_bb_ccs_new(gt, batch_size, read_write);
1145	if (IS_ERR(bb)) {	1154	if (IS_ERR(bb)) {
1146	drm_err(&xe->drm, "BB allocation failed.\n");	1155	drm_err(&xe->drm, "BB allocation failed.\n");
1147	err = PTR_ERR(bb);	1156	err = PTR_ERR(bb);
1148	goto err_ret;	1157	return err;
1149	}	1158	}
1150		1159
1151	batch_size_allocated = batch_size;	1160	batch_size_allocated = batch_size;
@@ -1194,10 +1203,52 @@ int xe_migrate_ccs_rw_copy(struct xe_tile tile, struct xe_exec_queue q, · 42 unmodified lines
1194	xe_assert(xe, (batch_size_allocated == bb->len));	1203	xe_assert(xe, (batch_size_allocated == bb->len));
1195	src_bo->bb_ccs[read_write] = bb;	1204	src_bo->bb_ccs[read_write] = bb;
1196		1205
		1206	xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
		1207	xe_sa_bo_sync_shadow(bb->bo);
1197	return 0;	1208	return 0;
		1209	}
1198		1210
1199	err_ret:	1211	/**
1200	return err;	1212	* xe_migrate_ccs_rw_copy_clear() - Clear the CCS read/write batch buffer
		1213	* content.
		1214	* @src_bo: The buffer object @src is currently bound to.
		1215	* @read_write : Creates BB commands for CCS read/write.
		1216	*
		1217	* Directly clearing the BB lacks atomicity and can lead to undefined
		1218	* behavior if the vCPU is halted mid-operation during the clearing
		1219	* process. To avoid this issue, we use a shadow buffer object approach.
		1220	*
		1221	* First swap the SA BO address with the shadow BO, perform the clearing
		1222	* operation on the BB, update the shadow BO in the ring buffer, then
		1223	* sync the shadow and the actual buffer to maintain consistency.
		1224	*
		1225	* Returns: None.
		1226	*/
		1227	void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
		1228	enum xe_sriov_vf_ccs_rw_ctxs read_write)
		1229	{
		1230	struct xe_bb *bb = src_bo->bb_ccs[read_write];
		1231	struct xe_device *xe = xe_bo_device(src_bo);
		1232	struct xe_sriov_vf_ccs_ctx *ctx;
		1233	struct xe_sa_manager *bb_pool;
		1234	u32 *cs;
		1235
		1236	xe_assert(xe, IS_SRIOV_VF(xe));
		1237
		1238	ctx = &xe->sriov.vf.ccs.contexts[read_write];
		1239	bb_pool = ctx->mem.ccs_bb_pool;
		1240
		1241	guard(mutex) (xe_sa_bo_swap_guard(bb_pool));
		1242	xe_sa_bo_swap_shadow(bb_pool);
		1243
		1244	cs = xe_sa_bo_cpu_addr(bb->bo);
		1245	memset(cs, MI_NOOP, bb->len * sizeof(u32));
		1246	xe_sriov_vf_ccs_rw_update_bb_addr(ctx);
		1247
		1248	xe_sa_bo_sync_shadow(bb->bo);
		1249
		1250	xe_bb_free(bb, NULL);
		1251	src_bo->bb_ccs[read_write] = NULL;
1201	}	1252	}
1202		1253
1203	/**	1254	/**

drivers/gpu/drm/xe/xe_migrate.h+3−0

@@ -134,6 +134,9 @@ int xe_migrate_ccs_rw_copy(struct xe_tile tile, struct xe_exec_queue q, · 133 unmodified lines
134	struct xe_bo *src_bo,	134	struct xe_bo *src_bo,
135	enum xe_sriov_vf_ccs_rw_ctxs read_write);	135	enum xe_sriov_vf_ccs_rw_ctxs read_write);
136		136
		137	void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
		138	enum xe_sriov_vf_ccs_rw_ctxs read_write);
		139
137	struct xe_lrc xe_migrate_lrc(struct xe_migrate migrate);	140	struct xe_lrc xe_migrate_lrc(struct xe_migrate migrate);
138	struct xe_exec_queue xe_migrate_exec_queue(struct xe_migrate migrate);	141	struct xe_exec_queue xe_migrate_exec_queue(struct xe_migrate migrate);
139	struct dma_fence xe_migrate_vram_copy_chunk(struct xe_bo vram_bo, u64 vram_offset,	142	struct dma_fence xe_migrate_vram_copy_chunk(struct xe_bo vram_bo, u64 vram_offset,

drivers/gpu/drm/xe/xe_mocs.c+6−12

 	struct xe_device *xe = gt_to_xe(gt);
 	enum xe_force_wake_domains domain;
 	struct xe_mocs_info table;
-	unsigned int fw_ref, flags;
+	unsigned int flags;
-	int err = 0;
 	flags = get_mocs_settings(xe, &table);
 	domain = flags & HAS_LNCF_MOCS ? XE_FORCEWAKE_ALL : XE_FW_GT;
-	xe_pm_runtime_get_noresume(xe);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), domain);
-	if (!xe_force_wake_ref_has_domain(fw_ref, domain)) {
+	guard(xe_pm_runtime_noresume)(xe);
-		err = -ETIMEDOUT;
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), domain);
-		goto err_fw;
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, domain))
-	}
+		return -ETIMEDOUT;
 	table.ops->dump(&table, flags, gt, p);
-err_fw:
+	return 0;
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_pm_runtime_put(xe);
-	return err;
 }
 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)

drivers/gpu/drm/xe/xe_nvm.c+23−11

 #include "xe_device_types.h"
 #include "xe_mmio.h"
 #include "xe_nvm.h"
+#include "xe_pcode_api.h"
 #include "regs/xe_gsc_regs.h"
 #include "xe_sriov.h"
 {
 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
-	if (xe->info.platform != XE_BATTLEMAGE)
+	switch (xe->info.platform) {
+	case XE_CRESCENTISLAND:
+	case XE_BATTLEMAGE:
+		return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) &
+			 NVM_NON_POSTED_ERASE_CHICKEN_BIT);
+	default:
 		return false;
-	return !(xe_mmio_read32(mmio, XE_REG(GEN12_CNTL_PROTECTED_NVM_REG)) &
+	}
-		 NVM_NON_POSTED_ERASE_CHICKEN_BIT);
 }
 static bool xe_nvm_writable_override(struct xe_device *xe)
 {
 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
 	bool writable_override;
-	resource_size_t base;
+	struct xe_reg reg;
+	u32 test_bit;
 	switch (xe->info.platform) {
+	case XE_CRESCENTISLAND:
+		reg = PCODE_SCRATCH(0);
+		test_bit = FDO_MODE;
+		break;
 	case XE_BATTLEMAGE:
-		base = DG2_GSC_HECI2_BASE;
+		reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE);
+		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
 		break;
 	case XE_PVC:
-		base = PVC_GSC_HECI2_BASE;
+		reg = HECI_FWSTS2(PVC_GSC_HECI2_BASE);
+		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
 		break;
 	case XE_DG2:
-		base = DG2_GSC_HECI2_BASE;
+		reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE);
+		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
 		break;
 	case XE_DG1:
-		base = DG1_GSC_HECI2_BASE;
+		reg = HECI_FWSTS2(DG1_GSC_HECI2_BASE);
+		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
 		break;
 	default:
 		drm_err(&xe->drm, "Unknown platform\n");
 		return true;
 	}
-	writable_override =
+	writable_override = !(xe_mmio_read32(mmio, reg) & test_bit);
-		!(xe_mmio_read32(mmio, HECI_FWSTS2(base)) &
-		  HECI_FW_STATUS_2_NVM_ACCESS_MODE);
 	if (writable_override)
 		drm_info(&xe->drm, "NVM access overridden by jumper\n");
 	return writable_override;

drivers/gpu/drm/xe/xe_oa.c+65−29

 			type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC;
 	case DRM_XE_OA_UNIT_TYPE_OAM:
 	case DRM_XE_OA_UNIT_TYPE_OAM_SAG:
+	case DRM_XE_OA_UNIT_TYPE_MERT:
 		return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC;
 	default:
 		return false;
 	enum xe_hw_engine_id id;
 	int ret = 0;
-	/* If not provided, OA unit defaults to OA unit 0 as per uapi */
-	if (!param->oa_unit)
-		param->oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0];
 	/* When we have an exec_q, get hwe from the exec_q */
 	if (param->exec_q) {
 		param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class,
 	if (ret)
 		return ret;
+	/* If not provided, OA unit defaults to OA unit 0 as per uapi */
+	if (!param.oa_unit)
+		param.oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0];
 	if (param.exec_queue_id > 0) {
+		/* An exec_queue is only needed for OAR/OAC functionality on OAG */
+		if (XE_IOCTL_DBG(oa->xe, param.oa_unit->type != DRM_XE_OA_UNIT_TYPE_OAG))
+			return -EINVAL;
 		param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id);
 		if (XE_IOCTL_DBG(oa->xe, !param.exec_q))
 			return -ENOENT;
 	{ .start = 0xE18C, .end = 0xE18C },	/* SAMPLER_MODE */
 	{ .start = 0xE590, .end = 0xE590 },	/* TDL_LSC_LAT_MEASURE_TDL_GFX */
 	{ .start = 0x13000, .end = 0x137FC },	/* PES_0_PESL0 - PES_63_UPPER_PESL3 */
+	{ .start = 0x145194, .end = 0x145194 },	/* SYS_MEM_LAT_MEASURE */
+	{ .start = 0x145340, .end = 0x14537C },	/* MERTSS_PES_0 - MERTSS_PES_7 */
 	{},
 };
 static u32 num_oa_units_per_gt(struct xe_gt *gt)
 {
 	if (xe_gt_is_main_type(gt) || GRAPHICS_VER(gt_to_xe(gt)) < 20)
-		return 1;
+		/*
+		 * Mert OA unit belongs to the SoC, not a gt, so should be accessed using
+		 * xe_root_tile_mmio(). However, for all known platforms this is the same as
+		 * accessing via xe_root_mmio_gt()->mmio.
+		 */
+		return xe_device_has_mert(gt_to_xe(gt)) ? 2 : 1;
 	else if (!IS_DGFX(gt_to_xe(gt)))
 		return XE_OAM_UNIT_SCMI_0 + 1; /* SAG + SCMI_0 */
 	else
 static struct xe_oa_regs __oam_regs(u32 base)
 {
 	return (struct xe_oa_regs) {
-		base,
+		.base		= base,
-		OAM_HEAD_POINTER(base),
+		.oa_head_ptr	= OAM_HEAD_POINTER(base),
-		OAM_TAIL_POINTER(base),
+		.oa_tail_ptr	= OAM_TAIL_POINTER(base),
-		OAM_BUFFER(base),
+		.oa_buffer	= OAM_BUFFER(base),
-		OAM_CONTEXT_CONTROL(base),
+		.oa_ctx_ctrl	= OAM_CONTEXT_CONTROL(base),
-		OAM_CONTROL(base),
+		.oa_ctrl	= OAM_CONTROL(base),
-		OAM_DEBUG(base),
+		.oa_debug	= OAM_DEBUG(base),
-		OAM_STATUS(base),
+		.oa_status	= OAM_STATUS(base),
-		OAM_CONTROL_COUNTER_SEL_MASK,
+		.oa_mmio_trg	= OAM_MMIO_TRG(base),
+		.oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK,
 	};
 }
 static struct xe_oa_regs __oag_regs(void)
 {
 	return (struct xe_oa_regs) {
-		0,
+		.base		= 0,
-		OAG_OAHEADPTR,
+		.oa_head_ptr	= OAG_OAHEADPTR,
-		OAG_OATAILPTR,
+		.oa_tail_ptr	= OAG_OATAILPTR,
-		OAG_OABUFFER,
+		.oa_buffer	= OAG_OABUFFER,
-		OAG_OAGLBCTXCTRL,
+		.oa_ctx_ctrl	= OAG_OAGLBCTXCTRL,
-		OAG_OACONTROL,
+		.oa_ctrl	= OAG_OACONTROL,
-		OAG_OA_DEBUG,
+		.oa_debug	= OAG_OA_DEBUG,
-		OAG_OASTATUS,
+		.oa_status	= OAG_OASTATUS,
-		OAG_OACONTROL_OA_COUNTER_SEL_MASK,
+		.oa_mmio_trg	= OAG_MMIOTRIGGER,
+		.oa_ctrl_counter_select_mask = OAG_OACONTROL_OA_COUNTER_SEL_MASK,
+	};
+}
+static struct xe_oa_regs __oamert_regs(void)
+{
+	return (struct xe_oa_regs) {
+		.base		= 0,
+		.oa_head_ptr	= OAMERT_HEAD_POINTER,
+		.oa_tail_ptr	= OAMERT_TAIL_POINTER,
+		.oa_buffer	= OAMERT_BUFFER,
+		.oa_ctx_ctrl	= OAMERT_CONTEXT_CONTROL,
+		.oa_ctrl	= OAMERT_CONTROL,
+		.oa_debug	= OAMERT_DEBUG,
+		.oa_status	= OAMERT_STATUS,
+		.oa_mmio_trg	= OAMERT_MMIO_TRG,
+		.oa_ctrl_counter_select_mask = OAM_CONTROL_COUNTER_SEL_MASK,
 	};
 }
 static void __xe_oa_init_oa_units(struct xe_gt *gt)
 {
-	/* Actual address is MEDIA_GT_GSI_OFFSET + oam_base_addr[i] */
 	const u32 oam_base_addr[] = {
-		[XE_OAM_UNIT_SAG]    = 0x13000,
+		[XE_OAM_UNIT_SAG]    = XE_OAM_SAG_BASE,
-		[XE_OAM_UNIT_SCMI_0] = 0x14000,
+		[XE_OAM_UNIT_SCMI_0] = XE_OAM_SCMI_0_BASE,
-		[XE_OAM_UNIT_SCMI_1] = 0x14800,
+		[XE_OAM_UNIT_SCMI_1] = XE_OAM_SCMI_1_BASE,
 	};
 	int i, num_units = gt->oa.num_oa_units;
 		struct xe_oa_unit *u = &gt->oa.oa_unit[i];
 		if (xe_gt_is_main_type(gt)) {
-			u->regs = __oag_regs();
+			if (!i) {
-			u->type = DRM_XE_OA_UNIT_TYPE_OAG;
+				u->regs = __oag_regs();
+				u->type = DRM_XE_OA_UNIT_TYPE_OAG;
+			} else {
+				xe_gt_assert(gt, xe_device_has_mert(gt_to_xe(gt)));
+				xe_gt_assert(gt, gt == xe_root_mmio_gt(gt_to_xe(gt)));
+				u->regs = __oamert_regs();
+				u->type = DRM_XE_OA_UNIT_TYPE_MERT;
+			}
 		} else {
 			xe_gt_assert(gt, GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270);
 			u->regs = __oam_regs(oam_base_addr[i]);

drivers/gpu/drm/xe/xe_oa_types.h+1−0

@@ -87,6 +87,7 @@ struct xe_oa_regs { · 86 unmodified lines
87	struct xe_reg oa_ctrl;	87	struct xe_reg oa_ctrl;
88	struct xe_reg oa_debug;	88	struct xe_reg oa_debug;
89	struct xe_reg oa_status;	89	struct xe_reg oa_status;
		90	struct xe_reg oa_mmio_trg;
90	u32 oa_ctrl_counter_select_mask;	91	u32 oa_ctrl_counter_select_mask;
91	};	92	};
92		93

drivers/gpu/drm/xe/xe_page_reclaim.c+136−0

@@ -0,0 +1,136 @@
	1	// SPDX-License-Identifier: MIT
	2	/*
	3	* Copyright © 2025 Intel Corporation
	4	*/
	5
	6	#include <linux/bitfield.h>
	7	#include <linux/kref.h>
	8	#include <linux/mm.h>
	9	#include <linux/slab.h>
	10
	11	#include "xe_page_reclaim.h"
	12
	13	#include "regs/xe_gt_regs.h"
	14	#include "xe_assert.h"
	15	#include "xe_macros.h"
	16	#include "xe_mmio.h"
	17	#include "xe_pat.h"
	18	#include "xe_sa.h"
	19	#include "xe_tlb_inval_types.h"
	20	#include "xe_vm.h"
	21
	22	/**
	23	* xe_page_reclaim_skip() - Decide whether PRL should be skipped for a VMA
	24	* @tile: Tile owning the VMA
	25	* @vma: VMA under consideration
	26	*
	27	* PPC flushing may be handled by HW for specific PAT encodings.
	28	* Skip PPC flushing/Page Reclaim for scenarios below due to redundant
	29	* flushes.
	30	* - pat_index is transient display (1)
	31	*
	32	* Return: true when page reclamation is unnecessary, false otherwise.
	33	*/
	34	bool xe_page_reclaim_skip(struct xe_tile tile, struct xe_vma vma)
	35	{
	36	u8 l3_policy;
	37
	38	l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index);
	39
	40	/*
	41	* - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC
	42	* Transient display flushes is taken care by HW, l3_policy = 1.
	43	*
	44	* HW will sequence these transient flushes at various sync points so
	45	* any event of page reclamation will hit these sync points before
	46	* page reclamation could execute.
	47	*/
	48	return (l3_policy == XE_L3_POLICY_XD);
	49	}
	50
	51	/**
	52	* xe_page_reclaim_create_prl_bo() - Back a PRL with a suballocated GGTT BO
	53	* @tlb_inval: TLB invalidation frontend associated with the request
	54	* @prl: page reclaim list data that bo will copy from
	55	* @fence: tlb invalidation fence that page reclaim action is paired to
	56	*
	57	* Suballocates a 4K BO out of the tile reclaim pool, copies the PRL CPU
	58	* copy into the BO and queues the buffer for release when @fence signals.
	59	*
	60	* Return: struct drm_suballoc pointer on success or ERR_PTR on failure.
	61	*/
	62	struct drm_suballoc xe_page_reclaim_create_prl_bo(struct xe_tlb_inval tlb_inval,
	63	struct xe_page_reclaim_list *prl,
	64	struct xe_tlb_inval_fence *fence)
	65	{
	66	struct xe_gt *gt = container_of(tlb_inval, struct xe_gt, tlb_inval);
	67	struct xe_tile *tile = gt_to_tile(gt);
	68	/* (+1) for NULL page_reclaim_entry to indicate end of list */
	69	int prl_size = min(prl->num_entries + 1, XE_PAGE_RECLAIM_MAX_ENTRIES) *
	70	sizeof(struct xe_guc_page_reclaim_entry);
	71	struct drm_suballoc *prl_sa;
	72
	73	/* Maximum size of PRL is 1 4K-page */
	74	prl_sa = __xe_sa_bo_new(tile->mem.reclaim_pool,
	75	prl_size, GFP_ATOMIC);
	76	if (IS_ERR(prl_sa))
	77	return prl_sa;
	78
	79	memcpy(xe_sa_bo_cpu_addr(prl_sa), prl->entries,
	80	prl_size);
	81	xe_sa_bo_flush_write(prl_sa);
	82	/* Queue up sa_bo_free on tlb invalidation fence signal */
	83	xe_sa_bo_free(prl_sa, &fence->base);
	84
	85	return prl_sa;
	86	}
	87
	88	/**
	89	* xe_page_reclaim_list_invalidate() - Mark a PRL as invalid
	90	* @prl: Page reclaim list to reset
	91	*
	92	* Clears the entries pointer and marks the list as invalid so
	93	* future use knows PRL is unusable. It is expected that the entries
	94	* have already been released.
	95	*/
	96	void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl)
	97	{
	98	xe_page_reclaim_entries_put(prl->entries);
	99	prl->entries = NULL;
	100	prl->num_entries = XE_PAGE_RECLAIM_INVALID_LIST;
	101	}
	102
	103	/**
	104	* xe_page_reclaim_list_init() - Initialize a page reclaim list
	105	* @prl: Page reclaim list to initialize
	106	*
	107	* NULLs both values in list to prepare on initalization.
	108	*/
	109	void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl)
	110	{
	111	// xe_page_reclaim_list_invalidate(prl);
	112	prl->entries = NULL;
	113	prl->num_entries = 0;
	114	}
	115
	116	/**
	117	* xe_page_reclaim_list_alloc_entries() - Allocate page reclaim list entries
	118	* @prl: Page reclaim list to allocate entries for
	119	*
	120	* Allocate one 4K page for the PRL entries, otherwise assign prl->entries to NULL.
	121	*/
	122	int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl)
	123	{
	124	struct page *page;
	125
	126	if (XE_WARN_ON(prl->entries))
	127	return 0;
	128
	129	page = alloc_page(GFP_KERNEL \| __GFP_ZERO);
	130	if (page) {
	131	prl->entries = page_address(page);
	132	prl->num_entries = 0;
	133	}
	134
	135	return page ? 0 : -ENOMEM;
	136	}

drivers/gpu/drm/xe/xe_page_reclaim.h+105−0

@@ -0,0 +1,105 @@
	1	/* SPDX-License-Identifier: MIT */
	2	/*
	3	* Copyright © 2025 Intel Corporation
	4	*/
	5
	6	#ifndef _XE_PAGE_RECLAIM_H_
	7	#define _XE_PAGE_RECLAIM_H_
	8
	9	#include <linux/kref.h>
	10	#include <linux/mm.h>
	11	#include <linux/slab.h>
	12	#include <linux/types.h>
	13	#include <linux/workqueue.h>
	14	#include <linux/bits.h>
	15
	16	#define XE_PAGE_RECLAIM_MAX_ENTRIES 512
	17	#define XE_PAGE_RECLAIM_LIST_MAX_SIZE SZ_4K
	18
	19	struct xe_tlb_inval;
	20	struct xe_tlb_inval_fence;
	21	struct xe_tile;
	22	struct xe_vma;
	23
	24	struct xe_guc_page_reclaim_entry {
	25	u64 qw;
	26	/* valid reclaim entry bit */
	27	#define XE_PAGE_RECLAIM_VALID BIT_ULL(0)
	28	/*
	29	* offset order of page size to be reclaimed
	30	* page_size = 1 << (XE_PTE_SHIFT + reclamation_size)
	31	*/
	32	#define XE_PAGE_RECLAIM_SIZE GENMASK_ULL(6, 1)
	33	#define XE_PAGE_RECLAIM_RSVD_0 GENMASK_ULL(11, 7)
	34	/* lower 20 bits of the physical address */
	35	#define XE_PAGE_RECLAIM_ADDR_LO GENMASK_ULL(31, 12)
	36	/* upper 20 bits of the physical address */
	37	#define XE_PAGE_RECLAIM_ADDR_HI GENMASK_ULL(51, 32)
	38	#define XE_PAGE_RECLAIM_RSVD_1 GENMASK_ULL(63, 52)
	39	} __packed;
	40
	41	struct xe_page_reclaim_list {
	42	/** @entries: array of page reclaim entries, page allocated */
	43	struct xe_guc_page_reclaim_entry *entries;
	44	/** @num_entries: number of entries */
	45	int num_entries;
	46	#define XE_PAGE_RECLAIM_INVALID_LIST -1
	47	};
	48
	49	/**
	50	* xe_page_reclaim_list_is_new() - Check if PRL is new allocation
	51	* @prl: Pointer to page reclaim list
	52	*
	53	* PRL indicates it hasn't been allocated through both values being NULL
	54	*/
	55	static inline bool xe_page_reclaim_list_is_new(struct xe_page_reclaim_list *prl)
	56	{
	57	return !prl->entries && prl->num_entries == 0;
	58	}
	59
	60	/**
	61	* xe_page_reclaim_list_valid() - Check if the page reclaim list is valid
	62	* @prl: Pointer to page reclaim list
	63	*
	64	* PRL uses the XE_PAGE_RECLAIM_INVALID_LIST to indicate that a PRL
	65	* is unusable.
	66	*/
	67	static inline bool xe_page_reclaim_list_valid(struct xe_page_reclaim_list *prl)
	68	{
	69	return !xe_page_reclaim_list_is_new(prl) &&
	70	prl->num_entries != XE_PAGE_RECLAIM_INVALID_LIST;
	71	}
	72
	73	bool xe_page_reclaim_skip(struct xe_tile tile, struct xe_vma vma);
	74	struct drm_suballoc xe_page_reclaim_create_prl_bo(struct xe_tlb_inval tlb_inval,
	75	struct xe_page_reclaim_list *prl,
	76	struct xe_tlb_inval_fence *fence);
	77	void xe_page_reclaim_list_invalidate(struct xe_page_reclaim_list *prl);
	78	void xe_page_reclaim_list_init(struct xe_page_reclaim_list *prl);
	79	int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl);
	80	/**
	81	* xe_page_reclaim_entries_get() - Increment the reference count of page reclaim entries.
	82	* @entries: Pointer to the array of page reclaim entries.
	83	*
	84	* This function increments the reference count of the backing page.
	85	*/
	86	static inline void xe_page_reclaim_entries_get(struct xe_guc_page_reclaim_entry *entries)
	87	{
	88	if (entries)
	89	get_page(virt_to_page(entries));
	90	}
	91
	92	/**
	93	* xe_page_reclaim_entries_put() - Decrement the reference count of page reclaim entries.
	94	* @entries: Pointer to the array of page reclaim entries.
	95	*
	96	* This function decrements the reference count of the backing page
	97	* and frees it if the count reaches zero.
	98	*/
	99	static inline void xe_page_reclaim_entries_put(struct xe_guc_page_reclaim_entry *entries)
	100	{
	101	if (entries)
	102	put_page(virt_to_page(entries));
	103	}
	104
	105	#endif /* _XE_PAGE_RECLAIM_H_ */

drivers/gpu/drm/xe/xe_pagefault.c+18−18

@@ -223,22 +223,22 @@ static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue, · 222 unmodified lines
223		223
224	static void xe_pagefault_print(struct xe_pagefault *pf)	224	static void xe_pagefault_print(struct xe_pagefault *pf)
225	{	225	{
226	xe_gt_dbg(pf->gt, "\n\tASID: %d\n"	226	xe_gt_info(pf->gt, "\n\tASID: %d\n"
227	"\tFaulted Address: 0x%08x%08x\n"	227	"\tFaulted Address: 0x%08x%08x\n"
228	"\tFaultType: %d\n"	228	"\tFaultType: %d\n"
229	"\tAccessType: %d\n"	229	"\tAccessType: %d\n"
230	"\tFaultLevel: %d\n"	230	"\tFaultLevel: %d\n"
231	"\tEngineClass: %d %s\n"	231	"\tEngineClass: %d %s\n"
232	"\tEngineInstance: %d\n",	232	"\tEngineInstance: %d\n",
233	pf->consumer.asid,	233	pf->consumer.asid,
234	upper_32_bits(pf->consumer.page_addr),	234	upper_32_bits(pf->consumer.page_addr),
235	lower_32_bits(pf->consumer.page_addr),	235	lower_32_bits(pf->consumer.page_addr),
236	pf->consumer.fault_type,	236	pf->consumer.fault_type,
237	pf->consumer.access_type,	237	pf->consumer.access_type,
238	pf->consumer.fault_level,	238	pf->consumer.fault_level,
239	pf->consumer.engine_class,	239	pf->consumer.engine_class,
240	xe_hw_engine_class_to_str(pf->consumer.engine_class),	240	xe_hw_engine_class_to_str(pf->consumer.engine_class),
241	pf->consumer.engine_instance);	241	pf->consumer.engine_instance);
242	}	242	}
243		243
244	static void xe_pagefault_queue_work(struct work_struct *w)	244	static void xe_pagefault_queue_work(struct work_struct *w)
@@ -260,8 +260,8 @@ static void xe_pagefault_queue_work(struct work_struct *w) · 15 unmodified lines
260	err = xe_pagefault_service(&pf);	260	err = xe_pagefault_service(&pf);
261	if (err) {	261	if (err) {
262	xe_pagefault_print(&pf);	262	xe_pagefault_print(&pf);
263	xe_gt_dbg(pf.gt, "Fault response: Unsuccessful %pe\n",	263	xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n",
264	ERR_PTR(err));	264	ERR_PTR(err));
265	}	265	}
266		266
267	pf.producer.ops->ack_fault(&pf, err);	267	pf.producer.ops->ack_fault(&pf, err);

drivers/gpu/drm/xe/xe_pat.c+152−68

 #include <generated/xe_wa_oob.h>
+#include "regs/xe_gt_regs.h"
 #include "regs/xe_reg_defs.h"
 #include "xe_assert.h"
 #include "xe_device.h"
 #define XELP_PAT_WC				REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1)
 #define XELP_PAT_UC				REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0)
+#define PAT_LABEL_LEN 20
 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" };
+static void xe_pat_index_label(char *label, size_t len, int index)
+{
+	snprintf(label, len, "PAT[%2d] ", index);
+}
+static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
+{
+	u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
+	drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index,
+		   XELP_MEM_TYPE_STR_MAP[mem_type], pat);
+}
+static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
+{
+	drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index,
+		   REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
+		   REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
+}
+static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat)
+{
+	drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index,
+		   REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
+		   REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
+}
 struct xe_pat_ops {
 	void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
 				 int n_entries);
 	return xe->pat.table[pat_index].coh_mode;
 }
+bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index)
+{
+	WARN_ON(pat_index >= xe->pat.n_entries);
+	return !!(xe->pat.table[pat_index].value & XE2_COMP_EN);
+}
+u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index)
+{
+	WARN_ON(pat_index >= xe->pat.n_entries);
+	return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value);
+}
 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
 			int n_entries)
 {
 static int xelp_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	int i;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	drm_printf(p, "PAT table:\n");
 	for (i = 0; i < xe->pat.n_entries; i++) {
 		u32 pat = xe_mmio_read32(&gt->mmio, XE_REG(_PAT_INDEX(i)));
-		u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
-		drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
+		xelp_pat_entry_dump(p, i, pat);
-			   XELP_MEM_TYPE_STR_MAP[mem_type], pat);
 	}
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 static int xehp_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	int i;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	drm_printf(p, "PAT table:\n");
 	for (i = 0; i < xe->pat.n_entries; i++) {
 		u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
-		u8 mem_type;
-		mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
-		drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
+		xelp_pat_entry_dump(p, i, pat);
-			   XELP_MEM_TYPE_STR_MAP[mem_type], pat);
 	}
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	int i;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	drm_printf(p, "PAT table:\n");
 	for (i = 0; i < xe->pat.n_entries; i++) {
 		u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
-		drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
+		xehpc_pat_entry_dump(p, i, pat);
-			   REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
-			   REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
 	}
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	int i;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	drm_printf(p, "PAT table:\n");
 		else
 			pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
-		drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
+		xelpg_pat_entry_dump(p, i, pat);
-			   REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
-			   REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
 	}
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 	.dump = xelpg_dump,
 };
+static void xe2_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd)
+{
+	drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ]  (%#8x)%s\n", label,
+		   !!(pat & XE2_NO_PROMOTE),
+		   !!(pat & XE2_COMP_EN),
+		   REG_FIELD_GET(XE2_L3_CLOS, pat),
+		   REG_FIELD_GET(XE2_L3_POLICY, pat),
+		   REG_FIELD_GET(XE2_L4_POLICY, pat),
+		   REG_FIELD_GET(XE2_COH_MODE, pat),
+		   pat, rsvd ? " *" : "");
+}
+static void xe3p_xpc_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd)
+{
+	drm_printf(p, "%s= [ %u, %u, %u, %u, %u ]  (%#8x)%s\n", label,
+		   !!(pat & XE2_NO_PROMOTE),
+		   REG_FIELD_GET(XE2_L3_CLOS, pat),
+		   REG_FIELD_GET(XE2_L3_POLICY, pat),
+		   REG_FIELD_GET(XE2_L4_POLICY, pat),
+		   REG_FIELD_GET(XE2_COH_MODE, pat),
+		   pat, rsvd ? " *" : "");
+}
 static int xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u32 pat;
 	int i;
+	char label[PAT_LABEL_LEN];
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	drm_printf(p, "PAT table: (* = reserved entry)\n");
 		else
 			pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
-		drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ]  (%#8x)%s\n", i,
+		xe_pat_index_label(label, sizeof(label), i);
-			   !!(pat & XE2_NO_PROMOTE),
+		xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
-			   !!(pat & XE2_COMP_EN),
-			   REG_FIELD_GET(XE2_L3_CLOS, pat),
-			   REG_FIELD_GET(XE2_L3_POLICY, pat),
-			   REG_FIELD_GET(XE2_L4_POLICY, pat),
-			   REG_FIELD_GET(XE2_COH_MODE, pat),
-			   pat, xe->pat.table[i].valid ? "" : " *");
 	}
 	/*
 		pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
 	drm_printf(p, "Page Table Access:\n");
-	drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ]  (%#8x)\n",
+	xe2_pat_entry_dump(p, "PTA_MODE", pat, false);
-		   !!(pat & XE2_NO_PROMOTE),
-		   !!(pat & XE2_COMP_EN),
-		   REG_FIELD_GET(XE2_L3_CLOS, pat),
-		   REG_FIELD_GET(XE2_L3_POLICY, pat),
-		   REG_FIELD_GET(XE2_L4_POLICY, pat),
-		   REG_FIELD_GET(XE2_COH_MODE, pat),
-		   pat);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 static int xe3p_xpc_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u32 pat;
 	int i;
+	char label[PAT_LABEL_LEN];
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	drm_printf(p, "PAT table: (* = reserved entry)\n");
 	for (i = 0; i < xe->pat.n_entries; i++) {
 		pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
-		drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u ]  (%#8x)%s\n", i,
+		xe_pat_index_label(label, sizeof(label), i);
-			   !!(pat & XE2_NO_PROMOTE),
+		xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
-			   REG_FIELD_GET(XE2_L3_CLOS, pat),
-			   REG_FIELD_GET(XE2_L3_POLICY, pat),
-			   REG_FIELD_GET(XE2_L4_POLICY, pat),
-			   REG_FIELD_GET(XE2_COH_MODE, pat),
-			   pat, xe->pat.table[i].valid ? "" : " *");
 	}
 	/*
 	pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
 	drm_printf(p, "Page Table Access:\n");
-	drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u ]  (%#8x)\n",
+	xe3p_xpc_pat_entry_dump(p, "PTA_MODE", pat, false);
-		   !!(pat & XE2_NO_PROMOTE),
-		   REG_FIELD_GET(XE2_L3_CLOS, pat),
-		   REG_FIELD_GET(XE2_L3_POLICY, pat),
-		   REG_FIELD_GET(XE2_L4_POLICY, pat),
-		   REG_FIELD_GET(XE2_COH_MODE, pat),
-		   pat);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	return 0;
 }
 	return xe->pat.ops->dump(gt, p);
 }
+/**
+ * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer.
+ * @gt: the &xe_gt
+ * @p: the &drm_printer
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p)
+{
+	struct xe_device *xe = gt_to_xe(gt);
+	char label[PAT_LABEL_LEN];
+	if (!xe->pat.table || !xe->pat.n_entries)
+		return -EOPNOTSUPP;
+	drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : "");
+	for (u32 i = 0; i < xe->pat.n_entries; i++) {
+		u32 pat = xe->pat.table[i].value;
+		if (GRAPHICS_VERx100(xe) == 3511) {
+			xe_pat_index_label(label, sizeof(label), i);
+			xe3p_xpc_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
+		} else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) {
+			xe_pat_index_label(label, sizeof(label), i);
+			xe2_pat_entry_dump(p, label, pat, !xe->pat.table[i].valid);
+		} else if (xe->info.platform == XE_METEORLAKE) {
+			xelpg_pat_entry_dump(p, i, pat);
+		} else if (xe->info.platform == XE_PVC) {
+			xehpc_pat_entry_dump(p, i, pat);
+		} else if (xe->info.platform == XE_DG2 || GRAPHICS_VERx100(xe) <= 1210) {
+			xelp_pat_entry_dump(p, i, pat);
+		} else {
+			return -EOPNOTSUPP;
+		}
+	}
+	if (xe->pat.pat_pta) {
+		u32 pat = xe->pat.pat_pta->value;
+		drm_printf(p, "Page Table Access:\n");
+		xe2_pat_entry_dump(p, "PTA_MODE", pat, false);
+	}
+	if (xe->pat.pat_ats) {
+		u32 pat = xe->pat.pat_ats->value;
+		drm_printf(p, "PCIe ATS/PASID:\n");
+		xe2_pat_entry_dump(p, "PAT_ATS ", pat, false);
+	}
+	drm_printf(p, "Cache Level:\n");
+	drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]);
+	drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]);
+	drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]);
+	if (GRAPHICS_VER(xe) >= 20) {
+		drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n",
+			   xe->pat.idx[XE_CACHE_NONE_COMPRESSION]);
+	}
+	return 0;
+}

drivers/gpu/drm/xe/xe_pat.h+21−0

@@ -49,6 +49,7 @@ void xe_pat_init_early(struct xe_device *xe); · 48 unmodified lines
49	void xe_pat_init(struct xe_gt *gt);	49	void xe_pat_init(struct xe_gt *gt);
50		50
51	int xe_pat_dump(struct xe_gt gt, struct drm_printer p);	51	int xe_pat_dump(struct xe_gt gt, struct drm_printer p);
		52	int xe_pat_dump_sw_config(struct xe_gt gt, struct drm_printer p);
52		53
53	/**	54	/**
54	* xe_pat_index_get_coh_mode - Extract the coherency mode for the given	55	* xe_pat_index_get_coh_mode - Extract the coherency mode for the given
@@ -58,4 +59,24 @@ int xe_pat_dump(struct xe_gt gt, struct drm_printer p); · 3 unmodified lines
58	*/	59	*/
59	u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index);	60	u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index);
60		61
		62	/**
		63	* xe_pat_index_get_comp_en - Extract the compression enable flag for
		64	* the given pat_index.
		65	* @xe: xe device
		66	* @pat_index: The pat_index to query
		67	*
		68	* Return: true if compression is enabled for this pat_index, false otherwise.
		69	*/
		70	bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index);
		71
		72	#define XE_L3_POLICY_WB 0 /* Write-back */
		73	#define XE_L3_POLICY_XD 1 /* WB - Transient Display */
		74	#define XE_L3_POLICY_UC 3 /* Uncached */
		75	/**
		76	* xe_pat_index_get_l3_policy - Extract the L3 policy for the given pat_index.
		77	* @xe: xe device
		78	* @pat_index: The pat_index to query
		79	*/
		80	u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index);
		81
61	#endif	82	#endif

drivers/gpu/drm/xe/xe_pci.c+29−0

 static const struct xe_graphics_desc graphics_xe3p_xpc = {
 	XE2_GFX_FEATURES,
+	.has_indirect_ring_state = 1,
 	.hw_engine_mask =
 		GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) |
 		GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0),
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(TIGERLAKE),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.has_sriov = true,
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(ROCKETLAKE),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.max_gt_per_tile = 1,
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(ALDERLAKE_S),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.has_sriov = true,
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(ALDERLAKE_P),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.has_sriov = true,
 	.pre_gmdid_media_ip = &media_ip_xem,
 	PLATFORM(ALDERLAKE_N),
 	.dma_mask_size = 39,
+	.has_cached_pt = true,
 	.has_display = true,
 	.has_llc = true,
 	.has_sriov = true,
 	.has_mbx_power_limits = true,
 	.has_gsc_nvm = 1,
 	.has_heci_cscfi = 1,
+	.has_i2c = true,
 	.has_late_bind = true,
+	.has_pre_prod_wa = 1,
 	.has_sriov = true,
 	.has_mem_copy_instr = true,
 	.max_gt_per_tile = 2,
 	.has_flat_ccs = 1,
 	.has_sriov = true,
 	.has_mem_copy_instr = true,
+	.has_pre_prod_wa = 1,
 	.max_gt_per_tile = 2,
 	.needs_scratch = true,
 	.needs_shared_vf_gt_wq = true,
 	.has_display = true,
 	.has_flat_ccs = 1,
 	.has_mem_copy_instr = true,
+	.has_pre_prod_wa = 1,
 	.max_gt_per_tile = 2,
 	.require_force_probe = true,
 	.va_bits = 48,
 	.dma_mask_size = 52,
 	.has_display = false,
 	.has_flat_ccs = false,
+	.has_gsc_nvm = 1,
+	.has_i2c = true,
 	.has_mbx_power_limits = true,
+	.has_mert = true,
+	.has_pre_prod_wa = 1,
 	.has_sriov = true,
 	.max_gt_per_tile = 2,
 	.require_force_probe = true,
 	xe->info.vram_flags = desc->vram_flags;
 	xe->info.is_dgfx = desc->is_dgfx;
+	xe->info.has_cached_pt = desc->has_cached_pt;
 	xe->info.has_fan_control = desc->has_fan_control;
 	/* runtime fusing may force flat_ccs to disabled later */
 	xe->info.has_flat_ccs = desc->has_flat_ccs;
 	xe->info.has_gsc_nvm = desc->has_gsc_nvm;
 	xe->info.has_heci_gscfi = desc->has_heci_gscfi;
 	xe->info.has_heci_cscfi = desc->has_heci_cscfi;
+	xe->info.has_i2c = desc->has_i2c;
 	xe->info.has_late_bind = desc->has_late_bind;
 	xe->info.has_llc = desc->has_llc;
+	xe->info.has_mert = desc->has_mert;
+	xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist;
+	xe->info.has_pre_prod_wa = desc->has_pre_prod_wa;
 	xe->info.has_pxp = desc->has_pxp;
 	xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) &&
 		desc->has_sriov;
 	gt->info.type = XE_GT_TYPE_MAIN;
 	gt->info.id = tile->id * xe->info.max_gt_per_tile;
 	gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state;
+	gt->info.multi_queue_engine_class_mask = graphics_desc->multi_queue_engine_class_mask;
 	gt->info.engine_mask = graphics_desc->hw_engine_mask;
 	/*
 	struct xe_device *xe = pdev_to_xe_device(pdev);
 	int err;
+	/*
+	 * We hold an additional reference to the runtime PM to keep PF in D0
+	 * during VFs lifetime, as our VFs do not implement the PM capability.
+	 * This means we should never be runtime suspending as long as VFs are
+	 * enabled.
+	 */
+	xe_assert(xe, !IS_SRIOV_VF(xe));
+	xe_assert(xe, !pci_num_vf(pdev));
 	err = xe_pm_runtime_suspend(xe);
 	if (err)
 		return err;

drivers/gpu/drm/xe/xe_pci_sriov.c+3−7

 int xe_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
 {
 	struct xe_device *xe = pdev_to_xe_device(pdev);
-	int ret;
 	if (!IS_SRIOV_PF(xe))
 		return -ENODEV;
 	if (num_vfs && pci_num_vf(pdev))
 		return -EBUSY;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	if (num_vfs > 0)
-		ret = pf_enable_vfs(xe, num_vfs);
+		return pf_enable_vfs(xe, num_vfs);
 	else
-		ret = pf_disable_vfs(xe);
+		return pf_disable_vfs(xe);
-	xe_pm_runtime_put(xe);
-	return ret;
 }
 /**

drivers/gpu/drm/xe/xe_pci_types.h+6−0

 	u8 require_force_probe:1;
 	u8 is_dgfx:1;
+	u8 has_cached_pt:1;
 	u8 has_display:1;
 	u8 has_fan_control:1;
 	u8 has_flat_ccs:1;
 	u8 has_gsc_nvm:1;
 	u8 has_heci_gscfi:1;
 	u8 has_heci_cscfi:1;
+	u8 has_i2c:1;
 	u8 has_late_bind:1;
 	u8 has_llc:1;
 	u8 has_mbx_power_limits:1;
 	u8 has_mem_copy_instr:1;
+	u8 has_mert:1;
+	u8 has_pre_prod_wa:1;
+	u8 has_page_reclaim_hw_assist:1;
 	u8 has_pxp:1;
 	u8 has_sriov:1;
 	u8 needs_scratch:1;
 struct xe_graphics_desc {
 	u64 hw_engine_mask;	/* hardware engines provided by graphics IP */
+	u16 multi_queue_engine_class_mask; /* bitmask of engine classes which support multi queue */
 	u8 has_asid:1;
 	u8 has_atomic_enable_pte_bit:1;

drivers/gpu/drm/xe/xe_pcode_api.h+2−0

 #define PCODE_SCRATCH(x)		XE_REG(0x138320 + ((x) * 4))
 /* PCODE_SCRATCH0 */
+#define   BREADCRUMB_VERSION		REG_GENMASK(31, 29)
 #define   AUXINFO_REG_OFFSET		REG_GENMASK(17, 15)
 #define   OVERFLOW_REG_OFFSET		REG_GENMASK(14, 12)
 #define   HISTORY_TRACKING		REG_BIT(11)
 #define   OVERFLOW_SUPPORT		REG_BIT(10)
 #define   AUXINFO_SUPPORT		REG_BIT(9)
+#define   FDO_MODE			REG_BIT(4)
 #define   BOOT_STATUS			REG_GENMASK(3, 1)
 #define      CRITICAL_FAILURE		4
 #define      NON_CRITICAL_FAILURE	7

drivers/gpu/drm/xe/xe_pm.c+5−5

 	}
 	for_each_gt(gt, xe, id) {
-		err = xe_gt_suspend(gt);
+		err = xe->d3cold.allowed ? xe_gt_suspend(gt) : xe_gt_runtime_suspend(gt);
 		if (err)
 			goto out_resume;
 	}
 	xe_rpm_lockmap_acquire(xe);
-	for_each_gt(gt, xe, id)
-		xe_gt_idle_disable_c6(gt);
 	if (xe->d3cold.allowed) {
+		for_each_gt(gt, xe, id)
+			xe_gt_idle_disable_c6(gt);
 		err = xe_pcode_ready(xe, true);
 		if (err)
 			goto out;
 	xe_irq_resume(xe);
 	for_each_gt(gt, xe, id)
-		xe_gt_resume(gt);
+		xe->d3cold.allowed ? xe_gt_resume(gt) : xe_gt_runtime_resume(gt);
 	xe_display_pm_runtime_resume(xe);

drivers/gpu/drm/xe/xe_pmu.c+1−1

@@ -425,7 +425,7 @@ static ssize_t event_attr_show(struct device *dev, · 424 unmodified lines
425	struct perf_pmu_events_attr *pmu_attr =	425	struct perf_pmu_events_attr *pmu_attr =
426	container_of(attr, struct perf_pmu_events_attr, attr);	426	container_of(attr, struct perf_pmu_events_attr, attr);
427		427
428	return sprintf(buf, "event=%#04llx\n", pmu_attr->id);	428	return sysfs_emit(buf, "event=%#04llx\n", pmu_attr->id);
429	}	429	}
430		430
431	#define XE_EVENT_ATTR(name_, v_, id_) \	431	#define XE_EVENT_ATTR(name_, v_, id_) \

drivers/gpu/drm/xe/xe_pt.c+134−1

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	#include "xe_exec_queue.h"	12	#include "xe_exec_queue.h"
13	#include "xe_gt.h"	13	#include "xe_gt.h"
14	#include "xe_migrate.h"	14	#include "xe_migrate.h"
		15	#include "xe_page_reclaim.h"
15	#include "xe_pt_types.h"	16	#include "xe_pt_types.h"
16	#include "xe_pt_walk.h"	17	#include "xe_pt_walk.h"
17	#include "xe_res_cursor.h"	18	#include "xe_res_cursor.h"
@@ -1535,6 +1536,9 @@ struct xe_pt_stage_unbind_walk { · 1517 unmodified lines
1535	/** @modified_end: Walk range start, modified like @modified_start. */	1536	/** @modified_end: Walk range start, modified like @modified_start. */
1536	u64 modified_end;	1537	u64 modified_end;
1537		1538
		1539	/** @prl: Backing pointer to page reclaim list in pt_update_ops */
		1540	struct xe_page_reclaim_list *prl;
		1541
1538	/* Output */	1542	/* Output */
1539	/* @wupd: Structure to track the page-table updates we're building */	1543	/* @wupd: Structure to track the page-table updates we're building */
1540	struct xe_walk_update wupd;	1544	struct xe_walk_update wupd;
@@ -1572,6 +1576,68 @@ static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, · 31 unmodified lines
1572	return false;	1576	return false;
1573	}	1577	}
1574		1578
		1579	/* Huge 2MB leaf lives directly in a level-1 table and has no children */
		1580	static bool is_2m_pte(struct xe_pt *pte)
		1581	{
		1582	return pte->level == 1 && !pte->base.children;
		1583	}
		1584
		1585	/* page_size = 2^(reclamation_size + XE_PTE_SHIFT) */
		1586	#define COMPUTE_RECLAIM_ADDRESS_MASK(page_size) \
		1587	({ \
		1588	BUILD_BUG_ON(!__builtin_constant_p(page_size)); \
		1589	ilog2(page_size) - XE_PTE_SHIFT; \
		1590	})
		1591
		1592	static int generate_reclaim_entry(struct xe_tile *tile,
		1593	struct xe_page_reclaim_list *prl,
		1594	u64 pte, struct xe_pt *xe_child)
		1595	{
		1596	struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries;
		1597	u64 phys_page = (pte & XE_PTE_ADDR_MASK) >> XE_PTE_SHIFT;
		1598	int num_entries = prl->num_entries;
		1599	u32 reclamation_size;
		1600
		1601	xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL);
		1602	xe_tile_assert(tile, reclaim_entries);
		1603	xe_tile_assert(tile, num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1);
		1604
		1605	if (!xe_page_reclaim_list_valid(prl))
		1606	return -EINVAL;
		1607
		1608	/**
		1609	* reclamation_size indicates the size of the page to be
		1610	* invalidated and flushed from non-coherent cache.
		1611	* Page size is computed as 2^(reclamation_size + XE_PTE_SHIFT) bytes.
		1612	* Only 4K, 64K (level 0), and 2M pages are supported by hardware for page reclaim
		1613	*/
		1614	if (xe_child->level == 0 && !(pte & XE_PTE_PS64)) {
		1615	reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_4K); /* reclamation_size = 0 */
		1616	} else if (xe_child->level == 0) {
		1617	reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_64K); /* reclamation_size = 4 */
		1618	} else if (is_2m_pte(xe_child)) {
		1619	reclamation_size = COMPUTE_RECLAIM_ADDRESS_MASK(SZ_2M); /* reclamation_size = 9 */
		1620	} else {
		1621	xe_page_reclaim_list_invalidate(prl);
		1622	vm_dbg(&tile_to_xe(tile)->drm,
		1623	"PRL invalidate: unsupported PTE level=%u pte=%#llx\n",
		1624	xe_child->level, pte);
		1625	return -EINVAL;
		1626	}
		1627
		1628	reclaim_entries[num_entries].qw =
		1629	FIELD_PREP(XE_PAGE_RECLAIM_VALID, 1) \|
		1630	FIELD_PREP(XE_PAGE_RECLAIM_SIZE, reclamation_size) \|
		1631	FIELD_PREP(XE_PAGE_RECLAIM_ADDR_LO, phys_page) \|
		1632	FIELD_PREP(XE_PAGE_RECLAIM_ADDR_HI, phys_page >> 20);
		1633	prl->num_entries++;
		1634	vm_dbg(&tile_to_xe(tile)->drm,
		1635	"PRL add entry: level=%u pte=%#llx reclamation_size=%u prl_idx=%d\n",
		1636	xe_child->level, pte, reclamation_size, num_entries);
		1637
		1638	return 0;
		1639	}
		1640
1575	static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,	1641	static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
1576	unsigned int level, u64 addr, u64 next,	1642	unsigned int level, u64 addr, u64 next,
1577	struct xe_ptw **child,	1643	struct xe_ptw **child,
@@ -1579,11 +1645,48 @@ static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, · 1 unmodified line
1579	struct xe_pt_walk *walk)	1645	struct xe_pt_walk *walk)
1580	{	1646	{
1581	struct xe_pt xe_child = container_of(child, typeof(*xe_child), base);	1647	struct xe_pt xe_child = container_of(child, typeof(*xe_child), base);
		1648	struct xe_pt_stage_unbind_walk *xe_walk =
		1649	container_of(walk, typeof(*xe_walk), base);
		1650	struct xe_device *xe = tile_to_xe(xe_walk->tile);
1582		1651
1583	XE_WARN_ON(!*child);	1652	XE_WARN_ON(!*child);
1584	XE_WARN_ON(!level);	1653	XE_WARN_ON(!level);
		1654	/* Check for leaf node */
		1655	if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) &&
		1656	!xe_child->base.children) {
		1657	struct iosys_map *leaf_map = &xe_child->bo->vmap;
		1658	pgoff_t first = xe_pt_offset(addr, 0, walk);
		1659	pgoff_t count = xe_pt_num_entries(addr, next, 0, walk);
		1660
		1661	for (pgoff_t i = 0; i < count; i++) {
		1662	u64 pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
		1663	int ret;
		1664
		1665	/* Account for NULL terminated entry on end (-1) */
		1666	if (xe_walk->prl->num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1) {
		1667	ret = generate_reclaim_entry(xe_walk->tile, xe_walk->prl,
		1668	pte, xe_child);
		1669	if (ret)
		1670	break;
		1671	} else {
		1672	/* overflow, mark as invalid */
		1673	xe_page_reclaim_list_invalidate(xe_walk->prl);
		1674	vm_dbg(&xe->drm,
		1675	"PRL invalidate: overflow while adding pte=%#llx",
		1676	pte);
		1677	break;
		1678	}
		1679	}
		1680	}
1585		1681
1586	xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);	1682	/* If aborting page walk early, invalidate PRL since PTE may be dropped from this abort */
		1683	if (xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk) &&
		1684	xe_walk->prl && level > 1 && xe_child->base.children && xe_child->num_live != 0) {
		1685	xe_page_reclaim_list_invalidate(xe_walk->prl);
		1686	vm_dbg(&xe->drm,
		1687	"PRL invalidate: kill at level=%u addr=%#llx next=%#llx num_live=%u\n",
		1688	level, addr, next, xe_child->num_live);
		1689	}
1587		1690
1588	return 0;	1691	return 0;
1589	}	1692	}
@@ -1654,6 +1757,8 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, · 64 unmodified lines
1654	{	1757	{
1655	u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma);	1758	u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma);
1656	u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma);	1759	u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma);
		1760	struct xe_vm_pgtable_update_op *pt_update_op =
		1761	container_of(entries, struct xe_vm_pgtable_update_op, entries[0]);
1657	struct xe_pt_stage_unbind_walk xe_walk = {	1762	struct xe_pt_stage_unbind_walk xe_walk = {
1658	.base = {	1763	.base = {
1659	.ops = &xe_pt_stage_unbind_ops,	1764	.ops = &xe_pt_stage_unbind_ops,
@@ -1665,6 +1770,7 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, · 5 unmodified lines
1665	.modified_start = start,	1770	.modified_start = start,
1666	.modified_end = end,	1771	.modified_end = end,
1667	.wupd.entries = entries,	1772	.wupd.entries = entries,
		1773	.prl = pt_update_op->prl,
1668	};	1774	};
1669	struct xe_pt *pt = vm->pt_root[tile->id];	1775	struct xe_pt *pt = vm->pt_root[tile->id];
1670		1776
@@ -1897,6 +2003,7 @@ static int unbind_op_prepare(struct xe_tile *tile, · 226 unmodified lines
1897	struct xe_vm_pgtable_update_ops *pt_update_ops,	2003	struct xe_vm_pgtable_update_ops *pt_update_ops,
1898	struct xe_vma *vma)	2004	struct xe_vma *vma)
1899	{	2005	{
		2006	struct xe_device *xe = tile_to_xe(tile);
1900	u32 current_op = pt_update_ops->current_op;	2007	u32 current_op = pt_update_ops->current_op;
1901	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];	2008	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1902	int err;	2009	int err;
@@ -1914,6 +2021,17 @@ static int unbind_op_prepare(struct xe_tile *tile, · 11 unmodified lines
1914	pt_op->vma = vma;	2021	pt_op->vma = vma;
1915	pt_op->bind = false;	2022	pt_op->bind = false;
1916	pt_op->rebind = false;	2023	pt_op->rebind = false;
		2024	/*
		2025	* Maintain one PRL located in pt_update_ops that all others in unbind op reference.
		2026	* Ensure that PRL is allocated only once, and if invalidated, remains an invalidated PRL.
		2027	*/
		2028	if (xe->info.has_page_reclaim_hw_assist &&
		2029	xe_page_reclaim_list_is_new(&pt_update_ops->prl))
		2030	xe_page_reclaim_list_alloc_entries(&pt_update_ops->prl);
		2031
		2032	/* Page reclaim may not be needed due to other features, so skip the corresponding VMA */
		2033	pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl) &&
		2034	!xe_page_reclaim_skip(tile, vma)) ? &pt_update_ops->prl : NULL;
1917		2035
1918	err = vma_reserve_fences(tile_to_xe(tile), vma);	2036	err = vma_reserve_fences(tile_to_xe(tile), vma);
1919	if (err)	2037	if (err)
@@ -1979,6 +2097,7 @@ static int unbind_range_prepare(struct xe_vm *vm, · 59 unmodified lines
1979	pt_op->vma = XE_INVALID_VMA;	2097	pt_op->vma = XE_INVALID_VMA;
1980	pt_op->bind = false;	2098	pt_op->bind = false;
1981	pt_op->rebind = false;	2099	pt_op->rebind = false;
		2100	pt_op->prl = NULL;
1982		2101
1983	pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,	2102	pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,
1984	pt_op->entries);	2103	pt_op->entries);
@@ -2096,6 +2215,7 @@ xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) · 111 unmodified lines
2096	init_llist_head(&pt_update_ops->deferred);	2215	init_llist_head(&pt_update_ops->deferred);
2097	pt_update_ops->start = ~0x0ull;	2216	pt_update_ops->start = ~0x0ull;
2098	pt_update_ops->last = 0x0ull;	2217	pt_update_ops->last = 0x0ull;
		2218	xe_page_reclaim_list_init(&pt_update_ops->prl);
2099	}	2219	}
2100		2220
2101	/**	2221	/**
@@ -2393,6 +2513,17 @@ xe_pt_update_ops_run(struct xe_tile tile, struct xe_vma_ops vops) · 291 unmodified lines
2393	goto kill_vm_tile1;	2513	goto kill_vm_tile1;
2394	}	2514	}
2395	update.ijob = ijob;	2515	update.ijob = ijob;
		2516	/*
		2517	* Only add page reclaim for the primary GT. Media GT does not have
		2518	* any PPC to flush, so enabling the PPC flush bit for media is
		2519	* effectively a NOP and provides no performance benefit nor
		2520	* interfere with primary GT.
		2521	*/
		2522	if (xe_page_reclaim_list_valid(&pt_update_ops->prl)) {
		2523	xe_tlb_inval_job_add_page_reclaim(ijob, &pt_update_ops->prl);
		2524	/* Release ref from alloc, job will now handle it */
		2525	xe_page_reclaim_list_invalidate(&pt_update_ops->prl);
		2526	}
2396		2527
2397	if (tile->media_gt) {	2528	if (tile->media_gt) {
2398	dep_scheduler = to_dep_scheduler(q, tile->media_gt);	2529	dep_scheduler = to_dep_scheduler(q, tile->media_gt);
@@ -2518,6 +2649,8 @@ void xe_pt_update_ops_fini(struct xe_tile tile, struct xe_vma_ops vops) · 119 unmodified lines
2518	&vops->pt_update_ops[tile->id];	2649	&vops->pt_update_ops[tile->id];
2519	int i;	2650	int i;
2520		2651
		2652	xe_page_reclaim_entries_put(pt_update_ops->prl.entries);
		2653
2521	lockdep_assert_held(&vops->vm->lock);	2654	lockdep_assert_held(&vops->vm->lock);
2522	xe_vm_assert_held(vops->vm);	2655	xe_vm_assert_held(vops->vm);
2523		2656

drivers/gpu/drm/xe/xe_pt_types.h+5−0

 #include <linux/types.h>
+#include "xe_page_reclaim.h"
 #include "xe_pt_walk.h"
 struct xe_bo;
 	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
 	/** @vma: VMA for operation, operation not valid if NULL */
 	struct xe_vma *vma;
+	/** @prl: Backing pointer to page reclaim list of pt_update_ops */
+	struct xe_page_reclaim_list *prl;
 	/** @num_entries: number of entries for this update operation */
 	u32 num_entries;
 	/** @bind: is a bind */
 	struct llist_head deferred;
 	/** @q: exec queue for PT operations */
 	struct xe_exec_queue *q;
+	/** @prl: embedded page reclaim list */
+	struct xe_page_reclaim_list prl;
 	/** @start: start address of ops */
 	u64 start;
 	/** @last: last address of ops */

drivers/gpu/drm/xe/xe_pxp.c+18−37

@@ -58,10 +58,9 @@ bool xe_pxp_is_enabled(const struct xe_pxp *pxp) · 57 unmodified lines
58	static bool pxp_prerequisites_done(const struct xe_pxp *pxp)	58	static bool pxp_prerequisites_done(const struct xe_pxp *pxp)
59	{	59	{
60	struct xe_gt *gt = pxp->gt;	60	struct xe_gt *gt = pxp->gt;
61	unsigned int fw_ref;
62	bool ready;	61	bool ready;
63		62
64	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);	63	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
65		64
66	/*	65	/*
67	* If force_wake fails we could falsely report the prerequisites as not	66	* If force_wake fails we could falsely report the prerequisites as not
@@ -71,14 +70,12 @@ static bool pxp_prerequisites_done(const struct xe_pxp *pxp) · 3 unmodified lines
71	* PXP. Therefore, we can just log the force_wake error and not escalate	70	* PXP. Therefore, we can just log the force_wake error and not escalate
72	* it.	71	* it.
73	*/	72	*/
74	XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL));	73	XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL));
75		74
76	/* PXP requires both HuC authentication via GSC and GSC proxy initialized */	75	/* PXP requires both HuC authentication via GSC and GSC proxy initialized */
77	ready = xe_huc_is_authenticated(&gt->uc.huc, XE_HUC_AUTH_VIA_GSC) &&	76	ready = xe_huc_is_authenticated(&gt->uc.huc, XE_HUC_AUTH_VIA_GSC) &&
78	xe_gsc_proxy_init_done(&gt->uc.gsc);	77	xe_gsc_proxy_init_done(&gt->uc.gsc);
79		78
80	xe_force_wake_put(gt_to_fw(gt), fw_ref);
81
82	return ready;	79	return ready;
83	}	80	}
84		81
@@ -104,13 +101,12 @@ int xe_pxp_get_readiness_status(struct xe_pxp *pxp) · 19 unmodified lines
104	xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status))	101	xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status))
105	return -EIO;	102	return -EIO;
106		103
107	xe_pm_runtime_get(pxp->xe);	104	guard(xe_pm_runtime)(pxp->xe);
108		105
109	/* PXP requires both HuC loaded and GSC proxy initialized */	106	/* PXP requires both HuC loaded and GSC proxy initialized */
110	if (pxp_prerequisites_done(pxp))	107	if (pxp_prerequisites_done(pxp))
111	ret = 1;	108	ret = 1;
112		109
113	xe_pm_runtime_put(pxp->xe);
114	return ret;	110	return ret;
115	}	111	}
116		112
@@ -135,35 +131,28 @@ static void pxp_invalidate_queues(struct xe_pxp *pxp); · 18 unmodified lines
135	static int pxp_terminate_hw(struct xe_pxp *pxp)	131	static int pxp_terminate_hw(struct xe_pxp *pxp)
136	{	132	{
137	struct xe_gt *gt = pxp->gt;	133	struct xe_gt *gt = pxp->gt;
138	unsigned int fw_ref;
139	int ret = 0;	134	int ret = 0;
140		135
141	drm_dbg(&pxp->xe->drm, "Terminating PXP\n");	136	drm_dbg(&pxp->xe->drm, "Terminating PXP\n");
142		137
143	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);	138	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
144	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {	139	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
145	ret = -EIO;	140	return -EIO;
146	goto out;
147	}
148		141
149	/* terminate the hw session */	142	/* terminate the hw session */
150	ret = xe_pxp_submit_session_termination(pxp, ARB_SESSION);	143	ret = xe_pxp_submit_session_termination(pxp, ARB_SESSION);
151	if (ret)	144	if (ret)
152	goto out;	145	return ret;
153		146
154	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, false);	147	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, false);
155	if (ret)	148	if (ret)
156	goto out;	149	return ret;
157		150
158	/* Trigger full HW cleanup */	151	/* Trigger full HW cleanup */
159	xe_mmio_write32(&gt->mmio, KCR_GLOBAL_TERMINATE, 1);	152	xe_mmio_write32(&gt->mmio, KCR_GLOBAL_TERMINATE, 1);
160		153
161	/* now we can tell the GSC to clean up its own state */	154	/* now we can tell the GSC to clean up its own state */
162	ret = xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION);	155	return xe_pxp_submit_session_invalidation(&pxp->gsc_res, ARB_SESSION);
163
164	out:
165	xe_force_wake_put(gt_to_fw(gt), fw_ref);
166	return ret;
167	}	156	}
168		157
169	static void mark_termination_in_progress(struct xe_pxp *pxp)	158	static void mark_termination_in_progress(struct xe_pxp *pxp)
@@ -326,14 +315,12 @@ static int kcr_pxp_set_status(const struct xe_pxp *pxp, bool enable) · 156 unmodified lines
326	{	315	{
327	u32 val = enable ? _MASKED_BIT_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :	316	u32 val = enable ? _MASKED_BIT_ENABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES) :
328	_MASKED_BIT_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);	317	_MASKED_BIT_DISABLE(KCR_INIT_ALLOW_DISPLAY_ME_WRITES);
329	unsigned int fw_ref;
330		318
331	fw_ref = xe_force_wake_get(gt_to_fw(pxp->gt), XE_FW_GT);	319	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
332	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT))	320	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
333	return -EIO;	321	return -EIO;
334		322
335	xe_mmio_write32(&pxp->gt->mmio, KCR_INIT, val);	323	xe_mmio_write32(&pxp->gt->mmio, KCR_INIT, val);
336	xe_force_wake_put(gt_to_fw(pxp->gt), fw_ref);
337		324
338	return 0;	325	return 0;
339	}	326	}
@@ -453,34 +440,28 @@ out: · 113 unmodified lines
453	static int __pxp_start_arb_session(struct xe_pxp *pxp)	440	static int __pxp_start_arb_session(struct xe_pxp *pxp)
454	{	441	{
455	int ret;	442	int ret;
456	unsigned int fw_ref;
457		443
458	fw_ref = xe_force_wake_get(gt_to_fw(pxp->gt), XE_FW_GT);	444	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pxp->gt), XE_FW_GT);
459	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT))	445	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT))
460	return -EIO;	446	return -EIO;
461		447
462	if (pxp_session_is_in_play(pxp, ARB_SESSION)) {	448	if (pxp_session_is_in_play(pxp, ARB_SESSION))
463	ret = -EEXIST;	449	return -EEXIST;
464	goto out_force_wake;
465	}
466		450
467	ret = xe_pxp_submit_session_init(&pxp->gsc_res, ARB_SESSION);	451	ret = xe_pxp_submit_session_init(&pxp->gsc_res, ARB_SESSION);
468	if (ret) {	452	if (ret) {
469	drm_err(&pxp->xe->drm, "Failed to init PXP arb session: %pe\n", ERR_PTR(ret));	453	drm_err(&pxp->xe->drm, "Failed to init PXP arb session: %pe\n", ERR_PTR(ret));
470	goto out_force_wake;	454	return ret;
471	}	455	}
472		456
473	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true);	457	ret = pxp_wait_for_session_state(pxp, ARB_SESSION, true);
474	if (ret) {	458	if (ret) {
475	drm_err(&pxp->xe->drm, "PXP ARB session failed to go in play%pe\n", ERR_PTR(ret));	459	drm_err(&pxp->xe->drm, "PXP ARB session failed to go in play%pe\n", ERR_PTR(ret));
476	goto out_force_wake;	460	return ret;
477	}	461	}
478		462
479	drm_dbg(&pxp->xe->drm, "PXP ARB session is active\n");	463	drm_dbg(&pxp->xe->drm, "PXP ARB session is active\n");
480		464	return 0;
481	out_force_wake:
482	xe_force_wake_put(gt_to_fw(pxp->gt), fw_ref);
483	return ret;
484	}	465	}
485		466
486	/**	467	/**

drivers/gpu/drm/xe/xe_query.c+12−11

 	__ktime_func_t cpu_clock;
 	struct xe_hw_engine *hwe;
 	struct xe_gt *gt;
-	unsigned int fw_ref;
 	if (IS_SRIOV_VF(xe))
 		return -EOPNOTSUPP;
 	if (!hwe)
 		return -EINVAL;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL) {
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))  {
+		if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
-		xe_force_wake_put(gt_to_fw(gt), fw_ref);
+			return -EIO;
-		return -EIO;
-	}
-	hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp,
-			   &resp.cpu_delta, cpu_clock);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
+		hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp,
+				   &resp.cpu_delta, cpu_clock);
+	}
 	if (GRAPHICS_VER(xe) >= 20)
 		resp.width = 64;
 	if (xe->info.has_usm && IS_ENABLED(CONFIG_DRM_XE_GPUSVM))
 		config->info[DRM_XE_QUERY_CONFIG_FLAGS] |=
 			DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR;
+	if (GRAPHICS_VER(xe) >= 20)
+		config->info[DRM_XE_QUERY_CONFIG_FLAGS] |=
+			DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT;
 	config->info[DRM_XE_QUERY_CONFIG_FLAGS] |=
 			DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY;
 	config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] =
 			du->capabilities = DRM_XE_OA_CAPS_BASE | DRM_XE_OA_CAPS_SYNCS |
 					   DRM_XE_OA_CAPS_OA_BUFFER_SIZE |
 					   DRM_XE_OA_CAPS_WAIT_NUM_REPORTS |
-					   DRM_XE_OA_CAPS_OAM;
+					   DRM_XE_OA_CAPS_OAM |
+					   DRM_XE_OA_CAPS_OA_UNIT_GT_ID;
+			du->gt_id = u->gt->info.id;
 			j = 0;
 			for_each_hw_engine(hwe, gt, hwe_id) {
 				if (!xe_hw_engine_is_reserved(hwe) &&

drivers/gpu/drm/xe/xe_reg_sr.c+5−12

 {
 	struct xe_reg_sr_entry *entry;
 	unsigned long reg;
-	unsigned int fw_ref;
 	if (xa_empty(&sr->xa))
 		return;
 	xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
-	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
+	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) {
-		goto err_force_wake;
+		xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n");
+		return;
+	}
 	xa_for_each(&sr->xa, reg, entry)
 		apply_one_mmio(gt, entry);
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	return;
-err_force_wake:
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
-	xe_gt_err(gt, "Failed to apply, err=-ETIMEDOUT\n");
 }
 /**

drivers/gpu/drm/xe/xe_reg_whitelist.c+59−22

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	#include "regs/xe_gt_regs.h"	9	#include "regs/xe_gt_regs.h"
10	#include "regs/xe_oa_regs.h"	10	#include "regs/xe_oa_regs.h"
11	#include "regs/xe_regs.h"	11	#include "regs/xe_regs.h"
		12	#include "xe_device.h"
12	#include "xe_gt_types.h"	13	#include "xe_gt_types.h"
13	#include "xe_gt_printk.h"	14	#include "xe_gt_printk.h"
14	#include "xe_platform_types.h"	15	#include "xe_platform_types.h"
@@ -26,6 +27,13 @@ static bool match_not_render(const struct xe_device *xe, · 11 unmodified lines
26	return hwe->class != XE_ENGINE_CLASS_RENDER;	27	return hwe->class != XE_ENGINE_CLASS_RENDER;
27	}	28	}
28		29
		30	static bool match_has_mert(const struct xe_device *xe,
		31	const struct xe_gt *gt,
		32	const struct xe_hw_engine *hwe)
		33	{
		34	return xe_device_has_mert((struct xe_device *)xe);
		35	}
		36
29	static const struct xe_rtp_entry_sr register_whitelist[] = {	37	static const struct xe_rtp_entry_sr register_whitelist[] = {
30	{ XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"),	38	{ XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"),
31	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),	39	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),
@@ -67,28 +75,6 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { · 35 unmodified lines
67	ENGINE_CLASS(RENDER)),	75	ENGINE_CLASS(RENDER)),
68	XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0))	76	XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0))
69	},	77	},
70	{ XE_RTP_NAME("oa_reg_render"),
71	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
72	ENGINE_CLASS(RENDER)),
73	XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER,
74	RING_FORCE_TO_NONPRIV_ACCESS_RW),
75	WHITELIST(OAG_OASTATUS,
76	RING_FORCE_TO_NONPRIV_ACCESS_RD),
77	WHITELIST(OAG_OAHEADPTR,
78	RING_FORCE_TO_NONPRIV_ACCESS_RD \|
79	RING_FORCE_TO_NONPRIV_RANGE_4))
80	},
81	{ XE_RTP_NAME("oa_reg_compute"),
82	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
83	ENGINE_CLASS(COMPUTE)),
84	XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER,
85	RING_FORCE_TO_NONPRIV_ACCESS_RW),
86	WHITELIST(OAG_OASTATUS,
87	RING_FORCE_TO_NONPRIV_ACCESS_RD),
88	WHITELIST(OAG_OAHEADPTR,
89	RING_FORCE_TO_NONPRIV_ACCESS_RD \|
90	RING_FORCE_TO_NONPRIV_RANGE_4))
91	},
92	{ XE_RTP_NAME("14024997852"),	78	{ XE_RTP_NAME("14024997852"),
93	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),	79	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),
94	XE_RTP_ACTIONS(WHITELIST(FF_MODE,	80	XE_RTP_ACTIONS(WHITELIST(FF_MODE,
@@ -96,6 +82,57 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { · 1 unmodified line
96	WHITELIST(VFLSKPD,	82	WHITELIST(VFLSKPD,
97	RING_FORCE_TO_NONPRIV_ACCESS_RW))	83	RING_FORCE_TO_NONPRIV_ACCESS_RW))
98	},	84	},
		85
		86	#define WHITELIST_OA_MMIO_TRG(trg, status, head) \
		87	WHITELIST(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \
		88	WHITELIST(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \
		89	WHITELIST(head, RING_FORCE_TO_NONPRIV_ACCESS_RD \| RING_FORCE_TO_NONPRIV_RANGE_4)
		90
		91	#define WHITELIST_OAG_MMIO_TRG \
		92	WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR)
		93
		94	#define WHITELIST_OAM_MMIO_TRG \
		95	WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SAG_BASE_ADJ), \
		96	OAM_STATUS(XE_OAM_SAG_BASE_ADJ), \
		97	OAM_HEAD_POINTER(XE_OAM_SAG_BASE_ADJ)), \
		98	WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_0_BASE_ADJ), \
		99	OAM_STATUS(XE_OAM_SCMI_0_BASE_ADJ), \
		100	OAM_HEAD_POINTER(XE_OAM_SCMI_0_BASE_ADJ)), \
		101	WHITELIST_OA_MMIO_TRG(OAM_MMIO_TRG(XE_OAM_SCMI_1_BASE_ADJ), \
		102	OAM_STATUS(XE_OAM_SCMI_1_BASE_ADJ), \
		103	OAM_HEAD_POINTER(XE_OAM_SCMI_1_BASE_ADJ))
		104
		105	#define WHITELIST_OA_MERT_MMIO_TRG \
		106	WHITELIST_OA_MMIO_TRG(OAMERT_MMIO_TRG, OAMERT_STATUS, OAMERT_HEAD_POINTER)
		107
		108	{ XE_RTP_NAME("oag_mmio_trg_rcs"),
		109	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
		110	ENGINE_CLASS(RENDER)),
		111	XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG)
		112	},
		113	{ XE_RTP_NAME("oag_mmio_trg_ccs"),
		114	XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
		115	ENGINE_CLASS(COMPUTE)),
		116	XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG)
		117	},
		118	{ XE_RTP_NAME("oam_mmio_trg_vcs"),
		119	XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED),
		120	ENGINE_CLASS(VIDEO_DECODE)),
		121	XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG)
		122	},
		123	{ XE_RTP_NAME("oam_mmio_trg_vecs"),
		124	XE_RTP_RULES(MEDIA_VERSION_RANGE(1300, XE_RTP_END_VERSION_UNDEFINED),
		125	ENGINE_CLASS(VIDEO_ENHANCE)),
		126	XE_RTP_ACTIONS(WHITELIST_OAM_MMIO_TRG)
		127	},
		128	{ XE_RTP_NAME("oa_mert_mmio_trg_ccs"),
		129	XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COMPUTE)),
		130	XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG)
		131	},
		132	{ XE_RTP_NAME("oa_mert_mmio_trg_bcs"),
		133	XE_RTP_RULES(FUNC(match_has_mert), ENGINE_CLASS(COPY)),
		134	XE_RTP_ACTIONS(WHITELIST_OA_MERT_MMIO_TRG)
		135	},
99	};	136	};
100		137
101	static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe)	138	static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe)

drivers/gpu/drm/xe/xe_ring_ops.c+34−34

@@ -12,7 +12,7 @@ · 11 unmodified lines
12	#include "regs/xe_engine_regs.h"	12	#include "regs/xe_engine_regs.h"
13	#include "regs/xe_gt_regs.h"	13	#include "regs/xe_gt_regs.h"
14	#include "regs/xe_lrc_layout.h"	14	#include "regs/xe_lrc_layout.h"
15	#include "xe_exec_queue_types.h"	15	#include "xe_exec_queue.h"
16	#include "xe_gt.h"	16	#include "xe_gt.h"
17	#include "xe_lrc.h"	17	#include "xe_lrc.h"
18	#include "xe_macros.h"	18	#include "xe_macros.h"
@@ -135,12 +135,11 @@ emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, · 116 unmodified lines
135	return i;	135	return i;
136	}	136	}
137		137
138	static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,	138	static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags,
139	int i)	139	bool invalidate_tlb, u32 *dw, int i)
140	{	140	{
141	u32 flags0 = 0;	141	u32 flags0 = 0;
142	u32 flags1 = PIPE_CONTROL_CS_STALL \|	142	u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE \|
143	PIPE_CONTROL_COMMAND_CACHE_INVALIDATE \|
144	PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE \|	143	PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE \|
145	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|	144	PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE \|
146	PIPE_CONTROL_VF_CACHE_INVALIDATE \|	145	PIPE_CONTROL_VF_CACHE_INVALIDATE \|
@@ -152,6 +151,11 @@ static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw, · 5 unmodified lines
152	if (invalidate_tlb)	151	if (invalidate_tlb)
153	flags1 \|= PIPE_CONTROL_TLB_INVALIDATE;	152	flags1 \|= PIPE_CONTROL_TLB_INVALIDATE;
154		153
		154	if (xe_exec_queue_is_multi_queue(q))
		155	flags0 \|= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
		156	else
		157	flags1 \|= PIPE_CONTROL_CS_STALL;
		158
155	flags1 &= ~mask_flags;	159	flags1 &= ~mask_flags;
156		160
157	if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)	161	if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)
@@ -175,54 +179,52 @@ static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, · 17 unmodified lines
175		179
176	static int emit_render_cache_flush(struct xe_sched_job job, u32 dw, int i)	180	static int emit_render_cache_flush(struct xe_sched_job job, u32 dw, int i)
177	{	181	{
178	struct xe_gt *gt = job->q->gt;	182	struct xe_exec_queue *q = job->q;
		183	struct xe_gt *gt = q->gt;
179	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);	184	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
180	u32 flags;	185	u32 flags0, flags1;
181		186
182	if (XE_GT_WA(gt, 14016712196))	187	if (XE_GT_WA(gt, 14016712196))
183	i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,	188	i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
184	LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);	189	LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
185		190
186	flags = (PIPE_CONTROL_CS_STALL \|	191	flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH;
187	PIPE_CONTROL_TILE_CACHE_FLUSH \|	192	flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH \|
188	PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH \|	193	PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH \|
189	PIPE_CONTROL_DEPTH_CACHE_FLUSH \|	194	PIPE_CONTROL_DEPTH_CACHE_FLUSH \|
190	PIPE_CONTROL_DC_FLUSH_ENABLE \|	195	PIPE_CONTROL_DC_FLUSH_ENABLE \|
191	PIPE_CONTROL_FLUSH_ENABLE);	196	PIPE_CONTROL_FLUSH_ENABLE);
192		197
193	if (XE_GT_WA(gt, 1409600907))	198	if (XE_GT_WA(gt, 1409600907))
194	flags \|= PIPE_CONTROL_DEPTH_STALL;	199	flags1 \|= PIPE_CONTROL_DEPTH_STALL;
195		200
196	if (lacks_render)	201	if (lacks_render)
197	flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;	202	flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
198	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)	203	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
199	flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;	204	flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
200		205
201	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);	206	if (xe_exec_queue_is_multi_queue(q))
202	}	207	flags0 \|= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
203		208	else
204	static int emit_pipe_control_to_ring_end(struct xe_hw_engine hwe, u32 dw, int i)	209	flags1 \|= PIPE_CONTROL_CS_STALL;
205	{
206	if (hwe->class != XE_ENGINE_CLASS_RENDER)
207	return i;
208		210
209	if (XE_GT_WA(hwe->gt, 16020292621))	211	return emit_pipe_control(dw, i, flags0, flags1, 0, 0);
210	i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
211	RING_NOPID(hwe->mmio_base).addr, 0);
212
213	return i;
214	}	212	}
215		213
216	static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,	214	static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value,
217	int i)	215	bool stall_only, u32 *dw, int i)
218	{	216	{
219	u32 flags = PIPE_CONTROL_CS_STALL \| PIPE_CONTROL_GLOBAL_GTT_IVB \|	217	u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB \| PIPE_CONTROL_QW_WRITE;
220	PIPE_CONTROL_QW_WRITE;
221		218
222	if (!stall_only)	219	if (!stall_only)
223	flags \|= PIPE_CONTROL_FLUSH_ENABLE;	220	flags1 \|= PIPE_CONTROL_FLUSH_ENABLE;
224		221
225	return emit_pipe_control(dw, i, 0, flags, addr, value);	222	if (xe_exec_queue_is_multi_queue(q))
		223	flags0 \|= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
		224	else
		225	flags1 \|= PIPE_CONTROL_CS_STALL;
		226
		227	return emit_pipe_control(dw, i, flags0, flags1, addr, value);
226	}	228	}
227		229
228	static u32 get_ppgtt_flag(struct xe_sched_job *job)	230	static u32 get_ppgtt_flag(struct xe_sched_job *job)
@@ -371,7 +373,7 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, · 142 unmodified lines
371	mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;	373	mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
372		374
373	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */	375	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
374	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);	376	i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i);
375		377
376	/* hsdes: 1809175790 */	378	/* hsdes: 1809175790 */
377	if (has_aux_ccs(xe))	379	if (has_aux_ccs(xe))
@@ -391,12 +393,10 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, · 13 unmodified lines
391	job->user_fence.value,	393	job->user_fence.value,
392	dw, i);	394	dw, i);
393		395
394	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);	396	i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
395		397
396	i = emit_user_interrupt(dw, i);	398	i = emit_user_interrupt(dw, i);
397		399
398	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
399
400	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);	400	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
401		401
402	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));	402	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));

drivers/gpu/drm/xe/xe_sa.c+66−1

@@ -29,6 +29,7 @@ static void xe_sa_bo_manager_fini(struct drm_device drm, void arg) · 28 unmodified lines
29	kvfree(sa_manager->cpu_ptr);	29	kvfree(sa_manager->cpu_ptr);
30		30
31	sa_manager->bo = NULL;	31	sa_manager->bo = NULL;
		32	sa_manager->shadow = NULL;
32	}	33	}
33		34
34	/**	35	/**
@@ -37,12 +38,14 @@ static void xe_sa_bo_manager_fini(struct drm_device drm, void arg) · 2 unmodified lines
37	* @size: number of bytes to allocate	38	* @size: number of bytes to allocate
38	* @guard: number of bytes to exclude from suballocations	39	* @guard: number of bytes to exclude from suballocations
39	* @align: alignment for each suballocated chunk	40	* @align: alignment for each suballocated chunk
		41	* @flags: flags for suballocator
40	*	42	*
41	* Prepares the suballocation manager for suballocations.	43	* Prepares the suballocation manager for suballocations.
42	*	44	*
43	* Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure.	45	* Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure.
44	*/	46	*/
45	struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u32 guard, u32 align)	47	struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size,
		48	u32 guard, u32 align, u32 flags)
46	{	49	{
47	struct xe_device *xe = tile_to_xe(tile);	50	struct xe_device *xe = tile_to_xe(tile);
48	struct xe_sa_manager *sa_manager;	51	struct xe_sa_manager *sa_manager;
@@ -79,6 +82,26 @@ struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u3 · 30 unmodified lines
79	memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size);	82	memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size);
80	}	83	}
81		84
		85	if (flags & XE_SA_BO_MANAGER_FLAG_SHADOW) {
		86	struct xe_bo *shadow;
		87
		88	ret = drmm_mutex_init(&xe->drm, &sa_manager->swap_guard);
		89	if (ret)
		90	return ERR_PTR(ret);
		91
		92	shadow = xe_managed_bo_create_pin_map(xe, tile, size,
		93	XE_BO_FLAG_VRAM_IF_DGFX(tile) \|
		94	XE_BO_FLAG_GGTT \|
		95	XE_BO_FLAG_GGTT_INVALIDATE \|
		96	XE_BO_FLAG_PINNED_NORESTORE);
		97	if (IS_ERR(shadow)) {
		98	drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n",
		99	size / SZ_1K, shadow);
		100	return ERR_CAST(shadow);
		101	}
		102	sa_manager->shadow = shadow;
		103	}
		104
82	drm_suballoc_manager_init(&sa_manager->base, managed_size, align);	105	drm_suballoc_manager_init(&sa_manager->base, managed_size, align);
83	ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini,	106	ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini,
84	sa_manager);	107	sa_manager);
@@ -88,6 +111,48 @@ struct xe_sa_manager __xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u3 · 3 unmodified lines
88	return sa_manager;	111	return sa_manager;
89	}	112	}
90		113
		114	/**
		115	* xe_sa_bo_swap_shadow() - Swap the SA BO with shadow BO.
		116	* @sa_manager: the XE sub allocator manager
		117	*
		118	* Swaps the sub-allocator primary buffer object with shadow buffer object.
		119	*
		120	* Return: None.
		121	*/
		122	void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager)
		123	{
		124	struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
		125
		126	xe_assert(xe, sa_manager->shadow);
		127	lockdep_assert_held(&sa_manager->swap_guard);
		128
		129	swap(sa_manager->bo, sa_manager->shadow);
		130	if (!sa_manager->bo->vmap.is_iomem)
		131	sa_manager->cpu_ptr = sa_manager->bo->vmap.vaddr;
		132	}
		133
		134	/**
		135	* xe_sa_bo_sync_shadow() - Sync the SA Shadow BO with primary BO.
		136	* @sa_bo: the sub-allocator buffer object.
		137	*
		138	* Synchronize sub-allocator shadow buffer object with primary buffer object.
		139	*
		140	* Return: None.
		141	*/
		142	void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo)
		143	{
		144	struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager);
		145	struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
		146
		147	xe_assert(xe, sa_manager->shadow);
		148	lockdep_assert_held(&sa_manager->swap_guard);
		149
		150	xe_map_memcpy_to(xe, &sa_manager->shadow->vmap,
		151	drm_suballoc_soffset(sa_bo),
		152	xe_sa_bo_cpu_addr(sa_bo),
		153	drm_suballoc_size(sa_bo));
		154	}
		155
91	/**	156	/**
92	* __xe_sa_bo_new() - Make a suballocation but use custom gfp flags.	157	* __xe_sa_bo_new() - Make a suballocation but use custom gfp flags.
93	* @sa_manager: the &xe_sa_manager	158	* @sa_manager: the &xe_sa_manager

drivers/gpu/drm/xe/xe_sa.h+18−2

 struct dma_fence;
 struct xe_tile;
-struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 guard, u32 align);
+#define XE_SA_BO_MANAGER_FLAG_SHADOW	BIT(0)
+struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size,
+					      u32 guard, u32 align, u32 flags);
 struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size, gfp_t gfp);
 static inline struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 align)
 {
-	return __xe_sa_bo_manager_init(tile, size, SZ_4K, align);
+	return __xe_sa_bo_manager_init(tile, size, SZ_4K, align, 0);
 }
 /**
 		drm_suballoc_soffset(sa);
 }
+void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager);
+void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo);
+/**
+ * xe_sa_bo_swap_guard() - Retrieve the SA BO swap guard within sub-allocator.
+ * @sa_manager: the &xe_sa_manager
+ *
+ * Return: Sub alloctor swap guard mutex.
+ */
+static inline struct mutex *xe_sa_bo_swap_guard(struct xe_sa_manager *sa_manager)
+{
+	return &sa_manager->swap_guard;
+}
 #endif

drivers/gpu/drm/xe/xe_sa_types.h+3−0

 struct xe_sa_manager {
 	struct drm_suballoc_manager base;
 	struct xe_bo *bo;
+	struct xe_bo *shadow;
+	/** @swap_guard: Timeline guard updating @bo and @shadow */
+	struct mutex swap_guard;
 	void *cpu_ptr;
 	bool is_iomem;
 };

drivers/gpu/drm/xe/xe_sriov_packet.c+1−1

@@ -358,7 +358,7 @@ ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, · 357 unmodified lines
358		358
359	#define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \	359	#define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \
360	GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN)	360	GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN)
361	static size_t pf_descriptor_init(struct xe_device *xe, unsigned int vfid)	361	static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
362	{	362	{
363	struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid);	363	struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid);
364	struct xe_sriov_packet *data;	364	struct xe_sriov_packet *data;

drivers/gpu/drm/xe/xe_sriov_pf.c+4−0

  */
 int xe_sriov_pf_init_early(struct xe_device *xe)
 {
+	struct xe_mert *mert = &xe_device_get_root_tile(xe)->mert;
 	int err;
 	xe_assert(xe, IS_SRIOV_PF(xe));
 	xe_sriov_pf_service_init(xe);
+	spin_lock_init(&mert->lock);
+	init_completion(&mert->tlb_inv_done);
 	return 0;
 }

drivers/gpu/drm/xe/xe_sriov_pf_debugfs.c+2−4

 	if (ret < 0)
 		return ret;
 	if (yes) {
-		xe_pm_runtime_get(xe);
+		guard(xe_pm_runtime)(xe);
 		ret = call(xe);
-		xe_pm_runtime_put(xe);
 	}
 	if (ret < 0)
 		return ret;
 	if (ret < 0)
 		return ret;
 	if (yes) {
-		xe_pm_runtime_get(xe);
+		guard(xe_pm_runtime)(xe);
 		ret = call(xe, vfid);
-		xe_pm_runtime_put(xe);
 	}
 	if (ret < 0)
 		return ret;

drivers/gpu/drm/xe/xe_sriov_pf_sysfs.c+4−12

 	struct xe_sriov_dev_attr *vattr = to_xe_sriov_dev_attr(attr);
 	struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj);
 	struct xe_device *xe = vkobj->xe;
-	ssize_t ret;
 	if (!vattr->store)
 		return -EPERM;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
-	ret = xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count);
+	return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, buf, count);
-	xe_pm_runtime_put(xe);
-	return ret;
 }
 static ssize_t xe_sriov_vf_attr_show(struct kobject *kobj, struct attribute *attr, char *buf)
 	struct xe_sriov_kobj *vkobj = to_xe_sriov_kobj(kobj);
 	struct xe_device *xe = vkobj->xe;
 	unsigned int vfid = vkobj->vfid;
-	ssize_t ret;
 	xe_sriov_pf_assert_vfid(xe, vfid);
 	if (!vattr->store)
 		return -EPERM;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
-	ret = xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count);
+	return xe_sriov_pf_wait_ready(xe) ?: vattr->store(xe, vfid, buf, count);
-	xe_pm_runtime_get(xe);
-	return ret;
 }
 static const struct sysfs_ops xe_sriov_dev_sysfs_ops = {

drivers/gpu/drm/xe/xe_sriov_vf.c+80−4

@@ -49,11 +49,13 @@ · 48 unmodified lines
49	*	49	*
50	* As soon as Virtual GPU of the VM starts, the VF driver within receives	50	* As soon as Virtual GPU of the VM starts, the VF driver within receives
51	* the MIGRATED interrupt and schedules post-migration recovery worker.	51	* the MIGRATED interrupt and schedules post-migration recovery worker.
52	* That worker queries GuC for new provisioning (using MMIO communication),	52	* That worker sends `VF2GUC_RESFIX_START` action along with non-zero
		53	* marker, queries GuC for new provisioning (using MMIO communication),
53	* and applies fixups to any non-virtualized resources used by the VF.	54	* and applies fixups to any non-virtualized resources used by the VF.
54	*	55	*
55	* When the VF driver is ready to continue operation on the newly connected	56	* When the VF driver is ready to continue operation on the newly connected
56	* hardware, it sends `VF2GUC_NOTIFY_RESFIX_DONE` which causes it to	57	* hardware, it sends `VF2GUC_RESFIX_DONE` action along with the same
		58	* marker which was sent with `VF2GUC_RESFIX_START` which causes it to
57	* enter the long awaited `VF_RUNNING` state, and therefore start handling	59	* enter the long awaited `VF_RUNNING` state, and therefore start handling
58	* CTB messages and scheduling workloads from the VF::	60	* CTB messages and scheduling workloads from the VF::
59	*	61	*
@@ -102,12 +104,17 @@ · 42 unmodified lines
102	* \| [ ] new VF provisioning [ ]	104	* \| [ ] new VF provisioning [ ]
103	* \| [ ]---------------------------> [ ]	105	* \| [ ]---------------------------> [ ]
104	* \| \| [ ]	106	* \| \| [ ]
		107	* \| \| VF2GUC_RESFIX_START [ ]
		108	* \| [ ] <---------------------------[ ]
		109	* \| [ ] [ ]
		110	* \| [ ] success [ ]
		111	* \| [ ]---------------------------> [ ]
105	* \| \| VF driver applies post [ ]	112	* \| \| VF driver applies post [ ]
106	* \| \| migration fixups -------[ ]	113	* \| \| migration fixups -------[ ]
107	* \| \| \| [ ]	114	* \| \| \| [ ]
108	* \| \| -----> [ ]	115	* \| \| -----> [ ]
109	* \| \| [ ]	116	* \| \| [ ]
110	* \| \| VF2GUC_NOTIFY_RESFIX_DONE [ ]	117	* \| \| VF2GUC_RESFIX_DONE [ ]
111	* \| [ ] <---------------------------[ ]	118	* \| [ ] <---------------------------[ ]
112	* \| [ ] [ ]	119	* \| [ ] [ ]
113	* \| [ ] GuC sets new VF state to [ ]	120	* \| [ ] GuC sets new VF state to [ ]
@@ -118,6 +125,55 @@ · 4 unmodified lines
118	* \| [ ]---------------------------> [ ]	125	* \| [ ]---------------------------> [ ]
119	* \| \| \|	126	* \| \| \|
120	* \| \| \|	127	* \| \| \|
		128	*
		129	* Handling of VF double migration flow is shown below::
		130	*
		131	* GuC1 VF
		132	* \| \|
		133	* \| [ ]<--- start fixups
		134	* \| VF2GUC_RESFIX_START(marker) [ ]
		135	* [ ] <-------------------------------------------[ ]
		136	* [ ] [ ]
		137	* [ ]---\ [ ]
		138	* [ ] store marker [ ]
		139	* [ ]<--/ [ ]
		140	* [ ] [ ]
		141	* [ ] success [ ]
		142	* [ ] ------------------------------------------> [ ]
		143	* \| [ ]
		144	* \| [ ]---\
		145	* \| [ ] do fixups
		146	* \| [ ]<--/
		147	* \| [ ]
		148	* -------------- VF paused / saved ----------------
		149	* :
		150	*
		151	* GuC2
		152	* \|
		153	* ----------------- VF restored ------------------
		154	* \|
		155	* [ ]
		156	* [ ]---\
		157	* [ ] reset marker
		158	* [ ]<--/
		159	* [ ]
		160	* ----------------- VF resumed ------------------
		161	* \| [ ]
		162	* \| [ ]
		163	* \| VF2GUC_RESFIX_DONE(marker) [ ]
		164	* [ ] <-------------------------------------------[ ]
		165	* [ ] [ ]
		166	* [ ]---\ [ ]
		167	* [ ] check marker [ ]
		168	* [ ] (mismatch) [ ]
		169	* [ ]<--/ [ ]
		170	* [ ] [ ]
		171	* [ ] RESPONSE_VF_MIGRATED [ ]
		172	* [ ] ------------------------------------------> [ ]
		173	* \| [ ]---\
		174	* \| [ ] reschedule fixups
		175	* \| [ ]<--/
		176	* \| \|
121	*/	177	*/
122		178
123	/**	179	/**
@@ -170,6 +226,26 @@ void xe_sriov_vf_init_early(struct xe_device *xe) · 46 unmodified lines
170	vf_migration_init_early(xe);	226	vf_migration_init_early(xe);
171	}	227	}
172		228
		229	static int vf_migration_init_late(struct xe_device *xe)
		230	{
		231	struct xe_gt *gt = xe_root_mmio_gt(xe);
		232	struct xe_uc_fw_version guc_version;
		233
		234	if (!xe_sriov_vf_migration_supported(xe))
		235	return 0;
		236
		237	xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version);
		238	if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 27, 0)) {
		239	xe_sriov_vf_migration_disable(xe,
		240	"requires GuC ABI >= 1.27.0, but only %u.%u.%u found",
		241	guc_version.major, guc_version.minor,
		242	guc_version.patch);
		243	return 0;
		244	}
		245
		246	return xe_sriov_vf_ccs_init(xe);
		247	}
		248
173	/**	249	/**
174	* xe_sriov_vf_init_late() - SR-IOV VF late initialization functions.	250	* xe_sriov_vf_init_late() - SR-IOV VF late initialization functions.
175	* @xe: the &xe_device to initialize	251	* @xe: the &xe_device to initialize
@@ -180,7 +256,7 @@ void xe_sriov_vf_init_early(struct xe_device *xe) · 4 unmodified lines
180	*/	256	*/
181	int xe_sriov_vf_init_late(struct xe_device *xe)	257	int xe_sriov_vf_init_late(struct xe_device *xe)
182	{	258	{
183	return xe_sriov_vf_ccs_init(xe);	259	return vf_migration_init_late(xe);
184	}	260	}
185		261
186	static int sa_info_vf_ccs(struct seq_file m, void data)	262	static int sa_info_vf_ccs(struct seq_file m, void data)

drivers/gpu/drm/xe/xe_sriov_vf_ccs.c+19−8

 	xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n",
 		      ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M);
-	sa_manager = xe_sa_bo_manager_init(tile, bb_pool_size, SZ_16);
+	sa_manager = __xe_sa_bo_manager_init(tile, bb_pool_size, SZ_4K, SZ_16,
+					     XE_SA_BO_MANAGER_FLAG_SHADOW);
 	if (IS_ERR(sa_manager)) {
 		xe_sriov_err(xe, "Suballocator init failed with error: %pe\n",
 	offset = 0;
 	xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP,
 		      bb_pool_size);
+	xe_map_memset(xe, &sa_manager->shadow->vmap, offset, MI_NOOP,
+		      bb_pool_size);
 	offset = bb_pool_size - sizeof(u32);
 	xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END);
+	xe_map_wr(xe, &sa_manager->shadow->vmap, offset, u32, MI_BATCH_BUFFER_END);
 	ctx->mem.ccs_bb_pool = sa_manager;
 	return err;
 }
+#define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET	(2 * sizeof(u32))
+void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx)
+{
+	u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool);
+	struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
+	struct xe_device *xe = gt_to_xe(ctx->mig_q->gt);
+	xe_device_wmb(xe);
+	xe_map_wr(xe, &lrc->bo->vmap, XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET, u32, addr);
+	xe_device_wmb(xe);
+}
 /**
  * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO.
  * @bo: the &buffer object to which batch buffer commands will be added.
 		if (!bb)
 			continue;
-		memset(bb->cs, MI_NOOP, bb->len * sizeof(u32));
+		xe_migrate_ccs_rw_copy_clear(bo, ctx_id);
-		xe_bb_free(bb, NULL);
-		bo->bb_ccs[ctx_id] = NULL;
 	}
 	return 0;
 }
 	if (!IS_VF_CCS_READY(xe))
 		return;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
 	for_each_ccs_rw_ctx(ctx_id) {
 		bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool;
 		if (!bb_pool)
 		drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool));
 		drm_puts(p, "\n");
 	}
-	xe_pm_runtime_put(xe);
 }

drivers/gpu/drm/xe/xe_sriov_vf_ccs.h+1−0

@@ -20,6 +20,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo); · 19 unmodified lines
20	int xe_sriov_vf_ccs_register_context(struct xe_device *xe);	20	int xe_sriov_vf_ccs_register_context(struct xe_device *xe);
21	void xe_sriov_vf_ccs_rebase(struct xe_device *xe);	21	void xe_sriov_vf_ccs_rebase(struct xe_device *xe);
22	void xe_sriov_vf_ccs_print(struct xe_device xe, struct drm_printer p);	22	void xe_sriov_vf_ccs_print(struct xe_device xe, struct drm_printer p);
		23	void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx);
23		24
24	static inline bool xe_sriov_vf_ccs_ready(struct xe_device *xe)	25	static inline bool xe_sriov_vf_ccs_ready(struct xe_device *xe)
25	{	26	{

drivers/gpu/drm/xe/xe_survivability_mode.c+180−93

@@ -16,11 +16,10 @@ · 15 unmodified lines
16	#include "xe_heci_gsc.h"	16	#include "xe_heci_gsc.h"
17	#include "xe_i2c.h"	17	#include "xe_i2c.h"
18	#include "xe_mmio.h"	18	#include "xe_mmio.h"
		19	#include "xe_nvm.h"
19	#include "xe_pcode_api.h"	20	#include "xe_pcode_api.h"
20	#include "xe_vsec.h"	21	#include "xe_vsec.h"
21		22
22	#define MAX_SCRATCH_MMIO 8
23
24	/**	23	/**
25	* DOC: Survivability Mode	24	* DOC: Survivability Mode
26	*	25	*
@@ -48,19 +47,43 @@ · 21 unmodified lines
48	*	47	*
49	* Refer :ref:`xe_configfs` for more details on how to use configfs	48	* Refer :ref:`xe_configfs` for more details on how to use configfs
50	*	49	*
51	* Survivability mode is indicated by the below admin-only readable sysfs which provides additional	50	* Survivability mode is indicated by the below admin-only readable sysfs entry. It
52	* debug information::	51	* provides information about the type of survivability mode (Boot/Runtime).
		52	*
		53	* .. code-block:: shell
		54	*
		55	* # cat /sys/bus/pci/devices/<device>/survivability_mode
		56	* Boot
		57	*
		58	*
		59	* Any additional debug information if present will be visible under the directory
		60	* ``survivability_info``::
		61	*
		62	* /sys/bus/pci/devices/<device>/survivability_info/
		63	* ├── aux_info0
		64	* ├── aux_info1
		65	* ├── aux_info2
		66	* ├── aux_info3
		67	* ├── aux_info4
		68	* ├── capability_info
		69	* ├── fdo_mode
		70	* ├── postcode_trace
		71	* └── postcode_trace_overflow
		72	*
		73	* This directory has the following attributes
		74	*
		75	* - ``capability_info`` : Indicates Boot status and support for additional information
53	*	76	*
54	* /sys/bus/pci/devices/<device>/survivability_mode	77	* - ``postcode_trace``, ``postcode_trace_overflow`` : Each postcode is a 8bit value and
		78	* represents a boot failure event. When a new failure event is logged by PCODE the
		79	* existing postcodes are shifted left. These entries provide a history of 8 postcodes.
55	*	80	*
56	* Capability Information:	81	* - ``aux_info<n>`` : Some failures have additional debug information
57	* Provides boot status	82	*
58	* Postcode Information:	83	* - ``fdo_mode`` : To allow recovery in scenarios where MEI itself fails, a new SPI Flash
59	* Provides information about the failure	84	* Descriptor Override (FDO) mode is added in v2 survivability breadcrumbs. This mode is enabled
60	* Overflow Information	85	* by PCODE and provides the ability to directly update the firmware via SPI Driver without
61	* Provides history of previous failures	86	* any dependency on MEI. Xe KMD initializes the nvm aux driver if FDO mode is enabled.
62	* Auxiliary Information
63	* Certain failures may have information in addition to postcode information
64	*	87	*
65	* Runtime Survivability	88	* Runtime Survivability
66	* =====================	89	* =====================
@@ -68,61 +91,77 @@ · 1 unmodified line
68	* Certain runtime firmware errors can cause the device to enter a wedged state	91	* Certain runtime firmware errors can cause the device to enter a wedged state
69	* (:ref:`xe-device-wedging`) requiring a firmware flash to restore normal operation.	92	* (:ref:`xe-device-wedging`) requiring a firmware flash to restore normal operation.
70	* Runtime Survivability Mode indicates that a firmware flash is necessary to recover the device and	93	* Runtime Survivability Mode indicates that a firmware flash is necessary to recover the device and
71	* is indicated by the presence of survivability mode sysfs::	94	* is indicated by the presence of survivability mode sysfs.
		95	* Survivability mode sysfs provides information about the type of survivability mode.
72	*	96	*
73	* /sys/bus/pci/devices/<device>/survivability_mode	97	* .. code-block:: shell
74	*	98	*
75	* Survivability mode sysfs provides information about the type of survivability mode.	99	* # cat /sys/bus/pci/devices/<device>/survivability_mode
		100	* Runtime
76	*	101	*
77	* When such errors occur, userspace is notified with the drm device wedged uevent and runtime	102	* When such errors occur, userspace is notified with the drm device wedged uevent and runtime
78	* survivability mode. User can then initiate a firmware flash using userspace tools like fwupd	103	* survivability mode. User can then initiate a firmware flash using userspace tools like fwupd
79	* to restore device to normal operation.	104	* to restore device to normal operation.
80	*/	105	*/
81		106
82	static u32 aux_history_offset(u32 reg_value)	107	static const char * const reg_map[] = {
		108	[CAPABILITY_INFO] = "Capability Info",
		109	[POSTCODE_TRACE] = "Postcode trace",
		110	[POSTCODE_TRACE_OVERFLOW] = "Postcode trace overflow",
		111	[AUX_INFO0] = "Auxiliary Info 0",
		112	[AUX_INFO1] = "Auxiliary Info 1",
		113	[AUX_INFO2] = "Auxiliary Info 2",
		114	[AUX_INFO3] = "Auxiliary Info 3",
		115	[AUX_INFO4] = "Auxiliary Info 4",
		116	};
		117
		118	#define FDO_INFO (MAX_SCRATCH_REG + 1)
		119
		120	struct xe_survivability_attribute {
		121	struct device_attribute attr;
		122	u8 index;
		123	};
		124
		125	static struct
		126	xe_survivability_attribute dev_attr_to_survivability_attr(struct device_attribute attr)
83	{	127	{
84	return REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, reg_value);	128	return container_of(attr, struct xe_survivability_attribute, attr);
85	}	129	}
86		130
87	static void set_survivability_info(struct xe_mmio mmio, struct xe_survivability_info info,	131	static void set_survivability_info(struct xe_mmio mmio, u32 info, int id)
88	int id, char *name)
89	{	132	{
90	strscpy(info[id].name, name, sizeof(info[id].name));	133	info[id] = xe_mmio_read32(mmio, PCODE_SCRATCH(id));
91	info[id].reg = PCODE_SCRATCH(id).raw;
92	info[id].value = xe_mmio_read32(mmio, PCODE_SCRATCH(id));
93	}	134	}
94		135
95	static void populate_survivability_info(struct xe_device *xe)	136	static void populate_survivability_info(struct xe_device *xe)
96	{	137	{
97	struct xe_survivability *survivability = &xe->survivability;	138	struct xe_survivability *survivability = &xe->survivability;
98	struct xe_survivability_info *info = survivability->info;	139	u32 *info = survivability->info;
99	struct xe_mmio *mmio;	140	struct xe_mmio *mmio;
100	u32 id = 0, reg_value;	141	u32 id = 0, reg_value;
101	char name[NAME_MAX];
102	int index;
103		142
104	mmio = xe_root_tile_mmio(xe);	143	mmio = xe_root_tile_mmio(xe);
105	set_survivability_info(mmio, info, id, "Capability Info");	144	set_survivability_info(mmio, info, CAPABILITY_INFO);
106	reg_value = info[id].value;	145	reg_value = info[CAPABILITY_INFO];
		146
		147	survivability->version = REG_FIELD_GET(BREADCRUMB_VERSION, reg_value);
		148	/* FDO mode is exposed only from version 2 */
		149	if (survivability->version >= 2)
		150	survivability->fdo_mode = REG_FIELD_GET(FDO_MODE, reg_value);
107		151
108	if (reg_value & HISTORY_TRACKING) {	152	if (reg_value & HISTORY_TRACKING) {
109	id++;	153	set_survivability_info(mmio, info, POSTCODE_TRACE);
110	set_survivability_info(mmio, info, id, "Postcode Info");
111		154
112	if (reg_value & OVERFLOW_SUPPORT) {	155	if (reg_value & OVERFLOW_SUPPORT)
113	id = REG_FIELD_GET(OVERFLOW_REG_OFFSET, reg_value);	156	set_survivability_info(mmio, info, POSTCODE_TRACE_OVERFLOW);
114	set_survivability_info(mmio, info, id, "Overflow Info");
115	}
116	}	157	}
117		158
		159	/* Traverse the linked list of aux info registers */
118	if (reg_value & AUXINFO_SUPPORT) {	160	if (reg_value & AUXINFO_SUPPORT) {
119	id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value);	161	for (id = REG_FIELD_GET(AUXINFO_REG_OFFSET, reg_value);
120		162	id >= AUX_INFO0 && id < MAX_SCRATCH_REG;
121	for (index = 0; id && reg_value; index++, reg_value = info[id].value,	163	id = REG_FIELD_GET(AUXINFO_HISTORY_OFFSET, info[id]))
122	id = aux_history_offset(reg_value)) {	164	set_survivability_info(mmio, info, id);
123	snprintf(name, NAME_MAX, "Auxiliary Info %d", index);
124	set_survivability_info(mmio, info, id, name);
125	}
126	}	165	}
127	}	166	}
128		167
@@ -130,15 +169,14 @@ static void log_survivability_info(struct pci_dev *pdev) · 1 unmodified line
130	{	169	{
131	struct xe_device *xe = pdev_to_xe_device(pdev);	170	struct xe_device *xe = pdev_to_xe_device(pdev);
132	struct xe_survivability *survivability = &xe->survivability;	171	struct xe_survivability *survivability = &xe->survivability;
133	struct xe_survivability_info *info = survivability->info;	172	u32 *info = survivability->info;
134	int id;	173	int id;
135		174
136	dev_info(&pdev->dev, "Survivability Boot Status : Critical Failure (%d)\n",	175	dev_info(&pdev->dev, "Survivability Boot Status : Critical Failure (%d)\n",
137	survivability->boot_status);	176	survivability->boot_status);
138	for (id = 0; id < MAX_SCRATCH_MMIO; id++) {	177	for (id = 0; id < MAX_SCRATCH_REG; id++) {
139	if (info[id].reg)	178	if (info[id])
140	dev_info(&pdev->dev, "%s: 0x%x - 0x%x\n", info[id].name,	179	dev_info(&pdev->dev, "%s: 0x%x\n", reg_map[id], info[id]);
141	info[id].reg, info[id].value);
142	}	180	}
143	}	181	}
144		182
@@ -156,43 +194,103 @@ static ssize_t survivability_mode_show(struct device *dev, · 11 unmodified lines
156	struct pci_dev *pdev = to_pci_dev(dev);	194	struct pci_dev *pdev = to_pci_dev(dev);
157	struct xe_device *xe = pdev_to_xe_device(pdev);	195	struct xe_device *xe = pdev_to_xe_device(pdev);
158	struct xe_survivability *survivability = &xe->survivability;	196	struct xe_survivability *survivability = &xe->survivability;
159	struct xe_survivability_info *info = survivability->info;
160	int index = 0, count = 0;
161		197
162	count += sysfs_emit_at(buff, count, "Survivability mode type: %s\n",	198	return sysfs_emit(buff, "%s\n", survivability->type ? "Runtime" : "Boot");
163	survivability->type ? "Runtime" : "Boot");	199	}
		200
		201	static DEVICE_ATTR_ADMIN_RO(survivability_mode);
164		202
165	if (!check_boot_failure(xe))	203	static ssize_t survivability_info_show(struct device *dev,
166	return count;	204	struct device_attribute attr, char buff)
		205	{
		206	struct xe_survivability_attribute *sa = dev_attr_to_survivability_attr(attr);
		207	struct pci_dev *pdev = to_pci_dev(dev);
		208	struct xe_device *xe = pdev_to_xe_device(pdev);
		209	struct xe_survivability *survivability = &xe->survivability;
		210	u32 *info = survivability->info;
167		211
168	for (index = 0; index < MAX_SCRATCH_MMIO; index++) {	212	if (sa->index == FDO_INFO)
169	if (info[index].reg)	213	return sysfs_emit(buff, "%s\n", str_enabled_disabled(survivability->fdo_mode));
170	count += sysfs_emit_at(buff, count, "%s: 0x%x - 0x%x\n", info[index].name,
171	info[index].reg, info[index].value);
172	}
173		214
174	return count;	215	return sysfs_emit(buff, "0x%x\n", info[sa->index]);
175	}	216	}
176		217
177	static DEVICE_ATTR_ADMIN_RO(survivability_mode);	218	#define SURVIVABILITY_ATTR_RO(name, _index) \
		219	struct xe_survivability_attribute attr_##name = { \
		220	.attr = __ATTR(name, 0400, survivability_info_show, NULL), \
		221	.index = _index, \
		222	}
		223
		224	static SURVIVABILITY_ATTR_RO(capability_info, CAPABILITY_INFO);
		225	static SURVIVABILITY_ATTR_RO(postcode_trace, POSTCODE_TRACE);
		226	static SURVIVABILITY_ATTR_RO(postcode_trace_overflow, POSTCODE_TRACE_OVERFLOW);
		227	static SURVIVABILITY_ATTR_RO(aux_info0, AUX_INFO0);
		228	static SURVIVABILITY_ATTR_RO(aux_info1, AUX_INFO1);
		229	static SURVIVABILITY_ATTR_RO(aux_info2, AUX_INFO2);
		230	static SURVIVABILITY_ATTR_RO(aux_info3, AUX_INFO3);
		231	static SURVIVABILITY_ATTR_RO(aux_info4, AUX_INFO4);
		232	static SURVIVABILITY_ATTR_RO(fdo_mode, FDO_INFO);
178		233
179	static void xe_survivability_mode_fini(void *arg)	234	static void xe_survivability_mode_fini(void *arg)
180	{	235	{
181	struct xe_device *xe = arg;	236	struct xe_device *xe = arg;
		237	struct xe_survivability *survivability = &xe->survivability;
182	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);	238	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
183	struct device *dev = &pdev->dev;	239	struct device *dev = &pdev->dev;
184		240
185	sysfs_remove_file(&dev->kobj, &dev_attr_survivability_mode.attr);	241	if (survivability->fdo_mode)
		242	xe_nvm_fini(xe);
		243
		244	device_remove_file(dev, &dev_attr_survivability_mode);
		245	}
		246
		247	static umode_t survivability_info_attrs_visible(struct kobject kobj, struct attribute attr,
		248	int idx)
		249	{
		250	struct xe_device *xe = kdev_to_xe_device(kobj_to_dev(kobj));
		251	struct xe_survivability *survivability = &xe->survivability;
		252	u32 *info = survivability->info;
		253
		254	/*
		255	* Last index in survivability_info_attrs is fdo mode and is applicable only in
		256	* version 2 of survivability mode
		257	*/
		258	if (idx == MAX_SCRATCH_REG && survivability->version >= 2)
		259	return 0400;
		260
		261	if (idx < MAX_SCRATCH_REG && info[idx])
		262	return 0400;
		263
		264	return 0;
186	}	265	}
187		266
		267	/* Attributes are ordered according to enum scratch_reg */
		268	static struct attribute *survivability_info_attrs[] = {
		269	&attr_capability_info.attr.attr,
		270	&attr_postcode_trace.attr.attr,
		271	&attr_postcode_trace_overflow.attr.attr,
		272	&attr_aux_info0.attr.attr,
		273	&attr_aux_info1.attr.attr,
		274	&attr_aux_info2.attr.attr,
		275	&attr_aux_info3.attr.attr,
		276	&attr_aux_info4.attr.attr,
		277	&attr_fdo_mode.attr.attr,
		278	NULL,
		279	};
		280
		281	static const struct attribute_group survivability_info_group = {
		282	.name = "survivability_info",
		283	.attrs = survivability_info_attrs,
		284	.is_visible = survivability_info_attrs_visible,
		285	};
		286
188	static int create_survivability_sysfs(struct pci_dev *pdev)	287	static int create_survivability_sysfs(struct pci_dev *pdev)
189	{	288	{
190	struct device *dev = &pdev->dev;	289	struct device *dev = &pdev->dev;
191	struct xe_device *xe = pdev_to_xe_device(pdev);	290	struct xe_device *xe = pdev_to_xe_device(pdev);
192	int ret;	291	int ret;
193		292
194	/* create survivability mode sysfs */	293	ret = device_create_file(dev, &dev_attr_survivability_mode);
195	ret = sysfs_create_file(&dev->kobj, &dev_attr_survivability_mode.attr);
196	if (ret) {	294	if (ret) {
197	dev_warn(dev, "Failed to create survivability sysfs files\n");	295	dev_warn(dev, "Failed to create survivability sysfs files\n");
198	return ret;	296	return ret;
@@ -203,6 +301,12 @@ static int create_survivability_sysfs(struct pci_dev *pdev) · 4 unmodified lines
203	if (ret)	301	if (ret)
204	return ret;	302	return ret;
205		303
		304	if (check_boot_failure(xe)) {
		305	ret = devm_device_add_group(dev, &survivability_info_group);
		306	if (ret)
		307	return ret;
		308	}
		309
206	return 0;	310	return 0;
207	}	311	}
208		312
@@ -220,12 +324,16 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev) · 11 unmodified lines
220	/* Make sure xe_heci_gsc_init() knows about survivability mode */	324	/* Make sure xe_heci_gsc_init() knows about survivability mode */
221	survivability->mode = true;	325	survivability->mode = true;
222		326
223	ret = xe_heci_gsc_init(xe);	327	xe_heci_gsc_init(xe);
224	if (ret)
225	goto err;
226		328
227	xe_vsec_init(xe);	329	xe_vsec_init(xe);
228		330
		331	if (survivability->fdo_mode) {
		332	ret = xe_nvm_init(xe);
		333	if (ret)
		334	goto err;
		335	}
		336
229	ret = xe_i2c_probe(xe);	337	ret = xe_i2c_probe(xe);
230	if (ret)	338	if (ret)
231	goto err;	339	goto err;
@@ -235,29 +343,11 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev) · 3 unmodified lines
235	return 0;	343	return 0;
236		344
237	err:	345	err:
		346	dev_err(dev, "Failed to enable Survivability Mode\n");
238	survivability->mode = false;	347	survivability->mode = false;
239	return ret;	348	return ret;
240	}	349	}
241		350
242	static int init_survivability_mode(struct xe_device *xe)
243	{
244	struct xe_survivability *survivability = &xe->survivability;
245	struct xe_survivability_info *info;
246
247	survivability->size = MAX_SCRATCH_MMIO;
248
249	info = devm_kcalloc(xe->drm.dev, survivability->size, sizeof(*info),
250	GFP_KERNEL);
251	if (!info)
252	return -ENOMEM;
253
254	survivability->info = info;
255
256	populate_survivability_info(xe);
257
258	return 0;
259	}
260
261	/**	351	/**
262	* xe_survivability_mode_is_boot_enabled- check if boot survivability mode is enabled	352	* xe_survivability_mode_is_boot_enabled- check if boot survivability mode is enabled
263	* @xe: xe device instance	353	* @xe: xe device instance
@@ -325,9 +415,7 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe) · 61 unmodified lines
325	return -EINVAL;	415	return -EINVAL;
326	}	416	}
327		417
328	ret = init_survivability_mode(xe);	418	populate_survivability_info(xe);
329	if (ret)
330	return ret;
331		419
332	ret = create_survivability_sysfs(pdev);	420	ret = create_survivability_sysfs(pdev);
333	if (ret)	421	if (ret)
@@ -356,17 +444,16 @@ int xe_survivability_mode_boot_enable(struct xe_device *xe) · 22 unmodified lines
356	{	444	{
357	struct xe_survivability *survivability = &xe->survivability;	445	struct xe_survivability *survivability = &xe->survivability;
358	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);	446	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
359	int ret;
360		447
361	if (!xe_survivability_mode_is_requested(xe))	448	if (!xe_survivability_mode_is_requested(xe))
362	return 0;	449	return 0;
363		450
364	ret = init_survivability_mode(xe);	451	populate_survivability_info(xe);
365	if (ret)
366	return ret;
367		452
368	/* Log breadcrumbs but do not enter survivability mode for Critical boot errors */	453	/*
369	if (survivability->boot_status == CRITICAL_FAILURE) {	454	* v2 supports survivability mode for critical errors
		455	*/
		456	if (survivability->version < 2 && survivability->boot_status == CRITICAL_FAILURE) {
370	log_survivability_info(pdev);	457	log_survivability_info(pdev);
371	return -ENXIO;	458	return -ENXIO;
372	}	459	}

drivers/gpu/drm/xe/xe_survivability_mode_types.h+20−8

 #include <linux/limits.h>
 #include <linux/types.h>
+enum scratch_reg {
+	CAPABILITY_INFO,
+	POSTCODE_TRACE,
+	POSTCODE_TRACE_OVERFLOW,
+	AUX_INFO0,
+	AUX_INFO1,
+	AUX_INFO2,
+	AUX_INFO3,
+	AUX_INFO4,
+	MAX_SCRATCH_REG,
+};
 enum xe_survivability_type {
 	XE_SURVIVABILITY_TYPE_BOOT,
 	XE_SURVIVABILITY_TYPE_RUNTIME,
 };
-struct xe_survivability_info {
-	char name[NAME_MAX];
-	u32 reg;
-	u32 value;
-};
 /**
  * struct xe_survivability: Contains survivability mode information
  */
 struct xe_survivability {
-	/** @info: struct that holds survivability info from scratch registers */
+	/** @info: survivability debug info */
-	struct xe_survivability_info *info;
+	u32 info[MAX_SCRATCH_REG];
 	/** @size: number of scratch registers */
 	u32 size;
 	/** @type: survivability type */
 	enum xe_survivability_type type;
+	/** @fdo_mode: indicates if FDO mode is enabled */
+	bool fdo_mode;
+	/** @version: breadcrumb version of survivability mode  */
+	u8 version;
 };
 #endif /* _XE_SURVIVABILITY_MODE_TYPES_H_ */

drivers/gpu/drm/xe/xe_svm.c+42−46

@@ -176,24 +176,13 @@ xe_svm_range_notifier_event_end(struct xe_vm vm, struct drm_gpusvm_range r, · 175 unmodified lines
176	mmu_range);	176	mmu_range);
177	}	177	}
178		178
179	static s64 xe_svm_stats_ktime_us_delta(ktime_t start)
180	{
181	return IS_ENABLED(CONFIG_DEBUG_FS) ?
182	ktime_us_delta(ktime_get(), start) : 0;
183	}
184
185	static void xe_svm_tlb_inval_us_stats_incr(struct xe_gt *gt, ktime_t start)	179	static void xe_svm_tlb_inval_us_stats_incr(struct xe_gt *gt, ktime_t start)
186	{	180	{
187	s64 us_delta = xe_svm_stats_ktime_us_delta(start);	181	s64 us_delta = xe_gt_stats_ktime_us_delta(start);
188		182
189	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_US, us_delta);	183	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_US, us_delta);
190	}	184	}
191		185
192	static ktime_t xe_svm_stats_ktime_get(void)
193	{
194	return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0;
195	}
196
197	static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,	186	static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,
198	struct drm_gpusvm_notifier *notifier,	187	struct drm_gpusvm_notifier *notifier,
199	const struct mmu_notifier_range *mmu_range)	188	const struct mmu_notifier_range *mmu_range)
@@ -202,7 +191,7 @@ static void xe_svm_invalidate(struct drm_gpusvm *gpusvm, · 2 unmodified lines
202	struct xe_device *xe = vm->xe;	191	struct xe_device *xe = vm->xe;
203	struct drm_gpusvm_range r, first;	192	struct drm_gpusvm_range r, first;
204	struct xe_tile *tile;	193	struct xe_tile *tile;
205	ktime_t start = xe_svm_stats_ktime_get();	194	ktime_t start = xe_gt_stats_ktime_get();
206	u64 adj_start = mmu_range->start, adj_end = mmu_range->end;	195	u64 adj_start = mmu_range->start, adj_end = mmu_range->end;
207	u8 tile_mask = 0, id;	196	u8 tile_mask = 0, id;
208	long err;	197	long err;
@@ -285,19 +274,21 @@ static int __xe_svm_garbage_collector(struct xe_vm *vm, · 76 unmodified lines
285	return 0;	274	return 0;
286	}	275	}
287		276
288	static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64 range_end)	277	static void xe_vma_set_default_attributes(struct xe_vma *vma)
		278	{
		279	vma->attr.preferred_loc.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE;
		280	vma->attr.preferred_loc.migration_policy = DRM_XE_MIGRATE_ALL_PAGES;
		281	vma->attr.pat_index = vma->attr.default_pat_index;
		282	vma->attr.atomic_access = DRM_XE_ATOMIC_UNDEFINED;
		283	}
		284
		285	static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end)
289	{	286	{
290	struct xe_vma *vma;	287	struct xe_vma *vma;
291	struct xe_vma_mem_attr default_attr = {	288	bool has_default_attr;
292	.preferred_loc = {	289	int err;
293	.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE,
294	.migration_policy = DRM_XE_MIGRATE_ALL_PAGES,
295	},
296	.atomic_access = DRM_XE_ATOMIC_UNDEFINED,
297	};
298	int err = 0;
299		290
300	vma = xe_vm_find_vma_by_addr(vm, range_start);	291	vma = xe_vm_find_vma_by_addr(vm, start);
301	if (!vma)	292	if (!vma)
302	return -EINVAL;	293	return -EINVAL;
303		294
@@ -306,25 +297,30 @@ static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 range_start, u64 · 2 unmodified lines
306	return 0;	297	return 0;
307	}	298	}
308		299
309	if (xe_vma_has_default_mem_attrs(vma))
310	return 0;
311
312	vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx",	300	vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx",
313	xe_vma_start(vma), xe_vma_end(vma));	301	xe_vma_start(vma), xe_vma_end(vma));
314		302
315	if (xe_vma_start(vma) == range_start && xe_vma_end(vma) == range_end) {	303	has_default_attr = xe_vma_has_default_mem_attrs(vma);
316	default_attr.pat_index = vma->attr.default_pat_index;	304
317	default_attr.default_pat_index = vma->attr.default_pat_index;	305	if (has_default_attr) {
318	vma->attr = default_attr;	306	start = xe_vma_start(vma);
319	} else {	307	end = xe_vma_end(vma);
320	vm_dbg(&vm->xe->drm, "Split VMA start=0x%016llx, vma_end=0x%016llx",	308	} else if (xe_vma_start(vma) == start && xe_vma_end(vma) == end) {
321	range_start, range_end);	309	xe_vma_set_default_attributes(vma);
322	err = xe_vm_alloc_cpu_addr_mirror_vma(vm, range_start, range_end - range_start);	310	}
323	if (err) {	311
324	drm_warn(&vm->xe->drm, "VMA SPLIT failed: %pe\n", ERR_PTR(err));	312	xe_vm_find_cpu_addr_mirror_vma_range(vm, &start, &end);
325	xe_vm_kill(vm, true);	313
326	return err;	314	if (xe_vma_start(vma) == start && xe_vma_end(vma) == end && has_default_attr)
327	}	315	return 0;
		316
		317	vm_dbg(&vm->xe->drm, "New VMA start=0x%016llx, vma_end=0x%016llx", start, end);
		318
		319	err = xe_vm_alloc_cpu_addr_mirror_vma(vm, start, end - start);
		320	if (err) {
		321	drm_warn(&vm->xe->drm, "New VMA MAP failed: %pe\n", ERR_PTR(err));
		322	xe_vm_kill(vm, true);
		323	return err;
328	}	324	}
329		325
330	/*	326	/*
@@ -435,7 +431,7 @@ static void xe_svm_copy_us_stats_incr(struct xe_gt *gt, · 104 unmodified lines
435	unsigned long npages,	431	unsigned long npages,
436	ktime_t start)	432	ktime_t start)
437	{	433	{
438	s64 us_delta = xe_svm_stats_ktime_us_delta(start);	434	s64 us_delta = xe_gt_stats_ktime_us_delta(start);
439		435
440	if (dir == XE_SVM_COPY_TO_VRAM) {	436	if (dir == XE_SVM_COPY_TO_VRAM) {
441	switch (npages) {	437	switch (npages) {
@@ -487,7 +483,7 @@ static int xe_svm_copy(struct page **pages, · 45 unmodified lines
487	u64 vram_addr = XE_VRAM_ADDR_INVALID;	483	u64 vram_addr = XE_VRAM_ADDR_INVALID;
488	int err = 0, pos = 0;	484	int err = 0, pos = 0;
489	bool sram = dir == XE_SVM_COPY_TO_SRAM;	485	bool sram = dir == XE_SVM_COPY_TO_SRAM;
490	ktime_t start = xe_svm_stats_ktime_get();	486	ktime_t start = xe_gt_stats_ktime_get();
491		487
492	/*	488	/*
493	* This flow is complex: it locates physically contiguous device pages,	489	* This flow is complex: it locates physically contiguous device pages,
@@ -979,7 +975,7 @@ static void xe_svm_range_##elem##_us_stats_incr(struct xe_gt *gt, \ · 485 unmodified lines
979	struct xe_svm_range *range, \	975	struct xe_svm_range *range, \
980	ktime_t start) \	976	ktime_t start) \
981	{ \	977	{ \
982	s64 us_delta = xe_svm_stats_ktime_us_delta(start); \	978	s64 us_delta = xe_gt_stats_ktime_us_delta(start); \
983	\	979	\
984	switch (xe_svm_range_size(range)) { \	980	switch (xe_svm_range_size(range)) { \
985	case SZ_4K: \	981	case SZ_4K: \
@@ -1024,7 +1020,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm vm, struct xe_vma vma, · 38 unmodified lines
1024	struct drm_pagemap *dpagemap;	1020	struct drm_pagemap *dpagemap;
1025	struct xe_tile *tile = gt_to_tile(gt);	1021	struct xe_tile *tile = gt_to_tile(gt);
1026	int migrate_try_count = ctx.devmem_only ? 3 : 1;	1022	int migrate_try_count = ctx.devmem_only ? 3 : 1;
1027	ktime_t start = xe_svm_stats_ktime_get(), bind_start, get_pages_start;	1023	ktime_t start = xe_gt_stats_ktime_get(), bind_start, get_pages_start;
1028	int err;	1024	int err;
1029		1025
1030	lockdep_assert_held_write(&vm->lock);	1026	lockdep_assert_held_write(&vm->lock);
@@ -1063,7 +1059,7 @@ retry: · 32 unmodified lines
1063		1059
1064	if (--migrate_try_count >= 0 &&	1060	if (--migrate_try_count >= 0 &&
1065	xe_svm_range_needs_migrate_to_vram(range, vma, !!dpagemap \|\| ctx.devmem_only)) {	1061	xe_svm_range_needs_migrate_to_vram(range, vma, !!dpagemap \|\| ctx.devmem_only)) {
1066	ktime_t migrate_start = xe_svm_stats_ktime_get();	1062	ktime_t migrate_start = xe_gt_stats_ktime_get();
1067		1063
1068	/* TODO : For multi-device dpagemap will be used to find the	1064	/* TODO : For multi-device dpagemap will be used to find the
1069	* remote tile and remote device. Will need to modify	1065	* remote tile and remote device. Will need to modify
@@ -1100,7 +1096,7 @@ retry: · 30 unmodified lines
1100	}	1096	}
1101		1097
1102	get_pages:	1098	get_pages:
1103	get_pages_start = xe_svm_stats_ktime_get();	1099	get_pages_start = xe_gt_stats_ktime_get();
1104		1100
1105	range_debug(range, "GET PAGES");	1101	range_debug(range, "GET PAGES");
1106	err = xe_svm_range_get_pages(vm, range, &ctx);	1102	err = xe_svm_range_get_pages(vm, range, &ctx);
@@ -1127,7 +1123,7 @@ get_pages: · 20 unmodified lines
1127	xe_svm_range_get_pages_us_stats_incr(gt, range, get_pages_start);	1123	xe_svm_range_get_pages_us_stats_incr(gt, range, get_pages_start);
1128	range_debug(range, "PAGE FAULT - BIND");	1124	range_debug(range, "PAGE FAULT - BIND");
1129		1125
1130	bind_start = xe_svm_stats_ktime_get();	1126	bind_start = xe_gt_stats_ktime_get();
1131	xe_validation_guard(&vctx, &vm->xe->val, &exec, (struct xe_val_flags) {}, err) {	1127	xe_validation_guard(&vctx, &vm->xe->val, &exec, (struct xe_val_flags) {}, err) {
1132	err = xe_vm_drm_exec_lock(vm, &exec);	1128	err = xe_vm_drm_exec_lock(vm, &exec);
1133	drm_exec_retry_on_contention(&exec);	1129	drm_exec_retry_on_contention(&exec);

drivers/gpu/drm/xe/xe_sync.c+31−2

@@ -228,6 +228,32 @@ int xe_sync_entry_add_deps(struct xe_sync_entry sync, struct xe_sched_job job) · 227 unmodified lines
228	return 0;	228	return 0;
229	}	229	}
230		230
		231	/**
		232	* xe_sync_entry_wait() - Wait on in-sync
		233	* @sync: Sync object
		234	*
		235	* If the sync is in an in-sync, wait on the sync to signal.
		236	*
		237	* Return: 0 on success, -ERESTARTSYS on failure (interruption)
		238	*/
		239	int xe_sync_entry_wait(struct xe_sync_entry *sync)
		240	{
		241	return xe_sync_needs_wait(sync) ?
		242	dma_fence_wait(sync->fence, true) : 0;
		243	}
		244
		245	/**
		246	* xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled)
		247	* @sync: Sync object
		248	*
		249	* Return: True if sync needs a wait, False otherwise
		250	*/
		251	bool xe_sync_needs_wait(struct xe_sync_entry *sync)
		252	{
		253	return sync->fence &&
		254	!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags);
		255	}
		256
231	void xe_sync_entry_signal(struct xe_sync_entry sync, struct dma_fence fence)	257	void xe_sync_entry_signal(struct xe_sync_entry sync, struct dma_fence fence)
232	{	258	{
233	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))	259	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
@@ -311,8 +337,11 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync, · 77 unmodified lines
311	struct xe_tile *tile;	337	struct xe_tile *tile;
312	u8 id;	338	u8 id;
313		339
314	for_each_tile(tile, vm->xe, id)	340	for_each_tile(tile, vm->xe, id) {
315	num_fence += (1 + XE_MAX_GT_PER_TILE);	341	num_fence++;
		342	for_each_tlb_inval(i)
		343	num_fence++;
		344	}
316		345
317	fences = kmalloc_array(num_fence, sizeof(*fences),	346	fences = kmalloc_array(num_fence, sizeof(*fences),
318	GFP_KERNEL);	347	GFP_KERNEL);

drivers/gpu/drm/xe/xe_sync.h+2−0

 			   struct xe_sched_job *job);
 void xe_sync_entry_signal(struct xe_sync_entry *sync,
 			  struct dma_fence *fence);
+int xe_sync_entry_wait(struct xe_sync_entry *sync);
+bool xe_sync_needs_wait(struct xe_sync_entry *sync);
 void xe_sync_entry_cleanup(struct xe_sync_entry *sync);
 struct dma_fence *
 xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,

drivers/gpu/drm/xe/xe_tile.c+5−0

 	if (IS_ERR(tile->mem.kernel_bb_pool))
 		return PTR_ERR(tile->mem.kernel_bb_pool);
+	/* Optimistically anticipate at most 256 TLB fences with PRL */
+	tile->mem.reclaim_pool = xe_sa_bo_manager_init(tile, SZ_1M, XE_PAGE_RECLAIM_LIST_MAX_SIZE);
+	if (IS_ERR(tile->mem.reclaim_pool))
+		return PTR_ERR(tile->mem.reclaim_pool);
 	return 0;
 }
 void xe_tile_migrate_wait(struct xe_tile *tile)

drivers/gpu/drm/xe/xe_tile_debugfs.c+11−6

 	struct drm_info_node *node = m->private;
 	struct xe_tile *tile = node_to_tile(node);
 	struct xe_device *xe = tile_to_xe(tile);
-	int ret;
-	xe_pm_runtime_get(xe);
+	guard(xe_pm_runtime)(xe);
-	ret = xe_tile_debugfs_simple_show(m, data);
+	return xe_tile_debugfs_simple_show(m, data);
-	xe_pm_runtime_put(xe);
-	return ret;
 }
 static int ggtt(struct xe_tile *tile, struct drm_printer *p)
 	{ "sa_info", .show = xe_tile_debugfs_show_with_rpm, .data = sa_info },
 };
+static void tile_debugfs_create_vram_mm(struct xe_tile *tile)
+{
+	if (tile->mem.vram)
+		ttm_resource_manager_create_debugfs(&tile->mem.vram->ttm.manager, tile->debugfs,
+						    "vram_mm");
+}
 /**
  * xe_tile_debugfs_register - Register tile's debugfs attributes
  * @tile: the &xe_tile to register
 	drm_debugfs_create_files(vf_safe_debugfs_list,
 				 ARRAY_SIZE(vf_safe_debugfs_list),
 				 tile->debugfs, minor);
+	tile_debugfs_create_vram_mm(tile);
 }

drivers/gpu/drm/xe/xe_tile_sriov_pf_debugfs.c+1−2

 	if (val > (TYPE)~0ull)							\
 		return -EOVERFLOW;						\
 										\
-	xe_pm_runtime_get(xe);							\
+	guard(xe_pm_runtime)(xe);						\
 	err = xe_sriov_pf_wait_ready(xe) ?:					\
 	      xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val);		\
 	if (!err)								\
 		xe_sriov_pf_provision_set_custom_mode(xe);			\
-	xe_pm_runtime_put(xe);							\
 										\
 	return err;								\
 }										\

drivers/gpu/drm/xe/xe_tlb_inval.c+24−3

 	mutex_unlock(&tlb_inval->seqno_lock);
 }
+/**
+ * xe_tlb_inval_reset_timeout() - Reset TLB inval fence timeout
+ * @tlb_inval: TLB invalidation client
+ *
+ * Reset the TLB invalidation timeout timer.
+ */
+static void xe_tlb_inval_reset_timeout(struct xe_tlb_inval *tlb_inval)
+{
+	lockdep_assert_held(&tlb_inval->pending_lock);
+	mod_delayed_work(system_wq, &tlb_inval->fence_tdr,
+			 tlb_inval->ops->timeout_delay(tlb_inval));
+}
 static bool xe_tlb_inval_seqno_past(struct xe_tlb_inval *tlb_inval, int seqno)
 {
 	int seqno_recv = READ_ONCE(tlb_inval->seqno_recv);
  * @start: start address
  * @end: end address
  * @asid: address space id
+ * @prl_sa: suballocation of page reclaim list if used, NULL indicates PPC flush
  *
  * Issue a range based TLB invalidation if supported, if not fallback to a full
  * TLB invalidation. Completion of TLB is asynchronous and caller can use
  */
 int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
 		       struct xe_tlb_inval_fence *fence, u64 start, u64 end,
-		       u32 asid)
+		       u32 asid, struct drm_suballoc *prl_sa)
 {
 	return xe_tlb_inval_issue(tlb_inval, fence, tlb_inval->ops->ppgtt,
-				  start, end, asid);
+				  start, end, asid, prl_sa);
 }
 /**
 	u64 range = 1ull << vm->xe->info.va_bits;
 	xe_tlb_inval_fence_init(tlb_inval, &fence, true);
-	xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid);
+	xe_tlb_inval_range(tlb_inval, &fence, 0, range, vm->usm.asid, NULL);
 	xe_tlb_inval_fence_wait(&fence);
 }
 	 * process_g2h_msg().
 	 */
 	spin_lock_irqsave(&tlb_inval->pending_lock, flags);
+	if (seqno == TLB_INVALIDATION_SEQNO_INVALID) {
+		xe_tlb_inval_reset_timeout(tlb_inval);
+		spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
+		return;
+	}
 	if (xe_tlb_inval_seqno_past(tlb_inval, seqno)) {
 		spin_unlock_irqrestore(&tlb_inval->pending_lock, flags);
 		return;

drivers/gpu/drm/xe/xe_tlb_inval.h+1−1

@@ -23,7 +23,7 @@ int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval); · 22 unmodified lines
23	void xe_tlb_inval_vm(struct xe_tlb_inval tlb_inval, struct xe_vm vm);	23	void xe_tlb_inval_vm(struct xe_tlb_inval tlb_inval, struct xe_vm vm);
24	int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,	24	int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
25	struct xe_tlb_inval_fence *fence,	25	struct xe_tlb_inval_fence *fence,
26	u64 start, u64 end, u32 asid);	26	u64 start, u64 end, u32 asid, struct drm_suballoc *prl_sa);
27		27
28	void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,	28	void xe_tlb_inval_fence_init(struct xe_tlb_inval *tlb_inval,
29	struct xe_tlb_inval_fence *fence,	29	struct xe_tlb_inval_fence *fence,

drivers/gpu/drm/xe/xe_tlb_inval_job.c+35−1

 #include "xe_dep_job_types.h"
 #include "xe_dep_scheduler.h"
 #include "xe_exec_queue.h"
+#include "xe_gt_printk.h"
 #include "xe_gt_types.h"
+#include "xe_page_reclaim.h"
 #include "xe_tlb_inval.h"
 #include "xe_tlb_inval_job.h"
 #include "xe_migrate.h"
 	struct xe_exec_queue *q;
 	/** @vm: VM which TLB invalidation is being issued for */
 	struct xe_vm *vm;
+	/** @prl: Embedded copy of page reclaim list */
+	struct xe_page_reclaim_list prl;
 	/** @refcount: ref count of this job */
 	struct kref refcount;
 	/**
 		container_of(dep_job, typeof(*job), dep);
 	struct xe_tlb_inval_fence *ifence =
 		container_of(job->fence, typeof(*ifence), base);
+	struct drm_suballoc *prl_sa = NULL;
+	if (xe_page_reclaim_list_valid(&job->prl)) {
+		prl_sa = xe_page_reclaim_create_prl_bo(job->tlb_inval, &job->prl, ifence);
+		if (IS_ERR(prl_sa))
+			prl_sa = NULL; /* Indicate fall back PPC flush with NULL */
+	}
 	xe_tlb_inval_range(job->tlb_inval, ifence, job->start,
-			   job->end, job->vm->usm.asid);
+			   job->end, job->vm->usm.asid, prl_sa);
 	return job->fence;
 }
 	job->start = start;
 	job->end = end;
 	job->fence_armed = false;
+	xe_page_reclaim_list_init(&job->prl);
 	job->dep.ops = &dep_job_ops;
 	job->type = type;
 	kref_init(&job->refcount);
 	return ERR_PTR(err);
 }
+/**
+ * xe_tlb_inval_job_add_page_reclaim() - Embed PRL into a TLB job
+ * @job: TLB invalidation job that may trigger reclamation
+ * @prl: Page reclaim list populated during unbind
+ *
+ * Copies @prl into the job and takes an extra reference to the entry page so
+ * ownership can transfer to the TLB fence when the job is pushed.
+ */
+void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
+				       struct xe_page_reclaim_list *prl)
+{
+	struct xe_device *xe = gt_to_xe(job->q->gt);
+	xe_gt_WARN_ON(job->q->gt, !xe->info.has_page_reclaim_hw_assist);
+	job->prl = *prl;
+	/* Pair with put in job_destroy */
+	xe_page_reclaim_entries_get(job->prl.entries);
+}
 static void xe_tlb_inval_job_destroy(struct kref *ref)
 {
 	struct xe_tlb_inval_job *job = container_of(ref, typeof(*job),
 	struct xe_device *xe = gt_to_xe(q->gt);
 	struct xe_vm *vm = job->vm;
+	/* BO creation retains a copy (if used), so no longer needed */
+	xe_page_reclaim_entries_put(job->prl.entries);
 	if (!job->fence_armed)
 		kfree(ifence);
 	else

drivers/gpu/drm/xe/xe_tlb_inval_job.h+4−0

 struct xe_dep_scheduler;
 struct xe_exec_queue;
 struct xe_migrate;
+struct xe_page_reclaim_list;
 struct xe_tlb_inval;
 struct xe_tlb_inval_job;
 struct xe_vm;
 			struct xe_dep_scheduler *dep_scheduler,
 			struct xe_vm *vm, u64 start, u64 end, int type);
+void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
+				       struct xe_page_reclaim_list *prl);
 int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job);
 struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,

drivers/gpu/drm/xe/xe_tlb_inval_types.h+4−1

 #include <linux/workqueue.h>
 #include <linux/dma-fence.h>
+struct drm_suballoc;
 struct xe_tlb_inval;
 /** struct xe_tlb_inval_ops - TLB invalidation ops (backend) */
 	 * @start: Start address
 	 * @end: End address
 	 * @asid: Address space ID
+	 * @prl_sa: Suballocation for page reclaim list
 	 *
 	 * Return 0 on success, -ECANCELED if backend is mid-reset, error on
 	 * failure
 	 */
 	int (*ppgtt)(struct xe_tlb_inval *tlb_inval, u32 seqno, u64 start,
-		     u64 end, u32 asid);
+		     u64 end, u32 asid, struct drm_suballoc *prl_sa);
 	/**
 	 * @initialized: Backend is initialized
 	const struct xe_tlb_inval_ops *ops;
 	/** @tlb_inval.seqno: TLB invalidation seqno, protected by CT lock */
 #define TLB_INVALIDATION_SEQNO_MAX	0x100000
+#define TLB_INVALIDATION_SEQNO_INVALID	TLB_INVALIDATION_SEQNO_MAX
 	int seqno;
 	/** @tlb_invalidation.seqno_lock: protects @tlb_invalidation.seqno */
 	struct mutex seqno_lock;

drivers/gpu/drm/xe/xe_trace.h+46−0

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	#include <linux/types.h>	13	#include <linux/types.h>
14		14
15	#include "xe_exec_queue_types.h"	15	#include "xe_exec_queue_types.h"
		16	#include "xe_exec_queue.h"
16	#include "xe_gpu_scheduler_types.h"	17	#include "xe_gpu_scheduler_types.h"
17	#include "xe_gt_types.h"	18	#include "xe_gt_types.h"
18	#include "xe_guc_exec_queue_types.h"	19	#include "xe_guc_exec_queue_types.h"
@@ -97,11 +98,51 @@ DECLARE_EVENT_CLASS(xe_exec_queue, · 78 unmodified lines
97	__entry->guc_state, __entry->flags)	98	__entry->guc_state, __entry->flags)
98	);	99	);
99		100
		101	DECLARE_EVENT_CLASS(xe_exec_queue_multi_queue,
		102	TP_PROTO(struct xe_exec_queue *q),
		103	TP_ARGS(q),
		104
		105	TP_STRUCT__entry(
		106	__string(dev, __dev_name_eq(q))
		107	__field(enum xe_engine_class, class)
		108	__field(u32, logical_mask)
		109	__field(u8, gt_id)
		110	__field(u16, width)
		111	__field(u32, guc_id)
		112	__field(u32, guc_state)
		113	__field(u32, flags)
		114	__field(u32, primary)
		115	),
		116
		117	TP_fast_assign(
		118	__assign_str(dev);
		119	__entry->class = q->class;
		120	__entry->logical_mask = q->logical_mask;
		121	__entry->gt_id = q->gt->info.id;
		122	__entry->width = q->width;
		123	__entry->guc_id = q->guc->id;
		124	__entry->guc_state = atomic_read(&q->guc->state);
		125	__entry->flags = q->flags;
		126	__entry->primary = xe_exec_queue_multi_queue_primary(q)->guc->id;
		127	),
		128
		129	TP_printk("dev=%s, %d:0x%x, gt=%d, width=%d guc_id=%d, guc_state=0x%x, flags=0x%x, primary=%d",
		130	__get_str(dev), __entry->class, __entry->logical_mask,
		131	__entry->gt_id, __entry->width, __entry->guc_id,
		132	__entry->guc_state, __entry->flags,
		133	__entry->primary)
		134	);
		135
100	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_create,	136	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_create,
101	TP_PROTO(struct xe_exec_queue *q),	137	TP_PROTO(struct xe_exec_queue *q),
102	TP_ARGS(q)	138	TP_ARGS(q)
103	);	139	);
104		140
		141	DEFINE_EVENT(xe_exec_queue_multi_queue, xe_exec_queue_create_multi_queue,
		142	TP_PROTO(struct xe_exec_queue *q),
		143	TP_ARGS(q)
		144	);
		145
105	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_supress_resume,	146	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_supress_resume,
106	TP_PROTO(struct xe_exec_queue *q),	147	TP_PROTO(struct xe_exec_queue *q),
107	TP_ARGS(q)	148	TP_ARGS(q)
@@ -172,6 +213,11 @@ DEFINE_EVENT(xe_exec_queue, xe_exec_queue_memory_cat_error, · 64 unmodified lines
172	TP_ARGS(q)	213	TP_ARGS(q)
173	);	214	);
174		215
		216	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_cgp_context_error,
		217	TP_PROTO(struct xe_exec_queue *q),
		218	TP_ARGS(q)
		219	);
		220
175	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_stop,	221	DEFINE_EVENT(xe_exec_queue, xe_exec_queue_stop,
176	TP_PROTO(struct xe_exec_queue *q),	222	TP_PROTO(struct xe_exec_queue *q),
177	TP_ARGS(q)	223	TP_ARGS(q)

drivers/gpu/drm/xe/xe_uc.c+33−2

@@ -218,9 +218,12 @@ int xe_uc_load_hw(struct xe_uc *uc) · 217 unmodified lines
218		218
219	xe_guc_engine_activity_enable_stats(&uc->guc);	219	xe_guc_engine_activity_enable_stats(&uc->guc);
220		220
221	/* We don't fail the driver load if HuC fails to auth, but let's warn */	221	/* We don't fail the driver load if HuC fails to auth */
222	ret = xe_huc_auth(&uc->huc, XE_HUC_AUTH_VIA_GUC);	222	ret = xe_huc_auth(&uc->huc, XE_HUC_AUTH_VIA_GUC);
223	xe_gt_assert(uc_to_gt(uc), !ret);	223	if (ret)
		224	xe_gt_err(uc_to_gt(uc),
		225	"HuC authentication failed (%pe), continuing with no HuC\n",
		226	ERR_PTR(ret));
224		227
225	/* GSC load is async */	228	/* GSC load is async */
226	xe_gsc_load_start(&uc->gsc);	229	xe_gsc_load_start(&uc->gsc);
@@ -301,6 +304,34 @@ int xe_uc_suspend(struct xe_uc *uc) · 74 unmodified lines
301	return xe_guc_suspend(&uc->guc);	304	return xe_guc_suspend(&uc->guc);
302	}	305	}
303		306
		307	/**
		308	* xe_uc_runtime_suspend() - UC runtime suspend
		309	* @uc: the UC object
		310	*
		311	* Runtime suspend all UCs.
		312	*/
		313	void xe_uc_runtime_suspend(struct xe_uc *uc)
		314	{
		315	if (!xe_device_uc_enabled(uc_to_xe(uc)))
		316	return;
		317
		318	xe_guc_runtime_suspend(&uc->guc);
		319	}
		320
		321	/**
		322	* xe_uc_runtime_resume() - UC runtime resume
		323	* @uc: the UC object
		324	*
		325	* Runtime resume all UCs.
		326	*/
		327	void xe_uc_runtime_resume(struct xe_uc *uc)
		328	{
		329	if (!xe_device_uc_enabled(uc_to_xe(uc)))
		330	return;
		331
		332	xe_guc_runtime_resume(&uc->guc);
		333	}
		334
304	/**	335	/**
305	* xe_uc_declare_wedged() - Declare UC wedged	336	* xe_uc_declare_wedged() - Declare UC wedged
306	* @uc: the UC object	337	* @uc: the UC object

drivers/gpu/drm/xe/xe_uc.h+2−0

 int xe_uc_load_hw(struct xe_uc *uc);
 void xe_uc_gucrc_disable(struct xe_uc *uc);
 int xe_uc_reset_prepare(struct xe_uc *uc);
+void xe_uc_runtime_resume(struct xe_uc *uc);
+void xe_uc_runtime_suspend(struct xe_uc *uc);
 void xe_uc_stop_prepare(struct xe_uc *uc);
 void xe_uc_stop(struct xe_uc *uc);
 int xe_uc_start(struct xe_uc *uc);

drivers/gpu/drm/xe/xe_uc_fw.c+5−5

@@ -115,11 +115,11 @@ struct fw_blobs_by_type { · 114 unmodified lines
115	#define XE_GT_TYPE_ANY XE_GT_TYPE_UNINITIALIZED	115	#define XE_GT_TYPE_ANY XE_GT_TYPE_UNINITIALIZED
116		116
117	#define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver) \	117	#define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver) \
118	fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 49, 4)) \	118	fw_def(PANTHERLAKE, GT_TYPE_ANY, major_ver(xe, guc, ptl, 70, 54, 0)) \
119	fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 49, 4)) \	119	fw_def(BATTLEMAGE, GT_TYPE_ANY, major_ver(xe, guc, bmg, 70, 54, 0)) \
120	fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 45, 2)) \	120	fw_def(LUNARLAKE, GT_TYPE_ANY, major_ver(xe, guc, lnl, 70, 53, 0)) \
121	fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, guc, mtl, 70, 44, 1)) \	121	fw_def(METEORLAKE, GT_TYPE_ANY, major_ver(i915, guc, mtl, 70, 53, 0)) \
122	fw_def(DG2, GT_TYPE_ANY, major_ver(i915, guc, dg2, 70, 45, 2)) \	122	fw_def(DG2, GT_TYPE_ANY, major_ver(i915, guc, dg2, 70, 53, 0)) \
123	fw_def(DG1, GT_TYPE_ANY, major_ver(i915, guc, dg1, 70, 44, 1)) \	123	fw_def(DG1, GT_TYPE_ANY, major_ver(i915, guc, dg1, 70, 44, 1)) \
124	fw_def(ALDERLAKE_N, GT_TYPE_ANY, major_ver(i915, guc, tgl, 70, 44, 1)) \	124	fw_def(ALDERLAKE_N, GT_TYPE_ANY, major_ver(i915, guc, tgl, 70, 44, 1)) \
125	fw_def(ALDERLAKE_P, GT_TYPE_ANY, major_ver(i915, guc, adlp, 70, 44, 1)) \	125	fw_def(ALDERLAKE_P, GT_TYPE_ANY, major_ver(i915, guc, adlp, 70, 44, 1)) \

drivers/gpu/drm/xe/xe_vm.c+136−18

@@ -1509,9 +1509,9 @@ struct xe_vm xe_vm_create(struct xe_device xe, u32 flags, struct xe_file *xef) · 1508 unmodified lines
1509		1509
1510	INIT_LIST_HEAD(&vm->preempt.exec_queues);	1510	INIT_LIST_HEAD(&vm->preempt.exec_queues);
1511	if (flags & XE_VM_FLAG_FAULT_MODE)	1511	if (flags & XE_VM_FLAG_FAULT_MODE)
1512	vm->preempt.min_run_period_ms = 0;	1512	vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms;
1513	else	1513	else
1514	vm->preempt.min_run_period_ms = 5;	1514	vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms;
1515		1515
1516	for_each_tile(tile, xe, id)	1516	for_each_tile(tile, xe, id)
1517	xe_range_fence_tree_init(&vm->rftree[id]);	1517	xe_range_fence_tree_init(&vm->rftree[id]);
@@ -2236,6 +2236,7 @@ vm_bind_ioctl_ops_create(struct xe_vm vm, struct xe_vma_ops vops, · 718 unmodified lines
2236	struct drm_gpuva_ops *ops;	2236	struct drm_gpuva_ops *ops;
2237	struct drm_gpuva_op *__op;	2237	struct drm_gpuva_op *__op;
2238	struct drm_gpuvm_bo *vm_bo;	2238	struct drm_gpuvm_bo *vm_bo;
		2239	u64 range_start = addr;
2239	u64 range_end = addr + range;	2240	u64 range_end = addr + range;
2240	int err;	2241	int err;
2241		2242
@@ -2248,10 +2249,16 @@ vm_bind_ioctl_ops_create(struct xe_vm vm, struct xe_vma_ops vops, · 6 unmodified lines
2248		2249
2249	switch (operation) {	2250	switch (operation) {
2250	case DRM_XE_VM_BIND_OP_MAP:	2251	case DRM_XE_VM_BIND_OP_MAP:
		2252	if (flags & DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) {
		2253	xe_vm_find_cpu_addr_mirror_vma_range(vm, &range_start, &range_end);
		2254	vops->flags \|= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP;
		2255	}
		2256
		2257	fallthrough;
2251	case DRM_XE_VM_BIND_OP_MAP_USERPTR: {	2258	case DRM_XE_VM_BIND_OP_MAP_USERPTR: {
2252	struct drm_gpuvm_map_req map_req = {	2259	struct drm_gpuvm_map_req map_req = {
2253	.map.va.addr = addr,	2260	.map.va.addr = range_start,
2254	.map.va.range = range,	2261	.map.va.range = range_end - range_start,
2255	.map.gem.obj = obj,	2262	.map.gem.obj = obj,
2256	.map.gem.offset = bo_offset_or_userptr,	2263	.map.gem.offset = bo_offset_or_userptr,
2257	};	2264	};
@@ -2451,8 +2458,17 @@ static struct xe_vma new_vma(struct xe_vm vm, struct drm_gpuva_op_map *op, · 193 unmodified lines
2451	if (IS_ERR(vma))	2458	if (IS_ERR(vma))
2452	return vma;	2459	return vma;
2453		2460
2454	if (xe_vma_is_userptr(vma))	2461	if (xe_vma_is_userptr(vma)) {
2455	err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));	2462	err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
		2463	/*
		2464	* -EBUSY has dedicated meaning that a user fence
		2465	* attached to the VMA is busy, in practice
		2466	* xe_vma_userptr_pin_pages can only fail with -EBUSY if
		2467	* we are low on memory so convert this to -ENOMEM.
		2468	*/
		2469	if (err == -EBUSY)
		2470	err = -ENOMEM;
		2471	}
2456	}	2472	}
2457	if (err) {	2473	if (err) {
2458	prep_vma_destroy(vm, vma, false);	2474	prep_vma_destroy(vm, vma, false);
@@ -2727,7 +2743,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct drm_gpuva_ops ops, · 268 unmodified lines
2727		2743
2728	if (xe_vma_is_cpu_addr_mirror(vma) &&	2744	if (xe_vma_is_cpu_addr_mirror(vma) &&
2729	xe_svm_has_mapping(vm, xe_vma_start(vma),	2745	xe_svm_has_mapping(vm, xe_vma_start(vma),
2730	xe_vma_end(vma)))	2746	xe_vma_end(vma)) &&
		2747	!(vops->flags & XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP))
2731	return -EBUSY;	2748	return -EBUSY;
2732		2749
2733	if (!xe_vma_is_cpu_addr_mirror(vma))	2750	if (!xe_vma_is_cpu_addr_mirror(vma))
@@ -3107,19 +3124,19 @@ static struct dma_fence ops_execute(struct xe_vm vm, · 373 unmodified lines
3107	struct dma_fence *fence = NULL;	3124	struct dma_fence *fence = NULL;
3108	struct dma_fence **fences = NULL;	3125	struct dma_fence **fences = NULL;
3109	struct dma_fence_array *cf = NULL;	3126	struct dma_fence_array *cf = NULL;
3110	int number_tiles = 0, current_fence = 0, n_fence = 0, err;	3127	int number_tiles = 0, current_fence = 0, n_fence = 0, err, i;
3111	u8 id;	3128	u8 id;
3112		3129
3113	number_tiles = vm_ops_setup_tile_args(vm, vops);	3130	number_tiles = vm_ops_setup_tile_args(vm, vops);
3114	if (number_tiles == 0)	3131	if (number_tiles == 0)
3115	return ERR_PTR(-ENODATA);	3132	return ERR_PTR(-ENODATA);
3116		3133
3117	if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) {	3134	for_each_tile(tile, vm->xe, id) {
3118	for_each_tile(tile, vm->xe, id)	3135	++n_fence;
3119	++n_fence;	3136
3120	} else {	3137	if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT))
3121	for_each_tile(tile, vm->xe, id)	3138	for_each_tlb_inval(i)
3122	n_fence += (1 + XE_MAX_GT_PER_TILE);	3139	++n_fence;
3123	}	3140	}
3124		3141
3125	fences = kmalloc_array(n_fence, sizeof(*fences), GFP_KERNEL);	3142	fences = kmalloc_array(n_fence, sizeof(*fences), GFP_KERNEL);
@@ -3149,7 +3166,6 @@ static struct dma_fence ops_execute(struct xe_vm vm, · 23 unmodified lines
3149		3166
3150	for_each_tile(tile, vm->xe, id) {	3167	for_each_tile(tile, vm->xe, id) {
3151	struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q;	3168	struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q;
3152	int i;
3153		3169
3154	fence = NULL;	3170	fence = NULL;
3155	if (!vops->pt_update_ops[id].num_ops)	3171	if (!vops->pt_update_ops[id].num_ops)
@@ -3214,7 +3230,8 @@ static void op_add_ufence(struct xe_vm vm, struct xe_vma_op op, · 58 unmodified lines
3214	{	3230	{
3215	switch (op->base.op) {	3231	switch (op->base.op) {
3216	case DRM_GPUVA_OP_MAP:	3232	case DRM_GPUVA_OP_MAP:
3217	vma_add_ufence(op->map.vma, ufence);	3233	if (!xe_vma_is_cpu_addr_mirror(op->map.vma))
		3234	vma_add_ufence(op->map.vma, ufence);
3218	break;	3235	break;
3219	case DRM_GPUVA_OP_REMAP:	3236	case DRM_GPUVA_OP_REMAP:
3220	if (op->remap.prev)	3237	if (op->remap.prev)
@@ -3490,6 +3507,10 @@ static int xe_vm_bind_ioctl_validate_bo(struct xe_device xe, struct xe_bo bo, · 269 unmodified lines
3490	{	3507	{
3491	u16 coh_mode;	3508	u16 coh_mode;
3492		3509
		3510	if (XE_IOCTL_DBG(xe, (bo->flags & XE_BO_FLAG_NO_COMPRESSION) &&
		3511	xe_pat_index_get_comp_en(xe, pat_index)))
		3512	return -EINVAL;
		3513
3493	if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) \|\|	3514	if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) \|\|
3494	XE_IOCTL_DBG(xe, obj_offset >	3515	XE_IOCTL_DBG(xe, obj_offset >
3495	xe_bo_size(bo) - range)) {	3516	xe_bo_size(bo) - range)) {
@@ -3913,7 +3934,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, · 417 unmodified lines
3913		3934
3914	err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,	3935	err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval,
3915	&fence[fence_id], start, end,	3936	&fence[fence_id], start, end,
3916	vm->usm.asid);	3937	vm->usm.asid, NULL);
3917	if (err)	3938	if (err)
3918	goto wait;	3939	goto wait;
3919	++fence_id;	3940	++fence_id;
@@ -3926,7 +3947,7 @@ int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, · 6 unmodified lines
3926		3947
3927	err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,	3948	err = xe_tlb_inval_range(&tile->media_gt->tlb_inval,
3928	&fence[fence_id], start, end,	3949	&fence[fence_id], start, end,
3929	vm->usm.asid);	3950	vm->usm.asid, NULL);
3930	if (err)	3951	if (err)
3931	goto wait;	3952	goto wait;
3932	++fence_id;	3953	++fence_id;
@@ -4032,10 +4053,18 @@ int xe_vm_validate_protected(struct xe_vm *vm) · 99 unmodified lines
4032	}	4053	}
4033		4054
4034	struct xe_vm_snapshot {	4055	struct xe_vm_snapshot {
		4056	int uapi_flags;
4035	unsigned long num_snaps;	4057	unsigned long num_snaps;
4036	struct {	4058	struct {
4037	u64 ofs, bo_ofs;	4059	u64 ofs, bo_ofs;
4038	unsigned long len;	4060	unsigned long len;
		4061	#define XE_VM_SNAP_FLAG_USERPTR BIT(0)
		4062	#define XE_VM_SNAP_FLAG_READ_ONLY BIT(1)
		4063	#define XE_VM_SNAP_FLAG_IS_NULL BIT(2)
		4064	unsigned long flags;
		4065	int uapi_mem_region;
		4066	int pat_index;
		4067	int cpu_caching;
4039	struct xe_bo *bo;	4068	struct xe_bo *bo;
4040	void *data;	4069	void *data;
4041	struct mm_struct *mm;	4070	struct mm_struct *mm;
@@ -4064,6 +4093,13 @@ struct xe_vm_snapshot xe_vm_snapshot_capture(struct xe_vm vm) · 22 unmodified lines
4064	goto out_unlock;	4093	goto out_unlock;
4065	}	4094	}
4066		4095
		4096	if (vm->flags & XE_VM_FLAG_FAULT_MODE)
		4097	snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_FAULT_MODE;
		4098	if (vm->flags & XE_VM_FLAG_LR_MODE)
		4099	snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_LR_MODE;
		4100	if (vm->flags & XE_VM_FLAG_SCRATCH_PAGE)
		4101	snap->uapi_flags \|= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE;
		4102
4067	snap->num_snaps = num_snaps;	4103	snap->num_snaps = num_snaps;
4068	i = 0;	4104	i = 0;
4069	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {	4105	drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) {
@@ -4076,9 +4112,25 @@ struct xe_vm_snapshot xe_vm_snapshot_capture(struct xe_vm vm) · 6 unmodified lines
4076		4112
4077	snap->snap[i].ofs = xe_vma_start(vma);	4113	snap->snap[i].ofs = xe_vma_start(vma);
4078	snap->snap[i].len = xe_vma_size(vma);	4114	snap->snap[i].len = xe_vma_size(vma);
		4115	snap->snap[i].flags = xe_vma_read_only(vma) ?
		4116	XE_VM_SNAP_FLAG_READ_ONLY : 0;
		4117	snap->snap[i].pat_index = vma->attr.pat_index;
4079	if (bo) {	4118	if (bo) {
		4119	snap->snap[i].cpu_caching = bo->cpu_caching;
4080	snap->snap[i].bo = xe_bo_get(bo);	4120	snap->snap[i].bo = xe_bo_get(bo);
4081	snap->snap[i].bo_ofs = xe_vma_bo_offset(vma);	4121	snap->snap[i].bo_ofs = xe_vma_bo_offset(vma);
		4122	switch (bo->ttm.resource->mem_type) {
		4123	case XE_PL_SYSTEM:
		4124	case XE_PL_TT:
		4125	snap->snap[i].uapi_mem_region = 0;
		4126	break;
		4127	case XE_PL_VRAM0:
		4128	snap->snap[i].uapi_mem_region = 1;
		4129	break;
		4130	case XE_PL_VRAM1:
		4131	snap->snap[i].uapi_mem_region = 2;
		4132	break;
		4133	}
4082	} else if (xe_vma_is_userptr(vma)) {	4134	} else if (xe_vma_is_userptr(vma)) {
4083	struct mm_struct *mm =	4135	struct mm_struct *mm =
4084	to_userptr_vma(vma)->userptr.notifier.mm;	4136	to_userptr_vma(vma)->userptr.notifier.mm;
@@ -4089,8 +4141,14 @@ struct xe_vm_snapshot xe_vm_snapshot_capture(struct xe_vm vm) · 4 unmodified lines
4089	snap->snap[i].data = ERR_PTR(-EFAULT);	4141	snap->snap[i].data = ERR_PTR(-EFAULT);
4090		4142
4091	snap->snap[i].bo_ofs = xe_vma_userptr(vma);	4143	snap->snap[i].bo_ofs = xe_vma_userptr(vma);
		4144	snap->snap[i].flags \|= XE_VM_SNAP_FLAG_USERPTR;
		4145	snap->snap[i].uapi_mem_region = 0;
		4146	} else if (xe_vma_is_null(vma)) {
		4147	snap->snap[i].flags \|= XE_VM_SNAP_FLAG_IS_NULL;
		4148	snap->snap[i].uapi_mem_region = -1;
4092	} else {	4149	} else {
4093	snap->snap[i].data = ERR_PTR(-ENOENT);	4150	snap->snap[i].data = ERR_PTR(-ENOENT);
		4151	snap->snap[i].uapi_mem_region = -1;
4094	}	4152	}
4095	i++;	4153	i++;
4096	}	4154	}
@@ -4109,7 +4167,8 @@ void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap) · 12 unmodified lines
4109	struct xe_bo *bo = snap->snap[i].bo;	4167	struct xe_bo *bo = snap->snap[i].bo;
4110	int err;	4168	int err;
4111		4169
4112	if (IS_ERR(snap->snap[i].data))	4170	if (IS_ERR(snap->snap[i].data) \|\|
		4171	snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL)
4113	continue;	4172	continue;
4114		4173
4115	snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER);	4174	snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER);
@@ -4155,15 +4214,32 @@ void xe_vm_snapshot_print(struct xe_vm_snapshot snap, struct drm_printer p) · 39 unmodified lines
4155	return;	4214	return;
4156	}	4215	}
4157		4216
		4217	drm_printf(p, "VM.uapi_flags: 0x%x\n", snap->uapi_flags);
4158	for (i = 0; i < snap->num_snaps; i++) {	4218	for (i = 0; i < snap->num_snaps; i++) {
4159	drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len);	4219	drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len);
4160		4220
		4221	drm_printf(p, "[%llx].properties: %s\|%s\|mem_region=0x%lx\|pat_index=%d\|cpu_caching=%d\n",
		4222	snap->snap[i].ofs,
		4223	snap->snap[i].flags & XE_VM_SNAP_FLAG_READ_ONLY ?
		4224	"read_only" : "read_write",
		4225	snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL ?
		4226	"null_sparse" :
		4227	snap->snap[i].flags & XE_VM_SNAP_FLAG_USERPTR ?
		4228	"userptr" : "bo",
		4229	snap->snap[i].uapi_mem_region == -1 ? 0 :
		4230	BIT(snap->snap[i].uapi_mem_region),
		4231	snap->snap[i].pat_index,
		4232	snap->snap[i].cpu_caching);
		4233
4161	if (IS_ERR(snap->snap[i].data)) {	4234	if (IS_ERR(snap->snap[i].data)) {
4162	drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs,	4235	drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs,
4163	PTR_ERR(snap->snap[i].data));	4236	PTR_ERR(snap->snap[i].data));
4164	continue;	4237	continue;
4165	}	4238	}
4166		4239
		4240	if (snap->snap[i].flags & XE_VM_SNAP_FLAG_IS_NULL)
		4241	continue;
		4242
4167	drm_printf(p, "[%llx].data: ", snap->snap[i].ofs);	4243	drm_printf(p, "[%llx].data: ", snap->snap[i].ofs);
4168		4244
4169	for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) {	4245	for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) {
@@ -4317,6 +4393,8 @@ static int xe_vm_alloc_vma(struct xe_vm *vm, · 147 unmodified lines
4317		4393
4318	if (is_madvise)	4394	if (is_madvise)
4319	vops.flags \|= XE_VMA_OPS_FLAG_MADVISE;	4395	vops.flags \|= XE_VMA_OPS_FLAG_MADVISE;
		4396	else
		4397	vops.flags \|= XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP;
4320		4398
4321	err = vm_bind_ioctl_ops_parse(vm, ops, &vops);	4399	err = vm_bind_ioctl_ops_parse(vm, ops, &vops);
4322	if (err)	4400	if (err)
@@ -4390,6 +4468,46 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) · 67 unmodified lines
4390	return xe_vm_alloc_vma(vm, &map_req, true);	4468	return xe_vm_alloc_vma(vm, &map_req, true);
4391	}	4469	}
4392		4470
		4471	static bool is_cpu_addr_vma_with_default_attr(struct xe_vma *vma)
		4472	{
		4473	return vma && xe_vma_is_cpu_addr_mirror(vma) &&
		4474	xe_vma_has_default_mem_attrs(vma);
		4475	}
		4476
		4477	/**
		4478	* xe_vm_find_cpu_addr_mirror_vma_range - Extend a VMA range to include adjacent CPU-mirrored VMAs
		4479	* @vm: VM to search within
		4480	* @start: Input/output pointer to the starting address of the range
		4481	* @end: Input/output pointer to the end address of the range
		4482	*
		4483	* Given a range defined by @start and @range, this function checks the VMAs
		4484	* immediately before and after the range. If those neighboring VMAs are
		4485	* CPU-address-mirrored and have default memory attributes, the function
		4486	* updates @start and @range to include them. This extended range can then
		4487	* be used for merging or other operations that require a unified VMA.
		4488	*
		4489	* The function does not perform the merge itself; it only computes the
		4490	* mergeable boundaries.
		4491	*/
		4492	void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm vm, u64 start, u64 *end)
		4493	{
		4494	struct xe_vma prev, next;
		4495
		4496	lockdep_assert_held(&vm->lock);
		4497
		4498	if (*start >= SZ_4K) {
		4499	prev = xe_vm_find_vma_by_addr(vm, *start - SZ_4K);
		4500	if (is_cpu_addr_vma_with_default_attr(prev))
		4501	*start = xe_vma_start(prev);
		4502	}
		4503
		4504	if (*end < vm->size) {
		4505	next = xe_vm_find_vma_by_addr(vm, *end + 1);
		4506	if (is_cpu_addr_vma_with_default_attr(next))
		4507	*end = xe_vma_end(next);
		4508	}
		4509	}
		4510
4393	/**	4511	/**
4394	* xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma	4512	* xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma
4395	* @vm: Pointer to the xe_vm structure	4513	* @vm: Pointer to the xe_vm structure

drivers/gpu/drm/xe/xe_vm.h+3−0

@@ -68,6 +68,9 @@ xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range); · 67 unmodified lines
68		68
69	bool xe_vma_has_default_mem_attrs(struct xe_vma *vma);	69	bool xe_vma_has_default_mem_attrs(struct xe_vma *vma);
70		70
		71	void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm,
		72	u64 *start,
		73	u64 *end);
71	/**	74	/**
72	* xe_vm_has_scratch() - Whether the vm is configured for scratch PTEs	75	* xe_vm_has_scratch() - Whether the vm is configured for scratch PTEs
73	* @vm: The vm	76	* @vm: The vm

drivers/gpu/drm/xe/xe_vm_types.h+1−0

@@ -467,6 +467,7 @@ struct xe_vma_ops { · 466 unmodified lines
467	#define XE_VMA_OPS_FLAG_MADVISE BIT(1)	467	#define XE_VMA_OPS_FLAG_MADVISE BIT(1)
468	#define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2)	468	#define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2)
469	#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3)	469	#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3)
		470	#define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP BIT(4)
470	u32 flags;	471	u32 flags;
471	#ifdef TEST_VM_OPS_ERROR	472	#ifdef TEST_VM_OPS_ERROR
472	/** @inject_error: inject error to test error handling */	473	/** @inject_error: inject error to test error handling */

drivers/gpu/drm/xe/xe_vram.c+2−4

 static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset)
 {
 	struct xe_device *xe = gt_to_xe(gt);
-	unsigned int fw_ref;
 	u64 offset;
 	u32 reg;
-	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
-	if (!fw_ref)
+	if (!fw_ref.domains)
 		return -ETIMEDOUT;
 	if (GRAPHICS_VER(xe) >= 20) {
 		offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K;
 	}
-	xe_force_wake_put(gt_to_fw(gt), fw_ref);
 	*poffset = offset;
 	return 0;

drivers/gpu/drm/xe/xe_wa.c+6−46

 #include "regs/xe_engine_regs.h"
 #include "regs/xe_gt_regs.h"
+#include "regs/xe_guc_regs.h"
 #include "regs/xe_regs.h"
 #include "xe_device_types.h"
 #include "xe_force_wake.h"
 	  XE_RTP_ACTIONS(SET(XELPMP_SQCNT1, ENFORCE_RAR))
 	},
-	/* Xe2_LPG */
-	{ XE_RTP_NAME("16020975621"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)),
-	  XE_RTP_ACTIONS(SET(XEHP_SLICE_UNIT_LEVEL_CLKGATE, SBEUNIT_CLKGATE_DIS))
-	},
-	{ XE_RTP_NAME("14018157293"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)),
-	  XE_RTP_ACTIONS(SET(XEHPC_L3CLOS_MASK(0), ~0),
-			 SET(XEHPC_L3CLOS_MASK(1), ~0),
-			 SET(XEHPC_L3CLOS_MASK(2), ~0),
-			 SET(XEHPC_L3CLOS_MASK(3), ~0))
-	},
 	/* Xe2_LPM */
 	{ XE_RTP_NAME("14017421178"),
 	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F10(0), RAMDFTUNIT_CLKGATE_DIS)),
 	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 	},
+	{ XE_RTP_NAME("16028005424"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005)),
+	  XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES))
+	},
 };
 static const struct xe_rtp_entry_sr engine_was[] = {
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, XE2_ALLOC_DPA_STARVE_FIX_DIS))
 	},
-	{ XE_RTP_NAME("14018957109"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN5, DISABLE_SAMPLE_G_PERFORMANCE))
-	},
 	{ XE_RTP_NAME("14020338487"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, XE2_EUPEND_CHK_FLUSH_DIS))
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN4, DISABLE_TDL_PUSH))
 	},
-	{ XE_RTP_NAME("14019322943"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       FUNC(xe_rtp_match_first_render_or_compute)),
-	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0, TGM_WRITE_EOM_FORCE))
-	},
 	{ XE_RTP_NAME("14018471104"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, ENABLE_SMP_LD_RENDER_SURFACE_CONTROL))
 	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
 	},
 	{ XE_RTP_NAME("18041344222"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3001),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3000),
 		       FUNC(xe_rtp_match_first_render_or_compute),
 		       FUNC(xe_rtp_match_not_sriov_vf),
 		       FUNC(xe_rtp_match_gt_has_discontiguous_dss_groups)),
 	/* Xe2_LPG */
-	{ XE_RTP_NAME("16020518922"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(FF_MODE,
-			     DIS_TE_AUTOSTRIP |
-			     DIS_MESH_PARTIAL_AUTOSTRIP |
-			     DIS_MESH_AUTOSTRIP),
-			 SET(VFLSKPD,
-			     DIS_PARTIAL_AUTOSTRIP |
-			     DIS_AUTOSTRIP))
-	},
 	{ XE_RTP_NAME("14019386621"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE))
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT))
 	},
-	{ XE_RTP_NAME("14020013138"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS))
-	},
 	{ XE_RTP_NAME("14019988906"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD))
 	},
-	{ XE_RTP_NAME("16020183090"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0),
-		       ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(INSTPM(RENDER_RING_BASE), ENABLE_SEMAPHORE_POLL_BIT))
-	},
 	{ XE_RTP_NAME("18033852989"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))

drivers/gpu/drm/xe/xe_wa_oob.rules+1−5

 16017236439	PLATFORM(PVC)
 14019821291	MEDIA_VERSION_RANGE(1300, 2000)
 14015076503	MEDIA_VERSION(1300)
-16020292621	GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
+14018913170	GRAPHICS_VERSION_RANGE(1270, 1274)
-14018913170	GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
-		MEDIA_VERSION(2000), GRAPHICS_STEP(A0, A1)
-		GRAPHICS_VERSION_RANGE(1270, 1274)
 		MEDIA_VERSION(1300)
 		PLATFORM(DG2)
 14018094691	GRAPHICS_VERSION_RANGE(2001, 2002)
 		GRAPHICS_VERSION(2004)
-14019882105	GRAPHICS_VERSION(2004), GRAPHICS_STEP(A0, B0)
 18024947630	GRAPHICS_VERSION(2001)
 		GRAPHICS_VERSION(2004)
 		MEDIA_VERSION(2000)

include/uapi/drm/xe_drm.h+78−3

 #define DRM_XE_OBSERVATION		0x0b
 #define DRM_XE_MADVISE			0x0c
 #define DRM_XE_VM_QUERY_MEM_RANGE_ATTRS	0x0d
+#define DRM_XE_EXEC_QUEUE_SET_PROPERTY	0x0e
 /* Must be kept compact -- no holes */
 #define DRM_IOCTL_XE_OBSERVATION		DRM_IOW(DRM_COMMAND_BASE + DRM_XE_OBSERVATION, struct drm_xe_observation_param)
 #define DRM_IOCTL_XE_MADVISE			DRM_IOW(DRM_COMMAND_BASE + DRM_XE_MADVISE, struct drm_xe_madvise)
 #define DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS	DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_QUERY_MEM_RANGE_ATTRS, struct drm_xe_vm_query_mem_range_attr)
+#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY	DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property)
 /**
  * DOC: Xe IOCTL Extensions
 	/** @pad: MBZ */
 	__u32 pad;
-	/** @value: property value */
+	union {
-	__u64 value;
+		/** @value: property value */
+		__u64 value;
+		/** @ptr: pointer to user value */
+		__u64 ptr;
+	};
 	/** @reserved: Reserved */
 	__u64 reserved[2];
  *      has low latency hint support
  *    - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the
  *      device has CPU address mirroring support
+ *    - %DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT - Flag is set if the
+ *      device supports the userspace hint %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION.
+ *      This is exposed only on Xe2+.
  *  - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment
  *    required by this device, typically SZ_4K or SZ_64K
  *  - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM	(1 << 0)
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY	(1 << 1)
 	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR	(1 << 2)
+	#define DRM_XE_QUERY_CONFIG_FLAG_HAS_NO_COMPRESSION_HINT (1 << 3)
 #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT		2
 #define DRM_XE_QUERY_CONFIG_VA_BITS			3
 #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY	4
  *    need to use VRAM for display surfaces, therefore the kernel requires
  *    setting this flag for such objects, otherwise an error is thrown on
  *    small-bar systems.
+ *  - %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION - Allows userspace to
+ *    hint that compression (CCS) should be disabled for the buffer being
+ *    created. This can avoid unnecessary memory operations and CCS state
+ *    management.
+ *    On pre-Xe2 platforms, this flag is currently rejected as compression
+ *    control is not supported via PAT index. On Xe2+ platforms, compression
+ *    is controlled via PAT entries. If this flag is set, the driver will reject
+ *    any VM bind that requests a PAT index enabling compression for this BO.
+ *    Note: On dGPU platforms, there is currently no change in behavior with
+ *    this flag, but future improvements may leverage it. The current benefit is
+ *    primarily applicable to iGPU platforms.
  *
  * @cpu_caching supports the following values:
  *  - %DRM_XE_GEM_CPU_CACHING_WB - Allocate the pages with write-back
 #define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING		(1 << 0)
 #define DRM_XE_GEM_CREATE_FLAG_SCANOUT			(1 << 1)
 #define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM	(1 << 2)
+#define DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION		(1 << 3)
 	/**
 	 * @flags: Flags, currently a mask of memory instances of where BO can
 	 * be placed
  *    Given that going into a power-saving state kills PXP HWDRM sessions,
  *    runtime PM will be blocked while queues of this type are alive.
  *    All PXP queues will be killed if a PXP invalidation event occurs.
+ *  - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP - Create a multi-queue group
+ *    or add secondary queues to a multi-queue group.
+ *    If the extension's 'value' field has %DRM_XE_MULTI_GROUP_CREATE flag set,
+ *    then a new multi-queue group is created with this queue as the primary queue
+ *    (Q0). Otherwise, the queue gets added to the multi-queue group whose primary
+ *    queue's exec_queue_id is specified in the lower 32 bits of the 'value' field.
+ *    If the extension's 'value' field has %DRM_XE_MULTI_GROUP_KEEP_ACTIVE flag
+ *    set, then the multi-queue group is kept active after the primary queue is
+ *    destroyed.
+ *    All the other non-relevant bits of extension's 'value' field while adding the
+ *    primary or the secondary queues of the group must be set to 0.
+ *  - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY - Set the queue
+ *    priority within the multi-queue group. Current valid priority values are 0–2
+ *    (default is 1), with higher values indicating higher priority.
  *
  * The example below shows how to use @drm_xe_exec_queue_create to create
  * a simple exec_queue (no parallel submission) of class
 #define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY		0
 #define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE		1
 #define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE		2
+#define   DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE		3
+#define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP		4
+#define     DRM_XE_MULTI_GROUP_CREATE				(1ull << 63)
+#define     DRM_XE_MULTI_GROUP_KEEP_ACTIVE			(1ull << 62)
+#define   DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY	5
 	/** @extensions: Pointer to the first extension struct, if any */
 	__u64 extensions;
 	/** @DRM_XE_OA_UNIT_TYPE_OAM_SAG: OAM_SAG OA unit */
 	DRM_XE_OA_UNIT_TYPE_OAM_SAG,
+	/** @DRM_XE_OA_UNIT_TYPE_MERT: MERT OA unit */
+	DRM_XE_OA_UNIT_TYPE_MERT,
 };
 /**
 #define DRM_XE_OA_CAPS_OA_BUFFER_SIZE	(1 << 2)
 #define DRM_XE_OA_CAPS_WAIT_NUM_REPORTS	(1 << 3)
 #define DRM_XE_OA_CAPS_OAM		(1 << 4)
+#define DRM_XE_OA_CAPS_OA_UNIT_GT_ID	(1 << 5)
 	/** @oa_timestamp_freq: OA timestamp freq */
 	__u64 oa_timestamp_freq;
+	/** @gt_id: gt id for this OA unit */
+	__u16 gt_id;
+	/** @reserved1: MBZ */
+	__u16 reserved1[3];
 	/** @reserved: MBZ */
-	__u64 reserved[4];
+	__u64 reserved[3];
 	/** @num_engines: number of engines in @eci array */
 	__u64 num_engines;
 };
+/**
+ * struct drm_xe_exec_queue_set_property - exec queue set property
+ *
+ * Sets execution queue properties dynamically.
+ * Currently only %DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY
+ * property can be dynamically set.
+ */
+struct drm_xe_exec_queue_set_property {
+	/** @extensions: Pointer to the first extension struct, if any */
+	__u64 extensions;
+	/** @exec_queue_id: Exec queue ID */
+	__u32 exec_queue_id;
+	/** @property: property to set */
+	__u32 property;
+	/** @value: property value */
+	__u64 value;
+	/** @reserved: Reserved */
+	__u64 reserved[2];
+};
 #if defined(__cplusplus)
 }
 #endif

@@ -317,7 +317,7 @@ static void guc_waklv_init(struct xe_guc_ads *ads) · 316 unmodified lines
317	offset = guc_ads_waklv_offset(ads);	317	offset = guc_ads_waklv_offset(ads);
318	remain = guc_ads_waklv_size(ads);	318	remain = guc_ads_waklv_size(ads);
319		319
320	if (XE_GT_WA(gt, 14019882105) \|\| XE_GT_WA(gt, 16021333562))	320	if (XE_GT_WA(gt, 16021333562))
321	guc_waklv_enable(ads, NULL, 0, &offset, &remain,	321	guc_waklv_enable(ads, NULL, 0, &offset, &remain,
322	GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED);	322	GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED);
323	if (XE_GT_WA(gt, 18024947630))	323	if (XE_GT_WA(gt, 18024947630))
@@ -347,10 +347,10 @@ static void guc_waklv_init(struct xe_guc_ads *ads) · 23 unmodified lines
347	guc_waklv_enable(ads, NULL, 0, &offset, &remain,	347	guc_waklv_enable(ads, NULL, 0, &offset, &remain,
348	GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET);	348	GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET);
349		349
350	if (GUC_FIRMWARE_VER(&gt->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_GT_WA(gt, 16026508708))	350	if (GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 44) && XE_GT_WA(gt, 16026508708))
351	guc_waklv_enable(ads, NULL, 0, &offset, &remain,	351	guc_waklv_enable(ads, NULL, 0, &offset, &remain,
352	GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH);	352	GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH);
353	if (GUC_FIRMWARE_VER(&gt->uc.guc) >= MAKE_GUC_VER(70, 47, 0) && XE_GT_WA(gt, 16026007364)) {	353	if (GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 47) && XE_GT_WA(gt, 16026007364)) {
354	u32 data[] = {	354	u32 data[] = {
355	0x0,	355	0x0,
356	0xF,	356	0xF,