lael-labs/linux

drivers/gpu/drm/i915/display/intel_display_wa.h+8−0

 #ifndef __INTEL_DISPLAY_WA_H__
 #define __INTEL_DISPLAY_WA_H__
+#include <linux/types.h>
 struct drm_i915_private;
 void intel_display_wa_apply(struct drm_i915_private *i915);
+#ifdef I915
+static inline bool intel_display_needs_wa_16023588340(struct drm_i915_private *i915) { return false; }
+#else
+bool intel_display_needs_wa_16023588340(struct drm_i915_private *i915);
+#endif
 #endif

drivers/gpu/drm/i915/display/intel_fbc.c+6−0

 #include "intel_display_device.h"
 #include "intel_display_trace.h"
 #include "intel_display_types.h"
+#include "intel_display_wa.h"
 #include "intel_fbc.h"
 #include "intel_fbc_regs.h"
 #include "intel_frontbuffer.h"
 		return 0;
 	}
+	if (intel_display_needs_wa_16023588340(i915)) {
+		plane_state->no_fbc_reason = "Wa_16023588340";
+		return 0;
+	}
 	/* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */
 	if (i915_vtd_active(i915) && (IS_SKYLAKE(i915) || IS_BROXTON(i915))) {
 		plane_state->no_fbc_reason = "VT-d enabled";

drivers/gpu/drm/xe/Makefile+5−18

 subdir-ccflags-y += -I$(obj) -I$(src)
 # generated sources
-hostprogs := xe_gen_wa_oob
+hostprogs := xe_gen_wa_oob
 generated_oob := $(obj)/generated/xe_wa_oob.c $(obj)/generated/xe_wa_oob.h
 quiet_cmd_wa_oob = GEN     $(notdir $(generated_oob))
       cmd_wa_oob = mkdir -p $(@D); $^ $(generated_oob)
 $(obj)/generated/%_wa_oob.c $(obj)/generated/%_wa_oob.h: $(obj)/xe_gen_wa_oob \
 		 $(src)/xe_wa_oob.rules
 	$(call cmd,wa_oob)
-uses_generated_oob := \
-	$(obj)/xe_ggtt.o \
-	$(obj)/xe_gsc.o \
-	$(obj)/xe_gt.o \
-	$(obj)/xe_guc.o \
-	$(obj)/xe_guc_ads.o \
-	$(obj)/xe_guc_pc.o \
-	$(obj)/xe_migrate.o \
-	$(obj)/xe_ring_ops.o \
-	$(obj)/xe_vm.o \
-	$(obj)/xe_wa.o \
-	$(obj)/xe_ttm_stolen_mgr.o
-$(uses_generated_oob): $(generated_oob)
 # Please keep these build lists sorted!
 # core driver code
 	display/xe_display.o \
 	display/xe_display_misc.o \
 	display/xe_display_rps.o \
+	display/xe_display_wa.o \
 	display/xe_dsb_buffer.o \
 	display/xe_fb_pin.o \
 	display/xe_hdcp_gsc.o \
 $(obj)/%.hdrtest: $(src)/%.h FORCE
 	$(call if_changed_dep,hdrtest)
+uses_generated_oob := $(addprefix $(obj)/, $(xe-y))
+$(uses_generated_oob): $(obj)/generated/xe_wa_oob.h

drivers/gpu/drm/xe/display/intel_fbdev_fb.c+5−1

 #include "xe_bo.h"
 #include "xe_gt.h"
 #include "xe_ttm_stolen_mgr.h"
+#include "xe_wa.h"
+#include <generated/xe_wa_oob.h>
 struct intel_framebuffer *intel_fbdev_fb_alloc(struct drm_fb_helper *helper,
 					       struct drm_fb_helper_surface_size *sizes)
 	size = PAGE_ALIGN(size);
 	obj = ERR_PTR(-ENODEV);
-	if (!IS_DGFX(xe)) {
+	if (!IS_DGFX(xe) && !XE_WA(xe_root_mmio_gt(xe), 22019338487_display)) {
 		obj = xe_bo_create_pin_map(xe, xe_device_get_root_tile(xe),
 					   NULL, size,
 					   ttm_bo_type_kernel, XE_BO_FLAG_SCANOUT |
 		else
 			drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj));
 	}
 	if (IS_ERR(obj)) {
 		obj = xe_bo_create_pin_map(xe, xe_device_get_root_tile(xe), NULL, size,
 					   ttm_bo_type_kernel, XE_BO_FLAG_SCANOUT |

drivers/gpu/drm/xe/display/xe_display_wa.c+16−0

@@ -0,0 +1,16 @@
	1	// SPDX-License-Identifier: MIT
	2	/*
	3	* Copyright © 2024 Intel Corporation
	4	*/
	5
	6	#include "intel_display_wa.h"
	7
	8	#include "xe_device.h"
	9	#include "xe_wa.h"
	10
	11	#include <generated/xe_wa_oob.h>
	12
	13	bool intel_display_needs_wa_16023588340(struct drm_i915_private *i915)
	14	{
	15	return XE_WA(xe_root_mmio_gt(i915), 16023588340);
	16	}

drivers/gpu/drm/xe/display/xe_dsb_buffer.c+8−0

 #include "intel_display_types.h"
 #include "intel_dsb_buffer.h"
 #include "xe_bo.h"
+#include "xe_device.h"
+#include "xe_device_types.h"
 #include "xe_gt.h"
 u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf)
 void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val)
 {
+	struct xe_device *xe = dsb_buf->vma->bo->tile->xe;
 	iosys_map_wr(&dsb_buf->vma->bo->vmap, idx * 4, u32, val);
+	xe_device_l2_flush(xe);
 }
 u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx)
 void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size)
 {
+	struct xe_device *xe = dsb_buf->vma->bo->tile->xe;
 	WARN_ON(idx > (dsb_buf->buf_size - size) / sizeof(*dsb_buf->cmd_buf));
 	iosys_map_memset(&dsb_buf->vma->bo->vmap, idx * 4, val, size);
+	xe_device_l2_flush(xe);
 }
 bool intel_dsb_buffer_create(struct intel_crtc *crtc, struct intel_dsb_buffer *dsb_buf, size_t size)

drivers/gpu/drm/xe/display/xe_fb_pin.c+3−0

 #include "intel_fb.h"
 #include "intel_fb_pin.h"
 #include "xe_bo.h"
+#include "xe_device.h"
 #include "xe_ggtt.h"
 #include "xe_gt.h"
 #include "xe_pm.h"
 	if (ret)
 		goto err_unpin;
+	/* Ensure DPT writes are flushed */
+	xe_device_l2_flush(xe);
 	return vma;
 err_unpin:

drivers/gpu/drm/xe/display/xe_plane_initial.c+6−0

 #include "intel_frontbuffer.h"
 #include "intel_plane_initial.h"
 #include "xe_bo.h"
+#include "xe_wa.h"
+#include <generated/xe_wa_oob.h>
 static bool
 intel_reuse_initial_plane_obj(struct intel_crtc *this,
 		phys_base = base;
 		flags |= XE_BO_FLAG_STOLEN;
+		if (XE_WA(xe_root_mmio_gt(xe), 22019338487_display))
+			return NULL;
 		/*
 		 * If the FB is too big, just don't use it since fbdev is not very
 		 * important and we should probably use that space with FBC or other

drivers/gpu/drm/xe/regs/xe_gt_regs.h+15−0

 #define   LE_CACHEABILITY_MASK			REG_GENMASK(1, 0)
 #define   LE_CACHEABILITY(value)		REG_FIELD_PREP(LE_CACHEABILITY_MASK, value)
+#define XE2_GAMREQSTRM_CTRL			XE_REG(0x4194)
+#define   CG_DIS_CNTLBUS			REG_BIT(6)
 #define CCS_AUX_INV				XE_REG(0x4208)
 #define VD0_AUX_INV				XE_REG(0x4218)
 #define VE1_AUX_INV				XE_REG(0x42b8)
 #define   AUX_INV				REG_BIT(0)
+#define XE2_LMEM_CFG				XE_REG(0x48b0)
 #define XEHP_TILE_ADDR_RANGE(_idx)		XE_REG_MCR(0x4900 + (_idx) * 4)
 #define XEHP_FLAT_CCS_BASE_ADDR			XE_REG_MCR(0x4910)
 #define XEHP_FLAT_CCS_PTR			REG_GENMASK(31, 8)
 #define FF_MODE					XE_REG_MCR(0x6210)
 #define   DIS_TE_AUTOSTRIP			REG_BIT(31)
+#define   VS_HIT_MAX_VALUE_MASK			REG_GENMASK(25, 20)
 #define   DIS_MESH_PARTIAL_AUTOSTRIP		REG_BIT(16)
 #define   DIS_MESH_AUTOSTRIP			REG_BIT(15)
 #define XEHPC_L3CLOS_MASK(i)			XE_REG_MCR(0xb194 + (i) * 8)
+#define XE2_GLOBAL_INVAL			XE_REG(0xb404)
+#define SCRATCH1LPFC				XE_REG(0xb474)
+#define   EN_L3_RW_CCS_CACHE_FLUSH		REG_BIT(0)
 #define XE2LPM_L3SQCREG5			XE_REG_MCR(0xb658)
 #define XE2_TDF_CTRL				XE_REG(0xb418)
 #define   INVALIDATION_BROADCAST_MODE_DIS	REG_BIT(12)
 #define   GLOBAL_INVALIDATION_MODE		REG_BIT(2)
+#define LMEM_CFG				XE_REG(0xcf58)
+#define   LMEM_EN				REG_BIT(31)
+#define   LMTT_DIR_PTR				REG_GENMASK(30, 0) /* in multiples of 64KB */
 #define HALF_SLICE_CHICKEN5			XE_REG_MCR(0xe188, XE_REG_OPTION_MASKED)
 #define   DISABLE_SAMPLE_G_PERFORMANCE		REG_BIT(0)

drivers/gpu/drm/xe/regs/xe_regs.h+8−4

 #define GU_MISC_IRQ_OFFSET			0x444f0
 #define   GU_MISC_GSE				REG_BIT(27)
-#define SOFTWARE_FLAGS_SPR33			XE_REG(0x4f084)
 #define GU_CNTL_PROTECTED			XE_REG(0x10100C)
 #define   DRIVERINT_FLR_DIS			REG_BIT(31)
 #define   LMEM_INIT				REG_BIT(7)
 #define   DRIVERFLR				REG_BIT(31)
+#define XEHP_CLOCK_GATE_DIS			XE_REG(0x101014)
+#define   SGSI_SIDECLK_DIS			REG_BIT(17)
 #define GU_DEBUG				XE_REG(0x101018)
 #define   DRIVERFLR_STATUS			REG_BIT(31)
-#define XEHP_CLOCK_GATE_DIS			XE_REG(0x101014)
+#define VIRTUAL_CTRL_REG			XE_REG(0x10108c)
-#define   SGSI_SIDECLK_DIS			REG_BIT(17)
+#define   GUEST_GTT_UPDATE_EN			REG_BIT(8)
 #define XEHP_MTCFG_ADDR				XE_REG(0x101800)
 #define   TILE_COUNT				REG_GENMASK(15, 8)
 #define   DISPLAY_IRQ				REG_BIT(16)
 #define   GT_DW_IRQ(x)				REG_BIT(x)
+#define VF_CAP_REG				XE_REG(0x1901f8, XE_REG_OPTION_VF)
+#define   VF_CAP				REG_BIT(0)
 #define PVC_RP_STATE_CAP			XE_REG(0x281014)
 #endif

drivers/gpu/drm/xe/regs/xe_sriov_regs.h+0−23

@@ -1,23 +0,0 @@
1	/* SPDX-License-Identifier: MIT */
2	/*
3	* Copyright © 2023 Intel Corporation
4	*/
5
6	#ifndef _REGS_XE_SRIOV_REGS_H_
7	#define _REGS_XE_SRIOV_REGS_H_
8
9	#include "regs/xe_reg_defs.h"
10
11	#define XE2_LMEM_CFG XE_REG(0x48b0)
12
13	#define LMEM_CFG XE_REG(0xcf58)
14	#define LMEM_EN REG_BIT(31)
15	#define LMTT_DIR_PTR REG_GENMASK(30, 0) /* in multiples of 64KB */
16
17	#define VIRTUAL_CTRL_REG XE_REG(0x10108c)
18	#define GUEST_GTT_UPDATE_EN REG_BIT(8)
19
20	#define VF_CAP_REG XE_REG(0x1901f8, XE_REG_OPTION_VF)
21	#define VF_CAP REG_BIT(0)
22
23	#endif

drivers/gpu/drm/xe/tests/Makefile+1−5

 # "live" kunit tests
 obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_live_test.o
-xe_live_test-y = xe_live_test_mod.o \
+xe_live_test-y = xe_live_test_mod.o
-	xe_bo_test.o \
-	xe_dma_buf_test.o \
-	xe_migrate_test.o \
-	xe_mocs_test.o
 # Normal kunit tests
 obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_test.o

drivers/gpu/drm/xe/tests/xe_bo.c+33−12

@@ -6,7 +6,7 @@ · 5 unmodified lines
6	#include <kunit/test.h>	6	#include <kunit/test.h>
7	#include <kunit/visibility.h>	7	#include <kunit/visibility.h>
8		8
9	#include "tests/xe_bo_test.h"	9	#include "tests/xe_kunit_helpers.h"
10	#include "tests/xe_pci_test.h"	10	#include "tests/xe_pci_test.h"
11	#include "tests/xe_test.h"	11	#include "tests/xe_test.h"
12		12
@@ -154,12 +154,18 @@ out_unlock: · 141 unmodified lines
154		154
155	static int ccs_test_run_device(struct xe_device *xe)	155	static int ccs_test_run_device(struct xe_device *xe)
156	{	156	{
157	struct kunit *test = xe_cur_kunit();	157	struct kunit *test = kunit_get_current_test();
158	struct xe_tile *tile;	158	struct xe_tile *tile;
159	int id;	159	int id;
160		160
161	if (!xe_device_has_flat_ccs(xe)) {	161	if (!xe_device_has_flat_ccs(xe)) {
162	kunit_info(test, "Skipping non-flat-ccs device.\n");	162	kunit_skip(test, "non-flat-ccs device\n");
		163	return 0;
		164	}
		165
		166	/* For xe2+ dgfx, we don't handle ccs metadata */
		167	if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) {
		168	kunit_skip(test, "xe2+ dgfx device\n");
163	return 0;	169	return 0;
164	}	170	}
165		171
@@ -177,11 +183,12 @@ static int ccs_test_run_device(struct xe_device *xe) · 11 unmodified lines
177	return 0;	183	return 0;
178	}	184	}
179		185
180	void xe_ccs_migrate_kunit(struct kunit *test)	186	static void xe_ccs_migrate_kunit(struct kunit *test)
181	{	187	{
182	xe_call_for_each_device(ccs_test_run_device);	188	struct xe_device *xe = test->priv;
		189
		190	ccs_test_run_device(xe);
183	}	191	}
184	EXPORT_SYMBOL_IF_KUNIT(xe_ccs_migrate_kunit);
185		192
186	static int evict_test_run_tile(struct xe_device xe, struct xe_tile tile, struct kunit *test)	193	static int evict_test_run_tile(struct xe_device xe, struct xe_tile tile, struct kunit *test)
187	{	194	{
@@ -325,13 +332,12 @@ cleanup_bo: · 137 unmodified lines
325		332
326	static int evict_test_run_device(struct xe_device *xe)	333	static int evict_test_run_device(struct xe_device *xe)
327	{	334	{
328	struct kunit *test = xe_cur_kunit();	335	struct kunit *test = kunit_get_current_test();
329	struct xe_tile *tile;	336	struct xe_tile *tile;
330	int id;	337	int id;
331		338
332	if (!IS_DGFX(xe)) {	339	if (!IS_DGFX(xe)) {
333	kunit_info(test, "Skipping non-discrete device %s.\n",	340	kunit_skip(test, "non-discrete device\n");
334	dev_name(xe->drm.dev));
335	return 0;	341	return 0;
336	}	342	}
337		343
@@ -345,8 +351,23 @@ static int evict_test_run_device(struct xe_device *xe) · 7 unmodified lines
345	return 0;	351	return 0;
346	}	352	}
347		353
348	void xe_bo_evict_kunit(struct kunit *test)	354	static void xe_bo_evict_kunit(struct kunit *test)
349	{	355	{
350	xe_call_for_each_device(evict_test_run_device);	356	struct xe_device *xe = test->priv;
		357
		358	evict_test_run_device(xe);
351	}	359	}
352	EXPORT_SYMBOL_IF_KUNIT(xe_bo_evict_kunit);	360
		361	static struct kunit_case xe_bo_tests[] = {
		362	KUNIT_CASE_PARAM(xe_ccs_migrate_kunit, xe_pci_live_device_gen_param),
		363	KUNIT_CASE_PARAM(xe_bo_evict_kunit, xe_pci_live_device_gen_param),
		364	{}
		365	};
		366
		367	VISIBLE_IF_KUNIT
		368	struct kunit_suite xe_bo_test_suite = {
		369	.name = "xe_bo",
		370	.test_cases = xe_bo_tests,
		371	.init = xe_kunit_helper_xe_device_live_test_init,
		372	};
		373	EXPORT_SYMBOL_IF_KUNIT(xe_bo_test_suite);

drivers/gpu/drm/xe/tests/xe_bo_test.c+0−21

@@ -1,21 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright © 2022 Intel Corporation
4	*/
5
6	#include "xe_bo_test.h"
7
8	#include <kunit/test.h>
9
10	static struct kunit_case xe_bo_tests[] = {
11	KUNIT_CASE(xe_ccs_migrate_kunit),
12	KUNIT_CASE(xe_bo_evict_kunit),
13	{}
14	};
15
16	static struct kunit_suite xe_bo_test_suite = {
17	.name = "xe_bo",
18	.test_cases = xe_bo_tests,
19	};
20
21	kunit_test_suite(xe_bo_test_suite);

drivers/gpu/drm/xe/tests/xe_bo_test.h+0−14

@@ -1,14 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0 AND MIT */
2	/*
3	* Copyright © 2023 Intel Corporation
4	*/
5
6	#ifndef _XE_BO_TEST_H_
7	#define _XE_BO_TEST_H_
8
9	struct kunit;
10
11	void xe_ccs_migrate_kunit(struct kunit *test);
12	void xe_bo_evict_kunit(struct kunit *test);
13
14	#endif

drivers/gpu/drm/xe/tests/xe_dma_buf.c+20−6

@@ -8,7 +8,7 @@ · 7 unmodified lines
8	#include <kunit/test.h>	8	#include <kunit/test.h>
9	#include <kunit/visibility.h>	9	#include <kunit/visibility.h>
10		10
11	#include "tests/xe_dma_buf_test.h"	11	#include "tests/xe_kunit_helpers.h"
12	#include "tests/xe_pci_test.h"	12	#include "tests/xe_pci_test.h"
13		13
14	#include "xe_pci.h"	14	#include "xe_pci.h"
@@ -107,7 +107,7 @@ static void check_residency(struct kunit test, struct xe_bo exported, · 92 unmodified lines
107		107
108	static void xe_test_dmabuf_import_same_driver(struct xe_device *xe)	108	static void xe_test_dmabuf_import_same_driver(struct xe_device *xe)
109	{	109	{
110	struct kunit *test = xe_cur_kunit();	110	struct kunit *test = kunit_get_current_test();
111	struct dma_buf_test_params *params = to_dma_buf_test_params(test->priv);	111	struct dma_buf_test_params *params = to_dma_buf_test_params(test->priv);
112	struct drm_gem_object *import;	112	struct drm_gem_object *import;
113	struct dma_buf *dmabuf;	113	struct dma_buf *dmabuf;
@@ -258,7 +258,7 @@ static const struct dma_buf_test_params test_params[] = { · 144 unmodified lines
258	static int dma_buf_run_device(struct xe_device *xe)	258	static int dma_buf_run_device(struct xe_device *xe)
259	{	259	{
260	const struct dma_buf_test_params *params;	260	const struct dma_buf_test_params *params;
261	struct kunit *test = xe_cur_kunit();	261	struct kunit *test = kunit_get_current_test();
262		262
263	xe_pm_runtime_get(xe);	263	xe_pm_runtime_get(xe);
264	for (params = test_params; params->mem_mask; ++params) {	264	for (params = test_params; params->mem_mask; ++params) {
@@ -274,8 +274,22 @@ static int dma_buf_run_device(struct xe_device *xe) · 9 unmodified lines
274	return 0;	274	return 0;
275	}	275	}
276		276
277	void xe_dma_buf_kunit(struct kunit *test)	277	static void xe_dma_buf_kunit(struct kunit *test)
278	{	278	{
279	xe_call_for_each_device(dma_buf_run_device);	279	struct xe_device *xe = test->priv;
		280
		281	dma_buf_run_device(xe);
280	}	282	}
281	EXPORT_SYMBOL_IF_KUNIT(xe_dma_buf_kunit);	283
		284	static struct kunit_case xe_dma_buf_tests[] = {
		285	KUNIT_CASE_PARAM(xe_dma_buf_kunit, xe_pci_live_device_gen_param),
		286	{}
		287	};
		288
		289	VISIBLE_IF_KUNIT
		290	struct kunit_suite xe_dma_buf_test_suite = {
		291	.name = "xe_dma_buf",
		292	.test_cases = xe_dma_buf_tests,
		293	.init = xe_kunit_helper_xe_device_live_test_init,
		294	};
		295	EXPORT_SYMBOL_IF_KUNIT(xe_dma_buf_test_suite);

drivers/gpu/drm/xe/tests/xe_dma_buf_test.c+0−20

@@ -1,20 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright © 2022 Intel Corporation
4	*/
5
6	#include "xe_dma_buf_test.h"
7
8	#include <kunit/test.h>
9
10	static struct kunit_case xe_dma_buf_tests[] = {
11	KUNIT_CASE(xe_dma_buf_kunit),
12	{}
13	};
14
15	static struct kunit_suite xe_dma_buf_test_suite = {
16	.name = "xe_dma_buf",
17	.test_cases = xe_dma_buf_tests,
18	};
19
20	kunit_test_suite(xe_dma_buf_test_suite);

drivers/gpu/drm/xe/tests/xe_dma_buf_test.h+0−13

@@ -1,13 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0 AND MIT */
2	/*
3	* Copyright © 2023 Intel Corporation
4	*/
5
6	#ifndef _XE_DMA_BUF_TEST_H_
7	#define _XE_DMA_BUF_TEST_H_
8
9	struct kunit;
10
11	void xe_dma_buf_kunit(struct kunit *test);
12
13	#endif

drivers/gpu/drm/xe/tests/xe_kunit_helpers.c+39−0

@@ -12,7 +12,9 @@ · 11 unmodified lines
12		12
13	#include "tests/xe_kunit_helpers.h"	13	#include "tests/xe_kunit_helpers.h"
14	#include "tests/xe_pci_test.h"	14	#include "tests/xe_pci_test.h"
		15	#include "xe_device.h"
15	#include "xe_device_types.h"	16	#include "xe_device_types.h"
		17	#include "xe_pm.h"
16		18
17	/**	19	/**
18	* xe_kunit_helper_alloc_xe_device - Allocate a &xe_device for a KUnit test.	20	* xe_kunit_helper_alloc_xe_device - Allocate a &xe_device for a KUnit test.
@@ -88,3 +90,40 @@ int xe_kunit_helper_xe_device_test_init(struct kunit *test) · 69 unmodified lines
88	return 0;	90	return 0;
89	}	91	}
90	EXPORT_SYMBOL_IF_KUNIT(xe_kunit_helper_xe_device_test_init);	92	EXPORT_SYMBOL_IF_KUNIT(xe_kunit_helper_xe_device_test_init);
		93
		94	KUNIT_DEFINE_ACTION_WRAPPER(put_xe_pm_runtime, xe_pm_runtime_put, struct xe_device *);
		95
		96	/**
		97	* xe_kunit_helper_xe_device_live_test_init - Prepare a &xe_device for
		98	* use in a live KUnit test.
		99	* @test: the &kunit where live &xe_device will be used
		100	*
		101	* This function expects pointer to the &xe_device in the &test.param_value,
		102	* like it is prepared by the &xe_pci_live_device_gen_param and stores that
		103	* pointer as &kunit.priv to allow the test code to access it.
		104	*
		105	* This function makes sure that device is not wedged and then resumes it
		106	* to avoid waking up the device inside the test. It uses deferred cleanup
		107	* action to release a runtime_pm reference.
		108	*
		109	* This function can be used as custom implementation of &kunit_suite.init.
		110	*
		111	* This function uses KUNIT_ASSERT to detect any failures.
		112	*
		113	* Return: Always 0.
		114	*/
		115	int xe_kunit_helper_xe_device_live_test_init(struct kunit *test)
		116	{
		117	struct xe_device *xe = xe_device_const_cast(test->param_value);
		118
		119	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe);
		120	kunit_info(test, "running on %s device\n", xe->info.platform_name);
		121
		122	KUNIT_ASSERT_FALSE(test, xe_device_wedged(xe));
		123	xe_pm_runtime_get(xe);
		124	KUNIT_ASSERT_EQ(test, 0, kunit_add_action_or_reset(test, put_xe_pm_runtime, xe));
		125
		126	test->priv = xe;
		127	return 0;
		128	}
		129	EXPORT_SYMBOL_IF_KUNIT(xe_kunit_helper_xe_device_live_test_init);

drivers/gpu/drm/xe/tests/xe_kunit_helpers.h+2−0

@@ -14,4 +14,6 @@ struct xe_device xe_kunit_helper_alloc_xe_device(struct kunit test, · 13 unmodified lines
14	struct device *dev);	14	struct device *dev);
15	int xe_kunit_helper_xe_device_test_init(struct kunit *test);	15	int xe_kunit_helper_xe_device_test_init(struct kunit *test);
16		16
		17	int xe_kunit_helper_xe_device_live_test_init(struct kunit *test);
		18
17	#endif	19	#endif

drivers/gpu/drm/xe/tests/xe_live_test_mod.c+11−0

@@ -3,6 +3,17 @@ · 2 unmodified lines
3	* Copyright © 2023 Intel Corporation	3	* Copyright © 2023 Intel Corporation
4	*/	4	*/
5	#include <linux/module.h>	5	#include <linux/module.h>
		6	#include <kunit/test.h>
		7
		8	extern struct kunit_suite xe_bo_test_suite;
		9	extern struct kunit_suite xe_dma_buf_test_suite;
		10	extern struct kunit_suite xe_migrate_test_suite;
		11	extern struct kunit_suite xe_mocs_test_suite;
		12
		13	kunit_test_suite(xe_bo_test_suite);
		14	kunit_test_suite(xe_dma_buf_test_suite);
		15	kunit_test_suite(xe_migrate_test_suite);
		16	kunit_test_suite(xe_mocs_test_suite);
6		17
7	MODULE_AUTHOR("Intel Corporation");	18	MODULE_AUTHOR("Intel Corporation");
8	MODULE_LICENSE("GPL");	19	MODULE_LICENSE("GPL");

drivers/gpu/drm/xe/tests/xe_migrate.c+419−5

@@ -6,7 +6,7 @@ · 5 unmodified lines
6	#include <kunit/test.h>	6	#include <kunit/test.h>
7	#include <kunit/visibility.h>	7	#include <kunit/visibility.h>
8		8
9	#include "tests/xe_migrate_test.h"	9	#include "tests/xe_kunit_helpers.h"
10	#include "tests/xe_pci_test.h"	10	#include "tests/xe_pci_test.h"
11		11
12	#include "xe_pci.h"	12	#include "xe_pci.h"
@@ -334,7 +334,7 @@ vunmap: · 321 unmodified lines
334		334
335	static int migrate_test_run_device(struct xe_device *xe)	335	static int migrate_test_run_device(struct xe_device *xe)
336	{	336	{
337	struct kunit *test = xe_cur_kunit();	337	struct kunit *test = kunit_get_current_test();
338	struct xe_tile *tile;	338	struct xe_tile *tile;
339	int id;	339	int id;
340		340
@@ -354,8 +354,422 @@ static int migrate_test_run_device(struct xe_device *xe) · 13 unmodified lines
354	return 0;	354	return 0;
355	}	355	}
356		356
357	void xe_migrate_sanity_kunit(struct kunit *test)	357	static void xe_migrate_sanity_kunit(struct kunit *test)
358	{	358	{
359	xe_call_for_each_device(migrate_test_run_device);	359	struct xe_device *xe = test->priv;
		360
		361	migrate_test_run_device(xe);
		362	}
		363
		364	static struct dma_fence blt_copy(struct xe_tile tile,
		365	struct xe_bo src_bo, struct xe_bo dst_bo,
		366	bool copy_only_ccs, const char str, struct kunit test)
		367	{
		368	struct xe_gt *gt = tile->primary_gt;
		369	struct xe_migrate *m = tile->migrate;
		370	struct xe_device *xe = gt_to_xe(gt);
		371	struct dma_fence *fence = NULL;
		372	u64 size = src_bo->size;
		373	struct xe_res_cursor src_it, dst_it;
		374	struct ttm_resource src = src_bo->ttm.resource, dst = dst_bo->ttm.resource;
		375	u64 src_L0_ofs, dst_L0_ofs;
		376	u32 src_L0_pt, dst_L0_pt;
		377	u64 src_L0, dst_L0;
		378	int err;
		379	bool src_is_vram = mem_type_is_vram(src->mem_type);
		380	bool dst_is_vram = mem_type_is_vram(dst->mem_type);
		381
		382	if (!src_is_vram)
		383	xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
		384	else
		385	xe_res_first(src, 0, size, &src_it);
		386
		387	if (!dst_is_vram)
		388	xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it);
		389	else
		390	xe_res_first(dst, 0, size, &dst_it);
		391
		392	while (size) {
		393	u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */
		394	struct xe_sched_job *job;
		395	struct xe_bb *bb;
		396	u32 flush_flags = 0;
		397	u32 update_idx;
		398	u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
		399	u32 pte_flags;
		400
		401	src_L0 = xe_migrate_res_sizes(m, &src_it);
		402	dst_L0 = xe_migrate_res_sizes(m, &dst_it);
		403
		404	src_L0 = min(src_L0, dst_L0);
		405
		406	pte_flags = src_is_vram ? (PTE_UPDATE_FLAG_IS_VRAM \|
		407	PTE_UPDATE_FLAG_IS_COMP_PTE) : 0;
		408	batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0,
		409	&src_L0_ofs, &src_L0_pt, 0, 0,
		410	avail_pts);
		411
		412	pte_flags = dst_is_vram ? (PTE_UPDATE_FLAG_IS_VRAM \|
		413	PTE_UPDATE_FLAG_IS_COMP_PTE) : 0;
		414	batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0,
		415	&dst_L0_ofs, &dst_L0_pt, 0,
		416	avail_pts, avail_pts);
		417
		418	/* Add copy commands size here */
		419	batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +
		420	((xe_device_has_flat_ccs(xe) && copy_only_ccs) ? EMIT_COPY_CCS_DW : 0);
		421
		422	bb = xe_bb_new(gt, batch_size, xe->info.has_usm);
		423	if (IS_ERR(bb)) {
		424	err = PTR_ERR(bb);
		425	goto err_sync;
		426	}
		427
		428	if (src_is_vram)
		429	xe_res_next(&src_it, src_L0);
		430	else
		431	emit_pte(m, bb, src_L0_pt, src_is_vram, false,
		432	&src_it, src_L0, src);
		433
		434	if (dst_is_vram)
		435	xe_res_next(&dst_it, src_L0);
		436	else
		437	emit_pte(m, bb, dst_L0_pt, dst_is_vram, false,
		438	&dst_it, src_L0, dst);
		439
		440	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
		441	update_idx = bb->len;
		442	if (!copy_only_ccs)
		443	emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);
		444
		445	if (copy_only_ccs)
		446	flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,
		447	src_is_vram, dst_L0_ofs,
		448	dst_is_vram, src_L0, dst_L0_ofs,
		449	copy_only_ccs);
		450
		451	job = xe_bb_create_migration_job(m->q, bb,
		452	xe_migrate_batch_base(m, xe->info.has_usm),
		453	update_idx);
		454	if (IS_ERR(job)) {
		455	err = PTR_ERR(job);
		456	goto err;
		457	}
		458
		459	xe_sched_job_add_migrate_flush(job, flush_flags);
		460
		461	mutex_lock(&m->job_mutex);
		462	xe_sched_job_arm(job);
		463	dma_fence_put(fence);
		464	fence = dma_fence_get(&job->drm.s_fence->finished);
		465	xe_sched_job_push(job);
		466
		467	dma_fence_put(m->fence);
		468	m->fence = dma_fence_get(fence);
		469
		470	mutex_unlock(&m->job_mutex);
		471
		472	xe_bb_free(bb, fence);
		473	size -= src_L0;
		474	continue;
		475
		476	err:
		477	xe_bb_free(bb, NULL);
		478
		479	err_sync:
		480	if (fence) {
		481	dma_fence_wait(fence, false);
		482	dma_fence_put(fence);
		483	}
		484	return ERR_PTR(err);
		485	}
		486
		487	return fence;
		488	}
		489
		490	static void test_migrate(struct xe_device xe, struct xe_tile tile,
		491	struct xe_bo sys_bo, struct xe_bo vram_bo, struct xe_bo *ccs_bo,
		492	struct kunit *test)
		493	{
		494	struct dma_fence *fence;
		495	u64 expected, retval;
		496	long timeout;
		497	long ret;
		498
		499	expected = 0xd0d0d0d0d0d0d0d0;
		500	xe_map_memset(xe, &sys_bo->vmap, 0, 0xd0, sys_bo->size);
		501
		502	fence = blt_copy(tile, sys_bo, vram_bo, false, "Blit copy from sysmem to vram", test);
		503	if (!sanity_fence_failed(xe, fence, "Blit copy from sysmem to vram", test)) {
		504	retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
		505	if (retval == expected)
		506	KUNIT_FAIL(test, "Sanity check failed: VRAM must have compressed value\n");
		507	}
		508	dma_fence_put(fence);
		509
		510	kunit_info(test, "Evict vram buffer object\n");
		511	ret = xe_bo_evict(vram_bo, true);
		512	if (ret) {
		513	KUNIT_FAIL(test, "Failed to evict bo.\n");
		514	return;
		515	}
		516
		517	ret = xe_bo_vmap(vram_bo);
		518	if (ret) {
		519	KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
		520	return;
		521	}
		522
		523	retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
		524	check(retval, expected, "Clear evicted vram data first value", test);
		525	retval = xe_map_rd(xe, &vram_bo->vmap, vram_bo->size - 8, u64);
		526	check(retval, expected, "Clear evicted vram data last value", test);
		527
		528	fence = blt_copy(tile, vram_bo, ccs_bo,
		529	true, "Blit surf copy from vram to sysmem", test);
		530	if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) {
		531	retval = xe_map_rd(xe, &ccs_bo->vmap, 0, u64);
		532	check(retval, 0, "Clear ccs data first value", test);
		533
		534	retval = xe_map_rd(xe, &ccs_bo->vmap, ccs_bo->size - 8, u64);
		535	check(retval, 0, "Clear ccs data last value", test);
		536	}
		537	dma_fence_put(fence);
		538
		539	kunit_info(test, "Restore vram buffer object\n");
		540	ret = xe_bo_validate(vram_bo, NULL, false);
		541	if (ret) {
		542	KUNIT_FAIL(test, "Failed to validate vram bo for: %li\n", ret);
		543	return;
		544	}
		545
		546	/* Sync all migration blits */
		547	timeout = dma_resv_wait_timeout(vram_bo->ttm.base.resv,
		548	DMA_RESV_USAGE_KERNEL,
		549	true,
		550	5 * HZ);
		551	if (timeout <= 0) {
		552	KUNIT_FAIL(test, "Failed to sync bo eviction.\n");
		553	return;
		554	}
		555
		556	ret = xe_bo_vmap(vram_bo);
		557	if (ret) {
		558	KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
		559	return;
		560	}
		561
		562	retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
		563	check(retval, expected, "Restored value must be equal to initial value", test);
		564	retval = xe_map_rd(xe, &vram_bo->vmap, vram_bo->size - 8, u64);
		565	check(retval, expected, "Restored value must be equal to initial value", test);
		566
		567	fence = blt_copy(tile, vram_bo, ccs_bo,
		568	true, "Blit surf copy from vram to sysmem", test);
		569	if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) {
		570	retval = xe_map_rd(xe, &ccs_bo->vmap, 0, u64);
		571	check(retval, 0, "Clear ccs data first value", test);
		572	retval = xe_map_rd(xe, &ccs_bo->vmap, ccs_bo->size - 8, u64);
		573	check(retval, 0, "Clear ccs data last value", test);
		574	}
		575	dma_fence_put(fence);
		576	}
		577
		578	static void test_clear(struct xe_device xe, struct xe_tile tile,
		579	struct xe_bo sys_bo, struct xe_bo vram_bo, struct kunit *test)
		580	{
		581	struct dma_fence *fence;
		582	u64 expected, retval;
		583
		584	expected = 0xd0d0d0d0d0d0d0d0;
		585	xe_map_memset(xe, &sys_bo->vmap, 0, 0xd0, sys_bo->size);
		586
		587	fence = blt_copy(tile, sys_bo, vram_bo, false, "Blit copy from sysmem to vram", test);
		588	if (!sanity_fence_failed(xe, fence, "Blit copy from sysmem to vram", test)) {
		589	retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
		590	if (retval == expected)
		591	KUNIT_FAIL(test, "Sanity check failed: VRAM must have compressed value\n");
		592	}
		593	dma_fence_put(fence);
		594
		595	fence = blt_copy(tile, vram_bo, sys_bo, false, "Blit copy from vram to sysmem", test);
		596	if (!sanity_fence_failed(xe, fence, "Blit copy from vram to sysmem", test)) {
		597	retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
		598	check(retval, expected, "Decompressed value must be equal to initial value", test);
		599	retval = xe_map_rd(xe, &sys_bo->vmap, sys_bo->size - 8, u64);
		600	check(retval, expected, "Decompressed value must be equal to initial value", test);
		601	}
		602	dma_fence_put(fence);
		603
		604	kunit_info(test, "Clear vram buffer object\n");
		605	expected = 0x0000000000000000;
		606	fence = xe_migrate_clear(tile->migrate, vram_bo, vram_bo->ttm.resource);
		607	if (sanity_fence_failed(xe, fence, "Clear vram_bo", test))
		608	return;
		609	dma_fence_put(fence);
		610
		611	fence = blt_copy(tile, vram_bo, sys_bo,
		612	false, "Blit copy from vram to sysmem", test);
		613	if (!sanity_fence_failed(xe, fence, "Clear main buffer data", test)) {
		614	retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
		615	check(retval, expected, "Clear main buffer first value", test);
		616	retval = xe_map_rd(xe, &sys_bo->vmap, sys_bo->size - 8, u64);
		617	check(retval, expected, "Clear main buffer last value", test);
		618	}
		619	dma_fence_put(fence);
		620
		621	fence = blt_copy(tile, vram_bo, sys_bo,
		622	true, "Blit surf copy from vram to sysmem", test);
		623	if (!sanity_fence_failed(xe, fence, "Clear ccs buffer data", test)) {
		624	retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
		625	check(retval, expected, "Clear ccs data first value", test);
		626	retval = xe_map_rd(xe, &sys_bo->vmap, sys_bo->size - 8, u64);
		627	check(retval, expected, "Clear ccs data last value", test);
		628	}
		629	dma_fence_put(fence);
		630	}
		631
		632	static void validate_ccs_test_run_tile(struct xe_device xe, struct xe_tile tile,
		633	struct kunit *test)
		634	{
		635	struct xe_bo sys_bo, vram_bo = NULL, *ccs_bo = NULL;
		636	unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
		637	long ret;
		638
		639	sys_bo = xe_bo_create_user(xe, NULL, NULL, SZ_4M,
		640	DRM_XE_GEM_CPU_CACHING_WC, ttm_bo_type_device,
		641	XE_BO_FLAG_SYSTEM \| XE_BO_FLAG_NEEDS_CPU_ACCESS);
		642
		643	if (IS_ERR(sys_bo)) {
		644	KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
		645	PTR_ERR(sys_bo));
		646	return;
		647	}
		648
		649	xe_bo_lock(sys_bo, false);
		650	ret = xe_bo_validate(sys_bo, NULL, false);
		651	if (ret) {
		652	KUNIT_FAIL(test, "Failed to validate system bo for: %li\n", ret);
		653	goto free_sysbo;
		654	}
		655
		656	ret = xe_bo_vmap(sys_bo);
		657	if (ret) {
		658	KUNIT_FAIL(test, "Failed to vmap system bo: %li\n", ret);
		659	goto free_sysbo;
		660	}
		661	xe_bo_unlock(sys_bo);
		662
		663	ccs_bo = xe_bo_create_user(xe, NULL, NULL, SZ_4M, DRM_XE_GEM_CPU_CACHING_WC,
		664	ttm_bo_type_device, bo_flags \| XE_BO_FLAG_NEEDS_CPU_ACCESS);
		665
		666	if (IS_ERR(ccs_bo)) {
		667	KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
		668	PTR_ERR(ccs_bo));
		669	return;
		670	}
		671
		672	xe_bo_lock(ccs_bo, false);
		673	ret = xe_bo_validate(ccs_bo, NULL, false);
		674	if (ret) {
		675	KUNIT_FAIL(test, "Failed to validate system bo for: %li\n", ret);
		676	goto free_ccsbo;
		677	}
		678
		679	ret = xe_bo_vmap(ccs_bo);
		680	if (ret) {
		681	KUNIT_FAIL(test, "Failed to vmap system bo: %li\n", ret);
		682	goto free_ccsbo;
		683	}
		684	xe_bo_unlock(ccs_bo);
		685
		686	vram_bo = xe_bo_create_user(xe, NULL, NULL, SZ_4M, DRM_XE_GEM_CPU_CACHING_WC,
		687	ttm_bo_type_device, bo_flags \| XE_BO_FLAG_NEEDS_CPU_ACCESS);
		688	if (IS_ERR(vram_bo)) {
		689	KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
		690	PTR_ERR(vram_bo));
		691	return;
		692	}
		693
		694	xe_bo_lock(vram_bo, false);
		695	ret = xe_bo_validate(vram_bo, NULL, false);
		696	if (ret) {
		697	KUNIT_FAIL(test, "Failed to validate vram bo for: %li\n", ret);
		698	goto free_vrambo;
		699	}
		700
		701	ret = xe_bo_vmap(vram_bo);
		702	if (ret) {
		703	KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
		704	goto free_vrambo;
		705	}
		706
		707	test_clear(xe, tile, sys_bo, vram_bo, test);
		708	test_migrate(xe, tile, sys_bo, vram_bo, ccs_bo, test);
		709	xe_bo_unlock(vram_bo);
		710
		711	xe_bo_lock(vram_bo, false);
		712	xe_bo_vunmap(vram_bo);
		713	xe_bo_unlock(vram_bo);
		714
		715	xe_bo_lock(ccs_bo, false);
		716	xe_bo_vunmap(ccs_bo);
		717	xe_bo_unlock(ccs_bo);
		718
		719	xe_bo_lock(sys_bo, false);
		720	xe_bo_vunmap(sys_bo);
		721	xe_bo_unlock(sys_bo);
		722	free_vrambo:
		723	xe_bo_put(vram_bo);
		724	free_ccsbo:
		725	xe_bo_put(ccs_bo);
		726	free_sysbo:
		727	xe_bo_put(sys_bo);
		728	}
		729
		730	static int validate_ccs_test_run_device(struct xe_device *xe)
		731	{
		732	struct kunit *test = kunit_get_current_test();
		733	struct xe_tile *tile;
		734	int id;
		735
		736	if (!xe_device_has_flat_ccs(xe)) {
		737	kunit_skip(test, "non-flat-ccs device\n");
		738	return 0;
		739	}
		740
		741	if (!(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe))) {
		742	kunit_skip(test, "non-xe2 discrete device\n");
		743	return 0;
		744	}
		745
		746	xe_pm_runtime_get(xe);
		747
		748	for_each_tile(tile, xe, id)
		749	validate_ccs_test_run_tile(xe, tile, test);
		750
		751	xe_pm_runtime_put(xe);
		752
		753	return 0;
360	}	754	}
361	EXPORT_SYMBOL_IF_KUNIT(xe_migrate_sanity_kunit);	755
		756	static void xe_validate_ccs_kunit(struct kunit *test)
		757	{
		758	struct xe_device *xe = test->priv;
		759
		760	validate_ccs_test_run_device(xe);
		761	}
		762
		763	static struct kunit_case xe_migrate_tests[] = {
		764	KUNIT_CASE_PARAM(xe_migrate_sanity_kunit, xe_pci_live_device_gen_param),
		765	KUNIT_CASE_PARAM(xe_validate_ccs_kunit, xe_pci_live_device_gen_param),
		766	{}
		767	};
		768
		769	VISIBLE_IF_KUNIT
		770	struct kunit_suite xe_migrate_test_suite = {
		771	.name = "xe_migrate",
		772	.test_cases = xe_migrate_tests,
		773	.init = xe_kunit_helper_xe_device_live_test_init,
		774	};
		775	EXPORT_SYMBOL_IF_KUNIT(xe_migrate_test_suite);

drivers/gpu/drm/xe/tests/xe_migrate_test.c+0−20

@@ -1,20 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright © 2022 Intel Corporation
4	*/
5
6	#include "xe_migrate_test.h"
7
8	#include <kunit/test.h>
9
10	static struct kunit_case xe_migrate_tests[] = {
11	KUNIT_CASE(xe_migrate_sanity_kunit),
12	{}
13	};
14
15	static struct kunit_suite xe_migrate_test_suite = {
16	.name = "xe_migrate",
17	.test_cases = xe_migrate_tests,
18	};
19
20	kunit_test_suite(xe_migrate_test_suite);

drivers/gpu/drm/xe/tests/xe_migrate_test.h+0−13

@@ -1,13 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0 AND MIT */
2	/*
3	* Copyright © 2023 Intel Corporation
4	*/
5
6	#ifndef _XE_MIGRATE_TEST_H_
7	#define _XE_MIGRATE_TEST_H_
8
9	struct kunit;
10
11	void xe_migrate_sanity_kunit(struct kunit *test);
12
13	#endif

drivers/gpu/drm/xe/tests/xe_mocs.c+33−11

@@ -6,7 +6,7 @@ · 5 unmodified lines
6	#include <kunit/test.h>	6	#include <kunit/test.h>
7	#include <kunit/visibility.h>	7	#include <kunit/visibility.h>
8		8
9	#include "tests/xe_mocs_test.h"	9	#include "tests/xe_kunit_helpers.h"
10	#include "tests/xe_pci_test.h"	10	#include "tests/xe_pci_test.h"
11	#include "tests/xe_test.h"	11	#include "tests/xe_test.h"
12		12
@@ -23,7 +23,7 @@ struct live_mocs { · 10 unmodified lines
23	static int live_mocs_init(struct live_mocs arg, struct xe_gt gt)	23	static int live_mocs_init(struct live_mocs arg, struct xe_gt gt)
24	{	24	{
25	unsigned int flags;	25	unsigned int flags;
26	struct kunit *test = xe_cur_kunit();	26	struct kunit *test = kunit_get_current_test();
27		27
28	memset(arg, 0, sizeof(*arg));	28	memset(arg, 0, sizeof(*arg));
29		29
@@ -41,7 +41,7 @@ static int live_mocs_init(struct live_mocs arg, struct xe_gt gt) · 11 unmodified lines
41	static void read_l3cc_table(struct xe_gt *gt,	41	static void read_l3cc_table(struct xe_gt *gt,
42	const struct xe_mocs_info *info)	42	const struct xe_mocs_info *info)
43	{	43	{
44	struct kunit *test = xe_cur_kunit();	44	struct kunit *test = kunit_get_current_test();
45	u32 l3cc, l3cc_expected;	45	u32 l3cc, l3cc_expected;
46	unsigned int i;	46	unsigned int i;
47	u32 reg_val;	47	u32 reg_val;
@@ -78,7 +78,7 @@ static void read_l3cc_table(struct xe_gt *gt, · 30 unmodified lines
78	static void read_mocs_table(struct xe_gt *gt,	78	static void read_mocs_table(struct xe_gt *gt,
79	const struct xe_mocs_info *info)	79	const struct xe_mocs_info *info)
80	{	80	{
81	struct kunit *test = xe_cur_kunit();	81	struct kunit *test = kunit_get_current_test();
82	u32 mocs, mocs_expected;	82	u32 mocs, mocs_expected;
83	unsigned int i;	83	unsigned int i;
84	u32 reg_val;	84	u32 reg_val;
@@ -134,11 +134,15 @@ static int mocs_kernel_test_run_device(struct xe_device *xe) · 49 unmodified lines
134	return 0;	134	return 0;
135	}	135	}
136		136
137	void xe_live_mocs_kernel_kunit(struct kunit *test)	137	static void xe_live_mocs_kernel_kunit(struct kunit *test)
138	{	138	{
139	xe_call_for_each_device(mocs_kernel_test_run_device);	139	struct xe_device *xe = test->priv;
		140
		141	if (IS_SRIOV_VF(xe))
		142	kunit_skip(test, "this test is N/A for VF");
		143
		144	mocs_kernel_test_run_device(xe);
140	}	145	}
141	EXPORT_SYMBOL_IF_KUNIT(xe_live_mocs_kernel_kunit);
142		146
143	static int mocs_reset_test_run_device(struct xe_device *xe)	147	static int mocs_reset_test_run_device(struct xe_device *xe)
144	{	148	{
@@ -148,7 +152,7 @@ static int mocs_reset_test_run_device(struct xe_device *xe) · 3 unmodified lines
148	struct xe_gt *gt;	152	struct xe_gt *gt;
149	unsigned int flags;	153	unsigned int flags;
150	int id;	154	int id;
151	struct kunit *test = xe_cur_kunit();	155	struct kunit *test = kunit_get_current_test();
152		156
153	xe_pm_runtime_get(xe);	157	xe_pm_runtime_get(xe);
154		158
@@ -175,8 +179,26 @@ static int mocs_reset_test_run_device(struct xe_device *xe) · 20 unmodified lines
175	return 0;	179	return 0;
176	}	180	}
177		181
178	void xe_live_mocs_reset_kunit(struct kunit *test)	182	static void xe_live_mocs_reset_kunit(struct kunit *test)
179	{	183	{
180	xe_call_for_each_device(mocs_reset_test_run_device);	184	struct xe_device *xe = test->priv;
		185
		186	if (IS_SRIOV_VF(xe))
		187	kunit_skip(test, "this test is N/A for VF");
		188
		189	mocs_reset_test_run_device(xe);
181	}	190	}
182	EXPORT_SYMBOL_IF_KUNIT(xe_live_mocs_reset_kunit);	191
		192	static struct kunit_case xe_mocs_tests[] = {
		193	KUNIT_CASE_PARAM(xe_live_mocs_kernel_kunit, xe_pci_live_device_gen_param),
		194	KUNIT_CASE_PARAM(xe_live_mocs_reset_kunit, xe_pci_live_device_gen_param),
		195	{}
		196	};
		197
		198	VISIBLE_IF_KUNIT
		199	struct kunit_suite xe_mocs_test_suite = {
		200	.name = "xe_mocs",
		201	.test_cases = xe_mocs_tests,
		202	.init = xe_kunit_helper_xe_device_live_test_init,
		203	};
		204	EXPORT_SYMBOL_IF_KUNIT(xe_mocs_test_suite);

drivers/gpu/drm/xe/tests/xe_mocs_test.c+0−21

@@ -1,21 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright © 2022 Intel Corporation
4	*/
5
6	#include "xe_mocs_test.h"
7
8	#include <kunit/test.h>
9
10	static struct kunit_case xe_mocs_tests[] = {
11	KUNIT_CASE(xe_live_mocs_kernel_kunit),
12	KUNIT_CASE(xe_live_mocs_reset_kunit),
13	{}
14	};
15
16	static struct kunit_suite xe_mocs_test_suite = {
17	.name = "xe_mocs",
18	.test_cases = xe_mocs_tests,
19	};
20
21	kunit_test_suite(xe_mocs_test_suite);

drivers/gpu/drm/xe/tests/xe_mocs_test.h+0−14

@@ -1,14 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0 AND MIT */
2	/*
3	* Copyright © 2023 Intel Corporation
4	*/
5
6	#ifndef _XE_MOCS_TEST_H_
7	#define _XE_MOCS_TEST_H_
8
9	struct kunit;
10
11	void xe_live_mocs_kernel_kunit(struct kunit *test);
12	void xe_live_mocs_reset_kunit(struct kunit *test);
13
14	#endif

drivers/gpu/drm/xe/tests/xe_pci.c+30−0

@@ -167,3 +167,33 @@ done: · 166 unmodified lines
167	return 0;	167	return 0;
168	}	168	}
169	EXPORT_SYMBOL_IF_KUNIT(xe_pci_fake_device_init);	169	EXPORT_SYMBOL_IF_KUNIT(xe_pci_fake_device_init);
		170
		171	/**
		172	* xe_pci_live_device_gen_param - Helper to iterate Xe devices as KUnit parameters
		173	* @prev: the previously returned value, or NULL for the first iteration
		174	* @desc: the buffer for a parameter name
		175	*
		176	* Iterates over the available Xe devices on the system. Uses the device name
		177	* as the parameter name.
		178	*
		179	* To be used only as a parameter generator function in &KUNIT_CASE_PARAM.
		180	*
		181	* Return: pointer to the next &struct xe_device ready to be used as a parameter
		182	* or NULL if there are no more Xe devices on the system.
		183	*/
		184	const void xe_pci_live_device_gen_param(const void prev, char *desc)
		185	{
		186	const struct xe_device *xe = prev;
		187	struct device *dev = xe ? xe->drm.dev : NULL;
		188	struct device *next;
		189
		190	next = driver_find_next_device(&xe_pci_driver.driver, dev);
		191	if (dev)
		192	put_device(dev);
		193	if (!next)
		194	return NULL;
		195
		196	snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s", dev_name(next));
		197	return pdev_to_xe_device(to_pci_dev(next));
		198	}
		199	EXPORT_SYMBOL_IF_KUNIT(xe_pci_live_device_gen_param);

drivers/gpu/drm/xe/tests/xe_pci_test.c+2−2

@@ -16,7 +16,7 @@ · 15 unmodified lines
16		16
17	static void check_graphics_ip(const struct xe_graphics_desc *graphics)	17	static void check_graphics_ip(const struct xe_graphics_desc *graphics)
18	{	18	{
19	struct kunit *test = xe_cur_kunit();	19	struct kunit *test = kunit_get_current_test();
20	u64 mask = graphics->hw_engine_mask;	20	u64 mask = graphics->hw_engine_mask;
21		21
22	/* RCS, CCS, and BCS engines are allowed on the graphics IP */	22	/* RCS, CCS, and BCS engines are allowed on the graphics IP */
@@ -30,7 +30,7 @@ static void check_graphics_ip(const struct xe_graphics_desc *graphics) · 7 unmodified lines
30		30
31	static void check_media_ip(const struct xe_media_desc *media)	31	static void check_media_ip(const struct xe_media_desc *media)
32	{	32	{
33	struct kunit *test = xe_cur_kunit();	33	struct kunit *test = kunit_get_current_test();
34	u64 mask = media->hw_engine_mask;	34	u64 mask = media->hw_engine_mask;
35		35
36	/* VCS, VECS and GSCCS engines are allowed on the media IP */	36	/* VCS, VECS and GSCCS engines are allowed on the media IP */

drivers/gpu/drm/xe/tests/xe_pci_test.h+2−0

@@ -35,4 +35,6 @@ struct xe_pci_fake_data { · 34 unmodified lines
35		35
36	int xe_pci_fake_device_init(struct xe_device *xe);	36	int xe_pci_fake_device_init(struct xe_device *xe);
37		37
		38	const void xe_pci_live_device_gen_param(const void prev, char *desc);
		39
38	#endif	40	#endif

drivers/gpu/drm/xe/tests/xe_rtp_test.c+197−22

@@ -31,16 +31,23 @@ · 30 unmodified lines
31	#undef XE_REG_MCR	31	#undef XE_REG_MCR
32	#define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1)	32	#define XE_REG_MCR(...) XE_REG(__VA_ARGS__, .mcr = 1)
33		33
34	struct rtp_test_case {	34	struct rtp_to_sr_test_case {
35	const char *name;	35	const char *name;
36	struct xe_reg expected_reg;	36	struct xe_reg expected_reg;
37	u32 expected_set_bits;	37	u32 expected_set_bits;
38	u32 expected_clr_bits;	38	u32 expected_clr_bits;
39	unsigned long expected_count;	39	unsigned long expected_count_sr_entries;
40	unsigned int expected_sr_errors;	40	unsigned int expected_sr_errors;
		41	unsigned long expected_active;
41	const struct xe_rtp_entry_sr *entries;	42	const struct xe_rtp_entry_sr *entries;
42	};	43	};
43		44
		45	struct rtp_test_case {
		46	const char *name;
		47	unsigned long expected_active;
		48	const struct xe_rtp_entry *entries;
		49	};
		50
44	static bool match_yes(const struct xe_gt gt, const struct xe_hw_engine hwe)	51	static bool match_yes(const struct xe_gt gt, const struct xe_hw_engine hwe)
45	{	52	{
46	return true;	53	return true;
@@ -51,13 +58,14 @@ static bool match_no(const struct xe_gt gt, const struct xe_hw_engine hwe) · 4 unmodified lines
51	return false;	58	return false;
52	}	59	}
53		60
54	static const struct rtp_test_case cases[] = {	61	static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = {
55	{	62	{
56	.name = "coalesce-same-reg",	63	.name = "coalesce-same-reg",
57	.expected_reg = REGULAR_REG1,	64	.expected_reg = REGULAR_REG1,
58	.expected_set_bits = REG_BIT(0) \| REG_BIT(1),	65	.expected_set_bits = REG_BIT(0) \| REG_BIT(1),
59	.expected_clr_bits = REG_BIT(0) \| REG_BIT(1),	66	.expected_clr_bits = REG_BIT(0) \| REG_BIT(1),
60	.expected_count = 1,	67	.expected_active = BIT(0) \| BIT(1),
		68	.expected_count_sr_entries = 1,
61	/* Different bits on the same register: create a single entry */	69	/* Different bits on the same register: create a single entry */
62	.entries = (const struct xe_rtp_entry_sr[]) {	70	.entries = (const struct xe_rtp_entry_sr[]) {
63	{ XE_RTP_NAME("basic-1"),	71	{ XE_RTP_NAME("basic-1"),
@@ -76,7 +84,8 @@ static const struct rtp_test_case cases[] = { · 12 unmodified lines
76	.expected_reg = REGULAR_REG1,	84	.expected_reg = REGULAR_REG1,
77	.expected_set_bits = REG_BIT(0),	85	.expected_set_bits = REG_BIT(0),
78	.expected_clr_bits = REG_BIT(0),	86	.expected_clr_bits = REG_BIT(0),
79	.expected_count = 1,	87	.expected_active = BIT(0),
		88	.expected_count_sr_entries = 1,
80	/* Don't coalesce second entry since rules don't match */	89	/* Don't coalesce second entry since rules don't match */
81	.entries = (const struct xe_rtp_entry_sr[]) {	90	.entries = (const struct xe_rtp_entry_sr[]) {
82	{ XE_RTP_NAME("basic-1"),	91	{ XE_RTP_NAME("basic-1"),
@@ -95,7 +104,8 @@ static const struct rtp_test_case cases[] = { · 12 unmodified lines
95	.expected_reg = REGULAR_REG1,	104	.expected_reg = REGULAR_REG1,
96	.expected_set_bits = REG_BIT(0) \| REG_BIT(1) \| REG_BIT(2),	105	.expected_set_bits = REG_BIT(0) \| REG_BIT(1) \| REG_BIT(2),
97	.expected_clr_bits = REG_BIT(0) \| REG_BIT(1) \| REG_BIT(2),	106	.expected_clr_bits = REG_BIT(0) \| REG_BIT(1) \| REG_BIT(2),
98	.expected_count = 1,	107	.expected_active = BIT(0) \| BIT(1) \| BIT(2),
		108	.expected_count_sr_entries = 1,
99	.entries = (const struct xe_rtp_entry_sr[]) {	109	.entries = (const struct xe_rtp_entry_sr[]) {
100	{ XE_RTP_NAME("first"),	110	{ XE_RTP_NAME("first"),
101	XE_RTP_RULES(FUNC(match_yes), OR, FUNC(match_no)),	111	XE_RTP_RULES(FUNC(match_yes), OR, FUNC(match_no)),
@@ -121,7 +131,7 @@ static const struct rtp_test_case cases[] = { · 19 unmodified lines
121	{	131	{
122	.name = "match-or-xfail",	132	.name = "match-or-xfail",
123	.expected_reg = REGULAR_REG1,	133	.expected_reg = REGULAR_REG1,
124	.expected_count = 0,	134	.expected_count_sr_entries = 0,
125	.entries = (const struct xe_rtp_entry_sr[]) {	135	.entries = (const struct xe_rtp_entry_sr[]) {
126	{ XE_RTP_NAME("leading-or"),	136	{ XE_RTP_NAME("leading-or"),
127	XE_RTP_RULES(OR, FUNC(match_yes)),	137	XE_RTP_RULES(OR, FUNC(match_yes)),
@@ -148,7 +158,8 @@ static const struct rtp_test_case cases[] = { · 20 unmodified lines
148	.expected_reg = REGULAR_REG1,	158	.expected_reg = REGULAR_REG1,
149	.expected_set_bits = REG_BIT(0),	159	.expected_set_bits = REG_BIT(0),
150	.expected_clr_bits = REG_BIT(0),	160	.expected_clr_bits = REG_BIT(0),
151	.expected_count = 1,	161	.expected_active = BIT(0),
		162	.expected_count_sr_entries = 1,
152	/* Don't coalesce second entry due to one of the rules */	163	/* Don't coalesce second entry due to one of the rules */
153	.entries = (const struct xe_rtp_entry_sr[]) {	164	.entries = (const struct xe_rtp_entry_sr[]) {
154	{ XE_RTP_NAME("basic-1"),	165	{ XE_RTP_NAME("basic-1"),
@@ -167,7 +178,8 @@ static const struct rtp_test_case cases[] = { · 12 unmodified lines
167	.expected_reg = REGULAR_REG1,	178	.expected_reg = REGULAR_REG1,
168	.expected_set_bits = REG_BIT(0),	179	.expected_set_bits = REG_BIT(0),
169	.expected_clr_bits = REG_BIT(0),	180	.expected_clr_bits = REG_BIT(0),
170	.expected_count = 2,	181	.expected_active = BIT(0) \| BIT(1),
		182	.expected_count_sr_entries = 2,
171	/* Same bits on different registers are not coalesced */	183	/* Same bits on different registers are not coalesced */
172	.entries = (const struct xe_rtp_entry_sr[]) {	184	.entries = (const struct xe_rtp_entry_sr[]) {
173	{ XE_RTP_NAME("basic-1"),	185	{ XE_RTP_NAME("basic-1"),
@@ -186,7 +198,8 @@ static const struct rtp_test_case cases[] = { · 12 unmodified lines
186	.expected_reg = REGULAR_REG1,	198	.expected_reg = REGULAR_REG1,
187	.expected_set_bits = REG_BIT(0),	199	.expected_set_bits = REG_BIT(0),
188	.expected_clr_bits = REG_BIT(1) \| REG_BIT(0),	200	.expected_clr_bits = REG_BIT(1) \| REG_BIT(0),
189	.expected_count = 1,	201	.expected_active = BIT(0) \| BIT(1),
		202	.expected_count_sr_entries = 1,
190	/* Check clr vs set actions on different bits */	203	/* Check clr vs set actions on different bits */
191	.entries = (const struct xe_rtp_entry_sr[]) {	204	.entries = (const struct xe_rtp_entry_sr[]) {
192	{ XE_RTP_NAME("basic-1"),	205	{ XE_RTP_NAME("basic-1"),
@@ -207,7 +220,8 @@ static const struct rtp_test_case cases[] = { · 14 unmodified lines
207	.expected_reg = REGULAR_REG1,	220	.expected_reg = REGULAR_REG1,
208	.expected_set_bits = TEMP_FIELD,	221	.expected_set_bits = TEMP_FIELD,
209	.expected_clr_bits = TEMP_MASK,	222	.expected_clr_bits = TEMP_MASK,
210	.expected_count = 1,	223	.expected_active = BIT(0),
		224	.expected_count_sr_entries = 1,
211	/* Check FIELD_SET works */	225	/* Check FIELD_SET works */
212	.entries = (const struct xe_rtp_entry_sr[]) {	226	.entries = (const struct xe_rtp_entry_sr[]) {
213	{ XE_RTP_NAME("basic-1"),	227	{ XE_RTP_NAME("basic-1"),
@@ -225,7 +239,8 @@ static const struct rtp_test_case cases[] = { · 11 unmodified lines
225	.expected_reg = REGULAR_REG1,	239	.expected_reg = REGULAR_REG1,
226	.expected_set_bits = REG_BIT(0),	240	.expected_set_bits = REG_BIT(0),
227	.expected_clr_bits = REG_BIT(0),	241	.expected_clr_bits = REG_BIT(0),
228	.expected_count = 1,	242	.expected_active = BIT(0) \| BIT(1),
		243	.expected_count_sr_entries = 1,
229	.expected_sr_errors = 1,	244	.expected_sr_errors = 1,
230	.entries = (const struct xe_rtp_entry_sr[]) {	245	.entries = (const struct xe_rtp_entry_sr[]) {
231	{ XE_RTP_NAME("basic-1"),	246	{ XE_RTP_NAME("basic-1"),
@@ -245,7 +260,8 @@ static const struct rtp_test_case cases[] = { · 13 unmodified lines
245	.expected_reg = REGULAR_REG1,	260	.expected_reg = REGULAR_REG1,
246	.expected_set_bits = REG_BIT(0),	261	.expected_set_bits = REG_BIT(0),
247	.expected_clr_bits = REG_BIT(0),	262	.expected_clr_bits = REG_BIT(0),
248	.expected_count = 1,	263	.expected_active = BIT(0) \| BIT(1),
		264	.expected_count_sr_entries = 1,
249	.expected_sr_errors = 1,	265	.expected_sr_errors = 1,
250	.entries = (const struct xe_rtp_entry_sr[]) {	266	.entries = (const struct xe_rtp_entry_sr[]) {
251	{ XE_RTP_NAME("basic-1"),	267	{ XE_RTP_NAME("basic-1"),
@@ -265,7 +281,8 @@ static const struct rtp_test_case cases[] = { · 13 unmodified lines
265	.expected_reg = REGULAR_REG1,	281	.expected_reg = REGULAR_REG1,
266	.expected_set_bits = REG_BIT(0),	282	.expected_set_bits = REG_BIT(0),
267	.expected_clr_bits = REG_BIT(0),	283	.expected_clr_bits = REG_BIT(0),
268	.expected_count = 1,	284	.expected_active = BIT(0) \| BIT(1) \| BIT(2),
		285	.expected_count_sr_entries = 1,
269	.expected_sr_errors = 2,	286	.expected_sr_errors = 2,
270	.entries = (const struct xe_rtp_entry_sr[]) {	287	.entries = (const struct xe_rtp_entry_sr[]) {
271	{ XE_RTP_NAME("basic-1"),	288	{ XE_RTP_NAME("basic-1"),
@@ -287,28 +304,35 @@ static const struct rtp_test_case cases[] = { · 15 unmodified lines
287	},	304	},
288	};	305	};
289		306
290	static void xe_rtp_process_tests(struct kunit *test)	307	static void xe_rtp_process_to_sr_tests(struct kunit *test)
291	{	308	{
292	const struct rtp_test_case *param = test->param_value;	309	const struct rtp_to_sr_test_case *param = test->param_value;
293	struct xe_device *xe = test->priv;	310	struct xe_device *xe = test->priv;
294	struct xe_gt *gt = xe_device_get_root_tile(xe)->primary_gt;	311	struct xe_gt *gt = xe_device_get_root_tile(xe)->primary_gt;
295	struct xe_reg_sr *reg_sr = &gt->reg_sr;	312	struct xe_reg_sr *reg_sr = &gt->reg_sr;
296	const struct xe_reg_sr_entry sre, sr_entry = NULL;	313	const struct xe_reg_sr_entry sre, sr_entry = NULL;
297	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt);	314	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt);
298	unsigned long idx, count = 0;	315	unsigned long idx, count_sr_entries = 0, count_rtp_entries = 0, active = 0;
		316
		317	xe_reg_sr_init(reg_sr, "xe_rtp_to_sr_tests", xe);
		318
		319	while (param->entries[count_rtp_entries].rules)
		320	count_rtp_entries++;
299		321
300	xe_reg_sr_init(reg_sr, "xe_rtp_tests", xe);	322	xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, count_rtp_entries);
301	xe_rtp_process_to_sr(&ctx, param->entries, reg_sr);	323	xe_rtp_process_to_sr(&ctx, param->entries, reg_sr);
302		324
303	xa_for_each(&reg_sr->xa, idx, sre) {	325	xa_for_each(&reg_sr->xa, idx, sre) {
304	if (idx == param->expected_reg.addr)	326	if (idx == param->expected_reg.addr)
305	sr_entry = sre;	327	sr_entry = sre;
306		328
307	count++;	329	count_sr_entries++;
308	}	330	}
309		331
310	KUNIT_EXPECT_EQ(test, count, param->expected_count);	332	KUNIT_EXPECT_EQ(test, active, param->expected_active);
311	if (count) {	333
		334	KUNIT_EXPECT_EQ(test, count_sr_entries, param->expected_count_sr_entries);
		335	if (count_sr_entries) {
312	KUNIT_EXPECT_EQ(test, sr_entry->clr_bits, param->expected_clr_bits);	336	KUNIT_EXPECT_EQ(test, sr_entry->clr_bits, param->expected_clr_bits);
313	KUNIT_EXPECT_EQ(test, sr_entry->set_bits, param->expected_set_bits);	337	KUNIT_EXPECT_EQ(test, sr_entry->set_bits, param->expected_set_bits);
314	KUNIT_EXPECT_EQ(test, sr_entry->reg.raw, param->expected_reg.raw);	338	KUNIT_EXPECT_EQ(test, sr_entry->reg.raw, param->expected_reg.raw);
@@ -319,12 +343,162 @@ static void xe_rtp_process_tests(struct kunit *test) · 4 unmodified lines
319	KUNIT_EXPECT_EQ(test, reg_sr->errors, param->expected_sr_errors);	343	KUNIT_EXPECT_EQ(test, reg_sr->errors, param->expected_sr_errors);
320	}	344	}
321		345
		346	/*
		347	* Entries below follow the logic used with xe_wa_oob.rules:
		348	* 1) Entries with empty name are OR'ed: all entries marked active since the
		349	* last entry with a name
		350	* 2) There are no action associated with rules
		351	*/
		352	static const struct rtp_test_case rtp_cases[] = {
		353	{
		354	.name = "active1",
		355	.expected_active = BIT(0),
		356	.entries = (const struct xe_rtp_entry[]) {
		357	{ XE_RTP_NAME("r1"),
		358	XE_RTP_RULES(FUNC(match_yes)),
		359	},
		360	{}
		361	},
		362	},
		363	{
		364	.name = "active2",
		365	.expected_active = BIT(0) \| BIT(1),
		366	.entries = (const struct xe_rtp_entry[]) {
		367	{ XE_RTP_NAME("r1"),
		368	XE_RTP_RULES(FUNC(match_yes)),
		369	},
		370	{ XE_RTP_NAME("r2"),
		371	XE_RTP_RULES(FUNC(match_yes)),
		372	},
		373	{}
		374	},
		375	},
		376	{
		377	.name = "active-inactive",
		378	.expected_active = BIT(0),
		379	.entries = (const struct xe_rtp_entry[]) {
		380	{ XE_RTP_NAME("r1"),
		381	XE_RTP_RULES(FUNC(match_yes)),
		382	},
		383	{ XE_RTP_NAME("r2"),
		384	XE_RTP_RULES(FUNC(match_no)),
		385	},
		386	{}
		387	},
		388	},
		389	{
		390	.name = "inactive-active",
		391	.expected_active = BIT(1),
		392	.entries = (const struct xe_rtp_entry[]) {
		393	{ XE_RTP_NAME("r1"),
		394	XE_RTP_RULES(FUNC(match_no)),
		395	},
		396	{ XE_RTP_NAME("r2"),
		397	XE_RTP_RULES(FUNC(match_yes)),
		398	},
		399	{}
		400	},
		401	},
		402	{
		403	.name = "inactive-1st_or_active-inactive",
		404	.expected_active = BIT(1),
		405	.entries = (const struct xe_rtp_entry[]) {
		406	{ XE_RTP_NAME("r1"),
		407	XE_RTP_RULES(FUNC(match_no)),
		408	},
		409	{ XE_RTP_NAME("r2_or_conditions"),
		410	XE_RTP_RULES(FUNC(match_yes), OR,
		411	FUNC(match_no), OR,
		412	FUNC(match_no)) },
		413	{ XE_RTP_NAME("r3"),
		414	XE_RTP_RULES(FUNC(match_no)),
		415	},
		416	{}
		417	},
		418	},
		419	{
		420	.name = "inactive-2nd_or_active-inactive",
		421	.expected_active = BIT(1),
		422	.entries = (const struct xe_rtp_entry[]) {
		423	{ XE_RTP_NAME("r1"),
		424	XE_RTP_RULES(FUNC(match_no)),
		425	},
		426	{ XE_RTP_NAME("r2_or_conditions"),
		427	XE_RTP_RULES(FUNC(match_no), OR,
		428	FUNC(match_yes), OR,
		429	FUNC(match_no)) },
		430	{ XE_RTP_NAME("r3"),
		431	XE_RTP_RULES(FUNC(match_no)),
		432	},
		433	{}
		434	},
		435	},
		436	{
		437	.name = "inactive-last_or_active-inactive",
		438	.expected_active = BIT(1),
		439	.entries = (const struct xe_rtp_entry[]) {
		440	{ XE_RTP_NAME("r1"),
		441	XE_RTP_RULES(FUNC(match_no)),
		442	},
		443	{ XE_RTP_NAME("r2_or_conditions"),
		444	XE_RTP_RULES(FUNC(match_no), OR,
		445	FUNC(match_no), OR,
		446	FUNC(match_yes)) },
		447	{ XE_RTP_NAME("r3"),
		448	XE_RTP_RULES(FUNC(match_no)),
		449	},
		450	{}
		451	},
		452	},
		453	{
		454	.name = "inactive-no_or_active-inactive",
		455	.expected_active = 0,
		456	.entries = (const struct xe_rtp_entry[]) {
		457	{ XE_RTP_NAME("r1"),
		458	XE_RTP_RULES(FUNC(match_no)),
		459	},
		460	{ XE_RTP_NAME("r2_or_conditions"),
		461	XE_RTP_RULES(FUNC(match_no), OR,
		462	FUNC(match_no), OR,
		463	FUNC(match_no)) },
		464	{ XE_RTP_NAME("r3"),
		465	XE_RTP_RULES(FUNC(match_no)),
		466	},
		467	{}
		468	},
		469	},
		470	};
		471
		472	static void xe_rtp_process_tests(struct kunit *test)
		473	{
		474	const struct rtp_test_case *param = test->param_value;
		475	struct xe_device *xe = test->priv;
		476	struct xe_gt *gt = xe_device_get_root_tile(xe)->primary_gt;
		477	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(gt);
		478	unsigned long count_rtp_entries = 0, active = 0;
		479
		480	while (param->entries[count_rtp_entries].rules)
		481	count_rtp_entries++;
		482
		483	xe_rtp_process_ctx_enable_active_tracking(&ctx, &active, count_rtp_entries);
		484	xe_rtp_process(&ctx, param->entries);
		485
		486	KUNIT_EXPECT_EQ(test, active, param->expected_active);
		487	}
		488
		489	static void rtp_to_sr_desc(const struct rtp_to_sr_test_case t, char desc)
		490	{
		491	strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE);
		492	}
		493
		494	KUNIT_ARRAY_PARAM(rtp_to_sr, rtp_to_sr_cases, rtp_to_sr_desc);
		495
322	static void rtp_desc(const struct rtp_test_case t, char desc)	496	static void rtp_desc(const struct rtp_test_case t, char desc)
323	{	497	{
324	strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE);	498	strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE);
325	}	499	}
326		500
327	KUNIT_ARRAY_PARAM(rtp, cases, rtp_desc);	501	KUNIT_ARRAY_PARAM(rtp, rtp_cases, rtp_desc);
328		502
329	static int xe_rtp_test_init(struct kunit *test)	503	static int xe_rtp_test_init(struct kunit *test)
330	{	504	{
@@ -357,6 +531,7 @@ static void xe_rtp_test_exit(struct kunit *test) · 26 unmodified lines
357	}	531	}
358		532
359	static struct kunit_case xe_rtp_tests[] = {	533	static struct kunit_case xe_rtp_tests[] = {
		534	KUNIT_CASE_PARAM(xe_rtp_process_to_sr_tests, rtp_to_sr_gen_params),
360	KUNIT_CASE_PARAM(xe_rtp_process_tests, rtp_gen_params),	535	KUNIT_CASE_PARAM(xe_rtp_process_tests, rtp_gen_params),
361	{}	536	{}
362	};	537	};

drivers/gpu/drm/xe/tests/xe_test.h+3−7

 #include <linux/types.h>
 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
-#include <linux/sched.h>
 #include <kunit/test.h>
+#include <kunit/test-bug.h>
 /*
  * Each test that provides a kunit private test structure, place a test id
 #define XE_TEST_DECLARE(x) x
 #define XE_TEST_ONLY(x) unlikely(x)
-#define XE_TEST_EXPORT
-#define xe_cur_kunit() current->kunit_test
 /**
  * xe_cur_kunit_priv - Obtain the struct xe_test_priv pointed to by
 {
 	struct xe_test_priv *priv;
-	if (!xe_cur_kunit())
+	if (!kunit_get_current_test())
 		return NULL;
-	priv = xe_cur_kunit()->priv;
+	priv = kunit_get_current_test()->priv;
 	return priv->id == id ? priv : NULL;
 }
 #define XE_TEST_DECLARE(x)
 #define XE_TEST_ONLY(x) 0
-#define XE_TEST_EXPORT static
-#define xe_cur_kunit() NULL
 #define xe_cur_kunit_priv(_id) NULL
 #endif

drivers/gpu/drm/xe/tests/xe_wa_test.c+1−0

@@ -74,6 +74,7 @@ static const struct platform_test_case cases[] = { · 73 unmodified lines
74	GMDID_CASE(METEORLAKE, 1274, A0, 1300, A0),	74	GMDID_CASE(METEORLAKE, 1274, A0, 1300, A0),
75	GMDID_CASE(LUNARLAKE, 2004, A0, 2000, A0),	75	GMDID_CASE(LUNARLAKE, 2004, A0, 2000, A0),
76	GMDID_CASE(LUNARLAKE, 2004, B0, 2000, A0),	76	GMDID_CASE(LUNARLAKE, 2004, B0, 2000, A0),
		77	GMDID_CASE(BATTLEMAGE, 2001, A0, 1301, A1),
77	};	78	};
78		79
79	static void platform_desc(const struct platform_test_case t, char desc)	80	static void platform_desc(const struct platform_test_case t, char desc)

drivers/gpu/drm/xe/xe_bo.c+6−5

 	if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) &&
 	    !(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) &&
 	    ((xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) ||
-	     (flags & XE_BO_NEEDS_64K))) {
+	     (flags & (XE_BO_FLAG_NEEDS_64K | XE_BO_FLAG_NEEDS_2M)))) {
-		aligned_size = ALIGN(size, SZ_64K);
+		size_t align = flags & XE_BO_FLAG_NEEDS_2M ? SZ_2M : SZ_64K;
+		aligned_size = ALIGN(size, align);
 		if (type != ttm_bo_type_device)
-			size = ALIGN(size, SZ_64K);
+			size = ALIGN(size, align);
 		flags |= XE_BO_FLAG_INTERNAL_64K;
-		alignment = SZ_64K >> PAGE_SHIFT;
+		alignment = align >> PAGE_SHIFT;
 	} else {
 		aligned_size = ALIGN(size, SZ_4K);
 		flags &= ~XE_BO_FLAG_INTERNAL_64K;

drivers/gpu/drm/xe/xe_bo.h+3−2

 #define XE_BO_FLAG_PAGETABLE		BIT(12)
 #define XE_BO_FLAG_NEEDS_CPU_ACCESS	BIT(13)
 #define XE_BO_FLAG_NEEDS_UC		BIT(14)
-#define XE_BO_NEEDS_64K			BIT(15)
+#define XE_BO_FLAG_NEEDS_64K		BIT(15)
-#define XE_BO_FLAG_GGTT_INVALIDATE	BIT(16)
+#define XE_BO_FLAG_NEEDS_2M		BIT(16)
+#define XE_BO_FLAG_GGTT_INVALIDATE	BIT(17)
 /* this one is trigger internally only */
 #define XE_BO_FLAG_INTERNAL_TEST	BIT(30)
 #define XE_BO_FLAG_INTERNAL_64K		BIT(31)

drivers/gpu/drm/xe/xe_bo_types.h+2−0

@@ -58,6 +58,8 @@ struct xe_bo { · 57 unmodified lines
58	#endif	58	#endif
59	/** @freed: List node for delayed put. */	59	/** @freed: List node for delayed put. */
60	struct llist_node freed;	60	struct llist_node freed;
		61	/** @update_index: Update index if PT BO */
		62	int update_index;
61	/** @created: Whether the bo has passed initial creation */	63	/** @created: Whether the bo has passed initial creation */
62	bool created;	64	bool created;
63		65

drivers/gpu/drm/xe/xe_devcoredump.c+2−8

 	u32 adj_logical_mask = q->logical_mask;
 	u32 width_mask = (0x1 << q->width) - 1;
 	const char *process_name = "no process";
-	struct task_struct *task = NULL;
 	int i;
 	bool cookie;
 	ss->snapshot_time = ktime_get_real();
 	ss->boot_time = ktime_get_boottime();
-	if (q->vm && q->vm->xef) {
+	if (q->vm && q->vm->xef)
-		task = get_pid_task(q->vm->xef->drm->pid, PIDTYPE_PID);
+		process_name = q->vm->xef->process_name;
-		if (task)
-			process_name = task->comm;
-	}
 	strscpy(ss->process_name, process_name);
-	if (task)
-		put_task_struct(task);
 	ss->gt = q->gt;
 	INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work);

drivers/gpu/drm/xe/xe_device.c+101−10

@@ -54,6 +54,9 @@ · 53 unmodified lines
54	#include "xe_vm.h"	54	#include "xe_vm.h"
55	#include "xe_vram.h"	55	#include "xe_vram.h"
56	#include "xe_wait_user_fence.h"	56	#include "xe_wait_user_fence.h"
		57	#include "xe_wa.h"
		58
		59	#include <generated/xe_wa_oob.h>
57		60
58	static int xe_file_open(struct drm_device dev, struct drm_file file)	61	static int xe_file_open(struct drm_device dev, struct drm_file file)
59	{	62	{
@@ -61,6 +64,7 @@ static int xe_file_open(struct drm_device dev, struct drm_file file) · 1 unmodified line
61	struct xe_drm_client *client;	64	struct xe_drm_client *client;
62	struct xe_file *xef;	65	struct xe_file *xef;
63	int ret = -ENOMEM;	66	int ret = -ENOMEM;
		67	struct task_struct *task = NULL;
64		68
65	xef = kzalloc(sizeof(*xef), GFP_KERNEL);	69	xef = kzalloc(sizeof(*xef), GFP_KERNEL);
66	if (!xef)	70	if (!xef)
@@ -87,9 +91,63 @@ static int xe_file_open(struct drm_device dev, struct drm_file file) · 20 unmodified lines
87	spin_unlock(&xe->clients.lock);	91	spin_unlock(&xe->clients.lock);
88		92
89	file->driver_priv = xef;	93	file->driver_priv = xef;
		94	kref_init(&xef->refcount);
		95
		96	task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
		97	if (task) {
		98	xef->process_name = kstrdup(task->comm, GFP_KERNEL);
		99	xef->pid = task->pid;
		100	put_task_struct(task);
		101	}
		102
90	return 0;	103	return 0;
91	}	104	}
92		105
		106	static void xe_file_destroy(struct kref *ref)
		107	{
		108	struct xe_file *xef = container_of(ref, struct xe_file, refcount);
		109	struct xe_device *xe = xef->xe;
		110
		111	xa_destroy(&xef->exec_queue.xa);
		112	mutex_destroy(&xef->exec_queue.lock);
		113	xa_destroy(&xef->vm.xa);
		114	mutex_destroy(&xef->vm.lock);
		115
		116	spin_lock(&xe->clients.lock);
		117	xe->clients.count--;
		118	spin_unlock(&xe->clients.lock);
		119
		120	xe_drm_client_put(xef->client);
		121	kfree(xef->process_name);
		122	kfree(xef);
		123	}
		124
		125	/**
		126	* xe_file_get() - Take a reference to the xe file object
		127	* @xef: Pointer to the xe file
		128	*
		129	* Anyone with a pointer to xef must take a reference to the xe file
		130	* object using this call.
		131	*
		132	* Return: xe file pointer
		133	*/
		134	struct xe_file xe_file_get(struct xe_file xef)
		135	{
		136	kref_get(&xef->refcount);
		137	return xef;
		138	}
		139
		140	/**
		141	* xe_file_put() - Drop a reference to the xe file object
		142	* @xef: Pointer to the xe file
		143	*
		144	* Used to drop reference to the xef object
		145	*/
		146	void xe_file_put(struct xe_file *xef)
		147	{
		148	kref_put(&xef->refcount, xe_file_destroy);
		149	}
		150
93	static void xe_file_close(struct drm_device dev, struct drm_file file)	151	static void xe_file_close(struct drm_device dev, struct drm_file file)
94	{	152	{
95	struct xe_device *xe = to_xe_device(dev);	153	struct xe_device *xe = to_xe_device(dev);
@@ -98,6 +156,8 @@ static void xe_file_close(struct drm_device dev, struct drm_file file) · 2 unmodified lines
98	struct xe_exec_queue *q;	156	struct xe_exec_queue *q;
99	unsigned long idx;	157	unsigned long idx;
100		158
		159	xe_pm_runtime_get(xe);
		160
101	/*	161	/*
102	* No need for exec_queue.lock here as there is no contention for it	162	* No need for exec_queue.lock here as there is no contention for it
103	* when FD is closing as IOCTLs presumably can't be modifying the	163	* when FD is closing as IOCTLs presumably can't be modifying the
@@ -108,21 +168,14 @@ static void xe_file_close(struct drm_device dev, struct drm_file file) · 4 unmodified lines
108	xe_exec_queue_kill(q);	168	xe_exec_queue_kill(q);
109	xe_exec_queue_put(q);	169	xe_exec_queue_put(q);
110	}	170	}
111	xa_destroy(&xef->exec_queue.xa);
112	mutex_destroy(&xef->exec_queue.lock);
113	mutex_lock(&xef->vm.lock);	171	mutex_lock(&xef->vm.lock);
114	xa_for_each(&xef->vm.xa, idx, vm)	172	xa_for_each(&xef->vm.xa, idx, vm)
115	xe_vm_close_and_put(vm);	173	xe_vm_close_and_put(vm);
116	mutex_unlock(&xef->vm.lock);	174	mutex_unlock(&xef->vm.lock);
117	xa_destroy(&xef->vm.xa);
118	mutex_destroy(&xef->vm.lock);
119		175
120	spin_lock(&xe->clients.lock);	176	xe_file_put(xef);
121	xe->clients.count--;
122	spin_unlock(&xe->clients.lock);
123		177
124	xe_drm_client_put(xef->client);	178	xe_pm_runtime_put(xe);
125	kfree(xef);
126	}	179	}
127		180
128	static const struct drm_ioctl_desc xe_ioctls[] = {	181	static const struct drm_ioctl_desc xe_ioctls[] = {
@@ -744,13 +797,22 @@ void xe_device_shutdown(struct xe_device *xe) · 615 unmodified lines
744	{	797	{
745	}	798	}
746		799
		800	/**
		801	* xe_device_wmb() - Device specific write memory barrier
		802	* @xe: the &xe_device
		803	*
		804	* While wmb() is sufficient for a barrier if we use system memory, on discrete
		805	* platforms with device memory we additionally need to issue a register write.
		806	* Since it doesn't matter which register we write to, use the read-only VF_CAP
		807	* register that is also marked as accessible by the VFs.
		808	*/
747	void xe_device_wmb(struct xe_device *xe)	809	void xe_device_wmb(struct xe_device *xe)
748	{	810	{
749	struct xe_gt *gt = xe_root_mmio_gt(xe);	811	struct xe_gt *gt = xe_root_mmio_gt(xe);
750		812
751	wmb();	813	wmb();
752	if (IS_DGFX(xe))	814	if (IS_DGFX(xe))
753	xe_mmio_write32(gt, SOFTWARE_FLAGS_SPR33, 0);	815	xe_mmio_write32(gt, VF_CAP_REG, 0);
754	}	816	}
755		817
756	/**	818	/**
@@ -779,6 +841,11 @@ void xe_device_td_flush(struct xe_device *xe) · 22 unmodified lines
779	if (!IS_DGFX(xe) \|\| GRAPHICS_VER(xe) < 20)	841	if (!IS_DGFX(xe) \|\| GRAPHICS_VER(xe) < 20)
780	return;	842	return;
781		843
		844	if (XE_WA(xe_root_mmio_gt(xe), 16023588340)) {
		845	xe_device_l2_flush(xe);
		846	return;
		847	}
		848
782	for_each_gt(gt, xe, id) {	849	for_each_gt(gt, xe, id) {
783	if (xe_gt_is_media_type(gt))	850	if (xe_gt_is_media_type(gt))
784	continue;	851	continue;
@@ -802,6 +869,30 @@ void xe_device_td_flush(struct xe_device *xe) · 17 unmodified lines
802	}	869	}
803	}	870	}
804		871
		872	void xe_device_l2_flush(struct xe_device *xe)
		873	{
		874	struct xe_gt *gt;
		875	int err;
		876
		877	gt = xe_root_mmio_gt(xe);
		878
		879	if (!XE_WA(gt, 16023588340))
		880	return;
		881
		882	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
		883	if (err)
		884	return;
		885
		886	spin_lock(&gt->global_invl_lock);
		887	xe_mmio_write32(gt, XE2_GLOBAL_INVAL, 0x1);
		888
		889	if (xe_mmio_wait32(gt, XE2_GLOBAL_INVAL, 0x1, 0x0, 150, NULL, true))
		890	xe_gt_err_once(gt, "Global invalidation timeout\n");
		891	spin_unlock(&gt->global_invl_lock);
		892
		893	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
		894	}
		895
805	u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)	896	u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
806	{	897	{
807	return xe_device_has_flat_ccs(xe) ?	898	return xe_device_has_flat_ccs(xe) ?

drivers/gpu/drm/xe/xe_device.h+9−0

 	return pci_get_drvdata(pdev);
 }
+static inline struct xe_device *xe_device_const_cast(const struct xe_device *xe)
+{
+	return (struct xe_device *)xe;
+}
 static inline struct xe_device *ttm_to_xe_device(struct ttm_device *ttm)
 {
 	return container_of(ttm, struct xe_device, ttm);
 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address);
 void xe_device_td_flush(struct xe_device *xe);
+void xe_device_l2_flush(struct xe_device *xe);
 static inline bool xe_device_wedged(struct xe_device *xe)
 {
 void xe_device_declare_wedged(struct xe_device *xe);
+struct xe_file *xe_file_get(struct xe_file *xef);
+void xe_file_put(struct xe_file *xef);
 #endif

drivers/gpu/drm/xe/xe_device_types.h+30−0

 #include "xe_sriov_types.h"
 #include "xe_step_types.h"
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
+#define TEST_VM_OPS_ERROR
+#endif
 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
 #include "soc/intel_pch.h"
 #include "intel_display_core.h"
 #define MEDIA_VERx100(xe) ((xe)->info.media_verx100)
 #define IS_DGFX(xe) ((xe)->info.is_dgfx)
 #define HAS_HECI_GSCFI(xe) ((xe)->info.has_heci_gscfi)
+#define HAS_HECI_CSCFI(xe) ((xe)->info.has_heci_cscfi)
 #define XE_VRAM_FLAGS_NEED64K		BIT(0)
 		u8 skip_pcode:1;
 		/** @info.has_heci_gscfi: device has heci gscfi */
 		u8 has_heci_gscfi:1;
+		/** @info.has_heci_cscfi: device has heci cscfi */
+		u8 has_heci_cscfi:1;
 		/** @info.skip_guc_pc: Skip GuC based PM feature init */
 		u8 skip_guc_pc:1;
 		/** @info.has_atomic_enable_pte_bit: Device has atomic enable PTE bit */
 		int mode;
 	} wedged;
+#ifdef TEST_VM_OPS_ERROR
+	/**
+	 * @vm_inject_error_position: inject errors at different places in VM
+	 * bind IOCTL based on this value
+	 */
+	u8 vm_inject_error_position;
+#endif
 	/* private: */
 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
 	/** @client: drm client */
 	struct xe_drm_client *client;
+	/**
+	 * @process_name: process name for file handle, used to safely output
+	 * during error situations where xe file can outlive process
+	 */
+	char *process_name;
+	/**
+	 * @pid: pid for file handle, used to safely output uring error
+	 * situations where xe file can outlive process
+	 */
+	pid_t pid;
+	/** @refcount: ref count of this xe file */
+	struct kref refcount;
 };
 #endif

drivers/gpu/drm/xe/xe_drm_client.c+1−4

 	/* Accumulate all the exec queues from this client */
 	mutex_lock(&xef->exec_queue.lock);
-	xa_for_each(&xef->exec_queue.xa, i, q) {
+	xa_for_each(&xef->exec_queue.xa, i, q)
 		xe_exec_queue_update_run_ticks(q);
-		xef->run_ticks[q->class] += q->run_ticks - q->old_run_ticks;
-		q->old_run_ticks = q->run_ticks;
-	}
 	mutex_unlock(&xef->exec_queue.lock);
 	/* Get the total GPU cycles */

drivers/gpu/drm/xe/xe_exec_queue.c+32−1

 {
 	if (q->vm)
 		xe_vm_put(q->vm);
+	if (q->xef)
+		xe_file_put(q->xef);
 	kfree(q);
 }
 		goto kill_exec_queue;
 	args->exec_queue_id = id;
+	q->xef = xe_file_get(xef);
 	return 0;
  */
 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q)
 {
+	struct xe_file *xef;
 	struct xe_lrc *lrc;
 	u32 old_ts, new_ts;
 	if (!q->vm || !q->vm->xef)
 		return;
+	xef = q->vm->xef;
 	/*
 	 * Only sample the first LRC. For parallel submission, all of them are
 	 * scheduled together and we compensate that below by multiplying by
 	 */
 	lrc = q->lrc[0];
 	new_ts = xe_lrc_update_timestamp(lrc, &old_ts);
-	q->run_ticks += (new_ts - old_ts) * q->width;
+	xef->run_ticks[q->class] += (new_ts - old_ts) * q->width;
 }
 void xe_exec_queue_kill(struct xe_exec_queue *q)
 	xe_exec_queue_last_fence_put(q, vm);
 	q->last_fence = dma_fence_get(fence);
 }
+/**
+ * xe_exec_queue_last_fence_test_dep - Test last fence dependency of queue
+ * @q: The exec queue
+ * @vm: The VM the engine does a bind or exec for
+ *
+ * Returns:
+ * -ETIME if there exists an unsignalled last fence dependency, zero otherwise.
+ */
+int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q, struct xe_vm *vm)
+{
+	struct dma_fence *fence;
+	int err = 0;
+	fence = xe_exec_queue_last_fence_get(q, vm);
+	if (fence) {
+		err = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ?
+			0 : -ETIME;
+		dma_fence_put(fence);
+	}
+	return err;
+}

drivers/gpu/drm/xe/xe_exec_queue.h+2−0

@@ -75,6 +75,8 @@ struct dma_fence xe_exec_queue_last_fence_get(struct xe_exec_queue e, · 74 unmodified lines
75	struct xe_vm *vm);	75	struct xe_vm *vm);
76	void xe_exec_queue_last_fence_set(struct xe_exec_queue e, struct xe_vm vm,	76	void xe_exec_queue_last_fence_set(struct xe_exec_queue e, struct xe_vm vm,
77	struct dma_fence *fence);	77	struct dma_fence *fence);
		78	int xe_exec_queue_last_fence_test_dep(struct xe_exec_queue *q,
		79	struct xe_vm *vm);
78	void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q);	80	void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q);
79		81
80	#endif	82	#endif

drivers/gpu/drm/xe/xe_exec_queue_types.h+7−6

  * a kernel object.
  */
 struct xe_exec_queue {
+	/** @xef: Back pointer to xe file if this is user created exec queue */
+	struct xe_file *xef;
 	/** @gt: graphics tile this exec queue can submit to */
 	struct xe_gt *gt;
 	/**
 	 * Protected by @vm's resv. Unused if @vm == NULL.
 	 */
 	u64 tlb_flush_seqno;
-	/** @old_run_ticks: prior hw engine class run time in ticks for this exec queue */
-	u64 old_run_ticks;
-	/** @run_ticks: hw engine class run time in ticks for this exec queue */
-	u64 run_ticks;
 	/** @lrc: logical ring context for this exec queue */
 	struct xe_lrc *lrc[];
 };
 	int (*suspend)(struct xe_exec_queue *q);
 	/**
 	 * @suspend_wait: Wait for an exec queue to suspend executing, should be
-	 * call after suspend.
+	 * call after suspend. In dma-fencing path thus must return within a
+	 * reasonable amount of time. -ETIME return shall indicate an error
+	 * waiting for suspend resulting in associated VM getting killed.
 	 */
-	void (*suspend_wait)(struct xe_exec_queue *q);
+	int (*suspend_wait)(struct xe_exec_queue *q);
 	/**
 	 * @resume: Resume exec queue execution, exec queue must be in a suspended
 	 * state and dma fence returned from most recent suspend call must be

drivers/gpu/drm/xe/xe_execlist.c+2−1

 	return 0;
 }
-static void execlist_exec_queue_suspend_wait(struct xe_exec_queue *q)
+static int execlist_exec_queue_suspend_wait(struct xe_exec_queue *q)
 {
 	/* NIY */
+	return 0;
 }
 static void execlist_exec_queue_resume(struct xe_exec_queue *q)

drivers/gpu/drm/xe/xe_gen_wa_oob.c+12−4

@@ -97,19 +97,27 @@ static int parse(FILE input, FILE csource, FILE *cheader) · 96 unmodified lines
97		97
98	if (name) {	98	if (name) {
99	fprintf(cheader, "\tXE_WA_OOB_%s = %u,\n", name, idx);	99	fprintf(cheader, "\tXE_WA_OOB_%s = %u,\n", name, idx);
100	fprintf(csource, "{ XE_RTP_NAME(\"%s\"), XE_RTP_RULES(%s) },\n",	100
		101	/* Close previous entry before starting a new one */
		102	if (idx)
		103	fprintf(csource, ") },\n");
		104
		105	fprintf(csource, "{ XE_RTP_NAME(\"%s\"),\n XE_RTP_RULES(%s",
101	name, rules);	106	name, rules);
		107	idx++;
102	} else {	108	} else {
103	fprintf(csource, "{ XE_RTP_NAME(NULL), XE_RTP_RULES(%s) },\n",	109	fprintf(csource, ", OR,\n\t%s", rules);
104	rules);
105	}	110	}
106		111
107	idx++;
108	lineno++;	112	lineno++;
109	if (!is_continuation)	113	if (!is_continuation)
110	prev_name = name;	114	prev_name = name;
111	}	115	}
112		116
		117	/* Close last entry */
		118	if (idx)
		119	fprintf(csource, ") },\n");
		120
113	fprintf(cheader, "\t_XE_WA_OOB_COUNT = %u\n", idx);	121	fprintf(cheader, "\t_XE_WA_OOB_COUNT = %u\n", idx);
114		122
115	return 0;	123	return 0;

drivers/gpu/drm/xe/xe_gt.c+54−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9		9
10	#include <drm/drm_managed.h>	10	#include <drm/drm_managed.h>
11	#include <drm/xe_drm.h>	11	#include <drm/xe_drm.h>
		12
12	#include <generated/xe_wa_oob.h>	13	#include <generated/xe_wa_oob.h>
13		14
14	#include "instructions/xe_gfxpipe_commands.h"	15	#include "instructions/xe_gfxpipe_commands.h"
@@ -95,6 +96,51 @@ void xe_gt_sanitize(struct xe_gt *gt) · 80 unmodified lines
95	gt->uc.guc.submission_state.enabled = false;	96	gt->uc.guc.submission_state.enabled = false;
96	}	97	}
97		98
		99	static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
		100	{
		101	u32 reg;
		102	int err;
		103
		104	if (!XE_WA(gt, 16023588340))
		105	return;
		106
		107	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
		108	if (WARN_ON(err))
		109	return;
		110
		111	if (!xe_gt_is_media_type(gt)) {
		112	xe_mmio_write32(gt, SCRATCH1LPFC, EN_L3_RW_CCS_CACHE_FLUSH);
		113	reg = xe_mmio_read32(gt, XE2_GAMREQSTRM_CTRL);
		114	reg \|= CG_DIS_CNTLBUS;
		115	xe_mmio_write32(gt, XE2_GAMREQSTRM_CTRL, reg);
		116	}
		117
		118	xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0x3);
		119	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
		120	}
		121
		122	static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
		123	{
		124	u32 reg;
		125	int err;
		126
		127	if (!XE_WA(gt, 16023588340))
		128	return;
		129
		130	if (xe_gt_is_media_type(gt))
		131	return;
		132
		133	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
		134	if (WARN_ON(err))
		135	return;
		136
		137	reg = xe_mmio_read32(gt, XE2_GAMREQSTRM_CTRL);
		138	reg &= ~CG_DIS_CNTLBUS;
		139	xe_mmio_write32(gt, XE2_GAMREQSTRM_CTRL, reg);
		140
		141	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
		142	}
		143
98	/**	144	/**
99	* xe_gt_remove() - Clean up the GT structures before driver removal	145	* xe_gt_remove() - Clean up the GT structures before driver removal
100	* @gt: the GT object	146	* @gt: the GT object
@@ -111,6 +157,8 @@ void xe_gt_remove(struct xe_gt *gt) · 10 unmodified lines
111		157
112	for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)	158	for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
113	xe_hw_fence_irq_finish(&gt->fence_irq[i]);	159	xe_hw_fence_irq_finish(&gt->fence_irq[i]);
		160
		161	xe_gt_disable_host_l2_vram(gt);
114	}	162	}
115		163
116	static void gt_reset_worker(struct work_struct *w);	164	static void gt_reset_worker(struct work_struct *w);
@@ -339,6 +387,7 @@ int xe_gt_init_early(struct xe_gt *gt) · 222 unmodified lines
339		387
340	xe_force_wake_init_gt(gt, gt_to_fw(gt));	388	xe_force_wake_init_gt(gt, gt_to_fw(gt));
341	xe_pcode_init(gt);	389	xe_pcode_init(gt);
		390	spin_lock_init(&gt->global_invl_lock);
342		391
343	return 0;	392	return 0;
344	}	393	}
@@ -508,6 +557,7 @@ int xe_gt_init_hwconfig(struct xe_gt *gt) · 163 unmodified lines
508		557
509	xe_gt_mcr_init_early(gt);	558	xe_gt_mcr_init_early(gt);
510	xe_pat_init(gt);	559	xe_pat_init(gt);
		560	xe_gt_enable_host_l2_vram(gt);
511		561
512	err = xe_uc_init(&gt->uc);	562	err = xe_uc_init(&gt->uc);
513	if (err)	563	if (err)
@@ -643,6 +693,8 @@ static int do_gt_restart(struct xe_gt *gt) · 129 unmodified lines
643		693
644	xe_pat_init(gt);	694	xe_pat_init(gt);
645		695
		696	xe_gt_enable_host_l2_vram(gt);
		697
646	xe_gt_mcr_set_implicit_defaults(gt);	698	xe_gt_mcr_set_implicit_defaults(gt);
647	xe_reg_sr_apply_mmio(&gt->reg_sr, gt);	699	xe_reg_sr_apply_mmio(&gt->reg_sr, gt);
648		700
@@ -796,6 +848,8 @@ int xe_gt_suspend(struct xe_gt *gt) · 147 unmodified lines
796		848
797	xe_gt_idle_disable_pg(gt);	849	xe_gt_idle_disable_pg(gt);
798		850
		851	xe_gt_disable_host_l2_vram(gt);
		852
799	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));	853	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
800	xe_gt_dbg(gt, "suspended\n");	854	xe_gt_dbg(gt, "suspended\n");
801		855

drivers/gpu/drm/xe/xe_gt_sriov_pf.c+1−1

@@ -5,7 +5,7 @@ · 4 unmodified lines
5		5
6	#include <drm/drm_managed.h>	6	#include <drm/drm_managed.h>
7		7
8	#include "regs/xe_sriov_regs.h"	8	#include "regs/xe_regs.h"
9		9
10	#include "xe_gt_sriov_pf.h"	10	#include "xe_gt_sriov_pf.h"
11	#include "xe_gt_sriov_pf_config.h"	11	#include "xe_gt_sriov_pf_config.h"

drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c+1−0

@@ -1401,6 +1401,7 @@ static int pf_provision_vf_lmem(struct xe_gt *gt, unsigned int vfid, u64 size) · 1400 unmodified lines
1401	ALIGN(size, PAGE_SIZE),	1401	ALIGN(size, PAGE_SIZE),
1402	ttm_bo_type_kernel,	1402	ttm_bo_type_kernel,
1403	XE_BO_FLAG_VRAM_IF_DGFX(tile) \|	1403	XE_BO_FLAG_VRAM_IF_DGFX(tile) \|
		1404	XE_BO_FLAG_NEEDS_2M \|
1404	XE_BO_FLAG_PINNED);	1405	XE_BO_FLAG_PINNED);
1405	if (IS_ERR(bo))	1406	if (IS_ERR(bo))
1406	return PTR_ERR(bo);	1407	return PTR_ERR(bo);

drivers/gpu/drm/xe/xe_gt_sriov_vf.c+27−1

@@ -850,7 +850,7 @@ static struct vf_runtime_reg vf_lookup_reg(struct xe_gt gt, u32 addr) · 849 unmodified lines
850		850
851	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));	851	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
852		852
853	return bsearch(&key, runtime->regs, runtime->regs_size, sizeof(key),	853	return bsearch(&key, runtime->regs, runtime->num_regs, sizeof(key),
854	vf_runtime_reg_cmp);	854	vf_runtime_reg_cmp);
855	}	855	}
856		856
@@ -892,6 +892,32 @@ u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg) · 35 unmodified lines
892	return rr->value;	892	return rr->value;
893	}	893	}
894		894
		895	/**
		896	* xe_gt_sriov_vf_write32 - Handle a write to an inaccessible register.
		897	* @gt: the &xe_gt
		898	* @reg: the register to write
		899	* @val: value to write
		900	*
		901	* This function is for VF use only.
		902	* Currently it will trigger a WARN if running on debug build.
		903	*/
		904	void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val)
		905	{
		906	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);
		907
		908	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
		909	xe_gt_assert(gt, !reg.vf);
		910
		911	/*
		912	* In the future, we may want to handle selected writes to inaccessible
		913	* registers in some custom way, but for now let's just log a warning
		914	* about such attempt, as likely we might be doing something wrong.
		915	*/
		916	xe_gt_WARN(gt, IS_ENABLED(CONFIG_DRM_XE_DEBUG),
		917	"VF is trying to write %#x to an inaccessible register %#x+%#x\n",
		918	val, reg.addr, addr - reg.addr);
		919	}
		920
895	/**	921	/**
896	* xe_gt_sriov_vf_print_config - Print VF self config.	922	* xe_gt_sriov_vf_print_config - Print VF self config.
897	* @gt: the &xe_gt	923	* @gt: the &xe_gt

drivers/gpu/drm/xe/xe_gt_sriov_vf.h+1−0

@@ -22,6 +22,7 @@ u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt); · 21 unmodified lines
22	u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt);	22	u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt);
23	u64 xe_gt_sriov_vf_lmem(struct xe_gt *gt);	23	u64 xe_gt_sriov_vf_lmem(struct xe_gt *gt);
24	u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg);	24	u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg);
		25	void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val);
25		26
26	void xe_gt_sriov_vf_print_config(struct xe_gt gt, struct drm_printer p);	27	void xe_gt_sriov_vf_print_config(struct xe_gt gt, struct drm_printer p);
27	void xe_gt_sriov_vf_print_runtime(struct xe_gt gt, struct drm_printer p);	28	void xe_gt_sriov_vf_print_runtime(struct xe_gt gt, struct drm_printer p);

drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c+111−94

@@ -13,10 +13,13 @@ · 12 unmodified lines
13	#include "xe_guc.h"	13	#include "xe_guc.h"
14	#include "xe_guc_ct.h"	14	#include "xe_guc_ct.h"
15	#include "xe_mmio.h"	15	#include "xe_mmio.h"
		16	#include "xe_pm.h"
16	#include "xe_sriov.h"	17	#include "xe_sriov.h"
17	#include "xe_trace.h"	18	#include "xe_trace.h"
18	#include "regs/xe_guc_regs.h"	19	#include "regs/xe_guc_regs.h"
19		20
		21	#define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS
		22
20	/*	23	/*
21	* TLB inval depends on pending commands in the CT queue and then the real	24	* TLB inval depends on pending commands in the CT queue and then the real
22	* invalidation time. Double up the time to process full CT queue	25	* invalidation time. Double up the time to process full CT queue
@@ -33,6 +36,24 @@ static long tlb_timeout_jiffies(struct xe_gt *gt) · 10 unmodified lines
33	return hw_tlb_timeout + 2 * delay;	36	return hw_tlb_timeout + 2 * delay;
34	}	37	}
35		38
		39	static void
		40	__invalidation_fence_signal(struct xe_device xe, struct xe_gt_tlb_invalidation_fence fence)
		41	{
		42	bool stack = test_bit(FENCE_STACK_BIT, &fence->base.flags);
		43
		44	trace_xe_gt_tlb_invalidation_fence_signal(xe, fence);
		45	xe_gt_tlb_invalidation_fence_fini(fence);
		46	dma_fence_signal(&fence->base);
		47	if (!stack)
		48	dma_fence_put(&fence->base);
		49	}
		50
		51	static void
		52	invalidation_fence_signal(struct xe_device xe, struct xe_gt_tlb_invalidation_fence fence)
		53	{
		54	list_del(&fence->link);
		55	__invalidation_fence_signal(xe, fence);
		56	}
36		57
37	static void xe_gt_tlb_fence_timeout(struct work_struct *work)	58	static void xe_gt_tlb_fence_timeout(struct work_struct *work)
38	{	59	{
@@ -54,10 +75,8 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work) · 15 unmodified lines
54	xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d",	75	xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d",
55	fence->seqno, gt->tlb_invalidation.seqno_recv);	76	fence->seqno, gt->tlb_invalidation.seqno_recv);
56		77
57	list_del(&fence->link);
58	fence->base.error = -ETIME;	78	fence->base.error = -ETIME;
59	dma_fence_signal(&fence->base);	79	invalidation_fence_signal(xe, fence);
60	dma_fence_put(&fence->base);
61	}	80	}
62	if (!list_empty(&gt->tlb_invalidation.pending_fences))	81	if (!list_empty(&gt->tlb_invalidation.pending_fences))
63	queue_delayed_work(system_wq,	82	queue_delayed_work(system_wq,
@@ -87,21 +106,6 @@ int xe_gt_tlb_invalidation_init(struct xe_gt *gt) · 23 unmodified lines
87	return 0;	106	return 0;
88	}	107	}
89		108
90	static void
91	__invalidation_fence_signal(struct xe_device xe, struct xe_gt_tlb_invalidation_fence fence)
92	{
93	trace_xe_gt_tlb_invalidation_fence_signal(xe, fence);
94	dma_fence_signal(&fence->base);
95	dma_fence_put(&fence->base);
96	}
97
98	static void
99	invalidation_fence_signal(struct xe_device xe, struct xe_gt_tlb_invalidation_fence fence)
100	{
101	list_del(&fence->link);
102	__invalidation_fence_signal(xe, fence);
103	}
104
105	/**	109	/**
106	* xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset	110	* xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset
107	* @gt: graphics tile	111	* @gt: graphics tile
@@ -111,7 +115,6 @@ invalidation_fence_signal(struct xe_device *xe, struct xe_gt_tlb_invalidation_fe · 3 unmodified lines
111	void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)	115	void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)
112	{	116	{
113	struct xe_gt_tlb_invalidation_fence fence, next;	117	struct xe_gt_tlb_invalidation_fence fence, next;
114	struct xe_guc *guc = &gt->uc.guc;
115	int pending_seqno;	118	int pending_seqno;
116		119
117	/*	120	/*
@@ -134,7 +137,6 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt) · 16 unmodified lines
134	else	137	else
135	pending_seqno = gt->tlb_invalidation.seqno - 1;	138	pending_seqno = gt->tlb_invalidation.seqno - 1;
136	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, pending_seqno);	139	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, pending_seqno);
137	wake_up_all(&guc->ct.wq);
138		140
139	list_for_each_entry_safe(fence, next,	141	list_for_each_entry_safe(fence, next,
140	&gt->tlb_invalidation.pending_fences, link)	142	&gt->tlb_invalidation.pending_fences, link)
@@ -165,6 +167,8 @@ static int send_tlb_invalidation(struct xe_guc *guc, · 24 unmodified lines
165	int seqno;	167	int seqno;
166	int ret;	168	int ret;
167		169
		170	xe_gt_assert(gt, fence);
		171
168	/*	172	/*
169	* XXX: The seqno algorithm relies on TLB invalidation being processed	173	* XXX: The seqno algorithm relies on TLB invalidation being processed
170	* in order which they currently are, if that changes the algorithm will	174	* in order which they currently are, if that changes the algorithm will
@@ -173,14 +177,12 @@ static int send_tlb_invalidation(struct xe_guc *guc, · 2 unmodified lines
173		177
174	mutex_lock(&guc->ct.lock);	178	mutex_lock(&guc->ct.lock);
175	seqno = gt->tlb_invalidation.seqno;	179	seqno = gt->tlb_invalidation.seqno;
176	if (fence) {	180	fence->seqno = seqno;
177	fence->seqno = seqno;	181	trace_xe_gt_tlb_invalidation_fence_send(xe, fence);
178	trace_xe_gt_tlb_invalidation_fence_send(xe, fence);
179	}
180	action[1] = seqno;	182	action[1] = seqno;
181	ret = xe_guc_ct_send_locked(&guc->ct, action, len,	183	ret = xe_guc_ct_send_locked(&guc->ct, action, len,
182	G2H_LEN_DW_TLB_INVALIDATE, 1);	184	G2H_LEN_DW_TLB_INVALIDATE, 1);
183	if (!ret && fence) {	185	if (!ret) {
184	spin_lock_irq(&gt->tlb_invalidation.pending_lock);	186	spin_lock_irq(&gt->tlb_invalidation.pending_lock);
185	/*	187	/*
186	* We haven't actually published the TLB fence as per	188	* We haven't actually published the TLB fence as per
@@ -201,7 +203,7 @@ static int send_tlb_invalidation(struct xe_guc *guc, · 14 unmodified lines
201	tlb_timeout_jiffies(gt));	203	tlb_timeout_jiffies(gt));
202	}	204	}
203	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);	205	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);
204	} else if (ret < 0 && fence) {	206	} else if (ret < 0) {
205	__invalidation_fence_signal(xe, fence);	207	__invalidation_fence_signal(xe, fence);
206	}	208	}
207	if (!ret) {	209	if (!ret) {
@@ -209,7 +211,6 @@ static int send_tlb_invalidation(struct xe_guc *guc, · 1 unmodified line
209	TLB_INVALIDATION_SEQNO_MAX;	211	TLB_INVALIDATION_SEQNO_MAX;
210	if (!gt->tlb_invalidation.seqno)	212	if (!gt->tlb_invalidation.seqno)
211	gt->tlb_invalidation.seqno = 1;	213	gt->tlb_invalidation.seqno = 1;
212	ret = seqno;
213	}	214	}
214	mutex_unlock(&guc->ct.lock);	215	mutex_unlock(&guc->ct.lock);
215		216
@@ -223,14 +224,16 @@ static int send_tlb_invalidation(struct xe_guc *guc, · 7 unmodified lines
223	/**	224	/**
224	* xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC	225	* xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC
225	* @gt: graphics tile	226	* @gt: graphics tile
		227	* @fence: invalidation fence which will be signal on TLB invalidation
		228	* completion
226	*	229	*
227	* Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and	230	* Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and
228	* caller can use seqno + xe_gt_tlb_invalidation_wait to wait for completion.	231	* caller can use the invalidation fence to wait for completion.
229	*	232	*
230	* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,	233	* Return: 0 on success, negative error code on error
231	* negative error code on error.
232	*/	234	*/
233	static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)	235	static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt,
		236	struct xe_gt_tlb_invalidation_fence *fence)
234	{	237	{
235	u32 action[] = {	238	u32 action[] = {
236	XE_GUC_ACTION_TLB_INVALIDATION,	239	XE_GUC_ACTION_TLB_INVALIDATION,
@@ -238,7 +241,7 @@ static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt) · 1 unmodified line
238	MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),	241	MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
239	};	242	};
240		243
241	return send_tlb_invalidation(&gt->uc.guc, NULL, action,	244	return send_tlb_invalidation(&gt->uc.guc, fence, action,
242	ARRAY_SIZE(action));	245	ARRAY_SIZE(action));
243	}	246	}
244		247
@@ -257,13 +260,17 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt) · 12 unmodified lines
257		260
258	if (xe_guc_ct_enabled(&gt->uc.guc.ct) &&	261	if (xe_guc_ct_enabled(&gt->uc.guc.ct) &&
259	gt->uc.guc.submission_state.enabled) {	262	gt->uc.guc.submission_state.enabled) {
260	int seqno;	263	struct xe_gt_tlb_invalidation_fence fence;
261		264	int ret;
262	seqno = xe_gt_tlb_invalidation_guc(gt);	265
263	if (seqno <= 0)	266	xe_gt_tlb_invalidation_fence_init(gt, &fence, true);
264	return seqno;	267	ret = xe_gt_tlb_invalidation_guc(gt, &fence);
		268	if (ret < 0) {
		269	xe_gt_tlb_invalidation_fence_fini(&fence);
		270	return ret;
		271	}
265		272
266	xe_gt_tlb_invalidation_wait(gt, seqno);	273	xe_gt_tlb_invalidation_fence_wait(&fence);
267	} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {	274	} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
268	if (IS_SRIOV_VF(xe))	275	if (IS_SRIOV_VF(xe))
269	return 0;	276	return 0;
@@ -290,18 +297,16 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt) · 20 unmodified lines
290	*	297	*
291	* @gt: graphics tile	298	* @gt: graphics tile
292	* @fence: invalidation fence which will be signal on TLB invalidation	299	* @fence: invalidation fence which will be signal on TLB invalidation
293	* completion, can be NULL	300	* completion
294	* @start: start address	301	* @start: start address
295	* @end: end address	302	* @end: end address
296	* @asid: address space id	303	* @asid: address space id
297	*	304	*
298	* Issue a range based TLB invalidation if supported, if not fallback to a full	305	* Issue a range based TLB invalidation if supported, if not fallback to a full
299	* TLB invalidation. Completion of TLB is asynchronous and caller can either use	306	* TLB invalidation. Completion of TLB is asynchronous and caller can use
300	* the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for	307	* the invalidation fence to wait for completion.
301	* completion.
302	*	308	*
303	* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,	309	* Return: Negative error code on error, 0 on success
304	* negative error code on error.
305	*/	310	*/
306	int xe_gt_tlb_invalidation_range(struct xe_gt *gt,	311	int xe_gt_tlb_invalidation_range(struct xe_gt *gt,
307	struct xe_gt_tlb_invalidation_fence *fence,	312	struct xe_gt_tlb_invalidation_fence *fence,
@@ -312,11 +317,11 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt, · 4 unmodified lines
312	u32 action[MAX_TLB_INVALIDATION_LEN];	317	u32 action[MAX_TLB_INVALIDATION_LEN];
313	int len = 0;	318	int len = 0;
314		319
		320	xe_gt_assert(gt, fence);
		321
315	/* Execlists not supported */	322	/* Execlists not supported */
316	if (gt_to_xe(gt)->info.force_execlist) {	323	if (gt_to_xe(gt)->info.force_execlist) {
317	if (fence)	324	__invalidation_fence_signal(xe, fence);
318	__invalidation_fence_signal(xe, fence);
319
320	return 0;	325	return 0;
321	}	326	}
322		327
@@ -382,12 +387,10 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt, · 59 unmodified lines
382	* @vma: VMA to invalidate	387	* @vma: VMA to invalidate
383	*	388	*
384	* Issue a range based TLB invalidation if supported, if not fallback to a full	389	* Issue a range based TLB invalidation if supported, if not fallback to a full
385	* TLB invalidation. Completion of TLB is asynchronous and caller can either use	390	* TLB invalidation. Completion of TLB is asynchronous and caller can use
386	* the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for	391	* the invalidation fence to wait for completion.
387	* completion.
388	*	392	*
389	* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,	393	* Return: Negative error code on error, 0 on success
390	* negative error code on error.
391	*/	394	*/
392	int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,	395	int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
393	struct xe_gt_tlb_invalidation_fence *fence,	396	struct xe_gt_tlb_invalidation_fence *fence,
@@ -400,43 +403,6 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt, · 6 unmodified lines
400	xe_vma_vm(vma)->usm.asid);	403	xe_vma_vm(vma)->usm.asid);
401	}	404	}
402		405
403	/**
404	* xe_gt_tlb_invalidation_wait - Wait for TLB to complete
405	* @gt: graphics tile
406	* @seqno: seqno to wait which was returned from xe_gt_tlb_invalidation
407	*
408	* Wait for tlb_timeout_jiffies() for a TLB invalidation to complete.
409	*
410	* Return: 0 on success, -ETIME on TLB invalidation timeout
411	*/
412	int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno)
413	{
414	struct xe_guc *guc = &gt->uc.guc;
415	int ret;
416
417	/* Execlists not supported */
418	if (gt_to_xe(gt)->info.force_execlist)
419	return 0;
420
421	/*
422	* XXX: See above, this algorithm only works if seqno are always in
423	* order
424	*/
425	ret = wait_event_timeout(guc->ct.wq,
426	tlb_invalidation_seqno_past(gt, seqno),
427	tlb_timeout_jiffies(gt));
428	if (!ret) {
429	struct drm_printer p = xe_gt_err_printer(gt);
430
431	xe_gt_err(gt, "TLB invalidation time'd out, seqno=%d, recv=%d\n",
432	seqno, gt->tlb_invalidation.seqno_recv);
433	xe_guc_ct_print(&guc->ct, &p, true);
434	return -ETIME;
435	}
436
437	return 0;
438	}
439
440	/**	406	/**
441	* xe_guc_tlb_invalidation_done_handler - TLB invalidation done handler	407	* xe_guc_tlb_invalidation_done_handler - TLB invalidation done handler
442	* @guc: guc	408	* @guc: guc
@@ -480,12 +446,7 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc guc, u32 msg, u32 len) · 37 unmodified lines
480	return 0;	446	return 0;
481	}	447	}
482		448
483	/*
484	* wake_up_all() and wait_event_timeout() already have the correct
485	* barriers.
486	*/
487	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, msg[0]);	449	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, msg[0]);
488	wake_up_all(&guc->ct.wq);
489		450
490	list_for_each_entry_safe(fence, next,	451	list_for_each_entry_safe(fence, next,
491	&gt->tlb_invalidation.pending_fences, link) {	452	&gt->tlb_invalidation.pending_fences, link) {
@@ -508,3 +469,59 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc guc, u32 msg, u32 len) · 16 unmodified lines
508		469
509	return 0;	470	return 0;
510	}	471	}
		472
		473	static const char *
		474	invalidation_fence_get_driver_name(struct dma_fence *dma_fence)
		475	{
		476	return "xe";
		477	}
		478
		479	static const char *
		480	invalidation_fence_get_timeline_name(struct dma_fence *dma_fence)
		481	{
		482	return "invalidation_fence";
		483	}
		484
		485	static const struct dma_fence_ops invalidation_fence_ops = {
		486	.get_driver_name = invalidation_fence_get_driver_name,
		487	.get_timeline_name = invalidation_fence_get_timeline_name,
		488	};
		489
		490	/**
		491	* xe_gt_tlb_invalidation_fence_init - Initialize TLB invalidation fence
		492	* @gt: GT
		493	* @fence: TLB invalidation fence to initialize
		494	* @stack: fence is stack variable
		495	*
		496	* Initialize TLB invalidation fence for use. xe_gt_tlb_invalidation_fence_fini
		497	* must be called if fence is not signaled.
		498	*/
		499	void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt,
		500	struct xe_gt_tlb_invalidation_fence *fence,
		501	bool stack)
		502	{
		503	xe_pm_runtime_get_noresume(gt_to_xe(gt));
		504
		505	spin_lock_irq(&gt->tlb_invalidation.lock);
		506	dma_fence_init(&fence->base, &invalidation_fence_ops,
		507	&gt->tlb_invalidation.lock,
		508	dma_fence_context_alloc(1), 1);
		509	spin_unlock_irq(&gt->tlb_invalidation.lock);
		510	INIT_LIST_HEAD(&fence->link);
		511	if (stack)
		512	set_bit(FENCE_STACK_BIT, &fence->base.flags);
		513	else
		514	dma_fence_get(&fence->base);
		515	fence->gt = gt;
		516	}
		517
		518	/**
		519	* xe_gt_tlb_invalidation_fence_fini - Finalize TLB invalidation fence
		520	* @fence: TLB invalidation fence to finalize
		521	*
		522	* Drop PM ref which fence took durinig init.
		523	*/
		524	void xe_gt_tlb_invalidation_fence_fini(struct xe_gt_tlb_invalidation_fence *fence)
		525	{
		526	xe_pm_runtime_put(gt_to_xe(fence->gt));
		527	}

drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h+11−1

@@ -23,7 +23,17 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt, · 22 unmodified lines
23	int xe_gt_tlb_invalidation_range(struct xe_gt *gt,	23	int xe_gt_tlb_invalidation_range(struct xe_gt *gt,
24	struct xe_gt_tlb_invalidation_fence *fence,	24	struct xe_gt_tlb_invalidation_fence *fence,
25	u64 start, u64 end, u32 asid);	25	u64 start, u64 end, u32 asid);
26	int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno);
27	int xe_guc_tlb_invalidation_done_handler(struct xe_guc guc, u32 msg, u32 len);	26	int xe_guc_tlb_invalidation_done_handler(struct xe_guc guc, u32 msg, u32 len);
28		27
		28	void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt,
		29	struct xe_gt_tlb_invalidation_fence *fence,
		30	bool stack);
		31	void xe_gt_tlb_invalidation_fence_fini(struct xe_gt_tlb_invalidation_fence *fence);
		32
		33	static inline void
		34	xe_gt_tlb_invalidation_fence_wait(struct xe_gt_tlb_invalidation_fence *fence)
		35	{
		36	dma_fence_wait(&fence->base, false);
		37	}
		38
29	#endif /* _XE_GT_TLB_INVALIDATION_ */	39	#endif /* _XE_GT_TLB_INVALIDATION_ */

drivers/gpu/drm/xe/xe_gt_tlb_invalidation_types.h+4−0

 #include <linux/dma-fence.h>
+struct xe_gt;
 /**
  * struct xe_gt_tlb_invalidation_fence - XE GT TLB invalidation fence
  *
 struct xe_gt_tlb_invalidation_fence {
 	/** @base: dma fence base */
 	struct dma_fence base;
+	/** @gt: GT which fence belong to */
+	struct xe_gt *gt;
 	/** @link: link into list of pending tlb fences */
 	struct list_head link;
 	/** @seqno: seqno of TLB invalidation to signal fence one */

drivers/gpu/drm/xe/xe_gt_topology.c+22−5

 #include "xe_gt_topology.h"
 #include <linux/bitmap.h>
+#include <linux/compiler.h>
 #include "regs/xe_gt_regs.h"
 #include "xe_assert.h"
 }
 static void
-load_eu_mask(struct xe_gt *gt, xe_eu_mask_t mask)
+load_eu_mask(struct xe_gt *gt, xe_eu_mask_t mask, enum xe_gt_eu_type *eu_type)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 	u32 reg_val = xe_mmio_read32(gt, XELP_EU_ENABLE);
 	if (GRAPHICS_VERx100(xe) < 1250)
 		reg_val = ~reg_val & XELP_EU_MASK;
-	/* On PVC, one bit = one EU */
+	if (GRAPHICS_VERx100(xe) == 1260 || GRAPHICS_VER(xe) >= 20) {
-	if (GRAPHICS_VERx100(xe) == 1260) {
+		/* SIMD16 EUs, one bit == one EU */
+		*eu_type = XE_GT_EU_TYPE_SIMD16;
 		val = reg_val;
 	} else {
-		/* All other platforms, one bit = 2 EU */
+		/* SIMD8 EUs, one bit == 2 EU */
+		*eu_type = XE_GT_EU_TYPE_SIMD8;
 		for (i = 0; i < fls(reg_val); i++)
 			if (reg_val & BIT(i))
 				val |= 0x3 << 2 * i;
 		      XEHP_GT_COMPUTE_DSS_ENABLE,
 		      XEHPC_GT_COMPUTE_DSS_ENABLE_EXT,
 		      XE2_GT_COMPUTE_DSS_2);
-	load_eu_mask(gt, gt->fuse_topo.eu_mask_per_dss);
+	load_eu_mask(gt, gt->fuse_topo.eu_mask_per_dss, &gt->fuse_topo.eu_type);
 	load_l3_bank_mask(gt, gt->fuse_topo.l3_bank_mask);
 	p = drm_dbg_printer(&gt_to_xe(gt)->drm, DRM_UT_DRIVER, "GT topology");
 	xe_gt_topology_dump(gt, &p);
 }
+static const char *eu_type_to_str(enum xe_gt_eu_type eu_type)
+{
+	switch (eu_type) {
+	case XE_GT_EU_TYPE_SIMD16:
+		return "simd16";
+	case XE_GT_EU_TYPE_SIMD8:
+		return "simd8";
+	}
+	return NULL;
+}
 void
 xe_gt_topology_dump(struct xe_gt *gt, struct drm_printer *p)
 {
 	drm_printf(p, "EU mask per DSS:     %*pb\n", XE_MAX_EU_FUSE_BITS,
 		   gt->fuse_topo.eu_mask_per_dss);
+	drm_printf(p, "EU type:             %s\n",
+		   eu_type_to_str(gt->fuse_topo.eu_type));
 	drm_printf(p, "L3 bank mask:        %*pb\n", XE_MAX_L3_BANK_MASK_BITS,
 		   gt->fuse_topo.l3_bank_mask);

drivers/gpu/drm/xe/xe_gt_types.h+24−1

 	XE_GT_TYPE_MEDIA,
 };
+enum xe_gt_eu_type {
+	XE_GT_EU_TYPE_SIMD8,
+	XE_GT_EU_TYPE_SIMD16,
+};
 #define XE_MAX_DSS_FUSE_REGS		3
 #define XE_MAX_DSS_FUSE_BITS		(32 * XE_MAX_DSS_FUSE_REGS)
 #define XE_MAX_EU_FUSE_REGS		1
 		/** @fuse_topo.l3_bank_mask: L3 bank mask */
 		xe_l3_bank_mask_t l3_bank_mask;
+		/**
+		 * @fuse_topo.eu_type: type/width of EU stored in
+		 * fuse_topo.eu_mask_per_dss
+		 */
+		enum xe_gt_eu_type eu_type;
 	} fuse_topo;
 	/** @steering: register steering for individual HW units */
 	 */
 	spinlock_t mcr_lock;
+	/**
+	 * @global_invl_lock: protects the register for the duration
+	 *    of a global invalidation of l2 cache
+	 */
+	spinlock_t global_invl_lock;
 	/** @wa_active: keep track of active workarounds */
 	struct {
 		/** @wa_active.gt: bitmap with active GT workarounds */
 		unsigned long *engine;
 		/** @wa_active.lrc: bitmap with active LRC workarounds */
 		unsigned long *lrc;
-		/** @wa_active.oob: bitmap with active OOB workaroudns */
+		/** @wa_active.oob: bitmap with active OOB workarounds */
 		unsigned long *oob;
+		/**
+		 * @wa_active.oob_initialized: mark oob as initialized to help
+		 * detecting misuse of XE_WA() - it can only be called on
+		 * initialization after OOB WAs have being processed
+		 */
+		bool oob_initialized;
 	} wa_active;
 	/** @user_engines: engines present in GT and available to userspace */

drivers/gpu/drm/xe/xe_guc_ct.c+10−1

 	xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 ||
 		     state == XE_GUC_CT_STATE_STOPPED);
+	if (ct->g2h_outstanding)
+		xe_pm_runtime_put(ct_to_xe(ct));
 	ct->g2h_outstanding = 0;
 	ct->state = state;
 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
 {
 	xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space);
+	xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) ||
+		     (g2h_len && num_g2h));
 	if (g2h_len) {
 		lockdep_assert_held(&ct->fast_lock);
+		if (!ct->g2h_outstanding)
+			xe_pm_runtime_get_noresume(ct_to_xe(ct));
 		ct->ctbs.g2h.info.space -= g2h_len;
 		ct->g2h_outstanding += num_g2h;
 	}
 	lockdep_assert_held(&ct->fast_lock);
 	xe_gt_assert(ct_to_gt(ct), ct->ctbs.g2h.info.space + g2h_len <=
 		     ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space);
+	xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding);
 	ct->ctbs.g2h.info.space += g2h_len;
-	--ct->g2h_outstanding;
+	if (!--ct->g2h_outstanding)
+		xe_pm_runtime_put(ct_to_xe(ct));
 }
 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)

drivers/gpu/drm/xe/xe_guc_id_mgr.c+2−2

@@ -97,8 +97,8 @@ int xe_guc_id_mgr_init(struct xe_guc_id_mgr *idm, unsigned int limit) · 96 unmodified lines
97	if (ret)	97	if (ret)
98	return ret;	98	return ret;
99		99
100	xe_gt_info(idm_to_gt(idm), "using %u GUC ID%s\n",	100	xe_gt_dbg(idm_to_gt(idm), "using %u GuC ID%s\n",
101	idm->total, str_plural(idm->total));	101	idm->total, str_plural(idm->total));
102	return 0;	102	return 0;
103	}	103	}
104		104

drivers/gpu/drm/xe/xe_guc_submit.c+48−8

 	struct xe_exec_queue *q = job->q;
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
 	struct xe_guc *guc = exec_queue_to_guc(q);
+	const char *process_name = "no process";
 	int err = -ETIME;
+	pid_t pid = -1;
 	int i = 0;
 	bool wedged, skip_timeout_check;
 		goto sched_enable;
 	}
-	xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx",
+	if (q->vm && q->vm->xef) {
+		process_name = q->vm->xef->process_name;
+		pid = q->vm->xef->pid;
+	}
+	xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]",
 		     xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
-		     q->guc->id, q->flags);
+		     q->guc->id, q->flags, process_name, pid);
 	trace_xe_sched_job_timedout(job);
 	if (!exec_queue_killed(q))
 	kfree(msg);
 }
+static void __suspend_fence_signal(struct xe_exec_queue *q)
+{
+	if (!q->guc->suspend_pending)
+		return;
+	WRITE_ONCE(q->guc->suspend_pending, false);
+	wake_up(&q->guc->suspend_wait);
+}
 static void suspend_fence_signal(struct xe_exec_queue *q)
 {
 	struct xe_guc *guc = exec_queue_to_guc(q);
 		  guc_read_stopped(guc));
 	xe_assert(xe, q->guc->suspend_pending);
-	q->guc->suspend_pending = false;
+	__suspend_fence_signal(q);
-	smp_wmb();
-	wake_up(&q->guc->suspend_wait);
 }
 static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)
 static void guc_exec_queue_process_msg(struct xe_sched_msg *msg)
 {
+	struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data));
 	trace_xe_sched_msg_recv(msg);
 	switch (msg->opcode) {
 	default:
 		XE_WARN_ON("Unknown message type");
 	}
+	xe_pm_runtime_put(xe);
 }
 static const struct drm_sched_backend_ops drm_sched_ops = {
 {
 	trace_xe_exec_queue_kill(q);
 	set_exec_queue_killed(q);
+	__suspend_fence_signal(q);
 	xe_guc_exec_queue_trigger_cleanup(q);
 }
 static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg,
 				   u32 opcode)
 {
+	xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q)));
 	INIT_LIST_HEAD(&msg->link);
 	msg->opcode = opcode;
 	msg->private_data = q;
 	return 0;
 }
-static void guc_exec_queue_suspend_wait(struct xe_exec_queue *q)
+static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q)
 {
 	struct xe_guc *guc = exec_queue_to_guc(q);
+	int ret;
+	/*
+	 * Likely don't need to check exec_queue_killed() as we clear
+	 * suspend_pending upon kill but to be paranoid but races in which
+	 * suspend_pending is set after kill also check kill here.
+	 */
+	ret = wait_event_timeout(q->guc->suspend_wait,
+				 !READ_ONCE(q->guc->suspend_pending) ||
+				 exec_queue_killed(q) ||
+				 guc_read_stopped(guc),
+				 HZ * 5);
-	wait_event(q->guc->suspend_wait, !q->guc->suspend_pending ||
+	if (!ret) {
-		   guc_read_stopped(guc));
+		xe_gt_warn(guc_to_gt(guc),
+			   "Suspend fence, guc_id=%d, failed to respond",
+			   q->guc->id);
+		/* XXX: Trigger GT reset? */
+		return -ETIME;
+	}
+	return 0;
 }
 static void guc_exec_queue_resume(struct xe_exec_queue *q)

drivers/gpu/drm/xe/xe_heci_gsc.c+25−3

 {
 	struct xe_heci_gsc *heci_gsc = &xe->heci_gsc;
-	if (!HAS_HECI_GSCFI(xe))
+	if (!HAS_HECI_GSCFI(xe) && !HAS_HECI_CSCFI(xe))
 		return;
 	if (heci_gsc->adev) {
 	const struct heci_gsc_def *def;
 	int ret;
-	if (!HAS_HECI_GSCFI(xe))
+	if (!HAS_HECI_GSCFI(xe) && !HAS_HECI_CSCFI(xe))
 		return;
 	heci_gsc->irq = -1;
-	if (xe->info.platform == XE_PVC) {
+	if (xe->info.platform == XE_BATTLEMAGE) {
+		def = &heci_gsc_def_dg2;
+	} else if (xe->info.platform == XE_PVC) {
 		def = &heci_gsc_def_pvc;
 	} else if (xe->info.platform == XE_DG2) {
 		def = &heci_gsc_def_dg2;
 	if (ret)
 		drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret);
 }
+void xe_heci_csc_irq_handler(struct xe_device *xe, u32 iir)
+{
+	int ret;
+	if ((iir & CSC_IRQ_INTF(1)) == 0)
+		return;
+	if (!HAS_HECI_CSCFI(xe)) {
+		drm_warn_once(&xe->drm, "CSC irq: not supported");
+		return;
+	}
+	if (xe->heci_gsc.irq < 0)
+		return;
+	ret = generic_handle_irq(xe->heci_gsc.irq);
+	if (ret)
+		drm_err_ratelimited(&xe->drm, "error handling GSC irq: %d\n", ret);
+}

drivers/gpu/drm/xe/xe_heci_gsc.h+8−2

 struct mei_aux_device;
 /*
- * The HECI1 bit corresponds to bit15 and HECI2 to bit14.
+ * GSC HECI1 bit corresponds to bit15 and HECI2 to bit14.
  * The reason for this is to allow growth for more interfaces in the future.
  */
-#define GSC_IRQ_INTF(_x)  BIT(15 - (_x))
+#define GSC_IRQ_INTF(_x) BIT(15 - (_x))
+/*
+ * CSC HECI1 bit corresponds to bit9 and HECI2 to bit10.
+ */
+#define CSC_IRQ_INTF(_x) BIT(9 + (_x))
 /**
  * struct xe_heci_gsc - graphics security controller for xe, HECI interface
 void xe_heci_gsc_init(struct xe_device *xe);
 void xe_heci_gsc_fini(struct xe_device *xe);
 void xe_heci_gsc_irq_handler(struct xe_device *xe, u32 iir);
+void xe_heci_csc_irq_handler(struct xe_device *xe, u32 iir);
 #endif /* __XE_HECI_GSC_DEV_H__ */

drivers/gpu/drm/xe/xe_irq.c+2−0

 		 * the primary tile.
 		 */
 		if (id == 0) {
+			if (HAS_HECI_CSCFI(xe))
+				xe_heci_csc_irq_handler(xe, master_ctl);
 			xe_display_irq_handler(xe, master_ctl);
 			gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
 		}

drivers/gpu/drm/xe/xe_lmtt.c+2−2

@@ -7,7 +7,7 @@ · 6 unmodified lines
7		7
8	#include <drm/drm_managed.h>	8	#include <drm/drm_managed.h>
9		9
10	#include "regs/xe_sriov_regs.h"	10	#include "regs/xe_gt_regs.h"
11		11
12	#include "xe_assert.h"	12	#include "xe_assert.h"
13	#include "xe_bo.h"	13	#include "xe_bo.h"
@@ -71,7 +71,7 @@ static struct xe_lmtt_pt lmtt_pt_alloc(struct xe_lmtt lmtt, unsigned int level · 57 unmodified lines
71	lmtt->ops->lmtt_pte_num(level)),	71	lmtt->ops->lmtt_pte_num(level)),
72	ttm_bo_type_kernel,	72	ttm_bo_type_kernel,
73	XE_BO_FLAG_VRAM_IF_DGFX(lmtt_to_tile(lmtt)) \|	73	XE_BO_FLAG_VRAM_IF_DGFX(lmtt_to_tile(lmtt)) \|
74	XE_BO_NEEDS_64K \| XE_BO_FLAG_PINNED);	74	XE_BO_FLAG_NEEDS_64K \| XE_BO_FLAG_PINNED);
75	if (IS_ERR(bo)) {	75	if (IS_ERR(bo)) {
76	err = PTR_ERR(bo);	76	err = PTR_ERR(bo);
77	goto out_free_pt;	77	goto out_free_pt;

drivers/gpu/drm/xe/xe_migrate.c+290−238

@@ -73,6 +73,7 @@ struct xe_migrate { · 72 unmodified lines
73	#define NUM_PT_SLOTS 32	73	#define NUM_PT_SLOTS 32
74	#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M	74	#define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M
75	#define MAX_NUM_PTE 512	75	#define MAX_NUM_PTE 512
		76	#define IDENTITY_OFFSET 256ULL
76		77
77	/*	78	/*
78	* Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest	79	* Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest
@@ -84,15 +85,14 @@ struct xe_migrate { · 5 unmodified lines
84	#define MAX_PTE_PER_SDI 0x1FE	85	#define MAX_PTE_PER_SDI 0x1FE
85		86
86	/**	87	/**
87	* xe_tile_migrate_engine() - Get this tile's migrate engine.	88	* xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue.
88	* @tile: The tile.	89	* @tile: The tile.
89	*	90	*
90	* Returns the default migrate engine of this tile.	91	* Returns the default migrate exec queue of this tile.
91	* TODO: Perhaps this function is slightly misplaced, and even unneeded?
92	*	92	*
93	* Return: The default migrate engine	93	* Return: The default migrate exec queue
94	*/	94	*/
95	struct xe_exec_queue xe_tile_migrate_engine(struct xe_tile tile)	95	struct xe_exec_queue xe_tile_migrate_exec_queue(struct xe_tile tile)
96	{	96	{
97	return tile->migrate->q;	97	return tile->migrate->q;
98	}	98	}
@@ -121,14 +121,64 @@ static u64 xe_migrate_vm_addr(u64 slot, u32 level) · 22 unmodified lines
121	return (slot + 1ULL) << xe_pt_shift(level + 1);	121	return (slot + 1ULL) << xe_pt_shift(level + 1);
122	}	122	}
123		123
124	static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr)	124	static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte)
125	{	125	{
126	/*	126	/*
127	* Remove the DPA to get a correct offset into identity table for the	127	* Remove the DPA to get a correct offset into identity table for the
128	* migrate offset	128	* migrate offset
129	*/	129	*/
		130	u64 identity_offset = IDENTITY_OFFSET;
		131
		132	if (GRAPHICS_VER(xe) >= 20 && is_comp_pte)
		133	identity_offset += DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G);
		134
130	addr -= xe->mem.vram.dpa_base;	135	addr -= xe->mem.vram.dpa_base;
131	return addr + (256ULL << xe_pt_shift(2));	136	return addr + (identity_offset << xe_pt_shift(2));
		137	}
		138
		139	static void xe_migrate_program_identity(struct xe_device xe, struct xe_vm vm, struct xe_bo *bo,
		140	u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs)
		141	{
		142	u64 pos, ofs, flags;
		143	u64 entry;
		144	/* XXX: Unclear if this should be usable_size? */
		145	u64 vram_limit = xe->mem.vram.actual_physical_size +
		146	xe->mem.vram.dpa_base;
		147	u32 level = 2;
		148
		149	ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8;
		150	flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level,
		151	true, 0);
		152
		153	xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M));
		154
		155	/*
		156	* Use 1GB pages when possible, last chunk always use 2M
		157	* pages as mixing reserved memory (stolen, WOCPM) with a single
		158	* mapping is not allowed on certain platforms.
		159	*/
		160	for (pos = xe->mem.vram.dpa_base; pos < vram_limit;
		161	pos += SZ_1G, ofs += 8) {
		162	if (pos + SZ_1G >= vram_limit) {
		163	entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs,
		164	pat_index);
		165	xe_map_wr(xe, &bo->vmap, ofs, u64, entry);
		166
		167	flags = vm->pt_ops->pte_encode_addr(xe, 0,
		168	pat_index,
		169	level - 1,
		170	true, 0);
		171
		172	for (ofs = pt_2m_ofs; pos < vram_limit;
		173	pos += SZ_2M, ofs += 8)
		174	xe_map_wr(xe, &bo->vmap, ofs, u64, pos \| flags);
		175	break; /* Ensure pos == vram_limit assert correct */
		176	}
		177
		178	xe_map_wr(xe, &bo->vmap, ofs, u64, pos \| flags);
		179	}
		180
		181	xe_assert(xe, pos == vram_limit);
132	}	182	}
133		183
134	static int xe_migrate_prepare_vm(struct xe_tile tile, struct xe_migrate m,	184	static int xe_migrate_prepare_vm(struct xe_tile tile, struct xe_migrate m,
@@ -137,11 +187,13 @@ static int xe_migrate_prepare_vm(struct xe_tile tile, struct xe_migrate m, · 2 unmodified lines
137	struct xe_device *xe = tile_to_xe(tile);	187	struct xe_device *xe = tile_to_xe(tile);
138	u16 pat_index = xe->pat.idx[XE_CACHE_WB];	188	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
139	u8 id = tile->id;	189	u8 id = tile->id;
140	u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level,	190	u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;
141	num_setup = num_level + 1;	191	#define VRAM_IDENTITY_MAP_COUNT 2
		192	u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT;
		193	#undef VRAM_IDENTITY_MAP_COUNT
142	u32 map_ofs, level, i;	194	u32 map_ofs, level, i;
143	struct xe_bo bo, batch = tile->mem.kernel_bb_pool->bo;	195	struct xe_bo bo, batch = tile->mem.kernel_bb_pool->bo;
144	u64 entry, pt30_ofs;	196	u64 entry, pt29_ofs;
145		197
146	/* Can't bump NUM_PT_SLOTS too high */	198	/* Can't bump NUM_PT_SLOTS too high */
147	BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);	199	BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);
@@ -161,9 +213,9 @@ static int xe_migrate_prepare_vm(struct xe_tile tile, struct xe_migrate m, · 13 unmodified lines
161	if (IS_ERR(bo))	213	if (IS_ERR(bo))
162	return PTR_ERR(bo);	214	return PTR_ERR(bo);
163		215
164	/* PT31 reserved for 2M identity map */	216	/* PT30 & PT31 reserved for 2M identity map */
165	pt30_ofs = bo->size - 2 * XE_PAGE_SIZE;	217	pt29_ofs = bo->size - 3 * XE_PAGE_SIZE;
166	entry = vm->pt_ops->pde_encode_bo(bo, pt30_ofs, pat_index);	218	entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs, pat_index);
167	xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry);	219	xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry);
168		220
169	map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE;	221	map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE;
@@ -215,12 +267,12 @@ static int xe_migrate_prepare_vm(struct xe_tile tile, struct xe_migrate m, · 45 unmodified lines
215	} else {	267	} else {
216	u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);	268	u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);
217		269
218	m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr);	270	m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false);
219		271
220	if (xe->info.has_usm) {	272	if (xe->info.has_usm) {
221	batch = tile->primary_gt->usm.bb_pool->bo;	273	batch = tile->primary_gt->usm.bb_pool->bo;
222	batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);	274	batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE);
223	m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr);	275	m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false);
224	}	276	}
225	}	277	}
226		278
@@ -254,55 +306,36 @@ static int xe_migrate_prepare_vm(struct xe_tile tile, struct xe_migrate m, · 27 unmodified lines
254		306
255	/* Identity map the entire vram at 256GiB offset */	307	/* Identity map the entire vram at 256GiB offset */
256	if (IS_DGFX(xe)) {	308	if (IS_DGFX(xe)) {
257	u64 pos, ofs, flags;	309	u64 pt30_ofs = bo->size - 2 * XE_PAGE_SIZE;
258	/* XXX: Unclear if this should be usable_size? */
259	u64 vram_limit = xe->mem.vram.actual_physical_size +
260	xe->mem.vram.dpa_base;
261		310
262	level = 2;	311	xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET,
263	ofs = map_ofs + XE_PAGE_SIZE * level + 256 * 8;	312	pat_index, pt30_ofs);
264	flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level,	313	xe_assert(xe, xe->mem.vram.actual_physical_size <=
265	true, 0);	314	(MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G);
266
267	xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M));
268		315
269	/*	316	/*
270	* Use 1GB pages when possible, last chunk always use 2M	317	* Identity map the entire vram for compressed pat_index for xe2+
271	* pages as mixing reserved memory (stolen, WOCPM) with a single	318	* if flat ccs is enabled.
272	* mapping is not allowed on certain platforms.
273	*/	319	*/
274	for (pos = xe->mem.vram.dpa_base; pos < vram_limit;	320	if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) {
275	pos += SZ_1G, ofs += 8) {	321	u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION];
276	if (pos + SZ_1G >= vram_limit) {	322	u64 vram_offset = IDENTITY_OFFSET +
277	u64 pt31_ofs = bo->size - XE_PAGE_SIZE;	323	DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G);
278		324	u64 pt31_ofs = bo->size - XE_PAGE_SIZE;
279	entry = vm->pt_ops->pde_encode_bo(bo, pt31_ofs,	325
280	pat_index);	326	xe_assert(xe, xe->mem.vram.actual_physical_size <= (MAX_NUM_PTE -
281	xe_map_wr(xe, &bo->vmap, ofs, u64, entry);	327	IDENTITY_OFFSET - IDENTITY_OFFSET / 2) * SZ_1G);
282		328	xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset,
283	flags = vm->pt_ops->pte_encode_addr(xe, 0,	329	comp_pat_index, pt31_ofs);
284	pat_index,
285	level - 1,
286	true, 0);
287
288	for (ofs = pt31_ofs; pos < vram_limit;
289	pos += SZ_2M, ofs += 8)
290	xe_map_wr(xe, &bo->vmap, ofs, u64, pos \| flags);
291	break; /* Ensure pos == vram_limit assert correct */
292	}
293
294	xe_map_wr(xe, &bo->vmap, ofs, u64, pos \| flags);
295	}	330	}
296
297	xe_assert(xe, pos == vram_limit);
298	}	331	}
299		332
300	/*	333	/*
301	* Example layout created above, with root level = 3:	334	* Example layout created above, with root level = 3:
302	* [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's	335	* [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's
303	* [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's	336	* [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's
304	* [PT9...PT27]: Userspace PT's for VM_BIND, 4 KiB PTE's	337	* [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's
305	* [PT28 = PDE 0] [PT29 = PDE 1] [PT30 = PDE 2] [PT31 = 2M vram identity map]	338	* [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map]
306	*	339	*
307	* This makes the lowest part of the VM point to the pagetables.	340	* This makes the lowest part of the VM point to the pagetables.
308	* Hence the lowest 2M in the vm should point to itself, with a few writes	341	* Hence the lowest 2M in the vm should point to itself, with a few writes
@@ -348,6 +381,11 @@ static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt) · 39 unmodified lines
348	return logical_mask;	381	return logical_mask;
349	}	382	}
350		383
		384	static bool xe_migrate_needs_ccs_emit(struct xe_device *xe)
		385	{
		386	return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe));
		387	}
		388
351	/**	389	/**
352	* xe_migrate_init() - Initialize a migrate context	390	* xe_migrate_init() - Initialize a migrate context
353	* @tile: Back-pointer to the tile we're initializing for.	391	* @tile: Back-pointer to the tile we're initializing for.
@@ -421,7 +459,7 @@ struct xe_migrate xe_migrate_init(struct xe_tile tile) · 67 unmodified lines
421	return ERR_PTR(err);	459	return ERR_PTR(err);
422		460
423	if (IS_DGFX(xe)) {	461	if (IS_DGFX(xe)) {
424	if (xe_device_has_flat_ccs(xe))	462	if (xe_migrate_needs_ccs_emit(xe))
425	/* min chunk size corresponds to 4K of CCS Metadata */	463	/* min chunk size corresponds to 4K of CCS Metadata */
426	m->min_chunk_size = SZ_4K * SZ_64K /	464	m->min_chunk_size = SZ_4K * SZ_64K /
427	xe_device_ccs_bytes(xe, SZ_64K);	465	xe_device_ccs_bytes(xe, SZ_64K);
@@ -475,20 +513,26 @@ static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur) · 47 unmodified lines
475	return cur->size >= size;	513	return cur->size >= size;
476	}	514	}
477		515
		516	#define PTE_UPDATE_FLAG_IS_VRAM BIT(0)
		517	#define PTE_UPDATE_FLAG_IS_COMP_PTE BIT(1)
		518
478	static u32 pte_update_size(struct xe_migrate *m,	519	static u32 pte_update_size(struct xe_migrate *m,
479	bool is_vram,	520	u32 flags,
480	struct ttm_resource *res,	521	struct ttm_resource *res,
481	struct xe_res_cursor *cur,	522	struct xe_res_cursor *cur,
482	u64 L0, u64 L0_ofs, u32 *L0_pt,	523	u64 L0, u64 L0_ofs, u32 *L0_pt,
483	u32 cmd_size, u32 pt_ofs, u32 avail_pts)	524	u32 cmd_size, u32 pt_ofs, u32 avail_pts)
484	{	525	{
485	u32 cmds = 0;	526	u32 cmds = 0;
		527	bool is_vram = PTE_UPDATE_FLAG_IS_VRAM & flags;
		528	bool is_comp_pte = PTE_UPDATE_FLAG_IS_COMP_PTE & flags;
486		529
487	*L0_pt = pt_ofs;	530	*L0_pt = pt_ofs;
488	if (is_vram && xe_migrate_allow_identity(*L0, cur)) {	531	if (is_vram && xe_migrate_allow_identity(*L0, cur)) {
489	/* Offset into identity map. */	532	/* Offset into identity map. */
490	*L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile),	533	*L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile),
491	cur->start + vram_region_gpu_offset(res));	534	cur->start + vram_region_gpu_offset(res),
		535	is_comp_pte);
492	cmds += cmd_size;	536	cmds += cmd_size;
493	} else {	537	} else {
494	/* Clip L0 to available size */	538	/* Clip L0 to available size */
@@ -661,7 +705,7 @@ static u32 xe_migrate_ccs_copy(struct xe_migrate *m, · 166 unmodified lines
661	struct xe_gt *gt = m->tile->primary_gt;	705	struct xe_gt *gt = m->tile->primary_gt;
662	u32 flush_flags = 0;	706	u32 flush_flags = 0;
663		707
664	if (xe_device_has_flat_ccs(gt_to_xe(gt)) && !copy_ccs && dst_is_indirect) {	708	if (!copy_ccs && dst_is_indirect) {
665	/*	709	/*
666	* If the src is already in vram, then it should already	710	* If the src is already in vram, then it should already
667	* have been cleared by us, or has been populated by the	711	* have been cleared by us, or has been populated by the
@@ -737,6 +781,8 @@ struct dma_fence xe_migrate_copy(struct xe_migrate m, · 69 unmodified lines
737	bool copy_ccs = xe_device_has_flat_ccs(xe) &&	781	bool copy_ccs = xe_device_has_flat_ccs(xe) &&
738	xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo);	782	xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo);
739	bool copy_system_ccs = copy_ccs && (!src_is_vram \|\| !dst_is_vram);	783	bool copy_system_ccs = copy_ccs && (!src_is_vram \|\| !dst_is_vram);
		784	bool use_comp_pat = xe_device_has_flat_ccs(xe) &&
		785	GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram;
740		786
741	/* Copying CCS between two different BOs is not supported yet. */	787	/* Copying CCS between two different BOs is not supported yet. */
742	if (XE_WARN_ON(copy_ccs && src_bo != dst_bo))	788	if (XE_WARN_ON(copy_ccs && src_bo != dst_bo))
@@ -763,10 +809,11 @@ struct dma_fence xe_migrate_copy(struct xe_migrate m, · 20 unmodified lines
763	u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */	809	u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */
764	struct xe_sched_job *job;	810	struct xe_sched_job *job;
765	struct xe_bb *bb;	811	struct xe_bb *bb;
766	u32 flush_flags;	812	u32 flush_flags = 0;
767	u32 update_idx;	813	u32 update_idx;
768	u64 ccs_ofs, ccs_size;	814	u64 ccs_ofs, ccs_size;
769	u32 ccs_pt;	815	u32 ccs_pt;
		816	u32 pte_flags;
770		817
771	bool usm = xe->info.has_usm;	818	bool usm = xe->info.has_usm;
772	u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;	819	u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
@@ -779,17 +826,20 @@ struct dma_fence xe_migrate_copy(struct xe_migrate m, · 6 unmodified lines
779		826
780	src_L0 = min(src_L0, dst_L0);	827	src_L0 = min(src_L0, dst_L0);
781		828
782	batch_size += pte_update_size(m, src_is_vram, src, &src_it, &src_L0,	829	pte_flags = src_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
		830	pte_flags \|= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0;
		831	batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0,
783	&src_L0_ofs, &src_L0_pt, 0, 0,	832	&src_L0_ofs, &src_L0_pt, 0, 0,
784	avail_pts);	833	avail_pts);
785		834
786	batch_size += pte_update_size(m, dst_is_vram, dst, &dst_it, &src_L0,	835	pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
		836	batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0,
787	&dst_L0_ofs, &dst_L0_pt, 0,	837	&dst_L0_ofs, &dst_L0_pt, 0,
788	avail_pts, avail_pts);	838	avail_pts, avail_pts);
789		839
790	if (copy_system_ccs) {	840	if (copy_system_ccs) {
791	ccs_size = xe_device_ccs_bytes(xe, src_L0);	841	ccs_size = xe_device_ccs_bytes(xe, src_L0);
792	batch_size += pte_update_size(m, false, NULL, &ccs_it, &ccs_size,	842	batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size,
793	&ccs_ofs, &ccs_pt, 0,	843	&ccs_ofs, &ccs_pt, 0,
794	2 * avail_pts,	844	2 * avail_pts,
795	avail_pts);	845	avail_pts);
@@ -798,7 +848,7 @@ struct dma_fence xe_migrate_copy(struct xe_migrate m, · 2 unmodified lines
798		848
799	/* Add copy commands size here */	849	/* Add copy commands size here */
800	batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +	850	batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +
801	((xe_device_has_flat_ccs(xe) ? EMIT_COPY_CCS_DW : 0));	851	((xe_migrate_needs_ccs_emit(xe) ? EMIT_COPY_CCS_DW : 0));
802		852
803	bb = xe_bb_new(gt, batch_size, usm);	853	bb = xe_bb_new(gt, batch_size, usm);
804	if (IS_ERR(bb)) {	854	if (IS_ERR(bb)) {
@@ -827,11 +877,12 @@ struct dma_fence xe_migrate_copy(struct xe_migrate m, · 22 unmodified lines
827	if (!copy_only_ccs)	877	if (!copy_only_ccs)
828	emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);	878	emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);
829		879
830	flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,	880	if (xe_migrate_needs_ccs_emit(xe))
831	IS_DGFX(xe) ? src_is_vram : src_is_pltt,	881	flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,
832	dst_L0_ofs,	882	IS_DGFX(xe) ? src_is_vram : src_is_pltt,
833	IS_DGFX(xe) ? dst_is_vram : dst_is_pltt,	883	dst_L0_ofs,
834	src_L0, ccs_ofs, copy_ccs);	884	IS_DGFX(xe) ? dst_is_vram : dst_is_pltt,
		885	src_L0, ccs_ofs, copy_ccs);
835		886
836	job = xe_bb_create_migration_job(m->q, bb,	887	job = xe_bb_create_migration_job(m->q, bb,
837	xe_migrate_batch_base(m, usm),	888	xe_migrate_batch_base(m, usm),
@@ -1022,6 +1073,7 @@ struct dma_fence xe_migrate_clear(struct xe_migrate m, · 184 unmodified lines
1022	struct xe_sched_job *job;	1073	struct xe_sched_job *job;
1023	struct xe_bb *bb;	1074	struct xe_bb *bb;
1024	u32 batch_size, update_idx;	1075	u32 batch_size, update_idx;
		1076	u32 pte_flags;
1025		1077
1026	bool usm = xe->info.has_usm;	1078	bool usm = xe->info.has_usm;
1027	u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;	1079	u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
@@ -1029,13 +1081,14 @@ struct dma_fence xe_migrate_clear(struct xe_migrate m, · 1 unmodified line
1029	clear_L0 = xe_migrate_res_sizes(m, &src_it);	1081	clear_L0 = xe_migrate_res_sizes(m, &src_it);
1030		1082
1031	/* Calculate final sizes and batch size.. */	1083	/* Calculate final sizes and batch size.. */
		1084	pte_flags = clear_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
1032	batch_size = 2 +	1085	batch_size = 2 +
1033	pte_update_size(m, clear_vram, src, &src_it,	1086	pte_update_size(m, pte_flags, src, &src_it,
1034	&clear_L0, &clear_L0_ofs, &clear_L0_pt,	1087	&clear_L0, &clear_L0_ofs, &clear_L0_pt,
1035	clear_system_ccs ? 0 : emit_clear_cmd_len(gt), 0,	1088	clear_system_ccs ? 0 : emit_clear_cmd_len(gt), 0,
1036	avail_pts);	1089	avail_pts);
1037		1090
1038	if (xe_device_has_flat_ccs(xe))	1091	if (xe_migrate_needs_ccs_emit(xe))
1039	batch_size += EMIT_COPY_CCS_DW;	1092	batch_size += EMIT_COPY_CCS_DW;
1040		1093
1041	/* Clear commands */	1094	/* Clear commands */
@@ -1063,7 +1116,7 @@ struct dma_fence xe_migrate_clear(struct xe_migrate m, · 21 unmodified lines
1063	if (!clear_system_ccs)	1116	if (!clear_system_ccs)
1064	emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram);	1117	emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram);
1065		1118
1066	if (xe_device_has_flat_ccs(xe)) {	1119	if (xe_migrate_needs_ccs_emit(xe)) {
1067	emit_copy_ccs(gt, bb, clear_L0_ofs, true,	1120	emit_copy_ccs(gt, bb, clear_L0_ofs, true,
1068	m->cleared_mem_ofs, false, clear_L0);	1121	m->cleared_mem_ofs, false, clear_L0);
1069	flush_flags = MI_FLUSH_DW_CCS;	1122	flush_flags = MI_FLUSH_DW_CCS;
@@ -1126,6 +1179,7 @@ err_sync: · 56 unmodified lines
1126	}	1179	}
1127		1180
1128	static void write_pgtable(struct xe_tile tile, struct xe_bb bb, u64 ppgtt_ofs,	1181	static void write_pgtable(struct xe_tile tile, struct xe_bb bb, u64 ppgtt_ofs,
		1182	const struct xe_vm_pgtable_update_op *pt_op,
1129	const struct xe_vm_pgtable_update *update,	1183	const struct xe_vm_pgtable_update *update,
1130	struct xe_migrate_pt_update *pt_update)	1184	struct xe_migrate_pt_update *pt_update)
1131	{	1185	{
@@ -1146,7 +1200,7 @@ static void write_pgtable(struct xe_tile tile, struct xe_bb bb, u64 ppgtt_ofs, · 14 unmodified lines
1146	if (!ppgtt_ofs)	1200	if (!ppgtt_ofs)
1147	ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),	1201	ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),
1148	xe_bo_addr(update->pt_bo, 0,	1202	xe_bo_addr(update->pt_bo, 0,
1149	XE_PAGE_SIZE));	1203	XE_PAGE_SIZE), false);
1150		1204
1151	do {	1205	do {
1152	u64 addr = ppgtt_ofs + ofs * 8;	1206	u64 addr = ppgtt_ofs + ofs * 8;
@@ -1160,8 +1214,12 @@ static void write_pgtable(struct xe_tile tile, struct xe_bb bb, u64 ppgtt_ofs, · 7 unmodified lines
1160	bb->cs[bb->len++] = MI_STORE_DATA_IMM \| MI_SDI_NUM_QW(chunk);	1214	bb->cs[bb->len++] = MI_STORE_DATA_IMM \| MI_SDI_NUM_QW(chunk);
1161	bb->cs[bb->len++] = lower_32_bits(addr);	1215	bb->cs[bb->len++] = lower_32_bits(addr);
1162	bb->cs[bb->len++] = upper_32_bits(addr);	1216	bb->cs[bb->len++] = upper_32_bits(addr);
1163	ops->populate(pt_update, tile, NULL, bb->cs + bb->len, ofs, chunk,	1217	if (pt_op->bind)
1164	update);	1218	ops->populate(pt_update, tile, NULL, bb->cs + bb->len,
		1219	ofs, chunk, update);
		1220	else
		1221	ops->clear(pt_update, tile, NULL, bb->cs + bb->len,
		1222	ofs, chunk, update);
1165		1223
1166	bb->len += chunk * 2;	1224	bb->len += chunk * 2;
1167	ofs += chunk;	1225	ofs += chunk;
@@ -1186,114 +1244,58 @@ struct migrate_test_params { · 18 unmodified lines
1186		1244
1187	static struct dma_fence *	1245	static struct dma_fence *
1188	xe_migrate_update_pgtables_cpu(struct xe_migrate *m,	1246	xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
1189	struct xe_vm vm, struct xe_bo bo,
1190	const struct xe_vm_pgtable_update *updates,
1191	u32 num_updates, bool wait_vm,
1192	struct xe_migrate_pt_update *pt_update)	1247	struct xe_migrate_pt_update *pt_update)
1193	{	1248	{
1194	XE_TEST_DECLARE(struct migrate_test_params *test =	1249	XE_TEST_DECLARE(struct migrate_test_params *test =
1195	to_migrate_test_params	1250	to_migrate_test_params
1196	(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)	1251	(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)
1197	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;	1252	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1198	struct dma_fence *fence;	1253	struct xe_vm *vm = pt_update->vops->vm;
		1254	struct xe_vm_pgtable_update_ops *pt_update_ops =
		1255	&pt_update->vops->pt_update_ops[pt_update->tile_id];
1199	int err;	1256	int err;
1200	u32 i;	1257	u32 i, j;
1201		1258
1202	if (XE_TEST_ONLY(test && test->force_gpu))	1259	if (XE_TEST_ONLY(test && test->force_gpu))
1203	return ERR_PTR(-ETIME);	1260	return ERR_PTR(-ETIME);
1204		1261
1205	if (bo && !dma_resv_test_signaled(bo->ttm.base.resv,
1206	DMA_RESV_USAGE_KERNEL))
1207	return ERR_PTR(-ETIME);
1208
1209	if (wait_vm && !dma_resv_test_signaled(xe_vm_resv(vm),
1210	DMA_RESV_USAGE_BOOKKEEP))
1211	return ERR_PTR(-ETIME);
1212
1213	if (ops->pre_commit) {	1262	if (ops->pre_commit) {
1214	pt_update->job = NULL;	1263	pt_update->job = NULL;
1215	err = ops->pre_commit(pt_update);	1264	err = ops->pre_commit(pt_update);
1216	if (err)	1265	if (err)
1217	return ERR_PTR(err);	1266	return ERR_PTR(err);
1218	}	1267	}
1219	for (i = 0; i < num_updates; i++) {
1220	const struct xe_vm_pgtable_update *update = &updates[i];
1221
1222	ops->populate(pt_update, m->tile, &update->pt_bo->vmap, NULL,
1223	update->ofs, update->qwords, update);
1224	}
1225
1226	if (vm) {
1227	trace_xe_vm_cpu_bind(vm);
1228	xe_device_wmb(vm->xe);
1229	}
1230
1231	fence = dma_fence_get_stub();
1232
1233	return fence;
1234	}
1235
1236	static bool no_in_syncs(struct xe_vm vm, struct xe_exec_queue q,
1237	struct xe_sync_entry *syncs, u32 num_syncs)
1238	{
1239	struct dma_fence *fence;
1240	int i;
1241		1268
1242	for (i = 0; i < num_syncs; i++) {	1269	for (i = 0; i < pt_update_ops->num_ops; ++i) {
1243	fence = syncs[i].fence;	1270	const struct xe_vm_pgtable_update_op *pt_op =
1244		1271	&pt_update_ops->ops[i];
1245	if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,	1272
1246	&fence->flags))	1273	for (j = 0; j < pt_op->num_entries; j++) {
1247	return false;	1274	const struct xe_vm_pgtable_update *update =
1248	}	1275	&pt_op->entries[j];
1249	if (q) {	1276
1250	fence = xe_exec_queue_last_fence_get(q, vm);	1277	if (pt_op->bind)
1251	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {	1278	ops->populate(pt_update, m->tile,
1252	dma_fence_put(fence);	1279	&update->pt_bo->vmap, NULL,
1253	return false;	1280	update->ofs, update->qwords,
		1281	update);
		1282	else
		1283	ops->clear(pt_update, m->tile,
		1284	&update->pt_bo->vmap, NULL,
		1285	update->ofs, update->qwords, update);
1254	}	1286	}
1255	dma_fence_put(fence);
1256	}	1287	}
1257		1288
1258	return true;	1289	trace_xe_vm_cpu_bind(vm);
		1290	xe_device_wmb(vm->xe);
		1291
		1292	return dma_fence_get_stub();
1259	}	1293	}
1260		1294
1261	/**	1295	static struct dma_fence *
1262	* xe_migrate_update_pgtables() - Pipelined page-table update	1296	__xe_migrate_update_pgtables(struct xe_migrate *m,
1263	* @m: The migrate context.	1297	struct xe_migrate_pt_update *pt_update,
1264	* @vm: The vm we'll be updating.	1298	struct xe_vm_pgtable_update_ops *pt_update_ops)
1265	* @bo: The bo whose dma-resv we will await before updating, or NULL if userptr.
1266	* @q: The exec queue to be used for the update or NULL if the default
1267	* migration engine is to be used.
1268	* @updates: An array of update descriptors.
1269	* @num_updates: Number of descriptors in @updates.
1270	* @syncs: Array of xe_sync_entry to await before updating. Note that waits
1271	* will block the engine timeline.
1272	* @num_syncs: Number of entries in @syncs.
1273	* @pt_update: Pointer to a struct xe_migrate_pt_update, which contains
1274	* pointers to callback functions and, if subclassed, private arguments to
1275	* those.
1276	*
1277	* Perform a pipelined page-table update. The update descriptors are typically
1278	* built under the same lock critical section as a call to this function. If
1279	* using the default engine for the updates, they will be performed in the
1280	* order they grab the job_mutex. If different engines are used, external
1281	* synchronization is needed for overlapping updates to maintain page-table
1282	* consistency. Note that the meaing of "overlapping" is that the updates
1283	* touch the same page-table, which might be a higher-level page-directory.
1284	* If no pipelining is needed, then updates may be performed by the cpu.
1285	*
1286	* Return: A dma_fence that, when signaled, indicates the update completion.
1287	*/
1288	struct dma_fence *
1289	xe_migrate_update_pgtables(struct xe_migrate *m,
1290	struct xe_vm *vm,
1291	struct xe_bo *bo,
1292	struct xe_exec_queue *q,
1293	const struct xe_vm_pgtable_update *updates,
1294	u32 num_updates,
1295	struct xe_sync_entry *syncs, u32 num_syncs,
1296	struct xe_migrate_pt_update *pt_update)
1297	{	1299	{
1298	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;	1300	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
1299	struct xe_tile *tile = m->tile;	1301	struct xe_tile *tile = m->tile;
@@ -1302,59 +1304,53 @@ xe_migrate_update_pgtables(struct xe_migrate *m, · 2 unmodified lines
1302	struct xe_sched_job *job;	1304	struct xe_sched_job *job;
1303	struct dma_fence *fence;	1305	struct dma_fence *fence;
1304	struct drm_suballoc *sa_bo = NULL;	1306	struct drm_suballoc *sa_bo = NULL;
1305	struct xe_vma *vma = pt_update->vma;
1306	struct xe_bb *bb;	1307	struct xe_bb *bb;
1307	u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0;	1308	u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0;
		1309	u32 num_updates = 0, current_update = 0;
1308	u64 addr;	1310	u64 addr;
1309	int err = 0;	1311	int err = 0;
1310	bool usm = !q && xe->info.has_usm;	1312	bool is_migrate = pt_update_ops->q == m->q;
1311	bool first_munmap_rebind = vma &&	1313	bool usm = is_migrate && xe->info.has_usm;
1312	vma->gpuva.flags & XE_VMA_FIRST_REBIND;	1314
1313	struct xe_exec_queue *q_override = !q ? m->q : q;	1315	for (i = 0; i < pt_update_ops->num_ops; ++i) {
1314	u16 pat_index = xe->pat.idx[XE_CACHE_WB];	1316	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
		1317	struct xe_vm_pgtable_update *updates = pt_op->entries;
1315		1318
1316	/* Use the CPU if no in syncs and engine is idle */	1319	num_updates += pt_op->num_entries;
1317	if (no_in_syncs(vm, q, syncs, num_syncs) && xe_exec_queue_is_idle(q_override)) {	1320	for (j = 0; j < pt_op->num_entries; ++j) {
1318	fence = xe_migrate_update_pgtables_cpu(m, vm, bo, updates,	1321	u32 num_cmds = DIV_ROUND_UP(updates[j].qwords,
1319	num_updates,	1322	MAX_PTE_PER_SDI);
1320	first_munmap_rebind,	1323
1321	pt_update);	1324	/* align noop + MI_STORE_DATA_IMM cmd prefix */
1322	if (!IS_ERR(fence) \|\| fence == ERR_PTR(-EAGAIN))	1325	batch_size += 4 * num_cmds + updates[j].qwords * 2;
1323	return fence;	1326	}
1324	}	1327	}
1325		1328
1326	/* fixed + PTE entries */	1329	/* fixed + PTE entries */
1327	if (IS_DGFX(xe))	1330	if (IS_DGFX(xe))
1328	batch_size = 2;	1331	batch_size += 2;
1329	else	1332	else
1330	batch_size = 6 + num_updates * 2;	1333	batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) +
1331		1334	num_updates * 2;
1332	for (i = 0; i < num_updates; i++) {
1333	u32 num_cmds = DIV_ROUND_UP(updates[i].qwords, MAX_PTE_PER_SDI);
1334
1335	/* align noop + MI_STORE_DATA_IMM cmd prefix */
1336	batch_size += 4 * num_cmds + updates[i].qwords * 2;
1337	}
1338
1339	/*
1340	* XXX: Create temp bo to copy from, if batch_size becomes too big?
1341	*
1342	* Worst case: Sum(2 * (each lower level page size) + (top level page size))
1343	* Should be reasonably bound..
1344	*/
1345	xe_tile_assert(tile, batch_size < SZ_128K);
1346		1335
1347	bb = xe_bb_new(gt, batch_size, !q && xe->info.has_usm);	1336	bb = xe_bb_new(gt, batch_size, usm);
1348	if (IS_ERR(bb))	1337	if (IS_ERR(bb))
1349	return ERR_CAST(bb);	1338	return ERR_CAST(bb);
1350		1339
1351	/* For sysmem PTE's, need to map them in our hole.. */	1340	/* For sysmem PTE's, need to map them in our hole.. */
1352	if (!IS_DGFX(xe)) {	1341	if (!IS_DGFX(xe)) {
		1342	u32 ptes, ofs;
		1343
1353	ppgtt_ofs = NUM_KERNEL_PDE - 1;	1344	ppgtt_ofs = NUM_KERNEL_PDE - 1;
1354	if (q) {	1345	if (!is_migrate) {
1355	xe_tile_assert(tile, num_updates <= NUM_VMUSA_WRITES_PER_UNIT);	1346	u32 num_units = DIV_ROUND_UP(num_updates,
		1347	NUM_VMUSA_WRITES_PER_UNIT);
1356		1348
1357	sa_bo = drm_suballoc_new(&m->vm_update_sa, 1,	1349	if (num_units > m->vm_update_sa.size) {
		1350	err = -ENOBUFS;
		1351	goto err_bb;
		1352	}
		1353	sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units,
1358	GFP_KERNEL, true, 0);	1354	GFP_KERNEL, true, 0);
1359	if (IS_ERR(sa_bo)) {	1355	if (IS_ERR(sa_bo)) {
1360	err = PTR_ERR(sa_bo);	1356	err = PTR_ERR(sa_bo);
@@ -1370,18 +1366,49 @@ xe_migrate_update_pgtables(struct xe_migrate *m, · 9 unmodified lines
1370	}	1366	}
1371		1367
1372	/* Map our PT's to gtt */	1368	/* Map our PT's to gtt */
1373	bb->cs[bb->len++] = MI_STORE_DATA_IMM \| MI_SDI_NUM_QW(num_updates);	1369	i = 0;
1374	bb->cs[bb->len++] = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;	1370	j = 0;
1375	bb->cs[bb->len++] = 0; /* upper_32_bits */	1371	ptes = num_updates;
1376		1372	ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;
1377	for (i = 0; i < num_updates; i++) {	1373	while (ptes) {
1378	struct xe_bo *pt_bo = updates[i].pt_bo;	1374	u32 chunk = min(MAX_PTE_PER_SDI, ptes);
		1375	u32 idx = 0;
		1376
		1377	bb->cs[bb->len++] = MI_STORE_DATA_IMM \|
		1378	MI_SDI_NUM_QW(chunk);
		1379	bb->cs[bb->len++] = ofs;
		1380	bb->cs[bb->len++] = 0; /* upper_32_bits */
		1381
		1382	for (; i < pt_update_ops->num_ops; ++i) {
		1383	struct xe_vm_pgtable_update_op *pt_op =
		1384	&pt_update_ops->ops[i];
		1385	struct xe_vm_pgtable_update *updates = pt_op->entries;
		1386
		1387	for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) {
		1388	struct xe_vm *vm = pt_update->vops->vm;
		1389	struct xe_bo *pt_bo = updates[j].pt_bo;
		1390
		1391	if (idx == chunk)
		1392	goto next_cmd;
		1393
		1394	xe_tile_assert(tile, pt_bo->size == SZ_4K);
		1395
		1396	/* Map a PT at most once */
		1397	if (pt_bo->update_index < 0)
		1398	pt_bo->update_index = current_update;
		1399
		1400	addr = vm->pt_ops->pte_encode_bo(pt_bo, 0,
		1401	XE_CACHE_WB, 0);
		1402	bb->cs[bb->len++] = lower_32_bits(addr);
		1403	bb->cs[bb->len++] = upper_32_bits(addr);
		1404	}
1379		1405
1380	xe_tile_assert(tile, pt_bo->size == SZ_4K);	1406	j = 0;
		1407	}
1381		1408
1382	addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, pat_index, 0);	1409	next_cmd:
1383	bb->cs[bb->len++] = lower_32_bits(addr);	1410	ptes -= chunk;
1384	bb->cs[bb->len++] = upper_32_bits(addr);	1411	ofs += chunk * sizeof(u64);
1385	}	1412	}
1386		1413
1387	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;	1414	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
@@ -1389,19 +1416,36 @@ xe_migrate_update_pgtables(struct xe_migrate *m, · 1 unmodified line
1389		1416
1390	addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +	1417	addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
1391	(page_ofs / sizeof(u64)) * XE_PAGE_SIZE;	1418	(page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
1392	for (i = 0; i < num_updates; i++)	1419	for (i = 0; i < pt_update_ops->num_ops; ++i) {
1393	write_pgtable(tile, bb, addr + i * XE_PAGE_SIZE,	1420	struct xe_vm_pgtable_update_op *pt_op =
1394	&updates[i], pt_update);	1421	&pt_update_ops->ops[i];
		1422	struct xe_vm_pgtable_update *updates = pt_op->entries;
		1423
		1424	for (j = 0; j < pt_op->num_entries; ++j) {
		1425	struct xe_bo *pt_bo = updates[j].pt_bo;
		1426
		1427	write_pgtable(tile, bb, addr +
		1428	pt_bo->update_index * XE_PAGE_SIZE,
		1429	pt_op, &updates[j], pt_update);
		1430	}
		1431	}
1395	} else {	1432	} else {
1396	/* phys pages, no preamble required */	1433	/* phys pages, no preamble required */
1397	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;	1434	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
1398	update_idx = bb->len;	1435	update_idx = bb->len;
1399		1436
1400	for (i = 0; i < num_updates; i++)	1437	for (i = 0; i < pt_update_ops->num_ops; ++i) {
1401	write_pgtable(tile, bb, 0, &updates[i], pt_update);	1438	struct xe_vm_pgtable_update_op *pt_op =
		1439	&pt_update_ops->ops[i];
		1440	struct xe_vm_pgtable_update *updates = pt_op->entries;
		1441
		1442	for (j = 0; j < pt_op->num_entries; ++j)
		1443	write_pgtable(tile, bb, 0, pt_op, &updates[j],
		1444	pt_update);
		1445	}
1402	}	1446	}
1403		1447
1404	job = xe_bb_create_migration_job(q ?: m->q, bb,	1448	job = xe_bb_create_migration_job(pt_update_ops->q, bb,
1405	xe_migrate_batch_base(m, usm),	1449	xe_migrate_batch_base(m, usm),
1406	update_idx);	1450	update_idx);
1407	if (IS_ERR(job)) {	1451	if (IS_ERR(job)) {
@@ -1409,46 +1453,20 @@ xe_migrate_update_pgtables(struct xe_migrate *m, · 1 unmodified line
1409	goto err_sa;	1453	goto err_sa;
1410	}	1454	}
1411		1455
1412	/* Wait on BO move */
1413	if (bo) {
1414	err = xe_sched_job_add_deps(job, bo->ttm.base.resv,
1415	DMA_RESV_USAGE_KERNEL);
1416	if (err)
1417	goto err_job;
1418	}
1419
1420	/*
1421	* Munmap style VM unbind, need to wait for all jobs to be complete /
1422	* trigger preempts before moving forward
1423	*/
1424	if (first_munmap_rebind) {
1425	err = xe_sched_job_add_deps(job, xe_vm_resv(vm),
1426	DMA_RESV_USAGE_BOOKKEEP);
1427	if (err)
1428	goto err_job;
1429	}
1430
1431	err = xe_sched_job_last_fence_add_dep(job, vm);
1432	for (i = 0; !err && i < num_syncs; i++)
1433	err = xe_sync_entry_add_deps(&syncs[i], job);
1434
1435	if (err)
1436	goto err_job;
1437
1438	if (ops->pre_commit) {	1456	if (ops->pre_commit) {
1439	pt_update->job = job;	1457	pt_update->job = job;
1440	err = ops->pre_commit(pt_update);	1458	err = ops->pre_commit(pt_update);
1441	if (err)	1459	if (err)
1442	goto err_job;	1460	goto err_job;
1443	}	1461	}
1444	if (!q)	1462	if (is_migrate)
1445	mutex_lock(&m->job_mutex);	1463	mutex_lock(&m->job_mutex);
1446		1464
1447	xe_sched_job_arm(job);	1465	xe_sched_job_arm(job);
1448	fence = dma_fence_get(&job->drm.s_fence->finished);	1466	fence = dma_fence_get(&job->drm.s_fence->finished);
1449	xe_sched_job_push(job);	1467	xe_sched_job_push(job);
1450		1468
1451	if (!q)	1469	if (is_migrate)
1452	mutex_unlock(&m->job_mutex);	1470	mutex_unlock(&m->job_mutex);
1453		1471
1454	xe_bb_free(bb, fence);	1472	xe_bb_free(bb, fence);
@@ -1465,6 +1483,40 @@ err_bb: · 10 unmodified lines
1465	return ERR_PTR(err);	1483	return ERR_PTR(err);
1466	}	1484	}
1467		1485
		1486	/**
		1487	* xe_migrate_update_pgtables() - Pipelined page-table update
		1488	* @m: The migrate context.
		1489	* @pt_update: PT update arguments
		1490	*
		1491	* Perform a pipelined page-table update. The update descriptors are typically
		1492	* built under the same lock critical section as a call to this function. If
		1493	* using the default engine for the updates, they will be performed in the
		1494	* order they grab the job_mutex. If different engines are used, external
		1495	* synchronization is needed for overlapping updates to maintain page-table
		1496	* consistency. Note that the meaing of "overlapping" is that the updates
		1497	* touch the same page-table, which might be a higher-level page-directory.
		1498	* If no pipelining is needed, then updates may be performed by the cpu.
		1499	*
		1500	* Return: A dma_fence that, when signaled, indicates the update completion.
		1501	*/
		1502	struct dma_fence *
		1503	xe_migrate_update_pgtables(struct xe_migrate *m,
		1504	struct xe_migrate_pt_update *pt_update)
		1505
		1506	{
		1507	struct xe_vm_pgtable_update_ops *pt_update_ops =
		1508	&pt_update->vops->pt_update_ops[pt_update->tile_id];
		1509	struct dma_fence *fence;
		1510
		1511	fence = xe_migrate_update_pgtables_cpu(m, pt_update);
		1512
		1513	/* -ETIME indicates a job is needed, anything else is legit error */
		1514	if (!IS_ERR(fence) \|\| PTR_ERR(fence) != -ETIME)
		1515	return fence;
		1516
		1517	return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops);
		1518	}
		1519
1468	/**	1520	/**
1469	* xe_migrate_wait() - Complete all operations using the xe_migrate context	1521	* xe_migrate_wait() - Complete all operations using the xe_migrate context
1470	* @m: Migrate context to wait for.	1522	* @m: Migrate context to wait for.

drivers/gpu/drm/xe/xe_migrate.h+21−13

 			 struct xe_tile *tile, struct iosys_map *map,
 			 void *pos, u32 ofs, u32 num_qwords,
 			 const struct xe_vm_pgtable_update *update);
+	/**
+	 * @clear: Clear a command buffer or page-table with ptes.
+	 * @pt_update: Embeddable callback argument.
+	 * @tile: The tile for the current operation.
+	 * @map: struct iosys_map into the memory to be populated.
+	 * @pos: If @map is NULL, map into the memory to be populated.
+	 * @ofs: qword offset into @map, unused if @map is NULL.
+	 * @num_qwords: Number of qwords to write.
+	 * @update: Information about the PTEs to be inserted.
+	 *
+	 * This interface is intended to be used as a callback into the
+	 * page-table system to populate command buffers or shared
+	 * page-tables with PTEs.
+	 */
+	void (*clear)(struct xe_migrate_pt_update *pt_update,
+		      struct xe_tile *tile, struct iosys_map *map,
+		      void *pos, u32 ofs, u32 num_qwords,
+		      const struct xe_vm_pgtable_update *update);
 	/**
 	 * @pre_commit: Callback to be called just before arming the
 struct xe_migrate_pt_update {
 	/** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */
 	const struct xe_migrate_pt_update_ops *ops;
-	/** @vma: The vma we're updating the pagetable for. */
+	/** @vops: VMA operations */
-	struct xe_vma *vma;
+	struct xe_vma_ops *vops;
 	/** @job: The job if a GPU page-table update. NULL otherwise */
 	struct xe_sched_job *job;
-	/** @start: Start of update for the range fence */
-	u64 start;
-	/** @last: Last of update for the range fence */
-	u64 last;
 	/** @tile_id: Tile ID of the update */
 	u8 tile_id;
 };
 struct dma_fence *
 xe_migrate_update_pgtables(struct xe_migrate *m,
-			   struct xe_vm *vm,
-			   struct xe_bo *bo,
-			   struct xe_exec_queue *q,
-			   const struct xe_vm_pgtable_update *updates,
-			   u32 num_updates,
-			   struct xe_sync_entry *syncs, u32 num_syncs,
 			   struct xe_migrate_pt_update *pt_update);
 void xe_migrate_wait(struct xe_migrate *m);
-struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile);
+struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile);
 #endif

drivers/gpu/drm/xe/xe_mmio.c+144−87

@@ -33,29 +33,56 @@ static void tiles_fini(void *arg) · 32 unmodified lines
33	tile->mmio.regs = NULL;	33	tile->mmio.regs = NULL;
34	}	34	}
35		35
36	int xe_mmio_probe_tiles(struct xe_device *xe)	36	/*
		37	* On multi-tile devices, partition the BAR space for MMIO on each tile,
		38	* possibly accounting for register override on the number of tiles available.
		39	* Resulting memory layout is like below:
		40	*
		41	* .----------------------. <- tile_count * tile_mmio_size
		42	* \| .... \|
		43	* \|----------------------\| <- 2 * tile_mmio_size
		44	* \| tile1->mmio.regs \|
		45	* \|----------------------\| <- 1 * tile_mmio_size
		46	* \| tile0->mmio.regs \|
		47	* '----------------------' <- 0MB
		48	*/
		49	static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size)
37	{	50	{
38	size_t tile_mmio_size = SZ_16M, tile_mmio_ext_size = xe->info.tile_mmio_ext_size;
39	u8 id, tile_count = xe->info.tile_count;
40	struct xe_gt *gt = xe_root_mmio_gt(xe);
41	struct xe_tile *tile;	51	struct xe_tile *tile;
42	void __iomem *regs;	52	void __iomem *regs;
43	u32 mtcfg;	53	u8 id;
44		54
45	if (tile_count == 1)	55	/*
46	goto add_mmio_ext;	56	* Nothing to be done as tile 0 has already been setup earlier with the
		57	* entire BAR mapped - see xe_mmio_init()
		58	*/
		59	if (xe->info.tile_count == 1)
		60	return;
47		61
		62	/* Possibly override number of tile based on configuration register */
48	if (!xe->info.skip_mtcfg) {	63	if (!xe->info.skip_mtcfg) {
		64	struct xe_gt *gt = xe_root_mmio_gt(xe);
		65	u8 tile_count;
		66	u32 mtcfg;
		67
		68	/*
		69	* Although the per-tile mmio regs are not yet initialized, this
		70	* is fine as it's going to the root gt, that's guaranteed to be
		71	* initialized earlier in xe_mmio_init()
		72	*/
49	mtcfg = xe_mmio_read64_2x32(gt, XEHP_MTCFG_ADDR);	73	mtcfg = xe_mmio_read64_2x32(gt, XEHP_MTCFG_ADDR);
50	tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1;	74	tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1;
		75
51	if (tile_count < xe->info.tile_count) {	76	if (tile_count < xe->info.tile_count) {
52	drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n",	77	drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n",
53	xe->info.tile_count, tile_count);	78	xe->info.tile_count, tile_count);
54	xe->info.tile_count = tile_count;	79	xe->info.tile_count = tile_count;
55		80
56	/*	81	/*
57	* FIXME: Needs some work for standalone media, but should be impossible	82	* FIXME: Needs some work for standalone media, but
58	* with multi-tile for now.	83	* should be impossible with multi-tile for now:
		84	* multi-tile platform with standalone media doesn't
		85	* exist
59	*/	86	*/
60	xe->info.gt_count = xe->info.tile_count;	87	xe->info.gt_count = xe->info.tile_count;
61	}	88	}
@@ -67,23 +94,51 @@ int xe_mmio_probe_tiles(struct xe_device *xe) · 5 unmodified lines
67	tile->mmio.regs = regs;	94	tile->mmio.regs = regs;
68	regs += tile_mmio_size;	95	regs += tile_mmio_size;
69	}	96	}
		97	}
70		98
71	add_mmio_ext:	99	/*
72	/*	100	* On top of all the multi-tile MMIO space there can be a platform-dependent
73	* By design, there's a contiguous multi-tile MMIO space (16MB hard coded per tile).	101	* extension for each tile, resulting in a layout like below:
74	* When supported, there could be an additional contiguous multi-tile MMIO extension	102	*
75	* space ON TOP of it, and hence the necessity for distinguished MMIO spaces.	103	* .----------------------. <- ext_base + tile_count * tile_mmio_ext_size
76	*/	104	* \| .... \|
77	if (xe->info.has_mmio_ext) {	105	* \|----------------------\| <- ext_base + 2 * tile_mmio_ext_size
78	regs = xe->mmio.regs + tile_mmio_size * tile_count;	106	* \| tile1->mmio_ext.regs \|
		107	* \|----------------------\| <- ext_base + 1 * tile_mmio_ext_size
		108	* \| tile0->mmio_ext.regs \|
		109	* \|======================\| <- ext_base = tile_count * tile_mmio_size
		110	* \| \|
		111	* \| mmio.regs \|
		112	* \| \|
		113	* '----------------------' <- 0MB
		114	*
		115	* Set up the tile[]->mmio_ext pointers/sizes.
		116	*/
		117	static void mmio_extension_setup(struct xe_device *xe, size_t tile_mmio_size,
		118	size_t tile_mmio_ext_size)
		119	{
		120	struct xe_tile *tile;
		121	void __iomem *regs;
		122	u8 id;
79		123
80	for_each_tile(tile, xe, id) {	124	if (!xe->info.has_mmio_ext)
81	tile->mmio_ext.size = tile_mmio_ext_size;	125	return;
82	tile->mmio_ext.regs = regs;
83		126
84	regs += tile_mmio_ext_size;	127	regs = xe->mmio.regs + tile_mmio_size * xe->info.tile_count;
85	}	128	for_each_tile(tile, xe, id) {
		129	tile->mmio_ext.size = tile_mmio_ext_size;
		130	tile->mmio_ext.regs = regs;
		131	regs += tile_mmio_ext_size;
86	}	132	}
		133	}
		134
		135	int xe_mmio_probe_tiles(struct xe_device *xe)
		136	{
		137	size_t tile_mmio_size = SZ_16M;
		138	size_t tile_mmio_ext_size = xe->info.tile_mmio_ext_size;
		139
		140	mmio_multi_tile_setup(xe, tile_mmio_size);
		141	mmio_extension_setup(xe, tile_mmio_size, tile_mmio_ext_size);
87		142
88	return devm_add_action_or_reset(xe->drm.dev, tiles_fini, xe);	143	return devm_add_action_or_reset(xe->drm.dev, tiles_fini, xe);
89	}	144	}
@@ -121,12 +176,29 @@ int xe_mmio_init(struct xe_device *xe) · 31 unmodified lines
121	return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe);	176	return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe);
122	}	177	}
123		178
		179	static void mmio_flush_pending_writes(struct xe_gt *gt)
		180	{
		181	#define DUMMY_REG_OFFSET 0x130030
		182	struct xe_tile *tile = gt_to_tile(gt);
		183	int i;
		184
		185	if (tile->xe->info.platform != XE_LUNARLAKE)
		186	return;
		187
		188	/* 4 dummy writes */
		189	for (i = 0; i < 4; i++)
		190	writel(0, tile->mmio.regs + DUMMY_REG_OFFSET);
		191	}
		192
124	u8 xe_mmio_read8(struct xe_gt *gt, struct xe_reg reg)	193	u8 xe_mmio_read8(struct xe_gt *gt, struct xe_reg reg)
125	{	194	{
126	struct xe_tile *tile = gt_to_tile(gt);	195	struct xe_tile *tile = gt_to_tile(gt);
127	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);	196	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);
128	u8 val;	197	u8 val;
129		198
		199	/* Wa_15015404425 */
		200	mmio_flush_pending_writes(gt);
		201
130	val = readb((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr);	202	val = readb((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr);
131	trace_xe_reg_rw(gt, false, addr, val, sizeof(val));	203	trace_xe_reg_rw(gt, false, addr, val, sizeof(val));
132		204
@@ -139,6 +211,9 @@ u16 xe_mmio_read16(struct xe_gt *gt, struct xe_reg reg) · 6 unmodified lines
139	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);	211	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);
140	u16 val;	212	u16 val;
141		213
		214	/* Wa_15015404425 */
		215	mmio_flush_pending_writes(gt);
		216
142	val = readw((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr);	217	val = readw((reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr);
143	trace_xe_reg_rw(gt, false, addr, val, sizeof(val));	218	trace_xe_reg_rw(gt, false, addr, val, sizeof(val));
144		219
@@ -151,7 +226,11 @@ void xe_mmio_write32(struct xe_gt *gt, struct xe_reg reg, u32 val) · 6 unmodified lines
151	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);	226	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);
152		227
153	trace_xe_reg_rw(gt, true, addr, val, sizeof(val));	228	trace_xe_reg_rw(gt, true, addr, val, sizeof(val));
154	writel(val, (reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr);	229
		230	if (!reg.vf && IS_SRIOV_VF(gt_to_xe(gt)))
		231	xe_gt_sriov_vf_write32(gt, reg, val);
		232	else
		233	writel(val, (reg.ext ? tile->mmio_ext.regs : tile->mmio.regs) + addr);
155	}	234	}
156		235
157	u32 xe_mmio_read32(struct xe_gt *gt, struct xe_reg reg)	236	u32 xe_mmio_read32(struct xe_gt *gt, struct xe_reg reg)
@@ -160,6 +239,9 @@ u32 xe_mmio_read32(struct xe_gt *gt, struct xe_reg reg) · 2 unmodified lines
160	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);	239	u32 addr = xe_mmio_adjusted_addr(gt, reg.addr);
161	u32 val;	240	u32 val;
162		241
		242	/* Wa_15015404425 */
		243	mmio_flush_pending_writes(gt);
		244
163	if (!reg.vf && IS_SRIOV_VF(gt_to_xe(gt)))	245	if (!reg.vf && IS_SRIOV_VF(gt_to_xe(gt)))
164	val = xe_gt_sriov_vf_read32(gt, reg);	246	val = xe_gt_sriov_vf_read32(gt, reg);
165	else	247	else
@@ -251,37 +333,24 @@ u64 xe_mmio_read64_2x32(struct xe_gt *gt, struct xe_reg reg) · 85 unmodified lines
251	return (u64)udw << 32 \| ldw;	333	return (u64)udw << 32 \| ldw;
252	}	334	}
253		335
254	/**	336	static int __xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
255	* xe_mmio_wait32() - Wait for a register to match the desired masked value	337	u32 *out_val, bool atomic, bool expect_match)
256	* @gt: MMIO target GT
257	* @reg: register to read value from
258	* @mask: mask to be applied to the value read from the register
259	* @val: desired value after applying the mask
260	* @timeout_us: time out after this period of time. Wait logic tries to be
261	* smart, applying an exponential backoff until @timeout_us is reached.
262	* @out_val: if not NULL, points where to store the last unmasked value
263	* @atomic: needs to be true if calling from an atomic context
264	*
265	* This function polls for the desired masked value and returns zero on success
266	* or -ETIMEDOUT if timed out.
267	*
268	* Note that @timeout_us represents the minimum amount of time to wait before
269	* giving up. The actual time taken by this function can be a little more than
270	* @timeout_us for different reasons, specially in non-atomic contexts. Thus,
271	* it is possible that this function succeeds even after @timeout_us has passed.
272	*/
273	int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
274	u32 *out_val, bool atomic)
275	{	338	{
276	ktime_t cur = ktime_get_raw();	339	ktime_t cur = ktime_get_raw();
277	const ktime_t end = ktime_add_us(cur, timeout_us);	340	const ktime_t end = ktime_add_us(cur, timeout_us);
278	int ret = -ETIMEDOUT;	341	int ret = -ETIMEDOUT;
279	s64 wait = 10;	342	s64 wait = 10;
280	u32 read;	343	u32 read;
		344	bool check;
281		345
282	for (;;) {	346	for (;;) {
283	read = xe_mmio_read32(gt, reg);	347	read = xe_mmio_read32(gt, reg);
284	if ((read & mask) == val) {	348
		349	check = (read & mask) == val;
		350	if (!expect_match)
		351	check = !check;
		352
		353	if (check) {
285	ret = 0;	354	ret = 0;
286	break;	355	break;
287	}	356	}
@@ -302,7 +371,12 @@ int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 t · 14 unmodified lines
302		371
303	if (ret != 0) {	372	if (ret != 0) {
304	read = xe_mmio_read32(gt, reg);	373	read = xe_mmio_read32(gt, reg);
305	if ((read & mask) == val)	374
		375	check = (read & mask) == val;
		376	if (!expect_match)
		377	check = !check;
		378
		379	if (check)
306	ret = 0;	380	ret = 0;
307	}	381	}
308		382
@@ -313,62 +387,45 @@ int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 t · 4 unmodified lines
313	}	387	}
314		388
315	/**	389	/**
316	* xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value	390	* xe_mmio_wait32() - Wait for a register to match the desired masked value
317	* @gt: MMIO target GT	391	* @gt: MMIO target GT
318	* @reg: register to read value from	392	* @reg: register to read value from
319	* @mask: mask to be applied to the value read from the register	393	* @mask: mask to be applied to the value read from the register
320	* @val: value to match after applying the mask	394	* @val: desired value after applying the mask
321	* @timeout_us: time out after this period of time. Wait logic tries to be	395	* @timeout_us: time out after this period of time. Wait logic tries to be
322	* smart, applying an exponential backoff until @timeout_us is reached.	396	* smart, applying an exponential backoff until @timeout_us is reached.
323	* @out_val: if not NULL, points where to store the last unmasked value	397	* @out_val: if not NULL, points where to store the last unmasked value
324	* @atomic: needs to be true if calling from an atomic context	398	* @atomic: needs to be true if calling from an atomic context
325	*	399	*
326	* This function polls for a masked value to change from a given value and	400	* This function polls for the desired masked value and returns zero on success
327	* returns zero on success or -ETIMEDOUT if timed out.	401	* or -ETIMEDOUT if timed out.
328	*	402	*
329	* Note that @timeout_us represents the minimum amount of time to wait before	403	* Note that @timeout_us represents the minimum amount of time to wait before
330	* giving up. The actual time taken by this function can be a little more than	404	* giving up. The actual time taken by this function can be a little more than
331	* @timeout_us for different reasons, specially in non-atomic contexts. Thus,	405	* @timeout_us for different reasons, specially in non-atomic contexts. Thus,
332	* it is possible that this function succeeds even after @timeout_us has passed.	406	* it is possible that this function succeeds even after @timeout_us has passed.
333	*/	407	*/
		408	int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
		409	u32 *out_val, bool atomic)
		410	{
		411	return __xe_mmio_wait32(gt, reg, mask, val, timeout_us, out_val, atomic, true);
		412	}
		413
		414	/**
		415	* xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value
		416	* @gt: MMIO target GT
		417	* @reg: register to read value from
		418	* @mask: mask to be applied to the value read from the register
		419	* @val: value not to be matched after applying the mask
		420	* @timeout_us: time out after this period of time
		421	* @out_val: if not NULL, points where to store the last unmasked value
		422	* @atomic: needs to be true if calling from an atomic context
		423	*
		424	* This function works exactly like xe_mmio_wait32() with the exception that
		425	* @val is expected not to be matched.
		426	*/
334	int xe_mmio_wait32_not(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,	427	int xe_mmio_wait32_not(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
335	u32 *out_val, bool atomic)	428	u32 *out_val, bool atomic)
336	{	429	{
337	ktime_t cur = ktime_get_raw();	430	return __xe_mmio_wait32(gt, reg, mask, val, timeout_us, out_val, atomic, false);
338	const ktime_t end = ktime_add_us(cur, timeout_us);
339	int ret = -ETIMEDOUT;
340	s64 wait = 10;
341	u32 read;
342
343	for (;;) {
344	read = xe_mmio_read32(gt, reg);
345	if ((read & mask) != val) {
346	ret = 0;
347	break;
348	}
349
350	cur = ktime_get_raw();
351	if (!ktime_before(cur, end))
352	break;
353
354	if (ktime_after(ktime_add_us(cur, wait), end))
355	wait = ktime_us_delta(end, cur);
356
357	if (atomic)
358	udelay(wait);
359	else
360	usleep_range(wait, wait << 1);
361	wait <<= 1;
362	}
363
364	if (ret != 0) {
365	read = xe_mmio_read32(gt, reg);
366	if ((read & mask) != val)
367	ret = 0;
368	}
369
370	if (out_val)
371	*out_val = read;
372
373	return ret;
374	}	431	}

drivers/gpu/drm/xe/xe_mmio.h+0−1

@@ -22,7 +22,6 @@ u32 xe_mmio_rmw32(struct xe_gt *gt, struct xe_reg reg, u32 clr, u32 set); · 21 unmodified lines
22	int xe_mmio_write32_and_verify(struct xe_gt *gt, struct xe_reg reg, u32 val, u32 mask, u32 eval);	22	int xe_mmio_write32_and_verify(struct xe_gt *gt, struct xe_reg reg, u32 val, u32 mask, u32 eval);
23	bool xe_mmio_in_range(const struct xe_gt gt, const struct xe_mmio_range range, struct xe_reg reg);	23	bool xe_mmio_in_range(const struct xe_gt gt, const struct xe_mmio_range range, struct xe_reg reg);
24		24
25	int xe_mmio_probe_vram(struct xe_device *xe);
26	u64 xe_mmio_read64_2x32(struct xe_gt *gt, struct xe_reg reg);	25	u64 xe_mmio_read64_2x32(struct xe_gt *gt, struct xe_reg reg);
27	int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,	26	int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us,
28	u32 *out_val, bool atomic);	27	u32 *out_val, bool atomic);

drivers/gpu/drm/xe/xe_oa.c+1−1

@@ -641,7 +641,7 @@ static void xe_oa_store_flex(struct xe_oa_stream stream, struct xe_lrc lrc, · 640 unmodified lines
641	u32 offset = xe_bo_ggtt_addr(lrc->bo);	641	u32 offset = xe_bo_ggtt_addr(lrc->bo);
642		642
643	do {	643	do {
644	bb->cs[bb->len++] = MI_STORE_DATA_IMM \| BIT(22) /* GGTT */ \| 2;	644	bb->cs[bb->len++] = MI_STORE_DATA_IMM \| MI_SDI_GGTT \| MI_SDI_NUM_DW(1);
645	bb->cs[bb->len++] = offset + flex->offset * sizeof(u32);	645	bb->cs[bb->len++] = offset + flex->offset * sizeof(u32);
646	bb->cs[bb->len++] = 0;	646	bb->cs[bb->len++] = 0;
647	bb->cs[bb->len++] = flex->value;	647	bb->cs[bb->len++] = flex->value;

drivers/gpu/drm/xe/xe_pat.c+10−1

@@ -7,6 +7,8 @@ · 6 unmodified lines
7		7
8	#include <drm/xe_drm.h>	8	#include <drm/xe_drm.h>
9		9
		10	#include <generated/xe_wa_oob.h>
		11
10	#include "regs/xe_reg_defs.h"	12	#include "regs/xe_reg_defs.h"
11	#include "xe_assert.h"	13	#include "xe_assert.h"
12	#include "xe_device.h"	14	#include "xe_device.h"
@@ -15,6 +17,7 @@ · 2 unmodified lines
15	#include "xe_gt_mcr.h"	17	#include "xe_gt_mcr.h"
16	#include "xe_mmio.h"	18	#include "xe_mmio.h"
17	#include "xe_sriov.h"	19	#include "xe_sriov.h"
		20	#include "xe_wa.h"
18		21
19	#define _PAT_ATS 0x47fc	22	#define _PAT_ATS 0x47fc
20	#define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \	23	#define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \
@@ -382,7 +385,13 @@ void xe_pat_init_early(struct xe_device *xe) · 361 unmodified lines
382	if (GRAPHICS_VER(xe) == 20) {	385	if (GRAPHICS_VER(xe) == 20) {
383	xe->pat.ops = &xe2_pat_ops;	386	xe->pat.ops = &xe2_pat_ops;
384	xe->pat.table = xe2_pat_table;	387	xe->pat.table = xe2_pat_table;
385	xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table);	388
		389	/* Wa_16023588340. XXX: Should use XE_WA */
		390	if (GRAPHICS_VERx100(xe) == 2001)
		391	xe->pat.n_entries = 28; /* Disable CLOS3 */
		392	else
		393	xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table);
		394
386	xe->pat.idx[XE_CACHE_NONE] = 3;	395	xe->pat.idx[XE_CACHE_NONE] = 3;
387	xe->pat.idx[XE_CACHE_WT] = 15;	396	xe->pat.idx[XE_CACHE_WT] = 15;
388	xe->pat.idx[XE_CACHE_WB] = 2;	397	xe->pat.idx[XE_CACHE_WB] = 2;

drivers/gpu/drm/xe/xe_pci.c+5−2

 	u8 has_display:1;
 	u8 has_heci_gscfi:1;
+	u8 has_heci_cscfi:1;
 	u8 has_llc:1;
 	u8 has_mmio_ext:1;
 	u8 has_sriov:1;
 	PLATFORM(BATTLEMAGE),
 	.has_display = true,
 	.require_force_probe = true,
+	.has_heci_cscfi = 1,
 };
 #undef PLATFORM
 	xe->info.is_dgfx = desc->is_dgfx;
 	xe->info.has_heci_gscfi = desc->has_heci_gscfi;
+	xe->info.has_heci_cscfi = desc->has_heci_cscfi;
 	xe->info.has_llc = desc->has_llc;
 	xe->info.has_mmio_ext = desc->has_mmio_ext;
 	xe->info.has_sriov = desc->has_sriov;
 	if (err)
 		return err;
-	drm_dbg(&xe->drm, "%s %s %04x:%04x dgfx:%d gfx:%s (%d.%02d) media:%s (%d.%02d) display:%s dma_m_s:%d tc:%d gscfi:%d",
+	drm_dbg(&xe->drm, "%s %s %04x:%04x dgfx:%d gfx:%s (%d.%02d) media:%s (%d.%02d) display:%s dma_m_s:%d tc:%d gscfi:%d cscfi:%d",
 		desc->platform_name,
 		subplatform_desc ? subplatform_desc->name : "",
 		xe->info.devid, xe->info.revid,
 		xe->info.media_verx100 % 100,
 		str_yes_no(xe->info.enable_display),
 		xe->info.dma_mask_size, xe->info.tile_count,
-		xe->info.has_heci_gscfi);
+		xe->info.has_heci_gscfi, xe->info.has_heci_cscfi);
 	drm_dbg(&xe->drm, "Stepping = (G:%s, M:%s, D:%s, B:%s)\n",
 		xe_step_name(xe->info.step.graphics),

drivers/gpu/drm/xe/xe_pm.c+8−0

 #include "xe_guc.h"
 #include "xe_irq.h"
 #include "xe_pcode.h"
+#include "xe_trace.h"
 #include "xe_wa.h"
 /**
 	int err;
 	drm_dbg(&xe->drm, "Suspending device\n");
+	trace_xe_pm_suspend(xe, __builtin_return_address(0));
 	for_each_gt(gt, xe, id)
 		xe_gt_suspend_prepare(gt);
 	int err;
 	drm_dbg(&xe->drm, "Resuming device\n");
+	trace_xe_pm_resume(xe, __builtin_return_address(0));
 	for_each_tile(tile, xe, id)
 		xe_wa_apply_tile_workarounds(tile);
 	u8 id;
 	int err = 0;
+	trace_xe_pm_runtime_suspend(xe, __builtin_return_address(0));
 	/* Disable access_ongoing asserts and prevent recursive pm calls */
 	xe_pm_write_callback_task(xe, current);
 	u8 id;
 	int err = 0;
+	trace_xe_pm_runtime_resume(xe, __builtin_return_address(0));
 	/* Disable access_ongoing asserts and prevent recursive pm calls */
 	xe_pm_write_callback_task(xe, current);
  */
 void xe_pm_runtime_get(struct xe_device *xe)
 {
+	trace_xe_pm_runtime_get(xe, __builtin_return_address(0));
 	pm_runtime_get_noresume(xe->drm.dev);
 	if (xe_pm_read_callback_task(xe) == current)
  */
 void xe_pm_runtime_put(struct xe_device *xe)
 {
+	trace_xe_pm_runtime_put(xe, __builtin_return_address(0));
 	if (xe_pm_read_callback_task(xe) == current) {
 		pm_runtime_put_noidle(xe->drm.dev);
 	} else {
  */
 int xe_pm_runtime_get_ioctl(struct xe_device *xe)
 {
+	trace_xe_pm_runtime_get_ioctl(xe, __builtin_return_address(0));
 	if (WARN_ON(xe_pm_read_callback_task(xe) == current))
 		return -ELOOP;

drivers/gpu/drm/xe/xe_preempt_fence.c+9−3

 		container_of(w, typeof(*pfence), preempt_work);
 	struct xe_exec_queue *q = pfence->q;
-	if (pfence->error)
+	if (pfence->error) {
 		dma_fence_set_error(&pfence->base, pfence->error);
-	else
+	} else if (!q->ops->reset_status(q)) {
-		q->ops->suspend_wait(q);
+		int err = q->ops->suspend_wait(q);
+		if (err)
+			dma_fence_set_error(&pfence->base, err);
+	} else {
+		dma_fence_set_error(&pfence->base, -ENOENT);
+	}
 	dma_fence_signal(&pfence->base);
 	/*

drivers/gpu/drm/xe/xe_pt.c+848−462

@@ -9,12 +9,15 @@ · 8 unmodified lines
9	#include "xe_bo.h"	9	#include "xe_bo.h"
10	#include "xe_device.h"	10	#include "xe_device.h"
11	#include "xe_drm_client.h"	11	#include "xe_drm_client.h"
		12	#include "xe_exec_queue.h"
12	#include "xe_gt.h"	13	#include "xe_gt.h"
13	#include "xe_gt_tlb_invalidation.h"	14	#include "xe_gt_tlb_invalidation.h"
14	#include "xe_migrate.h"	15	#include "xe_migrate.h"
15	#include "xe_pt_types.h"	16	#include "xe_pt_types.h"
16	#include "xe_pt_walk.h"	17	#include "xe_pt_walk.h"
17	#include "xe_res_cursor.h"	18	#include "xe_res_cursor.h"
		19	#include "xe_sched_job.h"
		20	#include "xe_sync.h"
18	#include "xe_trace.h"	21	#include "xe_trace.h"
19	#include "xe_ttm_stolen_mgr.h"	22	#include "xe_ttm_stolen_mgr.h"
20	#include "xe_vm.h"	23	#include "xe_vm.h"
@@ -325,6 +328,7 @@ xe_pt_new_shared(struct xe_walk_update wupd, struct xe_pt parent, · 304 unmodified lines
325	entry->pt = parent;	328	entry->pt = parent;
326	entry->flags = 0;	329	entry->flags = 0;
327	entry->qwords = 0;	330	entry->qwords = 0;
		331	entry->pt_bo->update_index = -1;
328		332
329	if (alloc_entries) {	333	if (alloc_entries) {
330	entry->pt_entries = kmalloc_array(XE_PDES,	334	entry->pt_entries = kmalloc_array(XE_PDES,
@@ -842,19 +846,27 @@ xe_vm_populate_pgtable(struct xe_migrate_pt_update pt_update, struct xe_tile t · 511 unmodified lines
842	}	846	}
843	}	847	}
844		848
845	static void xe_pt_abort_bind(struct xe_vma *vma,	849	static void xe_pt_cancel_bind(struct xe_vma *vma,
846	struct xe_vm_pgtable_update *entries,	850	struct xe_vm_pgtable_update *entries,
847	u32 num_entries)	851	u32 num_entries)
848	{	852	{
849	u32 i, j;	853	u32 i, j;
850		854
851	for (i = 0; i < num_entries; i++) {	855	for (i = 0; i < num_entries; i++) {
852	if (!entries[i].pt_entries)	856	struct xe_pt *pt = entries[i].pt;
		857
		858	if (!pt)
853	continue;	859	continue;
854		860
855	for (j = 0; j < entries[i].qwords; j++)	861	if (pt->level) {
856	xe_pt_destroy(entries[i].pt_entries[j].pt, xe_vma_vm(vma)->flags, NULL);	862	for (j = 0; j < entries[i].qwords; j++)
		863	xe_pt_destroy(entries[i].pt_entries[j].pt,
		864	xe_vma_vm(vma)->flags, NULL);
		865	}
		866
857	kfree(entries[i].pt_entries);	867	kfree(entries[i].pt_entries);
		868	entries[i].pt_entries = NULL;
		869	entries[i].qwords = 0;
858	}	870	}
859	}	871	}
860		872
@@ -864,18 +876,15 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma) · 3 unmodified lines
864		876
865	lockdep_assert_held(&vm->lock);	877	lockdep_assert_held(&vm->lock);
866		878
867	if (xe_vma_is_userptr(vma))	879	if (!xe_vma_is_userptr(vma) && !xe_vma_is_null(vma))
868	lockdep_assert_held_read(&vm->userptr.notifier_lock);
869	else if (!xe_vma_is_null(vma))
870	dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);	880	dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
871		881
872	xe_vm_assert_held(vm);	882	xe_vm_assert_held(vm);
873	}	883	}
874		884
875	static void xe_pt_commit_bind(struct xe_vma *vma,	885	static void xe_pt_commit(struct xe_vma *vma,
876	struct xe_vm_pgtable_update *entries,	886	struct xe_vm_pgtable_update *entries,
877	u32 num_entries, bool rebind,	887	u32 num_entries, struct llist_head *deferred)
878	struct llist_head *deferred)
879	{	888	{
880	u32 i, j;	889	u32 i, j;
881		890
@@ -883,31 +892,90 @@ static void xe_pt_commit_bind(struct xe_vma *vma, · 1 unmodified line
883		892
884	for (i = 0; i < num_entries; i++) {	893	for (i = 0; i < num_entries; i++) {
885	struct xe_pt *pt = entries[i].pt;	894	struct xe_pt *pt = entries[i].pt;
		895
		896	if (!pt->level)
		897	continue;
		898
		899	for (j = 0; j < entries[i].qwords; j++) {
		900	struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
		901
		902	xe_pt_destroy(oldpte, xe_vma_vm(vma)->flags, deferred);
		903	}
		904	}
		905	}
		906
		907	static void xe_pt_abort_bind(struct xe_vma *vma,
		908	struct xe_vm_pgtable_update *entries,
		909	u32 num_entries, bool rebind)
		910	{
		911	int i, j;
		912
		913	xe_pt_commit_locks_assert(vma);
		914
		915	for (i = num_entries - 1; i >= 0; --i) {
		916	struct xe_pt *pt = entries[i].pt;
886	struct xe_pt_dir *pt_dir;	917	struct xe_pt_dir *pt_dir;
887		918
888	if (!rebind)	919	if (!rebind)
889	pt->num_live += entries[i].qwords;	920	pt->num_live -= entries[i].qwords;
890		921
891	if (!pt->level) {	922	if (!pt->level)
892	kfree(entries[i].pt_entries);
893	continue;	923	continue;
		924
		925	pt_dir = as_xe_pt_dir(pt);
		926	for (j = 0; j < entries[i].qwords; j++) {
		927	u32 j_ = j + entries[i].ofs;
		928	struct xe_pt *newpte = xe_pt_entry(pt_dir, j_);
		929	struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
		930
		931	pt_dir->children[j_] = oldpte ? &oldpte->base : 0;
		932	xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);
894	}	933	}
		934	}
		935	}
		936
		937	static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
		938	struct xe_vm_pgtable_update *entries,
		939	u32 num_entries, bool rebind)
		940	{
		941	u32 i, j;
		942
		943	xe_pt_commit_locks_assert(vma);
		944
		945	for (i = 0; i < num_entries; i++) {
		946	struct xe_pt *pt = entries[i].pt;
		947	struct xe_pt_dir *pt_dir;
		948
		949	if (!rebind)
		950	pt->num_live += entries[i].qwords;
		951
		952	if (!pt->level)
		953	continue;
895		954
896	pt_dir = as_xe_pt_dir(pt);	955	pt_dir = as_xe_pt_dir(pt);
897	for (j = 0; j < entries[i].qwords; j++) {	956	for (j = 0; j < entries[i].qwords; j++) {
898	u32 j_ = j + entries[i].ofs;	957	u32 j_ = j + entries[i].ofs;
899	struct xe_pt *newpte = entries[i].pt_entries[j].pt;	958	struct xe_pt *newpte = entries[i].pt_entries[j].pt;
		959	struct xe_pt *oldpte = NULL;
900		960
901	if (xe_pt_entry(pt_dir, j_))	961	if (xe_pt_entry(pt_dir, j_))
902	xe_pt_destroy(xe_pt_entry(pt_dir, j_),	962	oldpte = xe_pt_entry(pt_dir, j_);
903	xe_vma_vm(vma)->flags, deferred);
904		963
905	pt_dir->children[j_] = &newpte->base;	964	pt_dir->children[j_] = &newpte->base;
		965	entries[i].pt_entries[j].pt = oldpte;
906	}	966	}
907	kfree(entries[i].pt_entries);
908	}	967	}
909	}	968	}
910		969
		970	static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries,
		971	u32 num_entries)
		972	{
		973	u32 i;
		974
		975	for (i = 0; i < num_entries; i++)
		976	kfree(entries[i].pt_entries);
		977	}
		978
911	static int	979	static int
912	xe_pt_prepare_bind(struct xe_tile tile, struct xe_vma vma,	980	xe_pt_prepare_bind(struct xe_tile tile, struct xe_vma vma,
913	struct xe_vm_pgtable_update entries, u32 num_entries)	981	struct xe_vm_pgtable_update entries, u32 num_entries)
@@ -918,20 +986,19 @@ xe_pt_prepare_bind(struct xe_tile tile, struct xe_vma vma, · 4 unmodified lines
918	err = xe_pt_stage_bind(tile, vma, entries, num_entries);	986	err = xe_pt_stage_bind(tile, vma, entries, num_entries);
919	if (!err)	987	if (!err)
920	xe_tile_assert(tile, *num_entries);	988	xe_tile_assert(tile, *num_entries);
921	else /* abort! */
922	xe_pt_abort_bind(vma, entries, *num_entries);
923		989
924	return err;	990	return err;
925	}	991	}
926		992
927	static void xe_vm_dbg_print_entries(struct xe_device *xe,	993	static void xe_vm_dbg_print_entries(struct xe_device *xe,
928	const struct xe_vm_pgtable_update *entries,	994	const struct xe_vm_pgtable_update *entries,
929	unsigned int num_entries)	995	unsigned int num_entries, bool bind)
930	#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM))	996	#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM))
931	{	997	{
932	unsigned int i;	998	unsigned int i;
933		999
934	vm_dbg(&xe->drm, "%u entries to update\n", num_entries);	1000	vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind",
		1001	num_entries);
935	for (i = 0; i < num_entries; i++) {	1002	for (i = 0; i < num_entries; i++) {
936	const struct xe_vm_pgtable_update *entry = &entries[i];	1003	const struct xe_vm_pgtable_update *entry = &entries[i];
937	struct xe_pt *xe_pt = entry->pt;	1004	struct xe_pt *xe_pt = entry->pt;
@@ -952,66 +1019,108 @@ static void xe_vm_dbg_print_entries(struct xe_device *xe, · 14 unmodified lines
952	{}	1019	{}
953	#endif	1020	#endif
954		1021
955	#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT	1022	static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs)
956
957	static int xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
958	{	1023	{
959	u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2;	1024	int i;
960	static u32 count;
961		1025
962	if (count++ % divisor == divisor - 1) {	1026	for (i = 0; i < num_syncs; i++) {
963	struct xe_vm *vm = xe_vma_vm(&uvma->vma);	1027	struct dma_fence *fence = syncs[i].fence;
964		1028
965	uvma->userptr.divisor = divisor << 1;	1029	if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
966	spin_lock(&vm->userptr.invalidated_lock);	1030	&fence->flags))
967	list_move_tail(&uvma->userptr.invalidate_link,	1031	return false;
968	&vm->userptr.invalidated);
969	spin_unlock(&vm->userptr.invalidated_lock);
970	return true;
971	}	1032	}
972		1033
973	return false;	1034	return true;
974	}	1035	}
975		1036
976	#else	1037	static int job_test_add_deps(struct xe_sched_job *job,
		1038	struct dma_resv *resv,
		1039	enum dma_resv_usage usage)
		1040	{
		1041	if (!job) {
		1042	if (!dma_resv_test_signaled(resv, usage))
		1043	return -ETIME;
977		1044
978	static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)	1045	return 0;
		1046	}
		1047
		1048	return xe_sched_job_add_deps(job, resv, usage);
		1049	}
		1050
		1051	static int vma_add_deps(struct xe_vma vma, struct xe_sched_job job)
979	{	1052	{
980	return false;	1053	struct xe_bo *bo = xe_vma_bo(vma);
		1054
		1055	xe_bo_assert_held(bo);
		1056
		1057	if (bo && !bo->vm)
		1058	return job_test_add_deps(job, bo->ttm.base.resv,
		1059	DMA_RESV_USAGE_KERNEL);
		1060
		1061	return 0;
981	}	1062	}
982		1063
983	#endif	1064	static int op_add_deps(struct xe_vm vm, struct xe_vma_op op,
		1065	struct xe_sched_job *job)
		1066	{
		1067	int err = 0;
984		1068
985	/**	1069	switch (op->base.op) {
986	* struct xe_pt_migrate_pt_update - Callback argument for pre-commit callbacks	1070	case DRM_GPUVA_OP_MAP:
987	* @base: Base we derive from.	1071	if (!op->map.immediate && xe_vm_in_fault_mode(vm))
988	* @bind: Whether this is a bind or an unbind operation. A bind operation	1072	break;
989	* makes the pre-commit callback error with -EAGAIN if it detects a	1073
990	* pending invalidation.	1074	err = vma_add_deps(op->map.vma, job);
991	* @locked: Whether the pre-commit callback locked the userptr notifier lock	1075	break;
992	* and it needs unlocking.	1076	case DRM_GPUVA_OP_REMAP:
993	*/	1077	if (op->remap.prev)
994	struct xe_pt_migrate_pt_update {	1078	err = vma_add_deps(op->remap.prev, job);
995	struct xe_migrate_pt_update base;	1079	if (!err && op->remap.next)
996	bool bind;	1080	err = vma_add_deps(op->remap.next, job);
997	bool locked;	1081	break;
998	};	1082	case DRM_GPUVA_OP_UNMAP:
		1083	break;
		1084	case DRM_GPUVA_OP_PREFETCH:
		1085	err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job);
		1086	break;
		1087	default:
		1088	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
		1089	}
		1090
		1091	return err;
		1092	}
999		1093
1000	/*
1001	* This function adds the needed dependencies to a page-table update job
1002	* to make sure racing jobs for separate bind engines don't race writing
1003	* to the same page-table range, wreaking havoc. Initially use a single
1004	* fence for the entire VM. An optimization would use smaller granularity.
1005	*/
1006	static int xe_pt_vm_dependencies(struct xe_sched_job *job,	1094	static int xe_pt_vm_dependencies(struct xe_sched_job *job,
1007	struct xe_range_fence_tree *rftree,	1095	struct xe_vm *vm,
1008	u64 start, u64 last)	1096	struct xe_vma_ops *vops,
		1097	struct xe_vm_pgtable_update_ops *pt_update_ops,
		1098	struct xe_range_fence_tree *rftree)
1009	{	1099	{
1010	struct xe_range_fence *rtfence;	1100	struct xe_range_fence *rtfence;
1011	struct dma_fence *fence;	1101	struct dma_fence *fence;
1012	int err;	1102	struct xe_vma_op *op;
		1103	int err = 0, i;
		1104
		1105	xe_vm_assert_held(vm);
1013		1106
1014	rtfence = xe_range_fence_tree_first(rftree, start, last);	1107	if (!job && !no_in_syncs(vops->syncs, vops->num_syncs))
		1108	return -ETIME;
		1109
		1110	if (!job && !xe_exec_queue_is_idle(pt_update_ops->q))
		1111	return -ETIME;
		1112
		1113	if (pt_update_ops->wait_vm_bookkeep \|\| pt_update_ops->wait_vm_kernel) {
		1114	err = job_test_add_deps(job, xe_vm_resv(vm),
		1115	pt_update_ops->wait_vm_bookkeep ?
		1116	DMA_RESV_USAGE_BOOKKEEP :
		1117	DMA_RESV_USAGE_KERNEL);
		1118	if (err)
		1119	return err;
		1120	}
		1121
		1122	rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start,
		1123	pt_update_ops->last);
1015	while (rtfence) {	1124	while (rtfence) {
1016	fence = rtfence->fence;	1125	fence = rtfence->fence;
1017		1126
@@ -1029,80 +1138,173 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job, · 11 unmodified lines
1029	return err;	1138	return err;
1030	}	1139	}
1031		1140
1032	rtfence = xe_range_fence_tree_next(rtfence, start, last);	1141	rtfence = xe_range_fence_tree_next(rtfence,
		1142	pt_update_ops->start,
		1143	pt_update_ops->last);
1033	}	1144	}
1034		1145
1035	return 0;	1146	list_for_each_entry(op, &vops->list, link) {
		1147	err = op_add_deps(vm, op, job);
		1148	if (err)
		1149	return err;
		1150	}
		1151
		1152	if (job)
		1153	err = xe_sched_job_last_fence_add_dep(job, vm);
		1154	else
		1155	err = xe_exec_queue_last_fence_test_dep(pt_update_ops->q, vm);
		1156
		1157	for (i = 0; job && !err && i < vops->num_syncs; i++)
		1158	err = xe_sync_entry_add_deps(&vops->syncs[i], job);
		1159
		1160	return err;
1036	}	1161	}
1037		1162
1038	static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)	1163	static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)
1039	{	1164	{
1040	struct xe_range_fence_tree *rftree =	1165	struct xe_vma_ops *vops = pt_update->vops;
1041	&xe_vma_vm(pt_update->vma)->rftree[pt_update->tile_id];	1166	struct xe_vm *vm = vops->vm;
		1167	struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id];
		1168	struct xe_vm_pgtable_update_ops *pt_update_ops =
		1169	&vops->pt_update_ops[pt_update->tile_id];
		1170
		1171	return xe_pt_vm_dependencies(pt_update->job, vm, pt_update->vops,
		1172	pt_update_ops, rftree);
		1173	}
1042		1174
1043	return xe_pt_vm_dependencies(pt_update->job, rftree,	1175	#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
1044	pt_update->start, pt_update->last);	1176
		1177	static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
		1178	{
		1179	u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2;
		1180	static u32 count;
		1181
		1182	if (count++ % divisor == divisor - 1) {
		1183	uvma->userptr.divisor = divisor << 1;
		1184	return true;
		1185	}
		1186
		1187	return false;
1045	}	1188	}
1046		1189
1047	static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)	1190	#else
		1191
		1192	static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
1048	{	1193	{
1049	struct xe_pt_migrate_pt_update *userptr_update =	1194	return false;
1050	container_of(pt_update, typeof(*userptr_update), base);	1195	}
1051	struct xe_userptr_vma *uvma = to_userptr_vma(pt_update->vma);
1052	unsigned long notifier_seq = uvma->userptr.notifier_seq;
1053	struct xe_vm *vm = xe_vma_vm(&uvma->vma);
1054	int err = xe_pt_vm_dependencies(pt_update->job,
1055	&vm->rftree[pt_update->tile_id],
1056	pt_update->start,
1057	pt_update->last);
1058		1196
1059	if (err)	1197	#endif
1060	return err;
1061		1198
1062	userptr_update->locked = false;	1199	static int vma_check_userptr(struct xe_vm vm, struct xe_vma vma,
		1200	struct xe_vm_pgtable_update_ops *pt_update)
		1201	{
		1202	struct xe_userptr_vma *uvma;
		1203	unsigned long notifier_seq;
1063		1204
1064	/*	1205	lockdep_assert_held_read(&vm->userptr.notifier_lock);
1065	* Wait until nobody is running the invalidation notifier, and
1066	* since we're exiting the loop holding the notifier lock,
1067	* nobody can proceed invalidating either.
1068	*
1069	* Note that we don't update the vma->userptr.notifier_seq since
1070	* we don't update the userptr pages.
1071	*/
1072	do {
1073	down_read(&vm->userptr.notifier_lock);
1074	if (!mmu_interval_read_retry(&uvma->userptr.notifier,
1075	notifier_seq))
1076	break;
1077		1206
1078	up_read(&vm->userptr.notifier_lock);	1207	if (!xe_vma_is_userptr(vma))
		1208	return 0;
1079		1209
1080	if (userptr_update->bind)	1210	uvma = to_userptr_vma(vma);
1081	return -EAGAIN;	1211	notifier_seq = uvma->userptr.notifier_seq;
1082		1212
1083	notifier_seq = mmu_interval_read_begin(&uvma->userptr.notifier);	1213	if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode(vm))
1084	} while (true);	1214	return 0;
1085		1215
1086	/* Inject errors to test_whether they are handled correctly */	1216	if (!mmu_interval_read_retry(&uvma->userptr.notifier,
1087	if (userptr_update->bind && xe_pt_userptr_inject_eagain(uvma)) {	1217	notifier_seq) &&
1088	up_read(&vm->userptr.notifier_lock);	1218	!xe_pt_userptr_inject_eagain(uvma))
		1219	return 0;
		1220
		1221	if (xe_vm_in_fault_mode(vm)) {
1089	return -EAGAIN;	1222	return -EAGAIN;
1090	}	1223	} else {
		1224	spin_lock(&vm->userptr.invalidated_lock);
		1225	list_move_tail(&uvma->userptr.invalidate_link,
		1226	&vm->userptr.invalidated);
		1227	spin_unlock(&vm->userptr.invalidated_lock);
1091		1228
1092	userptr_update->locked = true;	1229	if (xe_vm_in_preempt_fence_mode(vm)) {
		1230	struct dma_resv_iter cursor;
		1231	struct dma_fence *fence;
		1232	long err;
		1233
		1234	dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
		1235	DMA_RESV_USAGE_BOOKKEEP);
		1236	dma_resv_for_each_fence_unlocked(&cursor, fence)
		1237	dma_fence_enable_sw_signaling(fence);
		1238	dma_resv_iter_end(&cursor);
		1239
		1240	err = dma_resv_wait_timeout(xe_vm_resv(vm),
		1241	DMA_RESV_USAGE_BOOKKEEP,
		1242	false, MAX_SCHEDULE_TIMEOUT);
		1243	XE_WARN_ON(err <= 0);
		1244	}
		1245	}
1093		1246
1094	return 0;	1247	return 0;
1095	}	1248	}
1096		1249
1097	static const struct xe_migrate_pt_update_ops bind_ops = {	1250	static int op_check_userptr(struct xe_vm vm, struct xe_vma_op op,
1098	.populate = xe_vm_populate_pgtable,	1251	struct xe_vm_pgtable_update_ops *pt_update)
1099	.pre_commit = xe_pt_pre_commit,	1252	{
1100	};	1253	int err = 0;
1101		1254
1102	static const struct xe_migrate_pt_update_ops userptr_bind_ops = {	1255	lockdep_assert_held_read(&vm->userptr.notifier_lock);
1103	.populate = xe_vm_populate_pgtable,	1256
1104	.pre_commit = xe_pt_userptr_pre_commit,	1257	switch (op->base.op) {
1105	};	1258	case DRM_GPUVA_OP_MAP:
		1259	if (!op->map.immediate && xe_vm_in_fault_mode(vm))
		1260	break;
		1261
		1262	err = vma_check_userptr(vm, op->map.vma, pt_update);
		1263	break;
		1264	case DRM_GPUVA_OP_REMAP:
		1265	if (op->remap.prev)
		1266	err = vma_check_userptr(vm, op->remap.prev, pt_update);
		1267	if (!err && op->remap.next)
		1268	err = vma_check_userptr(vm, op->remap.next, pt_update);
		1269	break;
		1270	case DRM_GPUVA_OP_UNMAP:
		1271	break;
		1272	case DRM_GPUVA_OP_PREFETCH:
		1273	err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va),
		1274	pt_update);
		1275	break;
		1276	default:
		1277	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
		1278	}
		1279
		1280	return err;
		1281	}
		1282
		1283	static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
		1284	{
		1285	struct xe_vm *vm = pt_update->vops->vm;
		1286	struct xe_vma_ops *vops = pt_update->vops;
		1287	struct xe_vm_pgtable_update_ops *pt_update_ops =
		1288	&vops->pt_update_ops[pt_update->tile_id];
		1289	struct xe_vma_op *op;
		1290	int err;
		1291
		1292	err = xe_pt_pre_commit(pt_update);
		1293	if (err)
		1294	return err;
		1295
		1296	down_read(&vm->userptr.notifier_lock);
		1297
		1298	list_for_each_entry(op, &vops->list, link) {
		1299	err = op_check_userptr(vm, op, pt_update_ops);
		1300	if (err) {
		1301	up_read(&vm->userptr.notifier_lock);
		1302	break;
		1303	}
		1304	}
		1305
		1306	return err;
		1307	}
1106		1308
1107	struct invalidation_fence {	1309	struct invalidation_fence {
1108	struct xe_gt_tlb_invalidation_fence base;	1310	struct xe_gt_tlb_invalidation_fence base;
@@ -1115,23 +1317,6 @@ struct invalidation_fence { · 6 unmodified lines
1115	u32 asid;	1317	u32 asid;
1116	};	1318	};
1117		1319
1118	static const char *
1119	invalidation_fence_get_driver_name(struct dma_fence *dma_fence)
1120	{
1121	return "xe";
1122	}
1123
1124	static const char *
1125	invalidation_fence_get_timeline_name(struct dma_fence *dma_fence)
1126	{
1127	return "invalidation_fence";
1128	}
1129
1130	static const struct dma_fence_ops invalidation_fence_ops = {
1131	.get_driver_name = invalidation_fence_get_driver_name,
1132	.get_timeline_name = invalidation_fence_get_timeline_name,
1133	};
1134
1135	static void invalidation_fence_cb(struct dma_fence *fence,	1320	static void invalidation_fence_cb(struct dma_fence *fence,
1136	struct dma_fence_cb *cb)	1321	struct dma_fence_cb *cb)
1137	{	1322	{
@@ -1161,24 +1346,17 @@ static void invalidation_fence_work_func(struct work_struct *w) · 23 unmodified lines
1161	ifence->end, ifence->asid);	1346	ifence->end, ifence->asid);
1162	}	1347	}
1163		1348
1164	static int invalidation_fence_init(struct xe_gt *gt,	1349	static void invalidation_fence_init(struct xe_gt *gt,
1165	struct invalidation_fence *ifence,	1350	struct invalidation_fence *ifence,
1166	struct dma_fence *fence,	1351	struct dma_fence *fence,
1167	u64 start, u64 end, u32 asid)	1352	u64 start, u64 end, u32 asid)
1168	{	1353	{
1169	int ret;	1354	int ret;
1170		1355
1171	trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base);	1356	trace_xe_gt_tlb_invalidation_fence_create(gt_to_xe(gt), &ifence->base);
1172		1357
1173	spin_lock_irq(&gt->tlb_invalidation.lock);	1358	xe_gt_tlb_invalidation_fence_init(gt, &ifence->base, false);
1174	dma_fence_init(&ifence->base.base, &invalidation_fence_ops,
1175	&gt->tlb_invalidation.lock,
1176	dma_fence_context_alloc(1), 1);
1177	spin_unlock_irq(&gt->tlb_invalidation.lock);
1178
1179	INIT_LIST_HEAD(&ifence->base.link);
1180		1359
1181	dma_fence_get(&ifence->base.base); /* Ref for caller */
1182	ifence->fence = fence;	1360	ifence->fence = fence;
1183	ifence->gt = gt;	1361	ifence->gt = gt;
1184	ifence->start = start;	1362	ifence->start = start;
@@ -1196,192 +1374,6 @@ static int invalidation_fence_init(struct xe_gt *gt, · 11 unmodified lines
1196	}	1374	}
1197		1375
1198	xe_gt_assert(gt, !ret \|\| ret == -ENOENT);	1376	xe_gt_assert(gt, !ret \|\| ret == -ENOENT);
1199
1200	return ret && ret != -ENOENT ? ret : 0;
1201	}
1202
1203	static void xe_pt_calc_rfence_interval(struct xe_vma *vma,
1204	struct xe_pt_migrate_pt_update *update,
1205	struct xe_vm_pgtable_update *entries,
1206	u32 num_entries)
1207	{
1208	int i, level = 0;
1209
1210	for (i = 0; i < num_entries; i++) {
1211	const struct xe_vm_pgtable_update *entry = &entries[i];
1212
1213	if (entry->pt->level > level)
1214	level = entry->pt->level;
1215	}
1216
1217	/* Greedy (non-optimal) calculation but simple */
1218	update->base.start = ALIGN_DOWN(xe_vma_start(vma),
1219	0x1ull << xe_pt_shift(level));
1220	update->base.last = ALIGN(xe_vma_end(vma),
1221	0x1ull << xe_pt_shift(level)) - 1;
1222	}
1223
1224	/**
1225	* __xe_pt_bind_vma() - Build and connect a page-table tree for the vma
1226	* address range.
1227	* @tile: The tile to bind for.
1228	* @vma: The vma to bind.
1229	* @q: The exec_queue with which to do pipelined page-table updates.
1230	* @syncs: Entries to sync on before binding the built tree to the live vm tree.
1231	* @num_syncs: Number of @sync entries.
1232	* @rebind: Whether we're rebinding this vma to the same address range without
1233	* an unbind in-between.
1234	*
1235	* This function builds a page-table tree (see xe_pt_stage_bind() for more
1236	* information on page-table building), and the xe_vm_pgtable_update entries
1237	* abstracting the operations needed to attach it to the main vm tree. It
1238	* then takes the relevant locks and updates the metadata side of the main
1239	* vm tree and submits the operations for pipelined attachment of the
1240	* gpu page-table to the vm main tree, (which can be done either by the
1241	* cpu and the GPU).
1242	*
1243	* Return: A valid dma-fence representing the pipelined attachment operation
1244	* on success, an error pointer on error.
1245	*/
1246	struct dma_fence *
1247	__xe_pt_bind_vma(struct xe_tile tile, struct xe_vma vma, struct xe_exec_queue *q,
1248	struct xe_sync_entry *syncs, u32 num_syncs,
1249	bool rebind)
1250	{
1251	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
1252	struct xe_pt_migrate_pt_update bind_pt_update = {
1253	.base = {
1254	.ops = xe_vma_is_userptr(vma) ? &userptr_bind_ops : &bind_ops,
1255	.vma = vma,
1256	.tile_id = tile->id,
1257	},
1258	.bind = true,
1259	};
1260	struct xe_vm *vm = xe_vma_vm(vma);
1261	u32 num_entries;
1262	struct dma_fence *fence;
1263	struct invalidation_fence *ifence = NULL;
1264	struct xe_range_fence *rfence;
1265	int err;
1266
1267	bind_pt_update.locked = false;
1268	xe_bo_assert_held(xe_vma_bo(vma));
1269	xe_vm_assert_held(vm);
1270
1271	vm_dbg(&xe_vma_vm(vma)->xe->drm,
1272	"Preparing bind, with range [%llx...%llx) engine %p.\n",
1273	xe_vma_start(vma), xe_vma_end(vma), q);
1274
1275	err = xe_pt_prepare_bind(tile, vma, entries, &num_entries);
1276	if (err)
1277	goto err;
1278
1279	err = dma_resv_reserve_fences(xe_vm_resv(vm), 1);
1280	if (!err && !xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
1281	err = dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, 1);
1282	if (err)
1283	goto err;
1284
1285	xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries));
1286
1287	xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries);
1288	xe_pt_calc_rfence_interval(vma, &bind_pt_update, entries,
1289	num_entries);
1290
1291	/*
1292	* If rebind, we have to invalidate TLB on !LR vms to invalidate
1293	* cached PTEs point to freed memory. on LR vms this is done
1294	* automatically when the context is re-enabled by the rebind worker,
1295	* or in fault mode it was invalidated on PTE zapping.
1296	*
1297	* If !rebind, and scratch enabled VMs, there is a chance the scratch
1298	* PTE is already cached in the TLB so it needs to be invalidated.
1299	* on !LR VMs this is done in the ring ops preceding a batch, but on
1300	* non-faulting LR, in particular on user-space batch buffer chaining,
1301	* it needs to be done here.
1302	*/
1303	if ((!rebind && xe_vm_has_scratch(vm) && xe_vm_in_preempt_fence_mode(vm))) {
1304	ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
1305	if (!ifence)
1306	return ERR_PTR(-ENOMEM);
1307	} else if (rebind && !xe_vm_in_lr_mode(vm)) {
1308	/* We bump also if batch_invalidate_tlb is true */
1309	vm->tlb_flush_seqno++;
1310	}
1311
1312	rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
1313	if (!rfence) {
1314	kfree(ifence);
1315	return ERR_PTR(-ENOMEM);
1316	}
1317
1318	fence = xe_migrate_update_pgtables(tile->migrate,
1319	vm, xe_vma_bo(vma), q,
1320	entries, num_entries,
1321	syncs, num_syncs,
1322	&bind_pt_update.base);
1323	if (!IS_ERR(fence)) {
1324	bool last_munmap_rebind = vma->gpuva.flags & XE_VMA_LAST_REBIND;
1325	LLIST_HEAD(deferred);
1326	int err;
1327
1328	err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
1329	&xe_range_fence_kfree_ops,
1330	bind_pt_update.base.start,
1331	bind_pt_update.base.last, fence);
1332	if (err)
1333	dma_fence_wait(fence, false);
1334
1335	/* TLB invalidation must be done before signaling rebind */
1336	if (ifence) {
1337	int err = invalidation_fence_init(tile->primary_gt,
1338	ifence, fence,
1339	xe_vma_start(vma),
1340	xe_vma_end(vma),
1341	xe_vma_vm(vma)->usm.asid);
1342	if (err) {
1343	dma_fence_put(fence);
1344	kfree(ifence);
1345	return ERR_PTR(err);
1346	}
1347	fence = &ifence->base.base;
1348	}
1349
1350	/* add shared fence now for pagetable delayed destroy */
1351	dma_resv_add_fence(xe_vm_resv(vm), fence, rebind \|\|
1352	last_munmap_rebind ?
1353	DMA_RESV_USAGE_KERNEL :
1354	DMA_RESV_USAGE_BOOKKEEP);
1355
1356	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
1357	dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
1358	DMA_RESV_USAGE_BOOKKEEP);
1359	xe_pt_commit_bind(vma, entries, num_entries, rebind,
1360	bind_pt_update.locked ? &deferred : NULL);
1361
1362	/* This vma is live (again?) now */
1363	vma->tile_present \|= BIT(tile->id);
1364
1365	if (bind_pt_update.locked) {
1366	to_userptr_vma(vma)->userptr.initial_bind = true;
1367	up_read(&vm->userptr.notifier_lock);
1368	xe_bo_put_commit(&deferred);
1369	}
1370	if (!rebind && last_munmap_rebind &&
1371	xe_vm_in_preempt_fence_mode(vm))
1372	xe_vm_queue_rebind_worker(vm);
1373	} else {
1374	kfree(rfence);
1375	kfree(ifence);
1376	if (bind_pt_update.locked)
1377	up_read(&vm->userptr.notifier_lock);
1378	xe_pt_abort_bind(vma, entries, num_entries);
1379	}
1380
1381	return fence;
1382
1383	err:
1384	return ERR_PTR(err);
1385	}	1377	}
1386		1378
1387	struct xe_pt_stage_unbind_walk {	1379	struct xe_pt_stage_unbind_walk {
@@ -1466,6 +1458,7 @@ xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, · 78 unmodified lines
1466	struct xe_pt xe_child = container_of(child, typeof(*xe_child), base);	1458	struct xe_pt xe_child = container_of(child, typeof(*xe_child), base);
1467	pgoff_t end_offset;	1459	pgoff_t end_offset;
1468	u64 size = 1ull << walk->shifts[--level];	1460	u64 size = 1ull << walk->shifts[--level];
		1461	int err;
1469		1462
1470	if (!IS_ALIGNED(addr, size))	1463	if (!IS_ALIGNED(addr, size))
1471	addr = xe_walk->modified_start;	1464	addr = xe_walk->modified_start;
@@ -1481,7 +1474,10 @@ xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, · 9 unmodified lines
1481	&end_offset))	1474	&end_offset))
1482	return 0;	1475	return 0;
1483		1476
1484	(void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, false);	1477	err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true);
		1478	if (err)
		1479	return err;
		1480
1485	xe_walk->wupd.updates[level].update->qwords = end_offset - offset;	1481	xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
1486		1482
1487	return 0;	1483	return 0;
@@ -1534,8 +1530,8 @@ xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, · 46 unmodified lines
1534	void *ptr, u32 qword_ofs, u32 num_qwords,	1530	void *ptr, u32 qword_ofs, u32 num_qwords,
1535	const struct xe_vm_pgtable_update *update)	1531	const struct xe_vm_pgtable_update *update)
1536	{	1532	{
1537	struct xe_vma *vma = pt_update->vma;	1533	struct xe_vm *vm = pt_update->vops->vm;
1538	u64 empty = __xe_pt_empty_pte(tile, xe_vma_vm(vma), update->pt->level);	1534	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
1539	int i;	1535	int i;
1540		1536
1541	if (map && map->is_iomem)	1537	if (map && map->is_iomem)
@@ -1549,181 +1545,571 @@ xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, · 7 unmodified lines
1549	memset64(ptr, empty, num_qwords);	1545	memset64(ptr, empty, num_qwords);
1550	}	1546	}
1551		1547
		1548	static void xe_pt_abort_unbind(struct xe_vma *vma,
		1549	struct xe_vm_pgtable_update *entries,
		1550	u32 num_entries)
		1551	{
		1552	int i, j;
		1553
		1554	xe_pt_commit_locks_assert(vma);
		1555
		1556	for (i = num_entries - 1; i >= 0; --i) {
		1557	struct xe_vm_pgtable_update *entry = &entries[i];
		1558	struct xe_pt *pt = entry->pt;
		1559	struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
		1560
		1561	pt->num_live += entry->qwords;
		1562
		1563	if (!pt->level)
		1564	continue;
		1565
		1566	for (j = entry->ofs; j < entry->ofs + entry->qwords; j++)
		1567	pt_dir->children[j] =
		1568	entries[i].pt_entries[j - entry->ofs].pt ?
		1569	&entries[i].pt_entries[j - entry->ofs].pt->base : NULL;
		1570	}
		1571	}
		1572
1552	static void	1573	static void
1553	xe_pt_commit_unbind(struct xe_vma *vma,	1574	xe_pt_commit_prepare_unbind(struct xe_vma *vma,
1554	struct xe_vm_pgtable_update *entries, u32 num_entries,	1575	struct xe_vm_pgtable_update *entries,
1555	struct llist_head *deferred)	1576	u32 num_entries)
1556	{	1577	{
1557	u32 j;	1578	int i, j;
1558		1579
1559	xe_pt_commit_locks_assert(vma);	1580	xe_pt_commit_locks_assert(vma);
1560		1581
1561	for (j = 0; j < num_entries; ++j) {	1582	for (i = 0; i < num_entries; ++i) {
1562	struct xe_vm_pgtable_update *entry = &entries[j];	1583	struct xe_vm_pgtable_update *entry = &entries[i];
1563	struct xe_pt *pt = entry->pt;	1584	struct xe_pt *pt = entry->pt;
		1585	struct xe_pt_dir *pt_dir;
1564		1586
1565	pt->num_live -= entry->qwords;	1587	pt->num_live -= entry->qwords;
1566	if (pt->level) {	1588	if (!pt->level)
1567	struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);	1589	continue;
1568	u32 i;	1590
		1591	pt_dir = as_xe_pt_dir(pt);
		1592	for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) {
		1593	entry->pt_entries[j - entry->ofs].pt =
		1594	xe_pt_entry(pt_dir, j);
		1595	pt_dir->children[j] = NULL;
		1596	}
		1597	}
		1598	}
		1599
		1600	static void
		1601	xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops,
		1602	struct xe_vma *vma)
		1603	{
		1604	u32 current_op = pt_update_ops->current_op;
		1605	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
		1606	int i, level = 0;
		1607	u64 start, last;
1569		1608
1570	for (i = entry->ofs; i < entry->ofs + entry->qwords;	1609	for (i = 0; i < pt_op->num_entries; i++) {
1571	i++) {	1610	const struct xe_vm_pgtable_update *entry = &pt_op->entries[i];
1572	if (xe_pt_entry(pt_dir, i))
1573	xe_pt_destroy(xe_pt_entry(pt_dir, i),
1574	xe_vma_vm(vma)->flags, deferred);
1575		1611
1576	pt_dir->children[i] = NULL;	1612	if (entry->pt->level > level)
1577	}	1613	level = entry->pt->level;
		1614	}
		1615
		1616	/* Greedy (non-optimal) calculation but simple */
		1617	start = ALIGN_DOWN(xe_vma_start(vma), 0x1ull << xe_pt_shift(level));
		1618	last = ALIGN(xe_vma_end(vma), 0x1ull << xe_pt_shift(level)) - 1;
		1619
		1620	if (start < pt_update_ops->start)
		1621	pt_update_ops->start = start;
		1622	if (last > pt_update_ops->last)
		1623	pt_update_ops->last = last;
		1624	}
		1625
		1626	static int vma_reserve_fences(struct xe_device xe, struct xe_vma vma)
		1627	{
		1628	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
		1629	return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv,
		1630	xe->info.tile_count);
		1631
		1632	return 0;
		1633	}
		1634
		1635	static int bind_op_prepare(struct xe_vm vm, struct xe_tile tile,
		1636	struct xe_vm_pgtable_update_ops *pt_update_ops,
		1637	struct xe_vma *vma)
		1638	{
		1639	u32 current_op = pt_update_ops->current_op;
		1640	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
		1641	int err;
		1642
		1643	xe_bo_assert_held(xe_vma_bo(vma));
		1644
		1645	vm_dbg(&xe_vma_vm(vma)->xe->drm,
		1646	"Preparing bind, with range [%llx...%llx)\n",
		1647	xe_vma_start(vma), xe_vma_end(vma) - 1);
		1648
		1649	pt_op->vma = NULL;
		1650	pt_op->bind = true;
		1651	pt_op->rebind = BIT(tile->id) & vma->tile_present;
		1652
		1653	err = vma_reserve_fences(tile_to_xe(tile), vma);
		1654	if (err)
		1655	return err;
		1656
		1657	err = xe_pt_prepare_bind(tile, vma, pt_op->entries,
		1658	&pt_op->num_entries);
		1659	if (!err) {
		1660	xe_tile_assert(tile, pt_op->num_entries <=
		1661	ARRAY_SIZE(pt_op->entries));
		1662	xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
		1663	pt_op->num_entries, true);
		1664
		1665	xe_pt_update_ops_rfence_interval(pt_update_ops, vma);
		1666	++pt_update_ops->current_op;
		1667	pt_update_ops->needs_userptr_lock \|= xe_vma_is_userptr(vma);
		1668
		1669	/*
		1670	* If rebind, we have to invalidate TLB on !LR vms to invalidate
		1671	* cached PTEs point to freed memory. On LR vms this is done
		1672	* automatically when the context is re-enabled by the rebind worker,
		1673	* or in fault mode it was invalidated on PTE zapping.
		1674	*
		1675	* If !rebind, and scratch enabled VMs, there is a chance the scratch
		1676	* PTE is already cached in the TLB so it needs to be invalidated.
		1677	* On !LR VMs this is done in the ring ops preceding a batch, but on
		1678	* non-faulting LR, in particular on user-space batch buffer chaining,
		1679	* it needs to be done here.
		1680	*/
		1681	if ((!pt_op->rebind && xe_vm_has_scratch(vm) &&
		1682	xe_vm_in_preempt_fence_mode(vm)))
		1683	pt_update_ops->needs_invalidation = true;
		1684	else if (pt_op->rebind && !xe_vm_in_lr_mode(vm))
		1685	/* We bump also if batch_invalidate_tlb is true */
		1686	vm->tlb_flush_seqno++;
		1687
		1688	vma->tile_staged \|= BIT(tile->id);
		1689	pt_op->vma = vma;
		1690	xe_pt_commit_prepare_bind(vma, pt_op->entries,
		1691	pt_op->num_entries, pt_op->rebind);
		1692	} else {
		1693	xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
		1694	}
		1695
		1696	return err;
		1697	}
		1698
		1699	static int unbind_op_prepare(struct xe_tile *tile,
		1700	struct xe_vm_pgtable_update_ops *pt_update_ops,
		1701	struct xe_vma *vma)
		1702	{
		1703	u32 current_op = pt_update_ops->current_op;
		1704	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
		1705	int err;
		1706
		1707	if (!((vma->tile_present \| vma->tile_staged) & BIT(tile->id)))
		1708	return 0;
		1709
		1710	xe_bo_assert_held(xe_vma_bo(vma));
		1711
		1712	vm_dbg(&xe_vma_vm(vma)->xe->drm,
		1713	"Preparing unbind, with range [%llx...%llx)\n",
		1714	xe_vma_start(vma), xe_vma_end(vma) - 1);
		1715
		1716	/*
		1717	* Wait for invalidation to complete. Can corrupt internal page table
		1718	* state if an invalidation is running while preparing an unbind.
		1719	*/
		1720	if (xe_vma_is_userptr(vma) && xe_vm_in_fault_mode(xe_vma_vm(vma)))
		1721	mmu_interval_read_begin(&to_userptr_vma(vma)->userptr.notifier);
		1722
		1723	pt_op->vma = vma;
		1724	pt_op->bind = false;
		1725	pt_op->rebind = false;
		1726
		1727	err = vma_reserve_fences(tile_to_xe(tile), vma);
		1728	if (err)
		1729	return err;
		1730
		1731	pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries);
		1732
		1733	xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
		1734	pt_op->num_entries, false);
		1735	xe_pt_update_ops_rfence_interval(pt_update_ops, vma);
		1736	++pt_update_ops->current_op;
		1737	pt_update_ops->needs_userptr_lock \|= xe_vma_is_userptr(vma);
		1738	pt_update_ops->needs_invalidation = true;
		1739
		1740	xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries);
		1741
		1742	return 0;
		1743	}
		1744
		1745	static int op_prepare(struct xe_vm *vm,
		1746	struct xe_tile *tile,
		1747	struct xe_vm_pgtable_update_ops *pt_update_ops,
		1748	struct xe_vma_op *op)
		1749	{
		1750	int err = 0;
		1751
		1752	xe_vm_assert_held(vm);
		1753
		1754	switch (op->base.op) {
		1755	case DRM_GPUVA_OP_MAP:
		1756	if (!op->map.immediate && xe_vm_in_fault_mode(vm))
		1757	break;
		1758
		1759	err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma);
		1760	pt_update_ops->wait_vm_kernel = true;
		1761	break;
		1762	case DRM_GPUVA_OP_REMAP:
		1763	err = unbind_op_prepare(tile, pt_update_ops,
		1764	gpuva_to_vma(op->base.remap.unmap->va));
		1765
		1766	if (!err && op->remap.prev) {
		1767	err = bind_op_prepare(vm, tile, pt_update_ops,
		1768	op->remap.prev);
		1769	pt_update_ops->wait_vm_bookkeep = true;
1578	}	1770	}
		1771	if (!err && op->remap.next) {
		1772	err = bind_op_prepare(vm, tile, pt_update_ops,
		1773	op->remap.next);
		1774	pt_update_ops->wait_vm_bookkeep = true;
		1775	}
		1776	break;
		1777	case DRM_GPUVA_OP_UNMAP:
		1778	err = unbind_op_prepare(tile, pt_update_ops,
		1779	gpuva_to_vma(op->base.unmap.va));
		1780	break;
		1781	case DRM_GPUVA_OP_PREFETCH:
		1782	err = bind_op_prepare(vm, tile, pt_update_ops,
		1783	gpuva_to_vma(op->base.prefetch.va));
		1784	pt_update_ops->wait_vm_kernel = true;
		1785	break;
		1786	default:
		1787	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1579	}	1788	}
		1789
		1790	return err;
		1791	}
		1792
		1793	static void
		1794	xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
		1795	{
		1796	init_llist_head(&pt_update_ops->deferred);
		1797	pt_update_ops->start = ~0x0ull;
		1798	pt_update_ops->last = 0x0ull;
1580	}	1799	}
1581		1800
1582	static const struct xe_migrate_pt_update_ops unbind_ops = {	1801	/**
1583	.populate = xe_migrate_clear_pgtable_callback,	1802	* xe_pt_update_ops_prepare() - Prepare PT update operations
		1803	* @tile: Tile of PT update operations
		1804	* @vops: VMA operationa
		1805	*
		1806	* Prepare PT update operations which includes updating internal PT state,
		1807	* allocate memory for page tables, populate page table being pruned in, and
		1808	* create PT update operations for leaf insertion / removal.
		1809	*
		1810	* Return: 0 on success, negative error code on error.
		1811	*/
		1812	int xe_pt_update_ops_prepare(struct xe_tile tile, struct xe_vma_ops vops)
		1813	{
		1814	struct xe_vm_pgtable_update_ops *pt_update_ops =
		1815	&vops->pt_update_ops[tile->id];
		1816	struct xe_vma_op *op;
		1817	int err;
		1818
		1819	lockdep_assert_held(&vops->vm->lock);
		1820	xe_vm_assert_held(vops->vm);
		1821
		1822	xe_pt_update_ops_init(pt_update_ops);
		1823
		1824	err = dma_resv_reserve_fences(xe_vm_resv(vops->vm),
		1825	tile_to_xe(tile)->info.tile_count);
		1826	if (err)
		1827	return err;
		1828
		1829	list_for_each_entry(op, &vops->list, link) {
		1830	err = op_prepare(vops->vm, tile, pt_update_ops, op);
		1831
		1832	if (err)
		1833	return err;
		1834	}
		1835
		1836	xe_tile_assert(tile, pt_update_ops->current_op <=
		1837	pt_update_ops->num_ops);
		1838
		1839	#ifdef TEST_VM_OPS_ERROR
		1840	if (vops->inject_error &&
		1841	vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE)
		1842	return -ENOSPC;
		1843	#endif
		1844
		1845	return 0;
		1846	}
		1847
		1848	static void bind_op_commit(struct xe_vm vm, struct xe_tile tile,
		1849	struct xe_vm_pgtable_update_ops *pt_update_ops,
		1850	struct xe_vma vma, struct dma_fence fence)
		1851	{
		1852	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
		1853	dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
		1854	pt_update_ops->wait_vm_bookkeep ?
		1855	DMA_RESV_USAGE_KERNEL :
		1856	DMA_RESV_USAGE_BOOKKEEP);
		1857	vma->tile_present \|= BIT(tile->id);
		1858	vma->tile_staged &= ~BIT(tile->id);
		1859	if (xe_vma_is_userptr(vma)) {
		1860	lockdep_assert_held_read(&vm->userptr.notifier_lock);
		1861	to_userptr_vma(vma)->userptr.initial_bind = true;
		1862	}
		1863
		1864	/*
		1865	* Kick rebind worker if this bind triggers preempt fences and not in
		1866	* the rebind worker
		1867	*/
		1868	if (pt_update_ops->wait_vm_bookkeep &&
		1869	xe_vm_in_preempt_fence_mode(vm) &&
		1870	!current->mm)
		1871	xe_vm_queue_rebind_worker(vm);
		1872	}
		1873
		1874	static void unbind_op_commit(struct xe_vm vm, struct xe_tile tile,
		1875	struct xe_vm_pgtable_update_ops *pt_update_ops,
		1876	struct xe_vma vma, struct dma_fence fence)
		1877	{
		1878	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
		1879	dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
		1880	pt_update_ops->wait_vm_bookkeep ?
		1881	DMA_RESV_USAGE_KERNEL :
		1882	DMA_RESV_USAGE_BOOKKEEP);
		1883	vma->tile_present &= ~BIT(tile->id);
		1884	if (!vma->tile_present) {
		1885	list_del_init(&vma->combined_links.rebind);
		1886	if (xe_vma_is_userptr(vma)) {
		1887	lockdep_assert_held_read(&vm->userptr.notifier_lock);
		1888
		1889	spin_lock(&vm->userptr.invalidated_lock);
		1890	list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link);
		1891	spin_unlock(&vm->userptr.invalidated_lock);
		1892	}
		1893	}
		1894	}
		1895
		1896	static void op_commit(struct xe_vm *vm,
		1897	struct xe_tile *tile,
		1898	struct xe_vm_pgtable_update_ops *pt_update_ops,
		1899	struct xe_vma_op op, struct dma_fence fence)
		1900	{
		1901	xe_vm_assert_held(vm);
		1902
		1903	switch (op->base.op) {
		1904	case DRM_GPUVA_OP_MAP:
		1905	if (!op->map.immediate && xe_vm_in_fault_mode(vm))
		1906	break;
		1907
		1908	bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence);
		1909	break;
		1910	case DRM_GPUVA_OP_REMAP:
		1911	unbind_op_commit(vm, tile, pt_update_ops,
		1912	gpuva_to_vma(op->base.remap.unmap->va), fence);
		1913
		1914	if (op->remap.prev)
		1915	bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
		1916	fence);
		1917	if (op->remap.next)
		1918	bind_op_commit(vm, tile, pt_update_ops, op->remap.next,
		1919	fence);
		1920	break;
		1921	case DRM_GPUVA_OP_UNMAP:
		1922	unbind_op_commit(vm, tile, pt_update_ops,
		1923	gpuva_to_vma(op->base.unmap.va), fence);
		1924	break;
		1925	case DRM_GPUVA_OP_PREFETCH:
		1926	bind_op_commit(vm, tile, pt_update_ops,
		1927	gpuva_to_vma(op->base.prefetch.va), fence);
		1928	break;
		1929	default:
		1930	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
		1931	}
		1932	}
		1933
		1934	static const struct xe_migrate_pt_update_ops migrate_ops = {
		1935	.populate = xe_vm_populate_pgtable,
		1936	.clear = xe_migrate_clear_pgtable_callback,
1584	.pre_commit = xe_pt_pre_commit,	1937	.pre_commit = xe_pt_pre_commit,
1585	};	1938	};
1586		1939
1587	static const struct xe_migrate_pt_update_ops userptr_unbind_ops = {	1940	static const struct xe_migrate_pt_update_ops userptr_migrate_ops = {
1588	.populate = xe_migrate_clear_pgtable_callback,	1941	.populate = xe_vm_populate_pgtable,
		1942	.clear = xe_migrate_clear_pgtable_callback,
1589	.pre_commit = xe_pt_userptr_pre_commit,	1943	.pre_commit = xe_pt_userptr_pre_commit,
1590	};	1944	};
1591		1945
1592	/**	1946	/**
1593	* __xe_pt_unbind_vma() - Disconnect and free a page-table tree for the vma	1947	* xe_pt_update_ops_run() - Run PT update operations
1594	* address range.	1948	* @tile: Tile of PT update operations
1595	* @tile: The tile to unbind for.	1949	* @vops: VMA operationa
1596	* @vma: The vma to unbind.
1597	* @q: The exec_queue with which to do pipelined page-table updates.
1598	* @syncs: Entries to sync on before disconnecting the tree to be destroyed.
1599	* @num_syncs: Number of @sync entries.
1600	*	1950	*
1601	* This function builds a the xe_vm_pgtable_update entries abstracting the	1951	* Run PT update operations which includes committing internal PT state changes,
1602	* operations needed to detach the page-table tree to be destroyed from the	1952	* creating job for PT update operations for leaf insertion / removal, and
1603	* man vm tree.	1953	* installing job fence in various places.
1604	* It then takes the relevant locks and submits the operations for
1605	* pipelined detachment of the gpu page-table from the vm main tree,
1606	* (which can be done either by the cpu and the GPU), Finally it frees the
1607	* detached page-table tree.
1608	*	1954	*
1609	* Return: A valid dma-fence representing the pipelined detachment operation	1955	* Return: fence on success, negative ERR_PTR on error.
1610	* on success, an error pointer on error.
1611	*/	1956	*/
1612	struct dma_fence *	1957	struct dma_fence *
1613	__xe_pt_unbind_vma(struct xe_tile tile, struct xe_vma vma, struct xe_exec_queue *q,	1958	xe_pt_update_ops_run(struct xe_tile tile, struct xe_vma_ops vops)
1614	struct xe_sync_entry *syncs, u32 num_syncs)
1615	{	1959	{
1616	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];	1960	struct xe_vm *vm = vops->vm;
1617	struct xe_pt_migrate_pt_update unbind_pt_update = {	1961	struct xe_vm_pgtable_update_ops *pt_update_ops =
1618	.base = {	1962	&vops->pt_update_ops[tile->id];
1619	.ops = xe_vma_is_userptr(vma) ? &userptr_unbind_ops :	1963	struct dma_fence *fence;
1620	&unbind_ops,	1964	struct invalidation_fence *ifence = NULL;
1621	.vma = vma,
1622	.tile_id = tile->id,
1623	},
1624	};
1625	struct xe_vm *vm = xe_vma_vm(vma);
1626	u32 num_entries;
1627	struct dma_fence *fence = NULL;
1628	struct invalidation_fence *ifence;
1629	struct xe_range_fence *rfence;	1965	struct xe_range_fence *rfence;
1630	int err;	1966	struct xe_vma_op *op;
1631		1967	int err = 0, i;
1632	LLIST_HEAD(deferred);	1968	struct xe_migrate_pt_update update = {
		1969	.ops = pt_update_ops->needs_userptr_lock ?
		1970	&userptr_migrate_ops :
		1971	&migrate_ops,
		1972	.vops = vops,
		1973	.tile_id = tile->id,
		1974	};
1633		1975
1634	xe_bo_assert_held(xe_vma_bo(vma));	1976	lockdep_assert_held(&vm->lock);
1635	xe_vm_assert_held(vm);	1977	xe_vm_assert_held(vm);
1636		1978
1637	vm_dbg(&xe_vma_vm(vma)->xe->drm,	1979	if (!pt_update_ops->current_op) {
1638	"Preparing unbind, with range [%llx...%llx) engine %p.\n",	1980	xe_tile_assert(tile, xe_vm_in_fault_mode(vm));
1639	xe_vma_start(vma), xe_vma_end(vma), q);
1640
1641	num_entries = xe_pt_stage_unbind(tile, vma, entries);
1642	xe_tile_assert(tile, num_entries <= ARRAY_SIZE(entries));
1643		1981
1644	xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries);	1982	return dma_fence_get_stub();
1645	xe_pt_calc_rfence_interval(vma, &unbind_pt_update, entries,	1983	}
1646	num_entries);
1647		1984
1648	err = dma_resv_reserve_fences(xe_vm_resv(vm), 1);	1985	#ifdef TEST_VM_OPS_ERROR
1649	if (!err && !xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)	1986	if (vops->inject_error &&
1650	err = dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv, 1);	1987	vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
1651	if (err)	1988	return ERR_PTR(-ENOSPC);
1652	return ERR_PTR(err);	1989	#endif
1653		1990
1654	ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);	1991	if (pt_update_ops->needs_invalidation) {
1655	if (!ifence)	1992	ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
1656	return ERR_PTR(-ENOMEM);	1993	if (!ifence) {
		1994	err = -ENOMEM;
		1995	goto kill_vm_tile1;
		1996	}
		1997	}
1657		1998
1658	rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);	1999	rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
1659	if (!rfence) {	2000	if (!rfence) {
1660	kfree(ifence);	2001	err = -ENOMEM;
1661	return ERR_PTR(-ENOMEM);	2002	goto free_ifence;
1662	}	2003	}
1663		2004
1664	/*	2005	fence = xe_migrate_update_pgtables(tile->migrate, &update);
1665	* Even if we were already evicted and unbind to destroy, we need to	2006	if (IS_ERR(fence)) {
1666	* clear again here. The eviction may have updated pagetables at a	2007	err = PTR_ERR(fence);
1667	* lower level, because it needs to be more conservative.	2008	goto free_rfence;
1668	*/	2009	}
1669	fence = xe_migrate_update_pgtables(tile->migrate,
1670	vm, NULL, q ? q :
1671	vm->q[tile->id],
1672	entries, num_entries,
1673	syncs, num_syncs,
1674	&unbind_pt_update.base);
1675	if (!IS_ERR(fence)) {
1676	int err;
1677
1678	err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
1679	&xe_range_fence_kfree_ops,
1680	unbind_pt_update.base.start,
1681	unbind_pt_update.base.last, fence);
1682	if (err)
1683	dma_fence_wait(fence, false);
1684		2010
1685	/* TLB invalidation must be done before signaling unbind */	2011	/* Point of no return - VM killed if failure after this */
1686	err = invalidation_fence_init(tile->primary_gt, ifence, fence,	2012	for (i = 0; i < pt_update_ops->current_op; ++i) {
1687	xe_vma_start(vma),	2013	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
1688	xe_vma_end(vma),
1689	xe_vma_vm(vma)->usm.asid);
1690	if (err) {
1691	dma_fence_put(fence);
1692	kfree(ifence);
1693	return ERR_PTR(err);
1694	}
1695	fence = &ifence->base.base;
1696		2014
1697	/* add shared fence now for pagetable delayed destroy */	2015	xe_pt_commit(pt_op->vma, pt_op->entries,
1698	dma_resv_add_fence(xe_vm_resv(vm), fence,	2016	pt_op->num_entries, &pt_update_ops->deferred);
1699	DMA_RESV_USAGE_BOOKKEEP);	2017	pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */
		2018	}
1700		2019
1701	/* This fence will be installed by caller when doing eviction */	2020	if (xe_range_fence_insert(&vm->rftree[tile->id], rfence,
1702	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)	2021	&xe_range_fence_kfree_ops,
1703	dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,	2022	pt_update_ops->start,
1704	DMA_RESV_USAGE_BOOKKEEP);	2023	pt_update_ops->last, fence))
1705	xe_pt_commit_unbind(vma, entries, num_entries,	2024	dma_fence_wait(fence, false);
1706	unbind_pt_update.locked ? &deferred : NULL);	2025
1707	vma->tile_present &= ~BIT(tile->id);	2026	/* tlb invalidation must be done before signaling rebind */
1708	} else {	2027	if (ifence) {
1709	kfree(rfence);	2028	invalidation_fence_init(tile->primary_gt, ifence, fence,
1710	kfree(ifence);	2029	pt_update_ops->start,
		2030	pt_update_ops->last, vm->usm.asid);
		2031	fence = &ifence->base.base;
1711	}	2032	}
1712		2033
1713	if (!vma->tile_present)	2034	dma_resv_add_fence(xe_vm_resv(vm), fence,
1714	list_del_init(&vma->combined_links.rebind);	2035	pt_update_ops->wait_vm_bookkeep ?
		2036	DMA_RESV_USAGE_KERNEL :
		2037	DMA_RESV_USAGE_BOOKKEEP);
1715		2038
1716	if (unbind_pt_update.locked) {	2039	list_for_each_entry(op, &vops->list, link)
1717	xe_tile_assert(tile, xe_vma_is_userptr(vma));	2040	op_commit(vops->vm, tile, pt_update_ops, op, fence);
1718		2041
1719	if (!vma->tile_present) {	2042	if (pt_update_ops->needs_userptr_lock)
1720	spin_lock(&vm->userptr.invalidated_lock);
1721	list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link);
1722	spin_unlock(&vm->userptr.invalidated_lock);
1723	}
1724	up_read(&vm->userptr.notifier_lock);	2043	up_read(&vm->userptr.notifier_lock);
1725	xe_bo_put_commit(&deferred);
1726	}
1727		2044
1728	return fence;	2045	return fence;
		2046
		2047	free_rfence:
		2048	kfree(rfence);
		2049	free_ifence:
		2050	kfree(ifence);
		2051	kill_vm_tile1:
		2052	if (err != -EAGAIN && tile->id)
		2053	xe_vm_kill(vops->vm, false);
		2054
		2055	return ERR_PTR(err);
		2056	}
		2057
		2058	/**
		2059	* xe_pt_update_ops_fini() - Finish PT update operations
		2060	* @tile: Tile of PT update operations
		2061	* @vops: VMA operations
		2062	*
		2063	* Finish PT update operations by committing to destroy page table memory
		2064	*/
		2065	void xe_pt_update_ops_fini(struct xe_tile tile, struct xe_vma_ops vops)
		2066	{
		2067	struct xe_vm_pgtable_update_ops *pt_update_ops =
		2068	&vops->pt_update_ops[tile->id];
		2069	int i;
		2070
		2071	lockdep_assert_held(&vops->vm->lock);
		2072	xe_vm_assert_held(vops->vm);
		2073
		2074	for (i = 0; i < pt_update_ops->current_op; ++i) {
		2075	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
		2076
		2077	xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
		2078	}
		2079	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
		2080	}
		2081
		2082	/**
		2083	* xe_pt_update_ops_abort() - Abort PT update operations
		2084	* @tile: Tile of PT update operations
		2085	* @vops: VMA operationa
		2086	*
		2087	* Abort PT update operations by unwinding internal PT state
		2088	*/
		2089	void xe_pt_update_ops_abort(struct xe_tile tile, struct xe_vma_ops vops)
		2090	{
		2091	struct xe_vm_pgtable_update_ops *pt_update_ops =
		2092	&vops->pt_update_ops[tile->id];
		2093	int i;
		2094
		2095	lockdep_assert_held(&vops->vm->lock);
		2096	xe_vm_assert_held(vops->vm);
		2097
		2098	for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
		2099	struct xe_vm_pgtable_update_op *pt_op =
		2100	&pt_update_ops->ops[i];
		2101
		2102	if (!pt_op->vma \|\| i >= pt_update_ops->current_op)
		2103	continue;
		2104
		2105	if (pt_op->bind)
		2106	xe_pt_abort_bind(pt_op->vma, pt_op->entries,
		2107	pt_op->num_entries,
		2108	pt_op->rebind);
		2109	else
		2110	xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
		2111	pt_op->num_entries);
		2112	}
		2113
		2114	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
1729	}	2115	}

drivers/gpu/drm/xe/xe_pt.h+6−8

@@ -17,6 +17,7 @@ struct xe_sync_entry; · 16 unmodified lines
17	struct xe_tile;	17	struct xe_tile;
18	struct xe_vm;	18	struct xe_vm;
19	struct xe_vma;	19	struct xe_vma;
		20	struct xe_vma_ops;
20		21
21	/* Largest huge pte is currently 1GiB. May become device dependent. */	22	/* Largest huge pte is currently 1GiB. May become device dependent. */
22	#define MAX_HUGEPTE_LEVEL 2	23	#define MAX_HUGEPTE_LEVEL 2
@@ -34,14 +35,11 @@ void xe_pt_populate_empty(struct xe_tile tile, struct xe_vm vm, · 11 unmodified lines
34		35
35	void xe_pt_destroy(struct xe_pt pt, u32 flags, struct llist_head deferred);	36	void xe_pt_destroy(struct xe_pt pt, u32 flags, struct llist_head deferred);
36		37
37	struct dma_fence *	38	int xe_pt_update_ops_prepare(struct xe_tile tile, struct xe_vma_ops vops);
38	__xe_pt_bind_vma(struct xe_tile tile, struct xe_vma vma, struct xe_exec_queue *q,	39	struct dma_fence xe_pt_update_ops_run(struct xe_tile tile,
39	struct xe_sync_entry *syncs, u32 num_syncs,	40	struct xe_vma_ops *vops);
40	bool rebind);	41	void xe_pt_update_ops_fini(struct xe_tile tile, struct xe_vma_ops vops);
41		42	void xe_pt_update_ops_abort(struct xe_tile tile, struct xe_vma_ops vops);
42	struct dma_fence *
43	__xe_pt_unbind_vma(struct xe_tile tile, struct xe_vma vma, struct xe_exec_queue *q,
44	struct xe_sync_entry *syncs, u32 num_syncs);
45		43
46	bool xe_pt_zap_ptes(struct xe_tile tile, struct xe_vma vma);	44	bool xe_pt_zap_ptes(struct xe_tile tile, struct xe_vma vma);
47		45

drivers/gpu/drm/xe/xe_pt_types.h+48−0

@@ -74,4 +74,52 @@ struct xe_vm_pgtable_update { · 73 unmodified lines
74	u32 flags;	74	u32 flags;
75	};	75	};
76		76
		77	/** struct xe_vm_pgtable_update_op - Page table update operation */
		78	struct xe_vm_pgtable_update_op {
		79	/** @entries: entries to update for this operation */
		80	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
		81	/** @vma: VMA for operation, operation not valid if NULL */
		82	struct xe_vma *vma;
		83	/** @num_entries: number of entries for this update operation */
		84	u32 num_entries;
		85	/** @bind: is a bind */
		86	bool bind;
		87	/** @rebind: is a rebind */
		88	bool rebind;
		89	};
		90
		91	/** struct xe_vm_pgtable_update_ops: page table update operations */
		92	struct xe_vm_pgtable_update_ops {
		93	/** @ops: operations */
		94	struct xe_vm_pgtable_update_op *ops;
		95	/** @deferred: deferred list to destroy PT entries */
		96	struct llist_head deferred;
		97	/** @q: exec queue for PT operations */
		98	struct xe_exec_queue *q;
		99	/** @start: start address of ops */
		100	u64 start;
		101	/** @last: last address of ops */
		102	u64 last;
		103	/** @num_ops: number of operations */
		104	u32 num_ops;
		105	/** @current_op: current operations */
		106	u32 current_op;
		107	/** @needs_userptr_lock: Needs userptr lock */
		108	bool needs_userptr_lock;
		109	/** @needs_invalidation: Needs invalidation */
		110	bool needs_invalidation;
		111	/**
		112	* @wait_vm_bookkeep: PT operations need to wait until VM is idle
		113	* (bookkeep dma-resv slots are idle) and stage all future VM activity
		114	* behind these operations (install PT operations into VM kernel
		115	* dma-resv slot).
		116	*/
		117	bool wait_vm_bookkeep;
		118	/**
		119	* @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv
		120	* slots are idle.
		121	*/
		122	bool wait_vm_kernel;
		123	};
		124
77	#endif	125	#endif

drivers/gpu/drm/xe/xe_query.c+3−1

 		if (err)
 			return err;
-		topo.type = DRM_XE_TOPO_EU_PER_DSS;
+		topo.type = gt->fuse_topo.eu_type == XE_GT_EU_TYPE_SIMD16 ?
+			DRM_XE_TOPO_SIMD16_EU_PER_DSS :
+			DRM_XE_TOPO_EU_PER_DSS;
 		err = copy_mask(&query_ptr, &topo,
 				gt->fuse_topo.eu_mask_per_dss,
 				sizeof(gt->fuse_topo.eu_mask_per_dss));

drivers/gpu/drm/xe/xe_rtp.c+12−30

 	ctx->active_entries = active_entries;
 	ctx->n_entries = n_entries;
 }
+EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process_ctx_enable_active_tracking);
 static void rtp_mark_active(struct xe_device *xe,
 			    struct xe_rtp_process_ctx *ctx,
-			    unsigned int first, unsigned int last)
+			    unsigned int idx)
 {
 	if (!ctx->active_entries)
 		return;
-	if (drm_WARN_ON(&xe->drm, last > ctx->n_entries))
+	if (drm_WARN_ON(&xe->drm, idx >= ctx->n_entries))
 		return;
-	if (first == last)
+	bitmap_set(ctx->active_entries, idx, 1);
-		bitmap_set(ctx->active_entries, first, 1);
-	else
-		bitmap_set(ctx->active_entries, first, last - first + 2);
 }
 /**
 		}
 		if (match)
-			rtp_mark_active(xe, ctx, entry - entries,
+			rtp_mark_active(xe, ctx, entry - entries);
-					entry - entries);
 	}
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process_to_sr);
  * @entries: Table with RTP definitions
  *
  * Walk the table pointed by @entries (with an empty sentinel), executing the
- * rules. A few differences from xe_rtp_process_to_sr():
+ * rules. One difference from xe_rtp_process_to_sr(): there is no action
- *
+ * associated with each entry since this uses struct xe_rtp_entry. Its main use
- * 1. There is no action associated with each entry since this uses
+ * is for marking active workarounds via
- *    struct xe_rtp_entry. Its main use is for marking active workarounds via
+ * xe_rtp_process_ctx_enable_active_tracking().
- *    xe_rtp_process_ctx_enable_active_tracking().
- * 2. There is support for OR operations by having entries with no name.
  */
 void xe_rtp_process(struct xe_rtp_process_ctx *ctx,
 		    const struct xe_rtp_entry *entries)
 {
-	const struct xe_rtp_entry *entry, *first_entry;
+	const struct xe_rtp_entry *entry;
 	struct xe_hw_engine *hwe;
 	struct xe_gt *gt;
 	struct xe_device *xe;
 	rtp_get_context(ctx, &hwe, &gt, &xe);
-	first_entry = entries;
-	if (drm_WARN_ON(&xe->drm, !first_entry->name))
-		return;
 	for (entry = entries; entry && entry->rules; entry++) {
-		if (entry->name)
-			first_entry = entry;
 		if (!rule_matches(xe, gt, hwe, entry->rules, entry->n_rules))
 			continue;
-		/* Fast-forward entry, eliminating the OR'ed entries */
+		rtp_mark_active(xe, ctx, entry - entries);
-		for (entry++; entry && entry->rules; entry++)
-			if (entry->name)
-				break;
-		entry--;
-		rtp_mark_active(xe, ctx, first_entry - entries,
-				entry - entries);
 	}
 }
+EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process);
 bool xe_rtp_match_even_instance(const struct xe_gt *gt,
 				const struct xe_hw_engine *hwe)

drivers/gpu/drm/xe/xe_rtp.h+2−2

@@ -374,7 +374,7 @@ struct xe_reg_sr; · 373 unmodified lines
374	* XE_RTP_RULES - Helper to set multiple rules to a struct xe_rtp_entry_sr entry	374	* XE_RTP_RULES - Helper to set multiple rules to a struct xe_rtp_entry_sr entry
375	* @...: Rules	375	* @...: Rules
376	*	376	*
377	* At least one rule is needed and up to 6 are supported. Multiple rules are	377	* At least one rule is needed and up to 12 are supported. Multiple rules are
378	* AND'ed together, i.e. all the rules must evaluate to true for the entry to	378	* AND'ed together, i.e. all the rules must evaluate to true for the entry to
379	* be processed. See XE_RTP_MATCH_* for the possible match rules. Example:	379	* be processed. See XE_RTP_MATCH_* for the possible match rules. Example:
380	*	380	*
@@ -399,7 +399,7 @@ struct xe_reg_sr; · 18 unmodified lines
399	* XE_RTP_ACTIONS - Helper to set multiple actions to a struct xe_rtp_entry_sr	399	* XE_RTP_ACTIONS - Helper to set multiple actions to a struct xe_rtp_entry_sr
400	* @...: Actions to be taken	400	* @...: Actions to be taken
401	*	401	*
402	* At least one action is needed and up to 6 are supported. See XE_RTP_ACTION_*	402	* At least one action is needed and up to 12 are supported. See XE_RTP_ACTION_*
403	* for the possible actions. Example:	403	* for the possible actions. Example:
404	*	404	*
405	* .. code-block:: c	405	* .. code-block:: c

drivers/gpu/drm/xe/xe_rtp_helpers.h+6−0

 #define XE_RTP_PASTE_4(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_3(prefix_, sep_, _XE_TUPLE_TAIL args_)
 #define XE_RTP_PASTE_5(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_4(prefix_, sep_, _XE_TUPLE_TAIL args_)
 #define XE_RTP_PASTE_6(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_5(prefix_, sep_, _XE_TUPLE_TAIL args_)
+#define XE_RTP_PASTE_7(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_6(prefix_, sep_, _XE_TUPLE_TAIL args_)
+#define XE_RTP_PASTE_8(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_7(prefix_, sep_, _XE_TUPLE_TAIL args_)
+#define XE_RTP_PASTE_9(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_8(prefix_, sep_, _XE_TUPLE_TAIL args_)
+#define XE_RTP_PASTE_10(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_9(prefix_, sep_, _XE_TUPLE_TAIL args_)
+#define XE_RTP_PASTE_11(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_10(prefix_, sep_, _XE_TUPLE_TAIL args_)
+#define XE_RTP_PASTE_12(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_11(prefix_, sep_, _XE_TUPLE_TAIL args_)
 /*
  * XE_RTP_DROP_CAST - Drop cast to convert a compound statement to a initializer

drivers/gpu/drm/xe/xe_sa.c+7−0

@@ -84,6 +84,13 @@ struct xe_sa_manager xe_sa_bo_manager_init(struct xe_tile tile, u32 size, u32 · 83 unmodified lines
84	struct drm_suballoc xe_sa_bo_new(struct xe_sa_manager sa_manager,	84	struct drm_suballoc xe_sa_bo_new(struct xe_sa_manager sa_manager,
85	unsigned int size)	85	unsigned int size)
86	{	86	{
		87	/*
		88	* BB to large, return -ENOBUFS indicating user should split
		89	* array of binds into smaller chunks.
		90	*/
		91	if (size > sa_manager->base.size)
		92	return ERR_PTR(-ENOBUFS);
		93
87	return drm_suballoc_new(&sa_manager->base, size, GFP_KERNEL, true, 0);	94	return drm_suballoc_new(&sa_manager->base, size, GFP_KERNEL, true, 0);
88	}	95	}
89		96

drivers/gpu/drm/xe/xe_sriov.c+1−1

@@ -5,7 +5,7 @@ · 4 unmodified lines
5		5
6	#include <drm/drm_managed.h>	6	#include <drm/drm_managed.h>
7		7
8	#include "regs/xe_sriov_regs.h"	8	#include "regs/xe_regs.h"
9		9
10	#include "xe_assert.h"	10	#include "xe_assert.h"
11	#include "xe_device.h"	11	#include "xe_device.h"

drivers/gpu/drm/xe/xe_sync.c+8−12

 					       u64 value)
 {
 	struct xe_user_fence *ufence;
+	u64 __user *ptr = u64_to_user_ptr(addr);
+	if (!access_ok(ptr, sizeof(ptr)))
+		return ERR_PTR(-EFAULT);
 	ufence = kmalloc(sizeof(*ufence), GFP_KERNEL);
 	if (!ufence)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 	ufence->xe = xe;
 	kref_init(&ufence->refcount);
-	ufence->addr = u64_to_user_ptr(addr);
+	ufence->addr = ptr;
 	ufence->value = value;
 	ufence->mm = current->mm;
 	mmgrab(ufence->mm);
 		} else {
 			sync->ufence = user_fence_create(xe, sync_in.addr,
 							 sync_in.timeline_value);
-			if (XE_IOCTL_DBG(xe, !sync->ufence))
+			if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence)))
-				return -ENOMEM;
+				return PTR_ERR(sync->ufence);
 		}
 		break;
 	return 0;
 }
-int xe_sync_entry_wait(struct xe_sync_entry *sync)
-{
-	if (sync->fence)
-		dma_fence_wait(sync->fence, true);
-	return 0;
-}
 int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
 {
 	int err;

drivers/gpu/drm/xe/xe_sync.h+0−1

@@ -22,7 +22,6 @@ int xe_sync_entry_parse(struct xe_device xe, struct xe_file xef, · 21 unmodified lines
22	struct xe_sync_entry *sync,	22	struct xe_sync_entry *sync,
23	struct drm_xe_sync __user *sync_user,	23	struct drm_xe_sync __user *sync_user,
24	unsigned int flags);	24	unsigned int flags);
25	int xe_sync_entry_wait(struct xe_sync_entry *sync);
26	int xe_sync_entry_add_deps(struct xe_sync_entry *sync,	25	int xe_sync_entry_add_deps(struct xe_sync_entry *sync,
27	struct xe_sched_job *job);	26	struct xe_sched_job *job);
28	void xe_sync_entry_signal(struct xe_sync_entry *sync,	27	void xe_sync_entry_signal(struct xe_sync_entry *sync,

drivers/gpu/drm/xe/xe_trace.h+52−0

@@ -369,6 +369,58 @@ TRACE_EVENT(xe_reg_rw, · 368 unmodified lines
369	(u32)(__entry->val >> 32))	369	(u32)(__entry->val >> 32))
370	);	370	);
371		371
		372	DECLARE_EVENT_CLASS(xe_pm_runtime,
		373	TP_PROTO(struct xe_device xe, void caller),
		374	TP_ARGS(xe, caller),
		375
		376	TP_STRUCT__entry(
		377	__string(dev, __dev_name_xe(xe))
		378	__field(void *, caller)
		379	),
		380
		381	TP_fast_assign(
		382	__assign_str(dev);
		383	__entry->caller = caller;
		384	),
		385
		386	TP_printk("dev=%s caller_function=%pS", __get_str(dev), __entry->caller)
		387	);
		388
		389	DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_get,
		390	TP_PROTO(struct xe_device xe, void caller),
		391	TP_ARGS(xe, caller)
		392	);
		393
		394	DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_put,
		395	TP_PROTO(struct xe_device xe, void caller),
		396	TP_ARGS(xe, caller)
		397	);
		398
		399	DEFINE_EVENT(xe_pm_runtime, xe_pm_resume,
		400	TP_PROTO(struct xe_device xe, void caller),
		401	TP_ARGS(xe, caller)
		402	);
		403
		404	DEFINE_EVENT(xe_pm_runtime, xe_pm_suspend,
		405	TP_PROTO(struct xe_device xe, void caller),
		406	TP_ARGS(xe, caller)
		407	);
		408
		409	DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_resume,
		410	TP_PROTO(struct xe_device xe, void caller),
		411	TP_ARGS(xe, caller)
		412	);
		413
		414	DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_suspend,
		415	TP_PROTO(struct xe_device xe, void caller),
		416	TP_ARGS(xe, caller)
		417	);
		418
		419	DEFINE_EVENT(xe_pm_runtime, xe_pm_runtime_get_ioctl,
		420	TP_PROTO(struct xe_device xe, void caller),
		421	TP_ARGS(xe, caller)
		422	);
		423
372	#endif	424	#endif
373		425
374	/* This part must be outside protection */	426	/* This part must be outside protection */

drivers/gpu/drm/xe/xe_trace_bo.h+5−5

 	     TP_ARGS(vma)
 );
-DEFINE_EVENT(xe_vma, xe_vma_fail,
-	     TP_PROTO(struct xe_vma *vma),
-	     TP_ARGS(vma)
-);
 DEFINE_EVENT(xe_vma, xe_vma_bind,
 	     TP_PROTO(struct xe_vma *vma),
 	     TP_ARGS(vma)
 	     TP_ARGS(vm)
 );
+DEFINE_EVENT(xe_vm, xe_vm_ops_fail,
+	     TP_PROTO(struct xe_vm *vm),
+	     TP_ARGS(vm)
+);
 #endif
 /* This part must be outside protection */

drivers/gpu/drm/xe/xe_tuning.c+8−0

@@ -93,6 +93,14 @@ static const struct xe_rtp_entry_sr lrc_tunings[] = { · 92 unmodified lines
93	REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)))	93	REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)))
94	},	94	},
95		95
		96	/* Xe2_HPG */
		97
		98	{ XE_RTP_NAME("Tuning: vs hit max value"),
		99	XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)),
		100	XE_RTP_ACTIONS(FIELD_SET(FF_MODE, VS_HIT_MAX_VALUE_MASK,
		101	REG_FIELD_PREP(VS_HIT_MAX_VALUE_MASK, 0x3f)))
		102	},
		103
96	{}	104	{}
97	};	105	};
98		106

drivers/gpu/drm/xe/xe_uc_fw.c+3−0

 	fw_def(TIGERLAKE,	major_ver(i915,	guc,	tgl,	70, 19, 2))
 #define XE_HUC_FIRMWARE_DEFS(fw_def, mmp_ver, no_ver)		\
+	fw_def(BATTLEMAGE,	no_ver(xe,	huc,		bmg))		\
+	fw_def(LUNARLAKE,	no_ver(xe,	huc,		lnl))		\
 	fw_def(METEORLAKE,	no_ver(i915,	huc_gsc,	mtl))		\
 	fw_def(DG1,		no_ver(i915,	huc,		dg1))		\
 	fw_def(ALDERLAKE_P,	no_ver(i915,	huc,		tgl))		\
 /* for the GSC FW we match the compatibility version and not the release one */
 #define XE_GSC_FIRMWARE_DEFS(fw_def, major_ver)		\
+	fw_def(LUNARLAKE,	major_ver(xe,	gsc,	lnl,	1, 0, 0)) \
 	fw_def(METEORLAKE,	major_ver(i915,	gsc,	mtl,	1, 0, 0))
 #define MAKE_FW_PATH(dir__, uc__, shortname__, version__)			\

drivers/gpu/drm/xe/xe_vm.c+273−426

@@ -133,8 +133,10 @@ static int wait_for_existing_preempt_fences(struct xe_vm *vm) · 132 unmodified lines
133	if (q->lr.pfence) {	133	if (q->lr.pfence) {
134	long timeout = dma_fence_wait(q->lr.pfence, false);	134	long timeout = dma_fence_wait(q->lr.pfence, false);
135		135
136	if (timeout < 0)	136	/* Only -ETIME on fence indicates VM needs to be killed */
		137	if (timeout < 0 \|\| q->lr.pfence->error == -ETIME)
137	return -ETIME;	138	return -ETIME;
		139
138	dma_fence_put(q->lr.pfence);	140	dma_fence_put(q->lr.pfence);
139	q->lr.pfence = NULL;	141	q->lr.pfence = NULL;
140	}	142	}
@@ -311,7 +313,15 @@ int __xe_vm_userptr_needs_repin(struct xe_vm *vm) · 170 unmodified lines
311		313
312	#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000	314	#define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
313		315
314	static void xe_vm_kill(struct xe_vm *vm, bool unlocked)	316	/**
		317	* xe_vm_kill() - VM Kill
		318	* @vm: The VM.
		319	* @unlocked: Flag indicates the VM's dma-resv is not held
		320	*
		321	* Kill the VM by setting banned flag indicated VM is no longer available for
		322	* use. If in preempt fence mode, also kill all exec queue attached to the VM.
		323	*/
		324	void xe_vm_kill(struct xe_vm *vm, bool unlocked)
315	{	325	{
316	struct xe_exec_queue *q;	326	struct xe_exec_queue *q;
317		327
@@ -708,6 +718,42 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm) · 390 unmodified lines
708	list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;	718	list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
709	}	719	}
710		720
		721	static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds)
		722	{
		723	int i;
		724
		725	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
		726	if (!vops->pt_update_ops[i].num_ops)
		727	continue;
		728
		729	vops->pt_update_ops[i].ops =
		730	kmalloc_array(vops->pt_update_ops[i].num_ops,
		731	sizeof(*vops->pt_update_ops[i].ops),
		732	GFP_KERNEL);
		733	if (!vops->pt_update_ops[i].ops)
		734	return array_of_binds ? -ENOBUFS : -ENOMEM;
		735	}
		736
		737	return 0;
		738	}
		739
		740	static void xe_vma_ops_fini(struct xe_vma_ops *vops)
		741	{
		742	int i;
		743
		744	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
		745	kfree(vops->pt_update_ops[i].ops);
		746	}
		747
		748	static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask)
		749	{
		750	int i;
		751
		752	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
		753	if (BIT(i) & tile_mask)
		754	++vops->pt_update_ops[i].num_ops;
		755	}
		756
711	static void xe_vm_populate_rebind(struct xe_vma_op op, struct xe_vma vma,	757	static void xe_vm_populate_rebind(struct xe_vma_op op, struct xe_vma vma,
712	u8 tile_mask)	758	u8 tile_mask)
713	{	759	{
@@ -735,6 +781,7 @@ static int xe_vm_ops_add_rebind(struct xe_vma_ops vops, struct xe_vma vma, · 21 unmodified lines
735		781
736	xe_vm_populate_rebind(op, vma, tile_mask);	782	xe_vm_populate_rebind(op, vma, tile_mask);
737	list_add_tail(&op->link, &vops->list);	783	list_add_tail(&op->link, &vops->list);
		784	xe_vma_ops_incr_pt_update_ops(vops, tile_mask);
738		785
739	return 0;	786	return 0;
740	}	787	}
@@ -751,7 +798,7 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) · 10 unmodified lines
751	struct xe_vma vma, next;	798	struct xe_vma vma, next;
752	struct xe_vma_ops vops;	799	struct xe_vma_ops vops;
753	struct xe_vma_op op, next_op;	800	struct xe_vma_op op, next_op;
754	int err;	801	int err, i;
755		802
756	lockdep_assert_held(&vm->lock);	803	lockdep_assert_held(&vm->lock);
757	if ((xe_vm_in_lr_mode(vm) && !rebind_worker) \|\|	804	if ((xe_vm_in_lr_mode(vm) && !rebind_worker) \|\|
@@ -759,6 +806,8 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) · 1 unmodified line
759	return 0;	806	return 0;
760		807
761	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);	808	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
		809	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
		810	vops.pt_update_ops[i].wait_vm_bookkeep = true;
762		811
763	xe_vm_assert_held(vm);	812	xe_vm_assert_held(vm);
764	list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {	813	list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
@@ -775,6 +824,10 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) · 10 unmodified lines
775	goto free_ops;	824	goto free_ops;
776	}	825	}
777		826
		827	err = xe_vma_ops_alloc(&vops, false);
		828	if (err)
		829	goto free_ops;
		830
778	fence = ops_execute(vm, &vops);	831	fence = ops_execute(vm, &vops);
779	if (IS_ERR(fence)) {	832	if (IS_ERR(fence)) {
780	err = PTR_ERR(fence);	833	err = PTR_ERR(fence);
@@ -789,6 +842,7 @@ free_ops: · 8 unmodified lines
789	list_del(&op->link);	842	list_del(&op->link);
790	kfree(op);	843	kfree(op);
791	}	844	}
		845	xe_vma_ops_fini(&vops);
792		846
793	return err;	847	return err;
794	}	848	}
@@ -798,6 +852,8 @@ struct dma_fence xe_vma_rebind(struct xe_vm vm, struct xe_vma *vma, u8 tile_ma · 3 unmodified lines
798	struct dma_fence *fence = NULL;	852	struct dma_fence *fence = NULL;
799	struct xe_vma_ops vops;	853	struct xe_vma_ops vops;
800	struct xe_vma_op op, next_op;	854	struct xe_vma_op op, next_op;
		855	struct xe_tile *tile;
		856	u8 id;
801	int err;	857	int err;
802		858
803	lockdep_assert_held(&vm->lock);	859	lockdep_assert_held(&vm->lock);
@@ -805,17 +861,30 @@ struct dma_fence xe_vma_rebind(struct xe_vm vm, struct xe_vma *vma, u8 tile_ma · 1 unmodified line
805	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));	861	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
806		862
807	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);	863	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
		864	for_each_tile(tile, vm->xe, id) {
		865	vops.pt_update_ops[id].wait_vm_bookkeep = true;
		866	vops.pt_update_ops[tile->id].q =
		867	xe_tile_migrate_exec_queue(tile);
		868	}
808		869
809	err = xe_vm_ops_add_rebind(&vops, vma, tile_mask);	870	err = xe_vm_ops_add_rebind(&vops, vma, tile_mask);
810	if (err)	871	if (err)
811	return ERR_PTR(err);	872	return ERR_PTR(err);
812		873
		874	err = xe_vma_ops_alloc(&vops, false);
		875	if (err) {
		876	fence = ERR_PTR(err);
		877	goto free_ops;
		878	}
		879
813	fence = ops_execute(vm, &vops);	880	fence = ops_execute(vm, &vops);
814		881
		882	free_ops:
815	list_for_each_entry_safe(op, next_op, &vops.list, link) {	883	list_for_each_entry_safe(op, next_op, &vops.list, link) {
816	list_del(&op->link);	884	list_del(&op->link);
817	kfree(op);	885	kfree(op);
818	}	886	}
		887	xe_vma_ops_fini(&vops);
819		888
820	return fence;	889	return fence;
821	}	890	}
@@ -1601,6 +1670,10 @@ static void vm_destroy_work_func(struct work_struct *w) · 779 unmodified lines
1601	XE_WARN_ON(vm->pt_root[id]);	1670	XE_WARN_ON(vm->pt_root[id]);
1602		1671
1603	trace_xe_vm_free(vm);	1672	trace_xe_vm_free(vm);
		1673
		1674	if (vm->xef)
		1675	xe_file_put(vm->xef);
		1676
1604	kfree(vm);	1677	kfree(vm);
1605	}	1678	}
1606		1679
@@ -1637,147 +1710,6 @@ to_wait_exec_queue(struct xe_vm vm, struct xe_exec_queue q) · 30 unmodified lines
1637	return q ? q : vm->q[0];	1710	return q ? q : vm->q[0];
1638	}	1711	}
1639		1712
1640	static struct dma_fence *
1641	xe_vm_unbind_vma(struct xe_vma vma, struct xe_exec_queue q,
1642	struct xe_sync_entry *syncs, u32 num_syncs,
1643	bool first_op, bool last_op)
1644	{
1645	struct xe_vm *vm = xe_vma_vm(vma);
1646	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
1647	struct xe_tile *tile;
1648	struct dma_fence *fence = NULL;
1649	struct dma_fence **fences = NULL;
1650	struct dma_fence_array *cf = NULL;
1651	int cur_fence = 0;
1652	int number_tiles = hweight8(vma->tile_present);
1653	int err;
1654	u8 id;
1655
1656	trace_xe_vma_unbind(vma);
1657
1658	if (number_tiles > 1) {
1659	fences = kmalloc_array(number_tiles, sizeof(*fences),
1660	GFP_KERNEL);
1661	if (!fences)
1662	return ERR_PTR(-ENOMEM);
1663	}
1664
1665	for_each_tile(tile, vm->xe, id) {
1666	if (!(vma->tile_present & BIT(id)))
1667	goto next;
1668
1669	fence = __xe_pt_unbind_vma(tile, vma, q ? q : vm->q[id],
1670	first_op ? syncs : NULL,
1671	first_op ? num_syncs : 0);
1672	if (IS_ERR(fence)) {
1673	err = PTR_ERR(fence);
1674	goto err_fences;
1675	}
1676
1677	if (fences)
1678	fences[cur_fence++] = fence;
1679
1680	next:
1681	if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list))
1682	q = list_next_entry(q, multi_gt_list);
1683	}
1684
1685	if (fences) {
1686	cf = dma_fence_array_create(number_tiles, fences,
1687	vm->composite_fence_ctx,
1688	vm->composite_fence_seqno++,
1689	false);
1690	if (!cf) {
1691	--vm->composite_fence_seqno;
1692	err = -ENOMEM;
1693	goto err_fences;
1694	}
1695	}
1696
1697	fence = cf ? &cf->base : !fence ?
1698	xe_exec_queue_last_fence_get(wait_exec_queue, vm) : fence;
1699
1700	return fence;
1701
1702	err_fences:
1703	if (fences) {
1704	while (cur_fence)
1705	dma_fence_put(fences[--cur_fence]);
1706	kfree(fences);
1707	}
1708
1709	return ERR_PTR(err);
1710	}
1711
1712	static struct dma_fence *
1713	xe_vm_bind_vma(struct xe_vma vma, struct xe_exec_queue q,
1714	struct xe_sync_entry *syncs, u32 num_syncs,
1715	u8 tile_mask, bool first_op, bool last_op)
1716	{
1717	struct xe_tile *tile;
1718	struct dma_fence *fence;
1719	struct dma_fence **fences = NULL;
1720	struct dma_fence_array *cf = NULL;
1721	struct xe_vm *vm = xe_vma_vm(vma);
1722	int cur_fence = 0;
1723	int number_tiles = hweight8(tile_mask);
1724	int err;
1725	u8 id;
1726
1727	trace_xe_vma_bind(vma);
1728
1729	if (number_tiles > 1) {
1730	fences = kmalloc_array(number_tiles, sizeof(*fences),
1731	GFP_KERNEL);
1732	if (!fences)
1733	return ERR_PTR(-ENOMEM);
1734	}
1735
1736	for_each_tile(tile, vm->xe, id) {
1737	if (!(tile_mask & BIT(id)))
1738	goto next;
1739
1740	fence = __xe_pt_bind_vma(tile, vma, q ? q : vm->q[id],
1741	first_op ? syncs : NULL,
1742	first_op ? num_syncs : 0,
1743	vma->tile_present & BIT(id));
1744	if (IS_ERR(fence)) {
1745	err = PTR_ERR(fence);
1746	goto err_fences;
1747	}
1748
1749	if (fences)
1750	fences[cur_fence++] = fence;
1751
1752	next:
1753	if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list))
1754	q = list_next_entry(q, multi_gt_list);
1755	}
1756
1757	if (fences) {
1758	cf = dma_fence_array_create(number_tiles, fences,
1759	vm->composite_fence_ctx,
1760	vm->composite_fence_seqno++,
1761	false);
1762	if (!cf) {
1763	--vm->composite_fence_seqno;
1764	err = -ENOMEM;
1765	goto err_fences;
1766	}
1767	}
1768
1769	return cf ? &cf->base : fence;
1770
1771	err_fences:
1772	if (fences) {
1773	while (cur_fence)
1774	dma_fence_put(fences[--cur_fence]);
1775	kfree(fences);
1776	}
1777
1778	return ERR_PTR(err);
1779	}
1780
1781	static struct xe_user_fence *	1713	static struct xe_user_fence *
1782	find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs)	1714	find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs)
1783	{	1715	{
@@ -1793,48 +1725,6 @@ find_ufence_get(struct xe_sync_entry *syncs, u32 num_syncs) · 9 unmodified lines
1793	return NULL;	1725	return NULL;
1794	}	1726	}
1795		1727
1796	static struct dma_fence *
1797	xe_vm_bind(struct xe_vm vm, struct xe_vma vma, struct xe_exec_queue *q,
1798	struct xe_bo bo, struct xe_sync_entry syncs, u32 num_syncs,
1799	u8 tile_mask, bool immediate, bool first_op, bool last_op)
1800	{
1801	struct dma_fence *fence;
1802	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
1803
1804	xe_vm_assert_held(vm);
1805	xe_bo_assert_held(bo);
1806
1807	if (immediate) {
1808	fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, tile_mask,
1809	first_op, last_op);
1810	if (IS_ERR(fence))
1811	return fence;
1812	} else {
1813	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
1814
1815	fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm);
1816	}
1817
1818	return fence;
1819	}
1820
1821	static struct dma_fence *
1822	xe_vm_unbind(struct xe_vm vm, struct xe_vma vma,
1823	struct xe_exec_queue q, struct xe_sync_entry syncs,
1824	u32 num_syncs, bool first_op, bool last_op)
1825	{
1826	struct dma_fence *fence;
1827
1828	xe_vm_assert_held(vm);
1829	xe_bo_assert_held(xe_vma_bo(vma));
1830
1831	fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op);
1832	if (IS_ERR(fence))
1833	return fence;
1834
1835	return fence;
1836	}
1837
1838	#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE \| \	1728	#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE \| \
1839	DRM_XE_VM_CREATE_FLAG_LR_MODE \| \	1729	DRM_XE_VM_CREATE_FLAG_LR_MODE \| \
1840	DRM_XE_VM_CREATE_FLAG_FAULT_MODE)	1730	DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
@@ -1916,7 +1806,7 @@ int xe_vm_create_ioctl(struct drm_device dev, void data, · 75 unmodified lines
1916	}	1806	}
1917		1807
1918	args->vm_id = id;	1808	args->vm_id = id;
1919	vm->xef = xef;	1809	vm->xef = xe_file_get(xef);
1920		1810
1921	/* Record BO memory for VM pagetable created against client */	1811	/* Record BO memory for VM pagetable created against client */
1922	for_each_tile(tile, xe, id)	1812	for_each_tile(tile, xe, id)
@@ -1975,21 +1865,6 @@ static const u32 region_to_mem_type[] = { · 52 unmodified lines
1975	XE_PL_VRAM1,	1865	XE_PL_VRAM1,
1976	};	1866	};
1977		1867
1978	static struct dma_fence *
1979	xe_vm_prefetch(struct xe_vm vm, struct xe_vma vma,
1980	struct xe_exec_queue q, struct xe_sync_entry syncs,
1981	u32 num_syncs, bool first_op, bool last_op)
1982	{
1983	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
1984
1985	if (vma->tile_mask != (vma->tile_present & ~vma->tile_invalidated)) {
1986	return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
1987	vma->tile_mask, true, first_op, last_op);
1988	} else {
1989	return xe_exec_queue_last_fence_get(wait_exec_queue, vm);
1990	}
1991	}
1992
1993	static void prep_vma_destroy(struct xe_vm vm, struct xe_vma vma,	1868	static void prep_vma_destroy(struct xe_vm vm, struct xe_vma vma,
1994	bool post_commit)	1869	bool post_commit)
1995	{	1870	{
@@ -2277,14 +2152,10 @@ static int xe_vma_op_commit(struct xe_vm vm, struct xe_vma_op op) · 281 unmodified lines
2277	return err;	2152	return err;
2278	}	2153	}
2279		2154
2280		2155	static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct drm_gpuva_ops ops,
2281	static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct xe_exec_queue q,	2156	struct xe_vma_ops *vops)
2282	struct drm_gpuva_ops *ops,
2283	struct xe_sync_entry *syncs, u32 num_syncs,
2284	struct xe_vma_ops *vops, bool last)
2285	{	2157	{
2286	struct xe_device *xe = vm->xe;	2158	struct xe_device *xe = vm->xe;
2287	struct xe_vma_op *last_op = NULL;
2288	struct drm_gpuva_op *__op;	2159	struct drm_gpuva_op *__op;
2289	struct xe_tile *tile;	2160	struct xe_tile *tile;
2290	u8 id, tile_mask = 0;	2161	u8 id, tile_mask = 0;
@@ -2298,19 +2169,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct xe_exec_queue q, · 7 unmodified lines
2298	drm_gpuva_for_each_op(__op, ops) {	2169	drm_gpuva_for_each_op(__op, ops) {
2299	struct xe_vma_op *op = gpuva_op_to_vma_op(__op);	2170	struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
2300	struct xe_vma *vma;	2171	struct xe_vma *vma;
2301	bool first = list_empty(&vops->list);
2302	unsigned int flags = 0;	2172	unsigned int flags = 0;
2303		2173
2304	INIT_LIST_HEAD(&op->link);	2174	INIT_LIST_HEAD(&op->link);
2305	list_add_tail(&op->link, &vops->list);	2175	list_add_tail(&op->link, &vops->list);
2306
2307	if (first) {
2308	op->flags \|= XE_VMA_OP_FIRST;
2309	op->num_syncs = num_syncs;
2310	op->syncs = syncs;
2311	}
2312
2313	op->q = q;
2314	op->tile_mask = tile_mask;	2176	op->tile_mask = tile_mask;
2315		2177
2316	switch (op->base.op) {	2178	switch (op->base.op) {
@@ -2329,6 +2191,9 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct xe_exec_queue q, · 12 unmodified lines
2329	return PTR_ERR(vma);	2191	return PTR_ERR(vma);
2330		2192
2331	op->map.vma = vma;	2193	op->map.vma = vma;
		2194	if (op->map.immediate \|\| !xe_vm_in_fault_mode(vm))
		2195	xe_vma_ops_incr_pt_update_ops(vops,
		2196	op->tile_mask);
2332	break;	2197	break;
2333	}	2198	}
2334	case DRM_GPUVA_OP_REMAP:	2199	case DRM_GPUVA_OP_REMAP:
@@ -2373,6 +2238,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct xe_exec_queue q, · 38 unmodified lines
2373	vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx",	2238	vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx",
2374	(ULL)op->remap.start,	2239	(ULL)op->remap.start,
2375	(ULL)op->remap.range);	2240	(ULL)op->remap.range);
		2241	} else {
		2242	xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
2376	}	2243	}
2377	}	2244	}
2378		2245
@@ -2409,203 +2276,30 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct xe_exec_queue q, · 30 unmodified lines
2409	vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx",	2276	vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx",
2410	(ULL)op->remap.start,	2277	(ULL)op->remap.start,
2411	(ULL)op->remap.range);	2278	(ULL)op->remap.range);
		2279	} else {
		2280	xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
2412	}	2281	}
2413	}	2282	}
		2283	xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
2414	break;	2284	break;
2415	}	2285	}
2416	case DRM_GPUVA_OP_UNMAP:	2286	case DRM_GPUVA_OP_UNMAP:
2417	case DRM_GPUVA_OP_PREFETCH:	2287	case DRM_GPUVA_OP_PREFETCH:
2418	/* Nothing to do */	2288	/* FIXME: Need to skip some prefetch ops */
		2289	xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask);
2419	break;	2290	break;
2420	default:	2291	default:
2421	drm_warn(&vm->xe->drm, "NOT POSSIBLE");	2292	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2422	}	2293	}
2423		2294
2424	last_op = op;
2425
2426	err = xe_vma_op_commit(vm, op);	2295	err = xe_vma_op_commit(vm, op);
2427	if (err)	2296	if (err)
2428	return err;	2297	return err;
2429	}	2298	}
2430		2299
2431	/* FIXME: Unhandled corner case */
2432	XE_WARN_ON(!last_op && last && !list_empty(&vops->list));
2433
2434	if (!last_op)
2435	return 0;
2436
2437	if (last) {
2438	last_op->flags \|= XE_VMA_OP_LAST;
2439	last_op->num_syncs = num_syncs;
2440	last_op->syncs = syncs;
2441	}
2442
2443	return 0;	2300	return 0;
2444	}	2301	}
2445		2302
2446	static struct dma_fence op_execute(struct xe_vm vm, struct xe_vma *vma,
2447	struct xe_vma_op *op)
2448	{
2449	struct dma_fence *fence = NULL;
2450
2451	lockdep_assert_held(&vm->lock);
2452
2453	xe_vm_assert_held(vm);
2454	xe_bo_assert_held(xe_vma_bo(vma));
2455
2456	switch (op->base.op) {
2457	case DRM_GPUVA_OP_MAP:
2458	fence = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
2459	op->syncs, op->num_syncs,
2460	op->tile_mask,
2461	op->map.immediate \|\| !xe_vm_in_fault_mode(vm),
2462	op->flags & XE_VMA_OP_FIRST,
2463	op->flags & XE_VMA_OP_LAST);
2464	break;
2465	case DRM_GPUVA_OP_REMAP:
2466	{
2467	bool prev = !!op->remap.prev;
2468	bool next = !!op->remap.next;
2469
2470	if (!op->remap.unmap_done) {
2471	if (prev \|\| next)
2472	vma->gpuva.flags \|= XE_VMA_FIRST_REBIND;
2473	fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
2474	op->num_syncs,
2475	op->flags & XE_VMA_OP_FIRST,
2476	op->flags & XE_VMA_OP_LAST &&
2477	!prev && !next);
2478	if (IS_ERR(fence))
2479	break;
2480	op->remap.unmap_done = true;
2481	}
2482
2483	if (prev) {
2484	op->remap.prev->gpuva.flags \|= XE_VMA_LAST_REBIND;
2485	dma_fence_put(fence);
2486	fence = xe_vm_bind(vm, op->remap.prev, op->q,
2487	xe_vma_bo(op->remap.prev), op->syncs,
2488	op->num_syncs,
2489	op->remap.prev->tile_mask, true,
2490	false,
2491	op->flags & XE_VMA_OP_LAST && !next);
2492	op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND;
2493	if (IS_ERR(fence))
2494	break;
2495	op->remap.prev = NULL;
2496	}
2497
2498	if (next) {
2499	op->remap.next->gpuva.flags \|= XE_VMA_LAST_REBIND;
2500	dma_fence_put(fence);
2501	fence = xe_vm_bind(vm, op->remap.next, op->q,
2502	xe_vma_bo(op->remap.next),
2503	op->syncs, op->num_syncs,
2504	op->remap.next->tile_mask, true,
2505	false, op->flags & XE_VMA_OP_LAST);
2506	op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND;
2507	if (IS_ERR(fence))
2508	break;
2509	op->remap.next = NULL;
2510	}
2511
2512	break;
2513	}
2514	case DRM_GPUVA_OP_UNMAP:
2515	fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
2516	op->num_syncs, op->flags & XE_VMA_OP_FIRST,
2517	op->flags & XE_VMA_OP_LAST);
2518	break;
2519	case DRM_GPUVA_OP_PREFETCH:
2520	fence = xe_vm_prefetch(vm, vma, op->q, op->syncs, op->num_syncs,
2521	op->flags & XE_VMA_OP_FIRST,
2522	op->flags & XE_VMA_OP_LAST);
2523	break;
2524	default:
2525	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2526	}
2527
2528	if (IS_ERR(fence))
2529	trace_xe_vma_fail(vma);
2530
2531	return fence;
2532	}
2533
2534	static struct dma_fence *
2535	__xe_vma_op_execute(struct xe_vm vm, struct xe_vma vma,
2536	struct xe_vma_op *op)
2537	{
2538	struct dma_fence *fence;
2539	int err;
2540
2541	retry_userptr:
2542	fence = op_execute(vm, vma, op);
2543	if (IS_ERR(fence) && PTR_ERR(fence) == -EAGAIN) {
2544	lockdep_assert_held_write(&vm->lock);
2545
2546	if (op->base.op == DRM_GPUVA_OP_REMAP) {
2547	if (!op->remap.unmap_done)
2548	vma = gpuva_to_vma(op->base.remap.unmap->va);
2549	else if (op->remap.prev)
2550	vma = op->remap.prev;
2551	else
2552	vma = op->remap.next;
2553	}
2554
2555	if (xe_vma_is_userptr(vma)) {
2556	err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
2557	if (!err)
2558	goto retry_userptr;
2559
2560	fence = ERR_PTR(err);
2561	trace_xe_vma_fail(vma);
2562	}
2563	}
2564
2565	return fence;
2566	}
2567
2568	static struct dma_fence *
2569	xe_vma_op_execute(struct xe_vm vm, struct xe_vma_op op)
2570	{
2571	struct dma_fence *fence = ERR_PTR(-ENOMEM);
2572
2573	lockdep_assert_held(&vm->lock);
2574
2575	switch (op->base.op) {
2576	case DRM_GPUVA_OP_MAP:
2577	fence = __xe_vma_op_execute(vm, op->map.vma, op);
2578	break;
2579	case DRM_GPUVA_OP_REMAP:
2580	{
2581	struct xe_vma *vma;
2582
2583	if (!op->remap.unmap_done)
2584	vma = gpuva_to_vma(op->base.remap.unmap->va);
2585	else if (op->remap.prev)
2586	vma = op->remap.prev;
2587	else
2588	vma = op->remap.next;
2589
2590	fence = __xe_vma_op_execute(vm, vma, op);
2591	break;
2592	}
2593	case DRM_GPUVA_OP_UNMAP:
2594	fence = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
2595	op);
2596	break;
2597	case DRM_GPUVA_OP_PREFETCH:
2598	fence = __xe_vma_op_execute(vm,
2599	gpuva_to_vma(op->base.prefetch.va),
2600	op);
2601	break;
2602	default:
2603	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2604	}
2605
2606	return fence;
2607	}
2608
2609	static void xe_vma_op_unwind(struct xe_vm vm, struct xe_vma_op op,	2303	static void xe_vma_op_unwind(struct xe_vm vm, struct xe_vma_op op,
2610	bool post_commit, bool prev_post_commit,	2304	bool post_commit, bool prev_post_commit,
2611	bool next_post_commit)	2305	bool next_post_commit)
@@ -2788,26 +2482,157 @@ static int vm_bind_ioctl_ops_lock_and_prep(struct drm_exec *exec, · 176 unmodified lines
2788	return err;	2482	return err;
2789	}	2483	}
2790		2484
		2485	#ifdef TEST_VM_OPS_ERROR
		2486	if (vops->inject_error &&
		2487	vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK)
		2488	return -ENOSPC;
		2489	#endif
		2490
2791	return 0;	2491	return 0;
2792	}	2492	}
2793		2493
		2494	static void op_trace(struct xe_vma_op *op)
		2495	{
		2496	switch (op->base.op) {
		2497	case DRM_GPUVA_OP_MAP:
		2498	trace_xe_vma_bind(op->map.vma);
		2499	break;
		2500	case DRM_GPUVA_OP_REMAP:
		2501	trace_xe_vma_unbind(gpuva_to_vma(op->base.remap.unmap->va));
		2502	if (op->remap.prev)
		2503	trace_xe_vma_bind(op->remap.prev);
		2504	if (op->remap.next)
		2505	trace_xe_vma_bind(op->remap.next);
		2506	break;
		2507	case DRM_GPUVA_OP_UNMAP:
		2508	trace_xe_vma_unbind(gpuva_to_vma(op->base.unmap.va));
		2509	break;
		2510	case DRM_GPUVA_OP_PREFETCH:
		2511	trace_xe_vma_bind(gpuva_to_vma(op->base.prefetch.va));
		2512	break;
		2513	default:
		2514	XE_WARN_ON("NOT POSSIBLE");
		2515	}
		2516	}
		2517
		2518	static void trace_xe_vm_ops_execute(struct xe_vma_ops *vops)
		2519	{
		2520	struct xe_vma_op *op;
		2521
		2522	list_for_each_entry(op, &vops->list, link)
		2523	op_trace(op);
		2524	}
		2525
		2526	static int vm_ops_setup_tile_args(struct xe_vm vm, struct xe_vma_ops vops)
		2527	{
		2528	struct xe_exec_queue *q = vops->q;
		2529	struct xe_tile *tile;
		2530	int number_tiles = 0;
		2531	u8 id;
		2532
		2533	for_each_tile(tile, vm->xe, id) {
		2534	if (vops->pt_update_ops[id].num_ops)
		2535	++number_tiles;
		2536
		2537	if (vops->pt_update_ops[id].q)
		2538	continue;
		2539
		2540	if (q) {
		2541	vops->pt_update_ops[id].q = q;
		2542	if (vm->pt_root[id] && !list_empty(&q->multi_gt_list))
		2543	q = list_next_entry(q, multi_gt_list);
		2544	} else {
		2545	vops->pt_update_ops[id].q = vm->q[id];
		2546	}
		2547	}
		2548
		2549	return number_tiles;
		2550	}
		2551
2794	static struct dma_fence ops_execute(struct xe_vm vm,	2552	static struct dma_fence ops_execute(struct xe_vm vm,
2795	struct xe_vma_ops *vops)	2553	struct xe_vma_ops *vops)
2796	{	2554	{
2797	struct xe_vma_op op, next;	2555	struct xe_tile *tile;
2798	struct dma_fence *fence = NULL;	2556	struct dma_fence *fence = NULL;
		2557	struct dma_fence **fences = NULL;
		2558	struct dma_fence_array *cf = NULL;
		2559	int number_tiles = 0, current_fence = 0, err;
		2560	u8 id;
2799		2561
2800	list_for_each_entry_safe(op, next, &vops->list, link) {	2562	number_tiles = vm_ops_setup_tile_args(vm, vops);
2801	dma_fence_put(fence);	2563	if (number_tiles == 0)
2802	fence = xe_vma_op_execute(vm, op);	2564	return ERR_PTR(-ENODATA);
2803	if (IS_ERR(fence)) {	2565
2804	drm_warn(&vm->xe->drm, "VM op(%d) failed with %ld",	2566	if (number_tiles > 1) {
2805	op->base.op, PTR_ERR(fence));	2567	fences = kmalloc_array(number_tiles, sizeof(*fences),
2806	fence = ERR_PTR(-ENOSPC);	2568	GFP_KERNEL);
2807	break;	2569	if (!fences) {
		2570	fence = ERR_PTR(-ENOMEM);
		2571	goto err_trace;
		2572	}
		2573	}
		2574
		2575	for_each_tile(tile, vm->xe, id) {
		2576	if (!vops->pt_update_ops[id].num_ops)
		2577	continue;
		2578
		2579	err = xe_pt_update_ops_prepare(tile, vops);
		2580	if (err) {
		2581	fence = ERR_PTR(err);
		2582	goto err_out;
2808	}	2583	}
2809	}	2584	}
2810		2585
		2586	trace_xe_vm_ops_execute(vops);
		2587
		2588	for_each_tile(tile, vm->xe, id) {
		2589	if (!vops->pt_update_ops[id].num_ops)
		2590	continue;
		2591
		2592	fence = xe_pt_update_ops_run(tile, vops);
		2593	if (IS_ERR(fence))
		2594	goto err_out;
		2595
		2596	if (fences)
		2597	fences[current_fence++] = fence;
		2598	}
		2599
		2600	if (fences) {
		2601	cf = dma_fence_array_create(number_tiles, fences,
		2602	vm->composite_fence_ctx,
		2603	vm->composite_fence_seqno++,
		2604	false);
		2605	if (!cf) {
		2606	--vm->composite_fence_seqno;
		2607	fence = ERR_PTR(-ENOMEM);
		2608	goto err_out;
		2609	}
		2610	fence = &cf->base;
		2611	}
		2612
		2613	for_each_tile(tile, vm->xe, id) {
		2614	if (!vops->pt_update_ops[id].num_ops)
		2615	continue;
		2616
		2617	xe_pt_update_ops_fini(tile, vops);
		2618	}
		2619
		2620	return fence;
		2621
		2622	err_out:
		2623	for_each_tile(tile, vm->xe, id) {
		2624	if (!vops->pt_update_ops[id].num_ops)
		2625	continue;
		2626
		2627	xe_pt_update_ops_abort(tile, vops);
		2628	}
		2629	while (current_fence)
		2630	dma_fence_put(fences[--current_fence]);
		2631	kfree(fences);
		2632	kfree(cf);
		2633
		2634	err_trace:
		2635	trace_xe_vm_ops_fail(vm);
2811	return fence;	2636	return fence;
2812	}	2637	}
2813		2638
@@ -2888,12 +2713,10 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, · 74 unmodified lines
2888	fence = ops_execute(vm, vops);	2713	fence = ops_execute(vm, vops);
2889	if (IS_ERR(fence)) {	2714	if (IS_ERR(fence)) {
2890	err = PTR_ERR(fence);	2715	err = PTR_ERR(fence);
2891	/* FIXME: Killing VM rather than proper error handling */
2892	xe_vm_kill(vm, false);
2893	goto unlock;	2716	goto unlock;
2894	} else {
2895	vm_bind_ioctl_ops_fini(vm, vops, fence);
2896	}	2717	}
		2718
		2719	vm_bind_ioctl_ops_fini(vm, vops, fence);
2897	}	2720	}
2898		2721
2899	unlock:	2722	unlock:
@@ -2901,11 +2724,18 @@ unlock: · 1 unmodified line
2901	return err;	2724	return err;
2902	}	2725	}
2903		2726
2904	#define SUPPORTED_FLAGS \	2727	#define SUPPORTED_FLAGS_STUB \
2905	(DRM_XE_VM_BIND_FLAG_READONLY \| \	2728	(DRM_XE_VM_BIND_FLAG_READONLY \| \
2906	DRM_XE_VM_BIND_FLAG_IMMEDIATE \| \	2729	DRM_XE_VM_BIND_FLAG_IMMEDIATE \| \
2907	DRM_XE_VM_BIND_FLAG_NULL \| \	2730	DRM_XE_VM_BIND_FLAG_NULL \| \
2908	DRM_XE_VM_BIND_FLAG_DUMPABLE)	2731	DRM_XE_VM_BIND_FLAG_DUMPABLE)
		2732
		2733	#ifdef TEST_VM_OPS_ERROR
		2734	#define SUPPORTED_FLAGS (SUPPORTED_FLAGS_STUB \| FORCE_OP_ERROR)
		2735	#else
		2736	#define SUPPORTED_FLAGS SUPPORTED_FLAGS_STUB
		2737	#endif
		2738
2909	#define XE_64K_PAGE_MASK 0xffffull	2739	#define XE_64K_PAGE_MASK 0xffffull
2910	#define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP)	2740	#define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP)
2911		2741
@@ -2931,7 +2761,7 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, · 19 unmodified lines
2931	sizeof(struct drm_xe_vm_bind_op),	2761	sizeof(struct drm_xe_vm_bind_op),
2932	GFP_KERNEL \| __GFP_ACCOUNT);	2762	GFP_KERNEL \| __GFP_ACCOUNT);
2933	if (!*bind_ops)	2763	if (!*bind_ops)
2934	return -ENOMEM;	2764	return args->num_binds > 1 ? -ENOBUFS : -ENOMEM;
2935		2765
2936	err = __copy_from_user(*bind_ops, bind_user,	2766	err = __copy_from_user(*bind_ops, bind_user,
2937	sizeof(struct drm_xe_vm_bind_op) *	2767	sizeof(struct drm_xe_vm_bind_op) *
@@ -3250,10 +3080,18 @@ int xe_vm_bind_ioctl(struct drm_device dev, void data, struct drm_file *file) · 312 unmodified lines
3250	goto unwind_ops;	3080	goto unwind_ops;
3251	}	3081	}
3252		3082
3253	err = vm_bind_ioctl_ops_parse(vm, q, ops[i], syncs, num_syncs,	3083	err = vm_bind_ioctl_ops_parse(vm, ops[i], &vops);
3254	&vops, i == args->num_binds - 1);
3255	if (err)	3084	if (err)
3256	goto unwind_ops;	3085	goto unwind_ops;
		3086
		3087	#ifdef TEST_VM_OPS_ERROR
		3088	if (flags & FORCE_OP_ERROR) {
		3089	vops.inject_error = true;
		3090	vm->xe->vm_inject_error_position =
		3091	(vm->xe->vm_inject_error_position + 1) %
		3092	FORCE_OP_ERROR_COUNT;
		3093	}
		3094	#endif
3257	}	3095	}
3258		3096
3259	/* Nothing to do */	3097	/* Nothing to do */
@@ -3262,11 +3100,16 @@ int xe_vm_bind_ioctl(struct drm_device dev, void data, struct drm_file *file) · 2 unmodified lines
3262	goto unwind_ops;	3100	goto unwind_ops;
3263	}	3101	}
3264		3102
		3103	err = xe_vma_ops_alloc(&vops, args->num_binds > 1);
		3104	if (err)
		3105	goto unwind_ops;
		3106
3265	err = vm_bind_ioctl_ops_execute(vm, &vops);	3107	err = vm_bind_ioctl_ops_execute(vm, &vops);
3266		3108
3267	unwind_ops:	3109	unwind_ops:
3268	if (err && err != -ENODATA)	3110	if (err && err != -ENODATA)
3269	vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);	3111	vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
		3112	xe_vma_ops_fini(&vops);
3270	for (i = args->num_binds - 1; i >= 0; --i)	3113	for (i = args->num_binds - 1; i >= 0; --i)
3271	if (ops[i])	3114	if (ops[i])
3272	drm_gpuva_ops_free(&vm->gpuvm, ops[i]);	3115	drm_gpuva_ops_free(&vm->gpuvm, ops[i]);
@@ -3337,10 +3180,10 @@ int xe_vm_invalidate_vma(struct xe_vma *vma) · 64 unmodified lines
3337	{	3180	{
3338	struct xe_device *xe = xe_vma_vm(vma)->xe;	3181	struct xe_device *xe = xe_vma_vm(vma)->xe;
3339	struct xe_tile *tile;	3182	struct xe_tile *tile;
		3183	struct xe_gt_tlb_invalidation_fence fence[XE_MAX_TILES_PER_DEVICE];
3340	u32 tile_needs_invalidate = 0;	3184	u32 tile_needs_invalidate = 0;
3341	int seqno[XE_MAX_TILES_PER_DEVICE];
3342	u8 id;	3185	u8 id;
3343	int ret;	3186	int ret = 0;
3344		3187
3345	xe_assert(xe, !xe_vma_is_null(vma));	3188	xe_assert(xe, !xe_vma_is_null(vma));
3346	trace_xe_vma_invalidate(vma);	3189	trace_xe_vma_invalidate(vma);
@@ -3365,29 +3208,33 @@ int xe_vm_invalidate_vma(struct xe_vma *vma) · 18 unmodified lines
3365		3208
3366	for_each_tile(tile, xe, id) {	3209	for_each_tile(tile, xe, id) {
3367	if (xe_pt_zap_ptes(tile, vma)) {	3210	if (xe_pt_zap_ptes(tile, vma)) {
3368	tile_needs_invalidate \|= BIT(id);
3369	xe_device_wmb(xe);	3211	xe_device_wmb(xe);
		3212	xe_gt_tlb_invalidation_fence_init(tile->primary_gt,
		3213	&fence[id], true);
		3214
3370	/*	3215	/*
3371	* FIXME: We potentially need to invalidate multiple	3216	* FIXME: We potentially need to invalidate multiple
3372	* GTs within the tile	3217	* GTs within the tile
3373	*/	3218	*/
3374	seqno[id] = xe_gt_tlb_invalidation_vma(tile->primary_gt, NULL, vma);	3219	ret = xe_gt_tlb_invalidation_vma(tile->primary_gt,
3375	if (seqno[id] < 0)	3220	&fence[id], vma);
3376	return seqno[id];	3221	if (ret < 0) {
3377	}	3222	xe_gt_tlb_invalidation_fence_fini(&fence[id]);
3378	}	3223	goto wait;
		3224	}
3379		3225
3380	for_each_tile(tile, xe, id) {	3226	tile_needs_invalidate \|= BIT(id);
3381	if (tile_needs_invalidate & BIT(id)) {
3382	ret = xe_gt_tlb_invalidation_wait(tile->primary_gt, seqno[id]);
3383	if (ret < 0)
3384	return ret;
3385	}	3227	}
3386	}	3228	}
3387		3229
		3230	wait:
		3231	for_each_tile(tile, xe, id)
		3232	if (tile_needs_invalidate & BIT(id))
		3233	xe_gt_tlb_invalidation_fence_wait(&fence[id]);
		3234
3388	vma->tile_invalidated = vma->tile_mask;	3235	vma->tile_invalidated = vma->tile_mask;
3389		3236
3390	return 0;	3237	return ret;
3391	}	3238	}
3392		3239
3393	struct xe_vm_snapshot {	3240	struct xe_vm_snapshot {

drivers/gpu/drm/xe/xe_vm.h+2−0

@@ -259,6 +259,8 @@ static inline struct dma_resv xe_vm_resv(struct xe_vm vm) · 258 unmodified lines
259	return drm_gpuvm_resv(&vm->gpuvm);	259	return drm_gpuvm_resv(&vm->gpuvm);
260	}	260	}
261		261
		262	void xe_vm_kill(struct xe_vm *vm, bool unlocked);
		263
262	/**	264	/**
263	* xe_vm_assert_held(vm) - Assert that the vm's reservation object is held.	265	* xe_vm_assert_held(vm) - Assert that the vm's reservation object is held.
264	* @vm: The vm	266	* @vm: The vm

drivers/gpu/drm/xe/xe_vm_types.h+30−25

 struct xe_sync_entry;
 struct xe_user_fence;
 struct xe_vm;
+struct xe_vm_pgtable_update_op;
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
+#define TEST_VM_OPS_ERROR
+#define FORCE_OP_ERROR	BIT(31)
+#define FORCE_OP_ERROR_LOCK	0
+#define FORCE_OP_ERROR_PREPARE	1
+#define FORCE_OP_ERROR_RUN	2
+#define FORCE_OP_ERROR_COUNT	3
+#endif
 #define XE_VMA_READ_ONLY	DRM_GPUVA_USERBITS
 #define XE_VMA_DESTROYED	(DRM_GPUVA_USERBITS << 1)
 #define XE_VMA_ATOMIC_PTE_BIT	(DRM_GPUVA_USERBITS << 2)
-#define XE_VMA_FIRST_REBIND	(DRM_GPUVA_USERBITS << 3)
+#define XE_VMA_PTE_4K		(DRM_GPUVA_USERBITS << 3)
-#define XE_VMA_LAST_REBIND	(DRM_GPUVA_USERBITS << 4)
+#define XE_VMA_PTE_2M		(DRM_GPUVA_USERBITS << 4)
-#define XE_VMA_PTE_4K		(DRM_GPUVA_USERBITS << 5)
+#define XE_VMA_PTE_1G		(DRM_GPUVA_USERBITS << 5)
-#define XE_VMA_PTE_2M		(DRM_GPUVA_USERBITS << 6)
+#define XE_VMA_PTE_64K		(DRM_GPUVA_USERBITS << 6)
-#define XE_VMA_PTE_1G		(DRM_GPUVA_USERBITS << 7)
+#define XE_VMA_PTE_COMPACT	(DRM_GPUVA_USERBITS << 7)
-#define XE_VMA_PTE_64K		(DRM_GPUVA_USERBITS << 8)
+#define XE_VMA_DUMPABLE		(DRM_GPUVA_USERBITS << 8)
-#define XE_VMA_PTE_COMPACT	(DRM_GPUVA_USERBITS << 9)
-#define XE_VMA_DUMPABLE		(DRM_GPUVA_USERBITS << 10)
 /** struct xe_userptr - User pointer */
 struct xe_userptr {
 	 */
 	u8 tile_present;
+	/** @tile_staged: bind is staged for this VMA */
+	u8 tile_staged;
 	/**
 	 * @pat_index: The pat index to use when encoding the PTEs for this vma.
 	 */
 /** enum xe_vma_op_flags - flags for VMA operation */
 enum xe_vma_op_flags {
-	/** @XE_VMA_OP_FIRST: first VMA operation for a set of syncs */
-	XE_VMA_OP_FIRST			= BIT(0),
-	/** @XE_VMA_OP_LAST: last VMA operation for a set of syncs */
-	XE_VMA_OP_LAST			= BIT(1),
 	/** @XE_VMA_OP_COMMITTED: VMA operation committed */
-	XE_VMA_OP_COMMITTED		= BIT(2),
+	XE_VMA_OP_COMMITTED		= BIT(0),
 	/** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */
-	XE_VMA_OP_PREV_COMMITTED	= BIT(3),
+	XE_VMA_OP_PREV_COMMITTED	= BIT(1),
 	/** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */
-	XE_VMA_OP_NEXT_COMMITTED	= BIT(4),
+	XE_VMA_OP_NEXT_COMMITTED	= BIT(2),
 };
 /** struct xe_vma_op - VMA operation */
 struct xe_vma_op {
 	/** @base: GPUVA base operation */
 	struct drm_gpuva_op base;
-	/** @q: exec queue for this operation */
-	struct xe_exec_queue *q;
-	/**
-	 * @syncs: syncs for this operation, only used on first and last
-	 * operation
-	 */
-	struct xe_sync_entry *syncs;
-	/** @num_syncs: number of syncs */
-	u32 num_syncs;
 	/** @link: async operation link */
 	struct list_head link;
 	/** @flags: operation flags */
 	struct list_head list;
 	/** @vm: VM */
 	struct xe_vm *vm;
-	/** @q: exec queue these operations */
+	/** @q: exec queue for VMA operations */
 	struct xe_exec_queue *q;
 	/** @syncs: syncs these operation */
 	struct xe_sync_entry *syncs;
 	/** @num_syncs: number of syncs */
 	u32 num_syncs;
+	/** @pt_update_ops: page table update operations */
+	struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
+#ifdef TEST_VM_OPS_ERROR
+	/** @inject_error: inject error to test error handling */
+	bool inject_error;
+#endif
 };
 #endif

drivers/gpu/drm/xe/xe_wa.c+15−0

 	  XE_RTP_RULES(GRAPHICS_VERSION(2004), FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN, SLM_WMTP_RESTORE))
 	},
+	{ XE_RTP_NAME("14021402888"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
+	},
 	/* Xe2_HPG */
 	  XE_RTP_ACTIONS(SET(HALF_SLICE_CHICKEN7, CLEAR_OPTIMIZATION_DISABLE))
 	},
+	/* Xe2_LPM */
+	{ XE_RTP_NAME("16021639441"),
+	  XE_RTP_RULES(MEDIA_VERSION(2000)),
+	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0),
+			     GHWSP_CSB_REPORT_DIS |
+			     PPHWSP_CSB_AND_TIMESTAMP_REPORT_DIS,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
 	/* Xe2_HPM */
 	{ XE_RTP_NAME("16021639441"),
 	xe_rtp_process_ctx_enable_active_tracking(&ctx, gt->wa_active.oob,
 						  ARRAY_SIZE(oob_was));
+	gt->wa_active.oob_initialized = true;
 	xe_rtp_process(&ctx, oob_was);
 }

drivers/gpu/drm/xe/xe_wa.h+6−1

 #ifndef _XE_WA_
 #define _XE_WA_
+#include "xe_assert.h"
 struct drm_printer;
 struct xe_gt;
 struct xe_hw_engine;
  * @gt__: gt instance
  * @id__: XE_OOB_<id__>, as generated by build system in generated/xe_wa_oob.h
  */
-#define XE_WA(gt__, id__) test_bit(XE_WA_OOB_ ## id__, (gt__)->wa_active.oob)
+#define XE_WA(gt__, id__) ({						\
+	xe_gt_assert(gt__, (gt__)->wa_active.oob_initialized);		\
+	test_bit(XE_WA_OOB_ ## id__, (gt__)->wa_active.oob);		\
+})
 #endif

drivers/gpu/drm/xe/xe_wa_oob.rules+2−0

@@ -29,3 +29,5 @@ · 28 unmodified lines
29	13011645652 GRAPHICS_VERSION(2004)	29	13011645652 GRAPHICS_VERSION(2004)
30	22019338487 MEDIA_VERSION(2000)	30	22019338487 MEDIA_VERSION(2000)
31	GRAPHICS_VERSION(2001)	31	GRAPHICS_VERSION(2001)
		32	22019338487_display PLATFORM(LUNARLAKE)
		33	16023588340 GRAPHICS_VERSION(2001)

include/uapi/drm/xe_drm.h+13−5

  *    available per Dual Sub Slices (DSS). For example a query response
  *    containing the following in mask:
  *    ``EU_PER_DSS    ff ff 00 00 00 00 00 00``
- *    means each DSS has 16 EU.
+ *    means each DSS has 16 SIMD8 EUs. This type may be omitted if device
+ *    doesn't have SIMD8 EUs.
+ *  - %DRM_XE_TOPO_SIMD16_EU_PER_DSS - To query the mask of SIMD16 Execution
+ *    Units (EU) available per Dual Sub Slices (DSS). For example a query
+ *    response containing the following in mask:
+ *    ``SIMD16_EU_PER_DSS    ff ff 00 00 00 00 00 00``
+ *    means each DSS has 16 SIMD16 EUs. This type may be omitted if device
+ *    doesn't have SIMD16 EUs.
  */
 struct drm_xe_query_topology_mask {
 	/** @gt_id: GT ID the mask is associated with */
 #define DRM_XE_TOPO_DSS_COMPUTE		2
 #define DRM_XE_TOPO_L3_BANK		3
 #define DRM_XE_TOPO_EU_PER_DSS		4
+#define DRM_XE_TOPO_SIMD16_EU_PER_DSS	5
 	/** @type: type of mask */
 	__u16 type;
 	 * b. Counter select c. Counter size and d. BC report. Also refer to the
 	 * oa_formats array in drivers/gpu/drm/xe/xe_oa.c.
 	 */
-#define DRM_XE_OA_FORMAT_MASK_FMT_TYPE		(0xff << 0)
+#define DRM_XE_OA_FORMAT_MASK_FMT_TYPE		(0xffu << 0)
-#define DRM_XE_OA_FORMAT_MASK_COUNTER_SEL	(0xff << 8)
+#define DRM_XE_OA_FORMAT_MASK_COUNTER_SEL	(0xffu << 8)
-#define DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE	(0xff << 16)
+#define DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE	(0xffu << 16)
-#define DRM_XE_OA_FORMAT_MASK_BC_REPORT		(0xff << 24)
+#define DRM_XE_OA_FORMAT_MASK_BC_REPORT		(0xffu << 24)
 	/**
 	 * @DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT: Requests periodic OA unit

Merge tag 'drm-xe-next-2024-07-30' of https://gitlab.freedesktop.org/drm/xe/kernel into drm-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code