lael-labs/linux

drivers/gpu/drm/drm_gpusvm.c+5−2

 		order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
 		if (is_device_private_page(page) ||
 		    is_device_coherent_page(page)) {
+			struct drm_pagemap_zdd *__zdd =
+				drm_pagemap_page_zone_device_data(page);
 			if (!ctx->allow_mixed &&
-			    zdd != page->zone_device_data && i > 0) {
+			    zdd != __zdd && i > 0) {
 				err = -EOPNOTSUPP;
 				goto err_unmap;
 			}
-			zdd = page->zone_device_data;
+			zdd = __zdd;
 			if (pagemap != page_pgmap(page)) {
 				if (pagemap) {
 					err = -EOPNOTSUPP;

drivers/gpu/drm/drm_pagemap.c+128−29

@@ -154,15 +154,15 @@ static void drm_pagemap_zdd_put(struct drm_pagemap_zdd *zdd) · 153 unmodified lines
154	}	154	}
155		155
156	/**	156	/**
157	* drm_pagemap_migration_unlock_put_page() - Put a migration page	157	* drm_pagemap_migration_unlock_put_folio() - Put a migration folio
158	* @page: Pointer to the page to put	158	* @folio: Pointer to the folio to put
159	*	159	*
160	* This function unlocks and puts a page.	160	* This function unlocks and puts a folio.
161	*/	161	*/
162	static void drm_pagemap_migration_unlock_put_page(struct page *page)	162	static void drm_pagemap_migration_unlock_put_folio(struct folio *folio)
163	{	163	{
164	unlock_page(page);	164	folio_unlock(folio);
165	put_page(page);	165	folio_put(folio);
166	}	166	}
167		167
168	/**	168	/**
@@ -177,31 +177,42 @@ static void drm_pagemap_migration_unlock_put_pages(unsigned long npages, · 8 unmodified lines
177	{	177	{
178	unsigned long i;	178	unsigned long i;
179		179
180	for (i = 0; i < npages; ++i) {	180	for (i = 0; i < npages;) {
181	struct page *page;	181	struct page *page;
		182	struct folio *folio;
		183	unsigned int order = 0;
182		184
183	if (!migrate_pfn[i])	185	if (!migrate_pfn[i])
184	continue;	186	goto next;
185		187
186	page = migrate_pfn_to_page(migrate_pfn[i]);	188	page = migrate_pfn_to_page(migrate_pfn[i]);
187	drm_pagemap_migration_unlock_put_page(page);	189	folio = page_folio(page);
		190	order = folio_order(folio);
		191
		192	drm_pagemap_migration_unlock_put_folio(folio);
188	migrate_pfn[i] = 0;	193	migrate_pfn[i] = 0;
		194
		195	next:
		196	i += NR_PAGES(order);
189	}	197	}
190	}	198	}
191		199
192	/**	200	/**
193	* drm_pagemap_get_devmem_page() - Get a reference to a device memory page	201	* drm_pagemap_get_devmem_page() - Get a reference to a device memory page
194	* @page: Pointer to the page	202	* @page: Pointer to the page
		203	* @order: Order
195	* @zdd: Pointer to the GPU SVM zone device data	204	* @zdd: Pointer to the GPU SVM zone device data
196	*	205	*
197	* This function associates the given page with the specified GPU SVM zone	206	* This function associates the given page with the specified GPU SVM zone
198	* device data and initializes it for zone device usage.	207	* device data and initializes it for zone device usage.
199	*/	208	*/
200	static void drm_pagemap_get_devmem_page(struct page *page,	209	static void drm_pagemap_get_devmem_page(struct page *page,
		210	unsigned int order,
201	struct drm_pagemap_zdd *zdd)	211	struct drm_pagemap_zdd *zdd)
202	{	212	{
203	page->zone_device_data = drm_pagemap_zdd_get(zdd);	213	zone_device_folio_init((struct folio *)page, zdd->dpagemap->pagemap,
204	zone_device_page_init(page, page_pgmap(page), 0);	214	order);
		215	folio_set_zone_device_data(page_folio(page), drm_pagemap_zdd_get(zdd));
205	}	216	}
206		217
207	/**	218	/**
@@ -244,7 +255,7 @@ static int drm_pagemap_migrate_map_pages(struct device *dev, · 36 unmodified lines
244	order = folio_order(folio);	255	order = folio_order(folio);
245		256
246	if (is_device_private_page(page)) {	257	if (is_device_private_page(page)) {
247	struct drm_pagemap_zdd *zdd = page->zone_device_data;	258	struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page);
248	struct drm_pagemap *dpagemap = zdd->dpagemap;	259	struct drm_pagemap *dpagemap = zdd->dpagemap;
249	struct drm_pagemap_addr addr;	260	struct drm_pagemap_addr addr;
250		261
@@ -315,7 +326,7 @@ static void drm_pagemap_migrate_unmap_pages(struct device *dev, · 64 unmodified lines
315	goto next;	326	goto next;
316		327
317	if (is_zone_device_page(page)) {	328	if (is_zone_device_page(page)) {
318	struct drm_pagemap_zdd *zdd = page->zone_device_data;	329	struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page);
319	struct drm_pagemap *dpagemap = zdd->dpagemap;	330	struct drm_pagemap *dpagemap = zdd->dpagemap;
320		331
321	dpagemap->ops->device_unmap(dpagemap, dev, &pagemap_addr[i]);	332	dpagemap->ops->device_unmap(dpagemap, dev, &pagemap_addr[i]);
@@ -444,6 +455,41 @@ out: · 122 unmodified lines
444	return ret;	455	return ret;
445	}	456	}
446		457
		458	/**
		459	* drm_pagemap_cpages() - Count collected pages
		460	* @migrate_pfn: Array of migrate_pfn entries to account
		461	* @npages: Number of entries in @migrate_pfn
		462	*
		463	* Compute the total number of minimum-sized pages represented by the
		464	* collected entries in @migrate_pfn. The total is derived from the
		465	* order encoded in each entry.
		466	*
		467	* Return: Total number of minimum-sized pages.
		468	*/
		469	static int drm_pagemap_cpages(unsigned long *migrate_pfn, unsigned long npages)
		470	{
		471	unsigned long i, cpages = 0;
		472
		473	for (i = 0; i < npages;) {
		474	struct page *page = migrate_pfn_to_page(migrate_pfn[i]);
		475	struct folio *folio;
		476	unsigned int order = 0;
		477
		478	if (page) {
		479	folio = page_folio(page);
		480	order = folio_order(folio);
		481	cpages += NR_PAGES(order);
		482	} else if (migrate_pfn[i] & MIGRATE_PFN_COMPOUND) {
		483	order = HPAGE_PMD_ORDER;
		484	cpages += NR_PAGES(order);
		485	}
		486
		487	i += NR_PAGES(order);
		488	}
		489
		490	return cpages;
		491	}
		492
447	/**	493	/**
448	* drm_pagemap_migrate_to_devmem() - Migrate a struct mm_struct range to device memory	494	* drm_pagemap_migrate_to_devmem() - Migrate a struct mm_struct range to device memory
449	* @devmem_allocation: The device memory allocation to migrate to.	495	* @devmem_allocation: The device memory allocation to migrate to.
@@ -481,7 +527,7 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, · 31 unmodified lines
481	.end = end,	527	.end = end,
482	.pgmap_owner = pagemap->owner,	528	.pgmap_owner = pagemap->owner,
483	.flags = MIGRATE_VMA_SELECT_SYSTEM \| MIGRATE_VMA_SELECT_DEVICE_COHERENT \|	529	.flags = MIGRATE_VMA_SELECT_SYSTEM \| MIGRATE_VMA_SELECT_DEVICE_COHERENT \|
484	MIGRATE_VMA_SELECT_DEVICE_PRIVATE,	530	MIGRATE_VMA_SELECT_DEVICE_PRIVATE \| MIGRATE_VMA_SELECT_COMPOUND,
485	};	531	};
486	unsigned long i, npages = npages_in_range(start, end);	532	unsigned long i, npages = npages_in_range(start, end);
487	unsigned long own_pages = 0, migrated_pages = 0;	533	unsigned long own_pages = 0, migrated_pages = 0;
@@ -546,7 +592,8 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, · 58 unmodified lines
546	goto err_free;	592	goto err_free;
547	}	593	}
548		594
549	if (migrate.cpages != npages) {	595	if (migrate.cpages != npages &&
		596	drm_pagemap_cpages(migrate.src, npages) != npages) {
550	/*	597	/*
551	* Some pages to migrate. But we want to migrate all or	598	* Some pages to migrate. But we want to migrate all or
552	* nothing. Raced or unknown device pages.	599	* nothing. Raced or unknown device pages.
@@ -586,20 +633,23 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, · 33 unmodified lines
586		633
587	own_pages = 0;	634	own_pages = 0;
588		635
589	for (i = 0; i < npages; ++i) {	636	for (i = 0; i < npages;) {
		637	unsigned long j;
590	struct page *page = pfn_to_page(migrate.dst[i]);	638	struct page *page = pfn_to_page(migrate.dst[i]);
591	struct page *src_page = migrate_pfn_to_page(migrate.src[i]);	639	struct page *src_page = migrate_pfn_to_page(migrate.src[i]);
592	cur.start = i;	640	unsigned int order = 0;
593		641
		642	cur.start = i;
594	pages[i] = NULL;	643	pages[i] = NULL;
595	if (src_page && is_device_private_page(src_page)) {	644	if (src_page && is_device_private_page(src_page)) {
596	struct drm_pagemap_zdd *src_zdd = src_page->zone_device_data;	645	struct drm_pagemap_zdd *src_zdd =
		646	drm_pagemap_page_zone_device_data(src_page);
597		647
598	if (page_pgmap(src_page) == pagemap &&	648	if (page_pgmap(src_page) == pagemap &&
599	!mdetails->can_migrate_same_pagemap) {	649	!mdetails->can_migrate_same_pagemap) {
600	migrate.dst[i] = 0;	650	migrate.dst[i] = 0;
601	own_pages++;	651	own_pages++;
602	continue;	652	goto next;
603	}	653	}
604	if (mdetails->source_peer_migrates) {	654	if (mdetails->source_peer_migrates) {
605	cur.dpagemap = src_zdd->dpagemap;	655	cur.dpagemap = src_zdd->dpagemap;
@@ -615,7 +665,20 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, · 9 unmodified lines
615	pages[i] = page;	665	pages[i] = page;
616	}	666	}
617	migrate.dst[i] = migrate_pfn(migrate.dst[i]);	667	migrate.dst[i] = migrate_pfn(migrate.dst[i]);
618	drm_pagemap_get_devmem_page(page, zdd);	668
		669	if (migrate.src[i] & MIGRATE_PFN_COMPOUND) {
		670	drm_WARN_ONCE(dpagemap->drm, src_page &&
		671	folio_order(page_folio(src_page)) != HPAGE_PMD_ORDER,
		672	"Unexpected folio order\n");
		673
		674	order = HPAGE_PMD_ORDER;
		675	migrate.dst[i] \|= MIGRATE_PFN_COMPOUND;
		676
		677	for (j = 1; j < NR_PAGES(order) && i + j < npages; j++)
		678	migrate.dst[i + j] = 0;
		679	}
		680
		681	drm_pagemap_get_devmem_page(page, order, zdd);
619		682
620	/* If we switched the migrating drm_pagemap, migrate previous pages now */	683	/* If we switched the migrating drm_pagemap, migrate previous pages now */
621	err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst,	684	err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst,
@@ -625,7 +688,11 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, · 3 unmodified lines
625	npages = i + 1;	688	npages = i + 1;
626	goto err_finalize;	689	goto err_finalize;
627	}	690	}
		691
		692	next:
		693	i += NR_PAGES(order);
628	}	694	}
		695
629	cur.start = npages;	696	cur.start = npages;
630	cur.ops = NULL; /* Force migration */	697	cur.ops = NULL; /* Force migration */
631	err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst,	698	err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst,
@@ -715,8 +782,8 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas, · 83 unmodified lines
715	goto next;	782	goto next;
716		783
717	if (fault_page) {	784	if (fault_page) {
718	if (src_page->zone_device_data !=	785	if (drm_pagemap_page_zone_device_data(src_page) !=
719	fault_page->zone_device_data)	786	drm_pagemap_page_zone_device_data(fault_page))
720	goto next;	787	goto next;
721	}	788	}
722		789
@@ -734,6 +801,8 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas, · 11 unmodified lines
734	page = folio_page(folio, 0);	801	page = folio_page(folio, 0);
735	mpfn[i] = migrate_pfn(page_to_pfn(page));	802	mpfn[i] = migrate_pfn(page_to_pfn(page));
736		803
		804	if (order)
		805	mpfn[i] \|= MIGRATE_PFN_COMPOUND;
737	next:	806	next:
738	if (page)	807	if (page)
739	addr += page_size(page);	808	addr += page_size(page);
@@ -989,8 +1058,15 @@ retry: · 249 unmodified lines
989	if (err)	1058	if (err)
990	goto err_finalize;	1059	goto err_finalize;
991		1060
992	for (i = 0; i < npages; ++i)	1061	for (i = 0; i < npages;) {
		1062	unsigned int order = 0;
		1063
993	pages[i] = migrate_pfn_to_page(src[i]);	1064	pages[i] = migrate_pfn_to_page(src[i]);
		1065	if (pages[i])
		1066	order = folio_order(page_folio(pages[i]));
		1067
		1068	i += NR_PAGES(order);
		1069	}
994		1070
995	err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL);	1071	err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL);
996	if (err)	1072	if (err)
@@ -1043,7 +1119,8 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, · 46 unmodified lines
1043	.vma = vas,	1119	.vma = vas,
1044	.pgmap_owner = page_pgmap(page)->owner,	1120	.pgmap_owner = page_pgmap(page)->owner,
1045	.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE \|	1121	.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE \|
1046	MIGRATE_VMA_SELECT_DEVICE_COHERENT,	1122	MIGRATE_VMA_SELECT_DEVICE_COHERENT \|
		1123	MIGRATE_VMA_SELECT_COMPOUND,
1047	.fault_page = page,	1124	.fault_page = page,
1048	};	1125	};
1049	struct drm_pagemap_migrate_details mdetails = {};	1126	struct drm_pagemap_migrate_details mdetails = {};
@@ -1057,7 +1134,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, · 7 unmodified lines
1057	void *buf;	1134	void *buf;
1058	int i, err = 0;	1135	int i, err = 0;
1059		1136
1060	zdd = page->zone_device_data;	1137	zdd = drm_pagemap_page_zone_device_data(page);
1061	if (time_before64(get_jiffies_64(), zdd->devmem_allocation->timeslice_expiration))	1138	if (time_before64(get_jiffies_64(), zdd->devmem_allocation->timeslice_expiration))
1062	return 0;	1139	return 0;
1063		1140
@@ -1109,8 +1186,15 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, · 45 unmodified lines
1109	if (err)	1186	if (err)
1110	goto err_finalize;	1187	goto err_finalize;
1111		1188
1112	for (i = 0; i < npages; ++i)	1189	for (i = 0; i < npages;) {
		1190	unsigned int order = 0;
		1191
1113	pages[i] = migrate_pfn_to_page(migrate.src[i]);	1192	pages[i] = migrate_pfn_to_page(migrate.src[i]);
		1193	if (pages[i])
		1194	order = folio_order(page_folio(pages[i]));
		1195
		1196	i += NR_PAGES(order);
		1197	}
1114		1198
1115	err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL);	1199	err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL);
1116	if (err)	1200	if (err)
@@ -1140,7 +1224,9 @@ err_out: · 23 unmodified lines
1140	*/	1224	*/
1141	static void drm_pagemap_folio_free(struct folio *folio)	1225	static void drm_pagemap_folio_free(struct folio *folio)
1142	{	1226	{
1143	drm_pagemap_zdd_put(folio->page.zone_device_data);	1227	struct page *page = folio_page(folio, 0);
		1228
		1229	drm_pagemap_zdd_put(drm_pagemap_page_zone_device_data(page));
1144	}	1230	}
1145		1231
1146	/**	1232	/**
@@ -1156,7 +1242,7 @@ static void drm_pagemap_folio_free(struct folio *folio) · 9 unmodified lines
1156	*/	1242	*/
1157	static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf)	1243	static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf)
1158	{	1244	{
1159	struct drm_pagemap_zdd *zdd = vmf->page->zone_device_data;	1245	struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(vmf->page);
1160	int err;	1246	int err;
1161		1247
1162	err = __drm_pagemap_migrate_to_ram(vmf->vma,	1248	err = __drm_pagemap_migrate_to_ram(vmf->vma,
@@ -1166,9 +1252,22 @@ static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf) · 3 unmodified lines
1166	return err ? VM_FAULT_SIGBUS : 0;	1252	return err ? VM_FAULT_SIGBUS : 0;
1167	}	1253	}
1168		1254
		1255	static void drm_pagemap_folio_split(struct folio orig_folio, struct folio new_folio)
		1256	{
		1257	struct drm_pagemap_zdd *zdd;
		1258
		1259	if (!new_folio)
		1260	return;
		1261
		1262	new_folio->pgmap = orig_folio->pgmap;
		1263	zdd = folio_zone_device_data(orig_folio);
		1264	folio_set_zone_device_data(new_folio, drm_pagemap_zdd_get(zdd));
		1265	}
		1266
1169	static const struct dev_pagemap_ops drm_pagemap_pagemap_ops = {	1267	static const struct dev_pagemap_ops drm_pagemap_pagemap_ops = {
1170	.folio_free = drm_pagemap_folio_free,	1268	.folio_free = drm_pagemap_folio_free,
1171	.migrate_to_ram = drm_pagemap_migrate_to_ram,	1269	.migrate_to_ram = drm_pagemap_migrate_to_ram,
		1270	.folio_split = drm_pagemap_folio_split,
1172	};	1271	};
1173		1272
1174	/**	1273	/**
@@ -1222,7 +1321,7 @@ EXPORT_SYMBOL_GPL(drm_pagemap_devmem_init); · 47 unmodified lines
1222	*/	1321	*/
1223	struct drm_pagemap drm_pagemap_page_to_dpagemap(struct page page)	1322	struct drm_pagemap drm_pagemap_page_to_dpagemap(struct page page)
1224	{	1323	{
1225	struct drm_pagemap_zdd *zdd = page->zone_device_data;	1324	struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page);
1226		1325
1227	return zdd->devmem_allocation->dpagemap;	1326	return zdd->devmem_allocation->dpagemap;
1228	}	1327	}

drivers/gpu/drm/ttm/ttm_resource.c+14−14

@@ -37,7 +37,7 @@ · 36 unmodified lines
37	#include <drm/drm_print.h>	37	#include <drm/drm_print.h>
38	#include <drm/drm_util.h>	38	#include <drm/drm_util.h>
39		39
40	/* Detach the cursor from the bulk move list*/	40	/* Detach the cursor from the bulk move list */
41	static void	41	static void
42	ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)	42	ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)
43	{	43	{
@@ -105,9 +105,9 @@ void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor, · 61 unmodified lines
105	* ttm_resource_cursor_fini() - Finalize the LRU list cursor usage	105	* ttm_resource_cursor_fini() - Finalize the LRU list cursor usage
106	* @cursor: The struct ttm_resource_cursor to finalize.	106	* @cursor: The struct ttm_resource_cursor to finalize.
107	*	107	*
108	* The function pulls the LRU list cursor off any lists it was previusly	108	* The function pulls the LRU list cursor off any lists it was previously
109	* attached to. Needs to be called with the LRU lock held. The function	109	* attached to. Needs to be called with the LRU lock held. The function
110	* can be called multiple times after eachother.	110	* can be called multiple times after each other.
111	*/	111	*/
112	void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)	112	void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
113	{	113	{
@@ -317,10 +317,10 @@ void ttm_resource_move_to_lru_tail(struct ttm_resource *res) · 203 unmodified lines
317	}	317	}
318		318
319	/**	319	/**
320	* ttm_resource_init - resource object constructure	320	* ttm_resource_init - resource object constructor
321	* @bo: buffer object this resources is allocated for	321	* @bo: buffer object this resource is allocated for
322	* @place: placement of the resource	322	* @place: placement of the resource
323	* @res: the resource object to inistilize	323	* @res: the resource object to initialize
324	*	324	*
325	* Initialize a new resource object. Counterpart of ttm_resource_fini().	325	* Initialize a new resource object. Counterpart of ttm_resource_fini().
326	*/	326	*/
@@ -435,7 +435,7 @@ EXPORT_SYMBOL(ttm_resource_free); · 108 unmodified lines
435	* @size: How many bytes the new allocation needs.	435	* @size: How many bytes the new allocation needs.
436	*	436	*
437	* Test if @res intersects with @place and @size. Used for testing if evictions	437	* Test if @res intersects with @place and @size. Used for testing if evictions
438	* are valueable or not.	438	* are valuable or not.
439	*	439	*
440	* Returns true if the res placement intersects with @place and @size.	440	* Returns true if the res placement intersects with @place and @size.
441	*/	441	*/
@@ -513,7 +513,7 @@ void ttm_resource_set_bo(struct ttm_resource *res, · 71 unmodified lines
513	* @bdev: ttm device this manager belongs to	513	* @bdev: ttm device this manager belongs to
514	* @size: size of managed resources in arbitrary units	514	* @size: size of managed resources in arbitrary units
515	*	515	*
516	* Initialise core parts of a manager object.	516	* Initialize core parts of a manager object.
517	*/	517	*/
518	void ttm_resource_manager_init(struct ttm_resource_manager *man,	518	void ttm_resource_manager_init(struct ttm_resource_manager *man,
519	struct ttm_device *bdev,	519	struct ttm_device *bdev,
@@ -536,8 +536,8 @@ EXPORT_SYMBOL(ttm_resource_manager_init); · 16 unmodified lines
536	/*	536	/*
537	* ttm_resource_manager_evict_all	537	* ttm_resource_manager_evict_all
538	*	538	*
539	* @bdev - device to use	539	* @bdev: device to use
540	* @man - manager to use	540	* @man: manager to use
541	*	541	*
542	* Evict all the objects out of a memory manager until it is empty.	542	* Evict all the objects out of a memory manager until it is empty.
543	* Part of memory manager cleanup sequence.	543	* Part of memory manager cleanup sequence.
@@ -882,7 +882,7 @@ out_err: · 338 unmodified lines
882		882
883	/**	883	/**
884	* ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory	884	* ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
885	* @iter_io: The iterator to initialize	885	* @iter_io: The iterator to finalize
886	* @bdev: The TTM device	886	* @bdev: The TTM device
887	* @mem: The ttm resource representing the iomap.	887	* @mem: The ttm resource representing the iomap.
888	*	888	*
@@ -921,15 +921,15 @@ DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager); · 32 unmodified lines
921	/**	921	/**
922	* ttm_resource_manager_create_debugfs - Create debugfs entry for specified	922	* ttm_resource_manager_create_debugfs - Create debugfs entry for specified
923	* resource manager.	923	* resource manager.
924	* @man: The TTM resource manager for which the debugfs stats file be creates	924	* @man: The TTM resource manager for which the debugfs stats file to be created
925	* @parent: debugfs directory in which the file will reside	925	* @parent: debugfs directory in which the file will reside
926	* @name: The filename to create.	926	* @name: The filename to create.
927	*	927	*
928	* This function setups up a debugfs file that can be used to look	928	* This function sets up a debugfs file that can be used to look
929	* at debug statistics of the specified ttm_resource_manager.	929	* at debug statistics of the specified ttm_resource_manager.
930	*/	930	*/
931	void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,	931	void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
932	struct dentry * parent,	932	struct dentry *parent,
933	const char *name)	933	const char *name)
934	{	934	{
935	#if defined(CONFIG_DEBUG_FS)	935	#if defined(CONFIG_DEBUG_FS)

drivers/gpu/drm/xe/display/intel_fbdev_fb.c+8−4

 	if (intel_fbdev_fb_prefer_stolen(drm, size)) {
 		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe),
 						size,
-						ttm_bo_type_kernel, XE_BO_FLAG_SCANOUT |
+						ttm_bo_type_kernel,
+						XE_BO_FLAG_FORCE_WC |
 						XE_BO_FLAG_STOLEN |
-						XE_BO_FLAG_GGTT, false);
+						XE_BO_FLAG_GGTT,
+						false);
 		if (!IS_ERR(obj))
 			drm_info(&xe->drm, "Allocated fbdev into stolen\n");
 		else
 	if (IS_ERR(obj)) {
 		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size,
-						ttm_bo_type_kernel, XE_BO_FLAG_SCANOUT |
+						ttm_bo_type_kernel,
+						XE_BO_FLAG_FORCE_WC |
 						XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
-						XE_BO_FLAG_GGTT, false);
+						XE_BO_FLAG_GGTT,
+						false);
 	}
 	if (IS_ERR(obj)) {

drivers/gpu/drm/xe/display/xe_display.c+8−0

 	.synchronize = irq_synchronize,
 };
+static bool has_auxccs(struct drm_device *drm)
+{
+	struct xe_device *xe = to_xe_device(drm);
+	return xe->info.platform == XE_ALDERLAKE_P;
+}
 static const struct intel_display_parent_interface parent = {
 	.bo = &xe_display_bo_interface,
 	.dsb = &xe_display_dsb_interface,
 	.pcode = &xe_display_pcode_interface,
 	.rpm = &xe_display_rpm_interface,
 	.stolen = &xe_display_stolen_interface,
+	.has_auxccs = has_auxccs,
 };
 /**

drivers/gpu/drm/xe/display/xe_display_bo.c+3−3

@@ -42,9 +42,9 @@ static int xe_display_bo_framebuffer_init(struct drm_gem_object *obj, · 41 unmodified lines
42	if (ret)	42	if (ret)
43	goto err;	43	goto err;
44		44
45	if (!(bo->flags & XE_BO_FLAG_SCANOUT)) {	45	if (!(bo->flags & XE_BO_FLAG_FORCE_WC)) {
46	/*	46	/*
47	* XE_BO_FLAG_SCANOUT should ideally be set at creation, or is	47	* XE_BO_FLAG_FORCE_WC should ideally be set at creation, or is
48	* automatically set when creating FB. We cannot change caching	48	* automatically set when creating FB. We cannot change caching
49	* mode when the bo is VM_BINDed, so we can only set	49	* mode when the bo is VM_BINDed, so we can only set
50	* coherency with display when unbound.	50	* coherency with display when unbound.
@@ -54,7 +54,7 @@ static int xe_display_bo_framebuffer_init(struct drm_gem_object *obj, · 3 unmodified lines
54	ret = -EINVAL;	54	ret = -EINVAL;
55	goto err;	55	goto err;
56	}	56	}
57	bo->flags \|= XE_BO_FLAG_SCANOUT;	57	bo->flags \|= XE_BO_FLAG_FORCE_WC;
58	}	58	}
59	ttm_bo_unreserve(&bo->ttm);	59	ttm_bo_unreserve(&bo->ttm);
60	return 0;	60	return 0;

drivers/gpu/drm/xe/display/xe_dsb_buffer.c+3−1

 					PAGE_ALIGN(size),
 					ttm_bo_type_kernel,
 					XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
-					XE_BO_FLAG_SCANOUT | XE_BO_FLAG_GGTT, false);
+					XE_BO_FLAG_FORCE_WC |
+					XE_BO_FLAG_GGTT,
+					false);
 	if (IS_ERR(obj)) {
 		ret = PTR_ERR(obj);
 		goto err_pin_map;

drivers/gpu/drm/xe/display/xe_fb_pin.c+82−30

 	*dpt_ofs = ALIGN(*dpt_ofs, 4096);
 }
-static void
+static unsigned int
-write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs,
+write_dpt_padding(struct iosys_map *map, unsigned int dest, unsigned int pad)
-		   u32 bo_ofs, u32 width, u32 height, u32 src_stride,
+{
-		   u32 dst_stride)
+	/* The DE ignores the PTEs for the padding tiles */
+	return dest + pad * sizeof(u64);
+}
+static unsigned int
+write_dpt_remapped_linear(struct xe_bo *bo, struct iosys_map *map,
+			  unsigned int dest,
+			  const struct intel_remapped_plane_info *plane)
 {
 	struct xe_device *xe = xe_bo_device(bo);
 	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
-	u32 column, row;
+	const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo,
-	u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
+						 xe->pat.idx[XE_CACHE_NONE]);
+	unsigned int offset = plane->offset * XE_PAGE_SIZE;
+	unsigned int size = plane->size;
-	for (row = 0; row < height; row++) {
+	while (size--) {
-		u32 src_idx = src_stride * row + bo_ofs;
+		u64 addr = xe_bo_addr(bo, offset, XE_PAGE_SIZE);
-		for (column = 0; column < width; column++) {
+		iosys_map_wr(map, dest, u64, addr | pte);
-			u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE);
+		dest += sizeof(u64);
-			iosys_map_wr(map, *dpt_ofs, u64, pte | addr);
+		offset += XE_PAGE_SIZE;
+	}
-			*dpt_ofs += 8;
+	return dest;
-			src_idx++;
+}
+static unsigned int
+write_dpt_remapped_tiled(struct xe_bo *bo, struct iosys_map *map,
+			 unsigned int dest,
+			 const struct intel_remapped_plane_info *plane)
+{
+	struct xe_device *xe = xe_bo_device(bo);
+	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
+	const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo,
+						 xe->pat.idx[XE_CACHE_NONE]);
+	unsigned int offset, column, row;
+	for (row = 0; row < plane->height; row++) {
+		offset = (plane->offset + plane->src_stride * row) *
+			 XE_PAGE_SIZE;
+		for (column = 0; column < plane->width; column++) {
+			u64 addr = xe_bo_addr(bo, offset, XE_PAGE_SIZE);
+			iosys_map_wr(map, dest, u64, addr | pte);
+			dest += sizeof(u64);
+			offset += XE_PAGE_SIZE;
 		}
-		/* The DE ignores the PTEs for the padding tiles */
+		dest = write_dpt_padding(map, dest,
-		*dpt_ofs += (dst_stride - width) * 8;
+					 plane->dst_stride - plane->width);
 	}
-	/* Align to next page */
+	return dest;
-	*dpt_ofs = ALIGN(*dpt_ofs, 4096);
+}
+static void
+write_dpt_remapped(struct xe_bo *bo,
+		   const struct intel_remapped_info *remap_info,
+		   struct iosys_map *map)
+{
+	unsigned int i, dest = 0;
+	for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) {
+		const struct intel_remapped_plane_info *plane =
+				&remap_info->plane[i];
+		if (!plane->linear && !plane->width && !plane->height)
+			continue;
+		if (dest && remap_info->plane_alignment) {
+			const unsigned int index = dest / sizeof(u64);
+			const unsigned int pad =
+				ALIGN(index, remap_info->plane_alignment) -
+				index;
+			dest = write_dpt_padding(map, dest, pad);
+		}
+		if (plane->linear)
+			dest = write_dpt_remapped_linear(bo, map, dest, plane);
+		else
+			dest = write_dpt_remapped_tiled(bo, map, dest, plane);
+	}
 }
 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
 						   ttm_bo_type_kernel,
 						   XE_BO_FLAG_SYSTEM |
 						   XE_BO_FLAG_GGTT |
-						   XE_BO_FLAG_PAGETABLE,
+						   XE_BO_FLAG_PAGETABLE |
+						   XE_BO_FLAG_FORCE_WC,
 						   alignment, false);
 	if (IS_ERR(dpt))
 		return PTR_ERR(dpt);
 			iosys_map_wr(&dpt->vmap, x * 8, u64, pte | addr);
 		}
 	} else if (view->type == I915_GTT_VIEW_REMAPPED) {
-		const struct intel_remapped_info *remap_info = &view->remapped;
+		write_dpt_remapped(bo, &view->remapped, &dpt->vmap);
-		u32 i, dpt_ofs = 0;
-		for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++)
-			write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs,
-					   remap_info->plane[i].offset,
-					   remap_info->plane[i].width,
-					   remap_info->plane[i].height,
-					   remap_info->plane[i].src_stride,
-					   remap_info->plane[i].dst_stride);
 	} else {
 		const struct intel_rotation_info *rot_info = &view->rotated;
 		u32 i, dpt_ofs = 0;
 		return 0;
 	/* We reject creating !SCANOUT fb's, so this is weird.. */
-	drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT));
+	drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC));
 	vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment);

drivers/gpu/drm/xe/display/xe_initial_plane.c+1−1

@@ -48,7 +48,7 @@ initial_plane_bo(struct xe_device *xe, · 47 unmodified lines
48	if (plane_config->size == 0)	48	if (plane_config->size == 0)
49	return NULL;	49	return NULL;
50		50
51	flags = XE_BO_FLAG_SCANOUT \| XE_BO_FLAG_GGTT;	51	flags = XE_BO_FLAG_FORCE_WC \| XE_BO_FLAG_GGTT;
52		52
53	base = round_down(plane_config->base, page_size);	53	base = round_down(plane_config->base, page_size);
54	if (IS_DGFX(xe)) {	54	if (IS_DGFX(xe)) {

drivers/gpu/drm/xe/instructions/xe_mi_commands.h+19−0

@@ -34,6 +34,19 @@ · 33 unmodified lines
34	#define MI_FORCE_WAKEUP __MI_INSTR(0x1D)	34	#define MI_FORCE_WAKEUP __MI_INSTR(0x1D)
35	#define MI_MATH(n) (__MI_INSTR(0x1A) \| XE_INSTR_NUM_DW((n) + 1))	35	#define MI_MATH(n) (__MI_INSTR(0x1A) \| XE_INSTR_NUM_DW((n) + 1))
36		36
		37	#define MI_SEMAPHORE_WAIT (__MI_INSTR(0x1c) \| XE_INSTR_NUM_DW(5))
		38	#define MI_SEMW_GGTT REG_BIT(22)
		39	#define MI_SEMW_POLL REG_BIT(15)
		40	#define MI_SEMW_COMPARE_OP_MASK REG_GENMASK(14, 12)
		41	#define COMPARE_OP_SAD_GT_SDD 0
		42	#define COMPARE_OP_SAD_GTE_SDD 1
		43	#define COMPARE_OP_SAD_LT_SDD 2
		44	#define COMPARE_OP_SAD_LTE_SDD 3
		45	#define COMPARE_OP_SAD_EQ_SDD 4
		46	#define COMPARE_OP_SAD_NEQ_SDD 5
		47	#define MI_SEMW_COMPARE(OP) REG_FIELD_PREP(MI_SEMW_COMPARE_OP_MASK, COMPARE_OP_##OP)
		48	#define MI_SEMW_TOKEN(token) REG_FIELD_PREP(REG_GENMASK(9, 2), (token))
		49
37	#define MI_STORE_DATA_IMM __MI_INSTR(0x20)	50	#define MI_STORE_DATA_IMM __MI_INSTR(0x20)
38	#define MI_SDI_GGTT REG_BIT(22)	51	#define MI_SDI_GGTT REG_BIT(22)
39	#define MI_SDI_LEN_DW GENMASK(9, 0)	52	#define MI_SDI_LEN_DW GENMASK(9, 0)
@@ -81,4 +94,10 @@ · 41 unmodified lines
81	#define MI_SET_APPID_SESSION_ID_MASK REG_GENMASK(6, 0)	94	#define MI_SET_APPID_SESSION_ID_MASK REG_GENMASK(6, 0)
82	#define MI_SET_APPID_SESSION_ID(x) REG_FIELD_PREP(MI_SET_APPID_SESSION_ID_MASK, x)	95	#define MI_SET_APPID_SESSION_ID(x) REG_FIELD_PREP(MI_SET_APPID_SESSION_ID_MASK, x)
83		96
		97	#define MI_SEMAPHORE_WAIT_TOKEN (__MI_INSTR(0x1c) \| XE_INSTR_NUM_DW(5)) /* XeLP+ */
		98	#define MI_SEMAPHORE_REGISTER_POLL REG_BIT(16)
		99	#define MI_SEMAPHORE_POLL REG_BIT(15)
		100	#define MI_SEMAPHORE_CMP_OP_MASK REG_GENMASK(14, 12)
		101	#define MI_SEMAPHORE_SAD_EQ_SDD REG_FIELD_PREP(MI_SEMAPHORE_CMP_OP_MASK, 4)
		102
84	#endif	103	#endif

drivers/gpu/drm/xe/regs/xe_engine_regs.h+8−0

@@ -132,6 +132,14 @@ · 131 unmodified lines
132	#define RING_BBADDR(base) XE_REG((base) + 0x140)	132	#define RING_BBADDR(base) XE_REG((base) + 0x140)
133	#define RING_BBADDR_UDW(base) XE_REG((base) + 0x168)	133	#define RING_BBADDR_UDW(base) XE_REG((base) + 0x168)
134		134
		135	#define PR_CTR_CTRL(base) XE_REG((base) + 0x178)
		136	#define CTR_COUNT_SELECT_FF REG_BIT(31)
		137	#define CTR_LOGIC_OP_MASK REG_GENMASK(30, 0)
		138	#define CTR_START 0
		139	#define CTR_STOP 1
		140	#define CTR_LOGIC_OP(OP) REG_FIELD_PREP(CTR_LOGIC_OP_MASK, CTR_##OP)
		141	#define PR_CTR_THRSH(base) XE_REG((base) + 0x17c)
		142
135	#define BCS_SWCTRL(base) XE_REG((base) + 0x200, XE_REG_OPTION_MASKED)	143	#define BCS_SWCTRL(base) XE_REG((base) + 0x200, XE_REG_OPTION_MASKED)
136	#define BCS_SWCTRL_DISABLE_256B REG_BIT(2)	144	#define BCS_SWCTRL_DISABLE_256B REG_BIT(2)
137		145

drivers/gpu/drm/xe/regs/xe_gt_regs.h+1−0

@@ -578,6 +578,7 @@ · 577 unmodified lines
578	#define ENABLE_SMP_LD_RENDER_SURFACE_CONTROL REG_BIT(44 - 32)	578	#define ENABLE_SMP_LD_RENDER_SURFACE_CONTROL REG_BIT(44 - 32)
579	#define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32)	579	#define FORCE_SLM_FENCE_SCOPE_TO_TILE REG_BIT(42 - 32)
580	#define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32)	580	#define FORCE_UGM_FENCE_SCOPE_TO_TILE REG_BIT(41 - 32)
		581	#define L3_128B_256B_WRT_DIS REG_BIT(40 - 32)
581	#define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32)	582	#define MAXREQS_PER_BANK REG_GENMASK(39 - 32, 37 - 32)
582	#define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32)	583	#define DISABLE_128B_EVICTION_COMMAND_UDW REG_BIT(36 - 32)
583	#define LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE REG_BIT(35 - 32)	584	#define LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE REG_BIT(35 - 32)

drivers/gpu/drm/xe/xe_bo.c+15−9

 		WARN_ON((bo->flags & XE_BO_FLAG_USER) && !bo->cpu_caching);
 		/*
-		 * Display scanout is always non-coherent with the CPU cache.
-		 *
 		 * For Xe_LPG and beyond up to NVL-P (excluding), PPGTT PTE
 		 * lookups are also non-coherent and require a CPU:WC mapping.
 		 */
-		if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_SCANOUT) ||
+		if ((!bo->cpu_caching && bo->flags & XE_BO_FLAG_FORCE_WC) ||
-		     (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE))
+		    (!xe->info.has_cached_pt && bo->flags & XE_BO_FLAG_PAGETABLE))
 			caching = ttm_write_combined;
 	}
 		if (!xe_vm_in_fault_mode(vm)) {
 			drm_gpuvm_bo_evict(vm_bo, true);
-			continue;
+			/*
+			 * L2 cache may not be flushed, so ensure that is done in
+			 * xe_vm_invalidate_vma() below
+			 */
+			if (!xe_device_is_l2_flush_optimized(xe))
+				continue;
 		}
 		if (!idle) {
 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING)
 		bo_flags |= XE_BO_FLAG_DEFER_BACKING;
+	/*
+	 * Display scanout is always non-coherent with the CPU cache.
+	 */
 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT)
-		bo_flags |= XE_BO_FLAG_SCANOUT;
+		bo_flags |= XE_BO_FLAG_FORCE_WC;
 	if (args->flags & DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION) {
 		if (XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20))
 	/* CCS formats need physical placement at a 64K alignment in VRAM. */
 	if ((bo_flags & XE_BO_FLAG_VRAM_MASK) &&
-	    (bo_flags & XE_BO_FLAG_SCANOUT) &&
+	    (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT) &&
 	    !(xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) &&
 	    IS_ALIGNED(args->size, SZ_64K))
 		bo_flags |= XE_BO_FLAG_NEEDS_64K;
 			 args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC))
 		return -EINVAL;
-	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_SCANOUT &&
+	if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_FLAG_FORCE_WC &&
 			 args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB))
 		return -EINVAL;
 	bo = xe_bo_create_user(xe, NULL, args->size,
 			       DRM_XE_GEM_CPU_CACHING_WC,
 			       XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
-			       XE_BO_FLAG_SCANOUT |
+			       XE_BO_FLAG_FORCE_WC |
 			       XE_BO_FLAG_NEEDS_CPU_ACCESS, NULL);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);

drivers/gpu/drm/xe/xe_bo.h+1−1

@@ -35,7 +35,7 @@ · 34 unmodified lines
35	#define XE_BO_FLAG_PINNED BIT(7)	35	#define XE_BO_FLAG_PINNED BIT(7)
36	#define XE_BO_FLAG_NO_RESV_EVICT BIT(8)	36	#define XE_BO_FLAG_NO_RESV_EVICT BIT(8)
37	#define XE_BO_FLAG_DEFER_BACKING BIT(9)	37	#define XE_BO_FLAG_DEFER_BACKING BIT(9)
38	#define XE_BO_FLAG_SCANOUT BIT(10)	38	#define XE_BO_FLAG_FORCE_WC BIT(10)
39	#define XE_BO_FLAG_FIXED_PLACEMENT BIT(11)	39	#define XE_BO_FLAG_FIXED_PLACEMENT BIT(11)
40	#define XE_BO_FLAG_PAGETABLE BIT(12)	40	#define XE_BO_FLAG_PAGETABLE BIT(12)
41	#define XE_BO_FLAG_NEEDS_CPU_ACCESS BIT(13)	41	#define XE_BO_FLAG_NEEDS_CPU_ACCESS BIT(13)

drivers/gpu/drm/xe/xe_device.c+33−0

@@ -211,6 +211,8 @@ static const struct drm_ioctl_desc xe_ioctls[] = { · 210 unmodified lines
211	DRM_RENDER_ALLOW),	211	DRM_RENDER_ALLOW),
212	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,	212	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl,
213	DRM_RENDER_ALLOW),	213	DRM_RENDER_ALLOW),
		214	DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl,
		215	DRM_RENDER_ALLOW),
214	};	216	};
215		217
216	static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)	218	static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
@@ -1094,6 +1096,29 @@ static void tdf_request_sync(struct xe_device *xe) · 877 unmodified lines
1094	}	1096	}
1095	}	1097	}
1096		1098
		1099	/**
		1100	* xe_device_is_l2_flush_optimized - if L2 flush is optimized by HW
		1101	* @xe: The device to check.
		1102	*
		1103	* Return: true if the HW device optimizing L2 flush, false otherwise.
		1104	*/
		1105	bool xe_device_is_l2_flush_optimized(struct xe_device *xe)
		1106	{
		1107	/* XA is always flushed, like at the end-of-submssion (and maybe other
		1108	* places), just that internally as an optimisation hw doesn't need to make
		1109	* that a full flush (which will also include XA) when Media is
		1110	* off/powergated, since it doesn't need to worry about GT caches vs Media
		1111	* coherency, and only CPU vs GPU coherency, so can make that flush a
		1112	* targeted XA flush, since stuff tagged with XA now means it's shared with
		1113	* the CPU. The main implication is that we now need to somehow flush non-XA before
		1114	* freeing system memory pages, otherwise dirty cachelines could be flushed after the free
		1115	* (like if Media suddenly turns on and does a full flush)
		1116	*/
		1117	if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe))
		1118	return true;
		1119	return false;
		1120	}
		1121
1097	void xe_device_l2_flush(struct xe_device *xe)	1122	void xe_device_l2_flush(struct xe_device *xe)
1098	{	1123	{
1099	struct xe_gt *gt;	1124	struct xe_gt *gt;
@@ -1140,6 +1165,14 @@ void xe_device_td_flush(struct xe_device *xe) · 40 unmodified lines
1140	{	1165	{
1141	struct xe_gt *root_gt;	1166	struct xe_gt *root_gt;
1142		1167
		1168	/*
		1169	* From Xe3p onward the HW takes care of flush of TD entries also along
		1170	* with flushing XA entries, which will be at the usual sync points,
		1171	* like at the end of submission, so no manual flush is needed here.
		1172	*/
		1173	if (GRAPHICS_VER(xe) >= 35)
		1174	return;
		1175
1143	if (!IS_DGFX(xe) \|\| GRAPHICS_VER(xe) < 20)	1176	if (!IS_DGFX(xe) \|\| GRAPHICS_VER(xe) < 20)
1144	return;	1177	return;
1145		1178

drivers/gpu/drm/xe/xe_device.h+1−0

@@ -188,6 +188,7 @@ void xe_device_snapshot_print(struct xe_device xe, struct drm_printer p); · 187 unmodified lines
188	u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address);	188	u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address);
189	u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address);	189	u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address);
190		190
		191	bool xe_device_is_l2_flush_optimized(struct xe_device *xe);
191	void xe_device_td_flush(struct xe_device *xe);	192	void xe_device_td_flush(struct xe_device *xe);
192	void xe_device_l2_flush(struct xe_device *xe);	193	void xe_device_l2_flush(struct xe_device *xe);
193		194

drivers/gpu/drm/xe/xe_ggtt.c+9−6

  * give us the correct placement for free.
  */
+#define XE_GGTT_FLAGS_64K	BIT(0)
+#define XE_GGTT_FLAGS_ONLINE	BIT(1)
 /**
  * struct xe_ggtt_node - A node in GGTT.
  *
 	 * @flags: Flags for this GGTT
 	 * Acceptable flags:
 	 * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K.
+	 * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock
+	 *   after init
 	 */
 	unsigned int flags;
 	/** @scratch: Internal object allocation used as a scratch page */
 {
 	struct xe_ggtt *ggtt = arg;
+	scoped_guard(mutex, &ggtt->lock)
+		ggtt->flags &= ~XE_GGTT_FLAGS_ONLINE;
 	drain_workqueue(ggtt->wq);
 }
 	if (err)
 		return err;
+	ggtt->flags |= XE_GGTT_FLAGS_ONLINE;
 	return devm_add_action_or_reset(xe->drm.dev, dev_fini_ggtt, ggtt);
 }
 ALLOW_ERROR_INJECTION(xe_ggtt_init_early, ERRNO); /* See xe_pci_probe() */
 static void ggtt_node_remove(struct xe_ggtt_node *node)
 {
 	struct xe_ggtt *ggtt = node->ggtt;
-	struct xe_device *xe = tile_to_xe(ggtt->tile);
 	bool bound;
-	int idx;
-	bound = drm_dev_enter(&xe->drm, &idx);
 	mutex_lock(&ggtt->lock);
+	bound = ggtt->flags & XE_GGTT_FLAGS_ONLINE;
 	if (bound)
 		xe_ggtt_clear(ggtt, xe_ggtt_node_addr(node), xe_ggtt_node_size(node));
 	drm_mm_remove_node(&node->base);
 	if (node->invalidate_on_remove)
 		xe_ggtt_invalidate(ggtt);
-	drm_dev_exit(idx);
 free_node:
 	ggtt_node_fini(node);
 }

drivers/gpu/drm/xe/xe_gt.c+6−3

 static void gt_reset_worker(struct work_struct *w);
 static int emit_job_sync(struct xe_exec_queue *q, struct xe_bb *bb,
-			 long timeout_jiffies)
+			 long timeout_jiffies, bool force_reset)
 {
 	struct xe_sched_job *job;
 	struct dma_fence *fence;
 	if (IS_ERR(job))
 		return PTR_ERR(job);
+	job->ring_ops_force_reset = force_reset;
 	xe_sched_job_arm(job);
 	fence = dma_fence_get(&job->drm.s_fence->finished);
 	xe_sched_job_push(job);
 	if (IS_ERR(bb))
 		return PTR_ERR(bb);
-	ret = emit_job_sync(q, bb, HZ);
+	ret = emit_job_sync(q, bb, HZ, false);
 	xe_bb_free(bb, NULL);
 	return ret;
 	bb->len = cs - bb->cs;
-	ret = emit_job_sync(q, bb, HZ);
+	/* only VFs need to trigger reset to get a clean NULL context */
+	ret = emit_job_sync(q, bb, HZ, IS_SRIOV_VF(gt_to_xe(gt)));
 	xe_bb_free(bb, NULL);

drivers/gpu/drm/xe/xe_gt_ccs_mode.c+2−0

 #include "xe_gt_printk.h"
 #include "xe_gt_sysfs.h"
 #include "xe_mmio.h"
+#include "xe_pm.h"
 #include "xe_sriov.h"
 #include "xe_sriov_pf.h"
 	xe_gt_info(gt, "Setting compute mode to %d\n", num_engines);
 	gt->ccs_mode = num_engines;
 	xe_gt_record_user_engines(gt);
+	guard(xe_pm_runtime)(xe);
 	xe_gt_reset(gt);
 	/* We may end PF lockdown once CCS mode is default again */

drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c+62−16

 	case XE_GT_SRIOV_STATE_##_X: return #_X
 	CASE2STR(WIP);
 	CASE2STR(FLR_WIP);
+	CASE2STR(FLR_PREPARE);
 	CASE2STR(FLR_SEND_START);
 	CASE2STR(FLR_WAIT_GUC);
 	CASE2STR(FLR_GUC_DONE);
  * The VF FLR state machine looks like::
  *
  *	 (READY,PAUSED,STOPPED)<------------<--------------o
- *	    |                                               \
+ *	    |             |                                 \
- *	   flr                                               \
+ *	   flr           prepare                             \
- *	    |                                                 \
+ *	    |             |                                   \
- *	....V..........................FLR_WIP...........      \
+ *	....V.............V............FLR_WIP...........      \
- *	:    \                                          :       \
+ *	:   |             |                             :       \
+ *	:   |    FLR_PREPARE                            :        |
+ *	:   |    /                                      :        |
+ *	:   \   flr                                     :        |
+ *	:    \ /                                        :        |
  *	:     \   o----<----busy                        :        |
  *	:      \ /            /                         :        |
  *	:       FLR_SEND_START---failed----->-----------o--->(FLR_FAILED)<---o
 	pf_queue_vf(gt, vfid);
 }
-static void pf_enter_vf_flr_wip(struct xe_gt *gt, unsigned int vfid)
+static bool pf_exit_vf_flr_prepare(struct xe_gt *gt, unsigned int vfid)
 {
-	if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP)) {
+	if (!pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_PREPARE))
-		xe_gt_sriov_dbg(gt, "VF%u FLR is already in progress\n", vfid);
+		return false;
-		return;
-	}
-	pf_enter_vf_wip(gt, vfid);
 	pf_enter_vf_flr_send_start(gt, vfid);
+	return true;
+}
+static bool pf_enter_vf_flr_wip(struct xe_gt *gt, unsigned int vfid)
+{
+	if (!pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP))
+		return false;
+	pf_enter_vf_wip(gt, vfid);
+	return true;
 }
 static void pf_exit_vf_flr_wip(struct xe_gt *gt, unsigned int vfid)
 {
 	if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_WIP)) {
+		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_PREPARE);
 		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_SEND_FINISH);
 		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_MMIO);
 		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_RESET_DATA);
 }
 /**
- * xe_gt_sriov_pf_control_trigger_flr - Start a VF FLR sequence.
+ * xe_gt_sriov_pf_control_prepare_flr() - Notify PF that VF FLR request was issued.
  * @gt: the &xe_gt
  * @vfid: the VF identifier
  *
+ * This is an optional early notification path used to mark pending FLR before
+ * the GuC notifies the PF with a FLR event.
+ *
  * This function is for PF only.
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_control_trigger_flr(struct xe_gt *gt, unsigned int vfid)
+int xe_gt_sriov_pf_control_prepare_flr(struct xe_gt *gt, unsigned int vfid)
 {
-	pf_enter_vf_flr_wip(gt, vfid);
+	if (!pf_enter_vf_flr_wip(gt, vfid))
+		return -EALREADY;
+	pf_enter_vf_state(gt, vfid, XE_GT_SRIOV_STATE_FLR_PREPARE);
 	return 0;
 }
+static int pf_begin_vf_flr(struct xe_gt *gt, unsigned int vfid)
+{
+	if (pf_enter_vf_flr_wip(gt, vfid)) {
+		pf_enter_vf_flr_send_start(gt, vfid);
+		return 0;
+	}
+	if (pf_exit_vf_flr_prepare(gt, vfid))
+		return 0;
+	xe_gt_sriov_dbg(gt, "VF%u FLR is already in progress\n", vfid);
+	return -EALREADY;
+}
+/**
+ * xe_gt_sriov_pf_control_trigger_flr - Start a VF FLR sequence.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ *
+ * This function is for PF only.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_control_trigger_flr(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_begin_vf_flr(gt, vfid);
+}
 /**
  * xe_gt_sriov_pf_control_sync_flr() - Synchronize on the VF FLR checkpoint.
  * @gt: the &xe_gt
 	if (needs_dispatch_flr(xe)) {
 		for_each_gt(gtit, xe, gtid)
-			pf_enter_vf_flr_wip(gtit, vfid);
+			pf_begin_vf_flr(gtit, vfid);
 	} else {
-		pf_enter_vf_flr_wip(gt, vfid);
+		pf_begin_vf_flr(gt, vfid);
 	}
 }

drivers/gpu/drm/xe/xe_gt_sriov_pf_control.h+1−0

@@ -27,6 +27,7 @@ int xe_gt_sriov_pf_control_process_restore_data(struct xe_gt *gt, unsigned int v · 26 unmodified lines
27	int xe_gt_sriov_pf_control_trigger_restore_vf(struct xe_gt *gt, unsigned int vfid);	27	int xe_gt_sriov_pf_control_trigger_restore_vf(struct xe_gt *gt, unsigned int vfid);
28	int xe_gt_sriov_pf_control_finish_restore_vf(struct xe_gt *gt, unsigned int vfid);	28	int xe_gt_sriov_pf_control_finish_restore_vf(struct xe_gt *gt, unsigned int vfid);
29	int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid);	29	int xe_gt_sriov_pf_control_stop_vf(struct xe_gt *gt, unsigned int vfid);
		30	int xe_gt_sriov_pf_control_prepare_flr(struct xe_gt *gt, unsigned int vfid);
30	int xe_gt_sriov_pf_control_trigger_flr(struct xe_gt *gt, unsigned int vfid);	31	int xe_gt_sriov_pf_control_trigger_flr(struct xe_gt *gt, unsigned int vfid);
31	int xe_gt_sriov_pf_control_sync_flr(struct xe_gt *gt, unsigned int vfid, bool sync);	32	int xe_gt_sriov_pf_control_sync_flr(struct xe_gt *gt, unsigned int vfid, bool sync);
32	int xe_gt_sriov_pf_control_wait_flr(struct xe_gt *gt, unsigned int vfid);	33	int xe_gt_sriov_pf_control_wait_flr(struct xe_gt *gt, unsigned int vfid);

drivers/gpu/drm/xe/xe_gt_sriov_pf_control_types.h+2−0

  *
  * @XE_GT_SRIOV_STATE_WIP: indicates that some operations are in progress.
  * @XE_GT_SRIOV_STATE_FLR_WIP: indicates that a VF FLR is in progress.
+ * @XE_GT_SRIOV_STATE_FLR_PREPARE: indicates that the PF received early VF FLR prepare notification.
  * @XE_GT_SRIOV_STATE_FLR_SEND_START: indicates that the PF wants to send a FLR START command.
  * @XE_GT_SRIOV_STATE_FLR_WAIT_GUC: indicates that the PF awaits for a response from the GuC.
  * @XE_GT_SRIOV_STATE_FLR_GUC_DONE: indicates that the PF has received a response from the GuC.
 	XE_GT_SRIOV_STATE_WIP = 1,
 	XE_GT_SRIOV_STATE_FLR_WIP,
+	XE_GT_SRIOV_STATE_FLR_PREPARE,
 	XE_GT_SRIOV_STATE_FLR_SEND_START,
 	XE_GT_SRIOV_STATE_FLR_WAIT_GUC,
 	XE_GT_SRIOV_STATE_FLR_GUC_DONE,

drivers/gpu/drm/xe/xe_gt_stats.c+6−0

 	DEF_STAT_STR(SVM_64K_CPU_COPY_US, "svm_64K_cpu_copy_us"),
 	DEF_STAT_STR(SVM_2M_CPU_COPY_US, "svm_2M_cpu_copy_us"),
 	DEF_STAT_STR(SVM_DEVICE_COPY_KB, "svm_device_copy_kb"),
+	DEF_STAT_STR(SVM_4K_DEVICE_COPY_KB, "svm_4K_device_copy_kb"),
+	DEF_STAT_STR(SVM_64K_DEVICE_COPY_KB, "svm_64K_device_copy_kb"),
+	DEF_STAT_STR(SVM_2M_DEVICE_COPY_KB, "svm_2M_device_copy_kb"),
 	DEF_STAT_STR(SVM_CPU_COPY_KB, "svm_cpu_copy_kb"),
+	DEF_STAT_STR(SVM_4K_CPU_COPY_KB, "svm_4K_cpu_copy_kb"),
+	DEF_STAT_STR(SVM_64K_CPU_COPY_KB, "svm_64K_cpu_copy_kb"),
+	DEF_STAT_STR(SVM_2M_CPU_COPY_KB, "svm_2M_cpu_copy_kb"),
 	DEF_STAT_STR(SVM_4K_GET_PAGES_US, "svm_4K_get_pages_us"),
 	DEF_STAT_STR(SVM_64K_GET_PAGES_US, "svm_64K_get_pages_us"),
 	DEF_STAT_STR(SVM_2M_GET_PAGES_US, "svm_2M_get_pages_us"),

drivers/gpu/drm/xe/xe_gt_stats_types.h+6−0

 	XE_GT_STATS_ID_SVM_64K_CPU_COPY_US,
 	XE_GT_STATS_ID_SVM_2M_CPU_COPY_US,
 	XE_GT_STATS_ID_SVM_DEVICE_COPY_KB,
+	XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_KB,
+	XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_KB,
+	XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_KB,
 	XE_GT_STATS_ID_SVM_CPU_COPY_KB,
+	XE_GT_STATS_ID_SVM_4K_CPU_COPY_KB,
+	XE_GT_STATS_ID_SVM_64K_CPU_COPY_KB,
+	XE_GT_STATS_ID_SVM_2M_CPU_COPY_KB,
 	XE_GT_STATS_ID_SVM_4K_GET_PAGES_US,
 	XE_GT_STATS_ID_SVM_64K_GET_PAGES_US,
 	XE_GT_STATS_ID_SVM_2M_GET_PAGES_US,

drivers/gpu/drm/xe/xe_guc.c+30−5

 	if (xe_guc_using_main_gamctrl_queues(guc))
 		flags |= GUC_CTL_MAIN_GAMCTRL_QUEUES;
+	if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe) && xe_gt_is_media_type(guc_to_gt(guc)))
+		flags |= GUC_CTL_ENABLE_L2FLUSH_OPT;
 	return flags;
 }
 	struct xe_guc_pc *guc_pc = &gt->uc.guc.pc;
 	u32 before_freq, act_freq, cur_freq;
 	u32 status = 0, tries = 0;
+	int load_result, ret;
 	ktime_t before;
 	u64 delta_ms;
-	int ret;
 	before_freq = xe_guc_pc_get_act_freq(guc_pc);
 	before = ktime_get();
-	ret = poll_timeout_us(ret = guc_load_done(gt, &status, &tries), ret,
+	ret = poll_timeout_us(load_result = guc_load_done(gt, &status, &tries), load_result,
 			      10 * USEC_PER_MSEC,
 			      GUC_LOAD_TIMEOUT_SEC * USEC_PER_SEC, false);
 	act_freq = xe_guc_pc_get_act_freq(guc_pc);
 	cur_freq = xe_guc_pc_get_cur_freq_fw(guc_pc);
-	if (ret) {
+	if (ret || load_result <= 0) {
 		xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz)\n",
 			  status, delta_ms, xe_guc_pc_get_act_freq(guc_pc),
 			  xe_guc_pc_get_cur_freq_fw(guc_pc));
 	return 0;
 }
-int xe_guc_suspend(struct xe_guc *guc)
+/**
+ * xe_guc_softreset() - Soft reset GuC
+ * @guc: The GuC object
+ *
+ * Send soft reset command to GuC through mmio send.
+ *
+ * Return: 0 if success, otherwise error code
+ */
+int xe_guc_softreset(struct xe_guc *guc)
 {
-	struct xe_gt *gt = guc_to_gt(guc);
 	u32 action[] = {
 		XE_GUC_ACTION_CLIENT_SOFT_RESET,
 	};
 	int ret;
+	if (!xe_uc_fw_is_running(&guc->fw))
+		return 0;
 	ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action));
+	if (ret)
+		return ret;
+	return 0;
+}
+int xe_guc_suspend(struct xe_guc *guc)
+{
+	struct xe_gt *gt = guc_to_gt(guc);
+	int ret;
+	ret = xe_guc_softreset(guc);
 	if (ret) {
 		xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret));
 		return ret;

drivers/gpu/drm/xe/xe_guc.h+1−0

@@ -44,6 +44,7 @@ int xe_guc_opt_in_features_enable(struct xe_guc *guc); · 43 unmodified lines
44	void xe_guc_runtime_suspend(struct xe_guc *guc);	44	void xe_guc_runtime_suspend(struct xe_guc *guc);
45	void xe_guc_runtime_resume(struct xe_guc *guc);	45	void xe_guc_runtime_resume(struct xe_guc *guc);
46	int xe_guc_suspend(struct xe_guc *guc);	46	int xe_guc_suspend(struct xe_guc *guc);
		47	int xe_guc_softreset(struct xe_guc *guc);
47	void xe_guc_notify(struct xe_guc *guc);	48	void xe_guc_notify(struct xe_guc *guc);
48	int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);	49	int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);
49	int xe_guc_mmio_send(struct xe_guc guc, const u32 request, u32 len);	50	int xe_guc_mmio_send(struct xe_guc guc, const u32 request, u32 len);

drivers/gpu/drm/xe/xe_guc_ct.c+9−15

 #include "xe_guc_submit.h"
 #include "xe_guc_tlb_inval.h"
 #include "xe_map.h"
+#include "xe_page_reclaim.h"
 #include "xe_pm.h"
 #include "xe_sleep.h"
 #include "xe_sriov_vf.h"
 {
 	struct xe_guc_ct *ct = arg;
+	xe_guc_ct_stop(ct);
 	guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED);
 }
 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
 		break;
 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
-	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
-		/*
-		 * Page reclamation is an extension of TLB invalidation. Both
-		 * operations share the same seqno and fence. When either
-		 * action completes, we need to signal the corresponding
-		 * fence. Since the handling logic (lookup fence by seqno,
-		 * fence signalling) is identical, we use the same handler
-		 * for both G2H events.
-		 */
 		ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
 		break;
+	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
+		ret = xe_guc_page_reclaim_done_handler(guc, payload, adj_len);
+		break;
 	case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF:
 		ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len);
 		break;
 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
 		break;
 	case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
-	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
-		/*
-		 * Seqno and fence handling of page reclamation and TLB
-		 * invalidation is identical, so we can use the same handler
-		 * for both actions.
-		 */
 		__g2h_release_space(ct, len);
 		ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len);
 		break;
+	case XE_GUC_ACTION_PAGE_RECLAMATION_DONE:
+		__g2h_release_space(ct, len);
+		ret = xe_guc_page_reclaim_done_handler(guc, payload, adj_len);
+		break;
 	default:
 		xe_gt_warn(gt, "NOT_POSSIBLE\n");
 	}

drivers/gpu/drm/xe/xe_guc_fwif.h+1−0

@@ -67,6 +67,7 @@ struct guc_update_exec_queue_policy { · 66 unmodified lines
67	#define GUC_CTL_ENABLE_PSMI_LOGGING BIT(7)	67	#define GUC_CTL_ENABLE_PSMI_LOGGING BIT(7)
68	#define GUC_CTL_MAIN_GAMCTRL_QUEUES BIT(9)	68	#define GUC_CTL_MAIN_GAMCTRL_QUEUES BIT(9)
69	#define GUC_CTL_DISABLE_SCHEDULER BIT(14)	69	#define GUC_CTL_DISABLE_SCHEDULER BIT(14)
		70	#define GUC_CTL_ENABLE_L2FLUSH_OPT BIT(15)
70		71
71	#define GUC_CTL_DEBUG 3	72	#define GUC_CTL_DEBUG 3
72	#define GUC_LOG_VERBOSITY REG_GENMASK(1, 0)	73	#define GUC_LOG_VERBOSITY REG_GENMASK(1, 0)

drivers/gpu/drm/xe/xe_guc_submit.c+62−25

 #define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN		6
+static int guc_submit_reset_prepare(struct xe_guc *guc);
 static struct xe_guc *
 exec_queue_to_guc(struct xe_exec_queue *q)
 {
 		 EXEC_QUEUE_STATE_BANNED));
 }
-static void guc_submit_fini(struct drm_device *drm, void *arg)
+static void guc_submit_sw_fini(struct drm_device *drm, void *arg)
 {
 	struct xe_guc *guc = arg;
 	struct xe_device *xe = guc_to_xe(guc);
 	xa_destroy(&guc->submission_state.exec_queue_lookup);
 }
+static void guc_submit_fini(void *arg)
+{
+	struct xe_guc *guc = arg;
+	/* Forcefully kill any remaining exec queues */
+	xe_guc_ct_stop(&guc->ct);
+	guc_submit_reset_prepare(guc);
+	xe_guc_softreset(guc);
+	xe_guc_submit_stop(guc);
+	xe_uc_fw_sanitize(&guc->fw);
+	xe_guc_submit_pause_abort(guc);
+}
 static void guc_submit_wedged_fini(void *arg)
 {
 	struct xe_guc *guc = arg;
 	guc->submission_state.initialized = true;
-	return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc);
+	err = drmm_add_action_or_reset(&xe->drm, guc_submit_sw_fini, guc);
+	if (err)
+		return err;
+	return devm_add_action_or_reset(xe->drm.dev, guc_submit_fini, guc);
 }
 /*
  */
 void xe_guc_submit_wedge(struct xe_guc *guc)
 {
+	struct xe_device *xe = guc_to_xe(guc);
 	struct xe_gt *gt = guc_to_gt(guc);
 	struct xe_exec_queue *q;
 	unsigned long index;
 	if (!guc->submission_state.initialized)
 		return;
-	err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev,
+	if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) {
-				       guc_submit_wedged_fini, guc);
+		err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev,
-	if (err) {
+					       guc_submit_wedged_fini, guc);
-		xe_gt_err(gt, "Failed to register clean-up in wedged.mode=%s; "
+		if (err) {
-			  "Although device is wedged.\n",
+			xe_gt_err(gt, "Failed to register clean-up on wedged.mode=%s; "
-			  xe_wedged_mode_to_string(XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET));
+				  "Although device is wedged.\n",
-		return;
+				  xe_wedged_mode_to_string(XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET));
-	}
+			return;
+		}
-	mutex_lock(&guc->submission_state.lock);
+		mutex_lock(&guc->submission_state.lock);
-	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
+		xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
-		if (xe_exec_queue_get_unless_zero(q))
+			if (xe_exec_queue_get_unless_zero(q))
-			set_exec_queue_wedged(q);
+				set_exec_queue_wedged(q);
-	mutex_unlock(&guc->submission_state.lock);
+		mutex_unlock(&guc->submission_state.lock);
+	} else {
+		/* Forcefully kill any remaining exec queues, signal fences */
+		guc_submit_reset_prepare(guc);
+		xe_guc_submit_stop(guc);
+		xe_guc_softreset(guc);
+		xe_uc_fw_sanitize(&guc->fw);
+		xe_guc_submit_pause_abort(guc);
+	}
 }
 static bool guc_submit_hint_wedged(struct xe_guc *guc)
 static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
 {
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
+	bool do_destroy = false;
 	/* Stop scheduling + flush any DRM scheduler operations */
 	xe_sched_submission_stop(sched);
 	/* Clean up lost G2H + reset engine state */
 	if (exec_queue_registered(q)) {
 		if (exec_queue_destroyed(q))
-			__guc_exec_queue_destroy(guc, q);
+			do_destroy = true;
 	}
 	if (q->guc->suspend_pending) {
 		set_exec_queue_suspended(q);
 			xe_guc_exec_queue_trigger_cleanup(q);
 		}
 	}
+	if (do_destroy)
+		__guc_exec_queue_destroy(guc, q);
 }
-int xe_guc_submit_reset_prepare(struct xe_guc *guc)
+static int guc_submit_reset_prepare(struct xe_guc *guc)
 {
 	int ret;
-	if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc)))
-		return 0;
-	if (!guc->submission_state.initialized)
-		return 0;
 	/*
 	 * Using an atomic here rather than submission_state.lock as this
 	 * function can be called while holding the CT lock (engine reset
 	return ret;
 }
+int xe_guc_submit_reset_prepare(struct xe_guc *guc)
+{
+	if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc)))
+		return 0;
+	if (!guc->submission_state.initialized)
+		return 0;
+	return guc_submit_reset_prepare(guc);
+}
 void xe_guc_submit_reset_wait(struct xe_guc *guc)
 {
 	wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) ||
 			continue;
 		xe_sched_submission_start(sched);
-		if (exec_queue_killed_or_banned_or_wedged(q))
+		guc_exec_queue_kill(q);
-			xe_guc_exec_queue_trigger_cleanup(q);
 	}
 	mutex_unlock(&guc->submission_state.lock);
 }

drivers/gpu/drm/xe/xe_i2c.c+12−3

  */
 void xe_i2c_irq_handler(struct xe_device *xe, u32 master_ctl)
 {
-	if (!xe_i2c_irq_present(xe))
+	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
+	if (!(master_ctl & I2C_IRQ) || !xe_i2c_irq_present(xe))
 		return;
-	if (master_ctl & I2C_IRQ)
+	/* Forward interrupt to I2C adapter */
-		generic_handle_irq_safe(xe->i2c->adapter_irq);
+	generic_handle_irq_safe(xe->i2c->adapter_irq);
+	/* Deassert after I2C adapter clears the interrupt */
+	xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, 0, PCI_COMMAND_INTX_DISABLE);
+	/* Reassert to allow subsequent interrupt generation */
+	xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0);
 }
 void xe_i2c_irq_reset(struct xe_device *xe)
 	if (!xe_i2c_irq_present(xe))
 		return;
+	xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, 0, PCI_COMMAND_INTX_DISABLE);
 	xe_mmio_rmw32(mmio, I2C_BRIDGE_PCICFGCTL, ACPI_INTR_EN, 0);
 }
 		return;
 	xe_mmio_rmw32(mmio, I2C_BRIDGE_PCICFGCTL, 0, ACPI_INTR_EN);
+	xe_mmio_rmw32(mmio, I2C_CONFIG_CMD, PCI_COMMAND_INTX_DISABLE, 0);
 }
 static int xe_i2c_irq_map(struct irq_domain *h, unsigned int virq,

drivers/gpu/drm/xe/xe_lrc.c+66−30

 #include "xe_map.h"
 #include "xe_memirq.h"
 #include "xe_mmio.h"
+#include "xe_ring_ops.h"
 #include "xe_sriov.h"
 #include "xe_trace_lrc.h"
 #include "xe_vm.h"
 					       class, NULL))
 		return true;
+	if (gt->ring_ops[class]->emit_aux_table_inv)
+		return true;
 	return false;
 }
 	return cmd - batch;
 }
+static ssize_t setup_invalidate_auxccs_wa(struct xe_lrc *lrc,
+					  struct xe_hw_engine *hwe,
+					  u32 *batch, size_t max_len)
+{
+	struct xe_gt *gt = lrc->gt;
+	u32 *(*emit)(struct xe_gt *gt, u32 *cmd) =
+		gt->ring_ops[hwe->class]->emit_aux_table_inv;
+	if (!emit)
+		return 0;
+	if (xe_gt_WARN_ON(gt, max_len < 8))
+		return -ENOSPC;
+	return emit(gt, batch) - batch;
+}
 struct bo_setup {
 	ssize_t (*setup)(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 			 u32 *batch, size_t max_size);
 {
 	static const struct bo_setup rcs_funcs[] = {
 		{ .setup = setup_timestamp_wa },
+		{ .setup = setup_invalidate_auxccs_wa },
 		{ .setup = setup_configfs_mid_ctx_restore_bb },
 	};
 	static const struct bo_setup xcs_funcs[] = {
+		{ .setup = setup_invalidate_auxccs_wa },
 		{ .setup = setup_configfs_mid_ctx_restore_bb },
 	};
 	struct bo_setup_state state = {
 	bo = xe_bo_create_pin_map_novm(xe, tile, bo_size,
 				       ttm_bo_type_kernel,
 				       bo_flags, false);
-	if (IS_ERR(lrc->bo))
+	if (IS_ERR(bo))
-		return PTR_ERR(lrc->bo);
+		return PTR_ERR(bo);
 	lrc->bo = bo;
 static int dump_mi_command(struct drm_printer *p,
 			   struct xe_gt *gt,
+			   u32 *start,
 			   u32 *dw,
 			   int remaining_dw)
 {
 		while (num_noop < remaining_dw &&
 		       (*(++dw) & REG_GENMASK(31, 23)) == MI_NOOP)
 			num_noop++;
-		drm_printf(p, "[%#010x] MI_NOOP (%d dwords)\n", inst_header, num_noop);
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] MI_NOOP (%d dwords)\n",
+			   dw - num_noop - start, inst_header, num_noop);
 		return num_noop;
 	case MI_TOPOLOGY_FILTER:
-		drm_printf(p, "[%#010x] MI_TOPOLOGY_FILTER\n", inst_header);
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] MI_TOPOLOGY_FILTER\n",
+			   dw - start, inst_header);
 		return 1;
 	case MI_BATCH_BUFFER_END:
-		drm_printf(p, "[%#010x] MI_BATCH_BUFFER_END\n", inst_header);
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] MI_BATCH_BUFFER_END\n",
+			   dw - start, inst_header);
 		/* Return 'remaining_dw' to consume the rest of the LRC */
 		return remaining_dw;
 	}
 	switch (inst_header & MI_OPCODE) {
 	case MI_LOAD_REGISTER_IMM:
-		drm_printf(p, "[%#010x] MI_LOAD_REGISTER_IMM: %d regs\n",
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] MI_LOAD_REGISTER_IMM: %d regs\n",
-			   inst_header, (numdw - 1) / 2);
+			   dw - start, inst_header, (numdw - 1) / 2);
 		for (int i = 1; i < numdw; i += 2)
-			drm_printf(p, " - %#6x = %#010x\n", dw[i], dw[i + 1]);
+			drm_printf(p, "LRC[%#5tx]  =  - %#6x = %#010x\n",
+				   &dw[i] - start, dw[i], dw[i + 1]);
 		return numdw;
 	case MI_LOAD_REGISTER_MEM & MI_OPCODE:
-		drm_printf(p, "[%#010x] MI_LOAD_REGISTER_MEM: %s%s\n",
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] MI_LOAD_REGISTER_MEM: %s%s\n",
-			   inst_header,
+			   dw - start, inst_header,
 			   dw[0] & MI_LRI_LRM_CS_MMIO ? "CS_MMIO " : "",
 			   dw[0] & MI_LRM_USE_GGTT ? "USE_GGTT " : "");
 		if (numdw == 4)
-			drm_printf(p, " - %#6x = %#010llx\n",
+			drm_printf(p, "LRC[%#5tx]  =  - %#6x = %#010llx\n",
+				   dw - start,
 				   dw[1], ((u64)(dw[3]) << 32 | (u64)(dw[2])));
 		else
-			drm_printf(p, " - %*ph (%s)\n",
+			drm_printf(p, "LRC[%#5tx]  =  - %*ph (%s)\n",
-				   (int)sizeof(u32) * (numdw - 1), dw + 1,
+				   dw - start, (int)sizeof(u32) * (numdw - 1),
-				   numdw < 4 ? "truncated" : "malformed");
+				   dw + 1, numdw < 4 ? "truncated" : "malformed");
 		return numdw;
 	case MI_FORCE_WAKEUP:
-		drm_printf(p, "[%#010x] MI_FORCE_WAKEUP\n", inst_header);
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] MI_FORCE_WAKEUP\n",
+			   dw - start, inst_header);
 		return numdw;
 	default:
-		drm_printf(p, "[%#010x] unknown MI opcode %#x, likely %d dwords\n",
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] unknown MI opcode %#x, likely %d dwords\n",
-			   inst_header, opcode, numdw);
+			   dw - start, inst_header, opcode, numdw);
 		return numdw;
 	}
 }
 static int dump_gfxpipe_command(struct drm_printer *p,
 				struct xe_gt *gt,
+				u32 *start,
 				u32 *dw,
 				int remaining_dw)
 {
 	switch (*dw & GFXPIPE_MATCH_MASK) {
 #define MATCH(cmd) \
 	case cmd: \
-		drm_printf(p, "[%#010x] " #cmd " (%d dwords)\n", *dw, numdw); \
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] " #cmd " (%d dwords)\n", \
+			   dw - start, *dw, numdw); \
 		return numdw
 #define MATCH3D(cmd) \
 	case CMD_##cmd: \
-		drm_printf(p, "[%#010x] " #cmd " (%d dwords)\n", *dw, numdw); \
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] " #cmd " (%d dwords)\n", \
+			   dw - start, *dw, numdw); \
 		return numdw
 	MATCH(STATE_BASE_ADDRESS);
 	MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTER_2);
 	default:
-		drm_printf(p, "[%#010x] unknown GFXPIPE command (pipeline=%#x, opcode=%#x, subopcode=%#x), likely %d dwords\n",
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] unknown GFXPIPE command (pipeline=%#x, opcode=%#x, subopcode=%#x), likely %d dwords\n",
-			   *dw, pipeline, opcode, subopcode, numdw);
+			   dw - start, *dw, pipeline, opcode, subopcode, numdw);
 		return numdw;
 	}
 }
 static int dump_gfx_state_command(struct drm_printer *p,
 				  struct xe_gt *gt,
+				  u32 *start,
 				  u32 *dw,
 				  int remaining_dw)
 {
 	MATCH(STATE_WRITE_INLINE);
 	default:
-		drm_printf(p, "[%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n",
+		drm_printf(p, "LRC[%#5tx]  =  [%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n",
-			   *dw, opcode, numdw);
+			   dw - start, *dw, opcode, numdw);
 		return numdw;
 	}
 }
 			 struct xe_gt *gt,
 			 enum xe_engine_class hwe_class)
 {
-	u32 *dw;
+	u32 *dw, *start;
 	int remaining_dw, num_dw;
 	if (!gt->default_lrc[hwe_class]) {
 	 * hardware status page.
 	 */
 	dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE;
+	start = dw;
 	remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4;
 	while (remaining_dw > 0) {
 		if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_MI) {
-			num_dw = dump_mi_command(p, gt, dw, remaining_dw);
+			num_dw = dump_mi_command(p, gt, start, dw, remaining_dw);
 		} else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE) {
-			num_dw = dump_gfxpipe_command(p, gt, dw, remaining_dw);
+			num_dw = dump_gfxpipe_command(p, gt, start, dw, remaining_dw);
 		} else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFX_STATE) {
-			num_dw = dump_gfx_state_command(p, gt, dw, remaining_dw);
+			num_dw = dump_gfx_state_command(p, gt, start, dw, remaining_dw);
 		} else {
 			num_dw = min(instr_dw(*dw), remaining_dw);
-			drm_printf(p, "[%#10x] Unknown instruction of type %#x, likely %d dwords\n",
+			drm_printf(p, "LRC[%#5tx]  =  [%#10x] Unknown instruction of type %#x, likely %d dwords\n",
+				   dw - start,
 				   *dw, REG_FIELD_GET(XE_INSTR_CMD_TYPE, *dw),
 				   num_dw);
 		}
  * @lrc: Pointer to the lrc.
  *
  * Return latest ctx timestamp. With support for active contexts, the
- * calculation may bb slightly racy, so follow a read-again logic to ensure that
+ * calculation may be slightly racy, so follow a read-again logic to ensure that
  * the context is still active before returning the right timestamp.
  *
  * Returns: New ctx timestamp value
  */
 u64 xe_lrc_timestamp(struct xe_lrc *lrc)
 {
-	u64 lrc_ts, reg_ts, new_ts;
+	u64 lrc_ts, reg_ts, new_ts = lrc->ctx_timestamp;
 	u32 engine_id;
 	lrc_ts = xe_lrc_ctx_timestamp(lrc);

drivers/gpu/drm/xe/xe_oa.c+5−2

 	size_t offset = 0;
 	int ret;
-	/* Can't read from disabled streams */
+	if (!stream->sample)
-	if (!stream->enabled || !stream->sample)
 		return -EINVAL;
 	if (!(file->f_flags & O_NONBLOCK)) {
 	if (stream->sample)
 		hrtimer_cancel(&stream->poll_check_timer);
+	/* Update stream->oa_buffer.tail to allow any final reports to be read */
+	if (xe_oa_buffer_check_unlocked(stream))
+		wake_up(&stream->poll_wq);
 }
 static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream)

drivers/gpu/drm/xe/xe_page_reclaim.c+26−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	#include "xe_page_reclaim.h"	11	#include "xe_page_reclaim.h"
12		12
13	#include "xe_gt_stats.h"	13	#include "xe_gt_stats.h"
		14	#include "xe_guc_tlb_inval.h"
14	#include "xe_macros.h"	15	#include "xe_macros.h"
15	#include "xe_pat.h"	16	#include "xe_pat.h"
16	#include "xe_sa.h"	17	#include "xe_sa.h"
@@ -26,12 +27,18 @@ · 9 unmodified lines
26	* flushes.	27	* flushes.
27	* - pat_index is transient display (1)	28	* - pat_index is transient display (1)
28	*	29	*
		30	* For cases of NULL VMA, there should be no corresponding PRL entry
		31	* so skip over.
		32	*
29	* Return: true when page reclamation is unnecessary, false otherwise.	33	* Return: true when page reclamation is unnecessary, false otherwise.
30	*/	34	*/
31	bool xe_page_reclaim_skip(struct xe_tile tile, struct xe_vma vma)	35	bool xe_page_reclaim_skip(struct xe_tile tile, struct xe_vma vma)
32	{	36	{
33	u8 l3_policy;	37	u8 l3_policy;
34		38
		39	if (xe_vma_is_null(vma))
		40	return true;
		41
35	l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index);	42	l3_policy = xe_pat_index_get_l3_policy(tile->xe, vma->attr.pat_index);
36		43
37	/*	44	/*
@@ -130,3 +137,22 @@ int xe_page_reclaim_list_alloc_entries(struct xe_page_reclaim_list *prl) · 92 unmodified lines
130		137
131	return page ? 0 : -ENOMEM;	138	return page ? 0 : -ENOMEM;
132	}	139	}
		140
		141	/**
		142	* xe_guc_page_reclaim_done_handler() - Page reclaim done handler
		143	* @guc: guc
		144	* @msg: message indicating page reclamation done
		145	* @len: length of message
		146	*
		147	* Page reclamation is an extension of TLB invalidation. Both
		148	* operations share the same seqno and fence. When either
		149	* action completes, we need to signal the corresponding
		150	* fence. Since the handling logic is currently identical, this
		151	* function delegates to the TLB invalidation handler.
		152	*
		153	* Return: 0 on success, -EPROTO for malformed messages.
		154	*/
		155	int xe_guc_page_reclaim_done_handler(struct xe_guc guc, u32 msg, u32 len)
		156	{
		157	return xe_guc_tlb_inval_done_handler(guc, msg, len);
		158	}

drivers/gpu/drm/xe/xe_page_reclaim.h+3−0

 struct xe_tlb_inval_fence;
 struct xe_tile;
 struct xe_gt;
+struct xe_guc;
 struct xe_vma;
 struct xe_guc_page_reclaim_entry {
 		put_page(virt_to_page(entries));
 }
+int xe_guc_page_reclaim_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
 #endif	/* _XE_PAGE_RECLAIM_H_ */

drivers/gpu/drm/xe/xe_pagefault.c+32−0

@@ -187,6 +187,12 @@ static int xe_pagefault_service(struct xe_pagefault *pf) · 186 unmodified lines
187	goto unlock_vm;	187	goto unlock_vm;
188	}	188	}
189		189
		190	if (xe_vma_read_only(vma) &&
		191	pf->consumer.access_type != XE_PAGEFAULT_ACCESS_TYPE_READ) {
		192	err = -EPERM;
		193	goto unlock_vm;
		194	}
		195
190	atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type);	196	atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type);
191		197
192	if (xe_vma_is_cpu_addr_mirror(vma))	198	if (xe_vma_is_cpu_addr_mirror(vma))
@@ -244,6 +250,31 @@ static void xe_pagefault_print(struct xe_pagefault *pf) · 51 unmodified lines
244	pf->consumer.engine_instance);	250	pf->consumer.engine_instance);
245	}	251	}
246		252
		253	static void xe_pagefault_save_to_vm(struct xe_device xe, struct xe_pagefault pf)
		254	{
		255	struct xe_vm *vm;
		256
		257	/*
		258	* Pagefault may be asociated to VM that is not in fault mode.
		259	* Perform asid_to_vm behavior, except if VM is not in fault
		260	* mode, return VM anyways.
		261	*/
		262	down_read(&xe->usm.lock);
		263	vm = xa_load(&xe->usm.asid_to_vm, pf->consumer.asid);
		264	if (vm)
		265	xe_vm_get(vm);
		266	else
		267	vm = ERR_PTR(-EINVAL);
		268	up_read(&xe->usm.lock);
		269
		270	if (IS_ERR(vm))
		271	return;
		272
		273	xe_vm_add_fault_entry_pf(vm, pf);
		274
		275	xe_vm_put(vm);
		276	}
		277
247	static void xe_pagefault_queue_work(struct work_struct *w)	278	static void xe_pagefault_queue_work(struct work_struct *w)
248	{	279	{
249	struct xe_pagefault_queue *pf_queue =	280	struct xe_pagefault_queue *pf_queue =
@@ -262,6 +293,7 @@ static void xe_pagefault_queue_work(struct work_struct *w) · 12 unmodified lines
262		293
263	err = xe_pagefault_service(&pf);	294	err = xe_pagefault_service(&pf);
264	if (err) {	295	if (err) {
		296	xe_pagefault_save_to_vm(gt_to_xe(pf.gt), &pf);
265	if (!(pf.consumer.access_type & XE_PAGEFAULT_ACCESS_PREFETCH)) {	297	if (!(pf.consumer.access_type & XE_PAGEFAULT_ACCESS_PREFETCH)) {
266	xe_pagefault_print(&pf);	298	xe_pagefault_print(&pf);
267	xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n",	299	xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n",

drivers/gpu/drm/xe/xe_pat.c+7−7

drivers/gpu/drm/xe/xe_pat.h+3−2

 	/**
 	 * @coh_mode: The GPU coherency mode that @value maps to.
 	 */
-#define XE_COH_NONE          1
+#define XE_COH_NONE		1
-#define XE_COH_AT_LEAST_1WAY 2
+#define XE_COH_1WAY		2
+#define XE_COH_2WAY		3
 	u16 coh_mode;
 	/**

drivers/gpu/drm/xe/xe_pt.c+35−15

 		err = vma_check_userptr(vm, op->map.vma, pt_update);
 		break;
 	case DRM_GPUVA_OP_REMAP:
-		if (op->remap.prev)
+		if (op->remap.prev && !op->remap.skip_prev)
 			err = vma_check_userptr(vm, op->remap.prev, pt_update);
-		if (!err && op->remap.next)
+		if (!err && op->remap.next && !op->remap.skip_next)
 			err = vma_check_userptr(vm, op->remap.next, pt_update);
 		break;
 	case DRM_GPUVA_OP_UNMAP:
 	XE_WARN_ON(!level);
 	/* Check for leaf node */
 	if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) &&
-	    (!xe_child->base.children || !xe_child->base.children[first])) {
+	    xe_child->level <= MAX_HUGEPTE_LEVEL) {
 		struct iosys_map *leaf_map = &xe_child->bo->vmap;
 		pgoff_t count = xe_pt_num_entries(addr, next, xe_child->level, walk);
 		for (pgoff_t i = 0; i < count; i++) {
-			u64 pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
+			u64 pte;
 			int ret;
+			/*
+			 * If not a leaf pt, skip unless non-leaf pt is interleaved between
+			 * leaf ptes which causes the page walk to skip over the child leaves
+			 */
+			if (xe_child->base.children && xe_child->base.children[first + i]) {
+				u64 pt_size = 1ULL << walk->shifts[xe_child->level];
+				bool edge_pt = (i == 0 && !IS_ALIGNED(addr, pt_size)) ||
+					       (i == count - 1 && !IS_ALIGNED(next, pt_size));
+				if (!edge_pt) {
+					xe_page_reclaim_list_abort(xe_walk->tile->primary_gt,
+								   xe_walk->prl,
+								   "PT is skipped by walk at level=%u offset=%lu",
+								   xe_child->level, first + i);
+					break;
+				}
+				continue;
+			}
+			pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
 			/*
 			 * In rare scenarios, pte may not be written yet due to racy conditions.
 			 * In such cases, invalidate the PRL and fallback to full PPC invalidation.
 			}
 			/* Ensure it is a defined page */
-			xe_tile_assert(xe_walk->tile,
+			xe_tile_assert(xe_walk->tile, xe_child->level == 0 ||
-				       xe_child->level == 0 ||
+				       (pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G)));
-				       (pte & (XE_PTE_PS64 | XE_PDE_PS_2M | XE_PDPE_PS_1G)));
 			/* An entry should be added for 64KB but contigious 4K have XE_PTE_PS64 */
 			if (pte & XE_PTE_PS64)
 	killed = xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
 	/*
-	 * Verify PRL is active and if entry is not a leaf pte (base.children conditions),
+	 * Verify if any PTE are potentially dropped at non-leaf levels, either from being
-	 * there is a potential need to invalidate the PRL if any PTE (num_live) are dropped.
+	 * killed or the page walk covers the region.
 	 */
-	if (xe_walk->prl && level > 1 && xe_child->num_live &&
+	if (xe_walk->prl && xe_page_reclaim_list_valid(xe_walk->prl) &&
-	    xe_child->base.children && xe_child->base.children[first]) {
+	    xe_child->level > MAX_HUGEPTE_LEVEL && xe_child->num_live) {
 		bool covered = xe_pt_covers(addr, next, xe_child->level, &xe_walk->base);
 		/*
 		err = unbind_op_prepare(tile, pt_update_ops, old);
-		if (!err && op->remap.prev) {
+		if (!err && op->remap.prev && !op->remap.skip_prev) {
 			err = bind_op_prepare(vm, tile, pt_update_ops,
 					      op->remap.prev, false);
 			pt_update_ops->wait_vm_bookkeep = true;
 		}
-		if (!err && op->remap.next) {
+		if (!err && op->remap.next && !op->remap.skip_next) {
 			err = bind_op_prepare(vm, tile, pt_update_ops,
 					      op->remap.next, false);
 			pt_update_ops->wait_vm_bookkeep = true;
 		unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2);
-		if (op->remap.prev)
+		if (op->remap.prev && !op->remap.skip_prev)
 			bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
 				       fence, fence2, false);
-		if (op->remap.next)
+		if (op->remap.next && !op->remap.skip_next)
 			bind_op_commit(vm, tile, pt_update_ops, op->remap.next,
 				       fence, fence2, false);
 		break;

drivers/gpu/drm/xe/xe_ring_ops.c+122−21

@@ -48,15 +48,48 @@ static u32 preparser_disable(bool state) · 47 unmodified lines
48	return MI_ARB_CHECK \| BIT(8) \| state;	48	return MI_ARB_CHECK \| BIT(8) \| state;
49	}	49	}
50		50
51	static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,	51	static u32 *
52	u32 *dw, int i)	52	__emit_aux_table_inv(u32 *cmd, const struct xe_reg reg, u32 adj_offset)
53	{	53	{
54	dw[i++] = MI_LOAD_REGISTER_IMM \| MI_LRI_NUM_REGS(1) \| MI_LRI_MMIO_REMAP_EN;	54	*cmd++ = MI_LOAD_REGISTER_IMM \| MI_LRI_NUM_REGS(1) \|
55	dw[i++] = reg.addr + gt->mmio.adj_offset;	55	MI_LRI_MMIO_REMAP_EN;
56	dw[i++] = AUX_INV;	56	*cmd++ = reg.addr + adj_offset;
57	dw[i++] = MI_NOOP;	57	*cmd++ = AUX_INV;
		58	*cmd++ = MI_SEMAPHORE_WAIT_TOKEN \| MI_SEMAPHORE_REGISTER_POLL \|
		59	MI_SEMAPHORE_POLL \| MI_SEMAPHORE_SAD_EQ_SDD;
		60	*cmd++ = 0;
		61	*cmd++ = reg.addr + adj_offset;
		62	*cmd++ = 0;
		63	*cmd++ = 0;
		64
		65	return cmd;
		66	}
58		67
59	return i;	68	static u32 emit_aux_table_inv_render_compute(struct xe_gt gt, u32 *cmd)
		69	{
		70	return __emit_aux_table_inv(cmd, CCS_AUX_INV, gt->mmio.adj_offset);
		71	}
		72
		73	static u32 emit_aux_table_inv_video_decode(struct xe_gt gt, u32 *cmd)
		74	{
		75	return __emit_aux_table_inv(cmd, VD0_AUX_INV, gt->mmio.adj_offset);
		76	}
		77
		78	static u32 emit_aux_table_inv_video_enhance(struct xe_gt gt, u32 *cmd)
		79	{
		80	return __emit_aux_table_inv(cmd, VE0_AUX_INV, gt->mmio.adj_offset);
		81	}
		82
		83	static int emit_aux_table_inv(struct xe_hw_engine hwe, u32 dw, int i)
		84	{
		85	struct xe_gt *gt = hwe->gt;
		86	u32 (emit)(struct xe_gt gt, u32 cmd) =
		87	gt->ring_ops[hwe->class]->emit_aux_table_inv;
		88
		89	if (emit)
		90	return emit(gt, dw + i) - dw;
		91	else
		92	return i;
60	}	93	}
61		94
62	static int emit_user_interrupt(u32 *dw, int i)	95	static int emit_user_interrupt(u32 *dw, int i)
@@ -256,6 +289,32 @@ static int emit_copy_timestamp(struct xe_device xe, struct xe_lrc lrc, · 193 unmodified lines
256	return i;	289	return i;
257	}	290	}
258		291
		292	static int emit_fake_watchdog(struct xe_lrc lrc, u32 dw, int i)
		293	{
		294	/*
		295	* Setup a watchdog with impossible condition to always trigger an
		296	* hardware interrupt that would force the GuC to reset the engine.
		297	*/
		298
		299	dw[i++] = MI_LOAD_REGISTER_IMM \| MI_LRI_NUM_REGS(2) \| MI_LRI_LRM_CS_MMIO;
		300	dw[i++] = PR_CTR_THRSH(0).addr;
		301	dw[i++] = 2; /* small threshold */
		302	dw[i++] = PR_CTR_CTRL(0).addr;
		303	dw[i++] = CTR_LOGIC_OP(START);
		304
		305	dw[i++] = MI_SEMAPHORE_WAIT \| MI_SEMW_GGTT \| MI_SEMW_POLL \| MI_SEMW_COMPARE(SAD_EQ_SDD);
		306	dw[i++] = 0xdead; /* this should never be seen */
		307	dw[i++] = lower_32_bits(xe_lrc_ggtt_addr(lrc));
		308	dw[i++] = upper_32_bits(xe_lrc_ggtt_addr(lrc));
		309	dw[i++] = 0; /* unused token */
		310
		311	dw[i++] = MI_LOAD_REGISTER_IMM \| MI_LRI_NUM_REGS(1) \| MI_LRI_LRM_CS_MMIO;
		312	dw[i++] = PR_CTR_CTRL(0).addr;
		313	dw[i++] = CTR_LOGIC_OP(STOP);
		314
		315	return i;
		316	}
		317
259	/* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */	318	/* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
260	static void __emit_job_gen12_simple(struct xe_sched_job job, struct xe_lrc lrc,	319	static void __emit_job_gen12_simple(struct xe_sched_job job, struct xe_lrc lrc,
261	u64 batch_addr, u32 *head, u32 seqno)	320	u64 batch_addr, u32 *head, u32 seqno)
@@ -266,6 +325,9 @@ static void __emit_job_gen12_simple(struct xe_sched_job job, struct xe_lrc lrc · 4 unmodified lines
266		325
267	*head = lrc->ring.tail;	326	*head = lrc->ring.tail;
268		327
		328	if (job->ring_ops_force_reset)
		329	i = emit_fake_watchdog(lrc, dw, i);
		330
269	i = emit_copy_timestamp(gt_to_xe(gt), lrc, dw, i);	331	i = emit_copy_timestamp(gt_to_xe(gt), lrc, dw, i);
270		332
271	if (job->ring_ops_flush_tlb) {	333	if (job->ring_ops_flush_tlb) {
@@ -305,9 +367,9 @@ static bool has_aux_ccs(struct xe_device *xe) · 33 unmodified lines
305	* PVC is a special case that has no compression of either type	367	* PVC is a special case that has no compression of either type
306	* (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2	368	* (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2
307	* onward, so any future platforms with no FlatCCS will not have	369	* onward, so any future platforms with no FlatCCS will not have
308	* AuxCCS either.	370	* AuxCCS, and we explicitly do not want to support it on MTL.
309	*/	371	*/
310	if (GRAPHICS_VER(xe) >= 20 \|\| xe->info.platform == XE_PVC)	372	if (GRAPHICS_VERx100(xe) >= 1270 \|\| xe->info.platform == XE_PVC)
311	return false;	373	return false;
312		374
313	return !xe->info.has_flat_ccs;	375	return !xe->info.has_flat_ccs;
@@ -320,21 +382,18 @@ static void __emit_job_gen12_video(struct xe_sched_job job, struct xe_lrc lrc, · 6 unmodified lines
320	u32 ppgtt_flag = get_ppgtt_flag(job);	382	u32 ppgtt_flag = get_ppgtt_flag(job);
321	struct xe_gt *gt = job->q->gt;	383	struct xe_gt *gt = job->q->gt;
322	struct xe_device *xe = gt_to_xe(gt);	384	struct xe_device *xe = gt_to_xe(gt);
323	bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
324		385
325	*head = lrc->ring.tail;	386	*head = lrc->ring.tail;
326		387
		388	if (job->ring_ops_force_reset)
		389	i = emit_fake_watchdog(lrc, dw, i);
		390
327	i = emit_copy_timestamp(xe, lrc, dw, i);	391	i = emit_copy_timestamp(xe, lrc, dw, i);
328		392
329	dw[i++] = preparser_disable(true);	393	dw[i++] = preparser_disable(true);
330		394
331	/* hsdes: 1809175790 */	395	/* hsdes: 1809175790 */
332	if (has_aux_ccs(xe)) {	396	i = emit_aux_table_inv(job->q->hwe, dw, i);
333	if (decode)
334	i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
335	else
336	i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
337	}
338		397
339	if (job->ring_ops_flush_tlb)	398	if (job->ring_ops_flush_tlb)
340	i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),	399	i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
@@ -381,8 +440,18 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, · 40 unmodified lines
381		440
382	*head = lrc->ring.tail;	441	*head = lrc->ring.tail;
383		442
		443	if (job->ring_ops_force_reset)
		444	i = emit_fake_watchdog(lrc, dw, i);
		445
384	i = emit_copy_timestamp(xe, lrc, dw, i);	446	i = emit_copy_timestamp(xe, lrc, dw, i);
385		447
		448	/*
		449	* On AuxCCS platforms the invalidation of the Aux table requires
		450	* quiescing the memory traffic beforehand.
		451	*/
		452	if (has_aux_ccs(xe))
		453	i = emit_render_cache_flush(job, dw, i);
		454
386	dw[i++] = preparser_disable(true);	455	dw[i++] = preparser_disable(true);
387	if (lacks_render)	456	if (lacks_render)
388	mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;	457	mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
@@ -393,8 +462,7 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, · 4 unmodified lines
393	i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i);	462	i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i);
394		463
395	/* hsdes: 1809175790 */	464	/* hsdes: 1809175790 */
396	if (has_aux_ccs(xe))	465	i = emit_aux_table_inv(job->q->hwe, dw, i);
397	i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
398		466
399	dw[i++] = preparser_disable(false);	467	dw[i++] = preparser_disable(false);
400		468
@@ -433,6 +501,8 @@ static void emit_migration_job_gen12(struct xe_sched_job *job, · 32 unmodified lines
433		501
434	*head = lrc->ring.tail;	502	*head = lrc->ring.tail;
435		503
		504	xe_gt_assert(gt, !job->ring_ops_force_reset);
		505
436	i = emit_copy_timestamp(xe, lrc, dw, i);	506	i = emit_copy_timestamp(xe, lrc, dw, i);
437		507
438	i = emit_store_imm_ggtt(saddr, seqno, dw, i);	508	i = emit_store_imm_ggtt(saddr, seqno, dw, i);
@@ -519,7 +589,11 @@ static const struct xe_ring_ops ring_ops_gen12_copy = { · 80 unmodified lines
519	.emit_job = emit_job_gen12_copy,	589	.emit_job = emit_job_gen12_copy,
520	};	590	};
521		591
522	static const struct xe_ring_ops ring_ops_gen12_video = {	592	static const struct xe_ring_ops ring_ops_gen12_video_decode = {
		593	.emit_job = emit_job_gen12_video,
		594	};
		595
		596	static const struct xe_ring_ops ring_ops_gen12_video_enhance = {
523	.emit_job = emit_job_gen12_video,	597	.emit_job = emit_job_gen12_video,
524	};	598	};
525		599
@@ -527,20 +601,47 @@ static const struct xe_ring_ops ring_ops_gen12_render_compute = { · 1 unmodified line
527	.emit_job = emit_job_gen12_render_compute,	601	.emit_job = emit_job_gen12_render_compute,
528	};	602	};
529		603
		604	static const struct xe_ring_ops auxccs_ring_ops_gen12_video_decode = {
		605	.emit_job = emit_job_gen12_video,
		606	.emit_aux_table_inv = emit_aux_table_inv_video_decode,
		607	};
		608
		609	static const struct xe_ring_ops auxccs_ring_ops_gen12_video_enhance = {
		610	.emit_job = emit_job_gen12_video,
		611	.emit_aux_table_inv = emit_aux_table_inv_video_enhance,
		612	};
		613
		614	static const struct xe_ring_ops auxccs_ring_ops_gen12_render_compute = {
		615	.emit_job = emit_job_gen12_render_compute,
		616	.emit_aux_table_inv = emit_aux_table_inv_render_compute,
		617	};
		618
530	const struct xe_ring_ops *	619	const struct xe_ring_ops *
531	xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)	620	xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
532	{	621	{
		622	struct xe_device *xe = gt_to_xe(gt);
		623
533	switch (class) {	624	switch (class) {
534	case XE_ENGINE_CLASS_OTHER:	625	case XE_ENGINE_CLASS_OTHER:
535	return &ring_ops_gen12_gsc;	626	return &ring_ops_gen12_gsc;
536	case XE_ENGINE_CLASS_COPY:	627	case XE_ENGINE_CLASS_COPY:
537	return &ring_ops_gen12_copy;	628	return &ring_ops_gen12_copy;
538	case XE_ENGINE_CLASS_VIDEO_DECODE:	629	case XE_ENGINE_CLASS_VIDEO_DECODE:
		630	if (has_aux_ccs(xe))
		631	return &auxccs_ring_ops_gen12_video_decode;
		632	else
		633	return &ring_ops_gen12_video_decode;
539	case XE_ENGINE_CLASS_VIDEO_ENHANCE:	634	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
540	return &ring_ops_gen12_video;	635	if (has_aux_ccs(xe))
		636	return &auxccs_ring_ops_gen12_video_enhance;
		637	else
		638	return &ring_ops_gen12_video_enhance;
541	case XE_ENGINE_CLASS_RENDER:	639	case XE_ENGINE_CLASS_RENDER:
542	case XE_ENGINE_CLASS_COMPUTE:	640	case XE_ENGINE_CLASS_COMPUTE:
543	return &ring_ops_gen12_render_compute;	641	if (has_aux_ccs(xe))
		642	return &auxccs_ring_ops_gen12_render_compute;
		643	else
		644	return &ring_ops_gen12_render_compute;
544	default:	645	default:
545	return NULL;	646	return NULL;
546	}	647	}

drivers/gpu/drm/xe/xe_ring_ops_types.h+7−1

 #ifndef _XE_RING_OPS_TYPES_H_
 #define _XE_RING_OPS_TYPES_H_
+#include <linux/types.h>
+struct xe_gt;
 struct xe_sched_job;
-#define MAX_JOB_SIZE_DW 58
+#define MAX_JOB_SIZE_DW 74
 #define MAX_JOB_SIZE_BYTES (MAX_JOB_SIZE_DW * 4)
 /**
 struct xe_ring_ops {
 	/** @emit_job: Write job to ring */
 	void (*emit_job)(struct xe_sched_job *job);
+	/** @emit_aux_table_inv: Emit aux table invalidation to the ring */
+	u32 *(*emit_aux_table_inv)(struct xe_gt *gt, u32 *cmd);
 };
 #endif

drivers/gpu/drm/xe/xe_sched_job_types.h+2−0

 	u64 sample_timestamp;
 	/** @ring_ops_flush_tlb: The ring ops need to flush TLB before payload. */
 	bool ring_ops_flush_tlb;
+	/** @ring_ops_force_reset: The ring ops need to trigger a reset before payload. */
+	bool ring_ops_force_reset;
 	/** @ggtt: mapped in ggtt. */
 	bool ggtt;
 	/** @restore_replay: job being replayed for restore */

drivers/gpu/drm/xe/xe_sriov_packet.c+2−0

@@ -341,6 +341,8 @@ ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, · 340 unmodified lines
341	ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data);	341	ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data);
342	if (ret) {	342	if (ret) {
343	xe_sriov_packet_free(*data);	343	xe_sriov_packet_free(*data);
		344	*data = NULL;
		345
344	return ret;	346	return ret;
345	}	347	}
346		348

drivers/gpu/drm/xe/xe_sriov_pf_control.c+24−0

@@ -123,6 +123,30 @@ int xe_sriov_pf_control_reset_vf(struct xe_device *xe, unsigned int vfid) · 122 unmodified lines
123	return result;	123	return result;
124	}	124	}
125		125
		126	/**
		127	* xe_sriov_pf_control_prepare_flr() - Notify PF that VF FLR prepare has started.
		128	* @xe: the &xe_device
		129	* @vfid: the VF identifier
		130	*
		131	* This function is for PF only.
		132	*
		133	* Return: 0 on success or a negative error code on failure.
		134	*/
		135	int xe_sriov_pf_control_prepare_flr(struct xe_device *xe, unsigned int vfid)
		136	{
		137	struct xe_gt *gt;
		138	unsigned int id;
		139	int result = 0;
		140	int err;
		141
		142	for_each_gt(gt, xe, id) {
		143	err = xe_gt_sriov_pf_control_prepare_flr(gt, vfid);
		144	result = result ? -EUCLEAN : err;
		145	}
		146
		147	return result;
		148	}
		149
126	/**	150	/**
127	* xe_sriov_pf_control_wait_flr() - Wait for a VF reset (FLR) to complete.	151	* xe_sriov_pf_control_wait_flr() - Wait for a VF reset (FLR) to complete.
128	* @xe: the &xe_device	152	* @xe: the &xe_device

drivers/gpu/drm/xe/xe_sriov_pf_control.h+1−0

@@ -12,6 +12,7 @@ int xe_sriov_pf_control_pause_vf(struct xe_device *xe, unsigned int vfid); · 11 unmodified lines
12	int xe_sriov_pf_control_resume_vf(struct xe_device *xe, unsigned int vfid);	12	int xe_sriov_pf_control_resume_vf(struct xe_device *xe, unsigned int vfid);
13	int xe_sriov_pf_control_stop_vf(struct xe_device *xe, unsigned int vfid);	13	int xe_sriov_pf_control_stop_vf(struct xe_device *xe, unsigned int vfid);
14	int xe_sriov_pf_control_reset_vf(struct xe_device *xe, unsigned int vfid);	14	int xe_sriov_pf_control_reset_vf(struct xe_device *xe, unsigned int vfid);
		15	int xe_sriov_pf_control_prepare_flr(struct xe_device *xe, unsigned int vfid);
15	int xe_sriov_pf_control_wait_flr(struct xe_device *xe, unsigned int vfid);	16	int xe_sriov_pf_control_wait_flr(struct xe_device *xe, unsigned int vfid);
16	int xe_sriov_pf_control_sync_flr(struct xe_device *xe, unsigned int vfid);	17	int xe_sriov_pf_control_sync_flr(struct xe_device *xe, unsigned int vfid);
17	int xe_sriov_pf_control_trigger_save_vf(struct xe_device *xe, unsigned int vfid);	18	int xe_sriov_pf_control_trigger_save_vf(struct xe_device *xe, unsigned int vfid);

drivers/gpu/drm/xe/xe_sriov_vfio.c+1−0

@@ -42,6 +42,7 @@ _type xe_sriov_vfio_##_func(struct xe_device *xe, unsigned int vfid) \ · 41 unmodified lines
42	EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_##_func, "xe-vfio-pci")	42	EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_##_func, "xe-vfio-pci")
43		43
44	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, wait_flr_done, control_wait_flr);	44	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, wait_flr_done, control_wait_flr);
		45	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, flr_prepare, control_prepare_flr);
45	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, suspend_device, control_pause_vf);	46	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, suspend_device, control_pause_vf);
46	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_device, control_resume_vf);	47	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_device, control_resume_vf);
47	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_enter, control_trigger_save_vf);	48	DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_enter, control_trigger_save_vf);

drivers/gpu/drm/xe/xe_svm.c+25−2

@@ -485,10 +485,33 @@ static void xe_svm_copy_kb_stats_incr(struct xe_gt *gt, · 484 unmodified lines
485	const enum xe_svm_copy_dir dir,	485	const enum xe_svm_copy_dir dir,
486	int kb)	486	int kb)
487	{	487	{
488	if (dir == XE_SVM_COPY_TO_VRAM)	488	if (dir == XE_SVM_COPY_TO_VRAM) {
		489	switch (kb) {
		490	case 4:
		491	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_KB, kb);
		492	break;
		493	case 64:
		494	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_KB, kb);
		495	break;
		496	case 2048:
		497	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_KB, kb);
		498	break;
		499	}
489	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_DEVICE_COPY_KB, kb);	500	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_DEVICE_COPY_KB, kb);
490	else	501	} else {
		502	switch (kb) {
		503	case 4:
		504	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_CPU_COPY_KB, kb);
		505	break;
		506	case 64:
		507	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_CPU_COPY_KB, kb);
		508	break;
		509	case 2048:
		510	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_CPU_COPY_KB, kb);
		511	break;
		512	}
491	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_CPU_COPY_KB, kb);	513	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_CPU_COPY_KB, kb);
		514	}
492	}	515	}
493		516
494	static void xe_svm_copy_us_stats_incr(struct xe_gt *gt,	517	static void xe_svm_copy_us_stats_incr(struct xe_gt *gt,

drivers/gpu/drm/xe/xe_uc.c+7−15

 	err = xe_gt_sriov_vf_connect(uc_to_gt(uc));
 	if (err)
-		goto err_out;
+		return err;
 	uc->guc.submission_state.enabled = true;
 	err = xe_guc_opt_in_features_enable(&uc->guc);
 	if (err)
-		goto err_out;
+		return err;
 	err = xe_gt_record_default_lrcs(uc_to_gt(uc));
 	if (err)
-		goto err_out;
+		return err;
 	return 0;
-err_out:
-	xe_guc_sanitize(&uc->guc);
-	return err;
 }
 /*
 	ret = xe_gt_record_default_lrcs(uc_to_gt(uc));
 	if (ret)
-		goto err_out;
+		return ret;
 	ret = xe_guc_post_load_init(&uc->guc);
 	if (ret)
-		goto err_out;
+		return ret;
 	ret = xe_guc_pc_start(&uc->guc.pc);
 	if (ret)
-		goto err_out;
+		return ret;
 	ret = xe_guc_rc_enable(&uc->guc);
 	if (ret)
-		goto err_out;
+		return ret;
 	xe_guc_engine_activity_enable_stats(&uc->guc);
 	xe_gsc_load_start(&uc->gsc);
 	return 0;
-err_out:
-	xe_guc_sanitize(&uc->guc);
-	return ret;
 }
 int xe_uc_reset_prepare(struct xe_uc *uc)

drivers/gpu/drm/xe/xe_vm.c+218−5

@@ -27,6 +27,7 @@ · 26 unmodified lines
27	#include "xe_device.h"	27	#include "xe_device.h"
28	#include "xe_drm_client.h"	28	#include "xe_drm_client.h"
29	#include "xe_exec_queue.h"	29	#include "xe_exec_queue.h"
		30	#include "xe_gt.h"
30	#include "xe_migrate.h"	31	#include "xe_migrate.h"
31	#include "xe_pat.h"	32	#include "xe_pat.h"
32	#include "xe_pm.h"	33	#include "xe_pm.h"
@@ -577,6 +578,74 @@ out_unlock_outer: · 544 unmodified lines
577	trace_xe_vm_rebind_worker_exit(vm);	578	trace_xe_vm_rebind_worker_exit(vm);
578	}	579	}
579		580
		581	/**
		582	* xe_vm_add_fault_entry_pf() - Add pagefault to vm fault list
		583	* @vm: The VM.
		584	* @pf: The pagefault.
		585	*
		586	* This function takes the data from the pagefault @pf and saves it to @vm->faults.list.
		587	*
		588	* The function exits silently if the list is full, and reports a warning if the pagefault
		589	* could not be saved to the list.
		590	*/
		591	void xe_vm_add_fault_entry_pf(struct xe_vm vm, struct xe_pagefault pf)
		592	{
		593	struct xe_vm_fault_entry *e;
		594	struct xe_hw_engine *hwe;
		595
		596	/* Do not report faults on reserved engines */
		597	hwe = xe_gt_hw_engine(pf->gt, pf->consumer.engine_class,
		598	pf->consumer.engine_instance, false);
		599	if (!hwe \|\| xe_hw_engine_is_reserved(hwe))
		600	return;
		601
		602	e = kzalloc_obj(*e);
		603	if (!e) {
		604	drm_warn(&vm->xe->drm,
		605	"Could not allocate memory for fault!\n");
		606	return;
		607	}
		608
		609	guard(spinlock)(&vm->faults.lock);
		610
		611	/*
		612	* Limit the number of faults in the fault list to prevent
		613	* memory overuse.
		614	*/
		615	if (vm->faults.len >= MAX_FAULTS_SAVED_PER_VM) {
		616	kfree(e);
		617	return;
		618	}
		619
		620	e->address = pf->consumer.page_addr;
		621	/*
		622	* TODO:
		623	* Address precision is currently always SZ_4K, but this may change
		624	* in the future.
		625	*/
		626	e->address_precision = SZ_4K;
		627	e->access_type = pf->consumer.access_type;
		628	e->fault_type = FIELD_GET(XE_PAGEFAULT_TYPE_MASK,
		629	pf->consumer.fault_type_level),
		630	e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK,
		631	pf->consumer.fault_type_level),
		632
		633	list_add_tail(&e->list, &vm->faults.list);
		634	vm->faults.len++;
		635	}
		636
		637	static void xe_vm_clear_fault_entries(struct xe_vm *vm)
		638	{
		639	struct xe_vm_fault_entry e, tmp;
		640
		641	guard(spinlock)(&vm->faults.lock);
		642	list_for_each_entry_safe(e, tmp, &vm->faults.list, list) {
		643	list_del(&e->list);
		644	kfree(e);
		645	}
		646	vm->faults.len = 0;
		647	}
		648
580	static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds)	649	static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds)
581	{	650	{
582	int i;	651	int i;
@@ -1538,6 +1607,9 @@ struct xe_vm xe_vm_create(struct xe_device xe, u32 flags, struct xe_file *xef) · 955 unmodified lines
1538	INIT_LIST_HEAD(&vm->userptr.invalidated);	1607	INIT_LIST_HEAD(&vm->userptr.invalidated);
1539	spin_lock_init(&vm->userptr.invalidated_lock);	1608	spin_lock_init(&vm->userptr.invalidated_lock);
1540		1609
		1610	INIT_LIST_HEAD(&vm->faults.list);
		1611	spin_lock_init(&vm->faults.lock);
		1612
1541	ttm_lru_bulk_move_init(&vm->lru_bulk_move);	1613	ttm_lru_bulk_move_init(&vm->lru_bulk_move);
1542		1614
1543	INIT_WORK(&vm->destroy_work, vm_destroy_work_func);	1615	INIT_WORK(&vm->destroy_work, vm_destroy_work_func);
@@ -1854,6 +1926,8 @@ void xe_vm_close_and_put(struct xe_vm *vm) · 310 unmodified lines
1854	}	1926	}
1855	up_write(&xe->usm.lock);	1927	up_write(&xe->usm.lock);
1856		1928
		1929	xe_vm_clear_fault_entries(vm);
		1930
1857	for_each_tile(tile, xe, id)	1931	for_each_tile(tile, xe, id)
1858	xe_range_fence_tree_fini(&vm->rftree[id]);	1932	xe_range_fence_tree_fini(&vm->rftree[id]);
1859		1933
@@ -2584,7 +2658,6 @@ static int xe_vma_op_commit(struct xe_vm vm, struct xe_vma_op op) · 724 unmodified lines
2584	if (!err && op->remap.skip_prev) {	2658	if (!err && op->remap.skip_prev) {
2585	op->remap.prev->tile_present =	2659	op->remap.prev->tile_present =
2586	tile_present;	2660	tile_present;
2587	op->remap.prev = NULL;
2588	}	2661	}
2589	}	2662	}
2590	if (op->remap.next) {	2663	if (op->remap.next) {
@@ -2594,11 +2667,13 @@ static int xe_vma_op_commit(struct xe_vm vm, struct xe_vma_op op) · 3 unmodified lines
2594	if (!err && op->remap.skip_next) {	2667	if (!err && op->remap.skip_next) {
2595	op->remap.next->tile_present =	2668	op->remap.next->tile_present =
2596	tile_present;	2669	tile_present;
2597	op->remap.next = NULL;
2598	}	2670	}
2599	}	2671	}
2600		2672
2601	/* Adjust for partial unbind after removing VMA from VM */	2673	/*
		2674	* Adjust for partial unbind after removing VMA from VM. In case
		2675	* of unwind we might need to undo this later.
		2676	*/
2602	if (!err) {	2677	if (!err) {
2603	op->base.remap.unmap->va->va.addr = op->remap.start;	2678	op->base.remap.unmap->va->va.addr = op->remap.start;
2604	op->base.remap.unmap->va->va.range = op->remap.range;	2679	op->base.remap.unmap->va->va.range = op->remap.range;
@@ -2717,6 +2792,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm vm, struct drm_gpuva_ops ops, · 112 unmodified lines
2717		2792
2718	op->remap.start = xe_vma_start(old);	2793	op->remap.start = xe_vma_start(old);
2719	op->remap.range = xe_vma_size(old);	2794	op->remap.range = xe_vma_size(old);
		2795	op->remap.old_start = op->remap.start;
		2796	op->remap.old_range = op->remap.range;
2720		2797
2721	flags \|= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK;	2798	flags \|= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK;
2722	if (op->base.remap.prev) {	2799	if (op->base.remap.prev) {
@@ -2865,8 +2942,19 @@ static void xe_vma_op_unwind(struct xe_vm vm, struct xe_vma_op op, · 142 unmodified lines
2865	xe_svm_notifier_lock(vm);	2942	xe_svm_notifier_lock(vm);
2866	vma->gpuva.flags &= ~XE_VMA_DESTROYED;	2943	vma->gpuva.flags &= ~XE_VMA_DESTROYED;
2867	xe_svm_notifier_unlock(vm);	2944	xe_svm_notifier_unlock(vm);
2868	if (post_commit)	2945	if (post_commit) {
		2946	/*
		2947	* Restore the old va range, in case of the
		2948	* prev/next skip optimisation. Otherwise what
		2949	* we re-insert here could be smaller than the
		2950	* original range.
		2951	*/
		2952	op->base.remap.unmap->va->va.addr =
		2953	op->remap.old_start;
		2954	op->base.remap.unmap->va->va.range =
		2955	op->remap.old_range;
2869	xe_vm_insert_vma(vm, vma);	2956	xe_vm_insert_vma(vm, vma);
		2957	}
2870	}	2958	}
2871	break;	2959	break;
2872	}	2960	}
@@ -3465,7 +3553,7 @@ static int vm_bind_ioctl_check_args(struct xe_device xe, struct xe_vm vm, · 592 unmodified lines
3465	goto free_bind_ops;	3553	goto free_bind_ops;
3466	}	3554	}
3467		3555
3468	if (XE_WARN_ON(coh_mode > XE_COH_AT_LEAST_1WAY)) {	3556	if (XE_WARN_ON(coh_mode > XE_COH_2WAY)) {
3469	err = -EINVAL;	3557	err = -EINVAL;
3470	goto free_bind_ops;	3558	goto free_bind_ops;
3471	}	3559	}
@@ -3492,6 +3580,10 @@ static int vm_bind_ioctl_check_args(struct xe_device xe, struct xe_vm vm, · 20 unmodified lines
3492	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|	3580	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|
3493	XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&	3581	XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE &&
3494	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|	3582	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|
		3583	XE_IOCTL_DBG(xe, xe_device_is_l2_flush_optimized(xe) &&
		3584	(op == DRM_XE_VM_BIND_OP_MAP_USERPTR \|\|
		3585	is_cpu_addr_mirror) &&
		3586	(pat_index != 19 && coh_mode != XE_COH_2WAY)) \|\|
3495	XE_IOCTL_DBG(xe, comp_en &&	3587	XE_IOCTL_DBG(xe, comp_en &&
3496	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|	3588	op == DRM_XE_VM_BIND_OP_MAP_USERPTR) \|\|
3497	XE_IOCTL_DBG(xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR &&	3589	XE_IOCTL_DBG(xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR &&
@@ -3633,6 +3725,10 @@ static int xe_vm_bind_ioctl_validate_bo(struct xe_device xe, struct xe_bo bo, · 135 unmodified lines
3633	if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && comp_en))	3725	if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && comp_en))
3634	return -EINVAL;	3726	return -EINVAL;
3635		3727
		3728	if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && xe_device_is_l2_flush_optimized(xe) &&
		3729	(pat_index != 19 && coh_mode != XE_COH_2WAY)))
		3730	return -EINVAL;
		3731
3636	/* If a BO is protected it can only be mapped if the key is still valid */	3732	/* If a BO is protected it can only be mapped if the key is still valid */
3637	if ((bind_flags & DRM_XE_VM_BIND_FLAG_CHECK_PXP) && xe_bo_is_protected(bo) &&	3733	if ((bind_flags & DRM_XE_VM_BIND_FLAG_CHECK_PXP) && xe_bo_is_protected(bo) &&
3638	op != DRM_XE_VM_BIND_OP_UNMAP && op != DRM_XE_VM_BIND_OP_UNMAP_ALL)	3734	op != DRM_XE_VM_BIND_OP_UNMAP && op != DRM_XE_VM_BIND_OP_UNMAP_ALL)
@@ -3878,6 +3974,123 @@ put_vm: · 239 unmodified lines
3878	return err;	3974	return err;
3879	}	3975	}
3880		3976
		3977	/*
		3978	* Map access type, fault type, and fault level from current bspec
		3979	* specification to user spec abstraction. The current mapping is
		3980	* approximately 1-to-1, with access type being the only notable
		3981	* exception as it carries additional data with respect to prefetch
		3982	* status that needs to be masked out.
		3983	*/
		3984	static u8 xe_to_user_access_type(u8 access_type)
		3985	{
		3986	return access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK;
		3987	}
		3988
		3989	static u8 xe_to_user_fault_type(u8 fault_type)
		3990	{
		3991	return fault_type;
		3992	}
		3993
		3994	static u8 xe_to_user_fault_level(u8 fault_level)
		3995	{
		3996	return fault_level;
		3997	}
		3998
		3999	static int fill_faults(struct xe_vm *vm,
		4000	struct drm_xe_vm_get_property *args)
		4001	{
		4002	struct xe_vm_fault __user *usr_ptr = u64_to_user_ptr(args->data);
		4003	struct xe_vm_fault *fault_list, fault_entry = { 0 };
		4004	struct xe_vm_fault_entry *entry;
		4005	int ret = 0, i = 0, count, entry_size;
		4006
		4007	entry_size = sizeof(struct xe_vm_fault);
		4008	count = args->size / entry_size;
		4009
		4010	fault_list = kcalloc(count, sizeof(struct xe_vm_fault), GFP_KERNEL);
		4011	if (!fault_list)
		4012	return -ENOMEM;
		4013
		4014	spin_lock(&vm->faults.lock);
		4015	list_for_each_entry(entry, &vm->faults.list, list) {
		4016	if (i == count)
		4017	break;
		4018
		4019	fault_entry.address = xe_device_canonicalize_addr(vm->xe, entry->address);
		4020	fault_entry.address_precision = entry->address_precision;
		4021
		4022	fault_entry.access_type = xe_to_user_access_type(entry->access_type);
		4023	fault_entry.fault_type = xe_to_user_fault_type(entry->fault_type);
		4024	fault_entry.fault_level = xe_to_user_fault_level(entry->fault_level);
		4025
		4026	memcpy(&fault_list[i], &fault_entry, entry_size);
		4027
		4028	i++;
		4029	}
		4030	spin_unlock(&vm->faults.lock);
		4031
		4032	ret = copy_to_user(usr_ptr, fault_list, args->size);
		4033
		4034	kfree(fault_list);
		4035	return ret ? -EFAULT : 0;
		4036	}
		4037
		4038	static int xe_vm_get_property_helper(struct xe_vm *vm,
		4039	struct drm_xe_vm_get_property *args)
		4040	{
		4041	size_t size;
		4042
		4043	switch (args->property) {
		4044	case DRM_XE_VM_GET_PROPERTY_FAULTS:
		4045	spin_lock(&vm->faults.lock);
		4046	size = size_mul(sizeof(struct xe_vm_fault), vm->faults.len);
		4047	spin_unlock(&vm->faults.lock);
		4048
		4049	if (!args->size) {
		4050	args->size = size;
		4051	return 0;
		4052	}
		4053
		4054	/*
		4055	* Number of faults may increase between calls to
		4056	* xe_vm_get_property_ioctl, so just report the number of
		4057	* faults the user requests if it's less than or equal to
		4058	* the number of faults in the VM fault array.
		4059	*
		4060	* We should also at least assert that the args->size value
		4061	* is a multiple of the xe_vm_fault struct size.
		4062	*/
		4063	if (args->size > size \|\| args->size % sizeof(struct xe_vm_fault))
		4064	return -EINVAL;
		4065
		4066	return fill_faults(vm, args);
		4067	}
		4068	return -EINVAL;
		4069	}
		4070
		4071	int xe_vm_get_property_ioctl(struct drm_device drm, void data,
		4072	struct drm_file *file)
		4073	{
		4074	struct xe_device *xe = to_xe_device(drm);
		4075	struct xe_file *xef = to_xe_file(file);
		4076	struct drm_xe_vm_get_property *args = data;
		4077	struct xe_vm *vm;
		4078	int ret = 0;
		4079
		4080	if (XE_IOCTL_DBG(xe, (args->reserved[0] \|\| args->reserved[1] \|\|
		4081	args->reserved[2])))
		4082	return -EINVAL;
		4083
		4084	vm = xe_vm_lookup(xef, args->vm_id);
		4085	if (XE_IOCTL_DBG(xe, !vm))
		4086	return -ENOENT;
		4087
		4088	ret = xe_vm_get_property_helper(vm, args);
		4089
		4090	xe_vm_put(vm);
		4091	return ret;
		4092	}
		4093
3881	/**	4094	/**
3882	* xe_vm_bind_kernel_bo - bind a kernel BO to a VM	4095	* xe_vm_bind_kernel_bo - bind a kernel BO to a VM
3883	* @vm: VM to bind the BO to	4096	* @vm: VM to bind the BO to

drivers/gpu/drm/xe/xe_vm.h+12−0

 #include "xe_map.h"
 #include "xe_vm_types.h"
+/**
+ * MAX_FAULTS_SAVED_PER_VM - Maximum number of faults each vm can store before future
+ * faults are discarded to prevent memory overuse
+ */
+#define MAX_FAULTS_SAVED_PER_VM	50
 struct drm_device;
 struct drm_printer;
 struct drm_file;
 struct xe_exec_queue;
 struct xe_file;
+struct xe_pagefault;
 struct xe_sync_entry;
 struct xe_svm_range;
 struct drm_exec;
 int xe_vm_bind_ioctl(struct drm_device *dev, void *data,
 		     struct drm_file *file);
 int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
+int xe_vm_get_property_ioctl(struct drm_device *dev, void *data,
+			     struct drm_file *file);
 void xe_vm_close_and_put(struct xe_vm *vm);
 static inline bool xe_vm_in_fault_mode(struct xe_vm *vm)
 void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p);
 void xe_vm_snapshot_free(struct xe_vm_snapshot *snap);
+void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf);
 /**
  * xe_vm_set_validating() - Register this task as currently making bos resident
  * @allow_res_evict: Allow eviction of buffer objects bound to @vm when

drivers/gpu/drm/xe/xe_vm_madvise.c+24−1

 		if (XE_IOCTL_DBG(xe, !coh_mode))
 			return false;
-		if (XE_WARN_ON(coh_mode > XE_COH_AT_LEAST_1WAY))
+		if (XE_WARN_ON(coh_mode > XE_COH_2WAY))
 			return false;
 		if (XE_IOCTL_DBG(xe, args->pat_index.pad))
 	struct xe_vmas_in_madvise_range madvise_range = {.addr = args->start,
 							 .range =  args->range, };
 	struct xe_madvise_details details;
+	u16 pat_index, coh_mode;
 	struct xe_vm *vm;
 	struct drm_exec exec;
 	int err, attr_type;
 	if (err || !madvise_range.num_vmas)
 		goto madv_fini;
+	if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) {
+		pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries);
+		coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
+		if (XE_IOCTL_DBG(xe, madvise_range.has_svm_userptr_vmas &&
+				 xe_device_is_l2_flush_optimized(xe) &&
+				 (pat_index != 19 && coh_mode != XE_COH_2WAY))) {
+			err = -EINVAL;
+			goto madv_fini;
+		}
+	}
 	if (madvise_range.has_bo_vmas) {
 		if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) {
 			if (!check_bo_args_are_sane(vm, madvise_range.vmas,
 				if (!bo)
 					continue;
+				if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) {
+					if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach &&
+							 xe_device_is_l2_flush_optimized(xe) &&
+							 (pat_index != 19 &&
+							  coh_mode != XE_COH_2WAY))) {
+						err = -EINVAL;
+						goto err_fini;
+					}
+				}
 				err = drm_exec_lock_obj(&exec, &bo->ttm.base);
 				drm_exec_retry_on_contention(&exec);
 				if (err)

drivers/gpu/drm/xe/xe_vm_types.h+33−0

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	struct drm_pagemap;	24	struct drm_pagemap;
25		25
26	struct xe_bo;	26	struct xe_bo;
		27	struct xe_pagefault;
27	struct xe_svm_range;	28	struct xe_svm_range;
28	struct xe_sync_entry;	29	struct xe_sync_entry;
29	struct xe_user_fence;	30	struct xe_user_fence;
@@ -176,6 +177,24 @@ struct xe_userptr_vma { · 146 unmodified lines
176		177
177	struct xe_device;	178	struct xe_device;
178		179
		180	/**
		181	* struct xe_vm_fault_entry - Elements of vm->faults.list
		182	* @list: link into @xe_vm.faults.list
		183	* @address: address of the fault
		184	* @address_precision: precision of faulted address
		185	* @access_type: type of address access that resulted in fault
		186	* @fault_type: type of fault reported
		187	* @fault_level: fault level of the fault
		188	*/
		189	struct xe_vm_fault_entry {
		190	struct list_head list;
		191	u64 address;
		192	u32 address_precision;
		193	u8 access_type;
		194	u8 fault_type;
		195	u8 fault_level;
		196	};
		197
179	struct xe_vm {	198	struct xe_vm {
180	/** @gpuvm: base GPUVM used to track VMAs */	199	/** @gpuvm: base GPUVM used to track VMAs */
181	struct drm_gpuvm gpuvm;	200	struct drm_gpuvm gpuvm;
@@ -333,6 +352,16 @@ struct xe_vm { · 151 unmodified lines
333	bool capture_once;	352	bool capture_once;
334	} error_capture;	353	} error_capture;
335		354
		355	/** @faults: List of all faults associated with this VM */
		356	struct {
		357	/** @faults.lock: lock protecting @faults.list */
		358	spinlock_t lock;
		359	/** @faults.list: list of xe_vm_fault_entry entries */
		360	struct list_head list;
		361	/** @faults.len: length of @faults.list */
		362	unsigned int len;
		363	} faults;
		364
336	/**	365	/**
337	* @validation: Validation data only valid with the vm resv held.	366	* @validation: Validation data only valid with the vm resv held.
338	* Note: This is really task state of the task holding the vm resv,	367	* Note: This is really task state of the task holding the vm resv,
@@ -393,6 +422,10 @@ struct xe_vma_op_remap { · 54 unmodified lines
393	u64 start;	422	u64 start;
394	/** @range: range of the VMA unmap */	423	/** @range: range of the VMA unmap */
395	u64 range;	424	u64 range;
		425	/** @old_start: Original start of the VMA we unmap */
		426	u64 old_start;
		427	/** @old_range: Original range of the VMA we unmap */
		428	u64 old_range;
396	/** @skip_prev: skip prev rebind */	429	/** @skip_prev: skip prev rebind */
397	bool skip_prev;	430	bool skip_prev;
398	/** @skip_next: skip next rebind */	431	/** @skip_next: skip next rebind */

drivers/gpu/drm/xe/xe_wa.c+7−20

 				   LSN_DIM_Z_WGT_MASK,
 				   LSN_LNI_WGT(1) | LSN_LNE_WGT(1) |
 				   LSN_DIM_X_WGT(1) | LSN_DIM_Y_WGT(1) |
-				   LSN_DIM_Z_WGT(1)))
+				   LSN_DIM_Z_WGT(1)),
-	},
+			SET(LSC_CHICKEN_BIT_0_UDW, L3_128B_256B_WRT_DIS))
-	/* Xe2_HPM */
-	{ XE_RTP_NAME("16021867713"),
-	  XE_RTP_RULES(MEDIA_VERSION(1301),
-		       ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F1C(0), MFXPIPE_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
-	},
-	{ XE_RTP_NAME("14019449301"),
-	  XE_RTP_RULES(MEDIA_VERSION(1301), ENGINE_CLASS(VIDEO_DECODE)),
-	  XE_RTP_ACTIONS(SET(VDBOX_CGCTL3F08(0), CG3DDISHRS_CLKGATE_DIS)),
-	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 	},
 	/* Xe3_LPG */
 	  XE_RTP_ACTIONS(SET(MMIOATSREQLIMIT_GAM_WALK_3D,
 			     DIS_ATS_WRONLY_PG))
 	},
-	{ XE_RTP_NAME("14026144927"),
+	{ XE_RTP_NAME("14026144927, 16029437861"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)),
 	  XE_RTP_ACTIONS(SET(L3SQCREG2, L3_SQ_DISABLE_COAMA_2WAY_COH |
 			     L3_SQ_DISABLE_COAMA))
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX))
 	},
+	{ XE_RTP_NAME("14026781792"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3510), ENGINE_CLASS(RENDER)),
+	  XE_RTP_ACTIONS(SET(FF_MODE, DIS_TE_PATCH_CTRL))
+	},
 	/* DG1 */
 		       ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_CLIP_NEGATIVE_BOUNDING_BOX))
 	},
-	{ XE_RTP_NAME("14026781792"),
-	  XE_RTP_RULES(GRAPHICS_VERSION(3510), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(FF_MODE, DIS_TE_PATCH_CTRL))
-	},
 };
 static __maybe_unused const struct xe_rtp_entry oob_was[] = {

drivers/vfio/pci/xe/main.c+14−0

@@ -85,6 +85,19 @@ again: · 84 unmodified lines
85	spin_unlock(&xe_vdev->reset_lock);	85	spin_unlock(&xe_vdev->reset_lock);
86	}	86	}
87		87
		88	static void xe_vfio_pci_reset_prepare(struct pci_dev *pdev)
		89	{
		90	struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev);
		91	int ret;
		92
		93	if (!pdev->is_virtfn)
		94	return;
		95
		96	ret = xe_sriov_vfio_flr_prepare(xe_vdev->xe, xe_vdev->vfid);
		97	if (ret)
		98	dev_err(&pdev->dev, "Failed to prepare FLR: %d\n", ret);
		99	}
		100
88	static void xe_vfio_pci_reset_done(struct pci_dev *pdev)	101	static void xe_vfio_pci_reset_done(struct pci_dev *pdev)
89	{	102	{
90	struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev);	103	struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev);
@@ -127,6 +140,7 @@ static void xe_vfio_pci_reset_done(struct pci_dev *pdev) · 36 unmodified lines
127	}	140	}
128		141
129	static const struct pci_error_handlers xe_vfio_pci_err_handlers = {	142	static const struct pci_error_handlers xe_vfio_pci_err_handlers = {
		143	.reset_prepare = xe_vfio_pci_reset_prepare,
130	.reset_done = xe_vfio_pci_reset_done,	144	.reset_done = xe_vfio_pci_reset_done,
131	.error_detected = vfio_pci_core_aer_err_detected,	145	.error_detected = vfio_pci_core_aer_err_detected,
132	};	146	};

include/drm/drm_pagemap.h+21−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4		4
5	#include <linux/dma-direction.h>	5	#include <linux/dma-direction.h>
6	#include <linux/hmm.h>	6	#include <linux/hmm.h>
		7	#include <linux/memremap.h>
7	#include <linux/types.h>	8	#include <linux/types.h>
8		9
9	#define NR_PAGES(order) (1U << (order))	10	#define NR_PAGES(order) (1U << (order))
@@ -367,6 +368,26 @@ void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim · 357 unmodified lines
367		368
368	int drm_pagemap_reinit(struct drm_pagemap *dpagemap);	369	int drm_pagemap_reinit(struct drm_pagemap *dpagemap);
369		370
		371	/**
		372	* drm_pagemap_page_zone_device_data() - Page to zone_device_data
		373	* @page: Pointer to the page
		374	*
		375	* Return: Page's zone_device_data
		376	*/
		377	static inline struct drm_pagemap_zdd drm_pagemap_page_zone_device_data(struct page page)
		378	{
		379	struct folio *folio = page_folio(page);
		380
		381	return folio_zone_device_data(folio);
		382	}
		383
		384	#else
		385
		386	static inline struct drm_pagemap_zdd drm_pagemap_page_zone_device_data(struct page page)
		387	{
		388	return NULL;
		389	}
		390
370	#endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */	391	#endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */
371		392
372	#endif	393	#endif

include/drm/intel/xe_sriov_vfio.h+11−0

@@ -27,6 +27,17 @@ struct xe_device xe_sriov_vfio_get_pf(struct pci_dev pdev); · 26 unmodified lines
27	*/	27	*/
28	bool xe_sriov_vfio_migration_supported(struct xe_device *xe);	28	bool xe_sriov_vfio_migration_supported(struct xe_device *xe);
29		29
		30	/**
		31	* xe_sriov_vfio_flr_prepare() - Notify PF that VF FLR prepare has started.
		32	* @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf()
		33	* @vfid: the VF identifier (can't be 0)
		34	*
		35	* This function marks VF FLR as pending before PF receives GuC FLR event.
		36	*
		37	* Return: 0 on success or a negative error code on failure.
		38	*/
		39	int xe_sriov_vfio_flr_prepare(struct xe_device *xe, unsigned int vfid);
		40
30	/**	41	/**
31	* xe_sriov_vfio_wait_flr_done() - Wait for VF FLR completion.	42	* xe_sriov_vfio_wait_flr_done() - Wait for VF FLR completion.
32	* @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf()	43	* @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf()

include/uapi/drm/xe_drm.h+90−2

@@ -83,6 +83,7 @@ extern "C" { · 82 unmodified lines
83	* - &DRM_IOCTL_XE_OBSERVATION	83	* - &DRM_IOCTL_XE_OBSERVATION
84	* - &DRM_IOCTL_XE_MADVISE	84	* - &DRM_IOCTL_XE_MADVISE
85	* - &DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS	85	* - &DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS
		86	* - &DRM_IOCTL_XE_VM_GET_PROPERTY
86	*/	87	*/
87		88
88	/*	89	/*
@@ -107,6 +108,7 @@ extern "C" { · 18 unmodified lines
107	#define DRM_XE_MADVISE 0x0c	108	#define DRM_XE_MADVISE 0x0c
108	#define DRM_XE_VM_QUERY_MEM_RANGE_ATTRS 0x0d	109	#define DRM_XE_VM_QUERY_MEM_RANGE_ATTRS 0x0d
109	#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x0e	110	#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x0e
		111	#define DRM_XE_VM_GET_PROPERTY 0x0f
110		112
111	/* Must be kept compact -- no holes */	113	/* Must be kept compact -- no holes */
112		114
@@ -125,6 +127,7 @@ extern "C" { · 12 unmodified lines
125	#define DRM_IOCTL_XE_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_MADVISE, struct drm_xe_madvise)	127	#define DRM_IOCTL_XE_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_MADVISE, struct drm_xe_madvise)
126	#define DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_QUERY_MEM_RANGE_ATTRS, struct drm_xe_vm_query_mem_range_attr)	128	#define DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_QUERY_MEM_RANGE_ATTRS, struct drm_xe_vm_query_mem_range_attr)
127	#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property)	129	#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property)
		130	#define DRM_IOCTL_XE_VM_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_GET_PROPERTY, struct drm_xe_vm_get_property)
128		131
129	/**	132	/**
130	* DOC: Xe IOCTL Extensions	133	* DOC: Xe IOCTL Extensions
@@ -1057,7 +1060,7 @@ struct drm_xe_vm_destroy { · 926 unmodified lines
1057	* not invoke autoreset. Neither will stack variables going out of scope.	1060	* not invoke autoreset. Neither will stack variables going out of scope.
1058	* Therefore it's recommended to always explicitly reset the madvises when	1061	* Therefore it's recommended to always explicitly reset the madvises when
1059	* freeing the memory backing a region used in a &DRM_IOCTL_XE_MADVISE call.	1062	* freeing the memory backing a region used in a &DRM_IOCTL_XE_MADVISE call.
1060	* - DRM_XE_VM_BIND_FLAG_DECOMPRESS - Request on-device decompression for a MAP.	1063	* - %DRM_XE_VM_BIND_FLAG_DECOMPRESS - Request on-device decompression for a MAP.
1061	* When set on a MAP bind operation, request the driver schedule an on-device	1064	* When set on a MAP bind operation, request the driver schedule an on-device
1062	* in-place decompression (via the migrate/resolve path) for the GPU mapping	1065	* in-place decompression (via the migrate/resolve path) for the GPU mapping
1063	* created by this bind. Only valid for DRM_XE_VM_BIND_OP_MAP; usage on	1066	* created by this bind. Only valid for DRM_XE_VM_BIND_OP_MAP; usage on
@@ -1114,7 +1117,9 @@ struct drm_xe_vm_bind_op { · 50 unmodified lines
1114	* incoherent GT access is possible.	1117	* incoherent GT access is possible.
1115	*	1118	*
1116	* Note: For userptr and externally imported dma-buf the kernel expects	1119	* Note: For userptr and externally imported dma-buf the kernel expects
1117	* either 1WAY or 2WAY for the @pat_index.	1120	* either 1WAY or 2WAY for the @pat_index. Starting from NVL-P, for
		1121	* userptr, svm, madvise and externally imported dma-buf the kernel expects
		1122	* either 2WAY or 1WAY and XA @pat_index.
1118	*	1123	*
1119	* For DRM_XE_VM_BIND_FLAG_NULL bindings there are no KMD restrictions	1124	* For DRM_XE_VM_BIND_FLAG_NULL bindings there are no KMD restrictions
1120	* on the @pat_index. For such mappings there is no actual memory being	1125	* on the @pat_index. For such mappings there is no actual memory being
@@ -1261,6 +1266,89 @@ struct drm_xe_vm_bind { · 140 unmodified lines
1261	__u64 reserved[2];	1266	__u64 reserved[2];
1262	};	1267	};
1263		1268
		1269	/** struct xe_vm_fault - Describes faults for %DRM_XE_VM_GET_PROPERTY_FAULTS */
		1270	struct xe_vm_fault {
		1271	/** @address: Canonical address of the fault */
		1272	__u64 address;
		1273	/** @address_precision: Precision of faulted address */
		1274	__u32 address_precision;
		1275	/** @access_type: Type of address access that resulted in fault */
		1276	#define FAULT_ACCESS_TYPE_READ 0
		1277	#define FAULT_ACCESS_TYPE_WRITE 1
		1278	#define FAULT_ACCESS_TYPE_ATOMIC 2
		1279	__u8 access_type;
		1280	/** @fault_type: Type of fault reported */
		1281	#define FAULT_TYPE_NOT_PRESENT 0
		1282	#define FAULT_TYPE_WRITE_ACCESS 1
		1283	#define FAULT_TYPE_ATOMIC_ACCESS 2
		1284	__u8 fault_type;
		1285	/** @fault_level: fault level of the fault */
		1286	#define FAULT_LEVEL_PTE 0
		1287	#define FAULT_LEVEL_PDE 1
		1288	#define FAULT_LEVEL_PDP 2
		1289	#define FAULT_LEVEL_PML4 3
		1290	#define FAULT_LEVEL_PML5 4
		1291	__u8 fault_level;
		1292	/** @pad: MBZ */
		1293	__u8 pad;
		1294	/** @reserved: MBZ */
		1295	__u64 reserved[4];
		1296	};
		1297
		1298	/**
		1299	* struct drm_xe_vm_get_property - Input of &DRM_IOCTL_XE_VM_GET_PROPERTY
		1300	*
		1301	* The user provides a VM and a property to query among DRM_XE_VM_GET_PROPERTY_*,
		1302	* and sets the values in the vm_id and property members, respectively. This
		1303	* determines both the VM to get the property of, as well as the property to
		1304	* report.
		1305	*
		1306	* If size is set to 0, the driver fills it with the required size for the
		1307	* requested property. The user is expected here to allocate memory for the
		1308	* property structure and to provide a pointer to the allocated memory using the
		1309	* data member. For some properties, this may be zero, in which case, the
		1310	* value of the property will be saved to the value member and size will remain
		1311	* zero on return.
		1312	*
		1313	* If size is not zero, then the IOCTL will attempt to copy the requested
		1314	* property into the data member.
		1315	*
		1316	* The IOCTL will return -ENOENT if the VM could not be identified from the
		1317	* provided VM ID, or -EINVAL if the IOCTL fails for any other reason, such as
		1318	* providing an invalid size for the given property or if the property data
		1319	* could not be copied to the memory allocated to the data member.
		1320	*
		1321	* The property member can be:
		1322	* - %DRM_XE_VM_GET_PROPERTY_FAULTS
		1323	*/
		1324	struct drm_xe_vm_get_property {
		1325	/** @extensions: Pointer to the first extension struct, if any */
		1326	__u64 extensions;
		1327
		1328	/** @vm_id: The ID of the VM to query the properties of */
		1329	__u32 vm_id;
		1330
		1331	#define DRM_XE_VM_GET_PROPERTY_FAULTS 0
		1332	/** @property: property to get */
		1333	__u32 property;
		1334
		1335	/** @size: Size to allocate for @data */
		1336	__u32 size;
		1337
		1338	/** @pad: MBZ */
		1339	__u32 pad;
		1340
		1341	union {
		1342	/** @data: Pointer to user-defined array of flexible size and type */
		1343	__u64 data;
		1344	/** @value: Return value for scalar queries */
		1345	__u64 value;
		1346	};
		1347
		1348	/** @reserved: MBZ */
		1349	__u64 reserved[3];
		1350	};
		1351
1264	/**	1352	/**
1265	* struct drm_xe_exec_queue_create - Input of &DRM_IOCTL_XE_EXEC_QUEUE_CREATE	1353	* struct drm_xe_exec_queue_create - Input of &DRM_IOCTL_XE_EXEC_QUEUE_CREATE
1266	*	1354	*

@@ -92,7 +92,7 @@ struct xe_pat_ops { · 91 unmodified lines
92	};	92	};
93		93
94	static const struct xe_pat_table_entry xelp_pat_table[] = {	94	static const struct xe_pat_table_entry xelp_pat_table[] = {
95	[0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },	95	[0] = { XELP_PAT_WB, XE_COH_1WAY },
96	[1] = { XELP_PAT_WC, XE_COH_NONE },	96	[1] = { XELP_PAT_WC, XE_COH_NONE },
97	[2] = { XELP_PAT_WT, XE_COH_NONE },	97	[2] = { XELP_PAT_WT, XE_COH_NONE },
98	[3] = { XELP_PAT_UC, XE_COH_NONE },	98	[3] = { XELP_PAT_UC, XE_COH_NONE },
@@ -102,19 +102,19 @@ static const struct xe_pat_table_entry xehpc_pat_table[] = { · 3 unmodified lines
102	[0] = { XELP_PAT_UC, XE_COH_NONE },	102	[0] = { XELP_PAT_UC, XE_COH_NONE },
103	[1] = { XELP_PAT_WC, XE_COH_NONE },	103	[1] = { XELP_PAT_WC, XE_COH_NONE },
104	[2] = { XELP_PAT_WT, XE_COH_NONE },	104	[2] = { XELP_PAT_WT, XE_COH_NONE },
105	[3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },	105	[3] = { XELP_PAT_WB, XE_COH_1WAY },
106	[4] = { XEHPC_PAT_CLOS(1) \| XELP_PAT_WT, XE_COH_NONE },	106	[4] = { XEHPC_PAT_CLOS(1) \| XELP_PAT_WT, XE_COH_NONE },
107	[5] = { XEHPC_PAT_CLOS(1) \| XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },	107	[5] = { XEHPC_PAT_CLOS(1) \| XELP_PAT_WB, XE_COH_1WAY },
108	[6] = { XEHPC_PAT_CLOS(2) \| XELP_PAT_WT, XE_COH_NONE },	108	[6] = { XEHPC_PAT_CLOS(2) \| XELP_PAT_WT, XE_COH_NONE },
109	[7] = { XEHPC_PAT_CLOS(2) \| XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },	109	[7] = { XEHPC_PAT_CLOS(2) \| XELP_PAT_WB, XE_COH_1WAY },
110	};	110	};
111		111
112	static const struct xe_pat_table_entry xelpg_pat_table[] = {	112	static const struct xe_pat_table_entry xelpg_pat_table[] = {
113	[0] = { XELPG_PAT_0_WB, XE_COH_NONE },	113	[0] = { XELPG_PAT_0_WB, XE_COH_NONE },
114	[1] = { XELPG_PAT_1_WT, XE_COH_NONE },	114	[1] = { XELPG_PAT_1_WT, XE_COH_NONE },
115	[2] = { XELPG_PAT_3_UC, XE_COH_NONE },	115	[2] = { XELPG_PAT_3_UC, XE_COH_NONE },
116	[3] = { XELPG_PAT_0_WB \| XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY },	116	[3] = { XELPG_PAT_0_WB \| XELPG_2_COH_1W, XE_COH_1WAY },
117	[4] = { XELPG_PAT_0_WB \| XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY },	117	[4] = { XELPG_PAT_0_WB \| XELPG_3_COH_2W, XE_COH_2WAY },
118	};	118	};
119		119
120	/*	120	/*
@@ -147,7 +147,7 @@ static const struct xe_pat_table_entry xelpg_pat_table[] = { · 26 unmodified lines
147	REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) \| \	147	REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) \| \
148	REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) \| \	148	REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) \| \
149	REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \	149	REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \
150	.coh_mode = __coh_mode ? XE_COH_AT_LEAST_1WAY : XE_COH_NONE, \	150	.coh_mode = __coh_mode ? __coh_mode : XE_COH_NONE, \
151	.valid = 1 \	151	.valid = 1 \
152	}	152	}
153		153