accel/ivpu: Add Intel VPU MMU support

VPU Memory Management Unit is based on ARM MMU-600. It allows the creation of multiple virtual address spaces for the device and map noncontinuous host memory (there is no dedicated memory on the VPU). Address space is implemented as a struct ivpu_mmu_context, it has an ID, drm_mm allocator for VPU addresses and struct ivpu_mmu_pgtable that holds actual 3-level, 4KB page table. Context with ID 0 (global context) is created upon driver initialization and it's mainly used for mapping memory required to execute the firmware. Contexts with non-zero IDs are user contexts allocated each time the devices is open()-ed and they map command buffers and other workload-related memory. Workloads executing in a given contexts have access only to the memory mapped in this context. This patch is has two main files: - ivpu_mmu_context.c handles MMU page tables and memory mapping - ivpu_mmu.c implements a driver that programs the MMU device

Jacek Lawrynowicz committed 3 years ago

Commit 263b2ba1 parent 35b13769 files · +1479 −3Browse files

Unified · Split

9 files

drivers/accel/ivpu/Makefile +3 −1
drivers/accel/ivpu/ivpu_drv.c +81 −2
drivers/accel/ivpu/ivpu_drv.h +7 −0
drivers/accel/ivpu/ivpu_hw_mtl.c +10 −0
drivers/accel/ivpu/ivpu_mmu.c +876 −0
drivers/accel/ivpu/ivpu_mmu.h +50 −0
drivers/accel/ivpu/ivpu_mmu_context.c +398 −0
drivers/accel/ivpu/ivpu_mmu_context.h +50 −0
include/uapi/drm/ivpu_accel.h +4 −0

drivers/accel/ivpu/Makefile+3−1

@@ -3,6 +3,8 @@ · 2 unmodified lines
3	3
4	4	intel_vpu-y := \
5	5	ivpu_drv.o \
6		− ivpu_hw_mtl.o
	6	+ ivpu_hw_mtl.o \
	7	+ ivpu_mmu.o \
	8	+ ivpu_mmu_context.o
7	9
8	10	obj-$(CONFIG_DRM_ACCEL_IVPU) += intel_vpu.o

drivers/accel/ivpu/ivpu_drv.c+81−2

@@ -15,6 +15,8 @@ · 14 unmodified lines
15	15
16	16	#include "ivpu_drv.h"
17	17	#include "ivpu_hw.h"
	18	+#include "ivpu_mmu.h"
	19	+#include "ivpu_mmu_context.h"
18	20
19	21	#ifndef DRIVER_VERSION_STR
20	22	#define DRIVER_VERSION_STR __stringify(DRM_IVPU_DRIVER_MAJOR) "." \
@@ -37,23 +39,38 @@ MODULE_PARM_DESC(pll_max_ratio, "Maximum PLL ratio used to set VPU frequency"); · 16 unmodified lines
37	39
38	40	struct ivpu_file_priv ivpu_file_priv_get(struct ivpu_file_priv file_priv)
39	41	{
	42	+ struct ivpu_device *vdev = file_priv->vdev;
	43	+
40	44	kref_get(&file_priv->ref);
	45	+
	46	+ ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n",
	47	+ file_priv->ctx.id, kref_read(&file_priv->ref));
	48	+
41	49	return file_priv;
42	50	}
43	51
44	52	static void file_priv_release(struct kref *ref)
45	53	{
46	54	struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref);
	55	+ struct ivpu_device *vdev = file_priv->vdev;
47	56
	57	+ ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id);
	58	+
	59	+ ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
	60	+ WARN_ON(xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);
48	61	kfree(file_priv);
49	62	}
50	63
51	64	void ivpu_file_priv_put(struct ivpu_file_priv **link)
52	65	{
53	66	struct ivpu_file_priv file_priv = link;
	67	+ struct ivpu_device *vdev = file_priv->vdev;
54	68
55	69	WARN_ON(!file_priv);
56	70
	71	+ ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",
	72	+ file_priv->ctx.id, kref_read(&file_priv->ref));
	73	+
57	74	*link = NULL;
58	75	kref_put(&file_priv->ref, file_priv_release);
59	76	}
@@ -88,6 +105,9 @@ static int ivpu_get_param_ioctl(struct drm_device dev, void data, struct drm_f · 28 unmodified lines
88	105	case DRM_IVPU_PARAM_CONTEXT_PRIORITY:
89	106	args->value = file_priv->priority;
90	107	break;
	108	+ case DRM_IVPU_PARAM_CONTEXT_ID:
	109	+ args->value = file_priv->ctx.id;
	110	+ break;
91	111	default:
92	112	ret = -EINVAL;
93	113	break;
@@ -120,22 +140,59 @@ static int ivpu_open(struct drm_device dev, struct drm_file file) · 26 unmodified lines
120	140	{
121	141	struct ivpu_device *vdev = to_ivpu_device(dev);
122	142	struct ivpu_file_priv *file_priv;
	143	+ u32 ctx_id;
	144	+ void *old;
	145	+ int ret;
	146	+
	147	+ ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, NULL, vdev->context_xa_limit, GFP_KERNEL);
	148	+ if (ret) {
	149	+ ivpu_err(vdev, "Failed to allocate context id: %d\n", ret);
	150	+ return ret;
	151	+ }
123	152
124	153	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
125		− if (!file_priv)
126		− return -ENOMEM;
	154	+ if (!file_priv) {
	155	+ ret = -ENOMEM;
	156	+ goto err_xa_erase;
	157	+ }
127	158
128	159	file_priv->vdev = vdev;
129	160	file_priv->priority = DRM_IVPU_CONTEXT_PRIORITY_NORMAL;
130	161	kref_init(&file_priv->ref);
131	162
	163	+ ret = ivpu_mmu_user_context_init(vdev, &file_priv->ctx, ctx_id);
	164	+ if (ret)
	165	+ goto err_free_file_priv;
	166	+
	167	+ old = xa_store_irq(&vdev->context_xa, ctx_id, file_priv, GFP_KERNEL);
	168	+ if (xa_is_err(old)) {
	169	+ ret = xa_err(old);
	170	+ ivpu_err(vdev, "Failed to store context %u: %d\n", ctx_id, ret);
	171	+ goto err_ctx_fini;
	172	+ }
	173	+
	174	+ ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n",
	175	+ ctx_id, current->comm, task_pid_nr(current));
	176	+
132	177	file->driver_priv = file_priv;
133	178	return 0;
	179	+
	180	+err_ctx_fini:
	181	+ ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
	182	+err_free_file_priv:
	183	+ kfree(file_priv);
	184	+err_xa_erase:
	185	+ xa_erase_irq(&vdev->context_xa, ctx_id);
	186	+ return ret;
134	187	}
135	188
136	189	static void ivpu_postclose(struct drm_device dev, struct drm_file file)
137	190	{
138	191	struct ivpu_file_priv *file_priv = file->driver_priv;
	192	+ struct ivpu_device *vdev = to_ivpu_device(dev);
	193	+
	194	+ ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n",
	195	+ file_priv->ctx.id, current->comm, task_pid_nr(current));
139	196
140	197	ivpu_file_priv_put(&file_priv);
141	198	}
@@ -150,6 +207,7 @@ int ivpu_shutdown(struct ivpu_device *vdev) · 8 unmodified lines
150	207	int ret;
151	208
152	209	ivpu_hw_irq_disable(vdev);
	210	+ ivpu_mmu_disable(vdev);
153	211
154	212	ret = ivpu_hw_power_down(vdev);
155	213	if (ret)
@@ -251,6 +309,10 @@ static int ivpu_dev_init(struct ivpu_device *vdev) · 95 unmodified lines
251	309	if (!vdev->hw)
252	310	return -ENOMEM;
253	311
	312	+ vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL);
	313	+ if (!vdev->mmu)
	314	+ return -ENOMEM;
	315	+
254	316	vdev->hw->ops = &ivpu_hw_mtl_ops;
255	317	vdev->platform = IVPU_PLATFORM_INVALID;
256	318	vdev->context_xa_limit.min = IVPU_GLOBAL_CONTEXT_MMU_SSID + 1;
@@ -283,8 +345,24 @@ static int ivpu_dev_init(struct ivpu_device *vdev) · 26 unmodified lines
283	345	goto err_xa_destroy;
284	346	}
285	347
	348	+ ret = ivpu_mmu_global_context_init(vdev);
	349	+ if (ret) {
	350	+ ivpu_err(vdev, "Failed to initialize global MMU context: %d\n", ret);
	351	+ goto err_power_down;
	352	+ }
	353	+
	354	+ ret = ivpu_mmu_init(vdev);
	355	+ if (ret) {
	356	+ ivpu_err(vdev, "Failed to initialize MMU device: %d\n", ret);
	357	+ goto err_mmu_gctx_fini;
	358	+ }
	359	+
286	360	return 0;
287	361
	362	+err_mmu_gctx_fini:
	363	+ ivpu_mmu_global_context_fini(vdev);
	364	+err_power_down:
	365	+ ivpu_hw_power_down(vdev);
288	366	err_xa_destroy:
289	367	xa_destroy(&vdev->context_xa);
290	368	return ret;
@@ -293,6 +371,7 @@ err_xa_destroy: · 2 unmodified lines
293	371	static void ivpu_dev_fini(struct ivpu_device *vdev)
294	372	{
295	373	ivpu_shutdown(vdev);
	374	+ ivpu_mmu_global_context_fini(vdev);
296	375
297	376	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa));
298	377	xa_destroy(&vdev->context_xa);

drivers/accel/ivpu/ivpu_drv.h+7−0

@@ -15,6 +15,8 @@ · 14 unmodified lines
15	15	#include <linux/xarray.h>
16	16	#include <uapi/drm/ivpu_accel.h>
17	17
	18	+#include "ivpu_mmu_context.h"
	19	+
18	20	#define DRIVER_NAME "intel_vpu"
19	21	#define DRIVER_DESC "Driver for Intel Versatile Processing Unit (VPU)"
20	22	#define DRIVER_DATE "20230117"
@@ -71,6 +73,7 @@ struct ivpu_wa_table { · 50 unmodified lines
71	73	};
72	74
73	75	struct ivpu_hw_info;
	76	+struct ivpu_mmu_info;
74	77
75	78	struct ivpu_device {
76	79	struct drm_device drm;
@@ -81,7 +84,9 @@ struct ivpu_device { · 4 unmodified lines
81	84
82	85	struct ivpu_wa_table wa;
83	86	struct ivpu_hw_info *hw;
	87	+ struct ivpu_mmu_info *mmu;
84	88
	89	+ struct ivpu_mmu_context gctx;
85	90	struct xarray context_xa;
86	91	struct xa_limit context_xa_limit;
87	92
@@ -100,7 +105,9 @@ struct ivpu_device { · 12 unmodified lines
100	105	struct ivpu_file_priv {
101	106	struct kref ref;
102	107	struct ivpu_device *vdev;
	108	+ struct ivpu_mmu_context ctx;
103	109	u32 priority;
	110	+ bool has_mmu_faults;
104	111	};
105	112
106	113	extern int ivpu_dbg_mask;

drivers/accel/ivpu/ivpu_hw_mtl.c+10−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include "ivpu_hw_mtl_reg.h"
8	8	#include "ivpu_hw_reg_io.h"
9	9	#include "ivpu_hw.h"
	10	+#include "ivpu_mmu.h"
10	11
11	12	#define TILE_FUSE_ENABLE_BOTH 0x0
12	13	#define TILE_FUSE_ENABLE_UPPER 0x1
@@ -930,6 +931,15 @@ static u32 ivpu_hw_mtl_irqv_handler(struct ivpu_device *vdev, int irq) · 917 unmodified lines
930	931
931	932	REGV_WR32(MTL_VPU_HOST_SS_ICB_CLEAR_0, status);
932	933
	934	+ if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
	935	+ ivpu_mmu_irq_evtq_handler(vdev);
	936	+
	937	+ if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
	938	+ ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
	939	+
	940	+ if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
	941	+ ivpu_mmu_irq_gerr_handler(vdev);
	942	+
933	943	if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
934	944	ivpu_hw_mtl_irq_wdt_mss_handler(vdev);
935	945

drivers/accel/ivpu/ivpu_mmu.c+876−0

@@ -0,0 +1,876 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (C) 2020-2023 Intel Corporation
	4	+ */
	5	+
	6	+#include <linux/circ_buf.h>
	7	+#include <linux/highmem.h>
	8	+
	9	+#include "ivpu_drv.h"
	10	+#include "ivpu_hw_mtl_reg.h"
	11	+#include "ivpu_hw_reg_io.h"
	12	+#include "ivpu_mmu.h"
	13	+#include "ivpu_mmu_context.h"
	14	+
	15	+#define IVPU_MMU_IDR0_REF 0x080f3e0f
	16	+#define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f
	17	+#define IVPU_MMU_IDR1_REF 0x0e739d18
	18	+#define IVPU_MMU_IDR3_REF 0x0000003c
	19	+#define IVPU_MMU_IDR5_REF 0x00040070
	20	+#define IVPU_MMU_IDR5_REF_SIMICS 0x00000075
	21	+#define IVPU_MMU_IDR5_REF_FPGA 0x00800075
	22	+
	23	+#define IVPU_MMU_CDTAB_ENT_SIZE 64
	24	+#define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */
	25	+#define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2)
	26	+
	27	+#define IVPU_MMU_STREAM_ID0 0
	28	+#define IVPU_MMU_STREAM_ID3 3
	29	+
	30	+#define IVPU_MMU_STRTAB_ENT_SIZE 64
	31	+#define IVPU_MMU_STRTAB_ENT_COUNT 4
	32	+#define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2
	33	+#define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE
	34	+
	35	+#define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */
	36	+#define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2)
	37	+#define IVPU_MMU_Q_WRAP_BIT (IVPU_MMU_Q_COUNT << 1)
	38	+#define IVPU_MMU_Q_WRAP_MASK (IVPU_MMU_Q_WRAP_BIT - 1)
	39	+#define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1)
	40	+#define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK)
	41	+
	42	+#define IVPU_MMU_CMDQ_CMD_SIZE 16
	43	+#define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE)
	44	+
	45	+#define IVPU_MMU_EVTQ_CMD_SIZE 32
	46	+#define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE)
	47	+
	48	+#define IVPU_MMU_CMD_OPCODE GENMASK(7, 0)
	49	+
	50	+#define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12)
	51	+#define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22)
	52	+#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)
	53	+#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)
	54	+#define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32)
	55	+
	56	+#define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10)
	57	+#define IVPU_MMU_CMD_CFGI_0_SSV BIT(11)
	58	+#define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12)
	59	+#define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32)
	60	+#define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0)
	61	+
	62	+#define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48)
	63	+#define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32)
	64	+
	65	+#define CMD_PREFETCH_CFG 0x1
	66	+#define CMD_CFGI_STE 0x3
	67	+#define CMD_CFGI_ALL 0x4
	68	+#define CMD_CFGI_CD 0x5
	69	+#define CMD_CFGI_CD_ALL 0x6
	70	+#define CMD_TLBI_NH_ASID 0x11
	71	+#define CMD_TLBI_EL2_ALL 0x20
	72	+#define CMD_TLBI_NSNH_ALL 0x30
	73	+#define CMD_SYNC 0x46
	74	+
	75	+#define IVPU_MMU_EVT_F_UUT 0x01
	76	+#define IVPU_MMU_EVT_C_BAD_STREAMID 0x02
	77	+#define IVPU_MMU_EVT_F_STE_FETCH 0x03
	78	+#define IVPU_MMU_EVT_C_BAD_STE 0x04
	79	+#define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05
	80	+#define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06
	81	+#define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07
	82	+#define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08
	83	+#define IVPU_MMU_EVT_F_CD_FETCH 0x09
	84	+#define IVPU_MMU_EVT_C_BAD_CD 0x0a
	85	+#define IVPU_MMU_EVT_F_WALK_EABT 0x0b
	86	+#define IVPU_MMU_EVT_F_TRANSLATION 0x10
	87	+#define IVPU_MMU_EVT_F_ADDR_SIZE 0x11
	88	+#define IVPU_MMU_EVT_F_ACCESS 0x12
	89	+#define IVPU_MMU_EVT_F_PERMISSION 0x13
	90	+#define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20
	91	+#define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21
	92	+#define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24
	93	+#define IVPU_MMU_EVT_F_VMS_FETCH 0x25
	94	+
	95	+#define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0)
	96	+#define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12)
	97	+
	98	+#define IVPU_MMU_Q_BASE_RWA BIT(62)
	99	+#define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
	100	+#define IVPU_MMU_STRTAB_BASE_RA BIT(62)
	101	+#define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
	102	+
	103	+#define IVPU_MMU_IRQ_EVTQ_EN BIT(2)
	104	+#define IVPU_MMU_IRQ_GERROR_EN BIT(0)
	105	+
	106	+#define IVPU_MMU_CR0_ATSCHK BIT(4)
	107	+#define IVPU_MMU_CR0_CMDQEN BIT(3)
	108	+#define IVPU_MMU_CR0_EVTQEN BIT(2)
	109	+#define IVPU_MMU_CR0_PRIQEN BIT(1)
	110	+#define IVPU_MMU_CR0_SMMUEN BIT(0)
	111	+
	112	+#define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10)
	113	+#define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8)
	114	+#define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6)
	115	+#define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4)
	116	+#define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2)
	117	+#define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0)
	118	+#define IVPU_MMU_CACHE_NC 0
	119	+#define IVPU_MMU_CACHE_WB 1
	120	+#define IVPU_MMU_CACHE_WT 2
	121	+#define IVPU_MMU_SH_NSH 0
	122	+#define IVPU_MMU_SH_OSH 2
	123	+#define IVPU_MMU_SH_ISH 3
	124	+
	125	+#define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0)
	126	+
	127	+#define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
	128	+#define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
	129	+#define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
	130	+#define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
	131	+#define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
	132	+#define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14)
	133	+#define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30)
	134	+#define IVPU_MMU_CD_0_ENDI BIT(15)
	135	+#define IVPU_MMU_CD_0_V BIT(31)
	136	+#define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32)
	137	+#define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38)
	138	+#define IVPU_MMU_CD_0_AA64 BIT(41)
	139	+#define IVPU_MMU_CD_0_S BIT(44)
	140	+#define IVPU_MMU_CD_0_R BIT(45)
	141	+#define IVPU_MMU_CD_0_A BIT(46)
	142	+#define IVPU_MMU_CD_0_ASET BIT(47)
	143	+#define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48)
	144	+
	145	+#define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
	146	+
	147	+#define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59)
	148	+#define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4)
	149	+#define IVPU_MMU_STE_0_S1FMT_LINEAR 0
	150	+#define IVPU_MMU_STE_DWORDS 8
	151	+#define IVPU_MMU_STE_0_CFG_S1_TRANS 5
	152	+#define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1)
	153	+#define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
	154	+#define IVPU_MMU_STE_0_V BIT(0)
	155	+
	156	+#define IVPU_MMU_STE_1_STRW_NSEL1 0ul
	157	+#define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13)
	158	+#define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30)
	159	+#define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48)
	160	+#define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul
	161	+#define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50)
	162	+#define IVPU_MMU_STE_1_INSTCFG_DATA 2ul
	163	+#define IVPU_MMU_STE_1_MEV BIT(19)
	164	+#define IVPU_MMU_STE_1_S1STALLD BIT(27)
	165	+#define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul
	166	+#define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul
	167	+#define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul
	168	+#define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul
	169	+#define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2)
	170	+#define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4)
	171	+#define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6)
	172	+#define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0)
	173	+#define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0
	174	+
	175	+#define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC)
	176	+#define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC)
	177	+
	178	+#define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(MTL_VPU_HOST_MMU_GERROR, CMDQ)) \| \
	179	+ (REG_FLD(MTL_VPU_HOST_MMU_GERROR, EVTQ_ABT)) \| \
	180	+ (REG_FLD(MTL_VPU_HOST_MMU_GERROR, PRIQ_ABT)) \| \
	181	+ (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_CMDQ_ABT)) \| \
	182	+ (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_EVTQ_ABT)) \| \
	183	+ (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_PRIQ_ABT)) \| \
	184	+ (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_ABT)))
	185	+
	186	+static char *ivpu_mmu_event_to_str(u32 cmd)
	187	+{
	188	+ switch (cmd) {
	189	+ case IVPU_MMU_EVT_F_UUT:
	190	+ return "Unsupported Upstream Transaction";
	191	+ case IVPU_MMU_EVT_C_BAD_STREAMID:
	192	+ return "Transaction StreamID out of range";
	193	+ case IVPU_MMU_EVT_F_STE_FETCH:
	194	+ return "Fetch of STE caused external abort";
	195	+ case IVPU_MMU_EVT_C_BAD_STE:
	196	+ return "Used STE invalid";
	197	+ case IVPU_MMU_EVT_F_BAD_ATS_TREQ:
	198	+ return "Address Request disallowed for a StreamID";
	199	+ case IVPU_MMU_EVT_F_STREAM_DISABLED:
	200	+ return "Transaction marks non-substream disabled";
	201	+ case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN:
	202	+ return "MMU bypass is disallowed for this StreamID";
	203	+ case IVPU_MMU_EVT_C_BAD_SUBSTREAMID:
	204	+ return "Invalid StreamID";
	205	+ case IVPU_MMU_EVT_F_CD_FETCH:
	206	+ return "Fetch of CD caused external abort";
	207	+ case IVPU_MMU_EVT_C_BAD_CD:
	208	+ return "Fetched CD invalid";
	209	+ case IVPU_MMU_EVT_F_WALK_EABT:
	210	+ return " An external abort occurred fetching a TLB";
	211	+ case IVPU_MMU_EVT_F_TRANSLATION:
	212	+ return "Translation fault";
	213	+ case IVPU_MMU_EVT_F_ADDR_SIZE:
	214	+ return " Output address caused address size fault";
	215	+ case IVPU_MMU_EVT_F_ACCESS:
	216	+ return "Access flag fault";
	217	+ case IVPU_MMU_EVT_F_PERMISSION:
	218	+ return "Permission fault occurred on page access";
	219	+ case IVPU_MMU_EVT_F_TLB_CONFLICT:
	220	+ return "A TLB conflict";
	221	+ case IVPU_MMU_EVT_F_CFG_CONFLICT:
	222	+ return "A configuration cache conflict";
	223	+ case IVPU_MMU_EVT_E_PAGE_REQUEST:
	224	+ return "Page request hint from a client device";
	225	+ case IVPU_MMU_EVT_F_VMS_FETCH:
	226	+ return "Fetch of VMS caused external abort";
	227	+ default:
	228	+ return "Unknown CMDQ command";
	229	+ }
	230	+}
	231	+
	232	+static void ivpu_mmu_config_check(struct ivpu_device *vdev)
	233	+{
	234	+ u32 val_ref;
	235	+ u32 val;
	236	+
	237	+ if (ivpu_is_simics(vdev))
	238	+ val_ref = IVPU_MMU_IDR0_REF_SIMICS;
	239	+ else
	240	+ val_ref = IVPU_MMU_IDR0_REF;
	241	+
	242	+ val = REGV_RD32(MTL_VPU_HOST_MMU_IDR0);
	243	+ if (val != val_ref)
	244	+ ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);
	245	+
	246	+ val = REGV_RD32(MTL_VPU_HOST_MMU_IDR1);
	247	+ if (val != IVPU_MMU_IDR1_REF)
	248	+ ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);
	249	+
	250	+ val = REGV_RD32(MTL_VPU_HOST_MMU_IDR3);
	251	+ if (val != IVPU_MMU_IDR3_REF)
	252	+ ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);
	253	+
	254	+ if (ivpu_is_simics(vdev))
	255	+ val_ref = IVPU_MMU_IDR5_REF_SIMICS;
	256	+ else if (ivpu_is_fpga(vdev))
	257	+ val_ref = IVPU_MMU_IDR5_REF_FPGA;
	258	+ else
	259	+ val_ref = IVPU_MMU_IDR5_REF;
	260	+
	261	+ val = REGV_RD32(MTL_VPU_HOST_MMU_IDR5);
	262	+ if (val != val_ref)
	263	+ ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);
	264	+}
	265	+
	266	+static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)
	267	+{
	268	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	269	+ struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
	270	+ size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE;
	271	+
	272	+ cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL);
	273	+ if (!cdtab->base)
	274	+ return -ENOMEM;
	275	+
	276	+ ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);
	277	+
	278	+ return 0;
	279	+}
	280	+
	281	+static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)
	282	+{
	283	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	284	+ struct ivpu_mmu_strtab *strtab = &mmu->strtab;
	285	+ size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE;
	286	+
	287	+ strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL);
	288	+ if (!strtab->base)
	289	+ return -ENOMEM;
	290	+
	291	+ strtab->base_cfg = IVPU_MMU_STRTAB_CFG;
	292	+ strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA;
	293	+ strtab->dma_q \|= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK;
	294	+
	295	+ ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",
	296	+ &strtab->dma, &strtab->dma_q, size);
	297	+
	298	+ return 0;
	299	+}
	300	+
	301	+static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)
	302	+{
	303	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	304	+ struct ivpu_mmu_queue *q = &mmu->cmdq;
	305	+
	306	+ q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL);
	307	+ if (!q->base)
	308	+ return -ENOMEM;
	309	+
	310	+ q->dma_q = IVPU_MMU_Q_BASE_RWA;
	311	+ q->dma_q \|= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
	312	+ q->dma_q \|= IVPU_MMU_Q_COUNT_LOG2;
	313	+
	314	+ ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",
	315	+ &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE);
	316	+
	317	+ return 0;
	318	+}
	319	+
	320	+static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)
	321	+{
	322	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	323	+ struct ivpu_mmu_queue *q = &mmu->evtq;
	324	+
	325	+ q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL);
	326	+ if (!q->base)
	327	+ return -ENOMEM;
	328	+
	329	+ q->dma_q = IVPU_MMU_Q_BASE_RWA;
	330	+ q->dma_q \|= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
	331	+ q->dma_q \|= IVPU_MMU_Q_COUNT_LOG2;
	332	+
	333	+ ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",
	334	+ &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE);
	335	+
	336	+ return 0;
	337	+}
	338	+
	339	+static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)
	340	+{
	341	+ int ret;
	342	+
	343	+ ret = ivpu_mmu_cdtab_alloc(vdev);
	344	+ if (ret) {
	345	+ ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret);
	346	+ return ret;
	347	+ }
	348	+
	349	+ ret = ivpu_mmu_strtab_alloc(vdev);
	350	+ if (ret) {
	351	+ ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret);
	352	+ return ret;
	353	+ }
	354	+
	355	+ ret = ivpu_mmu_cmdq_alloc(vdev);
	356	+ if (ret) {
	357	+ ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret);
	358	+ return ret;
	359	+ }
	360	+
	361	+ ret = ivpu_mmu_evtq_alloc(vdev);
	362	+ if (ret)
	363	+ ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret);
	364	+
	365	+ return ret;
	366	+}
	367	+
	368	+static int ivpu_mmu_reg_write(struct ivpu_device *vdev, u32 reg, u32 val)
	369	+{
	370	+ u32 reg_ack = reg + 4; /* ACK register is 4B after base register */
	371	+ u32 val_ack;
	372	+ int ret;
	373	+
	374	+ REGV_WR32(reg, val);
	375	+
	376	+ ret = REGV_POLL(reg_ack, val_ack, (val == val_ack), IVPU_MMU_REG_TIMEOUT_US);
	377	+ if (ret)
	378	+ ivpu_err(vdev, "Failed to write register 0x%x\n", reg);
	379	+
	380	+ return ret;
	381	+}
	382	+
	383	+static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)
	384	+{
	385	+ u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN \| IVPU_MMU_IRQ_GERROR_EN;
	386	+ int ret;
	387	+
	388	+ ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, 0);
	389	+ if (ret)
	390	+ return ret;
	391	+
	392	+ return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, irq_ctrl);
	393	+}
	394	+
	395	+static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)
	396	+{
	397	+ struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
	398	+
	399	+ return REGV_POLL(MTL_VPU_HOST_MMU_CMDQ_CONS, cmdq->cons, (cmdq->prod == cmdq->cons),
	400	+ IVPU_MMU_QUEUE_TIMEOUT_US);
	401	+}
	402	+
	403	+static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device vdev, const char name, u64 data0, u64 data1)
	404	+{
	405	+ struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
	406	+ u64 *queue_buffer = q->base;
	407	+ int idx = IVPU_MMU_Q_IDX(q->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer));
	408	+
	409	+ if (!CIRC_SPACE(IVPU_MMU_Q_IDX(q->prod), IVPU_MMU_Q_IDX(q->cons), IVPU_MMU_Q_COUNT)) {
	410	+ ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);
	411	+ return -EBUSY;
	412	+ }
	413	+
	414	+ queue_buffer[idx] = data0;
	415	+ queue_buffer[idx + 1] = data1;
	416	+ q->prod = (q->prod + 1) & IVPU_MMU_Q_WRAP_MASK;
	417	+
	418	+ ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);
	419	+
	420	+ return 0;
	421	+}
	422	+
	423	+static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
	424	+{
	425	+ struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
	426	+ u64 val;
	427	+ int ret;
	428	+
	429	+ val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC) \|
	430	+ FIELD_PREP(IVPU_MMU_CMD_SYNC_0_CS, 0x2) \|
	431	+ FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSH, 0x3) \|
	432	+ FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSI_ATTR, 0xf);
	433	+
	434	+ ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);
	435	+ if (ret)
	436	+ return ret;
	437	+
	438	+ clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE);
	439	+ REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_PROD, q->prod);
	440	+
	441	+ ret = ivpu_mmu_cmdq_wait_for_cons(vdev);
	442	+ if (ret)
	443	+ ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret);
	444	+
	445	+ return ret;
	446	+}
	447	+
	448	+static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)
	449	+{
	450	+ u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL);
	451	+ u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f);
	452	+
	453	+ return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1);
	454	+}
	455	+
	456	+static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)
	457	+{
	458	+ u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) \|
	459	+ FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid);
	460	+
	461	+ return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0);
	462	+}
	463	+
	464	+static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)
	465	+{
	466	+ u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL);
	467	+
	468	+ return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0);
	469	+}
	470	+
	471	+static int ivpu_mmu_reset(struct ivpu_device *vdev)
	472	+{
	473	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	474	+ u32 val;
	475	+ int ret;
	476	+
	477	+ memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE);
	478	+ clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE);
	479	+ mmu->cmdq.prod = 0;
	480	+ mmu->cmdq.cons = 0;
	481	+
	482	+ memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE);
	483	+ clflush_cache_range(mmu->evtq.base, IVPU_MMU_EVTQ_SIZE);
	484	+ mmu->evtq.prod = 0;
	485	+ mmu->evtq.cons = 0;
	486	+
	487	+ ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, 0);
	488	+ if (ret)
	489	+ return ret;
	490	+
	491	+ val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) \|
	492	+ FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) \|
	493	+ FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) \|
	494	+ FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) \|
	495	+ FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) \|
	496	+ FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB);
	497	+ REGV_WR32(MTL_VPU_HOST_MMU_CR1, val);
	498	+
	499	+ REGV_WR64(MTL_VPU_HOST_MMU_STRTAB_BASE, mmu->strtab.dma_q);
	500	+ REGV_WR32(MTL_VPU_HOST_MMU_STRTAB_BASE_CFG, mmu->strtab.base_cfg);
	501	+
	502	+ REGV_WR64(MTL_VPU_HOST_MMU_CMDQ_BASE, mmu->cmdq.dma_q);
	503	+ REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_PROD, 0);
	504	+ REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_CONS, 0);
	505	+
	506	+ val = IVPU_MMU_CR0_CMDQEN;
	507	+ ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
	508	+ if (ret)
	509	+ return ret;
	510	+
	511	+ ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
	512	+ if (ret)
	513	+ return ret;
	514	+
	515	+ ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
	516	+ if (ret)
	517	+ return ret;
	518	+
	519	+ ret = ivpu_mmu_cmdq_sync(vdev);
	520	+ if (ret)
	521	+ return ret;
	522	+
	523	+ REGV_WR64(MTL_VPU_HOST_MMU_EVTQ_BASE, mmu->evtq.dma_q);
	524	+ REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_PROD_SEC, 0);
	525	+ REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_CONS_SEC, 0);
	526	+
	527	+ val \|= IVPU_MMU_CR0_EVTQEN;
	528	+ ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
	529	+ if (ret)
	530	+ return ret;
	531	+
	532	+ val \|= IVPU_MMU_CR0_ATSCHK;
	533	+ ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
	534	+ if (ret)
	535	+ return ret;
	536	+
	537	+ ret = ivpu_mmu_irqs_setup(vdev);
	538	+ if (ret)
	539	+ return ret;
	540	+
	541	+ val \|= IVPU_MMU_CR0_SMMUEN;
	542	+ return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
	543	+}
	544	+
	545	+static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)
	546	+{
	547	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	548	+ struct ivpu_mmu_strtab *strtab = &mmu->strtab;
	549	+ struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
	550	+ u64 entry = strtab->base + (sid IVPU_MMU_STRTAB_ENT_SIZE);
	551	+ u64 str[2];
	552	+
	553	+ str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) \|
	554	+ FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) \|
	555	+ FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) \|
	556	+ IVPU_MMU_STE_0_V \|
	557	+ (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK);
	558	+
	559	+ str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) \|
	560	+ FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) \|
	561	+ FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) \|
	562	+ FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) \|
	563	+ FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) \|
	564	+ FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) \|
	565	+ FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) \|
	566	+ FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) \|
	567	+ IVPU_MMU_STE_1_MEV \|
	568	+ IVPU_MMU_STE_1_S1STALLD;
	569	+
	570	+ WRITE_ONCE(entry[1], str[1]);
	571	+ WRITE_ONCE(entry[0], str[0]);
	572	+
	573	+ clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE);
	574	+
	575	+ ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);
	576	+}
	577	+
	578	+static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)
	579	+{
	580	+ ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0);
	581	+ ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);
	582	+
	583	+ return 0;
	584	+}
	585	+
	586	+int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)
	587	+{
	588	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	589	+ int ret;
	590	+
	591	+ ret = mutex_lock_interruptible(&mmu->lock);
	592	+ if (ret)
	593	+ return ret;
	594	+
	595	+ if (!mmu->on) {
	596	+ ret = 0;
	597	+ goto unlock;
	598	+ }
	599	+
	600	+ ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);
	601	+ if (ret)
	602	+ goto unlock;
	603	+
	604	+ ret = ivpu_mmu_cmdq_sync(vdev);
	605	+unlock:
	606	+ mutex_unlock(&mmu->lock);
	607	+ return ret;
	608	+}
	609	+
	610	+static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)
	611	+{
	612	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	613	+ struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
	614	+ u64 *entry;
	615	+ u64 cd[4];
	616	+ int ret;
	617	+
	618	+ if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
	619	+ return -EINVAL;
	620	+
	621	+ entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
	622	+
	623	+ if (cd_dma != 0) {
	624	+ cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, 26) \|
	625	+ FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) \|
	626	+ FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) \|
	627	+ FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) \|
	628	+ FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) \|
	629	+ FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, 3) \|
	630	+ FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) \|
	631	+ IVPU_MMU_CD_0_TCR_EPD1 \|
	632	+ IVPU_MMU_CD_0_AA64 \|
	633	+ IVPU_MMU_CD_0_R \|
	634	+ IVPU_MMU_CD_0_ASET \|
	635	+ IVPU_MMU_CD_0_V;
	636	+ cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK;
	637	+ cd[2] = 0;
	638	+ cd[3] = 0x0000000000007444;
	639	+
	640	+ /* For global context generate memory fault on VPU */
	641	+ if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID)
	642	+ cd[0] \|= IVPU_MMU_CD_0_A;
	643	+ } else {
	644	+ memset(cd, 0, sizeof(cd));
	645	+ }
	646	+
	647	+ WRITE_ONCE(entry[1], cd[1]);
	648	+ WRITE_ONCE(entry[2], cd[2]);
	649	+ WRITE_ONCE(entry[3], cd[3]);
	650	+ WRITE_ONCE(entry[0], cd[0]);
	651	+
	652	+ clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
	653	+
	654	+ ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
	655	+ cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]);
	656	+
	657	+ ret = mutex_lock_interruptible(&mmu->lock);
	658	+ if (ret)
	659	+ return ret;
	660	+
	661	+ if (!mmu->on) {
	662	+ ret = 0;
	663	+ goto unlock;
	664	+ }
	665	+
	666	+ ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
	667	+ if (ret)
	668	+ goto unlock;
	669	+
	670	+ ret = ivpu_mmu_cmdq_sync(vdev);
	671	+unlock:
	672	+ mutex_unlock(&mmu->lock);
	673	+ return ret;
	674	+}
	675	+
	676	+static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev)
	677	+{
	678	+ int ret;
	679	+
	680	+ ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma);
	681	+ if (ret)
	682	+ ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret);
	683	+
	684	+ return ret;
	685	+}
	686	+
	687	+static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma)
	688	+{
	689	+ int ret;
	690	+
	691	+ if (ssid == 0) {
	692	+ ivpu_err(vdev, "Invalid SSID: %u\n", ssid);
	693	+ return -EINVAL;
	694	+ }
	695	+
	696	+ ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma);
	697	+ if (ret)
	698	+ ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret);
	699	+
	700	+ return ret;
	701	+}
	702	+
	703	+int ivpu_mmu_init(struct ivpu_device *vdev)
	704	+{
	705	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	706	+ int ret;
	707	+
	708	+ ivpu_dbg(vdev, MMU, "Init..\n");
	709	+
	710	+ drmm_mutex_init(&vdev->drm, &mmu->lock);
	711	+ ivpu_mmu_config_check(vdev);
	712	+
	713	+ ret = ivpu_mmu_structs_alloc(vdev);
	714	+ if (ret)
	715	+ return ret;
	716	+
	717	+ ret = ivpu_mmu_strtab_init(vdev);
	718	+ if (ret) {
	719	+ ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);
	720	+ return ret;
	721	+ }
	722	+
	723	+ ret = ivpu_mmu_cd_add_gbl(vdev);
	724	+ if (ret) {
	725	+ ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);
	726	+ return ret;
	727	+ }
	728	+
	729	+ ret = ivpu_mmu_enable(vdev);
	730	+ if (ret) {
	731	+ ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
	732	+ return ret;
	733	+ }
	734	+
	735	+ ivpu_dbg(vdev, MMU, "Init done\n");
	736	+
	737	+ return 0;
	738	+}
	739	+
	740	+int ivpu_mmu_enable(struct ivpu_device *vdev)
	741	+{
	742	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	743	+ int ret;
	744	+
	745	+ mutex_lock(&mmu->lock);
	746	+
	747	+ mmu->on = true;
	748	+
	749	+ ret = ivpu_mmu_reset(vdev);
	750	+ if (ret) {
	751	+ ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);
	752	+ goto err;
	753	+ }
	754	+
	755	+ ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
	756	+ if (ret)
	757	+ goto err;
	758	+
	759	+ ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
	760	+ if (ret)
	761	+ goto err;
	762	+
	763	+ ret = ivpu_mmu_cmdq_sync(vdev);
	764	+ if (ret)
	765	+ goto err;
	766	+
	767	+ mutex_unlock(&mmu->lock);
	768	+
	769	+ return 0;
	770	+err:
	771	+ mmu->on = false;
	772	+ mutex_unlock(&mmu->lock);
	773	+ return ret;
	774	+}
	775	+
	776	+void ivpu_mmu_disable(struct ivpu_device *vdev)
	777	+{
	778	+ struct ivpu_mmu_info *mmu = vdev->mmu;
	779	+
	780	+ mutex_lock(&mmu->lock);
	781	+ mmu->on = false;
	782	+ mutex_unlock(&mmu->lock);
	783	+}
	784	+
	785	+static void ivpu_mmu_dump_event(struct ivpu_device vdev, u32 event)
	786	+{
	787	+ u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
	788	+ u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]);
	789	+ u64 fetch_addr = ((u64)event[7]) << 32 \| event[6];
	790	+ u64 in_addr = ((u64)event[5]) << 32 \| event[4];
	791	+ u32 sid = event[1];
	792	+
	793	+ ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n",
	794	+ op, ivpu_mmu_event_to_str(op), ssid, sid, event[2], event[3], in_addr, fetch_addr);
	795	+}
	796	+
	797	+static u32 ivpu_mmu_get_event(struct ivpu_device vdev)
	798	+{
	799	+ struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;
	800	+ u32 idx = IVPU_MMU_Q_IDX(evtq->cons);
	801	+ u32 evt = evtq->base + (idx IVPU_MMU_EVTQ_CMD_SIZE);
	802	+
	803	+ evtq->prod = REGV_RD32(MTL_VPU_HOST_MMU_EVTQ_PROD_SEC);
	804	+ if (!CIRC_CNT(IVPU_MMU_Q_IDX(evtq->prod), IVPU_MMU_Q_IDX(evtq->cons), IVPU_MMU_Q_COUNT))
	805	+ return NULL;
	806	+
	807	+ clflush_cache_range(evt, IVPU_MMU_EVTQ_CMD_SIZE);
	808	+
	809	+ evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK;
	810	+ REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_CONS_SEC, evtq->cons);
	811	+
	812	+ return evt;
	813	+}
	814	+
	815	+void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
	816	+{
	817	+ u32 *event;
	818	+ u32 ssid;
	819	+
	820	+ ivpu_dbg(vdev, IRQ, "MMU event queue\n");
	821	+
	822	+ while ((event = ivpu_mmu_get_event(vdev)) != NULL) {
	823	+ ivpu_mmu_dump_event(vdev, event);
	824	+
	825	+ ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
	826	+ if (ssid != IVPU_GLOBAL_CONTEXT_MMU_SSID)
	827	+ ivpu_mmu_user_context_mark_invalid(vdev, ssid);
	828	+ }
	829	+}
	830	+
	831	+void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)
	832	+{
	833	+ u32 gerror_val, gerrorn_val, active;
	834	+
	835	+ ivpu_dbg(vdev, IRQ, "MMU error\n");
	836	+
	837	+ gerror_val = REGV_RD32(MTL_VPU_HOST_MMU_GERROR);
	838	+ gerrorn_val = REGV_RD32(MTL_VPU_HOST_MMU_GERRORN);
	839	+
	840	+ active = gerror_val ^ gerrorn_val;
	841	+ if (!(active & IVPU_MMU_GERROR_ERR_MASK))
	842	+ return;
	843	+
	844	+ if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_ABT, active))
	845	+ ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");
	846	+
	847	+ if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_PRIQ_ABT, active))
	848	+ ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");
	849	+
	850	+ if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_EVTQ_ABT, active))
	851	+ ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");
	852	+
	853	+ if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_CMDQ_ABT, active))
	854	+ ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");
	855	+
	856	+ if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, PRIQ_ABT, active))
	857	+ ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");
	858	+
	859	+ if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, EVTQ_ABT, active))
	860	+ ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");
	861	+
	862	+ if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, CMDQ, active))
	863	+ ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");
	864	+
	865	+ REGV_WR32(MTL_VPU_HOST_MMU_GERRORN, gerror_val);
	866	+}
	867	+
	868	+int ivpu_mmu_set_pgtable(struct ivpu_device vdev, int ssid, struct ivpu_mmu_pgtable pgtable)
	869	+{
	870	+ return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma);
	871	+}
	872	+
	873	+void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid)
	874	+{
	875	+ ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */
	876	+}

drivers/accel/ivpu/ivpu_mmu.h+50−0

@@ -0,0 +1,50 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
	2	+/*
	3	+ * Copyright (C) 2020-2023 Intel Corporation
	4	+ */
	5	+
	6	+#ifndef __IVPU_MMU_H__
	7	+#define __IVPU_MMU_H__
	8	+
	9	+struct ivpu_device;
	10	+
	11	+struct ivpu_mmu_cdtab {
	12	+ void *base;
	13	+ dma_addr_t dma;
	14	+};
	15	+
	16	+struct ivpu_mmu_strtab {
	17	+ void *base;
	18	+ dma_addr_t dma;
	19	+ u64 dma_q;
	20	+ u32 base_cfg;
	21	+};
	22	+
	23	+struct ivpu_mmu_queue {
	24	+ void *base;
	25	+ dma_addr_t dma;
	26	+ u64 dma_q;
	27	+ u32 prod;
	28	+ u32 cons;
	29	+};
	30	+
	31	+struct ivpu_mmu_info {
	32	+ struct mutex lock; /* Protects cdtab, strtab, cmdq, on */
	33	+ struct ivpu_mmu_cdtab cdtab;
	34	+ struct ivpu_mmu_strtab strtab;
	35	+ struct ivpu_mmu_queue cmdq;
	36	+ struct ivpu_mmu_queue evtq;
	37	+ bool on;
	38	+};
	39	+
	40	+int ivpu_mmu_init(struct ivpu_device *vdev);
	41	+void ivpu_mmu_disable(struct ivpu_device *vdev);
	42	+int ivpu_mmu_enable(struct ivpu_device *vdev);
	43	+int ivpu_mmu_set_pgtable(struct ivpu_device vdev, int ssid, struct ivpu_mmu_pgtable pgtable);
	44	+void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid);
	45	+int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid);
	46	+
	47	+void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev);
	48	+void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev);
	49	+
	50	+#endif /* __IVPU_MMU_H__ */

drivers/accel/ivpu/ivpu_mmu_context.c+398−0

@@ -0,0 +1,398 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (C) 2020-2023 Intel Corporation
	4	+ */
	5	+
	6	+#include <linux/bitfield.h>
	7	+#include <linux/highmem.h>
	8	+
	9	+#include "ivpu_drv.h"
	10	+#include "ivpu_hw.h"
	11	+#include "ivpu_mmu.h"
	12	+#include "ivpu_mmu_context.h"
	13	+
	14	+#define IVPU_MMU_PGD_INDEX_MASK GENMASK(38, 30)
	15	+#define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21)
	16	+#define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12)
	17	+#define IVPU_MMU_ENTRY_FLAGS_MASK GENMASK(11, 0)
	18	+#define IVPU_MMU_ENTRY_FLAG_NG BIT(11)
	19	+#define IVPU_MMU_ENTRY_FLAG_AF BIT(10)
	20	+#define IVPU_MMU_ENTRY_FLAG_USER BIT(6)
	21	+#define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2)
	22	+#define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1)
	23	+#define IVPU_MMU_ENTRY_FLAG_VALID BIT(0)
	24	+
	25	+#define IVPU_MMU_PAGE_SIZE SZ_4K
	26	+#define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE)
	27	+#define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE)
	28	+#define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64))
	29	+
	30	+#define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000
	31	+#define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE \| IVPU_MMU_ENTRY_FLAG_VALID)
	32	+#define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK)
	33	+#define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF \| IVPU_MMU_ENTRY_FLAG_USER \| \
	34	+ IVPU_MMU_ENTRY_FLAG_NG \| IVPU_MMU_ENTRY_VALID)
	35	+
	36	+static int ivpu_mmu_pgtable_init(struct ivpu_device vdev, struct ivpu_mmu_pgtable pgtable)
	37	+{
	38	+ dma_addr_t pgd_dma;
	39	+ u64 *pgd;
	40	+
	41	+ pgd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, GFP_KERNEL);
	42	+ if (!pgd)
	43	+ return -ENOMEM;
	44	+
	45	+ pgtable->pgd = pgd;
	46	+ pgtable->pgd_dma = pgd_dma;
	47	+
	48	+ return 0;
	49	+}
	50	+
	51	+static void ivpu_mmu_pgtable_free(struct ivpu_device vdev, struct ivpu_mmu_pgtable pgtable)
	52	+{
	53	+ int pgd_index, pmd_index;
	54	+
	55	+ for (pgd_index = 0; pgd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_index) {
	56	+ u64 **pmd_entries = pgtable->pgd_cpu_entries[pgd_index];
	57	+ u64 *pmd = pgtable->pgd_entries[pgd_index];
	58	+
	59	+ if (!pmd_entries)
	60	+ continue;
	61	+
	62	+ for (pmd_index = 0; pmd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_index) {
	63	+ if (pmd_entries[pmd_index])
	64	+ dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE,
	65	+ pmd_entries[pmd_index],
	66	+ pmd[pmd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
	67	+ }
	68	+
	69	+ kfree(pmd_entries);
	70	+ dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd_entries[pgd_index],
	71	+ pgtable->pgd[pgd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
	72	+ }
	73	+
	74	+ dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd,
	75	+ pgtable->pgd_dma & ~IVPU_MMU_ENTRY_FLAGS_MASK);
	76	+}
	77	+
	78	+static u64*
	79	+ivpu_mmu_ensure_pmd(struct ivpu_device vdev, struct ivpu_mmu_pgtable pgtable, u64 pgd_index)
	80	+{
	81	+ u64 **pmd_entries;
	82	+ dma_addr_t pmd_dma;
	83	+ u64 *pmd;
	84	+
	85	+ if (pgtable->pgd_entries[pgd_index])
	86	+ return pgtable->pgd_entries[pgd_index];
	87	+
	88	+ pmd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
	89	+ if (!pmd)
	90	+ return NULL;
	91	+
	92	+ pmd_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
	93	+ if (!pmd_entries)
	94	+ goto err_free_pgd;
	95	+
	96	+ pgtable->pgd_entries[pgd_index] = pmd;
	97	+ pgtable->pgd_cpu_entries[pgd_index] = pmd_entries;
	98	+ pgtable->pgd[pgd_index] = pmd_dma \| IVPU_MMU_ENTRY_VALID;
	99	+
	100	+ return pmd;
	101	+
	102	+err_free_pgd:
	103	+ dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pmd, pmd_dma);
	104	+ return NULL;
	105	+}
	106	+
	107	+static u64*
	108	+ivpu_mmu_ensure_pte(struct ivpu_device vdev, struct ivpu_mmu_pgtable pgtable,
	109	+ int pgd_index, int pmd_index)
	110	+{
	111	+ dma_addr_t pte_dma;
	112	+ u64 *pte;
	113	+
	114	+ if (pgtable->pgd_cpu_entries[pgd_index][pmd_index])
	115	+ return pgtable->pgd_cpu_entries[pgd_index][pmd_index];
	116	+
	117	+ pte = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
	118	+ if (!pte)
	119	+ return NULL;
	120	+
	121	+ pgtable->pgd_cpu_entries[pgd_index][pmd_index] = pte;
	122	+ pgtable->pgd_entries[pgd_index][pmd_index] = pte_dma \| IVPU_MMU_ENTRY_VALID;
	123	+
	124	+ return pte;
	125	+}
	126	+
	127	+static int
	128	+ivpu_mmu_context_map_page(struct ivpu_device vdev, struct ivpu_mmu_context ctx,
	129	+ u64 vpu_addr, dma_addr_t dma_addr, int prot)
	130	+{
	131	+ u64 *pte;
	132	+ int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
	133	+ int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
	134	+ int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
	135	+
	136	+ /* Allocate PMD - second level page table if needed */
	137	+ if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_index))
	138	+ return -ENOMEM;
	139	+
	140	+ /* Allocate PTE - third level page table if needed */
	141	+ pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_index, pmd_index);
	142	+ if (!pte)
	143	+ return -ENOMEM;
	144	+
	145	+ /* Update PTE - third level page table with DMA address */
	146	+ pte[pte_index] = dma_addr \| prot;
	147	+
	148	+ return 0;
	149	+}
	150	+
	151	+static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr)
	152	+{
	153	+ int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
	154	+ int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
	155	+ int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
	156	+
	157	+ /* Update PTE with dummy physical address and clear flags */
	158	+ ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index][pte_index] = IVPU_MMU_ENTRY_INVALID;
	159	+}
	160	+
	161	+static void
	162	+ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
	163	+{
	164	+ u64 end_addr = vpu_addr + size;
	165	+ u64 *pgd = ctx->pgtable.pgd;
	166	+
	167	+ /* Align to PMD entry (2 MB) */
	168	+ vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1);
	169	+
	170	+ while (vpu_addr < end_addr) {
	171	+ int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
	172	+ u64 pmd_end = (pgd_index + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE;
	173	+ u64 *pmd = ctx->pgtable.pgd_entries[pgd_index];
	174	+
	175	+ while (vpu_addr < end_addr && vpu_addr < pmd_end) {
	176	+ int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
	177	+ u64 *pte = ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index];
	178	+
	179	+ clflush_cache_range(pte, IVPU_MMU_PGTABLE_SIZE);
	180	+ vpu_addr += IVPU_MMU_PTE_MAP_SIZE;
	181	+ }
	182	+ clflush_cache_range(pmd, IVPU_MMU_PGTABLE_SIZE);
	183	+ }
	184	+ clflush_cache_range(pgd, IVPU_MMU_PGTABLE_SIZE);
	185	+}
	186	+
	187	+static int
	188	+ivpu_mmu_context_map_pages(struct ivpu_device vdev, struct ivpu_mmu_context ctx,
	189	+ u64 vpu_addr, dma_addr_t dma_addr, size_t size, int prot)
	190	+{
	191	+ while (size) {
	192	+ int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
	193	+
	194	+ if (ret)
	195	+ return ret;
	196	+
	197	+ vpu_addr += IVPU_MMU_PAGE_SIZE;
	198	+ dma_addr += IVPU_MMU_PAGE_SIZE;
	199	+ size -= IVPU_MMU_PAGE_SIZE;
	200	+ }
	201	+
	202	+ return 0;
	203	+}
	204	+
	205	+static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
	206	+{
	207	+ while (size) {
	208	+ ivpu_mmu_context_unmap_page(ctx, vpu_addr);
	209	+ vpu_addr += IVPU_MMU_PAGE_SIZE;
	210	+ size -= IVPU_MMU_PAGE_SIZE;
	211	+ }
	212	+}
	213	+
	214	+int
	215	+ivpu_mmu_context_map_sgt(struct ivpu_device vdev, struct ivpu_mmu_context ctx,
	216	+ u64 vpu_addr, struct sg_table *sgt, bool llc_coherent)
	217	+{
	218	+ struct scatterlist *sg;
	219	+ int prot;
	220	+ int ret;
	221	+ u64 i;
	222	+
	223	+ if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
	224	+ return -EINVAL;
	225	+ /*
	226	+ * VPU is only 32 bit, but DMA engine is 38 bit
	227	+ * Ranges < 2 GB are reserved for VPU internal registers
	228	+ * Limit range to 8 GB
	229	+ */
	230	+ if (vpu_addr < SZ_2G \|\| vpu_addr > SZ_8G)
	231	+ return -EINVAL;
	232	+
	233	+ prot = IVPU_MMU_ENTRY_MAPPED;
	234	+ if (llc_coherent)
	235	+ prot \|= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;
	236	+
	237	+ mutex_lock(&ctx->lock);
	238	+
	239	+ for_each_sgtable_dma_sg(sgt, sg, i) {
	240	+ u64 dma_addr = sg_dma_address(sg) - sg->offset;
	241	+ size_t size = sg_dma_len(sg) + sg->offset;
	242	+
	243	+ ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
	244	+ if (ret) {
	245	+ ivpu_err(vdev, "Failed to map context pages\n");
	246	+ mutex_unlock(&ctx->lock);
	247	+ return ret;
	248	+ }
	249	+ ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
	250	+ vpu_addr += size;
	251	+ }
	252	+
	253	+ mutex_unlock(&ctx->lock);
	254	+
	255	+ ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
	256	+ if (ret)
	257	+ ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
	258	+ return ret;
	259	+}
	260	+
	261	+void
	262	+ivpu_mmu_context_unmap_sgt(struct ivpu_device vdev, struct ivpu_mmu_context ctx,
	263	+ u64 vpu_addr, struct sg_table *sgt)
	264	+{
	265	+ struct scatterlist *sg;
	266	+ int ret;
	267	+ u64 i;
	268	+
	269	+ if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
	270	+ ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr);
	271	+
	272	+ mutex_lock(&ctx->lock);
	273	+
	274	+ for_each_sgtable_dma_sg(sgt, sg, i) {
	275	+ size_t size = sg_dma_len(sg) + sg->offset;
	276	+
	277	+ ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
	278	+ ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
	279	+ vpu_addr += size;
	280	+ }
	281	+
	282	+ mutex_unlock(&ctx->lock);
	283	+
	284	+ ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
	285	+ if (ret)
	286	+ ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
	287	+}
	288	+
	289	+int
	290	+ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
	291	+ const struct ivpu_addr_range *range,
	292	+ u64 size, struct drm_mm_node *node)
	293	+{
	294	+ lockdep_assert_held(&ctx->lock);
	295	+
	296	+ return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE,
	297	+ 0, range->start, range->end, DRM_MM_INSERT_BEST);
	298	+}
	299	+
	300	+void
	301	+ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context ctx, struct drm_mm_node node)
	302	+{
	303	+ lockdep_assert_held(&ctx->lock);
	304	+
	305	+ drm_mm_remove_node(node);
	306	+}
	307	+
	308	+static int
	309	+ivpu_mmu_context_init(struct ivpu_device vdev, struct ivpu_mmu_context ctx, u32 context_id)
	310	+{
	311	+ u64 start, end;
	312	+ int ret;
	313	+
	314	+ mutex_init(&ctx->lock);
	315	+ INIT_LIST_HEAD(&ctx->bo_list);
	316	+
	317	+ ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);
	318	+ if (ret)
	319	+ return ret;
	320	+
	321	+ if (!context_id) {
	322	+ start = vdev->hw->ranges.global_low.start;
	323	+ end = vdev->hw->ranges.global_high.end;
	324	+ } else {
	325	+ start = vdev->hw->ranges.user_low.start;
	326	+ end = vdev->hw->ranges.user_high.end;
	327	+ }
	328	+
	329	+ drm_mm_init(&ctx->mm, start, end - start);
	330	+ ctx->id = context_id;
	331	+
	332	+ return 0;
	333	+}
	334	+
	335	+static void ivpu_mmu_context_fini(struct ivpu_device vdev, struct ivpu_mmu_context ctx)
	336	+{
	337	+ drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd);
	338	+
	339	+ mutex_destroy(&ctx->lock);
	340	+ ivpu_mmu_pgtable_free(vdev, &ctx->pgtable);
	341	+ drm_mm_takedown(&ctx->mm);
	342	+}
	343	+
	344	+int ivpu_mmu_global_context_init(struct ivpu_device *vdev)
	345	+{
	346	+ return ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID);
	347	+}
	348	+
	349	+void ivpu_mmu_global_context_fini(struct ivpu_device *vdev)
	350	+{
	351	+ return ivpu_mmu_context_fini(vdev, &vdev->gctx);
	352	+}
	353	+
	354	+void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid)
	355	+{
	356	+ struct ivpu_file_priv *file_priv;
	357	+
	358	+ xa_lock(&vdev->context_xa);
	359	+
	360	+ file_priv = xa_load(&vdev->context_xa, ssid);
	361	+ if (file_priv)
	362	+ file_priv->has_mmu_faults = true;
	363	+
	364	+ xa_unlock(&vdev->context_xa);
	365	+}
	366	+
	367	+int ivpu_mmu_user_context_init(struct ivpu_device vdev, struct ivpu_mmu_context ctx, u32 ctx_id)
	368	+{
	369	+ int ret;
	370	+
	371	+ drm_WARN_ON(&vdev->drm, !ctx_id);
	372	+
	373	+ ret = ivpu_mmu_context_init(vdev, ctx, ctx_id);
	374	+ if (ret) {
	375	+ ivpu_err(vdev, "Failed to initialize context: %d\n", ret);
	376	+ return ret;
	377	+ }
	378	+
	379	+ ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable);
	380	+ if (ret) {
	381	+ ivpu_err(vdev, "Failed to set page table: %d\n", ret);
	382	+ goto err_context_fini;
	383	+ }
	384	+
	385	+ return 0;
	386	+
	387	+err_context_fini:
	388	+ ivpu_mmu_context_fini(vdev, ctx);
	389	+ return ret;
	390	+}
	391	+
	392	+void ivpu_mmu_user_context_fini(struct ivpu_device vdev, struct ivpu_mmu_context ctx)
	393	+{
	394	+ drm_WARN_ON(&vdev->drm, !ctx->id);
	395	+
	396	+ ivpu_mmu_clear_pgtable(vdev, ctx->id);
	397	+ ivpu_mmu_context_fini(vdev, ctx);
	398	+}

drivers/accel/ivpu/ivpu_mmu_context.h+50−0

@@ -0,0 +1,50 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
	2	+/*
	3	+ * Copyright (C) 2020-2023 Intel Corporation
	4	+ */
	5	+
	6	+#ifndef __IVPU_MMU_CONTEXT_H__
	7	+#define __IVPU_MMU_CONTEXT_H__
	8	+
	9	+#include <drm/drm_mm.h>
	10	+
	11	+struct ivpu_device;
	12	+struct ivpu_file_priv;
	13	+struct ivpu_addr_range;
	14	+
	15	+#define IVPU_MMU_PGTABLE_ENTRIES 512
	16	+
	17	+struct ivpu_mmu_pgtable {
	18	+ u64 **pgd_cpu_entries[IVPU_MMU_PGTABLE_ENTRIES];
	19	+ u64 *pgd_entries[IVPU_MMU_PGTABLE_ENTRIES];
	20	+ u64 *pgd;
	21	+ dma_addr_t pgd_dma;
	22	+};
	23	+
	24	+struct ivpu_mmu_context {
	25	+ struct mutex lock; /* protects: mm, pgtable, bo_list */
	26	+ struct drm_mm mm;
	27	+ struct ivpu_mmu_pgtable pgtable;
	28	+ struct list_head bo_list;
	29	+ u32 id;
	30	+};
	31	+
	32	+int ivpu_mmu_global_context_init(struct ivpu_device *vdev);
	33	+void ivpu_mmu_global_context_fini(struct ivpu_device *vdev);
	34	+
	35	+int ivpu_mmu_user_context_init(struct ivpu_device vdev, struct ivpu_mmu_context ctx, u32 ctx_id);
	36	+void ivpu_mmu_user_context_fini(struct ivpu_device vdev, struct ivpu_mmu_context ctx);
	37	+void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid);
	38	+
	39	+int ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
	40	+ const struct ivpu_addr_range *range,
	41	+ u64 size, struct drm_mm_node *node);
	42	+void ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx,
	43	+ struct drm_mm_node *node);
	44	+
	45	+int ivpu_mmu_context_map_sgt(struct ivpu_device vdev, struct ivpu_mmu_context ctx,
	46	+ u64 vpu_addr, struct sg_table *sgt, bool llc_coherent);
	47	+void ivpu_mmu_context_unmap_sgt(struct ivpu_device vdev, struct ivpu_mmu_context ctx,
	48	+ u64 vpu_addr, struct sg_table *sgt);
	49	+
	50	+#endif /* __IVPU_MMU_CONTEXT_H__ */

include/uapi/drm/ivpu_accel.h+4−0

@@ -38,6 +38,7 @@ extern "C" { · 37 unmodified lines
38	38	#define DRM_IVPU_PARAM_NUM_CONTEXTS 4
39	39	#define DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS 5
40	40	#define DRM_IVPU_PARAM_CONTEXT_PRIORITY 6
	41	+#define DRM_IVPU_PARAM_CONTEXT_ID 7
41	42
42	43	#define DRM_IVPU_PLATFORM_TYPE_SILICON 0
43	44
@@ -78,6 +79,9 @@ struct drm_ivpu_param { · 34 unmodified lines
78	79	* Value of current context scheduling priority (read-write).
79	80	* See DRM_IVPU_CONTEXT_PRIORITY_* for possible values.
80	81	*
	82	+ * %DRM_IVPU_PARAM_CONTEXT_ID:
	83	+ * Current context ID, always greater than 0 (read-only)
	84	+ *
81	85	*/
82	86	__u32 param;
83	87

accel/ivpu: Add Intel VPU MMU support

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code