History

accel/ivpu: Add Intel VPU MMU support

VPU Memory Management Unit is based on ARM MMU-600. It allows the creation of multiple virtual address spaces for the device and map noncontinuous host memory (there is no dedicated memory on the VPU). Address space is implemented as a struct ivpu_mmu_context, it has an ID, drm_mm allocator for VPU addresses and struct ivpu_mmu_pgtable that holds actual 3-level, 4KB page table. Context with ID 0 (global context) is created upon driver initialization and it's mainly used for mapping memory required to execute the firmware. Contexts with non-zero IDs are user contexts allocated each time the devices is open()-ed and they map command buffers and other workload-related memory. Workloads executing in a given contexts have access only to the memory mapped in this context. This patch is has two main files: - ivpu_mmu_context.c handles MMU page tables and memory mapping - ivpu_mmu.c implements a driver that programs the MMU device

Jacek Lawrynowicz committed 3 years ago

Commit 263b2ba1 parent 35b13769 files · +1479 −3Browse files

Unified · Split

9 files
  1. drivers/accel/ivpu/Makefile +3 −1
  2. drivers/accel/ivpu/ivpu_drv.c +81 −2
  3. drivers/accel/ivpu/ivpu_drv.h +7 −0
  4. drivers/accel/ivpu/ivpu_hw_mtl.c +10 −0
  5. drivers/accel/ivpu/ivpu_mmu.c +876 −0
  6. drivers/accel/ivpu/ivpu_mmu.h +50 −0
  7. drivers/accel/ivpu/ivpu_mmu_context.c +398 −0
  8. drivers/accel/ivpu/ivpu_mmu_context.h +50 −0
  9. include/uapi/drm/ivpu_accel.h +4 −0
drivers/accel/ivpu/Makefile+3−1
@@ -3,6 +3,8 @@ · 2 unmodified lines
3 3
4intel_vpu-y := \4intel_vpu-y := \
5 ivpu_drv.o \5 ivpu_drv.o \
6 ivpu_hw_mtl.o6 ivpu_hw_mtl.o \
7 ivpu_mmu.o \
8 ivpu_mmu_context.o
7 9
8obj-$(CONFIG_DRM_ACCEL_IVPU) += intel_vpu.o10obj-$(CONFIG_DRM_ACCEL_IVPU) += intel_vpu.o
drivers/accel/ivpu/ivpu_drv.c+81−2
@@ -15,6 +15,8 @@ · 14 unmodified lines
15 15
16#include "ivpu_drv.h"16#include "ivpu_drv.h"
17#include "ivpu_hw.h"17#include "ivpu_hw.h"
18#include "ivpu_mmu.h"
19#include "ivpu_mmu_context.h"
18 20
19#ifndef DRIVER_VERSION_STR21#ifndef DRIVER_VERSION_STR
20#define DRIVER_VERSION_STR __stringify(DRM_IVPU_DRIVER_MAJOR) "." \22#define DRIVER_VERSION_STR __stringify(DRM_IVPU_DRIVER_MAJOR) "." \
@@ -37,23 +39,38 @@ MODULE_PARM_DESC(pll_max_ratio, "Maximum PLL ratio used to set VPU frequency"); · 16 unmodified lines
37 39
38struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv)40struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv)
39{41{
42 struct ivpu_device *vdev = file_priv->vdev;
43
40 kref_get(&file_priv->ref);44 kref_get(&file_priv->ref);
45
46 ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n",
47 file_priv->ctx.id, kref_read(&file_priv->ref));
48
41 return file_priv;49 return file_priv;
42}50}
43 51
44static void file_priv_release(struct kref *ref)52static void file_priv_release(struct kref *ref)
45{53{
46 struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref);54 struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref);
55 struct ivpu_device *vdev = file_priv->vdev;
47 56
57 ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id);
58
59 ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
60 WARN_ON(xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);
48 kfree(file_priv);61 kfree(file_priv);
49}62}
50 63
51void ivpu_file_priv_put(struct ivpu_file_priv **link)64void ivpu_file_priv_put(struct ivpu_file_priv **link)
52{65{
53 struct ivpu_file_priv *file_priv = *link;66 struct ivpu_file_priv *file_priv = *link;
67 struct ivpu_device *vdev = file_priv->vdev;
54 68
55 WARN_ON(!file_priv);69 WARN_ON(!file_priv);
56 70
71 ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",
72 file_priv->ctx.id, kref_read(&file_priv->ref));
73
57 *link = NULL;74 *link = NULL;
58 kref_put(&file_priv->ref, file_priv_release);75 kref_put(&file_priv->ref, file_priv_release);
59}76}
@@ -88,6 +105,9 @@ static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_f · 28 unmodified lines
88 case DRM_IVPU_PARAM_CONTEXT_PRIORITY:105 case DRM_IVPU_PARAM_CONTEXT_PRIORITY:
89 args->value = file_priv->priority;106 args->value = file_priv->priority;
90 break;107 break;
108 case DRM_IVPU_PARAM_CONTEXT_ID:
109 args->value = file_priv->ctx.id;
110 break;
91 default:111 default:
92 ret = -EINVAL;112 ret = -EINVAL;
93 break;113 break;
@@ -120,22 +140,59 @@ static int ivpu_open(struct drm_device *dev, struct drm_file *file) · 26 unmodified lines
120{140{
121 struct ivpu_device *vdev = to_ivpu_device(dev);141 struct ivpu_device *vdev = to_ivpu_device(dev);
122 struct ivpu_file_priv *file_priv;142 struct ivpu_file_priv *file_priv;
143 u32 ctx_id;
144 void *old;
145 int ret;
146
147 ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, NULL, vdev->context_xa_limit, GFP_KERNEL);
148 if (ret) {
149 ivpu_err(vdev, "Failed to allocate context id: %d\n", ret);
150 return ret;
151 }
123 152
124 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);153 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
125 if (!file_priv)154 if (!file_priv) {
126 return -ENOMEM;155 ret = -ENOMEM;
156 goto err_xa_erase;
157 }
127 158
128 file_priv->vdev = vdev;159 file_priv->vdev = vdev;
129 file_priv->priority = DRM_IVPU_CONTEXT_PRIORITY_NORMAL;160 file_priv->priority = DRM_IVPU_CONTEXT_PRIORITY_NORMAL;
130 kref_init(&file_priv->ref);161 kref_init(&file_priv->ref);
131 162
163 ret = ivpu_mmu_user_context_init(vdev, &file_priv->ctx, ctx_id);
164 if (ret)
165 goto err_free_file_priv;
166
167 old = xa_store_irq(&vdev->context_xa, ctx_id, file_priv, GFP_KERNEL);
168 if (xa_is_err(old)) {
169 ret = xa_err(old);
170 ivpu_err(vdev, "Failed to store context %u: %d\n", ctx_id, ret);
171 goto err_ctx_fini;
172 }
173
174 ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n",
175 ctx_id, current->comm, task_pid_nr(current));
176
132 file->driver_priv = file_priv;177 file->driver_priv = file_priv;
133 return 0;178 return 0;
179
180err_ctx_fini:
181 ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
182err_free_file_priv:
183 kfree(file_priv);
184err_xa_erase:
185 xa_erase_irq(&vdev->context_xa, ctx_id);
186 return ret;
134}187}
135 188
136static void ivpu_postclose(struct drm_device *dev, struct drm_file *file)189static void ivpu_postclose(struct drm_device *dev, struct drm_file *file)
137{190{
138 struct ivpu_file_priv *file_priv = file->driver_priv;191 struct ivpu_file_priv *file_priv = file->driver_priv;
192 struct ivpu_device *vdev = to_ivpu_device(dev);
193
194 ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n",
195 file_priv->ctx.id, current->comm, task_pid_nr(current));
139 196
140 ivpu_file_priv_put(&file_priv);197 ivpu_file_priv_put(&file_priv);
141}198}
@@ -150,6 +207,7 @@ int ivpu_shutdown(struct ivpu_device *vdev) · 8 unmodified lines
150 int ret;207 int ret;
151 208
152 ivpu_hw_irq_disable(vdev);209 ivpu_hw_irq_disable(vdev);
210 ivpu_mmu_disable(vdev);
153 211
154 ret = ivpu_hw_power_down(vdev);212 ret = ivpu_hw_power_down(vdev);
155 if (ret)213 if (ret)
@@ -251,6 +309,10 @@ static int ivpu_dev_init(struct ivpu_device *vdev) · 95 unmodified lines
251 if (!vdev->hw)309 if (!vdev->hw)
252 return -ENOMEM;310 return -ENOMEM;
253 311
312 vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL);
313 if (!vdev->mmu)
314 return -ENOMEM;
315
254 vdev->hw->ops = &ivpu_hw_mtl_ops;316 vdev->hw->ops = &ivpu_hw_mtl_ops;
255 vdev->platform = IVPU_PLATFORM_INVALID;317 vdev->platform = IVPU_PLATFORM_INVALID;
256 vdev->context_xa_limit.min = IVPU_GLOBAL_CONTEXT_MMU_SSID + 1;318 vdev->context_xa_limit.min = IVPU_GLOBAL_CONTEXT_MMU_SSID + 1;
@@ -283,8 +345,24 @@ static int ivpu_dev_init(struct ivpu_device *vdev) · 26 unmodified lines
283 goto err_xa_destroy;345 goto err_xa_destroy;
284 }346 }
285 347
348 ret = ivpu_mmu_global_context_init(vdev);
349 if (ret) {
350 ivpu_err(vdev, "Failed to initialize global MMU context: %d\n", ret);
351 goto err_power_down;
352 }
353
354 ret = ivpu_mmu_init(vdev);
355 if (ret) {
356 ivpu_err(vdev, "Failed to initialize MMU device: %d\n", ret);
357 goto err_mmu_gctx_fini;
358 }
359
286 return 0;360 return 0;
287 361
362err_mmu_gctx_fini:
363 ivpu_mmu_global_context_fini(vdev);
364err_power_down:
365 ivpu_hw_power_down(vdev);
288err_xa_destroy:366err_xa_destroy:
289 xa_destroy(&vdev->context_xa);367 xa_destroy(&vdev->context_xa);
290 return ret;368 return ret;
@@ -293,6 +371,7 @@ err_xa_destroy: · 2 unmodified lines
293static void ivpu_dev_fini(struct ivpu_device *vdev)371static void ivpu_dev_fini(struct ivpu_device *vdev)
294{372{
295 ivpu_shutdown(vdev);373 ivpu_shutdown(vdev);
374 ivpu_mmu_global_context_fini(vdev);
296 375
297 drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa));376 drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa));
298 xa_destroy(&vdev->context_xa);377 xa_destroy(&vdev->context_xa);
drivers/accel/ivpu/ivpu_drv.h+7−0
@@ -15,6 +15,8 @@ · 14 unmodified lines
15#include <linux/xarray.h>15#include <linux/xarray.h>
16#include <uapi/drm/ivpu_accel.h>16#include <uapi/drm/ivpu_accel.h>
17 17
18#include "ivpu_mmu_context.h"
19
18#define DRIVER_NAME "intel_vpu"20#define DRIVER_NAME "intel_vpu"
19#define DRIVER_DESC "Driver for Intel Versatile Processing Unit (VPU)"21#define DRIVER_DESC "Driver for Intel Versatile Processing Unit (VPU)"
20#define DRIVER_DATE "20230117"22#define DRIVER_DATE "20230117"
@@ -71,6 +73,7 @@ struct ivpu_wa_table { · 50 unmodified lines
71};73};
72 74
73struct ivpu_hw_info;75struct ivpu_hw_info;
76struct ivpu_mmu_info;
74 77
75struct ivpu_device {78struct ivpu_device {
76 struct drm_device drm;79 struct drm_device drm;
@@ -81,7 +84,9 @@ struct ivpu_device { · 4 unmodified lines
81 84
82 struct ivpu_wa_table wa;85 struct ivpu_wa_table wa;
83 struct ivpu_hw_info *hw;86 struct ivpu_hw_info *hw;
87 struct ivpu_mmu_info *mmu;
84 88
89 struct ivpu_mmu_context gctx;
85 struct xarray context_xa;90 struct xarray context_xa;
86 struct xa_limit context_xa_limit;91 struct xa_limit context_xa_limit;
87 92
@@ -100,7 +105,9 @@ struct ivpu_device { · 12 unmodified lines
100struct ivpu_file_priv {105struct ivpu_file_priv {
101 struct kref ref;106 struct kref ref;
102 struct ivpu_device *vdev;107 struct ivpu_device *vdev;
108 struct ivpu_mmu_context ctx;
103 u32 priority;109 u32 priority;
110 bool has_mmu_faults;
104};111};
105 112
106extern int ivpu_dbg_mask;113extern int ivpu_dbg_mask;
drivers/accel/ivpu/ivpu_hw_mtl.c+10−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#include "ivpu_hw_mtl_reg.h"7#include "ivpu_hw_mtl_reg.h"
8#include "ivpu_hw_reg_io.h"8#include "ivpu_hw_reg_io.h"
9#include "ivpu_hw.h"9#include "ivpu_hw.h"
10#include "ivpu_mmu.h"
10 11
11#define TILE_FUSE_ENABLE_BOTH 0x012#define TILE_FUSE_ENABLE_BOTH 0x0
12#define TILE_FUSE_ENABLE_UPPER 0x113#define TILE_FUSE_ENABLE_UPPER 0x1
@@ -930,6 +931,15 @@ static u32 ivpu_hw_mtl_irqv_handler(struct ivpu_device *vdev, int irq) · 917 unmodified lines
930 931
931 REGV_WR32(MTL_VPU_HOST_SS_ICB_CLEAR_0, status);932 REGV_WR32(MTL_VPU_HOST_SS_ICB_CLEAR_0, status);
932 933
934 if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
935 ivpu_mmu_irq_evtq_handler(vdev);
936
937 if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
938 ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
939
940 if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
941 ivpu_mmu_irq_gerr_handler(vdev);
942
933 if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))943 if (REG_TEST_FLD(MTL_VPU_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
934 ivpu_hw_mtl_irq_wdt_mss_handler(vdev);944 ivpu_hw_mtl_irq_wdt_mss_handler(vdev);
935 945
drivers/accel/ivpu/ivpu_mmu.c+876−0
@@ -0,0 +1,876 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7#include <linux/highmem.h>
8
9#include "ivpu_drv.h"
10#include "ivpu_hw_mtl_reg.h"
11#include "ivpu_hw_reg_io.h"
12#include "ivpu_mmu.h"
13#include "ivpu_mmu_context.h"
14
15#define IVPU_MMU_IDR0_REF 0x080f3e0f
16#define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f
17#define IVPU_MMU_IDR1_REF 0x0e739d18
18#define IVPU_MMU_IDR3_REF 0x0000003c
19#define IVPU_MMU_IDR5_REF 0x00040070
20#define IVPU_MMU_IDR5_REF_SIMICS 0x00000075
21#define IVPU_MMU_IDR5_REF_FPGA 0x00800075
22
23#define IVPU_MMU_CDTAB_ENT_SIZE 64
24#define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */
25#define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2)
26
27#define IVPU_MMU_STREAM_ID0 0
28#define IVPU_MMU_STREAM_ID3 3
29
30#define IVPU_MMU_STRTAB_ENT_SIZE 64
31#define IVPU_MMU_STRTAB_ENT_COUNT 4
32#define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2
33#define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE
34
35#define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */
36#define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2)
37#define IVPU_MMU_Q_WRAP_BIT (IVPU_MMU_Q_COUNT << 1)
38#define IVPU_MMU_Q_WRAP_MASK (IVPU_MMU_Q_WRAP_BIT - 1)
39#define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1)
40#define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK)
41
42#define IVPU_MMU_CMDQ_CMD_SIZE 16
43#define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE)
44
45#define IVPU_MMU_EVTQ_CMD_SIZE 32
46#define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE)
47
48#define IVPU_MMU_CMD_OPCODE GENMASK(7, 0)
49
50#define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12)
51#define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22)
52#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)
53#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)
54#define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32)
55
56#define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10)
57#define IVPU_MMU_CMD_CFGI_0_SSV BIT(11)
58#define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12)
59#define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32)
60#define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0)
61
62#define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48)
63#define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32)
64
65#define CMD_PREFETCH_CFG 0x1
66#define CMD_CFGI_STE 0x3
67#define CMD_CFGI_ALL 0x4
68#define CMD_CFGI_CD 0x5
69#define CMD_CFGI_CD_ALL 0x6
70#define CMD_TLBI_NH_ASID 0x11
71#define CMD_TLBI_EL2_ALL 0x20
72#define CMD_TLBI_NSNH_ALL 0x30
73#define CMD_SYNC 0x46
74
75#define IVPU_MMU_EVT_F_UUT 0x01
76#define IVPU_MMU_EVT_C_BAD_STREAMID 0x02
77#define IVPU_MMU_EVT_F_STE_FETCH 0x03
78#define IVPU_MMU_EVT_C_BAD_STE 0x04
79#define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05
80#define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06
81#define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07
82#define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08
83#define IVPU_MMU_EVT_F_CD_FETCH 0x09
84#define IVPU_MMU_EVT_C_BAD_CD 0x0a
85#define IVPU_MMU_EVT_F_WALK_EABT 0x0b
86#define IVPU_MMU_EVT_F_TRANSLATION 0x10
87#define IVPU_MMU_EVT_F_ADDR_SIZE 0x11
88#define IVPU_MMU_EVT_F_ACCESS 0x12
89#define IVPU_MMU_EVT_F_PERMISSION 0x13
90#define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20
91#define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21
92#define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24
93#define IVPU_MMU_EVT_F_VMS_FETCH 0x25
94
95#define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0)
96#define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12)
97
98#define IVPU_MMU_Q_BASE_RWA BIT(62)
99#define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
100#define IVPU_MMU_STRTAB_BASE_RA BIT(62)
101#define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
102
103#define IVPU_MMU_IRQ_EVTQ_EN BIT(2)
104#define IVPU_MMU_IRQ_GERROR_EN BIT(0)
105
106#define IVPU_MMU_CR0_ATSCHK BIT(4)
107#define IVPU_MMU_CR0_CMDQEN BIT(3)
108#define IVPU_MMU_CR0_EVTQEN BIT(2)
109#define IVPU_MMU_CR0_PRIQEN BIT(1)
110#define IVPU_MMU_CR0_SMMUEN BIT(0)
111
112#define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10)
113#define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8)
114#define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6)
115#define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4)
116#define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2)
117#define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0)
118#define IVPU_MMU_CACHE_NC 0
119#define IVPU_MMU_CACHE_WB 1
120#define IVPU_MMU_CACHE_WT 2
121#define IVPU_MMU_SH_NSH 0
122#define IVPU_MMU_SH_OSH 2
123#define IVPU_MMU_SH_ISH 3
124
125#define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0)
126
127#define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
128#define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
129#define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
130#define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
131#define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
132#define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14)
133#define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30)
134#define IVPU_MMU_CD_0_ENDI BIT(15)
135#define IVPU_MMU_CD_0_V BIT(31)
136#define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32)
137#define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38)
138#define IVPU_MMU_CD_0_AA64 BIT(41)
139#define IVPU_MMU_CD_0_S BIT(44)
140#define IVPU_MMU_CD_0_R BIT(45)
141#define IVPU_MMU_CD_0_A BIT(46)
142#define IVPU_MMU_CD_0_ASET BIT(47)
143#define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48)
144
145#define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
146
147#define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59)
148#define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4)
149#define IVPU_MMU_STE_0_S1FMT_LINEAR 0
150#define IVPU_MMU_STE_DWORDS 8
151#define IVPU_MMU_STE_0_CFG_S1_TRANS 5
152#define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1)
153#define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
154#define IVPU_MMU_STE_0_V BIT(0)
155
156#define IVPU_MMU_STE_1_STRW_NSEL1 0ul
157#define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13)
158#define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30)
159#define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48)
160#define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul
161#define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50)
162#define IVPU_MMU_STE_1_INSTCFG_DATA 2ul
163#define IVPU_MMU_STE_1_MEV BIT(19)
164#define IVPU_MMU_STE_1_S1STALLD BIT(27)
165#define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul
166#define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul
167#define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul
168#define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul
169#define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2)
170#define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4)
171#define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6)
172#define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0)
173#define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0
174
175#define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC)
176#define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC)
177
178#define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(MTL_VPU_HOST_MMU_GERROR, CMDQ)) | \
179 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, EVTQ_ABT)) | \
180 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, PRIQ_ABT)) | \
181 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_CMDQ_ABT)) | \
182 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_EVTQ_ABT)) | \
183 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_PRIQ_ABT)) | \
184 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_ABT)))
185
186static char *ivpu_mmu_event_to_str(u32 cmd)
187{
188 switch (cmd) {
189 case IVPU_MMU_EVT_F_UUT:
190 return "Unsupported Upstream Transaction";
191 case IVPU_MMU_EVT_C_BAD_STREAMID:
192 return "Transaction StreamID out of range";
193 case IVPU_MMU_EVT_F_STE_FETCH:
194 return "Fetch of STE caused external abort";
195 case IVPU_MMU_EVT_C_BAD_STE:
196 return "Used STE invalid";
197 case IVPU_MMU_EVT_F_BAD_ATS_TREQ:
198 return "Address Request disallowed for a StreamID";
199 case IVPU_MMU_EVT_F_STREAM_DISABLED:
200 return "Transaction marks non-substream disabled";
201 case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN:
202 return "MMU bypass is disallowed for this StreamID";
203 case IVPU_MMU_EVT_C_BAD_SUBSTREAMID:
204 return "Invalid StreamID";
205 case IVPU_MMU_EVT_F_CD_FETCH:
206 return "Fetch of CD caused external abort";
207 case IVPU_MMU_EVT_C_BAD_CD:
208 return "Fetched CD invalid";
209 case IVPU_MMU_EVT_F_WALK_EABT:
210 return " An external abort occurred fetching a TLB";
211 case IVPU_MMU_EVT_F_TRANSLATION:
212 return "Translation fault";
213 case IVPU_MMU_EVT_F_ADDR_SIZE:
214 return " Output address caused address size fault";
215 case IVPU_MMU_EVT_F_ACCESS:
216 return "Access flag fault";
217 case IVPU_MMU_EVT_F_PERMISSION:
218 return "Permission fault occurred on page access";
219 case IVPU_MMU_EVT_F_TLB_CONFLICT:
220 return "A TLB conflict";
221 case IVPU_MMU_EVT_F_CFG_CONFLICT:
222 return "A configuration cache conflict";
223 case IVPU_MMU_EVT_E_PAGE_REQUEST:
224 return "Page request hint from a client device";
225 case IVPU_MMU_EVT_F_VMS_FETCH:
226 return "Fetch of VMS caused external abort";
227 default:
228 return "Unknown CMDQ command";
229 }
230}
231
232static void ivpu_mmu_config_check(struct ivpu_device *vdev)
233{
234 u32 val_ref;
235 u32 val;
236
237 if (ivpu_is_simics(vdev))
238 val_ref = IVPU_MMU_IDR0_REF_SIMICS;
239 else
240 val_ref = IVPU_MMU_IDR0_REF;
241
242 val = REGV_RD32(MTL_VPU_HOST_MMU_IDR0);
243 if (val != val_ref)
244 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);
245
246 val = REGV_RD32(MTL_VPU_HOST_MMU_IDR1);
247 if (val != IVPU_MMU_IDR1_REF)
248 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);
249
250 val = REGV_RD32(MTL_VPU_HOST_MMU_IDR3);
251 if (val != IVPU_MMU_IDR3_REF)
252 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);
253
254 if (ivpu_is_simics(vdev))
255 val_ref = IVPU_MMU_IDR5_REF_SIMICS;
256 else if (ivpu_is_fpga(vdev))
257 val_ref = IVPU_MMU_IDR5_REF_FPGA;
258 else
259 val_ref = IVPU_MMU_IDR5_REF;
260
261 val = REGV_RD32(MTL_VPU_HOST_MMU_IDR5);
262 if (val != val_ref)
263 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);
264}
265
266static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)
267{
268 struct ivpu_mmu_info *mmu = vdev->mmu;
269 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
270 size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE;
271
272 cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL);
273 if (!cdtab->base)
274 return -ENOMEM;
275
276 ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);
277
278 return 0;
279}
280
281static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)
282{
283 struct ivpu_mmu_info *mmu = vdev->mmu;
284 struct ivpu_mmu_strtab *strtab = &mmu->strtab;
285 size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE;
286
287 strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL);
288 if (!strtab->base)
289 return -ENOMEM;
290
291 strtab->base_cfg = IVPU_MMU_STRTAB_CFG;
292 strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA;
293 strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK;
294
295 ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",
296 &strtab->dma, &strtab->dma_q, size);
297
298 return 0;
299}
300
301static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)
302{
303 struct ivpu_mmu_info *mmu = vdev->mmu;
304 struct ivpu_mmu_queue *q = &mmu->cmdq;
305
306 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL);
307 if (!q->base)
308 return -ENOMEM;
309
310 q->dma_q = IVPU_MMU_Q_BASE_RWA;
311 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
312 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;
313
314 ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",
315 &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE);
316
317 return 0;
318}
319
320static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)
321{
322 struct ivpu_mmu_info *mmu = vdev->mmu;
323 struct ivpu_mmu_queue *q = &mmu->evtq;
324
325 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL);
326 if (!q->base)
327 return -ENOMEM;
328
329 q->dma_q = IVPU_MMU_Q_BASE_RWA;
330 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
331 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;
332
333 ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",
334 &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE);
335
336 return 0;
337}
338
339static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)
340{
341 int ret;
342
343 ret = ivpu_mmu_cdtab_alloc(vdev);
344 if (ret) {
345 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret);
346 return ret;
347 }
348
349 ret = ivpu_mmu_strtab_alloc(vdev);
350 if (ret) {
351 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret);
352 return ret;
353 }
354
355 ret = ivpu_mmu_cmdq_alloc(vdev);
356 if (ret) {
357 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret);
358 return ret;
359 }
360
361 ret = ivpu_mmu_evtq_alloc(vdev);
362 if (ret)
363 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret);
364
365 return ret;
366}
367
368static int ivpu_mmu_reg_write(struct ivpu_device *vdev, u32 reg, u32 val)
369{
370 u32 reg_ack = reg + 4; /* ACK register is 4B after base register */
371 u32 val_ack;
372 int ret;
373
374 REGV_WR32(reg, val);
375
376 ret = REGV_POLL(reg_ack, val_ack, (val == val_ack), IVPU_MMU_REG_TIMEOUT_US);
377 if (ret)
378 ivpu_err(vdev, "Failed to write register 0x%x\n", reg);
379
380 return ret;
381}
382
383static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)
384{
385 u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN;
386 int ret;
387
388 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, 0);
389 if (ret)
390 return ret;
391
392 return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, irq_ctrl);
393}
394
395static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)
396{
397 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
398
399 return REGV_POLL(MTL_VPU_HOST_MMU_CMDQ_CONS, cmdq->cons, (cmdq->prod == cmdq->cons),
400 IVPU_MMU_QUEUE_TIMEOUT_US);
401}
402
403static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)
404{
405 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
406 u64 *queue_buffer = q->base;
407 int idx = IVPU_MMU_Q_IDX(q->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer));
408
409 if (!CIRC_SPACE(IVPU_MMU_Q_IDX(q->prod), IVPU_MMU_Q_IDX(q->cons), IVPU_MMU_Q_COUNT)) {
410 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);
411 return -EBUSY;
412 }
413
414 queue_buffer[idx] = data0;
415 queue_buffer[idx + 1] = data1;
416 q->prod = (q->prod + 1) & IVPU_MMU_Q_WRAP_MASK;
417
418 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);
419
420 return 0;
421}
422
423static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
424{
425 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
426 u64 val;
427 int ret;
428
429 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC) |
430 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_CS, 0x2) |
431 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSH, 0x3) |
432 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSI_ATTR, 0xf);
433
434 ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);
435 if (ret)
436 return ret;
437
438 clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE);
439 REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_PROD, q->prod);
440
441 ret = ivpu_mmu_cmdq_wait_for_cons(vdev);
442 if (ret)
443 ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret);
444
445 return ret;
446}
447
448static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)
449{
450 u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL);
451 u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f);
452
453 return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1);
454}
455
456static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)
457{
458 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) |
459 FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid);
460
461 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0);
462}
463
464static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)
465{
466 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL);
467
468 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0);
469}
470
471static int ivpu_mmu_reset(struct ivpu_device *vdev)
472{
473 struct ivpu_mmu_info *mmu = vdev->mmu;
474 u32 val;
475 int ret;
476
477 memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE);
478 clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE);
479 mmu->cmdq.prod = 0;
480 mmu->cmdq.cons = 0;
481
482 memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE);
483 clflush_cache_range(mmu->evtq.base, IVPU_MMU_EVTQ_SIZE);
484 mmu->evtq.prod = 0;
485 mmu->evtq.cons = 0;
486
487 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, 0);
488 if (ret)
489 return ret;
490
491 val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) |
492 FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) |
493 FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) |
494 FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) |
495 FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) |
496 FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB);
497 REGV_WR32(MTL_VPU_HOST_MMU_CR1, val);
498
499 REGV_WR64(MTL_VPU_HOST_MMU_STRTAB_BASE, mmu->strtab.dma_q);
500 REGV_WR32(MTL_VPU_HOST_MMU_STRTAB_BASE_CFG, mmu->strtab.base_cfg);
501
502 REGV_WR64(MTL_VPU_HOST_MMU_CMDQ_BASE, mmu->cmdq.dma_q);
503 REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_PROD, 0);
504 REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_CONS, 0);
505
506 val = IVPU_MMU_CR0_CMDQEN;
507 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
508 if (ret)
509 return ret;
510
511 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
512 if (ret)
513 return ret;
514
515 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
516 if (ret)
517 return ret;
518
519 ret = ivpu_mmu_cmdq_sync(vdev);
520 if (ret)
521 return ret;
522
523 REGV_WR64(MTL_VPU_HOST_MMU_EVTQ_BASE, mmu->evtq.dma_q);
524 REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_PROD_SEC, 0);
525 REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_CONS_SEC, 0);
526
527 val |= IVPU_MMU_CR0_EVTQEN;
528 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
529 if (ret)
530 return ret;
531
532 val |= IVPU_MMU_CR0_ATSCHK;
533 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
534 if (ret)
535 return ret;
536
537 ret = ivpu_mmu_irqs_setup(vdev);
538 if (ret)
539 return ret;
540
541 val |= IVPU_MMU_CR0_SMMUEN;
542 return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
543}
544
545static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)
546{
547 struct ivpu_mmu_info *mmu = vdev->mmu;
548 struct ivpu_mmu_strtab *strtab = &mmu->strtab;
549 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
550 u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE);
551 u64 str[2];
552
553 str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) |
554 FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) |
555 FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) |
556 IVPU_MMU_STE_0_V |
557 (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK);
558
559 str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) |
560 FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) |
561 FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) |
562 FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) |
563 FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) |
564 FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) |
565 FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) |
566 FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) |
567 IVPU_MMU_STE_1_MEV |
568 IVPU_MMU_STE_1_S1STALLD;
569
570 WRITE_ONCE(entry[1], str[1]);
571 WRITE_ONCE(entry[0], str[0]);
572
573 clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE);
574
575 ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);
576}
577
578static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)
579{
580 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0);
581 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);
582
583 return 0;
584}
585
586int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)
587{
588 struct ivpu_mmu_info *mmu = vdev->mmu;
589 int ret;
590
591 ret = mutex_lock_interruptible(&mmu->lock);
592 if (ret)
593 return ret;
594
595 if (!mmu->on) {
596 ret = 0;
597 goto unlock;
598 }
599
600 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);
601 if (ret)
602 goto unlock;
603
604 ret = ivpu_mmu_cmdq_sync(vdev);
605unlock:
606 mutex_unlock(&mmu->lock);
607 return ret;
608}
609
610static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)
611{
612 struct ivpu_mmu_info *mmu = vdev->mmu;
613 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
614 u64 *entry;
615 u64 cd[4];
616 int ret;
617
618 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
619 return -EINVAL;
620
621 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
622
623 if (cd_dma != 0) {
624 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, 26) |
625 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) |
626 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) |
627 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) |
628 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) |
629 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, 3) |
630 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) |
631 IVPU_MMU_CD_0_TCR_EPD1 |
632 IVPU_MMU_CD_0_AA64 |
633 IVPU_MMU_CD_0_R |
634 IVPU_MMU_CD_0_ASET |
635 IVPU_MMU_CD_0_V;
636 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK;
637 cd[2] = 0;
638 cd[3] = 0x0000000000007444;
639
640 /* For global context generate memory fault on VPU */
641 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID)
642 cd[0] |= IVPU_MMU_CD_0_A;
643 } else {
644 memset(cd, 0, sizeof(cd));
645 }
646
647 WRITE_ONCE(entry[1], cd[1]);
648 WRITE_ONCE(entry[2], cd[2]);
649 WRITE_ONCE(entry[3], cd[3]);
650 WRITE_ONCE(entry[0], cd[0]);
651
652 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
653
654 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
655 cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]);
656
657 ret = mutex_lock_interruptible(&mmu->lock);
658 if (ret)
659 return ret;
660
661 if (!mmu->on) {
662 ret = 0;
663 goto unlock;
664 }
665
666 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
667 if (ret)
668 goto unlock;
669
670 ret = ivpu_mmu_cmdq_sync(vdev);
671unlock:
672 mutex_unlock(&mmu->lock);
673 return ret;
674}
675
676static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev)
677{
678 int ret;
679
680 ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma);
681 if (ret)
682 ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret);
683
684 return ret;
685}
686
687static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma)
688{
689 int ret;
690
691 if (ssid == 0) {
692 ivpu_err(vdev, "Invalid SSID: %u\n", ssid);
693 return -EINVAL;
694 }
695
696 ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma);
697 if (ret)
698 ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret);
699
700 return ret;
701}
702
703int ivpu_mmu_init(struct ivpu_device *vdev)
704{
705 struct ivpu_mmu_info *mmu = vdev->mmu;
706 int ret;
707
708 ivpu_dbg(vdev, MMU, "Init..\n");
709
710 drmm_mutex_init(&vdev->drm, &mmu->lock);
711 ivpu_mmu_config_check(vdev);
712
713 ret = ivpu_mmu_structs_alloc(vdev);
714 if (ret)
715 return ret;
716
717 ret = ivpu_mmu_strtab_init(vdev);
718 if (ret) {
719 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);
720 return ret;
721 }
722
723 ret = ivpu_mmu_cd_add_gbl(vdev);
724 if (ret) {
725 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);
726 return ret;
727 }
728
729 ret = ivpu_mmu_enable(vdev);
730 if (ret) {
731 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
732 return ret;
733 }
734
735 ivpu_dbg(vdev, MMU, "Init done\n");
736
737 return 0;
738}
739
740int ivpu_mmu_enable(struct ivpu_device *vdev)
741{
742 struct ivpu_mmu_info *mmu = vdev->mmu;
743 int ret;
744
745 mutex_lock(&mmu->lock);
746
747 mmu->on = true;
748
749 ret = ivpu_mmu_reset(vdev);
750 if (ret) {
751 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);
752 goto err;
753 }
754
755 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
756 if (ret)
757 goto err;
758
759 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
760 if (ret)
761 goto err;
762
763 ret = ivpu_mmu_cmdq_sync(vdev);
764 if (ret)
765 goto err;
766
767 mutex_unlock(&mmu->lock);
768
769 return 0;
770err:
771 mmu->on = false;
772 mutex_unlock(&mmu->lock);
773 return ret;
774}
775
776void ivpu_mmu_disable(struct ivpu_device *vdev)
777{
778 struct ivpu_mmu_info *mmu = vdev->mmu;
779
780 mutex_lock(&mmu->lock);
781 mmu->on = false;
782 mutex_unlock(&mmu->lock);
783}
784
785static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event)
786{
787 u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
788 u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]);
789 u64 fetch_addr = ((u64)event[7]) << 32 | event[6];
790 u64 in_addr = ((u64)event[5]) << 32 | event[4];
791 u32 sid = event[1];
792
793 ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n",
794 op, ivpu_mmu_event_to_str(op), ssid, sid, event[2], event[3], in_addr, fetch_addr);
795}
796
797static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)
798{
799 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;
800 u32 idx = IVPU_MMU_Q_IDX(evtq->cons);
801 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE);
802
803 evtq->prod = REGV_RD32(MTL_VPU_HOST_MMU_EVTQ_PROD_SEC);
804 if (!CIRC_CNT(IVPU_MMU_Q_IDX(evtq->prod), IVPU_MMU_Q_IDX(evtq->cons), IVPU_MMU_Q_COUNT))
805 return NULL;
806
807 clflush_cache_range(evt, IVPU_MMU_EVTQ_CMD_SIZE);
808
809 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK;
810 REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_CONS_SEC, evtq->cons);
811
812 return evt;
813}
814
815void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
816{
817 u32 *event;
818 u32 ssid;
819
820 ivpu_dbg(vdev, IRQ, "MMU event queue\n");
821
822 while ((event = ivpu_mmu_get_event(vdev)) != NULL) {
823 ivpu_mmu_dump_event(vdev, event);
824
825 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
826 if (ssid != IVPU_GLOBAL_CONTEXT_MMU_SSID)
827 ivpu_mmu_user_context_mark_invalid(vdev, ssid);
828 }
829}
830
831void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)
832{
833 u32 gerror_val, gerrorn_val, active;
834
835 ivpu_dbg(vdev, IRQ, "MMU error\n");
836
837 gerror_val = REGV_RD32(MTL_VPU_HOST_MMU_GERROR);
838 gerrorn_val = REGV_RD32(MTL_VPU_HOST_MMU_GERRORN);
839
840 active = gerror_val ^ gerrorn_val;
841 if (!(active & IVPU_MMU_GERROR_ERR_MASK))
842 return;
843
844 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_ABT, active))
845 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");
846
847 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_PRIQ_ABT, active))
848 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");
849
850 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_EVTQ_ABT, active))
851 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");
852
853 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_CMDQ_ABT, active))
854 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");
855
856 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, PRIQ_ABT, active))
857 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");
858
859 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, EVTQ_ABT, active))
860 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");
861
862 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, CMDQ, active))
863 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");
864
865 REGV_WR32(MTL_VPU_HOST_MMU_GERRORN, gerror_val);
866}
867
868int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable)
869{
870 return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma);
871}
872
873void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid)
874{
875 ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */
876}
drivers/accel/ivpu/ivpu_mmu.h+50−0
@@ -0,0 +1,50 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#ifndef __IVPU_MMU_H__
7#define __IVPU_MMU_H__
8
9struct ivpu_device;
10
11struct ivpu_mmu_cdtab {
12 void *base;
13 dma_addr_t dma;
14};
15
16struct ivpu_mmu_strtab {
17 void *base;
18 dma_addr_t dma;
19 u64 dma_q;
20 u32 base_cfg;
21};
22
23struct ivpu_mmu_queue {
24 void *base;
25 dma_addr_t dma;
26 u64 dma_q;
27 u32 prod;
28 u32 cons;
29};
30
31struct ivpu_mmu_info {
32 struct mutex lock; /* Protects cdtab, strtab, cmdq, on */
33 struct ivpu_mmu_cdtab cdtab;
34 struct ivpu_mmu_strtab strtab;
35 struct ivpu_mmu_queue cmdq;
36 struct ivpu_mmu_queue evtq;
37 bool on;
38};
39
40int ivpu_mmu_init(struct ivpu_device *vdev);
41void ivpu_mmu_disable(struct ivpu_device *vdev);
42int ivpu_mmu_enable(struct ivpu_device *vdev);
43int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable);
44void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid);
45int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid);
46
47void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev);
48void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev);
49
50#endif /* __IVPU_MMU_H__ */
drivers/accel/ivpu/ivpu_mmu_context.c+398−0
@@ -0,0 +1,398 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/bitfield.h>
7#include <linux/highmem.h>
8
9#include "ivpu_drv.h"
10#include "ivpu_hw.h"
11#include "ivpu_mmu.h"
12#include "ivpu_mmu_context.h"
13
14#define IVPU_MMU_PGD_INDEX_MASK GENMASK(38, 30)
15#define IVPU_MMU_PMD_INDEX_MASK GENMASK(29, 21)
16#define IVPU_MMU_PTE_INDEX_MASK GENMASK(20, 12)
17#define IVPU_MMU_ENTRY_FLAGS_MASK GENMASK(11, 0)
18#define IVPU_MMU_ENTRY_FLAG_NG BIT(11)
19#define IVPU_MMU_ENTRY_FLAG_AF BIT(10)
20#define IVPU_MMU_ENTRY_FLAG_USER BIT(6)
21#define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2)
22#define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE BIT(1)
23#define IVPU_MMU_ENTRY_FLAG_VALID BIT(0)
24
25#define IVPU_MMU_PAGE_SIZE SZ_4K
26#define IVPU_MMU_PTE_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE)
27#define IVPU_MMU_PMD_MAP_SIZE (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE)
28#define IVPU_MMU_PGTABLE_SIZE (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64))
29
30#define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000
31#define IVPU_MMU_ENTRY_VALID (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID)
32#define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK)
33#define IVPU_MMU_ENTRY_MAPPED (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \
34 IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)
35
36static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
37{
38 dma_addr_t pgd_dma;
39 u64 *pgd;
40
41 pgd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, GFP_KERNEL);
42 if (!pgd)
43 return -ENOMEM;
44
45 pgtable->pgd = pgd;
46 pgtable->pgd_dma = pgd_dma;
47
48 return 0;
49}
50
51static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
52{
53 int pgd_index, pmd_index;
54
55 for (pgd_index = 0; pgd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_index) {
56 u64 **pmd_entries = pgtable->pgd_cpu_entries[pgd_index];
57 u64 *pmd = pgtable->pgd_entries[pgd_index];
58
59 if (!pmd_entries)
60 continue;
61
62 for (pmd_index = 0; pmd_index < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_index) {
63 if (pmd_entries[pmd_index])
64 dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE,
65 pmd_entries[pmd_index],
66 pmd[pmd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
67 }
68
69 kfree(pmd_entries);
70 dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd_entries[pgd_index],
71 pgtable->pgd[pgd_index] & ~IVPU_MMU_ENTRY_FLAGS_MASK);
72 }
73
74 dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pgtable->pgd,
75 pgtable->pgd_dma & ~IVPU_MMU_ENTRY_FLAGS_MASK);
76}
77
78static u64*
79ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, u64 pgd_index)
80{
81 u64 **pmd_entries;
82 dma_addr_t pmd_dma;
83 u64 *pmd;
84
85 if (pgtable->pgd_entries[pgd_index])
86 return pgtable->pgd_entries[pgd_index];
87
88 pmd = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
89 if (!pmd)
90 return NULL;
91
92 pmd_entries = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
93 if (!pmd_entries)
94 goto err_free_pgd;
95
96 pgtable->pgd_entries[pgd_index] = pmd;
97 pgtable->pgd_cpu_entries[pgd_index] = pmd_entries;
98 pgtable->pgd[pgd_index] = pmd_dma | IVPU_MMU_ENTRY_VALID;
99
100 return pmd;
101
102err_free_pgd:
103 dma_free_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, pmd, pmd_dma);
104 return NULL;
105}
106
107static u64*
108ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
109 int pgd_index, int pmd_index)
110{
111 dma_addr_t pte_dma;
112 u64 *pte;
113
114 if (pgtable->pgd_cpu_entries[pgd_index][pmd_index])
115 return pgtable->pgd_cpu_entries[pgd_index][pmd_index];
116
117 pte = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
118 if (!pte)
119 return NULL;
120
121 pgtable->pgd_cpu_entries[pgd_index][pmd_index] = pte;
122 pgtable->pgd_entries[pgd_index][pmd_index] = pte_dma | IVPU_MMU_ENTRY_VALID;
123
124 return pte;
125}
126
127static int
128ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
129 u64 vpu_addr, dma_addr_t dma_addr, int prot)
130{
131 u64 *pte;
132 int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
133 int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
134 int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
135
136 /* Allocate PMD - second level page table if needed */
137 if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_index))
138 return -ENOMEM;
139
140 /* Allocate PTE - third level page table if needed */
141 pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_index, pmd_index);
142 if (!pte)
143 return -ENOMEM;
144
145 /* Update PTE - third level page table with DMA address */
146 pte[pte_index] = dma_addr | prot;
147
148 return 0;
149}
150
151static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr)
152{
153 int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
154 int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
155 int pte_index = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
156
157 /* Update PTE with dummy physical address and clear flags */
158 ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index][pte_index] = IVPU_MMU_ENTRY_INVALID;
159}
160
161static void
162ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
163{
164 u64 end_addr = vpu_addr + size;
165 u64 *pgd = ctx->pgtable.pgd;
166
167 /* Align to PMD entry (2 MB) */
168 vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1);
169
170 while (vpu_addr < end_addr) {
171 int pgd_index = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
172 u64 pmd_end = (pgd_index + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE;
173 u64 *pmd = ctx->pgtable.pgd_entries[pgd_index];
174
175 while (vpu_addr < end_addr && vpu_addr < pmd_end) {
176 int pmd_index = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
177 u64 *pte = ctx->pgtable.pgd_cpu_entries[pgd_index][pmd_index];
178
179 clflush_cache_range(pte, IVPU_MMU_PGTABLE_SIZE);
180 vpu_addr += IVPU_MMU_PTE_MAP_SIZE;
181 }
182 clflush_cache_range(pmd, IVPU_MMU_PGTABLE_SIZE);
183 }
184 clflush_cache_range(pgd, IVPU_MMU_PGTABLE_SIZE);
185}
186
187static int
188ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
189 u64 vpu_addr, dma_addr_t dma_addr, size_t size, int prot)
190{
191 while (size) {
192 int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
193
194 if (ret)
195 return ret;
196
197 vpu_addr += IVPU_MMU_PAGE_SIZE;
198 dma_addr += IVPU_MMU_PAGE_SIZE;
199 size -= IVPU_MMU_PAGE_SIZE;
200 }
201
202 return 0;
203}
204
205static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
206{
207 while (size) {
208 ivpu_mmu_context_unmap_page(ctx, vpu_addr);
209 vpu_addr += IVPU_MMU_PAGE_SIZE;
210 size -= IVPU_MMU_PAGE_SIZE;
211 }
212}
213
214int
215ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
216 u64 vpu_addr, struct sg_table *sgt, bool llc_coherent)
217{
218 struct scatterlist *sg;
219 int prot;
220 int ret;
221 u64 i;
222
223 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
224 return -EINVAL;
225 /*
226 * VPU is only 32 bit, but DMA engine is 38 bit
227 * Ranges < 2 GB are reserved for VPU internal registers
228 * Limit range to 8 GB
229 */
230 if (vpu_addr < SZ_2G || vpu_addr > SZ_8G)
231 return -EINVAL;
232
233 prot = IVPU_MMU_ENTRY_MAPPED;
234 if (llc_coherent)
235 prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;
236
237 mutex_lock(&ctx->lock);
238
239 for_each_sgtable_dma_sg(sgt, sg, i) {
240 u64 dma_addr = sg_dma_address(sg) - sg->offset;
241 size_t size = sg_dma_len(sg) + sg->offset;
242
243 ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
244 if (ret) {
245 ivpu_err(vdev, "Failed to map context pages\n");
246 mutex_unlock(&ctx->lock);
247 return ret;
248 }
249 ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
250 vpu_addr += size;
251 }
252
253 mutex_unlock(&ctx->lock);
254
255 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
256 if (ret)
257 ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
258 return ret;
259}
260
261void
262ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
263 u64 vpu_addr, struct sg_table *sgt)
264{
265 struct scatterlist *sg;
266 int ret;
267 u64 i;
268
269 if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
270 ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr);
271
272 mutex_lock(&ctx->lock);
273
274 for_each_sgtable_dma_sg(sgt, sg, i) {
275 size_t size = sg_dma_len(sg) + sg->offset;
276
277 ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
278 ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
279 vpu_addr += size;
280 }
281
282 mutex_unlock(&ctx->lock);
283
284 ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
285 if (ret)
286 ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
287}
288
289int
290ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
291 const struct ivpu_addr_range *range,
292 u64 size, struct drm_mm_node *node)
293{
294 lockdep_assert_held(&ctx->lock);
295
296 return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE,
297 0, range->start, range->end, DRM_MM_INSERT_BEST);
298}
299
300void
301ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
302{
303 lockdep_assert_held(&ctx->lock);
304
305 drm_mm_remove_node(node);
306}
307
308static int
309ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id)
310{
311 u64 start, end;
312 int ret;
313
314 mutex_init(&ctx->lock);
315 INIT_LIST_HEAD(&ctx->bo_list);
316
317 ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);
318 if (ret)
319 return ret;
320
321 if (!context_id) {
322 start = vdev->hw->ranges.global_low.start;
323 end = vdev->hw->ranges.global_high.end;
324 } else {
325 start = vdev->hw->ranges.user_low.start;
326 end = vdev->hw->ranges.user_high.end;
327 }
328
329 drm_mm_init(&ctx->mm, start, end - start);
330 ctx->id = context_id;
331
332 return 0;
333}
334
335static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
336{
337 drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd);
338
339 mutex_destroy(&ctx->lock);
340 ivpu_mmu_pgtable_free(vdev, &ctx->pgtable);
341 drm_mm_takedown(&ctx->mm);
342}
343
344int ivpu_mmu_global_context_init(struct ivpu_device *vdev)
345{
346 return ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID);
347}
348
349void ivpu_mmu_global_context_fini(struct ivpu_device *vdev)
350{
351 return ivpu_mmu_context_fini(vdev, &vdev->gctx);
352}
353
354void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid)
355{
356 struct ivpu_file_priv *file_priv;
357
358 xa_lock(&vdev->context_xa);
359
360 file_priv = xa_load(&vdev->context_xa, ssid);
361 if (file_priv)
362 file_priv->has_mmu_faults = true;
363
364 xa_unlock(&vdev->context_xa);
365}
366
367int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id)
368{
369 int ret;
370
371 drm_WARN_ON(&vdev->drm, !ctx_id);
372
373 ret = ivpu_mmu_context_init(vdev, ctx, ctx_id);
374 if (ret) {
375 ivpu_err(vdev, "Failed to initialize context: %d\n", ret);
376 return ret;
377 }
378
379 ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable);
380 if (ret) {
381 ivpu_err(vdev, "Failed to set page table: %d\n", ret);
382 goto err_context_fini;
383 }
384
385 return 0;
386
387err_context_fini:
388 ivpu_mmu_context_fini(vdev, ctx);
389 return ret;
390}
391
392void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
393{
394 drm_WARN_ON(&vdev->drm, !ctx->id);
395
396 ivpu_mmu_clear_pgtable(vdev, ctx->id);
397 ivpu_mmu_context_fini(vdev, ctx);
398}
drivers/accel/ivpu/ivpu_mmu_context.h+50−0
@@ -0,0 +1,50 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#ifndef __IVPU_MMU_CONTEXT_H__
7#define __IVPU_MMU_CONTEXT_H__
8
9#include <drm/drm_mm.h>
10
11struct ivpu_device;
12struct ivpu_file_priv;
13struct ivpu_addr_range;
14
15#define IVPU_MMU_PGTABLE_ENTRIES 512
16
17struct ivpu_mmu_pgtable {
18 u64 **pgd_cpu_entries[IVPU_MMU_PGTABLE_ENTRIES];
19 u64 *pgd_entries[IVPU_MMU_PGTABLE_ENTRIES];
20 u64 *pgd;
21 dma_addr_t pgd_dma;
22};
23
24struct ivpu_mmu_context {
25 struct mutex lock; /* protects: mm, pgtable, bo_list */
26 struct drm_mm mm;
27 struct ivpu_mmu_pgtable pgtable;
28 struct list_head bo_list;
29 u32 id;
30};
31
32int ivpu_mmu_global_context_init(struct ivpu_device *vdev);
33void ivpu_mmu_global_context_fini(struct ivpu_device *vdev);
34
35int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id);
36void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx);
37void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid);
38
39int ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
40 const struct ivpu_addr_range *range,
41 u64 size, struct drm_mm_node *node);
42void ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx,
43 struct drm_mm_node *node);
44
45int ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
46 u64 vpu_addr, struct sg_table *sgt, bool llc_coherent);
47void ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
48 u64 vpu_addr, struct sg_table *sgt);
49
50#endif /* __IVPU_MMU_CONTEXT_H__ */
include/uapi/drm/ivpu_accel.h+4−0
@@ -38,6 +38,7 @@ extern "C" { · 37 unmodified lines
38#define DRM_IVPU_PARAM_NUM_CONTEXTS 438#define DRM_IVPU_PARAM_NUM_CONTEXTS 4
39#define DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS 539#define DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS 5
40#define DRM_IVPU_PARAM_CONTEXT_PRIORITY 640#define DRM_IVPU_PARAM_CONTEXT_PRIORITY 6
41#define DRM_IVPU_PARAM_CONTEXT_ID 7
41 42
42#define DRM_IVPU_PLATFORM_TYPE_SILICON 043#define DRM_IVPU_PLATFORM_TYPE_SILICON 0
43 44
@@ -78,6 +79,9 @@ struct drm_ivpu_param { · 34 unmodified lines
78 * Value of current context scheduling priority (read-write).79 * Value of current context scheduling priority (read-write).
79 * See DRM_IVPU_CONTEXT_PRIORITY_* for possible values.80 * See DRM_IVPU_CONTEXT_PRIORITY_* for possible values.
80 *81 *
82 * %DRM_IVPU_PARAM_CONTEXT_ID:
83 * Current context ID, always greater than 0 (read-only)
84 *
81 */85 */
82 __u32 param;86 __u32 param;
83 87

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t