History

accel/ivpu: Add GEM buffer object management

Adds four types of GEM-based BOs for the VPU: - shmem - internal - prime All types are implemented as struct ivpu_bo, based on struct drm_gem_object. VPU address is allocated when buffer is created except for imported prime buffers that allocate it in BO_INFO IOCTL due to missing file_priv arg in gem_prime_import callback. Internal buffers are pinned on creation, the rest of buffers types can be pinned on demand (in SUBMIT IOCTL). Buffer VPU address, allocated pages and mappings are released when the buffer is destroyed. Eviction mechanism is planned for future versions. Add two new IOCTLs: BO_CREATE, BO_INFO

Jacek Lawrynowicz committed 3 years ago

Commit 647371a1 parent 263b2ba6 files · +977 −2Browse files

Unified · Split

6 files
  1. drivers/accel/ivpu/Makefile +1 −0
  2. drivers/accel/ivpu/ivpu_drv.c +28 −2
  3. drivers/accel/ivpu/ivpu_drv.h +1 −0
  4. drivers/accel/ivpu/ivpu_gem.c +727 −0
  5. drivers/accel/ivpu/ivpu_gem.h +126 −0
  6. include/uapi/drm/ivpu_accel.h +94 −0
drivers/accel/ivpu/Makefile+1−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
3 3
4intel_vpu-y := \4intel_vpu-y := \
5 ivpu_drv.o \5 ivpu_drv.o \
6 ivpu_gem.o \
6 ivpu_hw_mtl.o \7 ivpu_hw_mtl.o \
7 ivpu_mmu.o \8 ivpu_mmu.o \
8 ivpu_mmu_context.o9 ivpu_mmu_context.o
drivers/accel/ivpu/ivpu_drv.c+28−2
@@ -12,8 +12,10 @@ · 11 unmodified lines
12#include <drm/drm_file.h>12#include <drm/drm_file.h>
13#include <drm/drm_gem.h>13#include <drm/drm_gem.h>
14#include <drm/drm_ioctl.h>14#include <drm/drm_ioctl.h>
15#include <drm/drm_prime.h>
15 16
16#include "ivpu_drv.h"17#include "ivpu_drv.h"
18#include "ivpu_gem.h"
17#include "ivpu_hw.h"19#include "ivpu_hw.h"
18#include "ivpu_mmu.h"20#include "ivpu_mmu.h"
19#include "ivpu_mmu_context.h"21#include "ivpu_mmu_context.h"
@@ -49,6 +51,24 @@ struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv) · 29 unmodified lines
49 return file_priv;51 return file_priv;
50}52}
51 53
54struct ivpu_file_priv *ivpu_file_priv_get_by_ctx_id(struct ivpu_device *vdev, unsigned long id)
55{
56 struct ivpu_file_priv *file_priv;
57
58 xa_lock_irq(&vdev->context_xa);
59 file_priv = xa_load(&vdev->context_xa, id);
60 /* file_priv may still be in context_xa during file_priv_release() */
61 if (file_priv && !kref_get_unless_zero(&file_priv->ref))
62 file_priv = NULL;
63 xa_unlock_irq(&vdev->context_xa);
64
65 if (file_priv)
66 ivpu_dbg(vdev, KREF, "file_priv get by id: ctx %u refcount %u\n",
67 file_priv->ctx.id, kref_read(&file_priv->ref));
68
69 return file_priv;
70}
71
52static void file_priv_release(struct kref *ref)72static void file_priv_release(struct kref *ref)
53{73{
54 struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref);74 struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref);
@@ -57,7 +77,7 @@ static void file_priv_release(struct kref *ref) · 2 unmodified lines
57 ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id);77 ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id);
58 78
59 ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);79 ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
60 WARN_ON(xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);80 drm_WARN_ON(&vdev->drm, xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);
61 kfree(file_priv);81 kfree(file_priv);
62}82}
63 83
@@ -66,7 +86,7 @@ void ivpu_file_priv_put(struct ivpu_file_priv **link) · 2 unmodified lines
66 struct ivpu_file_priv *file_priv = *link;86 struct ivpu_file_priv *file_priv = *link;
67 struct ivpu_device *vdev = file_priv->vdev;87 struct ivpu_device *vdev = file_priv->vdev;
68 88
69 WARN_ON(!file_priv);89 drm_WARN_ON(&vdev->drm, !file_priv);
70 90
71 ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",91 ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",
72 file_priv->ctx.id, kref_read(&file_priv->ref));92 file_priv->ctx.id, kref_read(&file_priv->ref));
@@ -200,6 +220,8 @@ static void ivpu_postclose(struct drm_device *dev, struct drm_file *file) · 127 unmodified lines
200static const struct drm_ioctl_desc ivpu_drm_ioctls[] = {220static const struct drm_ioctl_desc ivpu_drm_ioctls[] = {
201 DRM_IOCTL_DEF_DRV(IVPU_GET_PARAM, ivpu_get_param_ioctl, 0),221 DRM_IOCTL_DEF_DRV(IVPU_GET_PARAM, ivpu_get_param_ioctl, 0),
202 DRM_IOCTL_DEF_DRV(IVPU_SET_PARAM, ivpu_set_param_ioctl, 0),222 DRM_IOCTL_DEF_DRV(IVPU_SET_PARAM, ivpu_set_param_ioctl, 0),
223 DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE, ivpu_bo_create_ioctl, 0),
224 DRM_IOCTL_DEF_DRV(IVPU_BO_INFO, ivpu_bo_info_ioctl, 0),
203};225};
204 226
205int ivpu_shutdown(struct ivpu_device *vdev)227int ivpu_shutdown(struct ivpu_device *vdev)
@@ -227,6 +249,10 @@ static const struct drm_driver driver = { · 21 unmodified lines
227 249
228 .open = ivpu_open,250 .open = ivpu_open,
229 .postclose = ivpu_postclose,251 .postclose = ivpu_postclose,
252 .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
253 .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
254 .gem_prime_import = ivpu_gem_prime_import,
255 .gem_prime_mmap = drm_gem_prime_mmap,
230 256
231 .ioctls = ivpu_drm_ioctls,257 .ioctls = ivpu_drm_ioctls,
232 .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls),258 .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls),
drivers/accel/ivpu/ivpu_drv.h+1−0
@@ -115,6 +115,7 @@ extern u8 ivpu_pll_min_ratio; · 114 unmodified lines
115extern u8 ivpu_pll_max_ratio;115extern u8 ivpu_pll_max_ratio;
116 116
117struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv);117struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv);
118struct ivpu_file_priv *ivpu_file_priv_get_by_ctx_id(struct ivpu_device *vdev, unsigned long id);
118void ivpu_file_priv_put(struct ivpu_file_priv **link);119void ivpu_file_priv_put(struct ivpu_file_priv **link);
119int ivpu_shutdown(struct ivpu_device *vdev);120int ivpu_shutdown(struct ivpu_device *vdev);
120 121
drivers/accel/ivpu/ivpu_gem.c+727−0
@@ -0,0 +1,727 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/dma-buf.h>
7#include <linux/highmem.h>
8#include <linux/module.h>
9#include <linux/set_memory.h>
10#include <linux/xarray.h>
11
12#include <drm/drm_cache.h>
13#include <drm/drm_debugfs.h>
14#include <drm/drm_file.h>
15#include <drm/drm_utils.h>
16
17#include "ivpu_drv.h"
18#include "ivpu_gem.h"
19#include "ivpu_hw.h"
20#include "ivpu_mmu.h"
21#include "ivpu_mmu_context.h"
22
23MODULE_IMPORT_NS(DMA_BUF);
24
25static const struct drm_gem_object_funcs ivpu_gem_funcs;
26
27static struct lock_class_key prime_bo_lock_class_key;
28
29static int __must_check prime_alloc_pages_locked(struct ivpu_bo *bo)
30{
31 /* Pages are managed by the underlying dma-buf */
32 return 0;
33}
34
35static void prime_free_pages_locked(struct ivpu_bo *bo)
36{
37 /* Pages are managed by the underlying dma-buf */
38}
39
40static int prime_map_pages_locked(struct ivpu_bo *bo)
41{
42 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
43 struct sg_table *sgt;
44
45 WARN_ON(!bo->base.import_attach);
46
47 sgt = dma_buf_map_attachment(bo->base.import_attach, DMA_BIDIRECTIONAL);
48 if (IS_ERR(sgt)) {
49 ivpu_err(vdev, "Failed to map attachment: %ld\n", PTR_ERR(sgt));
50 return PTR_ERR(sgt);
51 }
52
53 bo->sgt = sgt;
54 return 0;
55}
56
57static void prime_unmap_pages_locked(struct ivpu_bo *bo)
58{
59 WARN_ON(!bo->base.import_attach);
60
61 dma_buf_unmap_attachment(bo->base.import_attach, bo->sgt, DMA_BIDIRECTIONAL);
62 bo->sgt = NULL;
63}
64
65static const struct ivpu_bo_ops prime_ops = {
66 .type = IVPU_BO_TYPE_PRIME,
67 .name = "prime",
68 .alloc_pages = prime_alloc_pages_locked,
69 .free_pages = prime_free_pages_locked,
70 .map_pages = prime_map_pages_locked,
71 .unmap_pages = prime_unmap_pages_locked,
72};
73
74static int __must_check shmem_alloc_pages_locked(struct ivpu_bo *bo)
75{
76 int npages = bo->base.size >> PAGE_SHIFT;
77 struct page **pages;
78
79 pages = drm_gem_get_pages(&bo->base);
80 if (IS_ERR(pages))
81 return PTR_ERR(pages);
82
83 if (bo->flags & DRM_IVPU_BO_WC)
84 set_pages_array_wc(pages, npages);
85 else if (bo->flags & DRM_IVPU_BO_UNCACHED)
86 set_pages_array_uc(pages, npages);
87
88 bo->pages = pages;
89 return 0;
90}
91
92static void shmem_free_pages_locked(struct ivpu_bo *bo)
93{
94 if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
95 set_pages_array_wb(bo->pages, bo->base.size >> PAGE_SHIFT);
96
97 drm_gem_put_pages(&bo->base, bo->pages, true, false);
98 bo->pages = NULL;
99}
100
101static int ivpu_bo_map_pages_locked(struct ivpu_bo *bo)
102{
103 int npages = bo->base.size >> PAGE_SHIFT;
104 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
105 struct sg_table *sgt;
106 int ret;
107
108 sgt = drm_prime_pages_to_sg(&vdev->drm, bo->pages, npages);
109 if (IS_ERR(sgt)) {
110 ivpu_err(vdev, "Failed to allocate sgtable\n");
111 return PTR_ERR(sgt);
112 }
113
114 ret = dma_map_sgtable(vdev->drm.dev, sgt, DMA_BIDIRECTIONAL, 0);
115 if (ret) {
116 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
117 goto err_free_sgt;
118 }
119
120 bo->sgt = sgt;
121 return 0;
122
123err_free_sgt:
124 kfree(sgt);
125 return ret;
126}
127
128static void ivpu_bo_unmap_pages_locked(struct ivpu_bo *bo)
129{
130 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
131
132 dma_unmap_sgtable(vdev->drm.dev, bo->sgt, DMA_BIDIRECTIONAL, 0);
133 sg_free_table(bo->sgt);
134 kfree(bo->sgt);
135 bo->sgt = NULL;
136}
137
138static const struct ivpu_bo_ops shmem_ops = {
139 .type = IVPU_BO_TYPE_SHMEM,
140 .name = "shmem",
141 .alloc_pages = shmem_alloc_pages_locked,
142 .free_pages = shmem_free_pages_locked,
143 .map_pages = ivpu_bo_map_pages_locked,
144 .unmap_pages = ivpu_bo_unmap_pages_locked,
145};
146
147static int __must_check internal_alloc_pages_locked(struct ivpu_bo *bo)
148{
149 unsigned int i, npages = bo->base.size >> PAGE_SHIFT;
150 struct page **pages;
151 int ret;
152
153 pages = kvmalloc_array(npages, sizeof(*bo->pages), GFP_KERNEL);
154 if (!pages)
155 return -ENOMEM;
156
157 for (i = 0; i < npages; i++) {
158 pages[i] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
159 if (!pages[i]) {
160 ret = -ENOMEM;
161 goto err_free_pages;
162 }
163 cond_resched();
164 }
165
166 bo->pages = pages;
167 return 0;
168
169err_free_pages:
170 while (i--)
171 put_page(pages[i]);
172 kvfree(pages);
173 return ret;
174}
175
176static void internal_free_pages_locked(struct ivpu_bo *bo)
177{
178 unsigned int i, npages = bo->base.size >> PAGE_SHIFT;
179
180 for (i = 0; i < npages; i++)
181 put_page(bo->pages[i]);
182
183 kvfree(bo->pages);
184 bo->pages = NULL;
185}
186
187static const struct ivpu_bo_ops internal_ops = {
188 .type = IVPU_BO_TYPE_INTERNAL,
189 .name = "internal",
190 .alloc_pages = internal_alloc_pages_locked,
191 .free_pages = internal_free_pages_locked,
192 .map_pages = ivpu_bo_map_pages_locked,
193 .unmap_pages = ivpu_bo_unmap_pages_locked,
194};
195
196static int __must_check ivpu_bo_alloc_and_map_pages_locked(struct ivpu_bo *bo)
197{
198 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
199 int ret;
200
201 lockdep_assert_held(&bo->lock);
202 drm_WARN_ON(&vdev->drm, bo->sgt);
203
204 ret = bo->ops->alloc_pages(bo);
205 if (ret) {
206 ivpu_err(vdev, "Failed to allocate pages for BO: %d", ret);
207 return ret;
208 }
209
210 ret = bo->ops->map_pages(bo);
211 if (ret) {
212 ivpu_err(vdev, "Failed to map pages for BO: %d", ret);
213 goto err_free_pages;
214 }
215 return ret;
216
217err_free_pages:
218 bo->ops->free_pages(bo);
219 return ret;
220}
221
222static void ivpu_bo_unmap_and_free_pages(struct ivpu_bo *bo)
223{
224 mutex_lock(&bo->lock);
225
226 WARN_ON(!bo->sgt);
227 bo->ops->unmap_pages(bo);
228 WARN_ON(bo->sgt);
229 bo->ops->free_pages(bo);
230 WARN_ON(bo->pages);
231
232 mutex_unlock(&bo->lock);
233}
234
235/*
236 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
237 *
238 * This function pins physical memory pages, then maps the physical pages
239 * to IOMMU address space and finally updates the VPU MMU page tables
240 * to allow the VPU to translate VPU address to IOMMU address.
241 */
242int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
243{
244 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
245 int ret = 0;
246
247 mutex_lock(&bo->lock);
248
249 if (!bo->vpu_addr) {
250 ivpu_err(vdev, "vpu_addr not set for BO ctx_id: %d handle: %d\n",
251 bo->ctx->id, bo->handle);
252 ret = -EINVAL;
253 goto unlock;
254 }
255
256 if (!bo->sgt) {
257 ret = ivpu_bo_alloc_and_map_pages_locked(bo);
258 if (ret)
259 goto unlock;
260 }
261
262 if (!bo->mmu_mapped) {
263 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, bo->sgt,
264 ivpu_bo_is_snooped(bo));
265 if (ret) {
266 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
267 goto unlock;
268 }
269 bo->mmu_mapped = true;
270 }
271
272unlock:
273 mutex_unlock(&bo->lock);
274
275 return ret;
276}
277
278static int
279ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
280 const struct ivpu_addr_range *range)
281{
282 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
283 int ret;
284
285 if (!range) {
286 if (bo->flags & DRM_IVPU_BO_HIGH_MEM)
287 range = &vdev->hw->ranges.user_high;
288 else
289 range = &vdev->hw->ranges.user_low;
290 }
291
292 mutex_lock(&ctx->lock);
293 ret = ivpu_mmu_context_insert_node_locked(ctx, range, bo->base.size, &bo->mm_node);
294 if (!ret) {
295 bo->ctx = ctx;
296 bo->vpu_addr = bo->mm_node.start;
297 list_add_tail(&bo->ctx_node, &ctx->bo_list);
298 }
299 mutex_unlock(&ctx->lock);
300
301 return ret;
302}
303
304static void ivpu_bo_free_vpu_addr(struct ivpu_bo *bo)
305{
306 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
307 struct ivpu_mmu_context *ctx = bo->ctx;
308
309 ivpu_dbg(vdev, BO, "remove from ctx: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
310 ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
311
312 mutex_lock(&bo->lock);
313
314 if (bo->mmu_mapped) {
315 drm_WARN_ON(&vdev->drm, !bo->sgt);
316 ivpu_mmu_context_unmap_sgt(vdev, ctx, bo->vpu_addr, bo->sgt);
317 bo->mmu_mapped = false;
318 }
319
320 mutex_lock(&ctx->lock);
321 list_del(&bo->ctx_node);
322 bo->vpu_addr = 0;
323 bo->ctx = NULL;
324 ivpu_mmu_context_remove_node_locked(ctx, &bo->mm_node);
325 mutex_unlock(&ctx->lock);
326
327 mutex_unlock(&bo->lock);
328}
329
330void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx)
331{
332 struct ivpu_bo *bo, *tmp;
333
334 list_for_each_entry_safe(bo, tmp, &ctx->bo_list, ctx_node)
335 ivpu_bo_free_vpu_addr(bo);
336}
337
338static struct ivpu_bo *
339ivpu_bo_alloc(struct ivpu_device *vdev, struct ivpu_mmu_context *mmu_context,
340 u64 size, u32 flags, const struct ivpu_bo_ops *ops,
341 const struct ivpu_addr_range *range, u64 user_ptr)
342{
343 struct ivpu_bo *bo;
344 int ret = 0;
345
346 if (drm_WARN_ON(&vdev->drm, size == 0 || !PAGE_ALIGNED(size)))
347 return ERR_PTR(-EINVAL);
348
349 switch (flags & DRM_IVPU_BO_CACHE_MASK) {
350 case DRM_IVPU_BO_CACHED:
351 case DRM_IVPU_BO_UNCACHED:
352 case DRM_IVPU_BO_WC:
353 break;
354 default:
355 return ERR_PTR(-EINVAL);
356 }
357
358 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
359 if (!bo)
360 return ERR_PTR(-ENOMEM);
361
362 mutex_init(&bo->lock);
363 bo->base.funcs = &ivpu_gem_funcs;
364 bo->flags = flags;
365 bo->ops = ops;
366 bo->user_ptr = user_ptr;
367
368 if (ops->type == IVPU_BO_TYPE_SHMEM)
369 ret = drm_gem_object_init(&vdev->drm, &bo->base, size);
370 else
371 drm_gem_private_object_init(&vdev->drm, &bo->base, size);
372
373 if (ret) {
374 ivpu_err(vdev, "Failed to initialize drm object\n");
375 goto err_free;
376 }
377
378 if (flags & DRM_IVPU_BO_MAPPABLE) {
379 ret = drm_gem_create_mmap_offset(&bo->base);
380 if (ret) {
381 ivpu_err(vdev, "Failed to allocate mmap offset\n");
382 goto err_release;
383 }
384 }
385
386 if (mmu_context) {
387 ret = ivpu_bo_alloc_vpu_addr(bo, mmu_context, range);
388 if (ret) {
389 ivpu_err(vdev, "Failed to add BO to context: %d\n", ret);
390 goto err_release;
391 }
392 }
393
394 return bo;
395
396err_release:
397 drm_gem_object_release(&bo->base);
398err_free:
399 kfree(bo);
400 return ERR_PTR(ret);
401}
402
403static void ivpu_bo_free(struct drm_gem_object *obj)
404{
405 struct ivpu_bo *bo = to_ivpu_bo(obj);
406 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
407
408 if (bo->ctx)
409 ivpu_dbg(vdev, BO, "free: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
410 bo->ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
411 else
412 ivpu_dbg(vdev, BO, "free: ctx (released) allocated %d mmu_mapped %d\n",
413 (bool)bo->sgt, bo->mmu_mapped);
414
415 drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
416
417 vunmap(bo->kvaddr);
418
419 if (bo->ctx)
420 ivpu_bo_free_vpu_addr(bo);
421
422 if (bo->sgt)
423 ivpu_bo_unmap_and_free_pages(bo);
424
425 if (bo->base.import_attach)
426 drm_prime_gem_destroy(&bo->base, bo->sgt);
427
428 drm_gem_object_release(&bo->base);
429
430 mutex_destroy(&bo->lock);
431 kfree(bo);
432}
433
434static int ivpu_bo_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
435{
436 struct ivpu_bo *bo = to_ivpu_bo(obj);
437 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
438
439 ivpu_dbg(vdev, BO, "mmap: ctx %u handle %u vpu_addr 0x%llx size %zu type %s",
440 bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.size, bo->ops->name);
441
442 if (obj->import_attach) {
443 /* Drop the reference drm_gem_mmap_obj() acquired.*/
444 drm_gem_object_put(obj);
445 vma->vm_private_data = NULL;
446 return dma_buf_mmap(obj->dma_buf, vma, 0);
447 }
448
449 vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND;
450 vma->vm_page_prot = ivpu_bo_pgprot(bo, vm_get_page_prot(vma->vm_flags));
451
452 return 0;
453}
454
455static struct sg_table *ivpu_bo_get_sg_table(struct drm_gem_object *obj)
456{
457 struct ivpu_bo *bo = to_ivpu_bo(obj);
458 loff_t npages = obj->size >> PAGE_SHIFT;
459 int ret = 0;
460
461 mutex_lock(&bo->lock);
462
463 if (!bo->sgt)
464 ret = ivpu_bo_alloc_and_map_pages_locked(bo);
465
466 mutex_unlock(&bo->lock);
467
468 if (ret)
469 return ERR_PTR(ret);
470
471 return drm_prime_pages_to_sg(obj->dev, bo->pages, npages);
472}
473
474static vm_fault_t ivpu_vm_fault(struct vm_fault *vmf)
475{
476 struct vm_area_struct *vma = vmf->vma;
477 struct drm_gem_object *obj = vma->vm_private_data;
478 struct ivpu_bo *bo = to_ivpu_bo(obj);
479 loff_t npages = obj->size >> PAGE_SHIFT;
480 pgoff_t page_offset;
481 struct page *page;
482 vm_fault_t ret;
483 int err;
484
485 mutex_lock(&bo->lock);
486
487 if (!bo->sgt) {
488 err = ivpu_bo_alloc_and_map_pages_locked(bo);
489 if (err) {
490 ret = vmf_error(err);
491 goto unlock;
492 }
493 }
494
495 /* We don't use vmf->pgoff since that has the fake offset */
496 page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
497 if (page_offset >= npages) {
498 ret = VM_FAULT_SIGBUS;
499 } else {
500 page = bo->pages[page_offset];
501 ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page));
502 }
503
504unlock:
505 mutex_unlock(&bo->lock);
506
507 return ret;
508}
509
510static const struct vm_operations_struct ivpu_vm_ops = {
511 .fault = ivpu_vm_fault,
512 .open = drm_gem_vm_open,
513 .close = drm_gem_vm_close,
514};
515
516static const struct drm_gem_object_funcs ivpu_gem_funcs = {
517 .free = ivpu_bo_free,
518 .mmap = ivpu_bo_mmap,
519 .vm_ops = &ivpu_vm_ops,
520 .get_sg_table = ivpu_bo_get_sg_table,
521};
522
523int
524ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
525{
526 struct ivpu_file_priv *file_priv = file->driver_priv;
527 struct ivpu_device *vdev = file_priv->vdev;
528 struct drm_ivpu_bo_create *args = data;
529 u64 size = PAGE_ALIGN(args->size);
530 struct ivpu_bo *bo;
531 int ret;
532
533 if (args->flags & ~DRM_IVPU_BO_FLAGS)
534 return -EINVAL;
535
536 if (size == 0)
537 return -EINVAL;
538
539 bo = ivpu_bo_alloc(vdev, &file_priv->ctx, size, args->flags, &shmem_ops, NULL, 0);
540 if (IS_ERR(bo)) {
541 ivpu_err(vdev, "Failed to create BO: %pe (ctx %u size %llu flags 0x%x)",
542 bo, file_priv->ctx.id, args->size, args->flags);
543 return PTR_ERR(bo);
544 }
545
546 ret = drm_gem_handle_create(file, &bo->base, &bo->handle);
547 if (!ret) {
548 args->vpu_addr = bo->vpu_addr;
549 args->handle = bo->handle;
550 }
551
552 drm_gem_object_put(&bo->base);
553
554 ivpu_dbg(vdev, BO, "alloc shmem: ctx %u vpu_addr 0x%llx size %zu flags 0x%x\n",
555 file_priv->ctx.id, bo->vpu_addr, bo->base.size, bo->flags);
556
557 return ret;
558}
559
560struct ivpu_bo *
561ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags)
562{
563 const struct ivpu_addr_range *range;
564 struct ivpu_addr_range fixed_range;
565 struct ivpu_bo *bo;
566 pgprot_t prot;
567 int ret;
568
569 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(vpu_addr));
570 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
571
572 if (vpu_addr) {
573 fixed_range.start = vpu_addr;
574 fixed_range.end = vpu_addr + size;
575 range = &fixed_range;
576 } else {
577 range = &vdev->hw->ranges.global_low;
578 }
579
580 bo = ivpu_bo_alloc(vdev, &vdev->gctx, size, flags, &internal_ops, range, 0);
581 if (IS_ERR(bo)) {
582 ivpu_err(vdev, "Failed to create BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
583 bo, vpu_addr, size, flags);
584 return NULL;
585 }
586
587 ret = ivpu_bo_pin(bo);
588 if (ret)
589 goto err_put;
590
591 if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
592 drm_clflush_pages(bo->pages, bo->base.size >> PAGE_SHIFT);
593
594 prot = ivpu_bo_pgprot(bo, PAGE_KERNEL);
595 bo->kvaddr = vmap(bo->pages, bo->base.size >> PAGE_SHIFT, VM_MAP, prot);
596 if (!bo->kvaddr) {
597 ivpu_err(vdev, "Failed to map BO into kernel virtual memory\n");
598 goto err_put;
599 }
600
601 ivpu_dbg(vdev, BO, "alloc internal: ctx 0 vpu_addr 0x%llx size %zu flags 0x%x\n",
602 bo->vpu_addr, bo->base.size, flags);
603
604 return bo;
605
606err_put:
607 drm_gem_object_put(&bo->base);
608 return NULL;
609}
610
611void ivpu_bo_free_internal(struct ivpu_bo *bo)
612{
613 drm_gem_object_put(&bo->base);
614}
615
616struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *buf)
617{
618 struct ivpu_device *vdev = to_ivpu_device(dev);
619 struct dma_buf_attachment *attach;
620 struct ivpu_bo *bo;
621
622 attach = dma_buf_attach(buf, dev->dev);
623 if (IS_ERR(attach))
624 return ERR_CAST(attach);
625
626 get_dma_buf(buf);
627
628 bo = ivpu_bo_alloc(vdev, NULL, buf->size, DRM_IVPU_BO_MAPPABLE, &prime_ops, NULL, 0);
629 if (IS_ERR(bo)) {
630 ivpu_err(vdev, "Failed to import BO: %pe (size %lu)", bo, buf->size);
631 goto err_detach;
632 }
633
634 lockdep_set_class(&bo->lock, &prime_bo_lock_class_key);
635
636 bo->base.import_attach = attach;
637
638 return &bo->base;
639
640err_detach:
641 dma_buf_detach(buf, attach);
642 dma_buf_put(buf);
643 return ERR_CAST(bo);
644}
645
646int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
647{
648 struct ivpu_file_priv *file_priv = file->driver_priv;
649 struct ivpu_device *vdev = to_ivpu_device(dev);
650 struct drm_ivpu_bo_info *args = data;
651 struct drm_gem_object *obj;
652 struct ivpu_bo *bo;
653 int ret = 0;
654
655 obj = drm_gem_object_lookup(file, args->handle);
656 if (!obj)
657 return -ENOENT;
658
659 bo = to_ivpu_bo(obj);
660
661 mutex_lock(&bo->lock);
662
663 if (!bo->ctx) {
664 ret = ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, NULL);
665 if (ret) {
666 ivpu_err(vdev, "Failed to allocate vpu_addr: %d\n", ret);
667 goto unlock;
668 }
669 }
670
671 args->flags = bo->flags;
672 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
673 args->vpu_addr = bo->vpu_addr;
674 args->size = obj->size;
675unlock:
676 mutex_unlock(&bo->lock);
677 drm_gem_object_put(obj);
678 return ret;
679}
680
681static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
682{
683 unsigned long dma_refcount = 0;
684
685 if (bo->base.dma_buf && bo->base.dma_buf->file)
686 dma_refcount = atomic_long_read(&bo->base.dma_buf->file->f_count);
687
688 drm_printf(p, "%5u %6d %16llx %10lu %10u %12lu %14s\n",
689 bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.size,
690 kref_read(&bo->base.refcount), dma_refcount, bo->ops->name);
691}
692
693void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
694{
695 struct ivpu_device *vdev = to_ivpu_device(dev);
696 struct ivpu_file_priv *file_priv;
697 unsigned long ctx_id;
698 struct ivpu_bo *bo;
699
700 drm_printf(p, "%5s %6s %16s %10s %10s %12s %14s\n",
701 "ctx", "handle", "vpu_addr", "size", "refcount", "dma_refcount", "type");
702
703 mutex_lock(&vdev->gctx.lock);
704 list_for_each_entry(bo, &vdev->gctx.bo_list, ctx_node)
705 ivpu_bo_print_info(bo, p);
706 mutex_unlock(&vdev->gctx.lock);
707
708 xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
709 file_priv = ivpu_file_priv_get_by_ctx_id(vdev, ctx_id);
710 if (!file_priv)
711 continue;
712
713 mutex_lock(&file_priv->ctx.lock);
714 list_for_each_entry(bo, &file_priv->ctx.bo_list, ctx_node)
715 ivpu_bo_print_info(bo, p);
716 mutex_unlock(&file_priv->ctx.lock);
717
718 ivpu_file_priv_put(&file_priv);
719 }
720}
721
722void ivpu_bo_list_print(struct drm_device *dev)
723{
724 struct drm_printer p = drm_info_printer(dev->dev);
725
726 ivpu_bo_list(dev, &p);
727}
drivers/accel/ivpu/ivpu_gem.h+126−0
@@ -0,0 +1,126 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5#ifndef __IVPU_GEM_H__
6#define __IVPU_GEM_H__
7
8#include <drm/drm_gem.h>
9#include <drm/drm_mm.h>
10
11struct dma_buf;
12struct ivpu_bo_ops;
13struct ivpu_file_priv;
14
15struct ivpu_bo {
16 struct drm_gem_object base;
17 const struct ivpu_bo_ops *ops;
18
19 struct ivpu_mmu_context *ctx;
20 struct list_head ctx_node;
21 struct drm_mm_node mm_node;
22
23 struct mutex lock; /* Protects: pages, sgt, mmu_mapped */
24 struct sg_table *sgt;
25 struct page **pages;
26 bool mmu_mapped;
27
28 void *kvaddr;
29 u64 vpu_addr;
30 u32 handle;
31 u32 flags;
32 uintptr_t user_ptr;
33};
34
35enum ivpu_bo_type {
36 IVPU_BO_TYPE_SHMEM = 1,
37 IVPU_BO_TYPE_INTERNAL,
38 IVPU_BO_TYPE_PRIME,
39};
40
41struct ivpu_bo_ops {
42 enum ivpu_bo_type type;
43 const char *name;
44 int (*alloc_pages)(struct ivpu_bo *bo);
45 void (*free_pages)(struct ivpu_bo *bo);
46 int (*map_pages)(struct ivpu_bo *bo);
47 void (*unmap_pages)(struct ivpu_bo *bo);
48};
49
50int ivpu_bo_pin(struct ivpu_bo *bo);
51void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx);
52void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p);
53void ivpu_bo_list_print(struct drm_device *dev);
54
55struct ivpu_bo *
56ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags);
57void ivpu_bo_free_internal(struct ivpu_bo *bo);
58struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf);
59void ivpu_bo_unmap_sgt_and_remove_from_context(struct ivpu_bo *bo);
60
61int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
62int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
63int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
64
65static inline struct ivpu_bo *to_ivpu_bo(struct drm_gem_object *obj)
66{
67 return container_of(obj, struct ivpu_bo, base);
68}
69
70static inline struct page *ivpu_bo_get_page(struct ivpu_bo *bo, u64 offset)
71{
72 if (offset > bo->base.size || !bo->pages)
73 return NULL;
74
75 return bo->pages[offset / PAGE_SIZE];
76}
77
78static inline u32 ivpu_bo_cache_mode(struct ivpu_bo *bo)
79{
80 return bo->flags & DRM_IVPU_BO_CACHE_MASK;
81}
82
83static inline bool ivpu_bo_is_snooped(struct ivpu_bo *bo)
84{
85 return ivpu_bo_cache_mode(bo) == DRM_IVPU_BO_CACHED;
86}
87
88static inline pgprot_t ivpu_bo_pgprot(struct ivpu_bo *bo, pgprot_t prot)
89{
90 if (bo->flags & DRM_IVPU_BO_WC)
91 return pgprot_writecombine(prot);
92
93 if (bo->flags & DRM_IVPU_BO_UNCACHED)
94 return pgprot_noncached(prot);
95
96 return prot;
97}
98
99static inline struct ivpu_device *ivpu_bo_to_vdev(struct ivpu_bo *bo)
100{
101 return to_ivpu_device(bo->base.dev);
102}
103
104static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u32 vpu_addr)
105{
106 if (vpu_addr < bo->vpu_addr)
107 return NULL;
108
109 if (vpu_addr >= (bo->vpu_addr + bo->base.size))
110 return NULL;
111
112 return bo->kvaddr + (vpu_addr - bo->vpu_addr);
113}
114
115static inline u32 cpu_to_vpu_addr(struct ivpu_bo *bo, void *cpu_addr)
116{
117 if (cpu_addr < bo->kvaddr)
118 return 0;
119
120 if (cpu_addr >= (bo->kvaddr + bo->base.size))
121 return 0;
122
123 return bo->vpu_addr + (cpu_addr - bo->kvaddr);
124}
125
126#endif /* __IVPU_GEM_H__ */
include/uapi/drm/ivpu_accel.h+94−0
@@ -17,6 +17,8 @@ extern "C" { · 16 unmodified lines
17 17
18#define DRM_IVPU_GET_PARAM 0x0018#define DRM_IVPU_GET_PARAM 0x00
19#define DRM_IVPU_SET_PARAM 0x0119#define DRM_IVPU_SET_PARAM 0x01
20#define DRM_IVPU_BO_CREATE 0x02
21#define DRM_IVPU_BO_INFO 0x03
20 22
21#define DRM_IOCTL_IVPU_GET_PARAM \23#define DRM_IOCTL_IVPU_GET_PARAM \
22 DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_GET_PARAM, struct drm_ivpu_param)24 DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_GET_PARAM, struct drm_ivpu_param)
@@ -24,6 +26,12 @@ extern "C" { · 1 unmodified line
24#define DRM_IOCTL_IVPU_SET_PARAM \26#define DRM_IOCTL_IVPU_SET_PARAM \
25 DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_SET_PARAM, struct drm_ivpu_param)27 DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_SET_PARAM, struct drm_ivpu_param)
26 28
29#define DRM_IOCTL_IVPU_BO_CREATE \
30 DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_CREATE, struct drm_ivpu_bo_create)
31
32#define DRM_IOCTL_IVPU_BO_INFO \
33 DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_INFO, struct drm_ivpu_bo_info)
34
27/**35/**
28 * DOC: contexts36 * DOC: contexts
29 *37 *
@@ -92,6 +100,92 @@ struct drm_ivpu_param { · 62 unmodified lines
92 __u64 value;100 __u64 value;
93};101};
94 102
103#define DRM_IVPU_BO_HIGH_MEM 0x00000001
104#define DRM_IVPU_BO_MAPPABLE 0x00000002
105
106#define DRM_IVPU_BO_CACHED 0x00000000
107#define DRM_IVPU_BO_UNCACHED 0x00010000
108#define DRM_IVPU_BO_WC 0x00020000
109#define DRM_IVPU_BO_CACHE_MASK 0x00030000
110
111#define DRM_IVPU_BO_FLAGS \
112 (DRM_IVPU_BO_HIGH_MEM | \
113 DRM_IVPU_BO_MAPPABLE | \
114 DRM_IVPU_BO_CACHE_MASK)
115
116/**
117 * struct drm_ivpu_bo_create - Create BO backed by SHMEM
118 *
119 * Create GEM buffer object allocated in SHMEM memory.
120 */
121struct drm_ivpu_bo_create {
122 /** @size: The size in bytes of the allocated memory */
123 __u64 size;
124
125 /**
126 * @flags:
127 *
128 * Supported flags:
129 *
130 * %DRM_IVPU_BO_HIGH_MEM:
131 *
132 * Allocate VPU address from >4GB range.
133 * Buffer object with vpu address >4GB can be always accessed by the
134 * VPU DMA engine, but some HW generation may not be able to access
135 * this memory from then firmware running on the VPU management processor.
136 * Suitable for input, output and some scratch buffers.
137 *
138 * %DRM_IVPU_BO_MAPPABLE:
139 *
140 * Buffer object can be mapped using mmap().
141 *
142 * %DRM_IVPU_BO_CACHED:
143 *
144 * Allocated BO will be cached on host side (WB) and snooped on the VPU side.
145 * This is the default caching mode.
146 *
147 * %DRM_IVPU_BO_UNCACHED:
148 *
149 * Allocated BO will not be cached on host side nor snooped on the VPU side.
150 *
151 * %DRM_IVPU_BO_WC:
152 *
153 * Allocated BO will use write combining buffer for writes but reads will be
154 * uncached.
155 */
156 __u32 flags;
157
158 /** @handle: Returned GEM object handle */
159 __u32 handle;
160
161 /** @vpu_addr: Returned VPU virtual address */
162 __u64 vpu_addr;
163};
164
165/**
166 * struct drm_ivpu_bo_info - Query buffer object info
167 */
168struct drm_ivpu_bo_info {
169 /** @handle: Handle of the queried BO */
170 __u32 handle;
171
172 /** @flags: Returned flags used to create the BO */
173 __u32 flags;
174
175 /** @vpu_addr: Returned VPU virtual address */
176 __u64 vpu_addr;
177
178 /**
179 * @mmap_offset:
180 *
181 * Returned offset to be used in mmap(). 0 in case the BO is not mappable.
182 */
183 __u64 mmap_offset;
184
185 /** @size: Returned GEM object size, aligned to PAGE_SIZE */
186 __u64 size;
187};
188
95#if defined(__cplusplus)189#if defined(__cplusplus)
96}190}
97#endif191#endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t