History

accel/ivpu: Add GEM buffer object management

Adds four types of GEM-based BOs for the VPU: - shmem - internal - prime All types are implemented as struct ivpu_bo, based on struct drm_gem_object. VPU address is allocated when buffer is created except for imported prime buffers that allocate it in BO_INFO IOCTL due to missing file_priv arg in gem_prime_import callback. Internal buffers are pinned on creation, the rest of buffers types can be pinned on demand (in SUBMIT IOCTL). Buffer VPU address, allocated pages and mappings are released when the buffer is destroyed. Eviction mechanism is planned for future versions. Add two new IOCTLs: BO_CREATE, BO_INFO

Jacek Lawrynowicz committed 3 years ago

Commit 647371a1 parent 263b2ba6 files · +977 −2Browse files

Unified · Split

6 files
  1. drivers/accel/ivpu/Makefile +1 −0
  2. drivers/accel/ivpu/ivpu_drv.c +28 −2
  3. drivers/accel/ivpu/ivpu_drv.h +1 −0
  4. drivers/accel/ivpu/ivpu_gem.c +727 −0
  5. drivers/accel/ivpu/ivpu_gem.h +126 −0
  6. include/uapi/drm/ivpu_accel.h +94 −0
drivers/accel/ivpu/Makefile+1−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
33
44 intel_vpu-y := \
55 ivpu_drv.o \
6+ ivpu_gem.o \
67 ivpu_hw_mtl.o \
78 ivpu_mmu.o \
89 ivpu_mmu_context.o
drivers/accel/ivpu/ivpu_drv.c+28−2
@@ -12,8 +12,10 @@ · 11 unmodified lines
1212 #include <drm/drm_file.h>
1313 #include <drm/drm_gem.h>
1414 #include <drm/drm_ioctl.h>
15+#include <drm/drm_prime.h>
1516
1617 #include "ivpu_drv.h"
18+#include "ivpu_gem.h"
1719 #include "ivpu_hw.h"
1820 #include "ivpu_mmu.h"
1921 #include "ivpu_mmu_context.h"
@@ -49,6 +51,24 @@ struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv) · 29 unmodified lines
4951 return file_priv;
5052 }
5153
54+struct ivpu_file_priv *ivpu_file_priv_get_by_ctx_id(struct ivpu_device *vdev, unsigned long id)
55+{
56+ struct ivpu_file_priv *file_priv;
57+
58+ xa_lock_irq(&vdev->context_xa);
59+ file_priv = xa_load(&vdev->context_xa, id);
60+ /* file_priv may still be in context_xa during file_priv_release() */
61+ if (file_priv && !kref_get_unless_zero(&file_priv->ref))
62+ file_priv = NULL;
63+ xa_unlock_irq(&vdev->context_xa);
64+
65+ if (file_priv)
66+ ivpu_dbg(vdev, KREF, "file_priv get by id: ctx %u refcount %u\n",
67+ file_priv->ctx.id, kref_read(&file_priv->ref));
68+
69+ return file_priv;
70+}
71+
5272 static void file_priv_release(struct kref *ref)
5373 {
5474 struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref);
@@ -57,7 +77,7 @@ static void file_priv_release(struct kref *ref) · 2 unmodified lines
5777 ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id);
5878
5979 ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
60− WARN_ON(xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);
80+ drm_WARN_ON(&vdev->drm, xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);
6181 kfree(file_priv);
6282 }
6383
@@ -66,7 +86,7 @@ void ivpu_file_priv_put(struct ivpu_file_priv **link) · 2 unmodified lines
6686 struct ivpu_file_priv *file_priv = *link;
6787 struct ivpu_device *vdev = file_priv->vdev;
6888
69− WARN_ON(!file_priv);
89+ drm_WARN_ON(&vdev->drm, !file_priv);
7090
7191 ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",
7292 file_priv->ctx.id, kref_read(&file_priv->ref));
@@ -200,6 +220,8 @@ static void ivpu_postclose(struct drm_device *dev, struct drm_file *file) · 127 unmodified lines
200220 static const struct drm_ioctl_desc ivpu_drm_ioctls[] = {
201221 DRM_IOCTL_DEF_DRV(IVPU_GET_PARAM, ivpu_get_param_ioctl, 0),
202222 DRM_IOCTL_DEF_DRV(IVPU_SET_PARAM, ivpu_set_param_ioctl, 0),
223+ DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE, ivpu_bo_create_ioctl, 0),
224+ DRM_IOCTL_DEF_DRV(IVPU_BO_INFO, ivpu_bo_info_ioctl, 0),
203225 };
204226
205227 int ivpu_shutdown(struct ivpu_device *vdev)
@@ -227,6 +249,10 @@ static const struct drm_driver driver = { · 21 unmodified lines
227249
228250 .open = ivpu_open,
229251 .postclose = ivpu_postclose,
252+ .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
253+ .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
254+ .gem_prime_import = ivpu_gem_prime_import,
255+ .gem_prime_mmap = drm_gem_prime_mmap,
230256
231257 .ioctls = ivpu_drm_ioctls,
232258 .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls),
drivers/accel/ivpu/ivpu_drv.h+1−0
@@ -115,6 +115,7 @@ extern u8 ivpu_pll_min_ratio; · 114 unmodified lines
115115 extern u8 ivpu_pll_max_ratio;
116116
117117 struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv);
118+struct ivpu_file_priv *ivpu_file_priv_get_by_ctx_id(struct ivpu_device *vdev, unsigned long id);
118119 void ivpu_file_priv_put(struct ivpu_file_priv **link);
119120 int ivpu_shutdown(struct ivpu_device *vdev);
120121
drivers/accel/ivpu/ivpu_gem.c+727−0
@@ -0,0 +1,727 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Copyright (C) 2020-2023 Intel Corporation
4+ */
5+
6+#include <linux/dma-buf.h>
7+#include <linux/highmem.h>
8+#include <linux/module.h>
9+#include <linux/set_memory.h>
10+#include <linux/xarray.h>
11+
12+#include <drm/drm_cache.h>
13+#include <drm/drm_debugfs.h>
14+#include <drm/drm_file.h>
15+#include <drm/drm_utils.h>
16+
17+#include "ivpu_drv.h"
18+#include "ivpu_gem.h"
19+#include "ivpu_hw.h"
20+#include "ivpu_mmu.h"
21+#include "ivpu_mmu_context.h"
22+
23+MODULE_IMPORT_NS(DMA_BUF);
24+
25+static const struct drm_gem_object_funcs ivpu_gem_funcs;
26+
27+static struct lock_class_key prime_bo_lock_class_key;
28+
29+static int __must_check prime_alloc_pages_locked(struct ivpu_bo *bo)
30+{
31+ /* Pages are managed by the underlying dma-buf */
32+ return 0;
33+}
34+
35+static void prime_free_pages_locked(struct ivpu_bo *bo)
36+{
37+ /* Pages are managed by the underlying dma-buf */
38+}
39+
40+static int prime_map_pages_locked(struct ivpu_bo *bo)
41+{
42+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
43+ struct sg_table *sgt;
44+
45+ WARN_ON(!bo->base.import_attach);
46+
47+ sgt = dma_buf_map_attachment(bo->base.import_attach, DMA_BIDIRECTIONAL);
48+ if (IS_ERR(sgt)) {
49+ ivpu_err(vdev, "Failed to map attachment: %ld\n", PTR_ERR(sgt));
50+ return PTR_ERR(sgt);
51+ }
52+
53+ bo->sgt = sgt;
54+ return 0;
55+}
56+
57+static void prime_unmap_pages_locked(struct ivpu_bo *bo)
58+{
59+ WARN_ON(!bo->base.import_attach);
60+
61+ dma_buf_unmap_attachment(bo->base.import_attach, bo->sgt, DMA_BIDIRECTIONAL);
62+ bo->sgt = NULL;
63+}
64+
65+static const struct ivpu_bo_ops prime_ops = {
66+ .type = IVPU_BO_TYPE_PRIME,
67+ .name = "prime",
68+ .alloc_pages = prime_alloc_pages_locked,
69+ .free_pages = prime_free_pages_locked,
70+ .map_pages = prime_map_pages_locked,
71+ .unmap_pages = prime_unmap_pages_locked,
72+};
73+
74+static int __must_check shmem_alloc_pages_locked(struct ivpu_bo *bo)
75+{
76+ int npages = bo->base.size >> PAGE_SHIFT;
77+ struct page **pages;
78+
79+ pages = drm_gem_get_pages(&bo->base);
80+ if (IS_ERR(pages))
81+ return PTR_ERR(pages);
82+
83+ if (bo->flags & DRM_IVPU_BO_WC)
84+ set_pages_array_wc(pages, npages);
85+ else if (bo->flags & DRM_IVPU_BO_UNCACHED)
86+ set_pages_array_uc(pages, npages);
87+
88+ bo->pages = pages;
89+ return 0;
90+}
91+
92+static void shmem_free_pages_locked(struct ivpu_bo *bo)
93+{
94+ if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
95+ set_pages_array_wb(bo->pages, bo->base.size >> PAGE_SHIFT);
96+
97+ drm_gem_put_pages(&bo->base, bo->pages, true, false);
98+ bo->pages = NULL;
99+}
100+
101+static int ivpu_bo_map_pages_locked(struct ivpu_bo *bo)
102+{
103+ int npages = bo->base.size >> PAGE_SHIFT;
104+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
105+ struct sg_table *sgt;
106+ int ret;
107+
108+ sgt = drm_prime_pages_to_sg(&vdev->drm, bo->pages, npages);
109+ if (IS_ERR(sgt)) {
110+ ivpu_err(vdev, "Failed to allocate sgtable\n");
111+ return PTR_ERR(sgt);
112+ }
113+
114+ ret = dma_map_sgtable(vdev->drm.dev, sgt, DMA_BIDIRECTIONAL, 0);
115+ if (ret) {
116+ ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
117+ goto err_free_sgt;
118+ }
119+
120+ bo->sgt = sgt;
121+ return 0;
122+
123+err_free_sgt:
124+ kfree(sgt);
125+ return ret;
126+}
127+
128+static void ivpu_bo_unmap_pages_locked(struct ivpu_bo *bo)
129+{
130+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
131+
132+ dma_unmap_sgtable(vdev->drm.dev, bo->sgt, DMA_BIDIRECTIONAL, 0);
133+ sg_free_table(bo->sgt);
134+ kfree(bo->sgt);
135+ bo->sgt = NULL;
136+}
137+
138+static const struct ivpu_bo_ops shmem_ops = {
139+ .type = IVPU_BO_TYPE_SHMEM,
140+ .name = "shmem",
141+ .alloc_pages = shmem_alloc_pages_locked,
142+ .free_pages = shmem_free_pages_locked,
143+ .map_pages = ivpu_bo_map_pages_locked,
144+ .unmap_pages = ivpu_bo_unmap_pages_locked,
145+};
146+
147+static int __must_check internal_alloc_pages_locked(struct ivpu_bo *bo)
148+{
149+ unsigned int i, npages = bo->base.size >> PAGE_SHIFT;
150+ struct page **pages;
151+ int ret;
152+
153+ pages = kvmalloc_array(npages, sizeof(*bo->pages), GFP_KERNEL);
154+ if (!pages)
155+ return -ENOMEM;
156+
157+ for (i = 0; i < npages; i++) {
158+ pages[i] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
159+ if (!pages[i]) {
160+ ret = -ENOMEM;
161+ goto err_free_pages;
162+ }
163+ cond_resched();
164+ }
165+
166+ bo->pages = pages;
167+ return 0;
168+
169+err_free_pages:
170+ while (i--)
171+ put_page(pages[i]);
172+ kvfree(pages);
173+ return ret;
174+}
175+
176+static void internal_free_pages_locked(struct ivpu_bo *bo)
177+{
178+ unsigned int i, npages = bo->base.size >> PAGE_SHIFT;
179+
180+ for (i = 0; i < npages; i++)
181+ put_page(bo->pages[i]);
182+
183+ kvfree(bo->pages);
184+ bo->pages = NULL;
185+}
186+
187+static const struct ivpu_bo_ops internal_ops = {
188+ .type = IVPU_BO_TYPE_INTERNAL,
189+ .name = "internal",
190+ .alloc_pages = internal_alloc_pages_locked,
191+ .free_pages = internal_free_pages_locked,
192+ .map_pages = ivpu_bo_map_pages_locked,
193+ .unmap_pages = ivpu_bo_unmap_pages_locked,
194+};
195+
196+static int __must_check ivpu_bo_alloc_and_map_pages_locked(struct ivpu_bo *bo)
197+{
198+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
199+ int ret;
200+
201+ lockdep_assert_held(&bo->lock);
202+ drm_WARN_ON(&vdev->drm, bo->sgt);
203+
204+ ret = bo->ops->alloc_pages(bo);
205+ if (ret) {
206+ ivpu_err(vdev, "Failed to allocate pages for BO: %d", ret);
207+ return ret;
208+ }
209+
210+ ret = bo->ops->map_pages(bo);
211+ if (ret) {
212+ ivpu_err(vdev, "Failed to map pages for BO: %d", ret);
213+ goto err_free_pages;
214+ }
215+ return ret;
216+
217+err_free_pages:
218+ bo->ops->free_pages(bo);
219+ return ret;
220+}
221+
222+static void ivpu_bo_unmap_and_free_pages(struct ivpu_bo *bo)
223+{
224+ mutex_lock(&bo->lock);
225+
226+ WARN_ON(!bo->sgt);
227+ bo->ops->unmap_pages(bo);
228+ WARN_ON(bo->sgt);
229+ bo->ops->free_pages(bo);
230+ WARN_ON(bo->pages);
231+
232+ mutex_unlock(&bo->lock);
233+}
234+
235+/*
236+ * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
237+ *
238+ * This function pins physical memory pages, then maps the physical pages
239+ * to IOMMU address space and finally updates the VPU MMU page tables
240+ * to allow the VPU to translate VPU address to IOMMU address.
241+ */
242+int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
243+{
244+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
245+ int ret = 0;
246+
247+ mutex_lock(&bo->lock);
248+
249+ if (!bo->vpu_addr) {
250+ ivpu_err(vdev, "vpu_addr not set for BO ctx_id: %d handle: %d\n",
251+ bo->ctx->id, bo->handle);
252+ ret = -EINVAL;
253+ goto unlock;
254+ }
255+
256+ if (!bo->sgt) {
257+ ret = ivpu_bo_alloc_and_map_pages_locked(bo);
258+ if (ret)
259+ goto unlock;
260+ }
261+
262+ if (!bo->mmu_mapped) {
263+ ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, bo->sgt,
264+ ivpu_bo_is_snooped(bo));
265+ if (ret) {
266+ ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
267+ goto unlock;
268+ }
269+ bo->mmu_mapped = true;
270+ }
271+
272+unlock:
273+ mutex_unlock(&bo->lock);
274+
275+ return ret;
276+}
277+
278+static int
279+ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
280+ const struct ivpu_addr_range *range)
281+{
282+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
283+ int ret;
284+
285+ if (!range) {
286+ if (bo->flags & DRM_IVPU_BO_HIGH_MEM)
287+ range = &vdev->hw->ranges.user_high;
288+ else
289+ range = &vdev->hw->ranges.user_low;
290+ }
291+
292+ mutex_lock(&ctx->lock);
293+ ret = ivpu_mmu_context_insert_node_locked(ctx, range, bo->base.size, &bo->mm_node);
294+ if (!ret) {
295+ bo->ctx = ctx;
296+ bo->vpu_addr = bo->mm_node.start;
297+ list_add_tail(&bo->ctx_node, &ctx->bo_list);
298+ }
299+ mutex_unlock(&ctx->lock);
300+
301+ return ret;
302+}
303+
304+static void ivpu_bo_free_vpu_addr(struct ivpu_bo *bo)
305+{
306+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
307+ struct ivpu_mmu_context *ctx = bo->ctx;
308+
309+ ivpu_dbg(vdev, BO, "remove from ctx: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
310+ ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
311+
312+ mutex_lock(&bo->lock);
313+
314+ if (bo->mmu_mapped) {
315+ drm_WARN_ON(&vdev->drm, !bo->sgt);
316+ ivpu_mmu_context_unmap_sgt(vdev, ctx, bo->vpu_addr, bo->sgt);
317+ bo->mmu_mapped = false;
318+ }
319+
320+ mutex_lock(&ctx->lock);
321+ list_del(&bo->ctx_node);
322+ bo->vpu_addr = 0;
323+ bo->ctx = NULL;
324+ ivpu_mmu_context_remove_node_locked(ctx, &bo->mm_node);
325+ mutex_unlock(&ctx->lock);
326+
327+ mutex_unlock(&bo->lock);
328+}
329+
330+void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx)
331+{
332+ struct ivpu_bo *bo, *tmp;
333+
334+ list_for_each_entry_safe(bo, tmp, &ctx->bo_list, ctx_node)
335+ ivpu_bo_free_vpu_addr(bo);
336+}
337+
338+static struct ivpu_bo *
339+ivpu_bo_alloc(struct ivpu_device *vdev, struct ivpu_mmu_context *mmu_context,
340+ u64 size, u32 flags, const struct ivpu_bo_ops *ops,
341+ const struct ivpu_addr_range *range, u64 user_ptr)
342+{
343+ struct ivpu_bo *bo;
344+ int ret = 0;
345+
346+ if (drm_WARN_ON(&vdev->drm, size == 0 || !PAGE_ALIGNED(size)))
347+ return ERR_PTR(-EINVAL);
348+
349+ switch (flags & DRM_IVPU_BO_CACHE_MASK) {
350+ case DRM_IVPU_BO_CACHED:
351+ case DRM_IVPU_BO_UNCACHED:
352+ case DRM_IVPU_BO_WC:
353+ break;
354+ default:
355+ return ERR_PTR(-EINVAL);
356+ }
357+
358+ bo = kzalloc(sizeof(*bo), GFP_KERNEL);
359+ if (!bo)
360+ return ERR_PTR(-ENOMEM);
361+
362+ mutex_init(&bo->lock);
363+ bo->base.funcs = &ivpu_gem_funcs;
364+ bo->flags = flags;
365+ bo->ops = ops;
366+ bo->user_ptr = user_ptr;
367+
368+ if (ops->type == IVPU_BO_TYPE_SHMEM)
369+ ret = drm_gem_object_init(&vdev->drm, &bo->base, size);
370+ else
371+ drm_gem_private_object_init(&vdev->drm, &bo->base, size);
372+
373+ if (ret) {
374+ ivpu_err(vdev, "Failed to initialize drm object\n");
375+ goto err_free;
376+ }
377+
378+ if (flags & DRM_IVPU_BO_MAPPABLE) {
379+ ret = drm_gem_create_mmap_offset(&bo->base);
380+ if (ret) {
381+ ivpu_err(vdev, "Failed to allocate mmap offset\n");
382+ goto err_release;
383+ }
384+ }
385+
386+ if (mmu_context) {
387+ ret = ivpu_bo_alloc_vpu_addr(bo, mmu_context, range);
388+ if (ret) {
389+ ivpu_err(vdev, "Failed to add BO to context: %d\n", ret);
390+ goto err_release;
391+ }
392+ }
393+
394+ return bo;
395+
396+err_release:
397+ drm_gem_object_release(&bo->base);
398+err_free:
399+ kfree(bo);
400+ return ERR_PTR(ret);
401+}
402+
403+static void ivpu_bo_free(struct drm_gem_object *obj)
404+{
405+ struct ivpu_bo *bo = to_ivpu_bo(obj);
406+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
407+
408+ if (bo->ctx)
409+ ivpu_dbg(vdev, BO, "free: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
410+ bo->ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
411+ else
412+ ivpu_dbg(vdev, BO, "free: ctx (released) allocated %d mmu_mapped %d\n",
413+ (bool)bo->sgt, bo->mmu_mapped);
414+
415+ drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
416+
417+ vunmap(bo->kvaddr);
418+
419+ if (bo->ctx)
420+ ivpu_bo_free_vpu_addr(bo);
421+
422+ if (bo->sgt)
423+ ivpu_bo_unmap_and_free_pages(bo);
424+
425+ if (bo->base.import_attach)
426+ drm_prime_gem_destroy(&bo->base, bo->sgt);
427+
428+ drm_gem_object_release(&bo->base);
429+
430+ mutex_destroy(&bo->lock);
431+ kfree(bo);
432+}
433+
434+static int ivpu_bo_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
435+{
436+ struct ivpu_bo *bo = to_ivpu_bo(obj);
437+ struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
438+
439+ ivpu_dbg(vdev, BO, "mmap: ctx %u handle %u vpu_addr 0x%llx size %zu type %s",
440+ bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.size, bo->ops->name);
441+
442+ if (obj->import_attach) {
443+ /* Drop the reference drm_gem_mmap_obj() acquired.*/
444+ drm_gem_object_put(obj);
445+ vma->vm_private_data = NULL;
446+ return dma_buf_mmap(obj->dma_buf, vma, 0);
447+ }
448+
449+ vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND;
450+ vma->vm_page_prot = ivpu_bo_pgprot(bo, vm_get_page_prot(vma->vm_flags));
451+
452+ return 0;
453+}
454+
455+static struct sg_table *ivpu_bo_get_sg_table(struct drm_gem_object *obj)
456+{
457+ struct ivpu_bo *bo = to_ivpu_bo(obj);
458+ loff_t npages = obj->size >> PAGE_SHIFT;
459+ int ret = 0;
460+
461+ mutex_lock(&bo->lock);
462+
463+ if (!bo->sgt)
464+ ret = ivpu_bo_alloc_and_map_pages_locked(bo);
465+
466+ mutex_unlock(&bo->lock);
467+
468+ if (ret)
469+ return ERR_PTR(ret);
470+
471+ return drm_prime_pages_to_sg(obj->dev, bo->pages, npages);
472+}
473+
474+static vm_fault_t ivpu_vm_fault(struct vm_fault *vmf)
475+{
476+ struct vm_area_struct *vma = vmf->vma;
477+ struct drm_gem_object *obj = vma->vm_private_data;
478+ struct ivpu_bo *bo = to_ivpu_bo(obj);
479+ loff_t npages = obj->size >> PAGE_SHIFT;
480+ pgoff_t page_offset;
481+ struct page *page;
482+ vm_fault_t ret;
483+ int err;
484+
485+ mutex_lock(&bo->lock);
486+
487+ if (!bo->sgt) {
488+ err = ivpu_bo_alloc_and_map_pages_locked(bo);
489+ if (err) {
490+ ret = vmf_error(err);
491+ goto unlock;
492+ }
493+ }
494+
495+ /* We don't use vmf->pgoff since that has the fake offset */
496+ page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
497+ if (page_offset >= npages) {
498+ ret = VM_FAULT_SIGBUS;
499+ } else {
500+ page = bo->pages[page_offset];
501+ ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page));
502+ }
503+
504+unlock:
505+ mutex_unlock(&bo->lock);
506+
507+ return ret;
508+}
509+
510+static const struct vm_operations_struct ivpu_vm_ops = {
511+ .fault = ivpu_vm_fault,
512+ .open = drm_gem_vm_open,
513+ .close = drm_gem_vm_close,
514+};
515+
516+static const struct drm_gem_object_funcs ivpu_gem_funcs = {
517+ .free = ivpu_bo_free,
518+ .mmap = ivpu_bo_mmap,
519+ .vm_ops = &ivpu_vm_ops,
520+ .get_sg_table = ivpu_bo_get_sg_table,
521+};
522+
523+int
524+ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
525+{
526+ struct ivpu_file_priv *file_priv = file->driver_priv;
527+ struct ivpu_device *vdev = file_priv->vdev;
528+ struct drm_ivpu_bo_create *args = data;
529+ u64 size = PAGE_ALIGN(args->size);
530+ struct ivpu_bo *bo;
531+ int ret;
532+
533+ if (args->flags & ~DRM_IVPU_BO_FLAGS)
534+ return -EINVAL;
535+
536+ if (size == 0)
537+ return -EINVAL;
538+
539+ bo = ivpu_bo_alloc(vdev, &file_priv->ctx, size, args->flags, &shmem_ops, NULL, 0);
540+ if (IS_ERR(bo)) {
541+ ivpu_err(vdev, "Failed to create BO: %pe (ctx %u size %llu flags 0x%x)",
542+ bo, file_priv->ctx.id, args->size, args->flags);
543+ return PTR_ERR(bo);
544+ }
545+
546+ ret = drm_gem_handle_create(file, &bo->base, &bo->handle);
547+ if (!ret) {
548+ args->vpu_addr = bo->vpu_addr;
549+ args->handle = bo->handle;
550+ }
551+
552+ drm_gem_object_put(&bo->base);
553+
554+ ivpu_dbg(vdev, BO, "alloc shmem: ctx %u vpu_addr 0x%llx size %zu flags 0x%x\n",
555+ file_priv->ctx.id, bo->vpu_addr, bo->base.size, bo->flags);
556+
557+ return ret;
558+}
559+
560+struct ivpu_bo *
561+ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags)
562+{
563+ const struct ivpu_addr_range *range;
564+ struct ivpu_addr_range fixed_range;
565+ struct ivpu_bo *bo;
566+ pgprot_t prot;
567+ int ret;
568+
569+ drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(vpu_addr));
570+ drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
571+
572+ if (vpu_addr) {
573+ fixed_range.start = vpu_addr;
574+ fixed_range.end = vpu_addr + size;
575+ range = &fixed_range;
576+ } else {
577+ range = &vdev->hw->ranges.global_low;
578+ }
579+
580+ bo = ivpu_bo_alloc(vdev, &vdev->gctx, size, flags, &internal_ops, range, 0);
581+ if (IS_ERR(bo)) {
582+ ivpu_err(vdev, "Failed to create BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
583+ bo, vpu_addr, size, flags);
584+ return NULL;
585+ }
586+
587+ ret = ivpu_bo_pin(bo);
588+ if (ret)
589+ goto err_put;
590+
591+ if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
592+ drm_clflush_pages(bo->pages, bo->base.size >> PAGE_SHIFT);
593+
594+ prot = ivpu_bo_pgprot(bo, PAGE_KERNEL);
595+ bo->kvaddr = vmap(bo->pages, bo->base.size >> PAGE_SHIFT, VM_MAP, prot);
596+ if (!bo->kvaddr) {
597+ ivpu_err(vdev, "Failed to map BO into kernel virtual memory\n");
598+ goto err_put;
599+ }
600+
601+ ivpu_dbg(vdev, BO, "alloc internal: ctx 0 vpu_addr 0x%llx size %zu flags 0x%x\n",
602+ bo->vpu_addr, bo->base.size, flags);
603+
604+ return bo;
605+
606+err_put:
607+ drm_gem_object_put(&bo->base);
608+ return NULL;
609+}
610+
611+void ivpu_bo_free_internal(struct ivpu_bo *bo)
612+{
613+ drm_gem_object_put(&bo->base);
614+}
615+
616+struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *buf)
617+{
618+ struct ivpu_device *vdev = to_ivpu_device(dev);
619+ struct dma_buf_attachment *attach;
620+ struct ivpu_bo *bo;
621+
622+ attach = dma_buf_attach(buf, dev->dev);
623+ if (IS_ERR(attach))
624+ return ERR_CAST(attach);
625+
626+ get_dma_buf(buf);
627+
628+ bo = ivpu_bo_alloc(vdev, NULL, buf->size, DRM_IVPU_BO_MAPPABLE, &prime_ops, NULL, 0);
629+ if (IS_ERR(bo)) {
630+ ivpu_err(vdev, "Failed to import BO: %pe (size %lu)", bo, buf->size);
631+ goto err_detach;
632+ }
633+
634+ lockdep_set_class(&bo->lock, &prime_bo_lock_class_key);
635+
636+ bo->base.import_attach = attach;
637+
638+ return &bo->base;
639+
640+err_detach:
641+ dma_buf_detach(buf, attach);
642+ dma_buf_put(buf);
643+ return ERR_CAST(bo);
644+}
645+
646+int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
647+{
648+ struct ivpu_file_priv *file_priv = file->driver_priv;
649+ struct ivpu_device *vdev = to_ivpu_device(dev);
650+ struct drm_ivpu_bo_info *args = data;
651+ struct drm_gem_object *obj;
652+ struct ivpu_bo *bo;
653+ int ret = 0;
654+
655+ obj = drm_gem_object_lookup(file, args->handle);
656+ if (!obj)
657+ return -ENOENT;
658+
659+ bo = to_ivpu_bo(obj);
660+
661+ mutex_lock(&bo->lock);
662+
663+ if (!bo->ctx) {
664+ ret = ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, NULL);
665+ if (ret) {
666+ ivpu_err(vdev, "Failed to allocate vpu_addr: %d\n", ret);
667+ goto unlock;
668+ }
669+ }
670+
671+ args->flags = bo->flags;
672+ args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
673+ args->vpu_addr = bo->vpu_addr;
674+ args->size = obj->size;
675+unlock:
676+ mutex_unlock(&bo->lock);
677+ drm_gem_object_put(obj);
678+ return ret;
679+}
680+
681+static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
682+{
683+ unsigned long dma_refcount = 0;
684+
685+ if (bo->base.dma_buf && bo->base.dma_buf->file)
686+ dma_refcount = atomic_long_read(&bo->base.dma_buf->file->f_count);
687+
688+ drm_printf(p, "%5u %6d %16llx %10lu %10u %12lu %14s\n",
689+ bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.size,
690+ kref_read(&bo->base.refcount), dma_refcount, bo->ops->name);
691+}
692+
693+void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
694+{
695+ struct ivpu_device *vdev = to_ivpu_device(dev);
696+ struct ivpu_file_priv *file_priv;
697+ unsigned long ctx_id;
698+ struct ivpu_bo *bo;
699+
700+ drm_printf(p, "%5s %6s %16s %10s %10s %12s %14s\n",
701+ "ctx", "handle", "vpu_addr", "size", "refcount", "dma_refcount", "type");
702+
703+ mutex_lock(&vdev->gctx.lock);
704+ list_for_each_entry(bo, &vdev->gctx.bo_list, ctx_node)
705+ ivpu_bo_print_info(bo, p);
706+ mutex_unlock(&vdev->gctx.lock);
707+
708+ xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
709+ file_priv = ivpu_file_priv_get_by_ctx_id(vdev, ctx_id);
710+ if (!file_priv)
711+ continue;
712+
713+ mutex_lock(&file_priv->ctx.lock);
714+ list_for_each_entry(bo, &file_priv->ctx.bo_list, ctx_node)
715+ ivpu_bo_print_info(bo, p);
716+ mutex_unlock(&file_priv->ctx.lock);
717+
718+ ivpu_file_priv_put(&file_priv);
719+ }
720+}
721+
722+void ivpu_bo_list_print(struct drm_device *dev)
723+{
724+ struct drm_printer p = drm_info_printer(dev->dev);
725+
726+ ivpu_bo_list(dev, &p);
727+}
drivers/accel/ivpu/ivpu_gem.h+126−0
@@ -0,0 +1,126 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/*
3+ * Copyright (C) 2020-2023 Intel Corporation
4+ */
5+#ifndef __IVPU_GEM_H__
6+#define __IVPU_GEM_H__
7+
8+#include <drm/drm_gem.h>
9+#include <drm/drm_mm.h>
10+
11+struct dma_buf;
12+struct ivpu_bo_ops;
13+struct ivpu_file_priv;
14+
15+struct ivpu_bo {
16+ struct drm_gem_object base;
17+ const struct ivpu_bo_ops *ops;
18+
19+ struct ivpu_mmu_context *ctx;
20+ struct list_head ctx_node;
21+ struct drm_mm_node mm_node;
22+
23+ struct mutex lock; /* Protects: pages, sgt, mmu_mapped */
24+ struct sg_table *sgt;
25+ struct page **pages;
26+ bool mmu_mapped;
27+
28+ void *kvaddr;
29+ u64 vpu_addr;
30+ u32 handle;
31+ u32 flags;
32+ uintptr_t user_ptr;
33+};
34+
35+enum ivpu_bo_type {
36+ IVPU_BO_TYPE_SHMEM = 1,
37+ IVPU_BO_TYPE_INTERNAL,
38+ IVPU_BO_TYPE_PRIME,
39+};
40+
41+struct ivpu_bo_ops {
42+ enum ivpu_bo_type type;
43+ const char *name;
44+ int (*alloc_pages)(struct ivpu_bo *bo);
45+ void (*free_pages)(struct ivpu_bo *bo);
46+ int (*map_pages)(struct ivpu_bo *bo);
47+ void (*unmap_pages)(struct ivpu_bo *bo);
48+};
49+
50+int ivpu_bo_pin(struct ivpu_bo *bo);
51+void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx);
52+void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p);
53+void ivpu_bo_list_print(struct drm_device *dev);
54+
55+struct ivpu_bo *
56+ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags);
57+void ivpu_bo_free_internal(struct ivpu_bo *bo);
58+struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf);
59+void ivpu_bo_unmap_sgt_and_remove_from_context(struct ivpu_bo *bo);
60+
61+int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
62+int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
63+int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
64+
65+static inline struct ivpu_bo *to_ivpu_bo(struct drm_gem_object *obj)
66+{
67+ return container_of(obj, struct ivpu_bo, base);
68+}
69+
70+static inline struct page *ivpu_bo_get_page(struct ivpu_bo *bo, u64 offset)
71+{
72+ if (offset > bo->base.size || !bo->pages)
73+ return NULL;
74+
75+ return bo->pages[offset / PAGE_SIZE];
76+}
77+
78+static inline u32 ivpu_bo_cache_mode(struct ivpu_bo *bo)
79+{
80+ return bo->flags & DRM_IVPU_BO_CACHE_MASK;
81+}
82+
83+static inline bool ivpu_bo_is_snooped(struct ivpu_bo *bo)
84+{
85+ return ivpu_bo_cache_mode(bo) == DRM_IVPU_BO_CACHED;
86+}
87+
88+static inline pgprot_t ivpu_bo_pgprot(struct ivpu_bo *bo, pgprot_t prot)
89+{
90+ if (bo->flags & DRM_IVPU_BO_WC)
91+ return pgprot_writecombine(prot);
92+
93+ if (bo->flags & DRM_IVPU_BO_UNCACHED)
94+ return pgprot_noncached(prot);
95+
96+ return prot;
97+}
98+
99+static inline struct ivpu_device *ivpu_bo_to_vdev(struct ivpu_bo *bo)
100+{
101+ return to_ivpu_device(bo->base.dev);
102+}
103+
104+static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u32 vpu_addr)
105+{
106+ if (vpu_addr < bo->vpu_addr)
107+ return NULL;
108+
109+ if (vpu_addr >= (bo->vpu_addr + bo->base.size))
110+ return NULL;
111+
112+ return bo->kvaddr + (vpu_addr - bo->vpu_addr);
113+}
114+
115+static inline u32 cpu_to_vpu_addr(struct ivpu_bo *bo, void *cpu_addr)
116+{
117+ if (cpu_addr < bo->kvaddr)
118+ return 0;
119+
120+ if (cpu_addr >= (bo->kvaddr + bo->base.size))
121+ return 0;
122+
123+ return bo->vpu_addr + (cpu_addr - bo->kvaddr);
124+}
125+
126+#endif /* __IVPU_GEM_H__ */
include/uapi/drm/ivpu_accel.h+94−0
@@ -17,6 +17,8 @@ extern "C" { · 16 unmodified lines
1717
1818 #define DRM_IVPU_GET_PARAM 0x00
1919 #define DRM_IVPU_SET_PARAM 0x01
20+#define DRM_IVPU_BO_CREATE 0x02
21+#define DRM_IVPU_BO_INFO 0x03
2022
2123 #define DRM_IOCTL_IVPU_GET_PARAM \
2224 DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_GET_PARAM, struct drm_ivpu_param)
@@ -24,6 +26,12 @@ extern "C" { · 1 unmodified line
2426 #define DRM_IOCTL_IVPU_SET_PARAM \
2527 DRM_IOW(DRM_COMMAND_BASE + DRM_IVPU_SET_PARAM, struct drm_ivpu_param)
2628
29+#define DRM_IOCTL_IVPU_BO_CREATE \
30+ DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_CREATE, struct drm_ivpu_bo_create)
31+
32+#define DRM_IOCTL_IVPU_BO_INFO \
33+ DRM_IOWR(DRM_COMMAND_BASE + DRM_IVPU_BO_INFO, struct drm_ivpu_bo_info)
34+
2735 /**
2836 * DOC: contexts
2937 *
@@ -92,6 +100,92 @@ struct drm_ivpu_param { · 62 unmodified lines
92100 __u64 value;
93101 };
94102
103+#define DRM_IVPU_BO_HIGH_MEM 0x00000001
104+#define DRM_IVPU_BO_MAPPABLE 0x00000002
105+
106+#define DRM_IVPU_BO_CACHED 0x00000000
107+#define DRM_IVPU_BO_UNCACHED 0x00010000
108+#define DRM_IVPU_BO_WC 0x00020000
109+#define DRM_IVPU_BO_CACHE_MASK 0x00030000
110+
111+#define DRM_IVPU_BO_FLAGS \
112+ (DRM_IVPU_BO_HIGH_MEM | \
113+ DRM_IVPU_BO_MAPPABLE | \
114+ DRM_IVPU_BO_CACHE_MASK)
115+
116+/**
117+ * struct drm_ivpu_bo_create - Create BO backed by SHMEM
118+ *
119+ * Create GEM buffer object allocated in SHMEM memory.
120+ */
121+struct drm_ivpu_bo_create {
122+ /** @size: The size in bytes of the allocated memory */
123+ __u64 size;
124+
125+ /**
126+ * @flags:
127+ *
128+ * Supported flags:
129+ *
130+ * %DRM_IVPU_BO_HIGH_MEM:
131+ *
132+ * Allocate VPU address from >4GB range.
133+ * Buffer object with vpu address >4GB can be always accessed by the
134+ * VPU DMA engine, but some HW generation may not be able to access
135+ * this memory from then firmware running on the VPU management processor.
136+ * Suitable for input, output and some scratch buffers.
137+ *
138+ * %DRM_IVPU_BO_MAPPABLE:
139+ *
140+ * Buffer object can be mapped using mmap().
141+ *
142+ * %DRM_IVPU_BO_CACHED:
143+ *
144+ * Allocated BO will be cached on host side (WB) and snooped on the VPU side.
145+ * This is the default caching mode.
146+ *
147+ * %DRM_IVPU_BO_UNCACHED:
148+ *
149+ * Allocated BO will not be cached on host side nor snooped on the VPU side.
150+ *
151+ * %DRM_IVPU_BO_WC:
152+ *
153+ * Allocated BO will use write combining buffer for writes but reads will be
154+ * uncached.
155+ */
156+ __u32 flags;
157+
158+ /** @handle: Returned GEM object handle */
159+ __u32 handle;
160+
161+ /** @vpu_addr: Returned VPU virtual address */
162+ __u64 vpu_addr;
163+};
164+
165+/**
166+ * struct drm_ivpu_bo_info - Query buffer object info
167+ */
168+struct drm_ivpu_bo_info {
169+ /** @handle: Handle of the queried BO */
170+ __u32 handle;
171+
172+ /** @flags: Returned flags used to create the BO */
173+ __u32 flags;
174+
175+ /** @vpu_addr: Returned VPU virtual address */
176+ __u64 vpu_addr;
177+
178+ /**
179+ * @mmap_offset:
180+ *
181+ * Returned offset to be used in mmap(). 0 in case the BO is not mappable.
182+ */
183+ __u64 mmap_offset;
184+
185+ /** @size: Returned GEM object size, aligned to PAGE_SIZE */
186+ __u64 size;
187+};
188+
95189 #if defined(__cplusplus)
96190 }
97191 #endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t