This patch updates the VM_IOCTL to allow userspace to synchronize the mapping/unmapping of a BO in the page table. The major changes are: - it adds a drm_timeline object as an input parameter to the VM IOCTL. - this object is used by the kernel to sync the update of the BO in the page table during the mapping of the object. - the kernel also synchronizes the tlb flush of the page table entry of this object during the unmapping (Added in this series: https://patchwork.freedesktop.org/series/131276/ and https://patchwork.freedesktop.org/patch/584182/) - the userspace can wait on this timeline, and then the BO is ready to be consumed by the GPU. The UAPI for the same has been approved here: https://gitlab.freedesktop.org/mesa/drm/-/merge_requests/392 V2: - remove the eviction fence coupling V3: - added the drm timeline support instead of input/output fence (Christian) V4: - made timeline 64-bit (Christian) - bug fix (Arvind) V5: GLCTS bug fix (Arvind) V6: Rename syncobj_handle -> timeline_syncobj_out Rename point -> timeline_point_in (Marek) V7: Addressed review comments from Christian: - do not send last_update fence in case of vm_clear_freed, instead return the fence from gen_va_update_vm - move the functions to update bo_mapping to amdgpu_gem.c - do not use amdgpu_userq_update_vm anymore in userq_create() V8: Addressed review comments from Christian: - Split amdgpu_gem_update_bo_mapping function. - amdgpu_gem_va_update_vm should return stub for error. V9: Addressed review comments from Christian: - Rename the function amdgpu_gem_update_timeline_node. - amdgpu_gem_update_timeline_node should be void function. - when timeline_point is zero don't allocate a chain and call drm_syncobj_replace_fence() instead of drm_syncobj_add_point(). V11: rebase V12: Fix 32-bit holes issue in sturct drm_amdgpu_gem_va. V13: Fix the review comment by renaming timeline syncobj (Marek)
Unified · Split
| @@ -36,6 +36,7 @@ · 35 unmodified lines | ||
| 36 | 36 | #include <drm/drm_exec.h> |
| 37 | 37 | #include <drm/drm_gem_ttm_helper.h> |
| 38 | 38 | #include <drm/ttm/ttm_tt.h> |
| 39 | +#include <drm/drm_syncobj.h> | |
| 39 | 40 | |
| 40 | 41 | #include "amdgpu.h" |
| 41 | 42 | #include "amdgpu_display.h" |
| @@ -44,6 +45,75 @@ · 2 unmodified lines | ||
| 44 | 45 | #include "amdgpu_xgmi.h" |
| 45 | 46 | #include "amdgpu_vm.h" |
| 46 | 47 | |
| 48 | +static int | |
| 49 | +amdgpu_gem_update_timeline_node(struct drm_file *filp, | |
| 50 | + uint32_t syncobj_handle, | |
| 51 | + uint64_t point, | |
| 52 | + struct drm_syncobj **syncobj, | |
| 53 | + struct dma_fence_chain **chain) | |
| 54 | +{ | |
| 55 | + if (!syncobj_handle) | |
| 56 | + return 0; | |
| 57 | + | |
| 58 | + /* Find the sync object */ | |
| 59 | + *syncobj = drm_syncobj_find(filp, syncobj_handle); | |
| 60 | + if (!*syncobj) | |
| 61 | + return -ENOENT; | |
| 62 | + | |
| 63 | + if (!point) | |
| 64 | + return 0; | |
| 65 | + | |
| 66 | + /* Allocate the chain node */ | |
| 67 | + *chain = dma_fence_chain_alloc(); | |
| 68 | + if (!*chain) { | |
| 69 | + drm_syncobj_put(*syncobj); | |
| 70 | + return -ENOMEM; | |
| 71 | + } | |
| 72 | + | |
| 73 | + return 0; | |
| 74 | +} | |
| 75 | + | |
| 76 | +static void | |
| 77 | +amdgpu_gem_update_bo_mapping(struct drm_file *filp, | |
| 78 | + struct amdgpu_bo_va *bo_va, | |
| 79 | + uint32_t operation, | |
| 80 | + uint64_t point, | |
| 81 | + struct dma_fence *fence, | |
| 82 | + struct drm_syncobj *syncobj, | |
| 83 | + struct dma_fence_chain *chain) | |
| 84 | +{ | |
| 85 | + struct amdgpu_bo *bo = bo_va ? bo_va->base.bo : NULL; | |
| 86 | + struct amdgpu_fpriv *fpriv = filp->driver_priv; | |
| 87 | + struct amdgpu_vm *vm = &fpriv->vm; | |
| 88 | + struct dma_fence *last_update; | |
| 89 | + | |
| 90 | + if (!syncobj) | |
| 91 | + return; | |
| 92 | + | |
| 93 | + /* Find the last update fence */ | |
| 94 | + switch (operation) { | |
| 95 | + case AMDGPU_VA_OP_MAP: | |
| 96 | + case AMDGPU_VA_OP_REPLACE: | |
| 97 | + if (bo && (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)) | |
| 98 | + last_update = vm->last_update; | |
| 99 | + else | |
| 100 | + last_update = bo_va->last_pt_update; | |
| 101 | + break; | |
| 102 | + case AMDGPU_VA_OP_UNMAP: | |
| 103 | + case AMDGPU_VA_OP_CLEAR: | |
| 104 | + last_update = fence; | |
| 105 | + break; | |
| 106 | + default: | |
| 107 | + return; | |
| 108 | + } | |
| 109 | + | |
| 110 | + /* Add fence to timeline */ | |
| 111 | + if (!point) | |
| 112 | + drm_syncobj_replace_fence(syncobj, last_update); | |
| 113 | + else | |
| 114 | + drm_syncobj_add_point(syncobj, chain, last_update, point); | |
| 115 | +} | |
| 116 | + | |
| 47 | 117 | static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf) |
| 48 | 118 | { |
| 49 | 119 | struct ttm_buffer_object *bo = vmf->vma->vm_private_data; |
| @@ -638,18 +708,23 @@ out: · 588 unmodified lines | ||
| 638 | 708 | * |
| 639 | 709 | * Update the bo_va directly after setting its address. Errors are not |
| 640 | 710 | * vital here, so they are not reported back to userspace. |
| 711 | + * | |
| 712 | + * Returns resulting fence if freed BO(s) got cleared from the PT. | |
| 713 | + * otherwise stub fence in case of error. | |
| 641 | 714 | */ |
| 642 | −static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev, | |
| 643 | − struct amdgpu_vm *vm, | |
| 644 | − struct amdgpu_bo_va *bo_va, | |
| 645 | − uint32_t operation) | |
| 715 | +static struct dma_fence * | |
| 716 | +amdgpu_gem_va_update_vm(struct amdgpu_device *adev, | |
| 717 | + struct amdgpu_vm *vm, | |
| 718 | + struct amdgpu_bo_va *bo_va, | |
| 719 | + uint32_t operation) | |
| 646 | 720 | { |
| 721 | + struct dma_fence *fence = dma_fence_get_stub(); | |
| 647 | 722 | int r; |
| 648 | 723 | |
| 649 | 724 | if (!amdgpu_vm_ready(vm)) |
| 650 | − return; | |
| 725 | + return fence; | |
| 651 | 726 | |
| 652 | − r = amdgpu_vm_clear_freed(adev, vm, NULL); | |
| 727 | + r = amdgpu_vm_clear_freed(adev, vm, &fence); | |
| 653 | 728 | if (r) |
| 654 | 729 | goto error; |
| 655 | 730 | |
| @@ -665,6 +740,8 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev, · 9 unmodified lines | ||
| 665 | 740 | error: |
| 666 | 741 | if (r && r != -ERESTARTSYS) |
| 667 | 742 | DRM_ERROR("Couldn't update BO_VA (%d)\n", r); |
| 743 | + | |
| 744 | + return fence; | |
| 668 | 745 | } |
| 669 | 746 | |
| 670 | 747 | /** |
| @@ -713,6 +790,9 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, · 42 unmodified lines | ||
| 713 | 790 | struct amdgpu_fpriv *fpriv = filp->driver_priv; |
| 714 | 791 | struct amdgpu_bo *abo; |
| 715 | 792 | struct amdgpu_bo_va *bo_va; |
| 793 | + struct drm_syncobj *timeline_syncobj = NULL; | |
| 794 | + struct dma_fence_chain *timeline_chain = NULL; | |
| 795 | + struct dma_fence *fence; | |
| 716 | 796 | struct drm_exec exec; |
| 717 | 797 | uint64_t va_flags; |
| 718 | 798 | uint64_t vm_size; |
| @@ -827,9 +907,24 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, · 108 unmodified lines | ||
| 827 | 907 | default: |
| 828 | 908 | break; |
| 829 | 909 | } |
| 830 | − if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) | |
| 831 | − amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, | |
| 832 | − args->operation); | |
| 910 | + if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) { | |
| 911 | + | |
| 912 | + r = amdgpu_gem_update_timeline_node(filp, | |
| 913 | + args->vm_timeline_syncobj_out, | |
| 914 | + args->vm_timeline_point, | |
| 915 | + &timeline_syncobj, | |
| 916 | + &timeline_chain); | |
| 917 | + | |
| 918 | + fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, | |
| 919 | + args->operation); | |
| 920 | + | |
| 921 | + if (!r) | |
| 922 | + amdgpu_gem_update_bo_mapping(filp, bo_va, | |
| 923 | + args->operation, | |
| 924 | + args->vm_timeline_point, | |
| 925 | + fence, timeline_syncobj, | |
| 926 | + timeline_chain); | |
| 927 | + } | |
| 833 | 928 | |
| 834 | 929 | error: |
| 835 | 930 | drm_exec_fini(&exec); |
| @@ -857,6 +857,15 @@ struct drm_amdgpu_gem_va { · 856 unmodified lines | ||
| 857 | 857 | __u64 offset_in_bo; |
| 858 | 858 | /** Specify mapping size. Must be correctly aligned. */ |
| 859 | 859 | __u64 map_size; |
| 860 | + /** | |
| 861 | + * vm_timeline_point is a sequence number used to add new timeline point. | |
| 862 | + */ | |
| 863 | + __u64 vm_timeline_point; | |
| 864 | + /** | |
| 865 | + * The vm page table update fence is installed in given vm_timeline_syncobj_out | |
| 866 | + * at vm_timeline_point. | |
| 867 | + */ | |
| 868 | + __u32 vm_timeline_syncobj_out; | |
| 860 | 869 | }; |
| 861 | 870 | |
| 862 | 871 | #define AMDGPU_HW_IP_GFX 0 |