This patch updates the VM_IOCTL to allow userspace to synchronize the mapping/unmapping of a BO in the page table. The major changes are: - it adds a drm_timeline object as an input parameter to the VM IOCTL. - this object is used by the kernel to sync the update of the BO in the page table during the mapping of the object. - the kernel also synchronizes the tlb flush of the page table entry of this object during the unmapping (Added in this series: https://patchwork.freedesktop.org/series/131276/ and https://patchwork.freedesktop.org/patch/584182/) - the userspace can wait on this timeline, and then the BO is ready to be consumed by the GPU. The UAPI for the same has been approved here: https://gitlab.freedesktop.org/mesa/drm/-/merge_requests/392 V2: - remove the eviction fence coupling V3: - added the drm timeline support instead of input/output fence (Christian) V4: - made timeline 64-bit (Christian) - bug fix (Arvind) V5: GLCTS bug fix (Arvind) V6: Rename syncobj_handle -> timeline_syncobj_out Rename point -> timeline_point_in (Marek) V7: Addressed review comments from Christian: - do not send last_update fence in case of vm_clear_freed, instead return the fence from gen_va_update_vm - move the functions to update bo_mapping to amdgpu_gem.c - do not use amdgpu_userq_update_vm anymore in userq_create() V8: Addressed review comments from Christian: - Split amdgpu_gem_update_bo_mapping function. - amdgpu_gem_va_update_vm should return stub for error. V9: Addressed review comments from Christian: - Rename the function amdgpu_gem_update_timeline_node. - amdgpu_gem_update_timeline_node should be void function. - when timeline_point is zero don't allocate a chain and call drm_syncobj_replace_fence() instead of drm_syncobj_add_point(). V11: rebase V12: Fix 32-bit holes issue in sturct drm_amdgpu_gem_va. V13: Fix the review comment by renaming timeline syncobj (Marek)
Unified · Split
| @@ -36,6 +36,7 @@ · 35 unmodified lines | |||
| 36 | #include <drm/drm_exec.h> | 36 | #include <drm/drm_exec.h> |
| 37 | #include <drm/drm_gem_ttm_helper.h> | 37 | #include <drm/drm_gem_ttm_helper.h> |
| 38 | #include <drm/ttm/ttm_tt.h> | 38 | #include <drm/ttm/ttm_tt.h> |
| 39 | #include <drm/drm_syncobj.h> | ||
| 39 | 40 | ||
| 40 | #include "amdgpu.h" | 41 | #include "amdgpu.h" |
| 41 | #include "amdgpu_display.h" | 42 | #include "amdgpu_display.h" |
| @@ -44,6 +45,75 @@ · 2 unmodified lines | |||
| 44 | #include "amdgpu_xgmi.h" | 45 | #include "amdgpu_xgmi.h" |
| 45 | #include "amdgpu_vm.h" | 46 | #include "amdgpu_vm.h" |
| 46 | 47 | ||
| 48 | static int | ||
| 49 | amdgpu_gem_update_timeline_node(struct drm_file *filp, | ||
| 50 | uint32_t syncobj_handle, | ||
| 51 | uint64_t point, | ||
| 52 | struct drm_syncobj **syncobj, | ||
| 53 | struct dma_fence_chain **chain) | ||
| 54 | { | ||
| 55 | if (!syncobj_handle) | ||
| 56 | return 0; | ||
| 57 | |||
| 58 | /* Find the sync object */ | ||
| 59 | *syncobj = drm_syncobj_find(filp, syncobj_handle); | ||
| 60 | if (!*syncobj) | ||
| 61 | return -ENOENT; | ||
| 62 | |||
| 63 | if (!point) | ||
| 64 | return 0; | ||
| 65 | |||
| 66 | /* Allocate the chain node */ | ||
| 67 | *chain = dma_fence_chain_alloc(); | ||
| 68 | if (!*chain) { | ||
| 69 | drm_syncobj_put(*syncobj); | ||
| 70 | return -ENOMEM; | ||
| 71 | } | ||
| 72 | |||
| 73 | return 0; | ||
| 74 | } | ||
| 75 | |||
| 76 | static void | ||
| 77 | amdgpu_gem_update_bo_mapping(struct drm_file *filp, | ||
| 78 | struct amdgpu_bo_va *bo_va, | ||
| 79 | uint32_t operation, | ||
| 80 | uint64_t point, | ||
| 81 | struct dma_fence *fence, | ||
| 82 | struct drm_syncobj *syncobj, | ||
| 83 | struct dma_fence_chain *chain) | ||
| 84 | { | ||
| 85 | struct amdgpu_bo *bo = bo_va ? bo_va->base.bo : NULL; | ||
| 86 | struct amdgpu_fpriv *fpriv = filp->driver_priv; | ||
| 87 | struct amdgpu_vm *vm = &fpriv->vm; | ||
| 88 | struct dma_fence *last_update; | ||
| 89 | |||
| 90 | if (!syncobj) | ||
| 91 | return; | ||
| 92 | |||
| 93 | /* Find the last update fence */ | ||
| 94 | switch (operation) { | ||
| 95 | case AMDGPU_VA_OP_MAP: | ||
| 96 | case AMDGPU_VA_OP_REPLACE: | ||
| 97 | if (bo && (bo->tbo.base.resv == vm->root.bo->tbo.base.resv)) | ||
| 98 | last_update = vm->last_update; | ||
| 99 | else | ||
| 100 | last_update = bo_va->last_pt_update; | ||
| 101 | break; | ||
| 102 | case AMDGPU_VA_OP_UNMAP: | ||
| 103 | case AMDGPU_VA_OP_CLEAR: | ||
| 104 | last_update = fence; | ||
| 105 | break; | ||
| 106 | default: | ||
| 107 | return; | ||
| 108 | } | ||
| 109 | |||
| 110 | /* Add fence to timeline */ | ||
| 111 | if (!point) | ||
| 112 | drm_syncobj_replace_fence(syncobj, last_update); | ||
| 113 | else | ||
| 114 | drm_syncobj_add_point(syncobj, chain, last_update, point); | ||
| 115 | } | ||
| 116 | |||
| 47 | static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf) | 117 | static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf) |
| 48 | { | 118 | { |
| 49 | struct ttm_buffer_object *bo = vmf->vma->vm_private_data; | 119 | struct ttm_buffer_object *bo = vmf->vma->vm_private_data; |
| @@ -638,18 +708,23 @@ out: · 588 unmodified lines | |||
| 638 | * | 708 | * |
| 639 | * Update the bo_va directly after setting its address. Errors are not | 709 | * Update the bo_va directly after setting its address. Errors are not |
| 640 | * vital here, so they are not reported back to userspace. | 710 | * vital here, so they are not reported back to userspace. |
| 711 | * | ||
| 712 | * Returns resulting fence if freed BO(s) got cleared from the PT. | ||
| 713 | * otherwise stub fence in case of error. | ||
| 641 | */ | 714 | */ |
| 642 | static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev, | 715 | static struct dma_fence * |
| 643 | struct amdgpu_vm *vm, | 716 | amdgpu_gem_va_update_vm(struct amdgpu_device *adev, |
| 644 | struct amdgpu_bo_va *bo_va, | 717 | struct amdgpu_vm *vm, |
| 645 | uint32_t operation) | 718 | struct amdgpu_bo_va *bo_va, |
| 719 | uint32_t operation) | ||
| 646 | { | 720 | { |
| 721 | struct dma_fence *fence = dma_fence_get_stub(); | ||
| 647 | int r; | 722 | int r; |
| 648 | 723 | ||
| 649 | if (!amdgpu_vm_ready(vm)) | 724 | if (!amdgpu_vm_ready(vm)) |
| 650 | return; | 725 | return fence; |
| 651 | 726 | ||
| 652 | r = amdgpu_vm_clear_freed(adev, vm, NULL); | 727 | r = amdgpu_vm_clear_freed(adev, vm, &fence); |
| 653 | if (r) | 728 | if (r) |
| 654 | goto error; | 729 | goto error; |
| 655 | 730 | ||
| @@ -665,6 +740,8 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev, · 9 unmodified lines | |||
| 665 | error: | 740 | error: |
| 666 | if (r && r != -ERESTARTSYS) | 741 | if (r && r != -ERESTARTSYS) |
| 667 | DRM_ERROR("Couldn't update BO_VA (%d)\n", r); | 742 | DRM_ERROR("Couldn't update BO_VA (%d)\n", r); |
| 743 | |||
| 744 | return fence; | ||
| 668 | } | 745 | } |
| 669 | 746 | ||
| 670 | /** | 747 | /** |
| @@ -713,6 +790,9 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, · 42 unmodified lines | |||
| 713 | struct amdgpu_fpriv *fpriv = filp->driver_priv; | 790 | struct amdgpu_fpriv *fpriv = filp->driver_priv; |
| 714 | struct amdgpu_bo *abo; | 791 | struct amdgpu_bo *abo; |
| 715 | struct amdgpu_bo_va *bo_va; | 792 | struct amdgpu_bo_va *bo_va; |
| 793 | struct drm_syncobj *timeline_syncobj = NULL; | ||
| 794 | struct dma_fence_chain *timeline_chain = NULL; | ||
| 795 | struct dma_fence *fence; | ||
| 716 | struct drm_exec exec; | 796 | struct drm_exec exec; |
| 717 | uint64_t va_flags; | 797 | uint64_t va_flags; |
| 718 | uint64_t vm_size; | 798 | uint64_t vm_size; |
| @@ -827,9 +907,24 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, · 108 unmodified lines | |||
| 827 | default: | 907 | default: |
| 828 | break; | 908 | break; |
| 829 | } | 909 | } |
| 830 | if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) | 910 | if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) { |
| 831 | amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, | 911 | |
| 832 | args->operation); | 912 | r = amdgpu_gem_update_timeline_node(filp, |
| 913 | args->vm_timeline_syncobj_out, | ||
| 914 | args->vm_timeline_point, | ||
| 915 | &timeline_syncobj, | ||
| 916 | &timeline_chain); | ||
| 917 | |||
| 918 | fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, | ||
| 919 | args->operation); | ||
| 920 | |||
| 921 | if (!r) | ||
| 922 | amdgpu_gem_update_bo_mapping(filp, bo_va, | ||
| 923 | args->operation, | ||
| 924 | args->vm_timeline_point, | ||
| 925 | fence, timeline_syncobj, | ||
| 926 | timeline_chain); | ||
| 927 | } | ||
| 833 | 928 | ||
| 834 | error: | 929 | error: |
| 835 | drm_exec_fini(&exec); | 930 | drm_exec_fini(&exec); |
| @@ -857,6 +857,15 @@ struct drm_amdgpu_gem_va { · 856 unmodified lines | |||
| 857 | __u64 offset_in_bo; | 857 | __u64 offset_in_bo; |
| 858 | /** Specify mapping size. Must be correctly aligned. */ | 858 | /** Specify mapping size. Must be correctly aligned. */ |
| 859 | __u64 map_size; | 859 | __u64 map_size; |
| 860 | /** | ||
| 861 | * vm_timeline_point is a sequence number used to add new timeline point. | ||
| 862 | */ | ||
| 863 | __u64 vm_timeline_point; | ||
| 864 | /** | ||
| 865 | * The vm page table update fence is installed in given vm_timeline_syncobj_out | ||
| 866 | * at vm_timeline_point. | ||
| 867 | */ | ||
| 868 | __u32 vm_timeline_syncobj_out; | ||
| 860 | }; | 869 | }; |
| 861 | 870 | ||
| 862 | #define AMDGPU_HW_IP_GFX 0 | 871 | #define AMDGPU_HW_IP_GFX 0 |