/* * Copyright 2008 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. * * Authors: * Jerome Glisse <glisse@freedesktop.org> */ #include <linux/file.h>#include <linux/pagemap.h>#include <linux/sync_file.h>#include <linux/dma-buf.h> #include <drm/amdgpu_drm.h>#include <drm/drm_syncobj.h>#include "amdgpu.h"#include "amdgpu_trace.h"#include "amdgpu_gmc.h"#include "amdgpu_gem.h"#include "amdgpu_ras.h" static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, struct drm_amdgpu_cs_chunk_fence *data, uint32_t *offset){ struct drm_gem_object *gobj; struct amdgpu_bo *bo; unsigned long size; int r; gobj = drm_gem_object_lookup(p->filp, data->handle); if (gobj == NULL) return -EINVAL; bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); p->uf_entry.priority = 0; p->uf_entry.tv.bo = &bo->tbo; /* One for TTM and one for the CS job */ p->uf_entry.tv.num_shared = 2; drm_gem_object_put(gobj); size = amdgpu_bo_size(bo); if (size != PAGE_SIZE || (data->offset + 8) > size) { r = -EINVAL; goto error_unref; } if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { r = -EINVAL; goto error_unref; } *offset = data->offset; return 0; error_unref: amdgpu_bo_unref(&bo); return r;} static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, struct drm_amdgpu_bo_list_in *data){ int r; struct drm_amdgpu_bo_list_entry *info = NULL; r = amdgpu_bo_create_list_entry_array(data, &info); if (r) return r; r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number, &p->bo_list); if (r) goto error_free; kvfree(info); return 0; error_free: kvfree(info); return r;} static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs){ struct amdgpu_fpriv *fpriv = p->filp->driver_priv; struct amdgpu_vm *vm = &fpriv->vm; uint64_t *chunk_array_user; uint64_t *chunk_array; unsigned size, num_ibs = 0; uint32_t uf_offset = 0; int i; int ret; if (cs->in.num_chunks == 0) return 0; chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); if (!chunk_array) return -ENOMEM; p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); if (!p->ctx) { ret = -EINVAL; goto free_chunk; } mutex_lock(&p->ctx->lock); /* skip guilty context job */ if (atomic_read(&p->ctx->guilty) == 1) { ret = -ECANCELED; goto free_chunk; } /* get chunks */ chunk_array_user = u64_to_user_ptr(cs->in.chunks); if (copy_from_user(chunk_array, chunk_array_user, sizeof(uint64_t)*cs->in.num_chunks)) { ret = -EFAULT; goto free_chunk; } p->nchunks = cs->in.num_chunks; p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), GFP_KERNEL); if (!p->chunks) { ret = -ENOMEM; goto free_chunk; } for (i = 0; i < p->nchunks; i++) { struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; struct drm_amdgpu_cs_chunk user_chunk; uint32_t __user *cdata; chunk_ptr = u64_to_user_ptr(chunk_array[i]); if (copy_from_user(&user_chunk, chunk_ptr, sizeof(struct drm_amdgpu_cs_chunk))) { ret = -EFAULT; i--; goto free_partial_kdata; } p->chunks[i].chunk_id = user_chunk.chunk_id; p->chunks[i].length_dw = user_chunk.length_dw; size = p->chunks[i].length_dw; cdata = u64_to_user_ptr(user_chunk.chunk_data); p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); if (p->chunks[i].kdata == NULL) { ret = -ENOMEM; i--; goto free_partial_kdata; } size *= sizeof(uint32_t); if (copy_from_user(p->chunks[i].kdata, cdata, size)) { ret = -EFAULT; goto free_partial_kdata; } switch (p->chunks[i].chunk_id) { case AMDGPU_CHUNK_ID_IB: ++num_ibs; break; case AMDGPU_CHUNK_ID_FENCE: size = sizeof(struct drm_amdgpu_cs_chunk_fence); if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { ret = -EINVAL; goto free_partial_kdata; } ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata, &uf_offset); if (ret) goto free_partial_kdata; break; case AMDGPU_CHUNK_ID_BO_HANDLES: size = sizeof(struct drm_amdgpu_bo_list_in); if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { ret = -EINVAL; goto free_partial_kdata; } ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata); if (ret) goto free_partial_kdata; break; case AMDGPU_CHUNK_ID_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_IN: case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: break; default: ret = -EINVAL; goto free_partial_kdata; } } ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm); if (ret) goto free_all_kdata; if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) { ret = -ECANCELED; goto free_all_kdata; } if (p->uf_entry.tv.bo) p->job->uf_addr = uf_offset; kvfree(chunk_array); /* Use this opportunity to fill in task info for the vm */ amdgpu_vm_set_task_info(vm); return 0; free_all_kdata: i = p->nchunks - 1;free_partial_kdata: for (; i >= 0; i--) kvfree(p->chunks[i].kdata); kvfree(p->chunks); p->chunks = NULL; p->nchunks = 0;free_chunk: kvfree(chunk_array); return ret;} /* Convert microseconds to bytes. */static u64 us_to_bytes(struct amdgpu_device *adev, s64 us){ if (us <= 0 || !adev->mm_stats.log2_max_MBps) return 0; /* Since accum_us is incremented by a million per second, just * multiply it by the number of MB/s to get the number of bytes. */ return us << adev->mm_stats.log2_max_MBps;} static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes){ if (!adev->mm_stats.log2_max_MBps) return 0; return bytes >> adev->mm_stats.log2_max_MBps;} /* Returns how many bytes TTM can move right now. If no bytes can be moved, * it returns 0. If it returns non-zero, it's OK to move at least one buffer, * which means it can go over the threshold once. If that happens, the driver * will be in debt and no other buffer migrations can be done until that debt * is repaid. * * This approach allows moving a buffer of any size (it's important to allow * that). * * The currency is simply time in microseconds and it increases as the clock * ticks. The accumulated microseconds (us) are converted to bytes and * returned. */static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, u64 *max_bytes, u64 *max_vis_bytes){ s64 time_us, increment_us; u64 free_vram, total_vram, used_vram; struct ttm_resource_manager *vram_man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); /* Allow a maximum of 200 accumulated ms. This is basically per-IB * throttling. * * It means that in order to get full max MBps, at least 5 IBs per * second must be submitted and not more than 200ms apart from each * other. */ const s64 us_upper_bound = 200000; if (!adev->mm_stats.log2_max_MBps) { *max_bytes = 0; *max_vis_bytes = 0; return; } total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); used_vram = amdgpu_vram_mgr_usage(vram_man); free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; spin_lock(&adev->mm_stats.lock); /* Increase the amount of accumulated us. */ time_us = ktime_to_us(ktime_get()); increment_us = time_us - adev->mm_stats.last_update_us; adev->mm_stats.last_update_us = time_us; adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, us_upper_bound); /* This prevents the short period of low performance when the VRAM * usage is low and the driver is in debt or doesn't have enough * accumulated us to fill VRAM quickly. * * The situation can occur in these cases: * - a lot of VRAM is freed by userspace * - the presence of a big buffer causes a lot of evictions * (solution: split buffers into smaller ones) * * If 128 MB or 1/8th of VRAM is free, start filling it now by setting * accum_us to a positive number. */ if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { s64 min_us; /* Be more aggresive on dGPUs. Try to fill a portion of free * VRAM now. */ if (!(adev->flags & AMD_IS_APU)) min_us = bytes_to_us(adev, free_vram / 4); else min_us = 0; /* Reset accum_us on APUs. */ adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); } /* This is set to 0 if the driver is in debt to disallow (optional) * buffer moves. */ *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); /* Do the same for visible VRAM if half of it is free */ if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { u64 total_vis_vram = adev->gmc.visible_vram_size; u64 used_vis_vram = amdgpu_vram_mgr_vis_usage(vram_man); if (used_vis_vram < total_vis_vram) { u64 free_vis_vram = total_vis_vram - used_vis_vram; adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + increment_us, us_upper_bound); if (free_vis_vram >= total_vis_vram / 2) adev->mm_stats.accum_us_vis = max(bytes_to_us(adev, free_vis_vram / 2), adev->mm_stats.accum_us_vis); } *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); } else { *max_vis_bytes = 0; } spin_unlock(&adev->mm_stats.lock);} /* Report how many bytes have really been moved for the last command * submission. This can result in a debt that can stop buffer migrations * temporarily. */void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, u64 num_vis_bytes){ spin_lock(&adev->mm_stats.lock); adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); spin_unlock(&adev->mm_stats.lock);} static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p, struct amdgpu_bo *bo){ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct ttm_operation_ctx ctx = { .interruptible = true, .no_wait_gpu = false, .resv = bo->tbo.base.resv }; uint32_t domain; int r; if (bo->tbo.pin_count) return 0; /* Don't move this buffer if we have depleted our allowance * to move it. Don't move anything if the threshold is zero. */ if (p->bytes_moved < p->bytes_moved_threshold && (!bo->tbo.base.dma_buf || list_empty(&bo->tbo.base.dma_buf->attachments))) { if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { /* And don't move a CPU_ACCESS_REQUIRED BO to limited * visible VRAM if we've depleted our allowance to do * that. */ if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) domain = bo->preferred_domains; else domain = bo->allowed_domains; } else { domain = bo->preferred_domains; } } else { domain = bo->allowed_domains; } retry: amdgpu_bo_placement_from_domain(bo, domain); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); p->bytes_moved += ctx.bytes_moved; if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && amdgpu_bo_in_cpu_visible_vram(bo)) p->bytes_moved_vis += ctx.bytes_moved; if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { domain = bo->allowed_domains; goto retry; } return r;} static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo){ struct amdgpu_cs_parser *p = param; int r; r = amdgpu_cs_bo_validate(p, bo); if (r) return r; if (bo->shadow) r = amdgpu_cs_bo_validate(p, bo->shadow); return r;} static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, struct list_head *validated){ struct ttm_operation_ctx ctx = { true, false }; struct amdgpu_bo_list_entry *lobj; int r; list_for_each_entry(lobj, validated, tv.head) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo); struct mm_struct *usermm; usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); if (usermm && usermm != current->mm) return -EPERM; if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) && lobj->user_invalidated && lobj->user_pages) { amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (r) return r; amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, lobj->user_pages); } r = amdgpu_cs_validate(p, bo); if (r) return r; kvfree(lobj->user_pages);Lines 1–500 of 1685. Load more
/* * Copyright 2008 Jerome Glisse. * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. * * Authors: * Jerome Glisse <glisse@freedesktop.org> */ #include <linux/file.h> #include <linux/pagemap.h> #include <linux/sync_file.h> #include <linux/dma-buf.h> #include <drm/amdgpu_drm.h> #include <drm/drm_syncobj.h> #include "amdgpu.h" #include "amdgpu_trace.h" #include "amdgpu_gmc.h" #include "amdgpu_gem.h" #include "amdgpu_ras.h" static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, struct drm_amdgpu_cs_chunk_fence *data, uint32_t *offset) { struct drm_gem_object *gobj; struct amdgpu_bo *bo; unsigned long size; int r; gobj = drm_gem_object_lookup(p->filp, data->handle); if (gobj == NULL) return -EINVAL; bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); p->uf_entry.priority = 0; p->uf_entry.tv.bo = &bo->tbo; /* One for TTM and one for the CS job */ p->uf_entry.tv.num_shared = 2; drm_gem_object_put(gobj); size = amdgpu_bo_size(bo); if (size != PAGE_SIZE || (data->offset + 8) > size) { r = -EINVAL; goto error_unref; } if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { r = -EINVAL; goto error_unref; } *offset = data->offset; return 0; error_unref: amdgpu_bo_unref(&bo); return r; } static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, struct drm_amdgpu_bo_list_in *data) { int r; struct drm_amdgpu_bo_list_entry *info = NULL; r = amdgpu_bo_create_list_entry_array(data, &info); if (r) return r; r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number, &p->bo_list); if (r) goto error_free; kvfree(info); return 0; error_free: kvfree(info); return r; } static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; struct amdgpu_vm *vm = &fpriv->vm; uint64_t *chunk_array_user; uint64_t *chunk_array; unsigned size, num_ibs = 0; uint32_t uf_offset = 0; int i; int ret; if (cs->in.num_chunks == 0) return 0; chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); if (!chunk_array) return -ENOMEM; p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); if (!p->ctx) { ret = -EINVAL; goto free_chunk; } mutex_lock(&p->ctx->lock); /* skip guilty context job */ if (atomic_read(&p->ctx->guilty) == 1) { ret = -ECANCELED; goto free_chunk; } /* get chunks */ chunk_array_user = u64_to_user_ptr(cs->in.chunks); if (copy_from_user(chunk_array, chunk_array_user, sizeof(uint64_t)*cs->in.num_chunks)) { ret = -EFAULT; goto free_chunk; } p->nchunks = cs->in.num_chunks; p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), GFP_KERNEL); if (!p->chunks) { ret = -ENOMEM; goto free_chunk; } for (i = 0; i < p->nchunks; i++) { struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; struct drm_amdgpu_cs_chunk user_chunk; uint32_t __user *cdata; chunk_ptr = u64_to_user_ptr(chunk_array[i]); if (copy_from_user(&user_chunk, chunk_ptr, sizeof(struct drm_amdgpu_cs_chunk))) { ret = -EFAULT; i--; goto free_partial_kdata; } p->chunks[i].chunk_id = user_chunk.chunk_id; p->chunks[i].length_dw = user_chunk.length_dw; size = p->chunks[i].length_dw; cdata = u64_to_user_ptr(user_chunk.chunk_data); p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); if (p->chunks[i].kdata == NULL) { ret = -ENOMEM; i--; goto free_partial_kdata; } size *= sizeof(uint32_t); if (copy_from_user(p->chunks[i].kdata, cdata, size)) { ret = -EFAULT; goto free_partial_kdata; } switch (p->chunks[i].chunk_id) { case AMDGPU_CHUNK_ID_IB: ++num_ibs; break; case AMDGPU_CHUNK_ID_FENCE: size = sizeof(struct drm_amdgpu_cs_chunk_fence); if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { ret = -EINVAL; goto free_partial_kdata; } ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata, &uf_offset); if (ret) goto free_partial_kdata; break; case AMDGPU_CHUNK_ID_BO_HANDLES: size = sizeof(struct drm_amdgpu_bo_list_in); if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { ret = -EINVAL; goto free_partial_kdata; } ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata); if (ret) goto free_partial_kdata; break; case AMDGPU_CHUNK_ID_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_IN: case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: break; default: ret = -EINVAL; goto free_partial_kdata; } } ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm); if (ret) goto free_all_kdata; if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) { ret = -ECANCELED; goto free_all_kdata; } if (p->uf_entry.tv.bo) p->job->uf_addr = uf_offset; kvfree(chunk_array); /* Use this opportunity to fill in task info for the vm */ amdgpu_vm_set_task_info(vm); return 0; free_all_kdata: i = p->nchunks - 1; free_partial_kdata: for (; i >= 0; i--) kvfree(p->chunks[i].kdata); kvfree(p->chunks); p->chunks = NULL; p->nchunks = 0; free_chunk: kvfree(chunk_array); return ret; } /* Convert microseconds to bytes. */ static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) { if (us <= 0 || !adev->mm_stats.log2_max_MBps) return 0; /* Since accum_us is incremented by a million per second, just * multiply it by the number of MB/s to get the number of bytes. */ return us << adev->mm_stats.log2_max_MBps; } static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) { if (!adev->mm_stats.log2_max_MBps) return 0; return bytes >> adev->mm_stats.log2_max_MBps; } /* Returns how many bytes TTM can move right now. If no bytes can be moved, * it returns 0. If it returns non-zero, it's OK to move at least one buffer, * which means it can go over the threshold once. If that happens, the driver * will be in debt and no other buffer migrations can be done until that debt * is repaid. * * This approach allows moving a buffer of any size (it's important to allow * that). * * The currency is simply time in microseconds and it increases as the clock * ticks. The accumulated microseconds (us) are converted to bytes and * returned. */ static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, u64 *max_bytes, u64 *max_vis_bytes) { s64 time_us, increment_us; u64 free_vram, total_vram, used_vram; struct ttm_resource_manager *vram_man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); /* Allow a maximum of 200 accumulated ms. This is basically per-IB * throttling. * * It means that in order to get full max MBps, at least 5 IBs per * second must be submitted and not more than 200ms apart from each * other. */ const s64 us_upper_bound = 200000; if (!adev->mm_stats.log2_max_MBps) { *max_bytes = 0; *max_vis_bytes = 0; return; } total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); used_vram = amdgpu_vram_mgr_usage(vram_man); free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; spin_lock(&adev->mm_stats.lock); /* Increase the amount of accumulated us. */ time_us = ktime_to_us(ktime_get()); increment_us = time_us - adev->mm_stats.last_update_us; adev->mm_stats.last_update_us = time_us; adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, us_upper_bound); /* This prevents the short period of low performance when the VRAM * usage is low and the driver is in debt or doesn't have enough * accumulated us to fill VRAM quickly. * * The situation can occur in these cases: * - a lot of VRAM is freed by userspace * - the presence of a big buffer causes a lot of evictions * (solution: split buffers into smaller ones) * * If 128 MB or 1/8th of VRAM is free, start filling it now by setting * accum_us to a positive number. */ if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { s64 min_us; /* Be more aggresive on dGPUs. Try to fill a portion of free * VRAM now. */ if (!(adev->flags & AMD_IS_APU)) min_us = bytes_to_us(adev, free_vram / 4); else min_us = 0; /* Reset accum_us on APUs. */ adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); } /* This is set to 0 if the driver is in debt to disallow (optional) * buffer moves. */ *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); /* Do the same for visible VRAM if half of it is free */ if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { u64 total_vis_vram = adev->gmc.visible_vram_size; u64 used_vis_vram = amdgpu_vram_mgr_vis_usage(vram_man); if (used_vis_vram < total_vis_vram) { u64 free_vis_vram = total_vis_vram - used_vis_vram; adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + increment_us, us_upper_bound); if (free_vis_vram >= total_vis_vram / 2) adev->mm_stats.accum_us_vis = max(bytes_to_us(adev, free_vis_vram / 2), adev->mm_stats.accum_us_vis); } *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); } else { *max_vis_bytes = 0; } spin_unlock(&adev->mm_stats.lock); } /* Report how many bytes have really been moved for the last command * submission. This can result in a debt that can stop buffer migrations * temporarily. */ void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, u64 num_vis_bytes) { spin_lock(&adev->mm_stats.lock); adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); spin_unlock(&adev->mm_stats.lock); } static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p, struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct ttm_operation_ctx ctx = { .interruptible = true, .no_wait_gpu = false, .resv = bo->tbo.base.resv }; uint32_t domain; int r; if (bo->tbo.pin_count) return 0; /* Don't move this buffer if we have depleted our allowance * to move it. Don't move anything if the threshold is zero. */ if (p->bytes_moved < p->bytes_moved_threshold && (!bo->tbo.base.dma_buf || list_empty(&bo->tbo.base.dma_buf->attachments))) { if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { /* And don't move a CPU_ACCESS_REQUIRED BO to limited * visible VRAM if we've depleted our allowance to do * that. */ if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) domain = bo->preferred_domains; else domain = bo->allowed_domains; } else { domain = bo->preferred_domains; } } else { domain = bo->allowed_domains; } retry: amdgpu_bo_placement_from_domain(bo, domain); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); p->bytes_moved += ctx.bytes_moved; if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && amdgpu_bo_in_cpu_visible_vram(bo)) p->bytes_moved_vis += ctx.bytes_moved; if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { domain = bo->allowed_domains; goto retry; } return r; } static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo) { struct amdgpu_cs_parser *p = param; int r; r = amdgpu_cs_bo_validate(p, bo); if (r) return r; if (bo->shadow) r = amdgpu_cs_bo_validate(p, bo->shadow); return r; } static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, struct list_head *validated) { struct ttm_operation_ctx ctx = { true, false }; struct amdgpu_bo_list_entry *lobj; int r; list_for_each_entry(lobj, validated, tv.head) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo); struct mm_struct *usermm; usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); if (usermm && usermm != current->mm) return -EPERM; if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) && lobj->user_invalidated && lobj->user_pages) { amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (r) return r; amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, lobj->user_pages); } r = amdgpu_cs_validate(p, bo); if (r) return r; kvfree(lobj->user_pages);