| @@ -6,6 +6,7 @@ · 5 unmodified lines |
| 6 | | 6 | |
| 7 | #include <linux/adreno-smmu-priv.h> | 7 | #include <linux/adreno-smmu-priv.h> |
| 8 | #include <linux/io-pgtable.h> | 8 | #include <linux/io-pgtable.h> |
| | 9 | #include <linux/kmemleak.h> |
| 9 | #include "msm_drv.h" | 10 | #include "msm_drv.h" |
| 10 | #include "msm_mmu.h" | 11 | #include "msm_mmu.h" |
| 11 | | 12 | |
| @@ -14,6 +15,8 @@ struct msm_iommu { · 2 unmodified lines |
| 14 | struct iommu_domain *domain; | 15 | struct iommu_domain *domain; |
| 15 | atomic_t pagetables; | 16 | atomic_t pagetables; |
| 16 | struct page *prr_page; | 17 | struct page *prr_page; |
| | 18 | |
| | 19 | struct kmem_cache *pt_cache; |
| 17 | }; | 20 | }; |
| 18 | | 21 | |
| 19 | #define to_msm_iommu(x) container_of(x, struct msm_iommu, base) | 22 | #define to_msm_iommu(x) container_of(x, struct msm_iommu, base) |
| @@ -27,6 +30,9 @@ struct msm_iommu_pagetable { · 7 unmodified lines |
| 27 | unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ | 30 | unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ |
| 28 | phys_addr_t ttbr; | 31 | phys_addr_t ttbr; |
| 29 | u32 asid; | 32 | u32 asid; |
| | 33 | |
| | 34 | /** @root_page_table: Stores the root page table pointer. */ |
| | 35 | void *root_page_table; |
| 30 | }; | 36 | }; |
| 31 | static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu) | 37 | static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu) |
| 32 | { | 38 | { |
| @@ -282,7 +288,145 @@ msm_iommu_pagetable_walk(struct msm_mmu *mmu, unsigned long iova, uint64_t ptes[ · 249 unmodified lines |
| 282 | return 0; | 288 | return 0; |
| 283 | } | 289 | } |
| 284 | | 290 | |
| | 291 | static void |
| | 292 | msm_iommu_pagetable_prealloc_count(struct msm_mmu *mmu, struct msm_mmu_prealloc *p, |
| | 293 | uint64_t iova, size_t len) |
| | 294 | { |
| | 295 | u64 pt_count; |
| | 296 | |
| | 297 | /* |
| | 298 | * L1, L2 and L3 page tables. |
| | 299 | * |
| | 300 | * We could optimize L3 allocation by iterating over the sgt and merging |
| | 301 | * 2M contiguous blocks, but it's simpler to over-provision and return |
| | 302 | * the pages if they're not used. |
| | 303 | * |
| | 304 | * The first level descriptor (v8 / v7-lpae page table format) encodes |
| | 305 | * 30 bits of address. The second level encodes 29. For the 3rd it is |
| | 306 | * 39. |
| | 307 | * |
| | 308 | * https://developer.arm.com/documentation/ddi0406/c/System-Level-Architecture/Virtual-Memory-System-Architecture--VMSA-/Long-descriptor-translation-table-format/Long-descriptor-translation-table-format-descriptors?lang=en#BEIHEFFB |
| | 309 | */ |
| | 310 | pt_count = ((ALIGN(iova + len, 1ull << 39) - ALIGN_DOWN(iova, 1ull << 39)) >> 39) + |
| | 311 | ((ALIGN(iova + len, 1ull << 30) - ALIGN_DOWN(iova, 1ull << 30)) >> 30) + |
| | 312 | ((ALIGN(iova + len, 1ull << 21) - ALIGN_DOWN(iova, 1ull << 21)) >> 21); |
| | 313 | |
| | 314 | p->count += pt_count; |
| | 315 | } |
| | 316 | |
| | 317 | static struct kmem_cache * |
| | 318 | get_pt_cache(struct msm_mmu *mmu) |
| | 319 | { |
| | 320 | struct msm_iommu_pagetable *pagetable = to_pagetable(mmu); |
| | 321 | return to_msm_iommu(pagetable->parent)->pt_cache; |
| | 322 | } |
| | 323 | |
| | 324 | static int |
| | 325 | msm_iommu_pagetable_prealloc_allocate(struct msm_mmu *mmu, struct msm_mmu_prealloc *p) |
| | 326 | { |
| | 327 | struct kmem_cache *pt_cache = get_pt_cache(mmu); |
| | 328 | int ret; |
| | 329 | |
| | 330 | p->pages = kvmalloc_array(p->count, sizeof(p->pages), GFP_KERNEL); |
| | 331 | if (!p->pages) |
| | 332 | return -ENOMEM; |
| | 333 | |
| | 334 | ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages); |
| | 335 | if (ret != p->count) { |
| | 336 | p->count = ret; |
| | 337 | return -ENOMEM; |
| | 338 | } |
| | 339 | |
| | 340 | return 0; |
| | 341 | } |
| | 342 | |
| | 343 | static void |
| | 344 | msm_iommu_pagetable_prealloc_cleanup(struct msm_mmu *mmu, struct msm_mmu_prealloc *p) |
| | 345 | { |
| | 346 | struct kmem_cache *pt_cache = get_pt_cache(mmu); |
| | 347 | uint32_t remaining_pt_count = p->count - p->ptr; |
| | 348 | |
| | 349 | kmem_cache_free_bulk(pt_cache, remaining_pt_count, &p->pages[p->ptr]); |
| | 350 | kvfree(p->pages); |
| | 351 | } |
| | 352 | |
| | 353 | /** |
| | 354 | * alloc_pt() - Custom page table allocator |
| | 355 | * @cookie: Cookie passed at page table allocation time. |
| | 356 | * @size: Size of the page table. This size should be fixed, |
| | 357 | * and determined at creation time based on the granule size. |
| | 358 | * @gfp: GFP flags. |
| | 359 | * |
| | 360 | * We want a custom allocator so we can use a cache for page table |
| | 361 | * allocations and amortize the cost of the over-reservation that's |
| | 362 | * done to allow asynchronous VM operations. |
| | 363 | * |
| | 364 | * Return: non-NULL on success, NULL if the allocation failed for any |
| | 365 | * reason. |
| | 366 | */ |
| | 367 | static void * |
| | 368 | msm_iommu_pagetable_alloc_pt(void *cookie, size_t size, gfp_t gfp) |
| | 369 | { |
| | 370 | struct msm_iommu_pagetable *pagetable = cookie; |
| | 371 | struct msm_mmu_prealloc *p = pagetable->base.prealloc; |
| | 372 | void *page; |
| | 373 | |
| | 374 | /* Allocation of the root page table happening during init. */ |
| | 375 | if (unlikely(!pagetable->root_page_table)) { |
| | 376 | struct page *p; |
| | 377 | |
| | 378 | p = alloc_pages_node(dev_to_node(pagetable->iommu_dev), |
| | 379 | gfp | __GFP_ZERO, get_order(size)); |
| | 380 | page = p ? page_address(p) : NULL; |
| | 381 | pagetable->root_page_table = page; |
| | 382 | return page; |
| | 383 | } |
| | 384 | |
| | 385 | if (WARN_ON(!p) || WARN_ON(p->ptr >= p->count)) |
| | 386 | return NULL; |
| | 387 | |
| | 388 | page = p->pages[p->ptr++]; |
| | 389 | memset(page, 0, size); |
| | 390 | |
| | 391 | /* |
| | 392 | * Page table entries don't use virtual addresses, which trips out |
| | 393 | * kmemleak. kmemleak_alloc_phys() might work, but physical addresses |
| | 394 | * are mixed with other fields, and I fear kmemleak won't detect that |
| | 395 | * either. |
| | 396 | * |
| | 397 | * Let's just ignore memory passed to the page-table driver for now. |
| | 398 | */ |
| | 399 | kmemleak_ignore(page); |
| | 400 | |
| | 401 | return page; |
| | 402 | } |
| | 403 | |
| | 404 | |
| | 405 | /** |
| | 406 | * free_pt() - Custom page table free function |
| | 407 | * @cookie: Cookie passed at page table allocation time. |
| | 408 | * @data: Page table to free. |
| | 409 | * @size: Size of the page table. This size should be fixed, |
| | 410 | * and determined at creation time based on the granule size. |
| | 411 | */ |
| | 412 | static void |
| | 413 | msm_iommu_pagetable_free_pt(void *cookie, void *data, size_t size) |
| | 414 | { |
| | 415 | struct msm_iommu_pagetable *pagetable = cookie; |
| | 416 | |
| | 417 | if (unlikely(pagetable->root_page_table == data)) { |
| | 418 | free_pages((unsigned long)data, get_order(size)); |
| | 419 | pagetable->root_page_table = NULL; |
| | 420 | return; |
| | 421 | } |
| | 422 | |
| | 423 | kmem_cache_free(get_pt_cache(&pagetable->base), data); |
| | 424 | } |
| | 425 | |
| 285 | static const struct msm_mmu_funcs pagetable_funcs = { | 426 | static const struct msm_mmu_funcs pagetable_funcs = { |
| | 427 | .prealloc_count = msm_iommu_pagetable_prealloc_count, |
| | 428 | .prealloc_allocate = msm_iommu_pagetable_prealloc_allocate, |
| | 429 | .prealloc_cleanup = msm_iommu_pagetable_prealloc_cleanup, |
| 286 | .map = msm_iommu_pagetable_map, | 430 | .map = msm_iommu_pagetable_map, |
| 287 | .unmap = msm_iommu_pagetable_unmap, | 431 | .unmap = msm_iommu_pagetable_unmap, |
| 288 | .destroy = msm_iommu_pagetable_destroy, | 432 | .destroy = msm_iommu_pagetable_destroy, |
| @@ -333,6 +477,17 @@ static const struct iommu_flush_ops tlb_ops = { · 44 unmodified lines |
| 333 | static int msm_gpu_fault_handler(struct iommu_domain *domain, struct device *dev, | 477 | static int msm_gpu_fault_handler(struct iommu_domain *domain, struct device *dev, |
| 334 | unsigned long iova, int flags, void *arg); | 478 | unsigned long iova, int flags, void *arg); |
| 335 | | 479 | |
| | 480 | static size_t get_tblsz(const struct io_pgtable_cfg *cfg) |
| | 481 | { |
| | 482 | int pg_shift, bits_per_level; |
| | 483 | |
| | 484 | pg_shift = __ffs(cfg->pgsize_bitmap); |
| | 485 | /* arm_lpae_iopte is u64: */ |
| | 486 | bits_per_level = pg_shift - ilog2(sizeof(u64)); |
| | 487 | |
| | 488 | return sizeof(u64) << bits_per_level; |
| | 489 | } |
| | 490 | |
| 336 | struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent, bool kernel_managed) | 491 | struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent, bool kernel_managed) |
| 337 | { | 492 | { |
| 338 | struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(parent->dev); | 493 | struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(parent->dev); |
| @@ -369,8 +524,34 @@ struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent, bool kernel_m · 30 unmodified lines |
| 369 | | 524 | |
| 370 | if (!kernel_managed) { | 525 | if (!kernel_managed) { |
| 371 | ttbr0_cfg.quirks |= IO_PGTABLE_QUIRK_NO_WARN; | 526 | ttbr0_cfg.quirks |= IO_PGTABLE_QUIRK_NO_WARN; |
| | 527 | |
| | 528 | /* |
| | 529 | * With userspace managed VM (aka VM_BIND), we need to pre- |
| | 530 | * allocate pages ahead of time for map/unmap operations, |
| | 531 | * handing them to io-pgtable via custom alloc/free ops as |
| | 532 | * needed: |
| | 533 | */ |
| | 534 | ttbr0_cfg.alloc = msm_iommu_pagetable_alloc_pt; |
| | 535 | ttbr0_cfg.free = msm_iommu_pagetable_free_pt; |
| | 536 | |
| | 537 | /* |
| | 538 | * Restrict to single page granules. Otherwise we may run |
| | 539 | * into a situation where userspace wants to unmap/remap |
| | 540 | * only a part of a larger block mapping, which is not |
| | 541 | * possible without unmapping the entire block. Which in |
| | 542 | * turn could cause faults if the GPU is accessing other |
| | 543 | * parts of the block mapping. |
| | 544 | * |
| | 545 | * Note that prior to commit 33729a5fc0ca ("iommu/io-pgtable-arm: |
| | 546 | * Remove split on unmap behavior)" this was handled in |
| | 547 | * io-pgtable-arm. But this apparently does not work |
| | 548 | * correctly on SMMUv3. |
| | 549 | */ |
| | 550 | WARN_ON(!(ttbr0_cfg.pgsize_bitmap & PAGE_SIZE)); |
| | 551 | ttbr0_cfg.pgsize_bitmap = PAGE_SIZE; |
| 372 | } | 552 | } |
| 373 | | 553 | |
| | 554 | pagetable->iommu_dev = ttbr1_cfg->iommu_dev; |
| 374 | pagetable->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, | 555 | pagetable->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, |
| 375 | &ttbr0_cfg, pagetable); | 556 | &ttbr0_cfg, pagetable); |
| 376 | | 557 | |
| @@ -414,7 +595,6 @@ struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent, bool kernel_m · 37 unmodified lines |
| 414 | /* Needed later for TLB flush */ | 595 | /* Needed later for TLB flush */ |
| 415 | pagetable->parent = parent; | 596 | pagetable->parent = parent; |
| 416 | pagetable->tlb = ttbr1_cfg->tlb; | 597 | pagetable->tlb = ttbr1_cfg->tlb; |
| 417 | pagetable->iommu_dev = ttbr1_cfg->iommu_dev; | | |
| 418 | pagetable->pgsize_bitmap = ttbr0_cfg.pgsize_bitmap; | 598 | pagetable->pgsize_bitmap = ttbr0_cfg.pgsize_bitmap; |
| 419 | pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr; | 599 | pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr; |
| 420 | | 600 | |
| @@ -510,6 +690,7 @@ static void msm_iommu_destroy(struct msm_mmu *mmu) · 89 unmodified lines |
| 510 | { | 690 | { |
| 511 | struct msm_iommu *iommu = to_msm_iommu(mmu); | 691 | struct msm_iommu *iommu = to_msm_iommu(mmu); |
| 512 | iommu_domain_free(iommu->domain); | 692 | iommu_domain_free(iommu->domain); |
| | 693 | kmem_cache_destroy(iommu->pt_cache); |
| 513 | kfree(iommu); | 694 | kfree(iommu); |
| 514 | } | 695 | } |
| 515 | | 696 | |
| @@ -583,6 +764,14 @@ struct msm_mmu *msm_iommu_gpu_new(struct device *dev, struct msm_gpu *gpu, unsig · 67 unmodified lines |
| 583 | return mmu; | 764 | return mmu; |
| 584 | | 765 | |
| 585 | iommu = to_msm_iommu(mmu); | 766 | iommu = to_msm_iommu(mmu); |
| | 767 | if (adreno_smmu && adreno_smmu->cookie) { |
| | 768 | const struct io_pgtable_cfg *cfg = |
| | 769 | adreno_smmu->get_ttbr1_cfg(adreno_smmu->cookie); |
| | 770 | size_t tblsz = get_tblsz(cfg); |
| | 771 | |
| | 772 | iommu->pt_cache = |
| | 773 | kmem_cache_create("msm-mmu-pt", tblsz, tblsz, 0, NULL); |
| | 774 | } |
| 586 | iommu_set_fault_handler(iommu->domain, msm_gpu_fault_handler, iommu); | 775 | iommu_set_fault_handler(iommu->domain, msm_gpu_fault_handler, iommu); |
| 587 | | 776 | |
| 588 | /* Enable stall on iommu fault: */ | 777 | /* Enable stall on iommu fault: */ |