(Commit message majorly collected from Jason Gunthorpe) Reduce the chance of false positive from page_maybe_dma_pinned() by keeping track if the mm_struct has ever been used with pin_user_pages(). This allows cases that might drive up the page ref_count to avoid any penalty from handling dma_pinned pages. Future work is planned, to provide a more sophisticated solution, likely to turn it into a real counter. For now, make it atomic_t but use it as a boolean for simplicity.
Unified · Split
| @@ -436,6 +436,16 @@ struct mm_struct { · 435 unmodified lines | |||
| 436 | */ | 436 | */ |
| 437 | atomic_t mm_count; | 437 | atomic_t mm_count; |
| 438 | 438 | ||
| 439 | /** | ||
| 440 | * @has_pinned: Whether this mm has pinned any pages. This can | ||
| 441 | * be either replaced in the future by @pinned_vm when it | ||
| 442 | * becomes stable, or grow into a counter on its own. We're | ||
| 443 | * aggresive on this bit now - even if the pinned pages were | ||
| 444 | * unpinned later on, we'll still keep this bit set for the | ||
| 445 | * lifecycle of this mm just for simplicity. | ||
| 446 | */ | ||
| 447 | atomic_t has_pinned; | ||
| 448 | |||
| 439 | #ifdef CONFIG_MMU | 449 | #ifdef CONFIG_MMU |
| 440 | atomic_long_t pgtables_bytes; /* PTE page table pages */ | 450 | atomic_long_t pgtables_bytes; /* PTE page table pages */ |
| 441 | #endif | 451 | #endif |
| @@ -1011,6 +1011,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, · 1010 unmodified lines | |||
| 1011 | mm_pgtables_bytes_init(mm); | 1011 | mm_pgtables_bytes_init(mm); |
| 1012 | mm->map_count = 0; | 1012 | mm->map_count = 0; |
| 1013 | mm->locked_vm = 0; | 1013 | mm->locked_vm = 0; |
| 1014 | atomic_set(&mm->has_pinned, 0); | ||
| 1014 | atomic64_set(&mm->pinned_vm, 0); | 1015 | atomic64_set(&mm->pinned_vm, 0); |
| 1015 | memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); | 1016 | memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); |
| 1016 | spin_lock_init(&mm->page_table_lock); | 1017 | spin_lock_init(&mm->page_table_lock); |
| @@ -1255,6 +1255,9 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, · 1254 unmodified lines | |||
| 1255 | BUG_ON(*locked != 1); | 1255 | BUG_ON(*locked != 1); |
| 1256 | } | 1256 | } |
| 1257 | 1257 | ||
| 1258 | if (flags & FOLL_PIN) | ||
| 1259 | atomic_set(¤t->mm->has_pinned, 1); | ||
| 1260 | |||
| 1258 | /* | 1261 | /* |
| 1259 | * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior | 1262 | * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior |
| 1260 | * is to set FOLL_GET if the caller wants pages[] filled in (but has | 1263 | * is to set FOLL_GET if the caller wants pages[] filled in (but has |
| @@ -2660,6 +2663,9 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, · 1399 unmodified lines | |||
| 2660 | FOLL_FAST_ONLY))) | 2663 | FOLL_FAST_ONLY))) |
| 2661 | return -EINVAL; | 2664 | return -EINVAL; |
| 2662 | 2665 | ||
| 2666 | if (gup_flags & FOLL_PIN) | ||
| 2667 | atomic_set(¤t->mm->has_pinned, 1); | ||
| 2668 | |||
| 2663 | if (!(gup_flags & FOLL_FAST_ONLY)) | 2669 | if (!(gup_flags & FOLL_FAST_ONLY)) |
| 2664 | might_lock_read(¤t->mm->mmap_lock); | 2670 | might_lock_read(¤t->mm->mmap_lock); |
| 2665 | 2671 | ||