The recent anon_vma fixes cause many anonymous pages to end up in the parent process anon_vma, even when the page is exclusively owned by the current process. Adding exclusively owned anonymous pages to the top anon_vma reduces rmap scanning overhead, especially in workloads with forking servers. This patch adds a parameter to __page_set_anon_rmap that can be used to indicate whether or not the added page is exclusively owned by the current process. Pages added through page_add_new_anon_rmap are exclusively owned by the current process, and can be added to the top anon_vma. Pages added through page_add_anon_rmap can be either shared or exclusively owned, so we do the conservative thing and add it to the oldest anon_vma. A next step would be to add the exclusive parameter to page_add_anon_rmap, to be used from functions where we do know for sure whether a page is exclusively owned.
Unified · Split
| @@ -730,23 +730,28 @@ void page_move_anon_rmap(struct page *page, · 729 unmodified lines | |||
| 730 | * @page: the page to add the mapping to | 730 | * @page: the page to add the mapping to |
| 731 | * @vma: the vm area in which the mapping is added | 731 | * @vma: the vm area in which the mapping is added |
| 732 | * @address: the user virtual address mapped | 732 | * @address: the user virtual address mapped |
| 733 | * @exclusive: the page is exclusively owned by the current process | ||
| 733 | */ | 734 | */ |
| 734 | static void __page_set_anon_rmap(struct page *page, | 735 | static void __page_set_anon_rmap(struct page *page, |
| 735 | struct vm_area_struct *vma, unsigned long address) | 736 | struct vm_area_struct *vma, unsigned long address, int exclusive) |
| 736 | { | 737 | { |
| 737 | struct anon_vma_chain *avc; | 738 | struct anon_vma *anon_vma = vma->anon_vma; |
| 738 | struct anon_vma *anon_vma; | ||
| 739 | 739 | ||
| 740 | BUG_ON(!vma->anon_vma); | 740 | BUG_ON(!anon_vma); |
| 741 | 741 | ||
| 742 | /* | 742 | /* |
| 743 | * We must use the _oldest_ possible anon_vma for the page mapping! | 743 | * If the page isn't exclusively mapped into this vma, |
| 744 | * we must use the _oldest_ possible anon_vma for the | ||
| 745 | * page mapping! | ||
| 744 | * | 746 | * |
| 745 | * So take the last AVC chain entry in the vma, which is the deepest | 747 | * So take the last AVC chain entry in the vma, which is |
| 746 | * ancestor, and use the anon_vma from that. | 748 | * the deepest ancestor, and use the anon_vma from that. |
| 747 | */ | 749 | */ |
| 748 | avc = list_entry(vma->anon_vma_chain.prev, struct anon_vma_chain, same_vma); | 750 | if (!exclusive) { |
| 749 | anon_vma = avc->anon_vma; | 751 | struct anon_vma_chain *avc; |
| 752 | avc = list_entry(vma->anon_vma_chain.prev, struct anon_vma_chain, same_vma); | ||
| 753 | anon_vma = avc->anon_vma; | ||
| 754 | } | ||
| 750 | 755 | ||
| 751 | anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; | 756 | anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; |
| 752 | page->mapping = (struct address_space *) anon_vma; | 757 | page->mapping = (struct address_space *) anon_vma; |
| @@ -802,7 +807,7 @@ void page_add_anon_rmap(struct page *page, · 49 unmodified lines | |||
| 802 | VM_BUG_ON(!PageLocked(page)); | 807 | VM_BUG_ON(!PageLocked(page)); |
| 803 | VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); | 808 | VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); |
| 804 | if (first) | 809 | if (first) |
| 805 | __page_set_anon_rmap(page, vma, address); | 810 | __page_set_anon_rmap(page, vma, address, 0); |
| 806 | else | 811 | else |
| 807 | __page_check_anon_rmap(page, vma, address); | 812 | __page_check_anon_rmap(page, vma, address); |
| 808 | } | 813 | } |
| @@ -824,7 +829,7 @@ void page_add_new_anon_rmap(struct page *page, · 15 unmodified lines | |||
| 824 | SetPageSwapBacked(page); | 829 | SetPageSwapBacked(page); |
| 825 | atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */ | 830 | atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */ |
| 826 | __inc_zone_page_state(page, NR_ANON_PAGES); | 831 | __inc_zone_page_state(page, NR_ANON_PAGES); |
| 827 | __page_set_anon_rmap(page, vma, address); | 832 | __page_set_anon_rmap(page, vma, address, 1); |
| 828 | if (page_evictable(page, vma)) | 833 | if (page_evictable(page, vma)) |
| 829 | lru_cache_add_lru(page, LRU_ACTIVE_ANON); | 834 | lru_cache_add_lru(page, LRU_ACTIVE_ANON); |
| 830 | else | 835 | else |