Although shmem_fault() has been careful to count a major fault to vm_mm, shmem_getpage_gfp() has been careless in charging a remote access fault to current->mm owner's memcg instead of to vma->vm_mm owner's memcg: that is inconsistent with all the mem_cgroup charging on remote access faults in mm/memory.c. Fix it by passing fault_mm along with fault_type to shmem_get_page_gfp(); but in that case, now knowing the right mm, it's better for it to handle the PGMAJFAULT updates itself. And let's keep this clutter out of most callers' way: change the common shmem_getpage() wrapper to hide fault_mm and fault_type as well as gfp.
Unified · Split
| @@ -121,13 +121,14 @@ static bool shmem_should_replace_page(struct page *page, gfp_t gfp); · 120 unmodified lines | ||
| 121 | 121 | static int shmem_replace_page(struct page **pagep, gfp_t gfp, |
| 122 | 122 | struct shmem_inode_info *info, pgoff_t index); |
| 123 | 123 | static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, |
| 124 | − struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type); | |
| 124 | + struct page **pagep, enum sgp_type sgp, | |
| 125 | + gfp_t gfp, struct mm_struct *fault_mm, int *fault_type); | |
| 125 | 126 | |
| 126 | 127 | static inline int shmem_getpage(struct inode *inode, pgoff_t index, |
| 127 | − struct page **pagep, enum sgp_type sgp, int *fault_type) | |
| 128 | + struct page **pagep, enum sgp_type sgp) | |
| 128 | 129 | { |
| 129 | 130 | return shmem_getpage_gfp(inode, index, pagep, sgp, |
| 130 | − mapping_gfp_mask(inode->i_mapping), fault_type); | |
| 131 | + mapping_gfp_mask(inode->i_mapping), NULL, NULL); | |
| 131 | 132 | } |
| 132 | 133 | |
| 133 | 134 | static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) |
| @@ -527,7 +528,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, · 393 unmodified lines | ||
| 527 | 528 | |
| 528 | 529 | if (partial_start) { |
| 529 | 530 | struct page *page = NULL; |
| 530 | − shmem_getpage(inode, start - 1, &page, SGP_READ, NULL); | |
| 531 | + shmem_getpage(inode, start - 1, &page, SGP_READ); | |
| 531 | 532 | if (page) { |
| 532 | 533 | unsigned int top = PAGE_SIZE; |
| 533 | 534 | if (start > end) { |
| @@ -542,7 +543,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, · 8 unmodified lines | ||
| 542 | 543 | } |
| 543 | 544 | if (partial_end) { |
| 544 | 545 | struct page *page = NULL; |
| 545 | − shmem_getpage(inode, end, &page, SGP_READ, NULL); | |
| 546 | + shmem_getpage(inode, end, &page, SGP_READ); | |
| 546 | 547 | if (page) { |
| 547 | 548 | zero_user_segment(page, 0, partial_end); |
| 548 | 549 | set_page_dirty(page); |
| @@ -1115,14 +1116,19 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp, · 566 unmodified lines | ||
| 1115 | 1116 | * |
| 1116 | 1117 | * If we allocate a new one we do not mark it dirty. That's up to the |
| 1117 | 1118 | * vm. If we swap it in we mark it dirty since we also free the swap |
| 1118 | − * entry since a page cannot live in both the swap and page cache | |
| 1119 | + * entry since a page cannot live in both the swap and page cache. | |
| 1120 | + * | |
| 1121 | + * fault_mm and fault_type are only supplied by shmem_fault: | |
| 1122 | + * otherwise they are NULL. | |
| 1119 | 1123 | */ |
| 1120 | 1124 | static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, |
| 1121 | − struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type) | |
| 1125 | + struct page **pagep, enum sgp_type sgp, gfp_t gfp, | |
| 1126 | + struct mm_struct *fault_mm, int *fault_type) | |
| 1122 | 1127 | { |
| 1123 | 1128 | struct address_space *mapping = inode->i_mapping; |
| 1124 | 1129 | struct shmem_inode_info *info; |
| 1125 | 1130 | struct shmem_sb_info *sbinfo; |
| 1131 | + struct mm_struct *charge_mm; | |
| 1126 | 1132 | struct mem_cgroup *memcg; |
| 1127 | 1133 | struct page *page; |
| 1128 | 1134 | swp_entry_t swap; |
| @@ -1168,14 +1174,19 @@ repeat: · 39 unmodified lines | ||
| 1168 | 1174 | */ |
| 1169 | 1175 | info = SHMEM_I(inode); |
| 1170 | 1176 | sbinfo = SHMEM_SB(inode->i_sb); |
| 1177 | + charge_mm = fault_mm ? : current->mm; | |
| 1171 | 1178 | |
| 1172 | 1179 | if (swap.val) { |
| 1173 | 1180 | /* Look it up and read it in.. */ |
| 1174 | 1181 | page = lookup_swap_cache(swap); |
| 1175 | 1182 | if (!page) { |
| 1176 | − /* here we actually do the io */ | |
| 1177 | − if (fault_type) | |
| 1183 | + /* Or update major stats only when swapin succeeds?? */ | |
| 1184 | + if (fault_type) { | |
| 1178 | 1185 | *fault_type |= VM_FAULT_MAJOR; |
| 1186 | + count_vm_event(PGMAJFAULT); | |
| 1187 | + mem_cgroup_count_vm_event(fault_mm, PGMAJFAULT); | |
| 1188 | + } | |
| 1189 | + /* Here we actually start the io */ | |
| 1179 | 1190 | page = shmem_swapin(swap, gfp, info, index); |
| 1180 | 1191 | if (!page) { |
| 1181 | 1192 | error = -ENOMEM; |
| @@ -1202,7 +1213,7 @@ repeat: · 20 unmodified lines | ||
| 1202 | 1213 | goto failed; |
| 1203 | 1214 | } |
| 1204 | 1215 | |
| 1205 | − error = mem_cgroup_try_charge(page, current->mm, gfp, &memcg, | |
| 1216 | + error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, | |
| 1206 | 1217 | false); |
| 1207 | 1218 | if (!error) { |
| 1208 | 1219 | error = shmem_add_to_page_cache(page, mapping, index, |
| @@ -1263,7 +1274,7 @@ repeat: · 54 unmodified lines | ||
| 1263 | 1274 | if (sgp == SGP_WRITE) |
| 1264 | 1275 | __SetPageReferenced(page); |
| 1265 | 1276 | |
| 1266 | − error = mem_cgroup_try_charge(page, current->mm, gfp, &memcg, | |
| 1277 | + error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, | |
| 1267 | 1278 | false); |
| 1268 | 1279 | if (error) |
| 1269 | 1280 | goto decused; |
| @@ -1352,6 +1363,7 @@ unlock: · 82 unmodified lines | ||
| 1352 | 1363 | static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) |
| 1353 | 1364 | { |
| 1354 | 1365 | struct inode *inode = file_inode(vma->vm_file); |
| 1366 | + gfp_t gfp = mapping_gfp_mask(inode->i_mapping); | |
| 1355 | 1367 | int error; |
| 1356 | 1368 | int ret = VM_FAULT_LOCKED; |
| 1357 | 1369 | |
| @@ -1413,14 +1425,10 @@ static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) · 55 unmodified lines | ||
| 1413 | 1425 | spin_unlock(&inode->i_lock); |
| 1414 | 1426 | } |
| 1415 | 1427 | |
| 1416 | − error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret); | |
| 1428 | + error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, SGP_CACHE, | |
| 1429 | + gfp, vma->vm_mm, &ret); | |
| 1417 | 1430 | if (error) |
| 1418 | 1431 | return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); |
| 1419 | − | |
| 1420 | − if (ret & VM_FAULT_MAJOR) { | |
| 1421 | − count_vm_event(PGMAJFAULT); | |
| 1422 | − mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); | |
| 1423 | − } | |
| 1424 | 1432 | return ret; |
| 1425 | 1433 | } |
| 1426 | 1434 | |
| @@ -1567,7 +1575,7 @@ shmem_write_begin(struct file *file, struct address_space *mapping, · 140 unmodified lines | ||
| 1567 | 1575 | return -EPERM; |
| 1568 | 1576 | } |
| 1569 | 1577 | |
| 1570 | − return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL); | |
| 1578 | + return shmem_getpage(inode, index, pagep, SGP_WRITE); | |
| 1571 | 1579 | } |
| 1572 | 1580 | |
| 1573 | 1581 | static int |
| @@ -1633,7 +1641,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) · 59 unmodified lines | ||
| 1633 | 1641 | break; |
| 1634 | 1642 | } |
| 1635 | 1643 | |
| 1636 | − error = shmem_getpage(inode, index, &page, sgp, NULL); | |
| 1644 | + error = shmem_getpage(inode, index, &page, sgp); | |
| 1637 | 1645 | if (error) { |
| 1638 | 1646 | if (error == -EINVAL) |
| 1639 | 1647 | error = 0; |
| @@ -1749,7 +1757,7 @@ static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos, · 109 unmodified lines | ||
| 1749 | 1757 | error = 0; |
| 1750 | 1758 | |
| 1751 | 1759 | while (spd.nr_pages < nr_pages) { |
| 1752 | − error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL); | |
| 1760 | + error = shmem_getpage(inode, index, &page, SGP_CACHE); | |
| 1753 | 1761 | if (error) |
| 1754 | 1762 | break; |
| 1755 | 1763 | unlock_page(page); |
| @@ -1771,8 +1779,7 @@ static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos, · 15 unmodified lines | ||
| 1771 | 1779 | page = spd.pages[page_nr]; |
| 1772 | 1780 | |
| 1773 | 1781 | if (!PageUptodate(page) || page->mapping != mapping) { |
| 1774 | − error = shmem_getpage(inode, index, &page, | |
| 1775 | − SGP_CACHE, NULL); | |
| 1782 | + error = shmem_getpage(inode, index, &page, SGP_CACHE); | |
| 1776 | 1783 | if (error) |
| 1777 | 1784 | break; |
| 1778 | 1785 | unlock_page(page); |
| @@ -2215,8 +2222,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, · 436 unmodified lines | ||
| 2215 | 2222 | else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) |
| 2216 | 2223 | error = -ENOMEM; |
| 2217 | 2224 | else |
| 2218 | − error = shmem_getpage(inode, index, &page, SGP_FALLOC, | |
| 2219 | − NULL); | |
| 2225 | + error = shmem_getpage(inode, index, &page, SGP_FALLOC); | |
| 2220 | 2226 | if (error) { |
| 2221 | 2227 | /* Remove the !PageUptodate pages we added */ |
| 2222 | 2228 | shmem_undo_range(inode, |
| @@ -2534,7 +2540,7 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s · 311 unmodified lines | ||
| 2534 | 2540 | inode->i_op = &shmem_short_symlink_operations; |
| 2535 | 2541 | } else { |
| 2536 | 2542 | inode_nohighmem(inode); |
| 2537 | − error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL); | |
| 2543 | + error = shmem_getpage(inode, 0, &page, SGP_WRITE); | |
| 2538 | 2544 | if (error) { |
| 2539 | 2545 | iput(inode); |
| 2540 | 2546 | return error; |
| @@ -2575,7 +2581,7 @@ static const char *shmem_get_link(struct dentry *dentry, · 34 unmodified lines | ||
| 2575 | 2581 | return ERR_PTR(-ECHILD); |
| 2576 | 2582 | } |
| 2577 | 2583 | } else { |
| 2578 | − error = shmem_getpage(inode, 0, &page, SGP_READ, NULL); | |
| 2584 | + error = shmem_getpage(inode, 0, &page, SGP_READ); | |
| 2579 | 2585 | if (error) |
| 2580 | 2586 | return ERR_PTR(error); |
| 2581 | 2587 | unlock_page(page); |
| @@ -3479,7 +3485,8 @@ struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, · 897 unmodified lines | ||
| 3479 | 3485 | int error; |
| 3480 | 3486 | |
| 3481 | 3487 | BUG_ON(mapping->a_ops != &shmem_aops); |
| 3482 | − error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL); | |
| 3488 | + error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, | |
| 3489 | + gfp, NULL, NULL); | |
| 3483 | 3490 | if (error) |
| 3484 | 3491 | page = ERR_PTR(error); |
| 3485 | 3492 | else |