History

mm: do page fault accounting in handle_mm_fault

Patch series "mm: Page fault accounting cleanups", v5. This is v5 of the pf accounting cleanup series. It originates from Gerald Schaefer's report on an issue a week ago regarding to incorrect page fault accountings for retried page fault after commit 4064b9827063 ("mm: allow VM_FAULT_RETRY for multiple times"): https://lore.kernel.org/lkml/20200610174811.44b94525@thinkpad/ What this series did: - Correct page fault accounting: we do accounting for a page fault (no matter whether it's from #PF handling, or gup, or anything else) only with the one that completed the fault. For example, page fault retries should not be counted in page fault counters. Same to the perf events. - Unify definition of PERF_COUNT_SW_PAGE_FAULTS: currently this perf event is used in an adhoc way across different archs. Case (1): for many archs it's done at the entry of a page fault handler, so that it will also cover e.g. errornous faults. Case (2): for some other archs, it is only accounted when the page fault is resolved successfully. Case (3): there're still quite some archs that have not enabled this perf event. Since this series will touch merely all the archs, we unify this perf event to always follow case (1), which is the one that makes most sense. And since we moved the accounting into handle_mm_fault, the other two MAJ/MIN perf events are well taken care of naturally. - Unify definition of "major faults": the definition of "major fault" is slightly changed when used in accounting (not VM_FAULT_MAJOR). More information in patch 1. - Always account the page fault onto the one that triggered the page fault. This does not matter much for #PF handlings, but mostly for gup. More information on this in patch 25. Patchset layout: Patch 1: Introduced the accounting in handle_mm_fault(), not enabled. Patch 2-23: Enable the new accounting for arch #PF handlers one by one. Patch 24: Enable the new accounting for the rest outliers (gup, iommu, etc.) Patch 25: Cleanup GUP task_struct pointer since it's not needed any more This patch (of 25): This is a preparation patch to move page fault accountings into the general code in handle_mm_fault(). This includes both the per task flt_maj/flt_min counters, and the major/minor page fault perf events. To do this, the pt_regs pointer is passed into handle_mm_fault(). PERF_COUNT_SW_PAGE_FAULTS should still be kept in per-arch page fault handlers. So far, all the pt_regs pointer that passed into handle_mm_fault() is NULL, which means this patch should have no intented functional change.

Peter Xu committed 6 years ago

Commit bce617e1 parent ed03d9231 files · +103 −34Browse files

Unified · Split

31 files
  1. arch/alpha/mm/fault.c +1 −1
  2. arch/arc/mm/fault.c +1 −1
  3. arch/arm/mm/fault.c +1 −1
  4. arch/arm64/mm/fault.c +1 −1
  5. arch/csky/mm/fault.c +2 −1
  6. arch/hexagon/mm/vm_fault.c +1 −1
  7. arch/ia64/mm/fault.c +1 −1
  8. arch/m68k/mm/fault.c +1 −1
  9. arch/microblaze/mm/fault.c +1 −1
  10. arch/mips/mm/fault.c +1 −1
  11. arch/nds32/mm/fault.c +1 −1
  12. arch/nios2/mm/fault.c +1 −1
  13. arch/openrisc/mm/fault.c +1 −1
  14. arch/parisc/mm/fault.c +1 −1
  15. arch/powerpc/mm/copro_fault.c +1 −1
  16. arch/powerpc/mm/fault.c +1 −1
  17. arch/riscv/mm/fault.c +1 −1
  18. arch/s390/mm/fault.c +1 −1
  19. arch/sh/mm/fault.c +1 −1
  20. arch/sparc/mm/fault_32.c +2 −2
  21. arch/sparc/mm/fault_64.c +1 −1
  22. arch/um/kernel/trap.c +1 −1
  23. arch/x86/mm/fault.c +1 −1
  24. arch/xtensa/mm/fault.c +1 −1
  25. drivers/iommu/amd/iommu_v2.c +1 −1
  26. drivers/iommu/intel/svm.c +2 −1
  27. include/linux/mm.h +5 −2
  28. mm/gup.c +2 −2
  29. mm/hmm.c +2 −1
  30. mm/ksm.c +2 −1
  31. mm/memory.c +63 −1
arch/alpha/mm/fault.c+1−1
@@ -148,7 +148,7 @@ retry: · 147 unmodified lines
148 /* If for any reason at all we couldn't handle the fault,148 /* If for any reason at all we couldn't handle the fault,
149 make sure we exit gracefully rather than endlessly redo149 make sure we exit gracefully rather than endlessly redo
150 the fault. */150 the fault. */
151 fault = handle_mm_fault(vma, address, flags);151 fault = handle_mm_fault(vma, address, flags, NULL);
152 152
153 if (fault_signal_pending(fault, regs))153 if (fault_signal_pending(fault, regs))
154 return;154 return;
arch/arc/mm/fault.c+1−1
@@ -130,7 +130,7 @@ retry: · 129 unmodified lines
130 goto bad_area;130 goto bad_area;
131 }131 }
132 132
133 fault = handle_mm_fault(vma, address, flags);133 fault = handle_mm_fault(vma, address, flags, NULL);
134 134
135 /* Quick path to respond to signals */135 /* Quick path to respond to signals */
136 if (fault_signal_pending(fault, regs)) {136 if (fault_signal_pending(fault, regs)) {
arch/arm/mm/fault.c+1−1
@@ -224,7 +224,7 @@ good_area: · 223 unmodified lines
224 goto out;224 goto out;
225 }225 }
226 226
227 return handle_mm_fault(vma, addr & PAGE_MASK, flags);227 return handle_mm_fault(vma, addr & PAGE_MASK, flags, NULL);
228 228
229check_stack:229check_stack:
230 /* Don't allow expansion below FIRST_USER_ADDRESS */230 /* Don't allow expansion below FIRST_USER_ADDRESS */
arch/arm64/mm/fault.c+1−1
@@ -428,7 +428,7 @@ static vm_fault_t __do_page_fault(struct mm_struct *mm, unsigned long addr, · 427 unmodified lines
428 */428 */
429 if (!(vma->vm_flags & vm_flags))429 if (!(vma->vm_flags & vm_flags))
430 return VM_FAULT_BADACCESS;430 return VM_FAULT_BADACCESS;
431 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags);431 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags, NULL);
432}432}
433 433
434static bool is_el0_instruction_abort(unsigned int esr)434static bool is_el0_instruction_abort(unsigned int esr)
arch/csky/mm/fault.c+2−1
@@ -150,7 +150,8 @@ good_area: · 149 unmodified lines
150 * make sure we exit gracefully rather than endlessly redo150 * make sure we exit gracefully rather than endlessly redo
151 * the fault.151 * the fault.
152 */152 */
153 fault = handle_mm_fault(vma, address, write ? FAULT_FLAG_WRITE : 0);153 fault = handle_mm_fault(vma, address, write ? FAULT_FLAG_WRITE : 0,
154 NULL);
154 if (unlikely(fault & VM_FAULT_ERROR)) {155 if (unlikely(fault & VM_FAULT_ERROR)) {
155 if (fault & VM_FAULT_OOM)156 if (fault & VM_FAULT_OOM)
156 goto out_of_memory;157 goto out_of_memory;
arch/hexagon/mm/vm_fault.c+1−1
@@ -88,7 +88,7 @@ good_area: · 87 unmodified lines
88 break;88 break;
89 }89 }
90 90
91 fault = handle_mm_fault(vma, address, flags);91 fault = handle_mm_fault(vma, address, flags, NULL);
92 92
93 if (fault_signal_pending(fault, regs))93 if (fault_signal_pending(fault, regs))
94 return;94 return;
arch/ia64/mm/fault.c+1−1
@@ -143,7 +143,7 @@ retry: · 142 unmodified lines
143 * sure we exit gracefully rather than endlessly redo the143 * sure we exit gracefully rather than endlessly redo the
144 * fault.144 * fault.
145 */145 */
146 fault = handle_mm_fault(vma, address, flags);146 fault = handle_mm_fault(vma, address, flags, NULL);
147 147
148 if (fault_signal_pending(fault, regs))148 if (fault_signal_pending(fault, regs))
149 return;149 return;
arch/m68k/mm/fault.c+1−1
@@ -134,7 +134,7 @@ good_area: · 133 unmodified lines
134 * the fault.134 * the fault.
135 */135 */
136 136
137 fault = handle_mm_fault(vma, address, flags);137 fault = handle_mm_fault(vma, address, flags, NULL);
138 pr_debug("handle_mm_fault returns %x\n", fault);138 pr_debug("handle_mm_fault returns %x\n", fault);
139 139
140 if (fault_signal_pending(fault, regs))140 if (fault_signal_pending(fault, regs))
arch/microblaze/mm/fault.c+1−1
@@ -214,7 +214,7 @@ good_area: · 213 unmodified lines
214 * make sure we exit gracefully rather than endlessly redo214 * make sure we exit gracefully rather than endlessly redo
215 * the fault.215 * the fault.
216 */216 */
217 fault = handle_mm_fault(vma, address, flags);217 fault = handle_mm_fault(vma, address, flags, NULL);
218 218
219 if (fault_signal_pending(fault, regs))219 if (fault_signal_pending(fault, regs))
220 return;220 return;
arch/mips/mm/fault.c+1−1
@@ -152,7 +152,7 @@ good_area: · 151 unmodified lines
152 * make sure we exit gracefully rather than endlessly redo152 * make sure we exit gracefully rather than endlessly redo
153 * the fault.153 * the fault.
154 */154 */
155 fault = handle_mm_fault(vma, address, flags);155 fault = handle_mm_fault(vma, address, flags, NULL);
156 156
157 if (fault_signal_pending(fault, regs))157 if (fault_signal_pending(fault, regs))
158 return;158 return;
arch/nds32/mm/fault.c+1−1
@@ -206,7 +206,7 @@ good_area: · 205 unmodified lines
206 * the fault.206 * the fault.
207 */207 */
208 208
209 fault = handle_mm_fault(vma, addr, flags);209 fault = handle_mm_fault(vma, addr, flags, NULL);
210 210
211 /*211 /*
212 * If we need to retry but a fatal signal is pending, handle the212 * If we need to retry but a fatal signal is pending, handle the
arch/nios2/mm/fault.c+1−1
@@ -131,7 +131,7 @@ good_area: · 130 unmodified lines
131 * make sure we exit gracefully rather than endlessly redo131 * make sure we exit gracefully rather than endlessly redo
132 * the fault.132 * the fault.
133 */133 */
134 fault = handle_mm_fault(vma, address, flags);134 fault = handle_mm_fault(vma, address, flags, NULL);
135 135
136 if (fault_signal_pending(fault, regs))136 if (fault_signal_pending(fault, regs))
137 return;137 return;
arch/openrisc/mm/fault.c+1−1
@@ -159,7 +159,7 @@ good_area: · 158 unmodified lines
159 * the fault.159 * the fault.
160 */160 */
161 161
162 fault = handle_mm_fault(vma, address, flags);162 fault = handle_mm_fault(vma, address, flags, NULL);
163 163
164 if (fault_signal_pending(fault, regs))164 if (fault_signal_pending(fault, regs))
165 return;165 return;
arch/parisc/mm/fault.c+1−1
@@ -302,7 +302,7 @@ good_area: · 301 unmodified lines
302 * fault.302 * fault.
303 */303 */
304 304
305 fault = handle_mm_fault(vma, address, flags);305 fault = handle_mm_fault(vma, address, flags, NULL);
306 306
307 if (fault_signal_pending(fault, regs))307 if (fault_signal_pending(fault, regs))
308 return;308 return;
arch/powerpc/mm/copro_fault.c+1−1
@@ -64,7 +64,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, · 63 unmodified lines
64 }64 }
65 65
66 ret = 0;66 ret = 0;
67 *flt = handle_mm_fault(vma, ea, is_write ? FAULT_FLAG_WRITE : 0);67 *flt = handle_mm_fault(vma, ea, is_write ? FAULT_FLAG_WRITE : 0, NULL);
68 if (unlikely(*flt & VM_FAULT_ERROR)) {68 if (unlikely(*flt & VM_FAULT_ERROR)) {
69 if (*flt & VM_FAULT_OOM) {69 if (*flt & VM_FAULT_OOM) {
70 ret = -ENOMEM;70 ret = -ENOMEM;
arch/powerpc/mm/fault.c+1−1
@@ -511,7 +511,7 @@ retry: · 510 unmodified lines
511 * make sure we exit gracefully rather than endlessly redo511 * make sure we exit gracefully rather than endlessly redo
512 * the fault.512 * the fault.
513 */513 */
514 fault = handle_mm_fault(vma, address, flags);514 fault = handle_mm_fault(vma, address, flags, NULL);
515 515
516 major |= fault & VM_FAULT_MAJOR;516 major |= fault & VM_FAULT_MAJOR;
517 517
arch/riscv/mm/fault.c+1−1
@@ -109,7 +109,7 @@ good_area: · 108 unmodified lines
109 * make sure we exit gracefully rather than endlessly redo109 * make sure we exit gracefully rather than endlessly redo
110 * the fault.110 * the fault.
111 */111 */
112 fault = handle_mm_fault(vma, addr, flags);112 fault = handle_mm_fault(vma, addr, flags, NULL);
113 113
114 /*114 /*
115 * If we need to retry but a fatal signal is pending, handle the115 * If we need to retry but a fatal signal is pending, handle the
arch/s390/mm/fault.c+1−1
@@ -476,7 +476,7 @@ retry: · 475 unmodified lines
476 * make sure we exit gracefully rather than endlessly redo476 * make sure we exit gracefully rather than endlessly redo
477 * the fault.477 * the fault.
478 */478 */
479 fault = handle_mm_fault(vma, address, flags);479 fault = handle_mm_fault(vma, address, flags, NULL);
480 if (fault_signal_pending(fault, regs)) {480 if (fault_signal_pending(fault, regs)) {
481 fault = VM_FAULT_SIGNAL;481 fault = VM_FAULT_SIGNAL;
482 if (flags & FAULT_FLAG_RETRY_NOWAIT)482 if (flags & FAULT_FLAG_RETRY_NOWAIT)
arch/sh/mm/fault.c+1−1
@@ -482,7 +482,7 @@ good_area: · 481 unmodified lines
482 * make sure we exit gracefully rather than endlessly redo482 * make sure we exit gracefully rather than endlessly redo
483 * the fault.483 * the fault.
484 */484 */
485 fault = handle_mm_fault(vma, address, flags);485 fault = handle_mm_fault(vma, address, flags, NULL);
486 486
487 if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR)))487 if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR)))
488 if (mm_fault_error(regs, error_code, address, fault))488 if (mm_fault_error(regs, error_code, address, fault))
arch/sparc/mm/fault_32.c+2−2
@@ -234,7 +234,7 @@ good_area: · 233 unmodified lines
234 * make sure we exit gracefully rather than endlessly redo234 * make sure we exit gracefully rather than endlessly redo
235 * the fault.235 * the fault.
236 */236 */
237 fault = handle_mm_fault(vma, address, flags);237 fault = handle_mm_fault(vma, address, flags, NULL);
238 238
239 if (fault_signal_pending(fault, regs))239 if (fault_signal_pending(fault, regs))
240 return;240 return;
@@ -410,7 +410,7 @@ good_area: · 169 unmodified lines
410 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))410 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
411 goto bad_area;411 goto bad_area;
412 }412 }
413 switch (handle_mm_fault(vma, address, flags)) {413 switch (handle_mm_fault(vma, address, flags, NULL)) {
414 case VM_FAULT_SIGBUS:414 case VM_FAULT_SIGBUS:
415 case VM_FAULT_OOM:415 case VM_FAULT_OOM:
416 goto do_sigbus;416 goto do_sigbus;
arch/sparc/mm/fault_64.c+1−1
@@ -422,7 +422,7 @@ good_area: · 421 unmodified lines
422 goto bad_area;422 goto bad_area;
423 }423 }
424 424
425 fault = handle_mm_fault(vma, address, flags);425 fault = handle_mm_fault(vma, address, flags, NULL);
426 426
427 if (fault_signal_pending(fault, regs))427 if (fault_signal_pending(fault, regs))
428 goto exit_exception;428 goto exit_exception;
arch/um/kernel/trap.c+1−1
@@ -71,7 +71,7 @@ good_area: · 70 unmodified lines
71 do {71 do {
72 vm_fault_t fault;72 vm_fault_t fault;
73 73
74 fault = handle_mm_fault(vma, address, flags);74 fault = handle_mm_fault(vma, address, flags, NULL);
75 75
76 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))76 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
77 goto out_nosemaphore;77 goto out_nosemaphore;
arch/x86/mm/fault.c+1−1
@@ -1291,7 +1291,7 @@ good_area: · 1290 unmodified lines
1291 * userland). The return to userland is identified whenever1291 * userland). The return to userland is identified whenever
1292 * FAULT_FLAG_USER|FAULT_FLAG_KILLABLE are both set in flags.1292 * FAULT_FLAG_USER|FAULT_FLAG_KILLABLE are both set in flags.
1293 */1293 */
1294 fault = handle_mm_fault(vma, address, flags);1294 fault = handle_mm_fault(vma, address, flags, NULL);
1295 major |= fault & VM_FAULT_MAJOR;1295 major |= fault & VM_FAULT_MAJOR;
1296 1296
1297 /* Quick path to respond to signals */1297 /* Quick path to respond to signals */
arch/xtensa/mm/fault.c+1−1
@@ -107,7 +107,7 @@ good_area: · 106 unmodified lines
107 * make sure we exit gracefully rather than endlessly redo107 * make sure we exit gracefully rather than endlessly redo
108 * the fault.108 * the fault.
109 */109 */
110 fault = handle_mm_fault(vma, address, flags);110 fault = handle_mm_fault(vma, address, flags, NULL);
111 111
112 if (fault_signal_pending(fault, regs))112 if (fault_signal_pending(fault, regs))
113 return;113 return;
drivers/iommu/amd/iommu_v2.c+1−1
@@ -495,7 +495,7 @@ static void do_fault(struct work_struct *work) · 494 unmodified lines
495 if (access_error(vma, fault))495 if (access_error(vma, fault))
496 goto out;496 goto out;
497 497
498 ret = handle_mm_fault(vma, address, flags);498 ret = handle_mm_fault(vma, address, flags, NULL);
499out:499out:
500 mmap_read_unlock(mm);500 mmap_read_unlock(mm);
501 501
drivers/iommu/intel/svm.c+2−1
@@ -872,7 +872,8 @@ static irqreturn_t prq_event_thread(int irq, void *d) · 871 unmodified lines
872 goto invalid;872 goto invalid;
873 873
874 ret = handle_mm_fault(vma, address,874 ret = handle_mm_fault(vma, address,
875 req->wr_req ? FAULT_FLAG_WRITE : 0);875 req->wr_req ? FAULT_FLAG_WRITE : 0,
876 NULL);
876 if (ret & VM_FAULT_ERROR)877 if (ret & VM_FAULT_ERROR)
877 goto invalid;878 goto invalid;
878 879
include/linux/mm.h+5−2
@@ -38,6 +38,7 @@ struct file_ra_state; · 37 unmodified lines
38struct user_struct;38struct user_struct;
39struct writeback_control;39struct writeback_control;
40struct bdi_writeback;40struct bdi_writeback;
41struct pt_regs;
41 42
42void init_mm_internals(void);43void init_mm_internals(void);
43 44
@@ -1658,7 +1659,8 @@ int invalidate_inode_page(struct page *page); · 1614 unmodified lines
1658 1659
1659#ifdef CONFIG_MMU1660#ifdef CONFIG_MMU
1660extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,1661extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
1661 unsigned long address, unsigned int flags);1662 unsigned long address, unsigned int flags,
1663 struct pt_regs *regs);
1662extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,1664extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
1663 unsigned long address, unsigned int fault_flags,1665 unsigned long address, unsigned int fault_flags,
1664 bool *unlocked);1666 bool *unlocked);
@@ -1668,7 +1670,8 @@ void unmap_mapping_range(struct address_space *mapping, · 3 unmodified lines
1668 loff_t const holebegin, loff_t const holelen, int even_cows);1670 loff_t const holebegin, loff_t const holelen, int even_cows);
1669#else1671#else
1670static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,1672static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
1671 unsigned long address, unsigned int flags)1673 unsigned long address, unsigned int flags,
1674 struct pt_regs *regs)
1672{1675{
1673 /* should never happen if there's no MMU */1676 /* should never happen if there's no MMU */
1674 BUG();1677 BUG();
mm/gup.c+2−2
@@ -884,7 +884,7 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, · 883 unmodified lines
884 fault_flags |= FAULT_FLAG_TRIED;884 fault_flags |= FAULT_FLAG_TRIED;
885 }885 }
886 886
887 ret = handle_mm_fault(vma, address, fault_flags);887 ret = handle_mm_fault(vma, address, fault_flags, NULL);
888 if (ret & VM_FAULT_ERROR) {888 if (ret & VM_FAULT_ERROR) {
889 int err = vm_fault_to_errno(ret, *flags);889 int err = vm_fault_to_errno(ret, *flags);
890 890
@@ -1238,7 +1238,7 @@ retry: · 347 unmodified lines
1238 fatal_signal_pending(current))1238 fatal_signal_pending(current))
1239 return -EINTR;1239 return -EINTR;
1240 1240
1241 ret = handle_mm_fault(vma, address, fault_flags);1241 ret = handle_mm_fault(vma, address, fault_flags, NULL);
1242 major |= ret & VM_FAULT_MAJOR;1242 major |= ret & VM_FAULT_MAJOR;
1243 if (ret & VM_FAULT_ERROR) {1243 if (ret & VM_FAULT_ERROR) {
1244 int err = vm_fault_to_errno(ret, 0);1244 int err = vm_fault_to_errno(ret, 0);
mm/hmm.c+2−1
@@ -75,7 +75,8 @@ static int hmm_vma_fault(unsigned long addr, unsigned long end, · 74 unmodified lines
75 }75 }
76 76
77 for (; addr < end; addr += PAGE_SIZE)77 for (; addr < end; addr += PAGE_SIZE)
78 if (handle_mm_fault(vma, addr, fault_flags) & VM_FAULT_ERROR)78 if (handle_mm_fault(vma, addr, fault_flags, NULL) &
79 VM_FAULT_ERROR)
79 return -EFAULT;80 return -EFAULT;
80 return -EBUSY;81 return -EBUSY;
81}82}
mm/ksm.c+2−1
@@ -480,7 +480,8 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) · 479 unmodified lines
480 break;480 break;
481 if (PageKsm(page))481 if (PageKsm(page))
482 ret = handle_mm_fault(vma, addr,482 ret = handle_mm_fault(vma, addr,
483 FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE);483 FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE,
484 NULL);
484 else485 else
485 ret = VM_FAULT_WRITE;486 ret = VM_FAULT_WRITE;
486 put_page(page);487 put_page(page);
mm/memory.c+63−1
@@ -71,6 +71,8 @@ · 70 unmodified lines
71#include <linux/dax.h>71#include <linux/dax.h>
72#include <linux/oom.h>72#include <linux/oom.h>
73#include <linux/numa.h>73#include <linux/numa.h>
74#include <linux/perf_event.h>
75#include <linux/ptrace.h>
74 76
75#include <trace/events/kmem.h>77#include <trace/events/kmem.h>
76 78
@@ -4356,6 +4358,64 @@ retry_pud: · 4279 unmodified lines
4356 return handle_pte_fault(&vmf);4358 return handle_pte_fault(&vmf);
4357}4359}
4358 4360
4361/**
4362 * mm_account_fault - Do page fault accountings
4363 *
4364 * @regs: the pt_regs struct pointer. When set to NULL, will skip accounting
4365 * of perf event counters, but we'll still do the per-task accounting to
4366 * the task who triggered this page fault.
4367 * @address: the faulted address.
4368 * @flags: the fault flags.
4369 * @ret: the fault retcode.
4370 *
4371 * This will take care of most of the page fault accountings. Meanwhile, it
4372 * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
4373 * updates. However note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
4374 * still be in per-arch page fault handlers at the entry of page fault.
4375 */
4376static inline void mm_account_fault(struct pt_regs *regs,
4377 unsigned long address, unsigned int flags,
4378 vm_fault_t ret)
4379{
4380 bool major;
4381
4382 /*
4383 * We don't do accounting for some specific faults:
4384 *
4385 * - Unsuccessful faults (e.g. when the address wasn't valid). That
4386 * includes arch_vma_access_permitted() failing before reaching here.
4387 * So this is not a "this many hardware page faults" counter. We
4388 * should use the hw profiling for that.
4389 *
4390 * - Incomplete faults (VM_FAULT_RETRY). They will only be counted
4391 * once they're completed.
4392 */
4393 if (ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))
4394 return;
4395
4396 /*
4397 * We define the fault as a major fault when the final successful fault
4398 * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
4399 * handle it immediately previously).
4400 */
4401 major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
4402
4403 /*
4404 * If the fault is done for GUP, regs will be NULL, and we will skip
4405 * the fault accounting.
4406 */
4407 if (!regs)
4408 return;
4409
4410 if (major) {
4411 current->maj_flt++;
4412 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
4413 } else {
4414 current->min_flt++;
4415 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
4416 }
4417}
4418
4359/*4419/*
4360 * By the time we get here, we already hold the mm semaphore4420 * By the time we get here, we already hold the mm semaphore
4361 *4421 *
@@ -4363,7 +4423,7 @@ retry_pud: · 1 unmodified line
4363 * return value. See filemap_fault() and __lock_page_or_retry().4423 * return value. See filemap_fault() and __lock_page_or_retry().
4364 */4424 */
4365vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,4425vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
4366 unsigned int flags)4426 unsigned int flags, struct pt_regs *regs)
4367{4427{
4368 vm_fault_t ret;4428 vm_fault_t ret;
4369 4429
@@ -4404,6 +4464,8 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, · 34 unmodified lines
4404 mem_cgroup_oom_synchronize(false);4464 mem_cgroup_oom_synchronize(false);
4405 }4465 }
4406 4466
4467 mm_account_fault(regs, address, flags, ret);
4468
4407 return ret;4469 return ret;
4408}4470}
4409EXPORT_SYMBOL_GPL(handle_mm_fault);4471EXPORT_SYMBOL_GPL(handle_mm_fault);