History

Merge branch 'akpm' (patches from Andrew)

Merge fourth patch-bomb from Andrew Morton: "A lot more stuff than expected, sorry. A bunch of ocfs2 reviewing was finished off. - mhocko's oom-reaper out-of-memory-handler changes - ocfs2 fixes and features - KASAN feature work - various fixes" * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (42 commits) thp: fix typo in khugepaged_scan_pmd() MAINTAINERS: fill entries for KASAN mm/filemap: generic_file_read_iter(): check for zero reads unconditionally kasan: test fix: warn if the UAF could not be detected in kmalloc_uaf2 mm, kasan: stackdepot implementation. Enable stackdepot for SLAB arch, ftrace: for KASAN put hard/soft IRQ entries into separate sections mm, kasan: add GFP flags to KASAN API mm, kasan: SLAB support kasan: modify kmalloc_large_oob_right(), add kmalloc_pagealloc_oob_right() include/linux/oom.h: remove undefined oom_kills_count()/note_oom_kill() mm/page_alloc: prevent merging between isolated and other pageblocks drivers/memstick/host/r592.c: avoid gcc-6 warning ocfs2: extend enough credits for freeing one truncate record while replaying truncate records ocfs2: extend transaction for ocfs2_remove_rightmost_path() and ocfs2_update_edge_lengths() before to avoid inconsistency between inode and et ocfs2/dlm: move lock to the tail of grant queue while doing in-place convert ocfs2: solve a problem of crossing the boundary in updating backups ocfs2: fix occurring deadlock by changing ocfs2_wq from global to local ocfs2/dlm: fix BUG in dlm_move_lockres_to_recovery_list ocfs2/dlm: fix race between convert and recovery ocfs2: fix a deadlock issue in ocfs2_dio_end_io_write() ...

Linus Torvalds committed 10 years ago

Commit 606c61a2 parents 15dbc13 0fda27879 files · +1770 −962Browse files

Unified · Split

79 files
  1. Documentation/kasan.txt +2 −3
  2. MAINTAINERS +14 −0
  3. arch/arm/include/asm/exception.h +1 −1
  4. arch/arm/kernel/vmlinux.lds.S +1 −0
  5. arch/arm64/include/asm/exception.h +1 −1
  6. arch/arm64/kernel/vmlinux.lds.S +1 −0
  7. arch/blackfin/kernel/vmlinux.lds.S +1 −0
  8. arch/c6x/kernel/vmlinux.lds.S +1 −0
  9. arch/metag/kernel/vmlinux.lds.S +1 −0
  10. arch/microblaze/kernel/vmlinux.lds.S +1 −0
  11. arch/mips/kernel/vmlinux.lds.S +1 −0
  12. arch/nios2/kernel/vmlinux.lds.S +1 −0
  13. arch/openrisc/kernel/vmlinux.lds.S +1 −0
  14. arch/parisc/kernel/vmlinux.lds.S +1 −0
  15. arch/powerpc/kernel/vmlinux.lds.S +1 −0
  16. arch/s390/kernel/vmlinux.lds.S +1 −0
  17. arch/sh/kernel/vmlinux.lds.S +1 −0
  18. arch/sparc/kernel/vmlinux.lds.S +1 −0
  19. arch/tile/kernel/vmlinux.lds.S +1 −0
  20. arch/x86/kernel/Makefile +1 −0
  21. arch/x86/kernel/vmlinux.lds.S +1 −0
  22. drivers/input/input-compat.c +3 −3
  23. drivers/input/input-compat.h +1 −3
  24. drivers/input/input.c +1 −1
  25. drivers/input/misc/uinput.c +2 −2
  26. drivers/memstick/host/r592.c +1 −2
  27. fs/ocfs2/alloc.c +64 −41
  28. fs/ocfs2/aops.c +561 −585
  29. fs/ocfs2/aops.h +8 −11
  30. fs/ocfs2/cluster/heartbeat.c +2 −2
  31. fs/ocfs2/dlm/dlmconvert.c +29 −1
  32. fs/ocfs2/dlm/dlmrecovery.c +0 −1
  33. fs/ocfs2/file.c +15 −150
  34. fs/ocfs2/inode.c +3 −0
  35. fs/ocfs2/inode.h +3 −3
  36. fs/ocfs2/journal.c +4 −4
  37. fs/ocfs2/localalloc.c +2 −2
  38. fs/ocfs2/mmap.c +2 −2
  39. fs/ocfs2/ocfs2.h +8 −0
  40. fs/ocfs2/ocfs2_trace.h +4 −12
  41. fs/ocfs2/quota_global.c +1 −1
  42. fs/ocfs2/resize.c +1 −1
  43. fs/ocfs2/super.c +16 −23
  44. fs/ocfs2/super.h +0 −2
  45. include/asm-generic/vmlinux.lds.h +11 −1
  46. include/linux/ftrace.h +0 −11
  47. include/linux/interrupt.h +20 −0
  48. include/linux/kasan.h +23 −8
  49. include/linux/mm.h +2 −0
  50. include/linux/oom.h +1 −3
  51. include/linux/sched.h +4 −0
  52. include/linux/slab.h +8 −2
  53. include/linux/slab_def.h +14 −0
  54. include/linux/slub_def.h +11 −0
  55. include/linux/stackdepot.h +32 −0
  56. kernel/exit.c +1 −1
  57. kernel/softirq.c +1 −1
  58. kernel/time/timer.c +11 −0
  59. kernel/trace/trace_functions_graph.c +1 −0
  60. lib/Kconfig +4 −0
  61. lib/Kconfig.kasan +4 −1
  62. lib/Makefile +3 −0
  63. lib/stackdepot.c +284 −0
  64. lib/test_kasan.c +29 −1
  65. mm/Makefile +1 −0
  66. mm/filemap.c +4 −3
  67. mm/huge_memory.c +1 −1
  68. mm/internal.h +5 −0
  69. mm/kasan/kasan.c +154 −8
  70. mm/kasan/kasan.h +37 −0
  71. mm/kasan/report.c +51 −11
  72. mm/memory.c +10 −7
  73. mm/mempool.c +8 −8
  74. mm/oom_kill.c +187 −9
  75. mm/page_alloc.c +33 −13
  76. mm/slab.c +37 −5
  77. mm/slab.h +1 −1
  78. mm/slab_common.c +3 −3
  79. mm/slub.c +8 −7
Documentation/kasan.txt+2−3
@@ -12,8 +12,7 @@ KASAN uses compile-time instrumentation for checking every memory access, · 11 unmodified lines
1212 therefore you will need a GCC version 4.9.2 or later. GCC 5.0 or later is
1313 required for detection of out-of-bounds accesses to stack or global variables.
1414
15−Currently KASAN is supported only for x86_64 architecture and requires the
16−kernel to be built with the SLUB allocator.
15+Currently KASAN is supported only for x86_64 architecture.
1716
1817 1. Usage
1918 ========
@@ -27,7 +26,7 @@ inline are compiler instrumentation types. The former produces smaller binary · 7 unmodified lines
2726 the latter is 1.1 - 2 times faster. Inline instrumentation requires a GCC
2827 version 5.0 or later.
2928
30−Currently KASAN works only with the SLUB memory allocator.
29+KASAN works with both SLUB and SLAB memory allocators.
3130 For better bug detection and nicer reporting, enable CONFIG_STACKTRACE.
3231
3332 To disable instrumentation for specific files or directories, add a line
MAINTAINERS+14−0
@@ -6165,6 +6165,20 @@ S: Maintained · 6164 unmodified lines
61656165 F: Documentation/hwmon/k8temp
61666166 F: drivers/hwmon/k8temp.c
61676167
6168+KASAN
6169+M: Andrey Ryabinin <aryabinin@virtuozzo.com>
6170+R: Alexander Potapenko <glider@google.com>
6171+R: Dmitry Vyukov <dvyukov@google.com>
6172+L: kasan-dev@googlegroups.com
6173+S: Maintained
6174+F: arch/*/include/asm/kasan.h
6175+F: arch/*/mm/kasan_init*
6176+F: Documentation/kasan.txt
6177+F: include/linux/kasan.h
6178+F: lib/test_kasan.c
6179+F: mm/kasan/
6180+F: scripts/Makefile.kasan
6181+
61686182 KCONFIG
61696183 M: "Yann E. MORIN" <yann.morin.1998@free.fr>
61706184 L: linux-kbuild@vger.kernel.org
arch/arm/include/asm/exception.h+1−1
@@ -7,7 +7,7 @@ · 6 unmodified lines
77 #ifndef __ASM_ARM_EXCEPTION_H
88 #define __ASM_ARM_EXCEPTION_H
99
10−#include <linux/ftrace.h>
10+#include <linux/interrupt.h>
1111
1212 #define __exception __attribute__((section(".exception.text")))
1313 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
arch/arm/kernel/vmlinux.lds.S+1−0
@@ -108,6 +108,7 @@ SECTIONS · 107 unmodified lines
108108 *(.exception.text)
109109 __exception_text_end = .;
110110 IRQENTRY_TEXT
111+ SOFTIRQENTRY_TEXT
111112 TEXT_TEXT
112113 SCHED_TEXT
113114 LOCK_TEXT
arch/arm64/include/asm/exception.h+1−1
@@ -18,7 +18,7 @@ · 17 unmodified lines
1818 #ifndef __ASM_EXCEPTION_H
1919 #define __ASM_EXCEPTION_H
2020
21−#include <linux/ftrace.h>
21+#include <linux/interrupt.h>
2222
2323 #define __exception __attribute__((section(".exception.text")))
2424 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
arch/arm64/kernel/vmlinux.lds.S+1−0
@@ -103,6 +103,7 @@ SECTIONS · 102 unmodified lines
103103 *(.exception.text)
104104 __exception_text_end = .;
105105 IRQENTRY_TEXT
106+ SOFTIRQENTRY_TEXT
106107 TEXT_TEXT
107108 SCHED_TEXT
108109 LOCK_TEXT
arch/blackfin/kernel/vmlinux.lds.S+1−0
@@ -35,6 +35,7 @@ SECTIONS · 34 unmodified lines
3535 #endif
3636 LOCK_TEXT
3737 IRQENTRY_TEXT
38+ SOFTIRQENTRY_TEXT
3839 KPROBES_TEXT
3940 #ifdef CONFIG_ROMKERNEL
4041 __sinittext = .;
arch/c6x/kernel/vmlinux.lds.S+1−0
@@ -72,6 +72,7 @@ SECTIONS · 71 unmodified lines
7272 SCHED_TEXT
7373 LOCK_TEXT
7474 IRQENTRY_TEXT
75+ SOFTIRQENTRY_TEXT
7576 KPROBES_TEXT
7677 *(.fixup)
7778 *(.gnu.warning)
arch/metag/kernel/vmlinux.lds.S+1−0
@@ -24,6 +24,7 @@ SECTIONS · 23 unmodified lines
2424 LOCK_TEXT
2525 KPROBES_TEXT
2626 IRQENTRY_TEXT
27+ SOFTIRQENTRY_TEXT
2728 *(.text.*)
2829 *(.gnu.warning)
2930 }
arch/microblaze/kernel/vmlinux.lds.S+1−0
@@ -36,6 +36,7 @@ SECTIONS { · 35 unmodified lines
3636 LOCK_TEXT
3737 KPROBES_TEXT
3838 IRQENTRY_TEXT
39+ SOFTIRQENTRY_TEXT
3940 . = ALIGN (4) ;
4041 _etext = . ;
4142 }
arch/mips/kernel/vmlinux.lds.S+1−0
@@ -58,6 +58,7 @@ SECTIONS · 57 unmodified lines
5858 LOCK_TEXT
5959 KPROBES_TEXT
6060 IRQENTRY_TEXT
61+ SOFTIRQENTRY_TEXT
6162 *(.text.*)
6263 *(.fixup)
6364 *(.gnu.warning)
arch/nios2/kernel/vmlinux.lds.S+1−0
@@ -39,6 +39,7 @@ SECTIONS · 38 unmodified lines
3939 SCHED_TEXT
4040 LOCK_TEXT
4141 IRQENTRY_TEXT
42+ SOFTIRQENTRY_TEXT
4243 KPROBES_TEXT
4344 } =0
4445 _etext = .;
arch/openrisc/kernel/vmlinux.lds.S+1−0
@@ -50,6 +50,7 @@ SECTIONS · 49 unmodified lines
5050 LOCK_TEXT
5151 KPROBES_TEXT
5252 IRQENTRY_TEXT
53+ SOFTIRQENTRY_TEXT
5354 *(.fixup)
5455 *(.text.__*)
5556 _etext = .;
arch/parisc/kernel/vmlinux.lds.S+1−0
@@ -72,6 +72,7 @@ SECTIONS · 71 unmodified lines
7272 LOCK_TEXT
7373 KPROBES_TEXT
7474 IRQENTRY_TEXT
75+ SOFTIRQENTRY_TEXT
7576 *(.text.do_softirq)
7677 *(.text.sys_exit)
7778 *(.text.do_sigaltstack)
arch/powerpc/kernel/vmlinux.lds.S+1−0
@@ -55,6 +55,7 @@ SECTIONS · 54 unmodified lines
5555 LOCK_TEXT
5656 KPROBES_TEXT
5757 IRQENTRY_TEXT
58+ SOFTIRQENTRY_TEXT
5859
5960 #ifdef CONFIG_PPC32
6061 *(.got1)
arch/s390/kernel/vmlinux.lds.S+1−0
@@ -28,6 +28,7 @@ SECTIONS · 27 unmodified lines
2828 LOCK_TEXT
2929 KPROBES_TEXT
3030 IRQENTRY_TEXT
31+ SOFTIRQENTRY_TEXT
3132 *(.fixup)
3233 *(.gnu.warning)
3334 } :text = 0x0700
arch/sh/kernel/vmlinux.lds.S+1−0
@@ -39,6 +39,7 @@ SECTIONS · 38 unmodified lines
3939 LOCK_TEXT
4040 KPROBES_TEXT
4141 IRQENTRY_TEXT
42+ SOFTIRQENTRY_TEXT
4243 *(.fixup)
4344 *(.gnu.warning)
4445 _etext = .; /* End of text section */
arch/sparc/kernel/vmlinux.lds.S+1−0
@@ -48,6 +48,7 @@ SECTIONS · 47 unmodified lines
4848 LOCK_TEXT
4949 KPROBES_TEXT
5050 IRQENTRY_TEXT
51+ SOFTIRQENTRY_TEXT
5152 *(.gnu.warning)
5253 } = 0
5354 _etext = .;
arch/tile/kernel/vmlinux.lds.S+1−0
@@ -45,6 +45,7 @@ SECTIONS · 44 unmodified lines
4545 LOCK_TEXT
4646 KPROBES_TEXT
4747 IRQENTRY_TEXT
48+ SOFTIRQENTRY_TEXT
4849 __fix_text_end = .; /* tile-cpack won't rearrange before this */
4950 ALIGN_FUNCTION();
5051 *(.hottext*)
arch/x86/kernel/Makefile+1−0
@@ -19,6 +19,7 @@ endif · 18 unmodified lines
1919 KASAN_SANITIZE_head$(BITS).o := n
2020 KASAN_SANITIZE_dumpstack.o := n
2121 KASAN_SANITIZE_dumpstack_$(BITS).o := n
22+KASAN_SANITIZE_stacktrace.o := n
2223
2324 OBJECT_FILES_NON_STANDARD_head_$(BITS).o := y
2425 OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y
arch/x86/kernel/vmlinux.lds.S+1−0
@@ -101,6 +101,7 @@ SECTIONS · 100 unmodified lines
101101 KPROBES_TEXT
102102 ENTRY_TEXT
103103 IRQENTRY_TEXT
104+ SOFTIRQENTRY_TEXT
104105 *(.fixup)
105106 *(.gnu.warning)
106107 /* End of text section */
drivers/input/input-compat.c+3−3
@@ -17,7 +17,7 @@ · 16 unmodified lines
1717 int input_event_from_user(const char __user *buffer,
1818 struct input_event *event)
1919 {
20− if (INPUT_COMPAT_TEST && !COMPAT_USE_64BIT_TIME) {
20+ if (in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
2121 struct input_event_compat compat_event;
2222
2323 if (copy_from_user(&compat_event, buffer,
@@ -41,7 +41,7 @@ int input_event_from_user(const char __user *buffer, · 17 unmodified lines
4141 int input_event_to_user(char __user *buffer,
4242 const struct input_event *event)
4343 {
44− if (INPUT_COMPAT_TEST && !COMPAT_USE_64BIT_TIME) {
44+ if (in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
4545 struct input_event_compat compat_event;
4646
4747 compat_event.time.tv_sec = event->time.tv_sec;
@@ -65,7 +65,7 @@ int input_event_to_user(char __user *buffer, · 17 unmodified lines
6565 int input_ff_effect_from_user(const char __user *buffer, size_t size,
6666 struct ff_effect *effect)
6767 {
68− if (INPUT_COMPAT_TEST) {
68+ if (in_compat_syscall()) {
6969 struct ff_effect_compat *compat_effect;
7070
7171 if (size != sizeof(struct ff_effect_compat))
drivers/input/input-compat.h+1−3
@@ -17,8 +17,6 @@ · 16 unmodified lines
1717
1818 #ifdef CONFIG_COMPAT
1919
20−#define INPUT_COMPAT_TEST in_compat_syscall()
21−
2220 struct input_event_compat {
2321 struct compat_timeval time;
2422 __u16 type;
@@ -57,7 +55,7 @@ struct ff_effect_compat { · 32 unmodified lines
5755
5856 static inline size_t input_event_size(void)
5957 {
60− return (INPUT_COMPAT_TEST && !COMPAT_USE_64BIT_TIME) ?
58+ return (in_compat_syscall() && !COMPAT_USE_64BIT_TIME) ?
6159 sizeof(struct input_event_compat) : sizeof(struct input_event);
6260 }
6361
drivers/input/input.c+1−1
@@ -1015,7 +1015,7 @@ static int input_bits_to_string(char *buf, int buf_size, · 1014 unmodified lines
10151015 {
10161016 int len = 0;
10171017
1018− if (INPUT_COMPAT_TEST) {
1018+ if (in_compat_syscall()) {
10191019 u32 dword = bits >> 32;
10201020 if (dword || !skip_empty)
10211021 len += snprintf(buf, buf_size, "%x ", dword);
drivers/input/misc/uinput.c+2−2
@@ -664,7 +664,7 @@ struct uinput_ff_upload_compat { · 663 unmodified lines
664664 static int uinput_ff_upload_to_user(char __user *buffer,
665665 const struct uinput_ff_upload *ff_up)
666666 {
667− if (INPUT_COMPAT_TEST) {
667+ if (in_compat_syscall()) {
668668 struct uinput_ff_upload_compat ff_up_compat;
669669
670670 ff_up_compat.request_id = ff_up->request_id;
@@ -695,7 +695,7 @@ static int uinput_ff_upload_to_user(char __user *buffer, · 24 unmodified lines
695695 static int uinput_ff_upload_from_user(const char __user *buffer,
696696 struct uinput_ff_upload *ff_up)
697697 {
698− if (INPUT_COMPAT_TEST) {
698+ if (in_compat_syscall()) {
699699 struct uinput_ff_upload_compat ff_up_compat;
700700
701701 if (copy_from_user(&ff_up_compat, buffer,
drivers/memstick/host/r592.c+1−2
@@ -298,8 +298,7 @@ static int r592_transfer_fifo_dma(struct r592_device *dev) · 297 unmodified lines
298298 sg_count = dma_map_sg(&dev->pci_dev->dev, &dev->req->sg, 1, is_write ?
299299 PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE);
300300
301− if (sg_count != 1 ||
302− (sg_dma_len(&dev->req->sg) < dev->req->sg.length)) {
301+ if (sg_count != 1 || sg_dma_len(&dev->req->sg) < R592_LFIFO_SIZE) {
303302 message("problem in dma_map_sg");
304303 return -EIO;
305304 }
fs/ocfs2/alloc.c+64−41
@@ -2516,21 +2516,6 @@ static int ocfs2_update_edge_lengths(handle_t *handle, · 2515 unmodified lines
25162516 struct ocfs2_extent_block *eb;
25172517 u32 range;
25182518
2519− /*
2520− * In normal tree rotation process, we will never touch the
2521− * tree branch above subtree_index and ocfs2_extend_rotate_transaction
2522− * doesn't reserve the credits for them either.
2523− *
2524− * But we do have a special case here which will update the rightmost
2525− * records for all the bh in the path.
2526− * So we have to allocate extra credits and access them.
2527− */
2528− ret = ocfs2_extend_trans(handle, subtree_index);
2529− if (ret) {
2530− mlog_errno(ret);
2531− goto out;
2532− }
2533−
25342519 ret = ocfs2_journal_access_path(et->et_ci, handle, path);
25352520 if (ret) {
25362521 mlog_errno(ret);
@@ -2956,7 +2941,7 @@ static int __ocfs2_rotate_tree_left(handle_t *handle, · 419 unmodified lines
29562941 right_path->p_node[subtree_root].bh->b_blocknr,
29572942 right_path->p_tree_depth);
29582943
2959− ret = ocfs2_extend_rotate_transaction(handle, subtree_root,
2944+ ret = ocfs2_extend_rotate_transaction(handle, 0,
29602945 orig_credits, left_path);
29612946 if (ret) {
29622947 mlog_errno(ret);
@@ -3029,21 +3014,9 @@ static int ocfs2_remove_rightmost_path(handle_t *handle, · 66 unmodified lines
30293014 struct ocfs2_extent_block *eb;
30303015 struct ocfs2_extent_list *el;
30313016
3032−
30333017 ret = ocfs2_et_sanity_check(et);
30343018 if (ret)
30353019 goto out;
3036− /*
3037− * There's two ways we handle this depending on
3038− * whether path is the only existing one.
3039− */
3040− ret = ocfs2_extend_rotate_transaction(handle, 0,
3041− handle->h_buffer_credits,
3042− path);
3043− if (ret) {
3044− mlog_errno(ret);
3045− goto out;
3046− }
30473020
30483021 ret = ocfs2_journal_access_path(et->et_ci, handle, path);
30493022 if (ret) {
@@ -3641,6 +3614,14 @@ static int ocfs2_merge_rec_left(struct ocfs2_path *right_path, · 591 unmodified lines
36413614 */
36423615 if (le16_to_cpu(right_rec->e_leaf_clusters) == 0 &&
36433616 le16_to_cpu(el->l_next_free_rec) == 1) {
3617+ /* extend credit for ocfs2_remove_rightmost_path */
3618+ ret = ocfs2_extend_rotate_transaction(handle, 0,
3619+ handle->h_buffer_credits,
3620+ right_path);
3621+ if (ret) {
3622+ mlog_errno(ret);
3623+ goto out;
3624+ }
36443625
36453626 ret = ocfs2_remove_rightmost_path(handle, et,
36463627 right_path,
@@ -3679,6 +3660,14 @@ static int ocfs2_try_to_merge_extent(handle_t *handle, · 32 unmodified lines
36793660 BUG_ON(ctxt->c_contig_type == CONTIG_NONE);
36803661
36813662 if (ctxt->c_split_covers_rec && ctxt->c_has_empty_extent) {
3663+ /* extend credit for ocfs2_remove_rightmost_path */
3664+ ret = ocfs2_extend_rotate_transaction(handle, 0,
3665+ handle->h_buffer_credits,
3666+ path);
3667+ if (ret) {
3668+ mlog_errno(ret);
3669+ goto out;
3670+ }
36823671 /*
36833672 * The merge code will need to create an empty
36843673 * extent to take the place of the newly
@@ -3727,6 +3716,15 @@ static int ocfs2_try_to_merge_extent(handle_t *handle, · 42 unmodified lines
37273716 */
37283717 BUG_ON(!ocfs2_is_empty_extent(&el->l_recs[0]));
37293718
3719+ /* extend credit for ocfs2_remove_rightmost_path */
3720+ ret = ocfs2_extend_rotate_transaction(handle, 0,
3721+ handle->h_buffer_credits,
3722+ path);
3723+ if (ret) {
3724+ mlog_errno(ret);
3725+ goto out;
3726+ }
3727+
37303728 /* The merge left us with an empty extent, remove it. */
37313729 ret = ocfs2_rotate_tree_left(handle, et, path, dealloc);
37323730 if (ret) {
@@ -3748,6 +3746,15 @@ static int ocfs2_try_to_merge_extent(handle_t *handle, · 15 unmodified lines
37483746 goto out;
37493747 }
37503748
3749+ /* extend credit for ocfs2_remove_rightmost_path */
3750+ ret = ocfs2_extend_rotate_transaction(handle, 0,
3751+ handle->h_buffer_credits,
3752+ path);
3753+ if (ret) {
3754+ mlog_errno(ret);
3755+ goto out;
3756+ }
3757+
37513758 ret = ocfs2_rotate_tree_left(handle, et, path, dealloc);
37523759 /*
37533760 * Error from this last rotate is not critical, so
@@ -3783,6 +3790,16 @@ static int ocfs2_try_to_merge_extent(handle_t *handle, · 29 unmodified lines
37833790 }
37843791
37853792 if (ctxt->c_split_covers_rec) {
3793+ /* extend credit for ocfs2_remove_rightmost_path */
3794+ ret = ocfs2_extend_rotate_transaction(handle, 0,
3795+ handle->h_buffer_credits,
3796+ path);
3797+ if (ret) {
3798+ mlog_errno(ret);
3799+ ret = 0;
3800+ goto out;
3801+ }
3802+
37863803 /*
37873804 * The merge may have left an empty extent in
37883805 * our leaf. Try to rotate it away.
@@ -5342,6 +5359,15 @@ static int ocfs2_truncate_rec(handle_t *handle, · 1553 unmodified lines
53425359 struct ocfs2_extent_block *eb;
53435360
53445361 if (ocfs2_is_empty_extent(&el->l_recs[0]) && index > 0) {
5362+ /* extend credit for ocfs2_remove_rightmost_path */
5363+ ret = ocfs2_extend_rotate_transaction(handle, 0,
5364+ handle->h_buffer_credits,
5365+ path);
5366+ if (ret) {
5367+ mlog_errno(ret);
5368+ goto out;
5369+ }
5370+
53455371 ret = ocfs2_rotate_tree_left(handle, et, path, dealloc);
53465372 if (ret) {
53475373 mlog_errno(ret);
@@ -5928,16 +5954,6 @@ static int ocfs2_replay_truncate_records(struct ocfs2_super *osb, · 580 unmodified lines
59285954
59295955 ocfs2_journal_dirty(handle, tl_bh);
59305956
5931− /* TODO: Perhaps we can calculate the bulk of the
5932− * credits up front rather than extending like
5933− * this. */
5934− status = ocfs2_extend_trans(handle,
5935− OCFS2_TRUNCATE_LOG_FLUSH_ONE_REC);
5936− if (status < 0) {
5937− mlog_errno(status);
5938− goto bail;
5939− }
5940−
59415957 rec = tl->tl_recs[i];
59425958 start_blk = ocfs2_clusters_to_blocks(data_alloc_inode->i_sb,
59435959 le32_to_cpu(rec.t_start));
@@ -5958,6 +5974,13 @@ static int ocfs2_replay_truncate_records(struct ocfs2_super *osb, · 14 unmodified lines
59585974 goto bail;
59595975 }
59605976 }
5977+
5978+ status = ocfs2_extend_trans(handle,
5979+ OCFS2_TRUNCATE_LOG_FLUSH_ONE_REC);
5980+ if (status < 0) {
5981+ mlog_errno(status);
5982+ goto bail;
5983+ }
59615984 i--;
59625985 }
59635986
@@ -6016,7 +6039,7 @@ int __ocfs2_flush_truncate_log(struct ocfs2_super *osb) · 52 unmodified lines
60166039 goto out_mutex;
60176040 }
60186041
6019− handle = ocfs2_start_trans(osb, OCFS2_TRUNCATE_LOG_UPDATE);
6042+ handle = ocfs2_start_trans(osb, OCFS2_TRUNCATE_LOG_FLUSH_ONE_REC);
60206043 if (IS_ERR(handle)) {
60216044 status = PTR_ERR(handle);
60226045 mlog_errno(status);
@@ -6079,7 +6102,7 @@ void ocfs2_schedule_truncate_log_flush(struct ocfs2_super *osb, · 56 unmodified lines
60796102 if (cancel)
60806103 cancel_delayed_work(&osb->osb_truncate_log_wq);
60816104
6082− queue_delayed_work(ocfs2_wq, &osb->osb_truncate_log_wq,
6105+ queue_delayed_work(osb->ocfs2_wq, &osb->osb_truncate_log_wq,
60836106 OCFS2_TRUNCATE_LOG_FLUSH_INTERVAL);
60846107 }
60856108 }
@@ -6253,7 +6276,7 @@ void ocfs2_truncate_log_shutdown(struct ocfs2_super *osb) · 167 unmodified lines
62536276
62546277 if (tl_inode) {
62556278 cancel_delayed_work(&osb->osb_truncate_log_wq);
6256− flush_workqueue(ocfs2_wq);
6279+ flush_workqueue(osb->ocfs2_wq);
62576280
62586281 status = ocfs2_flush_truncate_log(osb);
62596282 if (status < 0)
fs/ocfs2/aops.c+561−585
@@ -499,158 +499,6 @@ bail: · 498 unmodified lines
499499 return status;
500500 }
501501
502−/*
503− * TODO: Make this into a generic get_blocks function.
504− *
505− * From do_direct_io in direct-io.c:
506− * "So what we do is to permit the ->get_blocks function to populate
507− * bh.b_size with the size of IO which is permitted at this offset and
508− * this i_blkbits."
509− *
510− * This function is called directly from get_more_blocks in direct-io.c.
511− *
512− * called like this: dio->get_blocks(dio->inode, fs_startblk,
513− * fs_count, map_bh, dio->rw == WRITE);
514− */
515−static int ocfs2_direct_IO_get_blocks(struct inode *inode, sector_t iblock,
516− struct buffer_head *bh_result, int create)
517−{
518− int ret;
519− u32 cpos = 0;
520− int alloc_locked = 0;
521− u64 p_blkno, inode_blocks, contig_blocks;
522− unsigned int ext_flags;
523− unsigned char blocksize_bits = inode->i_sb->s_blocksize_bits;
524− unsigned long max_blocks = bh_result->b_size >> inode->i_blkbits;
525− unsigned long len = bh_result->b_size;
526− unsigned int clusters_to_alloc = 0, contig_clusters = 0;
527−
528− cpos = ocfs2_blocks_to_clusters(inode->i_sb, iblock);
529−
530− /* This function won't even be called if the request isn't all
531− * nicely aligned and of the right size, so there's no need
532− * for us to check any of that. */
533−
534− inode_blocks = ocfs2_blocks_for_bytes(inode->i_sb, i_size_read(inode));
535−
536− down_read(&OCFS2_I(inode)->ip_alloc_sem);
537−
538− /* This figures out the size of the next contiguous block, and
539− * our logical offset */
540− ret = ocfs2_extent_map_get_blocks(inode, iblock, &p_blkno,
541− &contig_blocks, &ext_flags);
542− up_read(&OCFS2_I(inode)->ip_alloc_sem);
543−
544− if (ret) {
545− mlog(ML_ERROR, "get_blocks() failed iblock=%llu\n",
546− (unsigned long long)iblock);
547− ret = -EIO;
548− goto bail;
549− }
550−
551− /* We should already CoW the refcounted extent in case of create. */
552− BUG_ON(create && (ext_flags & OCFS2_EXT_REFCOUNTED));
553−
554− /* allocate blocks if no p_blkno is found, and create == 1 */
555− if (!p_blkno && create) {
556− ret = ocfs2_inode_lock(inode, NULL, 1);
557− if (ret < 0) {
558− mlog_errno(ret);
559− goto bail;
560− }
561−
562− alloc_locked = 1;
563−
564− down_write(&OCFS2_I(inode)->ip_alloc_sem);
565−
566− /* fill hole, allocate blocks can't be larger than the size
567− * of the hole */
568− clusters_to_alloc = ocfs2_clusters_for_bytes(inode->i_sb, len);
569− contig_clusters = ocfs2_clusters_for_blocks(inode->i_sb,
570− contig_blocks);
571− if (clusters_to_alloc > contig_clusters)
572− clusters_to_alloc = contig_clusters;
573−
574− /* allocate extent and insert them into the extent tree */
575− ret = ocfs2_extend_allocation(inode, cpos,
576− clusters_to_alloc, 0);
577− if (ret < 0) {
578− up_write(&OCFS2_I(inode)->ip_alloc_sem);
579− mlog_errno(ret);
580− goto bail;
581− }
582−
583− ret = ocfs2_extent_map_get_blocks(inode, iblock, &p_blkno,
584− &contig_blocks, &ext_flags);
585− if (ret < 0) {
586− up_write(&OCFS2_I(inode)->ip_alloc_sem);
587− mlog(ML_ERROR, "get_blocks() failed iblock=%llu\n",
588− (unsigned long long)iblock);
589− ret = -EIO;
590− goto bail;
591− }
592− set_buffer_new(bh_result);
593− up_write(&OCFS2_I(inode)->ip_alloc_sem);
594− }
595−
596− /*
597− * get_more_blocks() expects us to describe a hole by clearing
598− * the mapped bit on bh_result().
599− *
600− * Consider an unwritten extent as a hole.
601− */
602− if (p_blkno && !(ext_flags & OCFS2_EXT_UNWRITTEN))
603− map_bh(bh_result, inode->i_sb, p_blkno);
604− else
605− clear_buffer_mapped(bh_result);
606−
607− /* make sure we don't map more than max_blocks blocks here as
608− that's all the kernel will handle at this point. */
609− if (max_blocks < contig_blocks)
610− contig_blocks = max_blocks;
611− bh_result->b_size = contig_blocks << blocksize_bits;
612−bail:
613− if (alloc_locked)
614− ocfs2_inode_unlock(inode, 1);
615− return ret;
616−}
617−
618−/*
619− * ocfs2_dio_end_io is called by the dio core when a dio is finished. We're
620− * particularly interested in the aio/dio case. We use the rw_lock DLM lock
621− * to protect io on one node from truncation on another.
622− */
623−static int ocfs2_dio_end_io(struct kiocb *iocb,
624− loff_t offset,
625− ssize_t bytes,
626− void *private)
627−{
628− struct inode *inode = file_inode(iocb->ki_filp);
629− int level;
630−
631− if (bytes <= 0)
632− return 0;
633−
634− /* this io's submitter should not have unlocked this before we could */
635− BUG_ON(!ocfs2_iocb_is_rw_locked(iocb));
636−
637− if (ocfs2_iocb_is_unaligned_aio(iocb)) {
638− ocfs2_iocb_clear_unaligned_aio(iocb);
639−
640− mutex_unlock(&OCFS2_I(inode)->ip_unaligned_aio);
641− }
642−
643− /* Let rw unlock to be done later to protect append direct io write */
644− if (offset + bytes <= i_size_read(inode)) {
645− ocfs2_iocb_clear_rw_locked(iocb);
646−
647− level = ocfs2_iocb_rw_locked_level(iocb);
648− ocfs2_rw_unlock(inode, level);
649− }
650−
651− return 0;
652−}
653−
654502 static int ocfs2_releasepage(struct page *page, gfp_t wait)
655503 {
656504 if (!page_has_buffers(page))
@@ -658,363 +506,6 @@ static int ocfs2_releasepage(struct page *page, gfp_t wait) · 1 unmodified line
658506 return try_to_free_buffers(page);
659507 }
660508
661−static int ocfs2_is_overwrite(struct ocfs2_super *osb,
662− struct inode *inode, loff_t offset)
663−{
664− int ret = 0;
665− u32 v_cpos = 0;
666− u32 p_cpos = 0;
667− unsigned int num_clusters = 0;
668− unsigned int ext_flags = 0;
669−
670− v_cpos = ocfs2_bytes_to_clusters(osb->sb, offset);
671− ret = ocfs2_get_clusters(inode, v_cpos, &p_cpos,
672− &num_clusters, &ext_flags);
673− if (ret < 0) {
674− mlog_errno(ret);
675− return ret;
676− }
677−
678− if (p_cpos && !(ext_flags & OCFS2_EXT_UNWRITTEN))
679− return 1;
680−
681− return 0;
682−}
683−
684−static int ocfs2_direct_IO_zero_extend(struct ocfs2_super *osb,
685− struct inode *inode, loff_t offset,
686− u64 zero_len, int cluster_align)
687−{
688− u32 p_cpos = 0;
689− u32 v_cpos = ocfs2_bytes_to_clusters(osb->sb, i_size_read(inode));
690− unsigned int num_clusters = 0;
691− unsigned int ext_flags = 0;
692− int ret = 0;
693−
694− if (offset <= i_size_read(inode) || cluster_align)
695− return 0;
696−
697− ret = ocfs2_get_clusters(inode, v_cpos, &p_cpos, &num_clusters,
698− &ext_flags);
699− if (ret < 0) {
700− mlog_errno(ret);
701− return ret;
702− }
703−
704− if (p_cpos && !(ext_flags & OCFS2_EXT_UNWRITTEN)) {
705− u64 s = i_size_read(inode);
706− sector_t sector = ((u64)p_cpos << (osb->s_clustersize_bits - 9)) +
707− (do_div(s, osb->s_clustersize) >> 9);
708−
709− ret = blkdev_issue_zeroout(osb->sb->s_bdev, sector,
710− zero_len >> 9, GFP_NOFS, false);
711− if (ret < 0)
712− mlog_errno(ret);
713− }
714−
715− return ret;
716−}
717−
718−static int ocfs2_direct_IO_extend_no_holes(struct ocfs2_super *osb,
719− struct inode *inode, loff_t offset)
720−{
721− u64 zero_start, zero_len, total_zero_len;
722− u32 p_cpos = 0, clusters_to_add;
723− u32 v_cpos = ocfs2_bytes_to_clusters(osb->sb, i_size_read(inode));
724− unsigned int num_clusters = 0;
725− unsigned int ext_flags = 0;
726− u32 size_div, offset_div;
727− int ret = 0;
728−
729− {
730− u64 o = offset;
731− u64 s = i_size_read(inode);
732−
733− offset_div = do_div(o, osb->s_clustersize);
734− size_div = do_div(s, osb->s_clustersize);
735− }
736−
737− if (offset <= i_size_read(inode))
738− return 0;
739−
740− clusters_to_add = ocfs2_bytes_to_clusters(inode->i_sb, offset) -
741− ocfs2_bytes_to_clusters(inode->i_sb, i_size_read(inode));
742− total_zero_len = offset - i_size_read(inode);
743− if (clusters_to_add)
744− total_zero_len -= offset_div;
745−
746− /* Allocate clusters to fill out holes, and this is only needed
747− * when we add more than one clusters. Otherwise the cluster will
748− * be allocated during direct IO */
749− if (clusters_to_add > 1) {
750− ret = ocfs2_extend_allocation(inode,
751− OCFS2_I(inode)->ip_clusters,
752− clusters_to_add - 1, 0);
753− if (ret) {
754− mlog_errno(ret);
755− goto out;
756− }
757− }
758−
759− while (total_zero_len) {
760− ret = ocfs2_get_clusters(inode, v_cpos, &p_cpos, &num_clusters,
761− &ext_flags);
762− if (ret < 0) {
763− mlog_errno(ret);
764− goto out;
765− }
766−
767− zero_start = ocfs2_clusters_to_bytes(osb->sb, p_cpos) +
768− size_div;
769− zero_len = ocfs2_clusters_to_bytes(osb->sb, num_clusters) -
770− size_div;
771− zero_len = min(total_zero_len, zero_len);
772−
773− if (p_cpos && !(ext_flags & OCFS2_EXT_UNWRITTEN)) {
774− ret = blkdev_issue_zeroout(osb->sb->s_bdev,
775− zero_start >> 9, zero_len >> 9,
776− GFP_NOFS, false);
777− if (ret < 0) {
778− mlog_errno(ret);
779− goto out;
780− }
781− }
782−
783− total_zero_len -= zero_len;
784− v_cpos += ocfs2_bytes_to_clusters(osb->sb, zero_len + size_div);
785−
786− /* Only at first iteration can be cluster not aligned.
787− * So set size_div to 0 for the rest */
788− size_div = 0;
789− }
790−
791−out:
792− return ret;
793−}
794−
795−static ssize_t ocfs2_direct_IO_write(struct kiocb *iocb,
796− struct iov_iter *iter,
797− loff_t offset)
798−{
799− ssize_t ret = 0;
800− ssize_t written = 0;
801− bool orphaned = false;
802− int is_overwrite = 0;
803− struct file *file = iocb->ki_filp;
804− struct inode *inode = file_inode(file)->i_mapping->host;
805− struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
806− struct buffer_head *di_bh = NULL;
807− size_t count = iter->count;
808− journal_t *journal = osb->journal->j_journal;
809− u64 zero_len_head, zero_len_tail;
810− int cluster_align_head, cluster_align_tail;
811− loff_t final_size = offset + count;
812− int append_write = offset >= i_size_read(inode) ? 1 : 0;
813− unsigned int num_clusters = 0;
814− unsigned int ext_flags = 0;
815−
816− {
817− u64 o = offset;
818− u64 s = i_size_read(inode);
819−
820− zero_len_head = do_div(o, 1 << osb->s_clustersize_bits);
821− cluster_align_head = !zero_len_head;
822−
823− zero_len_tail = osb->s_clustersize -
824− do_div(s, osb->s_clustersize);
825− if ((offset - i_size_read(inode)) < zero_len_tail)
826− zero_len_tail = offset - i_size_read(inode);
827− cluster_align_tail = !zero_len_tail;
828− }
829−
830− /*
831− * when final_size > inode->i_size, inode->i_size will be
832− * updated after direct write, so add the inode to orphan
833− * dir first.
834− */
835− if (final_size > i_size_read(inode)) {
836− ret = ocfs2_add_inode_to_orphan(osb, inode);
837− if (ret < 0) {
838− mlog_errno(ret);
839− goto out;
840− }
841− orphaned = true;
842− }
843−
844− if (append_write) {
845− ret = ocfs2_inode_lock(inode, NULL, 1);
846− if (ret < 0) {
847− mlog_errno(ret);
848− goto clean_orphan;
849− }
850−
851− /* zeroing out the previously allocated cluster tail
852− * that but not zeroed */
853− if (ocfs2_sparse_alloc(OCFS2_SB(inode->i_sb))) {
854− down_read(&OCFS2_I(inode)->ip_alloc_sem);
855− ret = ocfs2_direct_IO_zero_extend(osb, inode, offset,
856− zero_len_tail, cluster_align_tail);
857− up_read(&OCFS2_I(inode)->ip_alloc_sem);
858− } else {
859− down_write(&OCFS2_I(inode)->ip_alloc_sem);
860− ret = ocfs2_direct_IO_extend_no_holes(osb, inode,
861− offset);
862− up_write(&OCFS2_I(inode)->ip_alloc_sem);
863− }
864− if (ret < 0) {
865− mlog_errno(ret);
866− ocfs2_inode_unlock(inode, 1);
867− goto clean_orphan;
868− }
869−
870− is_overwrite = ocfs2_is_overwrite(osb, inode, offset);
871− if (is_overwrite < 0) {
872− mlog_errno(is_overwrite);
873− ret = is_overwrite;
874− ocfs2_inode_unlock(inode, 1);
875− goto clean_orphan;
876− }
877−
878− ocfs2_inode_unlock(inode, 1);
879− }
880−
881− written = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter,
882− offset, ocfs2_direct_IO_get_blocks,
883− ocfs2_dio_end_io, NULL, 0);
884− /* overwrite aio may return -EIOCBQUEUED, and it is not an error */
885− if ((written < 0) && (written != -EIOCBQUEUED)) {
886− loff_t i_size = i_size_read(inode);
887−
888− if (offset + count > i_size) {
889− ret = ocfs2_inode_lock(inode, &di_bh, 1);
890− if (ret < 0) {
891− mlog_errno(ret);
892− goto clean_orphan;
893− }
894−
895− if (i_size == i_size_read(inode)) {
896− ret = ocfs2_truncate_file(inode, di_bh,
897− i_size);
898− if (ret < 0) {
899− if (ret != -ENOSPC)
900− mlog_errno(ret);
901−
902− ocfs2_inode_unlock(inode, 1);
903− brelse(di_bh);
904− di_bh = NULL;
905− goto clean_orphan;
906− }
907− }
908−
909− ocfs2_inode_unlock(inode, 1);
910− brelse(di_bh);
911− di_bh = NULL;
912−
913− ret = jbd2_journal_force_commit(journal);
914− if (ret < 0)
915− mlog_errno(ret);
916− }
917− } else if (written > 0 && append_write && !is_overwrite &&
918− !cluster_align_head) {
919− /* zeroing out the allocated cluster head */
920− u32 p_cpos = 0;
921− u32 v_cpos = ocfs2_bytes_to_clusters(osb->sb, offset);
922−
923− ret = ocfs2_inode_lock(inode, NULL, 0);
924− if (ret < 0) {
925− mlog_errno(ret);
926− goto clean_orphan;
927− }
928−
929− ret = ocfs2_get_clusters(inode, v_cpos, &p_cpos,
930− &num_clusters, &ext_flags);
931− if (ret < 0) {
932− mlog_errno(ret);
933− ocfs2_inode_unlock(inode, 0);
934− goto clean_orphan;
935− }
936−
937− BUG_ON(!p_cpos || (ext_flags & OCFS2_EXT_UNWRITTEN));
938−
939− ret = blkdev_issue_zeroout(osb->sb->s_bdev,
940− (u64)p_cpos << (osb->s_clustersize_bits - 9),
941− zero_len_head >> 9, GFP_NOFS, false);
942− if (ret < 0)
943− mlog_errno(ret);
944−
945− ocfs2_inode_unlock(inode, 0);
946− }
947−
948−clean_orphan:
949− if (orphaned) {
950− int tmp_ret;
951− int update_isize = written > 0 ? 1 : 0;
952− loff_t end = update_isize ? offset + written : 0;
953−
954− tmp_ret = ocfs2_inode_lock(inode, &di_bh, 1);
955− if (tmp_ret < 0) {
956− ret = tmp_ret;
957− mlog_errno(ret);
958− goto out;
959− }
960−
961− tmp_ret = ocfs2_del_inode_from_orphan(osb, inode, di_bh,
962− update_isize, end);
963− if (tmp_ret < 0) {
964− ocfs2_inode_unlock(inode, 1);
965− ret = tmp_ret;
966− mlog_errno(ret);
967− brelse(di_bh);
968− goto out;
969− }
970−
971− ocfs2_inode_unlock(inode, 1);
972− brelse(di_bh);
973−
974− tmp_ret = jbd2_journal_force_commit(journal);
975− if (tmp_ret < 0) {
976− ret = tmp_ret;
977− mlog_errno(tmp_ret);
978− }
979− }
980−
981−out:
982− if (ret >= 0)
983− ret = written;
984− return ret;
985−}
986−
987−static ssize_t ocfs2_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
988− loff_t offset)
989−{
990− struct file *file = iocb->ki_filp;
991− struct inode *inode = file_inode(file)->i_mapping->host;
992− struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
993− int full_coherency = !(osb->s_mount_opt &
994− OCFS2_MOUNT_COHERENCY_BUFFERED);
995−
996− /*
997− * Fallback to buffered I/O if we see an inode without
998− * extents.
999− */
1000− if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)
1001− return 0;
1002−
1003− /* Fallback to buffered I/O if we are appending and
1004− * concurrent O_DIRECT writes are allowed.
1005− */
1006− if (i_size_read(inode) <= offset && !full_coherency)
1007− return 0;
1008−
1009− if (iov_iter_rw(iter) == READ)
1010− return __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
1011− iter, offset,
1012− ocfs2_direct_IO_get_blocks,
1013− ocfs2_dio_end_io, NULL, 0);
1014− else
1015− return ocfs2_direct_IO_write(iocb, iter, offset);
1016−}
1017−
1018509 static void ocfs2_figure_cluster_boundaries(struct ocfs2_super *osb,
1019510 u32 cpos,
1020511 unsigned int *start,
@@ -1201,6 +692,13 @@ next_bh: · 180 unmodified lines
1201692
1202693 #define OCFS2_MAX_CLUSTERS_PER_PAGE (PAGE_CACHE_SIZE / OCFS2_MIN_CLUSTERSIZE)
1203694
695+struct ocfs2_unwritten_extent {
696+ struct list_head ue_node;
697+ struct list_head ue_ip_node;
698+ u32 ue_cpos;
699+ u32 ue_phys;
700+};
701+
1204702 /*
1205703 * Describe the state of a single cluster to be written to.
1206704 */
@@ -1212,7 +710,7 @@ struct ocfs2_write_cluster_desc { · 5 unmodified lines
1212710 * filled.
1213711 */
1214712 unsigned c_new;
1215− unsigned c_unwritten;
713+ unsigned c_clear_unwritten;
1216714 unsigned c_needs_zero;
1217715 };
1218716
@@ -1224,6 +722,9 @@ struct ocfs2_write_ctxt { · 5 unmodified lines
1224722 /* First cluster allocated in a nonsparse extend */
1225723 u32 w_first_new_cpos;
1226724
725+ /* Type of caller. Must be one of buffer, mmap, direct. */
726+ ocfs2_write_type_t w_type;
727+
1227728 struct ocfs2_write_cluster_desc w_desc[OCFS2_MAX_CLUSTERS_PER_PAGE];
1228729
1229730 /*
@@ -1272,6 +773,8 @@ struct ocfs2_write_ctxt { · 42 unmodified lines
1272773 struct buffer_head *w_di_bh;
1273774
1274775 struct ocfs2_cached_dealloc_ctxt w_dealloc;
776+
777+ struct list_head w_unwritten_list;
1275778 };
1276779
1277780 void ocfs2_unlock_and_free_pages(struct page **pages, int num_pages)
@@ -1310,8 +813,25 @@ static void ocfs2_unlock_pages(struct ocfs2_write_ctxt *wc) · 32 unmodified lines
1310813 ocfs2_unlock_and_free_pages(wc->w_pages, wc->w_num_pages);
1311814 }
1312815
1313−static void ocfs2_free_write_ctxt(struct ocfs2_write_ctxt *wc)
816+static void ocfs2_free_unwritten_list(struct inode *inode,
817+ struct list_head *head)
818+{
819+ struct ocfs2_inode_info *oi = OCFS2_I(inode);
820+ struct ocfs2_unwritten_extent *ue = NULL, *tmp = NULL;
821+
822+ list_for_each_entry_safe(ue, tmp, head, ue_node) {
823+ list_del(&ue->ue_node);
824+ spin_lock(&oi->ip_lock);
825+ list_del(&ue->ue_ip_node);
826+ spin_unlock(&oi->ip_lock);
827+ kfree(ue);
828+ }
829+}
830+
831+static void ocfs2_free_write_ctxt(struct inode *inode,
832+ struct ocfs2_write_ctxt *wc)
1314833 {
834+ ocfs2_free_unwritten_list(inode, &wc->w_unwritten_list);
1315835 ocfs2_unlock_pages(wc);
1316836 brelse(wc->w_di_bh);
1317837 kfree(wc);
@@ -1319,7 +839,8 @@ static void ocfs2_free_write_ctxt(struct ocfs2_write_ctxt *wc) · 1 unmodified line
1319839
1320840 static int ocfs2_alloc_write_ctxt(struct ocfs2_write_ctxt **wcp,
1321841 struct ocfs2_super *osb, loff_t pos,
1322− unsigned len, struct buffer_head *di_bh)
842+ unsigned len, ocfs2_write_type_t type,
843+ struct buffer_head *di_bh)
1323844 {
1324845 u32 cend;
1325846 struct ocfs2_write_ctxt *wc;
@@ -1334,6 +855,7 @@ static int ocfs2_alloc_write_ctxt(struct ocfs2_write_ctxt **wcp, · 8 unmodified lines
1334855 wc->w_clen = cend - wc->w_cpos + 1;
1335856 get_bh(di_bh);
1336857 wc->w_di_bh = di_bh;
858+ wc->w_type = type;
1337859
1338860 if (unlikely(PAGE_CACHE_SHIFT > osb->s_clustersize_bits))
1339861 wc->w_large_pages = 1;
@@ -1341,6 +863,7 @@ static int ocfs2_alloc_write_ctxt(struct ocfs2_write_ctxt **wcp, · 1 unmodified line
1341863 wc->w_large_pages = 0;
1342864
1343865 ocfs2_init_dealloc_ctxt(&wc->w_dealloc);
866+ INIT_LIST_HEAD(&wc->w_unwritten_list);
1344867
1345868 *wcp = wc;
1346869
@@ -1401,12 +924,13 @@ static void ocfs2_write_failure(struct inode *inode, · 54 unmodified lines
1401924 to = user_pos + user_len;
1402925 struct page *tmppage;
1403926
1404− ocfs2_zero_new_buffers(wc->w_target_page, from, to);
927+ if (wc->w_target_page)
928+ ocfs2_zero_new_buffers(wc->w_target_page, from, to);
1405929
1406930 for(i = 0; i < wc->w_num_pages; i++) {
1407931 tmppage = wc->w_pages[i];
1408932
1409− if (page_has_buffers(tmppage)) {
933+ if (tmppage && page_has_buffers(tmppage)) {
1410934 if (ocfs2_should_order_data(inode))
1411935 ocfs2_jbd2_file_inode(wc->w_handle, inode);
1412936
@@ -1536,11 +1060,13 @@ static int ocfs2_grab_pages_for_write(struct address_space *mapping, · 123 unmodified lines
15361060 wc->w_num_pages = 1;
15371061 start = target_index;
15381062 }
1063+ end_index = (user_pos + user_len - 1) >> PAGE_CACHE_SHIFT;
15391064
15401065 for(i = 0; i < wc->w_num_pages; i++) {
15411066 index = start + i;
15421067
1543− if (index == target_index && mmap_page) {
1068+ if (index >= target_index && index <= end_index &&
1069+ wc->w_type == OCFS2_WRITE_MMAP) {
15441070 /*
15451071 * ocfs2_pagemkwrite() is a little different
15461072 * and wants us to directly use the page
@@ -1559,6 +1085,11 @@ static int ocfs2_grab_pages_for_write(struct address_space *mapping, · 12 unmodified lines
15591085 page_cache_get(mmap_page);
15601086 wc->w_pages[i] = mmap_page;
15611087 wc->w_target_locked = true;
1088+ } else if (index >= target_index && index <= end_index &&
1089+ wc->w_type == OCFS2_WRITE_DIRECT) {
1090+ /* Direct write has no mapping page. */
1091+ wc->w_pages[i] = NULL;
1092+ continue;
15621093 } else {
15631094 wc->w_pages[i] = find_or_create_page(mapping, index,
15641095 GFP_NOFS);
@@ -1583,19 +1114,20 @@ out: · 18 unmodified lines
15831114 * Prepare a single cluster for write one cluster into the file.
15841115 */
15851116 static int ocfs2_write_cluster(struct address_space *mapping,
1586− u32 phys, unsigned int unwritten,
1117+ u32 *phys, unsigned int new,
1118+ unsigned int clear_unwritten,
15871119 unsigned int should_zero,
15881120 struct ocfs2_alloc_context *data_ac,
15891121 struct ocfs2_alloc_context *meta_ac,
15901122 struct ocfs2_write_ctxt *wc, u32 cpos,
15911123 loff_t user_pos, unsigned user_len)
15921124 {
1593− int ret, i, new;
1594− u64 v_blkno, p_blkno;
1125+ int ret, i;
1126+ u64 p_blkno;
15951127 struct inode *inode = mapping->host;
15961128 struct ocfs2_extent_tree et;
1129+ int bpc = ocfs2_clusters_to_blocks(inode->i_sb, 1);
15971130
1598− new = phys == 0 ? 1 : 0;
15991131 if (new) {
16001132 u32 tmp_pos;
16011133
@@ -1605,9 +1137,9 @@ static int ocfs2_write_cluster(struct address_space *mapping, · 3 unmodified lines
16051137 */
16061138 tmp_pos = cpos;
16071139 ret = ocfs2_add_inode_data(OCFS2_SB(inode->i_sb), inode,
1608− &tmp_pos, 1, 0, wc->w_di_bh,
1609− wc->w_handle, data_ac,
1610− meta_ac, NULL);
1140+ &tmp_pos, 1, !clear_unwritten,
1141+ wc->w_di_bh, wc->w_handle,
1142+ data_ac, meta_ac, NULL);
16111143 /*
16121144 * This shouldn't happen because we must have already
16131145 * calculated the correct meta data allocation required. The
@@ -1624,11 +1156,11 @@ static int ocfs2_write_cluster(struct address_space *mapping, · 10 unmodified lines
16241156 mlog_errno(ret);
16251157 goto out;
16261158 }
1627− } else if (unwritten) {
1159+ } else if (clear_unwritten) {
16281160 ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode),
16291161 wc->w_di_bh);
16301162 ret = ocfs2_mark_extent_written(inode, &et,
1631− wc->w_handle, cpos, 1, phys,
1163+ wc->w_handle, cpos, 1, *phys,
16321164 meta_ac, &wc->w_dealloc);
16331165 if (ret < 0) {
16341166 mlog_errno(ret);
@@ -1636,30 +1168,33 @@ static int ocfs2_write_cluster(struct address_space *mapping, · 1 unmodified line
16361168 }
16371169 }
16381170
1639− if (should_zero)
1640− v_blkno = ocfs2_clusters_to_blocks(inode->i_sb, cpos);
1641− else
1642− v_blkno = user_pos >> inode->i_sb->s_blocksize_bits;
1643−
16441171 /*
16451172 * The only reason this should fail is due to an inability to
16461173 * find the extent added.
16471174 */
1648− ret = ocfs2_extent_map_get_blocks(inode, v_blkno, &p_blkno, NULL,
1649− NULL);
1175+ ret = ocfs2_get_clusters(inode, cpos, phys, NULL, NULL);
16501176 if (ret < 0) {
16511177 mlog(ML_ERROR, "Get physical blkno failed for inode %llu, "
1652− "at logical block %llu",
1653− (unsigned long long)OCFS2_I(inode)->ip_blkno,
1654− (unsigned long long)v_blkno);
1178+ "at logical cluster %u",
1179+ (unsigned long long)OCFS2_I(inode)->ip_blkno, cpos);
16551180 goto out;
16561181 }
16571182
1658− BUG_ON(p_blkno == 0);
1183+ BUG_ON(*phys == 0);
1184+
1185+ p_blkno = ocfs2_clusters_to_blocks(inode->i_sb, *phys);
1186+ if (!should_zero)
1187+ p_blkno += (user_pos >> inode->i_sb->s_blocksize_bits) & (u64)(bpc - 1);
16591188
16601189 for(i = 0; i < wc->w_num_pages; i++) {
16611190 int tmpret;
16621191
1192+ /* This is the direct io target page. */
1193+ if (wc->w_pages[i] == NULL) {
1194+ p_blkno++;
1195+ continue;
1196+ }
1197+
16631198 tmpret = ocfs2_prepare_page_for_write(inode, &p_blkno, wc,
16641199 wc->w_pages[i], cpos,
16651200 user_pos, user_len,
@@ -1706,8 +1241,9 @@ static int ocfs2_write_cluster_by_desc(struct address_space *mapping, · 40 unmodified lines
17061241 if ((cluster_off + local_len) > osb->s_clustersize)
17071242 local_len = osb->s_clustersize - cluster_off;
17081243
1709− ret = ocfs2_write_cluster(mapping, desc->c_phys,
1710− desc->c_unwritten,
1244+ ret = ocfs2_write_cluster(mapping, &desc->c_phys,
1245+ desc->c_new,
1246+ desc->c_clear_unwritten,
17111247 desc->c_needs_zero,
17121248 data_ac, meta_ac,
17131249 wc, desc->c_cpos, pos, local_len);
@@ -1777,6 +1313,66 @@ static void ocfs2_set_target_boundaries(struct ocfs2_super *osb, · 63 unmodified lines
17771313 }
17781314 }
17791315
1316+/*
1317+ * Check if this extent is marked UNWRITTEN by direct io. If so, we need not to
1318+ * do the zero work. And should not to clear UNWRITTEN since it will be cleared
1319+ * by the direct io procedure.
1320+ * If this is a new extent that allocated by direct io, we should mark it in
1321+ * the ip_unwritten_list.
1322+ */
1323+static int ocfs2_unwritten_check(struct inode *inode,
1324+ struct ocfs2_write_ctxt *wc,
1325+ struct ocfs2_write_cluster_desc *desc)
1326+{
1327+ struct ocfs2_inode_info *oi = OCFS2_I(inode);
1328+ struct ocfs2_unwritten_extent *ue = NULL, *new = NULL;
1329+ int ret = 0;
1330+
1331+ if (!desc->c_needs_zero)
1332+ return 0;
1333+
1334+retry:
1335+ spin_lock(&oi->ip_lock);
1336+ /* Needs not to zero no metter buffer or direct. The one who is zero
1337+ * the cluster is doing zero. And he will clear unwritten after all
1338+ * cluster io finished. */
1339+ list_for_each_entry(ue, &oi->ip_unwritten_list, ue_ip_node) {
1340+ if (desc->c_cpos == ue->ue_cpos) {
1341+ BUG_ON(desc->c_new);
1342+ desc->c_needs_zero = 0;
1343+ desc->c_clear_unwritten = 0;
1344+ goto unlock;
1345+ }
1346+ }
1347+
1348+ if (wc->w_type != OCFS2_WRITE_DIRECT)
1349+ goto unlock;
1350+
1351+ if (new == NULL) {
1352+ spin_unlock(&oi->ip_lock);
1353+ new = kmalloc(sizeof(struct ocfs2_unwritten_extent),
1354+ GFP_NOFS);
1355+ if (new == NULL) {
1356+ ret = -ENOMEM;
1357+ goto out;
1358+ }
1359+ goto retry;
1360+ }
1361+ /* This direct write will doing zero. */
1362+ new->ue_cpos = desc->c_cpos;
1363+ new->ue_phys = desc->c_phys;
1364+ desc->c_clear_unwritten = 0;
1365+ list_add_tail(&new->ue_ip_node, &oi->ip_unwritten_list);
1366+ list_add_tail(&new->ue_node, &wc->w_unwritten_list);
1367+ new = NULL;
1368+unlock:
1369+ spin_unlock(&oi->ip_lock);
1370+out:
1371+ if (new)
1372+ kfree(new);
1373+ return ret;
1374+}
1375+
17801376 /*
17811377 * Populate each single-cluster write descriptor in the write context
17821378 * with information about the i/o to be done.
@@ -1852,14 +1448,21 @@ static int ocfs2_populate_write_desc(struct inode *inode, · 69 unmodified lines
18521448 if (phys == 0) {
18531449 desc->c_new = 1;
18541450 desc->c_needs_zero = 1;
1451+ desc->c_clear_unwritten = 1;
18551452 *clusters_to_alloc = *clusters_to_alloc + 1;
18561453 }
18571454
18581455 if (ext_flags & OCFS2_EXT_UNWRITTEN) {
1859− desc->c_unwritten = 1;
1456+ desc->c_clear_unwritten = 1;
18601457 desc->c_needs_zero = 1;
18611458 }
18621459
1460+ ret = ocfs2_unwritten_check(inode, wc, desc);
1461+ if (ret) {
1462+ mlog_errno(ret);
1463+ goto out;
1464+ }
1465+
18631466 num_clusters--;
18641467 }
18651468
@@ -2022,8 +1625,10 @@ static int ocfs2_expand_nonsparse_inode(struct inode *inode, · 156 unmodified lines
20221625 if (ret)
20231626 mlog_errno(ret);
20241627
2025− wc->w_first_new_cpos =
2026− ocfs2_clusters_for_bytes(inode->i_sb, i_size_read(inode));
1628+ /* There is no wc if this is call from direct. */
1629+ if (wc)
1630+ wc->w_first_new_cpos =
1631+ ocfs2_clusters_for_bytes(inode->i_sb, i_size_read(inode));
20271632
20281633 return ret;
20291634 }
@@ -2077,9 +1682,8 @@ out: · 47 unmodified lines
20771682 return ret;
20781683 }
20791684
2080−int ocfs2_write_begin_nolock(struct file *filp,
2081− struct address_space *mapping,
2082− loff_t pos, unsigned len, unsigned flags,
1685+int ocfs2_write_begin_nolock(struct address_space *mapping,
1686+ loff_t pos, unsigned len, ocfs2_write_type_t type,
20831687 struct page **pagep, void **fsdata,
20841688 struct buffer_head *di_bh, struct page *mmap_page)
20851689 {
@@ -2096,7 +1700,7 @@ int ocfs2_write_begin_nolock(struct file *filp, · 10 unmodified lines
20961700 int try_free = 1, ret1;
20971701
20981702 try_again:
2099− ret = ocfs2_alloc_write_ctxt(&wc, osb, pos, len, di_bh);
1703+ ret = ocfs2_alloc_write_ctxt(&wc, osb, pos, len, type, di_bh);
21001704 if (ret) {
21011705 mlog_errno(ret);
21021706 return ret;
@@ -2115,14 +1719,17 @@ try_again: · 12 unmodified lines
21151719 }
21161720 }
21171721
2118− if (ocfs2_sparse_alloc(osb))
2119− ret = ocfs2_zero_tail(inode, di_bh, pos);
2120− else
2121− ret = ocfs2_expand_nonsparse_inode(inode, di_bh, pos, len,
2122− wc);
2123− if (ret) {
2124− mlog_errno(ret);
2125− goto out;
1722+ /* Direct io change i_size late, should not zero tail here. */
1723+ if (type != OCFS2_WRITE_DIRECT) {
1724+ if (ocfs2_sparse_alloc(osb))
1725+ ret = ocfs2_zero_tail(inode, di_bh, pos);
1726+ else
1727+ ret = ocfs2_expand_nonsparse_inode(inode, di_bh, pos,
1728+ len, wc);
1729+ if (ret) {
1730+ mlog_errno(ret);
1731+ goto out;
1732+ }
21261733 }
21271734
21281735 ret = ocfs2_check_range_for_refcount(inode, pos, len);
@@ -2153,7 +1760,7 @@ try_again: · 24 unmodified lines
21531760 (unsigned long long)OCFS2_I(inode)->ip_blkno,
21541761 (long long)i_size_read(inode),
21551762 le32_to_cpu(di->i_clusters),
2156− pos, len, flags, mmap_page,
1763+ pos, len, type, mmap_page,
21571764 clusters_to_alloc, extents_to_split);
21581765
21591766 /*
@@ -2183,17 +1790,17 @@ try_again: · 23 unmodified lines
21831790
21841791 credits = ocfs2_calc_extend_credits(inode->i_sb,
21851792 &di->id2.i_list);
2186−
2187− }
1793+ } else if (type == OCFS2_WRITE_DIRECT)
1794+ /* direct write needs not to start trans if no extents alloc. */
1795+ goto success;
21881796
21891797 /*
21901798 * We have to zero sparse allocated clusters, unwritten extent clusters,
21911799 * and non-sparse clusters we just extended. For non-sparse writes,
21921800 * we know zeros will only be needed in the first and/or last cluster.
21931801 */
2194− if (clusters_to_alloc || extents_to_split ||
2195− (wc->w_clen && (wc->w_desc[0].c_needs_zero ||
2196− wc->w_desc[wc->w_clen - 1].c_needs_zero)))
1802+ if (wc->w_clen && (wc->w_desc[0].c_needs_zero ||
1803+ wc->w_desc[wc->w_clen - 1].c_needs_zero))
21971804 cluster_of_pages = 1;
21981805 else
21991806 cluster_of_pages = 0;
@@ -2260,7 +1867,8 @@ try_again: · 60 unmodified lines
22601867 ocfs2_free_alloc_context(meta_ac);
22611868
22621869 success:
2263− *pagep = wc->w_target_page;
1870+ if (pagep)
1871+ *pagep = wc->w_target_page;
22641872 *fsdata = wc;
22651873 return 0;
22661874 out_quota:
@@ -2271,7 +1879,7 @@ out_commit: · 4 unmodified lines
22711879 ocfs2_commit_trans(osb, handle);
22721880
22731881 out:
2274− ocfs2_free_write_ctxt(wc);
1882+ ocfs2_free_write_ctxt(inode, wc);
22751883
22761884 if (data_ac) {
22771885 ocfs2_free_alloc_context(data_ac);
@@ -2323,8 +1931,8 @@ static int ocfs2_write_begin(struct file *file, struct address_space *mapping, · 45 unmodified lines
23231931 */
23241932 down_write(&OCFS2_I(inode)->ip_alloc_sem);
23251933
2326− ret = ocfs2_write_begin_nolock(file, mapping, pos, len, flags, pagep,
2327− fsdata, di_bh, NULL);
1934+ ret = ocfs2_write_begin_nolock(mapping, pos, len, OCFS2_WRITE_BUFFER,
1935+ pagep, fsdata, di_bh, NULL);
23281936 if (ret) {
23291937 mlog_errno(ret);
23301938 goto out_fail;
@@ -2381,12 +1989,16 @@ int ocfs2_write_end_nolock(struct address_space *mapping, · 50 unmodified lines
23811989 handle_t *handle = wc->w_handle;
23821990 struct page *tmppage;
23831991
2384− ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), wc->w_di_bh,
2385− OCFS2_JOURNAL_ACCESS_WRITE);
2386− if (ret) {
2387− copied = ret;
2388− mlog_errno(ret);
2389− goto out;
1992+ BUG_ON(!list_empty(&wc->w_unwritten_list));
1993+
1994+ if (handle) {
1995+ ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode),
1996+ wc->w_di_bh, OCFS2_JOURNAL_ACCESS_WRITE);
1997+ if (ret) {
1998+ copied = ret;
1999+ mlog_errno(ret);
2000+ goto out;
2001+ }
23902002 }
23912003
23922004 if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) {
@@ -2394,18 +2006,23 @@ int ocfs2_write_end_nolock(struct address_space *mapping, · 1 unmodified line
23942006 goto out_write_size;
23952007 }
23962008
2397− if (unlikely(copied < len)) {
2009+ if (unlikely(copied < len) && wc->w_target_page) {
23982010 if (!PageUptodate(wc->w_target_page))
23992011 copied = 0;
24002012
24012013 ocfs2_zero_new_buffers(wc->w_target_page, start+copied,
24022014 start+len);
24032015 }
2404− flush_dcache_page(wc->w_target_page);
2016+ if (wc->w_target_page)
2017+ flush_dcache_page(wc->w_target_page);
24052018
24062019 for(i = 0; i < wc->w_num_pages; i++) {
24072020 tmppage = wc->w_pages[i];
24082021
2022+ /* This is the direct io target page. */
2023+ if (tmppage == NULL)
2024+ continue;
2025+
24092026 if (tmppage == wc->w_target_page) {
24102027 from = wc->w_target_from;
24112028 to = wc->w_target_to;
@@ -2424,25 +2041,29 @@ int ocfs2_write_end_nolock(struct address_space *mapping, · 12 unmodified lines
24242041 }
24252042
24262043 if (page_has_buffers(tmppage)) {
2427− if (ocfs2_should_order_data(inode))
2428− ocfs2_jbd2_file_inode(wc->w_handle, inode);
2044+ if (handle && ocfs2_should_order_data(inode))
2045+ ocfs2_jbd2_file_inode(handle, inode);
24292046 block_commit_write(tmppage, from, to);
24302047 }
24312048 }
24322049
24332050 out_write_size:
2434− pos += copied;
2435− if (pos > i_size_read(inode)) {
2436− i_size_write(inode, pos);
2437− mark_inode_dirty(inode);
2438− }
2439− inode->i_blocks = ocfs2_inode_sector_count(inode);
2440− di->i_size = cpu_to_le64((u64)i_size_read(inode));
2441− inode->i_mtime = inode->i_ctime = CURRENT_TIME;
2442− di->i_mtime = di->i_ctime = cpu_to_le64(inode->i_mtime.tv_sec);
2443− di->i_mtime_nsec = di->i_ctime_nsec = cpu_to_le32(inode->i_mtime.tv_nsec);
2444− ocfs2_update_inode_fsync_trans(handle, inode, 1);
2445− ocfs2_journal_dirty(handle, wc->w_di_bh);
2051+ /* Direct io do not update i_size here. */
2052+ if (wc->w_type != OCFS2_WRITE_DIRECT) {
2053+ pos += copied;
2054+ if (pos > i_size_read(inode)) {
2055+ i_size_write(inode, pos);
2056+ mark_inode_dirty(inode);
2057+ }
2058+ inode->i_blocks = ocfs2_inode_sector_count(inode);
2059+ di->i_size = cpu_to_le64((u64)i_size_read(inode));
2060+ inode->i_mtime = inode->i_ctime = CURRENT_TIME;
2061+ di->i_mtime = di->i_ctime = cpu_to_le64(inode->i_mtime.tv_sec);
2062+ di->i_mtime_nsec = di->i_ctime_nsec = cpu_to_le32(inode->i_mtime.tv_nsec);
2063+ ocfs2_update_inode_fsync_trans(handle, inode, 1);
2064+ }
2065+ if (handle)
2066+ ocfs2_journal_dirty(handle, wc->w_di_bh);
24462067
24472068 out:
24482069 /* unlock pages before dealloc since it needs acquiring j_trans_barrier
@@ -2452,7 +2073,8 @@ out: · 3 unmodified lines
24522073 */
24532074 ocfs2_unlock_pages(wc);
24542075
2455− ocfs2_commit_trans(osb, handle);
2076+ if (handle)
2077+ ocfs2_commit_trans(osb, handle);
24562078
24572079 ocfs2_run_deallocs(osb, &wc->w_dealloc);
24582080
@@ -2477,6 +2099,360 @@ static int ocfs2_write_end(struct file *file, struct address_space *mapping, · 18 unmodified lines
24772099 return ret;
24782100 }
24792101
2102+struct ocfs2_dio_write_ctxt {
2103+ struct list_head dw_zero_list;
2104+ unsigned dw_zero_count;
2105+ int dw_orphaned;
2106+ pid_t dw_writer_pid;
2107+};
2108+
2109+static struct ocfs2_dio_write_ctxt *
2110+ocfs2_dio_alloc_write_ctx(struct buffer_head *bh, int *alloc)
2111+{
2112+ struct ocfs2_dio_write_ctxt *dwc = NULL;
2113+
2114+ if (bh->b_private)
2115+ return bh->b_private;
2116+
2117+ dwc = kmalloc(sizeof(struct ocfs2_dio_write_ctxt), GFP_NOFS);
2118+ if (dwc == NULL)
2119+ return NULL;
2120+ INIT_LIST_HEAD(&dwc->dw_zero_list);
2121+ dwc->dw_zero_count = 0;
2122+ dwc->dw_orphaned = 0;
2123+ dwc->dw_writer_pid = task_pid_nr(current);
2124+ bh->b_private = dwc;
2125+ *alloc = 1;
2126+
2127+ return dwc;
2128+}
2129+
2130+static void ocfs2_dio_free_write_ctx(struct inode *inode,
2131+ struct ocfs2_dio_write_ctxt *dwc)
2132+{
2133+ ocfs2_free_unwritten_list(inode, &dwc->dw_zero_list);
2134+ kfree(dwc);
2135+}
2136+
2137+/*
2138+ * TODO: Make this into a generic get_blocks function.
2139+ *
2140+ * From do_direct_io in direct-io.c:
2141+ * "So what we do is to permit the ->get_blocks function to populate
2142+ * bh.b_size with the size of IO which is permitted at this offset and
2143+ * this i_blkbits."
2144+ *
2145+ * This function is called directly from get_more_blocks in direct-io.c.
2146+ *
2147+ * called like this: dio->get_blocks(dio->inode, fs_startblk,
2148+ * fs_count, map_bh, dio->rw == WRITE);
2149+ */
2150+static int ocfs2_dio_get_block(struct inode *inode, sector_t iblock,
2151+ struct buffer_head *bh_result, int create)
2152+{
2153+ struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
2154+ struct ocfs2_inode_info *oi = OCFS2_I(inode);
2155+ struct ocfs2_write_ctxt *wc;
2156+ struct ocfs2_write_cluster_desc *desc = NULL;
2157+ struct ocfs2_dio_write_ctxt *dwc = NULL;
2158+ struct buffer_head *di_bh = NULL;
2159+ u64 p_blkno;
2160+ loff_t pos = iblock << inode->i_sb->s_blocksize_bits;
2161+ unsigned len, total_len = bh_result->b_size;
2162+ int ret = 0, first_get_block = 0;
2163+
2164+ len = osb->s_clustersize - (pos & (osb->s_clustersize - 1));
2165+ len = min(total_len, len);
2166+
2167+ mlog(0, "get block of %lu at %llu:%u req %u\n",
2168+ inode->i_ino, pos, len, total_len);
2169+
2170+ /*
2171+ * Because we need to change file size in ocfs2_dio_end_io_write(), or
2172+ * we may need to add it to orphan dir. So can not fall to fast path
2173+ * while file size will be changed.
2174+ */
2175+ if (pos + total_len <= i_size_read(inode)) {
2176+ down_read(&oi->ip_alloc_sem);
2177+ /* This is the fast path for re-write. */
2178+ ret = ocfs2_get_block(inode, iblock, bh_result, create);
2179+
2180+ up_read(&oi->ip_alloc_sem);
2181+
2182+ if (buffer_mapped(bh_result) &&
2183+ !buffer_new(bh_result) &&
2184+ ret == 0)
2185+ goto out;
2186+
2187+ /* Clear state set by ocfs2_get_block. */
2188+ bh_result->b_state = 0;
2189+ }
2190+
2191+ dwc = ocfs2_dio_alloc_write_ctx(bh_result, &first_get_block);
2192+ if (unlikely(dwc == NULL)) {
2193+ ret = -ENOMEM;
2194+ mlog_errno(ret);
2195+ goto out;
2196+ }
2197+
2198+ if (ocfs2_clusters_for_bytes(inode->i_sb, pos + total_len) >
2199+ ocfs2_clusters_for_bytes(inode->i_sb, i_size_read(inode)) &&
2200+ !dwc->dw_orphaned) {
2201+ /*
2202+ * when we are going to alloc extents beyond file size, add the
2203+ * inode to orphan dir, so we can recall those spaces when
2204+ * system crashed during write.
2205+ */
2206+ ret = ocfs2_add_inode_to_orphan(osb, inode);
2207+ if (ret < 0) {
2208+ mlog_errno(ret);
2209+ goto out;
2210+ }
2211+ dwc->dw_orphaned = 1;
2212+ }
2213+
2214+ ret = ocfs2_inode_lock(inode, &di_bh, 1);
2215+ if (ret) {
2216+ mlog_errno(ret);
2217+ goto out;
2218+ }
2219+
2220+ down_write(&oi->ip_alloc_sem);
2221+
2222+ if (first_get_block) {
2223+ if (ocfs2_sparse_alloc(OCFS2_SB(inode->i_sb)))
2224+ ret = ocfs2_zero_tail(inode, di_bh, pos);
2225+ else
2226+ ret = ocfs2_expand_nonsparse_inode(inode, di_bh, pos,
2227+ total_len, NULL);
2228+ if (ret < 0) {
2229+ mlog_errno(ret);
2230+ goto unlock;
2231+ }
2232+ }
2233+
2234+ ret = ocfs2_write_begin_nolock(inode->i_mapping, pos, len,
2235+ OCFS2_WRITE_DIRECT, NULL,
2236+ (void **)&wc, di_bh, NULL);
2237+ if (ret) {
2238+ mlog_errno(ret);
2239+ goto unlock;
2240+ }
2241+
2242+ desc = &wc->w_desc[0];
2243+
2244+ p_blkno = ocfs2_clusters_to_blocks(inode->i_sb, desc->c_phys);
2245+ BUG_ON(p_blkno == 0);
2246+ p_blkno += iblock & (u64)(ocfs2_clusters_to_blocks(inode->i_sb, 1) - 1);
2247+
2248+ map_bh(bh_result, inode->i_sb, p_blkno);
2249+ bh_result->b_size = len;
2250+ if (desc->c_needs_zero)
2251+ set_buffer_new(bh_result);
2252+
2253+ /* May sleep in end_io. It should not happen in a irq context. So defer
2254+ * it to dio work queue. */
2255+ set_buffer_defer_completion(bh_result);
2256+
2257+ if (!list_empty(&wc->w_unwritten_list)) {
2258+ struct ocfs2_unwritten_extent *ue = NULL;
2259+
2260+ ue = list_first_entry(&wc->w_unwritten_list,
2261+ struct ocfs2_unwritten_extent,
2262+ ue_node);
2263+ BUG_ON(ue->ue_cpos != desc->c_cpos);
2264+ /* The physical address may be 0, fill it. */
2265+ ue->ue_phys = desc->c_phys;
2266+
2267+ list_splice_tail_init(&wc->w_unwritten_list, &dwc->dw_zero_list);
2268+ dwc->dw_zero_count++;
2269+ }
2270+
2271+ ret = ocfs2_write_end_nolock(inode->i_mapping, pos, len, len, NULL, wc);
2272+ BUG_ON(ret != len);
2273+ ret = 0;
2274+unlock:
2275+ up_write(&oi->ip_alloc_sem);
2276+ ocfs2_inode_unlock(inode, 1);
2277+ brelse(di_bh);
2278+out:
2279+ if (ret < 0)
2280+ ret = -EIO;
2281+ return ret;
2282+}
2283+
2284+static void ocfs2_dio_end_io_write(struct inode *inode,
2285+ struct ocfs2_dio_write_ctxt *dwc,
2286+ loff_t offset,
2287+ ssize_t bytes)
2288+{
2289+ struct ocfs2_cached_dealloc_ctxt dealloc;
2290+ struct ocfs2_extent_tree et;
2291+ struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
2292+ struct ocfs2_inode_info *oi = OCFS2_I(inode);
2293+ struct ocfs2_unwritten_extent *ue = NULL;
2294+ struct buffer_head *di_bh = NULL;
2295+ struct ocfs2_dinode *di;
2296+ struct ocfs2_alloc_context *data_ac = NULL;
2297+ struct ocfs2_alloc_context *meta_ac = NULL;
2298+ handle_t *handle = NULL;
2299+ loff_t end = offset + bytes;
2300+ int ret = 0, credits = 0, locked = 0;
2301+
2302+ ocfs2_init_dealloc_ctxt(&dealloc);
2303+
2304+ /* We do clear unwritten, delete orphan, change i_size here. If neither
2305+ * of these happen, we can skip all this. */
2306+ if (list_empty(&dwc->dw_zero_list) &&
2307+ end <= i_size_read(inode) &&
2308+ !dwc->dw_orphaned)
2309+ goto out;
2310+
2311+ /* ocfs2_file_write_iter will get i_mutex, so we need not lock if we
2312+ * are in that context. */
2313+ if (dwc->dw_writer_pid != task_pid_nr(current)) {
2314+ mutex_lock(&inode->i_mutex);
2315+ locked = 1;
2316+ }
2317+
2318+ ret = ocfs2_inode_lock(inode, &di_bh, 1);
2319+ if (ret < 0) {
2320+ mlog_errno(ret);
2321+ goto out;
2322+ }
2323+
2324+ down_write(&oi->ip_alloc_sem);
2325+
2326+ /* Delete orphan before acquire i_mutex. */
2327+ if (dwc->dw_orphaned) {
2328+ BUG_ON(dwc->dw_writer_pid != task_pid_nr(current));
2329+
2330+ end = end > i_size_read(inode) ? end : 0;
2331+
2332+ ret = ocfs2_del_inode_from_orphan(osb, inode, di_bh,
2333+ !!end, end);
2334+ if (ret < 0)
2335+ mlog_errno(ret);
2336+ }
2337+
2338+ di = (struct ocfs2_dinode *)di_bh;
2339+
2340+ ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh);
2341+
2342+ ret = ocfs2_lock_allocators(inode, &et, 0, dwc->dw_zero_count*2,
2343+ &data_ac, &meta_ac);
2344+ if (ret) {
2345+ mlog_errno(ret);
2346+ goto unlock;
2347+ }
2348+
2349+ credits = ocfs2_calc_extend_credits(inode->i_sb, &di->id2.i_list);
2350+
2351+ handle = ocfs2_start_trans(osb, credits);
2352+ if (IS_ERR(handle)) {
2353+ ret = PTR_ERR(handle);
2354+ mlog_errno(ret);
2355+ goto unlock;
2356+ }
2357+ ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh,
2358+ OCFS2_JOURNAL_ACCESS_WRITE);
2359+ if (ret) {
2360+ mlog_errno(ret);
2361+ goto commit;
2362+ }
2363+
2364+ list_for_each_entry(ue, &dwc->dw_zero_list, ue_node) {
2365+ ret = ocfs2_mark_extent_written(inode, &et, handle,
2366+ ue->ue_cpos, 1,
2367+ ue->ue_phys,
2368+ meta_ac, &dealloc);
2369+ if (ret < 0) {
2370+ mlog_errno(ret);
2371+ break;
2372+ }
2373+ }
2374+
2375+ if (end > i_size_read(inode)) {
2376+ ret = ocfs2_set_inode_size(handle, inode, di_bh, end);
2377+ if (ret < 0)
2378+ mlog_errno(ret);
2379+ }
2380+commit:
2381+ ocfs2_commit_trans(osb, handle);
2382+unlock:
2383+ up_write(&oi->ip_alloc_sem);
2384+ ocfs2_inode_unlock(inode, 1);
2385+ brelse(di_bh);
2386+out:
2387+ if (data_ac)
2388+ ocfs2_free_alloc_context(data_ac);
2389+ if (meta_ac)
2390+ ocfs2_free_alloc_context(meta_ac);
2391+ ocfs2_run_deallocs(osb, &dealloc);
2392+ if (locked)
2393+ mutex_unlock(&inode->i_mutex);
2394+ ocfs2_dio_free_write_ctx(inode, dwc);
2395+}
2396+
2397+/*
2398+ * ocfs2_dio_end_io is called by the dio core when a dio is finished. We're
2399+ * particularly interested in the aio/dio case. We use the rw_lock DLM lock
2400+ * to protect io on one node from truncation on another.
2401+ */
2402+static int ocfs2_dio_end_io(struct kiocb *iocb,
2403+ loff_t offset,
2404+ ssize_t bytes,
2405+ void *private)
2406+{
2407+ struct inode *inode = file_inode(iocb->ki_filp);
2408+ int level;
2409+
2410+ if (bytes <= 0)
2411+ return 0;
2412+
2413+ /* this io's submitter should not have unlocked this before we could */
2414+ BUG_ON(!ocfs2_iocb_is_rw_locked(iocb));
2415+
2416+ if (private)
2417+ ocfs2_dio_end_io_write(inode, private, offset, bytes);
2418+
2419+ ocfs2_iocb_clear_rw_locked(iocb);
2420+
2421+ level = ocfs2_iocb_rw_locked_level(iocb);
2422+ ocfs2_rw_unlock(inode, level);
2423+ return 0;
2424+}
2425+
2426+static ssize_t ocfs2_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
2427+ loff_t offset)
2428+{
2429+ struct file *file = iocb->ki_filp;
2430+ struct inode *inode = file_inode(file)->i_mapping->host;
2431+ struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
2432+ loff_t end = offset + iter->count;
2433+ get_block_t *get_block;
2434+
2435+ /*
2436+ * Fallback to buffered I/O if we see an inode without
2437+ * extents.
2438+ */
2439+ if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)
2440+ return 0;
2441+
2442+ /* Fallback to buffered I/O if we do not support append dio. */
2443+ if (end > i_size_read(inode) && !ocfs2_supports_append_dio(osb))
2444+ return 0;
2445+
2446+ if (iov_iter_rw(iter) == READ)
2447+ get_block = ocfs2_get_block;
2448+ else
2449+ get_block = ocfs2_dio_get_block;
2450+
2451+ return __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
2452+ iter, offset, get_block,
2453+ ocfs2_dio_end_io, NULL, 0);
2454+}
2455+
24802456 const struct address_space_operations ocfs2_aops = {
24812457 .readpage = ocfs2_readpage,
24822458 .readpages = ocfs2_readpages,
fs/ocfs2/aops.h+8−11
@@ -47,9 +47,14 @@ int ocfs2_write_end_nolock(struct address_space *mapping, · 46 unmodified lines
4747 loff_t pos, unsigned len, unsigned copied,
4848 struct page *page, void *fsdata);
4949
50−int ocfs2_write_begin_nolock(struct file *filp,
51− struct address_space *mapping,
52− loff_t pos, unsigned len, unsigned flags,
50+typedef enum {
51+ OCFS2_WRITE_BUFFER = 0,
52+ OCFS2_WRITE_DIRECT,
53+ OCFS2_WRITE_MMAP,
54+} ocfs2_write_type_t;
55+
56+int ocfs2_write_begin_nolock(struct address_space *mapping,
57+ loff_t pos, unsigned len, ocfs2_write_type_t type,
5358 struct page **pagep, void **fsdata,
5459 struct buffer_head *di_bh, struct page *mmap_page);
5560
@@ -79,7 +84,6 @@ static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) · 23 unmodified lines
7984 enum ocfs2_iocb_lock_bits {
8085 OCFS2_IOCB_RW_LOCK = 0,
8186 OCFS2_IOCB_RW_LOCK_LEVEL,
82− OCFS2_IOCB_UNALIGNED_IO,
8387 OCFS2_IOCB_NUM_LOCKS
8488 };
8589
@@ -88,11 +92,4 @@ enum ocfs2_iocb_lock_bits { · 2 unmodified lines
8892 #define ocfs2_iocb_rw_locked_level(iocb) \
8993 test_bit(OCFS2_IOCB_RW_LOCK_LEVEL, (unsigned long *)&iocb->private)
9094
91−#define ocfs2_iocb_set_unaligned_aio(iocb) \
92− set_bit(OCFS2_IOCB_UNALIGNED_IO, (unsigned long *)&iocb->private)
93−#define ocfs2_iocb_clear_unaligned_aio(iocb) \
94− clear_bit(OCFS2_IOCB_UNALIGNED_IO, (unsigned long *)&iocb->private)
95−#define ocfs2_iocb_is_unaligned_aio(iocb) \
96− test_bit(OCFS2_IOCB_UNALIGNED_IO, (unsigned long *)&iocb->private)
97−
9895 #endif /* OCFS2_FILE_H */
fs/ocfs2/cluster/heartbeat.c+2−2
@@ -1444,8 +1444,8 @@ static void o2hb_region_release(struct config_item *item) · 1443 unmodified lines
14441444 debugfs_remove(reg->hr_debug_dir);
14451445 kfree(reg->hr_db_livenodes);
14461446 kfree(reg->hr_db_regnum);
1447− kfree(reg->hr_debug_elapsed_time);
1448− kfree(reg->hr_debug_pinned);
1447+ kfree(reg->hr_db_elapsed_time);
1448+ kfree(reg->hr_db_pinned);
14491449
14501450 spin_lock(&o2hb_live_lock);
14511451 list_del(&reg->hr_all_item);
fs/ocfs2/dlm/dlmconvert.c+29−1
@@ -212,6 +212,12 @@ grant: · 211 unmodified lines
212212 if (lock->lksb->flags & DLM_LKSB_PUT_LVB)
213213 memcpy(res->lvb, lock->lksb->lvb, DLM_LVB_LEN);
214214
215+ /*
216+ * Move the lock to the tail because it may be the only lock which has
217+ * an invalid lvb.
218+ */
219+ list_move_tail(&lock->list, &res->granted);
220+
215221 status = DLM_NORMAL;
216222 *call_ast = 1;
217223 goto unlock_exit;
@@ -262,6 +268,7 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, · 44 unmodified lines
262268 struct dlm_lock *lock, int flags, int type)
263269 {
264270 enum dlm_status status;
271+ u8 old_owner = res->owner;
265272
266273 mlog(0, "type=%d, convert_type=%d, busy=%d\n", lock->ml.type,
267274 lock->ml.convert_type, res->state & DLM_LOCK_RES_IN_PROGRESS);
@@ -287,6 +294,19 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, · 19 unmodified lines
287294 status = DLM_DENIED;
288295 goto bail;
289296 }
297+
298+ if (lock->ml.type == type && lock->ml.convert_type == LKM_IVMODE) {
299+ mlog(0, "last convert request returned DLM_RECOVERING, but "
300+ "owner has already queued and sent ast to me. res %.*s, "
301+ "(cookie=%u:%llu, type=%d, conv=%d)\n",
302+ res->lockname.len, res->lockname.name,
303+ dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)),
304+ dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)),
305+ lock->ml.type, lock->ml.convert_type);
306+ status = DLM_NORMAL;
307+ goto bail;
308+ }
309+
290310 res->state |= DLM_LOCK_RES_IN_PROGRESS;
291311 /* move lock to local convert queue */
292312 /* do not alter lock refcount. switching lists. */
@@ -316,11 +336,19 @@ enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, · 23 unmodified lines
316336 spin_lock(&res->spinlock);
317337 res->state &= ~DLM_LOCK_RES_IN_PROGRESS;
318338 lock->convert_pending = 0;
319− /* if it failed, move it back to granted queue */
339+ /* if it failed, move it back to granted queue.
340+ * if master returns DLM_NORMAL and then down before sending ast,
341+ * it may have already been moved to granted queue, reset to
342+ * DLM_RECOVERING and retry convert */
320343 if (status != DLM_NORMAL) {
321344 if (status != DLM_NOTQUEUED)
322345 dlm_error(status);
323346 dlm_revert_pending_convert(res, lock);
347+ } else if ((res->state & DLM_LOCK_RES_RECOVERING) ||
348+ (old_owner != res->owner)) {
349+ mlog(0, "res %.*s is in recovering or has been recovered.\n",
350+ res->lockname.len, res->lockname.name);
351+ status = DLM_RECOVERING;
324352 }
325353 bail:
326354 spin_unlock(&res->spinlock);
fs/ocfs2/dlm/dlmrecovery.c+0−1
@@ -2083,7 +2083,6 @@ void dlm_move_lockres_to_recovery_list(struct dlm_ctxt *dlm, · 2082 unmodified lines
20832083 dlm_lock_get(lock);
20842084 if (lock->convert_pending) {
20852085 /* move converting lock back to granted */
2086− BUG_ON(i != DLM_CONVERTING_LIST);
20872086 mlog(0, "node died with convert pending "
20882087 "on %.*s. move back to granted list.\n",
20892088 res->lockname.len, res->lockname.name);
fs/ocfs2/file.c+15−150
@@ -1381,44 +1381,6 @@ out: · 1380 unmodified lines
13811381 return ret;
13821382 }
13831383
1384−/*
1385− * Will look for holes and unwritten extents in the range starting at
1386− * pos for count bytes (inclusive).
1387− */
1388−static int ocfs2_check_range_for_holes(struct inode *inode, loff_t pos,
1389− size_t count)
1390−{
1391− int ret = 0;
1392− unsigned int extent_flags;
1393− u32 cpos, clusters, extent_len, phys_cpos;
1394− struct super_block *sb = inode->i_sb;
1395−
1396− cpos = pos >> OCFS2_SB(sb)->s_clustersize_bits;
1397− clusters = ocfs2_clusters_for_bytes(sb, pos + count) - cpos;
1398−
1399− while (clusters) {
1400− ret = ocfs2_get_clusters(inode, cpos, &phys_cpos, &extent_len,
1401− &extent_flags);
1402− if (ret < 0) {
1403− mlog_errno(ret);
1404− goto out;
1405− }
1406−
1407− if (phys_cpos == 0 || (extent_flags & OCFS2_EXT_UNWRITTEN)) {
1408− ret = 1;
1409− break;
1410− }
1411−
1412− if (extent_len > clusters)
1413− extent_len = clusters;
1414−
1415− clusters -= extent_len;
1416− cpos += extent_len;
1417− }
1418−out:
1419− return ret;
1420−}
1421−
14221384 static int ocfs2_write_remove_suid(struct inode *inode)
14231385 {
14241386 int ret;
@@ -2129,18 +2091,12 @@ out: · 704 unmodified lines
21292091
21302092 static int ocfs2_prepare_inode_for_write(struct file *file,
21312093 loff_t pos,
2132− size_t count,
2133− int appending,
2134− int *direct_io,
2135− int *has_refcount)
2094+ size_t count)
21362095 {
21372096 int ret = 0, meta_level = 0;
21382097 struct dentry *dentry = file->f_path.dentry;
21392098 struct inode *inode = d_inode(dentry);
21402099 loff_t end;
2141− struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
2142− int full_coherency = !(osb->s_mount_opt &
2143− OCFS2_MOUNT_COHERENCY_BUFFERED);
21442100
21452101 /*
21462102 * We start with a read level meta lock and only jump to an ex
@@ -2189,10 +2145,6 @@ static int ocfs2_prepare_inode_for_write(struct file *file, · 42 unmodified lines
21892145 pos,
21902146 count,
21912147 &meta_level);
2192− if (has_refcount)
2193− *has_refcount = 1;
2194− if (direct_io)
2195− *direct_io = 0;
21962148 }
21972149
21982150 if (ret < 0) {
@@ -2200,67 +2152,12 @@ static int ocfs2_prepare_inode_for_write(struct file *file, · 1 unmodified line
22002152 goto out_unlock;
22012153 }
22022154
2203− /*
2204− * Skip the O_DIRECT checks if we don't need
2205− * them.
2206− */
2207− if (!direct_io || !(*direct_io))
2208− break;
2209−
2210− /*
2211− * There's no sane way to do direct writes to an inode
2212− * with inline data.
2213− */
2214− if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) {
2215− *direct_io = 0;
2216− break;
2217− }
2218−
2219− /*
2220− * Allowing concurrent direct writes means
2221− * i_size changes wouldn't be synchronized, so
2222− * one node could wind up truncating another
2223− * nodes writes.
2224− */
2225− if (end > i_size_read(inode) && !full_coherency) {
2226− *direct_io = 0;
2227− break;
2228− }
2229−
2230− /*
2231− * Fallback to old way if the feature bit is not set.
2232− */
2233− if (end > i_size_read(inode) &&
2234− !ocfs2_supports_append_dio(osb)) {
2235− *direct_io = 0;
2236− break;
2237− }
2238−
2239− /*
2240− * We don't fill holes during direct io, so
2241− * check for them here. If any are found, the
2242− * caller will have to retake some cluster
2243− * locks and initiate the io as buffered.
2244− */
2245− ret = ocfs2_check_range_for_holes(inode, pos, count);
2246− if (ret == 1) {
2247− /*
2248− * Fallback to old way if the feature bit is not set.
2249− * Otherwise try dio first and then complete the rest
2250− * request through buffer io.
2251− */
2252− if (!ocfs2_supports_append_dio(osb))
2253− *direct_io = 0;
2254− ret = 0;
2255− } else if (ret < 0)
2256− mlog_errno(ret);
22572155 break;
22582156 }
22592157
22602158 out_unlock:
22612159 trace_ocfs2_prepare_inode_for_write(OCFS2_I(inode)->ip_blkno,
2262− pos, appending, count,
2263− direct_io, has_refcount);
2160+ pos, count);
22642161
22652162 if (meta_level >= 0)
22662163 ocfs2_inode_unlock(inode, meta_level);
@@ -2272,18 +2169,16 @@ out: · 5 unmodified lines
22722169 static ssize_t ocfs2_file_write_iter(struct kiocb *iocb,
22732170 struct iov_iter *from)
22742171 {
2275− int direct_io, appending, rw_level;
2276− int can_do_direct, has_refcount = 0;
2172+ int direct_io, rw_level;
22772173 ssize_t written = 0;
22782174 ssize_t ret;
2279− size_t count = iov_iter_count(from), orig_count;
2175+ size_t count = iov_iter_count(from);
22802176 struct file *file = iocb->ki_filp;
22812177 struct inode *inode = file_inode(file);
22822178 struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
22832179 int full_coherency = !(osb->s_mount_opt &
22842180 OCFS2_MOUNT_COHERENCY_BUFFERED);
2285− int unaligned_dio = 0;
2286− int dropped_dio = 0;
2181+ void *saved_ki_complete = NULL;
22872182 int append_write = ((iocb->ki_pos + count) >=
22882183 i_size_read(inode) ? 1 : 0);
22892184
@@ -2296,12 +2191,10 @@ static ssize_t ocfs2_file_write_iter(struct kiocb *iocb, · 6 unmodified lines
22962191 if (count == 0)
22972192 return 0;
22982193
2299− appending = iocb->ki_flags & IOCB_APPEND ? 1 : 0;
23002194 direct_io = iocb->ki_flags & IOCB_DIRECT ? 1 : 0;
23012195
23022196 inode_lock(inode);
23032197
2304−relock:
23052198 /*
23062199 * Concurrent O_DIRECT writes are allowed with
23072200 * mount_option "coherency=buffered".
@@ -2334,7 +2227,6 @@ relock: · 26 unmodified lines
23342227 ocfs2_inode_unlock(inode, 1);
23352228 }
23362229
2337− orig_count = iov_iter_count(from);
23382230 ret = generic_write_checks(iocb, from);
23392231 if (ret <= 0) {
23402232 if (ret)
@@ -2343,41 +2235,18 @@ relock: · 2 unmodified lines
23432235 }
23442236 count = ret;
23452237
2346− can_do_direct = direct_io;
2347− ret = ocfs2_prepare_inode_for_write(file, iocb->ki_pos, count, appending,
2348− &can_do_direct, &has_refcount);
2238+ ret = ocfs2_prepare_inode_for_write(file, iocb->ki_pos, count);
23492239 if (ret < 0) {
23502240 mlog_errno(ret);
23512241 goto out;
23522242 }
23532243
2354− if (direct_io && !is_sync_kiocb(iocb))
2355− unaligned_dio = ocfs2_is_io_unaligned(inode, count, iocb->ki_pos);
2356−
2357− /*
2358− * We can't complete the direct I/O as requested, fall back to
2359− * buffered I/O.
2360− */
2361− if (direct_io && !can_do_direct) {
2362− ocfs2_rw_unlock(inode, rw_level);
2363−
2364− rw_level = -1;
2365−
2366− direct_io = 0;
2367− iocb->ki_flags &= ~IOCB_DIRECT;
2368− iov_iter_reexpand(from, orig_count);
2369− dropped_dio = 1;
2370− goto relock;
2371− }
2372−
2373− if (unaligned_dio) {
2244+ if (direct_io && !is_sync_kiocb(iocb) &&
2245+ ocfs2_is_io_unaligned(inode, count, iocb->ki_pos)) {
23742246 /*
2375− * Wait on previous unaligned aio to complete before
2376− * proceeding.
2247+ * Make it a sync io if it's an unaligned aio.
23772248 */
2378− mutex_lock(&OCFS2_I(inode)->ip_unaligned_aio);
2379− /* Mark the iocb as needing an unlock in ocfs2_dio_end_io */
2380− ocfs2_iocb_set_unaligned_aio(iocb);
2249+ saved_ki_complete = xchg(&iocb->ki_complete, NULL);
23812250 }
23822251
23832252 /* communicate with ocfs2_dio_end_io */
@@ -2398,14 +2267,13 @@ relock: · 14 unmodified lines
23982267 */
23992268 if ((written == -EIOCBQUEUED) || (!ocfs2_iocb_is_rw_locked(iocb))) {
24002269 rw_level = -1;
2401− unaligned_dio = 0;
24022270 }
24032271
24042272 if (unlikely(written <= 0))
2405− goto no_sync;
2273+ goto out;
24062274
24072275 if (((file->f_flags & O_DSYNC) && !direct_io) ||
2408− IS_SYNC(inode) || dropped_dio) {
2276+ IS_SYNC(inode)) {
24092277 ret = filemap_fdatawrite_range(file->f_mapping,
24102278 iocb->ki_pos - written,
24112279 iocb->ki_pos - 1);
@@ -2424,13 +2292,10 @@ relock: · 12 unmodified lines
24242292 iocb->ki_pos - 1);
24252293 }
24262294
2427−no_sync:
2428− if (unaligned_dio && ocfs2_iocb_is_unaligned_aio(iocb)) {
2429− ocfs2_iocb_clear_unaligned_aio(iocb);
2430− mutex_unlock(&OCFS2_I(inode)->ip_unaligned_aio);
2431− }
2432−
24332295 out:
2296+ if (saved_ki_complete)
2297+ xchg(&iocb->ki_complete, saved_ki_complete);
2298+
24342299 if (rw_level != -1)
24352300 ocfs2_rw_unlock(inode, rw_level);
24362301
fs/ocfs2/inode.c+3−0
@@ -1170,6 +1170,9 @@ static void ocfs2_clear_inode(struct inode *inode) · 1169 unmodified lines
11701170 mlog_bug_on_msg(!list_empty(&oi->ip_io_markers),
11711171 "Clear inode of %llu, inode has io markers\n",
11721172 (unsigned long long)oi->ip_blkno);
1173+ mlog_bug_on_msg(!list_empty(&oi->ip_unwritten_list),
1174+ "Clear inode of %llu, inode has unwritten extents\n",
1175+ (unsigned long long)oi->ip_blkno);
11731176
11741177 ocfs2_extent_map_trunc(inode, 0);
11751178
fs/ocfs2/inode.h+3−3
@@ -43,9 +43,6 @@ struct ocfs2_inode_info · 42 unmodified lines
4343 /* protects extended attribute changes on this inode */
4444 struct rw_semaphore ip_xattr_sem;
4545
46− /* Number of outstanding AIO's which are not page aligned */
47− struct mutex ip_unaligned_aio;
48−
4946 /* These fields are protected by ip_lock */
5047 spinlock_t ip_lock;
5148 u32 ip_open_count;
@@ -57,6 +54,9 @@ struct ocfs2_inode_info · 5 unmodified lines
5754 u32 ip_flags; /* see below */
5855 u32 ip_attr; /* inode attributes */
5956
57+ /* Record unwritten extents during direct io. */
58+ struct list_head ip_unwritten_list;
59+
6060 /* protected by recovery_lock. */
6161 struct inode *ip_next_orphan;
6262
fs/ocfs2/journal.c+4−4
@@ -231,7 +231,7 @@ void ocfs2_recovery_exit(struct ocfs2_super *osb) · 230 unmodified lines
231231 /* At this point, we know that no more recovery threads can be
232232 * launched, so wait for any recovery completion work to
233233 * complete. */
234− flush_workqueue(ocfs2_wq);
234+ flush_workqueue(osb->ocfs2_wq);
235235
236236 /*
237237 * Now that recovery is shut down, and the osb is about to be
@@ -1326,7 +1326,7 @@ static void ocfs2_queue_recovery_completion(struct ocfs2_journal *journal, · 1088 unmodified lines
13261326
13271327 spin_lock(&journal->j_lock);
13281328 list_add_tail(&item->lri_list, &journal->j_la_cleanups);
1329− queue_work(ocfs2_wq, &journal->j_recovery_work);
1329+ queue_work(journal->j_osb->ocfs2_wq, &journal->j_recovery_work);
13301330 spin_unlock(&journal->j_lock);
13311331 }
13321332
@@ -1968,7 +1968,7 @@ static void ocfs2_orphan_scan_work(struct work_struct *work) · 635 unmodified lines
19681968 mutex_lock(&os->os_lock);
19691969 ocfs2_queue_orphan_scan(osb);
19701970 if (atomic_read(&os->os_state) == ORPHAN_SCAN_ACTIVE)
1971− queue_delayed_work(ocfs2_wq, &os->os_orphan_scan_work,
1971+ queue_delayed_work(osb->ocfs2_wq, &os->os_orphan_scan_work,
19721972 ocfs2_orphan_scan_timeout());
19731973 mutex_unlock(&os->os_lock);
19741974 }
@@ -2008,7 +2008,7 @@ void ocfs2_orphan_scan_start(struct ocfs2_super *osb) · 33 unmodified lines
20082008 atomic_set(&os->os_state, ORPHAN_SCAN_INACTIVE);
20092009 else {
20102010 atomic_set(&os->os_state, ORPHAN_SCAN_ACTIVE);
2011− queue_delayed_work(ocfs2_wq, &os->os_orphan_scan_work,
2011+ queue_delayed_work(osb->ocfs2_wq, &os->os_orphan_scan_work,
20122012 ocfs2_orphan_scan_timeout());
20132013 }
20142014 }
fs/ocfs2/localalloc.c+2−2
@@ -386,7 +386,7 @@ void ocfs2_shutdown_local_alloc(struct ocfs2_super *osb) · 385 unmodified lines
386386 struct ocfs2_dinode *alloc = NULL;
387387
388388 cancel_delayed_work(&osb->la_enable_wq);
389− flush_workqueue(ocfs2_wq);
389+ flush_workqueue(osb->ocfs2_wq);
390390
391391 if (osb->local_alloc_state == OCFS2_LA_UNUSED)
392392 goto out;
@@ -1085,7 +1085,7 @@ static int ocfs2_recalc_la_window(struct ocfs2_super *osb, · 692 unmodified lines
10851085 } else {
10861086 osb->local_alloc_state = OCFS2_LA_DISABLED;
10871087 }
1088− queue_delayed_work(ocfs2_wq, &osb->la_enable_wq,
1088+ queue_delayed_work(osb->ocfs2_wq, &osb->la_enable_wq,
10891089 OCFS2_LA_ENABLE_INTERVAL);
10901090 goto out_unlock;
10911091 }
fs/ocfs2/mmap.c+2−2
@@ -104,8 +104,8 @@ static int __ocfs2_page_mkwrite(struct file *file, struct buffer_head *di_bh, · 103 unmodified lines
104104 if (page->index == last_index)
105105 len = ((size - 1) & ~PAGE_CACHE_MASK) + 1;
106106
107− ret = ocfs2_write_begin_nolock(file, mapping, pos, len, 0, &locked_page,
108− &fsdata, di_bh, page);
107+ ret = ocfs2_write_begin_nolock(mapping, pos, len, OCFS2_WRITE_MMAP,
108+ &locked_page, &fsdata, di_bh, page);
109109 if (ret) {
110110 if (ret != -ENOSPC)
111111 mlog_errno(ret);
fs/ocfs2/ocfs2.h+8−0
@@ -464,6 +464,14 @@ struct ocfs2_super · 463 unmodified lines
464464 struct ocfs2_refcount_tree *osb_ref_tree_lru;
465465
466466 struct mutex system_file_mutex;
467+
468+ /*
469+ * OCFS2 needs to schedule several different types of work which
470+ * require cluster locking, disk I/O, recovery waits, etc. Since these
471+ * types of work tend to be heavy we avoid using the kernel events
472+ * workqueue and schedule on our own.
473+ */
474+ struct workqueue_struct *ocfs2_wq;
467475 };
468476
469477 #define OCFS2_SB(sb) ((struct ocfs2_super *)(sb)->s_fs_info)
fs/ocfs2/ocfs2_trace.h+4−12
@@ -1450,28 +1450,20 @@ DEFINE_OCFS2_ULL_ULL_ULL_EVENT(ocfs2_remove_inode_range); · 1449 unmodified lines
14501450
14511451 TRACE_EVENT(ocfs2_prepare_inode_for_write,
14521452 TP_PROTO(unsigned long long ino, unsigned long long saved_pos,
1453− int appending, unsigned long count,
1454− int *direct_io, int *has_refcount),
1455− TP_ARGS(ino, saved_pos, appending, count, direct_io, has_refcount),
1453+ unsigned long count),
1454+ TP_ARGS(ino, saved_pos, count),
14561455 TP_STRUCT__entry(
14571456 __field(unsigned long long, ino)
14581457 __field(unsigned long long, saved_pos)
1459− __field(int, appending)
14601458 __field(unsigned long, count)
1461− __field(int, direct_io)
1462− __field(int, has_refcount)
14631459 ),
14641460 TP_fast_assign(
14651461 __entry->ino = ino;
14661462 __entry->saved_pos = saved_pos;
1467− __entry->appending = appending;
14681463 __entry->count = count;
1469− __entry->direct_io = direct_io ? *direct_io : -1;
1470− __entry->has_refcount = has_refcount ? *has_refcount : -1;
14711464 ),
1472− TP_printk("%llu %llu %d %lu %d %d", __entry->ino,
1473− __entry->saved_pos, __entry->appending, __entry->count,
1474− __entry->direct_io, __entry->has_refcount)
1465+ TP_printk("%llu %llu %lu", __entry->ino,
1466+ __entry->saved_pos, __entry->count)
14751467 );
14761468
14771469 DEFINE_OCFS2_INT_EVENT(generic_file_aio_read_ret);
fs/ocfs2/quota_global.c+1−1
@@ -726,7 +726,7 @@ static int ocfs2_release_dquot(struct dquot *dquot) · 725 unmodified lines
726726 dqgrab(dquot);
727727 /* First entry on list -> queue work */
728728 if (llist_add(&OCFS2_DQUOT(dquot)->list, &osb->dquot_drop_list))
729− queue_work(ocfs2_wq, &osb->dquot_drop_work);
729+ queue_work(osb->ocfs2_wq, &osb->dquot_drop_work);
730730 goto out;
731731 }
732732 status = ocfs2_lock_global_qf(oinfo, 1);
fs/ocfs2/resize.c+1−1
@@ -196,7 +196,7 @@ static int update_backups(struct inode * inode, u32 clusters, char *data) · 195 unmodified lines
196196 for (i = 0; i < OCFS2_MAX_BACKUP_SUPERBLOCKS; i++) {
197197 blkno = ocfs2_backup_super_blkno(inode->i_sb, i);
198198 cluster = ocfs2_blocks_to_clusters(inode->i_sb, blkno);
199− if (cluster > clusters)
199+ if (cluster >= clusters)
200200 break;
201201
202202 ret = ocfs2_read_blocks_sync(osb, blkno, 1, &backup);
fs/ocfs2/super.c+16−23
@@ -80,12 +80,6 @@ static struct kmem_cache *ocfs2_inode_cachep; · 79 unmodified lines
8080 struct kmem_cache *ocfs2_dquot_cachep;
8181 struct kmem_cache *ocfs2_qf_chunk_cachep;
8282
83−/* OCFS2 needs to schedule several different types of work which
84− * require cluster locking, disk I/O, recovery waits, etc. Since these
85− * types of work tend to be heavy we avoid using the kernel events
86− * workqueue and schedule on our own. */
87−struct workqueue_struct *ocfs2_wq = NULL;
88−
8983 static struct dentry *ocfs2_debugfs_root;
9084
9185 MODULE_AUTHOR("Oracle");
@@ -1613,33 +1607,25 @@ static int __init ocfs2_init(void) · 1521 unmodified lines
16131607 if (status < 0)
16141608 goto out2;
16151609
1616− ocfs2_wq = create_singlethread_workqueue("ocfs2_wq");
1617− if (!ocfs2_wq) {
1618− status = -ENOMEM;
1619− goto out3;
1620− }
1621−
16221610 ocfs2_debugfs_root = debugfs_create_dir("ocfs2", NULL);
16231611 if (!ocfs2_debugfs_root) {
16241612 status = -ENOMEM;
16251613 mlog(ML_ERROR, "Unable to create ocfs2 debugfs root.\n");
1626− goto out4;
1614+ goto out3;
16271615 }
16281616
16291617 ocfs2_set_locking_protocol();
16301618
16311619 status = register_quota_format(&ocfs2_quota_format);
16321620 if (status < 0)
1633− goto out4;
1621+ goto out3;
16341622 status = register_filesystem(&ocfs2_fs_type);
16351623 if (!status)
16361624 return 0;
16371625
16381626 unregister_quota_format(&ocfs2_quota_format);
1639−out4:
1640− destroy_workqueue(ocfs2_wq);
1641− debugfs_remove(ocfs2_debugfs_root);
16421627 out3:
1628+ debugfs_remove(ocfs2_debugfs_root);
16431629 ocfs2_free_mem_caches();
16441630 out2:
16451631 exit_ocfs2_uptodate_cache();
@@ -1650,11 +1636,6 @@ out1: · 4 unmodified lines
16501636
16511637 static void __exit ocfs2_exit(void)
16521638 {
1653− if (ocfs2_wq) {
1654− flush_workqueue(ocfs2_wq);
1655− destroy_workqueue(ocfs2_wq);
1656− }
1657−
16581639 unregister_quota_format(&ocfs2_quota_format);
16591640
16601641 debugfs_remove(ocfs2_debugfs_root);
@@ -1745,8 +1726,8 @@ static void ocfs2_inode_init_once(void *data) · 84 unmodified lines
17451726 spin_lock_init(&oi->ip_lock);
17461727 ocfs2_extent_map_init(&oi->vfs_inode);
17471728 INIT_LIST_HEAD(&oi->ip_io_markers);
1729+ INIT_LIST_HEAD(&oi->ip_unwritten_list);
17481730 oi->ip_dir_start_lookup = 0;
1749− mutex_init(&oi->ip_unaligned_aio);
17501731 init_rwsem(&oi->ip_alloc_sem);
17511732 init_rwsem(&oi->ip_xattr_sem);
17521733 mutex_init(&oi->ip_io_mutex);
@@ -2349,6 +2330,12 @@ static int ocfs2_initialize_super(struct super_block *sb, · 596 unmodified lines
23492330 }
23502331 cleancache_init_shared_fs(sb);
23512332
2333+ osb->ocfs2_wq = create_singlethread_workqueue("ocfs2_wq");
2334+ if (!osb->ocfs2_wq) {
2335+ status = -ENOMEM;
2336+ mlog_errno(status);
2337+ }
2338+
23522339 bail:
23532340 return status;
23542341 }
@@ -2536,6 +2523,12 @@ static void ocfs2_delete_osb(struct ocfs2_super *osb) · 181 unmodified lines
25362523 {
25372524 /* This function assumes that the caller has the main osb resource */
25382525
2526+ /* ocfs2_initializer_super have already created this workqueue */
2527+ if (osb->ocfs2_wq) {
2528+ flush_workqueue(osb->ocfs2_wq);
2529+ destroy_workqueue(osb->ocfs2_wq);
2530+ }
2531+
25392532 ocfs2_free_slot_info(osb);
25402533
25412534 kfree(osb->osb_orphan_wipes);
fs/ocfs2/super.h+0−2
@@ -26,8 +26,6 @@ · 25 unmodified lines
2626 #ifndef OCFS2_SUPER_H
2727 #define OCFS2_SUPER_H
2828
29−extern struct workqueue_struct *ocfs2_wq;
30−
3129 int ocfs2_publish_get_mount_state(struct ocfs2_super *osb,
3230 int node_num);
3331
include/asm-generic/vmlinux.lds.h+11−1
@@ -456,7 +456,7 @@ · 455 unmodified lines
456456 *(.entry.text) \
457457 VMLINUX_SYMBOL(__entry_text_end) = .;
458458
459−#ifdef CONFIG_FUNCTION_GRAPH_TRACER
459+#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
460460 #define IRQENTRY_TEXT \
461461 ALIGN_FUNCTION(); \
462462 VMLINUX_SYMBOL(__irqentry_text_start) = .; \
@@ -466,6 +466,16 @@ · 3 unmodified lines
466466 #define IRQENTRY_TEXT
467467 #endif
468468
469+#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
470+#define SOFTIRQENTRY_TEXT \
471+ ALIGN_FUNCTION(); \
472+ VMLINUX_SYMBOL(__softirqentry_text_start) = .; \
473+ *(.softirqentry.text) \
474+ VMLINUX_SYMBOL(__softirqentry_text_end) = .;
475+#else
476+#define SOFTIRQENTRY_TEXT
477+#endif
478+
469479 /* Section used for early init (in .S files) */
470480 #define HEAD_TEXT *(.head.text)
471481
include/linux/ftrace.h+0−11
@@ -811,16 +811,6 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth, · 810 unmodified lines
811811 */
812812 #define __notrace_funcgraph notrace
813813
814−/*
815− * We want to which function is an entrypoint of a hardirq.
816− * That will help us to put a signal on output.
817− */
818−#define __irq_entry __attribute__((__section__(".irqentry.text")))
819−
820−/* Limits of hardirq entrypoints */
821−extern char __irqentry_text_start[];
822−extern char __irqentry_text_end[];
823−
824814 #define FTRACE_NOTRACE_DEPTH 65536
825815 #define FTRACE_RETFUNC_DEPTH 50
826816 #define FTRACE_RETSTACK_ALLOC_SIZE 32
@@ -857,7 +847,6 @@ static inline void unpause_graph_tracing(void) · 30 unmodified lines
857847 #else /* !CONFIG_FUNCTION_GRAPH_TRACER */
858848
859849 #define __notrace_funcgraph
860−#define __irq_entry
861850 #define INIT_FTRACE_GRAPH
862851
863852 static inline void ftrace_graph_init_task(struct task_struct *t) { }
include/linux/interrupt.h+20−0
@@ -683,4 +683,24 @@ extern int early_irq_init(void); · 682 unmodified lines
683683 extern int arch_probe_nr_irqs(void);
684684 extern int arch_early_irq_init(void);
685685
686+#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN)
687+/*
688+ * We want to know which function is an entrypoint of a hardirq or a softirq.
689+ */
690+#define __irq_entry __attribute__((__section__(".irqentry.text")))
691+#define __softirq_entry \
692+ __attribute__((__section__(".softirqentry.text")))
693+
694+/* Limits of hardirq entrypoints */
695+extern char __irqentry_text_start[];
696+extern char __irqentry_text_end[];
697+/* Limits of softirq entrypoints */
698+extern char __softirqentry_text_start[];
699+extern char __softirqentry_text_end[];
700+
701+#else
702+#define __irq_entry
703+#define __softirq_entry
704+#endif
705+
686706 #endif
include/linux/kasan.h+23−8
@@ -48,19 +48,28 @@ void kasan_unpoison_task_stack(struct task_struct *task); · 47 unmodified lines
4848 void kasan_alloc_pages(struct page *page, unsigned int order);
4949 void kasan_free_pages(struct page *page, unsigned int order);
5050
51+void kasan_cache_create(struct kmem_cache *cache, size_t *size,
52+ unsigned long *flags);
53+
5154 void kasan_poison_slab(struct page *page);
5255 void kasan_unpoison_object_data(struct kmem_cache *cache, void *object);
5356 void kasan_poison_object_data(struct kmem_cache *cache, void *object);
5457
55−void kasan_kmalloc_large(const void *ptr, size_t size);
58+void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags);
5659 void kasan_kfree_large(const void *ptr);
5760 void kasan_kfree(void *ptr);
58−void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size);
59−void kasan_krealloc(const void *object, size_t new_size);
61+void kasan_kmalloc(struct kmem_cache *s, const void *object, size_t size,
62+ gfp_t flags);
63+void kasan_krealloc(const void *object, size_t new_size, gfp_t flags);
6064
61−void kasan_slab_alloc(struct kmem_cache *s, void *object);
65+void kasan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags);
6266 void kasan_slab_free(struct kmem_cache *s, void *object);
6367
68+struct kasan_cache {
69+ int alloc_meta_offset;
70+ int free_meta_offset;
71+};
72+
6473 int kasan_module_alloc(void *addr, size_t size);
6574 void kasan_free_shadow(const struct vm_struct *vm);
6675
@@ -76,20 +85,26 @@ static inline void kasan_disable_current(void) {} · 9 unmodified lines
7685 static inline void kasan_alloc_pages(struct page *page, unsigned int order) {}
7786 static inline void kasan_free_pages(struct page *page, unsigned int order) {}
7887
88+static inline void kasan_cache_create(struct kmem_cache *cache,
89+ size_t *size,
90+ unsigned long *flags) {}
91+
7992 static inline void kasan_poison_slab(struct page *page) {}
8093 static inline void kasan_unpoison_object_data(struct kmem_cache *cache,
8194 void *object) {}
8295 static inline void kasan_poison_object_data(struct kmem_cache *cache,
8396 void *object) {}
8497
85−static inline void kasan_kmalloc_large(void *ptr, size_t size) {}
98+static inline void kasan_kmalloc_large(void *ptr, size_t size, gfp_t flags) {}
8699 static inline void kasan_kfree_large(const void *ptr) {}
87100 static inline void kasan_kfree(void *ptr) {}
88101 static inline void kasan_kmalloc(struct kmem_cache *s, const void *object,
89− size_t size) {}
90−static inline void kasan_krealloc(const void *object, size_t new_size) {}
102+ size_t size, gfp_t flags) {}
103+static inline void kasan_krealloc(const void *object, size_t new_size,
104+ gfp_t flags) {}
91105
92−static inline void kasan_slab_alloc(struct kmem_cache *s, void *object) {}
106+static inline void kasan_slab_alloc(struct kmem_cache *s, void *object,
107+ gfp_t flags) {}
93108 static inline void kasan_slab_free(struct kmem_cache *s, void *object) {}
94109
95110 static inline int kasan_module_alloc(void *addr, size_t size) { return 0; }
include/linux/mm.h+2−0
@@ -1132,6 +1132,8 @@ struct zap_details { · 1131 unmodified lines
11321132 struct address_space *check_mapping; /* Check page->mapping if set */
11331133 pgoff_t first_index; /* Lowest page->index to unmap */
11341134 pgoff_t last_index; /* Highest page->index to unmap */
1135+ bool ignore_dirty; /* Ignore dirty pages */
1136+ bool check_swap_entries; /* Check also swap entries */
11351137 };
11361138
11371139 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
include/linux/oom.h+1−3
@@ -76,8 +76,6 @@ extern unsigned long oom_badness(struct task_struct *p, · 75 unmodified lines
7676 struct mem_cgroup *memcg, const nodemask_t *nodemask,
7777 unsigned long totalpages);
7878
79−extern int oom_kills_count(void);
80−extern void note_oom_kill(void);
8179 extern void oom_kill_process(struct oom_control *oc, struct task_struct *p,
8280 unsigned int points, unsigned long totalpages,
8381 struct mem_cgroup *memcg, const char *message);
@@ -91,7 +89,7 @@ extern enum oom_scan_t oom_scan_process_thread(struct oom_control *oc, · 7 unmodified lines
9189
9290 extern bool out_of_memory(struct oom_control *oc);
9391
94−extern void exit_oom_victim(void);
92+extern void exit_oom_victim(struct task_struct *tsk);
9593
9694 extern int register_oom_notifier(struct notifier_block *nb);
9795 extern int unregister_oom_notifier(struct notifier_block *nb);
include/linux/sched.h+4−0
@@ -426,6 +426,7 @@ extern signed long schedule_timeout(signed long timeout); · 425 unmodified lines
426426 extern signed long schedule_timeout_interruptible(signed long timeout);
427427 extern signed long schedule_timeout_killable(signed long timeout);
428428 extern signed long schedule_timeout_uninterruptible(signed long timeout);
429+extern signed long schedule_timeout_idle(signed long timeout);
429430 asmlinkage void schedule(void);
430431 extern void schedule_preempt_disabled(void);
431432
@@ -1848,6 +1849,9 @@ struct task_struct { · 1416 unmodified lines
18481849 unsigned long task_state_change;
18491850 #endif
18501851 int pagefault_disabled;
1852+#ifdef CONFIG_MMU
1853+ struct task_struct *oom_reaper_list;
1854+#endif
18511855 /* CPU-specific state of this task */
18521856 struct thread_struct thread;
18531857 /*
include/linux/slab.h+8−2
@@ -92,6 +92,12 @@ · 91 unmodified lines
9292 # define SLAB_ACCOUNT 0x00000000UL
9393 #endif
9494
95+#ifdef CONFIG_KASAN
96+#define SLAB_KASAN 0x08000000UL
97+#else
98+#define SLAB_KASAN 0x00000000UL
99+#endif
100+
95101 /* The following flags affect the page allocator grouping pages by mobility */
96102 #define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */
97103 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
@@ -370,7 +376,7 @@ static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s, · 272 unmodified lines
370376 {
371377 void *ret = kmem_cache_alloc(s, flags);
372378
373− kasan_kmalloc(s, ret, size);
379+ kasan_kmalloc(s, ret, size, flags);
374380 return ret;
375381 }
376382
@@ -381,7 +387,7 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, · 4 unmodified lines
381387 {
382388 void *ret = kmem_cache_alloc_node(s, gfpflags, node);
383389
384− kasan_kmalloc(s, ret, size);
390+ kasan_kmalloc(s, ret, size, gfpflags);
385391 return ret;
386392 }
387393 #endif /* CONFIG_TRACING */
include/linux/slab_def.h+14−0
@@ -76,8 +76,22 @@ struct kmem_cache { · 75 unmodified lines
7676 #ifdef CONFIG_MEMCG
7777 struct memcg_cache_params memcg_params;
7878 #endif
79+#ifdef CONFIG_KASAN
80+ struct kasan_cache kasan_info;
81+#endif
7982
8083 struct kmem_cache_node *node[MAX_NUMNODES];
8184 };
8285
86+static inline void *nearest_obj(struct kmem_cache *cache, struct page *page,
87+ void *x) {
88+ void *object = x - (x - page->s_mem) % cache->size;
89+ void *last_object = page->s_mem + (cache->num - 1) * cache->size;
90+
91+ if (unlikely(object > last_object))
92+ return last_object;
93+ else
94+ return object;
95+}
96+
8397 #endif /* _LINUX_SLAB_DEF_H */
include/linux/slub_def.h+11−0
@@ -130,4 +130,15 @@ static inline void *virt_to_obj(struct kmem_cache *s, · 129 unmodified lines
130130 void object_err(struct kmem_cache *s, struct page *page,
131131 u8 *object, char *reason);
132132
133+static inline void *nearest_obj(struct kmem_cache *cache, struct page *page,
134+ void *x) {
135+ void *object = x - (x - page_address(page)) % cache->size;
136+ void *last_object = page_address(page) +
137+ (page->objects - 1) * cache->size;
138+ if (unlikely(object > last_object))
139+ return last_object;
140+ else
141+ return object;
142+}
143+
133144 #endif /* _LINUX_SLUB_DEF_H */
include/linux/stackdepot.h+32−0
@@ -0,0 +1,32 @@
1+/*
2+ * A generic stack depot implementation
3+ *
4+ * Author: Alexander Potapenko <glider@google.com>
5+ * Copyright (C) 2016 Google, Inc.
6+ *
7+ * Based on code by Dmitry Chernenkov.
8+ *
9+ * This program is free software; you can redistribute it and/or modify
10+ * it under the terms of the GNU General Public License as published by
11+ * the Free Software Foundation; either version 2 of the License, or
12+ * (at your option) any later version.
13+ *
14+ * This program is distributed in the hope that it will be useful,
15+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
16+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+ * GNU General Public License for more details.
18+ *
19+ */
20+
21+#ifndef _LINUX_STACKDEPOT_H
22+#define _LINUX_STACKDEPOT_H
23+
24+typedef u32 depot_stack_handle_t;
25+
26+struct stack_trace;
27+
28+depot_stack_handle_t depot_save_stack(struct stack_trace *trace, gfp_t flags);
29+
30+void depot_fetch_stack(depot_stack_handle_t handle, struct stack_trace *trace);
31+
32+#endif
kernel/exit.c+1−1
@@ -435,7 +435,7 @@ static void exit_mm(struct task_struct *tsk) · 434 unmodified lines
435435 mm_update_next_owner(mm);
436436 mmput(mm);
437437 if (test_thread_flag(TIF_MEMDIE))
438− exit_oom_victim();
438+ exit_oom_victim(tsk);
439439 }
440440
441441 static struct task_struct *find_alive_thread(struct task_struct *p)
kernel/softirq.c+1−1
@@ -227,7 +227,7 @@ static inline bool lockdep_softirq_start(void) { return false; } · 226 unmodified lines
227227 static inline void lockdep_softirq_end(bool in_hardirq) { }
228228 #endif
229229
230−asmlinkage __visible void __do_softirq(void)
230+asmlinkage __visible void __softirq_entry __do_softirq(void)
231231 {
232232 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
233233 unsigned long old_flags = current->flags;
kernel/time/timer.c+11−0
@@ -1566,6 +1566,17 @@ signed long __sched schedule_timeout_uninterruptible(signed long timeout) · 1565 unmodified lines
15661566 }
15671567 EXPORT_SYMBOL(schedule_timeout_uninterruptible);
15681568
1569+/*
1570+ * Like schedule_timeout_uninterruptible(), except this task will not contribute
1571+ * to load average.
1572+ */
1573+signed long __sched schedule_timeout_idle(signed long timeout)
1574+{
1575+ __set_current_state(TASK_IDLE);
1576+ return schedule_timeout(timeout);
1577+}
1578+EXPORT_SYMBOL(schedule_timeout_idle);
1579+
15691580 #ifdef CONFIG_HOTPLUG_CPU
15701581 static void migrate_timer_list(struct tvec_base *new_base, struct hlist_head *head)
15711582 {
kernel/trace/trace_functions_graph.c+1−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 */
99 #include <linux/uaccess.h>
1010 #include <linux/ftrace.h>
11+#include <linux/interrupt.h>
1112 #include <linux/slab.h>
1213 #include <linux/fs.h>
1314
lib/Kconfig+4−0
@@ -536,4 +536,8 @@ config ARCH_HAS_PMEM_API · 535 unmodified lines
536536 config ARCH_HAS_MMIO_FLUSH
537537 bool
538538
539+config STACKDEPOT
540+ bool
541+ select STACKTRACE
542+
539543 endmenu
lib/Kconfig.kasan+4−1
@@ -5,8 +5,9 @@ if HAVE_ARCH_KASAN · 4 unmodified lines
55
66 config KASAN
77 bool "KASan: runtime memory debugger"
8− depends on SLUB_DEBUG
8+ depends on SLUB_DEBUG || (SLAB && !DEBUG_SLAB)
99 select CONSTRUCTORS
10+ select STACKDEPOT if SLAB
1011 help
1112 Enables kernel address sanitizer - runtime memory debugger,
1213 designed to find out-of-bounds accesses and use-after-free bugs.
@@ -16,6 +17,8 @@ config KASAN · 3 unmodified lines
1617 This feature consumes about 1/8 of available memory and brings about
1718 ~x3 performance slowdown.
1819 For better error detection enable CONFIG_STACKTRACE.
20+ Currently CONFIG_KASAN doesn't work with CONFIG_DEBUG_SLAB
21+ (the resulting kernel does not boot).
1922
2023 choice
2124 prompt "Instrumentation type"
lib/Makefile+3−0
@@ -181,6 +181,9 @@ obj-$(CONFIG_SG_SPLIT) += sg_split.o · 180 unmodified lines
181181 obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
182182 obj-$(CONFIG_IRQ_POLL) += irq_poll.o
183183
184+obj-$(CONFIG_STACKDEPOT) += stackdepot.o
185+KASAN_SANITIZE_stackdepot.o := n
186+
184187 libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \
185188 fdt_empty_tree.o
186189 $(foreach file, $(libfdt_files), \
lib/stackdepot.c+284−0
@@ -0,0 +1,284 @@
1+/*
2+ * Generic stack depot for storing stack traces.
3+ *
4+ * Some debugging tools need to save stack traces of certain events which can
5+ * be later presented to the user. For example, KASAN needs to safe alloc and
6+ * free stacks for each object, but storing two stack traces per object
7+ * requires too much memory (e.g. SLUB_DEBUG needs 256 bytes per object for
8+ * that).
9+ *
10+ * Instead, stack depot maintains a hashtable of unique stacktraces. Since alloc
11+ * and free stacks repeat a lot, we save about 100x space.
12+ * Stacks are never removed from depot, so we store them contiguously one after
13+ * another in a contiguos memory allocation.
14+ *
15+ * Author: Alexander Potapenko <glider@google.com>
16+ * Copyright (C) 2016 Google, Inc.
17+ *
18+ * Based on code by Dmitry Chernenkov.
19+ *
20+ * This program is free software; you can redistribute it and/or
21+ * modify it under the terms of the GNU General Public License
22+ * version 2 as published by the Free Software Foundation.
23+ *
24+ * This program is distributed in the hope that it will be useful, but
25+ * WITHOUT ANY WARRANTY; without even the implied warranty of
26+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
27+ * General Public License for more details.
28+ *
29+ */
30+
31+#include <linux/gfp.h>
32+#include <linux/jhash.h>
33+#include <linux/kernel.h>
34+#include <linux/mm.h>
35+#include <linux/percpu.h>
36+#include <linux/printk.h>
37+#include <linux/slab.h>
38+#include <linux/stacktrace.h>
39+#include <linux/stackdepot.h>
40+#include <linux/string.h>
41+#include <linux/types.h>
42+
43+#define DEPOT_STACK_BITS (sizeof(depot_stack_handle_t) * 8)
44+
45+#define STACK_ALLOC_ORDER 2 /* 'Slab' size order for stack depot, 4 pages */
46+#define STACK_ALLOC_SIZE (1LL << (PAGE_SHIFT + STACK_ALLOC_ORDER))
47+#define STACK_ALLOC_ALIGN 4
48+#define STACK_ALLOC_OFFSET_BITS (STACK_ALLOC_ORDER + PAGE_SHIFT - \
49+ STACK_ALLOC_ALIGN)
50+#define STACK_ALLOC_INDEX_BITS (DEPOT_STACK_BITS - STACK_ALLOC_OFFSET_BITS)
51+#define STACK_ALLOC_SLABS_CAP 1024
52+#define STACK_ALLOC_MAX_SLABS \
53+ (((1LL << (STACK_ALLOC_INDEX_BITS)) < STACK_ALLOC_SLABS_CAP) ? \
54+ (1LL << (STACK_ALLOC_INDEX_BITS)) : STACK_ALLOC_SLABS_CAP)
55+
56+/* The compact structure to store the reference to stacks. */
57+union handle_parts {
58+ depot_stack_handle_t handle;
59+ struct {
60+ u32 slabindex : STACK_ALLOC_INDEX_BITS;
61+ u32 offset : STACK_ALLOC_OFFSET_BITS;
62+ };
63+};
64+
65+struct stack_record {
66+ struct stack_record *next; /* Link in the hashtable */
67+ u32 hash; /* Hash in the hastable */
68+ u32 size; /* Number of frames in the stack */
69+ union handle_parts handle;
70+ unsigned long entries[1]; /* Variable-sized array of entries. */
71+};
72+
73+static void *stack_slabs[STACK_ALLOC_MAX_SLABS];
74+
75+static int depot_index;
76+static int next_slab_inited;
77+static size_t depot_offset;
78+static DEFINE_SPINLOCK(depot_lock);
79+
80+static bool init_stack_slab(void **prealloc)
81+{
82+ if (!*prealloc)
83+ return false;
84+ /*
85+ * This smp_load_acquire() pairs with smp_store_release() to
86+ * |next_slab_inited| below and in depot_alloc_stack().
87+ */
88+ if (smp_load_acquire(&next_slab_inited))
89+ return true;
90+ if (stack_slabs[depot_index] == NULL) {
91+ stack_slabs[depot_index] = *prealloc;
92+ } else {
93+ stack_slabs[depot_index + 1] = *prealloc;
94+ /*
95+ * This smp_store_release pairs with smp_load_acquire() from
96+ * |next_slab_inited| above and in depot_save_stack().
97+ */
98+ smp_store_release(&next_slab_inited, 1);
99+ }
100+ *prealloc = NULL;
101+ return true;
102+}
103+
104+/* Allocation of a new stack in raw storage */
105+static struct stack_record *depot_alloc_stack(unsigned long *entries, int size,
106+ u32 hash, void **prealloc, gfp_t alloc_flags)
107+{
108+ int required_size = offsetof(struct stack_record, entries) +
109+ sizeof(unsigned long) * size;
110+ struct stack_record *stack;
111+
112+ required_size = ALIGN(required_size, 1 << STACK_ALLOC_ALIGN);
113+
114+ if (unlikely(depot_offset + required_size > STACK_ALLOC_SIZE)) {
115+ if (unlikely(depot_index + 1 >= STACK_ALLOC_MAX_SLABS)) {
116+ WARN_ONCE(1, "Stack depot reached limit capacity");
117+ return NULL;
118+ }
119+ depot_index++;
120+ depot_offset = 0;
121+ /*
122+ * smp_store_release() here pairs with smp_load_acquire() from
123+ * |next_slab_inited| in depot_save_stack() and
124+ * init_stack_slab().
125+ */
126+ if (depot_index + 1 < STACK_ALLOC_MAX_SLABS)
127+ smp_store_release(&next_slab_inited, 0);
128+ }
129+ init_stack_slab(prealloc);
130+ if (stack_slabs[depot_index] == NULL)
131+ return NULL;
132+
133+ stack = stack_slabs[depot_index] + depot_offset;
134+
135+ stack->hash = hash;
136+ stack->size = size;
137+ stack->handle.slabindex = depot_index;
138+ stack->handle.offset = depot_offset >> STACK_ALLOC_ALIGN;
139+ memcpy(stack->entries, entries, size * sizeof(unsigned long));
140+ depot_offset += required_size;
141+
142+ return stack;
143+}
144+
145+#define STACK_HASH_ORDER 20
146+#define STACK_HASH_SIZE (1L << STACK_HASH_ORDER)
147+#define STACK_HASH_MASK (STACK_HASH_SIZE - 1)
148+#define STACK_HASH_SEED 0x9747b28c
149+
150+static struct stack_record *stack_table[STACK_HASH_SIZE] = {
151+ [0 ... STACK_HASH_SIZE - 1] = NULL
152+};
153+
154+/* Calculate hash for a stack */
155+static inline u32 hash_stack(unsigned long *entries, unsigned int size)
156+{
157+ return jhash2((u32 *)entries,
158+ size * sizeof(unsigned long) / sizeof(u32),
159+ STACK_HASH_SEED);
160+}
161+
162+/* Find a stack that is equal to the one stored in entries in the hash */
163+static inline struct stack_record *find_stack(struct stack_record *bucket,
164+ unsigned long *entries, int size,
165+ u32 hash)
166+{
167+ struct stack_record *found;
168+
169+ for (found = bucket; found; found = found->next) {
170+ if (found->hash == hash &&
171+ found->size == size &&
172+ !memcmp(entries, found->entries,
173+ size * sizeof(unsigned long))) {
174+ return found;
175+ }
176+ }
177+ return NULL;
178+}
179+
180+void depot_fetch_stack(depot_stack_handle_t handle, struct stack_trace *trace)
181+{
182+ union handle_parts parts = { .handle = handle };
183+ void *slab = stack_slabs[parts.slabindex];
184+ size_t offset = parts.offset << STACK_ALLOC_ALIGN;
185+ struct stack_record *stack = slab + offset;
186+
187+ trace->nr_entries = trace->max_entries = stack->size;
188+ trace->entries = stack->entries;
189+ trace->skip = 0;
190+}
191+
192+/**
193+ * depot_save_stack - save stack in a stack depot.
194+ * @trace - the stacktrace to save.
195+ * @alloc_flags - flags for allocating additional memory if required.
196+ *
197+ * Returns the handle of the stack struct stored in depot.
198+ */
199+depot_stack_handle_t depot_save_stack(struct stack_trace *trace,
200+ gfp_t alloc_flags)
201+{
202+ u32 hash;
203+ depot_stack_handle_t retval = 0;
204+ struct stack_record *found = NULL, **bucket;
205+ unsigned long flags;
206+ struct page *page = NULL;
207+ void *prealloc = NULL;
208+
209+ if (unlikely(trace->nr_entries == 0))
210+ goto fast_exit;
211+
212+ hash = hash_stack(trace->entries, trace->nr_entries);
213+ /* Bad luck, we won't store this stack. */
214+ if (hash == 0)
215+ goto exit;
216+
217+ bucket = &stack_table[hash & STACK_HASH_MASK];
218+
219+ /*
220+ * Fast path: look the stack trace up without locking.
221+ * The smp_load_acquire() here pairs with smp_store_release() to
222+ * |bucket| below.
223+ */
224+ found = find_stack(smp_load_acquire(bucket), trace->entries,
225+ trace->nr_entries, hash);
226+ if (found)
227+ goto exit;
228+
229+ /*
230+ * Check if the current or the next stack slab need to be initialized.
231+ * If so, allocate the memory - we won't be able to do that under the
232+ * lock.
233+ *
234+ * The smp_load_acquire() here pairs with smp_store_release() to
235+ * |next_slab_inited| in depot_alloc_stack() and init_stack_slab().
236+ */
237+ if (unlikely(!smp_load_acquire(&next_slab_inited))) {
238+ /*
239+ * Zero out zone modifiers, as we don't have specific zone
240+ * requirements. Keep the flags related to allocation in atomic
241+ * contexts and I/O.
242+ */
243+ alloc_flags &= ~GFP_ZONEMASK;
244+ alloc_flags &= (GFP_ATOMIC | GFP_KERNEL);
245+ page = alloc_pages(alloc_flags, STACK_ALLOC_ORDER);
246+ if (page)
247+ prealloc = page_address(page);
248+ }
249+
250+ spin_lock_irqsave(&depot_lock, flags);
251+
252+ found = find_stack(*bucket, trace->entries, trace->nr_entries, hash);
253+ if (!found) {
254+ struct stack_record *new =
255+ depot_alloc_stack(trace->entries, trace->nr_entries,
256+ hash, &prealloc, alloc_flags);
257+ if (new) {
258+ new->next = *bucket;
259+ /*
260+ * This smp_store_release() pairs with
261+ * smp_load_acquire() from |bucket| above.
262+ */
263+ smp_store_release(bucket, new);
264+ found = new;
265+ }
266+ } else if (prealloc) {
267+ /*
268+ * We didn't need to store this stack trace, but let's keep
269+ * the preallocated memory for the future.
270+ */
271+ WARN_ON(!init_stack_slab(&prealloc));
272+ }
273+
274+ spin_unlock_irqrestore(&depot_lock, flags);
275+exit:
276+ if (prealloc) {
277+ /* Nobody used this memory, ok to free it. */
278+ free_pages((unsigned long)prealloc, STACK_ALLOC_ORDER);
279+ }
280+ if (found)
281+ retval = found->handle.handle;
282+fast_exit:
283+ return retval;
284+}
lib/test_kasan.c+29−1
@@ -65,11 +65,34 @@ static noinline void __init kmalloc_node_oob_right(void) · 64 unmodified lines
6565 kfree(ptr);
6666 }
6767
68−static noinline void __init kmalloc_large_oob_right(void)
68+#ifdef CONFIG_SLUB
69+static noinline void __init kmalloc_pagealloc_oob_right(void)
6970 {
7071 char *ptr;
7172 size_t size = KMALLOC_MAX_CACHE_SIZE + 10;
7273
74+ /* Allocate a chunk that does not fit into a SLUB cache to trigger
75+ * the page allocator fallback.
76+ */
77+ pr_info("kmalloc pagealloc allocation: out-of-bounds to right\n");
78+ ptr = kmalloc(size, GFP_KERNEL);
79+ if (!ptr) {
80+ pr_err("Allocation failed\n");
81+ return;
82+ }
83+
84+ ptr[size] = 0;
85+ kfree(ptr);
86+}
87+#endif
88+
89+static noinline void __init kmalloc_large_oob_right(void)
90+{
91+ char *ptr;
92+ size_t size = KMALLOC_MAX_CACHE_SIZE - 256;
93+ /* Allocate a chunk that is large enough, but still fits into a slab
94+ * and does not trigger the page allocator fallback in SLUB.
95+ */
7396 pr_info("kmalloc large allocation: out-of-bounds to right\n");
7497 ptr = kmalloc(size, GFP_KERNEL);
7598 if (!ptr) {
@@ -271,6 +294,8 @@ static noinline void __init kmalloc_uaf2(void) · 195 unmodified lines
271294 }
272295
273296 ptr1[40] = 'x';
297+ if (ptr1 == ptr2)
298+ pr_err("Could not detect use-after-free: ptr1 == ptr2\n");
274299 kfree(ptr2);
275300 }
276301
@@ -324,6 +349,9 @@ static int __init kmalloc_tests_init(void) · 47 unmodified lines
324349 kmalloc_oob_right();
325350 kmalloc_oob_left();
326351 kmalloc_node_oob_right();
352+#ifdef CONFIG_SLUB
353+ kmalloc_pagealloc_oob_right();
354+#endif
327355 kmalloc_large_oob_right();
328356 kmalloc_oob_krealloc_more();
329357 kmalloc_oob_krealloc_less();
mm/Makefile+1−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
33 #
44
55 KASAN_SANITIZE_slab_common.o := n
6+KASAN_SANITIZE_slab.o := n
67 KASAN_SANITIZE_slub.o := n
78
89 # These files are disabled because they produce non-interesting and/or
mm/filemap.c+4−3
@@ -1840,15 +1840,16 @@ generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) · 1839 unmodified lines
18401840 ssize_t retval = 0;
18411841 loff_t *ppos = &iocb->ki_pos;
18421842 loff_t pos = *ppos;
1843+ size_t count = iov_iter_count(iter);
1844+
1845+ if (!count)
1846+ goto out; /* skip atime */
18431847
18441848 if (iocb->ki_flags & IOCB_DIRECT) {
18451849 struct address_space *mapping = file->f_mapping;
18461850 struct inode *inode = mapping->host;
1847− size_t count = iov_iter_count(iter);
18481851 loff_t size;
18491852
1850− if (!count)
1851− goto out; /* skip atime */
18521853 size = i_size_read(inode);
18531854 retval = filemap_write_and_wait_range(mapping, pos,
18541855 pos + count - 1);
mm/huge_memory.c+1−1
@@ -2578,7 +2578,7 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, · 2577 unmodified lines
25782578 }
25792579 khugepaged_node_load[node]++;
25802580 if (!PageLRU(page)) {
2581− result = SCAN_SCAN_ABORT;
2581+ result = SCAN_PAGE_LRU;
25822582 goto out_unmap;
25832583 }
25842584 if (PageLocked(page)) {
mm/internal.h+5−0
@@ -38,6 +38,11 @@ · 37 unmodified lines
3838 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
3939 unsigned long floor, unsigned long ceiling);
4040
41+void unmap_page_range(struct mmu_gather *tlb,
42+ struct vm_area_struct *vma,
43+ unsigned long addr, unsigned long end,
44+ struct zap_details *details);
45+
4146 extern int __do_page_cache_readahead(struct address_space *mapping,
4247 struct file *filp, pgoff_t offset, unsigned long nr_to_read,
4348 unsigned long lookahead_size);
mm/kasan/kasan.c+154−8
@@ -17,7 +17,9 @@ · 16 unmodified lines
1717 #define DISABLE_BRANCH_PROFILING
1818
1919 #include <linux/export.h>
20+#include <linux/interrupt.h>
2021 #include <linux/init.h>
22+#include <linux/kasan.h>
2123 #include <linux/kernel.h>
2224 #include <linux/kmemleak.h>
2325 #include <linux/linkage.h>
@@ -32,7 +34,6 @@ · 8 unmodified lines
3234 #include <linux/string.h>
3335 #include <linux/types.h>
3436 #include <linux/vmalloc.h>
35−#include <linux/kasan.h>
3637
3738 #include "kasan.h"
3839 #include "../slab.h"
@@ -334,6 +335,59 @@ void kasan_free_pages(struct page *page, unsigned int order) · 295 unmodified lines
334335 KASAN_FREE_PAGE);
335336 }
336337
338+#ifdef CONFIG_SLAB
339+/*
340+ * Adaptive redzone policy taken from the userspace AddressSanitizer runtime.
341+ * For larger allocations larger redzones are used.
342+ */
343+static size_t optimal_redzone(size_t object_size)
344+{
345+ int rz =
346+ object_size <= 64 - 16 ? 16 :
347+ object_size <= 128 - 32 ? 32 :
348+ object_size <= 512 - 64 ? 64 :
349+ object_size <= 4096 - 128 ? 128 :
350+ object_size <= (1 << 14) - 256 ? 256 :
351+ object_size <= (1 << 15) - 512 ? 512 :
352+ object_size <= (1 << 16) - 1024 ? 1024 : 2048;
353+ return rz;
354+}
355+
356+void kasan_cache_create(struct kmem_cache *cache, size_t *size,
357+ unsigned long *flags)
358+{
359+ int redzone_adjust;
360+ /* Make sure the adjusted size is still less than
361+ * KMALLOC_MAX_CACHE_SIZE.
362+ * TODO: this check is only useful for SLAB, but not SLUB. We'll need
363+ * to skip it for SLUB when it starts using kasan_cache_create().
364+ */
365+ if (*size > KMALLOC_MAX_CACHE_SIZE -
366+ sizeof(struct kasan_alloc_meta) -
367+ sizeof(struct kasan_free_meta))
368+ return;
369+ *flags |= SLAB_KASAN;
370+ /* Add alloc meta. */
371+ cache->kasan_info.alloc_meta_offset = *size;
372+ *size += sizeof(struct kasan_alloc_meta);
373+
374+ /* Add free meta. */
375+ if (cache->flags & SLAB_DESTROY_BY_RCU || cache->ctor ||
376+ cache->object_size < sizeof(struct kasan_free_meta)) {
377+ cache->kasan_info.free_meta_offset = *size;
378+ *size += sizeof(struct kasan_free_meta);
379+ }
380+ redzone_adjust = optimal_redzone(cache->object_size) -
381+ (*size - cache->object_size);
382+ if (redzone_adjust > 0)
383+ *size += redzone_adjust;
384+ *size = min(KMALLOC_MAX_CACHE_SIZE,
385+ max(*size,
386+ cache->object_size +
387+ optimal_redzone(cache->object_size)));
388+}
389+#endif
390+
337391 void kasan_poison_slab(struct page *page)
338392 {
339393 kasan_poison_shadow(page_address(page),
@@ -351,11 +405,81 @@ void kasan_poison_object_data(struct kmem_cache *cache, void *object) · 11 unmodified lines
351405 kasan_poison_shadow(object,
352406 round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE),
353407 KASAN_KMALLOC_REDZONE);
408+#ifdef CONFIG_SLAB
409+ if (cache->flags & SLAB_KASAN) {
410+ struct kasan_alloc_meta *alloc_info =
411+ get_alloc_info(cache, object);
412+ alloc_info->state = KASAN_STATE_INIT;
413+ }
414+#endif
354415 }
355416
356−void kasan_slab_alloc(struct kmem_cache *cache, void *object)
417+#ifdef CONFIG_SLAB
418+static inline int in_irqentry_text(unsigned long ptr)
357419 {
358− kasan_kmalloc(cache, object, cache->object_size);
420+ return (ptr >= (unsigned long)&__irqentry_text_start &&
421+ ptr < (unsigned long)&__irqentry_text_end) ||
422+ (ptr >= (unsigned long)&__softirqentry_text_start &&
423+ ptr < (unsigned long)&__softirqentry_text_end);
424+}
425+
426+static inline void filter_irq_stacks(struct stack_trace *trace)
427+{
428+ int i;
429+
430+ if (!trace->nr_entries)
431+ return;
432+ for (i = 0; i < trace->nr_entries; i++)
433+ if (in_irqentry_text(trace->entries[i])) {
434+ /* Include the irqentry function into the stack. */
435+ trace->nr_entries = i + 1;
436+ break;
437+ }
438+}
439+
440+static inline depot_stack_handle_t save_stack(gfp_t flags)
441+{
442+ unsigned long entries[KASAN_STACK_DEPTH];
443+ struct stack_trace trace = {
444+ .nr_entries = 0,
445+ .entries = entries,
446+ .max_entries = KASAN_STACK_DEPTH,
447+ .skip = 0
448+ };
449+
450+ save_stack_trace(&trace);
451+ filter_irq_stacks(&trace);
452+ if (trace.nr_entries != 0 &&
453+ trace.entries[trace.nr_entries-1] == ULONG_MAX)
454+ trace.nr_entries--;
455+
456+ return depot_save_stack(&trace, flags);
457+}
458+
459+static inline void set_track(struct kasan_track *track, gfp_t flags)
460+{
461+ track->pid = current->pid;
462+ track->stack = save_stack(flags);
463+}
464+
465+struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache,
466+ const void *object)
467+{
468+ BUILD_BUG_ON(sizeof(struct kasan_alloc_meta) > 32);
469+ return (void *)object + cache->kasan_info.alloc_meta_offset;
470+}
471+
472+struct kasan_free_meta *get_free_info(struct kmem_cache *cache,
473+ const void *object)
474+{
475+ BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32);
476+ return (void *)object + cache->kasan_info.free_meta_offset;
477+}
478+#endif
479+
480+void kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags)
481+{
482+ kasan_kmalloc(cache, object, cache->object_size, flags);
359483 }
360484
361485 void kasan_slab_free(struct kmem_cache *cache, void *object)
@@ -367,10 +491,22 @@ void kasan_slab_free(struct kmem_cache *cache, void *object) · 5 unmodified lines
367491 if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU))
368492 return;
369493
494+#ifdef CONFIG_SLAB
495+ if (cache->flags & SLAB_KASAN) {
496+ struct kasan_free_meta *free_info =
497+ get_free_info(cache, object);
498+ struct kasan_alloc_meta *alloc_info =
499+ get_alloc_info(cache, object);
500+ alloc_info->state = KASAN_STATE_FREE;
501+ set_track(&free_info->track);
502+ }
503+#endif
504+
370505 kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
371506 }
372507
373−void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size)
508+void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size,
509+ gfp_t flags)
374510 {
375511 unsigned long redzone_start;
376512 unsigned long redzone_end;
@@ -386,10 +522,20 @@ void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size) · 9 unmodified lines
386522 kasan_unpoison_shadow(object, size);
387523 kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
388524 KASAN_KMALLOC_REDZONE);
525+#ifdef CONFIG_SLAB
526+ if (cache->flags & SLAB_KASAN) {
527+ struct kasan_alloc_meta *alloc_info =
528+ get_alloc_info(cache, object);
529+
530+ alloc_info->state = KASAN_STATE_ALLOC;
531+ alloc_info->alloc_size = size;
532+ set_track(&alloc_info->track, flags);
533+ }
534+#endif
389535 }
390536 EXPORT_SYMBOL(kasan_kmalloc);
391537
392−void kasan_kmalloc_large(const void *ptr, size_t size)
538+void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
393539 {
394540 struct page *page;
395541 unsigned long redzone_start;
@@ -408,7 +554,7 @@ void kasan_kmalloc_large(const void *ptr, size_t size) · 12 unmodified lines
408554 KASAN_PAGE_REDZONE);
409555 }
410556
411−void kasan_krealloc(const void *object, size_t size)
557+void kasan_krealloc(const void *object, size_t size, gfp_t flags)
412558 {
413559 struct page *page;
414560
@@ -418,9 +564,9 @@ void kasan_krealloc(const void *object, size_t size) · 3 unmodified lines
418564 page = virt_to_head_page(object);
419565
420566 if (unlikely(!PageSlab(page)))
421− kasan_kmalloc_large(object, size);
567+ kasan_kmalloc_large(object, size, flags);
422568 else
423− kasan_kmalloc(page->slab_cache, object, size);
569+ kasan_kmalloc(page->slab_cache, object, size, flags);
424570 }
425571
426572 void kasan_kfree(void *ptr)
mm/kasan/kasan.h+37−0
@@ -2,6 +2,7 @@ · 1 unmodified line
22 #define __MM_KASAN_KASAN_H
33
44 #include <linux/kasan.h>
5+#include <linux/stackdepot.h>
56
67 #define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT)
78 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1)
@@ -54,6 +55,42 @@ struct kasan_global { · 46 unmodified lines
5455 #endif
5556 };
5657
58+/**
59+ * Structures to keep alloc and free tracks *
60+ */
61+
62+enum kasan_state {
63+ KASAN_STATE_INIT,
64+ KASAN_STATE_ALLOC,
65+ KASAN_STATE_FREE
66+};
67+
68+#define KASAN_STACK_DEPTH 64
69+
70+struct kasan_track {
71+ u32 pid;
72+ depot_stack_handle_t stack;
73+};
74+
75+struct kasan_alloc_meta {
76+ struct kasan_track track;
77+ u32 state : 2; /* enum kasan_state */
78+ u32 alloc_size : 30;
79+ u32 reserved;
80+};
81+
82+struct kasan_free_meta {
83+ /* Allocator freelist pointer, unused by KASAN. */
84+ void **freelist;
85+ struct kasan_track track;
86+};
87+
88+struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache,
89+ const void *object);
90+struct kasan_free_meta *get_free_info(struct kmem_cache *cache,
91+ const void *object);
92+
93+
5794 static inline const void *kasan_shadow_to_mem(const void *shadow_addr)
5895 {
5996 return (void *)(((unsigned long)shadow_addr - KASAN_SHADOW_OFFSET)
mm/kasan/report.c+51−11
@@ -18,6 +18,7 @@ · 17 unmodified lines
1818 #include <linux/printk.h>
1919 #include <linux/sched.h>
2020 #include <linux/slab.h>
21+#include <linux/stackdepot.h>
2122 #include <linux/stacktrace.h>
2223 #include <linux/string.h>
2324 #include <linux/types.h>
@@ -115,6 +116,53 @@ static inline bool init_task_stack_addr(const void *addr) · 91 unmodified lines
115116 sizeof(init_thread_union.stack));
116117 }
117118
119+#ifdef CONFIG_SLAB
120+static void print_track(struct kasan_track *track)
121+{
122+ pr_err("PID = %u\n", track->pid);
123+ if (track->stack) {
124+ struct stack_trace trace;
125+
126+ depot_fetch_stack(track->stack, &trace);
127+ print_stack_trace(&trace, 0);
128+ } else {
129+ pr_err("(stack is not available)\n");
130+ }
131+}
132+
133+static void object_err(struct kmem_cache *cache, struct page *page,
134+ void *object, char *unused_reason)
135+{
136+ struct kasan_alloc_meta *alloc_info = get_alloc_info(cache, object);
137+ struct kasan_free_meta *free_info;
138+
139+ dump_stack();
140+ pr_err("Object at %p, in cache %s\n", object, cache->name);
141+ if (!(cache->flags & SLAB_KASAN))
142+ return;
143+ switch (alloc_info->state) {
144+ case KASAN_STATE_INIT:
145+ pr_err("Object not allocated yet\n");
146+ break;
147+ case KASAN_STATE_ALLOC:
148+ pr_err("Object allocated with size %u bytes.\n",
149+ alloc_info->alloc_size);
150+ pr_err("Allocation:\n");
151+ print_track(&alloc_info->track);
152+ break;
153+ case KASAN_STATE_FREE:
154+ pr_err("Object freed, allocated with size %u bytes\n",
155+ alloc_info->alloc_size);
156+ free_info = get_free_info(cache, object);
157+ pr_err("Allocation:\n");
158+ print_track(&alloc_info->track);
159+ pr_err("Deallocation:\n");
160+ print_track(&free_info->track);
161+ break;
162+ }
163+}
164+#endif
165+
118166 static void print_address_description(struct kasan_access_info *info)
119167 {
120168 const void *addr = info->access_addr;
@@ -126,17 +174,10 @@ static void print_address_description(struct kasan_access_info *info) · 5 unmodified lines
126174 if (PageSlab(page)) {
127175 void *object;
128176 struct kmem_cache *cache = page->slab_cache;
129− void *last_object;
130−
131− object = virt_to_obj(cache, page_address(page), addr);
132− last_object = page_address(page) +
133− page->objects * cache->size;
134−
135− if (unlikely(object > last_object))
136− object = last_object; /* we hit into padding */
137−
177+ object = nearest_obj(cache, page,
178+ (void *)info->access_addr);
138179 object_err(cache, page, object,
139− "kasan: bad access detected");
180+ "kasan: bad access detected");
140181 return;
141182 }
142183 dump_page(page, "kasan: bad access detected");
@@ -146,7 +187,6 @@ static void print_address_description(struct kasan_access_info *info) · 3 unmodified lines
146187 if (!init_task_stack_addr(addr))
147188 pr_err("Address belongs to variable %pS\n", addr);
148189 }
149−
150190 dump_stack();
151191 }
152192
mm/memory.c+10−7
@@ -1102,6 +1102,12 @@ again: · 1101 unmodified lines
11021102
11031103 if (!PageAnon(page)) {
11041104 if (pte_dirty(ptent)) {
1105+ /*
1106+ * oom_reaper cannot tear down dirty
1107+ * pages
1108+ */
1109+ if (unlikely(details && details->ignore_dirty))
1110+ continue;
11051111 force_flush = 1;
11061112 set_page_dirty(page);
11071113 }
@@ -1120,8 +1126,8 @@ again: · 12 unmodified lines
11201126 }
11211127 continue;
11221128 }
1123− /* If details->check_mapping, we leave swap entries. */
1124− if (unlikely(details))
1129+ /* only check swap_entries if explicitly asked for in details */
1130+ if (unlikely(details && !details->check_swap_entries))
11251131 continue;
11261132
11271133 entry = pte_to_swp_entry(ptent);
@@ -1226,7 +1232,7 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, · 98 unmodified lines
12261232 return addr;
12271233 }
12281234
1229−static void unmap_page_range(struct mmu_gather *tlb,
1235+void unmap_page_range(struct mmu_gather *tlb,
12301236 struct vm_area_struct *vma,
12311237 unsigned long addr, unsigned long end,
12321238 struct zap_details *details)
@@ -1234,9 +1240,6 @@ static void unmap_page_range(struct mmu_gather *tlb, · 1 unmodified line
12341240 pgd_t *pgd;
12351241 unsigned long next;
12361242
1237− if (details && !details->check_mapping)
1238− details = NULL;
1239−
12401243 BUG_ON(addr >= end);
12411244 tlb_start_vma(tlb, vma);
12421245 pgd = pgd_offset(vma->vm_mm, addr);
@@ -2432,7 +2435,7 @@ static inline void unmap_mapping_range_tree(struct rb_root *root, · 1189 unmodified lines
24322435 void unmap_mapping_range(struct address_space *mapping,
24332436 loff_t const holebegin, loff_t const holelen, int even_cows)
24342437 {
2435− struct zap_details details;
2438+ struct zap_details details = { };
24362439 pgoff_t hba = holebegin >> PAGE_SHIFT;
24372440 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
24382441
mm/mempool.c+8−8
@@ -112,12 +112,12 @@ static void kasan_poison_element(mempool_t *pool, void *element) · 111 unmodified lines
112112 kasan_free_pages(element, (unsigned long)pool->pool_data);
113113 }
114114
115−static void kasan_unpoison_element(mempool_t *pool, void *element)
115+static void kasan_unpoison_element(mempool_t *pool, void *element, gfp_t flags)
116116 {
117117 if (pool->alloc == mempool_alloc_slab)
118− kasan_slab_alloc(pool->pool_data, element);
118+ kasan_slab_alloc(pool->pool_data, element, flags);
119119 if (pool->alloc == mempool_kmalloc)
120− kasan_krealloc(element, (size_t)pool->pool_data);
120+ kasan_krealloc(element, (size_t)pool->pool_data, flags);
121121 if (pool->alloc == mempool_alloc_pages)
122122 kasan_alloc_pages(element, (unsigned long)pool->pool_data);
123123 }
@@ -130,12 +130,12 @@ static void add_element(mempool_t *pool, void *element) · 6 unmodified lines
130130 pool->elements[pool->curr_nr++] = element;
131131 }
132132
133−static void *remove_element(mempool_t *pool)
133+static void *remove_element(mempool_t *pool, gfp_t flags)
134134 {
135135 void *element = pool->elements[--pool->curr_nr];
136136
137137 BUG_ON(pool->curr_nr < 0);
138− kasan_unpoison_element(pool, element);
138+ kasan_unpoison_element(pool, element, flags);
139139 check_element(pool, element);
140140 return element;
141141 }
@@ -154,7 +154,7 @@ void mempool_destroy(mempool_t *pool) · 12 unmodified lines
154154 return;
155155
156156 while (pool->curr_nr) {
157− void *element = remove_element(pool);
157+ void *element = remove_element(pool, GFP_KERNEL);
158158 pool->free(element, pool->pool_data);
159159 }
160160 kfree(pool->elements);
@@ -250,7 +250,7 @@ int mempool_resize(mempool_t *pool, int new_min_nr) · 89 unmodified lines
250250 spin_lock_irqsave(&pool->lock, flags);
251251 if (new_min_nr <= pool->min_nr) {
252252 while (new_min_nr < pool->curr_nr) {
253− element = remove_element(pool);
253+ element = remove_element(pool, GFP_KERNEL);
254254 spin_unlock_irqrestore(&pool->lock, flags);
255255 pool->free(element, pool->pool_data);
256256 spin_lock_irqsave(&pool->lock, flags);
@@ -347,7 +347,7 @@ repeat_alloc: · 90 unmodified lines
347347
348348 spin_lock_irqsave(&pool->lock, flags);
349349 if (likely(pool->curr_nr)) {
350− element = remove_element(pool);
350+ element = remove_element(pool, gfp_temp);
351351 spin_unlock_irqrestore(&pool->lock, flags);
352352 /* paired with rmb in mempool_free(), read comment there */
353353 smp_wmb();
mm/oom_kill.c+187−9
@@ -35,6 +35,11 @@ · 34 unmodified lines
3535 #include <linux/freezer.h>
3636 #include <linux/ftrace.h>
3737 #include <linux/ratelimit.h>
38+#include <linux/kthread.h>
39+#include <linux/init.h>
40+
41+#include <asm/tlb.h>
42+#include "internal.h"
3843
3944 #define CREATE_TRACE_POINTS
4045 #include <trace/events/oom.h>
@@ -405,6 +410,172 @@ static DECLARE_WAIT_QUEUE_HEAD(oom_victims_wait); · 364 unmodified lines
405410
406411 bool oom_killer_disabled __read_mostly;
407412
413+#define K(x) ((x) << (PAGE_SHIFT-10))
414+
415+#ifdef CONFIG_MMU
416+/*
417+ * OOM Reaper kernel thread which tries to reap the memory used by the OOM
418+ * victim (if that is possible) to help the OOM killer to move on.
419+ */
420+static struct task_struct *oom_reaper_th;
421+static DECLARE_WAIT_QUEUE_HEAD(oom_reaper_wait);
422+static struct task_struct *oom_reaper_list;
423+static DEFINE_SPINLOCK(oom_reaper_lock);
424+
425+
426+static bool __oom_reap_task(struct task_struct *tsk)
427+{
428+ struct mmu_gather tlb;
429+ struct vm_area_struct *vma;
430+ struct mm_struct *mm;
431+ struct task_struct *p;
432+ struct zap_details details = {.check_swap_entries = true,
433+ .ignore_dirty = true};
434+ bool ret = true;
435+
436+ /*
437+ * Make sure we find the associated mm_struct even when the particular
438+ * thread has already terminated and cleared its mm.
439+ * We might have race with exit path so consider our work done if there
440+ * is no mm.
441+ */
442+ p = find_lock_task_mm(tsk);
443+ if (!p)
444+ return true;
445+
446+ mm = p->mm;
447+ if (!atomic_inc_not_zero(&mm->mm_users)) {
448+ task_unlock(p);
449+ return true;
450+ }
451+
452+ task_unlock(p);
453+
454+ if (!down_read_trylock(&mm->mmap_sem)) {
455+ ret = false;
456+ goto out;
457+ }
458+
459+ tlb_gather_mmu(&tlb, mm, 0, -1);
460+ for (vma = mm->mmap ; vma; vma = vma->vm_next) {
461+ if (is_vm_hugetlb_page(vma))
462+ continue;
463+
464+ /*
465+ * mlocked VMAs require explicit munlocking before unmap.
466+ * Let's keep it simple here and skip such VMAs.
467+ */
468+ if (vma->vm_flags & VM_LOCKED)
469+ continue;
470+
471+ /*
472+ * Only anonymous pages have a good chance to be dropped
473+ * without additional steps which we cannot afford as we
474+ * are OOM already.
475+ *
476+ * We do not even care about fs backed pages because all
477+ * which are reclaimable have already been reclaimed and
478+ * we do not want to block exit_mmap by keeping mm ref
479+ * count elevated without a good reason.
480+ */
481+ if (vma_is_anonymous(vma) || !(vma->vm_flags & VM_SHARED))
482+ unmap_page_range(&tlb, vma, vma->vm_start, vma->vm_end,
483+ &details);
484+ }
485+ tlb_finish_mmu(&tlb, 0, -1);
486+ pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n",
487+ task_pid_nr(tsk), tsk->comm,
488+ K(get_mm_counter(mm, MM_ANONPAGES)),
489+ K(get_mm_counter(mm, MM_FILEPAGES)),
490+ K(get_mm_counter(mm, MM_SHMEMPAGES)));
491+ up_read(&mm->mmap_sem);
492+
493+ /*
494+ * Clear TIF_MEMDIE because the task shouldn't be sitting on a
495+ * reasonably reclaimable memory anymore. OOM killer can continue
496+ * by selecting other victim if unmapping hasn't led to any
497+ * improvements. This also means that selecting this task doesn't
498+ * make any sense.
499+ */
500+ tsk->signal->oom_score_adj = OOM_SCORE_ADJ_MIN;
501+ exit_oom_victim(tsk);
502+out:
503+ mmput(mm);
504+ return ret;
505+}
506+
507+#define MAX_OOM_REAP_RETRIES 10
508+static void oom_reap_task(struct task_struct *tsk)
509+{
510+ int attempts = 0;
511+
512+ /* Retry the down_read_trylock(mmap_sem) a few times */
513+ while (attempts++ < MAX_OOM_REAP_RETRIES && !__oom_reap_task(tsk))
514+ schedule_timeout_idle(HZ/10);
515+
516+ if (attempts > MAX_OOM_REAP_RETRIES) {
517+ pr_info("oom_reaper: unable to reap pid:%d (%s)\n",
518+ task_pid_nr(tsk), tsk->comm);
519+ debug_show_all_locks();
520+ }
521+
522+ /* Drop a reference taken by wake_oom_reaper */
523+ put_task_struct(tsk);
524+}
525+
526+static int oom_reaper(void *unused)
527+{
528+ set_freezable();
529+
530+ while (true) {
531+ struct task_struct *tsk = NULL;
532+
533+ wait_event_freezable(oom_reaper_wait, oom_reaper_list != NULL);
534+ spin_lock(&oom_reaper_lock);
535+ if (oom_reaper_list != NULL) {
536+ tsk = oom_reaper_list;
537+ oom_reaper_list = tsk->oom_reaper_list;
538+ }
539+ spin_unlock(&oom_reaper_lock);
540+
541+ if (tsk)
542+ oom_reap_task(tsk);
543+ }
544+
545+ return 0;
546+}
547+
548+static void wake_oom_reaper(struct task_struct *tsk)
549+{
550+ if (!oom_reaper_th || tsk->oom_reaper_list)
551+ return;
552+
553+ get_task_struct(tsk);
554+
555+ spin_lock(&oom_reaper_lock);
556+ tsk->oom_reaper_list = oom_reaper_list;
557+ oom_reaper_list = tsk;
558+ spin_unlock(&oom_reaper_lock);
559+ wake_up(&oom_reaper_wait);
560+}
561+
562+static int __init oom_init(void)
563+{
564+ oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper");
565+ if (IS_ERR(oom_reaper_th)) {
566+ pr_err("Unable to start OOM reaper %ld. Continuing regardless\n",
567+ PTR_ERR(oom_reaper_th));
568+ oom_reaper_th = NULL;
569+ }
570+ return 0;
571+}
572+subsys_initcall(oom_init)
573+#else
574+static void wake_oom_reaper(struct task_struct *tsk)
575+{
576+}
577+#endif
578+
408579 /**
409580 * mark_oom_victim - mark the given task as OOM victim
410581 * @tsk: task to mark
@@ -431,9 +602,10 @@ void mark_oom_victim(struct task_struct *tsk) · 20 unmodified lines
431602 /**
432603 * exit_oom_victim - note the exit of an OOM victim
433604 */
434−void exit_oom_victim(void)
605+void exit_oom_victim(struct task_struct *tsk)
435606 {
436− clear_thread_flag(TIF_MEMDIE);
607+ if (!test_and_clear_tsk_thread_flag(tsk, TIF_MEMDIE))
608+ return;
437609
438610 if (!atomic_dec_return(&oom_victims))
439611 wake_up_all(&oom_victims_wait);
@@ -494,7 +666,6 @@ static bool process_shares_mm(struct task_struct *p, struct mm_struct *mm) · 54 unmodified lines
494666 return false;
495667 }
496668
497−#define K(x) ((x) << (PAGE_SHIFT-10))
498669 /*
499670 * Must be called while holding a reference to p, which will be released upon
500671 * returning.
@@ -510,6 +681,7 @@ void oom_kill_process(struct oom_control *oc, struct task_struct *p, · 9 unmodified lines
510681 unsigned int victim_points = 0;
511682 static DEFINE_RATELIMIT_STATE(oom_rs, DEFAULT_RATELIMIT_INTERVAL,
512683 DEFAULT_RATELIMIT_BURST);
684+ bool can_oom_reap = true;
513685
514686 /*
515687 * If the task is already exiting, don't alarm the sysadmin or kill
@@ -600,17 +772,23 @@ void oom_kill_process(struct oom_control *oc, struct task_struct *p, · 84 unmodified lines
600772 continue;
601773 if (same_thread_group(p, victim))
602774 continue;
603− if (unlikely(p->flags & PF_KTHREAD))
604− continue;
605− if (is_global_init(p))
606− continue;
607− if (p->signal->oom_score_adj == OOM_SCORE_ADJ_MIN)
775+ if (unlikely(p->flags & PF_KTHREAD) || is_global_init(p) ||
776+ p->signal->oom_score_adj == OOM_SCORE_ADJ_MIN) {
777+ /*
778+ * We cannot use oom_reaper for the mm shared by this
779+ * process because it wouldn't get killed and so the
780+ * memory might be still used.
781+ */
782+ can_oom_reap = false;
608783 continue;
609−
784+ }
610785 do_send_sig_info(SIGKILL, SEND_SIG_FORCED, p, true);
611786 }
612787 rcu_read_unlock();
613788
789+ if (can_oom_reap)
790+ wake_oom_reaper(victim);
791+
614792 mmdrop(mm);
615793 put_task_struct(victim);
616794 }
mm/page_alloc.c+33−13
@@ -692,34 +692,28 @@ static inline void __free_one_page(struct page *page, · 691 unmodified lines
692692 unsigned long combined_idx;
693693 unsigned long uninitialized_var(buddy_idx);
694694 struct page *buddy;
695− unsigned int max_order = MAX_ORDER;
695+ unsigned int max_order;
696+
697+ max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
696698
697699 VM_BUG_ON(!zone_is_initialized(zone));
698700 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
699701
700702 VM_BUG_ON(migratetype == -1);
701− if (is_migrate_isolate(migratetype)) {
702− /*
703− * We restrict max order of merging to prevent merge
704− * between freepages on isolate pageblock and normal
705− * pageblock. Without this, pageblock isolation
706− * could cause incorrect freepage accounting.
707− */
708− max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
709− } else {
703+ if (likely(!is_migrate_isolate(migratetype)))
710704 __mod_zone_freepage_state(zone, 1 << order, migratetype);
711− }
712705
713− page_idx = pfn & ((1 << max_order) - 1);
706+ page_idx = pfn & ((1 << MAX_ORDER) - 1);
714707
715708 VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
716709 VM_BUG_ON_PAGE(bad_range(zone, page), page);
717710
711+continue_merging:
718712 while (order < max_order - 1) {
719713 buddy_idx = __find_buddy_index(page_idx, order);
720714 buddy = page + (buddy_idx - page_idx);
721715 if (!page_is_buddy(page, buddy, order))
722− break;
716+ goto done_merging;
723717 /*
724718 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
725719 * merge with it and move up one order.
@@ -736,6 +730,32 @@ static inline void __free_one_page(struct page *page, · 10 unmodified lines
736730 page_idx = combined_idx;
737731 order++;
738732 }
733+ if (max_order < MAX_ORDER) {
734+ /* If we are here, it means order is >= pageblock_order.
735+ * We want to prevent merge between freepages on isolate
736+ * pageblock and normal pageblock. Without this, pageblock
737+ * isolation could cause incorrect freepage or CMA accounting.
738+ *
739+ * We don't want to hit this code for the more frequent
740+ * low-order merging.
741+ */
742+ if (unlikely(has_isolate_pageblock(zone))) {
743+ int buddy_mt;
744+
745+ buddy_idx = __find_buddy_index(page_idx, order);
746+ buddy = page + (buddy_idx - page_idx);
747+ buddy_mt = get_pageblock_migratetype(buddy);
748+
749+ if (migratetype != buddy_mt
750+ && (is_migrate_isolate(migratetype) ||
751+ is_migrate_isolate(buddy_mt)))
752+ goto done_merging;
753+ }
754+ max_order++;
755+ goto continue_merging;
756+ }
757+
758+done_merging:
739759 set_page_order(page, order);
740760
741761 /*
mm/slab.c+37−5
@@ -2086,6 +2086,8 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) · 2085 unmodified lines
20862086 }
20872087 #endif
20882088
2089+ kasan_cache_create(cachep, &size, &flags);
2090+
20892091 size = ALIGN(size, cachep->align);
20902092 /*
20912093 * We should restrict the number of objects in a slab to implement
@@ -2387,8 +2389,13 @@ static void cache_init_objs_debug(struct kmem_cache *cachep, struct page *page) · 295 unmodified lines
23872389 * cache which they are a constructor for. Otherwise, deadlock.
23882390 * They must also be threaded.
23892391 */
2390− if (cachep->ctor && !(cachep->flags & SLAB_POISON))
2392+ if (cachep->ctor && !(cachep->flags & SLAB_POISON)) {
2393+ kasan_unpoison_object_data(cachep,
2394+ objp + obj_offset(cachep));
23912395 cachep->ctor(objp + obj_offset(cachep));
2396+ kasan_poison_object_data(
2397+ cachep, objp + obj_offset(cachep));
2398+ }
23922399
23932400 if (cachep->flags & SLAB_RED_ZONE) {
23942401 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
@@ -2409,6 +2416,7 @@ static void cache_init_objs(struct kmem_cache *cachep, · 14 unmodified lines
24092416 struct page *page)
24102417 {
24112418 int i;
2419+ void *objp;
24122420
24132421 cache_init_objs_debug(cachep, page);
24142422
@@ -2419,8 +2427,12 @@ static void cache_init_objs(struct kmem_cache *cachep, · 4 unmodified lines
24192427
24202428 for (i = 0; i < cachep->num; i++) {
24212429 /* constructor could break poison info */
2422− if (DEBUG == 0 && cachep->ctor)
2423− cachep->ctor(index_to_obj(cachep, page, i));
2430+ if (DEBUG == 0 && cachep->ctor) {
2431+ objp = index_to_obj(cachep, page, i);
2432+ kasan_unpoison_object_data(cachep, objp);
2433+ cachep->ctor(objp);
2434+ kasan_poison_object_data(cachep, objp);
2435+ }
24242436
24252437 set_free_obj(page, i, i);
24262438 }
@@ -2550,6 +2562,7 @@ static int cache_grow(struct kmem_cache *cachep, · 123 unmodified lines
25502562
25512563 slab_map_pages(cachep, page, freelist);
25522564
2565+ kasan_poison_slab(page);
25532566 cache_init_objs(cachep, page);
25542567
25552568 if (gfpflags_allow_blocking(local_flags))
@@ -3316,6 +3329,8 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp, · 760 unmodified lines
33163329 {
33173330 struct array_cache *ac = cpu_cache_get(cachep);
33183331
3332+ kasan_slab_free(cachep, objp);
3333+
33193334 check_irq_off();
33203335 kmemleak_free_recursive(objp, cachep->flags);
33213336 objp = cache_free_debugcheck(cachep, objp, caller);
@@ -3363,6 +3378,7 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) · 41 unmodified lines
33633378 {
33643379 void *ret = slab_alloc(cachep, flags, _RET_IP_);
33653380
3381+ kasan_slab_alloc(cachep, ret, flags);
33663382 trace_kmem_cache_alloc(_RET_IP_, ret,
33673383 cachep->object_size, cachep->size, flags);
33683384
@@ -3428,6 +3444,7 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) · 59 unmodified lines
34283444
34293445 ret = slab_alloc(cachep, flags, _RET_IP_);
34303446
3447+ kasan_kmalloc(cachep, ret, size, flags);
34313448 trace_kmalloc(_RET_IP_, ret,
34323449 size, cachep->size, flags);
34333450 return ret;
@@ -3451,6 +3468,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) · 17 unmodified lines
34513468 {
34523469 void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
34533470
3471+ kasan_slab_alloc(cachep, ret, flags);
34543472 trace_kmem_cache_alloc_node(_RET_IP_, ret,
34553473 cachep->object_size, cachep->size,
34563474 flags, nodeid);
@@ -3469,6 +3487,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, · 12 unmodified lines
34693487
34703488 ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
34713489
3490+ kasan_kmalloc(cachep, ret, size, flags);
34723491 trace_kmalloc_node(_RET_IP_, ret,
34733492 size, cachep->size,
34743493 flags, nodeid);
@@ -3481,11 +3500,15 @@ static __always_inline void * · 6 unmodified lines
34813500 __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
34823501 {
34833502 struct kmem_cache *cachep;
3503+ void *ret;
34843504
34853505 cachep = kmalloc_slab(size, flags);
34863506 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
34873507 return cachep;
3488− return kmem_cache_alloc_node_trace(cachep, flags, node, size);
3508+ ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);
3509+ kasan_kmalloc(cachep, ret, size, flags);
3510+
3511+ return ret;
34893512 }
34903513
34913514 void *__kmalloc_node(size_t size, gfp_t flags, int node)
@@ -3519,6 +3542,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, · 27 unmodified lines
35193542 return cachep;
35203543 ret = slab_alloc(cachep, flags, caller);
35213544
3545+ kasan_kmalloc(cachep, ret, size, flags);
35223546 trace_kmalloc(caller, ret,
35233547 size, cachep->size, flags);
35243548
@@ -4290,10 +4314,18 @@ module_init(slab_proc_init); · 765 unmodified lines
42904314 */
42914315 size_t ksize(const void *objp)
42924316 {
4317+ size_t size;
4318+
42934319 BUG_ON(!objp);
42944320 if (unlikely(objp == ZERO_SIZE_PTR))
42954321 return 0;
42964322
4297− return virt_to_cache(objp)->object_size;
4323+ size = virt_to_cache(objp)->object_size;
4324+ /* We assume that ksize callers could use the whole allocated area,
4325+ * so we need to unpoison this area.
4326+ */
4327+ kasan_krealloc(objp, size, GFP_NOWAIT);
4328+
4329+ return size;
42984330 }
42994331 EXPORT_SYMBOL(ksize);
mm/slab.h+1−1
@@ -405,7 +405,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, · 404 unmodified lines
405405 kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
406406 kmemleak_alloc_recursive(object, s->object_size, 1,
407407 s->flags, flags);
408− kasan_slab_alloc(s, object);
408+ kasan_slab_alloc(s, object, flags);
409409 }
410410 memcg_kmem_put_cache(s);
411411 }
mm/slab_common.c+3−3
@@ -35,7 +35,7 @@ struct kmem_cache *kmem_cache; · 34 unmodified lines
3535 */
3636 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
3737 SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
38− SLAB_FAILSLAB)
38+ SLAB_FAILSLAB | SLAB_KASAN)
3939
4040 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
4141 SLAB_NOTRACK | SLAB_ACCOUNT)
@@ -1013,7 +1013,7 @@ void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) · 971 unmodified lines
10131013 page = alloc_kmem_pages(flags, order);
10141014 ret = page ? page_address(page) : NULL;
10151015 kmemleak_alloc(ret, size, 1, flags);
1016− kasan_kmalloc_large(ret, size);
1016+ kasan_kmalloc_large(ret, size, flags);
10171017 return ret;
10181018 }
10191019 EXPORT_SYMBOL(kmalloc_order);
@@ -1192,7 +1192,7 @@ static __always_inline void *__do_krealloc(const void *p, size_t new_size, · 172 unmodified lines
11921192 ks = ksize(p);
11931193
11941194 if (ks >= new_size) {
1195− kasan_krealloc((void *)p, new_size);
1195+ kasan_krealloc((void *)p, new_size, flags);
11961196 return (void *)p;
11971197 }
11981198
mm/slub.c+8−7
@@ -1313,7 +1313,7 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node, · 1312 unmodified lines
13131313 static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
13141314 {
13151315 kmemleak_alloc(ptr, size, 1, flags);
1316− kasan_kmalloc_large(ptr, size);
1316+ kasan_kmalloc_large(ptr, size, flags);
13171317 }
13181318
13191319 static inline void kfree_hook(const void *x)
@@ -2596,7 +2596,7 @@ void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) · 1276 unmodified lines
25962596 {
25972597 void *ret = slab_alloc(s, gfpflags, _RET_IP_);
25982598 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
2599− kasan_kmalloc(s, ret, size);
2599+ kasan_kmalloc(s, ret, size, gfpflags);
26002600 return ret;
26012601 }
26022602 EXPORT_SYMBOL(kmem_cache_alloc_trace);
@@ -2624,7 +2624,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s, · 21 unmodified lines
26242624 trace_kmalloc_node(_RET_IP_, ret,
26252625 size, s->size, gfpflags, node);
26262626
2627− kasan_kmalloc(s, ret, size);
2627+ kasan_kmalloc(s, ret, size, gfpflags);
26282628 return ret;
26292629 }
26302630 EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
@@ -3182,7 +3182,8 @@ static void early_kmem_cache_node_alloc(int node) · 551 unmodified lines
31823182 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
31833183 init_tracking(kmem_cache_node, n);
31843184 #endif
3185− kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node));
3185+ kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node),
3186+ GFP_KERNEL);
31863187 init_kmem_cache_node(n);
31873188 inc_slabs_node(kmem_cache_node, node, page->objects);
31883189
@@ -3561,7 +3562,7 @@ void *__kmalloc(size_t size, gfp_t flags) · 372 unmodified lines
35613562
35623563 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
35633564
3564− kasan_kmalloc(s, ret, size);
3565+ kasan_kmalloc(s, ret, size, flags);
35653566
35663567 return ret;
35673568 }
@@ -3606,7 +3607,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) · 38 unmodified lines
36063607
36073608 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
36083609
3609− kasan_kmalloc(s, ret, size);
3610+ kasan_kmalloc(s, ret, size, flags);
36103611
36113612 return ret;
36123613 }
@@ -3635,7 +3636,7 @@ size_t ksize(const void *object) · 22 unmodified lines
36353636 size_t size = __ksize(object);
36363637 /* We assume that ksize callers could use whole allocated area,
36373638 so we need unpoison this area. */
3638− kasan_krealloc(object, size);
3639+ kasan_krealloc(object, size, GFP_NOWAIT);
36393640 return size;
36403641 }
36413642 EXPORT_SYMBOL(ksize);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t