Pull uaccess futex cleanups for Al Viro: Consolidate access_ok() usage and the futex uaccess function zoo.
Unified · Split
| @@ -31,7 +31,8 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 30 unmodified lines | |||
| 31 | { | 31 | { |
| 32 | int oldval = 0, ret; | 32 | int oldval = 0, ret; |
| 33 | 33 | ||
| 34 | pagefault_disable(); | 34 | if (!access_ok(uaddr, sizeof(u32))) |
| 35 | return -EFAULT; | ||
| 35 | 36 | ||
| 36 | switch (op) { | 37 | switch (op) { |
| 37 | case FUTEX_OP_SET: | 38 | case FUTEX_OP_SET: |
| @@ -53,8 +54,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 15 unmodified lines | |||
| 53 | ret = -ENOSYS; | 54 | ret = -ENOSYS; |
| 54 | } | 55 | } |
| 55 | 56 | ||
| 56 | pagefault_enable(); | ||
| 57 | |||
| 58 | if (!ret) | 57 | if (!ret) |
| 59 | *oval = oldval; | 58 | *oval = oldval; |
| 60 | 59 | ||
| @@ -75,10 +75,12 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 74 unmodified lines | |||
| 75 | { | 75 | { |
| 76 | int oldval = 0, ret; | 76 | int oldval = 0, ret; |
| 77 | 77 | ||
| 78 | if (!access_ok(uaddr, sizeof(u32))) | ||
| 79 | return -EFAULT; | ||
| 80 | |||
| 78 | #ifndef CONFIG_ARC_HAS_LLSC | 81 | #ifndef CONFIG_ARC_HAS_LLSC |
| 79 | preempt_disable(); /* to guarantee atomic r-m-w of futex op */ | 82 | preempt_disable(); /* to guarantee atomic r-m-w of futex op */ |
| 80 | #endif | 83 | #endif |
| 81 | pagefault_disable(); | ||
| 82 | 84 | ||
| 83 | switch (op) { | 85 | switch (op) { |
| 84 | case FUTEX_OP_SET: | 86 | case FUTEX_OP_SET: |
| @@ -101,7 +103,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 16 unmodified lines | |||
| 101 | ret = -ENOSYS; | 103 | ret = -ENOSYS; |
| 102 | } | 104 | } |
| 103 | 105 | ||
| 104 | pagefault_enable(); | ||
| 105 | #ifndef CONFIG_ARC_HAS_LLSC | 106 | #ifndef CONFIG_ARC_HAS_LLSC |
| 106 | preempt_enable(); | 107 | preempt_enable(); |
| 107 | #endif | 108 | #endif |
| @@ -134,10 +134,12 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 133 unmodified lines | |||
| 134 | { | 134 | { |
| 135 | int oldval = 0, ret, tmp; | 135 | int oldval = 0, ret, tmp; |
| 136 | 136 | ||
| 137 | if (!access_ok(uaddr, sizeof(u32))) | ||
| 138 | return -EFAULT; | ||
| 139 | |||
| 137 | #ifndef CONFIG_SMP | 140 | #ifndef CONFIG_SMP |
| 138 | preempt_disable(); | 141 | preempt_disable(); |
| 139 | #endif | 142 | #endif |
| 140 | pagefault_disable(); | ||
| 141 | 143 | ||
| 142 | switch (op) { | 144 | switch (op) { |
| 143 | case FUTEX_OP_SET: | 145 | case FUTEX_OP_SET: |
| @@ -159,7 +161,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 15 unmodified lines | |||
| 159 | ret = -ENOSYS; | 161 | ret = -ENOSYS; |
| 160 | } | 162 | } |
| 161 | 163 | ||
| 162 | pagefault_enable(); | ||
| 163 | #ifndef CONFIG_SMP | 164 | #ifndef CONFIG_SMP |
| 164 | preempt_enable(); | 165 | preempt_enable(); |
| 165 | #endif | 166 | #endif |
| @@ -48,7 +48,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr) · 47 unmodified lines | |||
| 48 | int oldval = 0, ret, tmp; | 48 | int oldval = 0, ret, tmp; |
| 49 | u32 __user *uaddr = __uaccess_mask_ptr(_uaddr); | 49 | u32 __user *uaddr = __uaccess_mask_ptr(_uaddr); |
| 50 | 50 | ||
| 51 | pagefault_disable(); | 51 | if (!access_ok(_uaddr, sizeof(u32))) |
| 52 | return -EFAULT; | ||
| 52 | 53 | ||
| 53 | switch (op) { | 54 | switch (op) { |
| 54 | case FUTEX_OP_SET: | 55 | case FUTEX_OP_SET: |
| @@ -75,8 +76,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr) · 20 unmodified lines | |||
| 75 | ret = -ENOSYS; | 76 | ret = -ENOSYS; |
| 76 | } | 77 | } |
| 77 | 78 | ||
| 78 | pagefault_enable(); | ||
| 79 | |||
| 80 | if (!ret) | 79 | if (!ret) |
| 81 | *oval = oldval; | 80 | *oval = oldval; |
| 82 | 81 | ||
| @@ -36,7 +36,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 35 unmodified lines | |||
| 36 | { | 36 | { |
| 37 | int oldval = 0, ret; | 37 | int oldval = 0, ret; |
| 38 | 38 | ||
| 39 | pagefault_disable(); | 39 | if (!access_ok(uaddr, sizeof(u32))) |
| 40 | return -EFAULT; | ||
| 40 | 41 | ||
| 41 | switch (op) { | 42 | switch (op) { |
| 42 | case FUTEX_OP_SET: | 43 | case FUTEX_OP_SET: |
| @@ -62,8 +63,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 19 unmodified lines | |||
| 62 | ret = -ENOSYS; | 63 | ret = -ENOSYS; |
| 63 | } | 64 | } |
| 64 | 65 | ||
| 65 | pagefault_enable(); | ||
| 66 | |||
| 67 | if (!ret) | 66 | if (!ret) |
| 68 | *oval = oldval; | 67 | *oval = oldval; |
| 69 | 68 | ||
| @@ -50,7 +50,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 49 unmodified lines | |||
| 50 | { | 50 | { |
| 51 | int oldval = 0, ret; | 51 | int oldval = 0, ret; |
| 52 | 52 | ||
| 53 | pagefault_disable(); | 53 | if (!access_ok(uaddr, sizeof(u32))) |
| 54 | return -EFAULT; | ||
| 54 | 55 | ||
| 55 | switch (op) { | 56 | switch (op) { |
| 56 | case FUTEX_OP_SET: | 57 | case FUTEX_OP_SET: |
| @@ -74,8 +75,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 17 unmodified lines | |||
| 74 | ret = -ENOSYS; | 75 | ret = -ENOSYS; |
| 75 | } | 76 | } |
| 76 | 77 | ||
| 77 | pagefault_enable(); | ||
| 78 | |||
| 79 | if (!ret) | 78 | if (!ret) |
| 80 | *oval = oldval; | 79 | *oval = oldval; |
| 81 | 80 | ||
| @@ -34,7 +34,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 33 unmodified lines | |||
| 34 | { | 34 | { |
| 35 | int oldval = 0, ret; | 35 | int oldval = 0, ret; |
| 36 | 36 | ||
| 37 | pagefault_disable(); | 37 | if (!access_ok(uaddr, sizeof(u32))) |
| 38 | return -EFAULT; | ||
| 38 | 39 | ||
| 39 | switch (op) { | 40 | switch (op) { |
| 40 | case FUTEX_OP_SET: | 41 | case FUTEX_OP_SET: |
| @@ -56,8 +57,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 15 unmodified lines | |||
| 56 | ret = -ENOSYS; | 57 | ret = -ENOSYS; |
| 57 | } | 58 | } |
| 58 | 59 | ||
| 59 | pagefault_enable(); | ||
| 60 | |||
| 61 | if (!ret) | 60 | if (!ret) |
| 62 | *oval = oldval; | 61 | *oval = oldval; |
| 63 | 62 | ||
| @@ -89,7 +89,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 88 unmodified lines | |||
| 89 | { | 89 | { |
| 90 | int oldval = 0, ret; | 90 | int oldval = 0, ret; |
| 91 | 91 | ||
| 92 | pagefault_disable(); | 92 | if (!access_ok(uaddr, sizeof(u32))) |
| 93 | return -EFAULT; | ||
| 93 | 94 | ||
| 94 | switch (op) { | 95 | switch (op) { |
| 95 | case FUTEX_OP_SET: | 96 | case FUTEX_OP_SET: |
| @@ -116,8 +117,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 20 unmodified lines | |||
| 116 | ret = -ENOSYS; | 117 | ret = -ENOSYS; |
| 117 | } | 118 | } |
| 118 | 119 | ||
| 119 | pagefault_enable(); | ||
| 120 | |||
| 121 | if (!ret) | 120 | if (!ret) |
| 122 | *oval = oldval; | 121 | *oval = oldval; |
| 123 | 122 | ||
| @@ -66,8 +66,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 65 unmodified lines | |||
| 66 | { | 66 | { |
| 67 | int oldval = 0, ret; | 67 | int oldval = 0, ret; |
| 68 | 68 | ||
| 69 | 69 | if (!access_ok(uaddr, sizeof(u32))) | |
| 70 | pagefault_disable(); | 70 | return -EFAULT; |
| 71 | switch (op) { | 71 | switch (op) { |
| 72 | case FUTEX_OP_SET: | 72 | case FUTEX_OP_SET: |
| 73 | __futex_atomic_op("move %0, %3", ret, oldval, tmp, uaddr, | 73 | __futex_atomic_op("move %0, %3", ret, oldval, tmp, uaddr, |
| @@ -93,8 +93,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 19 unmodified lines | |||
| 93 | ret = -ENOSYS; | 93 | ret = -ENOSYS; |
| 94 | } | 94 | } |
| 95 | 95 | ||
| 96 | pagefault_enable(); | ||
| 97 | |||
| 98 | if (!ret) | 96 | if (!ret) |
| 99 | *oval = oldval; | 97 | *oval = oldval; |
| 100 | 98 | ||
| @@ -35,7 +35,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 34 unmodified lines | |||
| 35 | { | 35 | { |
| 36 | int oldval = 0, ret; | 36 | int oldval = 0, ret; |
| 37 | 37 | ||
| 38 | pagefault_disable(); | 38 | if (!access_ok(uaddr, sizeof(u32))) |
| 39 | return -EFAULT; | ||
| 39 | 40 | ||
| 40 | switch (op) { | 41 | switch (op) { |
| 41 | case FUTEX_OP_SET: | 42 | case FUTEX_OP_SET: |
| @@ -57,8 +58,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 15 unmodified lines | |||
| 57 | ret = -ENOSYS; | 58 | ret = -ENOSYS; |
| 58 | } | 59 | } |
| 59 | 60 | ||
| 60 | pagefault_enable(); | ||
| 61 | |||
| 62 | if (!ret) | 61 | if (!ret) |
| 63 | *oval = oldval; | 62 | *oval = oldval; |
| 64 | 63 | ||
| @@ -40,7 +40,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 39 unmodified lines | |||
| 40 | u32 tmp; | 40 | u32 tmp; |
| 41 | 41 | ||
| 42 | _futex_spin_lock_irqsave(uaddr, &flags); | 42 | _futex_spin_lock_irqsave(uaddr, &flags); |
| 43 | pagefault_disable(); | ||
| 44 | 43 | ||
| 45 | ret = -EFAULT; | 44 | ret = -EFAULT; |
| 46 | if (unlikely(get_user(oldval, uaddr) != 0)) | 45 | if (unlikely(get_user(oldval, uaddr) != 0)) |
| @@ -73,7 +72,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 26 unmodified lines | |||
| 73 | ret = -EFAULT; | 72 | ret = -EFAULT; |
| 74 | 73 | ||
| 75 | out_pagefault_enable: | 74 | out_pagefault_enable: |
| 76 | pagefault_enable(); | ||
| 77 | _futex_spin_unlock_irqrestore(uaddr, &flags); | 75 | _futex_spin_unlock_irqrestore(uaddr, &flags); |
| 78 | 76 | ||
| 79 | if (!ret) | 77 | if (!ret) |
| @@ -35,8 +35,9 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 34 unmodified lines | |||
| 35 | { | 35 | { |
| 36 | int oldval = 0, ret; | 36 | int oldval = 0, ret; |
| 37 | 37 | ||
| 38 | if (!access_ok(uaddr, sizeof(u32))) | ||
| 39 | return -EFAULT; | ||
| 38 | allow_read_write_user(uaddr, uaddr, sizeof(*uaddr)); | 40 | allow_read_write_user(uaddr, uaddr, sizeof(*uaddr)); |
| 39 | pagefault_disable(); | ||
| 40 | 41 | ||
| 41 | switch (op) { | 42 | switch (op) { |
| 42 | case FUTEX_OP_SET: | 43 | case FUTEX_OP_SET: |
| @@ -58,8 +59,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 15 unmodified lines | |||
| 58 | ret = -ENOSYS; | 59 | ret = -ENOSYS; |
| 59 | } | 60 | } |
| 60 | 61 | ||
| 61 | pagefault_enable(); | ||
| 62 | |||
| 63 | *oval = oldval; | 62 | *oval = oldval; |
| 64 | 63 | ||
| 65 | prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr)); | 64 | prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr)); |
| @@ -46,7 +46,8 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 45 unmodified lines | |||
| 46 | { | 46 | { |
| 47 | int oldval = 0, ret = 0; | 47 | int oldval = 0, ret = 0; |
| 48 | 48 | ||
| 49 | pagefault_disable(); | 49 | if (!access_ok(uaddr, sizeof(u32))) |
| 50 | return -EFAULT; | ||
| 50 | 51 | ||
| 51 | switch (op) { | 52 | switch (op) { |
| 52 | case FUTEX_OP_SET: | 53 | case FUTEX_OP_SET: |
| @@ -73,8 +74,6 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) · 20 unmodified lines | |||
| 73 | ret = -ENOSYS; | 74 | ret = -ENOSYS; |
| 74 | } | 75 | } |
| 75 | 76 | ||
| 76 | pagefault_enable(); | ||
| 77 | |||
| 78 | if (!ret) | 77 | if (!ret) |
| 79 | *oval = oldval; | 78 | *oval = oldval; |
| 80 | 79 | ||
| @@ -29,7 +29,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 28 unmodified lines | |||
| 29 | mm_segment_t old_fs; | 29 | mm_segment_t old_fs; |
| 30 | 30 | ||
| 31 | old_fs = enable_sacf_uaccess(); | 31 | old_fs = enable_sacf_uaccess(); |
| 32 | pagefault_disable(); | ||
| 33 | switch (op) { | 32 | switch (op) { |
| 34 | case FUTEX_OP_SET: | 33 | case FUTEX_OP_SET: |
| 35 | __futex_atomic_op("lr %2,%5\n", | 34 | __futex_atomic_op("lr %2,%5\n", |
| @@ -54,7 +53,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 18 unmodified lines | |||
| 54 | default: | 53 | default: |
| 55 | ret = -ENOSYS; | 54 | ret = -ENOSYS; |
| 56 | } | 55 | } |
| 57 | pagefault_enable(); | ||
| 58 | disable_sacf_uaccess(old_fs); | 56 | disable_sacf_uaccess(old_fs); |
| 59 | 57 | ||
| 60 | if (!ret) | 58 | if (!ret) |
| @@ -34,8 +34,6 @@ static inline int arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, · 33 unmodified lines | |||
| 34 | u32 oldval, newval, prev; | 34 | u32 oldval, newval, prev; |
| 35 | int ret; | 35 | int ret; |
| 36 | 36 | ||
| 37 | pagefault_disable(); | ||
| 38 | |||
| 39 | do { | 37 | do { |
| 40 | ret = get_user(oldval, uaddr); | 38 | ret = get_user(oldval, uaddr); |
| 41 | 39 | ||
| @@ -67,8 +65,6 @@ static inline int arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, · 25 unmodified lines | |||
| 67 | ret = futex_atomic_cmpxchg_inatomic(&prev, uaddr, oldval, newval); | 65 | ret = futex_atomic_cmpxchg_inatomic(&prev, uaddr, oldval, newval); |
| 68 | } while (!ret && prev != oldval); | 66 | } while (!ret && prev != oldval); |
| 69 | 67 | ||
| 70 | pagefault_enable(); | ||
| 71 | |||
| 72 | if (!ret) | 68 | if (!ret) |
| 73 | *oval = oldval; | 69 | *oval = oldval; |
| 74 | 70 | ||
| @@ -38,8 +38,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 37 unmodified lines | |||
| 38 | if (unlikely((((unsigned long) uaddr) & 0x3UL))) | 38 | if (unlikely((((unsigned long) uaddr) & 0x3UL))) |
| 39 | return -EINVAL; | 39 | return -EINVAL; |
| 40 | 40 | ||
| 41 | pagefault_disable(); | ||
| 42 | |||
| 43 | switch (op) { | 41 | switch (op) { |
| 44 | case FUTEX_OP_SET: | 42 | case FUTEX_OP_SET: |
| 45 | __futex_cas_op("mov\t%4, %1", ret, oldval, uaddr, oparg); | 43 | __futex_cas_op("mov\t%4, %1", ret, oldval, uaddr, oparg); |
| @@ -60,8 +58,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 14 unmodified lines | |||
| 60 | ret = -ENOSYS; | 58 | ret = -ENOSYS; |
| 61 | } | 59 | } |
| 62 | 60 | ||
| 63 | pagefault_enable(); | ||
| 64 | |||
| 65 | if (!ret) | 61 | if (!ret) |
| 66 | *oval = oldval; | 62 | *oval = oldval; |
| 67 | 63 | ||
| @@ -12,76 +12,103 @@ · 11 unmodified lines | |||
| 12 | #include <asm/processor.h> | 12 | #include <asm/processor.h> |
| 13 | #include <asm/smap.h> | 13 | #include <asm/smap.h> |
| 14 | 14 | ||
| 15 | #define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \ | 15 | #define unsafe_atomic_op1(insn, oval, uaddr, oparg, label) \ |
| 16 | asm volatile("\t" ASM_STAC "\n" \ | 16 | do { \ |
| 17 | "1:\t" insn "\n" \ | 17 | int oldval = 0, ret; \ |
| 18 | "2:\t" ASM_CLAC "\n" \ | 18 | asm volatile("1:\t" insn "\n" \ |
| 19 | "2:\n" \ | ||
| 19 | "\t.section .fixup,\"ax\"\n" \ | 20 | "\t.section .fixup,\"ax\"\n" \ |
| 20 | "3:\tmov\t%3, %1\n" \ | 21 | "3:\tmov\t%3, %1\n" \ |
| 21 | "\tjmp\t2b\n" \ | 22 | "\tjmp\t2b\n" \ |
| 22 | "\t.previous\n" \ | 23 | "\t.previous\n" \ |
| 23 | _ASM_EXTABLE_UA(1b, 3b) \ | 24 | _ASM_EXTABLE_UA(1b, 3b) \ |
| 24 | : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ | 25 | : "=r" (oldval), "=r" (ret), "+m" (*uaddr) \ |
| 25 | : "i" (-EFAULT), "0" (oparg), "1" (0)) | 26 | : "i" (-EFAULT), "0" (oparg), "1" (0)); \ |
| 27 | if (ret) \ | ||
| 28 | goto label; \ | ||
| 29 | *oval = oldval; \ | ||
| 30 | } while(0) | ||
| 26 | 31 | ||
| 27 | #define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \ | 32 | |
| 28 | asm volatile("\t" ASM_STAC "\n" \ | 33 | #define unsafe_atomic_op2(insn, oval, uaddr, oparg, label) \ |
| 29 | "1:\tmovl %2, %0\n" \ | 34 | do { \ |
| 30 | "\tmovl\t%0, %3\n" \ | 35 | int oldval = 0, ret, tem; \ |
| 36 | asm volatile("1:\tmovl %2, %0\n" \ | ||
| 37 | "2:\tmovl\t%0, %3\n" \ | ||
| 31 | "\t" insn "\n" \ | 38 | "\t" insn "\n" \ |
| 32 | "2:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ | 39 | "3:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ |
| 33 | "\tjnz\t1b\n" \ | 40 | "\tjnz\t2b\n" \ |
| 34 | "3:\t" ASM_CLAC "\n" \ | 41 | "4:\n" \ |
| 35 | "\t.section .fixup,\"ax\"\n" \ | 42 | "\t.section .fixup,\"ax\"\n" \ |
| 36 | "4:\tmov\t%5, %1\n" \ | 43 | "5:\tmov\t%5, %1\n" \ |
| 37 | "\tjmp\t3b\n" \ | 44 | "\tjmp\t4b\n" \ |
| 38 | "\t.previous\n" \ | 45 | "\t.previous\n" \ |
| 39 | _ASM_EXTABLE_UA(1b, 4b) \ | 46 | _ASM_EXTABLE_UA(1b, 5b) \ |
| 40 | _ASM_EXTABLE_UA(2b, 4b) \ | 47 | _ASM_EXTABLE_UA(3b, 5b) \ |
| 41 | : "=&a" (oldval), "=&r" (ret), \ | 48 | : "=&a" (oldval), "=&r" (ret), \ |
| 42 | "+m" (*uaddr), "=&r" (tem) \ | 49 | "+m" (*uaddr), "=&r" (tem) \ |
| 43 | : "r" (oparg), "i" (-EFAULT), "1" (0)) | 50 | : "r" (oparg), "i" (-EFAULT), "1" (0)); \ |
| 51 | if (ret) \ | ||
| 52 | goto label; \ | ||
| 53 | *oval = oldval; \ | ||
| 54 | } while(0) | ||
| 44 | 55 | ||
| 45 | static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, | 56 | static __always_inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, |
| 46 | u32 __user *uaddr) | 57 | u32 __user *uaddr) |
| 47 | { | 58 | { |
| 48 | int oldval = 0, ret, tem; | 59 | if (!user_access_begin(uaddr, sizeof(u32))) |
| 49 | 60 | return -EFAULT; | |
| 50 | pagefault_disable(); | ||
| 51 | 61 | ||
| 52 | switch (op) { | 62 | switch (op) { |
| 53 | case FUTEX_OP_SET: | 63 | case FUTEX_OP_SET: |
| 54 | __futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg); | 64 | unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault); |
| 55 | break; | 65 | break; |
| 56 | case FUTEX_OP_ADD: | 66 | case FUTEX_OP_ADD: |
| 57 | __futex_atomic_op1(LOCK_PREFIX "xaddl %0, %2", ret, oldval, | 67 | unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval, |
| 58 | uaddr, oparg); | 68 | uaddr, oparg, Efault); |
| 59 | break; | 69 | break; |
| 60 | case FUTEX_OP_OR: | 70 | case FUTEX_OP_OR: |
| 61 | __futex_atomic_op2("orl %4, %3", ret, oldval, uaddr, oparg); | 71 | unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault); |
| 62 | break; | 72 | break; |
| 63 | case FUTEX_OP_ANDN: | 73 | case FUTEX_OP_ANDN: |
| 64 | __futex_atomic_op2("andl %4, %3", ret, oldval, uaddr, ~oparg); | 74 | unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault); |
| 65 | break; | 75 | break; |
| 66 | case FUTEX_OP_XOR: | 76 | case FUTEX_OP_XOR: |
| 67 | __futex_atomic_op2("xorl %4, %3", ret, oldval, uaddr, oparg); | 77 | unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault); |
| 68 | break; | 78 | break; |
| 69 | default: | 79 | default: |
| 70 | ret = -ENOSYS; | 80 | user_access_end(); |
| 81 | return -ENOSYS; | ||
| 71 | } | 82 | } |
| 72 | 83 | user_access_end(); | |
| 73 | pagefault_enable(); | 84 | return 0; |
| 74 | 85 | Efault: | |
| 75 | if (!ret) | 86 | user_access_end(); |
| 76 | *oval = oldval; | 87 | return -EFAULT; |
| 77 | |||
| 78 | return ret; | ||
| 79 | } | 88 | } |
| 80 | 89 | ||
| 81 | static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, | 90 | static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, |
| 82 | u32 oldval, u32 newval) | 91 | u32 oldval, u32 newval) |
| 83 | { | 92 | { |
| 84 | return user_atomic_cmpxchg_inatomic(uval, uaddr, oldval, newval); | 93 | int ret = 0; |
| 94 | |||
| 95 | if (!user_access_begin(uaddr, sizeof(u32))) | ||
| 96 | return -EFAULT; | ||
| 97 | asm volatile("\n" | ||
| 98 | "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" | ||
| 99 | "2:\n" | ||
| 100 | "\t.section .fixup, \"ax\"\n" | ||
| 101 | "3:\tmov %3, %0\n" | ||
| 102 | "\tjmp 2b\n" | ||
| 103 | "\t.previous\n" | ||
| 104 | _ASM_EXTABLE_UA(1b, 3b) | ||
| 105 | : "+r" (ret), "=a" (oldval), "+m" (*uaddr) | ||
| 106 | : "i" (-EFAULT), "r" (newval), "1" (oldval) | ||
| 107 | : "memory" | ||
| 108 | ); | ||
| 109 | user_access_end(); | ||
| 110 | *uval = oldval; | ||
| 111 | return ret; | ||
| 85 | } | 112 | } |
| 86 | 113 | ||
| 87 | #endif | 114 | #endif |
| @@ -584,99 +584,6 @@ extern __must_check long strnlen_user(const char __user *str, long n); · 583 unmodified lines | |||
| 584 | unsigned long __must_check clear_user(void __user *mem, unsigned long len); | 584 | unsigned long __must_check clear_user(void __user *mem, unsigned long len); |
| 585 | unsigned long __must_check __clear_user(void __user *mem, unsigned long len); | 585 | unsigned long __must_check __clear_user(void __user *mem, unsigned long len); |
| 586 | 586 | ||
| 587 | extern void __cmpxchg_wrong_size(void) | ||
| 588 | __compiletime_error("Bad argument size for cmpxchg"); | ||
| 589 | |||
| 590 | #define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size) \ | ||
| 591 | ({ \ | ||
| 592 | int __ret = 0; \ | ||
| 593 | __typeof__(*(ptr)) __old = (old); \ | ||
| 594 | __typeof__(*(ptr)) __new = (new); \ | ||
| 595 | __uaccess_begin_nospec(); \ | ||
| 596 | switch (size) { \ | ||
| 597 | case 1: \ | ||
| 598 | { \ | ||
| 599 | asm volatile("\n" \ | ||
| 600 | "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \ | ||
| 601 | "2:\n" \ | ||
| 602 | "\t.section .fixup, \"ax\"\n" \ | ||
| 603 | "3:\tmov %3, %0\n" \ | ||
| 604 | "\tjmp 2b\n" \ | ||
| 605 | "\t.previous\n" \ | ||
| 606 | _ASM_EXTABLE_UA(1b, 3b) \ | ||
| 607 | : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ | ||
| 608 | : "i" (-EFAULT), "q" (__new), "1" (__old) \ | ||
| 609 | : "memory" \ | ||
| 610 | ); \ | ||
| 611 | break; \ | ||
| 612 | } \ | ||
| 613 | case 2: \ | ||
| 614 | { \ | ||
| 615 | asm volatile("\n" \ | ||
| 616 | "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \ | ||
| 617 | "2:\n" \ | ||
| 618 | "\t.section .fixup, \"ax\"\n" \ | ||
| 619 | "3:\tmov %3, %0\n" \ | ||
| 620 | "\tjmp 2b\n" \ | ||
| 621 | "\t.previous\n" \ | ||
| 622 | _ASM_EXTABLE_UA(1b, 3b) \ | ||
| 623 | : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ | ||
| 624 | : "i" (-EFAULT), "r" (__new), "1" (__old) \ | ||
| 625 | : "memory" \ | ||
| 626 | ); \ | ||
| 627 | break; \ | ||
| 628 | } \ | ||
| 629 | case 4: \ | ||
| 630 | { \ | ||
| 631 | asm volatile("\n" \ | ||
| 632 | "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \ | ||
| 633 | "2:\n" \ | ||
| 634 | "\t.section .fixup, \"ax\"\n" \ | ||
| 635 | "3:\tmov %3, %0\n" \ | ||
| 636 | "\tjmp 2b\n" \ | ||
| 637 | "\t.previous\n" \ | ||
| 638 | _ASM_EXTABLE_UA(1b, 3b) \ | ||
| 639 | : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ | ||
| 640 | : "i" (-EFAULT), "r" (__new), "1" (__old) \ | ||
| 641 | : "memory" \ | ||
| 642 | ); \ | ||
| 643 | break; \ | ||
| 644 | } \ | ||
| 645 | case 8: \ | ||
| 646 | { \ | ||
| 647 | if (!IS_ENABLED(CONFIG_X86_64)) \ | ||
| 648 | __cmpxchg_wrong_size(); \ | ||
| 649 | \ | ||
| 650 | asm volatile("\n" \ | ||
| 651 | "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ | ||
| 652 | "2:\n" \ | ||
| 653 | "\t.section .fixup, \"ax\"\n" \ | ||
| 654 | "3:\tmov %3, %0\n" \ | ||
| 655 | "\tjmp 2b\n" \ | ||
| 656 | "\t.previous\n" \ | ||
| 657 | _ASM_EXTABLE_UA(1b, 3b) \ | ||
| 658 | : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ | ||
| 659 | : "i" (-EFAULT), "r" (__new), "1" (__old) \ | ||
| 660 | : "memory" \ | ||
| 661 | ); \ | ||
| 662 | break; \ | ||
| 663 | } \ | ||
| 664 | default: \ | ||
| 665 | __cmpxchg_wrong_size(); \ | ||
| 666 | } \ | ||
| 667 | __uaccess_end(); \ | ||
| 668 | *(uval) = __old; \ | ||
| 669 | __ret; \ | ||
| 670 | }) | ||
| 671 | |||
| 672 | #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \ | ||
| 673 | ({ \ | ||
| 674 | access_ok((ptr), sizeof(*(ptr))) ? \ | ||
| 675 | __user_atomic_cmpxchg_inatomic((uval), (ptr), \ | ||
| 676 | (old), (new), sizeof(*(ptr))) : \ | ||
| 677 | -EFAULT; \ | ||
| 678 | }) | ||
| 679 | |||
| 680 | /* | 587 | /* |
| 681 | * movsl can be slow when source and dest are not both 8-byte aligned | 588 | * movsl can be slow when source and dest are not both 8-byte aligned |
| 682 | */ | 589 | */ |
| @@ -72,7 +72,8 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 71 unmodified lines | |||
| 72 | #if XCHAL_HAVE_S32C1I || XCHAL_HAVE_EXCLUSIVE | 72 | #if XCHAL_HAVE_S32C1I || XCHAL_HAVE_EXCLUSIVE |
| 73 | int oldval = 0, ret; | 73 | int oldval = 0, ret; |
| 74 | 74 | ||
| 75 | pagefault_disable(); | 75 | if (!access_ok(uaddr, sizeof(u32))) |
| 76 | return -EFAULT; | ||
| 76 | 77 | ||
| 77 | switch (op) { | 78 | switch (op) { |
| 78 | case FUTEX_OP_SET: | 79 | case FUTEX_OP_SET: |
| @@ -99,8 +100,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, · 20 unmodified lines | |||
| 99 | ret = -ENOSYS; | 100 | ret = -ENOSYS; |
| 100 | } | 101 | } |
| 101 | 102 | ||
| 102 | pagefault_enable(); | ||
| 103 | |||
| 104 | if (!ret) | 103 | if (!ret) |
| 105 | *oval = oldval; | 104 | *oval = oldval; |
| 106 | 105 | ||
| @@ -34,7 +34,6 @@ arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr) · 33 unmodified lines | |||
| 34 | u32 tmp; | 34 | u32 tmp; |
| 35 | 35 | ||
| 36 | preempt_disable(); | 36 | preempt_disable(); |
| 37 | pagefault_disable(); | ||
| 38 | 37 | ||
| 39 | ret = -EFAULT; | 38 | ret = -EFAULT; |
| 40 | if (unlikely(get_user(oldval, uaddr) != 0)) | 39 | if (unlikely(get_user(oldval, uaddr) != 0)) |
| @@ -67,7 +66,6 @@ arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr) · 26 unmodified lines | |||
| 67 | ret = -EFAULT; | 66 | ret = -EFAULT; |
| 68 | 67 | ||
| 69 | out_pagefault_enable: | 68 | out_pagefault_enable: |
| 70 | pagefault_enable(); | ||
| 71 | preempt_enable(); | 69 | preempt_enable(); |
| 72 | 70 | ||
| 73 | if (ret == 0) | 71 | if (ret == 0) |
| @@ -1665,10 +1665,9 @@ static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr) · 1664 unmodified lines | |||
| 1665 | oparg = 1 << oparg; | 1665 | oparg = 1 << oparg; |
| 1666 | } | 1666 | } |
| 1667 | 1667 | ||
| 1668 | if (!access_ok(uaddr, sizeof(u32))) | 1668 | pagefault_disable(); |
| 1669 | return -EFAULT; | ||
| 1670 | |||
| 1671 | ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr); | 1669 | ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr); |
| 1670 | pagefault_enable(); | ||
| 1672 | if (ret) | 1671 | if (ret) |
| 1673 | return ret; | 1672 | return ret; |
| 1674 | 1673 | ||
| @@ -478,6 +478,7 @@ static const char *uaccess_safe_builtin[] = { · 477 unmodified lines | |||
| 478 | "__sanitizer_cov_trace_cmp2", | 478 | "__sanitizer_cov_trace_cmp2", |
| 479 | "__sanitizer_cov_trace_cmp4", | 479 | "__sanitizer_cov_trace_cmp4", |
| 480 | "__sanitizer_cov_trace_cmp8", | 480 | "__sanitizer_cov_trace_cmp8", |
| 481 | "__sanitizer_cov_trace_switch", | ||
| 481 | /* UBSAN */ | 482 | /* UBSAN */ |
| 482 | "ubsan_type_mismatch_common", | 483 | "ubsan_type_mismatch_common", |
| 483 | "__ubsan_handle_type_mismatch", | 484 | "__ubsan_handle_type_mismatch", |