History

Merge tag 'locking-urgent-2020-06-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull atomics rework from Thomas Gleixner: "Peter Zijlstras rework of atomics and fallbacks. This solves two problems: 1) Compilers uninline small atomic_* static inline functions which can expose them to instrumentation. 2) The instrumentation of atomic primitives was done at the architecture level while composites or fallbacks were provided at the generic level. As a result there are no uninstrumented variants of the fallbacks. Both issues were in the way of fully isolating fragile entry code pathes and especially the text poke int3 handler which is prone to an endless recursion problem when anything in that code path is about to be instrumented. This was always a problem, but got elevated due to the new batch mode updates of tracing. The solution is to mark the functions __always_inline and to flip the fallback and instrumentation so the non-instrumented variants are at the architecture level and the instrumentation is done in generic code. The latter introduces another fallback variant which will go away once all architectures have been moved over to arch_atomic_*" * tag 'locking-urgent-2020-06-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/atomics: Flip fallbacks and instrumentation asm-generic/atomic: Use __always_inline for fallback wrappers

Linus Torvalds committed 6 years ago

Commit 9716e572 parents b1a6274 37f817328 files · +2594 −269Browse files

Showing the changes to scripts (21 of 28 files). Show all

Unified · Split

21 files
  1. scripts/atomic/fallbacks/acquire +3 −3
  2. scripts/atomic/fallbacks/add_negative +4 −4
  3. scripts/atomic/fallbacks/add_unless +4 −4
  4. scripts/atomic/fallbacks/andnot +3 −3
  5. scripts/atomic/fallbacks/dec +3 −3
  6. scripts/atomic/fallbacks/dec_and_test +4 −4
  7. scripts/atomic/fallbacks/dec_if_positive +4 −4
  8. scripts/atomic/fallbacks/dec_unless_positive +4 −4
  9. scripts/atomic/fallbacks/fence +3 −3
  10. scripts/atomic/fallbacks/fetch_add_unless +5 −5
  11. scripts/atomic/fallbacks/inc +3 −3
  12. scripts/atomic/fallbacks/inc_and_test +4 −4
  13. scripts/atomic/fallbacks/inc_not_zero +4 −4
  14. scripts/atomic/fallbacks/inc_unless_negative +4 −4
  15. scripts/atomic/fallbacks/read_acquire +2 −2
  16. scripts/atomic/fallbacks/release +3 −3
  17. scripts/atomic/fallbacks/set_release +2 −2
  18. scripts/atomic/fallbacks/sub_and_test +4 −4
  19. scripts/atomic/fallbacks/try_cmpxchg +3 −3
  20. scripts/atomic/gen-atomic-fallback.sh +15 −16
  21. scripts/atomic/gen-atomics.sh +3 −2
scripts/atomic/fallbacks/acquire+3−3
@@ -1,8 +1,8 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}${name}${sfx}_acquire(${params})
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}${name}${sfx}_acquire(${params})
44 {
5− ${ret} ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});
5+ ${ret} ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
66 __atomic_acquire_fence();
77 return ret;
88 }
scripts/atomic/fallbacks/add_negative+4−4
@@ -1,6 +1,6 @@
11 cat <<EOF
22 /**
3− * ${atomic}_add_negative - add and test if negative
3+ * ${arch}${atomic}_add_negative - add and test if negative
44 * @i: integer value to add
55 * @v: pointer of type ${atomic}_t
66 *
@@ -8,9 +8,9 @@ cat <<EOF · 1 unmodified line
88 * if the result is negative, or false when
99 * result is greater than or equal to zero.
1010 */
11−static inline bool
12−${atomic}_add_negative(${int} i, ${atomic}_t *v)
11+static __always_inline bool
12+${arch}${atomic}_add_negative(${int} i, ${atomic}_t *v)
1313 {
14− return ${atomic}_add_return(i, v) < 0;
14+ return ${arch}${atomic}_add_return(i, v) < 0;
1515 }
1616 EOF
scripts/atomic/fallbacks/add_unless+4−4
@@ -1,6 +1,6 @@
11 cat << EOF
22 /**
3− * ${atomic}_add_unless - add unless the number is already a given value
3+ * ${arch}${atomic}_add_unless - add unless the number is already a given value
44 * @v: pointer of type ${atomic}_t
55 * @a: the amount to add to v...
66 * @u: ...unless v is equal to u.
@@ -8,9 +8,9 @@ cat << EOF · 1 unmodified line
88 * Atomically adds @a to @v, if @v was not already @u.
99 * Returns true if the addition was done.
1010 */
11−static inline bool
12−${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
11+static __always_inline bool
12+${arch}${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
1313 {
14− return ${atomic}_fetch_add_unless(v, a, u) != u;
14+ return ${arch}${atomic}_fetch_add_unless(v, a, u) != u;
1515 }
1616 EOF
scripts/atomic/fallbacks/andnot+3−3
@@ -1,7 +1,7 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
44 {
5− ${retstmt}${atomic}_${pfx}and${sfx}${order}(~i, v);
5+ ${retstmt}${arch}${atomic}_${pfx}and${sfx}${order}(~i, v);
66 }
77 EOF
scripts/atomic/fallbacks/dec+3−3
@@ -1,7 +1,7 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
44 {
5− ${retstmt}${atomic}_${pfx}sub${sfx}${order}(1, v);
5+ ${retstmt}${arch}${atomic}_${pfx}sub${sfx}${order}(1, v);
66 }
77 EOF
scripts/atomic/fallbacks/dec_and_test+4−4
@@ -1,15 +1,15 @@
11 cat <<EOF
22 /**
3− * ${atomic}_dec_and_test - decrement and test
3+ * ${arch}${atomic}_dec_and_test - decrement and test
44 * @v: pointer of type ${atomic}_t
55 *
66 * Atomically decrements @v by 1 and
77 * returns true if the result is 0, or false for all other
88 * cases.
99 */
10−static inline bool
11−${atomic}_dec_and_test(${atomic}_t *v)
10+static __always_inline bool
11+${arch}${atomic}_dec_and_test(${atomic}_t *v)
1212 {
13− return ${atomic}_dec_return(v) == 0;
13+ return ${arch}${atomic}_dec_return(v) == 0;
1414 }
1515 EOF
scripts/atomic/fallbacks/dec_if_positive+4−4
@@ -1,14 +1,14 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_dec_if_positive(${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_dec_if_positive(${atomic}_t *v)
44 {
5− ${int} dec, c = ${atomic}_read(v);
5+ ${int} dec, c = ${arch}${atomic}_read(v);
66
77 do {
88 dec = c - 1;
99 if (unlikely(dec < 0))
1010 break;
11− } while (!${atomic}_try_cmpxchg(v, &c, dec));
11+ } while (!${arch}${atomic}_try_cmpxchg(v, &c, dec));
1212
1313 return dec;
1414 }
scripts/atomic/fallbacks/dec_unless_positive+4−4
@@ -1,13 +1,13 @@
11 cat <<EOF
2−static inline bool
3−${atomic}_dec_unless_positive(${atomic}_t *v)
2+static __always_inline bool
3+${arch}${atomic}_dec_unless_positive(${atomic}_t *v)
44 {
5− ${int} c = ${atomic}_read(v);
5+ ${int} c = ${arch}${atomic}_read(v);
66
77 do {
88 if (unlikely(c > 0))
99 return false;
10− } while (!${atomic}_try_cmpxchg(v, &c, c - 1));
10+ } while (!${arch}${atomic}_try_cmpxchg(v, &c, c - 1));
1111
1212 return true;
1313 }
scripts/atomic/fallbacks/fence+3−3
@@ -1,10 +1,10 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}${name}${sfx}(${params})
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}${name}${sfx}(${params})
44 {
55 ${ret} ret;
66 __atomic_pre_full_fence();
7− ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});
7+ ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
88 __atomic_post_full_fence();
99 return ret;
1010 }
scripts/atomic/fallbacks/fetch_add_unless+5−5
@@ -1,6 +1,6 @@
11 cat << EOF
22 /**
3− * ${atomic}_fetch_add_unless - add unless the number is already a given value
3+ * ${arch}${atomic}_fetch_add_unless - add unless the number is already a given value
44 * @v: pointer of type ${atomic}_t
55 * @a: the amount to add to v...
66 * @u: ...unless v is equal to u.
@@ -8,15 +8,15 @@ cat << EOF · 1 unmodified line
88 * Atomically adds @a to @v, so long as @v was not already @u.
99 * Returns original value of @v
1010 */
11−static inline ${int}
12−${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
11+static __always_inline ${int}
12+${arch}${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
1313 {
14− ${int} c = ${atomic}_read(v);
14+ ${int} c = ${arch}${atomic}_read(v);
1515
1616 do {
1717 if (unlikely(c == u))
1818 break;
19− } while (!${atomic}_try_cmpxchg(v, &c, c + a));
19+ } while (!${arch}${atomic}_try_cmpxchg(v, &c, c + a));
2020
2121 return c;
2222 }
scripts/atomic/fallbacks/inc+3−3
@@ -1,7 +1,7 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
44 {
5− ${retstmt}${atomic}_${pfx}add${sfx}${order}(1, v);
5+ ${retstmt}${arch}${atomic}_${pfx}add${sfx}${order}(1, v);
66 }
77 EOF
scripts/atomic/fallbacks/inc_and_test+4−4
@@ -1,15 +1,15 @@
11 cat <<EOF
22 /**
3− * ${atomic}_inc_and_test - increment and test
3+ * ${arch}${atomic}_inc_and_test - increment and test
44 * @v: pointer of type ${atomic}_t
55 *
66 * Atomically increments @v by 1
77 * and returns true if the result is zero, or false for all
88 * other cases.
99 */
10−static inline bool
11−${atomic}_inc_and_test(${atomic}_t *v)
10+static __always_inline bool
11+${arch}${atomic}_inc_and_test(${atomic}_t *v)
1212 {
13− return ${atomic}_inc_return(v) == 0;
13+ return ${arch}${atomic}_inc_return(v) == 0;
1414 }
1515 EOF
scripts/atomic/fallbacks/inc_not_zero+4−4
@@ -1,14 +1,14 @@
11 cat <<EOF
22 /**
3− * ${atomic}_inc_not_zero - increment unless the number is zero
3+ * ${arch}${atomic}_inc_not_zero - increment unless the number is zero
44 * @v: pointer of type ${atomic}_t
55 *
66 * Atomically increments @v by 1, if @v is non-zero.
77 * Returns true if the increment was done.
88 */
9−static inline bool
10−${atomic}_inc_not_zero(${atomic}_t *v)
9+static __always_inline bool
10+${arch}${atomic}_inc_not_zero(${atomic}_t *v)
1111 {
12− return ${atomic}_add_unless(v, 1, 0);
12+ return ${arch}${atomic}_add_unless(v, 1, 0);
1313 }
1414 EOF
scripts/atomic/fallbacks/inc_unless_negative+4−4
@@ -1,13 +1,13 @@
11 cat <<EOF
2−static inline bool
3−${atomic}_inc_unless_negative(${atomic}_t *v)
2+static __always_inline bool
3+${arch}${atomic}_inc_unless_negative(${atomic}_t *v)
44 {
5− ${int} c = ${atomic}_read(v);
5+ ${int} c = ${arch}${atomic}_read(v);
66
77 do {
88 if (unlikely(c < 0))
99 return false;
10− } while (!${atomic}_try_cmpxchg(v, &c, c + 1));
10+ } while (!${arch}${atomic}_try_cmpxchg(v, &c, c + 1));
1111
1212 return true;
1313 }
scripts/atomic/fallbacks/read_acquire+2−2
@@ -1,6 +1,6 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_read_acquire(const ${atomic}_t *v)
2+static __always_inline ${ret}
3+${arch}${atomic}_read_acquire(const ${atomic}_t *v)
44 {
55 return smp_load_acquire(&(v)->counter);
66 }
scripts/atomic/fallbacks/release+3−3
@@ -1,8 +1,8 @@
11 cat <<EOF
2−static inline ${ret}
3−${atomic}_${pfx}${name}${sfx}_release(${params})
2+static __always_inline ${ret}
3+${arch}${atomic}_${pfx}${name}${sfx}_release(${params})
44 {
55 __atomic_release_fence();
6− ${retstmt}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
6+ ${retstmt}${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args});
77 }
88 EOF
scripts/atomic/fallbacks/set_release+2−2
@@ -1,6 +1,6 @@
11 cat <<EOF
2−static inline void
3−${atomic}_set_release(${atomic}_t *v, ${int} i)
2+static __always_inline void
3+${arch}${atomic}_set_release(${atomic}_t *v, ${int} i)
44 {
55 smp_store_release(&(v)->counter, i);
66 }
scripts/atomic/fallbacks/sub_and_test+4−4
@@ -1,6 +1,6 @@
11 cat <<EOF
22 /**
3− * ${atomic}_sub_and_test - subtract value from variable and test result
3+ * ${arch}${atomic}_sub_and_test - subtract value from variable and test result
44 * @i: integer value to subtract
55 * @v: pointer of type ${atomic}_t
66 *
@@ -8,9 +8,9 @@ cat <<EOF · 1 unmodified line
88 * true if the result is zero, or false for all
99 * other cases.
1010 */
11−static inline bool
12−${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
11+static __always_inline bool
12+${arch}${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
1313 {
14− return ${atomic}_sub_return(i, v) == 0;
14+ return ${arch}${atomic}_sub_return(i, v) == 0;
1515 }
1616 EOF
scripts/atomic/fallbacks/try_cmpxchg+3−3
@@ -1,9 +1,9 @@
11 cat <<EOF
2−static inline bool
3−${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
2+static __always_inline bool
3+${arch}${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
44 {
55 ${int} r, o = *old;
6− r = ${atomic}_cmpxchg${order}(v, o, new);
6+ r = ${arch}${atomic}_cmpxchg${order}(v, o, new);
77 if (unlikely(r != o))
88 *old = r;
99 return likely(r == o);
scripts/atomic/gen-atomic-fallback.sh+15−16
@@ -2,10 +2,11 @@ · 1 unmodified line
22 # SPDX-License-Identifier: GPL-2.0
33
44 ATOMICDIR=$(dirname $0)
5+ARCH=$2
56
67 . ${ATOMICDIR}/atomic-tbl.sh
78
8−#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...)
9+#gen_template_fallback(template, meta, pfx, name, sfx, order, arch, atomic, int, args...)
910 gen_template_fallback()
1011 {
1112 local template="$1"; shift
@@ -14,10 +15,11 @@ gen_template_fallback() · 2 unmodified lines
1415 local name="$1"; shift
1516 local sfx="$1"; shift
1617 local order="$1"; shift
18+ local arch="$1"; shift
1719 local atomic="$1"; shift
1820 local int="$1"; shift
1921
20− local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
22+ local atomicname="${arch}${atomic}_${pfx}${name}${sfx}${order}"
2123
2224 local ret="$(gen_ret_type "${meta}" "${int}")"
2325 local retstmt="$(gen_ret_stmt "${meta}")"
@@ -32,7 +34,7 @@ gen_template_fallback() · 8 unmodified lines
3234 fi
3335 }
3436
35−#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
37+#gen_proto_fallback(meta, pfx, name, sfx, order, arch, atomic, int, args...)
3638 gen_proto_fallback()
3739 {
3840 local meta="$1"; shift
@@ -56,16 +58,17 @@ cat << EOF · 17 unmodified lines
5658 EOF
5759 }
5860
59−#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
61+#gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...)
6062 gen_proto_order_variants()
6163 {
6264 local meta="$1"; shift
6365 local pfx="$1"; shift
6466 local name="$1"; shift
6567 local sfx="$1"; shift
66− local atomic="$1"
68+ local arch="$1"
69+ local atomic="$2"
6770
68− local basename="${atomic}_${pfx}${name}${sfx}"
71+ local basename="${arch}${atomic}_${pfx}${name}${sfx}"
6972
7073 local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
7174
@@ -94,7 +97,7 @@ gen_proto_order_variants() · 22 unmodified lines
9497 gen_basic_fallbacks "${basename}"
9598
9699 if [ ! -z "${template}" ]; then
97− printf "#endif /* ${atomic}_${pfx}${name}${sfx} */\n\n"
100+ printf "#endif /* ${arch}${atomic}_${pfx}${name}${sfx} */\n\n"
98101 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
99102 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
100103 gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
@@ -149,20 +152,19 @@ cat << EOF · 48 unmodified lines
149152 #ifndef _LINUX_ATOMIC_FALLBACK_H
150153 #define _LINUX_ATOMIC_FALLBACK_H
151154
155+#include <linux/compiler.h>
156+
152157 EOF
153158
154−for xchg in "xchg" "cmpxchg" "cmpxchg64"; do
159+for xchg in "${ARCH}xchg" "${ARCH}cmpxchg" "${ARCH}cmpxchg64"; do
155160 gen_xchg_fallbacks "${xchg}"
156161 done
157162
158163 grep '^[a-z]' "$1" | while read name meta args; do
159− gen_proto "${meta}" "${name}" "atomic" "int" ${args}
164+ gen_proto "${meta}" "${name}" "${ARCH}" "atomic" "int" ${args}
160165 done
161166
162167 cat <<EOF
163−#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
164−#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
165−
166168 #ifdef CONFIG_GENERIC_ATOMIC64
167169 #include <asm-generic/atomic64.h>
168170 #endif
@@ -170,12 +172,9 @@ cat <<EOF · 1 unmodified line
170172 EOF
171173
172174 grep '^[a-z]' "$1" | while read name meta args; do
173− gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
175+ gen_proto "${meta}" "${name}" "${ARCH}" "atomic64" "s64" ${args}
174176 done
175177
176178 cat <<EOF
177−#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
178−#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
179−
180179 #endif /* _LINUX_ATOMIC_FALLBACK_H */
181180 EOF
scripts/atomic/gen-atomics.sh+3−2
@@ -10,10 +10,11 @@ LINUXDIR=${ATOMICDIR}/../.. · 9 unmodified lines
1010 cat <<EOF |
1111 gen-atomic-instrumented.sh asm-generic/atomic-instrumented.h
1212 gen-atomic-long.sh asm-generic/atomic-long.h
13+gen-atomic-fallback.sh linux/atomic-arch-fallback.h arch_
1314 gen-atomic-fallback.sh linux/atomic-fallback.h
1415 EOF
15−while read script header; do
16− /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} > ${LINUXDIR}/include/${header}
16+while read script header args; do
17+ /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header}
1718 HASH="$(sha1sum ${LINUXDIR}/include/${header})"
1819 HASH="${HASH%% *}"
1920 printf "// %s\n" "${HASH}" >> ${LINUXDIR}/include/${header}