asm-generic/atomic: Use __always_inline for fallback wrappers

Use __always_inline for atomic fallback wrappers. When building for size (CC_OPTIMIZE_FOR_SIZE), some compilers appear to be less inclined to inline even relatively small static inline functions that are assumed to be inlinable such as atomic ops. This can cause problems, for example in UACCESS regions. While the fallback wrappers aren't pure wrappers, they are trivial nonetheless, and the function they wrap should determine the final inlining policy. For x86 tinyconfig we observe: - vmlinux baseline: 1315988 - vmlinux with patch: 1315928 (-60 bytes)

Marco Elver committed 6 years ago

Commit 944bc9c1 parent c02039521 files · +192 −188Browse files

Showing the changes to scripts (20 of 21 files). Show all

Unified · Split

20 files

scripts/atomic/fallbacks/acquire +1 −1
scripts/atomic/fallbacks/add_negative +1 −1
scripts/atomic/fallbacks/add_unless +1 −1
scripts/atomic/fallbacks/andnot +1 −1
scripts/atomic/fallbacks/dec +1 −1
scripts/atomic/fallbacks/dec_and_test +1 −1
scripts/atomic/fallbacks/dec_if_positive +1 −1
scripts/atomic/fallbacks/dec_unless_positive +1 −1
scripts/atomic/fallbacks/fence +1 −1
scripts/atomic/fallbacks/fetch_add_unless +1 −1
scripts/atomic/fallbacks/inc +1 −1
scripts/atomic/fallbacks/inc_and_test +1 −1
scripts/atomic/fallbacks/inc_not_zero +1 −1
scripts/atomic/fallbacks/inc_unless_negative +1 −1
scripts/atomic/fallbacks/read_acquire +1 −1
scripts/atomic/fallbacks/release +1 −1
scripts/atomic/fallbacks/set_release +1 −1
scripts/atomic/fallbacks/sub_and_test +1 −1
scripts/atomic/fallbacks/try_cmpxchg +1 −1
scripts/atomic/gen-atomic-fallback.sh +2 −0

scripts/atomic/fallbacks/acquire+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline ${ret}	2	static __always_inline ${ret}
3	${atomic}_${pfx}${name}${sfx}_acquire(${params})	3	${atomic}_${pfx}${name}${sfx}_acquire(${params})
4	{	4	{
5	${ret} ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});	5	${ret} ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args});

scripts/atomic/fallbacks/add_negative+1−1

@@ -8,7 +8,7 @@ cat <<EOF · 7 unmodified lines
8	* if the result is negative, or false when	8	* if the result is negative, or false when
9	* result is greater than or equal to zero.	9	* result is greater than or equal to zero.
10	*/	10	*/
11	static inline bool	11	static __always_inline bool
12	${atomic}_add_negative(${int} i, ${atomic}_t *v)	12	${atomic}_add_negative(${int} i, ${atomic}_t *v)
13	{	13	{
14	return ${atomic}_add_return(i, v) < 0;	14	return ${atomic}_add_return(i, v) < 0;

scripts/atomic/fallbacks/add_unless+1−1

@@ -8,7 +8,7 @@ cat << EOF · 7 unmodified lines
8	* Atomically adds @a to @v, if @v was not already @u.	8	* Atomically adds @a to @v, if @v was not already @u.
9	* Returns true if the addition was done.	9	* Returns true if the addition was done.
10	*/	10	*/
11	static inline bool	11	static __always_inline bool
12	${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)	12	${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13	{	13	{
14	return ${atomic}_fetch_add_unless(v, a, u) != u;	14	return ${atomic}_fetch_add_unless(v, a, u) != u;

scripts/atomic/fallbacks/andnot+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline ${ret}	2	static __always_inline ${ret}
3	${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)	3	${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
4	{	4	{
5	${retstmt}${atomic}_${pfx}and${sfx}${order}(~i, v);	5	${retstmt}${atomic}_${pfx}and${sfx}${order}(~i, v);

scripts/atomic/fallbacks/dec+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline ${ret}	2	static __always_inline ${ret}
3	${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)	3	${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
4	{	4	{
5	${retstmt}${atomic}_${pfx}sub${sfx}${order}(1, v);	5	${retstmt}${atomic}_${pfx}sub${sfx}${order}(1, v);

scripts/atomic/fallbacks/dec_and_test+1−1

@@ -7,7 +7,7 @@ cat <<EOF · 6 unmodified lines
7	* returns true if the result is 0, or false for all other	7	* returns true if the result is 0, or false for all other
8	* cases.	8	* cases.
9	*/	9	*/
10	static inline bool	10	static __always_inline bool
11	${atomic}_dec_and_test(${atomic}_t *v)	11	${atomic}_dec_and_test(${atomic}_t *v)
12	{	12	{
13	return ${atomic}_dec_return(v) == 0;	13	return ${atomic}_dec_return(v) == 0;

scripts/atomic/fallbacks/dec_if_positive+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline ${ret}	2	static __always_inline ${ret}
3	${atomic}_dec_if_positive(${atomic}_t *v)	3	${atomic}_dec_if_positive(${atomic}_t *v)
4	{	4	{
5	${int} dec, c = ${atomic}_read(v);	5	${int} dec, c = ${atomic}_read(v);

scripts/atomic/fallbacks/dec_unless_positive+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline bool	2	static __always_inline bool
3	${atomic}_dec_unless_positive(${atomic}_t *v)	3	${atomic}_dec_unless_positive(${atomic}_t *v)
4	{	4	{
5	${int} c = ${atomic}_read(v);	5	${int} c = ${atomic}_read(v);

scripts/atomic/fallbacks/fence+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline ${ret}	2	static __always_inline ${ret}
3	${atomic}_${pfx}${name}${sfx}(${params})	3	${atomic}_${pfx}${name}${sfx}(${params})
4	{	4	{
5	${ret} ret;	5	${ret} ret;

scripts/atomic/fallbacks/fetch_add_unless+1−1

@@ -8,7 +8,7 @@ cat << EOF · 7 unmodified lines
8	* Atomically adds @a to @v, so long as @v was not already @u.	8	* Atomically adds @a to @v, so long as @v was not already @u.
9	* Returns original value of @v	9	* Returns original value of @v
10	*/	10	*/
11	static inline ${int}	11	static __always_inline ${int}
12	${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)	12	${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13	{	13	{
14	${int} c = ${atomic}_read(v);	14	${int} c = ${atomic}_read(v);

scripts/atomic/fallbacks/inc+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline ${ret}	2	static __always_inline ${ret}
3	${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)	3	${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
4	{	4	{
5	${retstmt}${atomic}_${pfx}add${sfx}${order}(1, v);	5	${retstmt}${atomic}_${pfx}add${sfx}${order}(1, v);

scripts/atomic/fallbacks/inc_and_test+1−1

@@ -7,7 +7,7 @@ cat <<EOF · 6 unmodified lines
7	* and returns true if the result is zero, or false for all	7	* and returns true if the result is zero, or false for all
8	* other cases.	8	* other cases.
9	*/	9	*/
10	static inline bool	10	static __always_inline bool
11	${atomic}_inc_and_test(${atomic}_t *v)	11	${atomic}_inc_and_test(${atomic}_t *v)
12	{	12	{
13	return ${atomic}_inc_return(v) == 0;	13	return ${atomic}_inc_return(v) == 0;

scripts/atomic/fallbacks/inc_not_zero+1−1

@@ -6,7 +6,7 @@ cat <<EOF · 5 unmodified lines
6	* Atomically increments @v by 1, if @v is non-zero.	6	* Atomically increments @v by 1, if @v is non-zero.
7	* Returns true if the increment was done.	7	* Returns true if the increment was done.
8	*/	8	*/
9	static inline bool	9	static __always_inline bool
10	${atomic}_inc_not_zero(${atomic}_t *v)	10	${atomic}_inc_not_zero(${atomic}_t *v)
11	{	11	{
12	return ${atomic}_add_unless(v, 1, 0);	12	return ${atomic}_add_unless(v, 1, 0);

scripts/atomic/fallbacks/inc_unless_negative+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline bool	2	static __always_inline bool
3	${atomic}_inc_unless_negative(${atomic}_t *v)	3	${atomic}_inc_unless_negative(${atomic}_t *v)
4	{	4	{
5	${int} c = ${atomic}_read(v);	5	${int} c = ${atomic}_read(v);

scripts/atomic/fallbacks/read_acquire+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline ${ret}	2	static __always_inline ${ret}
3	${atomic}_read_acquire(const ${atomic}_t *v)	3	${atomic}_read_acquire(const ${atomic}_t *v)
4	{	4	{
5	return smp_load_acquire(&(v)->counter);	5	return smp_load_acquire(&(v)->counter);

scripts/atomic/fallbacks/release+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline ${ret}	2	static __always_inline ${ret}
3	${atomic}_${pfx}${name}${sfx}_release(${params})	3	${atomic}_${pfx}${name}${sfx}_release(${params})
4	{	4	{
5	__atomic_release_fence();	5	__atomic_release_fence();

scripts/atomic/fallbacks/set_release+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline void	2	static __always_inline void
3	${atomic}_set_release(${atomic}_t *v, ${int} i)	3	${atomic}_set_release(${atomic}_t *v, ${int} i)
4	{	4	{
5	smp_store_release(&(v)->counter, i);	5	smp_store_release(&(v)->counter, i);

scripts/atomic/fallbacks/sub_and_test+1−1

@@ -8,7 +8,7 @@ cat <<EOF · 7 unmodified lines
8	* true if the result is zero, or false for all	8	* true if the result is zero, or false for all
9	* other cases.	9	* other cases.
10	*/	10	*/
11	static inline bool	11	static __always_inline bool
12	${atomic}_sub_and_test(${int} i, ${atomic}_t *v)	12	${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
13	{	13	{
14	return ${atomic}_sub_return(i, v) == 0;	14	return ${atomic}_sub_return(i, v) == 0;

scripts/atomic/fallbacks/try_cmpxchg+1−1

@@ -1,5 +1,5 @@
1	cat <<EOF	1	cat <<EOF
2	static inline bool	2	static __always_inline bool
3	${atomic}_try_cmpxchg${order}(${atomic}_t v, ${int} old, ${int} new)	3	${atomic}_try_cmpxchg${order}(${atomic}_t v, ${int} old, ${int} new)
4	{	4	{
5	${int} r, o = *old;	5	${int} r, o = *old;

scripts/atomic/gen-atomic-fallback.sh+2−0

@@ -149,6 +149,8 @@ cat << EOF · 148 unmodified lines
149	#ifndef _LINUX_ATOMIC_FALLBACK_H	149	#ifndef _LINUX_ATOMIC_FALLBACK_H
150	#define _LINUX_ATOMIC_FALLBACK_H	150	#define _LINUX_ATOMIC_FALLBACK_H
151		151
		152	#include <linux/compiler.h>
		153
152	EOF	154	EOF
153		155
154	for xchg in "xchg" "cmpxchg" "cmpxchg64"; do	156	for xchg in "xchg" "cmpxchg" "cmpxchg64"; do

asm-generic/atomic: Use __always_inline for fallback wrappers

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code