/* SPDX-License-Identifier: GPL-2.0 *//* Atomic operations usable in machine independent code */#ifndef _LINUX_ATOMIC_H#define _LINUX_ATOMIC_H#include <linux/types.h> #include <asm/atomic.h>#include <asm/barrier.h> /* * Relaxed variants of xchg, cmpxchg and some atomic operations. * * We support four variants: * * - Fully ordered: The default implementation, no suffix required. * - Acquire: Provides ACQUIRE semantics, _acquire suffix. * - Release: Provides RELEASE semantics, _release suffix. * - Relaxed: No ordering guarantees, _relaxed suffix. * * For compound atomics performing both a load and a store, ACQUIRE * semantics apply only to the load and RELEASE semantics only to the * store portion of the operation. Note that a failed cmpxchg_acquire * does -not- imply any memory ordering constraints. * * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. */ #ifndef atomic_read_acquire#define  atomic_read_acquire(v)		smp_load_acquire(&(v)->counter)#endif #ifndef atomic_set_release#define  atomic_set_release(v, i)	smp_store_release(&(v)->counter, (i))#endif /* * The idea here is to build acquire/release variants by adding explicit * barriers on top of the relaxed variant. In the case where the relaxed * variant is already fully ordered, no additional barriers are needed. * * If an architecture overrides __atomic_acquire_fence() it will probably * want to define smp_mb__after_spinlock(). */#ifndef __atomic_acquire_fence#define __atomic_acquire_fence		smp_mb__after_atomic#endif #ifndef __atomic_release_fence#define __atomic_release_fence		smp_mb__before_atomic#endif #ifndef __atomic_pre_full_fence#define __atomic_pre_full_fence		smp_mb__before_atomic#endif #ifndef __atomic_post_full_fence#define __atomic_post_full_fence	smp_mb__after_atomic#endif #define __atomic_op_acquire(op, args...)				\({									\	typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);		\	__atomic_acquire_fence();					\	__ret;								\}) #define __atomic_op_release(op, args...)				\({									\	__atomic_release_fence();					\	op##_relaxed(args);						\}) #define __atomic_op_fence(op, args...)					\({									\	typeof(op##_relaxed(args)) __ret;				\	__atomic_pre_full_fence();					\	__ret = op##_relaxed(args);					\	__atomic_post_full_fence();					\	__ret;								\}) /* atomic_add_return_relaxed */#ifndef atomic_add_return_relaxed#define  atomic_add_return_relaxed	atomic_add_return#define  atomic_add_return_acquire	atomic_add_return#define  atomic_add_return_release	atomic_add_return #else /* atomic_add_return_relaxed */ #ifndef atomic_add_return_acquire#define  atomic_add_return_acquire(...)					\	__atomic_op_acquire(atomic_add_return, __VA_ARGS__)#endif #ifndef atomic_add_return_release#define  atomic_add_return_release(...)					\	__atomic_op_release(atomic_add_return, __VA_ARGS__)#endif #ifndef atomic_add_return#define  atomic_add_return(...)						\	__atomic_op_fence(atomic_add_return, __VA_ARGS__)#endif#endif /* atomic_add_return_relaxed */ #ifndef atomic_inc#define atomic_inc(v)			atomic_add(1, (v))#endif /* atomic_inc_return_relaxed */#ifndef atomic_inc_return_relaxed #ifndef atomic_inc_return#define atomic_inc_return(v)		atomic_add_return(1, (v))#define atomic_inc_return_relaxed(v)	atomic_add_return_relaxed(1, (v))#define atomic_inc_return_acquire(v)	atomic_add_return_acquire(1, (v))#define atomic_inc_return_release(v)	atomic_add_return_release(1, (v))#else /* atomic_inc_return */#define  atomic_inc_return_relaxed	atomic_inc_return#define  atomic_inc_return_acquire	atomic_inc_return#define  atomic_inc_return_release	atomic_inc_return#endif /* atomic_inc_return */ #else /* atomic_inc_return_relaxed */ #ifndef atomic_inc_return_acquire#define  atomic_inc_return_acquire(...)					\	__atomic_op_acquire(atomic_inc_return, __VA_ARGS__)#endif #ifndef atomic_inc_return_release#define  atomic_inc_return_release(...)					\	__atomic_op_release(atomic_inc_return, __VA_ARGS__)#endif #ifndef atomic_inc_return#define  atomic_inc_return(...)						\	__atomic_op_fence(atomic_inc_return, __VA_ARGS__)#endif#endif /* atomic_inc_return_relaxed */ /* atomic_sub_return_relaxed */#ifndef atomic_sub_return_relaxed#define  atomic_sub_return_relaxed	atomic_sub_return#define  atomic_sub_return_acquire	atomic_sub_return#define  atomic_sub_return_release	atomic_sub_return #else /* atomic_sub_return_relaxed */ #ifndef atomic_sub_return_acquire#define  atomic_sub_return_acquire(...)					\	__atomic_op_acquire(atomic_sub_return, __VA_ARGS__)#endif #ifndef atomic_sub_return_release#define  atomic_sub_return_release(...)					\	__atomic_op_release(atomic_sub_return, __VA_ARGS__)#endif #ifndef atomic_sub_return#define  atomic_sub_return(...)						\	__atomic_op_fence(atomic_sub_return, __VA_ARGS__)#endif#endif /* atomic_sub_return_relaxed */ #ifndef atomic_dec#define atomic_dec(v)			atomic_sub(1, (v))#endif /* atomic_dec_return_relaxed */#ifndef atomic_dec_return_relaxed #ifndef atomic_dec_return#define atomic_dec_return(v)		atomic_sub_return(1, (v))#define atomic_dec_return_relaxed(v)	atomic_sub_return_relaxed(1, (v))#define atomic_dec_return_acquire(v)	atomic_sub_return_acquire(1, (v))#define atomic_dec_return_release(v)	atomic_sub_return_release(1, (v))#else /* atomic_dec_return */#define  atomic_dec_return_relaxed	atomic_dec_return#define  atomic_dec_return_acquire	atomic_dec_return#define  atomic_dec_return_release	atomic_dec_return#endif /* atomic_dec_return */ #else /* atomic_dec_return_relaxed */ #ifndef atomic_dec_return_acquire#define  atomic_dec_return_acquire(...)					\	__atomic_op_acquire(atomic_dec_return, __VA_ARGS__)#endif #ifndef atomic_dec_return_release#define  atomic_dec_return_release(...)					\	__atomic_op_release(atomic_dec_return, __VA_ARGS__)#endif #ifndef atomic_dec_return#define  atomic_dec_return(...)						\	__atomic_op_fence(atomic_dec_return, __VA_ARGS__)#endif#endif /* atomic_dec_return_relaxed */  /* atomic_fetch_add_relaxed */#ifndef atomic_fetch_add_relaxed#define atomic_fetch_add_relaxed	atomic_fetch_add#define atomic_fetch_add_acquire	atomic_fetch_add#define atomic_fetch_add_release	atomic_fetch_add #else /* atomic_fetch_add_relaxed */ #ifndef atomic_fetch_add_acquire#define atomic_fetch_add_acquire(...)					\	__atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)#endif #ifndef atomic_fetch_add_release#define atomic_fetch_add_release(...)					\	__atomic_op_release(atomic_fetch_add, __VA_ARGS__)#endif #ifndef atomic_fetch_add#define atomic_fetch_add(...)						\	__atomic_op_fence(atomic_fetch_add, __VA_ARGS__)#endif#endif /* atomic_fetch_add_relaxed */ /* atomic_fetch_inc_relaxed */#ifndef atomic_fetch_inc_relaxed #ifndef atomic_fetch_inc#define atomic_fetch_inc(v)	        atomic_fetch_add(1, (v))#define atomic_fetch_inc_relaxed(v)	atomic_fetch_add_relaxed(1, (v))#define atomic_fetch_inc_acquire(v)	atomic_fetch_add_acquire(1, (v))#define atomic_fetch_inc_release(v)	atomic_fetch_add_release(1, (v))#else /* atomic_fetch_inc */#define atomic_fetch_inc_relaxed	atomic_fetch_inc#define atomic_fetch_inc_acquire	atomic_fetch_inc#define atomic_fetch_inc_release	atomic_fetch_inc#endif /* atomic_fetch_inc */ #else /* atomic_fetch_inc_relaxed */ #ifndef atomic_fetch_inc_acquire#define atomic_fetch_inc_acquire(...)					\	__atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)#endif #ifndef atomic_fetch_inc_release#define atomic_fetch_inc_release(...)					\	__atomic_op_release(atomic_fetch_inc, __VA_ARGS__)#endif #ifndef atomic_fetch_inc#define atomic_fetch_inc(...)						\	__atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)#endif#endif /* atomic_fetch_inc_relaxed */ /* atomic_fetch_sub_relaxed */#ifndef atomic_fetch_sub_relaxed#define atomic_fetch_sub_relaxed	atomic_fetch_sub#define atomic_fetch_sub_acquire	atomic_fetch_sub#define atomic_fetch_sub_release	atomic_fetch_sub #else /* atomic_fetch_sub_relaxed */ #ifndef atomic_fetch_sub_acquire#define atomic_fetch_sub_acquire(...)					\	__atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)#endif #ifndef atomic_fetch_sub_release#define atomic_fetch_sub_release(...)					\	__atomic_op_release(atomic_fetch_sub, __VA_ARGS__)#endif #ifndef atomic_fetch_sub#define atomic_fetch_sub(...)						\	__atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)#endif#endif /* atomic_fetch_sub_relaxed */ /* atomic_fetch_dec_relaxed */#ifndef atomic_fetch_dec_relaxed #ifndef atomic_fetch_dec#define atomic_fetch_dec(v)	        atomic_fetch_sub(1, (v))#define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))#define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))#define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))#else /* atomic_fetch_dec */#define atomic_fetch_dec_relaxed	atomic_fetch_dec#define atomic_fetch_dec_acquire	atomic_fetch_dec#define atomic_fetch_dec_release	atomic_fetch_dec#endif /* atomic_fetch_dec */ #else /* atomic_fetch_dec_relaxed */ #ifndef atomic_fetch_dec_acquire#define atomic_fetch_dec_acquire(...)					\	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)#endif #ifndef atomic_fetch_dec_release#define atomic_fetch_dec_release(...)					\	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)#endif #ifndef atomic_fetch_dec#define atomic_fetch_dec(...)						\	__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)#endif#endif /* atomic_fetch_dec_relaxed */ /* atomic_fetch_or_relaxed */#ifndef atomic_fetch_or_relaxed#define atomic_fetch_or_relaxed	atomic_fetch_or#define atomic_fetch_or_acquire	atomic_fetch_or#define atomic_fetch_or_release	atomic_fetch_or #else /* atomic_fetch_or_relaxed */ #ifndef atomic_fetch_or_acquire#define atomic_fetch_or_acquire(...)					\	__atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)#endif #ifndef atomic_fetch_or_release#define atomic_fetch_or_release(...)					\	__atomic_op_release(atomic_fetch_or, __VA_ARGS__)#endif #ifndef atomic_fetch_or#define atomic_fetch_or(...)						\	__atomic_op_fence(atomic_fetch_or, __VA_ARGS__)#endif#endif /* atomic_fetch_or_relaxed */ /* atomic_fetch_and_relaxed */#ifndef atomic_fetch_and_relaxed#define atomic_fetch_and_relaxed	atomic_fetch_and#define atomic_fetch_and_acquire	atomic_fetch_and#define atomic_fetch_and_release	atomic_fetch_and #else /* atomic_fetch_and_relaxed */ #ifndef atomic_fetch_and_acquire#define atomic_fetch_and_acquire(...)					\	__atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)#endif #ifndef atomic_fetch_and_release#define atomic_fetch_and_release(...)					\	__atomic_op_release(atomic_fetch_and, __VA_ARGS__)#endif #ifndef atomic_fetch_and#define atomic_fetch_and(...)						\	__atomic_op_fence(atomic_fetch_and, __VA_ARGS__)#endif#endif /* atomic_fetch_and_relaxed */ #ifndef atomic_andnot#define atomic_andnot(i, v)		atomic_and(~(int)(i), (v))#endif #ifndef atomic_fetch_andnot_relaxed #ifndef atomic_fetch_andnot#define atomic_fetch_andnot(i, v)		atomic_fetch_and(~(int)(i), (v))#define atomic_fetch_andnot_relaxed(i, v)	atomic_fetch_and_relaxed(~(int)(i), (v))#define atomic_fetch_andnot_acquire(i, v)	atomic_fetch_and_acquire(~(int)(i), (v))#define atomic_fetch_andnot_release(i, v)	atomic_fetch_and_release(~(int)(i), (v))#else /* atomic_fetch_andnot */#define atomic_fetch_andnot_relaxed		atomic_fetch_andnot#define atomic_fetch_andnot_acquire		atomic_fetch_andnot#define atomic_fetch_andnot_release		atomic_fetch_andnot#endif /* atomic_fetch_andnot */ #else /* atomic_fetch_andnot_relaxed */ #ifndef atomic_fetch_andnot_acquire#define atomic_fetch_andnot_acquire(...)					\	__atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)#endif #ifndef atomic_fetch_andnot_release#define atomic_fetch_andnot_release(...)					\	__atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)#endif #ifndef atomic_fetch_andnot#define atomic_fetch_andnot(...)						\	__atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)#endif#endif /* atomic_fetch_andnot_relaxed */ /* atomic_fetch_xor_relaxed */#ifndef atomic_fetch_xor_relaxed#define atomic_fetch_xor_relaxed	atomic_fetch_xor#define atomic_fetch_xor_acquire	atomic_fetch_xor#define atomic_fetch_xor_release	atomic_fetch_xor #else /* atomic_fetch_xor_relaxed */ #ifndef atomic_fetch_xor_acquire#define atomic_fetch_xor_acquire(...)					\	__atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)#endif #ifndef atomic_fetch_xor_release#define atomic_fetch_xor_release(...)					\	__atomic_op_release(atomic_fetch_xor, __VA_ARGS__)#endif #ifndef atomic_fetch_xor#define atomic_fetch_xor(...)						\	__atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)#endif#endif /* atomic_fetch_xor_relaxed */  /* atomic_xchg_relaxed */#ifndef atomic_xchg_relaxed#define  atomic_xchg_relaxed		atomic_xchg#define  atomic_xchg_acquire		atomic_xchg#define  atomic_xchg_release		atomic_xchg #else /* atomic_xchg_relaxed */ #ifndef atomic_xchg_acquire#define  atomic_xchg_acquire(...)					\	__atomic_op_acquire(atomic_xchg, __VA_ARGS__)#endif #ifndef atomic_xchg_release#define  atomic_xchg_release(...)					\	__atomic_op_release(atomic_xchg, __VA_ARGS__)#endif #ifndef atomic_xchg#define  atomic_xchg(...)						\	__atomic_op_fence(atomic_xchg, __VA_ARGS__)#endif#endif /* atomic_xchg_relaxed */ /* atomic_cmpxchg_relaxed */#ifndef atomic_cmpxchg_relaxed#define  atomic_cmpxchg_relaxed		atomic_cmpxchg#define  atomic_cmpxchg_acquire		atomic_cmpxchg#define  atomic_cmpxchg_release		atomic_cmpxchg #else /* atomic_cmpxchg_relaxed */ #ifndef atomic_cmpxchg_acquire#define  atomic_cmpxchg_acquire(...)					\	__atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)#endif #ifndef atomic_cmpxchg_release#define  atomic_cmpxchg_release(...)					\	__atomic_op_release(atomic_cmpxchg, __VA_ARGS__)#endif #ifndef atomic_cmpxchg#define  atomic_cmpxchg(...)						\	__atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)#endif#endif /* atomic_cmpxchg_relaxed */ #ifndef atomic_try_cmpxchg #define __atomic_try_cmpxchg(type, _p, _po, _n)				\({									\	typeof(_po) __po = (_po);					\	typeof(*(_po)) __r, __o = *__po;				\	__r = atomic_cmpxchg##type((_p), __o, (_n));			\	if (unlikely(__r != __o))					\		*__po = __r;						\	likely(__r == __o);						\}) #define atomic_try_cmpxchg(_p, _po, _n)		__atomic_try_cmpxchg(, _p, _po, _n)#define atomic_try_cmpxchg_relaxed(_p, _po, _n)	__atomic_try_cmpxchg(_relaxed, _p, _po, _n)#define atomic_try_cmpxchg_acquire(_p, _po, _n)	__atomic_try_cmpxchg(_acquire, _p, _po, _n)#define atomic_try_cmpxchg_release(_p, _po, _n)	__atomic_try_cmpxchg(_release, _p, _po, _n) #else /* atomic_try_cmpxchg */#define atomic_try_cmpxchg_relaxed	atomic_try_cmpxchg#define atomic_try_cmpxchg_acquire	atomic_try_cmpxchg#define atomic_try_cmpxchg_release	atomic_try_cmpxchg#endif /* atomic_try_cmpxchg */ /* cmpxchg_relaxed */#ifndef cmpxchg_relaxed#define  cmpxchg_relaxed		cmpxchg#define  cmpxchg_acquire		cmpxchg#define  cmpxchg_release		cmpxchg #else /* cmpxchg_relaxed */

Lines 1–500 of 1317. Load more