// SPDX-License-Identifier: GPL-2.0 //! Atomic internal implementations.//!//! Provides 1:1 mapping to the C atomic operations. use crate::{ bindings, build_assert::static_assert, macros::paste, //};use core::cell::UnsafeCell;use ffi::c_void; mod private { /// Sealed trait marker to disable customized impls on atomic implementation traits. pub trait Sealed {}} // The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`),// while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of// lower-level C primitives.impl private::Sealed for i8 {}impl private::Sealed for i16 {}impl private::Sealed for *const c_void {}impl private::Sealed for i32 {}impl private::Sealed for i64 {} /// A marker trait for types that implement atomic operations with C side primitives.////// This trait is sealed, and only types that map directly to the C side atomics/// or can be implemented with lower-level C primitives are allowed to implement this:////// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives./// - `i32` map to `atomic_t`/// - `i64` map to `atomic64_t`pub trait AtomicImpl: Sized + Copy + private::Sealed { /// The type of the delta in arithmetic or logical operations. /// /// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of /// [`Self`], but it may be different for the atomic pointer type. type Delta;} // The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()`// hence the atomicity is only guaranteed against read-modify-write operations if the architecture// supports native atomic RmW.//// In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the// load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to// be added.static_assert!( cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW), "The current implementation of atomic i8/i16/ptr relies on the architecure being \ ARCH_SUPPORTS_ATOMIC_RMW"); impl AtomicImpl for i8 { type Delta = Self;} impl AtomicImpl for i16 { type Delta = Self;} impl AtomicImpl for *const c_void { type Delta = isize;} // `atomic_t` implements atomic operations on `i32`.impl AtomicImpl for i32 { type Delta = Self;} // `atomic64_t` implements atomic operations on `i64`.impl AtomicImpl for i64 { type Delta = Self;} /// Atomic representation.#[repr(transparent)]pub struct AtomicRepr<T: AtomicImpl>(UnsafeCell<T>); impl<T: AtomicImpl> AtomicRepr<T> { /// Creates a new atomic representation `T`. pub const fn new(v: T) -> Self { Self(UnsafeCell::new(v)) } /// Returns a pointer to the underlying `T`. /// /// # Guarantees /// /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::<T>()`]). pub const fn as_ptr(&self) -> *mut T { // GUARANTEE: `self.0` is an `UnsafeCell<T>`, therefore the pointer returned by `.get()` // must be valid and properly aligned. self.0.get() }} // This macro generates the function signature with given argument list and return type.macro_rules! declare_atomic_method { ( $(#[doc=$doc:expr])* $func:ident($($arg:ident : $arg_type:ty),*) $(-> $ret:ty)? ) => { paste!( $(#[doc = $doc])* fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)?; ); }; ( $(#[doc=$doc:expr])* $func:ident [$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? ) => { paste!( declare_atomic_method!( $(#[doc = $doc])* [< $func _ $variant >]($($arg_sig)*) $(-> $ret)? ); ); declare_atomic_method!( $(#[doc = $doc])* $func [$($rest)*]($($arg_sig)*) $(-> $ret)? ); }; ( $(#[doc=$doc:expr])* $func:ident []($($arg_sig:tt)*) $(-> $ret:ty)? ) => { declare_atomic_method!( $(#[doc = $doc])* $func($($arg_sig)*) $(-> $ret)? ); }} // This macro generates the function implementation with given argument list and return type, and it// will replace "call(...)" expression with "$ctype _ $func" to call the real C function.macro_rules! impl_atomic_method { ( ($ctype:ident) $func:ident($($arg:ident: $arg_type:ty),*) $(-> $ret:ty)? { $unsafe:tt { call($($c_arg:expr),*) } } ) => { paste!( #[inline(always)] fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)? { // TODO: Ideally we want to use the SAFETY comments written at the macro invocation // (e.g. in `declare_and_impl_atomic_methods!()`, however, since SAFETY comments // are just comments, and they are not passed to macros as tokens, therefore we // cannot use them here. One potential improvement is that if we support using // attributes as an alternative for SAFETY comments, then we can use that for macro // generating code. // // SAFETY: specified on macro invocation. $unsafe { bindings::[< $ctype _ $func >]($($c_arg,)*) } } ); }; ( ($ctype:ident) $func:ident[$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? { $unsafe:tt { call($($arg:tt)*) } } ) => { paste!( impl_atomic_method!( ($ctype) [< $func _ $variant >]($($arg_sig)*) $( -> $ret)? { $unsafe { call($($arg)*) } } ); ); impl_atomic_method!( ($ctype) $func [$($rest)*]($($arg_sig)*) $( -> $ret)? { $unsafe { call($($arg)*) } } ); }; ( ($ctype:ident) $func:ident[]($($arg_sig:tt)*) $( -> $ret:ty)? { $unsafe:tt { call($($arg:tt)*) } } ) => { impl_atomic_method!( ($ctype) $func($($arg_sig)*) $(-> $ret)? { $unsafe { call($($arg)*) } } ); }} macro_rules! declare_atomic_ops_trait { ( $(#[$attr:meta])* $pub:vis trait $ops:ident { $( $(#[doc=$doc:expr])* fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { $unsafe:tt { bindings::#call($($arg:tt)*) } } )* } ) => { $(#[$attr])* $pub trait $ops: AtomicImpl { $( declare_atomic_method!( $(#[doc=$doc])* $func[$($variant)*]($($arg_sig)*) $(-> $ret)? ); )* } }} macro_rules! impl_atomic_ops_for_one { ( $ty:ty => $ctype:ident, $(#[$attr:meta])* $pub:vis trait $ops:ident { $( $(#[doc=$doc:expr])* fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { $unsafe:tt { bindings::#call($($arg:tt)*) } } )* } ) => { impl $ops for $ty { $( impl_atomic_method!( ($ctype) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? { $unsafe { call($($arg)*) } } ); )* } }} // Declares $ops trait with methods and implements the trait.macro_rules! declare_and_impl_atomic_methods { ( [ $($map:tt)* ] $(#[$attr:meta])* $pub:vis trait $ops:ident { $($body:tt)* } ) => { declare_and_impl_atomic_methods!( @with_ops_def [ $($map)* ] ( $(#[$attr])* $pub trait $ops { $($body)* } ) ); }; (@with_ops_def [ $($map:tt)* ] ( $($ops_def:tt)* )) => { declare_atomic_ops_trait!( $($ops_def)* ); declare_and_impl_atomic_methods!( @munch [ $($map)* ] ( $($ops_def)* ) ); }; (@munch [] ( $($ops_def:tt)* )) => {}; (@munch [ $ty:ty => $ctype:ident $(, $($rest:tt)*)? ] ( $($ops_def:tt)* )) => { impl_atomic_ops_for_one!( $ty => $ctype, $($ops_def)* ); declare_and_impl_atomic_methods!( @munch [ $($($rest)*)? ] ( $($ops_def)* ) ); };} declare_and_impl_atomic_methods!( [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] /// Basic atomic operations pub trait AtomicBasicOps { /// Atomic read (load). fn read[acquire](a: &AtomicRepr<Self>) -> Self { // SAFETY: `a.as_ptr()` is valid and properly aligned. unsafe { bindings::#call(a.as_ptr().cast()) } } /// Atomic set (store). fn set[release](a: &AtomicRepr<Self>, v: Self) { // SAFETY: `a.as_ptr()` is valid and properly aligned. unsafe { bindings::#call(a.as_ptr().cast(), v) } } }); declare_and_impl_atomic_methods!( [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ] /// Exchange and compare-and-exchange atomic operations pub trait AtomicExchangeOps { /// Atomic exchange. /// /// Atomically updates `*a` to `v` and returns the old value. fn xchg[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self) -> Self { // SAFETY: `a.as_ptr()` is valid and properly aligned. unsafe { bindings::#call(a.as_ptr().cast(), v) } } /// Atomic compare and exchange. /// /// If `*a` == `*old`, atomically updates `*a` to `new`. Otherwise, `*a` is not /// modified, `*old` is updated to the current value of `*a`. /// /// Return `true` if the update of `*a` occurred, `false` otherwise. fn try_cmpxchg[acquire, release, relaxed]( a: &AtomicRepr<Self>, old: &mut Self, new: Self ) -> bool { // SAFETY: `a.as_ptr()` is valid and properly aligned. `core::ptr::from_mut(old)` // is valid and properly aligned. unsafe { bindings::#call(a.as_ptr().cast(), core::ptr::from_mut(old), new) } } }); declare_and_impl_atomic_methods!( [ i32 => atomic, i64 => atomic64 ] /// Atomic arithmetic operations pub trait AtomicArithmeticOps { /// Atomic add (wrapping). /// /// Atomically updates `*a` to `(*a).wrapping_add(v)`. fn add[](a: &AtomicRepr<Self>, v: Self::Delta) { // SAFETY: `a.as_ptr()` is valid and properly aligned. unsafe { bindings::#call(v, a.as_ptr().cast()) } } /// Atomic fetch and add (wrapping). /// /// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a` /// before the update. fn fetch_add[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self { // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. unsafe { bindings::#call(v, a.as_ptr().cast()) } } fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self { // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned. unsafe { bindings::#call(v, a.as_ptr().cast()) } } });
// SPDX-License-Identifier: GPL-2.0
//! Atomic internal implementations.
//!
//! Provides 1:1 mapping to the C atomic operations.
use crate::{
bindings,
build_assert::static_assert,
macros::paste, //
};
use core::cell::UnsafeCell;
use ffi::c_void;
mod private {
/// Sealed trait marker to disable customized impls on atomic implementation traits.
pub trait Sealed {}
}
// The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`),
// while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of
// lower-level C primitives.
impl private::Sealed for i8 {}
impl private::Sealed for i16 {}
impl private::Sealed for *const c_void {}
impl private::Sealed for i32 {}
impl private::Sealed for i64 {}
/// A marker trait for types that implement atomic operations with C side primitives.
///
/// This trait is sealed, and only types that map directly to the C side atomics
/// or can be implemented with lower-level C primitives are allowed to implement this:
///
/// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives.
/// - `i32` map to `atomic_t`
/// - `i64` map to `atomic64_t`
pub trait AtomicImpl: Sized + Copy + private::Sealed {
/// The type of the delta in arithmetic or logical operations.
///
/// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of
/// [`Self`], but it may be different for the atomic pointer type.
type Delta;
}
// The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()`
// hence the atomicity is only guaranteed against read-modify-write operations if the architecture
// supports native atomic RmW.
//
// In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the
// load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to
// be added.
static_assert!(
cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW),
"The current implementation of atomic i8/i16/ptr relies on the architecure being \
ARCH_SUPPORTS_ATOMIC_RMW"
);
impl AtomicImpl for i8 {
type Delta = Self;
}
impl AtomicImpl for i16 {
type Delta = Self;
}
impl AtomicImpl for *const c_void {
type Delta = isize;
}
// `atomic_t` implements atomic operations on `i32`.
impl AtomicImpl for i32 {
type Delta = Self;
}
// `atomic64_t` implements atomic operations on `i64`.
impl AtomicImpl for i64 {
type Delta = Self;
}
/// Atomic representation.
#[repr(transparent)]
pub struct AtomicRepr<T: AtomicImpl>(UnsafeCell<T>);
impl<T: AtomicImpl> AtomicRepr<T> {
/// Creates a new atomic representation `T`.
pub const fn new(v: T) -> Self {
Self(UnsafeCell::new(v))
}
/// Returns a pointer to the underlying `T`.
///
/// # Guarantees
///
/// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::<T>()`]).
pub const fn as_ptr(&self) -> *mut T {
// GUARANTEE: `self.0` is an `UnsafeCell<T>`, therefore the pointer returned by `.get()`
// must be valid and properly aligned.
self.0.get()
}
}
// This macro generates the function signature with given argument list and return type.
macro_rules! declare_atomic_method {
(
$(#[doc=$doc:expr])*
$func:ident($($arg:ident : $arg_type:ty),*) $(-> $ret:ty)?
) => {
paste!(
$(#[doc = $doc])*
fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)?;
);
};
(
$(#[doc=$doc:expr])*
$func:ident [$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)?
) => {
paste!(
declare_atomic_method!(
$(#[doc = $doc])*
[< $func _ $variant >]($($arg_sig)*) $(-> $ret)?
);
);
declare_atomic_method!(
$(#[doc = $doc])*
$func [$($rest)*]($($arg_sig)*) $(-> $ret)?
);
};
(
$(#[doc=$doc:expr])*
$func:ident []($($arg_sig:tt)*) $(-> $ret:ty)?
) => {
declare_atomic_method!(
$(#[doc = $doc])*
$func($($arg_sig)*) $(-> $ret)?
);
}
}
// This macro generates the function implementation with given argument list and return type, and it
// will replace "call(...)" expression with "$ctype _ $func" to call the real C function.
macro_rules! impl_atomic_method {
(
($ctype:ident) $func:ident($($arg:ident: $arg_type:ty),*) $(-> $ret:ty)? {
$unsafe:tt { call($($c_arg:expr),*) }
}
) => {
paste!(
#[inline(always)]
fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)? {
// TODO: Ideally we want to use the SAFETY comments written at the macro invocation
// (e.g. in `declare_and_impl_atomic_methods!()`, however, since SAFETY comments
// are just comments, and they are not passed to macros as tokens, therefore we
// cannot use them here. One potential improvement is that if we support using
// attributes as an alternative for SAFETY comments, then we can use that for macro
// generating code.
//
// SAFETY: specified on macro invocation.
$unsafe { bindings::[< $ctype _ $func >]($($c_arg,)*) }
}
);
};
(
($ctype:ident) $func:ident[$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? {
$unsafe:tt { call($($arg:tt)*) }
}
) => {
paste!(
impl_atomic_method!(
($ctype) [< $func _ $variant >]($($arg_sig)*) $( -> $ret)? {
$unsafe { call($($arg)*) }
}
);
);
impl_atomic_method!(
($ctype) $func [$($rest)*]($($arg_sig)*) $( -> $ret)? {
$unsafe { call($($arg)*) }
}
);
};
(
($ctype:ident) $func:ident[]($($arg_sig:tt)*) $( -> $ret:ty)? {
$unsafe:tt { call($($arg:tt)*) }
}
) => {
impl_atomic_method!(
($ctype) $func($($arg_sig)*) $(-> $ret)? {
$unsafe { call($($arg)*) }
}
);
}
}
macro_rules! declare_atomic_ops_trait {
(
$(#[$attr:meta])* $pub:vis trait $ops:ident {
$(
$(#[doc=$doc:expr])*
fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? {
$unsafe:tt { bindings::#call($($arg:tt)*) }
}
)*
}
) => {
$(#[$attr])*
$pub trait $ops: AtomicImpl {
$(
declare_atomic_method!(
$(#[doc=$doc])*
$func[$($variant)*]($($arg_sig)*) $(-> $ret)?
);
)*
}
}
}
macro_rules! impl_atomic_ops_for_one {
(
$ty:ty => $ctype:ident,
$(#[$attr:meta])* $pub:vis trait $ops:ident {
$(
$(#[doc=$doc:expr])*
fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? {
$unsafe:tt { bindings::#call($($arg:tt)*) }
}
)*
}
) => {
impl $ops for $ty {
$(
impl_atomic_method!(
($ctype) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? {
$unsafe { call($($arg)*) }
}
);
)*
}
}
}
// Declares $ops trait with methods and implements the trait.
macro_rules! declare_and_impl_atomic_methods {
(
[ $($map:tt)* ]
$(#[$attr:meta])* $pub:vis trait $ops:ident { $($body:tt)* }
) => {
declare_and_impl_atomic_methods!(
@with_ops_def
[ $($map)* ]
( $(#[$attr])* $pub trait $ops { $($body)* } )
);
};
(@with_ops_def [ $($map:tt)* ] ( $($ops_def:tt)* )) => {
declare_atomic_ops_trait!( $($ops_def)* );
declare_and_impl_atomic_methods!(
@munch
[ $($map)* ]
( $($ops_def)* )
);
};
(@munch [] ( $($ops_def:tt)* )) => {};
(@munch [ $ty:ty => $ctype:ident $(, $($rest:tt)*)? ] ( $($ops_def:tt)* )) => {
impl_atomic_ops_for_one!(
$ty => $ctype,
$($ops_def)*
);
declare_and_impl_atomic_methods!(
@munch
[ $($($rest)*)? ]
( $($ops_def)* )
);
};
}
declare_and_impl_atomic_methods!(
[ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ]
/// Basic atomic operations
pub trait AtomicBasicOps {
/// Atomic read (load).
fn read[acquire](a: &AtomicRepr<Self>) -> Self {
// SAFETY: `a.as_ptr()` is valid and properly aligned.
unsafe { bindings::#call(a.as_ptr().cast()) }
}
/// Atomic set (store).
fn set[release](a: &AtomicRepr<Self>, v: Self) {
// SAFETY: `a.as_ptr()` is valid and properly aligned.
unsafe { bindings::#call(a.as_ptr().cast(), v) }
}
}
);
declare_and_impl_atomic_methods!(
[ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ]
/// Exchange and compare-and-exchange atomic operations
pub trait AtomicExchangeOps {
/// Atomic exchange.
///
/// Atomically updates `*a` to `v` and returns the old value.
fn xchg[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self) -> Self {
// SAFETY: `a.as_ptr()` is valid and properly aligned.
unsafe { bindings::#call(a.as_ptr().cast(), v) }
}
/// Atomic compare and exchange.
///
/// If `*a` == `*old`, atomically updates `*a` to `new`. Otherwise, `*a` is not
/// modified, `*old` is updated to the current value of `*a`.
///
/// Return `true` if the update of `*a` occurred, `false` otherwise.
fn try_cmpxchg[acquire, release, relaxed](
a: &AtomicRepr<Self>, old: &mut Self, new: Self
) -> bool {
// SAFETY: `a.as_ptr()` is valid and properly aligned. `core::ptr::from_mut(old)`
// is valid and properly aligned.
unsafe { bindings::#call(a.as_ptr().cast(), core::ptr::from_mut(old), new) }
}
}
);
declare_and_impl_atomic_methods!(
[ i32 => atomic, i64 => atomic64 ]
/// Atomic arithmetic operations
pub trait AtomicArithmeticOps {
/// Atomic add (wrapping).
///
/// Atomically updates `*a` to `(*a).wrapping_add(v)`.
fn add[](a: &AtomicRepr<Self>, v: Self::Delta) {
// SAFETY: `a.as_ptr()` is valid and properly aligned.
unsafe { bindings::#call(v, a.as_ptr().cast()) }
}
/// Atomic fetch and add (wrapping).
///
/// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a`
/// before the update.
fn fetch_add[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self {
// SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned.
unsafe { bindings::#call(v, a.as_ptr().cast()) }
}
fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self {
// SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned.
unsafe { bindings::#call(v, a.as_ptr().cast()) }
}
}
);