// SPDX-License-Identifier: GPL-2.0 //! A reference-counted pointer.//!//! This module implements a way for users to create reference-counted objects and pointers to//! them. Such a pointer automatically increments and decrements the count, and drops the//! underlying object when it reaches zero. It is also safe to use concurrently from multiple//! threads.//!//! It is different from the standard library's [`Arc`] in a few ways://! 1. It is backed by the kernel's `refcount_t` type.//! 2. It does not support weak references, which allows it to be half the size.//! 3. It saturates the reference count instead of aborting when it goes over a threshold.//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.//! 5. The object in [`Arc`] is pinned implicitly.//!//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html use crate::{ alloc::{AllocError, Flags, KBox}, bindings, init::{self, InPlaceInit, Init, PinInit}, try_init, types::{ForeignOwnable, Opaque},};use core::{ alloc::Layout, fmt, marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, pin::Pin, ptr::NonNull,};use macros::pin_data; mod std_vendor; /// A reference-counted pointer to an instance of `T`.////// The reference count is incremented when new instances of [`Arc`] are created, and decremented/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.////// # Invariants////// The reference count on an instance of [`Arc`] is always non-zero./// The object pointed to by [`Arc`] is always pinned.////// # Examples////// ```/// use kernel::sync::Arc;////// struct Example {/// a: u32,/// b: u32,/// }////// // Create a refcounted instance of `Example`./// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;////// // Get a new pointer to `obj` and increment the refcount./// let cloned = obj.clone();////// // Assert that both `obj` and `cloned` point to the same underlying object./// assert!(core::ptr::eq(&*obj, &*cloned));////// // Destroy `obj` and decrement its refcount./// drop(obj);////// // Check that the values are still accessible through `cloned`./// assert_eq!(cloned.a, 10);/// assert_eq!(cloned.b, 20);////// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed./// # Ok::<(), Error>(())/// ```////// Using `Arc<T>` as the type of `self`:////// ```/// use kernel::sync::Arc;////// struct Example {/// a: u32,/// b: u32,/// }////// impl Example {/// fn take_over(self: Arc<Self>) {/// // .../// }////// fn use_reference(self: &Arc<Self>) {/// // .../// }/// }////// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;/// obj.use_reference();/// obj.take_over();/// # Ok::<(), Error>(())/// ```////// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:////// ```/// use kernel::sync::{Arc, ArcBorrow};////// trait MyTrait {/// // Trait has a function whose `self` type is `Arc<Self>`./// fn example1(self: Arc<Self>) {}////// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`./// fn example2(self: ArcBorrow<'_, Self>) {}/// }////// struct Example;/// impl MyTrait for Example {}////// // `obj` has type `Arc<Example>`./// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;////// // `coerced` has type `Arc<dyn MyTrait>`./// let coerced: Arc<dyn MyTrait> = obj;/// # Ok::<(), Error>(())/// ```#[repr(transparent)]#[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))]pub struct Arc<T: ?Sized> { ptr: NonNull<ArcInner<T>>, // NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in // `<Arc<T> as Drop>::drop` and the distinction between `T` and `ArcInner<T>` is not presently // meaningful with respect to dropck - but this may change in the future so this is left here // out of an abundance of caution. // // See https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking // for more detail on the semantics of dropck in the presence of `PhantomData`. _p: PhantomData<ArcInner<T>>,} #[pin_data]#[repr(C)]struct ArcInner<T: ?Sized> { refcount: Opaque<bindings::refcount_t>, data: T,} impl<T: ?Sized> ArcInner<T> { /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`]. /// /// # Safety /// /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must /// not yet have been destroyed. unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { let refcount_layout = Layout::new::<bindings::refcount_t>(); // SAFETY: The caller guarantees that the pointer is valid. let val_layout = Layout::for_value(unsafe { &*ptr }); // SAFETY: We're computing the layout of a real struct that existed when compiling this // binary, so its layout is not so large that it can trigger arithmetic overflow. let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field. // // This is documented at: // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>. let ptr = ptr as *const ArcInner<T>; // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is // still valid. let ptr = unsafe { ptr.byte_sub(val_offset) }; // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null // address. unsafe { NonNull::new_unchecked(ptr.cast_mut()) } }} // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the// dynamically-sized type (DST) `U`.#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {} // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.#[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))]impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {} // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a// mutable reference when the reference count reaches zero and `T` is dropped.unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {} // SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`// because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,// it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an// `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when// the reference count reaches zero and `T` is dropped.unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {} impl<T> Arc<T> { /// Constructs a new reference counted instance of `T`. pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. let value = ArcInner { // SAFETY: There are no safety requirements for this FFI call. refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), data: contents, }; let inner = KBox::new(value, flags)?; let inner = KBox::leak(inner).into(); // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new // `Arc` object. Ok(unsafe { Self::from_inner(inner) }) }} impl<T: ?Sized> Arc<T> { /// Constructs a new [`Arc`] from an existing [`ArcInner`]. /// /// # Safety /// /// The caller must ensure that `inner` points to a valid location and has a non-zero reference /// count, one of which will be owned by the new [`Arc`] instance. unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self { // INVARIANT: By the safety requirements, the invariants hold. Arc { ptr: inner, _p: PhantomData, } } /// Convert the [`Arc`] into a raw pointer. /// /// The raw pointer has ownership of the refcount that this Arc object owned. pub fn into_raw(self) -> *const T { let ptr = self.ptr.as_ptr(); core::mem::forget(self); // SAFETY: The pointer is valid. unsafe { core::ptr::addr_of!((*ptr).data) } } /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. /// /// # Safety /// /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it /// must not be called more than once for each previous call to [`Arc::into_raw`]. pub unsafe fn from_raw(ptr: *const T) -> Self { // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an // `Arc` that is still valid. let ptr = unsafe { ArcInner::container_of(ptr) }; // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the // reference count held then will be owned by the new `Arc` object. unsafe { Self::from_inner(ptr) } } /// Returns an [`ArcBorrow`] from the given [`Arc`]. /// /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. #[inline] pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { // SAFETY: The constraint that the lifetime of the shared reference must outlive that of // the returned `ArcBorrow` ensures that the object remains alive and that no mutable // reference can be created. unsafe { ArcBorrow::new(self.ptr) } } /// Compare whether two [`Arc`] pointers reference the same underlying object. pub fn ptr_eq(this: &Self, other: &Self) -> bool { core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr()) } /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. /// /// When this destroys the `Arc`, it does so while properly avoiding races. This means that /// this method will never call the destructor of the value. /// /// # Examples /// /// ``` /// use kernel::sync::{Arc, UniqueArc}; /// /// let arc = Arc::new(42, GFP_KERNEL)?; /// let unique_arc = arc.into_unique_or_drop(); /// /// // The above conversion should succeed since refcount of `arc` is 1. /// assert!(unique_arc.is_some()); /// /// assert_eq!(*(unique_arc.unwrap()), 42); /// /// # Ok::<(), Error>(()) /// ``` /// /// ``` /// use kernel::sync::{Arc, UniqueArc}; /// /// let arc = Arc::new(42, GFP_KERNEL)?; /// let another = arc.clone(); /// /// let unique_arc = arc.into_unique_or_drop(); /// /// // The above conversion should fail since refcount of `arc` is >1. /// assert!(unique_arc.is_none()); /// /// # Ok::<(), Error>(()) /// ``` pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> { // We will manually manage the refcount in this method, so we disable the destructor. let me = ManuallyDrop::new(self); // SAFETY: We own a refcount, so the pointer is still valid. let refcount = unsafe { me.ptr.as_ref() }.refcount.get(); // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will // return without further touching the `Arc`. If the refcount reaches zero, then there are // no other arcs, and we can create a `UniqueArc`. // // SAFETY: We own a refcount, so the pointer is not dangling. let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; if is_zero { // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized // accesses to the refcount. unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) }; // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin // their values. Some(Pin::from(UniqueArc { inner: ManuallyDrop::into_inner(me), })) } else { None } }} impl<T: 'static> ForeignOwnable for Arc<T> { type Borrowed<'a> = ArcBorrow<'a, T>; type BorrowedMut<'a> = Self::Borrowed<'a>; fn into_foreign(self) -> *mut crate::ffi::c_void { ManuallyDrop::new(self).ptr.as_ptr().cast() } unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) }; // SAFETY: By the safety requirement of this function, we know that `ptr` came from // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and // holds a reference count increment that is transferrable to us. unsafe { Self::from_inner(inner) } } unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) }; // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive // for the lifetime of the returned value. unsafe { ArcBorrow::new(inner) } } unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> { // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety // requirements for `borrow`. unsafe { Self::borrow(ptr) } }} impl<T: ?Sized> Deref for Arc<T> { type Target = T; fn deref(&self) -> &Self::Target { // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is // safe to dereference it. unsafe { &self.ptr.as_ref().data } }} impl<T: ?Sized> AsRef<T> for Arc<T> { fn as_ref(&self) -> &T { self.deref() }} impl<T: ?Sized> Clone for Arc<T> { fn clone(&self) -> Self { // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is // safe to dereference it. let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero. // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is // safe to increment the refcount. unsafe { bindings::refcount_inc(refcount) }; // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`. unsafe { Self::from_inner(self.ptr) } }} impl<T: ?Sized> Drop for Arc<T> { fn drop(&mut self) { // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot // touch `refcount` after it's decremented to a non-zero value because another thread/CPU // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to // freed/invalid memory as long as it is never dereferenced. let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and // this instance is being dropped, so the broken invariant is not observable. // SAFETY: Also by the type invariant, we are allowed to decrement the refcount. let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; if is_zero { // The count reached zero, we must free the memory. // // SAFETY: The pointer was initialised from the result of `KBox::leak`. unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) }; } }} impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> { fn from(item: UniqueArc<T>) -> Self { item.inner }} impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { fn from(item: Pin<UniqueArc<T>>) -> Self { // SAFETY: The type invariants of `Arc` guarantee that the data is pinned. unsafe { Pin::into_inner_unchecked(item).inner } }} /// A borrowed reference to an [`Arc`] instance.////// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.////// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if/// needed.////// # Invariants////// There are no mutable references to the underlying [`Arc`], and it remains valid for the/// lifetime of the [`ArcBorrow`] instance.////// # Example////// ```/// use kernel::sync::{Arc, ArcBorrow};////// struct Example;////// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {/// e.into()/// }////// let obj = Arc::new(Example, GFP_KERNEL)?;/// let cloned = do_something(obj.as_arc_borrow());////// // Assert that both `obj` and `cloned` point to the same underlying object./// assert!(core::ptr::eq(&*obj, &*cloned));/// # Ok::<(), Error>(())/// ```////// Using `ArcBorrow<T>` as the type of `self`:////// ```/// use kernel::sync::{Arc, ArcBorrow};////// struct Example {/// a: u32,/// b: u32,/// }////// impl Example {/// fn use_reference(self: ArcBorrow<'_, Self>) {/// // .../// }/// }////// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;/// obj.as_arc_borrow().use_reference();/// # Ok::<(), Error>(())Lines 1–500 of 785. Load more
// SPDX-License-Identifier: GPL-2.0 //! A reference-counted pointer. //! //! This module implements a way for users to create reference-counted objects and pointers to //! them. Such a pointer automatically increments and decrements the count, and drops the //! underlying object when it reaches zero. It is also safe to use concurrently from multiple //! threads. //! //! It is different from the standard library's [`Arc`] in a few ways: //! 1. It is backed by the kernel's `refcount_t` type. //! 2. It does not support weak references, which allows it to be half the size. //! 3. It saturates the reference count instead of aborting when it goes over a threshold. //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned. //! 5. The object in [`Arc`] is pinned implicitly. //! //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html use crate::{ alloc::{AllocError, Flags, KBox}, bindings, init::{self, InPlaceInit, Init, PinInit}, try_init, types::{ForeignOwnable, Opaque}, }; use core::{ alloc::Layout, fmt, marker::PhantomData, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, pin::Pin, ptr::NonNull, }; use macros::pin_data; mod std_vendor; /// A reference-counted pointer to an instance of `T`. /// /// The reference count is incremented when new instances of [`Arc`] are created, and decremented /// when they are dropped. When the count reaches zero, the underlying `T` is also dropped. /// /// # Invariants /// /// The reference count on an instance of [`Arc`] is always non-zero. /// The object pointed to by [`Arc`] is always pinned. /// /// # Examples /// /// ``` /// use kernel::sync::Arc; /// /// struct Example { /// a: u32, /// b: u32, /// } /// /// // Create a refcounted instance of `Example`. /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// /// // Get a new pointer to `obj` and increment the refcount. /// let cloned = obj.clone(); /// /// // Assert that both `obj` and `cloned` point to the same underlying object. /// assert!(core::ptr::eq(&*obj, &*cloned)); /// /// // Destroy `obj` and decrement its refcount. /// drop(obj); /// /// // Check that the values are still accessible through `cloned`. /// assert_eq!(cloned.a, 10); /// assert_eq!(cloned.b, 20); /// /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed. /// # Ok::<(), Error>(()) /// ``` /// /// Using `Arc<T>` as the type of `self`: /// /// ``` /// use kernel::sync::Arc; /// /// struct Example { /// a: u32, /// b: u32, /// } /// /// impl Example { /// fn take_over(self: Arc<Self>) { /// // ... /// } /// /// fn use_reference(self: &Arc<Self>) { /// // ... /// } /// } /// /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// obj.use_reference(); /// obj.take_over(); /// # Ok::<(), Error>(()) /// ``` /// /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`: /// /// ``` /// use kernel::sync::{Arc, ArcBorrow}; /// /// trait MyTrait { /// // Trait has a function whose `self` type is `Arc<Self>`. /// fn example1(self: Arc<Self>) {} /// /// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`. /// fn example2(self: ArcBorrow<'_, Self>) {} /// } /// /// struct Example; /// impl MyTrait for Example {} /// /// // `obj` has type `Arc<Example>`. /// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?; /// /// // `coerced` has type `Arc<dyn MyTrait>`. /// let coerced: Arc<dyn MyTrait> = obj; /// # Ok::<(), Error>(()) /// ``` #[repr(transparent)] #[cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, derive(core::marker::CoercePointee))] pub struct Arc<T: ?Sized> { ptr: NonNull<ArcInner<T>>, // NB: this informs dropck that objects of type `ArcInner<T>` may be used in `<Arc<T> as // Drop>::drop`. Note that dropck already assumes that objects of type `T` may be used in // `<Arc<T> as Drop>::drop` and the distinction between `T` and `ArcInner<T>` is not presently // meaningful with respect to dropck - but this may change in the future so this is left here // out of an abundance of caution. // // See https://doc.rust-lang.org/nomicon/phantom-data.html#generic-parameters-and-drop-checking // for more detail on the semantics of dropck in the presence of `PhantomData`. _p: PhantomData<ArcInner<T>>, } #[pin_data] #[repr(C)] struct ArcInner<T: ?Sized> { refcount: Opaque<bindings::refcount_t>, data: T, } impl<T: ?Sized> ArcInner<T> { /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`]. /// /// # Safety /// /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must /// not yet have been destroyed. unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { let refcount_layout = Layout::new::<bindings::refcount_t>(); // SAFETY: The caller guarantees that the pointer is valid. let val_layout = Layout::for_value(unsafe { &*ptr }); // SAFETY: We're computing the layout of a real struct that existed when compiling this // binary, so its layout is not so large that it can trigger arithmetic overflow. let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field. // // This is documented at: // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>. let ptr = ptr as *const ArcInner<T>; // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is // still valid. let ptr = unsafe { ptr.byte_sub(val_offset) }; // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null // address. unsafe { NonNull::new_unchecked(ptr.cast_mut()) } } } // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the // dynamically-sized type (DST) `U`. #[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {} // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`. #[cfg(not(CONFIG_RUSTC_HAS_COERCE_POINTEE))] impl<T: ?Sized + core::marker::Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {} // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs // `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a // mutable reference when the reference count reaches zero and `T` is dropped. unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {} // SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync` // because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, // it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an // `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when // the reference count reaches zero and `T` is dropped. unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {} impl<T> Arc<T> { /// Constructs a new reference counted instance of `T`. pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. let value = ArcInner { // SAFETY: There are no safety requirements for this FFI call. refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), data: contents, }; let inner = KBox::new(value, flags)?; let inner = KBox::leak(inner).into(); // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new // `Arc` object. Ok(unsafe { Self::from_inner(inner) }) } } impl<T: ?Sized> Arc<T> { /// Constructs a new [`Arc`] from an existing [`ArcInner`]. /// /// # Safety /// /// The caller must ensure that `inner` points to a valid location and has a non-zero reference /// count, one of which will be owned by the new [`Arc`] instance. unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self { // INVARIANT: By the safety requirements, the invariants hold. Arc { ptr: inner, _p: PhantomData, } } /// Convert the [`Arc`] into a raw pointer. /// /// The raw pointer has ownership of the refcount that this Arc object owned. pub fn into_raw(self) -> *const T { let ptr = self.ptr.as_ptr(); core::mem::forget(self); // SAFETY: The pointer is valid. unsafe { core::ptr::addr_of!((*ptr).data) } } /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. /// /// # Safety /// /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it /// must not be called more than once for each previous call to [`Arc::into_raw`]. pub unsafe fn from_raw(ptr: *const T) -> Self { // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an // `Arc` that is still valid. let ptr = unsafe { ArcInner::container_of(ptr) }; // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the // reference count held then will be owned by the new `Arc` object. unsafe { Self::from_inner(ptr) } } /// Returns an [`ArcBorrow`] from the given [`Arc`]. /// /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. #[inline] pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { // SAFETY: The constraint that the lifetime of the shared reference must outlive that of // the returned `ArcBorrow` ensures that the object remains alive and that no mutable // reference can be created. unsafe { ArcBorrow::new(self.ptr) } } /// Compare whether two [`Arc`] pointers reference the same underlying object. pub fn ptr_eq(this: &Self, other: &Self) -> bool { core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr()) } /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. /// /// When this destroys the `Arc`, it does so while properly avoiding races. This means that /// this method will never call the destructor of the value. /// /// # Examples /// /// ``` /// use kernel::sync::{Arc, UniqueArc}; /// /// let arc = Arc::new(42, GFP_KERNEL)?; /// let unique_arc = arc.into_unique_or_drop(); /// /// // The above conversion should succeed since refcount of `arc` is 1. /// assert!(unique_arc.is_some()); /// /// assert_eq!(*(unique_arc.unwrap()), 42); /// /// # Ok::<(), Error>(()) /// ``` /// /// ``` /// use kernel::sync::{Arc, UniqueArc}; /// /// let arc = Arc::new(42, GFP_KERNEL)?; /// let another = arc.clone(); /// /// let unique_arc = arc.into_unique_or_drop(); /// /// // The above conversion should fail since refcount of `arc` is >1. /// assert!(unique_arc.is_none()); /// /// # Ok::<(), Error>(()) /// ``` pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> { // We will manually manage the refcount in this method, so we disable the destructor. let me = ManuallyDrop::new(self); // SAFETY: We own a refcount, so the pointer is still valid. let refcount = unsafe { me.ptr.as_ref() }.refcount.get(); // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will // return without further touching the `Arc`. If the refcount reaches zero, then there are // no other arcs, and we can create a `UniqueArc`. // // SAFETY: We own a refcount, so the pointer is not dangling. let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; if is_zero { // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized // accesses to the refcount. unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) }; // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin // their values. Some(Pin::from(UniqueArc { inner: ManuallyDrop::into_inner(me), })) } else { None } } } impl<T: 'static> ForeignOwnable for Arc<T> { type Borrowed<'a> = ArcBorrow<'a, T>; type BorrowedMut<'a> = Self::Borrowed<'a>; fn into_foreign(self) -> *mut crate::ffi::c_void { ManuallyDrop::new(self).ptr.as_ptr().cast() } unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) }; // SAFETY: By the safety requirement of this function, we know that `ptr` came from // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and // holds a reference count increment that is transferrable to us. unsafe { Self::from_inner(inner) } } unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> { // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous // call to `Self::into_foreign`. let inner = unsafe { NonNull::new_unchecked(ptr.cast::<ArcInner<T>>()) }; // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive // for the lifetime of the returned value. unsafe { ArcBorrow::new(inner) } } unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> ArcBorrow<'a, T> { // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety // requirements for `borrow`. unsafe { Self::borrow(ptr) } } } impl<T: ?Sized> Deref for Arc<T> { type Target = T; fn deref(&self) -> &Self::Target { // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is // safe to dereference it. unsafe { &self.ptr.as_ref().data } } } impl<T: ?Sized> AsRef<T> for Arc<T> { fn as_ref(&self) -> &T { self.deref() } } impl<T: ?Sized> Clone for Arc<T> { fn clone(&self) -> Self { // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is // safe to dereference it. let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero. // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is // safe to increment the refcount. unsafe { bindings::refcount_inc(refcount) }; // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`. unsafe { Self::from_inner(self.ptr) } } } impl<T: ?Sized> Drop for Arc<T> { fn drop(&mut self) { // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot // touch `refcount` after it's decremented to a non-zero value because another thread/CPU // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to // freed/invalid memory as long as it is never dereferenced. let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and // this instance is being dropped, so the broken invariant is not observable. // SAFETY: Also by the type invariant, we are allowed to decrement the refcount. let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; if is_zero { // The count reached zero, we must free the memory. // // SAFETY: The pointer was initialised from the result of `KBox::leak`. unsafe { drop(KBox::from_raw(self.ptr.as_ptr())) }; } } } impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> { fn from(item: UniqueArc<T>) -> Self { item.inner } } impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { fn from(item: Pin<UniqueArc<T>>) -> Self { // SAFETY: The type invariants of `Arc` guarantee that the data is pinned. unsafe { Pin::into_inner_unchecked(item).inner } } } /// A borrowed reference to an [`Arc`] instance. /// /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler /// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance. /// /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>` /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference) /// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double /// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if /// needed. /// /// # Invariants /// /// There are no mutable references to the underlying [`Arc`], and it remains valid for the /// lifetime of the [`ArcBorrow`] instance. /// /// # Example /// /// ``` /// use kernel::sync::{Arc, ArcBorrow}; /// /// struct Example; /// /// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> { /// e.into() /// } /// /// let obj = Arc::new(Example, GFP_KERNEL)?; /// let cloned = do_something(obj.as_arc_borrow()); /// /// // Assert that both `obj` and `cloned` point to the same underlying object. /// assert!(core::ptr::eq(&*obj, &*cloned)); /// # Ok::<(), Error>(()) /// ``` /// /// Using `ArcBorrow<T>` as the type of `self`: /// /// ``` /// use kernel::sync::{Arc, ArcBorrow}; /// /// struct Example { /// a: u32, /// b: u32, /// } /// /// impl Example { /// fn use_reference(self: ArcBorrow<'_, Self>) { /// // ... /// } /// } /// /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// obj.as_arc_borrow().use_reference(); /// # Ok::<(), Error>(())