// SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2025 Google LLC. use kernel::{ list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc}, prelude::*, seq_file::SeqFile, seq_print, sync::lock::{spinlock::SpinLockBackend, Guard}, sync::{Arc, LockedBy, SpinLock},}; use crate::{ defs::*, error::BinderError, process::{NodeRefInfo, Process, ProcessInner}, thread::Thread, transaction::Transaction, BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverToRead,}; use core::mem; mod wrapper;pub(crate) use self::wrapper::CritIncrWrapper; #[derive(Debug)]pub(crate) struct CouldNotDeliverCriticalIncrement; /// Keeps track of how this node is scheduled.////// There are two ways to schedule a node to a work list. Just schedule the node itself, or/// allocate a wrapper that references the node and schedule the wrapper. These wrappers exists to/// make it possible to "move" a node from one list to another - when `do_work` is called directly/// on the `Node`, then it's a no-op if there's also a pending wrapper.////// Wrappers are generally only needed for zero-to-one refcount increments, and there are two cases/// of this: weak increments and strong increments. We call such increments "critical" because it/// is critical that they are delivered to the thread doing the increment. Some examples:////// * One thread makes a zero-to-one strong increment, and another thread makes a zero-to-one weak/// increment. Delivering the node to the thread doing the weak increment is wrong, since the/// thread doing the strong increment may have ended a long time ago when the command is actually/// processed by userspace.////// * We have a weak reference and are about to drop it on one thread. But then another thread does/// a zero-to-one strong increment. If the strong increment gets sent to the thread that was/// about to drop the weak reference, then the strong increment could be processed after the/// other thread has already exited, which would be too late.////// Note that trying to create a `ListArc` to the node can succeed even if `has_normal_push` is/// set. This is because another thread might just have popped the node from a todo list, but not/// yet called `do_work`. However, if `has_normal_push` is false, then creating a `ListArc` should/// always succeed.////// Like the other fields in `NodeInner`, the delivery state is protected by the process lock.struct DeliveryState { /// Is the `Node` currently scheduled? has_pushed_node: bool, /// Is a wrapper currently scheduled? /// /// The wrapper is used only for strong zero2one increments. has_pushed_wrapper: bool, /// Is the currently scheduled `Node` scheduled due to a weak zero2one increment? /// /// Weak zero2one operations are always scheduled using the `Node`. has_weak_zero2one: bool, /// Is the currently scheduled wrapper/`Node` scheduled due to a strong zero2one increment? /// /// If `has_pushed_wrapper` is set, then the strong zero2one increment was scheduled using the /// wrapper. Otherwise, `has_pushed_node` must be set and it was scheduled using the `Node`. has_strong_zero2one: bool,} impl DeliveryState { fn should_normal_push(&self) -> bool { !self.has_pushed_node && !self.has_pushed_wrapper } fn did_normal_push(&mut self) { assert!(self.should_normal_push()); self.has_pushed_node = true; } fn should_push_weak_zero2one(&self) -> bool { !self.has_weak_zero2one && !self.has_strong_zero2one } fn can_push_weak_zero2one_normally(&self) -> bool { !self.has_pushed_node } fn did_push_weak_zero2one(&mut self) { assert!(self.should_push_weak_zero2one()); assert!(self.can_push_weak_zero2one_normally()); self.has_pushed_node = true; self.has_weak_zero2one = true; } fn should_push_strong_zero2one(&self) -> bool { !self.has_strong_zero2one } fn can_push_strong_zero2one_normally(&self) -> bool { !self.has_pushed_node } fn did_push_strong_zero2one(&mut self) { assert!(self.should_push_strong_zero2one()); assert!(self.can_push_strong_zero2one_normally()); self.has_pushed_node = true; self.has_strong_zero2one = true; } fn did_push_strong_zero2one_wrapper(&mut self) { assert!(self.should_push_strong_zero2one()); assert!(!self.can_push_strong_zero2one_normally()); self.has_pushed_wrapper = true; self.has_strong_zero2one = true; }} struct CountState { /// The reference count. count: usize, /// Whether the process that owns this node thinks that we hold a refcount on it. (Note that /// even if count is greater than one, we only increment it once in the owning process.) has_count: bool,} impl CountState { fn new() -> Self { Self { count: 0, has_count: false, } }} struct NodeInner { /// Strong refcounts held on this node by `NodeRef` objects. strong: CountState, /// Weak refcounts held on this node by `NodeRef` objects. weak: CountState, delivery_state: DeliveryState, /// The binder driver guarantees that oneway transactions sent to the same node are serialized, /// that is, userspace will not be given the next one until it has finished processing the /// previous oneway transaction. This is done to avoid the case where two oneway transactions /// arrive in opposite order from the order in which they were sent. (E.g., they could be /// delivered to two different threads, which could appear as-if they were sent in opposite /// order.) /// /// To fix that, we store pending oneway transactions in a separate list in the node, and don't /// deliver the next oneway transaction until userspace signals that it has finished processing /// the previous oneway transaction by calling the `BC_FREE_BUFFER` ioctl. oneway_todo: List<DTRWrap<Transaction>>, /// Keeps track of whether this node has a pending oneway transaction. /// /// When this is true, incoming oneway transactions are stored in `oneway_todo`, instead of /// being delivered directly to the process. has_oneway_transaction: bool, /// List of processes to deliver a notification to when this node is destroyed (usually due to /// the process dying). death_list: List<DTRWrap<NodeDeath>, 1>, /// List of processes to deliver freeze notifications to. freeze_list: KVVec<Arc<Process>>, /// The number of active BR_INCREFS or BR_ACQUIRE operations. (should be maximum two) /// /// If this is non-zero, then we postpone any BR_RELEASE or BR_DECREFS notifications until the /// active operations have ended. This avoids the situation an increment and decrement get /// reordered from userspace's perspective. active_inc_refs: u8, /// List of `NodeRefInfo` objects that reference this node. refs: List<NodeRefInfo, { NodeRefInfo::LIST_NODE }>,} #[pin_data]pub(crate) struct Node { pub(crate) debug_id: usize, ptr: u64, pub(crate) cookie: u64, pub(crate) flags: u32, pub(crate) owner: Arc<Process>, inner: LockedBy<NodeInner, ProcessInner>, #[pin] links_track: AtomicTracker,} kernel::list::impl_list_arc_safe! { impl ListArcSafe<0> for Node { tracked_by links_track: AtomicTracker; }} // Make `oneway_todo` work.kernel::list::impl_list_item! { impl ListItem<0> for DTRWrap<Transaction> { using ListLinks { self.links.inner }; }} impl Node { pub(crate) fn new( ptr: u64, cookie: u64, flags: u32, owner: Arc<Process>, ) -> impl PinInit<Self> { pin_init!(Self { inner: LockedBy::new( &owner.inner, NodeInner { strong: CountState::new(), weak: CountState::new(), delivery_state: DeliveryState { has_pushed_node: false, has_pushed_wrapper: false, has_weak_zero2one: false, has_strong_zero2one: false, }, death_list: List::new(), oneway_todo: List::new(), freeze_list: KVVec::new(), has_oneway_transaction: false, active_inc_refs: 0, refs: List::new(), }, ), debug_id: super::next_debug_id(), ptr, cookie, flags, owner, links_track <- AtomicTracker::new(), }) } pub(crate) fn has_oneway_transaction(&self, owner_inner: &mut ProcessInner) -> bool { let inner = self.inner.access_mut(owner_inner); inner.has_oneway_transaction } #[inline(never)] pub(crate) fn full_debug_print( &self, m: &SeqFile, owner_inner: &mut ProcessInner, ) -> Result<()> { let inner = self.inner.access_mut(owner_inner); seq_print!( m, " node {}: u{:016x} c{:016x} hs {} hw {} cs {} cw {}", self.debug_id, self.ptr, self.cookie, inner.strong.has_count, inner.weak.has_count, inner.strong.count, inner.weak.count, ); if !inner.refs.is_empty() { seq_print!(m, " proc"); for node_ref in &inner.refs { seq_print!(m, " {}", node_ref.process.task.pid()); } } seq_print!(m, "\n"); for t in &inner.oneway_todo { t.debug_print_inner(m, " pending async transaction "); } Ok(()) } /// Insert the `NodeRef` into this `refs` list. /// /// # Safety /// /// It must be the case that `info.node_ref.node` is this node. pub(crate) unsafe fn insert_node_info( &self, info: ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>, ) { self.inner .access_mut(&mut self.owner.inner.lock()) .refs .push_front(info); } /// Insert the `NodeRef` into this `refs` list. /// /// # Safety /// /// It must be the case that `info.node_ref.node` is this node. pub(crate) unsafe fn remove_node_info( &self, info: &NodeRefInfo, ) -> Option<ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>> { // SAFETY: We always insert `NodeRefInfo` objects into the `refs` list of the node that it // references in `info.node_ref.node`. That is this node, so `info` cannot possibly be in // the `refs` list of another node. unsafe { self.inner .access_mut(&mut self.owner.inner.lock()) .refs .remove(info) } } /// An id that is unique across all binder nodes on the system. Used as the key in the /// `by_node` map. pub(crate) fn global_id(&self) -> usize { self as *const Node as usize } pub(crate) fn get_id(&self) -> (u64, u64) { (self.ptr, self.cookie) } pub(crate) fn add_death( &self, death: ListArc<DTRWrap<NodeDeath>, 1>, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>, ) { self.inner.access_mut(guard).death_list.push_back(death); } pub(crate) fn inc_ref_done_locked( self: &DArc<Node>, _strong: bool, owner_inner: &mut ProcessInner, ) -> Option<DLArc<Node>> { let inner = self.inner.access_mut(owner_inner); if inner.active_inc_refs == 0 { pr_err!("inc_ref_done called when no active inc_refs"); return None; } inner.active_inc_refs -= 1; if inner.active_inc_refs == 0 { // Having active inc_refs can inhibit dropping of ref-counts. Calculate whether we // would send a refcount decrement, and if so, tell the caller to schedule us. let strong = inner.strong.count > 0; let has_strong = inner.strong.has_count; let weak = strong || inner.weak.count > 0; let has_weak = inner.weak.has_count; let should_drop_weak = !weak && has_weak; let should_drop_strong = !strong && has_strong; // If we want to drop the ref-count again, tell the caller to schedule a work node for // that. let need_push = should_drop_weak || should_drop_strong; if need_push && inner.delivery_state.should_normal_push() { let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); inner.delivery_state.did_normal_push(); Some(list_arc) } else { None } } else { None } } pub(crate) fn update_refcount_locked( self: &DArc<Node>, inc: bool, strong: bool, count: usize, owner_inner: &mut ProcessInner, ) -> Option<DLArc<Node>> { let is_dead = owner_inner.is_dead; let inner = self.inner.access_mut(owner_inner); // Get a reference to the state we'll update. let state = if strong { &mut inner.strong } else { &mut inner.weak }; // Update the count and determine whether we need to push work. let need_push = if inc { state.count += count; // TODO: This method shouldn't be used for zero-to-one increments. !is_dead && !state.has_count } else { if state.count < count { pr_err!("Failure: refcount underflow!"); return None; } state.count -= count; !is_dead && state.count == 0 && state.has_count }; if need_push && inner.delivery_state.should_normal_push() { let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); inner.delivery_state.did_normal_push(); Some(list_arc) } else { None } } pub(crate) fn incr_refcount_allow_zero2one( self: &DArc<Self>, strong: bool, owner_inner: &mut ProcessInner, ) -> Result<Option<DLArc<Node>>, CouldNotDeliverCriticalIncrement> { let is_dead = owner_inner.is_dead; let inner = self.inner.access_mut(owner_inner); // Get a reference to the state we'll update. let state = if strong { &mut inner.strong } else { &mut inner.weak }; // Update the count and determine whether we need to push work. state.count += 1; if is_dead || state.has_count { return Ok(None); } // Userspace needs to be notified of this. if !strong && inner.delivery_state.should_push_weak_zero2one() { assert!(inner.delivery_state.can_push_weak_zero2one_normally()); let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); inner.delivery_state.did_push_weak_zero2one(); Ok(Some(list_arc)) } else if strong && inner.delivery_state.should_push_strong_zero2one() { if inner.delivery_state.can_push_strong_zero2one_normally() { let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); inner.delivery_state.did_push_strong_zero2one(); Ok(Some(list_arc)) } else { state.count -= 1; Err(CouldNotDeliverCriticalIncrement) } } else { // Work is already pushed, and we don't need to push again. Ok(None) } } pub(crate) fn incr_refcount_allow_zero2one_with_wrapper( self: &DArc<Self>, strong: bool, wrapper: CritIncrWrapper, owner_inner: &mut ProcessInner, ) -> Option<DLArc<dyn DeliverToRead>> { match self.incr_refcount_allow_zero2one(strong, owner_inner) { Ok(Some(node)) => Some(node as _), Ok(None) => None, Err(CouldNotDeliverCriticalIncrement) => { assert!(strong); let inner = self.inner.access_mut(owner_inner); inner.strong.count += 1; inner.delivery_state.did_push_strong_zero2one_wrapper(); Some(wrapper.init(self.clone())) } } } pub(crate) fn update_refcount(self: &DArc<Self>, inc: bool, count: usize, strong: bool) { self.owner .inner .lock() .update_node_refcount(self, inc, strong, count, None); } pub(crate) fn populate_counts( &self, out: &mut BinderNodeInfoForRef, guard: &Guard<'_, ProcessInner, SpinLockBackend>, ) { let inner = self.inner.access(guard); out.strong_count = inner.strong.count as _; out.weak_count = inner.weak.count as _; } pub(crate) fn populate_debug_info( &self, out: &mut BinderNodeDebugInfo, guard: &Guard<'_, ProcessInner, SpinLockBackend>, ) { out.ptr = self.ptr as _; out.cookie = self.cookie as _; let inner = self.inner.access(guard); if inner.strong.has_count { out.has_strong_ref = 1; } if inner.weak.has_count { out.has_weak_ref = 1;Lines 1–500 of 1131. Load more
// SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2025 Google LLC. use kernel::{ list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc}, prelude::*, seq_file::SeqFile, seq_print, sync::lock::{spinlock::SpinLockBackend, Guard}, sync::{Arc, LockedBy, SpinLock}, }; use crate::{ defs::*, error::BinderError, process::{NodeRefInfo, Process, ProcessInner}, thread::Thread, transaction::Transaction, BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverToRead, }; use core::mem; mod wrapper; pub(crate) use self::wrapper::CritIncrWrapper; #[derive(Debug)] pub(crate) struct CouldNotDeliverCriticalIncrement; /// Keeps track of how this node is scheduled. /// /// There are two ways to schedule a node to a work list. Just schedule the node itself, or /// allocate a wrapper that references the node and schedule the wrapper. These wrappers exists to /// make it possible to "move" a node from one list to another - when `do_work` is called directly /// on the `Node`, then it's a no-op if there's also a pending wrapper. /// /// Wrappers are generally only needed for zero-to-one refcount increments, and there are two cases /// of this: weak increments and strong increments. We call such increments "critical" because it /// is critical that they are delivered to the thread doing the increment. Some examples: /// /// * One thread makes a zero-to-one strong increment, and another thread makes a zero-to-one weak /// increment. Delivering the node to the thread doing the weak increment is wrong, since the /// thread doing the strong increment may have ended a long time ago when the command is actually /// processed by userspace. /// /// * We have a weak reference and are about to drop it on one thread. But then another thread does /// a zero-to-one strong increment. If the strong increment gets sent to the thread that was /// about to drop the weak reference, then the strong increment could be processed after the /// other thread has already exited, which would be too late. /// /// Note that trying to create a `ListArc` to the node can succeed even if `has_normal_push` is /// set. This is because another thread might just have popped the node from a todo list, but not /// yet called `do_work`. However, if `has_normal_push` is false, then creating a `ListArc` should /// always succeed. /// /// Like the other fields in `NodeInner`, the delivery state is protected by the process lock. struct DeliveryState { /// Is the `Node` currently scheduled? has_pushed_node: bool, /// Is a wrapper currently scheduled? /// /// The wrapper is used only for strong zero2one increments. has_pushed_wrapper: bool, /// Is the currently scheduled `Node` scheduled due to a weak zero2one increment? /// /// Weak zero2one operations are always scheduled using the `Node`. has_weak_zero2one: bool, /// Is the currently scheduled wrapper/`Node` scheduled due to a strong zero2one increment? /// /// If `has_pushed_wrapper` is set, then the strong zero2one increment was scheduled using the /// wrapper. Otherwise, `has_pushed_node` must be set and it was scheduled using the `Node`. has_strong_zero2one: bool, } impl DeliveryState { fn should_normal_push(&self) -> bool { !self.has_pushed_node && !self.has_pushed_wrapper } fn did_normal_push(&mut self) { assert!(self.should_normal_push()); self.has_pushed_node = true; } fn should_push_weak_zero2one(&self) -> bool { !self.has_weak_zero2one && !self.has_strong_zero2one } fn can_push_weak_zero2one_normally(&self) -> bool { !self.has_pushed_node } fn did_push_weak_zero2one(&mut self) { assert!(self.should_push_weak_zero2one()); assert!(self.can_push_weak_zero2one_normally()); self.has_pushed_node = true; self.has_weak_zero2one = true; } fn should_push_strong_zero2one(&self) -> bool { !self.has_strong_zero2one } fn can_push_strong_zero2one_normally(&self) -> bool { !self.has_pushed_node } fn did_push_strong_zero2one(&mut self) { assert!(self.should_push_strong_zero2one()); assert!(self.can_push_strong_zero2one_normally()); self.has_pushed_node = true; self.has_strong_zero2one = true; } fn did_push_strong_zero2one_wrapper(&mut self) { assert!(self.should_push_strong_zero2one()); assert!(!self.can_push_strong_zero2one_normally()); self.has_pushed_wrapper = true; self.has_strong_zero2one = true; } } struct CountState { /// The reference count. count: usize, /// Whether the process that owns this node thinks that we hold a refcount on it. (Note that /// even if count is greater than one, we only increment it once in the owning process.) has_count: bool, } impl CountState { fn new() -> Self { Self { count: 0, has_count: false, } } } struct NodeInner { /// Strong refcounts held on this node by `NodeRef` objects. strong: CountState, /// Weak refcounts held on this node by `NodeRef` objects. weak: CountState, delivery_state: DeliveryState, /// The binder driver guarantees that oneway transactions sent to the same node are serialized, /// that is, userspace will not be given the next one until it has finished processing the /// previous oneway transaction. This is done to avoid the case where two oneway transactions /// arrive in opposite order from the order in which they were sent. (E.g., they could be /// delivered to two different threads, which could appear as-if they were sent in opposite /// order.) /// /// To fix that, we store pending oneway transactions in a separate list in the node, and don't /// deliver the next oneway transaction until userspace signals that it has finished processing /// the previous oneway transaction by calling the `BC_FREE_BUFFER` ioctl. oneway_todo: List<DTRWrap<Transaction>>, /// Keeps track of whether this node has a pending oneway transaction. /// /// When this is true, incoming oneway transactions are stored in `oneway_todo`, instead of /// being delivered directly to the process. has_oneway_transaction: bool, /// List of processes to deliver a notification to when this node is destroyed (usually due to /// the process dying). death_list: List<DTRWrap<NodeDeath>, 1>, /// List of processes to deliver freeze notifications to. freeze_list: KVVec<Arc<Process>>, /// The number of active BR_INCREFS or BR_ACQUIRE operations. (should be maximum two) /// /// If this is non-zero, then we postpone any BR_RELEASE or BR_DECREFS notifications until the /// active operations have ended. This avoids the situation an increment and decrement get /// reordered from userspace's perspective. active_inc_refs: u8, /// List of `NodeRefInfo` objects that reference this node. refs: List<NodeRefInfo, { NodeRefInfo::LIST_NODE }>, } #[pin_data] pub(crate) struct Node { pub(crate) debug_id: usize, ptr: u64, pub(crate) cookie: u64, pub(crate) flags: u32, pub(crate) owner: Arc<Process>, inner: LockedBy<NodeInner, ProcessInner>, #[pin] links_track: AtomicTracker, } kernel::list::impl_list_arc_safe! { impl ListArcSafe<0> for Node { tracked_by links_track: AtomicTracker; } } // Make `oneway_todo` work. kernel::list::impl_list_item! { impl ListItem<0> for DTRWrap<Transaction> { using ListLinks { self.links.inner }; } } impl Node { pub(crate) fn new( ptr: u64, cookie: u64, flags: u32, owner: Arc<Process>, ) -> impl PinInit<Self> { pin_init!(Self { inner: LockedBy::new( &owner.inner, NodeInner { strong: CountState::new(), weak: CountState::new(), delivery_state: DeliveryState { has_pushed_node: false, has_pushed_wrapper: false, has_weak_zero2one: false, has_strong_zero2one: false, }, death_list: List::new(), oneway_todo: List::new(), freeze_list: KVVec::new(), has_oneway_transaction: false, active_inc_refs: 0, refs: List::new(), }, ), debug_id: super::next_debug_id(), ptr, cookie, flags, owner, links_track <- AtomicTracker::new(), }) } pub(crate) fn has_oneway_transaction(&self, owner_inner: &mut ProcessInner) -> bool { let inner = self.inner.access_mut(owner_inner); inner.has_oneway_transaction } #[inline(never)] pub(crate) fn full_debug_print( &self, m: &SeqFile, owner_inner: &mut ProcessInner, ) -> Result<()> { let inner = self.inner.access_mut(owner_inner); seq_print!( m, " node {}: u{:016x} c{:016x} hs {} hw {} cs {} cw {}", self.debug_id, self.ptr, self.cookie, inner.strong.has_count, inner.weak.has_count, inner.strong.count, inner.weak.count, ); if !inner.refs.is_empty() { seq_print!(m, " proc"); for node_ref in &inner.refs { seq_print!(m, " {}", node_ref.process.task.pid()); } } seq_print!(m, "\n"); for t in &inner.oneway_todo { t.debug_print_inner(m, " pending async transaction "); } Ok(()) } /// Insert the `NodeRef` into this `refs` list. /// /// # Safety /// /// It must be the case that `info.node_ref.node` is this node. pub(crate) unsafe fn insert_node_info( &self, info: ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>, ) { self.inner .access_mut(&mut self.owner.inner.lock()) .refs .push_front(info); } /// Insert the `NodeRef` into this `refs` list. /// /// # Safety /// /// It must be the case that `info.node_ref.node` is this node. pub(crate) unsafe fn remove_node_info( &self, info: &NodeRefInfo, ) -> Option<ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>> { // SAFETY: We always insert `NodeRefInfo` objects into the `refs` list of the node that it // references in `info.node_ref.node`. That is this node, so `info` cannot possibly be in // the `refs` list of another node. unsafe { self.inner .access_mut(&mut self.owner.inner.lock()) .refs .remove(info) } } /// An id that is unique across all binder nodes on the system. Used as the key in the /// `by_node` map. pub(crate) fn global_id(&self) -> usize { self as *const Node as usize } pub(crate) fn get_id(&self) -> (u64, u64) { (self.ptr, self.cookie) } pub(crate) fn add_death( &self, death: ListArc<DTRWrap<NodeDeath>, 1>, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>, ) { self.inner.access_mut(guard).death_list.push_back(death); } pub(crate) fn inc_ref_done_locked( self: &DArc<Node>, _strong: bool, owner_inner: &mut ProcessInner, ) -> Option<DLArc<Node>> { let inner = self.inner.access_mut(owner_inner); if inner.active_inc_refs == 0 { pr_err!("inc_ref_done called when no active inc_refs"); return None; } inner.active_inc_refs -= 1; if inner.active_inc_refs == 0 { // Having active inc_refs can inhibit dropping of ref-counts. Calculate whether we // would send a refcount decrement, and if so, tell the caller to schedule us. let strong = inner.strong.count > 0; let has_strong = inner.strong.has_count; let weak = strong || inner.weak.count > 0; let has_weak = inner.weak.has_count; let should_drop_weak = !weak && has_weak; let should_drop_strong = !strong && has_strong; // If we want to drop the ref-count again, tell the caller to schedule a work node for // that. let need_push = should_drop_weak || should_drop_strong; if need_push && inner.delivery_state.should_normal_push() { let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); inner.delivery_state.did_normal_push(); Some(list_arc) } else { None } } else { None } } pub(crate) fn update_refcount_locked( self: &DArc<Node>, inc: bool, strong: bool, count: usize, owner_inner: &mut ProcessInner, ) -> Option<DLArc<Node>> { let is_dead = owner_inner.is_dead; let inner = self.inner.access_mut(owner_inner); // Get a reference to the state we'll update. let state = if strong { &mut inner.strong } else { &mut inner.weak }; // Update the count and determine whether we need to push work. let need_push = if inc { state.count += count; // TODO: This method shouldn't be used for zero-to-one increments. !is_dead && !state.has_count } else { if state.count < count { pr_err!("Failure: refcount underflow!"); return None; } state.count -= count; !is_dead && state.count == 0 && state.has_count }; if need_push && inner.delivery_state.should_normal_push() { let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); inner.delivery_state.did_normal_push(); Some(list_arc) } else { None } } pub(crate) fn incr_refcount_allow_zero2one( self: &DArc<Self>, strong: bool, owner_inner: &mut ProcessInner, ) -> Result<Option<DLArc<Node>>, CouldNotDeliverCriticalIncrement> { let is_dead = owner_inner.is_dead; let inner = self.inner.access_mut(owner_inner); // Get a reference to the state we'll update. let state = if strong { &mut inner.strong } else { &mut inner.weak }; // Update the count and determine whether we need to push work. state.count += 1; if is_dead || state.has_count { return Ok(None); } // Userspace needs to be notified of this. if !strong && inner.delivery_state.should_push_weak_zero2one() { assert!(inner.delivery_state.can_push_weak_zero2one_normally()); let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); inner.delivery_state.did_push_weak_zero2one(); Ok(Some(list_arc)) } else if strong && inner.delivery_state.should_push_strong_zero2one() { if inner.delivery_state.can_push_strong_zero2one_normally() { let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap(); inner.delivery_state.did_push_strong_zero2one(); Ok(Some(list_arc)) } else { state.count -= 1; Err(CouldNotDeliverCriticalIncrement) } } else { // Work is already pushed, and we don't need to push again. Ok(None) } } pub(crate) fn incr_refcount_allow_zero2one_with_wrapper( self: &DArc<Self>, strong: bool, wrapper: CritIncrWrapper, owner_inner: &mut ProcessInner, ) -> Option<DLArc<dyn DeliverToRead>> { match self.incr_refcount_allow_zero2one(strong, owner_inner) { Ok(Some(node)) => Some(node as _), Ok(None) => None, Err(CouldNotDeliverCriticalIncrement) => { assert!(strong); let inner = self.inner.access_mut(owner_inner); inner.strong.count += 1; inner.delivery_state.did_push_strong_zero2one_wrapper(); Some(wrapper.init(self.clone())) } } } pub(crate) fn update_refcount(self: &DArc<Self>, inc: bool, count: usize, strong: bool) { self.owner .inner .lock() .update_node_refcount(self, inc, strong, count, None); } pub(crate) fn populate_counts( &self, out: &mut BinderNodeInfoForRef, guard: &Guard<'_, ProcessInner, SpinLockBackend>, ) { let inner = self.inner.access(guard); out.strong_count = inner.strong.count as _; out.weak_count = inner.weak.count as _; } pub(crate) fn populate_debug_info( &self, out: &mut BinderNodeDebugInfo, guard: &Guard<'_, ProcessInner, SpinLockBackend>, ) { out.ptr = self.ptr as _; out.cookie = self.cookie as _; let inner = self.inner.access(guard); if inner.strong.has_count { out.has_strong_ref = 1; } if inner.weak.has_count { out.has_weak_ref = 1;