// SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2025 Google LLC. use kernel::{    list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc},    prelude::*,    seq_file::SeqFile,    seq_print,    sync::lock::{spinlock::SpinLockBackend, Guard},    sync::{Arc, LockedBy, SpinLock},}; use crate::{    defs::*,    error::BinderError,    process::{NodeRefInfo, Process, ProcessInner},    thread::Thread,    transaction::Transaction,    BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverToRead,}; use core::mem; mod wrapper;pub(crate) use self::wrapper::CritIncrWrapper; #[derive(Debug)]pub(crate) struct CouldNotDeliverCriticalIncrement; /// Keeps track of how this node is scheduled.////// There are two ways to schedule a node to a work list. Just schedule the node itself, or/// allocate a wrapper that references the node and schedule the wrapper. These wrappers exists to/// make it possible to "move" a node from one list to another - when `do_work` is called directly/// on the `Node`, then it's a no-op if there's also a pending wrapper.////// Wrappers are generally only needed for zero-to-one refcount increments, and there are two cases/// of this: weak increments and strong increments. We call such increments "critical" because it/// is critical that they are delivered to the thread doing the increment. Some examples:////// * One thread makes a zero-to-one strong increment, and another thread makes a zero-to-one weak///   increment. Delivering the node to the thread doing the weak increment is wrong, since the///   thread doing the strong increment may have ended a long time ago when the command is actually///   processed by userspace.////// * We have a weak reference and are about to drop it on one thread. But then another thread does///   a zero-to-one strong increment. If the strong increment gets sent to the thread that was///   about to drop the weak reference, then the strong increment could be processed after the///   other thread has already exited, which would be too late.////// Note that trying to create a `ListArc` to the node can succeed even if `has_normal_push` is/// set. This is because another thread might just have popped the node from a todo list, but not/// yet called `do_work`. However, if `has_normal_push` is false, then creating a `ListArc` should/// always succeed.////// Like the other fields in `NodeInner`, the delivery state is protected by the process lock.struct DeliveryState {    /// Is the `Node` currently scheduled?    has_pushed_node: bool,     /// Is a wrapper currently scheduled?    ///    /// The wrapper is used only for strong zero2one increments.    has_pushed_wrapper: bool,     /// Is the currently scheduled `Node` scheduled due to a weak zero2one increment?    ///    /// Weak zero2one operations are always scheduled using the `Node`.    has_weak_zero2one: bool,     /// Is the currently scheduled wrapper/`Node` scheduled due to a strong zero2one increment?    ///    /// If `has_pushed_wrapper` is set, then the strong zero2one increment was scheduled using the    /// wrapper. Otherwise, `has_pushed_node` must be set and it was scheduled using the `Node`.    has_strong_zero2one: bool,} impl DeliveryState {    fn should_normal_push(&self) -> bool {        !self.has_pushed_node && !self.has_pushed_wrapper    }     fn did_normal_push(&mut self) {        assert!(self.should_normal_push());        self.has_pushed_node = true;    }     fn should_push_weak_zero2one(&self) -> bool {        !self.has_weak_zero2one && !self.has_strong_zero2one    }     fn can_push_weak_zero2one_normally(&self) -> bool {        !self.has_pushed_node    }     fn did_push_weak_zero2one(&mut self) {        assert!(self.should_push_weak_zero2one());        assert!(self.can_push_weak_zero2one_normally());        self.has_pushed_node = true;        self.has_weak_zero2one = true;    }     fn should_push_strong_zero2one(&self) -> bool {        !self.has_strong_zero2one    }     fn can_push_strong_zero2one_normally(&self) -> bool {        !self.has_pushed_node    }     fn did_push_strong_zero2one(&mut self) {        assert!(self.should_push_strong_zero2one());        assert!(self.can_push_strong_zero2one_normally());        self.has_pushed_node = true;        self.has_strong_zero2one = true;    }     fn did_push_strong_zero2one_wrapper(&mut self) {        assert!(self.should_push_strong_zero2one());        assert!(!self.can_push_strong_zero2one_normally());        self.has_pushed_wrapper = true;        self.has_strong_zero2one = true;    }} struct CountState {    /// The reference count.    count: usize,    /// Whether the process that owns this node thinks that we hold a refcount on it. (Note that    /// even if count is greater than one, we only increment it once in the owning process.)    has_count: bool,} impl CountState {    fn new() -> Self {        Self {            count: 0,            has_count: false,        }    }} struct NodeInner {    /// Strong refcounts held on this node by `NodeRef` objects.    strong: CountState,    /// Weak refcounts held on this node by `NodeRef` objects.    weak: CountState,    delivery_state: DeliveryState,    /// The binder driver guarantees that oneway transactions sent to the same node are serialized,    /// that is, userspace will not be given the next one until it has finished processing the    /// previous oneway transaction. This is done to avoid the case where two oneway transactions    /// arrive in opposite order from the order in which they were sent. (E.g., they could be    /// delivered to two different threads, which could appear as-if they were sent in opposite    /// order.)    ///    /// To fix that, we store pending oneway transactions in a separate list in the node, and don't    /// deliver the next oneway transaction until userspace signals that it has finished processing    /// the previous oneway transaction by calling the `BC_FREE_BUFFER` ioctl.    oneway_todo: List<DTRWrap<Transaction>>,    /// Keeps track of whether this node has a pending oneway transaction.    ///    /// When this is true, incoming oneway transactions are stored in `oneway_todo`, instead of    /// being delivered directly to the process.    has_oneway_transaction: bool,    /// List of processes to deliver a notification to when this node is destroyed (usually due to    /// the process dying).    death_list: List<DTRWrap<NodeDeath>, 1>,    /// List of processes to deliver freeze notifications to.    freeze_list: KVVec<Arc<Process>>,    /// The number of active BR_INCREFS or BR_ACQUIRE operations. (should be maximum two)    ///    /// If this is non-zero, then we postpone any BR_RELEASE or BR_DECREFS notifications until the    /// active operations have ended. This avoids the situation an increment and decrement get    /// reordered from userspace's perspective.    active_inc_refs: u8,    /// List of `NodeRefInfo` objects that reference this node.    refs: List<NodeRefInfo, { NodeRefInfo::LIST_NODE }>,} use kernel::bindings::rb_node_layout;use mem::offset_of;pub(crate) const NODE_LAYOUT: rb_node_layout = rb_node_layout {    arc_offset: Arc::<Node>::DATA_OFFSET + offset_of!(DTRWrap<Node>, wrapped),    debug_id: offset_of!(Node, debug_id),    ptr: offset_of!(Node, ptr),}; #[pin_data]pub(crate) struct Node {    pub(crate) debug_id: usize,    ptr: u64,    pub(crate) cookie: u64,    pub(crate) flags: u32,    pub(crate) owner: Arc<Process>,    inner: LockedBy<NodeInner, ProcessInner>,    #[pin]    links_track: AtomicTracker,} kernel::list::impl_list_arc_safe! {    impl ListArcSafe<0> for Node {        tracked_by links_track: AtomicTracker;    }} // Make `oneway_todo` work.kernel::list::impl_list_item! {    impl ListItem<0> for DTRWrap<Transaction> {        using ListLinks { self.links.inner };    }} impl Node {    pub(crate) fn new(        ptr: u64,        cookie: u64,        flags: u32,        owner: Arc<Process>,    ) -> impl PinInit<Self> {        pin_init!(Self {            inner: LockedBy::new(                &owner.inner,                NodeInner {                    strong: CountState::new(),                    weak: CountState::new(),                    delivery_state: DeliveryState {                        has_pushed_node: false,                        has_pushed_wrapper: false,                        has_weak_zero2one: false,                        has_strong_zero2one: false,                    },                    death_list: List::new(),                    oneway_todo: List::new(),                    freeze_list: KVVec::new(),                    has_oneway_transaction: false,                    active_inc_refs: 0,                    refs: List::new(),                },            ),            debug_id: super::next_debug_id(),            ptr,            cookie,            flags,            owner,            links_track <- AtomicTracker::new(),        })    }     pub(crate) fn has_oneway_transaction(&self, owner_inner: &mut ProcessInner) -> bool {        let inner = self.inner.access_mut(owner_inner);        inner.has_oneway_transaction    }     #[inline(never)]    pub(crate) fn full_debug_print(        &self,        m: &SeqFile,        owner_inner: &mut ProcessInner,    ) -> Result<()> {        let inner = self.inner.access_mut(owner_inner);        seq_print!(            m,            "  node {}: u{:016x} c{:016x} hs {} hw {} cs {} cw {}",            self.debug_id,            self.ptr,            self.cookie,            inner.strong.has_count,            inner.weak.has_count,            inner.strong.count,            inner.weak.count,        );        if !inner.refs.is_empty() {            seq_print!(m, " proc");            for node_ref in &inner.refs {                seq_print!(m, " {}", node_ref.process.task.pid());            }        }        seq_print!(m, "\n");        for t in &inner.oneway_todo {            t.debug_print_inner(m, "    pending async transaction ");        }        Ok(())    }     /// Insert the `NodeRef` into this `refs` list.    ///    /// # Safety    ///    /// It must be the case that `info.node_ref.node` is this node.    pub(crate) unsafe fn insert_node_info(        &self,        info: ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>,    ) {        self.inner            .access_mut(&mut self.owner.inner.lock())            .refs            .push_front(info);    }     /// Insert the `NodeRef` into this `refs` list.    ///    /// # Safety    ///    /// It must be the case that `info.node_ref.node` is this node.    pub(crate) unsafe fn remove_node_info(        &self,        info: &NodeRefInfo,    ) -> Option<ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>> {        // SAFETY: We always insert `NodeRefInfo` objects into the `refs` list of the node that it        // references in `info.node_ref.node`. That is this node, so `info` cannot possibly be in        // the `refs` list of another node.        unsafe {            self.inner                .access_mut(&mut self.owner.inner.lock())                .refs                .remove(info)        }    }     /// An id that is unique across all binder nodes on the system. Used as the key in the    /// `by_node` map.    pub(crate) fn global_id(&self) -> usize {        self as *const Node as usize    }     pub(crate) fn get_id(&self) -> (u64, u64) {        (self.ptr, self.cookie)    }     pub(crate) fn add_death(        &self,        death: ListArc<DTRWrap<NodeDeath>, 1>,        guard: &mut Guard<'_, ProcessInner, SpinLockBackend>,    ) {        self.inner.access_mut(guard).death_list.push_back(death);    }     pub(crate) fn inc_ref_done_locked(        self: &DArc<Node>,        _strong: bool,        owner_inner: &mut ProcessInner,    ) -> Option<DLArc<Node>> {        let inner = self.inner.access_mut(owner_inner);        if inner.active_inc_refs == 0 {            pr_err!("inc_ref_done called when no active inc_refs");            return None;        }         inner.active_inc_refs -= 1;        if inner.active_inc_refs == 0 {            // Having active inc_refs can inhibit dropping of ref-counts. Calculate whether we            // would send a refcount decrement, and if so, tell the caller to schedule us.            let strong = inner.strong.count > 0;            let has_strong = inner.strong.has_count;            let weak = strong || inner.weak.count > 0;            let has_weak = inner.weak.has_count;             let should_drop_weak = !weak && has_weak;            let should_drop_strong = !strong && has_strong;             // If we want to drop the ref-count again, tell the caller to schedule a work node for            // that.            let need_push = should_drop_weak || should_drop_strong;             if need_push && inner.delivery_state.should_normal_push() {                let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();                inner.delivery_state.did_normal_push();                Some(list_arc)            } else {                None            }        } else {            None        }    }     pub(crate) fn update_refcount_locked(        self: &DArc<Node>,        inc: bool,        strong: bool,        count: usize,        owner_inner: &mut ProcessInner,    ) -> Option<DLArc<Node>> {        let is_dead = owner_inner.is_dead;        let inner = self.inner.access_mut(owner_inner);         // Get a reference to the state we'll update.        let state = if strong {            &mut inner.strong        } else {            &mut inner.weak        };         // Update the count and determine whether we need to push work.        let need_push = if inc {            state.count += count;            // TODO: This method shouldn't be used for zero-to-one increments.            !is_dead && !state.has_count        } else {            if state.count < count {                pr_err!("Failure: refcount underflow!");                return None;            }            state.count -= count;            !is_dead && state.count == 0 && state.has_count        };         if need_push && inner.delivery_state.should_normal_push() {            let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();            inner.delivery_state.did_normal_push();            Some(list_arc)        } else {            None        }    }     pub(crate) fn incr_refcount_allow_zero2one(        self: &DArc<Self>,        strong: bool,        owner_inner: &mut ProcessInner,    ) -> Result<Option<DLArc<Node>>, CouldNotDeliverCriticalIncrement> {        let is_dead = owner_inner.is_dead;        let inner = self.inner.access_mut(owner_inner);         // Get a reference to the state we'll update.        let state = if strong {            &mut inner.strong        } else {            &mut inner.weak        };         // Update the count and determine whether we need to push work.        state.count += 1;        if is_dead || state.has_count {            return Ok(None);        }         // Userspace needs to be notified of this.        if !strong && inner.delivery_state.should_push_weak_zero2one() {            assert!(inner.delivery_state.can_push_weak_zero2one_normally());            let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();            inner.delivery_state.did_push_weak_zero2one();            Ok(Some(list_arc))        } else if strong && inner.delivery_state.should_push_strong_zero2one() {            if inner.delivery_state.can_push_strong_zero2one_normally() {                let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();                inner.delivery_state.did_push_strong_zero2one();                Ok(Some(list_arc))            } else {                state.count -= 1;                Err(CouldNotDeliverCriticalIncrement)            }        } else {            // Work is already pushed, and we don't need to push again.            Ok(None)        }    }     pub(crate) fn incr_refcount_allow_zero2one_with_wrapper(        self: &DArc<Self>,        strong: bool,        wrapper: CritIncrWrapper,        owner_inner: &mut ProcessInner,    ) -> Option<DLArc<dyn DeliverToRead>> {        match self.incr_refcount_allow_zero2one(strong, owner_inner) {            Ok(Some(node)) => Some(node as _),            Ok(None) => None,            Err(CouldNotDeliverCriticalIncrement) => {                assert!(strong);                let inner = self.inner.access_mut(owner_inner);                inner.strong.count += 1;                inner.delivery_state.did_push_strong_zero2one_wrapper();                Some(wrapper.init(self.clone()))            }        }    }     pub(crate) fn update_refcount(self: &DArc<Self>, inc: bool, count: usize, strong: bool) {        self.owner            .inner            .lock()            .update_node_refcount(self, inc, strong, count, None);    }     pub(crate) fn populate_counts(        &self,        out: &mut BinderNodeInfoForRef,        guard: &Guard<'_, ProcessInner, SpinLockBackend>,    ) {        let inner = self.inner.access(guard);        out.strong_count = inner.strong.count as _;        out.weak_count = inner.weak.count as _;    }     pub(crate) fn populate_debug_info(        &self,        out: &mut BinderNodeDebugInfo,        guard: &Guard<'_, ProcessInner, SpinLockBackend>,    ) {

Lines 1–500 of 1139. Load more