// SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2025 Google LLC. //! This module has utilities for managing a page range where unused pages may be reclaimed by a//! vma shrinker. // To avoid deadlocks, locks are taken in the order://// 1. mmap lock// 2. spinlock// 3. lru spinlock//// The shrinker will use trylock methods because it locks them in a different order. use crate::AssertSync; use core::{ marker::PhantomPinned, mem::{size_of, size_of_val, MaybeUninit}, ptr,}; use kernel::{ bindings, error::Result, ffi::{c_ulong, c_void}, mm::{virt, Mm, MmWithUser}, new_mutex, new_spinlock, page::{Page, PAGE_SHIFT, PAGE_SIZE}, prelude::*, str::CStr, sync::{aref::ARef, Mutex, SpinLock}, task::Pid, transmute::FromBytes, types::Opaque, uaccess::UserSliceReader,}; /// Represents a shrinker that can be registered with the kernel.////// Each shrinker can be used by many `ShrinkablePageRange` objects.#[repr(C)]pub(crate) struct Shrinker { inner: Opaque<*mut bindings::shrinker>, list_lru: Opaque<bindings::list_lru>,} // SAFETY: The shrinker and list_lru are thread safe.unsafe impl Send for Shrinker {}// SAFETY: The shrinker and list_lru are thread safe.unsafe impl Sync for Shrinker {} impl Shrinker { /// Create a new shrinker. /// /// # Safety /// /// Before using this shrinker with a `ShrinkablePageRange`, the `register` method must have /// been called exactly once, and it must not have returned an error. pub(crate) const unsafe fn new() -> Self { Self { inner: Opaque::uninit(), list_lru: Opaque::uninit(), } } /// Register this shrinker with the kernel. pub(crate) fn register(&'static self, name: &CStr) -> Result<()> { // SAFETY: These fields are not yet used, so it's okay to zero them. unsafe { self.inner.get().write(ptr::null_mut()); self.list_lru.get().write_bytes(0, 1); } // SAFETY: The field is not yet used, so we can initialize it. let ret = unsafe { bindings::__list_lru_init(self.list_lru.get(), false, ptr::null_mut()) }; if ret != 0 { return Err(Error::from_errno(ret)); } // SAFETY: The `name` points at a valid c string. let shrinker = unsafe { bindings::shrinker_alloc(0, name.as_char_ptr()) }; if shrinker.is_null() { // SAFETY: We initialized it, so its okay to destroy it. unsafe { bindings::list_lru_destroy(self.list_lru.get()) }; return Err(Error::from_errno(ret)); } // SAFETY: We're about to register the shrinker, and these are the fields we need to // initialize. (All other fields are already zeroed.) unsafe { (&raw mut (*shrinker).count_objects).write(Some(rust_shrink_count)); (&raw mut (*shrinker).scan_objects).write(Some(rust_shrink_scan)); (&raw mut (*shrinker).private_data).write(self.list_lru.get().cast()); } // SAFETY: The new shrinker has been fully initialized, so we can register it. unsafe { bindings::shrinker_register(shrinker) }; // SAFETY: This initializes the pointer to the shrinker so that we can use it. unsafe { self.inner.get().write(shrinker) }; Ok(()) }} /// A container that manages a page range in a vma.////// The pages can be thought of as an array of booleans of whether the pages are usable. The/// methods `use_range` and `stop_using_range` set all booleans in a range to true or false/// respectively. Initially, no pages are allocated. When a page is not used, it is not freed/// immediately. Instead, it is made available to the memory shrinker to free it if the device is/// under memory pressure.////// It's okay for `use_range` and `stop_using_range` to race with each other, although there's no/// way to know whether an index ends up with true or false if a call to `use_range` races with/// another call to `stop_using_range` on a given index.////// It's also okay for the two methods to race with themselves, e.g. if two threads call/// `use_range` on the same index, then that's fine and neither call will return until the page is/// allocated and mapped.////// The methods that read or write to a range require that the page is marked as in use. So it is/// _not_ okay to call `stop_using_range` on a page that is in use by the methods that read or/// write to the page.#[pin_data(PinnedDrop)]pub(crate) struct ShrinkablePageRange { /// Shrinker object registered with the kernel. shrinker: &'static Shrinker, /// Pid using this page range. Only used as debugging information. pid: Pid, /// The mm for the relevant process. mm: ARef<Mm>, /// Used to synchronize calls to `vm_insert_page` and `zap_vma_range`. #[pin] mm_lock: Mutex<()>, /// Spinlock protecting changes to pages. #[pin] lock: SpinLock<Inner>, /// Must not move, since page info has pointers back. #[pin] _pin: PhantomPinned,} // We do not define any ops. For now, used only to check identity of vmas.static BINDER_VM_OPS: AssertSync<bindings::vm_operations_struct> = AssertSync(pin_init::zeroed()); // To ensure that we do not accidentally install pages into or zap pages from the wrong vma, we// check its vm_ops and private data before using it.fn check_vma(vma: &virt::VmaRef, owner: *const ShrinkablePageRange) -> Option<&virt::VmaMixedMap> { // SAFETY: Just reading the vm_ops pointer of any active vma is safe. let vm_ops = unsafe { (*vma.as_ptr()).vm_ops }; if !ptr::eq(vm_ops, &BINDER_VM_OPS.0) { return None; } // SAFETY: Reading the vm_private_data pointer of a binder-owned vma is safe. let vm_private_data = unsafe { (*vma.as_ptr()).vm_private_data }; // The ShrinkablePageRange is only dropped when the Process is dropped, which only happens once // the file's ->release handler is invoked, which means the ShrinkablePageRange outlives any // VMA associated with it, so there can't be any false positives due to pointer reuse here. if !ptr::eq(vm_private_data, owner.cast()) { return None; } vma.as_mixedmap_vma()} struct Inner { /// Array of pages. /// /// Since this is also accessed by the shrinker, we can't use a `Box`, which asserts exclusive /// ownership. To deal with that, we manage it using raw pointers. pages: *mut PageInfo, /// Length of the `pages` array. size: usize, /// The address of the vma to insert the pages into. vma_addr: usize,} // SAFETY: proper locking is in place for `Inner`unsafe impl Send for Inner {} type StableMmGuard = kernel::sync::lock::Guard<'static, (), kernel::sync::lock::mutex::MutexBackend>; /// An array element that describes the current state of a page.////// There are three states:////// * Free. The page is None. The `lru` element is not queued./// * Available. The page is Some. The `lru` element is queued to the shrinker's lru./// * Used. The page is Some. The `lru` element is not queued.////// When an element is available, the shrinker is able to free the page.#[repr(C)]struct PageInfo { lru: bindings::list_head, page: Option<Page>, range: *const ShrinkablePageRange,} impl PageInfo { /// # Safety /// /// The caller ensures that writing to `me.page` is ok, and that the page is not currently set. unsafe fn set_page(me: *mut PageInfo, page: Page) { // SAFETY: This pointer offset is in bounds. let ptr = unsafe { &raw mut (*me).page }; // SAFETY: The pointer is valid for writing, so also valid for reading. if unsafe { (*ptr).is_some() } { pr_err!("set_page called when there is already a page"); // SAFETY: We will initialize the page again below. unsafe { ptr::drop_in_place(ptr) }; } // SAFETY: The pointer is valid for writing. unsafe { ptr::write(ptr, Some(page)) }; } /// # Safety /// /// The caller ensures that reading from `me.page` is ok for the duration of 'a. unsafe fn get_page<'a>(me: *const PageInfo) -> Option<&'a Page> { // SAFETY: This pointer offset is in bounds. let ptr = unsafe { &raw const (*me).page }; // SAFETY: The pointer is valid for reading. unsafe { (*ptr).as_ref() } } /// # Safety /// /// The caller ensures that writing to `me.page` is ok for the duration of 'a. unsafe fn take_page(me: *mut PageInfo) -> Option<Page> { // SAFETY: This pointer offset is in bounds. let ptr = unsafe { &raw mut (*me).page }; // SAFETY: The pointer is valid for reading. unsafe { (*ptr).take() } } /// Add this page to the lru list, if not already in the list. /// /// # Safety /// /// The pointer must be valid, and it must be the right shrinker and nid. unsafe fn list_lru_add(me: *mut PageInfo, nid: i32, shrinker: &'static Shrinker) { // SAFETY: This pointer offset is in bounds. let lru_ptr = unsafe { &raw mut (*me).lru }; // SAFETY: The lru pointer is valid, and we're not using it with any other lru list. unsafe { bindings::list_lru_add(shrinker.list_lru.get(), lru_ptr, nid, ptr::null_mut()) }; } /// Remove this page from the lru list, if it is in the list. /// /// # Safety /// /// The pointer must be valid, and it must be the right shrinker and nid. unsafe fn list_lru_del(me: *mut PageInfo, nid: i32, shrinker: &'static Shrinker) { // SAFETY: This pointer offset is in bounds. let lru_ptr = unsafe { &raw mut (*me).lru }; // SAFETY: The lru pointer is valid, and we're not using it with any other lru list. unsafe { bindings::list_lru_del(shrinker.list_lru.get(), lru_ptr, nid, ptr::null_mut()) }; }} impl ShrinkablePageRange { /// Create a new `ShrinkablePageRange` using the given shrinker. pub(crate) fn new(shrinker: &'static Shrinker) -> impl PinInit<Self, Error> { try_pin_init!(Self { shrinker, pid: kernel::current!().pid(), mm: ARef::from(&**kernel::current!().mm().ok_or(ESRCH)?), mm_lock <- new_mutex!((), "ShrinkablePageRange::mm"), lock <- new_spinlock!(Inner { pages: ptr::null_mut(), size: 0, vma_addr: 0, }, "ShrinkablePageRange"), _pin: PhantomPinned, }) } pub(crate) fn stable_trylock_mm(&self) -> Option<StableMmGuard> { // SAFETY: This extends the duration of the reference. Since this call happens before // `mm_lock` is taken in the destructor of `ShrinkablePageRange`, the destructor will block // until the returned guard is dropped. This ensures that the guard is valid until dropped. let mm_lock = unsafe { &*ptr::from_ref(&self.mm_lock) }; mm_lock.try_lock() } /// Register a vma with this page range. Returns the size of the region. pub(crate) fn register_with_vma(&self, vma: &virt::VmaNew) -> Result<usize> { let num_bytes = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize); let num_pages = num_bytes >> PAGE_SHIFT; if !ptr::eq::<Mm>(&*self.mm, &**vma.mm()) { pr_debug!("Failed to register with vma: invalid vma->vm_mm"); return Err(EINVAL); } if num_pages == 0 { pr_debug!("Failed to register with vma: size zero"); return Err(EINVAL); } let mut pages = KVVec::<PageInfo>::with_capacity(num_pages, GFP_KERNEL)?; // SAFETY: This just initializes the pages array. unsafe { let self_ptr = ptr::from_ref(self); for i in 0..num_pages { let info = pages.as_mut_ptr().add(i); (&raw mut (*info).range).write(self_ptr); (&raw mut (*info).page).write(None); let lru = &raw mut (*info).lru; (&raw mut (*lru).next).write(lru); (&raw mut (*lru).prev).write(lru); } } let mut inner = self.lock.lock(); if inner.size > 0 { pr_debug!("Failed to register with vma: already registered"); drop(inner); return Err(EBUSY); } inner.pages = pages.into_raw_parts().0; inner.size = num_pages; inner.vma_addr = vma.start(); // This pointer is only used for comparison - it's not dereferenced. // // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on // `vm_private_data`. unsafe { (*vma.as_ptr()).vm_private_data = ptr::from_ref(self).cast_mut().cast::<c_void>() }; // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on // `vm_ops`. unsafe { (*vma.as_ptr()).vm_ops = &BINDER_VM_OPS.0 }; Ok(num_pages) } /// Make sure that the given pages are allocated and mapped. /// /// Must not be called from an atomic context. pub(crate) fn use_range(&self, start: usize, end: usize) -> Result<()> { if start >= end { return Ok(()); } let mut inner = self.lock.lock(); assert!(end <= inner.size); for i in start..end { // SAFETY: This pointer offset is in bounds. let page_info = unsafe { inner.pages.add(i) }; // SAFETY: The pointer is valid, and we hold the lock so reading from the page is okay. if let Some(page) = unsafe { PageInfo::get_page(page_info) } { // Since we're going to use the page, we should remove it from the lru list so that // the shrinker will not free it. // // SAFETY: The pointer is valid, and this is the right shrinker. // // The shrinker can't free the page between the check and this call to // `list_lru_del` because we hold the lock. unsafe { PageInfo::list_lru_del(page_info, page.nid(), self.shrinker) }; } else { // We have to allocate a new page. Use the slow path. drop(inner); // SAFETY: `i < end <= inner.size` so `i` is in bounds. match unsafe { self.use_page_slow(i) } { Ok(()) => {} Err(err) => { pr_warn!("Error in use_page_slow: {:?}", err); return Err(err); } } inner = self.lock.lock(); } } Ok(()) } /// Mark the given page as in use, slow path. /// /// Must not be called from an atomic context. /// /// # Safety /// /// Assumes that `i` is in bounds. #[cold] unsafe fn use_page_slow(&self, i: usize) -> Result<()> { let new_page = Page::alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO)?; let mm_mutex = self.mm_lock.lock(); let inner = self.lock.lock(); // SAFETY: This pointer offset is in bounds. let page_info = unsafe { inner.pages.add(i) }; // SAFETY: The pointer is valid, and we hold the lock so reading from the page is okay. if let Some(page) = unsafe { PageInfo::get_page(page_info) } { // The page was already there, or someone else added the page while we didn't hold the // spinlock. // // SAFETY: The pointer is valid, and this is the right shrinker. // // The shrinker can't free the page between the check and this call to // `list_lru_del` because we hold the lock. unsafe { PageInfo::list_lru_del(page_info, page.nid(), self.shrinker) }; return Ok(()); } let vma_addr = inner.vma_addr; // Release the spinlock while we insert the page into the vma. drop(inner); // No overflow since we stay in bounds of the vma. let user_page_addr = vma_addr + (i << PAGE_SHIFT); // We use `mmput_async` when dropping the `mm` because `use_page_slow` is usually used from // a remote process. If the call to `mmput` races with the process shutting down, then the // caller of `use_page_slow` becomes responsible for cleaning up the `mm`, which doesn't // happen until it returns to userspace. However, the caller might instead go to sleep and // wait for the owner of the `mm` to wake it up, which doesn't happen because it's in the // middle of a shutdown process that won't complete until the `mm` is dropped. This can // amount to a deadlock. // // Using `mmput_async` avoids this, because then the `mm` cleanup is instead queued to a // workqueue. let mm = MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?); { let vma_read; let mmap_read; let vma = if let Some(ret) = mm.lock_vma_under_rcu(vma_addr) { vma_read = ret; check_vma(&vma_read, self) } else { mmap_read = mm.mmap_read_lock(); mmap_read .vma_lookup(vma_addr) .and_then(|vma| check_vma(vma, self)) }; match vma { Some(vma) => vma.vm_insert_page(user_page_addr, &new_page)?, None => return Err(ESRCH), } } let inner = self.lock.lock(); // SAFETY: The `page_info` pointer is valid and currently does not have a page. The page // can be written to since we hold the lock. // // We released and reacquired the spinlock since we checked that the page is null, but we // always hold the mm_lock mutex when setting the page to a non-null value, so it's not // possible for someone else to have changed it since our check. unsafe { PageInfo::set_page(page_info, new_page) }; drop(inner); drop(mm_mutex); Ok(()) } /// If the given page is in use, then mark it as available so that the shrinker can free it. /// /// May be called from an atomic context. pub(crate) fn stop_using_range(&self, start: usize, end: usize) { if start >= end { return; } let inner = self.lock.lock(); assert!(end <= inner.size); for i in (start..end).rev() { // SAFETY: The pointer is in bounds. let page_info = unsafe { inner.pages.add(i) }; // SAFETY: Okay for reading since we have the lock. if let Some(page) = unsafe { PageInfo::get_page(page_info) } { // SAFETY: The pointer is valid, and it's the right shrinker. unsafe { PageInfo::list_lru_add(page_info, page.nid(), self.shrinker) }; } } } /// Helper for reading or writing to a range of bytes that may overlap with several pages. /// /// # SafetyLines 1–500 of 775. Load more
// SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2025 Google LLC. //! This module has utilities for managing a page range where unused pages may be reclaimed by a //! vma shrinker. // To avoid deadlocks, locks are taken in the order: // // 1. mmap lock // 2. spinlock // 3. lru spinlock // // The shrinker will use trylock methods because it locks them in a different order. use crate::AssertSync; use core::{ marker::PhantomPinned, mem::{size_of, size_of_val, MaybeUninit}, ptr, }; use kernel::{ bindings, error::Result, ffi::{c_ulong, c_void}, mm::{virt, Mm, MmWithUser}, new_mutex, new_spinlock, page::{Page, PAGE_SHIFT, PAGE_SIZE}, prelude::*, str::CStr, sync::{aref::ARef, Mutex, SpinLock}, task::Pid, transmute::FromBytes, types::Opaque, uaccess::UserSliceReader, }; /// Represents a shrinker that can be registered with the kernel. /// /// Each shrinker can be used by many `ShrinkablePageRange` objects. #[repr(C)] pub(crate) struct Shrinker { inner: Opaque<*mut bindings::shrinker>, list_lru: Opaque<bindings::list_lru>, } // SAFETY: The shrinker and list_lru are thread safe. unsafe impl Send for Shrinker {} // SAFETY: The shrinker and list_lru are thread safe. unsafe impl Sync for Shrinker {} impl Shrinker { /// Create a new shrinker. /// /// # Safety /// /// Before using this shrinker with a `ShrinkablePageRange`, the `register` method must have /// been called exactly once, and it must not have returned an error. pub(crate) const unsafe fn new() -> Self { Self { inner: Opaque::uninit(), list_lru: Opaque::uninit(), } } /// Register this shrinker with the kernel. pub(crate) fn register(&'static self, name: &CStr) -> Result<()> { // SAFETY: These fields are not yet used, so it's okay to zero them. unsafe { self.inner.get().write(ptr::null_mut()); self.list_lru.get().write_bytes(0, 1); } // SAFETY: The field is not yet used, so we can initialize it. let ret = unsafe { bindings::__list_lru_init(self.list_lru.get(), false, ptr::null_mut()) }; if ret != 0 { return Err(Error::from_errno(ret)); } // SAFETY: The `name` points at a valid c string. let shrinker = unsafe { bindings::shrinker_alloc(0, name.as_char_ptr()) }; if shrinker.is_null() { // SAFETY: We initialized it, so its okay to destroy it. unsafe { bindings::list_lru_destroy(self.list_lru.get()) }; return Err(Error::from_errno(ret)); } // SAFETY: We're about to register the shrinker, and these are the fields we need to // initialize. (All other fields are already zeroed.) unsafe { (&raw mut (*shrinker).count_objects).write(Some(rust_shrink_count)); (&raw mut (*shrinker).scan_objects).write(Some(rust_shrink_scan)); (&raw mut (*shrinker).private_data).write(self.list_lru.get().cast()); } // SAFETY: The new shrinker has been fully initialized, so we can register it. unsafe { bindings::shrinker_register(shrinker) }; // SAFETY: This initializes the pointer to the shrinker so that we can use it. unsafe { self.inner.get().write(shrinker) }; Ok(()) } } /// A container that manages a page range in a vma. /// /// The pages can be thought of as an array of booleans of whether the pages are usable. The /// methods `use_range` and `stop_using_range` set all booleans in a range to true or false /// respectively. Initially, no pages are allocated. When a page is not used, it is not freed /// immediately. Instead, it is made available to the memory shrinker to free it if the device is /// under memory pressure. /// /// It's okay for `use_range` and `stop_using_range` to race with each other, although there's no /// way to know whether an index ends up with true or false if a call to `use_range` races with /// another call to `stop_using_range` on a given index. /// /// It's also okay for the two methods to race with themselves, e.g. if two threads call /// `use_range` on the same index, then that's fine and neither call will return until the page is /// allocated and mapped. /// /// The methods that read or write to a range require that the page is marked as in use. So it is /// _not_ okay to call `stop_using_range` on a page that is in use by the methods that read or /// write to the page. #[pin_data(PinnedDrop)] pub(crate) struct ShrinkablePageRange { /// Shrinker object registered with the kernel. shrinker: &'static Shrinker, /// Pid using this page range. Only used as debugging information. pid: Pid, /// The mm for the relevant process. mm: ARef<Mm>, /// Used to synchronize calls to `vm_insert_page` and `zap_vma_range`. #[pin] mm_lock: Mutex<()>, /// Spinlock protecting changes to pages. #[pin] lock: SpinLock<Inner>, /// Must not move, since page info has pointers back. #[pin] _pin: PhantomPinned, } // We do not define any ops. For now, used only to check identity of vmas. static BINDER_VM_OPS: AssertSync<bindings::vm_operations_struct> = AssertSync(pin_init::zeroed()); // To ensure that we do not accidentally install pages into or zap pages from the wrong vma, we // check its vm_ops and private data before using it. fn check_vma(vma: &virt::VmaRef, owner: *const ShrinkablePageRange) -> Option<&virt::VmaMixedMap> { // SAFETY: Just reading the vm_ops pointer of any active vma is safe. let vm_ops = unsafe { (*vma.as_ptr()).vm_ops }; if !ptr::eq(vm_ops, &BINDER_VM_OPS.0) { return None; } // SAFETY: Reading the vm_private_data pointer of a binder-owned vma is safe. let vm_private_data = unsafe { (*vma.as_ptr()).vm_private_data }; // The ShrinkablePageRange is only dropped when the Process is dropped, which only happens once // the file's ->release handler is invoked, which means the ShrinkablePageRange outlives any // VMA associated with it, so there can't be any false positives due to pointer reuse here. if !ptr::eq(vm_private_data, owner.cast()) { return None; } vma.as_mixedmap_vma() } struct Inner { /// Array of pages. /// /// Since this is also accessed by the shrinker, we can't use a `Box`, which asserts exclusive /// ownership. To deal with that, we manage it using raw pointers. pages: *mut PageInfo, /// Length of the `pages` array. size: usize, /// The address of the vma to insert the pages into. vma_addr: usize, } // SAFETY: proper locking is in place for `Inner` unsafe impl Send for Inner {} type StableMmGuard = kernel::sync::lock::Guard<'static, (), kernel::sync::lock::mutex::MutexBackend>; /// An array element that describes the current state of a page. /// /// There are three states: /// /// * Free. The page is None. The `lru` element is not queued. /// * Available. The page is Some. The `lru` element is queued to the shrinker's lru. /// * Used. The page is Some. The `lru` element is not queued. /// /// When an element is available, the shrinker is able to free the page. #[repr(C)] struct PageInfo { lru: bindings::list_head, page: Option<Page>, range: *const ShrinkablePageRange, } impl PageInfo { /// # Safety /// /// The caller ensures that writing to `me.page` is ok, and that the page is not currently set. unsafe fn set_page(me: *mut PageInfo, page: Page) { // SAFETY: This pointer offset is in bounds. let ptr = unsafe { &raw mut (*me).page }; // SAFETY: The pointer is valid for writing, so also valid for reading. if unsafe { (*ptr).is_some() } { pr_err!("set_page called when there is already a page"); // SAFETY: We will initialize the page again below. unsafe { ptr::drop_in_place(ptr) }; } // SAFETY: The pointer is valid for writing. unsafe { ptr::write(ptr, Some(page)) }; } /// # Safety /// /// The caller ensures that reading from `me.page` is ok for the duration of 'a. unsafe fn get_page<'a>(me: *const PageInfo) -> Option<&'a Page> { // SAFETY: This pointer offset is in bounds. let ptr = unsafe { &raw const (*me).page }; // SAFETY: The pointer is valid for reading. unsafe { (*ptr).as_ref() } } /// # Safety /// /// The caller ensures that writing to `me.page` is ok for the duration of 'a. unsafe fn take_page(me: *mut PageInfo) -> Option<Page> { // SAFETY: This pointer offset is in bounds. let ptr = unsafe { &raw mut (*me).page }; // SAFETY: The pointer is valid for reading. unsafe { (*ptr).take() } } /// Add this page to the lru list, if not already in the list. /// /// # Safety /// /// The pointer must be valid, and it must be the right shrinker and nid. unsafe fn list_lru_add(me: *mut PageInfo, nid: i32, shrinker: &'static Shrinker) { // SAFETY: This pointer offset is in bounds. let lru_ptr = unsafe { &raw mut (*me).lru }; // SAFETY: The lru pointer is valid, and we're not using it with any other lru list. unsafe { bindings::list_lru_add(shrinker.list_lru.get(), lru_ptr, nid, ptr::null_mut()) }; } /// Remove this page from the lru list, if it is in the list. /// /// # Safety /// /// The pointer must be valid, and it must be the right shrinker and nid. unsafe fn list_lru_del(me: *mut PageInfo, nid: i32, shrinker: &'static Shrinker) { // SAFETY: This pointer offset is in bounds. let lru_ptr = unsafe { &raw mut (*me).lru }; // SAFETY: The lru pointer is valid, and we're not using it with any other lru list. unsafe { bindings::list_lru_del(shrinker.list_lru.get(), lru_ptr, nid, ptr::null_mut()) }; } } impl ShrinkablePageRange { /// Create a new `ShrinkablePageRange` using the given shrinker. pub(crate) fn new(shrinker: &'static Shrinker) -> impl PinInit<Self, Error> { try_pin_init!(Self { shrinker, pid: kernel::current!().pid(), mm: ARef::from(&**kernel::current!().mm().ok_or(ESRCH)?), mm_lock <- new_mutex!((), "ShrinkablePageRange::mm"), lock <- new_spinlock!(Inner { pages: ptr::null_mut(), size: 0, vma_addr: 0, }, "ShrinkablePageRange"), _pin: PhantomPinned, }) } pub(crate) fn stable_trylock_mm(&self) -> Option<StableMmGuard> { // SAFETY: This extends the duration of the reference. Since this call happens before // `mm_lock` is taken in the destructor of `ShrinkablePageRange`, the destructor will block // until the returned guard is dropped. This ensures that the guard is valid until dropped. let mm_lock = unsafe { &*ptr::from_ref(&self.mm_lock) }; mm_lock.try_lock() } /// Register a vma with this page range. Returns the size of the region. pub(crate) fn register_with_vma(&self, vma: &virt::VmaNew) -> Result<usize> { let num_bytes = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize); let num_pages = num_bytes >> PAGE_SHIFT; if !ptr::eq::<Mm>(&*self.mm, &**vma.mm()) { pr_debug!("Failed to register with vma: invalid vma->vm_mm"); return Err(EINVAL); } if num_pages == 0 { pr_debug!("Failed to register with vma: size zero"); return Err(EINVAL); } let mut pages = KVVec::<PageInfo>::with_capacity(num_pages, GFP_KERNEL)?; // SAFETY: This just initializes the pages array. unsafe { let self_ptr = ptr::from_ref(self); for i in 0..num_pages { let info = pages.as_mut_ptr().add(i); (&raw mut (*info).range).write(self_ptr); (&raw mut (*info).page).write(None); let lru = &raw mut (*info).lru; (&raw mut (*lru).next).write(lru); (&raw mut (*lru).prev).write(lru); } } let mut inner = self.lock.lock(); if inner.size > 0 { pr_debug!("Failed to register with vma: already registered"); drop(inner); return Err(EBUSY); } inner.pages = pages.into_raw_parts().0; inner.size = num_pages; inner.vma_addr = vma.start(); // This pointer is only used for comparison - it's not dereferenced. // // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on // `vm_private_data`. unsafe { (*vma.as_ptr()).vm_private_data = ptr::from_ref(self).cast_mut().cast::<c_void>() }; // SAFETY: We own the vma, and we don't use any methods on VmaNew that rely on // `vm_ops`. unsafe { (*vma.as_ptr()).vm_ops = &BINDER_VM_OPS.0 }; Ok(num_pages) } /// Make sure that the given pages are allocated and mapped. /// /// Must not be called from an atomic context. pub(crate) fn use_range(&self, start: usize, end: usize) -> Result<()> { if start >= end { return Ok(()); } let mut inner = self.lock.lock(); assert!(end <= inner.size); for i in start..end { // SAFETY: This pointer offset is in bounds. let page_info = unsafe { inner.pages.add(i) }; // SAFETY: The pointer is valid, and we hold the lock so reading from the page is okay. if let Some(page) = unsafe { PageInfo::get_page(page_info) } { // Since we're going to use the page, we should remove it from the lru list so that // the shrinker will not free it. // // SAFETY: The pointer is valid, and this is the right shrinker. // // The shrinker can't free the page between the check and this call to // `list_lru_del` because we hold the lock. unsafe { PageInfo::list_lru_del(page_info, page.nid(), self.shrinker) }; } else { // We have to allocate a new page. Use the slow path. drop(inner); // SAFETY: `i < end <= inner.size` so `i` is in bounds. match unsafe { self.use_page_slow(i) } { Ok(()) => {} Err(err) => { pr_warn!("Error in use_page_slow: {:?}", err); return Err(err); } } inner = self.lock.lock(); } } Ok(()) } /// Mark the given page as in use, slow path. /// /// Must not be called from an atomic context. /// /// # Safety /// /// Assumes that `i` is in bounds. #[cold] unsafe fn use_page_slow(&self, i: usize) -> Result<()> { let new_page = Page::alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO)?; let mm_mutex = self.mm_lock.lock(); let inner = self.lock.lock(); // SAFETY: This pointer offset is in bounds. let page_info = unsafe { inner.pages.add(i) }; // SAFETY: The pointer is valid, and we hold the lock so reading from the page is okay. if let Some(page) = unsafe { PageInfo::get_page(page_info) } { // The page was already there, or someone else added the page while we didn't hold the // spinlock. // // SAFETY: The pointer is valid, and this is the right shrinker. // // The shrinker can't free the page between the check and this call to // `list_lru_del` because we hold the lock. unsafe { PageInfo::list_lru_del(page_info, page.nid(), self.shrinker) }; return Ok(()); } let vma_addr = inner.vma_addr; // Release the spinlock while we insert the page into the vma. drop(inner); // No overflow since we stay in bounds of the vma. let user_page_addr = vma_addr + (i << PAGE_SHIFT); // We use `mmput_async` when dropping the `mm` because `use_page_slow` is usually used from // a remote process. If the call to `mmput` races with the process shutting down, then the // caller of `use_page_slow` becomes responsible for cleaning up the `mm`, which doesn't // happen until it returns to userspace. However, the caller might instead go to sleep and // wait for the owner of the `mm` to wake it up, which doesn't happen because it's in the // middle of a shutdown process that won't complete until the `mm` is dropped. This can // amount to a deadlock. // // Using `mmput_async` avoids this, because then the `mm` cleanup is instead queued to a // workqueue. let mm = MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?); { let vma_read; let mmap_read; let vma = if let Some(ret) = mm.lock_vma_under_rcu(vma_addr) { vma_read = ret; check_vma(&vma_read, self) } else { mmap_read = mm.mmap_read_lock(); mmap_read .vma_lookup(vma_addr) .and_then(|vma| check_vma(vma, self)) }; match vma { Some(vma) => vma.vm_insert_page(user_page_addr, &new_page)?, None => return Err(ESRCH), } } let inner = self.lock.lock(); // SAFETY: The `page_info` pointer is valid and currently does not have a page. The page // can be written to since we hold the lock. // // We released and reacquired the spinlock since we checked that the page is null, but we // always hold the mm_lock mutex when setting the page to a non-null value, so it's not // possible for someone else to have changed it since our check. unsafe { PageInfo::set_page(page_info, new_page) }; drop(inner); drop(mm_mutex); Ok(()) } /// If the given page is in use, then mark it as available so that the shrinker can free it. /// /// May be called from an atomic context. pub(crate) fn stop_using_range(&self, start: usize, end: usize) { if start >= end { return; } let inner = self.lock.lock(); assert!(end <= inner.size); for i in (start..end).rev() { // SAFETY: The pointer is in bounds. let page_info = unsafe { inner.pages.add(i) }; // SAFETY: Okay for reading since we have the lock. if let Some(page) = unsafe { PageInfo::get_page(page_info) } { // SAFETY: The pointer is valid, and it's the right shrinker. unsafe { PageInfo::list_lru_add(page_info, page.nid(), self.shrinker) }; } } } /// Helper for reading or writing to a range of bytes that may overlap with several pages. /// /// # Safety