// SPDX-License-Identifier: GPL-2.0-only/* binder_alloc.c * * Android IPC Subsystem * * Copyright (C) 2007-2017 Google, Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/list.h>#include <linux/sched/mm.h>#include <linux/module.h>#include <linux/rtmutex.h>#include <linux/rbtree.h>#include <linux/seq_file.h>#include <linux/vmalloc.h>#include <linux/slab.h>#include <linux/sched.h>#include <linux/list_lru.h>#include <linux/ratelimit.h>#include <asm/cacheflush.h>#include <linux/uaccess.h>#include <linux/highmem.h>#include <linux/sizes.h>#include <kunit/visibility.h>#include "binder_alloc.h"#include "binder_trace.h" static struct list_lru binder_freelist; static DEFINE_MUTEX(binder_alloc_mmap_lock); enum {	BINDER_DEBUG_USER_ERROR             = 1U << 0,	BINDER_DEBUG_OPEN_CLOSE             = 1U << 1,	BINDER_DEBUG_BUFFER_ALLOC           = 1U << 2,	BINDER_DEBUG_BUFFER_ALLOC_ASYNC     = 1U << 3,};static uint32_t binder_alloc_debug_mask = BINDER_DEBUG_USER_ERROR; module_param_named(debug_mask, binder_alloc_debug_mask,		   uint, 0644); #define binder_alloc_debug(mask, x...) \	do { \		if (binder_alloc_debug_mask & mask) \			pr_info_ratelimited(x); \	} while (0) static struct binder_buffer *binder_buffer_next(struct binder_buffer *buffer){	return list_entry(buffer->entry.next, struct binder_buffer, entry);} static struct binder_buffer *binder_buffer_prev(struct binder_buffer *buffer){	return list_entry(buffer->entry.prev, struct binder_buffer, entry);} VISIBLE_IF_KUNIT size_t binder_alloc_buffer_size(struct binder_alloc *alloc,						 struct binder_buffer *buffer){	if (list_is_last(&buffer->entry, &alloc->buffers))		return alloc->vm_start + alloc->buffer_size - buffer->user_data;	return binder_buffer_next(buffer)->user_data - buffer->user_data;}EXPORT_SYMBOL_IF_KUNIT(binder_alloc_buffer_size); static void binder_insert_free_buffer(struct binder_alloc *alloc,				      struct binder_buffer *new_buffer){	struct rb_node **p = &alloc->free_buffers.rb_node;	struct rb_node *parent = NULL;	struct binder_buffer *buffer;	size_t buffer_size;	size_t new_buffer_size; 	BUG_ON(!new_buffer->free); 	new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer); 	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,		     "%d: add free buffer, size %zd, at %p\n",		      alloc->pid, new_buffer_size, new_buffer); 	while (*p) {		parent = *p;		buffer = rb_entry(parent, struct binder_buffer, rb_node);		BUG_ON(!buffer->free); 		buffer_size = binder_alloc_buffer_size(alloc, buffer); 		if (new_buffer_size < buffer_size)			p = &parent->rb_left;		else			p = &parent->rb_right;	}	rb_link_node(&new_buffer->rb_node, parent, p);	rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers);} static void binder_insert_allocated_buffer_locked(		struct binder_alloc *alloc, struct binder_buffer *new_buffer){	struct rb_node **p = &alloc->allocated_buffers.rb_node;	struct rb_node *parent = NULL;	struct binder_buffer *buffer; 	BUG_ON(new_buffer->free); 	while (*p) {		parent = *p;		buffer = rb_entry(parent, struct binder_buffer, rb_node);		BUG_ON(buffer->free); 		if (new_buffer->user_data < buffer->user_data)			p = &parent->rb_left;		else if (new_buffer->user_data > buffer->user_data)			p = &parent->rb_right;		else			BUG();	}	rb_link_node(&new_buffer->rb_node, parent, p);	rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers);} static struct binder_buffer *binder_alloc_prepare_to_free_locked(		struct binder_alloc *alloc,		unsigned long user_ptr){	struct rb_node *n = alloc->allocated_buffers.rb_node;	struct binder_buffer *buffer; 	while (n) {		buffer = rb_entry(n, struct binder_buffer, rb_node);		BUG_ON(buffer->free); 		if (user_ptr < buffer->user_data) {			n = n->rb_left;		} else if (user_ptr > buffer->user_data) {			n = n->rb_right;		} else {			/*			 * Guard against user threads attempting to			 * free the buffer when in use by kernel or			 * after it's already been freed.			 */			if (!buffer->allow_user_free)				return ERR_PTR(-EPERM);			buffer->allow_user_free = 0;			return buffer;		}	}	return NULL;} /** * binder_alloc_prepare_to_free() - get buffer given user ptr * @alloc:	binder_alloc for this proc * @user_ptr:	User pointer to buffer data * * Validate userspace pointer to buffer data and return buffer corresponding to * that user pointer. Search the rb tree for buffer that matches user data * pointer. * * Return:	Pointer to buffer or NULL */struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc,						   unsigned long user_ptr){	guard(mutex)(&alloc->mutex);	return binder_alloc_prepare_to_free_locked(alloc, user_ptr);} static inline voidbinder_set_installed_page(struct binder_alloc *alloc,			  unsigned long index,			  struct page *page){	/* Pairs with acquire in binder_get_installed_page() */	smp_store_release(&alloc->pages[index], page);} static inline struct page *binder_get_installed_page(struct binder_alloc *alloc, unsigned long index){	/* Pairs with release in binder_set_installed_page() */	return smp_load_acquire(&alloc->pages[index]);} static void binder_lru_freelist_add(struct binder_alloc *alloc,				    unsigned long start, unsigned long end){	unsigned long page_addr;	struct page *page; 	trace_binder_update_page_range(alloc, false, start, end); 	for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {		size_t index;		int ret; 		index = (page_addr - alloc->vm_start) / PAGE_SIZE;		page = binder_get_installed_page(alloc, index);		if (!page)			continue; 		trace_binder_free_lru_start(alloc, index); 		ret = list_lru_add(alloc->freelist,				   page_to_lru(page),				   page_to_nid(page),				   NULL);		WARN_ON(!ret); 		trace_binder_free_lru_end(alloc, index);	}} static inlinevoid binder_alloc_set_mapped(struct binder_alloc *alloc, bool state){	/* pairs with smp_load_acquire in binder_alloc_is_mapped() */	smp_store_release(&alloc->mapped, state);} static inline bool binder_alloc_is_mapped(struct binder_alloc *alloc){	/* pairs with smp_store_release in binder_alloc_set_mapped() */	return smp_load_acquire(&alloc->mapped);} static struct page *binder_page_lookup(struct binder_alloc *alloc,				       unsigned long addr){	struct mm_struct *mm = alloc->mm;	struct page *page;	long npages = 0; 	/*	 * Find an existing page in the remote mm. If missing,	 * don't attempt to fault-in just propagate an error.	 */	mmap_read_lock(mm);	if (binder_alloc_is_mapped(alloc))		npages = get_user_pages_remote(mm, addr, 1, FOLL_NOFAULT,					       &page, NULL);	mmap_read_unlock(mm); 	return npages > 0 ? page : NULL;} static int binder_page_insert(struct binder_alloc *alloc,			      unsigned long addr,			      struct page *page){	struct mm_struct *mm = alloc->mm;	struct vm_area_struct *vma;	int ret = -ESRCH; 	/* attempt per-vma lock first */	vma = lock_vma_under_rcu(mm, addr);	if (vma) {		if (binder_alloc_is_mapped(alloc))			ret = vm_insert_page(vma, addr, page);		vma_end_read(vma);		return ret;	} 	/* fall back to mmap_lock */	mmap_read_lock(mm);	vma = vma_lookup(mm, addr);	if (vma && binder_alloc_is_mapped(alloc))		ret = vm_insert_page(vma, addr, page);	mmap_read_unlock(mm); 	return ret;} static struct page *binder_page_alloc(struct binder_alloc *alloc,				      unsigned long index){	struct binder_shrinker_mdata *mdata;	struct page *page; 	page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);	if (!page)		return NULL; 	/* allocate and install shrinker metadata under page->private */	mdata = kzalloc_obj(*mdata);	if (!mdata) {		__free_page(page);		return NULL;	} 	mdata->alloc = alloc;	mdata->page_index = index;	INIT_LIST_HEAD(&mdata->lru);	set_page_private(page, (unsigned long)mdata); 	return page;} static void binder_free_page(struct page *page){	kfree((struct binder_shrinker_mdata *)page_private(page));	__free_page(page);} static int binder_install_single_page(struct binder_alloc *alloc,				      unsigned long index,				      unsigned long addr){	struct page *page;	int ret; 	if (!mmget_not_zero(alloc->mm))		return -ESRCH; 	page = binder_page_alloc(alloc, index);	if (!page) {		ret = -ENOMEM;		goto out;	} 	ret = binder_page_insert(alloc, addr, page);	switch (ret) {	case -EBUSY:		/*		 * EBUSY is ok. Someone installed the pte first but the		 * alloc->pages[index] has not been updated yet. Discard		 * our page and look up the one already installed.		 */		ret = 0;		binder_free_page(page);		page = binder_page_lookup(alloc, addr);		if (!page) {			pr_err("%d: failed to find page at offset %lx\n",			       alloc->pid, addr - alloc->vm_start);			ret = -ESRCH;			break;		}		fallthrough;	case 0:		/* Mark page installation complete and safe to use */		binder_set_installed_page(alloc, index, page);		break;	default:		binder_free_page(page);		pr_err("%d: %s failed to insert page at offset %lx with %d\n",		       alloc->pid, __func__, addr - alloc->vm_start, ret);		break;	}out:	mmput_async(alloc->mm);	return ret;} static int binder_install_buffer_pages(struct binder_alloc *alloc,				       struct binder_buffer *buffer,				       size_t size){	unsigned long start, final;	unsigned long page_addr; 	start = buffer->user_data & PAGE_MASK;	final = PAGE_ALIGN(buffer->user_data + size); 	for (page_addr = start; page_addr < final; page_addr += PAGE_SIZE) {		unsigned long index;		int ret; 		index = (page_addr - alloc->vm_start) / PAGE_SIZE;		if (binder_get_installed_page(alloc, index))			continue; 		trace_binder_alloc_page_start(alloc, index); 		ret = binder_install_single_page(alloc, index, page_addr);		if (ret)			return ret; 		trace_binder_alloc_page_end(alloc, index);	} 	return 0;} /* The range of pages should exclude those shared with other buffers */static void binder_lru_freelist_del(struct binder_alloc *alloc,				    unsigned long start, unsigned long end){	unsigned long page_addr;	struct page *page; 	trace_binder_update_page_range(alloc, true, start, end); 	for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {		unsigned long index;		bool on_lru; 		index = (page_addr - alloc->vm_start) / PAGE_SIZE;		page = binder_get_installed_page(alloc, index); 		if (page) {			trace_binder_alloc_lru_start(alloc, index); 			on_lru = list_lru_del(alloc->freelist,					      page_to_lru(page),					      page_to_nid(page),					      NULL);			WARN_ON(!on_lru); 			trace_binder_alloc_lru_end(alloc, index);			continue;		} 		if (index + 1 > alloc->pages_high)			alloc->pages_high = index + 1;	}} static void debug_no_space_locked(struct binder_alloc *alloc){	size_t largest_alloc_size = 0;	struct binder_buffer *buffer;	size_t allocated_buffers = 0;	size_t largest_free_size = 0;	size_t total_alloc_size = 0;	size_t total_free_size = 0;	size_t free_buffers = 0;	size_t buffer_size;	struct rb_node *n; 	for (n = rb_first(&alloc->allocated_buffers); n; n = rb_next(n)) {		buffer = rb_entry(n, struct binder_buffer, rb_node);		buffer_size = binder_alloc_buffer_size(alloc, buffer);		allocated_buffers++;		total_alloc_size += buffer_size;		if (buffer_size > largest_alloc_size)			largest_alloc_size = buffer_size;	} 	for (n = rb_first(&alloc->free_buffers); n; n = rb_next(n)) {		buffer = rb_entry(n, struct binder_buffer, rb_node);		buffer_size = binder_alloc_buffer_size(alloc, buffer);		free_buffers++;		total_free_size += buffer_size;		if (buffer_size > largest_free_size)			largest_free_size = buffer_size;	} 	binder_alloc_debug(BINDER_DEBUG_USER_ERROR,			   "allocated: %zd (num: %zd largest: %zd), free: %zd (num: %zd largest: %zd)\n",			   total_alloc_size, allocated_buffers,			   largest_alloc_size, total_free_size,			   free_buffers, largest_free_size);} static bool debug_low_async_space_locked(struct binder_alloc *alloc){	/*	 * Find the amount and size of buffers allocated by the current caller;	 * The idea is that once we cross the threshold, whoever is responsible	 * for the low async space is likely to try to send another async txn,	 * and at some point we'll catch them in the act. This is more efficient	 * than keeping a map per pid.	 */	struct binder_buffer *buffer;	size_t total_alloc_size = 0;	int pid = current->tgid;	size_t num_buffers = 0;	struct rb_node *n; 	/*	 * Only start detecting spammers once we have less than 20% of async	 * space left (which is less than 10% of total buffer size).	 */	if (alloc->free_async_space >= alloc->buffer_size / 10) {		alloc->oneway_spam_detected = false;		return false;	} 	for (n = rb_first(&alloc->allocated_buffers); n != NULL;		 n = rb_next(n)) {		buffer = rb_entry(n, struct binder_buffer, rb_node);		if (buffer->pid != pid)			continue;		if (!buffer->async_transaction)			continue;		total_alloc_size += binder_alloc_buffer_size(alloc, buffer);		num_buffers++;	} 	/*	 * Warn if this pid has more than 50 transactions, or more than 50% of	 * async space (which is 25% of total buffer size). Oneway spam is only	 * detected when the threshold is exceeded.

Lines 1–500 of 1409. Load more