/* SPDX-License-Identifier: GPL-2.0 * * page_pool.c *	Author:	Jesper Dangaard Brouer <netoptimizer@brouer.com> *	Copyright (C) 2016 Red Hat, Inc. */ #include <linux/error-injection.h>#include <linux/types.h>#include <linux/kernel.h>#include <linux/slab.h>#include <linux/device.h> #include <net/netdev_lock.h>#include <net/netdev_rx_queue.h>#include <net/page_pool/helpers.h>#include <net/page_pool/memory_provider.h>#include <net/xdp.h> #include <linux/dma-direction.h>#include <linux/dma-mapping.h>#include <linux/page-flags.h>#include <linux/mm.h> /* for put_page() */#include <linux/poison.h>#include <linux/ethtool.h>#include <linux/netdevice.h> #include <trace/events/page_pool.h> #include "dev.h"#include "mp_dmabuf_devmem.h"#include "netmem_priv.h"#include "page_pool_priv.h" DEFINE_STATIC_KEY_FALSE(page_pool_mem_providers); #define DEFER_TIME (msecs_to_jiffies(1000))#define DEFER_WARN_INTERVAL (60 * HZ) #define BIAS_MAX	(LONG_MAX >> 1) #ifdef CONFIG_PAGE_POOL_STATSstatic DEFINE_PER_CPU(struct page_pool_recycle_stats, pp_system_recycle_stats); /* alloc_stat_inc is intended to be used in softirq context */#define alloc_stat_inc(pool, __stat)	(pool->alloc_stats.__stat++)/* recycle_stat_inc is safe to use when preemption is possible. */#define recycle_stat_inc(pool, __stat)							\	do {										\		struct page_pool_recycle_stats __percpu *s = pool->recycle_stats;	\		this_cpu_inc(s->__stat);						\	} while (0) #define recycle_stat_add(pool, __stat, val)						\	do {										\		struct page_pool_recycle_stats __percpu *s = pool->recycle_stats;	\		this_cpu_add(s->__stat, val);						\	} while (0) static const char pp_stats[][ETH_GSTRING_LEN] = {	"rx_pp_alloc_fast",	"rx_pp_alloc_slow",	"rx_pp_alloc_slow_ho",	"rx_pp_alloc_empty",	"rx_pp_alloc_refill",	"rx_pp_alloc_waive",	"rx_pp_recycle_cached",	"rx_pp_recycle_cache_full",	"rx_pp_recycle_ring",	"rx_pp_recycle_ring_full",	"rx_pp_recycle_released_ref",}; /** * page_pool_get_stats() - fetch page pool stats * @pool:	pool from which page was allocated * @stats:	struct page_pool_stats to fill in * * Deprecated driver API for querying stats. Page pool stats can be queried * via netdev Netlink. * * Retrieve statistics about the page_pool. This API is only available * if the kernel has been configured with ``CONFIG_PAGE_POOL_STATS=y``. * A pointer to a caller allocated struct page_pool_stats structure * is passed to this API which is filled in. The caller can then report * those stats to the user (perhaps via ethtool, debugfs, etc.). */void page_pool_get_stats(const struct page_pool *pool,			 struct page_pool_stats *stats){	int cpu = 0; 	/* The caller is responsible to initialize stats. */	stats->alloc_stats.fast += pool->alloc_stats.fast;	stats->alloc_stats.slow += pool->alloc_stats.slow;	stats->alloc_stats.slow_high_order += pool->alloc_stats.slow_high_order;	stats->alloc_stats.empty += pool->alloc_stats.empty;	stats->alloc_stats.refill += pool->alloc_stats.refill;	stats->alloc_stats.waive += pool->alloc_stats.waive; 	for_each_possible_cpu(cpu) {		const struct page_pool_recycle_stats *pcpu =			per_cpu_ptr(pool->recycle_stats, cpu); 		stats->recycle_stats.cached += pcpu->cached;		stats->recycle_stats.cache_full += pcpu->cache_full;		stats->recycle_stats.ring += pcpu->ring;		stats->recycle_stats.ring_full += pcpu->ring_full;		stats->recycle_stats.released_refcnt += pcpu->released_refcnt;	}}EXPORT_SYMBOL(page_pool_get_stats); u8 *page_pool_ethtool_stats_get_strings(u8 *data){	int i; 	for (i = 0; i < ARRAY_SIZE(pp_stats); i++) {		memcpy(data, pp_stats[i], ETH_GSTRING_LEN);		data += ETH_GSTRING_LEN;	} 	return data;}EXPORT_SYMBOL(page_pool_ethtool_stats_get_strings); int page_pool_ethtool_stats_get_count(void){	return ARRAY_SIZE(pp_stats);}EXPORT_SYMBOL(page_pool_ethtool_stats_get_count); u64 *page_pool_ethtool_stats_get(u64 *data, const void *stats){	const struct page_pool_stats *pool_stats = stats; 	*data++ = pool_stats->alloc_stats.fast;	*data++ = pool_stats->alloc_stats.slow;	*data++ = pool_stats->alloc_stats.slow_high_order;	*data++ = pool_stats->alloc_stats.empty;	*data++ = pool_stats->alloc_stats.refill;	*data++ = pool_stats->alloc_stats.waive;	*data++ = pool_stats->recycle_stats.cached;	*data++ = pool_stats->recycle_stats.cache_full;	*data++ = pool_stats->recycle_stats.ring;	*data++ = pool_stats->recycle_stats.ring_full;	*data++ = pool_stats->recycle_stats.released_refcnt; 	return data;}EXPORT_SYMBOL(page_pool_ethtool_stats_get); #else#define alloc_stat_inc(...)	do { } while (0)#define recycle_stat_inc(...)	do { } while (0)#define recycle_stat_add(...)	do { } while (0)#endif static bool page_pool_producer_lock(struct page_pool *pool)	__acquires(&pool->ring.producer_lock){	bool in_softirq = in_softirq(); 	if (in_softirq)		spin_lock(&pool->ring.producer_lock);	else		spin_lock_bh(&pool->ring.producer_lock); 	return in_softirq;} static void page_pool_producer_unlock(struct page_pool *pool,				      bool in_softirq)	__releases(&pool->ring.producer_lock){	if (in_softirq)		spin_unlock(&pool->ring.producer_lock);	else		spin_unlock_bh(&pool->ring.producer_lock);} static void page_pool_struct_check(void){	CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_users);	CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_page);	CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_offset);	CACHELINE_ASSERT_GROUP_SIZE(struct page_pool, frag,				    PAGE_POOL_FRAG_GROUP_ALIGN);} static int page_pool_init(struct page_pool *pool,			  const struct page_pool_params *params,			  int cpuid){	unsigned int ring_qsize = 1024; /* Default */	struct netdev_rx_queue *rxq;	int err; 	page_pool_struct_check(); 	memcpy(&pool->p, &params->fast, sizeof(pool->p));	memcpy(&pool->slow, &params->slow, sizeof(pool->slow)); 	pool->cpuid = cpuid;	pool->dma_sync_for_cpu = true; 	/* Validate only known flags were used */	if (pool->slow.flags & ~PP_FLAG_ALL)		return -EINVAL; 	if (pool->p.pool_size)		ring_qsize = min(pool->p.pool_size, 16384); 	/* DMA direction is either DMA_FROM_DEVICE or DMA_BIDIRECTIONAL.	 * DMA_BIDIRECTIONAL is for allowing page used for DMA sending,	 * which is the XDP_TX use-case.	 */	if (pool->slow.flags & PP_FLAG_DMA_MAP) {		if ((pool->p.dma_dir != DMA_FROM_DEVICE) &&		    (pool->p.dma_dir != DMA_BIDIRECTIONAL))			return -EINVAL; 		pool->dma_map = true;	} 	if (pool->slow.flags & PP_FLAG_DMA_SYNC_DEV) {		/* In order to request DMA-sync-for-device the page		 * needs to be mapped		 */		if (!(pool->slow.flags & PP_FLAG_DMA_MAP))			return -EINVAL; 		if (!pool->p.max_len)			return -EINVAL; 		pool->dma_sync = true; 		/* pool->p.offset has to be set according to the address		 * offset used by the DMA engine to start copying rx data		 */	} 	pool->has_init_callback = !!pool->slow.init_callback; #ifdef CONFIG_PAGE_POOL_STATS	if (!(pool->slow.flags & PP_FLAG_SYSTEM_POOL)) {		pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats);		if (!pool->recycle_stats)			return -ENOMEM;	} else {		/* For system page pool instance we use a singular stats object		 * instead of allocating a separate percpu variable for each		 * (also percpu) page pool instance.		 */		pool->recycle_stats = &pp_system_recycle_stats;		pool->system = true;	}#endif 	if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) {#ifdef CONFIG_PAGE_POOL_STATS		if (!pool->system)			free_percpu(pool->recycle_stats);#endif		return -ENOMEM;	} 	atomic_set(&pool->pages_state_release_cnt, 0); 	/* Driver calling page_pool_create() also call page_pool_destroy() */	refcount_set(&pool->user_cnt, 1); 	xa_init_flags(&pool->dma_mapped, XA_FLAGS_ALLOC1); 	if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) {		netdev_assert_locked(pool->slow.netdev);		rxq = __netif_get_rx_queue(pool->slow.netdev,					   pool->slow.queue_idx);		pool->mp_priv = rxq->mp_params.mp_priv;		pool->mp_ops = rxq->mp_params.mp_ops;	} 	if (pool->mp_ops) {		if (!pool->dma_map || !pool->dma_sync) {			err = -EOPNOTSUPP;			goto free_ptr_ring;		} 		if (WARN_ON(!is_kernel_rodata((unsigned long)pool->mp_ops))) {			err = -EFAULT;			goto free_ptr_ring;		} 		err = pool->mp_ops->init(pool);		if (err) {			pr_warn("%s() mem-provider init failed %d\n", __func__,				err);			goto free_ptr_ring;		} 		static_branch_inc(&page_pool_mem_providers);	} else if (pool->p.order > MAX_PAGE_ORDER) {		err = -EINVAL;		goto free_ptr_ring;	} 	return 0; free_ptr_ring:	ptr_ring_cleanup(&pool->ring, NULL);	xa_destroy(&pool->dma_mapped);#ifdef CONFIG_PAGE_POOL_STATS	if (!pool->system)		free_percpu(pool->recycle_stats);#endif	return err;} static void page_pool_uninit(struct page_pool *pool){	ptr_ring_cleanup(&pool->ring, NULL);	xa_destroy(&pool->dma_mapped); #ifdef CONFIG_PAGE_POOL_STATS	if (!pool->system)		free_percpu(pool->recycle_stats);#endif 	if (pool->mp_ops) {		pool->mp_ops->destroy(pool);		static_branch_dec(&page_pool_mem_providers);	}} /** * page_pool_create_percpu() - create a page pool for a given cpu. * @params: parameters, see struct page_pool_params * @cpuid: cpu identifier */struct page_pool *page_pool_create_percpu(const struct page_pool_params *params, int cpuid){	struct page_pool *pool;	int err; 	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid);	if (!pool)		return ERR_PTR(-ENOMEM); 	err = page_pool_init(pool, params, cpuid);	if (err < 0)		goto err_free; 	err = page_pool_list(pool);	if (err)		goto err_uninit; 	return pool; err_uninit:	page_pool_uninit(pool);err_free:	pr_warn("%s() gave up with errno %d\n", __func__, err);	kfree(pool);	return ERR_PTR(err);}EXPORT_SYMBOL(page_pool_create_percpu); /** * page_pool_create() - create a page pool * @params: parameters, see struct page_pool_params */struct page_pool *page_pool_create(const struct page_pool_params *params){	return page_pool_create_percpu(params, -1);}EXPORT_SYMBOL(page_pool_create); static void page_pool_return_netmem(struct page_pool *pool, netmem_ref netmem); static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool){	struct ptr_ring *r = &pool->ring;	netmem_ref netmem;	int pref_nid; /* preferred NUMA node */ 	/* Quicker fallback, avoid locks when ring is empty */	if (__ptr_ring_empty(r)) {		alloc_stat_inc(pool, empty);		return 0;	} 	/* Softirq guarantee CPU and thus NUMA node is stable. This,	 * assumes CPU refilling driver RX-ring will also run RX-NAPI.	 */#ifdef CONFIG_NUMA	pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid;#else	/* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */	pref_nid = numa_mem_id(); /* will be zero like page_to_nid() */#endif 	/* Refill alloc array, but only if NUMA match */	do {		netmem = (__force netmem_ref)__ptr_ring_consume(r);		if (unlikely(!netmem))			break; 		if (likely(netmem_is_pref_nid(netmem, pref_nid))) {			pool->alloc.cache[pool->alloc.count++] = netmem;		} else {			/* NUMA mismatch;			 * (1) release 1 page to page-allocator and			 * (2) break out to fallthrough to alloc_pages_node.			 * This limit stress on page buddy alloactor.			 */			page_pool_return_netmem(pool, netmem);			alloc_stat_inc(pool, waive);			netmem = 0;			break;		}	} while (pool->alloc.count < PP_ALLOC_CACHE_REFILL); 	/* Return last page */	if (likely(pool->alloc.count > 0)) {		netmem = pool->alloc.cache[--pool->alloc.count];		alloc_stat_inc(pool, refill);	} 	return netmem;} /* fast path */static netmem_ref __page_pool_get_cached(struct page_pool *pool){	netmem_ref netmem; 	/* Caller MUST guarantee safe non-concurrent access, e.g. softirq */	if (likely(pool->alloc.count)) {		/* Fast-path */		netmem = pool->alloc.cache[--pool->alloc.count];		alloc_stat_inc(pool, fast);	} else {		netmem = page_pool_refill_alloc_cache(pool);	} 	return netmem;} static void __page_pool_dma_sync_for_device(const struct page_pool *pool,					    netmem_ref netmem,					    u32 dma_sync_size){#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)	dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem); 	dma_sync_size = min(dma_sync_size, pool->p.max_len);	__dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset,				     dma_sync_size, pool->p.dma_dir);#endif} static __always_inline voidpage_pool_dma_sync_for_device(const struct page_pool *pool,			      netmem_ref netmem,			      u32 dma_sync_size){	if (pool->dma_sync && dma_dev_need_sync(pool->p.dev)) {		rcu_read_lock();		/* re-check under rcu_read_lock() to sync with page_pool_scrub() */		if (pool->dma_sync)			__page_pool_dma_sync_for_device(pool, netmem,							dma_sync_size);		rcu_read_unlock();	}} static int page_pool_register_dma_index(struct page_pool *pool,					netmem_ref netmem, gfp_t gfp){	int err = 0;	u32 id; 	if (unlikely(!PP_DMA_INDEX_BITS))		goto out; 	/*	 * Drivers request GFP flags according to both the current context and	 * the device constraints, but the XArray entry itself is by no mean	 * used by the device, so remove zone/policy flags.	 */	gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_COMP); 	if (in_softirq())		err = xa_alloc(&pool->dma_mapped, &id, netmem_to_page(netmem),			       PP_DMA_INDEX_LIMIT, gfp);	else		err = xa_alloc_bh(&pool->dma_mapped, &id, netmem_to_page(netmem),				  PP_DMA_INDEX_LIMIT, gfp);	if (err) {

Lines 1–500 of 1364. Load more