// SPDX-License-Identifier: GPL-2.0 #include <linux/mutex.h>#include <linux/netdevice.h>#include <linux/xarray.h>#include <net/busy_poll.h>#include <net/net_debug.h>#include <net/netdev_rx_queue.h>#include <net/page_pool/helpers.h>#include <net/page_pool/types.h>#include <net/page_pool/memory_provider.h>#include <net/sock.h> #include "page_pool_priv.h"#include "netdev-genl-gen.h" static DEFINE_XARRAY_FLAGS(page_pools, XA_FLAGS_ALLOC1);/* Protects: page_pools, netdevice->page_pools, pool->p.napi, pool->slow.netdev, *	pool->user. * Ordering: inside rtnl_lock */DEFINE_MUTEX(page_pools_lock); /* Page pools are only reachable from user space (via netlink) if they are * linked to a netdev at creation time. Following page pool "visibility" * states are possible: *  - normal *    - user.list: linked to real netdev, netdev: real netdev *  - orphaned - real netdev has disappeared *    - user.list: linked to lo, netdev: lo *  - invisible - either (a) created without netdev linking, (b) unlisted due *      to error, or (c) the entire namespace which owned this pool disappeared *    - user.list: unhashed, netdev: unknown */ typedef int (*pp_nl_fill_cb)(struct sk_buff *rsp, const struct page_pool *pool,			     const struct genl_info *info); static intnetdev_nl_page_pool_get_do(struct genl_info *info, u32 id, pp_nl_fill_cb fill){	struct page_pool *pool;	struct sk_buff *rsp;	int err; 	mutex_lock(&page_pools_lock);	pool = xa_load(&page_pools, id);	if (!pool || hlist_unhashed(&pool->user.list) ||	    !net_eq(dev_net(pool->slow.netdev), genl_info_net(info))) {		err = -ENOENT;		goto err_unlock;	} 	rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);	if (!rsp) {		err = -ENOMEM;		goto err_unlock;	} 	err = fill(rsp, pool, info);	if (err)		goto err_free_msg; 	mutex_unlock(&page_pools_lock); 	return genlmsg_reply(rsp, info); err_free_msg:	nlmsg_free(rsp);err_unlock:	mutex_unlock(&page_pools_lock);	return err;} struct page_pool_dump_cb {	unsigned long ifindex;	u32 pp_id;}; static intnetdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,			     pp_nl_fill_cb fill, struct nlattr *ifindex_attr){	struct page_pool_dump_cb *state = (void *)cb->ctx;	const struct genl_info *info = genl_info_dump(cb);	struct net *net = sock_net(skb->sk);	struct net_device *netdev;	struct page_pool *pool;	int err = 0; 	if (ifindex_attr)		state->ifindex = nla_get_u32(ifindex_attr); 	rtnl_lock();	mutex_lock(&page_pools_lock);	for_each_netdev_dump(net, netdev, state->ifindex) {		/* Either the provided ifindex doesn't exist or done dumping */		if (ifindex_attr &&		    netdev->ifindex != nla_get_u32(ifindex_attr))			break; 		hlist_for_each_entry(pool, &netdev->page_pools, user.list) {			if (state->pp_id && state->pp_id < pool->user.id)				continue; 			state->pp_id = pool->user.id;			err = fill(skb, pool, info);			if (err)				goto out;		} 		state->pp_id = 0;	}out:	mutex_unlock(&page_pools_lock);	rtnl_unlock(); 	return err;} static intpage_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool,			const struct genl_info *info){#ifdef CONFIG_PAGE_POOL_STATS	struct page_pool_stats stats = {};	struct nlattr *nest;	void *hdr; 	page_pool_get_stats(pool, &stats); 	hdr = genlmsg_iput(rsp, info);	if (!hdr)		return -EMSGSIZE; 	nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO);	if (!nest)		goto err_cancel_msg; 	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) ||	    (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&	     nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,			 pool->slow.netdev->ifindex)))		goto err_cancel_nest; 	nla_nest_end(rsp, nest); 	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_FAST,			 stats.alloc_stats.fast) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW,			 stats.alloc_stats.slow) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_SLOW_HIGH_ORDER,			 stats.alloc_stats.slow_high_order) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_EMPTY,			 stats.alloc_stats.empty) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_REFILL,			 stats.alloc_stats.refill) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_ALLOC_WAIVE,			 stats.alloc_stats.waive) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHED,			 stats.recycle_stats.cached) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_CACHE_FULL,			 stats.recycle_stats.cache_full) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING,			 stats.recycle_stats.ring) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RING_FULL,			 stats.recycle_stats.ring_full) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_STATS_RECYCLE_RELEASED_REFCNT,			 stats.recycle_stats.released_refcnt))		goto err_cancel_msg; 	genlmsg_end(rsp, hdr); 	return 0;err_cancel_nest:	nla_nest_cancel(rsp, nest);err_cancel_msg:	genlmsg_cancel(rsp, hdr);	return -EMSGSIZE;#else	GENL_SET_ERR_MSG(info, "kernel built without CONFIG_PAGE_POOL_STATS");	return -EOPNOTSUPP;#endif} int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb,				       struct genl_info *info){	struct nlattr *tb[ARRAY_SIZE(netdev_page_pool_info_nl_policy)];	struct nlattr *nest;	int err;	u32 id; 	if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_STATS_INFO))		return -EINVAL; 	nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];	err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,			       netdev_page_pool_info_nl_policy,			       info->extack);	if (err)		return err; 	if (NL_REQ_ATTR_CHECK(info->extack, nest, tb, NETDEV_A_PAGE_POOL_ID))		return -EINVAL;	if (tb[NETDEV_A_PAGE_POOL_IFINDEX]) {		NL_SET_ERR_MSG_ATTR(info->extack,				    tb[NETDEV_A_PAGE_POOL_IFINDEX],				    "selecting by ifindex not supported");		return -EINVAL;	} 	id = nla_get_uint(tb[NETDEV_A_PAGE_POOL_ID]); 	return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill);} static const struct netlink_range_validation page_pool_ifindex_range = {	.min	= 1ULL,	.max	= S32_MAX,}; static const struct nla_policypage_pool_stat_info_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = {	[NETDEV_A_PAGE_POOL_IFINDEX] =		NLA_POLICY_FULL_RANGE(NLA_U32, &page_pool_ifindex_range),}; int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb,					 struct netlink_callback *cb){	struct nlattr *tb[ARRAY_SIZE(page_pool_stat_info_policy)];	const struct genl_info *info = genl_info_dump(cb);	struct nlattr *ifindex_attr = NULL; 	if (info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]) {		struct nlattr *nest;		int err; 		nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];		err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,				       page_pool_stat_info_policy,				       info->extack);		if (err)			return err; 		ifindex_attr = tb[NETDEV_A_PAGE_POOL_IFINDEX];	} 	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill,					    ifindex_attr);} static intpage_pool_nl_fill(struct sk_buff *rsp, const struct page_pool *pool,		  const struct genl_info *info){	size_t inflight, refsz;	unsigned int napi_id;	void *hdr; 	hdr = genlmsg_iput(rsp, info);	if (!hdr)		return -EMSGSIZE; 	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id))		goto err_cancel; 	if (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX &&	    nla_put_u32(rsp, NETDEV_A_PAGE_POOL_IFINDEX,			pool->slow.netdev->ifindex))		goto err_cancel; 	napi_id = pool->p.napi ? READ_ONCE(pool->p.napi->napi_id) : 0;	if (napi_id_valid(napi_id) &&	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_NAPI_ID, napi_id))		goto err_cancel; 	inflight = page_pool_inflight(pool, false);	refsz =	PAGE_SIZE << pool->p.order;	if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT, inflight) ||	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_INFLIGHT_MEM,			 inflight * refsz))		goto err_cancel;	if (pool->user.detach_time &&	    nla_put_uint(rsp, NETDEV_A_PAGE_POOL_DETACH_TIME,			 ktime_divns(pool->user.detach_time, NSEC_PER_SEC)))		goto err_cancel; 	if (pool->mp_ops && pool->mp_ops->nl_fill(pool->mp_priv, rsp, NULL))		goto err_cancel; 	genlmsg_end(rsp, hdr); 	return 0;err_cancel:	genlmsg_cancel(rsp, hdr);	return -EMSGSIZE;} static void netdev_nl_page_pool_event(const struct page_pool *pool, u32 cmd){	struct genl_info info;	struct sk_buff *ntf;	struct net *net; 	lockdep_assert_held(&page_pools_lock); 	/* 'invisible' page pools don't matter */	if (hlist_unhashed(&pool->user.list))		return;	net = dev_net(pool->slow.netdev); 	if (!genl_has_listeners(&netdev_nl_family, net, NETDEV_NLGRP_PAGE_POOL))		return; 	genl_info_init_ntf(&info, &netdev_nl_family, cmd); 	ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);	if (!ntf)		return; 	if (page_pool_nl_fill(ntf, pool, &info)) {		nlmsg_free(ntf);		return;	} 	genlmsg_multicast_netns(&netdev_nl_family, net, ntf,				0, NETDEV_NLGRP_PAGE_POOL, GFP_KERNEL);} int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info){	u32 id; 	if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_PAGE_POOL_ID))		return -EINVAL; 	id = nla_get_uint(info->attrs[NETDEV_A_PAGE_POOL_ID]); 	return netdev_nl_page_pool_get_do(info, id, page_pool_nl_fill);} int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb,				   struct netlink_callback *cb){	const struct genl_info *info = genl_info_dump(cb); 	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill,					    info->attrs[NETDEV_A_PAGE_POOL_IFINDEX]);} int page_pool_list(struct page_pool *pool){	static u32 id_alloc_next;	int err; 	mutex_lock(&page_pools_lock);	err = xa_alloc_cyclic(&page_pools, &pool->user.id, pool, xa_limit_32b,			      &id_alloc_next, GFP_KERNEL);	if (err < 0)		goto err_unlock; 	INIT_HLIST_NODE(&pool->user.list);	if (pool->slow.netdev) {		hlist_add_head(&pool->user.list,			       &pool->slow.netdev->page_pools);		netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_ADD_NTF);	} 	mutex_unlock(&page_pools_lock);	return 0; err_unlock:	mutex_unlock(&page_pools_lock);	return err;} void page_pool_detached(struct page_pool *pool){	mutex_lock(&page_pools_lock);	pool->user.detach_time = ktime_get_boottime();	netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_CHANGE_NTF);	mutex_unlock(&page_pools_lock);} void page_pool_unlist(struct page_pool *pool){	mutex_lock(&page_pools_lock);	netdev_nl_page_pool_event(pool, NETDEV_CMD_PAGE_POOL_DEL_NTF);	xa_erase(&page_pools, pool->user.id);	if (!hlist_unhashed(&pool->user.list))		hlist_del(&pool->user.list);	mutex_unlock(&page_pools_lock);} int page_pool_check_memory_provider(struct net_device *dev,				    struct netdev_rx_queue *rxq){	void *binding = rxq->mp_params.mp_priv;	struct page_pool *pool;	struct hlist_node *n; 	if (!binding)		return 0; 	mutex_lock(&page_pools_lock);	hlist_for_each_entry_safe(pool, n, &dev->page_pools, user.list) {		if (pool->mp_priv != binding)			continue; 		if (pool->slow.queue_idx == get_netdev_rx_queue_index(rxq)) {			mutex_unlock(&page_pools_lock);			return 0;		}	}	mutex_unlock(&page_pools_lock);	return -ENODATA;} static void page_pool_unreg_netdev_wipe(struct net_device *netdev){	struct page_pool *pool;	struct hlist_node *n; 	mutex_lock(&page_pools_lock);	hlist_for_each_entry_safe(pool, n, &netdev->page_pools, user.list) {		hlist_del_init(&pool->user.list);		pool->slow.netdev = NET_PTR_POISON;	}	mutex_unlock(&page_pools_lock);} static void page_pool_unreg_netdev(struct net_device *netdev){	struct page_pool *pool, *last;	struct net_device *lo; 	lo = dev_net(netdev)->loopback_dev; 	mutex_lock(&page_pools_lock);	last = NULL;	hlist_for_each_entry(pool, &netdev->page_pools, user.list) {		pool->slow.netdev = lo;		netdev_nl_page_pool_event(pool,					  NETDEV_CMD_PAGE_POOL_CHANGE_NTF);		last = pool;	}	if (last)		hlist_splice_init(&netdev->page_pools, &last->user.list,				  &lo->page_pools);	mutex_unlock(&page_pools_lock);} static intpage_pool_netdevice_event(struct notifier_block *nb,			  unsigned long event, void *ptr){	struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 	if (event != NETDEV_UNREGISTER)		return NOTIFY_DONE; 	if (hlist_empty(&netdev->page_pools))		return NOTIFY_OK; 	if (netdev->ifindex != LOOPBACK_IFINDEX)		page_pool_unreg_netdev(netdev);	else		page_pool_unreg_netdev_wipe(netdev);	return NOTIFY_OK;} static struct notifier_block page_pool_netdevice_nb = {	.notifier_call = page_pool_netdevice_event,}; static int __init page_pool_user_init(void){	return register_netdevice_notifier(&page_pool_netdevice_nb);} subsys_initcall(page_pool_user_init);