// SPDX-License-Identifier: GPL-2.0-or-later #include <linux/ethtool_netlink.h>#include <linux/netdevice.h>#include <net/netdev_lock.h>#include <net/netdev_queues.h>#include <net/netdev_rx_queue.h>#include <net/page_pool/memory_provider.h> #include "dev.h"#include "page_pool_priv.h" void netdev_rx_queue_lease(struct netdev_rx_queue *rxq_dst,			   struct netdev_rx_queue *rxq_src){	netdev_assert_locked(rxq_src->dev);	netdev_assert_locked(rxq_dst->dev); 	netdev_hold(rxq_src->dev, &rxq_src->lease_tracker, GFP_KERNEL); 	WRITE_ONCE(rxq_src->lease, rxq_dst);	WRITE_ONCE(rxq_dst->lease, rxq_src);} void netdev_rx_queue_unlease(struct netdev_rx_queue *rxq_dst,			     struct netdev_rx_queue *rxq_src){	netdev_assert_locked(rxq_dst->dev);	netdev_assert_locked(rxq_src->dev); 	netif_rxq_cleanup_unlease(rxq_src, rxq_dst); 	WRITE_ONCE(rxq_src->lease, NULL);	WRITE_ONCE(rxq_dst->lease, NULL); 	netdev_put(rxq_src->dev, &rxq_src->lease_tracker);} bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx){	if (rxq_idx < dev->real_num_rx_queues)		return READ_ONCE(__netif_get_rx_queue(dev, rxq_idx)->lease);	return false;} /* Virtual devices eligible for leasing have no dev->dev.parent, while * physical devices always have one. Use this to enforce the correct * lease traversal direction. */static bool netif_lease_dir_ok(const struct net_device *dev,			       enum netif_lease_dir dir){	if (dir == NETIF_VIRT_TO_PHYS && !dev->dev.parent)		return true;	if (dir == NETIF_PHYS_TO_VIRT && dev->dev.parent)		return true;	return false;} bool netif_is_queue_leasee(const struct net_device *dev){	return netif_lease_dir_ok(dev, NETIF_VIRT_TO_PHYS);} struct netdev_rx_queue *__netif_get_rx_queue_lease(struct net_device **dev, unsigned int *rxq_idx,			   enum netif_lease_dir dir){	struct net_device *orig_dev = *dev;	struct netdev_rx_queue *rxq = __netif_get_rx_queue(orig_dev, *rxq_idx); 	if (rxq->lease) {		if (!netif_lease_dir_ok(orig_dev, dir))			return NULL;		rxq = rxq->lease;		*rxq_idx = get_netdev_rx_queue_index(rxq);		*dev = rxq->dev;	}	return rxq;} /* See also page_pool_is_unreadable() */bool netif_rxq_has_unreadable_mp(struct net_device *dev, unsigned int rxq_idx){	if (rxq_idx < dev->real_num_rx_queues)		return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_ops;	return false;}EXPORT_SYMBOL(netif_rxq_has_unreadable_mp); bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx){	if (rxq_idx < dev->real_num_rx_queues)		return __netif_get_rx_queue(dev, rxq_idx)->mp_params.mp_priv;	return false;} static int netdev_rx_queue_reconfig(struct net_device *dev,				    unsigned int rxq_idx,				    struct netdev_queue_config *qcfg_old,				    struct netdev_queue_config *qcfg_new){	struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx);	const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops;	void *new_mem, *old_mem;	int err; 	if (!qops || !qops->ndo_queue_stop || !qops->ndo_queue_mem_free ||	    !qops->ndo_queue_mem_alloc || !qops->ndo_queue_start)		return -EOPNOTSUPP; 	netdev_assert_locked(dev); 	new_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL);	if (!new_mem)		return -ENOMEM; 	old_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL);	if (!old_mem) {		err = -ENOMEM;		goto err_free_new_mem;	} 	err = qops->ndo_queue_mem_alloc(dev, qcfg_new, new_mem, rxq_idx);	if (err)		goto err_free_old_mem; 	err = page_pool_check_memory_provider(dev, rxq);	if (err)		goto err_free_new_queue_mem; 	if (netif_running(dev)) {		err = qops->ndo_queue_stop(dev, old_mem, rxq_idx);		if (err)			goto err_free_new_queue_mem; 		err = qops->ndo_queue_start(dev, qcfg_new, new_mem, rxq_idx);		if (err)			goto err_start_queue;	} else {		swap(new_mem, old_mem);	} 	qops->ndo_queue_mem_free(dev, old_mem); 	kvfree(old_mem);	kvfree(new_mem); 	return 0; err_start_queue:	/* Restarting the queue with old_mem should be successful as we haven't	 * changed any of the queue configuration, and there is not much we can	 * do to recover from a failure here.	 *	 * WARN if we fail to recover the old rx queue, and at least free	 * old_mem so we don't also leak that.	 */	if (qops->ndo_queue_start(dev, qcfg_old, old_mem, rxq_idx)) {		WARN(1,		     "Failed to restart old queue in error path. RX queue %d may be unhealthy.",		     rxq_idx);		qops->ndo_queue_mem_free(dev, old_mem);	} err_free_new_queue_mem:	qops->ndo_queue_mem_free(dev, new_mem); err_free_old_mem:	kvfree(old_mem); err_free_new_mem:	kvfree(new_mem); 	return err;} int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx){	struct netdev_queue_config qcfg; 	netdev_queue_config(dev, rxq_idx, &qcfg);	return netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg, &qcfg);}EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL"); static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,			       const struct pp_memory_provider_params *p,			       struct netlink_ext_ack *extack){	const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops;	struct netdev_queue_config qcfg[2];	struct netdev_rx_queue *rxq;	int ret; 	if (!qops)		return -EOPNOTSUPP; 	if (dev->cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED) {		NL_SET_ERR_MSG(extack, "tcp-data-split is disabled");		return -EINVAL;	}	if (dev->cfg->hds_thresh) {		NL_SET_ERR_MSG(extack, "hds-thresh is not zero");		return -EINVAL;	}	if (dev_xdp_prog_count(dev)) {		NL_SET_ERR_MSG(extack, "unable to custom memory provider to device with XDP program attached");		return -EEXIST;	}	if (p->rx_page_size && !(qops->supported_params & QCFG_RX_PAGE_SIZE)) {		NL_SET_ERR_MSG(extack, "device does not support: rx_page_size");		return -EOPNOTSUPP;	} 	rxq = __netif_get_rx_queue(dev, rxq_idx);	if (rxq->mp_params.mp_ops) {		NL_SET_ERR_MSG(extack, "designated queue already memory provider bound");		return -EEXIST;	}#ifdef CONFIG_XDP_SOCKETS	if (rxq->pool) {		NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP");		return -EBUSY;	}#endif 	netdev_queue_config(dev, rxq_idx, &qcfg[0]);	rxq->mp_params = *p;	ret = netdev_queue_config_validate(dev, rxq_idx, &qcfg[1], extack);	if (ret)		goto err_clear_mp; 	ret = netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg[0], &qcfg[1]);	if (ret)		goto err_clear_mp; 	return 0; err_clear_mp:	memset(&rxq->mp_params, 0, sizeof(rxq->mp_params));	return ret;} int netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,		      const struct pp_memory_provider_params *p,		      struct netlink_ext_ack *extack){	int ret; 	if (!netdev_need_ops_lock(dev))		return -EOPNOTSUPP; 	if (rxq_idx >= dev->real_num_rx_queues) {		NL_SET_ERR_MSG(extack, "rx queue index out of range");		return -ERANGE;	}	rxq_idx = array_index_nospec(rxq_idx, dev->real_num_rx_queues); 	if (!netif_rxq_is_leased(dev, rxq_idx))		return __netif_mp_open_rxq(dev, rxq_idx, p, extack); 	if (!__netif_get_rx_queue_lease(&dev, &rxq_idx, NETIF_VIRT_TO_PHYS)) {		NL_SET_ERR_MSG(extack, "rx queue leased to a virtual netdev");		return -EBUSY;	}	if (!dev->dev.parent) {		NL_SET_ERR_MSG(extack, "rx queue belongs to a virtual netdev");		return -EOPNOTSUPP;	} 	netdev_lock(dev);	ret = __netif_mp_open_rxq(dev, rxq_idx, p, extack);	netdev_unlock(dev);	return ret;} static void __netif_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx,				 const struct pp_memory_provider_params *old_p){	struct netdev_queue_config qcfg[2];	struct netdev_rx_queue *rxq;	int err; 	rxq = __netif_get_rx_queue(dev, rxq_idx); 	/* Callers holding a netdev ref may get here after we already	 * went thru shutdown via dev_memory_provider_uninstall().	 */	if (dev->reg_state > NETREG_REGISTERED &&	    !rxq->mp_params.mp_ops)		return; 	if (WARN_ON_ONCE(rxq->mp_params.mp_ops != old_p->mp_ops ||			 rxq->mp_params.mp_priv != old_p->mp_priv))		return; 	netdev_queue_config(dev, rxq_idx, &qcfg[0]);	memset(&rxq->mp_params, 0, sizeof(rxq->mp_params));	netdev_queue_config(dev, rxq_idx, &qcfg[1]); 	err = netdev_rx_queue_reconfig(dev, rxq_idx, &qcfg[0], &qcfg[1]);	WARN_ON(err && err != -ENETDOWN);} void netif_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx,			const struct pp_memory_provider_params *old_p){	if (WARN_ON_ONCE(rxq_idx >= dev->real_num_rx_queues))		return;	if (!netif_rxq_is_leased(dev, rxq_idx))		return __netif_mp_close_rxq(dev, rxq_idx, old_p); 	if (!__netif_get_rx_queue_lease(&dev, &rxq_idx, NETIF_VIRT_TO_PHYS)) {		WARN_ON_ONCE(1);		return;	}	netdev_lock(dev);	__netif_mp_close_rxq(dev, rxq_idx, old_p);	netdev_unlock(dev);} void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq,			      const struct pp_memory_provider_params *p){	if (p->mp_ops && p->mp_ops->uninstall)		p->mp_ops->uninstall(p->mp_priv, rxq);} /* Clean up memory provider state when a queue lease is torn down. If * a memory provider was installed on the physical queue via the lease, * close it now. The memory provider is a property of the queue itself, * and it was _guaranteed_ to be installed on the physical queue via * the lease redirection. The extra __netif_mp_close_rxq is needed * since the physical queue can outlive the virtual queue in the lease * case, so it needs to be reconfigured to clear the memory provider. */void netif_rxq_cleanup_unlease(struct netdev_rx_queue *phys_rxq,			       struct netdev_rx_queue *virt_rxq){	unsigned int rxq_idx = get_netdev_rx_queue_index(phys_rxq);	struct pp_memory_provider_params p = phys_rxq->mp_params; 	if (!p.mp_ops)		return; 	__netif_mp_close_rxq(phys_rxq->dev, rxq_idx, &p);	__netif_mp_uninstall_rxq(virt_rxq, &p);}