/* * Copyright (C) 2017 Netronome Systems, Inc. * * This software is licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this * source tree. * * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. */ #include <linux/debugfs.h>#include <linux/etherdevice.h>#include <linux/ethtool_netlink.h>#include <linux/kernel.h>#include <linux/module.h>#include <linux/netdevice.h>#include <linux/slab.h>#include <net/netdev_queues.h>#include <net/netdev_rx_queue.h>#include <net/page_pool/helpers.h>#include <net/netlink.h>#include <net/net_shaper.h>#include <net/netdev_lock.h>#include <net/pkt_cls.h>#include <net/rtnetlink.h>#include <net/udp_tunnel.h>#include <net/busy_poll.h> #include "netdevsim.h" MODULE_IMPORT_NS("NETDEV_INTERNAL"); #define NSIM_RING_SIZE		256 static void nsim_start_peer_tx_queue(struct net_device *dev, struct nsim_rq *rq){	struct netdevsim *ns = netdev_priv(dev);	struct net_device *peer_dev;	struct netdevsim *peer_ns;	struct netdev_queue *txq;	u16 idx; 	idx = rq->napi.index;	rcu_read_lock();	peer_ns = rcu_dereference(ns->peer);	if (!peer_ns)		goto out; 	/* TX device */	peer_dev = peer_ns->netdev;	if (dev->real_num_tx_queues != peer_dev->num_rx_queues)		goto out; 	txq = netdev_get_tx_queue(peer_dev, idx);	if (!netif_tx_queue_stopped(txq))		goto out; 	netif_tx_wake_queue(txq);out:	rcu_read_unlock();} static void nsim_stop_tx_queue(struct net_device *tx_dev,			       struct net_device *rx_dev,			       struct nsim_rq *rq,			       u16 idx){	/* If different queues size, do not stop, since it is not	 * easy to find which TX queue is mapped here	 */	if (rx_dev->real_num_tx_queues != tx_dev->num_rx_queues)		return; 	/* rq is the queue on the receive side */	netif_subqueue_try_stop(tx_dev, idx,				NSIM_RING_SIZE - skb_queue_len(&rq->skb_queue),				NSIM_RING_SIZE / 2);} static int nsim_napi_rx(struct net_device *tx_dev, struct net_device *rx_dev,			struct nsim_rq *rq, struct sk_buff *skb){	if (skb_queue_len(&rq->skb_queue) > NSIM_RING_SIZE) {		dev_kfree_skb_any(skb);		return NET_RX_DROP;	} 	skb_queue_tail(&rq->skb_queue, skb); 	/* Stop the peer TX queue avoiding dropping packets later */	if (skb_queue_len(&rq->skb_queue) >= NSIM_RING_SIZE)		nsim_stop_tx_queue(tx_dev, rx_dev, rq,				   skb_get_queue_mapping(skb)); 	return NET_RX_SUCCESS;} static int nsim_forward_skb(struct net_device *tx_dev,			    struct net_device *rx_dev,			    struct sk_buff *skb,			    struct nsim_rq *rq){	return __dev_forward_skb(rx_dev, skb) ?:		nsim_napi_rx(tx_dev, rx_dev, rq, skb);} static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev){	struct netdevsim *ns = netdev_priv(dev);	struct net_device *peer_dev;	unsigned int len = skb->len;	struct netdevsim *peer_ns;	struct netdev_config *cfg;	struct nsim_rq *rq;	int rxq; 	rcu_read_lock();	if (!nsim_ipsec_tx(ns, skb))		goto out_drop_free; 	peer_ns = rcu_dereference(ns->peer);	if (!peer_ns)		goto out_drop_free; 	peer_dev = peer_ns->netdev;	rxq = skb_get_queue_mapping(skb);	if (rxq >= peer_dev->num_rx_queues)		rxq = rxq % peer_dev->num_rx_queues;	rq = peer_ns->rq[rxq]; 	cfg = peer_dev->cfg;	if (skb_is_nonlinear(skb) &&	    (cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED ||	     (cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&	      cfg->hds_thresh > len)))		skb_linearize(skb); 	skb_tx_timestamp(skb);	if (unlikely(nsim_forward_skb(dev, peer_dev, skb, rq) == NET_RX_DROP))		goto out_drop_cnt; 	if (!hrtimer_active(&rq->napi_timer))		hrtimer_start(&rq->napi_timer, us_to_ktime(5), HRTIMER_MODE_REL); 	rcu_read_unlock();	dev_dstats_tx_add(dev, len);	return NETDEV_TX_OK; out_drop_free:	dev_kfree_skb(skb);out_drop_cnt:	rcu_read_unlock();	dev_dstats_tx_dropped(dev);	return NETDEV_TX_OK;} static void nsim_set_rx_mode(struct net_device *dev){} static int nsim_change_mtu(struct net_device *dev, int new_mtu){	struct netdevsim *ns = netdev_priv(dev); 	if (ns->xdp.prog && !ns->xdp.prog->aux->xdp_has_frags &&	    new_mtu > NSIM_XDP_MAX_MTU)		return -EBUSY; 	WRITE_ONCE(dev->mtu, new_mtu); 	return 0;} static intnsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv){	return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv);} static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac){	struct netdevsim *ns = netdev_priv(dev);	struct nsim_dev *nsim_dev = ns->nsim_dev; 	/* Only refuse multicast addresses, zero address can mean unset/any. */	if (vf >= nsim_dev_get_vfs(nsim_dev) || is_multicast_ether_addr(mac))		return -EINVAL;	memcpy(nsim_dev->vfconfigs[vf].vf_mac, mac, ETH_ALEN); 	return 0;} static int nsim_set_vf_vlan(struct net_device *dev, int vf,			    u16 vlan, u8 qos, __be16 vlan_proto){	struct netdevsim *ns = netdev_priv(dev);	struct nsim_dev *nsim_dev = ns->nsim_dev; 	if (vf >= nsim_dev_get_vfs(nsim_dev) || vlan > 4095 || qos > 7)		return -EINVAL; 	nsim_dev->vfconfigs[vf].vlan = vlan;	nsim_dev->vfconfigs[vf].qos = qos;	nsim_dev->vfconfigs[vf].vlan_proto = vlan_proto; 	return 0;} static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max){	struct netdevsim *ns = netdev_priv(dev);	struct nsim_dev *nsim_dev = ns->nsim_dev; 	if (nsim_esw_mode_is_switchdev(ns->nsim_dev)) {		pr_err("Not supported in switchdev mode. Please use devlink API.\n");		return -EOPNOTSUPP;	} 	if (vf >= nsim_dev_get_vfs(nsim_dev))		return -EINVAL; 	nsim_dev->vfconfigs[vf].min_tx_rate = min;	nsim_dev->vfconfigs[vf].max_tx_rate = max; 	return 0;} static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val){	struct netdevsim *ns = netdev_priv(dev);	struct nsim_dev *nsim_dev = ns->nsim_dev; 	if (vf >= nsim_dev_get_vfs(nsim_dev))		return -EINVAL;	nsim_dev->vfconfigs[vf].spoofchk_enabled = val; 	return 0;} static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val){	struct netdevsim *ns = netdev_priv(dev);	struct nsim_dev *nsim_dev = ns->nsim_dev; 	if (vf >= nsim_dev_get_vfs(nsim_dev))		return -EINVAL;	nsim_dev->vfconfigs[vf].rss_query_enabled = val; 	return 0;} static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val){	struct netdevsim *ns = netdev_priv(dev);	struct nsim_dev *nsim_dev = ns->nsim_dev; 	if (vf >= nsim_dev_get_vfs(nsim_dev))		return -EINVAL;	nsim_dev->vfconfigs[vf].trusted = val; 	return 0;} static intnsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi){	struct netdevsim *ns = netdev_priv(dev);	struct nsim_dev *nsim_dev = ns->nsim_dev; 	if (vf >= nsim_dev_get_vfs(nsim_dev))		return -EINVAL; 	ivi->vf = vf;	ivi->linkstate = nsim_dev->vfconfigs[vf].link_state;	ivi->min_tx_rate = nsim_dev->vfconfigs[vf].min_tx_rate;	ivi->max_tx_rate = nsim_dev->vfconfigs[vf].max_tx_rate;	ivi->vlan = nsim_dev->vfconfigs[vf].vlan;	ivi->vlan_proto = nsim_dev->vfconfigs[vf].vlan_proto;	ivi->qos = nsim_dev->vfconfigs[vf].qos;	memcpy(&ivi->mac, nsim_dev->vfconfigs[vf].vf_mac, ETH_ALEN);	ivi->spoofchk = nsim_dev->vfconfigs[vf].spoofchk_enabled;	ivi->trusted = nsim_dev->vfconfigs[vf].trusted;	ivi->rss_query_en = nsim_dev->vfconfigs[vf].rss_query_enabled; 	return 0;} static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state){	struct netdevsim *ns = netdev_priv(dev);	struct nsim_dev *nsim_dev = ns->nsim_dev; 	if (vf >= nsim_dev_get_vfs(nsim_dev))		return -EINVAL; 	switch (state) {	case IFLA_VF_LINK_STATE_AUTO:	case IFLA_VF_LINK_STATE_ENABLE:	case IFLA_VF_LINK_STATE_DISABLE:		break;	default:		return -EINVAL;	} 	nsim_dev->vfconfigs[vf].link_state = state; 	return 0;} static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats){	stats->window_drops = 0;	stats->tx_overruns = 0;} static int nsim_setup_tc_taprio(struct net_device *dev,				struct tc_taprio_qopt_offload *offload){	int err = 0; 	switch (offload->cmd) {	case TAPRIO_CMD_REPLACE:	case TAPRIO_CMD_DESTROY:		break;	case TAPRIO_CMD_STATS:		nsim_taprio_stats(&offload->stats);		break;	default:		err = -EOPNOTSUPP;	} 	return err;} static LIST_HEAD(nsim_block_cb_list); static intnsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data){	struct netdevsim *ns = netdev_priv(dev); 	switch (type) {	case TC_SETUP_QDISC_TAPRIO:		return nsim_setup_tc_taprio(dev, type_data);	case TC_SETUP_BLOCK:		return flow_block_cb_setup_simple(type_data,						  &nsim_block_cb_list,						  nsim_setup_tc_block_cb,						  ns, ns, true);	default:		return -EOPNOTSUPP;	}} static intnsim_set_features(struct net_device *dev, netdev_features_t features){	struct netdevsim *ns = netdev_priv(dev); 	if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC))		return nsim_bpf_disable_tc(ns); 	return 0;} static int nsim_get_iflink(const struct net_device *dev){	struct netdevsim *nsim, *peer;	int iflink; 	nsim = netdev_priv(dev); 	rcu_read_lock();	peer = rcu_dereference(nsim->peer);	iflink = peer ? READ_ONCE(peer->netdev->ifindex) :			READ_ONCE(dev->ifindex);	rcu_read_unlock(); 	return iflink;} static int nsim_rcv(struct nsim_rq *rq, int budget){	struct net_device *dev = rq->napi.dev;	struct bpf_prog *xdp_prog;	struct netdevsim *ns;	struct sk_buff *skb;	unsigned int skblen;	int i, ret; 	ns = netdev_priv(dev);	xdp_prog = READ_ONCE(ns->xdp.prog); 	for (i = 0; i < budget; i++) {		if (skb_queue_empty(&rq->skb_queue))			break; 		skb = skb_dequeue(&rq->skb_queue); 		if (xdp_prog) {			/* skb might be freed directly by XDP, save the len */			skblen = skb->len; 			if (skb->ip_summed == CHECKSUM_PARTIAL)				skb_checksum_help(skb);			ret = do_xdp_generic(xdp_prog, &skb);			if (ret != XDP_PASS) {				dev_dstats_rx_add(dev, skblen);				continue;			}		} 		/* skb might be discard at netif_receive_skb, save the len */		skblen = skb->len;		skb_mark_napi_id(skb, &rq->napi);		ret = netif_receive_skb(skb);		if (ret == NET_RX_SUCCESS)			dev_dstats_rx_add(dev, skblen);		else			dev_dstats_rx_dropped(dev);	} 	nsim_start_peer_tx_queue(dev, rq);	return i;} static int nsim_poll(struct napi_struct *napi, int budget){	struct nsim_rq *rq = container_of(napi, struct nsim_rq, napi);	int done; 	done = nsim_rcv(rq, budget);	if (done < budget)		napi_complete_done(napi, done); 	return done;} static int nsim_create_page_pool(struct page_pool **p, struct napi_struct *napi){	struct page_pool_params params = {		.order = 0,		.pool_size = NSIM_RING_SIZE,		.nid = NUMA_NO_NODE,		.dev = &napi->dev->dev,		.napi = napi,		.dma_dir = DMA_BIDIRECTIONAL,		.netdev = napi->dev,	};	struct page_pool *pool; 	pool = page_pool_create(&params);	if (IS_ERR(pool))		return PTR_ERR(pool); 	*p = pool;	return 0;} static int nsim_init_napi(struct netdevsim *ns){	struct net_device *dev = ns->netdev;	struct nsim_rq *rq;	int err, i; 	for (i = 0; i < dev->num_rx_queues; i++) {		rq = ns->rq[i]; 		netif_napi_add_config_locked(dev, &rq->napi, nsim_poll, i);	} 	for (i = 0; i < dev->num_rx_queues; i++) {		rq = ns->rq[i]; 		err = nsim_create_page_pool(&rq->page_pool, &rq->napi);		if (err)			goto err_pp_destroy;	} 	return 0; err_pp_destroy:	while (i--) {		page_pool_destroy(ns->rq[i]->page_pool);		ns->rq[i]->page_pool = NULL;	} 	for (i = 0; i < dev->num_rx_queues; i++)		__netif_napi_del_locked(&ns->rq[i]->napi); 	return err;} static enum hrtimer_restart nsim_napi_schedule(struct hrtimer *timer){

Lines 1–500 of 1215. Load more