// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2023 Isovalent */ #include <linux/netdevice.h>#include <linux/ethtool.h>#include <linux/etherdevice.h>#include <linux/filter.h>#include <linux/netfilter_netdev.h>#include <linux/bpf_mprog.h>#include <linux/indirect_call_wrapper.h> #include <net/netdev_lock.h>#include <net/netdev_queues.h>#include <net/netdev_rx_queue.h>#include <net/xdp_sock_drv.h>#include <net/netkit.h>#include <net/dst.h>#include <net/tcx.h> #define NETKIT_DRV_NAME "netkit" #define NETKIT_NUM_RX_QUEUES_MAX 1024#define NETKIT_NUM_TX_QUEUES_MAX 1 #define NETKIT_NUM_RX_QUEUES_REAL 1#define NETKIT_NUM_TX_QUEUES_REAL 1 struct netkit { __cacheline_group_begin(netkit_fastpath); struct net_device __rcu *peer; struct bpf_mprog_entry __rcu *active; enum netkit_action policy; enum netkit_scrub scrub; struct bpf_mprog_bundle bundle; __cacheline_group_end(netkit_fastpath); __cacheline_group_begin(netkit_slowpath); enum netkit_mode mode; enum netkit_pairing pair; bool primary; u32 headroom; __cacheline_group_end(netkit_slowpath);}; struct netkit_link { struct bpf_link link; struct net_device *dev;}; static struct rtnl_link_ops netkit_link_ops; static __always_inline intnetkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, enum netkit_action ret){ const struct bpf_mprog_fp *fp; const struct bpf_prog *prog; bpf_mprog_foreach_prog(entry, fp, prog) { bpf_compute_data_pointers(skb); ret = bpf_prog_run(prog, skb); if (ret != NETKIT_NEXT) break; } return ret;} static void netkit_xnet(struct sk_buff *skb){ skb->priority = 0; skb->mark = 0;} static void netkit_prep_forward(struct sk_buff *skb, bool xnet, bool xnet_scrub){ skb_scrub_packet(skb, false); nf_skip_egress(skb, true); skb_reset_mac_header(skb); if (!xnet) return; skb_clear_tstamp(skb); if (xnet_scrub) netkit_xnet(skb);} static struct netkit *netkit_priv(const struct net_device *dev){ return netdev_priv(dev);} static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev){ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; struct netkit *nk = netkit_priv(dev); enum netkit_action ret = READ_ONCE(nk->policy); netdev_tx_t ret_dev = NET_XMIT_SUCCESS; const struct bpf_mprog_entry *entry; struct net_device *peer; int len = skb->len; bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); rcu_read_lock(); peer = rcu_dereference(nk->peer); if (unlikely(!peer || !(peer->flags & IFF_UP) || !pskb_may_pull(skb, ETH_HLEN) || skb_orphan_frags(skb, GFP_ATOMIC))) goto drop; netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)), nk->scrub); eth_skb_pkt_type(skb, peer); skb->dev = peer; entry = rcu_dereference(nk->active); if (entry) ret = netkit_run(entry, skb, ret); switch (ret) { case NETKIT_NEXT: case NETKIT_PASS: eth_skb_pull_mac(skb); skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) { dev_sw_netstats_tx_add(dev, 1, len); dev_sw_netstats_rx_add(peer, len); } else { goto drop_stats; } break; case NETKIT_REDIRECT: dev_sw_netstats_tx_add(dev, 1, len); skb_do_redirect(skb); break; case NETKIT_DROP: default:drop: kfree_skb(skb);drop_stats: dev_core_stats_tx_dropped_inc(dev); ret_dev = NET_XMIT_DROP; break; } rcu_read_unlock(); bpf_net_ctx_clear(bpf_net_ctx); return ret_dev;} static int netkit_open(struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer); if (nk->pair == NETKIT_DEVICE_SINGLE) { netif_carrier_on(dev); return 0; } if (!peer) return -ENOTCONN; if (peer->flags & IFF_UP) { netif_carrier_on(dev); netif_carrier_on(peer); } return 0;} static int netkit_close(struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer); netif_carrier_off(dev); if (peer) netif_carrier_off(peer); return 0;} static int netkit_get_iflink(const struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer; int iflink = 0; rcu_read_lock(); peer = rcu_dereference(nk->peer); if (peer) iflink = READ_ONCE(peer->ifindex); rcu_read_unlock(); return iflink;} static int netkit_set_multicast(struct net_device *dev, struct netdev_hw_addr_list *uc, struct netdev_hw_addr_list *mc){ /* Nothing to do, we receive whatever gets pushed to us! */ return 0;} static int netkit_set_macaddr(struct net_device *dev, void *sa){ struct netkit *nk = netkit_priv(dev); if (nk->mode != NETKIT_L2) return -EOPNOTSUPP; return eth_mac_addr(dev, sa);} static void netkit_set_headroom(struct net_device *dev, int headroom){ struct netkit *nk = netkit_priv(dev), *nk2; struct net_device *peer; if (headroom < 0) headroom = NET_SKB_PAD; rcu_read_lock(); peer = rcu_dereference(nk->peer); if (!peer) { nk->headroom = headroom; dev->needed_headroom = headroom; } else { nk2 = netkit_priv(peer); nk->headroom = headroom; headroom = max(nk->headroom, nk2->headroom); peer->needed_headroom = headroom; dev->needed_headroom = headroom; } rcu_read_unlock();} INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev){ return rcu_dereference(netkit_priv(dev)->peer);} static void netkit_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats){ dev_fetch_sw_netstats(stats, dev->tstats); stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);} static bool netkit_xsk_supported_at_phys(const struct net_device *dev){ if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xdp_xmit || !dev->netdev_ops->ndo_xsk_wakeup) return false; return true;} static int netkit_xsk(struct net_device *dev, struct netdev_bpf *xdp){ struct netkit *nk = netkit_priv(dev); struct netdev_bpf xdp_lower; struct netdev_rx_queue *rxq; struct net_device *phys; bool create = false; int ret = -EBUSY; switch (xdp->command) { case XDP_SETUP_XSK_POOL: if (nk->pair == NETKIT_DEVICE_PAIR) return -EOPNOTSUPP; if (xdp->xsk.queue_id >= dev->real_num_rx_queues) return -EINVAL; rxq = __netif_get_rx_queue(dev, xdp->xsk.queue_id); if (!rxq->lease) return -EOPNOTSUPP; phys = rxq->lease->dev; if (!netkit_xsk_supported_at_phys(phys)) return -EOPNOTSUPP; create = xdp->xsk.pool; memcpy(&xdp_lower, xdp, sizeof(xdp_lower)); xdp_lower.xsk.queue_id = get_netdev_rx_queue_index(rxq->lease); break; case XDP_SETUP_PROG: return -EOPNOTSUPP; default: return -EINVAL; } netdev_lock(phys); if (create && (phys->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) { ret = -EOPNOTSUPP; goto out; } if (!create || !dev_get_min_mp_channel_count(phys)) ret = phys->netdev_ops->ndo_bpf(phys, &xdp_lower);out: netdev_unlock(phys); return ret;} static int netkit_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags){ struct netdev_rx_queue *rxq, *rxq_lease; struct net_device *phys; if (queue_id >= dev->real_num_rx_queues) return -EINVAL; rxq = __netif_get_rx_queue(dev, queue_id); rxq_lease = READ_ONCE(rxq->lease); if (unlikely(!rxq_lease)) return -EOPNOTSUPP; /* netkit_xsk already validated full xsk support, hence it's * fine to call into ndo_xsk_wakeup right away given this * was a prerequisite to get here in the first place. The * phys xsk support cannot change without tearing down the * device (which clears the lease first). */ phys = rxq_lease->dev; return phys->netdev_ops->ndo_xsk_wakeup(phys, get_netdev_rx_queue_index(rxq_lease), flags);} static int netkit_init(struct net_device *dev){ netdev_lockdep_set_classes(dev); return 0;} static void netkit_uninit(struct net_device *dev); static const struct net_device_ops netkit_netdev_ops = { .ndo_init = netkit_init, .ndo_open = netkit_open, .ndo_stop = netkit_close, .ndo_start_xmit = netkit_xmit, .ndo_set_rx_mode_async = netkit_set_multicast, .ndo_set_rx_headroom = netkit_set_headroom, .ndo_set_mac_address = netkit_set_macaddr, .ndo_get_iflink = netkit_get_iflink, .ndo_get_peer_dev = netkit_peer_dev, .ndo_get_stats64 = netkit_get_stats, .ndo_uninit = netkit_uninit, .ndo_bpf = netkit_xsk, .ndo_xsk_wakeup = netkit_xsk_wakeup, .ndo_features_check = passthru_features_check,}; static void netkit_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info){ strscpy(info->driver, NETKIT_DRV_NAME, sizeof(info->driver));} static const struct ethtool_ops netkit_ethtool_ops = { .get_drvinfo = netkit_get_drvinfo,}; static int netkit_queue_create(struct net_device *dev, struct netlink_ext_ack *extack){ struct netkit *nk = netkit_priv(dev); u32 rxq_count_old, rxq_count_new; int err; rxq_count_old = dev->real_num_rx_queues; rxq_count_new = rxq_count_old + 1; /* In paired mode, only the non-primary (peer) device can * create leased queues since the primary is the management * side. In single device mode, leasing is always allowed. */ if (nk->pair == NETKIT_DEVICE_PAIR && nk->primary) { NL_SET_ERR_MSG(extack, "netkit can only lease against the peer device"); return -EOPNOTSUPP; } err = netif_set_real_num_rx_queues(dev, rxq_count_new); if (err) { if (rxq_count_new > dev->num_rx_queues) NL_SET_ERR_MSG(extack, "netkit maximum queue limit reached"); else NL_SET_ERR_MSG_FMT(extack, "netkit cannot create more queues err=%d", err); return err; } return rxq_count_old;} static const struct netdev_queue_mgmt_ops netkit_queue_mgmt_ops = { .ndo_queue_create = netkit_queue_create,}; static struct net_device *netkit_alloc(struct nlattr *tb[], const char *ifname, unsigned char name_assign_type, unsigned int num_tx_queues, unsigned int num_rx_queues){ const struct rtnl_link_ops *ops = &netkit_link_ops; struct net_device *dev; if (num_tx_queues > NETKIT_NUM_TX_QUEUES_MAX || num_rx_queues > NETKIT_NUM_RX_QUEUES_MAX) return ERR_PTR(-EOPNOTSUPP); dev = alloc_netdev_mqs(ops->priv_size, ifname, name_assign_type, ops->setup, num_tx_queues, num_rx_queues); if (dev) { dev->real_num_tx_queues = NETKIT_NUM_TX_QUEUES_REAL; dev->real_num_rx_queues = NETKIT_NUM_RX_QUEUES_REAL; } return dev;} static void netkit_queue_unlease(struct net_device *dev){ struct netdev_rx_queue *rxq, *rxq_lease; struct net_device *dev_lease; int i; if (dev->real_num_rx_queues == 1) return; netdev_lock(dev); for (i = 1; i < dev->real_num_rx_queues; i++) { rxq = __netif_get_rx_queue(dev, i); rxq_lease = rxq->lease; dev_lease = rxq_lease->dev; netdev_lock(dev_lease); netdev_rx_queue_unlease(rxq, rxq_lease); netdev_unlock(dev_lease); } netdev_unlock(dev);} static void netkit_setup(struct net_device *dev){ static const netdev_features_t netkit_features_hw_vlan = NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX; static const netdev_features_t netkit_features = netkit_features_hw_vlan | NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL; ether_setup(dev); dev->max_mtu = ETH_MAX_MTU; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; dev->flags |= IFF_NOARP; dev->priv_flags &= ~IFF_TX_SKB_SHARING; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; dev->priv_flags |= IFF_PHONY_HEADROOM; dev->priv_flags |= IFF_NO_QUEUE; dev->priv_flags |= IFF_DISABLE_NETPOLL; dev->lltx = true; dev->netmem_tx = NETMEM_TX_NO_DMA; dev->netdev_ops = &netkit_netdev_ops; dev->ethtool_ops = &netkit_ethtool_ops; dev->queue_mgmt_ops = &netkit_queue_mgmt_ops; dev->features |= netkit_features; dev->hw_features = netkit_features; dev->hw_enc_features = netkit_features; dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; dev->vlan_features = dev->features & ~netkit_features_hw_vlan; dev->needs_free_netdev = true; netif_set_tso_max_size(dev, GSO_MAX_SIZE);} static struct net *netkit_get_link_net(const struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer); return peer ? dev_net(peer) : dev_net(dev);} static int netkit_check_policy(int policy, struct nlattr *tb, struct netlink_ext_ack *extack){ switch (policy) { case NETKIT_PASS: case NETKIT_DROP:Lines 1–500 of 1308. Load more
// SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2023 Isovalent */ #include <linux/netdevice.h> #include <linux/ethtool.h> #include <linux/etherdevice.h> #include <linux/filter.h> #include <linux/netfilter_netdev.h> #include <linux/bpf_mprog.h> #include <linux/indirect_call_wrapper.h> #include <net/netdev_lock.h> #include <net/netdev_queues.h> #include <net/netdev_rx_queue.h> #include <net/xdp_sock_drv.h> #include <net/netkit.h> #include <net/dst.h> #include <net/tcx.h> #define NETKIT_DRV_NAME "netkit" #define NETKIT_NUM_RX_QUEUES_MAX 1024 #define NETKIT_NUM_TX_QUEUES_MAX 1 #define NETKIT_NUM_RX_QUEUES_REAL 1 #define NETKIT_NUM_TX_QUEUES_REAL 1 struct netkit { __cacheline_group_begin(netkit_fastpath); struct net_device __rcu *peer; struct bpf_mprog_entry __rcu *active; enum netkit_action policy; enum netkit_scrub scrub; struct bpf_mprog_bundle bundle; __cacheline_group_end(netkit_fastpath); __cacheline_group_begin(netkit_slowpath); enum netkit_mode mode; enum netkit_pairing pair; bool primary; u32 headroom; __cacheline_group_end(netkit_slowpath); }; struct netkit_link { struct bpf_link link; struct net_device *dev; }; static struct rtnl_link_ops netkit_link_ops; static __always_inline int netkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, enum netkit_action ret) { const struct bpf_mprog_fp *fp; const struct bpf_prog *prog; bpf_mprog_foreach_prog(entry, fp, prog) { bpf_compute_data_pointers(skb); ret = bpf_prog_run(prog, skb); if (ret != NETKIT_NEXT) break; } return ret; } static void netkit_xnet(struct sk_buff *skb) { skb->priority = 0; skb->mark = 0; } static void netkit_prep_forward(struct sk_buff *skb, bool xnet, bool xnet_scrub) { skb_scrub_packet(skb, false); nf_skip_egress(skb, true); skb_reset_mac_header(skb); if (!xnet) return; skb_clear_tstamp(skb); if (xnet_scrub) netkit_xnet(skb); } static struct netkit *netkit_priv(const struct net_device *dev) { return netdev_priv(dev); } static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev) { struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; struct netkit *nk = netkit_priv(dev); enum netkit_action ret = READ_ONCE(nk->policy); netdev_tx_t ret_dev = NET_XMIT_SUCCESS; const struct bpf_mprog_entry *entry; struct net_device *peer; int len = skb->len; bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); rcu_read_lock(); peer = rcu_dereference(nk->peer); if (unlikely(!peer || !(peer->flags & IFF_UP) || !pskb_may_pull(skb, ETH_HLEN) || skb_orphan_frags(skb, GFP_ATOMIC))) goto drop; netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)), nk->scrub); eth_skb_pkt_type(skb, peer); skb->dev = peer; entry = rcu_dereference(nk->active); if (entry) ret = netkit_run(entry, skb, ret); switch (ret) { case NETKIT_NEXT: case NETKIT_PASS: eth_skb_pull_mac(skb); skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) { dev_sw_netstats_tx_add(dev, 1, len); dev_sw_netstats_rx_add(peer, len); } else { goto drop_stats; } break; case NETKIT_REDIRECT: dev_sw_netstats_tx_add(dev, 1, len); skb_do_redirect(skb); break; case NETKIT_DROP: default: drop: kfree_skb(skb); drop_stats: dev_core_stats_tx_dropped_inc(dev); ret_dev = NET_XMIT_DROP; break; } rcu_read_unlock(); bpf_net_ctx_clear(bpf_net_ctx); return ret_dev; } static int netkit_open(struct net_device *dev) { struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer); if (nk->pair == NETKIT_DEVICE_SINGLE) { netif_carrier_on(dev); return 0; } if (!peer) return -ENOTCONN; if (peer->flags & IFF_UP) { netif_carrier_on(dev); netif_carrier_on(peer); } return 0; } static int netkit_close(struct net_device *dev) { struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer); netif_carrier_off(dev); if (peer) netif_carrier_off(peer); return 0; } static int netkit_get_iflink(const struct net_device *dev) { struct netkit *nk = netkit_priv(dev); struct net_device *peer; int iflink = 0; rcu_read_lock(); peer = rcu_dereference(nk->peer); if (peer) iflink = READ_ONCE(peer->ifindex); rcu_read_unlock(); return iflink; } static int netkit_set_multicast(struct net_device *dev, struct netdev_hw_addr_list *uc, struct netdev_hw_addr_list *mc) { /* Nothing to do, we receive whatever gets pushed to us! */ return 0; } static int netkit_set_macaddr(struct net_device *dev, void *sa) { struct netkit *nk = netkit_priv(dev); if (nk->mode != NETKIT_L2) return -EOPNOTSUPP; return eth_mac_addr(dev, sa); } static void netkit_set_headroom(struct net_device *dev, int headroom) { struct netkit *nk = netkit_priv(dev), *nk2; struct net_device *peer; if (headroom < 0) headroom = NET_SKB_PAD; rcu_read_lock(); peer = rcu_dereference(nk->peer); if (!peer) { nk->headroom = headroom; dev->needed_headroom = headroom; } else { nk2 = netkit_priv(peer); nk->headroom = headroom; headroom = max(nk->headroom, nk2->headroom); peer->needed_headroom = headroom; dev->needed_headroom = headroom; } rcu_read_unlock(); } INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev) { return rcu_dereference(netkit_priv(dev)->peer); } static void netkit_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) { dev_fetch_sw_netstats(stats, dev->tstats); stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped); } static bool netkit_xsk_supported_at_phys(const struct net_device *dev) { if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xdp_xmit || !dev->netdev_ops->ndo_xsk_wakeup) return false; return true; } static int netkit_xsk(struct net_device *dev, struct netdev_bpf *xdp) { struct netkit *nk = netkit_priv(dev); struct netdev_bpf xdp_lower; struct netdev_rx_queue *rxq; struct net_device *phys; bool create = false; int ret = -EBUSY; switch (xdp->command) { case XDP_SETUP_XSK_POOL: if (nk->pair == NETKIT_DEVICE_PAIR) return -EOPNOTSUPP; if (xdp->xsk.queue_id >= dev->real_num_rx_queues) return -EINVAL; rxq = __netif_get_rx_queue(dev, xdp->xsk.queue_id); if (!rxq->lease) return -EOPNOTSUPP; phys = rxq->lease->dev; if (!netkit_xsk_supported_at_phys(phys)) return -EOPNOTSUPP; create = xdp->xsk.pool; memcpy(&xdp_lower, xdp, sizeof(xdp_lower)); xdp_lower.xsk.queue_id = get_netdev_rx_queue_index(rxq->lease); break; case XDP_SETUP_PROG: return -EOPNOTSUPP; default: return -EINVAL; } netdev_lock(phys); if (create && (phys->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) { ret = -EOPNOTSUPP; goto out; } if (!create || !dev_get_min_mp_channel_count(phys)) ret = phys->netdev_ops->ndo_bpf(phys, &xdp_lower); out: netdev_unlock(phys); return ret; } static int netkit_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) { struct netdev_rx_queue *rxq, *rxq_lease; struct net_device *phys; if (queue_id >= dev->real_num_rx_queues) return -EINVAL; rxq = __netif_get_rx_queue(dev, queue_id); rxq_lease = READ_ONCE(rxq->lease); if (unlikely(!rxq_lease)) return -EOPNOTSUPP; /* netkit_xsk already validated full xsk support, hence it's * fine to call into ndo_xsk_wakeup right away given this * was a prerequisite to get here in the first place. The * phys xsk support cannot change without tearing down the * device (which clears the lease first). */ phys = rxq_lease->dev; return phys->netdev_ops->ndo_xsk_wakeup(phys, get_netdev_rx_queue_index(rxq_lease), flags); } static int netkit_init(struct net_device *dev) { netdev_lockdep_set_classes(dev); return 0; } static void netkit_uninit(struct net_device *dev); static const struct net_device_ops netkit_netdev_ops = { .ndo_init = netkit_init, .ndo_open = netkit_open, .ndo_stop = netkit_close, .ndo_start_xmit = netkit_xmit, .ndo_set_rx_mode_async = netkit_set_multicast, .ndo_set_rx_headroom = netkit_set_headroom, .ndo_set_mac_address = netkit_set_macaddr, .ndo_get_iflink = netkit_get_iflink, .ndo_get_peer_dev = netkit_peer_dev, .ndo_get_stats64 = netkit_get_stats, .ndo_uninit = netkit_uninit, .ndo_bpf = netkit_xsk, .ndo_xsk_wakeup = netkit_xsk_wakeup, .ndo_features_check = passthru_features_check, }; static void netkit_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { strscpy(info->driver, NETKIT_DRV_NAME, sizeof(info->driver)); } static const struct ethtool_ops netkit_ethtool_ops = { .get_drvinfo = netkit_get_drvinfo, }; static int netkit_queue_create(struct net_device *dev, struct netlink_ext_ack *extack) { struct netkit *nk = netkit_priv(dev); u32 rxq_count_old, rxq_count_new; int err; rxq_count_old = dev->real_num_rx_queues; rxq_count_new = rxq_count_old + 1; /* In paired mode, only the non-primary (peer) device can * create leased queues since the primary is the management * side. In single device mode, leasing is always allowed. */ if (nk->pair == NETKIT_DEVICE_PAIR && nk->primary) { NL_SET_ERR_MSG(extack, "netkit can only lease against the peer device"); return -EOPNOTSUPP; } err = netif_set_real_num_rx_queues(dev, rxq_count_new); if (err) { if (rxq_count_new > dev->num_rx_queues) NL_SET_ERR_MSG(extack, "netkit maximum queue limit reached"); else NL_SET_ERR_MSG_FMT(extack, "netkit cannot create more queues err=%d", err); return err; } return rxq_count_old; } static const struct netdev_queue_mgmt_ops netkit_queue_mgmt_ops = { .ndo_queue_create = netkit_queue_create, }; static struct net_device *netkit_alloc(struct nlattr *tb[], const char *ifname, unsigned char name_assign_type, unsigned int num_tx_queues, unsigned int num_rx_queues) { const struct rtnl_link_ops *ops = &netkit_link_ops; struct net_device *dev; if (num_tx_queues > NETKIT_NUM_TX_QUEUES_MAX || num_rx_queues > NETKIT_NUM_RX_QUEUES_MAX) return ERR_PTR(-EOPNOTSUPP); dev = alloc_netdev_mqs(ops->priv_size, ifname, name_assign_type, ops->setup, num_tx_queues, num_rx_queues); if (dev) { dev->real_num_tx_queues = NETKIT_NUM_TX_QUEUES_REAL; dev->real_num_rx_queues = NETKIT_NUM_RX_QUEUES_REAL; } return dev; } static void netkit_queue_unlease(struct net_device *dev) { struct netdev_rx_queue *rxq, *rxq_lease; struct net_device *dev_lease; int i; if (dev->real_num_rx_queues == 1) return; netdev_lock(dev); for (i = 1; i < dev->real_num_rx_queues; i++) { rxq = __netif_get_rx_queue(dev, i); rxq_lease = rxq->lease; dev_lease = rxq_lease->dev; netdev_lock(dev_lease); netdev_rx_queue_unlease(rxq, rxq_lease); netdev_unlock(dev_lease); } netdev_unlock(dev); } static void netkit_setup(struct net_device *dev) { static const netdev_features_t netkit_features_hw_vlan = NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX; static const netdev_features_t netkit_features = netkit_features_hw_vlan | NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL; ether_setup(dev); dev->max_mtu = ETH_MAX_MTU; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; dev->flags |= IFF_NOARP; dev->priv_flags &= ~IFF_TX_SKB_SHARING; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; dev->priv_flags |= IFF_PHONY_HEADROOM; dev->priv_flags |= IFF_NO_QUEUE; dev->priv_flags |= IFF_DISABLE_NETPOLL; dev->lltx = true; dev->netmem_tx = NETMEM_TX_NO_DMA; dev->netdev_ops = &netkit_netdev_ops; dev->ethtool_ops = &netkit_ethtool_ops; dev->queue_mgmt_ops = &netkit_queue_mgmt_ops; dev->features |= netkit_features; dev->hw_features = netkit_features; dev->hw_enc_features = netkit_features; dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; dev->vlan_features = dev->features & ~netkit_features_hw_vlan; dev->needs_free_netdev = true; netif_set_tso_max_size(dev, GSO_MAX_SIZE); } static struct net *netkit_get_link_net(const struct net_device *dev) { struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer); return peer ? dev_net(peer) : dev_net(dev); } static int netkit_check_policy(int policy, struct nlattr *tb, struct netlink_ext_ack *extack) { switch (policy) { case NETKIT_PASS: case NETKIT_DROP: