7321 lines · 190.1 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0-or-later/* A network driver using virtio. * * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation *///#define DEBUG#include <linux/netdevice.h>#include <linux/etherdevice.h>#include <linux/ethtool.h>#include <linux/module.h>#include <linux/virtio.h>#include <linux/virtio_net.h>#include <linux/bpf.h>#include <linux/bpf_trace.h>#include <linux/scatterlist.h>#include <linux/if_vlan.h>#include <linux/slab.h>#include <linux/cpu.h>#include <linux/average.h>#include <linux/filter.h>#include <linux/kernel.h>#include <linux/dim.h>#include <net/route.h>#include <net/xdp.h>#include <net/net_failover.h>#include <net/netdev_rx_queue.h>#include <net/netdev_queues.h>#include <net/xdp_sock_drv.h>#include <net/page_pool/helpers.h> static int napi_weight = NAPI_POLL_WEIGHT;module_param(napi_weight, int, 0444); static bool csum = true, gso = true, napi_tx = true;module_param(csum, bool, 0444);module_param(gso, bool, 0444);module_param(napi_tx, bool, 0644); #define VIRTIO_OFFLOAD_MAP_MIN	46#define VIRTIO_OFFLOAD_MAP_MAX	47#define VIRTIO_FEATURES_MAP_MIN	65#define VIRTIO_O2F_DELTA	(VIRTIO_FEATURES_MAP_MIN - \				 VIRTIO_OFFLOAD_MAP_MIN) static bool virtio_is_mapped_offload(unsigned int obit){	return obit >= VIRTIO_OFFLOAD_MAP_MIN &&	       obit <= VIRTIO_OFFLOAD_MAP_MAX;} static unsigned int virtio_offload_to_feature(unsigned int obit){	return virtio_is_mapped_offload(obit) ? obit + VIRTIO_O2F_DELTA : obit;} /* FIXME: MTU in config. */#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)#define GOOD_COPY_LEN	128 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) /* Separating two types of XDP xmit */#define VIRTIO_XDP_TX		BIT(0)#define VIRTIO_XDP_REDIR	BIT(1) /* RX packet size EWMA. The average packet size is used to determine the packet * buffer size when refilling RX rings. As the entire RX ring may be refilled * at once, the weight is chosen so that the EWMA will be insensitive to short- * term, transient changes in packet size. */DECLARE_EWMA(pkt_len, 0, 64) #define VIRTNET_DRIVER_VERSION "1.0.0" static const unsigned long guest_offloads[] = {	VIRTIO_NET_F_GUEST_TSO4,	VIRTIO_NET_F_GUEST_TSO6,	VIRTIO_NET_F_GUEST_ECN,	VIRTIO_NET_F_GUEST_UFO,	VIRTIO_NET_F_GUEST_CSUM,	VIRTIO_NET_F_GUEST_USO4,	VIRTIO_NET_F_GUEST_USO6,	VIRTIO_NET_F_GUEST_HDRLEN,	VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_MAPPED,	VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM_MAPPED,}; #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \			(1ULL << VIRTIO_NET_F_GUEST_TSO6) | \			(1ULL << VIRTIO_NET_F_GUEST_ECN)  | \			(1ULL << VIRTIO_NET_F_GUEST_UFO)  | \			(1ULL << VIRTIO_NET_F_GUEST_USO4) | \			(1ULL << VIRTIO_NET_F_GUEST_USO6) | \			(1ULL << VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_MAPPED) | \			(1ULL << VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM_MAPPED)) struct virtnet_stat_desc {	char desc[ETH_GSTRING_LEN];	size_t offset;	size_t qstat_offset;}; struct virtnet_sq_free_stats {	u64 packets;	u64 bytes;	u64 napi_packets;	u64 napi_bytes;	u64 xsk;}; struct virtnet_sq_stats {	struct u64_stats_sync syncp;	u64_stats_t packets;	u64_stats_t bytes;	u64_stats_t xdp_tx;	u64_stats_t xdp_tx_drops;	u64_stats_t kicks;	u64_stats_t tx_timeouts;	u64_stats_t stop;	u64_stats_t wake;}; struct virtnet_rq_stats {	struct u64_stats_sync syncp;	u64_stats_t packets;	u64_stats_t bytes;	u64_stats_t drops;	u64_stats_t xdp_packets;	u64_stats_t xdp_tx;	u64_stats_t xdp_redirects;	u64_stats_t xdp_drops;	u64_stats_t kicks;}; #define VIRTNET_SQ_STAT(name, m) {name, offsetof(struct virtnet_sq_stats, m), -1}#define VIRTNET_RQ_STAT(name, m) {name, offsetof(struct virtnet_rq_stats, m), -1} #define VIRTNET_SQ_STAT_QSTAT(name, m)				\	{							\		name,						\		offsetof(struct virtnet_sq_stats, m),		\		offsetof(struct netdev_queue_stats_tx, m),	\	} #define VIRTNET_RQ_STAT_QSTAT(name, m)				\	{							\		name,						\		offsetof(struct virtnet_rq_stats, m),		\		offsetof(struct netdev_queue_stats_rx, m),	\	} static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {	VIRTNET_SQ_STAT("xdp_tx",       xdp_tx),	VIRTNET_SQ_STAT("xdp_tx_drops", xdp_tx_drops),	VIRTNET_SQ_STAT("kicks",        kicks),	VIRTNET_SQ_STAT("tx_timeouts",  tx_timeouts),}; static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {	VIRTNET_RQ_STAT("drops",         drops),	VIRTNET_RQ_STAT("xdp_packets",   xdp_packets),	VIRTNET_RQ_STAT("xdp_tx",        xdp_tx),	VIRTNET_RQ_STAT("xdp_redirects", xdp_redirects),	VIRTNET_RQ_STAT("xdp_drops",     xdp_drops),	VIRTNET_RQ_STAT("kicks",         kicks),}; static const struct virtnet_stat_desc virtnet_sq_stats_desc_qstat[] = {	VIRTNET_SQ_STAT_QSTAT("packets", packets),	VIRTNET_SQ_STAT_QSTAT("bytes",   bytes),	VIRTNET_SQ_STAT_QSTAT("stop",	 stop),	VIRTNET_SQ_STAT_QSTAT("wake",	 wake),}; static const struct virtnet_stat_desc virtnet_rq_stats_desc_qstat[] = {	VIRTNET_RQ_STAT_QSTAT("packets", packets),	VIRTNET_RQ_STAT_QSTAT("bytes",   bytes),}; #define VIRTNET_STATS_DESC_CQ(name) \	{#name, offsetof(struct virtio_net_stats_cvq, name), -1} #define VIRTNET_STATS_DESC_RX(class, name) \	{#name, offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name), -1} #define VIRTNET_STATS_DESC_TX(class, name) \	{#name, offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name), -1}  static const struct virtnet_stat_desc virtnet_stats_cvq_desc[] = {	VIRTNET_STATS_DESC_CQ(command_num),	VIRTNET_STATS_DESC_CQ(ok_num),}; static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc[] = {	VIRTNET_STATS_DESC_RX(basic, packets),	VIRTNET_STATS_DESC_RX(basic, bytes), 	VIRTNET_STATS_DESC_RX(basic, notifications),	VIRTNET_STATS_DESC_RX(basic, interrupts),}; static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc[] = {	VIRTNET_STATS_DESC_TX(basic, packets),	VIRTNET_STATS_DESC_TX(basic, bytes), 	VIRTNET_STATS_DESC_TX(basic, notifications),	VIRTNET_STATS_DESC_TX(basic, interrupts),}; static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc[] = {	VIRTNET_STATS_DESC_RX(csum, needs_csum),}; static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc[] = {	VIRTNET_STATS_DESC_TX(gso, gso_packets_noseg),	VIRTNET_STATS_DESC_TX(gso, gso_bytes_noseg),}; static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc[] = {	VIRTNET_STATS_DESC_RX(speed, ratelimit_bytes),}; static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc[] = {	VIRTNET_STATS_DESC_TX(speed, ratelimit_bytes),}; #define VIRTNET_STATS_DESC_RX_QSTAT(class, name, qstat_field)			\	{									\		#name,								\		offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name),	\		offsetof(struct netdev_queue_stats_rx, qstat_field),		\	} #define VIRTNET_STATS_DESC_TX_QSTAT(class, name, qstat_field)			\	{									\		#name,								\		offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name),	\		offsetof(struct netdev_queue_stats_tx, qstat_field),		\	} static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc_qstat[] = {	VIRTNET_STATS_DESC_RX_QSTAT(basic, drops,         hw_drops),	VIRTNET_STATS_DESC_RX_QSTAT(basic, drop_overruns, hw_drop_overruns),}; static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc_qstat[] = {	VIRTNET_STATS_DESC_TX_QSTAT(basic, drops,          hw_drops),	VIRTNET_STATS_DESC_TX_QSTAT(basic, drop_malformed, hw_drop_errors),}; static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc_qstat[] = {	VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_valid, csum_unnecessary),	VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_none,  csum_none),	VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_bad,   csum_bad),}; static const struct virtnet_stat_desc virtnet_stats_tx_csum_desc_qstat[] = {	VIRTNET_STATS_DESC_TX_QSTAT(csum, csum_none,  csum_none),	VIRTNET_STATS_DESC_TX_QSTAT(csum, needs_csum, needs_csum),}; static const struct virtnet_stat_desc virtnet_stats_rx_gso_desc_qstat[] = {	VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets,           hw_gro_packets),	VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes,             hw_gro_bytes),	VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets_coalesced, hw_gro_wire_packets),	VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes_coalesced,   hw_gro_wire_bytes),}; static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc_qstat[] = {	VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_packets,        hw_gso_packets),	VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_bytes,          hw_gso_bytes),	VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments,       hw_gso_wire_packets),	VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments_bytes, hw_gso_wire_bytes),}; static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc_qstat[] = {	VIRTNET_STATS_DESC_RX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits),}; static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc_qstat[] = {	VIRTNET_STATS_DESC_TX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits),}; #define VIRTNET_Q_TYPE_RX 0#define VIRTNET_Q_TYPE_TX 1#define VIRTNET_Q_TYPE_CQ 2 struct virtnet_interrupt_coalesce {	u32 max_packets;	u32 max_usecs;}; /* Internal representation of a send virtqueue */struct send_queue {	/* Virtqueue associated with this send _queue */	struct virtqueue *vq; 	/* TX: fragments + linear part + virtio header */	struct scatterlist sg[MAX_SKB_FRAGS + 2]; 	/* Name of the send queue: output.$index */	char name[16]; 	struct virtnet_sq_stats stats; 	struct virtnet_interrupt_coalesce intr_coal; 	struct napi_struct napi; 	/* Record whether sq is in reset state. */	bool reset; 	struct xsk_buff_pool *xsk_pool; 	dma_addr_t xsk_hdr_dma_addr;}; /* Internal representation of a receive virtqueue */struct receive_queue {	/* Virtqueue associated with this receive_queue */	struct virtqueue *vq; 	struct napi_struct napi; 	struct bpf_prog __rcu *xdp_prog; 	struct virtnet_rq_stats stats; 	/* The number of rx notifications */	u16 calls; 	/* Is dynamic interrupt moderation enabled? */	bool dim_enabled; 	/* Used to protect dim_enabled and inter_coal */	struct mutex dim_lock; 	/* Dynamic Interrupt Moderation */	struct dim dim; 	u32 packets_in_napi; 	struct virtnet_interrupt_coalesce intr_coal; 	/* Chain pages by the private ptr. */	struct page *pages; 	/* Average packet length for mergeable receive buffers. */	struct ewma_pkt_len mrg_avg_pkt_len; 	struct page_pool *page_pool; 	/* True if page_pool handles DMA mapping via PP_FLAG_DMA_MAP */	bool use_page_pool_dma; 	/* RX: fragments + linear part + virtio header */	struct scatterlist sg[MAX_SKB_FRAGS + 2]; 	/* Min single buffer size for mergeable buffers case. */	unsigned int min_buf_len; 	/* Name of this receive queue: input.$index */	char name[16]; 	struct xdp_rxq_info xdp_rxq; 	struct xsk_buff_pool *xsk_pool; 	/* xdp rxq used by xsk */	struct xdp_rxq_info xsk_rxq_info; 	struct xdp_buff **xsk_buffs;}; /* Control VQ buffers: protected by the rtnl lock */struct control_buf {	struct virtio_net_ctrl_hdr hdr;	virtio_net_ctrl_ack status;}; struct virtnet_info {	struct virtio_device *vdev;	struct virtqueue *cvq;	struct net_device *dev;	struct send_queue *sq;	struct receive_queue *rq;	unsigned int status; 	/* Max # of queue pairs supported by the device */	u16 max_queue_pairs; 	/* # of queue pairs currently used by the driver */	u16 curr_queue_pairs; 	/* # of XDP queue pairs currently used by the driver */	u16 xdp_queue_pairs; 	/* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */	bool xdp_enabled; 	/* I like... big packets and I cannot lie! */	bool big_packets; 	/* number of sg entries allocated for big packets */	unsigned int big_packets_num_skbfrags; 	/* Host will merge rx buffers for big packets (shake it! shake it!) */	bool mergeable_rx_bufs; 	/* Host supports rss and/or hash report */	bool has_rss;	bool has_rss_hash_report;	u8 rss_key_size;	u16 rss_indir_table_size;	u32 rss_hash_types_supported;	u32 rss_hash_types_saved; 	/* Has control virtqueue */	bool has_cvq; 	/* Lock to protect the control VQ */	struct mutex cvq_lock; 	/* Host can handle any s/g split between our header and packet data */	bool any_header_sg; 	/* Packet virtio header size */	u8 hdr_len; 	/* UDP tunnel support */	bool tx_tnl; 	bool rx_tnl; 	bool rx_tnl_csum; 	/* Work struct for config space updates */	struct work_struct config_work; 	/* Work struct for setting rx mode */	struct work_struct rx_mode_work; 	/* OK to queue work setting RX mode? */	bool rx_mode_work_enabled; 	/* Does the affinity hint is set for virtqueues? */	bool affinity_hint_set; 	/* CPU hotplug instances for online & dead */	struct hlist_node node;	struct hlist_node node_dead; 	struct control_buf *ctrl; 	/* Ethtool settings */	u8 duplex;	u32 speed; 	/* Is rx dynamic interrupt moderation enabled? */	bool rx_dim_enabled; 	/* Interrupt coalescing settings */	struct virtnet_interrupt_coalesce intr_coal_tx;	struct virtnet_interrupt_coalesce intr_coal_rx; 	unsigned long guest_offloads;	unsigned long guest_offloads_capable; 	/* failover when STANDBY feature enabled */	struct failover *failover; 	u64 device_stats_cap; 	struct virtio_net_rss_config_hdr *rss_hdr; 	/* Must be last as it ends in a flexible-array member. */	TRAILING_OVERLAP(struct virtio_net_rss_config_trailer, rss_trailer, hash_key_data,		u8 rss_hash_key_data[NETDEV_RSS_KEY_LEN];	);};static_assert(offsetof(struct virtnet_info, rss_trailer.hash_key_data) ==	      offsetof(struct virtnet_info, rss_hash_key_data)); struct padded_vnet_hdr {	struct virtio_net_hdr_v1_hash hdr;	/*	 * hdr is in a separate sg buffer, and data sg buffer shares same page	 * with this header sg. This padding makes next sg 16 byte aligned	 * after the header.	 */	char padding[12];}; struct virtio_net_common_hdr {	union {		struct virtio_net_hdr hdr;		struct virtio_net_hdr_mrg_rxbuf	mrg_hdr;		struct virtio_net_hdr_v1_hash hash_v1_hdr;		struct virtio_net_hdr_v1_hash_tunnel tnl_hdr;

Lines 1–500 of 7321.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t