3696 lines · 85.4 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0-or-later/* *  TUN - Universal TUN/TAP device driver. *  Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com> * *  $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $ */ /* *  Changes: * *  Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14 *    Add TUNSETLINK ioctl to set the link encapsulation * *  Mark Smith <markzzzsmith@yahoo.com.au> *    Use eth_random_addr() for tap MAC address. * *  Harald Roelle <harald.roelle@ifi.lmu.de>  2004/04/20 *    Fixes in packet dropping, queue length setting and queue wakeup. *    Increased default tx queue length. *    Added ethtool API. *    Minor cleanups * *  Daniel Podlejski <underley@underley.eu.org> *    Modifications for 2.3.99-pre5 kernel. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #define DRV_NAME	"tun"#define DRV_VERSION	"1.6"#define DRV_DESCRIPTION	"Universal TUN/TAP device driver"#define DRV_COPYRIGHT	"(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>" #include <linux/module.h>#include <linux/errno.h>#include <linux/kernel.h>#include <linux/sched/signal.h>#include <linux/major.h>#include <linux/slab.h>#include <linux/poll.h>#include <linux/fcntl.h>#include <linux/init.h>#include <linux/skbuff.h>#include <linux/netdevice.h>#include <linux/etherdevice.h>#include <linux/miscdevice.h>#include <linux/ethtool.h>#include <linux/rtnetlink.h>#include <linux/compat.h>#include <linux/if.h>#include <linux/if_arp.h>#include <linux/if_ether.h>#include <linux/if_tun.h>#include <linux/if_vlan.h>#include <linux/crc32.h>#include <linux/nsproxy.h>#include <linux/virtio_net.h>#include <linux/rcupdate.h>#include <net/net_namespace.h>#include <net/netns/generic.h>#include <net/rtnetlink.h>#include <net/sock.h>#include <net/xdp.h>#include <net/ip_tunnels.h>#include <linux/seq_file.h>#include <linux/uio.h>#include <linux/skb_array.h>#include <linux/bpf.h>#include <linux/bpf_trace.h>#include <linux/mutex.h> #include <linux/uaccess.h>#include <linux/proc_fs.h> static void tun_default_link_ksettings(struct net_device *dev,				       struct ethtool_link_ksettings *cmd); #define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) /* TUN device flags */ /* IFF_ATTACH_QUEUE is never stored in device flags, * overload it to mean fasync when stored there. */#define TUN_FASYNC	IFF_ATTACH_QUEUE/* High bits in flags field are unused. */#define TUN_VNET_LE     0x80000000#define TUN_VNET_BE     0x40000000 #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \		      IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS) #define GOODCOPY_LEN 128 #define FLT_EXACT_COUNT 8struct tap_filter {	unsigned int    count;    /* Number of addrs. Zero means disabled */	u32             mask[2];  /* Mask of the hashed addrs */	unsigned char	addr[FLT_EXACT_COUNT][ETH_ALEN];}; /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal * to max number of VCPUs in guest. */#define MAX_TAP_QUEUES 256#define MAX_TAP_FLOWS  4096 #define TUN_FLOW_EXPIRE (3 * HZ) struct tun_pcpu_stats {	u64_stats_t rx_packets;	u64_stats_t rx_bytes;	u64_stats_t tx_packets;	u64_stats_t tx_bytes;	struct u64_stats_sync syncp;	u32 rx_dropped;	u32 tx_dropped;	u32 rx_frame_errors;}; /* A tun_file connects an open character device to a tuntap netdevice. It * also contains all socket related structures (except sock_fprog and tap_filter) * to serve as one transmit queue for tuntap device. The sock_fprog and * tap_filter were kept in tun_struct since they were used for filtering for the * netdevice not for a specific queue (at least I didn't see the requirement for * this). * * RCU usage: * The tun_file and tun_struct are loosely coupled, the pointer from one to the * other can only be read while rcu_read_lock or rtnl_lock is held. */struct tun_file {	struct sock sk;	struct socket socket;	struct tun_struct __rcu *tun;	struct fasync_struct *fasync;	/* only used for fasnyc */	unsigned int flags;	union {		u16 queue_index;		unsigned int ifindex;	};	struct napi_struct napi;	bool napi_enabled;	bool napi_frags_enabled;	struct mutex napi_mutex;	/* Protects access to the above napi */	struct list_head next;	struct tun_struct *detached;	struct ptr_ring tx_ring;	struct xdp_rxq_info xdp_rxq;}; struct tun_page {	struct page *page;	int count;}; struct tun_flow_entry {	struct hlist_node hash_link;	struct rcu_head rcu;	struct tun_struct *tun; 	u32 rxhash;	u32 rps_rxhash;	int queue_index;	unsigned long updated ____cacheline_aligned_in_smp;}; #define TUN_NUM_FLOW_ENTRIES 1024#define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1) struct tun_prog {	struct rcu_head rcu;	struct bpf_prog *prog;}; /* Since the socket were moved to tun_file, to preserve the behavior of persist * device, socket filter, sndbuf and vnet header size were restore when the * file were attached to a persist device. */struct tun_struct {	struct tun_file __rcu	*tfiles[MAX_TAP_QUEUES];	unsigned int            numqueues;	unsigned int 		flags;	kuid_t			owner;	kgid_t			group; 	struct net_device	*dev;	netdev_features_t	set_features;#define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \			  NETIF_F_TSO6) 	int			align;	int			vnet_hdr_sz;	int			sndbuf;	struct tap_filter	txflt;	struct sock_fprog	fprog;	/* protected by rtnl lock */	bool			filter_attached;	u32			msg_enable;	spinlock_t lock;	struct hlist_head flows[TUN_NUM_FLOW_ENTRIES];	struct timer_list flow_gc_timer;	unsigned long ageing_time;	unsigned int numdisabled;	struct list_head disabled;	void *security;	u32 flow_count;	u32 rx_batched;	struct tun_pcpu_stats __percpu *pcpu_stats;	struct bpf_prog __rcu *xdp_prog;	struct tun_prog __rcu *steering_prog;	struct tun_prog __rcu *filter_prog;	struct ethtool_link_ksettings link_ksettings;}; struct veth {	__be16 h_vlan_proto;	__be16 h_vlan_TCI;}; bool tun_is_xdp_frame(void *ptr){	return (unsigned long)ptr & TUN_XDP_FLAG;}EXPORT_SYMBOL(tun_is_xdp_frame); void *tun_xdp_to_ptr(void *ptr){	return (void *)((unsigned long)ptr | TUN_XDP_FLAG);}EXPORT_SYMBOL(tun_xdp_to_ptr); void *tun_ptr_to_xdp(void *ptr){	return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG);}EXPORT_SYMBOL(tun_ptr_to_xdp); static int tun_napi_receive(struct napi_struct *napi, int budget){	struct tun_file *tfile = container_of(napi, struct tun_file, napi);	struct sk_buff_head *queue = &tfile->sk.sk_write_queue;	struct sk_buff_head process_queue;	struct sk_buff *skb;	int received = 0; 	__skb_queue_head_init(&process_queue); 	spin_lock(&queue->lock);	skb_queue_splice_tail_init(queue, &process_queue);	spin_unlock(&queue->lock); 	while (received < budget && (skb = __skb_dequeue(&process_queue))) {		napi_gro_receive(napi, skb);		++received;	} 	if (!skb_queue_empty(&process_queue)) {		spin_lock(&queue->lock);		skb_queue_splice(&process_queue, queue);		spin_unlock(&queue->lock);	} 	return received;} static int tun_napi_poll(struct napi_struct *napi, int budget){	unsigned int received; 	received = tun_napi_receive(napi, budget); 	if (received < budget)		napi_complete_done(napi, received); 	return received;} static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,			  bool napi_en, bool napi_frags){	tfile->napi_enabled = napi_en;	tfile->napi_frags_enabled = napi_en && napi_frags;	if (napi_en) {		netif_tx_napi_add(tun->dev, &tfile->napi, tun_napi_poll,				  NAPI_POLL_WEIGHT);		napi_enable(&tfile->napi);	}} static void tun_napi_disable(struct tun_file *tfile){	if (tfile->napi_enabled)		napi_disable(&tfile->napi);} static void tun_napi_del(struct tun_file *tfile){	if (tfile->napi_enabled)		netif_napi_del(&tfile->napi);} static bool tun_napi_frags_enabled(const struct tun_file *tfile){	return tfile->napi_frags_enabled;} #ifdef CONFIG_TUN_VNET_CROSS_LEstatic inline bool tun_legacy_is_little_endian(struct tun_struct *tun){	return tun->flags & TUN_VNET_BE ? false :		virtio_legacy_is_little_endian();} static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp){	int be = !!(tun->flags & TUN_VNET_BE); 	if (put_user(be, argp))		return -EFAULT; 	return 0;} static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp){	int be; 	if (get_user(be, argp))		return -EFAULT; 	if (be)		tun->flags |= TUN_VNET_BE;	else		tun->flags &= ~TUN_VNET_BE; 	return 0;}#elsestatic inline bool tun_legacy_is_little_endian(struct tun_struct *tun){	return virtio_legacy_is_little_endian();} static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp){	return -EINVAL;} static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp){	return -EINVAL;}#endif /* CONFIG_TUN_VNET_CROSS_LE */ static inline bool tun_is_little_endian(struct tun_struct *tun){	return tun->flags & TUN_VNET_LE ||		tun_legacy_is_little_endian(tun);} static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val){	return __virtio16_to_cpu(tun_is_little_endian(tun), val);} static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val){	return __cpu_to_virtio16(tun_is_little_endian(tun), val);} static inline u32 tun_hashfn(u32 rxhash){	return rxhash & TUN_MASK_FLOW_ENTRIES;} static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash){	struct tun_flow_entry *e; 	hlist_for_each_entry_rcu(e, head, hash_link) {		if (e->rxhash == rxhash)			return e;	}	return NULL;} static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,					      struct hlist_head *head,					      u32 rxhash, u16 queue_index){	struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC); 	if (e) {		netif_info(tun, tx_queued, tun->dev,			   "create flow: hash %u index %u\n",			   rxhash, queue_index);		e->updated = jiffies;		e->rxhash = rxhash;		e->rps_rxhash = 0;		e->queue_index = queue_index;		e->tun = tun;		hlist_add_head_rcu(&e->hash_link, head);		++tun->flow_count;	}	return e;} static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e){	netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n",		   e->rxhash, e->queue_index);	hlist_del_rcu(&e->hash_link);	kfree_rcu(e, rcu);	--tun->flow_count;} static void tun_flow_flush(struct tun_struct *tun){	int i; 	spin_lock_bh(&tun->lock);	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {		struct tun_flow_entry *e;		struct hlist_node *n; 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)			tun_flow_delete(tun, e);	}	spin_unlock_bh(&tun->lock);} static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index){	int i; 	spin_lock_bh(&tun->lock);	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {		struct tun_flow_entry *e;		struct hlist_node *n; 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {			if (e->queue_index == queue_index)				tun_flow_delete(tun, e);		}	}	spin_unlock_bh(&tun->lock);} static void tun_flow_cleanup(struct timer_list *t){	struct tun_struct *tun = from_timer(tun, t, flow_gc_timer);	unsigned long delay = tun->ageing_time;	unsigned long next_timer = jiffies + delay;	unsigned long count = 0;	int i; 	spin_lock(&tun->lock);	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {		struct tun_flow_entry *e;		struct hlist_node *n; 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {			unsigned long this_timer; 			this_timer = e->updated + delay;			if (time_before_eq(this_timer, jiffies)) {				tun_flow_delete(tun, e);				continue;			}			count++;			if (time_before(this_timer, next_timer))				next_timer = this_timer;		}	} 	if (count)		mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));	spin_unlock(&tun->lock);} static void tun_flow_update(struct tun_struct *tun, u32 rxhash,			    struct tun_file *tfile){	struct hlist_head *head;	struct tun_flow_entry *e;	unsigned long delay = tun->ageing_time;	u16 queue_index = tfile->queue_index; 	head = &tun->flows[tun_hashfn(rxhash)]; 	rcu_read_lock(); 	e = tun_flow_find(head, rxhash);	if (likely(e)) {		/* TODO: keep queueing to old queue until it's empty? */		if (READ_ONCE(e->queue_index) != queue_index)			WRITE_ONCE(e->queue_index, queue_index);		if (e->updated != jiffies)

Lines 1–500 of 3696.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t