3925 lines · 97.8 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0-or-later/* * INET		An implementation of the TCP/IP protocol suite for the LINUX *		operating system.  INET is implemented using the  BSD Socket *		interface as the means of communication with the user level. * *		The User Datagram Protocol (UDP). * * Authors:	Ross Biro *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> *		Arnt Gulbrandsen, <agulbra@nvg.unit.no> *		Alan Cox, <alan@lxorguk.ukuu.org.uk> *		Hirokazu Takahashi, <taka@valinux.co.jp> * * Fixes: *		Alan Cox	:	verify_area() calls *		Alan Cox	: 	stopped close while in use off icmp *					messages. Not a fix but a botch that *					for udp at least is 'valid'. *		Alan Cox	:	Fixed icmp handling properly *		Alan Cox	: 	Correct error for oversized datagrams *		Alan Cox	:	Tidied select() semantics. *		Alan Cox	:	udp_err() fixed properly, also now *					select and read wake correctly on errors *		Alan Cox	:	udp_send verify_area moved to avoid mem leak *		Alan Cox	:	UDP can count its memory *		Alan Cox	:	send to an unknown connection causes *					an ECONNREFUSED off the icmp, but *					does NOT close. *		Alan Cox	:	Switched to new sk_buff handlers. No more backlog! *		Alan Cox	:	Using generic datagram code. Even smaller and the PEEK *					bug no longer crashes it. *		Fred Van Kempen	: 	Net2e support for sk->broadcast. *		Alan Cox	:	Uses skb_free_datagram *		Alan Cox	:	Added get/set sockopt support. *		Alan Cox	:	Broadcasting without option set returns EACCES. *		Alan Cox	:	No wakeup calls. Instead we now use the callbacks. *		Alan Cox	:	Use ip_tos and ip_ttl *		Alan Cox	:	SNMP Mibs *		Alan Cox	:	MSG_DONTROUTE, and 0.0.0.0 support. *		Matt Dillon	:	UDP length checks. *		Alan Cox	:	Smarter af_inet used properly. *		Alan Cox	:	Use new kernel side addressing. *		Alan Cox	:	Incorrect return on truncated datagram receive. *	Arnt Gulbrandsen 	:	New udp_send and stuff *		Alan Cox	:	Cache last socket *		Alan Cox	:	Route cache *		Jon Peatfield	:	Minor efficiency fix to sendto(). *		Mike Shaver	:	RFC1122 checks. *		Alan Cox	:	Nonblocking error fix. *	Willy Konynenberg	:	Transparent proxying support. *		Mike McLagan	:	Routing by source *		David S. Miller	:	New socket lookup architecture. *					Last socket cache retained as it *					does have a high hit rate. *		Olaf Kirch	:	Don't linearise iovec on sendmsg. *		Andi Kleen	:	Some cleanups, cache destination entry *					for connect. *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma. *		Melvin Smith	:	Check msg_name not msg_namelen in sendto(), *					return ENOTCONN for unconnected sockets (POSIX) *		Janos Farkas	:	don't deliver multi/broadcasts to a different *					bound-to-device socket *	Hirokazu Takahashi	:	HW checksumming for outgoing UDP *					datagrams. *	Hirokazu Takahashi	:	sendfile() on UDP works now. *		Arnaldo C. Melo :	convert /proc/net/udp to seq_file *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which *	Alexey Kuznetsov:		allow both IPv4 and IPv6 sockets to bind *					a single port at the same time. *	Derek Atkins <derek@ihtfp.com>: Add Encapsulation Support *	James Chapman		:	Add L2TP encapsulation type. */ #define pr_fmt(fmt) "UDP: " fmt #include <linux/bpf-cgroup.h>#include <linux/uaccess.h>#include <linux/uio.h>#include <asm/ioctls.h>#include <linux/memblock.h>#include <linux/highmem.h>#include <linux/types.h>#include <linux/fcntl.h>#include <linux/module.h>#include <linux/socket.h>#include <linux/sockios.h>#include <linux/igmp.h>#include <linux/inetdevice.h>#include <linux/in.h>#include <linux/errno.h>#include <linux/timer.h>#include <linux/mm.h>#include <linux/inet.h>#include <linux/netdevice.h>#include <linux/slab.h>#include <linux/sock_diag.h>#include <net/tcp_states.h>#include <linux/skbuff.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <net/aligned_data.h>#include <net/net_namespace.h>#include <net/icmp.h>#include <net/inet_common.h>#include <net/inet_hashtables.h>#include <net/ip.h>#include <net/ip_tunnels.h>#include <net/route.h>#include <net/checksum.h>#include <net/gso.h>#include <net/xfrm.h>#include <trace/events/udp.h>#include <linux/static_key.h>#include <linux/btf_ids.h>#include <trace/events/skb.h>#include <net/busy_poll.h>#include <net/sock_reuseport.h>#include <net/addrconf.h>#include <net/udp_tunnel.h>#include <net/gro.h>#include <net/rps.h> struct udp_table udp_table __read_mostly; long sysctl_udp_mem[3] __read_mostly; DEFINE_PER_CPU(int, udp_memory_per_cpu_fw_alloc);EXPORT_PER_CPU_SYMBOL_GPL(udp_memory_per_cpu_fw_alloc); #define MAX_UDP_PORTS 65536#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN_PERNET) static int udp_lib_lport_inuse(struct net *net, __u16 num,			       const struct udp_hslot *hslot,			       unsigned long *bitmap,			       struct sock *sk, unsigned int log){	kuid_t uid = sk_uid(sk);	struct sock *sk2; 	sk_for_each(sk2, &hslot->head) {		if (net_eq(sock_net(sk2), net) &&		    sk2 != sk &&		    (bitmap || udp_sk(sk2)->udp_port_hash == num) &&		    (!sk2->sk_reuse || !sk->sk_reuse) &&		    (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||		     sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&		    inet_rcv_saddr_equal(sk, sk2, true)) {			if (sk2->sk_reuseport && sk->sk_reuseport &&			    !rcu_access_pointer(sk->sk_reuseport_cb) &&			    uid_eq(uid, sk_uid(sk2))) {				if (!bitmap)					return 0;			} else {				if (!bitmap)					return 1;				__set_bit(udp_sk(sk2)->udp_port_hash >> log,					  bitmap);			}		}	}	return 0;} /* * Note: we still hold spinlock of primary hash chain, so no other writer * can insert/delete a socket with local_port == num */static int udp_lib_lport_inuse2(struct net *net, __u16 num,				struct udp_hslot *hslot2,				struct sock *sk){	kuid_t uid = sk_uid(sk);	struct sock *sk2;	int res = 0; 	spin_lock(&hslot2->lock);	udp_portaddr_for_each_entry(sk2, &hslot2->head) {		if (net_eq(sock_net(sk2), net) &&		    sk2 != sk &&		    (udp_sk(sk2)->udp_port_hash == num) &&		    (!sk2->sk_reuse || !sk->sk_reuse) &&		    (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||		     sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&		    inet_rcv_saddr_equal(sk, sk2, true)) {			if (sk2->sk_reuseport && sk->sk_reuseport &&			    !rcu_access_pointer(sk->sk_reuseport_cb) &&			    uid_eq(uid, sk_uid(sk2))) {				res = 0;			} else {				res = 1;			}			break;		}	}	spin_unlock(&hslot2->lock);	return res;} static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot){	struct net *net = sock_net(sk);	kuid_t uid = sk_uid(sk);	struct sock *sk2; 	sk_for_each(sk2, &hslot->head) {		if (net_eq(sock_net(sk2), net) &&		    sk2 != sk &&		    sk2->sk_family == sk->sk_family &&		    ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&		    (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&		    (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&		    sk2->sk_reuseport && uid_eq(uid, sk_uid(sk2)) &&		    inet_rcv_saddr_equal(sk, sk2, false)) {			return reuseport_add_sock(sk, sk2,						  inet_rcv_saddr_any(sk));		}	} 	return reuseport_alloc(sk, inet_rcv_saddr_any(sk));} /** *  udp_lib_get_port  -  UDP port lookup for IPv4 and IPv6 * *  @sk:          socket struct in question *  @snum:        port number to look up *  @hash2_nulladdr: AF-dependent hash value in secondary hash chains, *                   with NULL address */int udp_lib_get_port(struct sock *sk, unsigned short snum,		     unsigned int hash2_nulladdr){	struct udp_hslot *hslot, *hslot2;	struct net *net = sock_net(sk);	struct udp_table *udptable;	int error = -EADDRINUSE; 	udptable = net->ipv4.udp_table; 	if (!snum) {		DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);		unsigned short first, last;		int low, high, remaining;		unsigned int rand; 		inet_sk_get_local_port_range(sk, &low, &high);		remaining = (high - low) + 1; 		rand = get_random_u32();		first = reciprocal_scale(rand, remaining) + low;		/*		 * force rand to be an odd multiple of UDP_HTABLE_SIZE		 */		rand = (rand | 1) * (udptable->mask + 1);		last = first + udptable->mask + 1;		do {			hslot = udp_hashslot(udptable, net, first);			bitmap_zero(bitmap, PORTS_PER_CHAIN);			spin_lock_bh(&hslot->lock);			udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,					    udptable->log); 			snum = first;			/*			 * Iterate on all possible values of snum for this hash.			 * Using steps of an odd multiple of UDP_HTABLE_SIZE			 * give us randomization and full range coverage.			 */			do {				if (low <= snum && snum <= high &&				    !test_bit(snum >> udptable->log, bitmap) &&				    !inet_is_local_reserved_port(net, snum))					goto found;				snum += rand;			} while (snum != first);			spin_unlock_bh(&hslot->lock);			cond_resched();		} while (++first != last);		goto fail;	} else {		hslot = udp_hashslot(udptable, net, snum);		spin_lock_bh(&hslot->lock);		if (inet_use_hash2_on_bind(sk) && hslot->count > 10) {			int exist;			unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; 			slot2          &= udptable->mask;			hash2_nulladdr &= udptable->mask; 			hslot2 = udp_hashslot2(udptable, slot2);			if (hslot->count < hslot2->count)				goto scan_primary_hash; 			exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);			if (!exist && (hash2_nulladdr != slot2)) {				hslot2 = udp_hashslot2(udptable, hash2_nulladdr);				exist = udp_lib_lport_inuse2(net, snum, hslot2,							     sk);			}			if (exist)				goto fail_unlock;			else				goto found;		}scan_primary_hash:		if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))			goto fail_unlock;	}found:	inet_sk(sk)->inet_num = snum;	udp_sk(sk)->udp_port_hash = snum;	udp_sk(sk)->udp_portaddr_hash ^= snum;	if (sk_unhashed(sk)) {		if (sk->sk_reuseport &&		    udp_reuseport_add_sock(sk, hslot)) {			inet_sk(sk)->inet_num = 0;			udp_sk(sk)->udp_port_hash = 0;			udp_sk(sk)->udp_portaddr_hash ^= snum;			goto fail_unlock;		} 		sock_set_flag(sk, SOCK_RCU_FREE); 		sk_add_node_rcu(sk, &hslot->head);		hslot->count++;		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 		hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);		spin_lock(&hslot2->lock);		if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&		    sk->sk_family == AF_INET6)			hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,					   &hslot2->head);		else			hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,					   &hslot2->head);		hslot2->count++;		spin_unlock(&hslot2->lock);	} 	error = 0;fail_unlock:	spin_unlock_bh(&hslot->lock);fail:	return error;} static int udp_v4_get_port(struct sock *sk, unsigned short snum){	unsigned int hash2_nulladdr =		ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);	unsigned int hash2_partial =		ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0); 	/* precompute partial secondary hash */	udp_sk(sk)->udp_portaddr_hash = hash2_partial;	return udp_lib_get_port(sk, snum, hash2_nulladdr);} static __always_inline intcompute_score(struct sock *sk, const struct net *net,	      __be32 saddr, __be16 sport, __be32 daddr,	      unsigned short hnum, int dif, int sdif){	int score;	struct inet_sock *inet;	bool dev_match; 	if (!net_eq(sock_net(sk), net) ||	    udp_sk(sk)->udp_port_hash != hnum ||	    ipv6_only_sock(sk))		return -1; 	if (sk->sk_rcv_saddr != daddr)		return -1; 	score = (sk->sk_family == PF_INET) ? 2 : 1; 	inet = inet_sk(sk);	if (inet->inet_daddr) {		if (inet->inet_daddr != saddr)			return -1;		score += 4;	} 	if (inet->inet_dport) {		if (inet->inet_dport != sport)			return -1;		score += 4;	} 	dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,					dif, sdif);	if (!dev_match)		return -1;	if (sk->sk_bound_dev_if)		score += 4; 	if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())		score++;	return score;} u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,		const __be32 faddr, const __be16 fport){	net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret)); 	return __inet_ehashfn(laddr, lport, faddr, fport,			      udp_ehash_secret + net_hash_mix(net));} /** * udp4_lib_lookup1() - Simplified lookup using primary hash (destination port) * @net:	Network namespace * @saddr:	Source address, network order * @sport:	Source port, network order * @daddr:	Destination address, network order * @hnum:	Destination port, host order * @dif:	Destination interface index * @sdif:	Destination bridge port index, if relevant * @udptable:	Set of UDP hash tables * * Simplified lookup to be used as fallback if no sockets are found due to a * potential race between (receive) address change, and lookup happening before * the rehash operation. This function ignores SO_REUSEPORT groups while scoring * result sockets, because if we have one, we don't need the fallback at all. * * Called under rcu_read_lock(). * * Return: socket with highest matching score if any, NULL if none */static struct sock *udp4_lib_lookup1(const struct net *net,				     __be32 saddr, __be16 sport,				     __be32 daddr, unsigned int hnum,				     int dif, int sdif,				     const struct udp_table *udptable){	unsigned int slot = udp_hashfn(net, hnum, udptable->mask);	struct udp_hslot *hslot = &udptable->hash[slot];	struct sock *sk, *result = NULL;	int score, badness = 0; 	sk_for_each_rcu(sk, &hslot->head) {		score = compute_score(sk, net,				      saddr, sport, daddr, hnum, dif, sdif);		if (score > badness) {			result = sk;			badness = score;		}	} 	return result;} /* called with rcu_read_lock() */static struct sock *udp4_lib_lookup2(const struct net *net,				     __be32 saddr, __be16 sport,				     __be32 daddr, unsigned int hnum,				     int dif, int sdif,				     struct udp_hslot *hslot2,				     struct sk_buff *skb){	struct sock *sk, *result;	int score, badness;	bool need_rescore; 	result = NULL;	badness = 0;	udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {		need_rescore = false;rescore:		score = compute_score(need_rescore ? result : sk, net, saddr,				      sport, daddr, hnum, dif, sdif);		if (score > badness) {			badness = score; 			if (need_rescore)				continue; 			if (sk->sk_state == TCP_ESTABLISHED) {				result = sk;				continue;			} 			result = inet_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),						       saddr, sport, daddr, hnum, udp_ehashfn);			if (!result) {				result = sk;				continue;			} 			/* Fall back to scoring if group has connections */			if (!reuseport_has_conns(sk))				return result; 			/* Reuseport logic returned an error, keep original score. */			if (IS_ERR(result))

Lines 1–500 of 3925.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t