History

Merge branch 'net-group-together-hot-data'

Eric Dumazet says: ==================== net: group together hot data While our recent structure reorganizations were focused on increasing max throughput, there is still an area where improvements are much needed. In many cases, a cpu handles one packet at a time, instead of a nice batch. Hardware interrupt. -> Software interrupt. -> Network/Protocol stacks. If the cpu was idle or busy in other layers, it has to pull many cache lines. This series adds a new net_hotdata structure, where some critical (and read-mostly) data used in rx and tx path is packed in a small number of cache lines. Synthetic benchmarks will not see much difference, but latency of single packet should improve. net_hodata current size on 64bit is 416 bytes, but might grow in the future. Also move RPS definitions to a new include file. ====================

Jakub Kicinski committed 2 years ago

Commit e8bb2cc2 parents d3423ed ce7f49a44 files · +391 −320Browse files

Unified · Split

44 files
  1. drivers/net/ethernet/intel/ice/ice_arfs.c +1 −0
  2. drivers/net/ethernet/mellanox/mlx4/en_netdev.c +1 −0
  3. drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +1 −0
  4. drivers/net/ethernet/sfc/rx_common.c +1 −0
  5. drivers/net/ethernet/sfc/siena/rx_common.c +1 −0
  6. drivers/net/tun.c +1 −0
  7. include/linux/netdevice.h +0 −88
  8. include/linux/skbuff.h +0 −1
  9. include/net/gro.h +2 −3
  10. include/net/hotdata.h +52 −0
  11. include/net/protocol.h +3 −0
  12. include/net/rps.h +125 −0
  13. include/net/sock.h +0 −35
  14. kernel/bpf/cpumap.c +3 −1
  15. net/bpf/test_run.c +3 −1
  16. net/core/Makefile +1 −0
  17. net/core/dev.c +22 −36
  18. net/core/dev.h +0 −3
  19. net/core/gro.c +6 −9
  20. net/core/gro_cells.c +2 −1
  21. net/core/gso.c +2 −2
  22. net/core/hotdata.c +22 −0
  23. net/core/net-procfs.c +4 −3
  24. net/core/net-sysfs.c +1 −0
  25. net/core/skbuff.c +20 −24
  26. net/core/sysctl_net_core.c +15 −10
  27. net/core/xdp.c +3 −2
  28. net/ipv4/af_inet.c +25 −24
  29. net/ipv4/inet_hashtables.c +1 −2
  30. net/ipv4/tcp.c +1 −0
  31. net/ipv4/tcp_offload.c +8 −9
  32. net/ipv4/udp.c +0 −2
  33. net/ipv4/udp_offload.c +8 −9
  34. net/ipv6/af_inet6.c +1 −0
  35. net/ipv6/inet6_hashtables.c +3 −5
  36. net/ipv6/ip6_offload.c +9 −9
  37. net/ipv6/tcp_ipv6.c +9 −8
  38. net/ipv6/tcpv6_offload.c +8 −8
  39. net/ipv6/udp.c +8 −11
  40. net/ipv6/udp_offload.c +10 −11
  41. net/sched/sch_generic.c +2 −1
  42. net/sctp/socket.c +1 −0
  43. net/xfrm/espintcp.c +3 −1
  44. net/xfrm/xfrm_input.c +2 −1
drivers/net/ethernet/intel/ice/ice_arfs.c+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
22 /* Copyright (C) 2018-2020, Intel Corporation. */
33
44 #include "ice.h"
5+#include <net/rps.h>
56
67 /**
78 * ice_is_arfs_active - helper to check is aRFS is active
drivers/net/ethernet/mellanox/mlx4/en_netdev.c+1−0
@@ -42,6 +42,7 @@ · 41 unmodified lines
4242 #include <net/ip.h>
4343 #include <net/vxlan.h>
4444 #include <net/devlink.h>
45+#include <net/rps.h>
4546
4647 #include <linux/mlx4/driver.h>
4748 #include <linux/mlx4/device.h>
drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c+1−0
@@ -34,6 +34,7 @@ · 33 unmodified lines
3434 #include <linux/mlx5/fs.h>
3535 #include <linux/ip.h>
3636 #include <linux/ipv6.h>
37+#include <net/rps.h>
3738 #include "en.h"
3839
3940 #define ARFS_HASH_SHIFT BITS_PER_BYTE
drivers/net/ethernet/sfc/rx_common.c+1−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 #include "net_driver.h"
1212 #include <linux/module.h>
1313 #include <linux/iommu.h>
14+#include <net/rps.h>
1415 #include "efx.h"
1516 #include "nic.h"
1617 #include "rx_common.h"
drivers/net/ethernet/sfc/siena/rx_common.c+1−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 #include "net_driver.h"
1212 #include <linux/module.h>
1313 #include <linux/iommu.h>
14+#include <net/rps.h>
1415 #include "efx.h"
1516 #include "nic.h"
1617 #include "rx_common.h"
drivers/net/tun.c+1−0
@@ -78,6 +78,7 @@ · 77 unmodified lines
7878 #include <net/ax25.h>
7979 #include <net/rose.h>
8080 #include <net/6lowpan.h>
81+#include <net/rps.h>
8182
8283 #include <linux/uaccess.h>
8384 #include <linux/proc_fs.h>
include/linux/netdevice.h+0−88
@@ -225,12 +225,6 @@ struct net_device_core_stats { · 224 unmodified lines
225225 #include <linux/cache.h>
226226 #include <linux/skbuff.h>
227227
228−#ifdef CONFIG_RPS
229−#include <linux/static_key.h>
230−extern struct static_key_false rps_needed;
231−extern struct static_key_false rfs_needed;
232−#endif
233−
234228 struct neighbour;
235229 struct neigh_parms;
236230 struct sk_buff;
@@ -730,86 +724,10 @@ static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node · 493 unmodified lines
730724 #endif
731725 }
732726
733−#ifdef CONFIG_RPS
734−/*
735− * This structure holds an RPS map which can be of variable length. The
736− * map is an array of CPUs.
737− */
738−struct rps_map {
739− unsigned int len;
740− struct rcu_head rcu;
741− u16 cpus[];
742−};
743−#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
744−
745−/*
746− * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
747− * tail pointer for that CPU's input queue at the time of last enqueue, and
748− * a hardware filter index.
749− */
750−struct rps_dev_flow {
751− u16 cpu;
752− u16 filter;
753− unsigned int last_qtail;
754−};
755−#define RPS_NO_FILTER 0xffff
756−
757−/*
758− * The rps_dev_flow_table structure contains a table of flow mappings.
759− */
760−struct rps_dev_flow_table {
761− unsigned int mask;
762− struct rcu_head rcu;
763− struct rps_dev_flow flows[];
764−};
765−#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
766− ((_num) * sizeof(struct rps_dev_flow)))
767−
768−/*
769− * The rps_sock_flow_table contains mappings of flows to the last CPU
770− * on which they were processed by the application (set in recvmsg).
771− * Each entry is a 32bit value. Upper part is the high-order bits
772− * of flow hash, lower part is CPU number.
773− * rps_cpu_mask is used to partition the space, depending on number of
774− * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1
775− * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f,
776− * meaning we use 32-6=26 bits for the hash.
777− */
778−struct rps_sock_flow_table {
779− u32 mask;
780−
781− u32 ents[] ____cacheline_aligned_in_smp;
782−};
783−#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
784−
785−#define RPS_NO_CPU 0xffff
786−
787−extern u32 rps_cpu_mask;
788−extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
789−
790−static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
791− u32 hash)
792−{
793− if (table && hash) {
794− unsigned int index = hash & table->mask;
795− u32 val = hash & ~rps_cpu_mask;
796−
797− /* We only give a hint, preemption can change CPU under us */
798− val |= raw_smp_processor_id();
799−
800− /* The following WRITE_ONCE() is paired with the READ_ONCE()
801− * here, and another one in get_rps_cpu().
802− */
803− if (READ_ONCE(table->ents[index]) != val)
804− WRITE_ONCE(table->ents[index], val);
805− }
806−}
807−
808727 #ifdef CONFIG_RFS_ACCEL
809728 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
810729 u16 filter_id);
811730 #endif
812−#endif /* CONFIG_RPS */
813731
814732 /* XPS map type and offset of the xps map within net_device->xps_maps[]. */
815733 enum xps_map_type {
@@ -4793,11 +4711,6 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, · 3977 unmodified lines
47934711 const struct pcpu_sw_netstats __percpu *netstats);
47944712 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
47954713
4796−extern int netdev_max_backlog;
4797−extern int dev_rx_weight;
4798−extern int dev_tx_weight;
4799−extern int gro_normal_batch;
4800−
48014714 enum {
48024715 NESTED_SYNC_IMM_BIT,
48034716 NESTED_SYNC_TODO_BIT,
@@ -5307,7 +5220,6 @@ static inline const char *netdev_reg_state(const struct net_device *dev) · 503 unmodified lines
53075220 #define PTYPE_HASH_SIZE (16)
53085221 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
53095222
5310−extern struct list_head ptype_all __read_mostly;
53115223 extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
53125224
53135225 extern struct net_device *blackhole_netdev;
include/linux/skbuff.h+0−1
@@ -1271,7 +1271,6 @@ static inline void consume_skb(struct sk_buff *skb) · 1270 unmodified lines
12711271
12721272 void __consume_stateless_skb(struct sk_buff *skb);
12731273 void __kfree_skb(struct sk_buff *skb);
1274−extern struct kmem_cache *skbuff_cache;
12751274
12761275 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
12771276 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
include/net/gro.h+2−3
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include <net/ip6_checksum.h>
1010 #include <linux/skbuff.h>
1111 #include <net/udp.h>
12+#include <net/hotdata.h>
1213
1314 struct napi_gro_cb {
1415 union {
@@ -446,7 +447,7 @@ static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, · 431 unmodified lines
446447 {
447448 list_add_tail(&skb->list, &napi->rx_list);
448449 napi->rx_count += segs;
449− if (napi->rx_count >= READ_ONCE(gro_normal_batch))
450+ if (napi->rx_count >= READ_ONCE(net_hotdata.gro_normal_batch))
450451 gro_normal_list(napi);
451452 }
452453
@@ -493,6 +494,4 @@ static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int * · 40 unmodified lines
493494 #endif
494495 }
495496
496−extern struct list_head offload_base;
497−
498497 #endif /* _NET_IPV6_GRO_H */
include/net/hotdata.h+52−0
@@ -0,0 +1,52 @@
1+/* SPDX-License-Identifier: GPL-2.0-or-later */
2+#ifndef _NET_HOTDATA_H
3+#define _NET_HOTDATA_H
4+
5+#include <linux/types.h>
6+#include <linux/netdevice.h>
7+#include <net/protocol.h>
8+
9+/* Read mostly data used in network fast paths. */
10+struct net_hotdata {
11+#if IS_ENABLED(CONFIG_INET)
12+ struct packet_offload ip_packet_offload;
13+ struct net_offload tcpv4_offload;
14+ struct net_protocol tcp_protocol;
15+ struct net_offload udpv4_offload;
16+ struct net_protocol udp_protocol;
17+ struct packet_offload ipv6_packet_offload;
18+ struct net_offload tcpv6_offload;
19+#if IS_ENABLED(CONFIG_IPV6)
20+ struct inet6_protocol tcpv6_protocol;
21+ struct inet6_protocol udpv6_protocol;
22+#endif
23+ struct net_offload udpv6_offload;
24+#endif
25+ struct list_head offload_base;
26+ struct list_head ptype_all;
27+ struct kmem_cache *skbuff_cache;
28+ struct kmem_cache *skbuff_fclone_cache;
29+ struct kmem_cache *skb_small_head_cache;
30+#ifdef CONFIG_RPS
31+ struct rps_sock_flow_table __rcu *rps_sock_flow_table;
32+ u32 rps_cpu_mask;
33+#endif
34+ int gro_normal_batch;
35+ int netdev_budget;
36+ int netdev_budget_usecs;
37+ int tstamp_prequeue;
38+ int max_backlog;
39+ int dev_tx_weight;
40+ int dev_rx_weight;
41+};
42+
43+#define inet_ehash_secret net_hotdata.tcp_protocol.secret
44+#define udp_ehash_secret net_hotdata.udp_protocol.secret
45+#define inet6_ehash_secret net_hotdata.tcpv6_protocol.secret
46+#define tcp_ipv6_hash_secret net_hotdata.tcpv6_offload.secret
47+#define udp6_ehash_secret net_hotdata.udpv6_protocol.secret
48+#define udp_ipv6_hash_secret net_hotdata.udpv6_offload.secret
49+
50+extern struct net_hotdata net_hotdata;
51+
52+#endif /* _NET_HOTDATA_H */
include/net/protocol.h+3−0
@@ -46,6 +46,7 @@ struct net_protocol { · 45 unmodified lines
4646 * socket lookup?
4747 */
4848 icmp_strict_tag_validation:1;
49+ u32 secret;
4950 };
5051
5152 #if IS_ENABLED(CONFIG_IPV6)
@@ -59,6 +60,7 @@ struct inet6_protocol { · 7 unmodified lines
5960 __be32 info);
6061
6162 unsigned int flags; /* INET6_PROTO_xxx */
63+ u32 secret;
6264 };
6365
6466 #define INET6_PROTO_NOPOLICY 0x1
@@ -68,6 +70,7 @@ struct inet6_protocol { · 3 unmodified lines
6870 struct net_offload {
6971 struct offload_callbacks callbacks;
7072 unsigned int flags; /* Flags used by IPv6 for now */
73+ u32 secret;
7174 };
7275 /* This should be set for any extension header which is compatible with GSO. */
7376 #define INET6_PROTO_GSO_EXTHDR 0x1
include/net/rps.h+125−0
@@ -0,0 +1,125 @@
1+/* SPDX-License-Identifier: GPL-2.0-or-later */
2+#ifndef _NET_RPS_H
3+#define _NET_RPS_H
4+
5+#include <linux/types.h>
6+#include <linux/static_key.h>
7+#include <net/sock.h>
8+#include <net/hotdata.h>
9+
10+#ifdef CONFIG_RPS
11+
12+extern struct static_key_false rps_needed;
13+extern struct static_key_false rfs_needed;
14+
15+/*
16+ * This structure holds an RPS map which can be of variable length. The
17+ * map is an array of CPUs.
18+ */
19+struct rps_map {
20+ unsigned int len;
21+ struct rcu_head rcu;
22+ u16 cpus[];
23+};
24+#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
25+
26+/*
27+ * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
28+ * tail pointer for that CPU's input queue at the time of last enqueue, and
29+ * a hardware filter index.
30+ */
31+struct rps_dev_flow {
32+ u16 cpu;
33+ u16 filter;
34+ unsigned int last_qtail;
35+};
36+#define RPS_NO_FILTER 0xffff
37+
38+/*
39+ * The rps_dev_flow_table structure contains a table of flow mappings.
40+ */
41+struct rps_dev_flow_table {
42+ unsigned int mask;
43+ struct rcu_head rcu;
44+ struct rps_dev_flow flows[];
45+};
46+#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
47+ ((_num) * sizeof(struct rps_dev_flow)))
48+
49+/*
50+ * The rps_sock_flow_table contains mappings of flows to the last CPU
51+ * on which they were processed by the application (set in recvmsg).
52+ * Each entry is a 32bit value. Upper part is the high-order bits
53+ * of flow hash, lower part is CPU number.
54+ * rps_cpu_mask is used to partition the space, depending on number of
55+ * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1
56+ * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f,
57+ * meaning we use 32-6=26 bits for the hash.
58+ */
59+struct rps_sock_flow_table {
60+ u32 mask;
61+
62+ u32 ents[] ____cacheline_aligned_in_smp;
63+};
64+#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
65+
66+#define RPS_NO_CPU 0xffff
67+
68+static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
69+ u32 hash)
70+{
71+ unsigned int index = hash & table->mask;
72+ u32 val = hash & ~net_hotdata.rps_cpu_mask;
73+
74+ /* We only give a hint, preemption can change CPU under us */
75+ val |= raw_smp_processor_id();
76+
77+ /* The following WRITE_ONCE() is paired with the READ_ONCE()
78+ * here, and another one in get_rps_cpu().
79+ */
80+ if (READ_ONCE(table->ents[index]) != val)
81+ WRITE_ONCE(table->ents[index], val);
82+}
83+
84+#endif /* CONFIG_RPS */
85+
86+static inline void sock_rps_record_flow_hash(__u32 hash)
87+{
88+#ifdef CONFIG_RPS
89+ struct rps_sock_flow_table *sock_flow_table;
90+
91+ if (!hash)
92+ return;
93+ rcu_read_lock();
94+ sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
95+ if (sock_flow_table)
96+ rps_record_sock_flow(sock_flow_table, hash);
97+ rcu_read_unlock();
98+#endif
99+}
100+
101+static inline void sock_rps_record_flow(const struct sock *sk)
102+{
103+#ifdef CONFIG_RPS
104+ if (static_branch_unlikely(&rfs_needed)) {
105+ /* Reading sk->sk_rxhash might incur an expensive cache line
106+ * miss.
107+ *
108+ * TCP_ESTABLISHED does cover almost all states where RFS
109+ * might be useful, and is cheaper [1] than testing :
110+ * IPv4: inet_sk(sk)->inet_daddr
111+ * IPv6: ipv6_addr_any(&sk->sk_v6_daddr)
112+ * OR an additional socket flag
113+ * [1] : sk_state and sk_prot are in the same cache line.
114+ */
115+ if (sk->sk_state == TCP_ESTABLISHED) {
116+ /* This READ_ONCE() is paired with the WRITE_ONCE()
117+ * from sock_rps_save_rxhash() and sock_rps_reset_rxhash().
118+ */
119+ sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash));
120+ }
121+ }
122+#endif
123+}
124+
125+#endif /* _NET_RPS_H */
include/net/sock.h+0−35
@@ -1117,41 +1117,6 @@ static inline void sk_incoming_cpu_update(struct sock *sk) · 1116 unmodified lines
11171117 WRITE_ONCE(sk->sk_incoming_cpu, cpu);
11181118 }
11191119
1120−static inline void sock_rps_record_flow_hash(__u32 hash)
1121−{
1122−#ifdef CONFIG_RPS
1123− struct rps_sock_flow_table *sock_flow_table;
1124−
1125− rcu_read_lock();
1126− sock_flow_table = rcu_dereference(rps_sock_flow_table);
1127− rps_record_sock_flow(sock_flow_table, hash);
1128− rcu_read_unlock();
1129−#endif
1130−}
1131−
1132−static inline void sock_rps_record_flow(const struct sock *sk)
1133−{
1134−#ifdef CONFIG_RPS
1135− if (static_branch_unlikely(&rfs_needed)) {
1136− /* Reading sk->sk_rxhash might incur an expensive cache line
1137− * miss.
1138− *
1139− * TCP_ESTABLISHED does cover almost all states where RFS
1140− * might be useful, and is cheaper [1] than testing :
1141− * IPv4: inet_sk(sk)->inet_daddr
1142− * IPv6: ipv6_addr_any(&sk->sk_v6_daddr)
1143− * OR an additional socket flag
1144− * [1] : sk_state and sk_prot are in the same cache line.
1145− */
1146− if (sk->sk_state == TCP_ESTABLISHED) {
1147− /* This READ_ONCE() is paired with the WRITE_ONCE()
1148− * from sock_rps_save_rxhash() and sock_rps_reset_rxhash().
1149− */
1150− sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash));
1151− }
1152− }
1153−#endif
1154−}
11551120
11561121 static inline void sock_rps_save_rxhash(struct sock *sk,
11571122 const struct sk_buff *skb)
kernel/bpf/cpumap.c+3−1
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424 #include <linux/filter.h>
2525 #include <linux/ptr_ring.h>
2626 #include <net/xdp.h>
27+#include <net/hotdata.h>
2728
2829 #include <linux/sched.h>
2930 #include <linux/workqueue.h>
@@ -326,7 +327,8 @@ static int cpu_map_kthread_run(void *data) · 296 unmodified lines
326327 /* Support running another XDP prog on this CPU */
327328 nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list);
328329 if (nframes) {
329− m = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, skbs);
330+ m = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
331+ gfp, nframes, skbs);
330332 if (unlikely(m == 0)) {
331333 for (i = 0; i < nframes; i++)
332334 skbs[i] = NULL; /* effect: xdp_return_frame */
net/bpf/test_run.c+3−1
@@ -12,6 +12,7 @@ · 11 unmodified lines
1212 #include <linux/rcupdate_trace.h>
1313 #include <linux/sched/signal.h>
1414 #include <net/bpf_sk_storage.h>
15+#include <net/hotdata.h>
1516 #include <net/sock.h>
1617 #include <net/tcp.h>
1718 #include <net/net_namespace.h>
@@ -254,7 +255,8 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes, · 236 unmodified lines
254255 int i, n;
255256 LIST_HEAD(list);
256257
257− n = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, (void **)skbs);
258+ n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
259+ (void **)skbs);
258260 if (unlikely(n == 0)) {
259261 for (i = 0; i < nframes; i++)
260262 xdp_return_frame(frames[i]);
net/core/Makefile+1−0
@@ -18,6 +18,7 @@ obj-y += dev.o dev_addr_lists.o dst.o netevent.o \ · 17 unmodified lines
1818 obj-$(CONFIG_NETDEV_ADDR_LIST_TEST) += dev_addr_lists_test.o
1919
2020 obj-y += net-sysfs.o
21+obj-y += hotdata.o
2122 obj-$(CONFIG_PAGE_POOL) += page_pool.o page_pool_user.o
2223 obj-$(CONFIG_PROC_FS) += net-procfs.o
2324 obj-$(CONFIG_NET_PKTGEN) += pktgen.o
net/core/dev.c+22−36
@@ -155,13 +155,13 @@ · 154 unmodified lines
155155 #include <net/netdev_rx_queue.h>
156156 #include <net/page_pool/types.h>
157157 #include <net/page_pool/helpers.h>
158+#include <net/rps.h>
158159
159160 #include "dev.h"
160161 #include "net-sysfs.h"
161162
162163 static DEFINE_SPINLOCK(ptype_lock);
163164 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
164−struct list_head ptype_all __read_mostly; /* Taps */
165165
166166 static int netif_rx_internal(struct sk_buff *skb);
167167 static int call_netdevice_notifiers_extack(unsigned long val,
@@ -540,7 +540,7 @@ static inline void netdev_set_addr_lockdep_class(struct net_device *dev) · 372 unmodified lines
540540 static inline struct list_head *ptype_head(const struct packet_type *pt)
541541 {
542542 if (pt->type == htons(ETH_P_ALL))
543− return pt->dev ? &pt->dev->ptype_all : &ptype_all;
543+ return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all;
544544 else
545545 return pt->dev ? &pt->dev->ptype_specific :
546546 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
@@ -2226,7 +2226,8 @@ static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) · 1679 unmodified lines
22262226 */
22272227 bool dev_nit_active(struct net_device *dev)
22282228 {
2229− return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2229+ return !list_empty(&net_hotdata.ptype_all) ||
2230+ !list_empty(&dev->ptype_all);
22302231 }
22312232 EXPORT_SYMBOL_GPL(dev_nit_active);
22322233
@@ -2237,10 +2238,9 @@ EXPORT_SYMBOL_GPL(dev_nit_active); · 4 unmodified lines
22372238
22382239 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
22392240 {
2240− struct packet_type *ptype;
2241+ struct list_head *ptype_list = &net_hotdata.ptype_all;
2242+ struct packet_type *ptype, *pt_prev = NULL;
22412243 struct sk_buff *skb2 = NULL;
2242− struct packet_type *pt_prev = NULL;
2243− struct list_head *ptype_list = &ptype_all;
22442244
22452245 rcu_read_lock();
22462246 again:
@@ -2286,7 +2286,7 @@ again: · 39 unmodified lines
22862286 pt_prev = ptype;
22872287 }
22882288
2289− if (ptype_list == &ptype_all) {
2289+ if (ptype_list == &net_hotdata.ptype_all) {
22902290 ptype_list = &dev->ptype_all;
22912291 goto again;
22922292 }
@@ -4405,19 +4405,10 @@ EXPORT_SYMBOL(__dev_direct_xmit); · 2112 unmodified lines
44054405 * Receiver routines
44064406 *************************************************************************/
44074407
4408−int netdev_max_backlog __read_mostly = 1000;
4409−EXPORT_SYMBOL(netdev_max_backlog);
4410−
4411−int netdev_tstamp_prequeue __read_mostly = 1;
44124408 unsigned int sysctl_skb_defer_max __read_mostly = 64;
4413−int netdev_budget __read_mostly = 300;
4414−/* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4415−unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
44164409 int weight_p __read_mostly = 64; /* old backlog weight */
44174410 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
44184411 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4419−int dev_rx_weight __read_mostly = 64;
4420−int dev_tx_weight __read_mostly = 64;
44214412
44224413 /* Called with irq disabled */
44234414 static inline void ____napi_schedule(struct softnet_data *sd,
@@ -4459,12 +4450,6 @@ static inline void ____napi_schedule(struct softnet_data *sd, · 35 unmodified lines
44594450
44604451 #ifdef CONFIG_RPS
44614452
4462−/* One global table that all flow-based protocols share. */
4463−struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4464−EXPORT_SYMBOL(rps_sock_flow_table);
4465−u32 rps_cpu_mask __read_mostly;
4466−EXPORT_SYMBOL(rps_cpu_mask);
4467−
44684453 struct static_key_false rps_needed __read_mostly;
44694454 EXPORT_SYMBOL(rps_needed);
44704455 struct static_key_false rfs_needed __read_mostly;
@@ -4556,7 +4541,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 85 unmodified lines
45564541 if (!hash)
45574542 goto done;
45584543
4559− sock_flow_table = rcu_dereference(rps_sock_flow_table);
4544+ sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
45604545 if (flow_table && sock_flow_table) {
45614546 struct rps_dev_flow *rflow;
45624547 u32 next_cpu;
@@ -4566,10 +4551,10 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 3 unmodified lines
45664551 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
45674552 */
45684553 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
4569− if ((ident ^ hash) & ~rps_cpu_mask)
4554+ if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask)
45704555 goto try_rps;
45714556
4572− next_cpu = ident & rps_cpu_mask;
4557+ next_cpu = ident & net_hotdata.rps_cpu_mask;
45734558
45744559 /* OK, now we know there is a match,
45754560 * we can look at the local (per receive queue) flow table
@@ -4718,7 +4703,7 @@ static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) · 142 unmodified lines
47184703 struct softnet_data *sd;
47194704 unsigned int old_flow, new_flow;
47204705
4721− if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))
4706+ if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1))
47224707 return false;
47234708
47244709 sd = this_cpu_ptr(&softnet_data);
@@ -4766,7 +4751,8 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, · 41 unmodified lines
47664751 if (!netif_running(skb->dev))
47674752 goto drop;
47684753 qlen = skb_queue_len(&sd->input_pkt_queue);
4769− if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {
4754+ if (qlen <= READ_ONCE(net_hotdata.max_backlog) &&
4755+ !skb_flow_limit(skb, qlen)) {
47704756 if (qlen) {
47714757 enqueue:
47724758 __skb_queue_tail(&sd->input_pkt_queue, skb);
@@ -5055,7 +5041,7 @@ static int netif_rx_internal(struct sk_buff *skb) · 282 unmodified lines
50555041 {
50565042 int ret;
50575043
5058− net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5044+ net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
50595045
50605046 trace_netif_rx(skb);
50615047
@@ -5347,7 +5333,7 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, · 285 unmodified lines
53475333 int ret = NET_RX_DROP;
53485334 __be16 type;
53495335
5350− net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb);
5336+ net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb);
53515337
53525338 trace_netif_receive_skb(skb);
53535339
@@ -5391,7 +5377,7 @@ another_round: · 37 unmodified lines
53915377 if (pfmemalloc)
53925378 goto skip_taps;
53935379
5394− list_for_each_entry_rcu(ptype, &ptype_all, list) {
5380+ list_for_each_entry_rcu(ptype, &net_hotdata.ptype_all, list) {
53955381 if (pt_prev)
53965382 ret = deliver_skb(skb, pt_prev, orig_dev);
53975383 pt_prev = ptype;
@@ -5731,7 +5717,7 @@ static int netif_receive_skb_internal(struct sk_buff *skb) · 333 unmodified lines
57315717 {
57325718 int ret;
57335719
5734− net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5720+ net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
57355721
57365722 if (skb_defer_rx_timestamp(skb))
57375723 return NET_RX_SUCCESS;
@@ -5761,7 +5747,8 @@ void netif_receive_skb_list_internal(struct list_head *head) · 23 unmodified lines
57615747
57625748 INIT_LIST_HEAD(&sublist);
57635749 list_for_each_entry_safe(skb, next, head, list) {
5764− net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);
5750+ net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue),
5751+ skb);
57655752 skb_list_del_init(skb);
57665753 if (!skb_defer_rx_timestamp(skb))
57675754 list_add_tail(&skb->list, &sublist);
@@ -5985,7 +5972,7 @@ static int process_backlog(struct napi_struct *napi, int quota) · 217 unmodified lines
59855972 net_rps_action_and_irq_enable(sd);
59865973 }
59875974
5988− napi->weight = READ_ONCE(dev_rx_weight);
5975+ napi->weight = READ_ONCE(net_hotdata.dev_rx_weight);
59895976 while (again) {
59905977 struct sk_buff *skb;
59915978
@@ -6790,8 +6777,8 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) · 798 unmodified lines
67906777 {
67916778 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
67926779 unsigned long time_limit = jiffies +
6793− usecs_to_jiffies(READ_ONCE(netdev_budget_usecs));
6794− int budget = READ_ONCE(netdev_budget);
6780+ usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs));
6781+ int budget = READ_ONCE(net_hotdata.netdev_budget);
67956782 LIST_HEAD(list);
67966783 LIST_HEAD(repoll);
67976784
@@ -11752,7 +11739,6 @@ static int __init net_dev_init(void) · 4954 unmodified lines
1175211739 if (netdev_kobject_init())
1175311740 goto out;
1175411741
11755− INIT_LIST_HEAD(&ptype_all);
1175611742 for (i = 0; i < PTYPE_HASH_SIZE; i++)
1175711743 INIT_LIST_HEAD(&ptype_base[i]);
1175811744
net/core/dev.h+0−3
@@ -38,10 +38,7 @@ int dev_addr_init(struct net_device *dev); · 37 unmodified lines
3838 void dev_addr_check(struct net_device *dev);
3939
4040 /* sysctls not referred to from outside net/core/ */
41−extern int netdev_budget;
42−extern unsigned int netdev_budget_usecs;
4341 extern unsigned int sysctl_skb_defer_max;
44−extern int netdev_tstamp_prequeue;
4542 extern int netdev_unregister_timeout_secs;
4643 extern int weight_p;
4744 extern int dev_weight_rx_bias;
net/core/gro.c+6−9
@@ -10,9 +10,6 @@ · 9 unmodified lines
1010 #define GRO_MAX_HEAD (MAX_HEADER + 128)
1111
1212 static DEFINE_SPINLOCK(offload_lock);
13−struct list_head offload_base __read_mostly = LIST_HEAD_INIT(offload_base);
14−/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
15−int gro_normal_batch __read_mostly = 8;
1613
1714 /**
1815 * dev_add_offload - register offload handlers
@@ -31,7 +28,7 @@ void dev_add_offload(struct packet_offload *po) · 12 unmodified lines
3128 struct packet_offload *elem;
3229
3330 spin_lock(&offload_lock);
34− list_for_each_entry(elem, &offload_base, list) {
31+ list_for_each_entry(elem, &net_hotdata.offload_base, list) {
3532 if (po->priority < elem->priority)
3633 break;
3734 }
@@ -55,7 +52,7 @@ EXPORT_SYMBOL(dev_add_offload); · 17 unmodified lines
5552 */
5653 static void __dev_remove_offload(struct packet_offload *po)
5754 {
58− struct list_head *head = &offload_base;
55+ struct list_head *head = &net_hotdata.offload_base;
5956 struct packet_offload *po1;
6057
6158 spin_lock(&offload_lock);
@@ -235,9 +232,9 @@ done: · 173 unmodified lines
235232
236233 static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
237234 {
235+ struct list_head *head = &net_hotdata.offload_base;
238236 struct packet_offload *ptype;
239237 __be16 type = skb->protocol;
240− struct list_head *head = &offload_base;
241238 int err = -ENOENT;
242239
243240 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
@@ -444,7 +441,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff · 200 unmodified lines
444441 {
445442 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
446443 struct gro_list *gro_list = &napi->gro_hash[bucket];
447− struct list_head *head = &offload_base;
444+ struct list_head *head = &net_hotdata.offload_base;
448445 struct packet_offload *ptype;
449446 __be16 type = skb->protocol;
450447 struct sk_buff *pp = NULL;
@@ -550,7 +547,7 @@ normal: · 99 unmodified lines
550547
551548 struct packet_offload *gro_find_receive_by_type(__be16 type)
552549 {
553− struct list_head *offload_head = &offload_base;
550+ struct list_head *offload_head = &net_hotdata.offload_base;
554551 struct packet_offload *ptype;
555552
556553 list_for_each_entry_rcu(ptype, offload_head, list) {
@@ -564,7 +561,7 @@ EXPORT_SYMBOL(gro_find_receive_by_type); · 7 unmodified lines
564561
565562 struct packet_offload *gro_find_complete_by_type(__be16 type)
566563 {
567− struct list_head *offload_head = &offload_base;
564+ struct list_head *offload_head = &net_hotdata.offload_base;
568565 struct packet_offload *ptype;
569566
570567 list_for_each_entry_rcu(ptype, offload_head, list) {
net/core/gro_cells.c+2−1
@@ -3,6 +3,7 @@ · 2 unmodified lines
33 #include <linux/slab.h>
44 #include <linux/netdevice.h>
55 #include <net/gro_cells.h>
6+#include <net/hotdata.h>
67
78 struct gro_cell {
89 struct sk_buff_head napi_skbs;
@@ -26,7 +27,7 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) · 17 unmodified lines
2627
2728 cell = this_cpu_ptr(gcells->cells);
2829
29− if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(netdev_max_backlog)) {
30+ if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(net_hotdata.max_backlog)) {
3031 drop:
3132 dev_core_stats_rx_dropped_inc(dev);
3233 kfree_skb(skb);
net/core/gso.c+2−2
@@ -17,7 +17,7 @@ struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb, · 16 unmodified lines
1717 struct packet_offload *ptype;
1818
1919 rcu_read_lock();
20− list_for_each_entry_rcu(ptype, &offload_base, list) {
20+ list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) {
2121 if (ptype->type == type && ptype->callbacks.gso_segment) {
2222 segs = ptype->callbacks.gso_segment(skb, features);
2323 break;
@@ -48,7 +48,7 @@ struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, · 24 unmodified lines
4848 __skb_pull(skb, vlan_depth);
4949
5050 rcu_read_lock();
51− list_for_each_entry_rcu(ptype, &offload_base, list) {
51+ list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) {
5252 if (ptype->type == type && ptype->callbacks.gso_segment) {
5353 segs = ptype->callbacks.gso_segment(skb, features);
5454 break;
net/core/hotdata.c+22−0
@@ -0,0 +1,22 @@
1+// SPDX-License-Identifier: GPL-2.0-or-later
2+#include <net/hotdata.h>
3+#include <linux/cache.h>
4+#include <linux/jiffies.h>
5+#include <linux/list.h>
6+
7+
8+struct net_hotdata net_hotdata __cacheline_aligned = {
9+ .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
10+ .ptype_all = LIST_HEAD_INIT(net_hotdata.ptype_all),
11+ .gro_normal_batch = 8,
12+
13+ .netdev_budget = 300,
14+ /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
15+ .netdev_budget_usecs = 2 * USEC_PER_SEC / HZ,
16+
17+ .tstamp_prequeue = 1,
18+ .max_backlog = 1000,
19+ .dev_tx_weight = 64,
20+ .dev_rx_weight = 64,
21+};
22+EXPORT_SYMBOL(net_hotdata);
net/core/net-procfs.c+4−3
@@ -3,6 +3,7 @@ · 2 unmodified lines
33 #include <linux/proc_fs.h>
44 #include <linux/seq_file.h>
55 #include <net/wext.h>
6+#include <net/hotdata.h>
67
78 #include "dev.h"
89
@@ -183,7 +184,7 @@ static void *ptype_get_idx(struct seq_file *seq, loff_t pos) · 174 unmodified lines
183184 }
184185 }
185186
186− list_for_each_entry_rcu(pt, &ptype_all, list) {
187+ list_for_each_entry_rcu(pt, &net_hotdata.ptype_all, list) {
187188 if (i == pos)
188189 return pt;
189190 ++i;
@@ -231,13 +232,13 @@ static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos) · 41 unmodified lines
231232 }
232233 }
233234
234− nxt = ptype_all.next;
235+ nxt = net_hotdata.ptype_all.next;
235236 goto ptype_all;
236237 }
237238
238239 if (pt->type == htons(ETH_P_ALL)) {
239240 ptype_all:
240− if (nxt != &ptype_all)
241+ if (nxt != &net_hotdata.ptype_all)
241242 goto found;
242243 hash = 0;
243244 nxt = ptype_base[0].next;
net/core/net-sysfs.c+1−0
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424 #include <linux/of_net.h>
2525 #include <linux/cpu.h>
2626 #include <net/netdev_rx_queue.h>
27+#include <net/rps.h>
2728
2829 #include "dev.h"
2930 #include "net-sysfs.h"
net/core/skbuff.c+20−24
@@ -69,6 +69,7 @@ · 68 unmodified lines
6969 #include <net/sock.h>
7070 #include <net/checksum.h>
7171 #include <net/gso.h>
72+#include <net/hotdata.h>
7273 #include <net/ip6_checksum.h>
7374 #include <net/xfrm.h>
7475 #include <net/mpls.h>
@@ -88,15 +89,10 @@ · 13 unmodified lines
8889 #include "dev.h"
8990 #include "sock_destructor.h"
9091
91−struct kmem_cache *skbuff_cache __ro_after_init;
92−static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
9392 #ifdef CONFIG_SKB_EXTENSIONS
9493 static struct kmem_cache *skbuff_ext_cache __ro_after_init;
9594 #endif
9695
97−
98−static struct kmem_cache *skb_small_head_cache __ro_after_init;
99−
10096 #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER)
10197
10298 /* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two.
@@ -349,7 +345,7 @@ static struct sk_buff *napi_skb_cache_get(void) · 246 unmodified lines
349345 struct sk_buff *skb;
350346
351347 if (unlikely(!nc->skb_count)) {
352− nc->skb_count = kmem_cache_alloc_bulk(skbuff_cache,
348+ nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
353349 GFP_ATOMIC,
354350 NAPI_SKB_CACHE_BULK,
355351 nc->skb_cache);
@@ -358,7 +354,7 @@ static struct sk_buff *napi_skb_cache_get(void) · 2 unmodified lines
358354 }
359355
360356 skb = nc->skb_cache[--nc->skb_count];
361− kasan_mempool_unpoison_object(skb, kmem_cache_size(skbuff_cache));
357+ kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache));
362358
363359 return skb;
364360 }
@@ -416,7 +412,7 @@ struct sk_buff *slab_build_skb(void *data) · 51 unmodified lines
416412 struct sk_buff *skb;
417413 unsigned int size;
418414
419− skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);
415+ skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
420416 if (unlikely(!skb))
421417 return NULL;
422418
@@ -467,7 +463,7 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size) · 44 unmodified lines
467463 {
468464 struct sk_buff *skb;
469465
470− skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);
466+ skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
471467 if (unlikely(!skb))
472468 return NULL;
473469
@@ -578,7 +574,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, · 104 unmodified lines
578574 obj_size = SKB_HEAD_ALIGN(*size);
579575 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&
580576 !(flags & KMALLOC_NOT_NORMAL_BITS)) {
581− obj = kmem_cache_alloc_node(skb_small_head_cache,
577+ obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
582578 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
583579 node);
584580 *size = SKB_SMALL_HEAD_CACHE_SIZE;
@@ -586,7 +582,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, · 1 unmodified line
586582 goto out;
587583 /* Try again but now we are using pfmemalloc reserves */
588584 ret_pfmemalloc = true;
589− obj = kmem_cache_alloc_node(skb_small_head_cache, flags, node);
585+ obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node);
590586 goto out;
591587 }
592588
@@ -649,7 +645,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, · 56 unmodified lines
649645 u8 *data;
650646
651647 cache = (flags & SKB_ALLOC_FCLONE)
652− ? skbuff_fclone_cache : skbuff_cache;
648+ ? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache;
653649
654650 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
655651 gfp_mask |= __GFP_MEMALLOC;
@@ -1095,7 +1091,7 @@ static int skb_pp_frag_ref(struct sk_buff *skb) · 439 unmodified lines
10951091 static void skb_kfree_head(void *head, unsigned int end_offset)
10961092 {
10971093 if (end_offset == SKB_SMALL_HEAD_HEADROOM)
1098− kmem_cache_free(skb_small_head_cache, head);
1094+ kmem_cache_free(net_hotdata.skb_small_head_cache, head);
10991095 else
11001096 kfree(head);
11011097 }
@@ -1162,7 +1158,7 @@ static void kfree_skbmem(struct sk_buff *skb) · 60 unmodified lines
11621158
11631159 switch (skb->fclone) {
11641160 case SKB_FCLONE_UNAVAILABLE:
1165− kmem_cache_free(skbuff_cache, skb);
1161+ kmem_cache_free(net_hotdata.skbuff_cache, skb);
11661162 return;
11671163
11681164 case SKB_FCLONE_ORIG:
@@ -1183,7 +1179,7 @@ static void kfree_skbmem(struct sk_buff *skb) · 14 unmodified lines
11831179 if (!refcount_dec_and_test(&fclones->fclone_ref))
11841180 return;
11851181 fastpath:
1186− kmem_cache_free(skbuff_fclone_cache, fclones);
1182+ kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);
11871183 }
11881184
11891185 void skb_release_head_state(struct sk_buff *skb)
@@ -1280,7 +1276,7 @@ static void kfree_skb_add_bulk(struct sk_buff *skb, · 90 unmodified lines
12801276 sa->skb_array[sa->skb_count++] = skb;
12811277
12821278 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
1283− kmem_cache_free_bulk(skbuff_cache, KFREE_SKB_BULK_SIZE,
1279+ kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,
12841280 sa->skb_array);
12851281 sa->skb_count = 0;
12861282 }
@@ -1305,7 +1301,7 @@ kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) · 18 unmodified lines
13051301 }
13061302
13071303 if (sa.skb_count)
1308− kmem_cache_free_bulk(skbuff_cache, sa.skb_count, sa.skb_array);
1304+ kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);
13091305 }
13101306 EXPORT_SYMBOL(kfree_skb_list_reason);
13111307
@@ -1467,9 +1463,9 @@ static void napi_skb_cache_put(struct sk_buff *skb) · 155 unmodified lines
14671463 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
14681464 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)
14691465 kasan_mempool_unpoison_object(nc->skb_cache[i],
1470− kmem_cache_size(skbuff_cache));
1466+ kmem_cache_size(net_hotdata.skbuff_cache));
14711467
1472− kmem_cache_free_bulk(skbuff_cache, NAPI_SKB_CACHE_HALF,
1468+ kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF,
14731469 nc->skb_cache + NAPI_SKB_CACHE_HALF);
14741470 nc->skb_count = NAPI_SKB_CACHE_HALF;
14751471 }
@@ -2066,7 +2062,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) · 590 unmodified lines
20662062 if (skb_pfmemalloc(skb))
20672063 gfp_mask |= __GFP_MEMALLOC;
20682064
2069− n = kmem_cache_alloc(skbuff_cache, gfp_mask);
2065+ n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);
20702066 if (!n)
20712067 return NULL;
20722068
@@ -5005,7 +5001,7 @@ static void skb_extensions_init(void) {} · 2932 unmodified lines
50055001
50065002 void __init skb_init(void)
50075003 {
5008− skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
5004+ net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
50095005 sizeof(struct sk_buff),
50105006 0,
50115007 SLAB_HWCACHE_ALIGN|SLAB_PANIC|
@@ -5013,7 +5009,7 @@ void __init skb_init(void) · 1 unmodified line
50135009 offsetof(struct sk_buff, cb),
50145010 sizeof_field(struct sk_buff, cb),
50155011 NULL);
5016− skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
5012+ net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
50175013 sizeof(struct sk_buff_fclones),
50185014 0,
50195015 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
@@ -5022,7 +5018,7 @@ void __init skb_init(void) · 2 unmodified lines
50225018 * struct skb_shared_info is located at the end of skb->head,
50235019 * and should not be copied to/from user.
50245020 */
5025− skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
5021+ net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
50265022 SKB_SMALL_HEAD_CACHE_SIZE,
50275023 0,
50285024 SLAB_HWCACHE_ALIGN | SLAB_PANIC,
@@ -5895,7 +5891,7 @@ void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) · 866 unmodified lines
58955891 {
58965892 if (head_stolen) {
58975893 skb_release_head_state(skb);
5898− kmem_cache_free(skbuff_cache, skb);
5894+ kmem_cache_free(net_hotdata.skbuff_cache, skb);
58995895 } else {
59005896 __kfree_skb(skb);
59015897 }
net/core/sysctl_net_core.c+15−10
@@ -23,6 +23,8 @@ · 22 unmodified lines
2323 #include <net/net_ratelimit.h>
2424 #include <net/busy_poll.h>
2525 #include <net/pkt_sched.h>
26+#include <net/hotdata.h>
27+#include <net/rps.h>
2628
2729 #include "dev.h"
2830
@@ -138,7 +140,8 @@ static int rps_sock_flow_sysctl(struct ctl_table *table, int write, · 109 unmodified lines
138140
139141 mutex_lock(&sock_flow_mutex);
140142
141− orig_sock_table = rcu_dereference_protected(rps_sock_flow_table,
143+ orig_sock_table = rcu_dereference_protected(
144+ net_hotdata.rps_sock_flow_table,
142145 lockdep_is_held(&sock_flow_mutex));
143146 size = orig_size = orig_sock_table ? orig_sock_table->mask + 1 : 0;
144147
@@ -159,7 +162,8 @@ static int rps_sock_flow_sysctl(struct ctl_table *table, int write, · 14 unmodified lines
159162 mutex_unlock(&sock_flow_mutex);
160163 return -ENOMEM;
161164 }
162− rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1;
165+ net_hotdata.rps_cpu_mask =
166+ roundup_pow_of_two(nr_cpu_ids) - 1;
163167 sock_table->mask = size - 1;
164168 } else
165169 sock_table = orig_sock_table;
@@ -170,7 +174,8 @@ static int rps_sock_flow_sysctl(struct ctl_table *table, int write, · 4 unmodified lines
170174 sock_table = NULL;
171175
172176 if (sock_table != orig_sock_table) {
173− rcu_assign_pointer(rps_sock_flow_table, sock_table);
177+ rcu_assign_pointer(net_hotdata.rps_sock_flow_table,
178+ sock_table);
174179 if (sock_table) {
175180 static_branch_inc(&rps_needed);
176181 static_branch_inc(&rfs_needed);
@@ -300,8 +305,8 @@ static int proc_do_dev_weight(struct ctl_table *table, int write, · 123 unmodified lines
300305 ret = proc_dointvec(table, write, buffer, lenp, ppos);
301306 if (!ret && write) {
302307 weight = READ_ONCE(weight_p);
303− WRITE_ONCE(dev_rx_weight, weight * dev_weight_rx_bias);
304− WRITE_ONCE(dev_tx_weight, weight * dev_weight_tx_bias);
308+ WRITE_ONCE(net_hotdata.dev_rx_weight, weight * dev_weight_rx_bias);
309+ WRITE_ONCE(net_hotdata.dev_tx_weight, weight * dev_weight_tx_bias);
305310 }
306311 mutex_unlock(&dev_weight_mutex);
307312
@@ -439,7 +444,7 @@ static struct ctl_table net_core_table[] = { · 131 unmodified lines
439444 },
440445 {
441446 .procname = "netdev_max_backlog",
442− .data = &netdev_max_backlog,
447+ .data = &net_hotdata.max_backlog,
443448 .maxlen = sizeof(int),
444449 .mode = 0644,
445450 .proc_handler = proc_dointvec
@@ -498,7 +503,7 @@ static struct ctl_table net_core_table[] = { · 52 unmodified lines
498503 #endif
499504 {
500505 .procname = "netdev_tstamp_prequeue",
501− .data = &netdev_tstamp_prequeue,
506+ .data = &net_hotdata.tstamp_prequeue,
502507 .maxlen = sizeof(int),
503508 .mode = 0644,
504509 .proc_handler = proc_dointvec
@@ -576,7 +581,7 @@ static struct ctl_table net_core_table[] = { · 71 unmodified lines
576581 #endif
577582 {
578583 .procname = "netdev_budget",
579− .data = &netdev_budget,
584+ .data = &net_hotdata.netdev_budget,
580585 .maxlen = sizeof(int),
581586 .mode = 0644,
582587 .proc_handler = proc_dointvec
@@ -599,7 +604,7 @@ static struct ctl_table net_core_table[] = { · 16 unmodified lines
599604 },
600605 {
601606 .procname = "netdev_budget_usecs",
602− .data = &netdev_budget_usecs,
607+ .data = &net_hotdata.netdev_budget_usecs,
603608 .maxlen = sizeof(unsigned int),
604609 .mode = 0644,
605610 .proc_handler = proc_dointvec_minmax,
@@ -632,7 +637,7 @@ static struct ctl_table net_core_table[] = { · 26 unmodified lines
632637 },
633638 {
634639 .procname = "gro_normal_batch",
635− .data = &gro_normal_batch,
640+ .data = &net_hotdata.gro_normal_batch,
636641 .maxlen = sizeof(unsigned int),
637642 .mode = 0644,
638643 .proc_handler = proc_dointvec_minmax,
net/core/xdp.c+3−2
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #include <linux/bug.h>
1717 #include <net/page_pool/helpers.h>
1818
19+#include <net/hotdata.h>
1920 #include <net/xdp.h>
2021 #include <net/xdp_priv.h> /* struct xdp_mem_allocator */
2122 #include <trace/events/xdp.h>
@@ -589,7 +590,7 @@ EXPORT_SYMBOL_GPL(xdp_warn); · 567 unmodified lines
589590
590591 int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp)
591592 {
592− n_skb = kmem_cache_alloc_bulk(skbuff_cache, gfp, n_skb, skbs);
593+ n_skb = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, n_skb, skbs);
593594 if (unlikely(!n_skb))
594595 return -ENOMEM;
595596
@@ -658,7 +659,7 @@ struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, · 62 unmodified lines
658659 {
659660 struct sk_buff *skb;
660661
661− skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);
662+ skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
662663 if (unlikely(!skb))
663664 return NULL;
664665
net/ipv4/af_inet.c+25−24
@@ -119,6 +119,7 @@ · 118 unmodified lines
119119 #endif
120120 #include <net/l3mdev.h>
121121 #include <net/compat.h>
122+#include <net/rps.h>
122123
123124 #include <trace/events/sock.h>
124125
@@ -1751,19 +1752,6 @@ static const struct net_protocol igmp_protocol = { · 1626 unmodified lines
17511752 };
17521753 #endif
17531754
1754−static const struct net_protocol tcp_protocol = {
1755− .handler = tcp_v4_rcv,
1756− .err_handler = tcp_v4_err,
1757− .no_policy = 1,
1758− .icmp_strict_tag_validation = 1,
1759−};
1760−
1761−static const struct net_protocol udp_protocol = {
1762− .handler = udp_rcv,
1763− .err_handler = udp_err,
1764− .no_policy = 1,
1765−};
1766−
17671755 static const struct net_protocol icmp_protocol = {
17681756 .handler = icmp_rcv,
17691757 .err_handler = icmp_err,
@@ -1904,14 +1892,6 @@ static int ipv4_proc_init(void); · 134 unmodified lines
19041892 * IP protocol layer initialiser
19051893 */
19061894
1907−static struct packet_offload ip_packet_offload __read_mostly = {
1908− .type = cpu_to_be16(ETH_P_IP),
1909− .callbacks = {
1910− .gso_segment = inet_gso_segment,
1911− .gro_receive = inet_gro_receive,
1912− .gro_complete = inet_gro_complete,
1913− },
1914−};
19151895
19161896 static const struct net_offload ipip_offload = {
19171897 .callbacks = {
@@ -1938,7 +1918,15 @@ static int __init ipv4_offload_init(void) · 20 unmodified lines
19381918 if (ipip_offload_init() < 0)
19391919 pr_crit("%s: Cannot add IPIP protocol offload\n", __func__);
19401920
1941− dev_add_offload(&ip_packet_offload);
1921+ net_hotdata.ip_packet_offload = (struct packet_offload) {
1922+ .type = cpu_to_be16(ETH_P_IP),
1923+ .callbacks = {
1924+ .gso_segment = inet_gso_segment,
1925+ .gro_receive = inet_gro_receive,
1926+ .gro_complete = inet_gro_complete,
1927+ },
1928+ };
1929+ dev_add_offload(&net_hotdata.ip_packet_offload);
19421930 return 0;
19431931 }
19441932
@@ -1992,9 +1980,22 @@ static int __init inet_init(void) · 47 unmodified lines
19921980
19931981 if (inet_add_protocol(&icmp_protocol, IPPROTO_ICMP) < 0)
19941982 pr_crit("%s: Cannot add ICMP protocol\n", __func__);
1995− if (inet_add_protocol(&udp_protocol, IPPROTO_UDP) < 0)
1983+
1984+ net_hotdata.udp_protocol = (struct net_protocol) {
1985+ .handler = udp_rcv,
1986+ .err_handler = udp_err,
1987+ .no_policy = 1,
1988+ };
1989+ if (inet_add_protocol(&net_hotdata.udp_protocol, IPPROTO_UDP) < 0)
19961990 pr_crit("%s: Cannot add UDP protocol\n", __func__);
1997− if (inet_add_protocol(&tcp_protocol, IPPROTO_TCP) < 0)
1991+
1992+ net_hotdata.tcp_protocol = (struct net_protocol) {
1993+ .handler = tcp_v4_rcv,
1994+ .err_handler = tcp_v4_err,
1995+ .no_policy = 1,
1996+ .icmp_strict_tag_validation = 1,
1997+ };
1998+ if (inet_add_protocol(&net_hotdata.tcp_protocol, IPPROTO_TCP) < 0)
19981999 pr_crit("%s: Cannot add TCP protocol\n", __func__);
19992000 #ifdef CONFIG_IP_MULTICAST
20002001 if (inet_add_protocol(&igmp_protocol, IPPROTO_IGMP) < 0)
net/ipv4/inet_hashtables.c+1−2
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424 #include <net/inet6_hashtables.h>
2525 #endif
2626 #include <net/secure_seq.h>
27+#include <net/hotdata.h>
2728 #include <net/ip.h>
2829 #include <net/tcp.h>
2930 #include <net/sock_reuseport.h>
@@ -32,8 +33,6 @@ u32 inet_ehashfn(const struct net *net, const __be32 laddr, · 2 unmodified lines
3233 const __u16 lport, const __be32 faddr,
3334 const __be16 fport)
3435 {
35− static u32 inet_ehash_secret __read_mostly;
36−
3736 net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));
3837
3938 return __inet_ehashfn(laddr, lport, faddr, fport,
net/ipv4/tcp.c+1−0
@@ -279,6 +279,7 @@ · 278 unmodified lines
279279 #include <linux/uaccess.h>
280280 #include <asm/ioctls.h>
281281 #include <net/busy_poll.h>
282+#include <net/rps.h>
282283
283284 /* Track pending CMSGs. */
284285 enum {
net/ipv4/tcp_offload.c+8−9
@@ -345,15 +345,14 @@ INDIRECT_CALLABLE_SCOPE int tcp4_gro_complete(struct sk_buff *skb, int thoff) · 344 unmodified lines
345345 return 0;
346346 }
347347
348−static const struct net_offload tcpv4_offload = {
349− .callbacks = {
350− .gso_segment = tcp4_gso_segment,
351− .gro_receive = tcp4_gro_receive,
352− .gro_complete = tcp4_gro_complete,
353− },
354−};
355−
356348 int __init tcpv4_offload_init(void)
357349 {
358− return inet_add_offload(&tcpv4_offload, IPPROTO_TCP);
350+ net_hotdata.tcpv4_offload = (struct net_offload) {
351+ .callbacks = {
352+ .gso_segment = tcp4_gso_segment,
353+ .gro_receive = tcp4_gro_receive,
354+ .gro_complete = tcp4_gro_complete,
355+ },
356+ };
357+ return inet_add_offload(&net_hotdata.tcpv4_offload, IPPROTO_TCP);
359358 }
net/ipv4/udp.c+0−2
@@ -411,8 +411,6 @@ INDIRECT_CALLABLE_SCOPE · 410 unmodified lines
411411 u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
412412 const __be32 faddr, const __be16 fport)
413413 {
414− static u32 udp_ehash_secret __read_mostly;
415−
416414 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
417415
418416 return __inet_ehashfn(laddr, lport, faddr, fport,
net/ipv4/udp_offload.c+8−9
@@ -737,15 +737,14 @@ INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff) · 736 unmodified lines
737737 return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
738738 }
739739
740−static const struct net_offload udpv4_offload = {
741− .callbacks = {
742− .gso_segment = udp4_ufo_fragment,
743− .gro_receive = udp4_gro_receive,
744− .gro_complete = udp4_gro_complete,
745− },
746−};
747−
748740 int __init udpv4_offload_init(void)
749741 {
750− return inet_add_offload(&udpv4_offload, IPPROTO_UDP);
742+ net_hotdata.udpv4_offload = (struct net_offload) {
743+ .callbacks = {
744+ .gso_segment = udp4_ufo_fragment,
745+ .gro_receive = udp4_gro_receive,
746+ .gro_complete = udp4_gro_complete,
747+ },
748+ };
749+ return inet_add_offload(&net_hotdata.udpv4_offload, IPPROTO_UDP);
751750 }
net/ipv6/af_inet6.c+1−0
@@ -64,6 +64,7 @@ · 63 unmodified lines
6464 #include <net/xfrm.h>
6565 #include <net/ioam6.h>
6666 #include <net/rawv6.h>
67+#include <net/rps.h>
6768
6869 #include <linux/uaccess.h>
6970 #include <linux/mroute6.h>
net/ipv6/inet6_hashtables.c+3−5
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <linux/random.h>
1515
1616 #include <net/addrconf.h>
17+#include <net/hotdata.h>
1718 #include <net/inet_connection_sock.h>
1819 #include <net/inet_hashtables.h>
1920 #include <net/inet6_hashtables.h>
@@ -25,16 +26,13 @@ u32 inet6_ehashfn(const struct net *net, · 5 unmodified lines
2526 const struct in6_addr *laddr, const u16 lport,
2627 const struct in6_addr *faddr, const __be16 fport)
2728 {
28− static u32 inet6_ehash_secret __read_mostly;
29− static u32 ipv6_hash_secret __read_mostly;
30−
3129 u32 lhash, fhash;
3230
3331 net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
34− net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
32+ net_get_random_once(&tcp_ipv6_hash_secret, sizeof(tcp_ipv6_hash_secret));
3533
3634 lhash = (__force u32)laddr->s6_addr32[3];
37− fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);
35+ fhash = __ipv6_addr_jhash(faddr, tcp_ipv6_hash_secret);
3836
3937 return __inet6_ehashfn(lhash, lport, fhash, fport,
4038 inet6_ehash_secret + net_hash_mix(net));
net/ipv6/ip6_offload.c+9−9
@@ -419,14 +419,6 @@ static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff) · 418 unmodified lines
419419 return inet_gro_complete(skb, nhoff);
420420 }
421421
422−static struct packet_offload ipv6_packet_offload __read_mostly = {
423− .type = cpu_to_be16(ETH_P_IPV6),
424− .callbacks = {
425− .gso_segment = ipv6_gso_segment,
426− .gro_receive = ipv6_gro_receive,
427− .gro_complete = ipv6_gro_complete,
428− },
429−};
430422
431423 static struct sk_buff *sit_gso_segment(struct sk_buff *skb,
432424 netdev_features_t features)
@@ -486,7 +478,15 @@ static int __init ipv6_offload_init(void) · 53 unmodified lines
486478 if (ipv6_exthdrs_offload_init() < 0)
487479 pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);
488480
489− dev_add_offload(&ipv6_packet_offload);
481+ net_hotdata.ipv6_packet_offload = (struct packet_offload) {
482+ .type = cpu_to_be16(ETH_P_IPV6),
483+ .callbacks = {
484+ .gso_segment = ipv6_gso_segment,
485+ .gro_receive = ipv6_gro_receive,
486+ .gro_complete = ipv6_gro_complete,
487+ },
488+ };
489+ dev_add_offload(&net_hotdata.ipv6_packet_offload);
490490
491491 inet_add_offload(&sit_offload, IPPROTO_IPV6);
492492 inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6);
net/ipv6/tcp_ipv6.c+9−8
@@ -58,6 +58,7 @@ · 57 unmodified lines
5858 #include <net/timewait_sock.h>
5959 #include <net/inet_common.h>
6060 #include <net/secure_seq.h>
61+#include <net/hotdata.h>
6162 #include <net/busy_poll.h>
6263
6364 #include <linux/proc_fs.h>
@@ -2367,11 +2368,6 @@ struct proto tcpv6_prot = { · 2303 unmodified lines
23672368 };
23682369 EXPORT_SYMBOL_GPL(tcpv6_prot);
23692370
2370−static const struct inet6_protocol tcpv6_protocol = {
2371− .handler = tcp_v6_rcv,
2372− .err_handler = tcp_v6_err,
2373− .flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
2374−};
23752371
23762372 static struct inet_protosw tcpv6_protosw = {
23772373 .type = SOCK_STREAM,
@@ -2408,7 +2404,12 @@ int __init tcpv6_init(void) · 30 unmodified lines
24082404 {
24092405 int ret;
24102406
2411− ret = inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP);
2407+ net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
2408+ .handler = tcp_v6_rcv,
2409+ .err_handler = tcp_v6_err,
2410+ .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
2411+ };
2412+ ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
24122413 if (ret)
24132414 goto out;
24142415
@@ -2433,7 +2434,7 @@ out_tcpv6_pernet_subsys: · 18 unmodified lines
24332434 out_tcpv6_protosw:
24342435 inet6_unregister_protosw(&tcpv6_protosw);
24352436 out_tcpv6_protocol:
2436− inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2437+ inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
24372438 goto out;
24382439 }
24392440
@@ -2441,5 +2442,5 @@ void tcpv6_exit(void) · 1 unmodified line
24412442 {
24422443 unregister_pernet_subsys(&tcpv6_net_ops);
24432444 inet6_unregister_protosw(&tcpv6_protosw);
2444− inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);
2445+ inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
24452446 }
net/ipv6/tcpv6_offload.c+8−8
@@ -66,15 +66,15 @@ static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb, · 65 unmodified lines
6666
6767 return tcp_gso_segment(skb, features);
6868 }
69−static const struct net_offload tcpv6_offload = {
70− .callbacks = {
71− .gso_segment = tcp6_gso_segment,
72− .gro_receive = tcp6_gro_receive,
73− .gro_complete = tcp6_gro_complete,
74− },
75−};
7669
7770 int __init tcpv6_offload_init(void)
7871 {
79− return inet6_add_offload(&tcpv6_offload, IPPROTO_TCP);
72+ net_hotdata.tcpv6_offload = (struct net_offload) {
73+ .callbacks = {
74+ .gso_segment = tcp6_gso_segment,
75+ .gro_receive = tcp6_gro_receive,
76+ .gro_complete = tcp6_gro_complete,
77+ },
78+ };
79+ return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP);
8080 }
net/ipv6/udp.c+8−11
@@ -79,9 +79,6 @@ u32 udp6_ehashfn(const struct net *net, · 78 unmodified lines
7979 const struct in6_addr *faddr,
8080 const __be16 fport)
8181 {
82− static u32 udp6_ehash_secret __read_mostly;
83− static u32 udp_ipv6_hash_secret __read_mostly;
84−
8582 u32 lhash, fhash;
8683
8784 net_get_random_once(&udp6_ehash_secret,
@@ -1702,11 +1699,6 @@ int udpv6_getsockopt(struct sock *sk, int level, int optname, · 1614 unmodified lines
17021699 return ipv6_getsockopt(sk, level, optname, optval, optlen);
17031700 }
17041701
1705−static const struct inet6_protocol udpv6_protocol = {
1706− .handler = udpv6_rcv,
1707− .err_handler = udpv6_err,
1708− .flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
1709−};
17101702
17111703 /* ------------------------------------------------------------------------ */
17121704 #ifdef CONFIG_PROC_FS
@@ -1803,7 +1795,12 @@ int __init udpv6_init(void) · 90 unmodified lines
18031795 {
18041796 int ret;
18051797
1806− ret = inet6_add_protocol(&udpv6_protocol, IPPROTO_UDP);
1798+ net_hotdata.udpv6_protocol = (struct inet6_protocol) {
1799+ .handler = udpv6_rcv,
1800+ .err_handler = udpv6_err,
1801+ .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
1802+ };
1803+ ret = inet6_add_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
18071804 if (ret)
18081805 goto out;
18091806
@@ -1814,12 +1811,12 @@ out: · 4 unmodified lines
18141811 return ret;
18151812
18161813 out_udpv6_protocol:
1817− inet6_del_protocol(&udpv6_protocol, IPPROTO_UDP);
1814+ inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
18181815 goto out;
18191816 }
18201817
18211818 void udpv6_exit(void)
18221819 {
18231820 inet6_unregister_protosw(&udpv6_protosw);
1824− inet6_del_protocol(&udpv6_protocol, IPPROTO_UDP);
1821+ inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
18251822 }
net/ipv6/udp_offload.c+10−11
@@ -192,20 +192,19 @@ INDIRECT_CALLABLE_SCOPE int udp6_gro_complete(struct sk_buff *skb, int nhoff) · 191 unmodified lines
192192 return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
193193 }
194194
195−static const struct net_offload udpv6_offload = {
196− .callbacks = {
197− .gso_segment = udp6_ufo_fragment,
198− .gro_receive = udp6_gro_receive,
199− .gro_complete = udp6_gro_complete,
200− },
201−};
202−
203−int udpv6_offload_init(void)
195+int __init udpv6_offload_init(void)
204196 {
205− return inet6_add_offload(&udpv6_offload, IPPROTO_UDP);
197+ net_hotdata.udpv6_offload = (struct net_offload) {
198+ .callbacks = {
199+ .gso_segment = udp6_ufo_fragment,
200+ .gro_receive = udp6_gro_receive,
201+ .gro_complete = udp6_gro_complete,
202+ },
203+ };
204+ return inet6_add_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP);
206205 }
207206
208207 int udpv6_offload_exit(void)
209208 {
210− return inet6_del_offload(&udpv6_offload, IPPROTO_UDP);
209+ return inet6_del_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP);
211210 }
net/sched/sch_generic.c+2−1
@@ -27,6 +27,7 @@ · 26 unmodified lines
2727 #include <net/sch_generic.h>
2828 #include <net/pkt_sched.h>
2929 #include <net/dst.h>
30+#include <net/hotdata.h>
3031 #include <trace/events/qdisc.h>
3132 #include <trace/events/net.h>
3233 #include <net/xfrm.h>
@@ -409,7 +410,7 @@ static inline bool qdisc_restart(struct Qdisc *q, int *packets) · 376 unmodified lines
409410
410411 void __qdisc_run(struct Qdisc *q)
411412 {
412− int quota = READ_ONCE(dev_tx_weight);
413+ int quota = READ_ONCE(net_hotdata.dev_tx_weight);
413414 int packets;
414415
415416 while (qdisc_restart(q, &packets)) {
net/sctp/socket.c+1−0
@@ -67,6 +67,7 @@ · 66 unmodified lines
6767 #include <net/sctp/sctp.h>
6868 #include <net/sctp/sm.h>
6969 #include <net/sctp/stream_sched.h>
70+#include <net/rps.h>
7071
7172 /* Forward declarations for internal helper functions. */
7273 static bool sctp_writeable(const struct sock *sk);
net/xfrm/espintcp.c+3−1
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010 #if IS_ENABLED(CONFIG_IPV6)
1111 #include <net/ipv6_stubs.h>
1212 #endif
13+#include <net/hotdata.h>
1314
1415 static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb,
1516 struct sock *sk)
@@ -169,7 +170,8 @@ int espintcp_queue_out(struct sock *sk, struct sk_buff *skb) · 153 unmodified lines
169170 {
170171 struct espintcp_ctx *ctx = espintcp_getctx(sk);
171172
172− if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog))
173+ if (skb_queue_len(&ctx->out_queue) >=
174+ READ_ONCE(net_hotdata.max_backlog))
173175 return -ENOBUFS;
174176
175177 __skb_queue_tail(&ctx->out_queue, skb);
net/xfrm/xfrm_input.c+2−1
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 #include <net/ip_tunnels.h>
2222 #include <net/ip6_tunnel.h>
2323 #include <net/dst_metadata.h>
24+#include <net/hotdata.h>
2425
2526 #include "xfrm_inout.h"
2627
@@ -764,7 +765,7 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, · 737 unmodified lines
764765
765766 trans = this_cpu_ptr(&xfrm_trans_tasklet);
766767
767− if (skb_queue_len(&trans->queue) >= READ_ONCE(netdev_max_backlog))
768+ if (skb_queue_len(&trans->queue) >= READ_ONCE(net_hotdata.max_backlog))
768769 return -ENOBUFS;
769770
770771 BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));