History

Merge branch 'net-group-together-hot-data'

Eric Dumazet says: ==================== net: group together hot data While our recent structure reorganizations were focused on increasing max throughput, there is still an area where improvements are much needed. In many cases, a cpu handles one packet at a time, instead of a nice batch. Hardware interrupt. -> Software interrupt. -> Network/Protocol stacks. If the cpu was idle or busy in other layers, it has to pull many cache lines. This series adds a new net_hotdata structure, where some critical (and read-mostly) data used in rx and tx path is packed in a small number of cache lines. Synthetic benchmarks will not see much difference, but latency of single packet should improve. net_hodata current size on 64bit is 416 bytes, but might grow in the future. Also move RPS definitions to a new include file. ====================

Jakub Kicinski committed 2 years ago

Commit e8bb2cc2 parents d3423ed ce7f49a44 files · +391 −320Browse files

Unified · Split

44 files
  1. drivers/net/ethernet/intel/ice/ice_arfs.c +1 −0
  2. drivers/net/ethernet/mellanox/mlx4/en_netdev.c +1 −0
  3. drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +1 −0
  4. drivers/net/ethernet/sfc/rx_common.c +1 −0
  5. drivers/net/ethernet/sfc/siena/rx_common.c +1 −0
  6. drivers/net/tun.c +1 −0
  7. include/linux/netdevice.h +0 −88
  8. include/linux/skbuff.h +0 −1
  9. include/net/gro.h +2 −3
  10. include/net/hotdata.h +52 −0
  11. include/net/protocol.h +3 −0
  12. include/net/rps.h +125 −0
  13. include/net/sock.h +0 −35
  14. kernel/bpf/cpumap.c +3 −1
  15. net/bpf/test_run.c +3 −1
  16. net/core/Makefile +1 −0
  17. net/core/dev.c +22 −36
  18. net/core/dev.h +0 −3
  19. net/core/gro.c +6 −9
  20. net/core/gro_cells.c +2 −1
  21. net/core/gso.c +2 −2
  22. net/core/hotdata.c +22 −0
  23. net/core/net-procfs.c +4 −3
  24. net/core/net-sysfs.c +1 −0
  25. net/core/skbuff.c +20 −24
  26. net/core/sysctl_net_core.c +15 −10
  27. net/core/xdp.c +3 −2
  28. net/ipv4/af_inet.c +25 −24
  29. net/ipv4/inet_hashtables.c +1 −2
  30. net/ipv4/tcp.c +1 −0
  31. net/ipv4/tcp_offload.c +8 −9
  32. net/ipv4/udp.c +0 −2
  33. net/ipv4/udp_offload.c +8 −9
  34. net/ipv6/af_inet6.c +1 −0
  35. net/ipv6/inet6_hashtables.c +3 −5
  36. net/ipv6/ip6_offload.c +9 −9
  37. net/ipv6/tcp_ipv6.c +9 −8
  38. net/ipv6/tcpv6_offload.c +8 −8
  39. net/ipv6/udp.c +8 −11
  40. net/ipv6/udp_offload.c +10 −11
  41. net/sched/sch_generic.c +2 −1
  42. net/sctp/socket.c +1 −0
  43. net/xfrm/espintcp.c +3 −1
  44. net/xfrm/xfrm_input.c +2 −1
drivers/net/ethernet/intel/ice/ice_arfs.c+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
2/* Copyright (C) 2018-2020, Intel Corporation. */2/* Copyright (C) 2018-2020, Intel Corporation. */
3 3
4#include "ice.h"4#include "ice.h"
5#include <net/rps.h>
5 6
6/**7/**
7 * ice_is_arfs_active - helper to check is aRFS is active8 * ice_is_arfs_active - helper to check is aRFS is active
drivers/net/ethernet/mellanox/mlx4/en_netdev.c+1−0
@@ -42,6 +42,7 @@ · 41 unmodified lines
42#include <net/ip.h>42#include <net/ip.h>
43#include <net/vxlan.h>43#include <net/vxlan.h>
44#include <net/devlink.h>44#include <net/devlink.h>
45#include <net/rps.h>
45 46
46#include <linux/mlx4/driver.h>47#include <linux/mlx4/driver.h>
47#include <linux/mlx4/device.h>48#include <linux/mlx4/device.h>
drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c+1−0
@@ -34,6 +34,7 @@ · 33 unmodified lines
34#include <linux/mlx5/fs.h>34#include <linux/mlx5/fs.h>
35#include <linux/ip.h>35#include <linux/ip.h>
36#include <linux/ipv6.h>36#include <linux/ipv6.h>
37#include <net/rps.h>
37#include "en.h"38#include "en.h"
38 39
39#define ARFS_HASH_SHIFT BITS_PER_BYTE40#define ARFS_HASH_SHIFT BITS_PER_BYTE
drivers/net/ethernet/sfc/rx_common.c+1−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
11#include "net_driver.h"11#include "net_driver.h"
12#include <linux/module.h>12#include <linux/module.h>
13#include <linux/iommu.h>13#include <linux/iommu.h>
14#include <net/rps.h>
14#include "efx.h"15#include "efx.h"
15#include "nic.h"16#include "nic.h"
16#include "rx_common.h"17#include "rx_common.h"
drivers/net/ethernet/sfc/siena/rx_common.c+1−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
11#include "net_driver.h"11#include "net_driver.h"
12#include <linux/module.h>12#include <linux/module.h>
13#include <linux/iommu.h>13#include <linux/iommu.h>
14#include <net/rps.h>
14#include "efx.h"15#include "efx.h"
15#include "nic.h"16#include "nic.h"
16#include "rx_common.h"17#include "rx_common.h"
drivers/net/tun.c+1−0
@@ -78,6 +78,7 @@ · 77 unmodified lines
78#include <net/ax25.h>78#include <net/ax25.h>
79#include <net/rose.h>79#include <net/rose.h>
80#include <net/6lowpan.h>80#include <net/6lowpan.h>
81#include <net/rps.h>
81 82
82#include <linux/uaccess.h>83#include <linux/uaccess.h>
83#include <linux/proc_fs.h>84#include <linux/proc_fs.h>
include/linux/netdevice.h+0−88
@@ -225,12 +225,6 @@ struct net_device_core_stats { · 224 unmodified lines
225#include <linux/cache.h>225#include <linux/cache.h>
226#include <linux/skbuff.h>226#include <linux/skbuff.h>
227 227
228#ifdef CONFIG_RPS
229#include <linux/static_key.h>
230extern struct static_key_false rps_needed;
231extern struct static_key_false rfs_needed;
232#endif
233
234struct neighbour;228struct neighbour;
235struct neigh_parms;229struct neigh_parms;
236struct sk_buff;230struct sk_buff;
@@ -730,86 +724,10 @@ static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node · 493 unmodified lines
730#endif724#endif
731}725}
732 726
733#ifdef CONFIG_RPS
734/*
735 * This structure holds an RPS map which can be of variable length. The
736 * map is an array of CPUs.
737 */
738struct rps_map {
739 unsigned int len;
740 struct rcu_head rcu;
741 u16 cpus[];
742};
743#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
744
745/*
746 * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
747 * tail pointer for that CPU's input queue at the time of last enqueue, and
748 * a hardware filter index.
749 */
750struct rps_dev_flow {
751 u16 cpu;
752 u16 filter;
753 unsigned int last_qtail;
754};
755#define RPS_NO_FILTER 0xffff
756
757/*
758 * The rps_dev_flow_table structure contains a table of flow mappings.
759 */
760struct rps_dev_flow_table {
761 unsigned int mask;
762 struct rcu_head rcu;
763 struct rps_dev_flow flows[];
764};
765#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
766 ((_num) * sizeof(struct rps_dev_flow)))
767
768/*
769 * The rps_sock_flow_table contains mappings of flows to the last CPU
770 * on which they were processed by the application (set in recvmsg).
771 * Each entry is a 32bit value. Upper part is the high-order bits
772 * of flow hash, lower part is CPU number.
773 * rps_cpu_mask is used to partition the space, depending on number of
774 * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1
775 * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f,
776 * meaning we use 32-6=26 bits for the hash.
777 */
778struct rps_sock_flow_table {
779 u32 mask;
780
781 u32 ents[] ____cacheline_aligned_in_smp;
782};
783#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
784
785#define RPS_NO_CPU 0xffff
786
787extern u32 rps_cpu_mask;
788extern struct rps_sock_flow_table __rcu *rps_sock_flow_table;
789
790static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
791 u32 hash)
792{
793 if (table && hash) {
794 unsigned int index = hash & table->mask;
795 u32 val = hash & ~rps_cpu_mask;
796
797 /* We only give a hint, preemption can change CPU under us */
798 val |= raw_smp_processor_id();
799
800 /* The following WRITE_ONCE() is paired with the READ_ONCE()
801 * here, and another one in get_rps_cpu().
802 */
803 if (READ_ONCE(table->ents[index]) != val)
804 WRITE_ONCE(table->ents[index], val);
805 }
806}
807
808#ifdef CONFIG_RFS_ACCEL727#ifdef CONFIG_RFS_ACCEL
809bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,728bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
810 u16 filter_id);729 u16 filter_id);
811#endif730#endif
812#endif /* CONFIG_RPS */
813 731
814/* XPS map type and offset of the xps map within net_device->xps_maps[]. */732/* XPS map type and offset of the xps map within net_device->xps_maps[]. */
815enum xps_map_type {733enum xps_map_type {
@@ -4793,11 +4711,6 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, · 3977 unmodified lines
4793 const struct pcpu_sw_netstats __percpu *netstats);4711 const struct pcpu_sw_netstats __percpu *netstats);
4794void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);4712void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
4795 4713
4796extern int netdev_max_backlog;
4797extern int dev_rx_weight;
4798extern int dev_tx_weight;
4799extern int gro_normal_batch;
4800
4801enum {4714enum {
4802 NESTED_SYNC_IMM_BIT,4715 NESTED_SYNC_IMM_BIT,
4803 NESTED_SYNC_TODO_BIT,4716 NESTED_SYNC_TODO_BIT,
@@ -5307,7 +5220,6 @@ static inline const char *netdev_reg_state(const struct net_device *dev) · 503 unmodified lines
5307#define PTYPE_HASH_SIZE (16)5220#define PTYPE_HASH_SIZE (16)
5308#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)5221#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
5309 5222
5310extern struct list_head ptype_all __read_mostly;
5311extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;5223extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
5312 5224
5313extern struct net_device *blackhole_netdev;5225extern struct net_device *blackhole_netdev;
include/linux/skbuff.h+0−1
@@ -1271,7 +1271,6 @@ static inline void consume_skb(struct sk_buff *skb) · 1270 unmodified lines
1271 1271
1272void __consume_stateless_skb(struct sk_buff *skb);1272void __consume_stateless_skb(struct sk_buff *skb);
1273void __kfree_skb(struct sk_buff *skb);1273void __kfree_skb(struct sk_buff *skb);
1274extern struct kmem_cache *skbuff_cache;
1275 1274
1276void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);1275void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
1277bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,1276bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
include/net/gro.h+2−3
@@ -9,6 +9,7 @@ · 8 unmodified lines
9#include <net/ip6_checksum.h>9#include <net/ip6_checksum.h>
10#include <linux/skbuff.h>10#include <linux/skbuff.h>
11#include <net/udp.h>11#include <net/udp.h>
12#include <net/hotdata.h>
12 13
13struct napi_gro_cb {14struct napi_gro_cb {
14 union {15 union {
@@ -446,7 +447,7 @@ static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, · 431 unmodified lines
446{447{
447 list_add_tail(&skb->list, &napi->rx_list);448 list_add_tail(&skb->list, &napi->rx_list);
448 napi->rx_count += segs;449 napi->rx_count += segs;
449 if (napi->rx_count >= READ_ONCE(gro_normal_batch))450 if (napi->rx_count >= READ_ONCE(net_hotdata.gro_normal_batch))
450 gro_normal_list(napi);451 gro_normal_list(napi);
451}452}
452 453
@@ -493,6 +494,4 @@ static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int * · 40 unmodified lines
493#endif494#endif
494}495}
495 496
496extern struct list_head offload_base;
497
498#endif /* _NET_IPV6_GRO_H */497#endif /* _NET_IPV6_GRO_H */
include/net/hotdata.h+52−0
@@ -0,0 +1,52 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2#ifndef _NET_HOTDATA_H
3#define _NET_HOTDATA_H
4
5#include <linux/types.h>
6#include <linux/netdevice.h>
7#include <net/protocol.h>
8
9/* Read mostly data used in network fast paths. */
10struct net_hotdata {
11#if IS_ENABLED(CONFIG_INET)
12 struct packet_offload ip_packet_offload;
13 struct net_offload tcpv4_offload;
14 struct net_protocol tcp_protocol;
15 struct net_offload udpv4_offload;
16 struct net_protocol udp_protocol;
17 struct packet_offload ipv6_packet_offload;
18 struct net_offload tcpv6_offload;
19#if IS_ENABLED(CONFIG_IPV6)
20 struct inet6_protocol tcpv6_protocol;
21 struct inet6_protocol udpv6_protocol;
22#endif
23 struct net_offload udpv6_offload;
24#endif
25 struct list_head offload_base;
26 struct list_head ptype_all;
27 struct kmem_cache *skbuff_cache;
28 struct kmem_cache *skbuff_fclone_cache;
29 struct kmem_cache *skb_small_head_cache;
30#ifdef CONFIG_RPS
31 struct rps_sock_flow_table __rcu *rps_sock_flow_table;
32 u32 rps_cpu_mask;
33#endif
34 int gro_normal_batch;
35 int netdev_budget;
36 int netdev_budget_usecs;
37 int tstamp_prequeue;
38 int max_backlog;
39 int dev_tx_weight;
40 int dev_rx_weight;
41};
42
43#define inet_ehash_secret net_hotdata.tcp_protocol.secret
44#define udp_ehash_secret net_hotdata.udp_protocol.secret
45#define inet6_ehash_secret net_hotdata.tcpv6_protocol.secret
46#define tcp_ipv6_hash_secret net_hotdata.tcpv6_offload.secret
47#define udp6_ehash_secret net_hotdata.udpv6_protocol.secret
48#define udp_ipv6_hash_secret net_hotdata.udpv6_offload.secret
49
50extern struct net_hotdata net_hotdata;
51
52#endif /* _NET_HOTDATA_H */
include/net/protocol.h+3−0
@@ -46,6 +46,7 @@ struct net_protocol { · 45 unmodified lines
46 * socket lookup?46 * socket lookup?
47 */47 */
48 icmp_strict_tag_validation:1;48 icmp_strict_tag_validation:1;
49 u32 secret;
49};50};
50 51
51#if IS_ENABLED(CONFIG_IPV6)52#if IS_ENABLED(CONFIG_IPV6)
@@ -59,6 +60,7 @@ struct inet6_protocol { · 7 unmodified lines
59 __be32 info);60 __be32 info);
60 61
61 unsigned int flags; /* INET6_PROTO_xxx */62 unsigned int flags; /* INET6_PROTO_xxx */
63 u32 secret;
62};64};
63 65
64#define INET6_PROTO_NOPOLICY 0x166#define INET6_PROTO_NOPOLICY 0x1
@@ -68,6 +70,7 @@ struct inet6_protocol { · 3 unmodified lines
68struct net_offload {70struct net_offload {
69 struct offload_callbacks callbacks;71 struct offload_callbacks callbacks;
70 unsigned int flags; /* Flags used by IPv6 for now */72 unsigned int flags; /* Flags used by IPv6 for now */
73 u32 secret;
71};74};
72/* This should be set for any extension header which is compatible with GSO. */75/* This should be set for any extension header which is compatible with GSO. */
73#define INET6_PROTO_GSO_EXTHDR 0x176#define INET6_PROTO_GSO_EXTHDR 0x1
include/net/rps.h+125−0
@@ -0,0 +1,125 @@
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2#ifndef _NET_RPS_H
3#define _NET_RPS_H
4
5#include <linux/types.h>
6#include <linux/static_key.h>
7#include <net/sock.h>
8#include <net/hotdata.h>
9
10#ifdef CONFIG_RPS
11
12extern struct static_key_false rps_needed;
13extern struct static_key_false rfs_needed;
14
15/*
16 * This structure holds an RPS map which can be of variable length. The
17 * map is an array of CPUs.
18 */
19struct rps_map {
20 unsigned int len;
21 struct rcu_head rcu;
22 u16 cpus[];
23};
24#define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + ((_num) * sizeof(u16)))
25
26/*
27 * The rps_dev_flow structure contains the mapping of a flow to a CPU, the
28 * tail pointer for that CPU's input queue at the time of last enqueue, and
29 * a hardware filter index.
30 */
31struct rps_dev_flow {
32 u16 cpu;
33 u16 filter;
34 unsigned int last_qtail;
35};
36#define RPS_NO_FILTER 0xffff
37
38/*
39 * The rps_dev_flow_table structure contains a table of flow mappings.
40 */
41struct rps_dev_flow_table {
42 unsigned int mask;
43 struct rcu_head rcu;
44 struct rps_dev_flow flows[];
45};
46#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
47 ((_num) * sizeof(struct rps_dev_flow)))
48
49/*
50 * The rps_sock_flow_table contains mappings of flows to the last CPU
51 * on which they were processed by the application (set in recvmsg).
52 * Each entry is a 32bit value. Upper part is the high-order bits
53 * of flow hash, lower part is CPU number.
54 * rps_cpu_mask is used to partition the space, depending on number of
55 * possible CPUs : rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1
56 * For example, if 64 CPUs are possible, rps_cpu_mask = 0x3f,
57 * meaning we use 32-6=26 bits for the hash.
58 */
59struct rps_sock_flow_table {
60 u32 mask;
61
62 u32 ents[] ____cacheline_aligned_in_smp;
63};
64#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num]))
65
66#define RPS_NO_CPU 0xffff
67
68static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
69 u32 hash)
70{
71 unsigned int index = hash & table->mask;
72 u32 val = hash & ~net_hotdata.rps_cpu_mask;
73
74 /* We only give a hint, preemption can change CPU under us */
75 val |= raw_smp_processor_id();
76
77 /* The following WRITE_ONCE() is paired with the READ_ONCE()
78 * here, and another one in get_rps_cpu().
79 */
80 if (READ_ONCE(table->ents[index]) != val)
81 WRITE_ONCE(table->ents[index], val);
82}
83
84#endif /* CONFIG_RPS */
85
86static inline void sock_rps_record_flow_hash(__u32 hash)
87{
88#ifdef CONFIG_RPS
89 struct rps_sock_flow_table *sock_flow_table;
90
91 if (!hash)
92 return;
93 rcu_read_lock();
94 sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
95 if (sock_flow_table)
96 rps_record_sock_flow(sock_flow_table, hash);
97 rcu_read_unlock();
98#endif
99}
100
101static inline void sock_rps_record_flow(const struct sock *sk)
102{
103#ifdef CONFIG_RPS
104 if (static_branch_unlikely(&rfs_needed)) {
105 /* Reading sk->sk_rxhash might incur an expensive cache line
106 * miss.
107 *
108 * TCP_ESTABLISHED does cover almost all states where RFS
109 * might be useful, and is cheaper [1] than testing :
110 * IPv4: inet_sk(sk)->inet_daddr
111 * IPv6: ipv6_addr_any(&sk->sk_v6_daddr)
112 * OR an additional socket flag
113 * [1] : sk_state and sk_prot are in the same cache line.
114 */
115 if (sk->sk_state == TCP_ESTABLISHED) {
116 /* This READ_ONCE() is paired with the WRITE_ONCE()
117 * from sock_rps_save_rxhash() and sock_rps_reset_rxhash().
118 */
119 sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash));
120 }
121 }
122#endif
123}
124
125#endif /* _NET_RPS_H */
include/net/sock.h+0−35
@@ -1117,41 +1117,6 @@ static inline void sk_incoming_cpu_update(struct sock *sk) · 1116 unmodified lines
1117 WRITE_ONCE(sk->sk_incoming_cpu, cpu);1117 WRITE_ONCE(sk->sk_incoming_cpu, cpu);
1118}1118}
1119 1119
1120static inline void sock_rps_record_flow_hash(__u32 hash)
1121{
1122#ifdef CONFIG_RPS
1123 struct rps_sock_flow_table *sock_flow_table;
1124
1125 rcu_read_lock();
1126 sock_flow_table = rcu_dereference(rps_sock_flow_table);
1127 rps_record_sock_flow(sock_flow_table, hash);
1128 rcu_read_unlock();
1129#endif
1130}
1131
1132static inline void sock_rps_record_flow(const struct sock *sk)
1133{
1134#ifdef CONFIG_RPS
1135 if (static_branch_unlikely(&rfs_needed)) {
1136 /* Reading sk->sk_rxhash might incur an expensive cache line
1137 * miss.
1138 *
1139 * TCP_ESTABLISHED does cover almost all states where RFS
1140 * might be useful, and is cheaper [1] than testing :
1141 * IPv4: inet_sk(sk)->inet_daddr
1142 * IPv6: ipv6_addr_any(&sk->sk_v6_daddr)
1143 * OR an additional socket flag
1144 * [1] : sk_state and sk_prot are in the same cache line.
1145 */
1146 if (sk->sk_state == TCP_ESTABLISHED) {
1147 /* This READ_ONCE() is paired with the WRITE_ONCE()
1148 * from sock_rps_save_rxhash() and sock_rps_reset_rxhash().
1149 */
1150 sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash));
1151 }
1152 }
1153#endif
1154}
1155 1120
1156static inline void sock_rps_save_rxhash(struct sock *sk,1121static inline void sock_rps_save_rxhash(struct sock *sk,
1157 const struct sk_buff *skb)1122 const struct sk_buff *skb)
kernel/bpf/cpumap.c+3−1
@@ -24,6 +24,7 @@ · 23 unmodified lines
24#include <linux/filter.h>24#include <linux/filter.h>
25#include <linux/ptr_ring.h>25#include <linux/ptr_ring.h>
26#include <net/xdp.h>26#include <net/xdp.h>
27#include <net/hotdata.h>
27 28
28#include <linux/sched.h>29#include <linux/sched.h>
29#include <linux/workqueue.h>30#include <linux/workqueue.h>
@@ -326,7 +327,8 @@ static int cpu_map_kthread_run(void *data) · 296 unmodified lines
326 /* Support running another XDP prog on this CPU */327 /* Support running another XDP prog on this CPU */
327 nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list);328 nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list);
328 if (nframes) {329 if (nframes) {
329 m = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, skbs);330 m = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
331 gfp, nframes, skbs);
330 if (unlikely(m == 0)) {332 if (unlikely(m == 0)) {
331 for (i = 0; i < nframes; i++)333 for (i = 0; i < nframes; i++)
332 skbs[i] = NULL; /* effect: xdp_return_frame */334 skbs[i] = NULL; /* effect: xdp_return_frame */
net/bpf/test_run.c+3−1
@@ -12,6 +12,7 @@ · 11 unmodified lines
12#include <linux/rcupdate_trace.h>12#include <linux/rcupdate_trace.h>
13#include <linux/sched/signal.h>13#include <linux/sched/signal.h>
14#include <net/bpf_sk_storage.h>14#include <net/bpf_sk_storage.h>
15#include <net/hotdata.h>
15#include <net/sock.h>16#include <net/sock.h>
16#include <net/tcp.h>17#include <net/tcp.h>
17#include <net/net_namespace.h>18#include <net/net_namespace.h>
@@ -254,7 +255,8 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes, · 236 unmodified lines
254 int i, n;255 int i, n;
255 LIST_HEAD(list);256 LIST_HEAD(list);
256 257
257 n = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, (void **)skbs);258 n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
259 (void **)skbs);
258 if (unlikely(n == 0)) {260 if (unlikely(n == 0)) {
259 for (i = 0; i < nframes; i++)261 for (i = 0; i < nframes; i++)
260 xdp_return_frame(frames[i]);262 xdp_return_frame(frames[i]);
net/core/Makefile+1−0
@@ -18,6 +18,7 @@ obj-y += dev.o dev_addr_lists.o dst.o netevent.o \ · 17 unmodified lines
18obj-$(CONFIG_NETDEV_ADDR_LIST_TEST) += dev_addr_lists_test.o18obj-$(CONFIG_NETDEV_ADDR_LIST_TEST) += dev_addr_lists_test.o
19 19
20obj-y += net-sysfs.o20obj-y += net-sysfs.o
21obj-y += hotdata.o
21obj-$(CONFIG_PAGE_POOL) += page_pool.o page_pool_user.o22obj-$(CONFIG_PAGE_POOL) += page_pool.o page_pool_user.o
22obj-$(CONFIG_PROC_FS) += net-procfs.o23obj-$(CONFIG_PROC_FS) += net-procfs.o
23obj-$(CONFIG_NET_PKTGEN) += pktgen.o24obj-$(CONFIG_NET_PKTGEN) += pktgen.o
net/core/dev.c+22−36
@@ -155,13 +155,13 @@ · 154 unmodified lines
155#include <net/netdev_rx_queue.h>155#include <net/netdev_rx_queue.h>
156#include <net/page_pool/types.h>156#include <net/page_pool/types.h>
157#include <net/page_pool/helpers.h>157#include <net/page_pool/helpers.h>
158#include <net/rps.h>
158 159
159#include "dev.h"160#include "dev.h"
160#include "net-sysfs.h"161#include "net-sysfs.h"
161 162
162static DEFINE_SPINLOCK(ptype_lock);163static DEFINE_SPINLOCK(ptype_lock);
163struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;164struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
164struct list_head ptype_all __read_mostly; /* Taps */
165 165
166static int netif_rx_internal(struct sk_buff *skb);166static int netif_rx_internal(struct sk_buff *skb);
167static int call_netdevice_notifiers_extack(unsigned long val,167static int call_netdevice_notifiers_extack(unsigned long val,
@@ -540,7 +540,7 @@ static inline void netdev_set_addr_lockdep_class(struct net_device *dev) · 372 unmodified lines
540static inline struct list_head *ptype_head(const struct packet_type *pt)540static inline struct list_head *ptype_head(const struct packet_type *pt)
541{541{
542 if (pt->type == htons(ETH_P_ALL))542 if (pt->type == htons(ETH_P_ALL))
543 return pt->dev ? &pt->dev->ptype_all : &ptype_all;543 return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all;
544 else544 else
545 return pt->dev ? &pt->dev->ptype_specific :545 return pt->dev ? &pt->dev->ptype_specific :
546 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];546 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
@@ -2226,7 +2226,8 @@ static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) · 1679 unmodified lines
2226 */2226 */
2227bool dev_nit_active(struct net_device *dev)2227bool dev_nit_active(struct net_device *dev)
2228{2228{
2229 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);2229 return !list_empty(&net_hotdata.ptype_all) ||
2230 !list_empty(&dev->ptype_all);
2230}2231}
2231EXPORT_SYMBOL_GPL(dev_nit_active);2232EXPORT_SYMBOL_GPL(dev_nit_active);
2232 2233
@@ -2237,10 +2238,9 @@ EXPORT_SYMBOL_GPL(dev_nit_active); · 4 unmodified lines
2237 2238
2238void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)2239void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2239{2240{
2240 struct packet_type *ptype;2241 struct list_head *ptype_list = &net_hotdata.ptype_all;
2242 struct packet_type *ptype, *pt_prev = NULL;
2241 struct sk_buff *skb2 = NULL;2243 struct sk_buff *skb2 = NULL;
2242 struct packet_type *pt_prev = NULL;
2243 struct list_head *ptype_list = &ptype_all;
2244 2244
2245 rcu_read_lock();2245 rcu_read_lock();
2246again:2246again:
@@ -2286,7 +2286,7 @@ again: · 39 unmodified lines
2286 pt_prev = ptype;2286 pt_prev = ptype;
2287 }2287 }
2288 2288
2289 if (ptype_list == &ptype_all) {2289 if (ptype_list == &net_hotdata.ptype_all) {
2290 ptype_list = &dev->ptype_all;2290 ptype_list = &dev->ptype_all;
2291 goto again;2291 goto again;
2292 }2292 }
@@ -4405,19 +4405,10 @@ EXPORT_SYMBOL(__dev_direct_xmit); · 2112 unmodified lines
4405 * Receiver routines4405 * Receiver routines
4406 *************************************************************************/4406 *************************************************************************/
4407 4407
4408int netdev_max_backlog __read_mostly = 1000;
4409EXPORT_SYMBOL(netdev_max_backlog);
4410
4411int netdev_tstamp_prequeue __read_mostly = 1;
4412unsigned int sysctl_skb_defer_max __read_mostly = 64;4408unsigned int sysctl_skb_defer_max __read_mostly = 64;
4413int netdev_budget __read_mostly = 300;
4414/* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4415unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
4416int weight_p __read_mostly = 64; /* old backlog weight */4409int weight_p __read_mostly = 64; /* old backlog weight */
4417int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */4410int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4418int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */4411int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4419int dev_rx_weight __read_mostly = 64;
4420int dev_tx_weight __read_mostly = 64;
4421 4412
4422/* Called with irq disabled */4413/* Called with irq disabled */
4423static inline void ____napi_schedule(struct softnet_data *sd,4414static inline void ____napi_schedule(struct softnet_data *sd,
@@ -4459,12 +4450,6 @@ static inline void ____napi_schedule(struct softnet_data *sd, · 35 unmodified lines
4459 4450
4460#ifdef CONFIG_RPS4451#ifdef CONFIG_RPS
4461 4452
4462/* One global table that all flow-based protocols share. */
4463struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
4464EXPORT_SYMBOL(rps_sock_flow_table);
4465u32 rps_cpu_mask __read_mostly;
4466EXPORT_SYMBOL(rps_cpu_mask);
4467
4468struct static_key_false rps_needed __read_mostly;4453struct static_key_false rps_needed __read_mostly;
4469EXPORT_SYMBOL(rps_needed);4454EXPORT_SYMBOL(rps_needed);
4470struct static_key_false rfs_needed __read_mostly;4455struct static_key_false rfs_needed __read_mostly;
@@ -4556,7 +4541,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 85 unmodified lines
4556 if (!hash)4541 if (!hash)
4557 goto done;4542 goto done;
4558 4543
4559 sock_flow_table = rcu_dereference(rps_sock_flow_table);4544 sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table);
4560 if (flow_table && sock_flow_table) {4545 if (flow_table && sock_flow_table) {
4561 struct rps_dev_flow *rflow;4546 struct rps_dev_flow *rflow;
4562 u32 next_cpu;4547 u32 next_cpu;
@@ -4566,10 +4551,10 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 3 unmodified lines
4566 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().4551 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
4567 */4552 */
4568 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);4553 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
4569 if ((ident ^ hash) & ~rps_cpu_mask)4554 if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask)
4570 goto try_rps;4555 goto try_rps;
4571 4556
4572 next_cpu = ident & rps_cpu_mask;4557 next_cpu = ident & net_hotdata.rps_cpu_mask;
4573 4558
4574 /* OK, now we know there is a match,4559 /* OK, now we know there is a match,
4575 * we can look at the local (per receive queue) flow table4560 * we can look at the local (per receive queue) flow table
@@ -4718,7 +4703,7 @@ static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) · 142 unmodified lines
4718 struct softnet_data *sd;4703 struct softnet_data *sd;
4719 unsigned int old_flow, new_flow;4704 unsigned int old_flow, new_flow;
4720 4705
4721 if (qlen < (READ_ONCE(netdev_max_backlog) >> 1))4706 if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1))
4722 return false;4707 return false;
4723 4708
4724 sd = this_cpu_ptr(&softnet_data);4709 sd = this_cpu_ptr(&softnet_data);
@@ -4766,7 +4751,8 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, · 41 unmodified lines
4766 if (!netif_running(skb->dev))4751 if (!netif_running(skb->dev))
4767 goto drop;4752 goto drop;
4768 qlen = skb_queue_len(&sd->input_pkt_queue);4753 qlen = skb_queue_len(&sd->input_pkt_queue);
4769 if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) {4754 if (qlen <= READ_ONCE(net_hotdata.max_backlog) &&
4755 !skb_flow_limit(skb, qlen)) {
4770 if (qlen) {4756 if (qlen) {
4771enqueue:4757enqueue:
4772 __skb_queue_tail(&sd->input_pkt_queue, skb);4758 __skb_queue_tail(&sd->input_pkt_queue, skb);
@@ -5055,7 +5041,7 @@ static int netif_rx_internal(struct sk_buff *skb) · 282 unmodified lines
5055{5041{
5056 int ret;5042 int ret;
5057 5043
5058 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);5044 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5059 5045
5060 trace_netif_rx(skb);5046 trace_netif_rx(skb);
5061 5047
@@ -5347,7 +5333,7 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, · 285 unmodified lines
5347 int ret = NET_RX_DROP;5333 int ret = NET_RX_DROP;
5348 __be16 type;5334 __be16 type;
5349 5335
5350 net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb);5336 net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5351 5337
5352 trace_netif_receive_skb(skb);5338 trace_netif_receive_skb(skb);
5353 5339
@@ -5391,7 +5377,7 @@ another_round: · 37 unmodified lines
5391 if (pfmemalloc)5377 if (pfmemalloc)
5392 goto skip_taps;5378 goto skip_taps;
5393 5379
5394 list_for_each_entry_rcu(ptype, &ptype_all, list) {5380 list_for_each_entry_rcu(ptype, &net_hotdata.ptype_all, list) {
5395 if (pt_prev)5381 if (pt_prev)
5396 ret = deliver_skb(skb, pt_prev, orig_dev);5382 ret = deliver_skb(skb, pt_prev, orig_dev);
5397 pt_prev = ptype;5383 pt_prev = ptype;
@@ -5731,7 +5717,7 @@ static int netif_receive_skb_internal(struct sk_buff *skb) · 333 unmodified lines
5731{5717{
5732 int ret;5718 int ret;
5733 5719
5734 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);5720 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5735 5721
5736 if (skb_defer_rx_timestamp(skb))5722 if (skb_defer_rx_timestamp(skb))
5737 return NET_RX_SUCCESS;5723 return NET_RX_SUCCESS;
@@ -5761,7 +5747,8 @@ void netif_receive_skb_list_internal(struct list_head *head) · 23 unmodified lines
5761 5747
5762 INIT_LIST_HEAD(&sublist);5748 INIT_LIST_HEAD(&sublist);
5763 list_for_each_entry_safe(skb, next, head, list) {5749 list_for_each_entry_safe(skb, next, head, list) {
5764 net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb);5750 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue),
5751 skb);
5765 skb_list_del_init(skb);5752 skb_list_del_init(skb);
5766 if (!skb_defer_rx_timestamp(skb))5753 if (!skb_defer_rx_timestamp(skb))
5767 list_add_tail(&skb->list, &sublist);5754 list_add_tail(&skb->list, &sublist);
@@ -5985,7 +5972,7 @@ static int process_backlog(struct napi_struct *napi, int quota) · 217 unmodified lines
5985 net_rps_action_and_irq_enable(sd);5972 net_rps_action_and_irq_enable(sd);
5986 }5973 }
5987 5974
5988 napi->weight = READ_ONCE(dev_rx_weight);5975 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight);
5989 while (again) {5976 while (again) {
5990 struct sk_buff *skb;5977 struct sk_buff *skb;
5991 5978
@@ -6790,8 +6777,8 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) · 798 unmodified lines
6790{6777{
6791 struct softnet_data *sd = this_cpu_ptr(&softnet_data);6778 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6792 unsigned long time_limit = jiffies +6779 unsigned long time_limit = jiffies +
6793 usecs_to_jiffies(READ_ONCE(netdev_budget_usecs));6780 usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs));
6794 int budget = READ_ONCE(netdev_budget);6781 int budget = READ_ONCE(net_hotdata.netdev_budget);
6795 LIST_HEAD(list);6782 LIST_HEAD(list);
6796 LIST_HEAD(repoll);6783 LIST_HEAD(repoll);
6797 6784
@@ -11752,7 +11739,6 @@ static int __init net_dev_init(void) · 4954 unmodified lines
11752 if (netdev_kobject_init())11739 if (netdev_kobject_init())
11753 goto out;11740 goto out;
11754 11741
11755 INIT_LIST_HEAD(&ptype_all);
11756 for (i = 0; i < PTYPE_HASH_SIZE; i++)11742 for (i = 0; i < PTYPE_HASH_SIZE; i++)
11757 INIT_LIST_HEAD(&ptype_base[i]);11743 INIT_LIST_HEAD(&ptype_base[i]);
11758 11744
net/core/dev.h+0−3
@@ -38,10 +38,7 @@ int dev_addr_init(struct net_device *dev); · 37 unmodified lines
38void dev_addr_check(struct net_device *dev);38void dev_addr_check(struct net_device *dev);
39 39
40/* sysctls not referred to from outside net/core/ */40/* sysctls not referred to from outside net/core/ */
41extern int netdev_budget;
42extern unsigned int netdev_budget_usecs;
43extern unsigned int sysctl_skb_defer_max;41extern unsigned int sysctl_skb_defer_max;
44extern int netdev_tstamp_prequeue;
45extern int netdev_unregister_timeout_secs;42extern int netdev_unregister_timeout_secs;
46extern int weight_p;43extern int weight_p;
47extern int dev_weight_rx_bias;44extern int dev_weight_rx_bias;
net/core/gro.c+6−9
@@ -10,9 +10,6 @@ · 9 unmodified lines
10#define GRO_MAX_HEAD (MAX_HEADER + 128)10#define GRO_MAX_HEAD (MAX_HEADER + 128)
11 11
12static DEFINE_SPINLOCK(offload_lock);12static DEFINE_SPINLOCK(offload_lock);
13struct list_head offload_base __read_mostly = LIST_HEAD_INIT(offload_base);
14/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
15int gro_normal_batch __read_mostly = 8;
16 13
17/**14/**
18 * dev_add_offload - register offload handlers15 * dev_add_offload - register offload handlers
@@ -31,7 +28,7 @@ void dev_add_offload(struct packet_offload *po) · 12 unmodified lines
31 struct packet_offload *elem;28 struct packet_offload *elem;
32 29
33 spin_lock(&offload_lock);30 spin_lock(&offload_lock);
34 list_for_each_entry(elem, &offload_base, list) {31 list_for_each_entry(elem, &net_hotdata.offload_base, list) {
35 if (po->priority < elem->priority)32 if (po->priority < elem->priority)
36 break;33 break;
37 }34 }
@@ -55,7 +52,7 @@ EXPORT_SYMBOL(dev_add_offload); · 17 unmodified lines
55 */52 */
56static void __dev_remove_offload(struct packet_offload *po)53static void __dev_remove_offload(struct packet_offload *po)
57{54{
58 struct list_head *head = &offload_base;55 struct list_head *head = &net_hotdata.offload_base;
59 struct packet_offload *po1;56 struct packet_offload *po1;
60 57
61 spin_lock(&offload_lock);58 spin_lock(&offload_lock);
@@ -235,9 +232,9 @@ done: · 173 unmodified lines
235 232
236static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)233static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
237{234{
235 struct list_head *head = &net_hotdata.offload_base;
238 struct packet_offload *ptype;236 struct packet_offload *ptype;
239 __be16 type = skb->protocol;237 __be16 type = skb->protocol;
240 struct list_head *head = &offload_base;
241 int err = -ENOENT;238 int err = -ENOENT;
242 239
243 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));240 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
@@ -444,7 +441,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff · 200 unmodified lines
444{441{
445 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);442 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
446 struct gro_list *gro_list = &napi->gro_hash[bucket];443 struct gro_list *gro_list = &napi->gro_hash[bucket];
447 struct list_head *head = &offload_base;444 struct list_head *head = &net_hotdata.offload_base;
448 struct packet_offload *ptype;445 struct packet_offload *ptype;
449 __be16 type = skb->protocol;446 __be16 type = skb->protocol;
450 struct sk_buff *pp = NULL;447 struct sk_buff *pp = NULL;
@@ -550,7 +547,7 @@ normal: · 99 unmodified lines
550 547
551struct packet_offload *gro_find_receive_by_type(__be16 type)548struct packet_offload *gro_find_receive_by_type(__be16 type)
552{549{
553 struct list_head *offload_head = &offload_base;550 struct list_head *offload_head = &net_hotdata.offload_base;
554 struct packet_offload *ptype;551 struct packet_offload *ptype;
555 552
556 list_for_each_entry_rcu(ptype, offload_head, list) {553 list_for_each_entry_rcu(ptype, offload_head, list) {
@@ -564,7 +561,7 @@ EXPORT_SYMBOL(gro_find_receive_by_type); · 7 unmodified lines
564 561
565struct packet_offload *gro_find_complete_by_type(__be16 type)562struct packet_offload *gro_find_complete_by_type(__be16 type)
566{563{
567 struct list_head *offload_head = &offload_base;564 struct list_head *offload_head = &net_hotdata.offload_base;
568 struct packet_offload *ptype;565 struct packet_offload *ptype;
569 566
570 list_for_each_entry_rcu(ptype, offload_head, list) {567 list_for_each_entry_rcu(ptype, offload_head, list) {
net/core/gro_cells.c+2−1
@@ -3,6 +3,7 @@ · 2 unmodified lines
3#include <linux/slab.h>3#include <linux/slab.h>
4#include <linux/netdevice.h>4#include <linux/netdevice.h>
5#include <net/gro_cells.h>5#include <net/gro_cells.h>
6#include <net/hotdata.h>
6 7
7struct gro_cell {8struct gro_cell {
8 struct sk_buff_head napi_skbs;9 struct sk_buff_head napi_skbs;
@@ -26,7 +27,7 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) · 17 unmodified lines
26 27
27 cell = this_cpu_ptr(gcells->cells);28 cell = this_cpu_ptr(gcells->cells);
28 29
29 if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(netdev_max_backlog)) {30 if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(net_hotdata.max_backlog)) {
30drop:31drop:
31 dev_core_stats_rx_dropped_inc(dev);32 dev_core_stats_rx_dropped_inc(dev);
32 kfree_skb(skb);33 kfree_skb(skb);
net/core/gso.c+2−2
@@ -17,7 +17,7 @@ struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb, · 16 unmodified lines
17 struct packet_offload *ptype;17 struct packet_offload *ptype;
18 18
19 rcu_read_lock();19 rcu_read_lock();
20 list_for_each_entry_rcu(ptype, &offload_base, list) {20 list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) {
21 if (ptype->type == type && ptype->callbacks.gso_segment) {21 if (ptype->type == type && ptype->callbacks.gso_segment) {
22 segs = ptype->callbacks.gso_segment(skb, features);22 segs = ptype->callbacks.gso_segment(skb, features);
23 break;23 break;
@@ -48,7 +48,7 @@ struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, · 24 unmodified lines
48 __skb_pull(skb, vlan_depth);48 __skb_pull(skb, vlan_depth);
49 49
50 rcu_read_lock();50 rcu_read_lock();
51 list_for_each_entry_rcu(ptype, &offload_base, list) {51 list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) {
52 if (ptype->type == type && ptype->callbacks.gso_segment) {52 if (ptype->type == type && ptype->callbacks.gso_segment) {
53 segs = ptype->callbacks.gso_segment(skb, features);53 segs = ptype->callbacks.gso_segment(skb, features);
54 break;54 break;
net/core/hotdata.c+22−0
@@ -0,0 +1,22 @@
1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <net/hotdata.h>
3#include <linux/cache.h>
4#include <linux/jiffies.h>
5#include <linux/list.h>
6
7
8struct net_hotdata net_hotdata __cacheline_aligned = {
9 .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
10 .ptype_all = LIST_HEAD_INIT(net_hotdata.ptype_all),
11 .gro_normal_batch = 8,
12
13 .netdev_budget = 300,
14 /* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
15 .netdev_budget_usecs = 2 * USEC_PER_SEC / HZ,
16
17 .tstamp_prequeue = 1,
18 .max_backlog = 1000,
19 .dev_tx_weight = 64,
20 .dev_rx_weight = 64,
21};
22EXPORT_SYMBOL(net_hotdata);
net/core/net-procfs.c+4−3
@@ -3,6 +3,7 @@ · 2 unmodified lines
3#include <linux/proc_fs.h>3#include <linux/proc_fs.h>
4#include <linux/seq_file.h>4#include <linux/seq_file.h>
5#include <net/wext.h>5#include <net/wext.h>
6#include <net/hotdata.h>
6 7
7#include "dev.h"8#include "dev.h"
8 9
@@ -183,7 +184,7 @@ static void *ptype_get_idx(struct seq_file *seq, loff_t pos) · 174 unmodified lines
183 }184 }
184 }185 }
185 186
186 list_for_each_entry_rcu(pt, &ptype_all, list) {187 list_for_each_entry_rcu(pt, &net_hotdata.ptype_all, list) {
187 if (i == pos)188 if (i == pos)
188 return pt;189 return pt;
189 ++i;190 ++i;
@@ -231,13 +232,13 @@ static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos) · 41 unmodified lines
231 }232 }
232 }233 }
233 234
234 nxt = ptype_all.next;235 nxt = net_hotdata.ptype_all.next;
235 goto ptype_all;236 goto ptype_all;
236 }237 }
237 238
238 if (pt->type == htons(ETH_P_ALL)) {239 if (pt->type == htons(ETH_P_ALL)) {
239ptype_all:240ptype_all:
240 if (nxt != &ptype_all)241 if (nxt != &net_hotdata.ptype_all)
241 goto found;242 goto found;
242 hash = 0;243 hash = 0;
243 nxt = ptype_base[0].next;244 nxt = ptype_base[0].next;
net/core/net-sysfs.c+1−0
@@ -24,6 +24,7 @@ · 23 unmodified lines
24#include <linux/of_net.h>24#include <linux/of_net.h>
25#include <linux/cpu.h>25#include <linux/cpu.h>
26#include <net/netdev_rx_queue.h>26#include <net/netdev_rx_queue.h>
27#include <net/rps.h>
27 28
28#include "dev.h"29#include "dev.h"
29#include "net-sysfs.h"30#include "net-sysfs.h"
net/core/skbuff.c+20−24
@@ -69,6 +69,7 @@ · 68 unmodified lines
69#include <net/sock.h>69#include <net/sock.h>
70#include <net/checksum.h>70#include <net/checksum.h>
71#include <net/gso.h>71#include <net/gso.h>
72#include <net/hotdata.h>
72#include <net/ip6_checksum.h>73#include <net/ip6_checksum.h>
73#include <net/xfrm.h>74#include <net/xfrm.h>
74#include <net/mpls.h>75#include <net/mpls.h>
@@ -88,15 +89,10 @@ · 13 unmodified lines
88#include "dev.h"89#include "dev.h"
89#include "sock_destructor.h"90#include "sock_destructor.h"
90 91
91struct kmem_cache *skbuff_cache __ro_after_init;
92static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
93#ifdef CONFIG_SKB_EXTENSIONS92#ifdef CONFIG_SKB_EXTENSIONS
94static struct kmem_cache *skbuff_ext_cache __ro_after_init;93static struct kmem_cache *skbuff_ext_cache __ro_after_init;
95#endif94#endif
96 95
97
98static struct kmem_cache *skb_small_head_cache __ro_after_init;
99
100#define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER)96#define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER)
101 97
102/* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two.98/* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two.
@@ -349,7 +345,7 @@ static struct sk_buff *napi_skb_cache_get(void) · 246 unmodified lines
349 struct sk_buff *skb;345 struct sk_buff *skb;
350 346
351 if (unlikely(!nc->skb_count)) {347 if (unlikely(!nc->skb_count)) {
352 nc->skb_count = kmem_cache_alloc_bulk(skbuff_cache,348 nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
353 GFP_ATOMIC,349 GFP_ATOMIC,
354 NAPI_SKB_CACHE_BULK,350 NAPI_SKB_CACHE_BULK,
355 nc->skb_cache);351 nc->skb_cache);
@@ -358,7 +354,7 @@ static struct sk_buff *napi_skb_cache_get(void) · 2 unmodified lines
358 }354 }
359 355
360 skb = nc->skb_cache[--nc->skb_count];356 skb = nc->skb_cache[--nc->skb_count];
361 kasan_mempool_unpoison_object(skb, kmem_cache_size(skbuff_cache));357 kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache));
362 358
363 return skb;359 return skb;
364}360}
@@ -416,7 +412,7 @@ struct sk_buff *slab_build_skb(void *data) · 51 unmodified lines
416 struct sk_buff *skb;412 struct sk_buff *skb;
417 unsigned int size;413 unsigned int size;
418 414
419 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);415 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
420 if (unlikely(!skb))416 if (unlikely(!skb))
421 return NULL;417 return NULL;
422 418
@@ -467,7 +463,7 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size) · 44 unmodified lines
467{463{
468 struct sk_buff *skb;464 struct sk_buff *skb;
469 465
470 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);466 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
471 if (unlikely(!skb))467 if (unlikely(!skb))
472 return NULL;468 return NULL;
473 469
@@ -578,7 +574,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, · 104 unmodified lines
578 obj_size = SKB_HEAD_ALIGN(*size);574 obj_size = SKB_HEAD_ALIGN(*size);
579 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&575 if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&
580 !(flags & KMALLOC_NOT_NORMAL_BITS)) {576 !(flags & KMALLOC_NOT_NORMAL_BITS)) {
581 obj = kmem_cache_alloc_node(skb_small_head_cache,577 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
582 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,578 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
583 node);579 node);
584 *size = SKB_SMALL_HEAD_CACHE_SIZE;580 *size = SKB_SMALL_HEAD_CACHE_SIZE;
@@ -586,7 +582,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node, · 1 unmodified line
586 goto out;582 goto out;
587 /* Try again but now we are using pfmemalloc reserves */583 /* Try again but now we are using pfmemalloc reserves */
588 ret_pfmemalloc = true;584 ret_pfmemalloc = true;
589 obj = kmem_cache_alloc_node(skb_small_head_cache, flags, node);585 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node);
590 goto out;586 goto out;
591 }587 }
592 588
@@ -649,7 +645,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, · 56 unmodified lines
649 u8 *data;645 u8 *data;
650 646
651 cache = (flags & SKB_ALLOC_FCLONE)647 cache = (flags & SKB_ALLOC_FCLONE)
652 ? skbuff_fclone_cache : skbuff_cache;648 ? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache;
653 649
654 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))650 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
655 gfp_mask |= __GFP_MEMALLOC;651 gfp_mask |= __GFP_MEMALLOC;
@@ -1095,7 +1091,7 @@ static int skb_pp_frag_ref(struct sk_buff *skb) · 439 unmodified lines
1095static void skb_kfree_head(void *head, unsigned int end_offset)1091static void skb_kfree_head(void *head, unsigned int end_offset)
1096{1092{
1097 if (end_offset == SKB_SMALL_HEAD_HEADROOM)1093 if (end_offset == SKB_SMALL_HEAD_HEADROOM)
1098 kmem_cache_free(skb_small_head_cache, head);1094 kmem_cache_free(net_hotdata.skb_small_head_cache, head);
1099 else1095 else
1100 kfree(head);1096 kfree(head);
1101}1097}
@@ -1162,7 +1158,7 @@ static void kfree_skbmem(struct sk_buff *skb) · 60 unmodified lines
1162 1158
1163 switch (skb->fclone) {1159 switch (skb->fclone) {
1164 case SKB_FCLONE_UNAVAILABLE:1160 case SKB_FCLONE_UNAVAILABLE:
1165 kmem_cache_free(skbuff_cache, skb);1161 kmem_cache_free(net_hotdata.skbuff_cache, skb);
1166 return;1162 return;
1167 1163
1168 case SKB_FCLONE_ORIG:1164 case SKB_FCLONE_ORIG:
@@ -1183,7 +1179,7 @@ static void kfree_skbmem(struct sk_buff *skb) · 14 unmodified lines
1183 if (!refcount_dec_and_test(&fclones->fclone_ref))1179 if (!refcount_dec_and_test(&fclones->fclone_ref))
1184 return;1180 return;
1185fastpath:1181fastpath:
1186 kmem_cache_free(skbuff_fclone_cache, fclones);1182 kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);
1187}1183}
1188 1184
1189void skb_release_head_state(struct sk_buff *skb)1185void skb_release_head_state(struct sk_buff *skb)
@@ -1280,7 +1276,7 @@ static void kfree_skb_add_bulk(struct sk_buff *skb, · 90 unmodified lines
1280 sa->skb_array[sa->skb_count++] = skb;1276 sa->skb_array[sa->skb_count++] = skb;
1281 1277
1282 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {1278 if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
1283 kmem_cache_free_bulk(skbuff_cache, KFREE_SKB_BULK_SIZE,1279 kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,
1284 sa->skb_array);1280 sa->skb_array);
1285 sa->skb_count = 0;1281 sa->skb_count = 0;
1286 }1282 }
@@ -1305,7 +1301,7 @@ kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason) · 18 unmodified lines
1305 }1301 }
1306 1302
1307 if (sa.skb_count)1303 if (sa.skb_count)
1308 kmem_cache_free_bulk(skbuff_cache, sa.skb_count, sa.skb_array);1304 kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);
1309}1305}
1310EXPORT_SYMBOL(kfree_skb_list_reason);1306EXPORT_SYMBOL(kfree_skb_list_reason);
1311 1307
@@ -1467,9 +1463,9 @@ static void napi_skb_cache_put(struct sk_buff *skb) · 155 unmodified lines
1467 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {1463 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
1468 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)1464 for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)
1469 kasan_mempool_unpoison_object(nc->skb_cache[i],1465 kasan_mempool_unpoison_object(nc->skb_cache[i],
1470 kmem_cache_size(skbuff_cache));1466 kmem_cache_size(net_hotdata.skbuff_cache));
1471 1467
1472 kmem_cache_free_bulk(skbuff_cache, NAPI_SKB_CACHE_HALF,1468 kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF,
1473 nc->skb_cache + NAPI_SKB_CACHE_HALF);1469 nc->skb_cache + NAPI_SKB_CACHE_HALF);
1474 nc->skb_count = NAPI_SKB_CACHE_HALF;1470 nc->skb_count = NAPI_SKB_CACHE_HALF;
1475 }1471 }
@@ -2066,7 +2062,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) · 590 unmodified lines
2066 if (skb_pfmemalloc(skb))2062 if (skb_pfmemalloc(skb))
2067 gfp_mask |= __GFP_MEMALLOC;2063 gfp_mask |= __GFP_MEMALLOC;
2068 2064
2069 n = kmem_cache_alloc(skbuff_cache, gfp_mask);2065 n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);
2070 if (!n)2066 if (!n)
2071 return NULL;2067 return NULL;
2072 2068
@@ -5005,7 +5001,7 @@ static void skb_extensions_init(void) {} · 2932 unmodified lines
5005 5001
5006void __init skb_init(void)5002void __init skb_init(void)
5007{5003{
5008 skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",5004 net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
5009 sizeof(struct sk_buff),5005 sizeof(struct sk_buff),
5010 0,5006 0,
5011 SLAB_HWCACHE_ALIGN|SLAB_PANIC|5007 SLAB_HWCACHE_ALIGN|SLAB_PANIC|
@@ -5013,7 +5009,7 @@ void __init skb_init(void) · 1 unmodified line
5013 offsetof(struct sk_buff, cb),5009 offsetof(struct sk_buff, cb),
5014 sizeof_field(struct sk_buff, cb),5010 sizeof_field(struct sk_buff, cb),
5015 NULL);5011 NULL);
5016 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",5012 net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
5017 sizeof(struct sk_buff_fclones),5013 sizeof(struct sk_buff_fclones),
5018 0,5014 0,
5019 SLAB_HWCACHE_ALIGN|SLAB_PANIC,5015 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
@@ -5022,7 +5018,7 @@ void __init skb_init(void) · 2 unmodified lines
5022 * struct skb_shared_info is located at the end of skb->head,5018 * struct skb_shared_info is located at the end of skb->head,
5023 * and should not be copied to/from user.5019 * and should not be copied to/from user.
5024 */5020 */
5025 skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",5021 net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
5026 SKB_SMALL_HEAD_CACHE_SIZE,5022 SKB_SMALL_HEAD_CACHE_SIZE,
5027 0,5023 0,
5028 SLAB_HWCACHE_ALIGN | SLAB_PANIC,5024 SLAB_HWCACHE_ALIGN | SLAB_PANIC,
@@ -5895,7 +5891,7 @@ void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) · 866 unmodified lines
5895{5891{
5896 if (head_stolen) {5892 if (head_stolen) {
5897 skb_release_head_state(skb);5893 skb_release_head_state(skb);
5898 kmem_cache_free(skbuff_cache, skb);5894 kmem_cache_free(net_hotdata.skbuff_cache, skb);
5899 } else {5895 } else {
5900 __kfree_skb(skb);5896 __kfree_skb(skb);
5901 }5897 }
net/core/sysctl_net_core.c+15−10
@@ -23,6 +23,8 @@ · 22 unmodified lines
23#include <net/net_ratelimit.h>23#include <net/net_ratelimit.h>
24#include <net/busy_poll.h>24#include <net/busy_poll.h>
25#include <net/pkt_sched.h>25#include <net/pkt_sched.h>
26#include <net/hotdata.h>
27#include <net/rps.h>
26 28
27#include "dev.h"29#include "dev.h"
28 30
@@ -138,7 +140,8 @@ static int rps_sock_flow_sysctl(struct ctl_table *table, int write, · 109 unmodified lines
138 140
139 mutex_lock(&sock_flow_mutex);141 mutex_lock(&sock_flow_mutex);
140 142
141 orig_sock_table = rcu_dereference_protected(rps_sock_flow_table,143 orig_sock_table = rcu_dereference_protected(
144 net_hotdata.rps_sock_flow_table,
142 lockdep_is_held(&sock_flow_mutex));145 lockdep_is_held(&sock_flow_mutex));
143 size = orig_size = orig_sock_table ? orig_sock_table->mask + 1 : 0;146 size = orig_size = orig_sock_table ? orig_sock_table->mask + 1 : 0;
144 147
@@ -159,7 +162,8 @@ static int rps_sock_flow_sysctl(struct ctl_table *table, int write, · 14 unmodified lines
159 mutex_unlock(&sock_flow_mutex);162 mutex_unlock(&sock_flow_mutex);
160 return -ENOMEM;163 return -ENOMEM;
161 }164 }
162 rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1;165 net_hotdata.rps_cpu_mask =
166 roundup_pow_of_two(nr_cpu_ids) - 1;
163 sock_table->mask = size - 1;167 sock_table->mask = size - 1;
164 } else168 } else
165 sock_table = orig_sock_table;169 sock_table = orig_sock_table;
@@ -170,7 +174,8 @@ static int rps_sock_flow_sysctl(struct ctl_table *table, int write, · 4 unmodified lines
170 sock_table = NULL;174 sock_table = NULL;
171 175
172 if (sock_table != orig_sock_table) {176 if (sock_table != orig_sock_table) {
173 rcu_assign_pointer(rps_sock_flow_table, sock_table);177 rcu_assign_pointer(net_hotdata.rps_sock_flow_table,
178 sock_table);
174 if (sock_table) {179 if (sock_table) {
175 static_branch_inc(&rps_needed);180 static_branch_inc(&rps_needed);
176 static_branch_inc(&rfs_needed);181 static_branch_inc(&rfs_needed);
@@ -300,8 +305,8 @@ static int proc_do_dev_weight(struct ctl_table *table, int write, · 123 unmodified lines
300 ret = proc_dointvec(table, write, buffer, lenp, ppos);305 ret = proc_dointvec(table, write, buffer, lenp, ppos);
301 if (!ret && write) {306 if (!ret && write) {
302 weight = READ_ONCE(weight_p);307 weight = READ_ONCE(weight_p);
303 WRITE_ONCE(dev_rx_weight, weight * dev_weight_rx_bias);308 WRITE_ONCE(net_hotdata.dev_rx_weight, weight * dev_weight_rx_bias);
304 WRITE_ONCE(dev_tx_weight, weight * dev_weight_tx_bias);309 WRITE_ONCE(net_hotdata.dev_tx_weight, weight * dev_weight_tx_bias);
305 }310 }
306 mutex_unlock(&dev_weight_mutex);311 mutex_unlock(&dev_weight_mutex);
307 312
@@ -439,7 +444,7 @@ static struct ctl_table net_core_table[] = { · 131 unmodified lines
439 },444 },
440 {445 {
441 .procname = "netdev_max_backlog",446 .procname = "netdev_max_backlog",
442 .data = &netdev_max_backlog,447 .data = &net_hotdata.max_backlog,
443 .maxlen = sizeof(int),448 .maxlen = sizeof(int),
444 .mode = 0644,449 .mode = 0644,
445 .proc_handler = proc_dointvec450 .proc_handler = proc_dointvec
@@ -498,7 +503,7 @@ static struct ctl_table net_core_table[] = { · 52 unmodified lines
498#endif503#endif
499 {504 {
500 .procname = "netdev_tstamp_prequeue",505 .procname = "netdev_tstamp_prequeue",
501 .data = &netdev_tstamp_prequeue,506 .data = &net_hotdata.tstamp_prequeue,
502 .maxlen = sizeof(int),507 .maxlen = sizeof(int),
503 .mode = 0644,508 .mode = 0644,
504 .proc_handler = proc_dointvec509 .proc_handler = proc_dointvec
@@ -576,7 +581,7 @@ static struct ctl_table net_core_table[] = { · 71 unmodified lines
576#endif581#endif
577 {582 {
578 .procname = "netdev_budget",583 .procname = "netdev_budget",
579 .data = &netdev_budget,584 .data = &net_hotdata.netdev_budget,
580 .maxlen = sizeof(int),585 .maxlen = sizeof(int),
581 .mode = 0644,586 .mode = 0644,
582 .proc_handler = proc_dointvec587 .proc_handler = proc_dointvec
@@ -599,7 +604,7 @@ static struct ctl_table net_core_table[] = { · 16 unmodified lines
599 },604 },
600 {605 {
601 .procname = "netdev_budget_usecs",606 .procname = "netdev_budget_usecs",
602 .data = &netdev_budget_usecs,607 .data = &net_hotdata.netdev_budget_usecs,
603 .maxlen = sizeof(unsigned int),608 .maxlen = sizeof(unsigned int),
604 .mode = 0644,609 .mode = 0644,
605 .proc_handler = proc_dointvec_minmax,610 .proc_handler = proc_dointvec_minmax,
@@ -632,7 +637,7 @@ static struct ctl_table net_core_table[] = { · 26 unmodified lines
632 },637 },
633 {638 {
634 .procname = "gro_normal_batch",639 .procname = "gro_normal_batch",
635 .data = &gro_normal_batch,640 .data = &net_hotdata.gro_normal_batch,
636 .maxlen = sizeof(unsigned int),641 .maxlen = sizeof(unsigned int),
637 .mode = 0644,642 .mode = 0644,
638 .proc_handler = proc_dointvec_minmax,643 .proc_handler = proc_dointvec_minmax,
net/core/xdp.c+3−2
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#include <linux/bug.h>16#include <linux/bug.h>
17#include <net/page_pool/helpers.h>17#include <net/page_pool/helpers.h>
18 18
19#include <net/hotdata.h>
19#include <net/xdp.h>20#include <net/xdp.h>
20#include <net/xdp_priv.h> /* struct xdp_mem_allocator */21#include <net/xdp_priv.h> /* struct xdp_mem_allocator */
21#include <trace/events/xdp.h>22#include <trace/events/xdp.h>
@@ -589,7 +590,7 @@ EXPORT_SYMBOL_GPL(xdp_warn); · 567 unmodified lines
589 590
590int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp)591int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp)
591{592{
592 n_skb = kmem_cache_alloc_bulk(skbuff_cache, gfp, n_skb, skbs);593 n_skb = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, n_skb, skbs);
593 if (unlikely(!n_skb))594 if (unlikely(!n_skb))
594 return -ENOMEM;595 return -ENOMEM;
595 596
@@ -658,7 +659,7 @@ struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, · 62 unmodified lines
658{659{
659 struct sk_buff *skb;660 struct sk_buff *skb;
660 661
661 skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC);662 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
662 if (unlikely(!skb))663 if (unlikely(!skb))
663 return NULL;664 return NULL;
664 665
net/ipv4/af_inet.c+25−24
@@ -119,6 +119,7 @@ · 118 unmodified lines
119#endif119#endif
120#include <net/l3mdev.h>120#include <net/l3mdev.h>
121#include <net/compat.h>121#include <net/compat.h>
122#include <net/rps.h>
122 123
123#include <trace/events/sock.h>124#include <trace/events/sock.h>
124 125
@@ -1751,19 +1752,6 @@ static const struct net_protocol igmp_protocol = { · 1626 unmodified lines
1751};1752};
1752#endif1753#endif
1753 1754
1754static const struct net_protocol tcp_protocol = {
1755 .handler = tcp_v4_rcv,
1756 .err_handler = tcp_v4_err,
1757 .no_policy = 1,
1758 .icmp_strict_tag_validation = 1,
1759};
1760
1761static const struct net_protocol udp_protocol = {
1762 .handler = udp_rcv,
1763 .err_handler = udp_err,
1764 .no_policy = 1,
1765};
1766
1767static const struct net_protocol icmp_protocol = {1755static const struct net_protocol icmp_protocol = {
1768 .handler = icmp_rcv,1756 .handler = icmp_rcv,
1769 .err_handler = icmp_err,1757 .err_handler = icmp_err,
@@ -1904,14 +1892,6 @@ static int ipv4_proc_init(void); · 134 unmodified lines
1904 * IP protocol layer initialiser1892 * IP protocol layer initialiser
1905 */1893 */
1906 1894
1907static struct packet_offload ip_packet_offload __read_mostly = {
1908 .type = cpu_to_be16(ETH_P_IP),
1909 .callbacks = {
1910 .gso_segment = inet_gso_segment,
1911 .gro_receive = inet_gro_receive,
1912 .gro_complete = inet_gro_complete,
1913 },
1914};
1915 1895
1916static const struct net_offload ipip_offload = {1896static const struct net_offload ipip_offload = {
1917 .callbacks = {1897 .callbacks = {
@@ -1938,7 +1918,15 @@ static int __init ipv4_offload_init(void) · 20 unmodified lines
1938 if (ipip_offload_init() < 0)1918 if (ipip_offload_init() < 0)
1939 pr_crit("%s: Cannot add IPIP protocol offload\n", __func__);1919 pr_crit("%s: Cannot add IPIP protocol offload\n", __func__);
1940 1920
1941 dev_add_offload(&ip_packet_offload);1921 net_hotdata.ip_packet_offload = (struct packet_offload) {
1922 .type = cpu_to_be16(ETH_P_IP),
1923 .callbacks = {
1924 .gso_segment = inet_gso_segment,
1925 .gro_receive = inet_gro_receive,
1926 .gro_complete = inet_gro_complete,
1927 },
1928 };
1929 dev_add_offload(&net_hotdata.ip_packet_offload);
1942 return 0;1930 return 0;
1943}1931}
1944 1932
@@ -1992,9 +1980,22 @@ static int __init inet_init(void) · 47 unmodified lines
1992 1980
1993 if (inet_add_protocol(&icmp_protocol, IPPROTO_ICMP) < 0)1981 if (inet_add_protocol(&icmp_protocol, IPPROTO_ICMP) < 0)
1994 pr_crit("%s: Cannot add ICMP protocol\n", __func__);1982 pr_crit("%s: Cannot add ICMP protocol\n", __func__);
1995 if (inet_add_protocol(&udp_protocol, IPPROTO_UDP) < 0)1983
1984 net_hotdata.udp_protocol = (struct net_protocol) {
1985 .handler = udp_rcv,
1986 .err_handler = udp_err,
1987 .no_policy = 1,
1988 };
1989 if (inet_add_protocol(&net_hotdata.udp_protocol, IPPROTO_UDP) < 0)
1996 pr_crit("%s: Cannot add UDP protocol\n", __func__);1990 pr_crit("%s: Cannot add UDP protocol\n", __func__);
1997 if (inet_add_protocol(&tcp_protocol, IPPROTO_TCP) < 0)1991
1992 net_hotdata.tcp_protocol = (struct net_protocol) {
1993 .handler = tcp_v4_rcv,
1994 .err_handler = tcp_v4_err,
1995 .no_policy = 1,
1996 .icmp_strict_tag_validation = 1,
1997 };
1998 if (inet_add_protocol(&net_hotdata.tcp_protocol, IPPROTO_TCP) < 0)
1998 pr_crit("%s: Cannot add TCP protocol\n", __func__);1999 pr_crit("%s: Cannot add TCP protocol\n", __func__);
1999#ifdef CONFIG_IP_MULTICAST2000#ifdef CONFIG_IP_MULTICAST
2000 if (inet_add_protocol(&igmp_protocol, IPPROTO_IGMP) < 0)2001 if (inet_add_protocol(&igmp_protocol, IPPROTO_IGMP) < 0)
net/ipv4/inet_hashtables.c+1−2
@@ -24,6 +24,7 @@ · 23 unmodified lines
24#include <net/inet6_hashtables.h>24#include <net/inet6_hashtables.h>
25#endif25#endif
26#include <net/secure_seq.h>26#include <net/secure_seq.h>
27#include <net/hotdata.h>
27#include <net/ip.h>28#include <net/ip.h>
28#include <net/tcp.h>29#include <net/tcp.h>
29#include <net/sock_reuseport.h>30#include <net/sock_reuseport.h>
@@ -32,8 +33,6 @@ u32 inet_ehashfn(const struct net *net, const __be32 laddr, · 2 unmodified lines
32 const __u16 lport, const __be32 faddr,33 const __u16 lport, const __be32 faddr,
33 const __be16 fport)34 const __be16 fport)
34{35{
35 static u32 inet_ehash_secret __read_mostly;
36
37 net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));36 net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));
38 37
39 return __inet_ehashfn(laddr, lport, faddr, fport,38 return __inet_ehashfn(laddr, lport, faddr, fport,
net/ipv4/tcp.c+1−0
@@ -279,6 +279,7 @@ · 278 unmodified lines
279#include <linux/uaccess.h>279#include <linux/uaccess.h>
280#include <asm/ioctls.h>280#include <asm/ioctls.h>
281#include <net/busy_poll.h>281#include <net/busy_poll.h>
282#include <net/rps.h>
282 283
283/* Track pending CMSGs. */284/* Track pending CMSGs. */
284enum {285enum {
net/ipv4/tcp_offload.c+8−9
@@ -345,15 +345,14 @@ INDIRECT_CALLABLE_SCOPE int tcp4_gro_complete(struct sk_buff *skb, int thoff) · 344 unmodified lines
345 return 0;345 return 0;
346}346}
347 347
348static const struct net_offload tcpv4_offload = {
349 .callbacks = {
350 .gso_segment = tcp4_gso_segment,
351 .gro_receive = tcp4_gro_receive,
352 .gro_complete = tcp4_gro_complete,
353 },
354};
355
356int __init tcpv4_offload_init(void)348int __init tcpv4_offload_init(void)
357{349{
358 return inet_add_offload(&tcpv4_offload, IPPROTO_TCP);350 net_hotdata.tcpv4_offload = (struct net_offload) {
351 .callbacks = {
352 .gso_segment = tcp4_gso_segment,
353 .gro_receive = tcp4_gro_receive,
354 .gro_complete = tcp4_gro_complete,
355 },
356 };
357 return inet_add_offload(&net_hotdata.tcpv4_offload, IPPROTO_TCP);
359}358}
net/ipv4/udp.c+0−2
@@ -411,8 +411,6 @@ INDIRECT_CALLABLE_SCOPE · 410 unmodified lines
411u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,411u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
412 const __be32 faddr, const __be16 fport)412 const __be32 faddr, const __be16 fport)
413{413{
414 static u32 udp_ehash_secret __read_mostly;
415
416 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));414 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
417 415
418 return __inet_ehashfn(laddr, lport, faddr, fport,416 return __inet_ehashfn(laddr, lport, faddr, fport,
net/ipv4/udp_offload.c+8−9
@@ -737,15 +737,14 @@ INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff) · 736 unmodified lines
737 return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);737 return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
738}738}
739 739
740static const struct net_offload udpv4_offload = {
741 .callbacks = {
742 .gso_segment = udp4_ufo_fragment,
743 .gro_receive = udp4_gro_receive,
744 .gro_complete = udp4_gro_complete,
745 },
746};
747
748int __init udpv4_offload_init(void)740int __init udpv4_offload_init(void)
749{741{
750 return inet_add_offload(&udpv4_offload, IPPROTO_UDP);742 net_hotdata.udpv4_offload = (struct net_offload) {
743 .callbacks = {
744 .gso_segment = udp4_ufo_fragment,
745 .gro_receive = udp4_gro_receive,
746 .gro_complete = udp4_gro_complete,
747 },
748 };
749 return inet_add_offload(&net_hotdata.udpv4_offload, IPPROTO_UDP);
751}750}
net/ipv6/af_inet6.c+1−0
@@ -64,6 +64,7 @@ · 63 unmodified lines
64#include <net/xfrm.h>64#include <net/xfrm.h>
65#include <net/ioam6.h>65#include <net/ioam6.h>
66#include <net/rawv6.h>66#include <net/rawv6.h>
67#include <net/rps.h>
67 68
68#include <linux/uaccess.h>69#include <linux/uaccess.h>
69#include <linux/mroute6.h>70#include <linux/mroute6.h>
net/ipv6/inet6_hashtables.c+3−5
@@ -14,6 +14,7 @@ · 13 unmodified lines
14#include <linux/random.h>14#include <linux/random.h>
15 15
16#include <net/addrconf.h>16#include <net/addrconf.h>
17#include <net/hotdata.h>
17#include <net/inet_connection_sock.h>18#include <net/inet_connection_sock.h>
18#include <net/inet_hashtables.h>19#include <net/inet_hashtables.h>
19#include <net/inet6_hashtables.h>20#include <net/inet6_hashtables.h>
@@ -25,16 +26,13 @@ u32 inet6_ehashfn(const struct net *net, · 5 unmodified lines
25 const struct in6_addr *laddr, const u16 lport,26 const struct in6_addr *laddr, const u16 lport,
26 const struct in6_addr *faddr, const __be16 fport)27 const struct in6_addr *faddr, const __be16 fport)
27{28{
28 static u32 inet6_ehash_secret __read_mostly;
29 static u32 ipv6_hash_secret __read_mostly;
30
31 u32 lhash, fhash;29 u32 lhash, fhash;
32 30
33 net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));31 net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
34 net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));32 net_get_random_once(&tcp_ipv6_hash_secret, sizeof(tcp_ipv6_hash_secret));
35 33
36 lhash = (__force u32)laddr->s6_addr32[3];34 lhash = (__force u32)laddr->s6_addr32[3];
37 fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);35 fhash = __ipv6_addr_jhash(faddr, tcp_ipv6_hash_secret);
38 36
39 return __inet6_ehashfn(lhash, lport, fhash, fport,37 return __inet6_ehashfn(lhash, lport, fhash, fport,
40 inet6_ehash_secret + net_hash_mix(net));38 inet6_ehash_secret + net_hash_mix(net));
net/ipv6/ip6_offload.c+9−9
@@ -419,14 +419,6 @@ static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff) · 418 unmodified lines
419 return inet_gro_complete(skb, nhoff);419 return inet_gro_complete(skb, nhoff);
420}420}
421 421
422static struct packet_offload ipv6_packet_offload __read_mostly = {
423 .type = cpu_to_be16(ETH_P_IPV6),
424 .callbacks = {
425 .gso_segment = ipv6_gso_segment,
426 .gro_receive = ipv6_gro_receive,
427 .gro_complete = ipv6_gro_complete,
428 },
429};
430 422
431static struct sk_buff *sit_gso_segment(struct sk_buff *skb,423static struct sk_buff *sit_gso_segment(struct sk_buff *skb,
432 netdev_features_t features)424 netdev_features_t features)
@@ -486,7 +478,15 @@ static int __init ipv6_offload_init(void) · 53 unmodified lines
486 if (ipv6_exthdrs_offload_init() < 0)478 if (ipv6_exthdrs_offload_init() < 0)
487 pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);479 pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);
488 480
489 dev_add_offload(&ipv6_packet_offload);481 net_hotdata.ipv6_packet_offload = (struct packet_offload) {
482 .type = cpu_to_be16(ETH_P_IPV6),
483 .callbacks = {
484 .gso_segment = ipv6_gso_segment,
485 .gro_receive = ipv6_gro_receive,
486 .gro_complete = ipv6_gro_complete,
487 },
488 };
489 dev_add_offload(&net_hotdata.ipv6_packet_offload);
490 490
491 inet_add_offload(&sit_offload, IPPROTO_IPV6);491 inet_add_offload(&sit_offload, IPPROTO_IPV6);
492 inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6);492 inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6);
net/ipv6/tcp_ipv6.c+9−8
@@ -58,6 +58,7 @@ · 57 unmodified lines
58#include <net/timewait_sock.h>58#include <net/timewait_sock.h>
59#include <net/inet_common.h>59#include <net/inet_common.h>
60#include <net/secure_seq.h>60#include <net/secure_seq.h>
61#include <net/hotdata.h>
61#include <net/busy_poll.h>62#include <net/busy_poll.h>
62 63
63#include <linux/proc_fs.h>64#include <linux/proc_fs.h>
@@ -2367,11 +2368,6 @@ struct proto tcpv6_prot = { · 2303 unmodified lines
2367};2368};
2368EXPORT_SYMBOL_GPL(tcpv6_prot);2369EXPORT_SYMBOL_GPL(tcpv6_prot);
2369 2370
2370static const struct inet6_protocol tcpv6_protocol = {
2371 .handler = tcp_v6_rcv,
2372 .err_handler = tcp_v6_err,
2373 .flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
2374};
2375 2371
2376static struct inet_protosw tcpv6_protosw = {2372static struct inet_protosw tcpv6_protosw = {
2377 .type = SOCK_STREAM,2373 .type = SOCK_STREAM,
@@ -2408,7 +2404,12 @@ int __init tcpv6_init(void) · 30 unmodified lines
2408{2404{
2409 int ret;2405 int ret;
2410 2406
2411 ret = inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP);2407 net_hotdata.tcpv6_protocol = (struct inet6_protocol) {
2408 .handler = tcp_v6_rcv,
2409 .err_handler = tcp_v6_err,
2410 .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
2411 };
2412 ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2412 if (ret)2413 if (ret)
2413 goto out;2414 goto out;
2414 2415
@@ -2433,7 +2434,7 @@ out_tcpv6_pernet_subsys: · 18 unmodified lines
2433out_tcpv6_protosw:2434out_tcpv6_protosw:
2434 inet6_unregister_protosw(&tcpv6_protosw);2435 inet6_unregister_protosw(&tcpv6_protosw);
2435out_tcpv6_protocol:2436out_tcpv6_protocol:
2436 inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);2437 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2437 goto out;2438 goto out;
2438}2439}
2439 2440
@@ -2441,5 +2442,5 @@ void tcpv6_exit(void) · 1 unmodified line
2441{2442{
2442 unregister_pernet_subsys(&tcpv6_net_ops);2443 unregister_pernet_subsys(&tcpv6_net_ops);
2443 inet6_unregister_protosw(&tcpv6_protosw);2444 inet6_unregister_protosw(&tcpv6_protosw);
2444 inet6_del_protocol(&tcpv6_protocol, IPPROTO_TCP);2445 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
2445}2446}
net/ipv6/tcpv6_offload.c+8−8
@@ -66,15 +66,15 @@ static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb, · 65 unmodified lines
66 66
67 return tcp_gso_segment(skb, features);67 return tcp_gso_segment(skb, features);
68}68}
69static const struct net_offload tcpv6_offload = {
70 .callbacks = {
71 .gso_segment = tcp6_gso_segment,
72 .gro_receive = tcp6_gro_receive,
73 .gro_complete = tcp6_gro_complete,
74 },
75};
76 69
77int __init tcpv6_offload_init(void)70int __init tcpv6_offload_init(void)
78{71{
79 return inet6_add_offload(&tcpv6_offload, IPPROTO_TCP);72 net_hotdata.tcpv6_offload = (struct net_offload) {
73 .callbacks = {
74 .gso_segment = tcp6_gso_segment,
75 .gro_receive = tcp6_gro_receive,
76 .gro_complete = tcp6_gro_complete,
77 },
78 };
79 return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP);
80}80}
net/ipv6/udp.c+8−11
@@ -79,9 +79,6 @@ u32 udp6_ehashfn(const struct net *net, · 78 unmodified lines
79 const struct in6_addr *faddr,79 const struct in6_addr *faddr,
80 const __be16 fport)80 const __be16 fport)
81{81{
82 static u32 udp6_ehash_secret __read_mostly;
83 static u32 udp_ipv6_hash_secret __read_mostly;
84
85 u32 lhash, fhash;82 u32 lhash, fhash;
86 83
87 net_get_random_once(&udp6_ehash_secret,84 net_get_random_once(&udp6_ehash_secret,
@@ -1702,11 +1699,6 @@ int udpv6_getsockopt(struct sock *sk, int level, int optname, · 1614 unmodified lines
1702 return ipv6_getsockopt(sk, level, optname, optval, optlen);1699 return ipv6_getsockopt(sk, level, optname, optval, optlen);
1703}1700}
1704 1701
1705static const struct inet6_protocol udpv6_protocol = {
1706 .handler = udpv6_rcv,
1707 .err_handler = udpv6_err,
1708 .flags = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
1709};
1710 1702
1711/* ------------------------------------------------------------------------ */1703/* ------------------------------------------------------------------------ */
1712#ifdef CONFIG_PROC_FS1704#ifdef CONFIG_PROC_FS
@@ -1803,7 +1795,12 @@ int __init udpv6_init(void) · 90 unmodified lines
1803{1795{
1804 int ret;1796 int ret;
1805 1797
1806 ret = inet6_add_protocol(&udpv6_protocol, IPPROTO_UDP);1798 net_hotdata.udpv6_protocol = (struct inet6_protocol) {
1799 .handler = udpv6_rcv,
1800 .err_handler = udpv6_err,
1801 .flags = INET6_PROTO_NOPOLICY | INET6_PROTO_FINAL,
1802 };
1803 ret = inet6_add_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
1807 if (ret)1804 if (ret)
1808 goto out;1805 goto out;
1809 1806
@@ -1814,12 +1811,12 @@ out: · 4 unmodified lines
1814 return ret;1811 return ret;
1815 1812
1816out_udpv6_protocol:1813out_udpv6_protocol:
1817 inet6_del_protocol(&udpv6_protocol, IPPROTO_UDP);1814 inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
1818 goto out;1815 goto out;
1819}1816}
1820 1817
1821void udpv6_exit(void)1818void udpv6_exit(void)
1822{1819{
1823 inet6_unregister_protosw(&udpv6_protosw);1820 inet6_unregister_protosw(&udpv6_protosw);
1824 inet6_del_protocol(&udpv6_protocol, IPPROTO_UDP);1821 inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP);
1825}1822}
net/ipv6/udp_offload.c+10−11
@@ -192,20 +192,19 @@ INDIRECT_CALLABLE_SCOPE int udp6_gro_complete(struct sk_buff *skb, int nhoff) · 191 unmodified lines
192 return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);192 return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
193}193}
194 194
195static const struct net_offload udpv6_offload = {195int __init udpv6_offload_init(void)
196 .callbacks = {
197 .gso_segment = udp6_ufo_fragment,
198 .gro_receive = udp6_gro_receive,
199 .gro_complete = udp6_gro_complete,
200 },
201};
202
203int udpv6_offload_init(void)
204{196{
205 return inet6_add_offload(&udpv6_offload, IPPROTO_UDP);197 net_hotdata.udpv6_offload = (struct net_offload) {
198 .callbacks = {
199 .gso_segment = udp6_ufo_fragment,
200 .gro_receive = udp6_gro_receive,
201 .gro_complete = udp6_gro_complete,
202 },
203 };
204 return inet6_add_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP);
206}205}
207 206
208int udpv6_offload_exit(void)207int udpv6_offload_exit(void)
209{208{
210 return inet6_del_offload(&udpv6_offload, IPPROTO_UDP);209 return inet6_del_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP);
211}210}
net/sched/sch_generic.c+2−1
@@ -27,6 +27,7 @@ · 26 unmodified lines
27#include <net/sch_generic.h>27#include <net/sch_generic.h>
28#include <net/pkt_sched.h>28#include <net/pkt_sched.h>
29#include <net/dst.h>29#include <net/dst.h>
30#include <net/hotdata.h>
30#include <trace/events/qdisc.h>31#include <trace/events/qdisc.h>
31#include <trace/events/net.h>32#include <trace/events/net.h>
32#include <net/xfrm.h>33#include <net/xfrm.h>
@@ -409,7 +410,7 @@ static inline bool qdisc_restart(struct Qdisc *q, int *packets) · 376 unmodified lines
409 410
410void __qdisc_run(struct Qdisc *q)411void __qdisc_run(struct Qdisc *q)
411{412{
412 int quota = READ_ONCE(dev_tx_weight);413 int quota = READ_ONCE(net_hotdata.dev_tx_weight);
413 int packets;414 int packets;
414 415
415 while (qdisc_restart(q, &packets)) {416 while (qdisc_restart(q, &packets)) {
net/sctp/socket.c+1−0
@@ -67,6 +67,7 @@ · 66 unmodified lines
67#include <net/sctp/sctp.h>67#include <net/sctp/sctp.h>
68#include <net/sctp/sm.h>68#include <net/sctp/sm.h>
69#include <net/sctp/stream_sched.h>69#include <net/sctp/stream_sched.h>
70#include <net/rps.h>
70 71
71/* Forward declarations for internal helper functions. */72/* Forward declarations for internal helper functions. */
72static bool sctp_writeable(const struct sock *sk);73static bool sctp_writeable(const struct sock *sk);
net/xfrm/espintcp.c+3−1
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#if IS_ENABLED(CONFIG_IPV6)10#if IS_ENABLED(CONFIG_IPV6)
11#include <net/ipv6_stubs.h>11#include <net/ipv6_stubs.h>
12#endif12#endif
13#include <net/hotdata.h>
13 14
14static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb,15static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb,
15 struct sock *sk)16 struct sock *sk)
@@ -169,7 +170,8 @@ int espintcp_queue_out(struct sock *sk, struct sk_buff *skb) · 153 unmodified lines
169{170{
170 struct espintcp_ctx *ctx = espintcp_getctx(sk);171 struct espintcp_ctx *ctx = espintcp_getctx(sk);
171 172
172 if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog))173 if (skb_queue_len(&ctx->out_queue) >=
174 READ_ONCE(net_hotdata.max_backlog))
173 return -ENOBUFS;175 return -ENOBUFS;
174 176
175 __skb_queue_tail(&ctx->out_queue, skb);177 __skb_queue_tail(&ctx->out_queue, skb);
net/xfrm/xfrm_input.c+2−1
@@ -21,6 +21,7 @@ · 20 unmodified lines
21#include <net/ip_tunnels.h>21#include <net/ip_tunnels.h>
22#include <net/ip6_tunnel.h>22#include <net/ip6_tunnel.h>
23#include <net/dst_metadata.h>23#include <net/dst_metadata.h>
24#include <net/hotdata.h>
24 25
25#include "xfrm_inout.h"26#include "xfrm_inout.h"
26 27
@@ -764,7 +765,7 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, · 737 unmodified lines
764 765
765 trans = this_cpu_ptr(&xfrm_trans_tasklet);766 trans = this_cpu_ptr(&xfrm_trans_tasklet);
766 767
767 if (skb_queue_len(&trans->queue) >= READ_ONCE(netdev_max_backlog))768 if (skb_queue_len(&trans->queue) >= READ_ONCE(net_hotdata.max_backlog))
768 return -ENOBUFS;769 return -ENOBUFS;
769 770
770 BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));771 BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));