History

net-sysfs: use rps_tag_ptr and remove metadata from rps_dev_flow_table

Instead of storing the @log at the beginning of rps_dev_flow_table use 5 low order bits of the rps_tag_ptr to store the log of the size. This removes a potential cache line miss (for light traffic). This allows us to switch to one high-order allocation instead of vmalloc() when CONFIG_RFS_ACCEL is not set.

Eric Dumazet committed 8 months ago

Commit a4351631 parent b2cc6184 files · +67 −69Browse files

Unified · Split

4 files
  1. include/net/netdev_rx_queue.h +2 −1
  2. include/net/rps.h +0 −10
  3. net/core/dev.c +31 −22
  4. net/core/net-sysfs.c +34 −36
include/net/netdev_rx_queue.h+2−1
@@ -8,13 +8,14 @@ · 7 unmodified lines
88 #include <net/xdp.h>
99 #include <net/page_pool/types.h>
1010 #include <net/netdev_queues.h>
11+#include <net/rps-types.h>
1112
1213 /* This structure contains an instance of an RX queue. */
1314 struct netdev_rx_queue {
1415 struct xdp_rxq_info xdp_rxq;
1516 #ifdef CONFIG_RPS
1617 struct rps_map __rcu *rps_map;
17− struct rps_dev_flow_table __rcu *rps_flow_table;
18+ rps_tag_ptr rps_flow_table;
1819 #endif
1920 struct kobject kobj;
2021 const struct attribute_group **groups;
include/net/rps.h+0−10
@@ -39,16 +39,6 @@ struct rps_dev_flow { · 38 unmodified lines
3939 };
4040 #define RPS_NO_FILTER 0xffff
4141
42−/*
43− * The rps_dev_flow_table structure contains a table of flow mappings.
44− */
45−struct rps_dev_flow_table {
46− u8 log;
47− struct rps_dev_flow flows[];
48−};
49−#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
50− ((_num) * sizeof(struct rps_dev_flow)))
51−
5242 /*
5343 * The rps_sock_flow_table contains mappings of flows to the last CPU
5444 * on which they were processed by the application (set in recvmsg).
net/core/dev.c+31−22
@@ -4968,16 +4968,16 @@ EXPORT_SYMBOL(rps_needed); · 4967 unmodified lines
49684968 struct static_key_false rfs_needed __read_mostly;
49694969 EXPORT_SYMBOL(rfs_needed);
49704970
4971−static u32 rfs_slot(u32 hash, const struct rps_dev_flow_table *flow_table)
4971+static u32 rfs_slot(u32 hash, rps_tag_ptr tag_ptr)
49724972 {
4973− return hash_32(hash, flow_table->log);
4973+ return hash_32(hash, rps_tag_to_log(tag_ptr));
49744974 }
49754975
49764976 #ifdef CONFIG_RFS_ACCEL
49774977 /**
49784978 * rps_flow_is_active - check whether the flow is recently active.
49794979 * @rflow: Specific flow to check activity.
4980− * @flow_table: per-queue flowtable that @rflow belongs to.
4980+ * @log: ilog2(hashsize).
49814981 * @cpu: CPU saved in @rflow.
49824982 *
49834983 * If the CPU has processed many packets since the flow's last activity
@@ -4986,7 +4986,7 @@ static u32 rfs_slot(u32 hash, const struct rps_dev_flow_table *flow_table) · 2 unmodified lines
49864986 * Return: true if flow was recently active.
49874987 */
49884988 static bool rps_flow_is_active(struct rps_dev_flow *rflow,
4989− struct rps_dev_flow_table *flow_table,
4989+ u8 log,
49904990 unsigned int cpu)
49914991 {
49924992 unsigned int flow_last_active;
@@ -4999,7 +4999,7 @@ static bool rps_flow_is_active(struct rps_dev_flow *rflow, · 6 unmodified lines
49994999 flow_last_active = READ_ONCE(rflow->last_qtail);
50005000
50015001 return (int)(sd_input_head - flow_last_active) <
5002− (int)(10 << flow_table->log);
5002+ (int)(10 << log);
50035003 }
50045004 #endif
50055005
@@ -5011,9 +5011,10 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 5 unmodified lines
50115011 u32 head;
50125012 #ifdef CONFIG_RFS_ACCEL
50135013 struct netdev_rx_queue *rxqueue;
5014− struct rps_dev_flow_table *flow_table;
5014+ struct rps_dev_flow *flow_table;
50155015 struct rps_dev_flow *old_rflow;
50165016 struct rps_dev_flow *tmp_rflow;
5017+ rps_tag_ptr q_tag_ptr;
50175018 unsigned int tmp_cpu;
50185019 u16 rxq_index;
50195020 u32 flow_id;
@@ -5028,16 +5029,18 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 8 unmodified lines
50285029 goto out;
50295030
50305031 rxqueue = dev->_rx + rxq_index;
5031− flow_table = rcu_dereference(rxqueue->rps_flow_table);
5032− if (!flow_table)
5032+ q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5033+ if (!q_tag_ptr)
50335034 goto out;
50345035
5035− flow_id = rfs_slot(hash, flow_table);
5036− tmp_rflow = &flow_table->flows[flow_id];
5036+ flow_id = rfs_slot(hash, q_tag_ptr);
5037+ flow_table = rps_tag_to_table(q_tag_ptr);
5038+ tmp_rflow = flow_table + flow_id;
50375039 tmp_cpu = READ_ONCE(tmp_rflow->cpu);
50385040
50395041 if (READ_ONCE(tmp_rflow->filter) != RPS_NO_FILTER) {
5040− if (rps_flow_is_active(tmp_rflow, flow_table,
5042+ if (rps_flow_is_active(tmp_rflow,
5043+ rps_tag_to_log(q_tag_ptr),
50415044 tmp_cpu)) {
50425045 if (hash != READ_ONCE(tmp_rflow->hash) ||
50435046 next_cpu == tmp_cpu)
@@ -5076,8 +5079,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 32 unmodified lines
50765079 struct rps_dev_flow **rflowp)
50775080 {
50785081 struct netdev_rx_queue *rxqueue = dev->_rx;
5079− struct rps_dev_flow_table *flow_table;
5080− rps_tag_ptr global_tag_ptr;
5082+ rps_tag_ptr global_tag_ptr, q_tag_ptr;
50815083 struct rps_map *map;
50825084 int cpu = -1;
50835085 u32 tcpu;
@@ -5098,9 +5100,9 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 14 unmodified lines
50985100
50995101 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
51005102
5101− flow_table = rcu_dereference(rxqueue->rps_flow_table);
5103+ q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
51025104 map = rcu_dereference(rxqueue->rps_map);
5103− if (!flow_table && !map)
5105+ if (!q_tag_ptr && !map)
51045106 goto done;
51055107
51065108 skb_reset_network_header(skb);
@@ -5109,8 +5111,9 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 2 unmodified lines
51095111 goto done;
51105112
51115113 global_tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
5112− if (flow_table && global_tag_ptr) {
5114+ if (q_tag_ptr && global_tag_ptr) {
51135115 struct rps_sock_flow_table *sock_flow_table;
5116+ struct rps_dev_flow *flow_table;
51145117 struct rps_dev_flow *rflow;
51155118 u32 next_cpu;
51165119 u32 flow_id;
@@ -5130,7 +5133,9 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 13 unmodified lines
51305133 /* OK, now we know there is a match,
51315134 * we can look at the local (per receive queue) flow table
51325135 */
5133− rflow = &flow_table->flows[rfs_slot(hash, flow_table)];
5136+ flow_id = rfs_slot(hash, q_tag_ptr);
5137+ flow_table = rps_tag_to_table(q_tag_ptr);
5138+ rflow = flow_table + flow_id;
51345139 tcpu = rflow->cpu;
51355140
51365141 /*
@@ -5190,19 +5195,23 @@ bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, · 53 unmodified lines
51905195 u32 flow_id, u16 filter_id)
51915196 {
51925197 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
5193− struct rps_dev_flow_table *flow_table;
5198+ struct rps_dev_flow *flow_table;
51945199 struct rps_dev_flow *rflow;
5200+ rps_tag_ptr q_tag_ptr;
51955201 bool expire = true;
5202+ u8 log;
51965203
51975204 rcu_read_lock();
5198− flow_table = rcu_dereference(rxqueue->rps_flow_table);
5199− if (flow_table && flow_id < (1UL << flow_table->log)) {
5205+ q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5206+ log = rps_tag_to_log(q_tag_ptr);
5207+ if (q_tag_ptr && flow_id < (1UL << log)) {
52005208 unsigned int cpu;
52015209
5202− rflow = &flow_table->flows[flow_id];
5210+ flow_table = rps_tag_to_table(q_tag_ptr);
5211+ rflow = flow_table + flow_id;
52035212 cpu = READ_ONCE(rflow->cpu);
52045213 if (READ_ONCE(rflow->filter) == filter_id &&
5205− rps_flow_is_active(rflow, flow_table, cpu))
5214+ rps_flow_is_active(rflow, log, cpu))
52065215 expire = false;
52075216 }
52085217 rcu_read_unlock();
net/core/net-sysfs.c+34−36
@@ -1060,14 +1060,12 @@ out: · 1059 unmodified lines
10601060 static ssize_t show_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,
10611061 char *buf)
10621062 {
1063− struct rps_dev_flow_table *flow_table;
10641063 unsigned long val = 0;
1064+ rps_tag_ptr tag_ptr;
10651065
1066− rcu_read_lock();
1067− flow_table = rcu_dereference(queue->rps_flow_table);
1068− if (flow_table)
1069− val = 1UL << flow_table->log;
1070− rcu_read_unlock();
1066+ tag_ptr = READ_ONCE(queue->rps_flow_table);
1067+ if (tag_ptr)
1068+ val = 1UL << rps_tag_to_log(tag_ptr);
10711069
10721070 return sysfs_emit(buf, "%lu\n", val);
10731071 }
@@ -1075,8 +1073,10 @@ static ssize_t show_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue, · 1 unmodified line
10751073 static ssize_t store_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,
10761074 const char *buf, size_t len)
10771075 {
1076+ rps_tag_ptr otag, tag_ptr = 0UL;
1077+ struct rps_dev_flow *table;
10781078 unsigned long mask, count;
1079− struct rps_dev_flow_table *table, *old_table;
1079+ size_t sz;
10801080 int rc;
10811081
10821082 if (!capable(CAP_NET_ADMIN))
@@ -1093,38 +1093,36 @@ static ssize_t store_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue, · 10 unmodified lines
10931093 */
10941094 while ((mask | (mask >> 1)) != mask)
10951095 mask |= (mask >> 1);
1096− /* On 64 bit arches, must check mask fits in table->mask (u32),
1097− * and on 32bit arches, must check
1098− * RPS_DEV_FLOW_TABLE_SIZE(mask + 1) doesn't overflow.
1099− */
1100−#if BITS_PER_LONG > 32
1101− if (mask > (unsigned long)(u32)mask)
1102− return -EINVAL;
1103−#else
1104− if (mask > (ULONG_MAX - RPS_DEV_FLOW_TABLE_SIZE(1))
1105− / sizeof(struct rps_dev_flow)) {
1106− /* Enforce a limit to prevent overflow */
1096+
1097+ /* Do not accept too large tables. */
1098+ if (mask > (INT_MAX / sizeof(*table) - 1))
11071099 return -EINVAL;
1108− }
1109−#endif
1110− table = vmalloc(RPS_DEV_FLOW_TABLE_SIZE(mask + 1));
1100+
1101+ sz = max_t(size_t, sizeof(*table) * (mask + 1),
1102+ PAGE_SIZE);
1103+ if (sz <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER) ||
1104+ is_power_of_2(sizeof(*table)))
1105+ table = kvmalloc(sz, GFP_KERNEL);
1106+ else
1107+ table = vmalloc(sz);
11111108 if (!table)
11121109 return -ENOMEM;
1113−
1114− table->log = ilog2(mask) + 1;
1110+ tag_ptr = (rps_tag_ptr)table;
1111+ if (rps_tag_to_log(tag_ptr)) {
1112+ pr_err_once("store_rps_dev_flow_table_cnt() got a non page aligned allocation.\n");
1113+ kvfree(table);
1114+ return -ENOMEM;
1115+ }
1116+ tag_ptr |= (ilog2(mask) + 1);
11151117 for (count = 0; count <= mask; count++) {
1116− table->flows[count].cpu = RPS_NO_CPU;
1117− table->flows[count].filter = RPS_NO_FILTER;
1118+ table[count].cpu = RPS_NO_CPU;
1119+ table[count].filter = RPS_NO_FILTER;
11181120 }
1119− } else {
1120− table = NULL;
11211121 }
11221122
1123− old_table = unrcu_pointer(xchg(&queue->rps_flow_table,
1124− RCU_INITIALIZER(table)));
1125−
1126− if (old_table)
1127− kvfree_rcu_mightsleep(old_table);
1123+ otag = xchg(&queue->rps_flow_table, tag_ptr);
1124+ if (otag)
1125+ kvfree_rcu_mightsleep(rps_tag_to_table(otag));
11281126
11291127 return len;
11301128 }
@@ -1150,7 +1148,7 @@ static void rx_queue_release(struct kobject *kobj) · 19 unmodified lines
11501148 {
11511149 struct netdev_rx_queue *queue = to_rx_queue(kobj);
11521150 #ifdef CONFIG_RPS
1153− struct rps_dev_flow_table *old_table;
1151+ rps_tag_ptr tag_ptr;
11541152 struct rps_map *map;
11551153
11561154 map = rcu_dereference_protected(queue->rps_map, 1);
@@ -1159,9 +1157,9 @@ static void rx_queue_release(struct kobject *kobj) · 2 unmodified lines
11591157 kfree_rcu(map, rcu);
11601158 }
11611159
1162− old_table = unrcu_pointer(xchg(&queue->rps_flow_table, NULL));
1163− if (old_table)
1164− kvfree_rcu_mightsleep(old_table);
1160+ tag_ptr = xchg(&queue->rps_flow_table, 0UL);
1161+ if (tag_ptr)
1162+ kvfree_rcu_mightsleep(rps_tag_to_table(tag_ptr));
11651163 #endif
11661164
11671165 memset(kobj, 0, sizeof(*kobj));