History

net-sysfs: use rps_tag_ptr and remove metadata from rps_dev_flow_table

Instead of storing the @log at the beginning of rps_dev_flow_table use 5 low order bits of the rps_tag_ptr to store the log of the size. This removes a potential cache line miss (for light traffic). This allows us to switch to one high-order allocation instead of vmalloc() when CONFIG_RFS_ACCEL is not set.

Eric Dumazet committed 8 months ago

Commit a4351631 parent b2cc6184 files · +67 −69Browse files

Unified · Split

4 files
  1. include/net/netdev_rx_queue.h +2 −1
  2. include/net/rps.h +0 −10
  3. net/core/dev.c +31 −22
  4. net/core/net-sysfs.c +34 −36
include/net/netdev_rx_queue.h+2−1
@@ -8,13 +8,14 @@ · 7 unmodified lines
8#include <net/xdp.h>8#include <net/xdp.h>
9#include <net/page_pool/types.h>9#include <net/page_pool/types.h>
10#include <net/netdev_queues.h>10#include <net/netdev_queues.h>
11#include <net/rps-types.h>
11 12
12/* This structure contains an instance of an RX queue. */13/* This structure contains an instance of an RX queue. */
13struct netdev_rx_queue {14struct netdev_rx_queue {
14 struct xdp_rxq_info xdp_rxq;15 struct xdp_rxq_info xdp_rxq;
15#ifdef CONFIG_RPS16#ifdef CONFIG_RPS
16 struct rps_map __rcu *rps_map;17 struct rps_map __rcu *rps_map;
17 struct rps_dev_flow_table __rcu *rps_flow_table;18 rps_tag_ptr rps_flow_table;
18#endif19#endif
19 struct kobject kobj;20 struct kobject kobj;
20 const struct attribute_group **groups;21 const struct attribute_group **groups;
include/net/rps.h+0−10
@@ -39,16 +39,6 @@ struct rps_dev_flow { · 38 unmodified lines
39};39};
40#define RPS_NO_FILTER 0xffff40#define RPS_NO_FILTER 0xffff
41 41
42/*
43 * The rps_dev_flow_table structure contains a table of flow mappings.
44 */
45struct rps_dev_flow_table {
46 u8 log;
47 struct rps_dev_flow flows[];
48};
49#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
50 ((_num) * sizeof(struct rps_dev_flow)))
51
52/*42/*
53 * The rps_sock_flow_table contains mappings of flows to the last CPU43 * The rps_sock_flow_table contains mappings of flows to the last CPU
54 * on which they were processed by the application (set in recvmsg).44 * on which they were processed by the application (set in recvmsg).
net/core/dev.c+31−22
@@ -4968,16 +4968,16 @@ EXPORT_SYMBOL(rps_needed); · 4967 unmodified lines
4968struct static_key_false rfs_needed __read_mostly;4968struct static_key_false rfs_needed __read_mostly;
4969EXPORT_SYMBOL(rfs_needed);4969EXPORT_SYMBOL(rfs_needed);
4970 4970
4971static u32 rfs_slot(u32 hash, const struct rps_dev_flow_table *flow_table)4971static u32 rfs_slot(u32 hash, rps_tag_ptr tag_ptr)
4972{4972{
4973 return hash_32(hash, flow_table->log);4973 return hash_32(hash, rps_tag_to_log(tag_ptr));
4974}4974}
4975 4975
4976#ifdef CONFIG_RFS_ACCEL4976#ifdef CONFIG_RFS_ACCEL
4977/**4977/**
4978 * rps_flow_is_active - check whether the flow is recently active.4978 * rps_flow_is_active - check whether the flow is recently active.
4979 * @rflow: Specific flow to check activity.4979 * @rflow: Specific flow to check activity.
4980 * @flow_table: per-queue flowtable that @rflow belongs to.4980 * @log: ilog2(hashsize).
4981 * @cpu: CPU saved in @rflow.4981 * @cpu: CPU saved in @rflow.
4982 *4982 *
4983 * If the CPU has processed many packets since the flow's last activity4983 * If the CPU has processed many packets since the flow's last activity
@@ -4986,7 +4986,7 @@ static u32 rfs_slot(u32 hash, const struct rps_dev_flow_table *flow_table) · 2 unmodified lines
4986 * Return: true if flow was recently active.4986 * Return: true if flow was recently active.
4987 */4987 */
4988static bool rps_flow_is_active(struct rps_dev_flow *rflow,4988static bool rps_flow_is_active(struct rps_dev_flow *rflow,
4989 struct rps_dev_flow_table *flow_table,4989 u8 log,
4990 unsigned int cpu)4990 unsigned int cpu)
4991{4991{
4992 unsigned int flow_last_active;4992 unsigned int flow_last_active;
@@ -4999,7 +4999,7 @@ static bool rps_flow_is_active(struct rps_dev_flow *rflow, · 6 unmodified lines
4999 flow_last_active = READ_ONCE(rflow->last_qtail);4999 flow_last_active = READ_ONCE(rflow->last_qtail);
5000 5000
5001 return (int)(sd_input_head - flow_last_active) <5001 return (int)(sd_input_head - flow_last_active) <
5002 (int)(10 << flow_table->log);5002 (int)(10 << log);
5003}5003}
5004#endif5004#endif
5005 5005
@@ -5011,9 +5011,10 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 5 unmodified lines
5011 u32 head;5011 u32 head;
5012#ifdef CONFIG_RFS_ACCEL5012#ifdef CONFIG_RFS_ACCEL
5013 struct netdev_rx_queue *rxqueue;5013 struct netdev_rx_queue *rxqueue;
5014 struct rps_dev_flow_table *flow_table;5014 struct rps_dev_flow *flow_table;
5015 struct rps_dev_flow *old_rflow;5015 struct rps_dev_flow *old_rflow;
5016 struct rps_dev_flow *tmp_rflow;5016 struct rps_dev_flow *tmp_rflow;
5017 rps_tag_ptr q_tag_ptr;
5017 unsigned int tmp_cpu;5018 unsigned int tmp_cpu;
5018 u16 rxq_index;5019 u16 rxq_index;
5019 u32 flow_id;5020 u32 flow_id;
@@ -5028,16 +5029,18 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 8 unmodified lines
5028 goto out;5029 goto out;
5029 5030
5030 rxqueue = dev->_rx + rxq_index;5031 rxqueue = dev->_rx + rxq_index;
5031 flow_table = rcu_dereference(rxqueue->rps_flow_table);5032 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5032 if (!flow_table)5033 if (!q_tag_ptr)
5033 goto out;5034 goto out;
5034 5035
5035 flow_id = rfs_slot(hash, flow_table);5036 flow_id = rfs_slot(hash, q_tag_ptr);
5036 tmp_rflow = &flow_table->flows[flow_id];5037 flow_table = rps_tag_to_table(q_tag_ptr);
5038 tmp_rflow = flow_table + flow_id;
5037 tmp_cpu = READ_ONCE(tmp_rflow->cpu);5039 tmp_cpu = READ_ONCE(tmp_rflow->cpu);
5038 5040
5039 if (READ_ONCE(tmp_rflow->filter) != RPS_NO_FILTER) {5041 if (READ_ONCE(tmp_rflow->filter) != RPS_NO_FILTER) {
5040 if (rps_flow_is_active(tmp_rflow, flow_table,5042 if (rps_flow_is_active(tmp_rflow,
5043 rps_tag_to_log(q_tag_ptr),
5041 tmp_cpu)) {5044 tmp_cpu)) {
5042 if (hash != READ_ONCE(tmp_rflow->hash) ||5045 if (hash != READ_ONCE(tmp_rflow->hash) ||
5043 next_cpu == tmp_cpu)5046 next_cpu == tmp_cpu)
@@ -5076,8 +5079,7 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 32 unmodified lines
5076 struct rps_dev_flow **rflowp)5079 struct rps_dev_flow **rflowp)
5077{5080{
5078 struct netdev_rx_queue *rxqueue = dev->_rx;5081 struct netdev_rx_queue *rxqueue = dev->_rx;
5079 struct rps_dev_flow_table *flow_table;5082 rps_tag_ptr global_tag_ptr, q_tag_ptr;
5080 rps_tag_ptr global_tag_ptr;
5081 struct rps_map *map;5083 struct rps_map *map;
5082 int cpu = -1;5084 int cpu = -1;
5083 u32 tcpu;5085 u32 tcpu;
@@ -5098,9 +5100,9 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 14 unmodified lines
5098 5100
5099 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */5101 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
5100 5102
5101 flow_table = rcu_dereference(rxqueue->rps_flow_table);5103 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5102 map = rcu_dereference(rxqueue->rps_map);5104 map = rcu_dereference(rxqueue->rps_map);
5103 if (!flow_table && !map)5105 if (!q_tag_ptr && !map)
5104 goto done;5106 goto done;
5105 5107
5106 skb_reset_network_header(skb);5108 skb_reset_network_header(skb);
@@ -5109,8 +5111,9 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 2 unmodified lines
5109 goto done;5111 goto done;
5110 5112
5111 global_tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);5113 global_tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
5112 if (flow_table && global_tag_ptr) {5114 if (q_tag_ptr && global_tag_ptr) {
5113 struct rps_sock_flow_table *sock_flow_table;5115 struct rps_sock_flow_table *sock_flow_table;
5116 struct rps_dev_flow *flow_table;
5114 struct rps_dev_flow *rflow;5117 struct rps_dev_flow *rflow;
5115 u32 next_cpu;5118 u32 next_cpu;
5116 u32 flow_id;5119 u32 flow_id;
@@ -5130,7 +5133,9 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, · 13 unmodified lines
5130 /* OK, now we know there is a match,5133 /* OK, now we know there is a match,
5131 * we can look at the local (per receive queue) flow table5134 * we can look at the local (per receive queue) flow table
5132 */5135 */
5133 rflow = &flow_table->flows[rfs_slot(hash, flow_table)];5136 flow_id = rfs_slot(hash, q_tag_ptr);
5137 flow_table = rps_tag_to_table(q_tag_ptr);
5138 rflow = flow_table + flow_id;
5134 tcpu = rflow->cpu;5139 tcpu = rflow->cpu;
5135 5140
5136 /*5141 /*
@@ -5190,19 +5195,23 @@ bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, · 53 unmodified lines
5190 u32 flow_id, u16 filter_id)5195 u32 flow_id, u16 filter_id)
5191{5196{
5192 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;5197 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
5193 struct rps_dev_flow_table *flow_table;5198 struct rps_dev_flow *flow_table;
5194 struct rps_dev_flow *rflow;5199 struct rps_dev_flow *rflow;
5200 rps_tag_ptr q_tag_ptr;
5195 bool expire = true;5201 bool expire = true;
5202 u8 log;
5196 5203
5197 rcu_read_lock();5204 rcu_read_lock();
5198 flow_table = rcu_dereference(rxqueue->rps_flow_table);5205 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5199 if (flow_table && flow_id < (1UL << flow_table->log)) {5206 log = rps_tag_to_log(q_tag_ptr);
5207 if (q_tag_ptr && flow_id < (1UL << log)) {
5200 unsigned int cpu;5208 unsigned int cpu;
5201 5209
5202 rflow = &flow_table->flows[flow_id];5210 flow_table = rps_tag_to_table(q_tag_ptr);
5211 rflow = flow_table + flow_id;
5203 cpu = READ_ONCE(rflow->cpu);5212 cpu = READ_ONCE(rflow->cpu);
5204 if (READ_ONCE(rflow->filter) == filter_id &&5213 if (READ_ONCE(rflow->filter) == filter_id &&
5205 rps_flow_is_active(rflow, flow_table, cpu))5214 rps_flow_is_active(rflow, log, cpu))
5206 expire = false;5215 expire = false;
5207 }5216 }
5208 rcu_read_unlock();5217 rcu_read_unlock();
net/core/net-sysfs.c+34−36
@@ -1060,14 +1060,12 @@ out: · 1059 unmodified lines
1060static ssize_t show_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,1060static ssize_t show_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,
1061 char *buf)1061 char *buf)
1062{1062{
1063 struct rps_dev_flow_table *flow_table;
1064 unsigned long val = 0;1063 unsigned long val = 0;
1064 rps_tag_ptr tag_ptr;
1065 1065
1066 rcu_read_lock();1066 tag_ptr = READ_ONCE(queue->rps_flow_table);
1067 flow_table = rcu_dereference(queue->rps_flow_table);1067 if (tag_ptr)
1068 if (flow_table)1068 val = 1UL << rps_tag_to_log(tag_ptr);
1069 val = 1UL << flow_table->log;
1070 rcu_read_unlock();
1071 1069
1072 return sysfs_emit(buf, "%lu\n", val);1070 return sysfs_emit(buf, "%lu\n", val);
1073}1071}
@@ -1075,8 +1073,10 @@ static ssize_t show_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue, · 1 unmodified line
1075static ssize_t store_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,1073static ssize_t store_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,
1076 const char *buf, size_t len)1074 const char *buf, size_t len)
1077{1075{
1076 rps_tag_ptr otag, tag_ptr = 0UL;
1077 struct rps_dev_flow *table;
1078 unsigned long mask, count;1078 unsigned long mask, count;
1079 struct rps_dev_flow_table *table, *old_table;1079 size_t sz;
1080 int rc;1080 int rc;
1081 1081
1082 if (!capable(CAP_NET_ADMIN))1082 if (!capable(CAP_NET_ADMIN))
@@ -1093,38 +1093,36 @@ static ssize_t store_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue, · 10 unmodified lines
1093 */1093 */
1094 while ((mask | (mask >> 1)) != mask)1094 while ((mask | (mask >> 1)) != mask)
1095 mask |= (mask >> 1);1095 mask |= (mask >> 1);
1096 /* On 64 bit arches, must check mask fits in table->mask (u32),1096
1097 * and on 32bit arches, must check1097 /* Do not accept too large tables. */
1098 * RPS_DEV_FLOW_TABLE_SIZE(mask + 1) doesn't overflow.1098 if (mask > (INT_MAX / sizeof(*table) - 1))
1099 */
1100#if BITS_PER_LONG > 32
1101 if (mask > (unsigned long)(u32)mask)
1102 return -EINVAL;
1103#else
1104 if (mask > (ULONG_MAX - RPS_DEV_FLOW_TABLE_SIZE(1))
1105 / sizeof(struct rps_dev_flow)) {
1106 /* Enforce a limit to prevent overflow */
1107 return -EINVAL;1099 return -EINVAL;
1108 }1100
1109#endif1101 sz = max_t(size_t, sizeof(*table) * (mask + 1),
1110 table = vmalloc(RPS_DEV_FLOW_TABLE_SIZE(mask + 1));1102 PAGE_SIZE);
1103 if (sz <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER) ||
1104 is_power_of_2(sizeof(*table)))
1105 table = kvmalloc(sz, GFP_KERNEL);
1106 else
1107 table = vmalloc(sz);
1111 if (!table)1108 if (!table)
1112 return -ENOMEM;1109 return -ENOMEM;
1113 1110 tag_ptr = (rps_tag_ptr)table;
1114 table->log = ilog2(mask) + 1;1111 if (rps_tag_to_log(tag_ptr)) {
1112 pr_err_once("store_rps_dev_flow_table_cnt() got a non page aligned allocation.\n");
1113 kvfree(table);
1114 return -ENOMEM;
1115 }
1116 tag_ptr |= (ilog2(mask) + 1);
1115 for (count = 0; count <= mask; count++) {1117 for (count = 0; count <= mask; count++) {
1116 table->flows[count].cpu = RPS_NO_CPU;1118 table[count].cpu = RPS_NO_CPU;
1117 table->flows[count].filter = RPS_NO_FILTER;1119 table[count].filter = RPS_NO_FILTER;
1118 }1120 }
1119 } else {
1120 table = NULL;
1121 }1121 }
1122 1122
1123 old_table = unrcu_pointer(xchg(&queue->rps_flow_table,1123 otag = xchg(&queue->rps_flow_table, tag_ptr);
1124 RCU_INITIALIZER(table)));1124 if (otag)
1125 1125 kvfree_rcu_mightsleep(rps_tag_to_table(otag));
1126 if (old_table)
1127 kvfree_rcu_mightsleep(old_table);
1128 1126
1129 return len;1127 return len;
1130}1128}
@@ -1150,7 +1148,7 @@ static void rx_queue_release(struct kobject *kobj) · 19 unmodified lines
1150{1148{
1151 struct netdev_rx_queue *queue = to_rx_queue(kobj);1149 struct netdev_rx_queue *queue = to_rx_queue(kobj);
1152#ifdef CONFIG_RPS1150#ifdef CONFIG_RPS
1153 struct rps_dev_flow_table *old_table;1151 rps_tag_ptr tag_ptr;
1154 struct rps_map *map;1152 struct rps_map *map;
1155 1153
1156 map = rcu_dereference_protected(queue->rps_map, 1);1154 map = rcu_dereference_protected(queue->rps_map, 1);
@@ -1159,9 +1157,9 @@ static void rx_queue_release(struct kobject *kobj) · 2 unmodified lines
1159 kfree_rcu(map, rcu);1157 kfree_rcu(map, rcu);
1160 }1158 }
1161 1159
1162 old_table = unrcu_pointer(xchg(&queue->rps_flow_table, NULL));1160 tag_ptr = xchg(&queue->rps_flow_table, 0UL);
1163 if (old_table)1161 if (tag_ptr)
1164 kvfree_rcu_mightsleep(old_table);1162 kvfree_rcu_mightsleep(rps_tag_to_table(tag_ptr));
1165#endif1163#endif
1166 1164
1167 memset(kobj, 0, sizeof(*kobj));1165 memset(kobj, 0, sizeof(*kobj));