History

Merge branch 'net-mlx5-hw-counters-refactor'

Tariq Toukan says: ==================== net/mlx5: hw counters refactor This is a patchset re-post, see: https://lore.kernel.org/20240815054656.2210494-7-tariqt@nvidia.com In this patchset, Cosmin refactors hw counters and solves perf scaling issue. Series generated against: commit c824deb1a897 ("cxgb4: clip_tbl: Fix spelling mistake "wont" -> "won't"") HW counters are central to mlx5 driver operations. They are hardware objects created and used alongside most steering operations, and queried from a variety of places. Most counters are queried in bulk from a periodic task in fs_counters.c. Counter performance is important and as such, a variety of improvements have been done over the years. Currently, counters are allocated from pools, which are bulk allocated to amortize the cost of firmware commands. Counters are managed through an IDR, a doubly linked list and two atomic single linked lists. Adding/removing counters is a complex dance between user contexts requesting it and the mlx5_fc_stats_work task which does most of the work. Under high load (e.g. from connection tracking flow insertion/deletion), the counter code becomes a bottleneck, as seen on flame graphs. Whenever a counter is deleted, it gets added to a list and the wq task is scheduled to run immediately to actually delete it. This is done via mod_delayed_work which uses an internal spinlock. In some tests, waiting for this spinlock took up to 66% of all samples. This series refactors the counter code to use a more straight-forward approach, avoiding the mod_delayed_work problem and making the code easier to understand. For that: - patch #1 moves counters data structs to a more appropriate place. - patch #2 simplifies the bulk query allocation scheme by using vmalloc. - patch #3 replaces the IDR+3 lists with an xarray. This is the main patch of the series, solving the spinlock congestion issue. - patch #4 removes an unnecessary cacheline alignment causing a lot of memory to be wasted. - patches #5 and #6 are small cleanups enabled by the refactoring. ====================

Jakub Kicinski committed 2 years ago

Commit 34ea1df2 parents c55ff46 d1c9cff4 files · +147 −278Browse files

Unified · Split

4 files
  1. drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +1 −1
  2. drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c +145 −242
  3. include/linux/mlx5/driver.h +1 −32
  4. include/linux/mlx5/fs.h +0 −3
drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c+1−1
@@ -1026,7 +1026,7 @@ mlx5_tc_ct_counter_create(struct mlx5_tc_ct_priv *ct_priv) · 1025 unmodified lines
10261026 return ERR_PTR(-ENOMEM);
10271027
10281028 counter->is_shared = false;
1029− counter->counter = mlx5_fc_create_ex(ct_priv->dev, true);
1029+ counter->counter = mlx5_fc_create(ct_priv->dev, true);
10301030 if (IS_ERR(counter->counter)) {
10311031 ct_dbg("Failed to create counter for ct entry");
10321032 ret = PTR_ERR(counter->counter);
drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c+145−242
@@ -32,13 +32,11 @@ · 31 unmodified lines
3232
3333 #include <linux/mlx5/driver.h>
3434 #include <linux/mlx5/fs.h>
35−#include <linux/rbtree.h>
3635 #include "mlx5_core.h"
3736 #include "fs_core.h"
3837 #include "fs_cmd.h"
3938
4039 #define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000)
41−#define MLX5_FC_BULK_QUERY_ALLOC_PERIOD msecs_to_jiffies(180 * 1000)
4240 /* Max number of counters to query in bulk read is 32K */
4341 #define MLX5_SW_MAX_COUNTERS_BULK BIT(15)
4442 #define MLX5_INIT_COUNTERS_BULK 8
@@ -52,21 +50,37 @@ struct mlx5_fc_cache { · 7 unmodified lines
5250 };
5351
5452 struct mlx5_fc {
55− struct list_head list;
56− struct llist_node addlist;
57− struct llist_node dellist;
58−
59− /* last{packets,bytes} members are used when calculating the delta since
60− * last reading
61− */
53+ u32 id;
54+ bool aging;
55+ struct mlx5_fc_bulk *bulk;
56+ struct mlx5_fc_cache cache;
57+ /* last{packets,bytes} are used for calculating deltas since last reading. */
6258 u64 lastpackets;
6359 u64 lastbytes;
60+};
6461
65− struct mlx5_fc_bulk *bulk;
66− u32 id;
67− bool aging;
62+struct mlx5_fc_pool {
63+ struct mlx5_core_dev *dev;
64+ struct mutex pool_lock; /* protects pool lists */
65+ struct list_head fully_used;
66+ struct list_head partially_used;
67+ struct list_head unused;
68+ int available_fcs;
69+ int used_fcs;
70+ int threshold;
71+};
6872
69− struct mlx5_fc_cache cache ____cacheline_aligned_in_smp;
73+struct mlx5_fc_stats {
74+ struct xarray counters;
75+
76+ struct workqueue_struct *wq;
77+ struct delayed_work work;
78+ unsigned long sampling_interval; /* jiffies */
79+ u32 *bulk_query_out;
80+ int bulk_query_len;
81+ bool bulk_query_alloc_failed;
82+ unsigned long next_bulk_query_alloc;
83+ struct mlx5_fc_pool fc_pool;
7084 };
7185
7286 static void mlx5_fc_pool_init(struct mlx5_fc_pool *fc_pool, struct mlx5_core_dev *dev);
@@ -74,78 +88,6 @@ static void mlx5_fc_pool_cleanup(struct mlx5_fc_pool *fc_pool); · 1 unmodified line
7488 static struct mlx5_fc *mlx5_fc_pool_acquire_counter(struct mlx5_fc_pool *fc_pool);
7589 static void mlx5_fc_pool_release_counter(struct mlx5_fc_pool *fc_pool, struct mlx5_fc *fc);
7690
77−/* locking scheme:
78− *
79− * It is the responsibility of the user to prevent concurrent calls or bad
80− * ordering to mlx5_fc_create(), mlx5_fc_destroy() and accessing a reference
81− * to struct mlx5_fc.
82− * e.g en_tc.c is protected by RTNL lock of its caller, and will never call a
83− * dump (access to struct mlx5_fc) after a counter is destroyed.
84− *
85− * access to counter list:
86− * - create (user context)
87− * - mlx5_fc_create() only adds to an addlist to be used by
88− * mlx5_fc_stats_work(). addlist is a lockless single linked list
89− * that doesn't require any additional synchronization when adding single
90− * node.
91− * - spawn thread to do the actual destroy
92− *
93− * - destroy (user context)
94− * - add a counter to lockless dellist
95− * - spawn thread to do the actual del
96− *
97− * - dump (user context)
98− * user should not call dump after destroy
99− *
100− * - query (single thread workqueue context)
101− * destroy/dump - no conflict (see destroy)
102− * query/dump - packets and bytes might be inconsistent (since update is not
103− * atomic)
104− * query/create - no conflict (see create)
105− * since every create/destroy spawn the work, only after necessary time has
106− * elapsed, the thread will actually query the hardware.
107− */
108−
109−static struct list_head *mlx5_fc_counters_lookup_next(struct mlx5_core_dev *dev,
110− u32 id)
111−{
112− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
113− unsigned long next_id = (unsigned long)id + 1;
114− struct mlx5_fc *counter;
115− unsigned long tmp;
116−
117− rcu_read_lock();
118− /* skip counters that are in idr, but not yet in counters list */
119− idr_for_each_entry_continue_ul(&fc_stats->counters_idr,
120− counter, tmp, next_id) {
121− if (!list_empty(&counter->list))
122− break;
123− }
124− rcu_read_unlock();
125−
126− return counter ? &counter->list : &fc_stats->counters;
127−}
128−
129−static void mlx5_fc_stats_insert(struct mlx5_core_dev *dev,
130− struct mlx5_fc *counter)
131−{
132− struct list_head *next = mlx5_fc_counters_lookup_next(dev, counter->id);
133−
134− list_add_tail(&counter->list, next);
135−}
136−
137−static void mlx5_fc_stats_remove(struct mlx5_core_dev *dev,
138− struct mlx5_fc *counter)
139−{
140− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
141−
142− list_del(&counter->list);
143−
144− spin_lock(&fc_stats->counters_idr_lock);
145− WARN_ON(!idr_remove(&fc_stats->counters_idr, counter->id));
146− spin_unlock(&fc_stats->counters_idr_lock);
147−}
148−
14991 static int get_init_bulk_query_len(struct mlx5_core_dev *dev)
15092 {
15193 return min_t(int, MLX5_INIT_COUNTERS_BULK,
@@ -174,47 +116,64 @@ static void update_counter_cache(int index, u32 *bulk_raw_data, · 22 unmodified lines
174116 cache->lastuse = jiffies;
175117 }
176118
177−static void mlx5_fc_stats_query_counter_range(struct mlx5_core_dev *dev,
178− struct mlx5_fc *first,
179− u32 last_id)
119+/* Synchronization notes
120+ *
121+ * Access to counter array:
122+ * - create - mlx5_fc_create() (user context)
123+ * - inserts the counter into the xarray.
124+ *
125+ * - destroy - mlx5_fc_destroy() (user context)
126+ * - erases the counter from the xarray and releases it.
127+ *
128+ * - query mlx5_fc_query(), mlx5_fc_query_cached{,_raw}() (user context)
129+ * - user should not access a counter after destroy.
130+ *
131+ * - bulk query (single thread workqueue context)
132+ * - create: query relies on 'lastuse' to avoid updating counters added
133+ * around the same time as the current bulk cmd.
134+ * - destroy: destroyed counters will not be accessed, even if they are
135+ * destroyed during a bulk query command.
136+ */
137+static void mlx5_fc_stats_query_all_counters(struct mlx5_core_dev *dev)
180138 {
181− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
182− bool query_more_counters = (first->id <= last_id);
183− int cur_bulk_len = fc_stats->bulk_query_len;
139+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
140+ u32 bulk_len = fc_stats->bulk_query_len;
141+ XA_STATE(xas, &fc_stats->counters, 0);
184142 u32 *data = fc_stats->bulk_query_out;
185− struct mlx5_fc *counter = first;
143+ struct mlx5_fc *counter;
144+ u32 last_bulk_id = 0;
145+ u64 bulk_query_time;
186146 u32 bulk_base_id;
187− int bulk_len;
188147 int err;
189148
190− while (query_more_counters) {
191− /* first id must be aligned to 4 when using bulk query */
192− bulk_base_id = counter->id & ~0x3;
193−
194− /* number of counters to query inc. the last counter */
195− bulk_len = min_t(int, cur_bulk_len,
196− ALIGN(last_id - bulk_base_id + 1, 4));
197−
198− err = mlx5_cmd_fc_bulk_query(dev, bulk_base_id, bulk_len,
199− data);
200− if (err) {
201− mlx5_core_err(dev, "Error doing bulk query: %d\n", err);
202− return;
203− }
204− query_more_counters = false;
205−
206− list_for_each_entry_from(counter, &fc_stats->counters, list) {
207− int counter_index = counter->id - bulk_base_id;
208− struct mlx5_fc_cache *cache = &counter->cache;
209−
210− if (counter->id >= bulk_base_id + bulk_len) {
211− query_more_counters = true;
212− break;
149+ xas_lock(&xas);
150+ xas_for_each(&xas, counter, U32_MAX) {
151+ if (xas_retry(&xas, counter))
152+ continue;
153+ if (unlikely(counter->id >= last_bulk_id)) {
154+ /* Start new bulk query. */
155+ /* First id must be aligned to 4 when using bulk query. */
156+ bulk_base_id = counter->id & ~0x3;
157+ last_bulk_id = bulk_base_id + bulk_len;
158+ /* The lock is released while querying the hw and reacquired after. */
159+ xas_unlock(&xas);
160+ /* The same id needs to be processed again in the next loop iteration. */
161+ xas_reset(&xas);
162+ bulk_query_time = jiffies;
163+ err = mlx5_cmd_fc_bulk_query(dev, bulk_base_id, bulk_len, data);
164+ if (err) {
165+ mlx5_core_err(dev, "Error doing bulk query: %d\n", err);
166+ return;
213167 }
214−
215− update_counter_cache(counter_index, data, cache);
168+ xas_lock(&xas);
169+ continue;
216170 }
171+ /* Do not update counters added after bulk query was started. */
172+ if (time_after64(bulk_query_time, counter->cache.lastuse))
173+ update_counter_cache(counter->id - bulk_base_id, data,
174+ &counter->cache);
217175 }
176+ xas_unlock(&xas);
218177 }
219178
220179 static void mlx5_fc_free(struct mlx5_core_dev *dev, struct mlx5_fc *counter)
@@ -225,7 +184,7 @@ static void mlx5_fc_free(struct mlx5_core_dev *dev, struct mlx5_fc *counter) · 4 unmodified lines
225184
226185 static void mlx5_fc_release(struct mlx5_core_dev *dev, struct mlx5_fc *counter)
227186 {
228− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
187+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
229188
230189 if (counter->bulk)
231190 mlx5_fc_pool_release_counter(&fc_stats->fc_pool, counter);
@@ -233,85 +192,55 @@ static void mlx5_fc_release(struct mlx5_core_dev *dev, struct mlx5_fc *counter) · 1 unmodified line
233192 mlx5_fc_free(dev, counter);
234193 }
235194
236−static void mlx5_fc_stats_bulk_query_size_increase(struct mlx5_core_dev *dev)
195+static void mlx5_fc_stats_bulk_query_buf_realloc(struct mlx5_core_dev *dev,
196+ int bulk_query_len)
237197 {
238− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
239− int max_bulk_len = get_max_bulk_query_len(dev);
240− unsigned long now = jiffies;
198+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
241199 u32 *bulk_query_out_tmp;
242− int max_out_len;
243−
244− if (fc_stats->bulk_query_alloc_failed &&
245− time_before(now, fc_stats->next_bulk_query_alloc))
246− return;
200+ int out_len;
247201
248− max_out_len = mlx5_cmd_fc_get_bulk_query_out_len(max_bulk_len);
249− bulk_query_out_tmp = kzalloc(max_out_len, GFP_KERNEL);
202+ out_len = mlx5_cmd_fc_get_bulk_query_out_len(bulk_query_len);
203+ bulk_query_out_tmp = kvzalloc(out_len, GFP_KERNEL);
250204 if (!bulk_query_out_tmp) {
251205 mlx5_core_warn_once(dev,
252− "Can't increase flow counters bulk query buffer size, insufficient memory, bulk_size(%d)\n",
253− max_bulk_len);
254− fc_stats->bulk_query_alloc_failed = true;
255− fc_stats->next_bulk_query_alloc =
256− now + MLX5_FC_BULK_QUERY_ALLOC_PERIOD;
206+ "Can't increase flow counters bulk query buffer size, alloc failed, bulk_query_len(%d)\n",
207+ bulk_query_len);
257208 return;
258209 }
259210
260− kfree(fc_stats->bulk_query_out);
211+ kvfree(fc_stats->bulk_query_out);
261212 fc_stats->bulk_query_out = bulk_query_out_tmp;
262− fc_stats->bulk_query_len = max_bulk_len;
263− if (fc_stats->bulk_query_alloc_failed) {
264− mlx5_core_info(dev,
265− "Flow counters bulk query buffer size increased, bulk_size(%d)\n",
266− max_bulk_len);
267− fc_stats->bulk_query_alloc_failed = false;
268− }
213+ fc_stats->bulk_query_len = bulk_query_len;
214+ mlx5_core_info(dev,
215+ "Flow counters bulk query buffer size increased, bulk_query_len(%d)\n",
216+ bulk_query_len);
269217 }
270218
271−static void mlx5_fc_stats_work(struct work_struct *work)
219+static int mlx5_fc_num_counters(struct mlx5_fc_stats *fc_stats)
272220 {
273− struct mlx5_core_dev *dev = container_of(work, struct mlx5_core_dev,
274− priv.fc_stats.work.work);
275− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
276− /* Take dellist first to ensure that counters cannot be deleted before
277− * they are inserted.
278− */
279− struct llist_node *dellist = llist_del_all(&fc_stats->dellist);
280− struct llist_node *addlist = llist_del_all(&fc_stats->addlist);
281− struct mlx5_fc *counter = NULL, *last = NULL, *tmp;
282− unsigned long now = jiffies;
283−
284− if (addlist || !list_empty(&fc_stats->counters))
285− queue_delayed_work(fc_stats->wq, &fc_stats->work,
286− fc_stats->sampling_interval);
287−
288− llist_for_each_entry(counter, addlist, addlist) {
289− mlx5_fc_stats_insert(dev, counter);
290− fc_stats->num_counters++;
291− }
292−
293− llist_for_each_entry_safe(counter, tmp, dellist, dellist) {
294− mlx5_fc_stats_remove(dev, counter);
221+ struct mlx5_fc *counter;
222+ int num_counters = 0;
223+ unsigned long id;
295224
296− mlx5_fc_release(dev, counter);
297− fc_stats->num_counters--;
298− }
225+ xa_for_each(&fc_stats->counters, id, counter)
226+ num_counters++;
227+ return num_counters;
228+}
299229
300− if (fc_stats->bulk_query_len < get_max_bulk_query_len(dev) &&
301− fc_stats->num_counters > get_init_bulk_query_len(dev))
302− mlx5_fc_stats_bulk_query_size_increase(dev);
230+static void mlx5_fc_stats_work(struct work_struct *work)
231+{
232+ struct mlx5_fc_stats *fc_stats = container_of(work, struct mlx5_fc_stats,
233+ work.work);
234+ struct mlx5_core_dev *dev = fc_stats->fc_pool.dev;
303235
304− if (time_before(now, fc_stats->next_query) ||
305− list_empty(&fc_stats->counters))
306− return;
307− last = list_last_entry(&fc_stats->counters, struct mlx5_fc, list);
236+ queue_delayed_work(fc_stats->wq, &fc_stats->work, fc_stats->sampling_interval);
308237
309− counter = list_first_entry(&fc_stats->counters, struct mlx5_fc,
310− list);
311− if (counter)
312− mlx5_fc_stats_query_counter_range(dev, counter, last->id);
238+ /* Grow the bulk query buffer to max if not maxed and enough counters are present. */
239+ if (unlikely(fc_stats->bulk_query_len < get_max_bulk_query_len(dev) &&
240+ mlx5_fc_num_counters(fc_stats) > get_init_bulk_query_len(dev)))
241+ mlx5_fc_stats_bulk_query_buf_realloc(dev, get_max_bulk_query_len(dev));
313242
314− fc_stats->next_query = now + fc_stats->sampling_interval;
243+ mlx5_fc_stats_query_all_counters(dev);
315244 }
316245
317246 static struct mlx5_fc *mlx5_fc_single_alloc(struct mlx5_core_dev *dev)
@@ -334,7 +263,7 @@ static struct mlx5_fc *mlx5_fc_single_alloc(struct mlx5_core_dev *dev) · 16 unmodified lines
334263
335264 static struct mlx5_fc *mlx5_fc_acquire(struct mlx5_core_dev *dev, bool aging)
336265 {
337− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
266+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
338267 struct mlx5_fc *counter;
339268
340269 if (aging && MLX5_CAP_GEN(dev, flow_counter_bulk_alloc) != 0) {
@@ -346,16 +275,15 @@ static struct mlx5_fc *mlx5_fc_acquire(struct mlx5_core_dev *dev, bool aging) · 5 unmodified lines
346275 return mlx5_fc_single_alloc(dev);
347276 }
348277
349−struct mlx5_fc *mlx5_fc_create_ex(struct mlx5_core_dev *dev, bool aging)
278+struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging)
350279 {
351280 struct mlx5_fc *counter = mlx5_fc_acquire(dev, aging);
352− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
281+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
353282 int err;
354283
355284 if (IS_ERR(counter))
356285 return counter;
357286
358− INIT_LIST_HEAD(&counter->list);
359287 counter->aging = aging;
360288
361289 if (aging) {
@@ -365,18 +293,9 @@ struct mlx5_fc *mlx5_fc_create_ex(struct mlx5_core_dev *dev, bool aging) · 3 unmodified lines
365293 counter->lastbytes = counter->cache.bytes;
366294 counter->lastpackets = counter->cache.packets;
367295
368− idr_preload(GFP_KERNEL);
369− spin_lock(&fc_stats->counters_idr_lock);
370−
371− err = idr_alloc_u32(&fc_stats->counters_idr, counter, &id, id,
372− GFP_NOWAIT);
373−
374− spin_unlock(&fc_stats->counters_idr_lock);
375− idr_preload_end();
376− if (err)
296+ err = xa_err(xa_store(&fc_stats->counters, id, counter, GFP_KERNEL));
297+ if (err != 0)
377298 goto err_out_alloc;
378−
379− llist_add(&counter->addlist, &fc_stats->addlist);
380299 }
381300
382301 return counter;
@@ -385,16 +304,6 @@ err_out_alloc: · 2 unmodified lines
385304 mlx5_fc_release(dev, counter);
386305 return ERR_PTR(err);
387306 }
388−
389−struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging)
390−{
391− struct mlx5_fc *counter = mlx5_fc_create_ex(dev, aging);
392− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
393−
394− if (aging)
395− mod_delayed_work(fc_stats->wq, &fc_stats->work, 0);
396− return counter;
397−}
398307 EXPORT_SYMBOL(mlx5_fc_create);
399308
400309 u32 mlx5_fc_id(struct mlx5_fc *counter)
@@ -405,39 +314,32 @@ EXPORT_SYMBOL(mlx5_fc_id); · 4 unmodified lines
405314
406315 void mlx5_fc_destroy(struct mlx5_core_dev *dev, struct mlx5_fc *counter)
407316 {
408− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
317+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
409318
410319 if (!counter)
411320 return;
412321
413− if (counter->aging) {
414− llist_add(&counter->dellist, &fc_stats->dellist);
415− mod_delayed_work(fc_stats->wq, &fc_stats->work, 0);
416− return;
417− }
418−
322+ if (counter->aging)
323+ xa_erase(&fc_stats->counters, counter->id);
419324 mlx5_fc_release(dev, counter);
420325 }
421326 EXPORT_SYMBOL(mlx5_fc_destroy);
422327
423328 int mlx5_init_fc_stats(struct mlx5_core_dev *dev)
424329 {
425− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
426− int init_bulk_len;
427− int init_out_len;
330+ struct mlx5_fc_stats *fc_stats;
331+
332+ fc_stats = kzalloc(sizeof(*fc_stats), GFP_KERNEL);
333+ if (!fc_stats)
334+ return -ENOMEM;
335+ dev->priv.fc_stats = fc_stats;
428336
429− spin_lock_init(&fc_stats->counters_idr_lock);
430− idr_init(&fc_stats->counters_idr);
431− INIT_LIST_HEAD(&fc_stats->counters);
432− init_llist_head(&fc_stats->addlist);
433− init_llist_head(&fc_stats->dellist);
337+ xa_init(&fc_stats->counters);
434338
435− init_bulk_len = get_init_bulk_query_len(dev);
436− init_out_len = mlx5_cmd_fc_get_bulk_query_out_len(init_bulk_len);
437− fc_stats->bulk_query_out = kzalloc(init_out_len, GFP_KERNEL);
339+ /* Allocate initial (small) bulk query buffer. */
340+ mlx5_fc_stats_bulk_query_buf_realloc(dev, get_init_bulk_query_len(dev));
438341 if (!fc_stats->bulk_query_out)
439− return -ENOMEM;
440− fc_stats->bulk_query_len = init_bulk_len;
342+ goto err_bulk;
441343
442344 fc_stats->wq = create_singlethread_workqueue("mlx5_fc");
443345 if (!fc_stats->wq)
@@ -447,34 +349,35 @@ int mlx5_init_fc_stats(struct mlx5_core_dev *dev) · 3 unmodified lines
447349 INIT_DELAYED_WORK(&fc_stats->work, mlx5_fc_stats_work);
448350
449351 mlx5_fc_pool_init(&fc_stats->fc_pool, dev);
352+ queue_delayed_work(fc_stats->wq, &fc_stats->work, MLX5_FC_STATS_PERIOD);
450353 return 0;
451354
452355 err_wq_create:
453− kfree(fc_stats->bulk_query_out);
356+ kvfree(fc_stats->bulk_query_out);
357+err_bulk:
358+ kfree(fc_stats);
454359 return -ENOMEM;
455360 }
456361
457362 void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev)
458363 {
459− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
460− struct llist_node *tmplist;
364+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
461365 struct mlx5_fc *counter;
462− struct mlx5_fc *tmp;
463−
464− cancel_delayed_work_sync(&dev->priv.fc_stats.work);
465− destroy_workqueue(dev->priv.fc_stats.wq);
466− dev->priv.fc_stats.wq = NULL;
366+ unsigned long id;
467367
468− tmplist = llist_del_all(&fc_stats->addlist);
469− llist_for_each_entry_safe(counter, tmp, tmplist, addlist)
470− mlx5_fc_release(dev, counter);
368+ cancel_delayed_work_sync(&fc_stats->work);
369+ destroy_workqueue(fc_stats->wq);
370+ fc_stats->wq = NULL;
471371
472− list_for_each_entry_safe(counter, tmp, &fc_stats->counters, list)
372+ xa_for_each(&fc_stats->counters, id, counter) {
373+ xa_erase(&fc_stats->counters, id);
473374 mlx5_fc_release(dev, counter);
375+ }
376+ xa_destroy(&fc_stats->counters);
474377
475378 mlx5_fc_pool_cleanup(&fc_stats->fc_pool);
476− idr_destroy(&fc_stats->counters_idr);
477− kfree(fc_stats->bulk_query_out);
379+ kvfree(fc_stats->bulk_query_out);
380+ kfree(fc_stats);
478381 }
479382
480383 int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter,
@@ -518,7 +421,7 @@ void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, · 37 unmodified lines
518421 struct delayed_work *dwork,
519422 unsigned long delay)
520423 {
521− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
424+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
522425
523426 queue_delayed_work(fc_stats->wq, dwork, delay);
524427 }
@@ -526,7 +429,7 @@ void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, · 1 unmodified line
526429 void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
527430 unsigned long interval)
528431 {
529− struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
432+ struct mlx5_fc_stats *fc_stats = dev->priv.fc_stats;
530433
531434 fc_stats->sampling_interval = min_t(unsigned long, interval,
532435 fc_stats->sampling_interval);
include/linux/mlx5/driver.h+1−32
@@ -45,7 +45,6 @@ · 44 unmodified lines
4545 #include <linux/workqueue.h>
4646 #include <linux/mempool.h>
4747 #include <linux/interrupt.h>
48−#include <linux/idr.h>
4948 #include <linux/notifier.h>
5049 #include <linux/refcount.h>
5150 #include <linux/auxiliary_bus.h>
@@ -474,36 +473,6 @@ struct mlx5_core_sriov { · 422 unmodified lines
474473 u16 max_ec_vfs;
475474 };
476475
477−struct mlx5_fc_pool {
478− struct mlx5_core_dev *dev;
479− struct mutex pool_lock; /* protects pool lists */
480− struct list_head fully_used;
481− struct list_head partially_used;
482− struct list_head unused;
483− int available_fcs;
484− int used_fcs;
485− int threshold;
486−};
487−
488−struct mlx5_fc_stats {
489− spinlock_t counters_idr_lock; /* protects counters_idr */
490− struct idr counters_idr;
491− struct list_head counters;
492− struct llist_head addlist;
493− struct llist_head dellist;
494−
495− struct workqueue_struct *wq;
496− struct delayed_work work;
497− unsigned long next_query;
498− unsigned long sampling_interval; /* jiffies */
499− u32 *bulk_query_out;
500− int bulk_query_len;
501− size_t num_counters;
502− bool bulk_query_alloc_failed;
503− unsigned long next_bulk_query_alloc;
504− struct mlx5_fc_pool fc_pool;
505−};
506−
507476 struct mlx5_events;
508477 struct mlx5_mpfs;
509478 struct mlx5_eswitch;
@@ -630,7 +599,7 @@ struct mlx5_priv { · 120 unmodified lines
630599 struct mlx5_devcom_comp_dev *hca_devcom_comp;
631600 struct mlx5_fw_reset *fw_reset;
632601 struct mlx5_core_roce roce;
633− struct mlx5_fc_stats fc_stats;
602+ struct mlx5_fc_stats *fc_stats;
634603 struct mlx5_rl_table rl_table;
635604 struct mlx5_ft_pool *ft_pool;
636605
include/linux/mlx5/fs.h+0−3
@@ -298,9 +298,6 @@ int mlx5_modify_rule_destination(struct mlx5_flow_handle *handler, · 297 unmodified lines
298298
299299 struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging);
300300
301−/* As mlx5_fc_create() but doesn't queue stats refresh thread. */
302−struct mlx5_fc *mlx5_fc_create_ex(struct mlx5_core_dev *dev, bool aging);
303−
304301 void mlx5_fc_destroy(struct mlx5_core_dev *dev, struct mlx5_fc *counter);
305302 u64 mlx5_fc_query_lastuse(struct mlx5_fc *counter);
306303 void mlx5_fc_query_cached(struct mlx5_fc *counter,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t