History

net/mlx5: Hold pages RB tree per VF

Per page request event, FW request to allocated or release pages for a single function. Driver maintains FW pages object per function, so there is no need to hold one global page data-base. Instead, have a page data-base per function, which will improve performance release flow in all cases, especially for "release all pages". As the range of function IDs is large and not sequential, use xarray to store a per function ID page data-base, where the function ID is the key. Upon first allocation of a page to a function ID, create the page data-base per function. This data-base will be released only at pagealloc mechanism cleanup. NIC: ConnectX-4 Lx CPU: Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz Test case: 32 VFs, measure release pages on one VF as part of FLR Before: 0.021 Sec After: 0.014 Sec The improvement depends on amount of VFs and memory utilization by them. Time measurements above were taken from idle system.

Eran Ben Elisha committed 6 years ago

Commit d6945241 parent 5e619d72 files · +105 −39Browse files

Unified · Split

2 files
  1. drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +104 −38
  2. include/linux/mlx5/driver.h +1 −1
drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c+104−38
@@ -35,6 +35,7 @@ · 34 unmodified lines
3535 #include <linux/module.h>
3636 #include <linux/delay.h>
3737 #include <linux/mlx5/driver.h>
38+#include <linux/xarray.h>
3839 #include "mlx5_core.h"
3940 #include "lib/eq.h"
4041
@@ -73,15 +74,45 @@ enum { · 32 unmodified lines
7374 MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,
7475 };
7576
77+static struct rb_root *page_root_per_func_id(struct mlx5_core_dev *dev, u16 func_id)
78+{
79+ struct rb_root *root;
80+ int err;
81+
82+ root = xa_load(&dev->priv.page_root_xa, func_id);
83+ if (root)
84+ return root;
85+
86+ root = kzalloc(sizeof(*root), GFP_KERNEL);
87+ if (!root)
88+ return ERR_PTR(-ENOMEM);
89+
90+ err = xa_insert(&dev->priv.page_root_xa, func_id, root, GFP_KERNEL);
91+ if (err) {
92+ kfree(root);
93+ return ERR_PTR(err);
94+ }
95+
96+ *root = RB_ROOT;
97+
98+ return root;
99+}
100+
76101 static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id)
77102 {
78− struct rb_root *root = &dev->priv.page_root;
79− struct rb_node **new = &root->rb_node;
80103 struct rb_node *parent = NULL;
104+ struct rb_root *root;
105+ struct rb_node **new;
81106 struct fw_page *nfp;
82107 struct fw_page *tfp;
83108 int i;
84109
110+ root = page_root_per_func_id(dev, func_id);
111+ if (IS_ERR(root))
112+ return PTR_ERR(root);
113+
114+ new = &root->rb_node;
115+
85116 while (*new) {
86117 parent = *new;
87118 tfp = rb_entry(parent, struct fw_page, rb_node);
@@ -111,13 +142,20 @@ static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u · 23 unmodified lines
111142 return 0;
112143 }
113144
114−static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr)
145+static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr,
146+ u32 func_id)
115147 {
116− struct rb_root *root = &dev->priv.page_root;
117− struct rb_node *tmp = root->rb_node;
118148 struct fw_page *result = NULL;
149+ struct rb_root *root;
150+ struct rb_node *tmp;
119151 struct fw_page *tfp;
120152
153+ root = xa_load(&dev->priv.page_root_xa, func_id);
154+ if (WARN_ON_ONCE(!root))
155+ return NULL;
156+
157+ tmp = root->rb_node;
158+
121159 while (tmp) {
122160 tfp = rb_entry(tmp, struct fw_page, rb_node);
123161 if (tfp->addr < addr) {
@@ -191,7 +229,13 @@ static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id) · 67 unmodified lines
191229 static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp,
192230 bool in_free_list)
193231 {
194− rb_erase(&fwp->rb_node, &dev->priv.page_root);
232+ struct rb_root *root;
233+
234+ root = xa_load(&dev->priv.page_root_xa, fwp->func_id);
235+ if (WARN_ON_ONCE(!root))
236+ return;
237+
238+ rb_erase(&fwp->rb_node, root);
195239 if (in_free_list)
196240 list_del(&fwp->list);
197241 dma_unmap_page(dev->device, fwp->addr & MLX5_U64_4K_PAGE_MASK,
@@ -200,12 +244,12 @@ static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp, · 2 unmodified lines
200244 kfree(fwp);
201245 }
202246
203−static void free_4k(struct mlx5_core_dev *dev, u64 addr)
247+static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 func_id)
204248 {
205249 struct fw_page *fwp;
206250 int n;
207251
208− fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK);
252+ fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK, func_id);
209253 if (!fwp) {
210254 mlx5_core_warn_rl(dev, "page not found\n");
211255 return;
@@ -340,7 +384,7 @@ retry: · 128 unmodified lines
340384
341385 out_4k:
342386 for (i--; i >= 0; i--)
343− free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]));
387+ free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]), func_id);
344388 out_free:
345389 kvfree(in);
346390 if (notify_fail)
@@ -351,16 +395,19 @@ out_free: · 4 unmodified lines
351395 static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id,
352396 bool ec_function)
353397 {
398+ struct rb_root *root;
354399 struct rb_node *p;
355400 int npages = 0;
356401
357− p = rb_first(&dev->priv.page_root);
402+ root = xa_load(&dev->priv.page_root_xa, func_id);
403+ if (WARN_ON_ONCE(!root))
404+ return;
405+
406+ p = rb_first(root);
358407 while (p) {
359408 struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node);
360409
361410 p = rb_next(p);
362− if (fwp->func_id != func_id)
363− continue;
364411 npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count);
365412 free_fwp(dev, fwp, fwp->free_count);
366413 }
@@ -378,6 +425,7 @@ static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id, · 11 unmodified lines
378425 static int reclaim_pages_cmd(struct mlx5_core_dev *dev,
379426 u32 *in, int in_size, u32 *out, int out_size)
380427 {
428+ struct rb_root *root;
381429 struct fw_page *fwp;
382430 struct rb_node *p;
383431 u32 func_id;
@@ -391,12 +439,14 @@ static int reclaim_pages_cmd(struct mlx5_core_dev *dev, · 7 unmodified lines
391439 npages = MLX5_GET(manage_pages_in, in, input_num_entries);
392440 func_id = MLX5_GET(manage_pages_in, in, function_id);
393441
394− p = rb_first(&dev->priv.page_root);
442+ root = xa_load(&dev->priv.page_root_xa, func_id);
443+ if (WARN_ON_ONCE(!root))
444+ return -EEXIST;
445+
446+ p = rb_first(root);
395447 while (p && i < npages) {
396448 fwp = rb_entry(p, struct fw_page, rb_node);
397449 p = rb_next(p);
398− if (fwp->func_id != func_id)
399− continue;
400450
401451 MLX5_ARRAY_SET64(manage_pages_out, out, pas, i, fwp->addr);
402452 i++;
@@ -446,7 +496,7 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, · 43 unmodified lines
446496 }
447497
448498 for (i = 0; i < num_claimed; i++)
449− free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]));
499+ free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]), func_id);
450500
451501 if (nclaimed)
452502 *nclaimed = num_claimed;
@@ -560,35 +610,49 @@ static int optimal_reclaimed_pages(void) · 107 unmodified lines
560610 return ret;
561611 }
562612
563−int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
613+static int mlx5_reclaim_root_pages(struct mlx5_core_dev *dev,
614+ struct rb_root *root, u16 func_id)
564615 {
565616 unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
566− struct fw_page *fwp;
567− struct rb_node *p;
568− int nclaimed = 0;
569− int err = 0;
570617
571− do {
572− p = rb_first(&dev->priv.page_root);
573− if (p) {
574− fwp = rb_entry(p, struct fw_page, rb_node);
575− err = reclaim_pages(dev, fwp->func_id,
576− optimal_reclaimed_pages(),
577− &nclaimed, mlx5_core_is_ecpf(dev));
578−
579− if (err) {
580− mlx5_core_warn(dev, "failed reclaiming pages (%d)\n",
581− err);
582− return err;
583− }
584− if (nclaimed)
585− end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
618+ while (!RB_EMPTY_ROOT(root)) {
619+ int nclaimed;
620+ int err;
621+
622+ err = reclaim_pages(dev, func_id, optimal_reclaimed_pages(),
623+ &nclaimed, mlx5_core_is_ecpf(dev));
624+ if (err) {
625+ mlx5_core_warn(dev, "failed reclaiming pages (%d) for func id 0x%x\n",
626+ err, func_id);
627+ return err;
586628 }
629+
630+ if (nclaimed)
631+ end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
632+
587633 if (time_after(jiffies, end)) {
588634 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n");
589635 break;
590636 }
591− } while (p);
637+ }
638+
639+ return 0;
640+}
641+
642+int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
643+{
644+ struct rb_root *root;
645+ unsigned long id;
646+ void *entry;
647+
648+ xa_for_each(&dev->priv.page_root_xa, id, entry) {
649+ root = entry;
650+ mlx5_reclaim_root_pages(dev, root, id);
651+ xa_erase(&dev->priv.page_root_xa, id);
652+ kfree(root);
653+ }
654+
655+ WARN_ON(!xa_empty(&dev->priv.page_root_xa));
592656
593657 WARN(dev->priv.fw_pages,
594658 "FW pages counter is %d after reclaiming all pages\n",
@@ -605,17 +669,19 @@ int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) · 10 unmodified lines
605669
606670 int mlx5_pagealloc_init(struct mlx5_core_dev *dev)
607671 {
608− dev->priv.page_root = RB_ROOT;
609672 INIT_LIST_HEAD(&dev->priv.free_list);
610673 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator");
611674 if (!dev->priv.pg_wq)
612675 return -ENOMEM;
613676
677+ xa_init(&dev->priv.page_root_xa);
678+
614679 return 0;
615680 }
616681
617682 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev)
618683 {
684+ xa_destroy(&dev->priv.page_root_xa);
619685 destroy_workqueue(dev->priv.pg_wq);
620686 }
621687
include/linux/mlx5/driver.h+1−1
@@ -541,7 +541,7 @@ struct mlx5_priv { · 540 unmodified lines
541541 /* pages stuff */
542542 struct mlx5_nb pg_nb;
543543 struct workqueue_struct *pg_wq;
544− struct rb_root page_root;
544+ struct xarray page_root_xa;
545545 int fw_pages;
546546 atomic_t reg_pages;
547547 struct list_head free_list;