History

net/mlx5: Hold pages RB tree per VF

Per page request event, FW request to allocated or release pages for a single function. Driver maintains FW pages object per function, so there is no need to hold one global page data-base. Instead, have a page data-base per function, which will improve performance release flow in all cases, especially for "release all pages". As the range of function IDs is large and not sequential, use xarray to store a per function ID page data-base, where the function ID is the key. Upon first allocation of a page to a function ID, create the page data-base per function. This data-base will be released only at pagealloc mechanism cleanup. NIC: ConnectX-4 Lx CPU: Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz Test case: 32 VFs, measure release pages on one VF as part of FLR Before: 0.021 Sec After: 0.014 Sec The improvement depends on amount of VFs and memory utilization by them. Time measurements above were taken from idle system.

Eran Ben Elisha committed 6 years ago

Commit d6945241 parent 5e619d72 files · +105 −39Browse files

Unified · Split

2 files
  1. drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +104 −38
  2. include/linux/mlx5/driver.h +1 −1
drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c+104−38
@@ -35,6 +35,7 @@ · 34 unmodified lines
35#include <linux/module.h>35#include <linux/module.h>
36#include <linux/delay.h>36#include <linux/delay.h>
37#include <linux/mlx5/driver.h>37#include <linux/mlx5/driver.h>
38#include <linux/xarray.h>
38#include "mlx5_core.h"39#include "mlx5_core.h"
39#include "lib/eq.h"40#include "lib/eq.h"
40 41
@@ -73,15 +74,45 @@ enum { · 32 unmodified lines
73 MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,74 MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,
74};75};
75 76
77static struct rb_root *page_root_per_func_id(struct mlx5_core_dev *dev, u16 func_id)
78{
79 struct rb_root *root;
80 int err;
81
82 root = xa_load(&dev->priv.page_root_xa, func_id);
83 if (root)
84 return root;
85
86 root = kzalloc(sizeof(*root), GFP_KERNEL);
87 if (!root)
88 return ERR_PTR(-ENOMEM);
89
90 err = xa_insert(&dev->priv.page_root_xa, func_id, root, GFP_KERNEL);
91 if (err) {
92 kfree(root);
93 return ERR_PTR(err);
94 }
95
96 *root = RB_ROOT;
97
98 return root;
99}
100
76static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id)101static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id)
77{102{
78 struct rb_root *root = &dev->priv.page_root;
79 struct rb_node **new = &root->rb_node;
80 struct rb_node *parent = NULL;103 struct rb_node *parent = NULL;
104 struct rb_root *root;
105 struct rb_node **new;
81 struct fw_page *nfp;106 struct fw_page *nfp;
82 struct fw_page *tfp;107 struct fw_page *tfp;
83 int i;108 int i;
84 109
110 root = page_root_per_func_id(dev, func_id);
111 if (IS_ERR(root))
112 return PTR_ERR(root);
113
114 new = &root->rb_node;
115
85 while (*new) {116 while (*new) {
86 parent = *new;117 parent = *new;
87 tfp = rb_entry(parent, struct fw_page, rb_node);118 tfp = rb_entry(parent, struct fw_page, rb_node);
@@ -111,13 +142,20 @@ static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u · 23 unmodified lines
111 return 0;142 return 0;
112}143}
113 144
114static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr)145static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr,
146 u32 func_id)
115{147{
116 struct rb_root *root = &dev->priv.page_root;
117 struct rb_node *tmp = root->rb_node;
118 struct fw_page *result = NULL;148 struct fw_page *result = NULL;
149 struct rb_root *root;
150 struct rb_node *tmp;
119 struct fw_page *tfp;151 struct fw_page *tfp;
120 152
153 root = xa_load(&dev->priv.page_root_xa, func_id);
154 if (WARN_ON_ONCE(!root))
155 return NULL;
156
157 tmp = root->rb_node;
158
121 while (tmp) {159 while (tmp) {
122 tfp = rb_entry(tmp, struct fw_page, rb_node);160 tfp = rb_entry(tmp, struct fw_page, rb_node);
123 if (tfp->addr < addr) {161 if (tfp->addr < addr) {
@@ -191,7 +229,13 @@ static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id) · 67 unmodified lines
191static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp,229static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp,
192 bool in_free_list)230 bool in_free_list)
193{231{
194 rb_erase(&fwp->rb_node, &dev->priv.page_root);232 struct rb_root *root;
233
234 root = xa_load(&dev->priv.page_root_xa, fwp->func_id);
235 if (WARN_ON_ONCE(!root))
236 return;
237
238 rb_erase(&fwp->rb_node, root);
195 if (in_free_list)239 if (in_free_list)
196 list_del(&fwp->list);240 list_del(&fwp->list);
197 dma_unmap_page(dev->device, fwp->addr & MLX5_U64_4K_PAGE_MASK,241 dma_unmap_page(dev->device, fwp->addr & MLX5_U64_4K_PAGE_MASK,
@@ -200,12 +244,12 @@ static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp, · 2 unmodified lines
200 kfree(fwp);244 kfree(fwp);
201}245}
202 246
203static void free_4k(struct mlx5_core_dev *dev, u64 addr)247static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 func_id)
204{248{
205 struct fw_page *fwp;249 struct fw_page *fwp;
206 int n;250 int n;
207 251
208 fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK);252 fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK, func_id);
209 if (!fwp) {253 if (!fwp) {
210 mlx5_core_warn_rl(dev, "page not found\n");254 mlx5_core_warn_rl(dev, "page not found\n");
211 return;255 return;
@@ -340,7 +384,7 @@ retry: · 128 unmodified lines
340 384
341out_4k:385out_4k:
342 for (i--; i >= 0; i--)386 for (i--; i >= 0; i--)
343 free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]));387 free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]), func_id);
344out_free:388out_free:
345 kvfree(in);389 kvfree(in);
346 if (notify_fail)390 if (notify_fail)
@@ -351,16 +395,19 @@ out_free: · 4 unmodified lines
351static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id,395static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id,
352 bool ec_function)396 bool ec_function)
353{397{
398 struct rb_root *root;
354 struct rb_node *p;399 struct rb_node *p;
355 int npages = 0;400 int npages = 0;
356 401
357 p = rb_first(&dev->priv.page_root);402 root = xa_load(&dev->priv.page_root_xa, func_id);
403 if (WARN_ON_ONCE(!root))
404 return;
405
406 p = rb_first(root);
358 while (p) {407 while (p) {
359 struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node);408 struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node);
360 409
361 p = rb_next(p);410 p = rb_next(p);
362 if (fwp->func_id != func_id)
363 continue;
364 npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count);411 npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count);
365 free_fwp(dev, fwp, fwp->free_count);412 free_fwp(dev, fwp, fwp->free_count);
366 }413 }
@@ -378,6 +425,7 @@ static void release_all_pages(struct mlx5_core_dev *dev, u32 func_id, · 11 unmodified lines
378static int reclaim_pages_cmd(struct mlx5_core_dev *dev,425static int reclaim_pages_cmd(struct mlx5_core_dev *dev,
379 u32 *in, int in_size, u32 *out, int out_size)426 u32 *in, int in_size, u32 *out, int out_size)
380{427{
428 struct rb_root *root;
381 struct fw_page *fwp;429 struct fw_page *fwp;
382 struct rb_node *p;430 struct rb_node *p;
383 u32 func_id;431 u32 func_id;
@@ -391,12 +439,14 @@ static int reclaim_pages_cmd(struct mlx5_core_dev *dev, · 7 unmodified lines
391 npages = MLX5_GET(manage_pages_in, in, input_num_entries);439 npages = MLX5_GET(manage_pages_in, in, input_num_entries);
392 func_id = MLX5_GET(manage_pages_in, in, function_id);440 func_id = MLX5_GET(manage_pages_in, in, function_id);
393 441
394 p = rb_first(&dev->priv.page_root);442 root = xa_load(&dev->priv.page_root_xa, func_id);
443 if (WARN_ON_ONCE(!root))
444 return -EEXIST;
445
446 p = rb_first(root);
395 while (p && i < npages) {447 while (p && i < npages) {
396 fwp = rb_entry(p, struct fw_page, rb_node);448 fwp = rb_entry(p, struct fw_page, rb_node);
397 p = rb_next(p);449 p = rb_next(p);
398 if (fwp->func_id != func_id)
399 continue;
400 450
401 MLX5_ARRAY_SET64(manage_pages_out, out, pas, i, fwp->addr);451 MLX5_ARRAY_SET64(manage_pages_out, out, pas, i, fwp->addr);
402 i++;452 i++;
@@ -446,7 +496,7 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, · 43 unmodified lines
446 }496 }
447 497
448 for (i = 0; i < num_claimed; i++)498 for (i = 0; i < num_claimed; i++)
449 free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]));499 free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]), func_id);
450 500
451 if (nclaimed)501 if (nclaimed)
452 *nclaimed = num_claimed;502 *nclaimed = num_claimed;
@@ -560,35 +610,49 @@ static int optimal_reclaimed_pages(void) · 107 unmodified lines
560 return ret;610 return ret;
561}611}
562 612
563int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)613static int mlx5_reclaim_root_pages(struct mlx5_core_dev *dev,
614 struct rb_root *root, u16 func_id)
564{615{
565 unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);616 unsigned long end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
566 struct fw_page *fwp;
567 struct rb_node *p;
568 int nclaimed = 0;
569 int err = 0;
570 617
571 do {618 while (!RB_EMPTY_ROOT(root)) {
572 p = rb_first(&dev->priv.page_root);619 int nclaimed;
573 if (p) {620 int err;
574 fwp = rb_entry(p, struct fw_page, rb_node);621
575 err = reclaim_pages(dev, fwp->func_id,622 err = reclaim_pages(dev, func_id, optimal_reclaimed_pages(),
576 optimal_reclaimed_pages(),623 &nclaimed, mlx5_core_is_ecpf(dev));
577 &nclaimed, mlx5_core_is_ecpf(dev));624 if (err) {
578 625 mlx5_core_warn(dev, "failed reclaiming pages (%d) for func id 0x%x\n",
579 if (err) {626 err, func_id);
580 mlx5_core_warn(dev, "failed reclaiming pages (%d)\n",627 return err;
581 err);
582 return err;
583 }
584 if (nclaimed)
585 end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
586 }628 }
629
630 if (nclaimed)
631 end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
632
587 if (time_after(jiffies, end)) {633 if (time_after(jiffies, end)) {
588 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n");634 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n");
589 break;635 break;
590 }636 }
591 } while (p);637 }
638
639 return 0;
640}
641
642int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
643{
644 struct rb_root *root;
645 unsigned long id;
646 void *entry;
647
648 xa_for_each(&dev->priv.page_root_xa, id, entry) {
649 root = entry;
650 mlx5_reclaim_root_pages(dev, root, id);
651 xa_erase(&dev->priv.page_root_xa, id);
652 kfree(root);
653 }
654
655 WARN_ON(!xa_empty(&dev->priv.page_root_xa));
592 656
593 WARN(dev->priv.fw_pages,657 WARN(dev->priv.fw_pages,
594 "FW pages counter is %d after reclaiming all pages\n",658 "FW pages counter is %d after reclaiming all pages\n",
@@ -605,17 +669,19 @@ int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) · 10 unmodified lines
605 669
606int mlx5_pagealloc_init(struct mlx5_core_dev *dev)670int mlx5_pagealloc_init(struct mlx5_core_dev *dev)
607{671{
608 dev->priv.page_root = RB_ROOT;
609 INIT_LIST_HEAD(&dev->priv.free_list);672 INIT_LIST_HEAD(&dev->priv.free_list);
610 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator");673 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator");
611 if (!dev->priv.pg_wq)674 if (!dev->priv.pg_wq)
612 return -ENOMEM;675 return -ENOMEM;
613 676
677 xa_init(&dev->priv.page_root_xa);
678
614 return 0;679 return 0;
615}680}
616 681
617void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev)682void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev)
618{683{
684 xa_destroy(&dev->priv.page_root_xa);
619 destroy_workqueue(dev->priv.pg_wq);685 destroy_workqueue(dev->priv.pg_wq);
620}686}
621 687
include/linux/mlx5/driver.h+1−1
@@ -541,7 +541,7 @@ struct mlx5_priv { · 540 unmodified lines
541 /* pages stuff */541 /* pages stuff */
542 struct mlx5_nb pg_nb;542 struct mlx5_nb pg_nb;
543 struct workqueue_struct *pg_wq;543 struct workqueue_struct *pg_wq;
544 struct rb_root page_root;544 struct xarray page_root_xa;
545 int fw_pages;545 int fw_pages;
546 atomic_t reg_pages;546 atomic_t reg_pages;
547 struct list_head free_list;547 struct list_head free_list;