History

RDMA/nldev: Add command to get FRMR pools

Add support for a new command in netlink to dump to user the state of the FRMR pools on the devices. Expose each pool with its key and the usage statistics for it.

Michael Guralnik committed 8 months ago

Commit 50c03591 parent ba51cf92 files · +182 −0Browse files

Unified · Split

2 files
  1. drivers/infiniband/core/nldev.c +165 −0
  2. include/uapi/rdma/rdma_netlink.h +17 −0
drivers/infiniband/core/nldev.c+165−0
@@ -37,11 +37,13 @@ · 36 unmodified lines
3737 #include <net/netlink.h>
3838 #include <rdma/rdma_cm.h>
3939 #include <rdma/rdma_netlink.h>
40+#include <rdma/frmr_pools.h>
4041
4142 #include "core_priv.h"
4243 #include "cma_priv.h"
4344 #include "restrack.h"
4445 #include "uverbs.h"
46+#include "frmr_pools.h"
4547
4648 /*
4749 * This determines whether a non-privileged user is allowed to specify a
@@ -172,6 +174,16 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { · 124 unmodified lines
172174 [RDMA_NLDEV_ATTR_NAME_ASSIGN_TYPE] = { .type = NLA_U8 },
173175 [RDMA_NLDEV_ATTR_EVENT_TYPE] = { .type = NLA_U8 },
174176 [RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED] = { .type = NLA_U8 },
177+ [RDMA_NLDEV_ATTR_FRMR_POOLS] = { .type = NLA_NESTED },
178+ [RDMA_NLDEV_ATTR_FRMR_POOL_ENTRY] = { .type = NLA_NESTED },
179+ [RDMA_NLDEV_ATTR_FRMR_POOL_KEY] = { .type = NLA_NESTED },
180+ [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS] = { .type = NLA_U8 },
181+ [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS] = { .type = NLA_U32 },
182+ [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY] = { .type = NLA_U64 },
183+ [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS] = { .type = NLA_U64 },
184+ [RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES] = { .type = NLA_U32 },
185+ [RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE] = { .type = NLA_U64 },
186+ [RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE] = { .type = NLA_U64 },
175187 };
176188
177189 static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
@@ -2637,6 +2649,156 @@ static int nldev_deldev(struct sk_buff *skb, struct nlmsghdr *nlh, · 2459 unmodified lines
26372649 return ib_del_sub_device_and_put(device);
26382650 }
26392651
2652+static int fill_frmr_pool_key(struct sk_buff *msg, struct ib_frmr_key *key)
2653+{
2654+ struct nlattr *key_attr;
2655+
2656+ key_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY);
2657+ if (!key_attr)
2658+ return -EMSGSIZE;
2659+
2660+ if (nla_put_u8(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS, key->ats))
2661+ goto err;
2662+ if (nla_put_u32(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS,
2663+ key->access_flags))
2664+ goto err;
2665+ if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY,
2666+ key->vendor_key, RDMA_NLDEV_ATTR_PAD))
2667+ goto err;
2668+ if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS,
2669+ key->num_dma_blocks, RDMA_NLDEV_ATTR_PAD))
2670+ goto err;
2671+
2672+ nla_nest_end(msg, key_attr);
2673+ return 0;
2674+
2675+err:
2676+ return -EMSGSIZE;
2677+}
2678+
2679+static int fill_frmr_pool_entry(struct sk_buff *msg, struct ib_frmr_pool *pool)
2680+{
2681+ if (fill_frmr_pool_key(msg, &pool->key))
2682+ return -EMSGSIZE;
2683+
2684+ spin_lock(&pool->lock);
2685+ if (nla_put_u32(msg, RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES,
2686+ pool->queue.ci + pool->inactive_queue.ci))
2687+ goto err_unlock;
2688+ if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE,
2689+ pool->max_in_use, RDMA_NLDEV_ATTR_PAD))
2690+ goto err_unlock;
2691+ if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE,
2692+ pool->in_use, RDMA_NLDEV_ATTR_PAD))
2693+ goto err_unlock;
2694+ spin_unlock(&pool->lock);
2695+
2696+ return 0;
2697+
2698+err_unlock:
2699+ spin_unlock(&pool->lock);
2700+ return -EMSGSIZE;
2701+}
2702+
2703+static int nldev_frmr_pools_get_dumpit(struct sk_buff *skb,
2704+ struct netlink_callback *cb)
2705+{
2706+ struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
2707+ struct ib_frmr_pools *pools;
2708+ int err, ret = 0, idx = 0;
2709+ struct ib_frmr_pool *pool;
2710+ struct nlattr *table_attr;
2711+ struct nlattr *entry_attr;
2712+ struct ib_device *device;
2713+ int start = cb->args[0];
2714+ struct rb_node *node;
2715+ struct nlmsghdr *nlh;
2716+ bool filled = false;
2717+
2718+ err = __nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
2719+ nldev_policy, NL_VALIDATE_LIBERAL, NULL);
2720+ if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
2721+ return -EINVAL;
2722+
2723+ device = ib_device_get_by_index(
2724+ sock_net(skb->sk), nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]));
2725+ if (!device)
2726+ return -EINVAL;
2727+
2728+ pools = device->frmr_pools;
2729+ if (!pools) {
2730+ ib_device_put(device);
2731+ return 0;
2732+ }
2733+
2734+ nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
2735+ RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
2736+ RDMA_NLDEV_CMD_FRMR_POOLS_GET),
2737+ 0, NLM_F_MULTI);
2738+
2739+ if (!nlh || fill_nldev_handle(skb, device)) {
2740+ ret = -EMSGSIZE;
2741+ goto err;
2742+ }
2743+
2744+ table_attr = nla_nest_start_noflag(skb, RDMA_NLDEV_ATTR_FRMR_POOLS);
2745+ if (!table_attr) {
2746+ ret = -EMSGSIZE;
2747+ goto err;
2748+ }
2749+
2750+ read_lock(&pools->rb_lock);
2751+ for (node = rb_first(&pools->rb_root); node; node = rb_next(node)) {
2752+ pool = rb_entry(node, struct ib_frmr_pool, node);
2753+ if (pool->key.kernel_vendor_key)
2754+ continue;
2755+
2756+ if (idx < start) {
2757+ idx++;
2758+ continue;
2759+ }
2760+
2761+ filled = true;
2762+
2763+ entry_attr = nla_nest_start_noflag(
2764+ skb, RDMA_NLDEV_ATTR_FRMR_POOL_ENTRY);
2765+ if (!entry_attr) {
2766+ ret = -EMSGSIZE;
2767+ goto end_msg;
2768+ }
2769+
2770+ if (fill_frmr_pool_entry(skb, pool)) {
2771+ nla_nest_cancel(skb, entry_attr);
2772+ ret = -EMSGSIZE;
2773+ goto end_msg;
2774+ }
2775+
2776+ nla_nest_end(skb, entry_attr);
2777+ idx++;
2778+ }
2779+end_msg:
2780+ read_unlock(&pools->rb_lock);
2781+
2782+ nla_nest_end(skb, table_attr);
2783+ nlmsg_end(skb, nlh);
2784+ cb->args[0] = idx;
2785+
2786+ /*
2787+ * No more entries to fill, cancel the message and
2788+ * return 0 to mark end of dumpit.
2789+ */
2790+ if (!filled)
2791+ goto err;
2792+
2793+ ib_device_put(device);
2794+ return skb->len;
2795+
2796+err:
2797+ nlmsg_cancel(skb, nlh);
2798+ ib_device_put(device);
2799+ return ret;
2800+}
2801+
26402802 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
26412803 [RDMA_NLDEV_CMD_GET] = {
26422804 .doit = nldev_get_doit,
@@ -2743,6 +2905,9 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = { · 100 unmodified lines
27432905 .doit = nldev_deldev,
27442906 .flags = RDMA_NL_ADMIN_PERM,
27452907 },
2908+ [RDMA_NLDEV_CMD_FRMR_POOLS_GET] = {
2909+ .dump = nldev_frmr_pools_get_dumpit,
2910+ },
27462911 };
27472912
27482913 static int fill_mon_netdev_rename(struct sk_buff *msg,
include/uapi/rdma/rdma_netlink.h+17−0
@@ -308,6 +308,8 @@ enum rdma_nldev_command { · 307 unmodified lines
308308
309309 RDMA_NLDEV_CMD_MONITOR,
310310
311+ RDMA_NLDEV_CMD_FRMR_POOLS_GET, /* can dump */
312+
311313 RDMA_NLDEV_NUM_OPS
312314 };
313315
@@ -582,6 +584,21 @@ enum rdma_nldev_attr { · 268 unmodified lines
582584 RDMA_NLDEV_SYS_ATTR_MONITOR_MODE, /* u8 */
583585
584586 RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED, /* u8 */
587+
588+ /*
589+ * FRMR Pools attributes
590+ */
591+ RDMA_NLDEV_ATTR_FRMR_POOLS, /* nested table */
592+ RDMA_NLDEV_ATTR_FRMR_POOL_ENTRY, /* nested table */
593+ RDMA_NLDEV_ATTR_FRMR_POOL_KEY, /* nested table */
594+ RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS, /* u8 */
595+ RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS, /* u32 */
596+ RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY, /* u64 */
597+ RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS, /* u64 */
598+ RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES, /* u32 */
599+ RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE, /* u64 */
600+ RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE, /* u64 */
601+
585602 /*
586603 * Always the end
587604 */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t