/* SPDX-License-Identifier: GPL-2.0 *//* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ #ifndef _IONIC_IBDEV_H_#define _IONIC_IBDEV_H_ #include <rdma/ib_verbs.h>#include <rdma/ib_pack.h>#include <rdma/iter.h>#include <rdma/uverbs_ioctl.h> #include <rdma/ionic-abi.h>#include <ionic_api.h>#include <ionic_regs.h> #include "ionic_fw.h"#include "ionic_queue.h"#include "ionic_res.h" #include "ionic_lif_cfg.h" /* Config knobs */#define IONIC_EQ_DEPTH 511#define IONIC_EQ_COUNT 32#define IONIC_AQ_DEPTH 63#define IONIC_AQ_COUNT 4#define IONIC_EQ_ISR_BUDGET 10#define IONIC_EQ_WORK_BUDGET 1000#define IONIC_MAX_RD_ATOM 16#define IONIC_PKEY_TBL_LEN 1#define IONIC_GID_TBL_LEN 256 #define IONIC_MAX_QPID 0xffffff#define IONIC_SPEC_HIGH 8#define IONIC_MAX_PD 1024#define IONIC_SQCMB_ORDER 5#define IONIC_RQCMB_ORDER 0 #define IONIC_META_LAST ((void *)1ul)#define IONIC_META_POSTED ((void *)2ul) #define IONIC_CQ_GRACE 100 #define IONIC_ROCE_UDP_SPORT 28272#define IONIC_DMA_LKEY 0#define IONIC_DMA_RKEY IONIC_DMA_LKEY #define IONIC_CMB_SUPPORTED \ (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \ IONIC_CMB_WC | IONIC_CMB_UC) /* resource is not reserved on the device, indicated in tbl_order */#define IONIC_RES_INVALID -1 struct ionic_aq;struct ionic_cq;struct ionic_eq;struct ionic_vcq; enum ionic_admin_state { IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */ IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */ IONIC_ADMIN_KILLED, /* not submitting, locally completed */}; enum ionic_admin_flags { IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */ IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */ IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */}; enum ionic_mmap_flag { IONIC_MMAP_WC = BIT(0), IONIC_MMAP_PHC = BIT(1),}; struct ionic_mmap_entry { struct rdma_user_mmap_entry rdma_entry; unsigned long size; unsigned long pfn; u8 mmap_flags;}; struct ionic_ibdev { struct ib_device ibdev; struct ionic_lif_cfg lif_cfg; struct xarray qp_tbl; struct xarray cq_tbl; struct ionic_resid_bits inuse_dbid; struct ionic_resid_bits inuse_pdid; struct ionic_resid_bits inuse_ahid; struct ionic_resid_bits inuse_mrid; struct ionic_resid_bits inuse_qpid; struct ionic_resid_bits inuse_cqid; u8 half_cqid_udma_shift; u8 half_qpid_udma_shift; u8 next_qpid_udma_idx; u8 next_mrkey; struct work_struct reset_work; bool reset_posted; u32 reset_cnt; struct delayed_work admin_dwork; struct ionic_aq **aq_vec; atomic_t admin_state; struct ionic_eq **eq_vec; struct ionic_v1_stat *hw_stats; void *hw_stats_buf; struct rdma_stat_desc *hw_stats_hdrs; struct ionic_counter_stats *counter_stats; int hw_stats_count;}; struct ionic_eq { struct ionic_ibdev *dev; u32 eqid; u32 intr; struct ionic_queue q; int armed; bool enable; struct work_struct work; int irq; char name[32];}; struct ionic_admin_wr { struct completion work; struct list_head aq_ent; struct ionic_v1_admin_wqe wqe; struct ionic_v1_cqe cqe; struct ionic_aq *aq; int status;}; struct ionic_admin_wr_q { struct ionic_admin_wr *wr; int wqe_strides;}; struct ionic_aq { struct ionic_ibdev *dev; struct ionic_vcq *vcq; struct work_struct work; atomic_t admin_state; unsigned long stamp; bool armed; u32 aqid; u32 cqid; spinlock_t lock; /* for posting */ struct ionic_queue q; struct ionic_admin_wr_q *q_wr; struct list_head wr_prod; struct list_head wr_post;}; struct ionic_ctx { struct ib_ucontext ibctx; u32 dbid; struct rdma_user_mmap_entry *mmap_dbell; struct rdma_user_mmap_entry *mmap_phc;}; struct ionic_tbl_buf { u32 tbl_limit; u32 tbl_pages; size_t tbl_size; __le64 *tbl_buf; dma_addr_t tbl_dma; u8 page_size_log2;}; struct ionic_pd { struct ib_pd ibpd; u32 pdid; u32 flags;}; struct ionic_cq { struct ionic_vcq *vcq; u32 cqid; u32 eqid; spinlock_t lock; /* for polling */ struct list_head poll_sq; bool flush; struct list_head flush_sq; struct list_head flush_rq; struct list_head ibkill_flush_ent; struct ionic_queue q; bool color; int credit; u16 arm_any_prod; u16 arm_sol_prod; struct kref cq_kref; struct completion cq_rel_comp; /* infrequently accessed, keep at end */ struct ib_umem *umem;}; struct ionic_vcq { struct ib_cq ibcq; struct ionic_cq cq[2]; u8 udma_mask; u8 poll_idx;}; struct ionic_sq_meta { u64 wrid; u32 len; u16 seq; u8 ibop; u8 ibsts; u8 remote:1; u8 signal:1; u8 local_comp:1;}; struct ionic_rq_meta { struct ionic_rq_meta *next; u64 wrid;}; struct ionic_qp { struct ib_qp ibqp; enum ib_qp_state state; u32 qpid; u32 ahid; u32 sq_cqid; u32 rq_cqid; u8 udma_idx; u8 has_ah:1; u8 has_sq:1; u8 has_rq:1; u8 sig_all:1; struct list_head qp_list_counter; struct list_head cq_poll_sq; struct list_head cq_flush_sq; struct list_head cq_flush_rq; struct list_head ibkill_flush_ent; spinlock_t sq_lock; /* for posting and polling */ struct ionic_queue sq; struct ionic_sq_meta *sq_meta; u16 *sq_msn_idx; int sq_spec; u16 sq_old_prod; u16 sq_msn_prod; u16 sq_msn_cons; u8 sq_cmb; bool sq_flush; bool sq_flush_rcvd; spinlock_t rq_lock; /* for posting and polling */ struct ionic_queue rq; struct ionic_rq_meta *rq_meta; struct ionic_rq_meta *rq_meta_head; int rq_spec; u16 rq_old_prod; u8 rq_cmb; bool rq_flush; struct kref qp_kref; struct completion qp_rel_comp; /* infrequently accessed, keep at end */ int sgid_index; int sq_cmb_order; u32 sq_cmb_pgid; phys_addr_t sq_cmb_addr; struct rdma_user_mmap_entry *mmap_sq_cmb; struct ib_umem *sq_umem; int rq_cmb_order; u32 rq_cmb_pgid; phys_addr_t rq_cmb_addr; struct rdma_user_mmap_entry *mmap_rq_cmb; struct ib_umem *rq_umem; int dcqcn_profile; struct ib_ud_header *hdr;}; struct ionic_ah { struct ib_ah ibah; u32 ahid; int sgid_index; struct ib_ud_header hdr;}; struct ionic_mr { union { struct ib_mr ibmr; struct ib_mw ibmw; }; u32 mrid; int flags; struct ib_umem *umem; struct ionic_tbl_buf buf; bool created;}; struct ionic_counter_stats { int queue_stats_count; struct ionic_v1_stat *hdr; struct rdma_stat_desc *stats_hdrs; struct ida counter_ida;}; struct ionic_rdma_counter { struct rdma_counter rdma_counter; void *vals; struct list_head qp_list;}; static inline struct ionic_rdma_counter *to_ionic_rdma_counter(struct rdma_counter *counter){ return container_of(counter, struct ionic_rdma_counter, rdma_counter);} static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev){ return container_of(ibdev, struct ionic_ibdev, ibdev);} static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx){ return container_of(ibctx, struct ionic_ctx, ibctx);} static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj){ if (!uobj) return NULL; if (!uobj->context) return NULL; return to_ionic_ctx(uobj->context);} static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd){ return container_of(ibpd, struct ionic_pd, ibpd);} static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr){ return container_of(ibmr, struct ionic_mr, ibmr);} static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw){ return container_of(ibmw, struct ionic_mr, ibmw);} static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq){ return container_of(ibcq, struct ionic_vcq, ibcq);} static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq, uint8_t udma_idx){ return &to_ionic_vcq(ibcq)->cq[udma_idx];} static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp){ return container_of(ibqp, struct ionic_qp, ibqp);} static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah){ return container_of(ibah, struct ionic_ah, ibah);} static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev, struct ionic_ctx *ctx){ if (!ctx) return dev->lif_cfg.dbid; return ctx->dbid;} static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev, struct ib_uobject *uobj){ return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj));} static inline bool ionic_ibop_is_local(enum ib_wr_opcode op){ return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;} static inline void ionic_qp_complete(struct kref *kref){ struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref); complete(&qp->qp_rel_comp);} static inline void ionic_cq_complete(struct kref *kref){ struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref); complete(&cq->cq_rel_comp);} /* ionic_admin.c */extern struct workqueue_struct *ionic_evt_workq;void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, enum ionic_admin_flags); int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev); int ionic_create_rdma_admin(struct ionic_ibdev *dev);void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path); /* ionic_controlpath.c */int ionic_create_cq_common(struct ionic_vcq *vcq, struct ionic_tbl_buf *buf, const struct ib_cq_init_attr *attr, struct ionic_ctx *ctx, struct ib_udata *udata, struct ionic_qdesc *req_cq, __u32 *resp_cqid, int udma_idx);void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);void ionic_notify_flush_cq(struct ionic_cq *cq); int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata);int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 addr, int access, struct ib_dmah *dmah, struct ib_udata *udata);struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, u64 length, u64 addr, int fd, int access, struct ib_dmah *dmah, struct uverbs_attr_bundle *attrs);int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, u32 max_sg);int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset);int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);int ionic_dealloc_mw(struct ib_mw *ibmw);int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs);int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, struct ib_udata *udata);int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_udata *udata);int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_qp_init_attr *init_attr);int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);Lines 1–500 of 525. Load more
/* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ #ifndef _IONIC_IBDEV_H_ #define _IONIC_IBDEV_H_ #include <rdma/ib_verbs.h> #include <rdma/ib_pack.h> #include <rdma/iter.h> #include <rdma/uverbs_ioctl.h> #include <rdma/ionic-abi.h> #include <ionic_api.h> #include <ionic_regs.h> #include "ionic_fw.h" #include "ionic_queue.h" #include "ionic_res.h" #include "ionic_lif_cfg.h" /* Config knobs */ #define IONIC_EQ_DEPTH 511 #define IONIC_EQ_COUNT 32 #define IONIC_AQ_DEPTH 63 #define IONIC_AQ_COUNT 4 #define IONIC_EQ_ISR_BUDGET 10 #define IONIC_EQ_WORK_BUDGET 1000 #define IONIC_MAX_RD_ATOM 16 #define IONIC_PKEY_TBL_LEN 1 #define IONIC_GID_TBL_LEN 256 #define IONIC_MAX_QPID 0xffffff #define IONIC_SPEC_HIGH 8 #define IONIC_MAX_PD 1024 #define IONIC_SQCMB_ORDER 5 #define IONIC_RQCMB_ORDER 0 #define IONIC_META_LAST ((void *)1ul) #define IONIC_META_POSTED ((void *)2ul) #define IONIC_CQ_GRACE 100 #define IONIC_ROCE_UDP_SPORT 28272 #define IONIC_DMA_LKEY 0 #define IONIC_DMA_RKEY IONIC_DMA_LKEY #define IONIC_CMB_SUPPORTED \ (IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \ IONIC_CMB_WC | IONIC_CMB_UC) /* resource is not reserved on the device, indicated in tbl_order */ #define IONIC_RES_INVALID -1 struct ionic_aq; struct ionic_cq; struct ionic_eq; struct ionic_vcq; enum ionic_admin_state { IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */ IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */ IONIC_ADMIN_KILLED, /* not submitting, locally completed */ }; enum ionic_admin_flags { IONIC_ADMIN_F_BUSYWAIT = BIT(0), /* Don't sleep */ IONIC_ADMIN_F_TEARDOWN = BIT(1), /* In destroy path */ IONIC_ADMIN_F_INTERRUPT = BIT(2), /* Interruptible w/timeout */ }; enum ionic_mmap_flag { IONIC_MMAP_WC = BIT(0), IONIC_MMAP_PHC = BIT(1), }; struct ionic_mmap_entry { struct rdma_user_mmap_entry rdma_entry; unsigned long size; unsigned long pfn; u8 mmap_flags; }; struct ionic_ibdev { struct ib_device ibdev; struct ionic_lif_cfg lif_cfg; struct xarray qp_tbl; struct xarray cq_tbl; struct ionic_resid_bits inuse_dbid; struct ionic_resid_bits inuse_pdid; struct ionic_resid_bits inuse_ahid; struct ionic_resid_bits inuse_mrid; struct ionic_resid_bits inuse_qpid; struct ionic_resid_bits inuse_cqid; u8 half_cqid_udma_shift; u8 half_qpid_udma_shift; u8 next_qpid_udma_idx; u8 next_mrkey; struct work_struct reset_work; bool reset_posted; u32 reset_cnt; struct delayed_work admin_dwork; struct ionic_aq **aq_vec; atomic_t admin_state; struct ionic_eq **eq_vec; struct ionic_v1_stat *hw_stats; void *hw_stats_buf; struct rdma_stat_desc *hw_stats_hdrs; struct ionic_counter_stats *counter_stats; int hw_stats_count; }; struct ionic_eq { struct ionic_ibdev *dev; u32 eqid; u32 intr; struct ionic_queue q; int armed; bool enable; struct work_struct work; int irq; char name[32]; }; struct ionic_admin_wr { struct completion work; struct list_head aq_ent; struct ionic_v1_admin_wqe wqe; struct ionic_v1_cqe cqe; struct ionic_aq *aq; int status; }; struct ionic_admin_wr_q { struct ionic_admin_wr *wr; int wqe_strides; }; struct ionic_aq { struct ionic_ibdev *dev; struct ionic_vcq *vcq; struct work_struct work; atomic_t admin_state; unsigned long stamp; bool armed; u32 aqid; u32 cqid; spinlock_t lock; /* for posting */ struct ionic_queue q; struct ionic_admin_wr_q *q_wr; struct list_head wr_prod; struct list_head wr_post; }; struct ionic_ctx { struct ib_ucontext ibctx; u32 dbid; struct rdma_user_mmap_entry *mmap_dbell; struct rdma_user_mmap_entry *mmap_phc; }; struct ionic_tbl_buf { u32 tbl_limit; u32 tbl_pages; size_t tbl_size; __le64 *tbl_buf; dma_addr_t tbl_dma; u8 page_size_log2; }; struct ionic_pd { struct ib_pd ibpd; u32 pdid; u32 flags; }; struct ionic_cq { struct ionic_vcq *vcq; u32 cqid; u32 eqid; spinlock_t lock; /* for polling */ struct list_head poll_sq; bool flush; struct list_head flush_sq; struct list_head flush_rq; struct list_head ibkill_flush_ent; struct ionic_queue q; bool color; int credit; u16 arm_any_prod; u16 arm_sol_prod; struct kref cq_kref; struct completion cq_rel_comp; /* infrequently accessed, keep at end */ struct ib_umem *umem; }; struct ionic_vcq { struct ib_cq ibcq; struct ionic_cq cq[2]; u8 udma_mask; u8 poll_idx; }; struct ionic_sq_meta { u64 wrid; u32 len; u16 seq; u8 ibop; u8 ibsts; u8 remote:1; u8 signal:1; u8 local_comp:1; }; struct ionic_rq_meta { struct ionic_rq_meta *next; u64 wrid; }; struct ionic_qp { struct ib_qp ibqp; enum ib_qp_state state; u32 qpid; u32 ahid; u32 sq_cqid; u32 rq_cqid; u8 udma_idx; u8 has_ah:1; u8 has_sq:1; u8 has_rq:1; u8 sig_all:1; struct list_head qp_list_counter; struct list_head cq_poll_sq; struct list_head cq_flush_sq; struct list_head cq_flush_rq; struct list_head ibkill_flush_ent; spinlock_t sq_lock; /* for posting and polling */ struct ionic_queue sq; struct ionic_sq_meta *sq_meta; u16 *sq_msn_idx; int sq_spec; u16 sq_old_prod; u16 sq_msn_prod; u16 sq_msn_cons; u8 sq_cmb; bool sq_flush; bool sq_flush_rcvd; spinlock_t rq_lock; /* for posting and polling */ struct ionic_queue rq; struct ionic_rq_meta *rq_meta; struct ionic_rq_meta *rq_meta_head; int rq_spec; u16 rq_old_prod; u8 rq_cmb; bool rq_flush; struct kref qp_kref; struct completion qp_rel_comp; /* infrequently accessed, keep at end */ int sgid_index; int sq_cmb_order; u32 sq_cmb_pgid; phys_addr_t sq_cmb_addr; struct rdma_user_mmap_entry *mmap_sq_cmb; struct ib_umem *sq_umem; int rq_cmb_order; u32 rq_cmb_pgid; phys_addr_t rq_cmb_addr; struct rdma_user_mmap_entry *mmap_rq_cmb; struct ib_umem *rq_umem; int dcqcn_profile; struct ib_ud_header *hdr; }; struct ionic_ah { struct ib_ah ibah; u32 ahid; int sgid_index; struct ib_ud_header hdr; }; struct ionic_mr { union { struct ib_mr ibmr; struct ib_mw ibmw; }; u32 mrid; int flags; struct ib_umem *umem; struct ionic_tbl_buf buf; bool created; }; struct ionic_counter_stats { int queue_stats_count; struct ionic_v1_stat *hdr; struct rdma_stat_desc *stats_hdrs; struct ida counter_ida; }; struct ionic_rdma_counter { struct rdma_counter rdma_counter; void *vals; struct list_head qp_list; }; static inline struct ionic_rdma_counter * to_ionic_rdma_counter(struct rdma_counter *counter) { return container_of(counter, struct ionic_rdma_counter, rdma_counter); } static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev) { return container_of(ibdev, struct ionic_ibdev, ibdev); } static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx) { return container_of(ibctx, struct ionic_ctx, ibctx); } static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj) { if (!uobj) return NULL; if (!uobj->context) return NULL; return to_ionic_ctx(uobj->context); } static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd) { return container_of(ibpd, struct ionic_pd, ibpd); } static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr) { return container_of(ibmr, struct ionic_mr, ibmr); } static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw) { return container_of(ibmw, struct ionic_mr, ibmw); } static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq) { return container_of(ibcq, struct ionic_vcq, ibcq); } static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq, uint8_t udma_idx) { return &to_ionic_vcq(ibcq)->cq[udma_idx]; } static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp) { return container_of(ibqp, struct ionic_qp, ibqp); } static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah) { return container_of(ibah, struct ionic_ah, ibah); } static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev, struct ionic_ctx *ctx) { if (!ctx) return dev->lif_cfg.dbid; return ctx->dbid; } static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev, struct ib_uobject *uobj) { return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj)); } static inline bool ionic_ibop_is_local(enum ib_wr_opcode op) { return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR; } static inline void ionic_qp_complete(struct kref *kref) { struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref); complete(&qp->qp_rel_comp); } static inline void ionic_cq_complete(struct kref *kref) { struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref); complete(&cq->cq_rel_comp); } /* ionic_admin.c */ extern struct workqueue_struct *ionic_evt_workq; void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, enum ionic_admin_flags); int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev); int ionic_create_rdma_admin(struct ionic_ibdev *dev); void ionic_destroy_rdma_admin(struct ionic_ibdev *dev); void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path); /* ionic_controlpath.c */ int ionic_create_cq_common(struct ionic_vcq *vcq, struct ionic_tbl_buf *buf, const struct ib_cq_init_attr *attr, struct ionic_ctx *ctx, struct ib_udata *udata, struct ionic_qdesc *req_cq, __u32 *resp_cqid, int udma_idx); void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq); void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp); void ionic_notify_flush_cq(struct ionic_cq *cq); int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata); void ionic_dealloc_ucontext(struct ib_ucontext *ibctx); int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma); void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry); int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, struct ib_udata *udata); int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); int ionic_destroy_ah(struct ib_ah *ibah, u32 flags); struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access); struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 addr, int access, struct ib_dmah *dmah, struct ib_udata *udata); struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset, u64 length, u64 addr, int fd, int access, struct ib_dmah *dmah, struct uverbs_attr_bundle *attrs); int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type, u32 max_sg); int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata); int ionic_dealloc_mw(struct ib_mw *ibmw); int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs); int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, struct ib_udata *udata); int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_udata *udata); int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_qp_init_attr *init_attr); int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);