// SPDX-License-Identifier: GPL-2.0/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ #include <linux/module.h>#include <linux/printk.h>#include <rdma/ib_addr.h>#include <rdma/ib_cache.h>#include <rdma/ib_user_verbs.h>#include <ionic_api.h> #include "ionic_fw.h"#include "ionic_ibdev.h" #define ionic_set_ecn(tos)   (((tos) | 2u) & ~1u)#define ionic_clear_ecn(tos)  ((tos) & ~3u) static int ionic_validate_qdesc(struct ionic_qdesc *q){	if (!q->addr || !q->size || !q->mask ||	    !q->depth_log2 || !q->stride_log2)		return -EINVAL; 	if (q->addr & (PAGE_SIZE - 1))		return -EINVAL; 	if (q->mask != BIT(q->depth_log2) - 1)		return -EINVAL; 	if (q->size < BIT_ULL(q->depth_log2 + q->stride_log2))		return -EINVAL; 	return 0;} static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx){	/* EQ per vector per udma, and the first eqs reserved for async events.	 * The rest of the vectors can be requested for completions.	 */	u32 comp_vec_count = dev->lif_cfg.eq_count / dev->lif_cfg.udma_count - 1; 	return (comp_vector % comp_vec_count + 1) * dev->lif_cfg.udma_count + udma_idx;} static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx){	unsigned int size, base, bound;	int rc; 	size = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count;	base = size * udma_idx;	bound = base + size; 	rc = ionic_resid_get_shared(&dev->inuse_cqid, base, bound);	if (rc >= 0) {		/* cq_base is zero or a multiple of two queue groups */		*cqid = dev->lif_cfg.cq_base +			ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift,					   dev->half_cqid_udma_shift); 		rc = 0;	} 	return rc;} static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid){	u32 bitid = ionic_qid_to_bitid(cqid - dev->lif_cfg.cq_base,				       dev->lif_cfg.udma_qgrp_shift,				       dev->half_cqid_udma_shift); 	ionic_resid_put(&dev->inuse_cqid, bitid);} int ionic_create_cq_common(struct ionic_vcq *vcq,			   struct ionic_tbl_buf *buf,			   const struct ib_cq_init_attr *attr,			   struct ionic_ctx *ctx,			   struct ib_udata *udata,			   struct ionic_qdesc *req_cq,			   __u32 *resp_cqid,			   int udma_idx){	struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device);	struct ionic_cq *cq = &vcq->cq[udma_idx];	void *entry;	int rc; 	cq->vcq = vcq; 	if (attr->cqe < 1 || attr->cqe + IONIC_CQ_GRACE > 0xffff) {		rc = -EINVAL;		goto err_args;	} 	rc = ionic_get_cqid(dev, &cq->cqid, udma_idx);	if (rc)		goto err_args; 	cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx); 	spin_lock_init(&cq->lock);	INIT_LIST_HEAD(&cq->poll_sq);	INIT_LIST_HEAD(&cq->flush_sq);	INIT_LIST_HEAD(&cq->flush_rq); 	if (udata) {		rc = ionic_validate_qdesc(req_cq);		if (rc)			goto err_qdesc; 		cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr,					  req_cq->size, IB_ACCESS_LOCAL_WRITE);		if (IS_ERR(cq->umem)) {			rc = PTR_ERR(cq->umem);			goto err_qdesc;		} 		cq->q.ptr = NULL;		cq->q.size = req_cq->size;		cq->q.mask = req_cq->mask;		cq->q.depth_log2 = req_cq->depth_log2;		cq->q.stride_log2 = req_cq->stride_log2; 		*resp_cqid = cq->cqid;	} else {		rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev,				      attr->cqe + IONIC_CQ_GRACE,				      sizeof(struct ionic_v1_cqe));		if (rc)			goto err_q_init; 		ionic_queue_dbell_init(&cq->q, cq->cqid);		cq->color = true;		cq->credit = cq->q.mask;	} 	rc = ionic_pgtbl_init(dev, buf, cq->umem, cq->q.dma, 1, PAGE_SIZE);	if (rc)		goto err_pgtbl_init; 	init_completion(&cq->cq_rel_comp);	kref_init(&cq->cq_kref); 	entry = xa_store_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL);	if (entry) {		if (!xa_is_err(entry))			rc = -EINVAL;		else			rc = xa_err(entry); 		goto err_xa;	} 	return 0; err_xa:	ionic_pgtbl_unbuf(dev, buf);err_pgtbl_init:	if (!udata)		ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);err_q_init:	if (cq->umem)		ib_umem_release(cq->umem);err_qdesc:	ionic_put_cqid(dev, cq->cqid);err_args:	cq->vcq = NULL; 	return rc;} void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq){	if (!cq->vcq)		return; 	xa_erase_irq(&dev->cq_tbl, cq->cqid); 	kref_put(&cq->cq_kref, ionic_cq_complete);	wait_for_completion(&cq->cq_rel_comp); 	if (cq->umem)		ib_umem_release(cq->umem);	else		ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); 	ionic_put_cqid(dev, cq->cqid); 	cq->vcq = NULL;} static int ionic_validate_qdesc_zero(struct ionic_qdesc *q){	if (q->addr || q->size || q->mask || q->depth_log2 || q->stride_log2)		return -EINVAL; 	return 0;} static int ionic_get_pdid(struct ionic_ibdev *dev, u32 *pdid){	int rc; 	rc = ionic_resid_get(&dev->inuse_pdid);	if (rc < 0)		return rc; 	*pdid = rc;	return 0;} static int ionic_get_ahid(struct ionic_ibdev *dev, u32 *ahid){	int rc; 	rc = ionic_resid_get(&dev->inuse_ahid);	if (rc < 0)		return rc; 	*ahid = rc;	return 0;} static int ionic_get_mrid(struct ionic_ibdev *dev, u32 *mrid){	int rc; 	/* wrap to 1, skip reserved lkey */	rc = ionic_resid_get_shared(&dev->inuse_mrid, 1,				    dev->inuse_mrid.inuse_size);	if (rc < 0)		return rc; 	*mrid = ionic_mrid(rc, dev->next_mrkey++);	return 0;} static int ionic_get_gsi_qpid(struct ionic_ibdev *dev, u32 *qpid){	int rc = 0; 	rc = ionic_resid_get_shared(&dev->inuse_qpid, IB_QPT_GSI, IB_QPT_GSI + 1);	if (rc < 0)		return rc; 	*qpid = IB_QPT_GSI;	return 0;} static int ionic_get_qpid(struct ionic_ibdev *dev, u32 *qpid,			  u8 *udma_idx, u8 udma_mask){	unsigned int size, base, bound;	int udma_i, udma_x, udma_ix;	int rc = -EINVAL; 	udma_x = dev->next_qpid_udma_idx; 	dev->next_qpid_udma_idx ^= dev->lif_cfg.udma_count - 1; 	for (udma_i = 0; udma_i < dev->lif_cfg.udma_count; ++udma_i) {		udma_ix = udma_i ^ udma_x; 		if (!(udma_mask & BIT(udma_ix)))			continue; 		size = dev->lif_cfg.qp_count / dev->lif_cfg.udma_count;		base = size * udma_ix;		bound = base + size; 		/* skip reserved SMI and GSI qpids in group zero */		if (!base)			base = 2; 		rc = ionic_resid_get_shared(&dev->inuse_qpid, base, bound);		if (rc >= 0) {			*qpid = ionic_bitid_to_qid(rc,						   dev->lif_cfg.udma_qgrp_shift,						   dev->half_qpid_udma_shift);			*udma_idx = udma_ix; 			rc = 0;			break;		}	} 	return rc;} static int ionic_get_dbid(struct ionic_ibdev *dev, u32 *dbid, phys_addr_t *addr){	int rc, dbpage_num; 	/* wrap to 1, skip kernel reserved */	rc = ionic_resid_get_shared(&dev->inuse_dbid, 1,				    dev->inuse_dbid.inuse_size);	if (rc < 0)		return rc; 	dbpage_num = (dev->lif_cfg.lif_hw_index * dev->lif_cfg.dbid_count) + rc;	*addr = dev->lif_cfg.db_phys + ((phys_addr_t)dbpage_num << PAGE_SHIFT); 	*dbid = rc; 	return 0;} static void ionic_put_pdid(struct ionic_ibdev *dev, u32 pdid){	ionic_resid_put(&dev->inuse_pdid, pdid);} static void ionic_put_ahid(struct ionic_ibdev *dev, u32 ahid){	ionic_resid_put(&dev->inuse_ahid, ahid);} static void ionic_put_mrid(struct ionic_ibdev *dev, u32 mrid){	ionic_resid_put(&dev->inuse_mrid, ionic_mrid_index(mrid));} static void ionic_put_qpid(struct ionic_ibdev *dev, u32 qpid){	u32 bitid = ionic_qid_to_bitid(qpid,				       dev->lif_cfg.udma_qgrp_shift,				       dev->half_qpid_udma_shift); 	ionic_resid_put(&dev->inuse_qpid, bitid);} static void ionic_put_dbid(struct ionic_ibdev *dev, u32 dbid){	ionic_resid_put(&dev->inuse_dbid, dbid);} static struct rdma_user_mmap_entry*ionic_mmap_entry_insert(struct ionic_ctx *ctx, unsigned long size,			unsigned long pfn, u8 mmap_flags, u64 *offset){	struct ionic_mmap_entry *entry;	int rc; 	entry = kzalloc_obj(*entry);	if (!entry)		return NULL; 	entry->size = size;	entry->pfn = pfn;	entry->mmap_flags = mmap_flags; 	rc = rdma_user_mmap_entry_insert(&ctx->ibctx, &entry->rdma_entry,					 entry->size);	if (rc) {		kfree(entry);		return NULL;	} 	if (offset)		*offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 	return &entry->rdma_entry;} int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata){	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);	struct ionic_ctx *ctx = to_ionic_ctx(ibctx);	struct ionic_ctx_resp resp = {};	struct ionic_ctx_req req;	phys_addr_t db_phys = 0;	int rc; 	rc = ib_copy_validate_udata_in(udata, req, rsvd);	if (rc)		return rc; 	/* try to allocate dbid for user ctx */	rc = ionic_get_dbid(dev, &ctx->dbid, &db_phys);	if (rc < 0)		return rc; 	ibdev_dbg(&dev->ibdev, "user space dbid %u\n", ctx->dbid); 	ctx->mmap_dbell = ionic_mmap_entry_insert(ctx, PAGE_SIZE,						  PHYS_PFN(db_phys), 0, NULL);	if (!ctx->mmap_dbell) {		rc = -ENOMEM;		goto err_mmap_dbell;	} 	if (dev->lif_cfg.phc_state) {		ctx->mmap_phc = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 0,							IONIC_MMAP_PHC,							&resp.phc_offset);		if (!ctx->mmap_phc) {			rc = -ENOMEM;			goto err_mmap_phc;		}	} 	resp.page_shift = PAGE_SHIFT; 	resp.dbell_offset = db_phys & ~PAGE_MASK; 	resp.version = dev->lif_cfg.rdma_version;	resp.qp_opcodes = dev->lif_cfg.qp_opcodes;	resp.admin_opcodes = dev->lif_cfg.admin_opcodes; 	resp.sq_qtype = dev->lif_cfg.sq_qtype;	resp.rq_qtype = dev->lif_cfg.rq_qtype;	resp.cq_qtype = dev->lif_cfg.cq_qtype;	resp.admin_qtype = dev->lif_cfg.aq_qtype;	resp.max_stride = dev->lif_cfg.max_stride;	resp.max_spec = IONIC_SPEC_HIGH; 	resp.udma_count = dev->lif_cfg.udma_count;	resp.expdb_mask = dev->lif_cfg.expdb_mask; 	if (dev->lif_cfg.sq_expdb)		resp.expdb_qtypes |= IONIC_EXPDB_SQ;	if (dev->lif_cfg.rq_expdb)		resp.expdb_qtypes |= IONIC_EXPDB_RQ; 	rc = ib_respond_udata(udata, resp);	if (rc)		goto err_resp; 	return 0; err_resp:	rdma_user_mmap_entry_remove(ctx->mmap_phc);err_mmap_phc:	rdma_user_mmap_entry_remove(ctx->mmap_dbell);err_mmap_dbell:	ionic_put_dbid(dev, ctx->dbid); 	return rc;} void ionic_dealloc_ucontext(struct ib_ucontext *ibctx){	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);	struct ionic_ctx *ctx = to_ionic_ctx(ibctx); 	rdma_user_mmap_entry_remove(ctx->mmap_phc);	rdma_user_mmap_entry_remove(ctx->mmap_dbell);	ionic_put_dbid(dev, ctx->dbid);} static int ionic_mmap_phc_state(struct ionic_ibdev *dev,				struct vm_area_struct *vma){	if (!(vma->vm_flags & VM_SHARED))		return -EINVAL; 	if (vma->vm_flags & (VM_WRITE | VM_EXEC))		return -EPERM; 	vm_flags_clear(vma, VM_MAYWRITE); 	return vm_insert_page(vma, vma->vm_start,			      virt_to_page(dev->lif_cfg.phc_state));} int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma){	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);	struct ionic_ctx *ctx = to_ionic_ctx(ibctx);	struct rdma_user_mmap_entry *rdma_entry;	struct ionic_mmap_entry *ionic_entry;	int rc = 0; 	rdma_entry = rdma_user_mmap_entry_get(&ctx->ibctx, vma);	if (!rdma_entry) {		ibdev_dbg(&dev->ibdev, "not found %#lx\n",			  vma->vm_pgoff << PAGE_SHIFT);		return -EINVAL;	} 	ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry,				   rdma_entry); 	if (ionic_entry->mmap_flags & IONIC_MMAP_PHC) {		rc = ionic_mmap_phc_state(dev, vma);		rdma_user_mmap_entry_put(rdma_entry);		return rc;	} 	ibdev_dbg(&dev->ibdev, "writecombine? %d\n",		  ionic_entry->mmap_flags & IONIC_MMAP_WC);	if (ionic_entry->mmap_flags & IONIC_MMAP_WC)		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);	else		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 	ibdev_dbg(&dev->ibdev, "remap st %#lx pf %#lx sz %#lx\n",		  vma->vm_start, ionic_entry->pfn, ionic_entry->size);

Lines 1–500 of 2768. Load more