// SPDX-License-Identifier: GPL-2.0/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ #include <linux/module.h>#include <linux/printk.h>#include <rdma/ib_addr.h>#include <rdma/ib_user_verbs.h> #include "ionic_fw.h"#include "ionic_ibdev.h" #define IONIC_OP(version, opname) \ ((version) < 2 ? IONIC_V1_OP_##opname : IONIC_V2_OP_##opname) static bool ionic_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_v1_cqe **cqe){ struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q); if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) return false; /* Prevent out-of-order reads of the CQE */ dma_rmb(); *cqe = qcqe; return true;} static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc){ struct ionic_rq_meta *meta; struct ionic_v1_wqe *wqe; u64 wqe_idx; if (!qp->rq_flush) return 0; if (ionic_queue_empty(&qp->rq)) return 0; wqe = ionic_queue_at_cons(&qp->rq); wqe_idx = le64_to_cpu(wqe->base.wqe_idx); /* wqe_idx must be a valid queue index */ if (unlikely(wqe_idx >> qp->rq.depth_log2)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu invalid\n", qp->qpid, (unsigned long long)wqe_idx); return -EIO; } /* wqe_idx must indicate a request that is outstanding */ meta = &qp->rq_meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu not posted\n", qp->qpid, (unsigned long long)wqe_idx); return -EIO; } ionic_queue_consume(&qp->rq); memset(wc, 0, sizeof(*wc)); wc->status = IB_WC_WR_FLUSH_ERR; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp; meta->next = qp->rq_meta_head; qp->rq_meta_head = meta; return 1;} static int ionic_flush_recv_many(struct ionic_qp *qp, struct ib_wc *wc, int nwc){ int rc = 0, npolled = 0; while (npolled < nwc) { rc = ionic_flush_recv(qp, wc + npolled); if (rc <= 0) break; npolled += rc; } return npolled ?: rc;} static int ionic_flush_send(struct ionic_qp *qp, struct ib_wc *wc){ struct ionic_sq_meta *meta; if (!qp->sq_flush) return 0; if (ionic_queue_empty(&qp->sq)) return 0; meta = &qp->sq_meta[qp->sq.cons]; ionic_queue_consume(&qp->sq); memset(wc, 0, sizeof(*wc)); wc->status = IB_WC_WR_FLUSH_ERR; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp; return 1;} static int ionic_flush_send_many(struct ionic_qp *qp, struct ib_wc *wc, int nwc){ int rc = 0, npolled = 0; while (npolled < nwc) { rc = ionic_flush_send(qp, wc + npolled); if (rc <= 0) break; npolled += rc; } return npolled ?: rc;} static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *cqe_qp, struct ionic_v1_cqe *cqe, struct ib_wc *wc){ struct ionic_qp *qp = NULL; struct ionic_rq_meta *meta; u16 vlan_tag, wqe_idx; u32 src_qpn, st_len; u8 op; if (cqe_qp->rq_flush) return 0; qp = cqe_qp; st_len = be32_to_cpu(cqe->status_length); /* ignore wqe_idx in case of flush error */ if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { cqe_qp->rq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); /* posted recvs (if any) flushed by ionic_flush_recv */ return 0; } /* there had better be something in the recv queue to complete */ if (ionic_queue_empty(&qp->rq)) { ibdev_warn(&dev->ibdev, "qp %u is empty\n", qp->qpid); return -EIO; } wqe_idx = le64_to_cpu(cqe->recv.wqe_idx_timestamp) & IONIC_V1_CQE_WQE_IDX_MASK; /* wqe_idx must be a valid queue index */ if (unlikely(wqe_idx >> qp->rq.depth_log2)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u invalid\n", qp->qpid, wqe_idx); return -EIO; } /* wqe_idx must indicate a request that is outstanding */ meta = &qp->rq_meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u not posted\n", qp->qpid, wqe_idx); return -EIO; } meta->next = qp->rq_meta_head; qp->rq_meta_head = meta; memset(wc, 0, sizeof(*wc)); wc->wr_id = meta->wrid; wc->qp = &cqe_qp->ibqp; if (ionic_v1_cqe_error(cqe)) { wc->vendor_err = st_len; wc->status = ionic_to_ib_status(st_len); cqe_qp->rq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); ibdev_warn(&dev->ibdev, "qp %d recv cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, BIT(cq->q.stride_log2), true); goto out; } wc->vendor_err = 0; wc->status = IB_WC_SUCCESS; src_qpn = be32_to_cpu(cqe->recv.src_qpn_op); op = src_qpn >> IONIC_V1_CQE_RECV_OP_SHIFT; src_qpn &= IONIC_V1_CQE_RECV_QPN_MASK; op &= IONIC_V1_CQE_RECV_OP_MASK; wc->opcode = IB_WC_RECV; switch (op) { case IONIC_V1_CQE_RECV_OP_RDMA_IMM: wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; wc->wc_flags |= IB_WC_WITH_IMM; wc->ex.imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */ break; case IONIC_V1_CQE_RECV_OP_SEND_IMM: wc->wc_flags |= IB_WC_WITH_IMM; wc->ex.imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */ break; case IONIC_V1_CQE_RECV_OP_SEND_INV: wc->wc_flags |= IB_WC_WITH_INVALIDATE; wc->ex.invalidate_rkey = be32_to_cpu(cqe->recv.imm_data_rkey); break; } wc->byte_len = st_len; wc->src_qp = src_qpn; if (qp->ibqp.qp_type == IB_QPT_UD || qp->ibqp.qp_type == IB_QPT_GSI) { wc->wc_flags |= IB_WC_GRH | IB_WC_WITH_SMAC; ether_addr_copy(wc->smac, cqe->recv.src_mac); wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; if (ionic_v1_cqe_recv_is_ipv4(cqe)) wc->network_hdr_type = RDMA_NETWORK_IPV4; else wc->network_hdr_type = RDMA_NETWORK_IPV6; if (ionic_v1_cqe_recv_is_vlan(cqe)) wc->wc_flags |= IB_WC_WITH_VLAN; /* vlan_tag in cqe will be valid from dpath even if no vlan */ vlan_tag = be16_to_cpu(cqe->recv.vlan_tag); wc->vlan_id = vlan_tag & 0xfff; /* 802.1q VID */ wc->sl = vlan_tag >> VLAN_PRIO_SHIFT; /* 802.1q PCP */ } wc->pkey_index = 0; wc->port_num = 1; out: ionic_queue_consume(&qp->rq); return 1;} static bool ionic_peek_send(struct ionic_qp *qp){ struct ionic_sq_meta *meta; if (qp->sq_flush) return false; /* completed all send queue requests */ if (ionic_queue_empty(&qp->sq)) return false; meta = &qp->sq_meta[qp->sq.cons]; /* waiting for remote completion */ if (meta->remote && meta->seq == qp->sq_msn_cons) return false; /* waiting for local completion */ if (!meta->remote && !meta->local_comp) return false; return true;} static int ionic_poll_send(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *qp, struct ib_wc *wc){ struct ionic_sq_meta *meta; if (qp->sq_flush) return 0; do { /* completed all send queue requests */ if (ionic_queue_empty(&qp->sq)) goto out_empty; meta = &qp->sq_meta[qp->sq.cons]; /* waiting for remote completion */ if (meta->remote && meta->seq == qp->sq_msn_cons) goto out_empty; /* waiting for local completion */ if (!meta->remote && !meta->local_comp) goto out_empty; ionic_queue_consume(&qp->sq); /* produce wc only if signaled or error status */ } while (!meta->signal && meta->ibsts == IB_WC_SUCCESS); memset(wc, 0, sizeof(*wc)); wc->status = meta->ibsts; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp; if (meta->ibsts == IB_WC_SUCCESS) { wc->byte_len = meta->len; wc->opcode = meta->ibop; } else { wc->vendor_err = meta->len; qp->sq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); } return 1; out_empty: if (qp->sq_flush_rcvd) { qp->sq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); } return 0;} static int ionic_poll_send_many(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *qp, struct ib_wc *wc, int nwc){ int rc = 0, npolled = 0; while (npolled < nwc) { rc = ionic_poll_send(dev, cq, qp, wc + npolled); if (rc <= 0) break; npolled += rc; } return npolled ?: rc;} static int ionic_validate_cons(u16 prod, u16 cons, u16 comp, u16 mask){ if (((prod - cons) & mask) <= ((comp - cons) & mask)) return -EIO; return 0;} static int ionic_comp_msn(struct ionic_qp *qp, struct ionic_v1_cqe *cqe){ struct ionic_sq_meta *meta; u16 cqe_seq, cqe_idx; int rc; if (qp->sq_flush) return 0; cqe_seq = be32_to_cpu(cqe->send.msg_msn) & qp->sq.mask; rc = ionic_validate_cons(qp->sq_msn_prod, qp->sq_msn_cons, cqe_seq - 1, qp->sq.mask); if (rc) { ibdev_warn(qp->ibqp.device, "qp %u bad msn %#x seq %u for prod %u cons %u\n", qp->qpid, be32_to_cpu(cqe->send.msg_msn), cqe_seq, qp->sq_msn_prod, qp->sq_msn_cons); return rc; } qp->sq_msn_cons = cqe_seq; if (ionic_v1_cqe_error(cqe)) { cqe_idx = qp->sq_msn_idx[(cqe_seq - 1) & qp->sq.mask]; meta = &qp->sq_meta[cqe_idx]; meta->len = be32_to_cpu(cqe->status_length); meta->ibsts = ionic_to_ib_status(meta->len); ibdev_warn(qp->ibqp.device, "qp %d msn cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, sizeof(*cqe), true); } return 0;} static int ionic_comp_npg(struct ionic_qp *qp, struct ionic_v1_cqe *cqe){ struct ionic_sq_meta *meta; u16 wqe_idx; u32 st_len; if (qp->sq_flush) return 0; st_len = be32_to_cpu(cqe->status_length); if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { /* * Flush cqe does not consume a wqe on the device, and maybe * no such work request is posted. * * The driver should begin flushing after the last indicated * normal or error completion. Here, only set a hint that the * flush request was indicated. In poll_send, if nothing more * can be polled normally, then begin flushing. */ qp->sq_flush_rcvd = true; return 0; } wqe_idx = le64_to_cpu(cqe->send.npg_wqe_idx_timestamp) & qp->sq.mask; meta = &qp->sq_meta[wqe_idx]; meta->local_comp = true; if (ionic_v1_cqe_error(cqe)) { meta->len = st_len; meta->ibsts = ionic_to_ib_status(st_len); meta->remote = false; ibdev_warn(qp->ibqp.device, "qp %d npg cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, sizeof(*cqe), true); } return 0;} static void ionic_reserve_sync_cq(struct ionic_ibdev *dev, struct ionic_cq *cq){ if (!ionic_queue_empty(&cq->q)) { cq->credit += ionic_queue_length(&cq->q); cq->q.cons = cq->q.prod; ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, ionic_queue_dbell_val(&cq->q)); }} static void ionic_reserve_cq(struct ionic_ibdev *dev, struct ionic_cq *cq, int spend){ cq->credit -= spend; if (cq->credit <= 0) ionic_reserve_sync_cq(dev, cq);} static int ionic_poll_vcq_cq(struct ionic_ibdev *dev, struct ionic_cq *cq, int nwc, struct ib_wc *wc){ struct ionic_qp *qp, *qp_next; struct ionic_v1_cqe *cqe; int rc = 0, npolled = 0; unsigned long irqflags; u32 qtf, qid; bool peek; u8 type; if (nwc < 1) return 0; spin_lock_irqsave(&cq->lock, irqflags); /* poll already indicated work completions for send queue */ list_for_each_entry_safe(qp, qp_next, &cq->poll_sq, cq_poll_sq) { if (npolled == nwc) goto out; spin_lock(&qp->sq_lock); rc = ionic_poll_send_many(dev, cq, qp, wc + npolled, nwc - npolled); spin_unlock(&qp->sq_lock); if (rc > 0) npolled += rc; Lines 1–500 of 1400. Load more
// SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ #include <linux/module.h> #include <linux/printk.h> #include <rdma/ib_addr.h> #include <rdma/ib_user_verbs.h> #include "ionic_fw.h" #include "ionic_ibdev.h" #define IONIC_OP(version, opname) \ ((version) < 2 ? IONIC_V1_OP_##opname : IONIC_V2_OP_##opname) static bool ionic_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_v1_cqe **cqe) { struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q); if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) return false; /* Prevent out-of-order reads of the CQE */ dma_rmb(); *cqe = qcqe; return true; } static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) { struct ionic_rq_meta *meta; struct ionic_v1_wqe *wqe; u64 wqe_idx; if (!qp->rq_flush) return 0; if (ionic_queue_empty(&qp->rq)) return 0; wqe = ionic_queue_at_cons(&qp->rq); wqe_idx = le64_to_cpu(wqe->base.wqe_idx); /* wqe_idx must be a valid queue index */ if (unlikely(wqe_idx >> qp->rq.depth_log2)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu invalid\n", qp->qpid, (unsigned long long)wqe_idx); return -EIO; } /* wqe_idx must indicate a request that is outstanding */ meta = &qp->rq_meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu not posted\n", qp->qpid, (unsigned long long)wqe_idx); return -EIO; } ionic_queue_consume(&qp->rq); memset(wc, 0, sizeof(*wc)); wc->status = IB_WC_WR_FLUSH_ERR; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp; meta->next = qp->rq_meta_head; qp->rq_meta_head = meta; return 1; } static int ionic_flush_recv_many(struct ionic_qp *qp, struct ib_wc *wc, int nwc) { int rc = 0, npolled = 0; while (npolled < nwc) { rc = ionic_flush_recv(qp, wc + npolled); if (rc <= 0) break; npolled += rc; } return npolled ?: rc; } static int ionic_flush_send(struct ionic_qp *qp, struct ib_wc *wc) { struct ionic_sq_meta *meta; if (!qp->sq_flush) return 0; if (ionic_queue_empty(&qp->sq)) return 0; meta = &qp->sq_meta[qp->sq.cons]; ionic_queue_consume(&qp->sq); memset(wc, 0, sizeof(*wc)); wc->status = IB_WC_WR_FLUSH_ERR; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp; return 1; } static int ionic_flush_send_many(struct ionic_qp *qp, struct ib_wc *wc, int nwc) { int rc = 0, npolled = 0; while (npolled < nwc) { rc = ionic_flush_send(qp, wc + npolled); if (rc <= 0) break; npolled += rc; } return npolled ?: rc; } static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *cqe_qp, struct ionic_v1_cqe *cqe, struct ib_wc *wc) { struct ionic_qp *qp = NULL; struct ionic_rq_meta *meta; u16 vlan_tag, wqe_idx; u32 src_qpn, st_len; u8 op; if (cqe_qp->rq_flush) return 0; qp = cqe_qp; st_len = be32_to_cpu(cqe->status_length); /* ignore wqe_idx in case of flush error */ if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { cqe_qp->rq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); /* posted recvs (if any) flushed by ionic_flush_recv */ return 0; } /* there had better be something in the recv queue to complete */ if (ionic_queue_empty(&qp->rq)) { ibdev_warn(&dev->ibdev, "qp %u is empty\n", qp->qpid); return -EIO; } wqe_idx = le64_to_cpu(cqe->recv.wqe_idx_timestamp) & IONIC_V1_CQE_WQE_IDX_MASK; /* wqe_idx must be a valid queue index */ if (unlikely(wqe_idx >> qp->rq.depth_log2)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u invalid\n", qp->qpid, wqe_idx); return -EIO; } /* wqe_idx must indicate a request that is outstanding */ meta = &qp->rq_meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u not posted\n", qp->qpid, wqe_idx); return -EIO; } meta->next = qp->rq_meta_head; qp->rq_meta_head = meta; memset(wc, 0, sizeof(*wc)); wc->wr_id = meta->wrid; wc->qp = &cqe_qp->ibqp; if (ionic_v1_cqe_error(cqe)) { wc->vendor_err = st_len; wc->status = ionic_to_ib_status(st_len); cqe_qp->rq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); ibdev_warn(&dev->ibdev, "qp %d recv cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, BIT(cq->q.stride_log2), true); goto out; } wc->vendor_err = 0; wc->status = IB_WC_SUCCESS; src_qpn = be32_to_cpu(cqe->recv.src_qpn_op); op = src_qpn >> IONIC_V1_CQE_RECV_OP_SHIFT; src_qpn &= IONIC_V1_CQE_RECV_QPN_MASK; op &= IONIC_V1_CQE_RECV_OP_MASK; wc->opcode = IB_WC_RECV; switch (op) { case IONIC_V1_CQE_RECV_OP_RDMA_IMM: wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; wc->wc_flags |= IB_WC_WITH_IMM; wc->ex.imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */ break; case IONIC_V1_CQE_RECV_OP_SEND_IMM: wc->wc_flags |= IB_WC_WITH_IMM; wc->ex.imm_data = cqe->recv.imm_data_rkey; /* be32 in wc */ break; case IONIC_V1_CQE_RECV_OP_SEND_INV: wc->wc_flags |= IB_WC_WITH_INVALIDATE; wc->ex.invalidate_rkey = be32_to_cpu(cqe->recv.imm_data_rkey); break; } wc->byte_len = st_len; wc->src_qp = src_qpn; if (qp->ibqp.qp_type == IB_QPT_UD || qp->ibqp.qp_type == IB_QPT_GSI) { wc->wc_flags |= IB_WC_GRH | IB_WC_WITH_SMAC; ether_addr_copy(wc->smac, cqe->recv.src_mac); wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE; if (ionic_v1_cqe_recv_is_ipv4(cqe)) wc->network_hdr_type = RDMA_NETWORK_IPV4; else wc->network_hdr_type = RDMA_NETWORK_IPV6; if (ionic_v1_cqe_recv_is_vlan(cqe)) wc->wc_flags |= IB_WC_WITH_VLAN; /* vlan_tag in cqe will be valid from dpath even if no vlan */ vlan_tag = be16_to_cpu(cqe->recv.vlan_tag); wc->vlan_id = vlan_tag & 0xfff; /* 802.1q VID */ wc->sl = vlan_tag >> VLAN_PRIO_SHIFT; /* 802.1q PCP */ } wc->pkey_index = 0; wc->port_num = 1; out: ionic_queue_consume(&qp->rq); return 1; } static bool ionic_peek_send(struct ionic_qp *qp) { struct ionic_sq_meta *meta; if (qp->sq_flush) return false; /* completed all send queue requests */ if (ionic_queue_empty(&qp->sq)) return false; meta = &qp->sq_meta[qp->sq.cons]; /* waiting for remote completion */ if (meta->remote && meta->seq == qp->sq_msn_cons) return false; /* waiting for local completion */ if (!meta->remote && !meta->local_comp) return false; return true; } static int ionic_poll_send(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *qp, struct ib_wc *wc) { struct ionic_sq_meta *meta; if (qp->sq_flush) return 0; do { /* completed all send queue requests */ if (ionic_queue_empty(&qp->sq)) goto out_empty; meta = &qp->sq_meta[qp->sq.cons]; /* waiting for remote completion */ if (meta->remote && meta->seq == qp->sq_msn_cons) goto out_empty; /* waiting for local completion */ if (!meta->remote && !meta->local_comp) goto out_empty; ionic_queue_consume(&qp->sq); /* produce wc only if signaled or error status */ } while (!meta->signal && meta->ibsts == IB_WC_SUCCESS); memset(wc, 0, sizeof(*wc)); wc->status = meta->ibsts; wc->wr_id = meta->wrid; wc->qp = &qp->ibqp; if (meta->ibsts == IB_WC_SUCCESS) { wc->byte_len = meta->len; wc->opcode = meta->ibop; } else { wc->vendor_err = meta->len; qp->sq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); } return 1; out_empty: if (qp->sq_flush_rcvd) { qp->sq_flush = true; cq->flush = true; list_move_tail(&qp->cq_flush_sq, &cq->flush_sq); } return 0; } static int ionic_poll_send_many(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_qp *qp, struct ib_wc *wc, int nwc) { int rc = 0, npolled = 0; while (npolled < nwc) { rc = ionic_poll_send(dev, cq, qp, wc + npolled); if (rc <= 0) break; npolled += rc; } return npolled ?: rc; } static int ionic_validate_cons(u16 prod, u16 cons, u16 comp, u16 mask) { if (((prod - cons) & mask) <= ((comp - cons) & mask)) return -EIO; return 0; } static int ionic_comp_msn(struct ionic_qp *qp, struct ionic_v1_cqe *cqe) { struct ionic_sq_meta *meta; u16 cqe_seq, cqe_idx; int rc; if (qp->sq_flush) return 0; cqe_seq = be32_to_cpu(cqe->send.msg_msn) & qp->sq.mask; rc = ionic_validate_cons(qp->sq_msn_prod, qp->sq_msn_cons, cqe_seq - 1, qp->sq.mask); if (rc) { ibdev_warn(qp->ibqp.device, "qp %u bad msn %#x seq %u for prod %u cons %u\n", qp->qpid, be32_to_cpu(cqe->send.msg_msn), cqe_seq, qp->sq_msn_prod, qp->sq_msn_cons); return rc; } qp->sq_msn_cons = cqe_seq; if (ionic_v1_cqe_error(cqe)) { cqe_idx = qp->sq_msn_idx[(cqe_seq - 1) & qp->sq.mask]; meta = &qp->sq_meta[cqe_idx]; meta->len = be32_to_cpu(cqe->status_length); meta->ibsts = ionic_to_ib_status(meta->len); ibdev_warn(qp->ibqp.device, "qp %d msn cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, sizeof(*cqe), true); } return 0; } static int ionic_comp_npg(struct ionic_qp *qp, struct ionic_v1_cqe *cqe) { struct ionic_sq_meta *meta; u16 wqe_idx; u32 st_len; if (qp->sq_flush) return 0; st_len = be32_to_cpu(cqe->status_length); if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { /* * Flush cqe does not consume a wqe on the device, and maybe * no such work request is posted. * * The driver should begin flushing after the last indicated * normal or error completion. Here, only set a hint that the * flush request was indicated. In poll_send, if nothing more * can be polled normally, then begin flushing. */ qp->sq_flush_rcvd = true; return 0; } wqe_idx = le64_to_cpu(cqe->send.npg_wqe_idx_timestamp) & qp->sq.mask; meta = &qp->sq_meta[wqe_idx]; meta->local_comp = true; if (ionic_v1_cqe_error(cqe)) { meta->len = st_len; meta->ibsts = ionic_to_ib_status(st_len); meta->remote = false; ibdev_warn(qp->ibqp.device, "qp %d npg cqe with error\n", qp->qpid); print_hex_dump(KERN_WARNING, "cqe ", DUMP_PREFIX_OFFSET, 16, 1, cqe, sizeof(*cqe), true); } return 0; } static void ionic_reserve_sync_cq(struct ionic_ibdev *dev, struct ionic_cq *cq) { if (!ionic_queue_empty(&cq->q)) { cq->credit += ionic_queue_length(&cq->q); cq->q.cons = cq->q.prod; ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, ionic_queue_dbell_val(&cq->q)); } } static void ionic_reserve_cq(struct ionic_ibdev *dev, struct ionic_cq *cq, int spend) { cq->credit -= spend; if (cq->credit <= 0) ionic_reserve_sync_cq(dev, cq); } static int ionic_poll_vcq_cq(struct ionic_ibdev *dev, struct ionic_cq *cq, int nwc, struct ib_wc *wc) { struct ionic_qp *qp, *qp_next; struct ionic_v1_cqe *cqe; int rc = 0, npolled = 0; unsigned long irqflags; u32 qtf, qid; bool peek; u8 type; if (nwc < 1) return 0; spin_lock_irqsave(&cq->lock, irqflags); /* poll already indicated work completions for send queue */ list_for_each_entry_safe(qp, qp_next, &cq->poll_sq, cq_poll_sq) { if (npolled == nwc) goto out; spin_lock(&qp->sq_lock); rc = ionic_poll_send_many(dev, cq, qp, wc + npolled, nwc - npolled); spin_unlock(&qp->sq_lock); if (rc > 0) npolled += rc;