// SPDX-License-Identifier: GPL-2.0/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ #include <linux/interrupt.h>#include <linux/module.h>#include <linux/printk.h> #include "ionic_fw.h"#include "ionic_ibdev.h" #define IONIC_EQ_COUNT_MIN 4#define IONIC_AQ_COUNT_MIN 1 /* not a valid queue position or negative error status */#define IONIC_ADMIN_POSTED 0x10000 /* cpu can be held with irq disabled for COUNT * MS (for create/destroy_ah) */#define IONIC_ADMIN_BUSY_RETRY_COUNT 2000#define IONIC_ADMIN_BUSY_RETRY_MS 1 /* admin queue will be considered failed if a command takes longer */#define IONIC_ADMIN_TIMEOUT (HZ * 2)#define IONIC_ADMIN_WARN (HZ / 8) /* will poll for admin cq to tolerate and report from missed event */#define IONIC_ADMIN_DELAY (HZ / 8) /* work queue for polling the event queue and admin cq */struct workqueue_struct *ionic_evt_workq; static void ionic_admin_timedout(struct ionic_aq *aq){ struct ionic_ibdev *dev = aq->dev; unsigned long irqflags; u16 pos; spin_lock_irqsave(&aq->lock, irqflags); if (ionic_queue_empty(&aq->q)) goto out; /* Reset ALL adminq if any one times out */ if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) queue_work(ionic_evt_workq, &dev->reset_work); ibdev_err(&dev->ibdev, "admin command timed out, aq %d after: %ums\n", aq->aqid, (u32)jiffies_to_msecs(jiffies - aq->stamp)); pos = (aq->q.prod - 1) & aq->q.mask; if (pos == aq->q.cons) goto out; ibdev_warn(&dev->ibdev, "admin pos %u (last posted)\n", pos); print_hex_dump(KERN_WARNING, "cmd ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at(&aq->q, pos), BIT(aq->q.stride_log2), true); out: spin_unlock_irqrestore(&aq->lock, irqflags);} static void ionic_admin_reset_dwork(struct ionic_ibdev *dev){ if (atomic_read(&dev->admin_state) == IONIC_ADMIN_KILLED) return; queue_delayed_work(ionic_evt_workq, &dev->admin_dwork, IONIC_ADMIN_DELAY);} static void ionic_admin_reset_wdog(struct ionic_aq *aq){ if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) return; aq->stamp = jiffies; ionic_admin_reset_dwork(aq->dev);} static bool ionic_admin_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_v1_cqe **cqe){ struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q); if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) return false; /* Prevent out-of-order reads of the CQE */ dma_rmb(); *cqe = qcqe; return true;} static void ionic_admin_poll_locked(struct ionic_aq *aq){ struct ionic_cq *cq = &aq->vcq->cq[0]; struct ionic_admin_wr *wr, *wr_next; struct ionic_ibdev *dev = aq->dev; u32 wr_strides, avlbl_strides; struct ionic_v1_cqe *cqe; u32 qtf, qid; u16 old_prod; u8 type; lockdep_assert_held(&aq->lock); if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) { list_for_each_entry_safe(wr, wr_next, &aq->wr_prod, aq_ent) { INIT_LIST_HEAD(&wr->aq_ent); aq->q_wr[wr->status].wr = NULL; wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); } INIT_LIST_HEAD(&aq->wr_prod); list_for_each_entry_safe(wr, wr_next, &aq->wr_post, aq_ent) { INIT_LIST_HEAD(&wr->aq_ent); wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); } INIT_LIST_HEAD(&aq->wr_post); return; } old_prod = cq->q.prod; while (ionic_admin_next_cqe(dev, cq, &cqe)) { qtf = ionic_v1_cqe_qtf(cqe); qid = ionic_v1_cqe_qtf_qid(qtf); type = ionic_v1_cqe_qtf_type(qtf); if (unlikely(type != IONIC_V1_CQE_TYPE_ADMIN)) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe type %u\n", type); goto cq_next; } if (unlikely(qid != aq->aqid)) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe qid %u\n", qid); goto cq_next; } if (unlikely(be16_to_cpu(cqe->admin.cmd_idx) != aq->q.cons)) { ibdev_warn_ratelimited(&dev->ibdev, "bad idx %u cons %u qid %u\n", be16_to_cpu(cqe->admin.cmd_idx), aq->q.cons, qid); goto cq_next; } if (unlikely(ionic_queue_empty(&aq->q))) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe for empty adminq\n"); goto cq_next; } wr = aq->q_wr[aq->q.cons].wr; if (wr) { aq->q_wr[aq->q.cons].wr = NULL; list_del_init(&wr->aq_ent); wr->cqe = *cqe; wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); } ionic_queue_consume_entries(&aq->q, aq->q_wr[aq->q.cons].wqe_strides); cq_next: ionic_queue_produce(&cq->q); cq->color = ionic_color_wrap(cq->q.prod, cq->color); } if (old_prod != cq->q.prod) { ionic_admin_reset_wdog(aq); cq->q.cons = cq->q.prod; ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, ionic_queue_dbell_val(&cq->q)); queue_work(ionic_evt_workq, &aq->work); } else if (!aq->armed) { aq->armed = true; cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod); ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, cq->q.dbell | IONIC_CQ_RING_ARM | cq->arm_any_prod); queue_work(ionic_evt_workq, &aq->work); } if (atomic_read(&aq->admin_state) != IONIC_ADMIN_ACTIVE) return; old_prod = aq->q.prod; if (ionic_queue_empty(&aq->q) && !list_empty(&aq->wr_post)) ionic_admin_reset_wdog(aq); if (list_empty(&aq->wr_post)) return; do { u8 *src; int i, src_len; size_t stride_len; wr = list_first_entry(&aq->wr_post, struct ionic_admin_wr, aq_ent); wr_strides = (le16_to_cpu(wr->wqe.len) + ADMIN_WQE_HDR_LEN + (ADMIN_WQE_STRIDE - 1)) >> aq->q.stride_log2; avlbl_strides = ionic_queue_length_remaining(&aq->q); if (wr_strides > avlbl_strides) break; list_move(&wr->aq_ent, &aq->wr_prod); wr->status = aq->q.prod; aq->q_wr[aq->q.prod].wr = wr; aq->q_wr[aq->q.prod].wqe_strides = wr_strides; src_len = le16_to_cpu(wr->wqe.len); src = (uint8_t *)&wr->wqe.cmd; /* First stride */ memcpy(ionic_queue_at_prod(&aq->q), &wr->wqe, ADMIN_WQE_HDR_LEN); stride_len = ADMIN_WQE_STRIDE - ADMIN_WQE_HDR_LEN; if (stride_len > src_len) stride_len = src_len; memcpy(ionic_queue_at_prod(&aq->q) + ADMIN_WQE_HDR_LEN, src, stride_len); ibdev_dbg(&dev->ibdev, "post admin prod %u (%u strides)\n", aq->q.prod, wr_strides); print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at_prod(&aq->q), BIT(aq->q.stride_log2), true); ionic_queue_produce(&aq->q); /* Remaining strides */ for (i = stride_len; i < src_len; i += stride_len) { stride_len = ADMIN_WQE_STRIDE; if (i + stride_len > src_len) stride_len = src_len - i; memcpy(ionic_queue_at_prod(&aq->q), src + i, stride_len); print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at_prod(&aq->q), BIT(aq->q.stride_log2), true); ionic_queue_produce(&aq->q); } } while (!list_empty(&aq->wr_post)); if (old_prod != aq->q.prod) ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.aq_qtype, ionic_queue_dbell_val(&aq->q));} static void ionic_admin_dwork(struct work_struct *ws){ struct ionic_ibdev *dev = container_of(ws, struct ionic_ibdev, admin_dwork.work); struct ionic_aq *aq, *bad_aq = NULL; bool do_reschedule = false; unsigned long irqflags; bool do_reset = false; u16 pos; int i; for (i = 0; i < dev->lif_cfg.aq_count; i++) { aq = dev->aq_vec[i]; spin_lock_irqsave(&aq->lock, irqflags); if (ionic_queue_empty(&aq->q)) goto next_aq; /* Reschedule if any queue has outstanding work */ do_reschedule = true; if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_WARN)) /* Warning threshold not met, nothing to do */ goto next_aq; /* See if polling now makes some progress */ pos = aq->q.cons; ionic_admin_poll_locked(aq); if (pos != aq->q.cons) { ibdev_dbg(&dev->ibdev, "missed event for acq %d\n", aq->cqid); goto next_aq; } if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_TIMEOUT)) { /* Timeout threshold not met */ ibdev_dbg(&dev->ibdev, "no progress after %ums\n", (u32)jiffies_to_msecs(jiffies - aq->stamp)); goto next_aq; } /* Queue timed out */ bad_aq = aq; do_reset = true;next_aq: spin_unlock_irqrestore(&aq->lock, irqflags); } if (do_reset) /* Reset RDMA lif on a timeout */ ionic_admin_timedout(bad_aq); else if (do_reschedule) /* Try to poll again later */ ionic_admin_reset_dwork(dev);} static void ionic_admin_work(struct work_struct *ws){ struct ionic_aq *aq = container_of(ws, struct ionic_aq, work); unsigned long irqflags; spin_lock_irqsave(&aq->lock, irqflags); ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags);} static void ionic_admin_post_aq(struct ionic_aq *aq, struct ionic_admin_wr *wr){ unsigned long irqflags; bool poll; wr->status = IONIC_ADMIN_POSTED; wr->aq = aq; spin_lock_irqsave(&aq->lock, irqflags); poll = list_empty(&aq->wr_post); list_add(&wr->aq_ent, &aq->wr_post); if (poll) ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags);} void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr){ int aq_idx; /* Use cpu id for the adminq selection */ aq_idx = raw_smp_processor_id() % dev->lif_cfg.aq_count; ionic_admin_post_aq(dev->aq_vec[aq_idx], wr);} static void ionic_admin_cancel(struct ionic_admin_wr *wr){ struct ionic_aq *aq = wr->aq; unsigned long irqflags; spin_lock_irqsave(&aq->lock, irqflags); if (!list_empty(&wr->aq_ent)) { list_del(&wr->aq_ent); if (wr->status != IONIC_ADMIN_POSTED) aq->q_wr[wr->status].wr = NULL; } spin_unlock_irqrestore(&aq->lock, irqflags);} static int ionic_admin_busy_wait(struct ionic_admin_wr *wr){ struct ionic_aq *aq = wr->aq; unsigned long irqflags; int try_i; for (try_i = 0; try_i < IONIC_ADMIN_BUSY_RETRY_COUNT; ++try_i) { if (completion_done(&wr->work)) return 0; mdelay(IONIC_ADMIN_BUSY_RETRY_MS); spin_lock_irqsave(&aq->lock, irqflags); ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags); } /* * we timed out. Initiate RDMA LIF reset and indicate * error to caller. */ ionic_admin_timedout(aq); return -ETIMEDOUT;} int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, enum ionic_admin_flags flags){ int rc, timo; if (flags & IONIC_ADMIN_F_BUSYWAIT) { /* Spin */ rc = ionic_admin_busy_wait(wr); } else if (flags & IONIC_ADMIN_F_INTERRUPT) { /* * Interruptible sleep, 1s timeout * This is used for commands which are safe for the caller * to clean up without killing and resetting the adminq. */ timo = wait_for_completion_interruptible_timeout(&wr->work, HZ); if (timo > 0) rc = 0; else if (timo == 0) rc = -ETIMEDOUT; else rc = timo; } else { /* * Uninterruptible sleep * This is used for commands which are NOT safe for the * caller to clean up. Cleanup must be handled by the * adminq kill and reset process so that host memory is * not corrupted by the device. */ wait_for_completion(&wr->work); rc = 0; } if (rc) { ibdev_warn(&dev->ibdev, "wait status %d\n", rc); ionic_admin_cancel(wr); } else if (wr->status == IONIC_ADMIN_KILLED) { ibdev_dbg(&dev->ibdev, "admin killed\n"); /* No error if admin already killed during teardown */ rc = (flags & IONIC_ADMIN_F_TEARDOWN) ? 0 : -ENODEV; } else if (ionic_v1_cqe_error(&wr->cqe)) { ibdev_warn(&dev->ibdev, "opcode %u error %u\n", wr->wqe.op, be32_to_cpu(wr->cqe.status_length)); rc = -EINVAL; } return rc;} static int ionic_rdma_devcmd(struct ionic_ibdev *dev, struct ionic_admin_ctx *admin){ int rc; rc = ionic_adminq_post_wait(dev->lif_cfg.lif, admin); if (rc) return rc; return ionic_error_to_errno(admin->comp.comp.status);} int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev){ struct ionic_admin_ctx admin = { .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), .cmd.rdma_reset = { .opcode = IONIC_CMD_RDMA_RESET_LIF, .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), }, }; return ionic_rdma_devcmd(dev, &admin);} static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev, struct ionic_queue *q, u32 qid, u32 cid, u16 opcode){ struct ionic_admin_ctx admin = { .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), .cmd.rdma_queue = { .opcode = opcode, .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), .qid_ver = cpu_to_le32(qid), .cid = cpu_to_le32(cid), .dbid = cpu_to_le16(dev->lif_cfg.dbid), .depth_log2 = q->depth_log2, .stride_log2 = q->stride_log2, .dma_addr = cpu_to_le64(q->dma), }, }; return ionic_rdma_devcmd(dev, &admin);} static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context){ struct ionic_aq *aq = cq_context; unsigned long irqflags; spin_lock_irqsave(&aq->lock, irqflags); aq->armed = false; if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) queue_work(ionic_evt_workq, &aq->work);Lines 1–500 of 1227. Load more
// SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ #include <linux/interrupt.h> #include <linux/module.h> #include <linux/printk.h> #include "ionic_fw.h" #include "ionic_ibdev.h" #define IONIC_EQ_COUNT_MIN 4 #define IONIC_AQ_COUNT_MIN 1 /* not a valid queue position or negative error status */ #define IONIC_ADMIN_POSTED 0x10000 /* cpu can be held with irq disabled for COUNT * MS (for create/destroy_ah) */ #define IONIC_ADMIN_BUSY_RETRY_COUNT 2000 #define IONIC_ADMIN_BUSY_RETRY_MS 1 /* admin queue will be considered failed if a command takes longer */ #define IONIC_ADMIN_TIMEOUT (HZ * 2) #define IONIC_ADMIN_WARN (HZ / 8) /* will poll for admin cq to tolerate and report from missed event */ #define IONIC_ADMIN_DELAY (HZ / 8) /* work queue for polling the event queue and admin cq */ struct workqueue_struct *ionic_evt_workq; static void ionic_admin_timedout(struct ionic_aq *aq) { struct ionic_ibdev *dev = aq->dev; unsigned long irqflags; u16 pos; spin_lock_irqsave(&aq->lock, irqflags); if (ionic_queue_empty(&aq->q)) goto out; /* Reset ALL adminq if any one times out */ if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) queue_work(ionic_evt_workq, &dev->reset_work); ibdev_err(&dev->ibdev, "admin command timed out, aq %d after: %ums\n", aq->aqid, (u32)jiffies_to_msecs(jiffies - aq->stamp)); pos = (aq->q.prod - 1) & aq->q.mask; if (pos == aq->q.cons) goto out; ibdev_warn(&dev->ibdev, "admin pos %u (last posted)\n", pos); print_hex_dump(KERN_WARNING, "cmd ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at(&aq->q, pos), BIT(aq->q.stride_log2), true); out: spin_unlock_irqrestore(&aq->lock, irqflags); } static void ionic_admin_reset_dwork(struct ionic_ibdev *dev) { if (atomic_read(&dev->admin_state) == IONIC_ADMIN_KILLED) return; queue_delayed_work(ionic_evt_workq, &dev->admin_dwork, IONIC_ADMIN_DELAY); } static void ionic_admin_reset_wdog(struct ionic_aq *aq) { if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) return; aq->stamp = jiffies; ionic_admin_reset_dwork(aq->dev); } static bool ionic_admin_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq, struct ionic_v1_cqe **cqe) { struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q); if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) return false; /* Prevent out-of-order reads of the CQE */ dma_rmb(); *cqe = qcqe; return true; } static void ionic_admin_poll_locked(struct ionic_aq *aq) { struct ionic_cq *cq = &aq->vcq->cq[0]; struct ionic_admin_wr *wr, *wr_next; struct ionic_ibdev *dev = aq->dev; u32 wr_strides, avlbl_strides; struct ionic_v1_cqe *cqe; u32 qtf, qid; u16 old_prod; u8 type; lockdep_assert_held(&aq->lock); if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) { list_for_each_entry_safe(wr, wr_next, &aq->wr_prod, aq_ent) { INIT_LIST_HEAD(&wr->aq_ent); aq->q_wr[wr->status].wr = NULL; wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); } INIT_LIST_HEAD(&aq->wr_prod); list_for_each_entry_safe(wr, wr_next, &aq->wr_post, aq_ent) { INIT_LIST_HEAD(&wr->aq_ent); wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); } INIT_LIST_HEAD(&aq->wr_post); return; } old_prod = cq->q.prod; while (ionic_admin_next_cqe(dev, cq, &cqe)) { qtf = ionic_v1_cqe_qtf(cqe); qid = ionic_v1_cqe_qtf_qid(qtf); type = ionic_v1_cqe_qtf_type(qtf); if (unlikely(type != IONIC_V1_CQE_TYPE_ADMIN)) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe type %u\n", type); goto cq_next; } if (unlikely(qid != aq->aqid)) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe qid %u\n", qid); goto cq_next; } if (unlikely(be16_to_cpu(cqe->admin.cmd_idx) != aq->q.cons)) { ibdev_warn_ratelimited(&dev->ibdev, "bad idx %u cons %u qid %u\n", be16_to_cpu(cqe->admin.cmd_idx), aq->q.cons, qid); goto cq_next; } if (unlikely(ionic_queue_empty(&aq->q))) { ibdev_warn_ratelimited(&dev->ibdev, "bad cqe for empty adminq\n"); goto cq_next; } wr = aq->q_wr[aq->q.cons].wr; if (wr) { aq->q_wr[aq->q.cons].wr = NULL; list_del_init(&wr->aq_ent); wr->cqe = *cqe; wr->status = atomic_read(&aq->admin_state); complete_all(&wr->work); } ionic_queue_consume_entries(&aq->q, aq->q_wr[aq->q.cons].wqe_strides); cq_next: ionic_queue_produce(&cq->q); cq->color = ionic_color_wrap(cq->q.prod, cq->color); } if (old_prod != cq->q.prod) { ionic_admin_reset_wdog(aq); cq->q.cons = cq->q.prod; ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, ionic_queue_dbell_val(&cq->q)); queue_work(ionic_evt_workq, &aq->work); } else if (!aq->armed) { aq->armed = true; cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod); ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, cq->q.dbell | IONIC_CQ_RING_ARM | cq->arm_any_prod); queue_work(ionic_evt_workq, &aq->work); } if (atomic_read(&aq->admin_state) != IONIC_ADMIN_ACTIVE) return; old_prod = aq->q.prod; if (ionic_queue_empty(&aq->q) && !list_empty(&aq->wr_post)) ionic_admin_reset_wdog(aq); if (list_empty(&aq->wr_post)) return; do { u8 *src; int i, src_len; size_t stride_len; wr = list_first_entry(&aq->wr_post, struct ionic_admin_wr, aq_ent); wr_strides = (le16_to_cpu(wr->wqe.len) + ADMIN_WQE_HDR_LEN + (ADMIN_WQE_STRIDE - 1)) >> aq->q.stride_log2; avlbl_strides = ionic_queue_length_remaining(&aq->q); if (wr_strides > avlbl_strides) break; list_move(&wr->aq_ent, &aq->wr_prod); wr->status = aq->q.prod; aq->q_wr[aq->q.prod].wr = wr; aq->q_wr[aq->q.prod].wqe_strides = wr_strides; src_len = le16_to_cpu(wr->wqe.len); src = (uint8_t *)&wr->wqe.cmd; /* First stride */ memcpy(ionic_queue_at_prod(&aq->q), &wr->wqe, ADMIN_WQE_HDR_LEN); stride_len = ADMIN_WQE_STRIDE - ADMIN_WQE_HDR_LEN; if (stride_len > src_len) stride_len = src_len; memcpy(ionic_queue_at_prod(&aq->q) + ADMIN_WQE_HDR_LEN, src, stride_len); ibdev_dbg(&dev->ibdev, "post admin prod %u (%u strides)\n", aq->q.prod, wr_strides); print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at_prod(&aq->q), BIT(aq->q.stride_log2), true); ionic_queue_produce(&aq->q); /* Remaining strides */ for (i = stride_len; i < src_len; i += stride_len) { stride_len = ADMIN_WQE_STRIDE; if (i + stride_len > src_len) stride_len = src_len - i; memcpy(ionic_queue_at_prod(&aq->q), src + i, stride_len); print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, ionic_queue_at_prod(&aq->q), BIT(aq->q.stride_log2), true); ionic_queue_produce(&aq->q); } } while (!list_empty(&aq->wr_post)); if (old_prod != aq->q.prod) ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.aq_qtype, ionic_queue_dbell_val(&aq->q)); } static void ionic_admin_dwork(struct work_struct *ws) { struct ionic_ibdev *dev = container_of(ws, struct ionic_ibdev, admin_dwork.work); struct ionic_aq *aq, *bad_aq = NULL; bool do_reschedule = false; unsigned long irqflags; bool do_reset = false; u16 pos; int i; for (i = 0; i < dev->lif_cfg.aq_count; i++) { aq = dev->aq_vec[i]; spin_lock_irqsave(&aq->lock, irqflags); if (ionic_queue_empty(&aq->q)) goto next_aq; /* Reschedule if any queue has outstanding work */ do_reschedule = true; if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_WARN)) /* Warning threshold not met, nothing to do */ goto next_aq; /* See if polling now makes some progress */ pos = aq->q.cons; ionic_admin_poll_locked(aq); if (pos != aq->q.cons) { ibdev_dbg(&dev->ibdev, "missed event for acq %d\n", aq->cqid); goto next_aq; } if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_TIMEOUT)) { /* Timeout threshold not met */ ibdev_dbg(&dev->ibdev, "no progress after %ums\n", (u32)jiffies_to_msecs(jiffies - aq->stamp)); goto next_aq; } /* Queue timed out */ bad_aq = aq; do_reset = true; next_aq: spin_unlock_irqrestore(&aq->lock, irqflags); } if (do_reset) /* Reset RDMA lif on a timeout */ ionic_admin_timedout(bad_aq); else if (do_reschedule) /* Try to poll again later */ ionic_admin_reset_dwork(dev); } static void ionic_admin_work(struct work_struct *ws) { struct ionic_aq *aq = container_of(ws, struct ionic_aq, work); unsigned long irqflags; spin_lock_irqsave(&aq->lock, irqflags); ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags); } static void ionic_admin_post_aq(struct ionic_aq *aq, struct ionic_admin_wr *wr) { unsigned long irqflags; bool poll; wr->status = IONIC_ADMIN_POSTED; wr->aq = aq; spin_lock_irqsave(&aq->lock, irqflags); poll = list_empty(&aq->wr_post); list_add(&wr->aq_ent, &aq->wr_post); if (poll) ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags); } void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr) { int aq_idx; /* Use cpu id for the adminq selection */ aq_idx = raw_smp_processor_id() % dev->lif_cfg.aq_count; ionic_admin_post_aq(dev->aq_vec[aq_idx], wr); } static void ionic_admin_cancel(struct ionic_admin_wr *wr) { struct ionic_aq *aq = wr->aq; unsigned long irqflags; spin_lock_irqsave(&aq->lock, irqflags); if (!list_empty(&wr->aq_ent)) { list_del(&wr->aq_ent); if (wr->status != IONIC_ADMIN_POSTED) aq->q_wr[wr->status].wr = NULL; } spin_unlock_irqrestore(&aq->lock, irqflags); } static int ionic_admin_busy_wait(struct ionic_admin_wr *wr) { struct ionic_aq *aq = wr->aq; unsigned long irqflags; int try_i; for (try_i = 0; try_i < IONIC_ADMIN_BUSY_RETRY_COUNT; ++try_i) { if (completion_done(&wr->work)) return 0; mdelay(IONIC_ADMIN_BUSY_RETRY_MS); spin_lock_irqsave(&aq->lock, irqflags); ionic_admin_poll_locked(aq); spin_unlock_irqrestore(&aq->lock, irqflags); } /* * we timed out. Initiate RDMA LIF reset and indicate * error to caller. */ ionic_admin_timedout(aq); return -ETIMEDOUT; } int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, enum ionic_admin_flags flags) { int rc, timo; if (flags & IONIC_ADMIN_F_BUSYWAIT) { /* Spin */ rc = ionic_admin_busy_wait(wr); } else if (flags & IONIC_ADMIN_F_INTERRUPT) { /* * Interruptible sleep, 1s timeout * This is used for commands which are safe for the caller * to clean up without killing and resetting the adminq. */ timo = wait_for_completion_interruptible_timeout(&wr->work, HZ); if (timo > 0) rc = 0; else if (timo == 0) rc = -ETIMEDOUT; else rc = timo; } else { /* * Uninterruptible sleep * This is used for commands which are NOT safe for the * caller to clean up. Cleanup must be handled by the * adminq kill and reset process so that host memory is * not corrupted by the device. */ wait_for_completion(&wr->work); rc = 0; } if (rc) { ibdev_warn(&dev->ibdev, "wait status %d\n", rc); ionic_admin_cancel(wr); } else if (wr->status == IONIC_ADMIN_KILLED) { ibdev_dbg(&dev->ibdev, "admin killed\n"); /* No error if admin already killed during teardown */ rc = (flags & IONIC_ADMIN_F_TEARDOWN) ? 0 : -ENODEV; } else if (ionic_v1_cqe_error(&wr->cqe)) { ibdev_warn(&dev->ibdev, "opcode %u error %u\n", wr->wqe.op, be32_to_cpu(wr->cqe.status_length)); rc = -EINVAL; } return rc; } static int ionic_rdma_devcmd(struct ionic_ibdev *dev, struct ionic_admin_ctx *admin) { int rc; rc = ionic_adminq_post_wait(dev->lif_cfg.lif, admin); if (rc) return rc; return ionic_error_to_errno(admin->comp.comp.status); } int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev) { struct ionic_admin_ctx admin = { .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), .cmd.rdma_reset = { .opcode = IONIC_CMD_RDMA_RESET_LIF, .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), }, }; return ionic_rdma_devcmd(dev, &admin); } static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev, struct ionic_queue *q, u32 qid, u32 cid, u16 opcode) { struct ionic_admin_ctx admin = { .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), .cmd.rdma_queue = { .opcode = opcode, .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), .qid_ver = cpu_to_le32(qid), .cid = cpu_to_le32(cid), .dbid = cpu_to_le16(dev->lif_cfg.dbid), .depth_log2 = q->depth_log2, .stride_log2 = q->stride_log2, .dma_addr = cpu_to_le64(q->dma), }, }; return ionic_rdma_devcmd(dev, &admin); } static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context) { struct ionic_aq *aq = cq_context; unsigned long irqflags; spin_lock_irqsave(&aq->lock, irqflags); aq->armed = false; if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) queue_work(ionic_evt_workq, &aq->work);