polling the completion queue directly does not interfere with the existing polling logic, hence drop the requirement. Be aware that running ib_process_cq_direct with non IB_POLL_DIRECT CQ may trigger concurrent CQ processing. This can be used for polling mode ULPs.
Unified · Split
| @@ -25,9 +25,10 @@ · 24 unmodified lines | |||
| 25 | #define IB_POLL_FLAGS \ | 25 | #define IB_POLL_FLAGS \ |
| 26 | (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS) | 26 | (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS) |
| 27 | 27 | ||
| 28 | static int __ib_process_cq(struct ib_cq *cq, int budget) | 28 | static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *poll_wc) |
| 29 | { | 29 | { |
| 30 | int i, n, completed = 0; | 30 | int i, n, completed = 0; |
| 31 | struct ib_wc *wcs = poll_wc ? : cq->wc; | ||
| 31 | 32 | ||
| 32 | /* | 33 | /* |
| 33 | * budget might be (-1) if the caller does not | 34 | * budget might be (-1) if the caller does not |
| @@ -35,9 +36,9 @@ static int __ib_process_cq(struct ib_cq *cq, int budget) · 1 unmodified line | |||
| 35 | * minimum here. | 36 | * minimum here. |
| 36 | */ | 37 | */ |
| 37 | while ((n = ib_poll_cq(cq, min_t(u32, IB_POLL_BATCH, | 38 | while ((n = ib_poll_cq(cq, min_t(u32, IB_POLL_BATCH, |
| 38 | budget - completed), cq->wc)) > 0) { | 39 | budget - completed), wcs)) > 0) { |
| 39 | for (i = 0; i < n; i++) { | 40 | for (i = 0; i < n; i++) { |
| 40 | struct ib_wc *wc = &cq->wc[i]; | 41 | struct ib_wc *wc = &wcs[i]; |
| 41 | 42 | ||
| 42 | if (wc->wr_cqe) | 43 | if (wc->wr_cqe) |
| 43 | wc->wr_cqe->done(cq, wc); | 44 | wc->wr_cqe->done(cq, wc); |
| @@ -60,18 +61,20 @@ static int __ib_process_cq(struct ib_cq *cq, int budget) · 16 unmodified lines | |||
| 60 | * @cq: CQ to process | 61 | * @cq: CQ to process |
| 61 | * @budget: number of CQEs to poll for | 62 | * @budget: number of CQEs to poll for |
| 62 | * | 63 | * |
| 63 | * This function is used to process all outstanding CQ entries on a | 64 | * This function is used to process all outstanding CQ entries. |
| 64 | * %IB_POLL_DIRECT CQ. It does not offload CQ processing to a different | 65 | * It does not offload CQ processing to a different context and does |
| 65 | * context and does not ask for completion interrupts from the HCA. | 66 | * not ask for completion interrupts from the HCA. |
| 67 | * Using direct processing on CQ with non IB_POLL_DIRECT type may trigger | ||
| 68 | * concurrent processing. | ||
| 66 | * | 69 | * |
| 67 | * Note: do not pass -1 as %budget unless it is guaranteed that the number | 70 | * Note: do not pass -1 as %budget unless it is guaranteed that the number |
| 68 | * of completions that will be processed is small. | 71 | * of completions that will be processed is small. |
| 69 | */ | 72 | */ |
| 70 | int ib_process_cq_direct(struct ib_cq *cq, int budget) | 73 | int ib_process_cq_direct(struct ib_cq *cq, int budget) |
| 71 | { | 74 | { |
| 72 | WARN_ON_ONCE(cq->poll_ctx != IB_POLL_DIRECT); | 75 | struct ib_wc wcs[IB_POLL_BATCH]; |
| 73 | 76 | ||
| 74 | return __ib_process_cq(cq, budget); | 77 | return __ib_process_cq(cq, budget, wcs); |
| 75 | } | 78 | } |
| 76 | EXPORT_SYMBOL(ib_process_cq_direct); | 79 | EXPORT_SYMBOL(ib_process_cq_direct); |
| 77 | 80 | ||
| @@ -85,7 +88,7 @@ static int ib_poll_handler(struct irq_poll *iop, int budget) · 7 unmodified lines | |||
| 85 | struct ib_cq *cq = container_of(iop, struct ib_cq, iop); | 88 | struct ib_cq *cq = container_of(iop, struct ib_cq, iop); |
| 86 | int completed; | 89 | int completed; |
| 87 | 90 | ||
| 88 | completed = __ib_process_cq(cq, budget); | 91 | completed = __ib_process_cq(cq, budget, NULL); |
| 89 | if (completed < budget) { | 92 | if (completed < budget) { |
| 90 | irq_poll_complete(&cq->iop); | 93 | irq_poll_complete(&cq->iop); |
| 91 | if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) | 94 | if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) |
| @@ -105,7 +108,7 @@ static void ib_cq_poll_work(struct work_struct *work) · 13 unmodified lines | |||
| 105 | struct ib_cq *cq = container_of(work, struct ib_cq, work); | 108 | struct ib_cq *cq = container_of(work, struct ib_cq, work); |
| 106 | int completed; | 109 | int completed; |
| 107 | 110 | ||
| 108 | completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE); | 111 | completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, NULL); |
| 109 | if (completed >= IB_POLL_BUDGET_WORKQUEUE || | 112 | if (completed >= IB_POLL_BUDGET_WORKQUEUE || |
| 110 | ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) | 113 | ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) |
| 111 | queue_work(ib_comp_wq, &cq->work); | 114 | queue_work(ib_comp_wq, &cq->work); |