Add a helper for verifying a rqe and extracting a niov out of it. It'll be reused in following patches.
Unified · Split
| @@ -750,6 +750,28 @@ static struct io_uring_zcrx_rqe *io_zcrx_get_rqe(struct io_zcrx_ifq *ifq, · 749 unmodified lines | |||
| 750 | return &ifq->rqes[idx]; | 750 | return &ifq->rqes[idx]; |
| 751 | } | 751 | } |
| 752 | 752 | ||
| 753 | static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe, | ||
| 754 | struct io_zcrx_ifq *ifq, | ||
| 755 | struct net_iov **ret_niov) | ||
| 756 | { | ||
| 757 | unsigned niov_idx, area_idx; | ||
| 758 | struct io_zcrx_area *area; | ||
| 759 | |||
| 760 | area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT; | ||
| 761 | niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; | ||
| 762 | |||
| 763 | if (unlikely(rqe->__pad || area_idx)) | ||
| 764 | return false; | ||
| 765 | area = ifq->area; | ||
| 766 | |||
| 767 | if (unlikely(niov_idx >= area->nia.num_niovs)) | ||
| 768 | return false; | ||
| 769 | niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs); | ||
| 770 | |||
| 771 | *ret_niov = &area->nia.niovs[niov_idx]; | ||
| 772 | return true; | ||
| 773 | } | ||
| 774 | |||
| 753 | static void io_zcrx_ring_refill(struct page_pool *pp, | 775 | static void io_zcrx_ring_refill(struct page_pool *pp, |
| 754 | struct io_zcrx_ifq *ifq) | 776 | struct io_zcrx_ifq *ifq) |
| 755 | { | 777 | { |
| @@ -765,23 +787,11 @@ static void io_zcrx_ring_refill(struct page_pool *pp, · 9 unmodified lines | |||
| 765 | 787 | ||
| 766 | do { | 788 | do { |
| 767 | struct io_uring_zcrx_rqe *rqe = io_zcrx_get_rqe(ifq, mask); | 789 | struct io_uring_zcrx_rqe *rqe = io_zcrx_get_rqe(ifq, mask); |
| 768 | struct io_zcrx_area *area; | ||
| 769 | struct net_iov *niov; | 790 | struct net_iov *niov; |
| 770 | unsigned niov_idx, area_idx; | ||
| 771 | netmem_ref netmem; | 791 | netmem_ref netmem; |
| 772 | 792 | ||
| 773 | area_idx = rqe->off >> IORING_ZCRX_AREA_SHIFT; | 793 | if (!io_parse_rqe(rqe, ifq, &niov)) |
| 774 | niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift; | ||
| 775 | |||
| 776 | if (unlikely(rqe->__pad || area_idx)) | ||
| 777 | continue; | 794 | continue; |
| 778 | area = ifq->area; | ||
| 779 | |||
| 780 | if (unlikely(niov_idx >= area->nia.num_niovs)) | ||
| 781 | continue; | ||
| 782 | niov_idx = array_index_nospec(niov_idx, area->nia.num_niovs); | ||
| 783 | |||
| 784 | niov = &area->nia.niovs[niov_idx]; | ||
| 785 | if (!io_zcrx_put_niov_uref(niov)) | 795 | if (!io_zcrx_put_niov_uref(niov)) |
| 786 | continue; | 796 | continue; |
| 787 | 797 | ||