History

io_uring/zcrx: add dynamic area provisioning

It's not always possible for the user to predict during registration how much memory zcrx will need to sustain the traffic. Allow to dynamically add more areas with a new ctrl code ZCRX_CTRL_ADD_AREA.

Pavel Begunkov committed 3 months ago

Commit 3c8a5e21 parent 5a22bfd2 files · +57 −6Browse files

Unified · Split

2 files
  1. include/uapi/linux/io_uring/zcrx.h +7 −0
  2. io_uring/zcrx.c +50 −6
include/uapi/linux/io_uring/zcrx.h+7−0
@@ -116,6 +116,7 @@ enum zcrx_ctrl_op { · 115 unmodified lines
116116 ZCRX_CTRL_FLUSH_RQ,
117117 ZCRX_CTRL_EXPORT,
118118 ZCRX_CTRL_ARM_NOTIFICATION,
119+ ZCRX_CTRL_ADD_AREA,
119120
120121 __ZCRX_CTRL_LAST,
121122 };
@@ -134,6 +135,11 @@ struct zcrx_ctrl_arm_notif { · 12 unmodified lines
134135 __u32 __resv[11];
135136 };
136137
138+struct zcrx_ctrl_add_area {
139+ __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */
140+ __u64 __resv[5];
141+};
142+
137143 struct zcrx_ctrl {
138144 __u32 zcrx_id;
139145 __u32 op; /* see enum zcrx_ctrl_op */
@@ -143,6 +149,7 @@ struct zcrx_ctrl { · 3 unmodified lines
143149 struct zcrx_ctrl_export zc_export;
144150 struct zcrx_ctrl_flush_rq zc_flush;
145151 struct zcrx_ctrl_arm_notif zc_arm_notif;
152+ struct zcrx_ctrl_add_area zc_area;
146153 };
147154 };
148155
io_uring/zcrx.c+50−6
@@ -36,9 +36,15 @@ · 35 unmodified lines
3636 #define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024)
3737
3838 #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF)
39+#define ZCRX_MAX_AREAS 1024
3940
4041 #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
4142
43+static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx)
44+{
45+ return zcrx->nr_areas;
46+}
47+
4248 static inline u64 zcrx_area_id_to_token(u32 area_id)
4349 {
4450 return (u64)area_id << IORING_ZCRX_AREA_SHIFT;
@@ -334,7 +340,7 @@ static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq) · 289 unmodified lines
334340 {
335341 unsigned area_idx;
336342
337− guard(mutex)(&ifq->pp_lock);
343+ lockdep_assert_held(&ifq->pp_lock);
338344
339345 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++)
340346 __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]);
@@ -469,7 +475,9 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, · 128 unmodified lines
469475 struct io_zcrx_area **areas, **old_areas;
470476 unsigned old_nr;
471477
472− if (WARN_ON_ONCE(ifq->kern_readable != kern_readable))
478+ if (ifq->kern_readable != kern_readable)
479+ return -EINVAL;
480+ if (ifq->nr_areas + 1 > ZCRX_MAX_AREAS)
473481 return -EINVAL;
474482
475483 old_areas = ifq->areas;
@@ -508,7 +516,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, · 32 unmodified lines
508516 return -EINVAL;
509517 buf_size_shift = ilog2(rx_buf_len);
510518 }
511− if (WARN_ON_ONCE(ifq->niov_shift))
519+ if (ifq->niov_shift && ifq->niov_shift != buf_size_shift)
512520 return -EINVAL;
513521 if (!ifq->dev && buf_size_shift != PAGE_SHIFT)
514522 return -EOPNOTSUPP;
@@ -566,7 +574,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, · 51 unmodified lines
566574
567575 area->free_count = nr_iovs;
568576 /* we're only supporting one area per ifq for now */
569− area->area_id = 0;
577+ area->area_id = zcrx_next_area_id(ifq);
570578 area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);
571579
572580 ret = io_zcrx_append_area(ifq, area);
@@ -608,7 +616,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) · 35 unmodified lines
608616
609617 static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq)
610618 {
611− guard(mutex)(&ifq->pp_lock);
619+ lockdep_assert_held(&ifq->pp_lock);
612620
613621 if (!ifq->netdev)
614622 return;
@@ -636,7 +644,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) · 21 unmodified lines
636644 if (ifq->if_rxq != -1)
637645 netif_mp_close_rxq(netdev, ifq->if_rxq, &p);
638646
639− io_zcrx_unmap_areas(ifq);
647+ scoped_guard(mutex, &ifq->pp_lock)
648+ io_zcrx_unmap_areas(ifq);
640649 netdev_unlock(netdev);
641650 netdev_put(netdev, &netdev_tracker);
642651 }
@@ -994,6 +1003,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx, · 351 unmodified lines
9941003
9951004 if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area)))
9961005 return -EFAULT;
1006+ if (area.rq_area_token)
1007+ return -EINVAL;
9971008
9981009 memset(&notif, 0, sizeof(notif));
9991010 if (reg.notif_desc && copy_from_user(&notif, u64_to_user_ptr(reg.notif_desc),
@@ -1056,6 +1067,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx, · 56 unmodified lines
10561067 goto err;
10571068 }
10581069
1070+ WARN_ON_ONCE(!ifq->niov_shift);
1071+
10591072 reg.zcrx_id = id;
10601073
10611074 scoped_guard(mutex, &ctx->mmap_lock) {
@@ -1447,6 +1460,7 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) · 385 unmodified lines
14471460 struct pp_memory_provider_params *p = &rxq->mp_params;
14481461 struct io_zcrx_ifq *ifq = mp_priv;
14491462
1463+ guard(mutex)(&ifq->pp_lock);
14501464 io_zcrx_unmap_areas(ifq);
14511465 io_zcrx_drop_netdev(ifq);
14521466
@@ -1549,6 +1563,34 @@ static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, · 96 unmodified lines
15491563 return 0;
15501564 }
15511565
1566+static int zcrx_ctrl_add_area(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq,
1567+ struct zcrx_ctrl *ctrl)
1568+{
1569+ struct zcrx_ctrl_add_area *ctrl_add = &ctrl->zc_area;
1570+ struct io_uring_zcrx_area_reg __user *area_uptr;
1571+ struct io_uring_zcrx_area_reg area_reg;
1572+
1573+ area_uptr = u64_to_user_ptr(ctrl_add->area_ptr);
1574+ if (copy_from_user(&area_reg, area_uptr, sizeof(area_reg)))
1575+ return -EFAULT;
1576+ if (!mem_is_zero(&ctrl_add->__resv, sizeof(ctrl_add->__resv)))
1577+ return -EINVAL;
1578+ if (area_reg.rq_area_token)
1579+ return -EINVAL;
1580+
1581+ guard(mutex)(&ifq->pp_lock);
1582+ if (ifq->dev && !ifq->netdev)
1583+ return -EFAULT;
1584+
1585+ /* we can't safely roll back area append, copy it out first */
1586+ area_reg.rq_area_token = zcrx_area_id_to_token(zcrx_next_area_id(ifq));
1587+ if (copy_to_user(area_uptr, &area_reg, sizeof(area_reg)))
1588+ return -EFAULT;
1589+ area_reg.rq_area_token = 0;
1590+
1591+ return __zcrx_create_area(ifq, &area_reg, 1U << ifq->niov_shift);
1592+}
1593+
15521594 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
15531595 {
15541596 struct zcrx_ctrl ctrl;
@@ -1575,6 +1617,8 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) · 20 unmodified lines
15751617 return zcrx_export(ctx, zcrx, &ctrl, arg);
15761618 case ZCRX_CTRL_ARM_NOTIFICATION:
15771619 return zcrx_arm_notif(ctx, zcrx, &ctrl);
1620+ case ZCRX_CTRL_ADD_AREA:
1621+ return zcrx_ctrl_add_area(ctx, zcrx, &ctrl);
15781622 }
15791623
15801624 return -EOPNOTSUPP;