History

io_uring/zcrx: add dynamic area provisioning

It's not always possible for the user to predict during registration how much memory zcrx will need to sustain the traffic. Allow to dynamically add more areas with a new ctrl code ZCRX_CTRL_ADD_AREA.

Pavel Begunkov committed 3 months ago

Commit 3c8a5e21 parent 5a22bfd2 files · +57 −6Browse files

Unified · Split

2 files
  1. include/uapi/linux/io_uring/zcrx.h +7 −0
  2. io_uring/zcrx.c +50 −6
include/uapi/linux/io_uring/zcrx.h+7−0
@@ -116,6 +116,7 @@ enum zcrx_ctrl_op { · 115 unmodified lines
116 ZCRX_CTRL_FLUSH_RQ,116 ZCRX_CTRL_FLUSH_RQ,
117 ZCRX_CTRL_EXPORT,117 ZCRX_CTRL_EXPORT,
118 ZCRX_CTRL_ARM_NOTIFICATION,118 ZCRX_CTRL_ARM_NOTIFICATION,
119 ZCRX_CTRL_ADD_AREA,
119 120
120 __ZCRX_CTRL_LAST,121 __ZCRX_CTRL_LAST,
121};122};
@@ -134,6 +135,11 @@ struct zcrx_ctrl_arm_notif { · 12 unmodified lines
134 __u32 __resv[11];135 __u32 __resv[11];
135};136};
136 137
138struct zcrx_ctrl_add_area {
139 __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */
140 __u64 __resv[5];
141};
142
137struct zcrx_ctrl {143struct zcrx_ctrl {
138 __u32 zcrx_id;144 __u32 zcrx_id;
139 __u32 op; /* see enum zcrx_ctrl_op */145 __u32 op; /* see enum zcrx_ctrl_op */
@@ -143,6 +149,7 @@ struct zcrx_ctrl { · 3 unmodified lines
143 struct zcrx_ctrl_export zc_export;149 struct zcrx_ctrl_export zc_export;
144 struct zcrx_ctrl_flush_rq zc_flush;150 struct zcrx_ctrl_flush_rq zc_flush;
145 struct zcrx_ctrl_arm_notif zc_arm_notif;151 struct zcrx_ctrl_arm_notif zc_arm_notif;
152 struct zcrx_ctrl_add_area zc_area;
146 };153 };
147};154};
148 155
io_uring/zcrx.c+50−6
@@ -36,9 +36,15 @@ · 35 unmodified lines
36#define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024)36#define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024)
37 37
38#define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF)38#define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF)
39#define ZCRX_MAX_AREAS 1024
39 40
40#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)41#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
41 42
43static inline u32 zcrx_next_area_id(struct io_zcrx_ifq *zcrx)
44{
45 return zcrx->nr_areas;
46}
47
42static inline u64 zcrx_area_id_to_token(u32 area_id)48static inline u64 zcrx_area_id_to_token(u32 area_id)
43{49{
44 return (u64)area_id << IORING_ZCRX_AREA_SHIFT;50 return (u64)area_id << IORING_ZCRX_AREA_SHIFT;
@@ -334,7 +340,7 @@ static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq) · 289 unmodified lines
334{340{
335 unsigned area_idx;341 unsigned area_idx;
336 342
337 guard(mutex)(&ifq->pp_lock);343 lockdep_assert_held(&ifq->pp_lock);
338 344
339 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++)345 for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++)
340 __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]);346 __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]);
@@ -469,7 +475,9 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq, · 128 unmodified lines
469 struct io_zcrx_area **areas, **old_areas;475 struct io_zcrx_area **areas, **old_areas;
470 unsigned old_nr;476 unsigned old_nr;
471 477
472 if (WARN_ON_ONCE(ifq->kern_readable != kern_readable))478 if (ifq->kern_readable != kern_readable)
479 return -EINVAL;
480 if (ifq->nr_areas + 1 > ZCRX_MAX_AREAS)
473 return -EINVAL;481 return -EINVAL;
474 482
475 old_areas = ifq->areas;483 old_areas = ifq->areas;
@@ -508,7 +516,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, · 32 unmodified lines
508 return -EINVAL;516 return -EINVAL;
509 buf_size_shift = ilog2(rx_buf_len);517 buf_size_shift = ilog2(rx_buf_len);
510 }518 }
511 if (WARN_ON_ONCE(ifq->niov_shift))519 if (ifq->niov_shift && ifq->niov_shift != buf_size_shift)
512 return -EINVAL;520 return -EINVAL;
513 if (!ifq->dev && buf_size_shift != PAGE_SHIFT)521 if (!ifq->dev && buf_size_shift != PAGE_SHIFT)
514 return -EOPNOTSUPP;522 return -EOPNOTSUPP;
@@ -566,7 +574,7 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq, · 51 unmodified lines
566 574
567 area->free_count = nr_iovs;575 area->free_count = nr_iovs;
568 /* we're only supporting one area per ifq for now */576 /* we're only supporting one area per ifq for now */
569 area->area_id = 0;577 area->area_id = zcrx_next_area_id(ifq);
570 area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);578 area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);
571 579
572 ret = io_zcrx_append_area(ifq, area);580 ret = io_zcrx_append_area(ifq, area);
@@ -608,7 +616,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) · 35 unmodified lines
608 616
609static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq)617static void io_zcrx_drop_netdev(struct io_zcrx_ifq *ifq)
610{618{
611 guard(mutex)(&ifq->pp_lock);619 lockdep_assert_held(&ifq->pp_lock);
612 620
613 if (!ifq->netdev)621 if (!ifq->netdev)
614 return;622 return;
@@ -636,7 +644,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq) · 21 unmodified lines
636 if (ifq->if_rxq != -1)644 if (ifq->if_rxq != -1)
637 netif_mp_close_rxq(netdev, ifq->if_rxq, &p);645 netif_mp_close_rxq(netdev, ifq->if_rxq, &p);
638 646
639 io_zcrx_unmap_areas(ifq);647 scoped_guard(mutex, &ifq->pp_lock)
648 io_zcrx_unmap_areas(ifq);
640 netdev_unlock(netdev);649 netdev_unlock(netdev);
641 netdev_put(netdev, &netdev_tracker);650 netdev_put(netdev, &netdev_tracker);
642 }651 }
@@ -994,6 +1003,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx, · 351 unmodified lines
994 1003
995 if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area)))1004 if (copy_from_user(&area, u64_to_user_ptr(reg.area_ptr), sizeof(area)))
996 return -EFAULT;1005 return -EFAULT;
1006 if (area.rq_area_token)
1007 return -EINVAL;
997 1008
998 memset(&notif, 0, sizeof(notif));1009 memset(&notif, 0, sizeof(notif));
999 if (reg.notif_desc && copy_from_user(&notif, u64_to_user_ptr(reg.notif_desc),1010 if (reg.notif_desc && copy_from_user(&notif, u64_to_user_ptr(reg.notif_desc),
@@ -1056,6 +1067,8 @@ int io_register_zcrx(struct io_ring_ctx *ctx, · 56 unmodified lines
1056 goto err;1067 goto err;
1057 }1068 }
1058 1069
1070 WARN_ON_ONCE(!ifq->niov_shift);
1071
1059 reg.zcrx_id = id;1072 reg.zcrx_id = id;
1060 1073
1061 scoped_guard(mutex, &ctx->mmap_lock) {1074 scoped_guard(mutex, &ctx->mmap_lock) {
@@ -1447,6 +1460,7 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq) · 385 unmodified lines
1447 struct pp_memory_provider_params *p = &rxq->mp_params;1460 struct pp_memory_provider_params *p = &rxq->mp_params;
1448 struct io_zcrx_ifq *ifq = mp_priv;1461 struct io_zcrx_ifq *ifq = mp_priv;
1449 1462
1463 guard(mutex)(&ifq->pp_lock);
1450 io_zcrx_unmap_areas(ifq);1464 io_zcrx_unmap_areas(ifq);
1451 io_zcrx_drop_netdev(ifq);1465 io_zcrx_drop_netdev(ifq);
1452 1466
@@ -1549,6 +1563,34 @@ static int zcrx_arm_notif(struct io_ring_ctx *ctx, struct io_zcrx_ifq *zcrx, · 96 unmodified lines
1549 return 0;1563 return 0;
1550}1564}
1551 1565
1566static int zcrx_ctrl_add_area(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq,
1567 struct zcrx_ctrl *ctrl)
1568{
1569 struct zcrx_ctrl_add_area *ctrl_add = &ctrl->zc_area;
1570 struct io_uring_zcrx_area_reg __user *area_uptr;
1571 struct io_uring_zcrx_area_reg area_reg;
1572
1573 area_uptr = u64_to_user_ptr(ctrl_add->area_ptr);
1574 if (copy_from_user(&area_reg, area_uptr, sizeof(area_reg)))
1575 return -EFAULT;
1576 if (!mem_is_zero(&ctrl_add->__resv, sizeof(ctrl_add->__resv)))
1577 return -EINVAL;
1578 if (area_reg.rq_area_token)
1579 return -EINVAL;
1580
1581 guard(mutex)(&ifq->pp_lock);
1582 if (ifq->dev && !ifq->netdev)
1583 return -EFAULT;
1584
1585 /* we can't safely roll back area append, copy it out first */
1586 area_reg.rq_area_token = zcrx_area_id_to_token(zcrx_next_area_id(ifq));
1587 if (copy_to_user(area_uptr, &area_reg, sizeof(area_reg)))
1588 return -EFAULT;
1589 area_reg.rq_area_token = 0;
1590
1591 return __zcrx_create_area(ifq, &area_reg, 1U << ifq->niov_shift);
1592}
1593
1552int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)1594int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
1553{1595{
1554 struct zcrx_ctrl ctrl;1596 struct zcrx_ctrl ctrl;
@@ -1575,6 +1617,8 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) · 20 unmodified lines
1575 return zcrx_export(ctx, zcrx, &ctrl, arg);1617 return zcrx_export(ctx, zcrx, &ctrl, arg);
1576 case ZCRX_CTRL_ARM_NOTIFICATION:1618 case ZCRX_CTRL_ARM_NOTIFICATION:
1577 return zcrx_arm_notif(ctx, zcrx, &ctrl);1619 return zcrx_arm_notif(ctx, zcrx, &ctrl);
1620 case ZCRX_CTRL_ADD_AREA:
1621 return zcrx_ctrl_add_area(ctx, zcrx, &ctrl);
1578 }1622 }
1579 1623
1580 return -EOPNOTSUPP;1624 return -EOPNOTSUPP;