History

fuse: add io-uring buffer pools

Right now, ents and buffers are tightly coupled in fuse io-uring where each entry has its own dedicated payload buffer, requiring N buffers for N entries where each buffer must be large enough to accomodate the maximum payload size. This is suboptimal as most request types (lookup, open, release, getattr, etc) require vastly less bytes than the maximum payload size and some requests (unlink, rmdir, fsync, flush, etc) do not require payload buffers at all. Instead of requiring a 1:1 coupling between ents and payload buffers, allow the server to pass in a buffer pool (a contiguous chunk of memory) that the kernel will use as it wishes for servicing ents/requests. Entries only reserve a "buffer" from the pool while actively processing a request that requires a payload buffer. This decoupling and letting the kernel delegate memory from the pool for requests allows the kernel to optimize memory usage and reduces the memory usage requirements needed to use fuse-over-io-uring. A pool is registered per queue with the new FUSE_IO_URING_CMD_ADD_BUFPOOL command. The server passes the pool's base address and length in fuse_uring_cmd_req.bufpool.{uaddr,len}. Internally, the kernel splits the region into buffers of ring->max_payload_sz bytes each (nr_bufs = pool len / max_payload_sz). A queue commits to a payload mode on first use: registering an entry that carries its own payload selects the legacy per-entry mode, while ADD_BUFPOOL selects pool mode. The two are mutually exclusive, so ADD_BUFPOOL must be issued before any payload-carrying entries are registered on that queue. The queue must have been created before the bufpool is added, through the FUSE_IO_URING_CMD_ADD_QUEUE command. The kernel tracks free buffers with a bitmap (a set bit marks a free buffer). On dispatch, a request that needs a payload claims a free buffer (find_first_bit + clear). A request that needs none claims nothing. The buffer's byte offset within the pool is reported to the server in the new fuse_uring_ent_in_out.offset field so that the server can locate the payload. On completion the buffer is returned to the pool or reused directly if the next request on that entry also has a payload. The FUSE_HAS_IO_URING_BUFPOOL flag advertises kernel support to the server for bufpools. Buffer pool request flow ~~~~~~~~~~~~~~~~~~~~~~~~ | Kernel | FUSE daemon | | | [request arrives] | | [claim a free pool buffer] | | >fuse_uring_select_buffer() | | [copy headers to ring] | | [copy payload to buffer] | | [report buffer offset in ent_in_out] | | >io_uring_cmd_done() | | | [read headers] | | [read/write payload at offset] | | [process request] | | >io_uring_submit() | | COMMIT_AND_FETCH | >fuse_uring_commit_fetch() | | [copy reply from ring] | | [return buffer to the pool] | | >fuse_uring_recycle_buffer() |

Joanne Koong committed 2 months ago

Commit b45aaab1 parent ebed9ea4 files · +269 −30Browse files

Unified · Split

4 files
  1. fs/fuse/dev_uring.c +212 −27
  2. fs/fuse/dev_uring_i.h +36 −1
  3. fs/fuse/inode.c +1 −1
  4. include/uapi/linux/fuse.h +20 −1
fs/fuse/dev_uring.c+212−27
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include "dev_uring_i.h"
1010 #include "fuse_trace.h"
1111
12+#include <linux/bitmap.h>
1213 #include <linux/fs.h>
1314 #include <linux/io_uring/cmd.h>
1415
@@ -41,6 +42,11 @@ enum fuse_uring_header_type { · 26 unmodified lines
4142 FUSE_URING_HEADER_RING_ENT,
4243 };
4344
45+static inline bool bufpool_enabled(struct fuse_ring_queue *queue)
46+{
47+ return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL;
48+}
49+
4450 static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd,
4551 struct fuse_ring_ent *ring_ent)
4652 {
@@ -222,6 +228,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) · 175 unmodified lines
222228 }
223229
224230 kfree(queue->fpq.processing);
231+ kfree(queue->bufpool);
225232 kfree(queue);
226233 WRITE_ONCE(ring->queues[qid], NULL);
227234 }
@@ -316,6 +323,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, · 88 unmodified lines
316323 if (ring->queues[qid]) {
317324 spin_unlock(&fch->lock);
318325 kfree(queue->fpq.processing);
326+ kfree(queue->bufpool);
319327 kfree(queue);
320328 return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid];
321329 }
@@ -646,13 +654,14 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, · 324 unmodified lines
646654 }
647655
648656 static int setup_fuse_copy_state(struct fuse_copy_state *cs,
649− struct fuse_ring *ring, struct fuse_req *req,
657+ struct fuse_req *req,
650658 struct fuse_ring_ent *ent, int dir,
651659 struct iov_iter *iter)
652660 {
653661 int err;
654662
655− err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter);
663+ err = import_ubuf(dir, ent->payload.iov_base, ent->payload.iov_len,
664+ iter);
656665 if (err) {
657666 pr_info_ratelimited("fuse: Import of user buffer failed\n");
658667 return err;
@@ -666,8 +675,7 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, · 7 unmodified lines
666675 return 0;
667676 }
668677
669−static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
670− struct fuse_req *req,
678+static int fuse_uring_copy_from_ring(struct fuse_req *req,
671679 struct fuse_ring_ent *ent)
672680 {
673681 struct fuse_copy_state cs;
@@ -681,7 +689,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, · 7 unmodified lines
681689 if (err)
682690 return err;
683691
684− err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter);
692+ err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter);
685693 if (err)
686694 return err;
687695
@@ -693,7 +701,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, · 5 unmodified lines
693701 /*
694702 * Copy data from the req to the ring buffer
695703 */
696−static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
704+static int fuse_uring_args_to_ring(struct fuse_req *req,
697705 struct fuse_ring_ent *ent)
698706 {
699707 struct fuse_copy_state cs;
@@ -707,7 +715,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, · 7 unmodified lines
707715 .commit_id = req->in.h.unique,
708716 };
709717
710− err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter);
718+ err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter);
711719 if (err)
712720 return err;
713721
@@ -737,6 +745,10 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, · 23 unmodified lines
737745 }
738746
739747 ent_in_out.payload_sz = cs.ring.copied_sz;
748+ if (bufpool_enabled(ent->queue) && ent->payload.iov_base)
749+ ent_in_out.offset =
750+ (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr;
751+
740752 return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT,
741753 &ent_in_out, sizeof(ent_in_out));
742754 }
@@ -745,7 +757,6 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, · 2 unmodified lines
745757 struct fuse_req *req)
746758 {
747759 struct fuse_ring_queue *queue = ent->queue;
748− struct fuse_ring *ring = queue->ring;
749760 int err;
750761
751762 err = -EIO;
@@ -760,7 +771,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, · 8 unmodified lines
760771 return err;
761772
762773 /* copy the request */
763− err = fuse_uring_args_to_ring(ring, req, ent);
774+ err = fuse_uring_args_to_ring(req, ent);
764775 if (unlikely(err)) {
765776 pr_info_ratelimited("Copy to ring failed: %d\n", err);
766777 return err;
@@ -771,6 +782,91 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, · 4 unmodified lines
771782 sizeof(req->in.h));
772783 }
773784
785+static bool fuse_uring_req_has_payload(struct fuse_req *req)
786+{
787+ struct fuse_args *args = req->args;
788+
789+ return args->in_numargs > 1 || args->out_numargs;
790+}
791+
792+static int fuse_uring_select_buffer(struct fuse_ring_ent *ent)
793+{
794+ struct fuse_ring_queue *queue = ent->queue;
795+ struct fuse_bufpool *pool = queue->bufpool;
796+ unsigned int id;
797+
798+ lockdep_assert_held(&queue->lock);
799+
800+ id = find_first_bit(pool->free_map, pool->nr_bufs);
801+ if (id >= pool->nr_bufs)
802+ return -ENOBUFS;
803+
804+ WARN_ON_ONCE(ent->payload.iov_base);
805+ __clear_bit(id, pool->free_map);
806+
807+ ent->buf_id = id;
808+ ent->payload.iov_base =
809+ (void __user *)(pool->base_uaddr + id * pool->buf_size);
810+ ent->payload.iov_len = pool->buf_size;
811+
812+ return 0;
813+}
814+
815+static void fuse_uring_recycle_buffer(struct fuse_ring_ent *ent)
816+{
817+ struct iovec *ent_payload = &ent->payload;
818+ struct fuse_ring_queue *queue = ent->queue;
819+ struct fuse_bufpool *pool;
820+
821+ lockdep_assert_held(&queue->lock);
822+
823+ if (!bufpool_enabled(queue) || !ent_payload->iov_base)
824+ return;
825+
826+ pool = queue->bufpool;
827+
828+ /* a buffer should never be recycled twice */
829+ WARN_ON_ONCE(test_bit(ent->buf_id, pool->free_map));
830+ __set_bit(ent->buf_id, pool->free_map);
831+
832+ memset(ent_payload, 0, sizeof(*ent_payload));
833+ ent->buf_id = 0;
834+}
835+
836+static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent,
837+ struct fuse_req *req)
838+{
839+ bool buffer_selected;
840+ bool has_payload;
841+
842+ if (!bufpool_enabled(ent->queue))
843+ return 0;
844+
845+ buffer_selected = !!ent->payload.iov_base;
846+ has_payload = fuse_uring_req_has_payload(req);
847+
848+ if (has_payload && !buffer_selected)
849+ return fuse_uring_select_buffer(ent);
850+
851+ if (!has_payload && buffer_selected)
852+ fuse_uring_recycle_buffer(ent);
853+
854+ return 0;
855+}
856+
857+static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent,
858+ struct fuse_req *req)
859+{
860+ if (!bufpool_enabled(ent->queue))
861+ return 0;
862+
863+ /* no payload to copy, can skip selecting a buffer */
864+ if (!fuse_uring_req_has_payload(req))
865+ return 0;
866+
867+ return fuse_uring_select_buffer(ent);
868+}
869+
774870 static int fuse_uring_prepare_send(struct fuse_ring_ent *ent,
775871 struct fuse_req *req)
776872 {
@@ -858,9 +954,12 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) · 81 unmodified lines
858954
859955 /* get and assign the next entry while it is still holding the lock */
860956 req = list_first_entry_or_null(req_queue, struct fuse_req, list);
861− if (req)
862− fuse_uring_add_req_to_ring_ent(ent, req);
957+ if (!req || fuse_uring_next_req_update_buffer(ent, req)) {
958+ fuse_uring_recycle_buffer(ent);
959+ return NULL;
960+ }
863961
962+ fuse_uring_add_req_to_ring_ent(ent, req);
864963 return req;
865964 }
866965
@@ -872,7 +971,6 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) · 5 unmodified lines
872971 static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
873972 unsigned int issue_flags)
874973 {
875− struct fuse_ring *ring = ent->queue->ring;
876974 ssize_t err = -EFAULT;
877975
878976 if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h,
@@ -885,7 +983,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, · 6 unmodified lines
885983 goto out;
886984 }
887985
888− err = fuse_uring_copy_from_ring(ring, req, ent);
986+ err = fuse_uring_copy_from_ring(req, ent);
889987 out:
890988 fuse_uring_req_end(ent, req, err);
891989 }
@@ -1004,6 +1102,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, · 112 unmodified lines
10041102 if (err != 0) {
10051103 pr_info_ratelimited("qid=%d commit_id %llu state %d",
10061104 queue->qid, commit_id, ent->state);
1105+ fuse_uring_recycle_buffer(ent);
10071106 spin_unlock(&queue->lock);
10081107 fuse_uring_req_end(ent, req, err);
10091108 return err;
@@ -1021,6 +1120,11 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, · 11 unmodified lines
10211120 * fuse requests would otherwise not get processed - committing
10221121 * and fetching is done in one step vs legacy fuse, which has separated
10231122 * read (fetch request) and write (commit result).
1123+ *
1124+ * If there is no next request or if all buffers are busy (if using a
1125+ * bufpool), the cmd is not returned to userspace. The entry is left
1126+ * available and the cmd only returns to userspace when there's a
1127+ * next request and an available buffer.
10241128 */
10251129 if (fuse_uring_get_next_fuse_req(ent, queue))
10261130 fuse_uring_send(ent, cmd, 0, issue_flags);
@@ -1145,11 +1249,23 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, · 118 unmodified lines
11451249 }
11461250
11471251 payload = &iov[FUSE_URING_IOV_PAYLOAD];
1148− if (payload->iov_len < ring->max_payload_sz) {
1149− pr_info_ratelimited("Invalid req payload len %zu\n",
1150− payload->iov_len);
1151− return ERR_PTR(err);
1252+
1253+ spin_lock(&queue->lock);
1254+ if (bufpool_enabled(queue)) {
1255+ if (payload->iov_base || payload->iov_len) {
1256+ spin_unlock(&queue->lock);
1257+ return ERR_PTR(err);
1258+ }
1259+ } else {
1260+ if (payload->iov_len < ring->max_payload_sz) {
1261+ pr_info_ratelimited("Invalid req payload len %zu\n",
1262+ payload->iov_len);
1263+ spin_unlock(&queue->lock);
1264+ return ERR_PTR(err);
1265+ }
1266+ queue->payload_mode = FUSE_PAYLOAD_PER_ENT;
11521267 }
1268+ spin_unlock(&queue->lock);
11531269
11541270 err = -ENOMEM;
11551271 ent = kzalloc_obj(*ent, GFP_KERNEL_ACCOUNT);
@@ -1160,7 +1276,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, · 4 unmodified lines
11601276
11611277 ent->queue = queue;
11621278 ent->headers = headers->iov_base;
1163− ent->payload = payload->iov_base;
1279+ if (queue->payload_mode == FUSE_PAYLOAD_PER_ENT)
1280+ ent->payload = *payload;
11641281
11651282 atomic_inc(&ring->queue_refs);
11661283 return ent;
@@ -1231,6 +1348,67 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) · 64 unmodified lines
12311348 return 0;
12321349 }
12331350
1351+static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd,
1352+ struct fuse_chan *fch)
1353+{
1354+ const struct fuse_uring_cmd_req *cmd_req =
1355+ io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req);
1356+ unsigned int qid = READ_ONCE(cmd_req->qid);
1357+ uint64_t flags = READ_ONCE(cmd_req->flags);
1358+ /* paired with the smp_store_release() in fuse_uring_create */
1359+ struct fuse_ring *ring = smp_load_acquire(&fch->ring);
1360+ struct fuse_ring_queue *queue;
1361+ struct fuse_bufpool *pool;
1362+ uintptr_t pool_uaddr;
1363+ unsigned int pool_len, nr_bufs;
1364+ size_t pool_size, buf_size;
1365+
1366+ if (!ring || qid >= ring->nr_queues || flags)
1367+ return -EINVAL;
1368+
1369+ /* reserved for future use, must be zero */
1370+ if (READ_ONCE(cmd_req->bufpool.reserved))
1371+ return -EINVAL;
1372+
1373+ /* Pairs with smp_store_release() in fuse_uring_create_queue() */
1374+ queue = smp_load_acquire(&ring->queues[qid]);
1375+ if (!queue)
1376+ return -EINVAL;
1377+
1378+ pool_uaddr = READ_ONCE(cmd_req->bufpool.uaddr);
1379+ pool_len = READ_ONCE(cmd_req->bufpool.len);
1380+
1381+ /* each buffer holds the max payload size */
1382+ buf_size = queue->ring->max_payload_sz;
1383+
1384+ nr_bufs = pool_len / buf_size;
1385+ if (!nr_bufs)
1386+ return -EINVAL;
1387+
1388+ pool_size = struct_size(pool, free_map, BITS_TO_LONGS(nr_bufs));
1389+ pool = kzalloc(pool_size, GFP_KERNEL_ACCOUNT);
1390+ if (!pool)
1391+ return -ENOMEM;
1392+
1393+ pool->base_uaddr = pool_uaddr;
1394+ pool->buf_size = buf_size;
1395+ pool->nr_bufs = nr_bufs;
1396+ /* all buffers are free */
1397+ bitmap_set(pool->free_map, 0, nr_bufs);
1398+
1399+ spin_lock(&queue->lock);
1400+ if (queue->payload_mode != FUSE_PAYLOAD_UNSET) {
1401+ spin_unlock(&queue->lock);
1402+ kfree(pool);
1403+ return -EINVAL;
1404+ }
1405+ queue->bufpool = pool;
1406+ queue->payload_mode = FUSE_PAYLOAD_BUFPOOL;
1407+ spin_unlock(&queue->lock);
1408+
1409+ return 0;
1410+}
1411+
12341412 /*
12351413 * Entry function from io_uring to handle the given passthrough command
12361414 * (op code IORING_OP_URING_CMD)
@@ -1303,6 +1481,12 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) · 66 unmodified lines
13031481 pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n",
13041482 err);
13051483 return err;
1484+ case FUSE_IO_URING_CMD_ADD_BUFPOOL:
1485+ err = fuse_uring_add_bufpool(cmd, fch);
1486+ if (err)
1487+ pr_info_once("FUSE_IO_URING_ADD_BUFPOOL failed err=%d\n",
1488+ err);
1489+ return err;
13061490 default:
13071491 return -EINVAL;
13081492 }
@@ -1336,6 +1520,7 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) · 27 unmodified lines
13361520
13371521 spin_lock(&queue->lock);
13381522 list_del_init(&ent->list);
1523+ fuse_uring_recycle_buffer(ent);
13391524 spin_unlock(&queue->lock);
13401525
13411526 io_uring_cmd_done(cmd, err, issue_flags);
@@ -1397,15 +1582,16 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) · 55 unmodified lines
13971582 req->ring_queue = queue;
13981583 ent = list_first_entry_or_null(&queue->ent_avail_queue,
13991584 struct fuse_ring_ent, list);
1400− if (ent)
1401− fuse_uring_add_req_to_ring_ent(ent, req);
1402− else
1403− list_add_tail(&req->list, &queue->fuse_req_queue);
1404− spin_unlock(&queue->lock);
14051585
1406− if (ent)
1407− fuse_uring_dispatch_ent(ent);
1586+ if (!ent || fuse_uring_prep_buffer(ent, req)) {
1587+ list_add_tail(&req->list, &queue->fuse_req_queue);
1588+ spin_unlock(&queue->lock);
1589+ return;
1590+ }
14081591
1592+ fuse_uring_add_req_to_ring_ent(ent, req);
1593+ spin_unlock(&queue->lock);
1594+ fuse_uring_dispatch_ent(ent);
14091595 return;
14101596
14111597 err_unlock:
@@ -1453,10 +1639,9 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) · 41 unmodified lines
14531639 */
14541640 req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req,
14551641 list);
1456− if (ent && req) {
1642+ if (ent && req && !fuse_uring_prep_buffer(ent, req)) {
14571643 fuse_uring_add_req_to_ring_ent(ent, req);
14581644 spin_unlock(&queue->lock);
1459−
14601645 fuse_uring_dispatch_ent(ent);
14611646 } else {
14621647 spin_unlock(&queue->lock);
fs/fuse/dev_uring_i.h+36−1
@@ -7,6 +7,8 @@ · 6 unmodified lines
77 #ifndef _FS_FUSE_DEV_URING_I_H
88 #define _FS_FUSE_DEV_URING_I_H
99
10+#include <linux/uio.h>
11+
1012 #include "fuse_dev_i.h"
1113
1214 #ifdef CONFIG_FUSE_IO_URING
@@ -36,11 +38,38 @@ enum fuse_ring_req_state { · 23 unmodified lines
3638 FRRS_RELEASED,
3739 };
3840
41+/* how a queue's payload buffers are provided */
42+enum fuse_queue_payload_mode {
43+ /* not yet committed (a bufpool may still be added) */
44+ FUSE_PAYLOAD_UNSET = 0,
45+ /* each entry registers its own payload buffer */
46+ FUSE_PAYLOAD_PER_ENT,
47+ /* each entry's payload buffer is assigned from a bufpool */
48+ FUSE_PAYLOAD_BUFPOOL,
49+};
50+
51+struct fuse_bufpool {
52+ /* starting uaddr of the bufpool */
53+ uintptr_t base_uaddr;
54+
55+ /* size of each buffer in the pool */
56+ size_t buf_size;
57+
58+ /* total number of buffers in the pool */
59+ unsigned int nr_bufs;
60+
61+ /* bitmap tracking which buffers are free */
62+ unsigned long free_map[];
63+};
64+
3965 /** A fuse ring entry, part of the ring queue */
4066 struct fuse_ring_ent {
4167 /* userspace buffer */
4268 struct fuse_uring_req_header __user *headers;
43− void __user *payload;
69+ struct iovec payload;
70+
71+ /* buffer id in the pool, if bufpools are used. ignored otherwise */
72+ unsigned int buf_id;
4473
4574 /* the ring queue that owns the request */
4675 struct fuse_ring_queue *queue;
@@ -99,6 +128,12 @@ struct fuse_ring_queue { · 52 unmodified lines
99128 unsigned int active_background;
100129
101130 bool stopped;
131+
132+ /* how this queue's payload buffers are provided */
133+ enum fuse_queue_payload_mode payload_mode;
134+
135+ /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */
136+ struct fuse_bufpool *bufpool;
102137 };
103138
104139 /*
fs/fuse/inode.c+1−1
@@ -1482,7 +1482,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) · 1481 unmodified lines
14821482 * the reply - server is either sending IORING_OP_URING_CMD or not.
14831483 */
14841484 if (fuse_uring_enabled())
1485− flags |= FUSE_OVER_IO_URING;
1485+ flags |= FUSE_OVER_IO_URING | FUSE_HAS_IO_URING_BUFPOOL;
14861486
14871487 ia->in.flags = flags;
14881488 ia->in.flags2 = flags >> 32;
include/uapi/linux/fuse.h+20−1
@@ -243,6 +243,9 @@ · 242 unmodified lines
243243 *
244244 * 7.46
245245 * - add FUSE_IO_URING_CMD_ADD_QUEUE
246+ * - add FUSE_HAS_IO_URING_BUFPOOL
247+ * - add fuse_uring_cmd_req bufpool struct
248+ * - add bufpool offset field to fuse_uring_ent_in_out struct
246249 */
247250
248251 #ifndef _LINUX_FUSE_H
@@ -451,6 +454,7 @@ struct fuse_file_lock { · 202 unmodified lines
451454 * FUSE_OVER_IO_URING: Indicate that client supports io-uring
452455 * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests.
453456 * init_out.request_timeout contains the timeout (in secs)
457+ * FUSE_HAS_IO_URING_BUFPOOL: kernel supports io-uring buffer pools
454458 */
455459 #define FUSE_ASYNC_READ (1 << 0)
456460 #define FUSE_POSIX_LOCKS (1 << 1)
@@ -498,6 +502,7 @@ struct fuse_file_lock { · 41 unmodified lines
498502 #define FUSE_ALLOW_IDMAP (1ULL << 40)
499503 #define FUSE_OVER_IO_URING (1ULL << 41)
500504 #define FUSE_REQUEST_TIMEOUT (1ULL << 42)
505+#define FUSE_HAS_IO_URING_BUFPOOL (1ULL << 43)
501506
502507 /**
503508 * CUSE INIT request/reply flags
@@ -1266,7 +1271,9 @@ struct fuse_uring_ent_in_out { · 762 unmodified lines
12661271
12671272 /* size of user payload buffer */
12681273 uint32_t payload_sz;
1269− uint32_t padding;
1274+
1275+ /* Offset into the bufpool, if bufpools are used */
1276+ uint32_t offset;
12701277
12711278 uint64_t reserved;
12721279 };
@@ -1298,6 +1305,9 @@ enum fuse_uring_cmd { · 25 unmodified lines
12981305
12991306 /* add a queue */
13001307 FUSE_IO_URING_CMD_ADD_QUEUE = 3,
1308+
1309+ /* add a bufpool to a queue */
1310+ FUSE_IO_URING_CMD_ADD_BUFPOOL = 4,
13011311 };
13021312
13031313 /**
@@ -1312,6 +1322,15 @@ struct fuse_uring_cmd_req { · 8 unmodified lines
13121322 /* queue the command is for (queue index) */
13131323 uint16_t qid;
13141324 uint8_t padding[6];
1325+
1326+ union {
1327+ struct {
1328+ /* base address of bufpool */
1329+ uint64_t uaddr;
1330+ uint32_t len;
1331+ uint32_t reserved;
1332+ } bufpool;
1333+ };
13151334 };
13161335
13171336 #endif /* _LINUX_FUSE_H */