History

fuse: add io-uring buffer pools

Right now, ents and buffers are tightly coupled in fuse io-uring where each entry has its own dedicated payload buffer, requiring N buffers for N entries where each buffer must be large enough to accomodate the maximum payload size. This is suboptimal as most request types (lookup, open, release, getattr, etc) require vastly less bytes than the maximum payload size and some requests (unlink, rmdir, fsync, flush, etc) do not require payload buffers at all. Instead of requiring a 1:1 coupling between ents and payload buffers, allow the server to pass in a buffer pool (a contiguous chunk of memory) that the kernel will use as it wishes for servicing ents/requests. Entries only reserve a "buffer" from the pool while actively processing a request that requires a payload buffer. This decoupling and letting the kernel delegate memory from the pool for requests allows the kernel to optimize memory usage and reduces the memory usage requirements needed to use fuse-over-io-uring. A pool is registered per queue with the new FUSE_IO_URING_CMD_ADD_BUFPOOL command. The server passes the pool's base address and length in fuse_uring_cmd_req.bufpool.{uaddr,len}. Internally, the kernel splits the region into buffers of ring->max_payload_sz bytes each (nr_bufs = pool len / max_payload_sz). A queue commits to a payload mode on first use: registering an entry that carries its own payload selects the legacy per-entry mode, while ADD_BUFPOOL selects pool mode. The two are mutually exclusive, so ADD_BUFPOOL must be issued before any payload-carrying entries are registered on that queue. The queue must have been created before the bufpool is added, through the FUSE_IO_URING_CMD_ADD_QUEUE command. The kernel tracks free buffers with a bitmap (a set bit marks a free buffer). On dispatch, a request that needs a payload claims a free buffer (find_first_bit + clear). A request that needs none claims nothing. The buffer's byte offset within the pool is reported to the server in the new fuse_uring_ent_in_out.offset field so that the server can locate the payload. On completion the buffer is returned to the pool or reused directly if the next request on that entry also has a payload. The FUSE_HAS_IO_URING_BUFPOOL flag advertises kernel support to the server for bufpools. Buffer pool request flow ~~~~~~~~~~~~~~~~~~~~~~~~ | Kernel | FUSE daemon | | | [request arrives] | | [claim a free pool buffer] | | >fuse_uring_select_buffer() | | [copy headers to ring] | | [copy payload to buffer] | | [report buffer offset in ent_in_out] | | >io_uring_cmd_done() | | | [read headers] | | [read/write payload at offset] | | [process request] | | >io_uring_submit() | | COMMIT_AND_FETCH | >fuse_uring_commit_fetch() | | [copy reply from ring] | | [return buffer to the pool] | | >fuse_uring_recycle_buffer() |

Joanne Koong committed 2 months ago

Commit b45aaab1 parent ebed9ea4 files · +269 −30Browse files

Unified · Split

4 files
  1. fs/fuse/dev_uring.c +212 −27
  2. fs/fuse/dev_uring_i.h +36 −1
  3. fs/fuse/inode.c +1 −1
  4. include/uapi/linux/fuse.h +20 −1
fs/fuse/dev_uring.c+212−27
@@ -9,6 +9,7 @@ · 8 unmodified lines
9#include "dev_uring_i.h"9#include "dev_uring_i.h"
10#include "fuse_trace.h"10#include "fuse_trace.h"
11 11
12#include <linux/bitmap.h>
12#include <linux/fs.h>13#include <linux/fs.h>
13#include <linux/io_uring/cmd.h>14#include <linux/io_uring/cmd.h>
14 15
@@ -41,6 +42,11 @@ enum fuse_uring_header_type { · 26 unmodified lines
41 FUSE_URING_HEADER_RING_ENT,42 FUSE_URING_HEADER_RING_ENT,
42};43};
43 44
45static inline bool bufpool_enabled(struct fuse_ring_queue *queue)
46{
47 return queue->payload_mode == FUSE_PAYLOAD_BUFPOOL;
48}
49
44static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd,50static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd,
45 struct fuse_ring_ent *ring_ent)51 struct fuse_ring_ent *ring_ent)
46{52{
@@ -222,6 +228,7 @@ void fuse_uring_destruct(struct fuse_chan *fch) · 175 unmodified lines
222 }228 }
223 229
224 kfree(queue->fpq.processing);230 kfree(queue->fpq.processing);
231 kfree(queue->bufpool);
225 kfree(queue);232 kfree(queue);
226 WRITE_ONCE(ring->queues[qid], NULL);233 WRITE_ONCE(ring->queues[qid], NULL);
227 }234 }
@@ -316,6 +323,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, · 88 unmodified lines
316 if (ring->queues[qid]) {323 if (ring->queues[qid]) {
317 spin_unlock(&fch->lock);324 spin_unlock(&fch->lock);
318 kfree(queue->fpq.processing);325 kfree(queue->fpq.processing);
326 kfree(queue->bufpool);
319 kfree(queue);327 kfree(queue);
320 return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid];328 return fail_if_exists ? ERR_PTR(-EEXIST) : ring->queues[qid];
321 }329 }
@@ -646,13 +654,14 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, · 324 unmodified lines
646}654}
647 655
648static int setup_fuse_copy_state(struct fuse_copy_state *cs,656static int setup_fuse_copy_state(struct fuse_copy_state *cs,
649 struct fuse_ring *ring, struct fuse_req *req,657 struct fuse_req *req,
650 struct fuse_ring_ent *ent, int dir,658 struct fuse_ring_ent *ent, int dir,
651 struct iov_iter *iter)659 struct iov_iter *iter)
652{660{
653 int err;661 int err;
654 662
655 err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter);663 err = import_ubuf(dir, ent->payload.iov_base, ent->payload.iov_len,
664 iter);
656 if (err) {665 if (err) {
657 pr_info_ratelimited("fuse: Import of user buffer failed\n");666 pr_info_ratelimited("fuse: Import of user buffer failed\n");
658 return err;667 return err;
@@ -666,8 +675,7 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs, · 7 unmodified lines
666 return 0;675 return 0;
667}676}
668 677
669static int fuse_uring_copy_from_ring(struct fuse_ring *ring,678static int fuse_uring_copy_from_ring(struct fuse_req *req,
670 struct fuse_req *req,
671 struct fuse_ring_ent *ent)679 struct fuse_ring_ent *ent)
672{680{
673 struct fuse_copy_state cs;681 struct fuse_copy_state cs;
@@ -681,7 +689,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, · 7 unmodified lines
681 if (err)689 if (err)
682 return err;690 return err;
683 691
684 err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter);692 err = setup_fuse_copy_state(&cs, req, ent, ITER_SOURCE, &iter);
685 if (err)693 if (err)
686 return err;694 return err;
687 695
@@ -693,7 +701,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, · 5 unmodified lines
693/*701/*
694 * Copy data from the req to the ring buffer702 * Copy data from the req to the ring buffer
695 */703 */
696static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,704static int fuse_uring_args_to_ring(struct fuse_req *req,
697 struct fuse_ring_ent *ent)705 struct fuse_ring_ent *ent)
698{706{
699 struct fuse_copy_state cs;707 struct fuse_copy_state cs;
@@ -707,7 +715,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, · 7 unmodified lines
707 .commit_id = req->in.h.unique,715 .commit_id = req->in.h.unique,
708 };716 };
709 717
710 err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter);718 err = setup_fuse_copy_state(&cs, req, ent, ITER_DEST, &iter);
711 if (err)719 if (err)
712 return err;720 return err;
713 721
@@ -737,6 +745,10 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, · 23 unmodified lines
737 }745 }
738 746
739 ent_in_out.payload_sz = cs.ring.copied_sz;747 ent_in_out.payload_sz = cs.ring.copied_sz;
748 if (bufpool_enabled(ent->queue) && ent->payload.iov_base)
749 ent_in_out.offset =
750 (uintptr_t)ent->payload.iov_base - ent->queue->bufpool->base_uaddr;
751
740 return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT,752 return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT,
741 &ent_in_out, sizeof(ent_in_out));753 &ent_in_out, sizeof(ent_in_out));
742}754}
@@ -745,7 +757,6 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, · 2 unmodified lines
745 struct fuse_req *req)757 struct fuse_req *req)
746{758{
747 struct fuse_ring_queue *queue = ent->queue;759 struct fuse_ring_queue *queue = ent->queue;
748 struct fuse_ring *ring = queue->ring;
749 int err;760 int err;
750 761
751 err = -EIO;762 err = -EIO;
@@ -760,7 +771,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, · 8 unmodified lines
760 return err;771 return err;
761 772
762 /* copy the request */773 /* copy the request */
763 err = fuse_uring_args_to_ring(ring, req, ent);774 err = fuse_uring_args_to_ring(req, ent);
764 if (unlikely(err)) {775 if (unlikely(err)) {
765 pr_info_ratelimited("Copy to ring failed: %d\n", err);776 pr_info_ratelimited("Copy to ring failed: %d\n", err);
766 return err;777 return err;
@@ -771,6 +782,91 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, · 4 unmodified lines
771 sizeof(req->in.h));782 sizeof(req->in.h));
772}783}
773 784
785static bool fuse_uring_req_has_payload(struct fuse_req *req)
786{
787 struct fuse_args *args = req->args;
788
789 return args->in_numargs > 1 || args->out_numargs;
790}
791
792static int fuse_uring_select_buffer(struct fuse_ring_ent *ent)
793{
794 struct fuse_ring_queue *queue = ent->queue;
795 struct fuse_bufpool *pool = queue->bufpool;
796 unsigned int id;
797
798 lockdep_assert_held(&queue->lock);
799
800 id = find_first_bit(pool->free_map, pool->nr_bufs);
801 if (id >= pool->nr_bufs)
802 return -ENOBUFS;
803
804 WARN_ON_ONCE(ent->payload.iov_base);
805 __clear_bit(id, pool->free_map);
806
807 ent->buf_id = id;
808 ent->payload.iov_base =
809 (void __user *)(pool->base_uaddr + id * pool->buf_size);
810 ent->payload.iov_len = pool->buf_size;
811
812 return 0;
813}
814
815static void fuse_uring_recycle_buffer(struct fuse_ring_ent *ent)
816{
817 struct iovec *ent_payload = &ent->payload;
818 struct fuse_ring_queue *queue = ent->queue;
819 struct fuse_bufpool *pool;
820
821 lockdep_assert_held(&queue->lock);
822
823 if (!bufpool_enabled(queue) || !ent_payload->iov_base)
824 return;
825
826 pool = queue->bufpool;
827
828 /* a buffer should never be recycled twice */
829 WARN_ON_ONCE(test_bit(ent->buf_id, pool->free_map));
830 __set_bit(ent->buf_id, pool->free_map);
831
832 memset(ent_payload, 0, sizeof(*ent_payload));
833 ent->buf_id = 0;
834}
835
836static int fuse_uring_next_req_update_buffer(struct fuse_ring_ent *ent,
837 struct fuse_req *req)
838{
839 bool buffer_selected;
840 bool has_payload;
841
842 if (!bufpool_enabled(ent->queue))
843 return 0;
844
845 buffer_selected = !!ent->payload.iov_base;
846 has_payload = fuse_uring_req_has_payload(req);
847
848 if (has_payload && !buffer_selected)
849 return fuse_uring_select_buffer(ent);
850
851 if (!has_payload && buffer_selected)
852 fuse_uring_recycle_buffer(ent);
853
854 return 0;
855}
856
857static int fuse_uring_prep_buffer(struct fuse_ring_ent *ent,
858 struct fuse_req *req)
859{
860 if (!bufpool_enabled(ent->queue))
861 return 0;
862
863 /* no payload to copy, can skip selecting a buffer */
864 if (!fuse_uring_req_has_payload(req))
865 return 0;
866
867 return fuse_uring_select_buffer(ent);
868}
869
774static int fuse_uring_prepare_send(struct fuse_ring_ent *ent,870static int fuse_uring_prepare_send(struct fuse_ring_ent *ent,
775 struct fuse_req *req)871 struct fuse_req *req)
776{872{
@@ -858,9 +954,12 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) · 81 unmodified lines
858 954
859 /* get and assign the next entry while it is still holding the lock */955 /* get and assign the next entry while it is still holding the lock */
860 req = list_first_entry_or_null(req_queue, struct fuse_req, list);956 req = list_first_entry_or_null(req_queue, struct fuse_req, list);
861 if (req)957 if (!req || fuse_uring_next_req_update_buffer(ent, req)) {
862 fuse_uring_add_req_to_ring_ent(ent, req);958 fuse_uring_recycle_buffer(ent);
959 return NULL;
960 }
863 961
962 fuse_uring_add_req_to_ring_ent(ent, req);
864 return req;963 return req;
865}964}
866 965
@@ -872,7 +971,6 @@ static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) · 5 unmodified lines
872static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,971static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
873 unsigned int issue_flags)972 unsigned int issue_flags)
874{973{
875 struct fuse_ring *ring = ent->queue->ring;
876 ssize_t err = -EFAULT;974 ssize_t err = -EFAULT;
877 975
878 if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h,976 if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h,
@@ -885,7 +983,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, · 6 unmodified lines
885 goto out;983 goto out;
886 }984 }
887 985
888 err = fuse_uring_copy_from_ring(ring, req, ent);986 err = fuse_uring_copy_from_ring(req, ent);
889out:987out:
890 fuse_uring_req_end(ent, req, err);988 fuse_uring_req_end(ent, req, err);
891}989}
@@ -1004,6 +1102,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, · 112 unmodified lines
1004 if (err != 0) {1102 if (err != 0) {
1005 pr_info_ratelimited("qid=%d commit_id %llu state %d",1103 pr_info_ratelimited("qid=%d commit_id %llu state %d",
1006 queue->qid, commit_id, ent->state);1104 queue->qid, commit_id, ent->state);
1105 fuse_uring_recycle_buffer(ent);
1007 spin_unlock(&queue->lock);1106 spin_unlock(&queue->lock);
1008 fuse_uring_req_end(ent, req, err);1107 fuse_uring_req_end(ent, req, err);
1009 return err;1108 return err;
@@ -1021,6 +1120,11 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, · 11 unmodified lines
1021 * fuse requests would otherwise not get processed - committing1120 * fuse requests would otherwise not get processed - committing
1022 * and fetching is done in one step vs legacy fuse, which has separated1121 * and fetching is done in one step vs legacy fuse, which has separated
1023 * read (fetch request) and write (commit result).1122 * read (fetch request) and write (commit result).
1123 *
1124 * If there is no next request or if all buffers are busy (if using a
1125 * bufpool), the cmd is not returned to userspace. The entry is left
1126 * available and the cmd only returns to userspace when there's a
1127 * next request and an available buffer.
1024 */1128 */
1025 if (fuse_uring_get_next_fuse_req(ent, queue))1129 if (fuse_uring_get_next_fuse_req(ent, queue))
1026 fuse_uring_send(ent, cmd, 0, issue_flags);1130 fuse_uring_send(ent, cmd, 0, issue_flags);
@@ -1145,11 +1249,23 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, · 118 unmodified lines
1145 }1249 }
1146 1250
1147 payload = &iov[FUSE_URING_IOV_PAYLOAD];1251 payload = &iov[FUSE_URING_IOV_PAYLOAD];
1148 if (payload->iov_len < ring->max_payload_sz) {1252
1149 pr_info_ratelimited("Invalid req payload len %zu\n",1253 spin_lock(&queue->lock);
1150 payload->iov_len);1254 if (bufpool_enabled(queue)) {
1151 return ERR_PTR(err);1255 if (payload->iov_base || payload->iov_len) {
1256 spin_unlock(&queue->lock);
1257 return ERR_PTR(err);
1258 }
1259 } else {
1260 if (payload->iov_len < ring->max_payload_sz) {
1261 pr_info_ratelimited("Invalid req payload len %zu\n",
1262 payload->iov_len);
1263 spin_unlock(&queue->lock);
1264 return ERR_PTR(err);
1265 }
1266 queue->payload_mode = FUSE_PAYLOAD_PER_ENT;
1152 }1267 }
1268 spin_unlock(&queue->lock);
1153 1269
1154 err = -ENOMEM;1270 err = -ENOMEM;
1155 ent = kzalloc_obj(*ent, GFP_KERNEL_ACCOUNT);1271 ent = kzalloc_obj(*ent, GFP_KERNEL_ACCOUNT);
@@ -1160,7 +1276,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, · 4 unmodified lines
1160 1276
1161 ent->queue = queue;1277 ent->queue = queue;
1162 ent->headers = headers->iov_base;1278 ent->headers = headers->iov_base;
1163 ent->payload = payload->iov_base;1279 if (queue->payload_mode == FUSE_PAYLOAD_PER_ENT)
1280 ent->payload = *payload;
1164 1281
1165 atomic_inc(&ring->queue_refs);1282 atomic_inc(&ring->queue_refs);
1166 return ent;1283 return ent;
@@ -1231,6 +1348,67 @@ static int fuse_uring_add_queue(struct io_uring_cmd *cmd, struct fuse_chan *fch) · 64 unmodified lines
1231 return 0;1348 return 0;
1232}1349}
1233 1350
1351static int fuse_uring_add_bufpool(struct io_uring_cmd *cmd,
1352 struct fuse_chan *fch)
1353{
1354 const struct fuse_uring_cmd_req *cmd_req =
1355 io_uring_sqe128_cmd(cmd->sqe, struct fuse_uring_cmd_req);
1356 unsigned int qid = READ_ONCE(cmd_req->qid);
1357 uint64_t flags = READ_ONCE(cmd_req->flags);
1358 /* paired with the smp_store_release() in fuse_uring_create */
1359 struct fuse_ring *ring = smp_load_acquire(&fch->ring);
1360 struct fuse_ring_queue *queue;
1361 struct fuse_bufpool *pool;
1362 uintptr_t pool_uaddr;
1363 unsigned int pool_len, nr_bufs;
1364 size_t pool_size, buf_size;
1365
1366 if (!ring || qid >= ring->nr_queues || flags)
1367 return -EINVAL;
1368
1369 /* reserved for future use, must be zero */
1370 if (READ_ONCE(cmd_req->bufpool.reserved))
1371 return -EINVAL;
1372
1373 /* Pairs with smp_store_release() in fuse_uring_create_queue() */
1374 queue = smp_load_acquire(&ring->queues[qid]);
1375 if (!queue)
1376 return -EINVAL;
1377
1378 pool_uaddr = READ_ONCE(cmd_req->bufpool.uaddr);
1379 pool_len = READ_ONCE(cmd_req->bufpool.len);
1380
1381 /* each buffer holds the max payload size */
1382 buf_size = queue->ring->max_payload_sz;
1383
1384 nr_bufs = pool_len / buf_size;
1385 if (!nr_bufs)
1386 return -EINVAL;
1387
1388 pool_size = struct_size(pool, free_map, BITS_TO_LONGS(nr_bufs));
1389 pool = kzalloc(pool_size, GFP_KERNEL_ACCOUNT);
1390 if (!pool)
1391 return -ENOMEM;
1392
1393 pool->base_uaddr = pool_uaddr;
1394 pool->buf_size = buf_size;
1395 pool->nr_bufs = nr_bufs;
1396 /* all buffers are free */
1397 bitmap_set(pool->free_map, 0, nr_bufs);
1398
1399 spin_lock(&queue->lock);
1400 if (queue->payload_mode != FUSE_PAYLOAD_UNSET) {
1401 spin_unlock(&queue->lock);
1402 kfree(pool);
1403 return -EINVAL;
1404 }
1405 queue->bufpool = pool;
1406 queue->payload_mode = FUSE_PAYLOAD_BUFPOOL;
1407 spin_unlock(&queue->lock);
1408
1409 return 0;
1410}
1411
1234/*1412/*
1235 * Entry function from io_uring to handle the given passthrough command1413 * Entry function from io_uring to handle the given passthrough command
1236 * (op code IORING_OP_URING_CMD)1414 * (op code IORING_OP_URING_CMD)
@@ -1303,6 +1481,12 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) · 66 unmodified lines
1303 pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n",1481 pr_info_once("FUSE_IO_URING_CMD_ADD_QUEUE failed err=%d\n",
1304 err);1482 err);
1305 return err;1483 return err;
1484 case FUSE_IO_URING_CMD_ADD_BUFPOOL:
1485 err = fuse_uring_add_bufpool(cmd, fch);
1486 if (err)
1487 pr_info_once("FUSE_IO_URING_ADD_BUFPOOL failed err=%d\n",
1488 err);
1489 return err;
1306 default:1490 default:
1307 return -EINVAL;1491 return -EINVAL;
1308 }1492 }
@@ -1336,6 +1520,7 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) · 27 unmodified lines
1336 1520
1337 spin_lock(&queue->lock);1521 spin_lock(&queue->lock);
1338 list_del_init(&ent->list);1522 list_del_init(&ent->list);
1523 fuse_uring_recycle_buffer(ent);
1339 spin_unlock(&queue->lock);1524 spin_unlock(&queue->lock);
1340 1525
1341 io_uring_cmd_done(cmd, err, issue_flags);1526 io_uring_cmd_done(cmd, err, issue_flags);
@@ -1397,15 +1582,16 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) · 55 unmodified lines
1397 req->ring_queue = queue;1582 req->ring_queue = queue;
1398 ent = list_first_entry_or_null(&queue->ent_avail_queue,1583 ent = list_first_entry_or_null(&queue->ent_avail_queue,
1399 struct fuse_ring_ent, list);1584 struct fuse_ring_ent, list);
1400 if (ent)
1401 fuse_uring_add_req_to_ring_ent(ent, req);
1402 else
1403 list_add_tail(&req->list, &queue->fuse_req_queue);
1404 spin_unlock(&queue->lock);
1405 1585
1406 if (ent)1586 if (!ent || fuse_uring_prep_buffer(ent, req)) {
1407 fuse_uring_dispatch_ent(ent);1587 list_add_tail(&req->list, &queue->fuse_req_queue);
1588 spin_unlock(&queue->lock);
1589 return;
1590 }
1408 1591
1592 fuse_uring_add_req_to_ring_ent(ent, req);
1593 spin_unlock(&queue->lock);
1594 fuse_uring_dispatch_ent(ent);
1409 return;1595 return;
1410 1596
1411err_unlock:1597err_unlock:
@@ -1453,10 +1639,9 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) · 41 unmodified lines
1453 */1639 */
1454 req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req,1640 req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req,
1455 list);1641 list);
1456 if (ent && req) {1642 if (ent && req && !fuse_uring_prep_buffer(ent, req)) {
1457 fuse_uring_add_req_to_ring_ent(ent, req);1643 fuse_uring_add_req_to_ring_ent(ent, req);
1458 spin_unlock(&queue->lock);1644 spin_unlock(&queue->lock);
1459
1460 fuse_uring_dispatch_ent(ent);1645 fuse_uring_dispatch_ent(ent);
1461 } else {1646 } else {
1462 spin_unlock(&queue->lock);1647 spin_unlock(&queue->lock);
fs/fuse/dev_uring_i.h+36−1
@@ -7,6 +7,8 @@ · 6 unmodified lines
7#ifndef _FS_FUSE_DEV_URING_I_H7#ifndef _FS_FUSE_DEV_URING_I_H
8#define _FS_FUSE_DEV_URING_I_H8#define _FS_FUSE_DEV_URING_I_H
9 9
10#include <linux/uio.h>
11
10#include "fuse_dev_i.h"12#include "fuse_dev_i.h"
11 13
12#ifdef CONFIG_FUSE_IO_URING14#ifdef CONFIG_FUSE_IO_URING
@@ -36,11 +38,38 @@ enum fuse_ring_req_state { · 23 unmodified lines
36 FRRS_RELEASED,38 FRRS_RELEASED,
37};39};
38 40
41/* how a queue's payload buffers are provided */
42enum fuse_queue_payload_mode {
43 /* not yet committed (a bufpool may still be added) */
44 FUSE_PAYLOAD_UNSET = 0,
45 /* each entry registers its own payload buffer */
46 FUSE_PAYLOAD_PER_ENT,
47 /* each entry's payload buffer is assigned from a bufpool */
48 FUSE_PAYLOAD_BUFPOOL,
49};
50
51struct fuse_bufpool {
52 /* starting uaddr of the bufpool */
53 uintptr_t base_uaddr;
54
55 /* size of each buffer in the pool */
56 size_t buf_size;
57
58 /* total number of buffers in the pool */
59 unsigned int nr_bufs;
60
61 /* bitmap tracking which buffers are free */
62 unsigned long free_map[];
63};
64
39/** A fuse ring entry, part of the ring queue */65/** A fuse ring entry, part of the ring queue */
40struct fuse_ring_ent {66struct fuse_ring_ent {
41 /* userspace buffer */67 /* userspace buffer */
42 struct fuse_uring_req_header __user *headers;68 struct fuse_uring_req_header __user *headers;
43 void __user *payload;69 struct iovec payload;
70
71 /* buffer id in the pool, if bufpools are used. ignored otherwise */
72 unsigned int buf_id;
44 73
45 /* the ring queue that owns the request */74 /* the ring queue that owns the request */
46 struct fuse_ring_queue *queue;75 struct fuse_ring_queue *queue;
@@ -99,6 +128,12 @@ struct fuse_ring_queue { · 52 unmodified lines
99 unsigned int active_background;128 unsigned int active_background;
100 129
101 bool stopped;130 bool stopped;
131
132 /* how this queue's payload buffers are provided */
133 enum fuse_queue_payload_mode payload_mode;
134
135 /* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */
136 struct fuse_bufpool *bufpool;
102};137};
103 138
104/*139/*
fs/fuse/inode.c+1−1
@@ -1482,7 +1482,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) · 1481 unmodified lines
1482 * the reply - server is either sending IORING_OP_URING_CMD or not.1482 * the reply - server is either sending IORING_OP_URING_CMD or not.
1483 */1483 */
1484 if (fuse_uring_enabled())1484 if (fuse_uring_enabled())
1485 flags |= FUSE_OVER_IO_URING;1485 flags |= FUSE_OVER_IO_URING | FUSE_HAS_IO_URING_BUFPOOL;
1486 1486
1487 ia->in.flags = flags;1487 ia->in.flags = flags;
1488 ia->in.flags2 = flags >> 32;1488 ia->in.flags2 = flags >> 32;
include/uapi/linux/fuse.h+20−1
@@ -243,6 +243,9 @@ · 242 unmodified lines
243 *243 *
244 * 7.46244 * 7.46
245 * - add FUSE_IO_URING_CMD_ADD_QUEUE245 * - add FUSE_IO_URING_CMD_ADD_QUEUE
246 * - add FUSE_HAS_IO_URING_BUFPOOL
247 * - add fuse_uring_cmd_req bufpool struct
248 * - add bufpool offset field to fuse_uring_ent_in_out struct
246 */249 */
247 250
248#ifndef _LINUX_FUSE_H251#ifndef _LINUX_FUSE_H
@@ -451,6 +454,7 @@ struct fuse_file_lock { · 202 unmodified lines
451 * FUSE_OVER_IO_URING: Indicate that client supports io-uring454 * FUSE_OVER_IO_URING: Indicate that client supports io-uring
452 * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests.455 * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests.
453 * init_out.request_timeout contains the timeout (in secs)456 * init_out.request_timeout contains the timeout (in secs)
457 * FUSE_HAS_IO_URING_BUFPOOL: kernel supports io-uring buffer pools
454 */458 */
455#define FUSE_ASYNC_READ (1 << 0)459#define FUSE_ASYNC_READ (1 << 0)
456#define FUSE_POSIX_LOCKS (1 << 1)460#define FUSE_POSIX_LOCKS (1 << 1)
@@ -498,6 +502,7 @@ struct fuse_file_lock { · 41 unmodified lines
498#define FUSE_ALLOW_IDMAP (1ULL << 40)502#define FUSE_ALLOW_IDMAP (1ULL << 40)
499#define FUSE_OVER_IO_URING (1ULL << 41)503#define FUSE_OVER_IO_URING (1ULL << 41)
500#define FUSE_REQUEST_TIMEOUT (1ULL << 42)504#define FUSE_REQUEST_TIMEOUT (1ULL << 42)
505#define FUSE_HAS_IO_URING_BUFPOOL (1ULL << 43)
501 506
502/**507/**
503 * CUSE INIT request/reply flags508 * CUSE INIT request/reply flags
@@ -1266,7 +1271,9 @@ struct fuse_uring_ent_in_out { · 762 unmodified lines
1266 1271
1267 /* size of user payload buffer */1272 /* size of user payload buffer */
1268 uint32_t payload_sz;1273 uint32_t payload_sz;
1269 uint32_t padding;1274
1275 /* Offset into the bufpool, if bufpools are used */
1276 uint32_t offset;
1270 1277
1271 uint64_t reserved;1278 uint64_t reserved;
1272};1279};
@@ -1298,6 +1305,9 @@ enum fuse_uring_cmd { · 25 unmodified lines
1298 1305
1299 /* add a queue */1306 /* add a queue */
1300 FUSE_IO_URING_CMD_ADD_QUEUE = 3,1307 FUSE_IO_URING_CMD_ADD_QUEUE = 3,
1308
1309 /* add a bufpool to a queue */
1310 FUSE_IO_URING_CMD_ADD_BUFPOOL = 4,
1301};1311};
1302 1312
1303/**1313/**
@@ -1312,6 +1322,15 @@ struct fuse_uring_cmd_req { · 8 unmodified lines
1312 /* queue the command is for (queue index) */1322 /* queue the command is for (queue index) */
1313 uint16_t qid;1323 uint16_t qid;
1314 uint8_t padding[6];1324 uint8_t padding[6];
1325
1326 union {
1327 struct {
1328 /* base address of bufpool */
1329 uint64_t uaddr;
1330 uint32_t len;
1331 uint32_t reserved;
1332 } bufpool;
1333 };
1315};1334};
1316 1335
1317#endif /* _LINUX_FUSE_H */1336#endif /* _LINUX_FUSE_H */