History

Merge tag 'fuse-update-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse

Pull fuse updates from Miklos Szeredi: - Improve performance of the io-uring transport by introducing buffer pools and zero-copy (Joanne) - Fix lots of bugs (Baokun Li) - Fix io-uring initialization issues (Joanne, Bernd) - More prep work for large folios (Joanne) - Don't limit buffered read to 128k (Jim Harris) - Fix zeroing of page end (dirtied with mmap) on file size extension (Jimmy Zuber) - Improve performance in certain cases with wake_up_sync() when queuing request (Xuewen Yan) - Misc fixes and cleanups (Xuewen Yan) * tag 'fuse-update-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse: (35 commits) fuse: zero the partial EOF page when extending a file io_uring: Add missing include for ITER_SOURCE and ITER_DEST fuse: Fix the condition to enable over-io-uring fuse: invalidate the correct range after O_APPEND direct write selftests/fuse: test post-EOF page zeroing when a file is extended fuse: wake one waiter per freed slot when raising max_background fuse: use min_not_zero() in fuse_init_server_timeout() fuse: copy request headers via a stack buffer for io-uring fuse: give wakeup hints to the scheduler for synchronous requests fuse: check for NULL root inode in fuse_fill_super_submount fuse: reject a duplicate fd= mount option cuse: wait for pending RCU callbacks on module exit fuse: fix invalidate lock leak on open O_TRUNC DAX failure fuse: fix invalidate lock leak on setattr writeback failure fuse: wait for FR_FINISHED on abort_on_kill to prevent use-after-free fuse: make dentry_tree_work static docs: fuse: document io-uring buffer pool and zero-copy uapi fuse: add zero-copy over io-uring fuse: support registered buffer pools in io-uring fuse: add io-uring buffer pools ...

Linus Torvalds committed 2 months ago

Commit 9cebfe62 parents ce14fe4 34b5c4a27 files · +1474 −236Browse files

Showing the changes to include (3 of 27 files). Show all

Unified · Split

3 files
  1. include/linux/io_uring/cmd.h +32 −6
  2. include/linux/io_uring_types.h +5 −0
  3. include/uapi/linux/fuse.h +61 −2
include/linux/io_uring/cmd.h+32−6
@@ -91,6 +91,15 @@ struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd, · 90 unmodified lines
9191 bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,
9292 struct io_br_sel *sel, unsigned int issue_flags);
9393
94+int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq,
95+ void (*release)(void *), unsigned int index,
96+ unsigned int issue_flags);
97+int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs,
98+ unsigned int nr_bvecs, void (*release)(void *),
99+ void *priv, u8 dir, unsigned int index,
100+ unsigned int issue_flags);
101+int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index,
102+ unsigned int issue_flags);
94103 #else
95104 static inline int
96105 io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
@@ -133,6 +142,29 @@ static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, · 36 unmodified lines
133142 {
134143 return true;
135144 }
145+static inline int io_buffer_register_request(struct io_uring_cmd *cmd,
146+ struct request *rq,
147+ void (*release)(void *),
148+ unsigned int index,
149+ unsigned int issue_flags)
150+{
151+ return -EOPNOTSUPP;
152+}
153+static inline int io_buffer_register_bvec(struct io_uring_cmd *cmd,
154+ const struct bio_vec *bvs,
155+ unsigned int nr_bvecs,
156+ void (*release)(void *), void *priv,
157+ u8 dir, unsigned int index,
158+ unsigned int issue_flags)
159+{
160+ return -EOPNOTSUPP;
161+}
162+static inline int io_buffer_unregister(struct io_uring_cmd *cmd,
163+ unsigned int index,
164+ unsigned int issue_flags)
165+{
166+ return -EOPNOTSUPP;
167+}
136168 #endif
137169
138170 static inline struct io_uring_cmd *io_uring_cmd_from_tw(struct io_tw_req tw_req)
@@ -182,10 +214,4 @@ static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret, · 43 unmodified lines
182214 return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true);
183215 }
184216
185−int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
186− void (*release)(void *), unsigned int index,
187− unsigned int issue_flags);
188−int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index,
189− unsigned int issue_flags);
190−
191217 #endif /* _LINUX_IO_URING_CMD_H */
include/linux/io_uring_types.h+5−0
@@ -53,6 +53,11 @@ enum io_uring_cmd_flags { · 52 unmodified lines
5353 IO_URING_F_COMPAT = (1 << 12),
5454 };
5555
56+enum {
57+ IO_BUF_DEST = 1 << ITER_DEST,
58+ IO_BUF_SOURCE = 1 << ITER_SOURCE,
59+};
60+
5661 struct iou_loop_params;
5762
5863 struct io_wq_work_node {
include/uapi/linux/fuse.h+61−2
@@ -240,6 +240,14 @@ · 239 unmodified lines
240240 * - add FUSE_COPY_FILE_RANGE_64
241241 * - add struct fuse_copy_file_range_out
242242 * - add FUSE_NOTIFY_PRUNE
243+ *
244+ * 7.46
245+ * - add FUSE_IO_URING_CMD_ADD_QUEUE
246+ * - add FUSE_HAS_IO_URING_BUFPOOL
247+ * - add fuse_uring_cmd_req bufpool struct
248+ * - add bufpool offset field to fuse_uring_ent_in_out struct
249+ * - add FUSE_URING_ZERO_COPY, FUSE_URING_ENT_ZERO_COPY, and
250+ * FOPEN_IO_URING_ZERO_COPY flag
243251 */
244252
245253 #ifndef _LINUX_FUSE_H
@@ -275,7 +283,7 @@ · 29 unmodified lines
275283 #define FUSE_KERNEL_VERSION 7
276284
277285 /** Minor version number of this interface */
278−#define FUSE_KERNEL_MINOR_VERSION 45
286+#define FUSE_KERNEL_MINOR_VERSION 46
279287
280288 /** The node ID of the root inode */
281289 #define FUSE_ROOT_ID 1
@@ -383,6 +391,12 @@ struct fuse_file_lock { · 101 unmodified lines
383391 * FOPEN_NOFLUSH: don't flush data cache on close (unless FUSE_WRITEBACK_CACHE)
384392 * FOPEN_PARALLEL_DIRECT_WRITES: Allow concurrent direct writes on the same inode
385393 * FOPEN_PASSTHROUGH: passthrough read/write io for this open file
394+ * FOPEN_IO_URING_ZERO_COPY: use io-uring zero-copy for reads/writes on this
395+ * open file. Honored only when the serving io-uring
396+ * queue was set up for zero-copy
397+ * (FUSE_URING_ZERO_COPY) and the request carries page
398+ * payload. Otherwise reads/writes fall back to
399+ * copying.
386400 */
387401 #define FOPEN_DIRECT_IO (1 << 0)
388402 #define FOPEN_KEEP_CACHE (1 << 1)
@@ -392,6 +406,7 @@ struct fuse_file_lock { · 3 unmodified lines
392406 #define FOPEN_NOFLUSH (1 << 5)
393407 #define FOPEN_PARALLEL_DIRECT_WRITES (1 << 6)
394408 #define FOPEN_PASSTHROUGH (1 << 7)
409+#define FOPEN_IO_URING_ZERO_COPY (1 << 8)
395410
396411 /**
397412 * INIT request/reply flags
@@ -448,6 +463,7 @@ struct fuse_file_lock { · 50 unmodified lines
448463 * FUSE_OVER_IO_URING: Indicate that client supports io-uring
449464 * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests.
450465 * init_out.request_timeout contains the timeout (in secs)
466+ * FUSE_HAS_IO_URING_BUFPOOL: kernel supports io-uring buffer pools
451467 */
452468 #define FUSE_ASYNC_READ (1 << 0)
453469 #define FUSE_POSIX_LOCKS (1 << 1)
@@ -495,6 +511,7 @@ struct fuse_file_lock { · 41 unmodified lines
495511 #define FUSE_ALLOW_IDMAP (1ULL << 40)
496512 #define FUSE_OVER_IO_URING (1ULL << 41)
497513 #define FUSE_REQUEST_TIMEOUT (1ULL << 42)
514+#define FUSE_HAS_IO_URING_BUFPOOL (1ULL << 43)
498515
499516 /**
500517 * CUSE INIT request/reply flags
@@ -1251,6 +1268,13 @@ struct fuse_supp_groups { · 750 unmodified lines
12511268 #define FUSE_URING_IN_OUT_HEADER_SZ 128
12521269 #define FUSE_URING_OP_IN_OUT_SZ 128
12531270
1271+/**
1272+ * fuse_uring_ent_in_out flags
1273+ *
1274+ * FUSE_URING_ENT_ZERO_COPY: Set if the ent's payload is zero-copied
1275+ */
1276+#define FUSE_URING_ENT_ZERO_COPY (1 << 0)
1277+
12541278 /* Used as part of the fuse_uring_req_header */
12551279 struct fuse_uring_ent_in_out {
12561280 uint64_t flags;
@@ -1263,7 +1287,9 @@ struct fuse_uring_ent_in_out { · 6 unmodified lines
12631287
12641288 /* size of user payload buffer */
12651289 uint32_t payload_sz;
1266− uint32_t padding;
1290+
1291+ /* Offset into the bufpool, if bufpools are used */
1292+ uint32_t offset;
12671293
12681294 uint64_t reserved;
12691295 };
@@ -1292,8 +1318,22 @@ enum fuse_uring_cmd { · 22 unmodified lines
12921318
12931319 /* commit fuse request result and fetch next request */
12941320 FUSE_IO_URING_CMD_COMMIT_AND_FETCH = 2,
1321+
1322+ /* add a queue */
1323+ FUSE_IO_URING_CMD_ADD_QUEUE = 3,
1324+
1325+ /* add a bufpool to a queue */
1326+ FUSE_IO_URING_CMD_ADD_BUFPOOL = 4,
12951327 };
12961328
1329+/*
1330+ * fuse_uring_cmd_req flags for FUSE_IO_URING_CMD_ADD_QUEUE
1331+ *
1332+ * FUSE_URING_ZERO_COPY is only supported for queues with bufpools on privileged
1333+ * servers
1334+ */
1335+#define FUSE_URING_ZERO_COPY (1 << 0)
1336+
12971337 /**
12981338 * In the 80B command area of the SQE.
12991339 */
@@ -1306,6 +1346,25 @@ struct fuse_uring_cmd_req { · 6 unmodified lines
13061346 /* queue the command is for (queue index) */
13071347 uint16_t qid;
13081348 uint8_t padding[6];
1349+
1350+ union {
1351+ struct {
1352+ /* base address of bufpool */
1353+ uint64_t uaddr;
1354+ uint32_t len;
1355+ uint32_t reserved;
1356+ } bufpool;
1357+
1358+ /*
1359+ * Index of this entry's slot in the server's io_uring
1360+ * registered buffer table, where the kernel registers the
1361+ * request's pages for zero-copy. Set for
1362+ * FUSE_IO_URING_CMD_REGISTER cmds only, and only on queues
1363+ * created with FUSE_URING_ZERO_COPY. On a non-zero-copy queue
1364+ * this must be 0
1365+ */
1366+ uint16_t ent_zero_copy_buf_index;
1367+ };
13091368 };
13101369
13111370 #endif /* _LINUX_FUSE_H */