/* SPDX-License-Identifier: GPL-2.0 * * FUSE: Filesystem in Userspace * Copyright (c) 2023-2024 DataDirect Networks. */ #ifndef _FS_FUSE_DEV_URING_I_H#define _FS_FUSE_DEV_URING_I_H #include <linux/uio.h> #include "fuse_dev_i.h" #ifdef CONFIG_FUSE_IO_URING #define FUSE_URING_TEARDOWN_TIMEOUT (5 * HZ)#define FUSE_URING_TEARDOWN_INTERVAL (HZ/20) enum fuse_ring_req_state {	FRRS_INVALID = 0, 	/* The ring entry received from userspace and it is being processed */	FRRS_COMMIT, 	/* The ring entry is waiting for new fuse requests */	FRRS_AVAILABLE, 	/* The ring entry got assigned a fuse req */	FRRS_FUSE_REQ, 	/* The ring entry is in or on the way to user space */	FRRS_USERSPACE, 	/* The ring entry is in teardown */	FRRS_TEARDOWN, 	/* The ring entry is released, but not freed yet */	FRRS_RELEASED,}; /* how a queue's payload buffers are provided */enum fuse_queue_payload_mode {	/* not yet committed (a bufpool may still be added) */	FUSE_PAYLOAD_UNSET = 0,	/* each entry registers its own payload buffer */	FUSE_PAYLOAD_PER_ENT,	/* each entry's payload buffer is assigned from a bufpool */	FUSE_PAYLOAD_BUFPOOL,}; struct fuse_bufpool {	bool registered; 	/*	 * io_uring registered buffer table index for this pool, bound at	 * ADD_BUFPOOL time. Only valid if the bufpool is registered	 */	u16 registered_index; 	/* starting uaddr of the bufpool */	uintptr_t base_uaddr; 	/* size of each buffer in the pool */	size_t buf_size; 	/* total number of buffers in the pool */	unsigned int nr_bufs; 	/* bitmap tracking which buffers are free */	unsigned long free_map[];}; /** A fuse ring entry, part of the ring queue */struct fuse_ring_ent {	/* userspace buffer */	struct fuse_uring_req_header __user *headers;	struct iovec payload; 	/* buffer id in the pool, if bufpools are used. ignored otherwise */	unsigned int buf_id; 	/* true if the request's pages are being zero-copied */	bool zero_copied;	unsigned int zero_copy_index; 	/* the ring queue that owns the request */	struct fuse_ring_queue *queue; 	/* fields below are protected by queue->lock */ 	struct io_uring_cmd *cmd; 	struct list_head list; 	enum fuse_ring_req_state state; 	struct fuse_req *fuse_req;}; struct fuse_ring_queue {	/*	 * back pointer to the main fuse uring structure that holds this	 * queue	 */	struct fuse_ring *ring; 	/* queue id, corresponds to the cpu core */	unsigned int qid; 	/*	 * queue lock, taken when any value in the queue changes _and_ also	 * a ring entry state changes.	 */	spinlock_t lock; 	/* available ring entries (struct fuse_ring_ent) */	struct list_head ent_avail_queue; 	/*	 * entries in the process of being committed or in the process	 * to be sent to userspace	 */	struct list_head ent_w_req_queue;	struct list_head ent_commit_queue; 	/* entries in userspace */	struct list_head ent_in_userspace; 	/* entries that are released */	struct list_head ent_released; 	/* fuse requests waiting for an entry slot */	struct list_head fuse_req_queue; 	/* background fuse requests */	struct list_head fuse_req_bg_queue; 	struct fuse_pqueue fpq; 	unsigned int active_background; 	bool stopped; 	/* how this queue's payload buffers are provided */	enum fuse_queue_payload_mode payload_mode; 	/* only allocated when payload_mode == FUSE_PAYLOAD_BUFPOOL */	struct fuse_bufpool *bufpool; 	bool zero_copy;}; /* * Describes if uring is for communication and holds alls the data needed * for uring communication */struct fuse_ring {	/* back pointer */	struct fuse_chan *chan; 	/* number of ring queues */	size_t nr_queues; 	/* maximum payload/arg size */	size_t max_payload_sz; 	struct fuse_ring_queue **queues; 	/*	 * Log ring entry states on stop when entries cannot be released	 */	unsigned int stop_debug_log : 1; 	wait_queue_head_t stop_waitq; 	/* async tear down */	struct delayed_work async_teardown_work; 	/* log */	unsigned long teardown_time; 	atomic_t queue_refs; 	bool ready;}; void fuse_uring_conn_init(struct fuse_chan *fch);void fuse_uring_stop_queues(struct fuse_ring *ring);void fuse_uring_abort_end_requests(struct fuse_ring *ring);int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags);void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req);bool fuse_uring_queue_bq_req(struct fuse_req *req);bool fuse_uring_remove_pending_req(struct fuse_req *req);bool fuse_uring_request_expired(struct fuse_chan *fch); static inline void fuse_uring_abort(struct fuse_chan *fch){	struct fuse_ring *ring = fch->ring; 	if (ring == NULL)		return; 	fuse_uring_abort_end_requests(ring); 	if (atomic_read(&ring->queue_refs) > 0)		fuse_uring_stop_queues(ring);} static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch){	struct fuse_ring *ring = fch->ring; 	if (ring)		wait_event(ring->stop_waitq,			   atomic_read(&ring->queue_refs) == 0);} static inline bool fuse_uring_ready(struct fuse_chan *fch){	struct fuse_ring *ring = READ_ONCE(fch->ring); 	return ring && smp_load_acquire(&ring->ready);} #else /* CONFIG_FUSE_IO_URING */ static inline void fuse_uring_conn_init(struct fuse_chan *fch){} static inline void fuse_uring_abort(struct fuse_chan *fch){} static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch){} static inline bool fuse_uring_ready(struct fuse_chan *fch){	return false;} static inline bool fuse_uring_remove_pending_req(struct fuse_req *req){	return false;} static inline bool fuse_uring_request_expired(struct fuse_chan *fch){	return false;} #endif /* CONFIG_FUSE_IO_URING */ #endif /* _FS_FUSE_DEV_URING_I_H */