// SPDX-License-Identifier: GPL-2.0/*  FUSE: Filesystem in Userspace  Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>*/ #include "dev.h"#include "args.h"#include "dev_uring_i.h" #include <linux/init.h>#include <linux/module.h>#include <linux/poll.h>#include <linux/sched/signal.h>#include <linux/uio.h>#include <linux/miscdevice.h>#include <linux/pagemap.h>#include <linux/file.h>#include <linux/slab.h>#include <linux/pipe_fs_i.h>#include <linux/swap.h>#include <linux/splice.h>#include <linux/sched.h>#include <linux/seq_file.h> #include "fuse_trace.h" MODULE_ALIAS_MISCDEV(FUSE_MINOR);MODULE_ALIAS("devname:fuse"); static DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq); static struct kmem_cache *fuse_req_cachep; static void fuse_request_init(struct fuse_chan *fch, struct fuse_req *req){	INIT_LIST_HEAD(&req->list);	INIT_LIST_HEAD(&req->intr_entry);	init_waitqueue_head(&req->waitq);	refcount_set(&req->count, 1);	__set_bit(FR_PENDING, &req->flags);	req->chan = fch;	req->create_time = jiffies;} static struct fuse_req *fuse_request_alloc(struct fuse_chan *fch, gfp_t flags){	struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags);	if (req)		fuse_request_init(fch, req); 	return req;} static void fuse_request_free(struct fuse_req *req){	WARN_ON(!list_empty(&req->intr_entry));	kmem_cache_free(fuse_req_cachep, req);} static void __fuse_get_request(struct fuse_req *req){	refcount_inc(&req->count);} /* Must be called with > 1 refcount */static void __fuse_put_request(struct fuse_req *req){	refcount_dec(&req->count);} void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param){	if (param) {		fch->minor = param->minor;		fch->max_write = param->max_write;		fch->max_pages = param->max_pages; 		if (param->io_uring_enabled)			fuse_uring_conn_init(fch);	} 	/* Pairs with smp_load_acquire() readers of fch->initialized */	smp_store_release(&fch->initialized, 1);	wake_up_all(&fch->blocked_waitq);} static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background){	/* Pairs with smp_store_release() in fuse_chan_set_initialized() */	if (!smp_load_acquire(&fch->initialized))		return true; 	return (for_background && fch->blocked) ||	       (fch->io_uring && fch->connected && !fuse_uring_ready(fch));} static void fuse_drop_waiting(struct fuse_chan *fch){	/*	 * lockess check of fch->connected is okay, because atomic_dec_and_test()	 * provides a memory barrier matched with the one in fuse_chan_wait_aborted()	 * to ensure no wake-up is missed.	 */	if (atomic_dec_and_test(&fch->num_waiting) &&	    !READ_ONCE(fch->connected)) {		/* wake up aborters */		wake_up_all(&fch->blocked_waitq);	}} static void fuse_put_request(struct fuse_req *req); static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background){	struct fuse_req *req;	int err; 	atomic_inc(&fch->num_waiting); 	if (fuse_block_alloc(fch, for_background)) {		err = -EINTR;		if (wait_event_state_exclusive(fch->blocked_waitq,				!fuse_block_alloc(fch, for_background),				(TASK_KILLABLE | TASK_FREEZABLE)))			goto out;	} 	err = -ENOTCONN;	if (!fch->connected)		goto out; 	req = fuse_request_alloc(fch, GFP_KERNEL);	err = -ENOMEM;	if (!req) {		if (for_background)			wake_up(&fch->blocked_waitq);		goto out;	} 	__set_bit(FR_WAITING, &req->flags);	if (for_background)		__set_bit(FR_BACKGROUND, &req->flags); 	return req;  out:	fuse_drop_waiting(fch);	return ERR_PTR(err);} static void fuse_put_request(struct fuse_req *req){	struct fuse_chan *fch = req->chan; 	if (refcount_dec_and_test(&req->count)) {		if (test_bit(FR_BACKGROUND, &req->flags)) {			/*			 * We get here in the unlikely case that a background			 * request was allocated but not sent			 */			spin_lock(&fch->bg_lock);			if (!fch->blocked)				wake_up(&fch->blocked_waitq);			spin_unlock(&fch->bg_lock);		} 		if (test_bit(FR_WAITING, &req->flags)) {			__clear_bit(FR_WAITING, &req->flags);			fuse_drop_waiting(fch);		} 		fuse_request_free(req);	}} unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args){	unsigned nbytes = 0;	unsigned i; 	for (i = 0; i < numargs; i++)		nbytes += args[i].size; 	return nbytes;}EXPORT_SYMBOL_GPL(fuse_len_args); static u64 fuse_get_unique_locked(struct fuse_iqueue *fiq){	fiq->reqctr += FUSE_REQ_ID_STEP;	return fiq->reqctr;} u64 fuse_get_unique(struct fuse_iqueue *fiq){	u64 ret; 	spin_lock(&fiq->lock);	ret = fuse_get_unique_locked(fiq);	spin_unlock(&fiq->lock); 	return ret;}EXPORT_SYMBOL_GPL(fuse_get_unique); unsigned int fuse_req_hash(u64 unique){	return hash_long(unique & ~FUSE_INT_REQ_BIT, FUSE_PQ_HASH_BITS);}EXPORT_SYMBOL_GPL(fuse_req_hash); /* * A new request is available, wake fiq->waitq */static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq, bool sync)__releases(fiq->lock){	if (sync)		wake_up_sync(&fiq->waitq);	else		wake_up(&fiq->waitq);	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);	spin_unlock(&fiq->lock);} struct fuse_forget_link *fuse_alloc_forget(void){	return kzalloc_obj(struct fuse_forget_link, GFP_KERNEL_ACCOUNT);} void fuse_dev_queue_forget(struct fuse_iqueue *fiq,			   struct fuse_forget_link *forget){	spin_lock(&fiq->lock);	if (fiq->connected) {		fiq->forget_list_tail->next = forget;		fiq->forget_list_tail = forget;		fuse_dev_wake_and_unlock(fiq, false);	} else {		kfree(forget);		spin_unlock(&fiq->lock);	}} void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req){	spin_lock(&fiq->lock);	/* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */	if (list_empty(&req->intr_entry) && test_bit(FR_SENT, &req->flags)) {		list_add_tail(&req->intr_entry, &fiq->interrupts);		/*		 * Pairs with smp_mb() implied by test_and_set_bit()		 * from fuse_request_end().		 */		smp_mb();		if (test_bit(FR_FINISHED, &req->flags)) {			list_del_init(&req->intr_entry);			spin_unlock(&fiq->lock);		} else  {			fuse_dev_wake_and_unlock(fiq, false);		}	} else {		spin_unlock(&fiq->lock);	}} static inline void fuse_request_assign_unique_locked(struct fuse_iqueue *fiq,						     struct fuse_req *req){	if (req->in.h.opcode != FUSE_NOTIFY_REPLY)		req->in.h.unique = fuse_get_unique_locked(fiq); 	/* tracepoint captures in.h.unique and in.h.len */	trace_fuse_request_send(req);} inline void fuse_request_assign_unique(struct fuse_iqueue *fiq,				       struct fuse_req *req){	if (req->in.h.opcode != FUSE_NOTIFY_REPLY)		req->in.h.unique = fuse_get_unique(fiq); 	/* tracepoint captures in.h.unique and in.h.len */	trace_fuse_request_send(req);}EXPORT_SYMBOL_GPL(fuse_request_assign_unique); static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req){	bool sync = test_and_clear_bit(FR_SYNC_WAKEUP, &req->flags); 	spin_lock(&fiq->lock);	if (fiq->connected) {		fuse_request_assign_unique_locked(fiq, req);		list_add_tail(&req->list, &fiq->pending);		fuse_dev_wake_and_unlock(fiq, sync);	} else {		spin_unlock(&fiq->lock);		req->out.h.error = -ENOTCONN;		clear_bit(FR_PENDING, &req->flags);		fuse_request_end(req);	}} static const struct fuse_iqueue_ops fuse_dev_fiq_ops = {	.send_forget	= fuse_dev_queue_forget,	.send_interrupt	= fuse_dev_queue_interrupt,	.send_req	= fuse_dev_queue_req,}; void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv){	spin_lock_init(&fiq->lock);	init_waitqueue_head(&fiq->waitq);	INIT_LIST_HEAD(&fiq->pending);	INIT_LIST_HEAD(&fiq->interrupts);	fiq->forget_list_tail = &fiq->forget_list_head;	fiq->connected = 1;	fiq->ops = ops;	fiq->priv = priv;}EXPORT_SYMBOL_GPL(fuse_iqueue_init); void fuse_chan_release(struct fuse_chan *fch){	struct fuse_iqueue *fiq = &fch->iq; 	if (fiq->ops->release)		fiq->ops->release(fiq); 	if (fch->timeout.req_timeout)		cancel_delayed_work_sync(&fch->timeout.work);} void fuse_chan_free(struct fuse_chan *fch){	WARN_ON(!list_empty(&fch->devices));	kfree(fch->pq_prealloc);	kfree(fch);}EXPORT_SYMBOL_GPL(fuse_chan_free); struct fuse_chan *fuse_chan_new(void){	struct fuse_chan *fch = kzalloc_obj(struct fuse_chan);	if (!fch)		return NULL; 	spin_lock_init(&fch->lock);	INIT_LIST_HEAD(&fch->devices);	spin_lock_init(&fch->bg_lock);	INIT_LIST_HEAD(&fch->bg_queue);	init_waitqueue_head(&fch->blocked_waitq);	atomic_set(&fch->num_waiting, 0);	fch->max_background = FUSE_DEFAULT_MAX_BACKGROUND;	fch->initialized = 0;	fch->blocked = 0;	fch->connected = 1;	fch->timeout.req_timeout = 0; 	return fch;}EXPORT_SYMBOL_GPL(fuse_chan_new); struct list_head *fuse_pqueue_alloc(void){	struct list_head *pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE); 	if (pq) {		for (int i = 0; i < FUSE_PQ_HASH_SIZE; i++)			INIT_LIST_HEAD(&pq[i]);	}	return pq;} struct fuse_chan *fuse_dev_chan_new(void){	struct fuse_chan *fch __free(kfree) = fuse_chan_new();	if (!fch)		return NULL; 	fch->pq_prealloc = fuse_pqueue_alloc();	if (!fch->pq_prealloc)		return NULL; 	fuse_iqueue_init(&fch->iq, &fuse_dev_fiq_ops, NULL); 	return no_free_ptr(fch);}EXPORT_SYMBOL_GPL(fuse_dev_chan_new); unsigned int fuse_chan_num_background(struct fuse_chan *fch){	return READ_ONCE(fch->num_background);} unsigned int fuse_chan_max_background(struct fuse_chan *fch){	return READ_ONCE(fch->max_background);} void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val){	spin_lock(&fch->bg_lock);	fch->max_background = val;	fch->blocked = fch->num_background >= fch->max_background;	if (!fch->blocked)		wake_up_nr(&fch->blocked_waitq,			   fch->max_background - fch->num_background);	spin_unlock(&fch->bg_lock);} unsigned int fuse_chan_num_waiting(struct fuse_chan *fch){	return atomic_read(&fch->num_waiting);} void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc){	fch->conn = fc;} void fuse_pqueue_init(struct fuse_pqueue *fpq){	spin_lock_init(&fpq->lock);	INIT_LIST_HEAD(&fpq->io);	fpq->connected = 1;	fpq->processing = NULL;} static struct fuse_dev *fuse_dev_alloc_no_pq(void){	struct fuse_dev *fud; 	fud = kzalloc_obj(struct fuse_dev);	if (!fud)		return NULL; 	refcount_set(&fud->ref, 1);	fuse_pqueue_init(&fud->pq); 	return fud;} struct fuse_dev *fuse_dev_alloc(void){	struct fuse_dev *fud __free(kfree) = fuse_dev_alloc_no_pq();	if (!fud)		return NULL; 	fud->pq.processing = fuse_pqueue_alloc();	if (!fud->pq.processing)		return NULL; 	return no_free_ptr(fud);}EXPORT_SYMBOL_GPL(fuse_dev_alloc); /* * Installs @fch into @fud, return true on success.  "Consumes" @pq in either case. */static bool fuse_dev_install_with_pq(struct fuse_dev *fud, struct fuse_chan *fch,				     struct list_head *pq){	struct fuse_chan *old_fch; 	guard(spinlock)(&fch->lock);	/*	 * Pairs with:	 *  - xchg() in fuse_dev_release()	 *  - smp_load_acquire() in fuse_dev_fc_get()	 */	old_fch = cmpxchg(&fud->chan, NULL, fch);	if (old_fch) {		/*		 * failed to set fud->chan because		 *  - it was already set to a different fc		 *  - it was set to disconneted		 */		kfree(pq);		return false;	}	if (pq) {		WARN_ON(fud->pq.processing);		fud->pq.processing = pq;	}	list_add_tail(&fud->entry, &fch->devices);	fuse_conn_get(fch->conn);	wake_up_all(&fuse_dev_waitq);	return true;} void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch){	struct list_head *pq = fch->pq_prealloc; 	fch->pq_prealloc = NULL;	if (!fuse_dev_install_with_pq(fud, fch, pq)) {		/* Channel is not usable without a dev */

Lines 1–500 of 2460. Load more