// SPDX-License-Identifier: GPL-2.0-or-later/* * Copyright (C) 2006, 2007, 2009 Rusty Russell, IBM Corporation * Copyright (C) 2009, 2010, 2011 Red Hat, Inc. * Copyright (C) 2009, 2010, 2011 Amit Shah <amit.shah@redhat.com> */#include <linux/cdev.h>#include <linux/debugfs.h>#include <linux/completion.h>#include <linux/device.h>#include <linux/err.h>#include <linux/freezer.h>#include <linux/fs.h>#include <linux/splice.h>#include <linux/pagemap.h>#include <linux/idr.h>#include <linux/init.h>#include <linux/list.h>#include <linux/poll.h>#include <linux/sched.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/virtio.h>#include <linux/virtio_console.h>#include <linux/wait.h>#include <linux/workqueue.h>#include <linux/module.h>#include <linux/dma-mapping.h>#include <linux/string_choices.h>#include "../tty/hvc/hvc_console.h" #define is_rproc_enabled IS_ENABLED(CONFIG_REMOTEPROC)#define VIRTCONS_MAX_PORTS 0x8000 /* * This is a global struct for storing common data for all the devices * this driver handles. * * Mainly, it has a linked list for all the consoles in one place so * that callbacks from hvc for get_chars(), put_chars() work properly * across multiple devices and multiple ports per device. */struct ports_driver_data { /* Used for exporting per-port information to debugfs */ struct dentry *debugfs_dir; /* List of all the devices we're handling */ struct list_head portdevs; /* All the console devices handled by this driver */ struct list_head consoles;}; static struct ports_driver_data pdrvdata; static const struct class port_class = { .name = "virtio-ports",}; static DEFINE_SPINLOCK(pdrvdata_lock);static DECLARE_COMPLETION(early_console_added); /* This struct holds information that's relevant only for console ports */struct console { /* We'll place all consoles in a list in the pdrvdata struct */ struct list_head list; /* The hvc device associated with this console port */ struct hvc_struct *hvc; /* The size of the console */ struct winsize ws; /* * This number identifies the number that we used to register * with hvc in hvc_instantiate() and hvc_alloc(); this is the * number passed on by the hvc callbacks to us to * differentiate between the other console ports handled by * this driver */ u32 vtermno;}; static DEFINE_IDA(vtermno_ida); struct port_buffer { char *buf; /* size of the buffer in *buf above */ size_t size; /* used length of the buffer */ size_t len; /* offset in the buf from which to consume data */ size_t offset; /* DMA address of buffer */ dma_addr_t dma; /* Device we got DMA memory from */ struct device *dev; /* List of pending dma buffers to free */ struct list_head list; /* If sgpages == 0 then buf is used */ unsigned int sgpages; /* sg is used if spages > 0. sg must be the last in is struct */ struct scatterlist sg[] __counted_by(sgpages);}; /* * This is a per-device struct that stores data common to all the * ports for that device (vdev->priv). */struct ports_device { /* Next portdev in the list, head is in the pdrvdata struct */ struct list_head list; /* * Workqueue handlers where we process deferred work after * notification */ struct work_struct control_work; struct work_struct config_work; struct list_head ports; /* To protect the list of ports */ spinlock_t ports_lock; /* To protect the vq operations for the control channel */ spinlock_t c_ivq_lock; spinlock_t c_ovq_lock; /* max. number of ports this device can hold */ u32 max_nr_ports; /* The virtio device we're associated with */ struct virtio_device *vdev; /* * A couple of virtqueues for the control channel: one for * guest->host transfers, one for host->guest transfers */ struct virtqueue *c_ivq, *c_ovq; /* * A control packet buffer for guest->host requests, protected * by c_ovq_lock. */ struct virtio_console_control cpkt; /* Array of per-port IO virtqueues */ struct virtqueue **in_vqs, **out_vqs; /* Major number for this device. Ports will be created as minors. */ int chr_major; /* * Set to true during PM freeze to block TX paths that may race * with virtqueue teardown (e.g. hvc put_chars with no_console_suspend). */ bool pm_freezing;}; struct port_stats { unsigned long bytes_sent, bytes_received, bytes_discarded;}; /* This struct holds the per-port data */struct port { /* Next port in the list, head is in the ports_device */ struct list_head list; /* Pointer to the parent virtio_console device */ struct ports_device *portdev; /* The current buffer from which data has to be fed to readers */ struct port_buffer *inbuf; /* * To protect the operations on the in_vq associated with this * port. Has to be a spinlock because it can be called from * interrupt context (get_char()). */ spinlock_t inbuf_lock; /* Protect the operations on the out_vq. */ spinlock_t outvq_lock; /* The IO vqs for this port */ struct virtqueue *in_vq, *out_vq; /* File in the debugfs directory that exposes this port's information */ struct dentry *debugfs_file; /* * Keep count of the bytes sent, received and discarded for * this port for accounting and debugging purposes. These * counts are not reset across port open / close events. */ struct port_stats stats; /* * The entries in this struct will be valid if this port is * hooked up to an hvc console */ struct console cons; /* Each port associates with a separate char device */ struct cdev *cdev; struct device *dev; /* Reference-counting to handle port hot-unplugs and file operations */ struct kref kref; /* A waitqueue for poll() or blocking read operations */ wait_queue_head_t waitqueue; /* The 'name' of the port that we expose via sysfs properties */ char *name; /* We can notify apps of host connect / disconnect events via SIGIO */ struct fasync_struct *async_queue; /* The 'id' to identify the port with the Host */ u32 id; bool outvq_full; /* Is the host device open */ bool host_connected; /* We should allow only one process to open a port */ bool guest_connected;}; static struct port *find_port_by_vtermno(u32 vtermno){ struct port *port; struct console *cons; unsigned long flags; spin_lock_irqsave(&pdrvdata_lock, flags); list_for_each_entry(cons, &pdrvdata.consoles, list) { if (cons->vtermno == vtermno) { port = container_of(cons, struct port, cons); goto out; } } port = NULL;out: spin_unlock_irqrestore(&pdrvdata_lock, flags); return port;} static struct port *find_port_by_devt_in_portdev(struct ports_device *portdev, dev_t dev){ struct port *port; unsigned long flags; spin_lock_irqsave(&portdev->ports_lock, flags); list_for_each_entry(port, &portdev->ports, list) { if (port->cdev->dev == dev) { kref_get(&port->kref); goto out; } } port = NULL;out: spin_unlock_irqrestore(&portdev->ports_lock, flags); return port;} static struct port *find_port_by_devt(dev_t dev){ struct ports_device *portdev; struct port *port; unsigned long flags; spin_lock_irqsave(&pdrvdata_lock, flags); list_for_each_entry(portdev, &pdrvdata.portdevs, list) { port = find_port_by_devt_in_portdev(portdev, dev); if (port) goto out; } port = NULL;out: spin_unlock_irqrestore(&pdrvdata_lock, flags); return port;} static struct port *find_port_by_id(struct ports_device *portdev, u32 id){ struct port *port; unsigned long flags; spin_lock_irqsave(&portdev->ports_lock, flags); list_for_each_entry(port, &portdev->ports, list) if (port->id == id) goto out; port = NULL;out: spin_unlock_irqrestore(&portdev->ports_lock, flags); return port;} /* * Finds a port by the virtqueue and returns a pointer to struct port * with the reference count incremented. * * Callers MUST decrement it when finished. */static struct port *find_port_by_vq(struct ports_device *portdev, struct virtqueue *vq){ struct port *port; unsigned long flags; spin_lock_irqsave(&portdev->ports_lock, flags); list_for_each_entry(port, &portdev->ports, list) if (port->in_vq == vq || port->out_vq == vq) { kref_get(&port->kref); goto out; } port = NULL;out: spin_unlock_irqrestore(&portdev->ports_lock, flags); return port;} static bool is_console_port(struct port *port){ if (port->cons.hvc) return true; return false;} static bool is_rproc_serial(const struct virtio_device *vdev){ return is_rproc_enabled && vdev->id.device == VIRTIO_ID_RPROC_SERIAL;} static inline bool use_multiport(struct ports_device *portdev){ /* * This condition can be true when put_chars is called from * early_init */ if (!portdev->vdev) return false; return __virtio_test_bit(portdev->vdev, VIRTIO_CONSOLE_F_MULTIPORT);} static DEFINE_SPINLOCK(dma_bufs_lock);static LIST_HEAD(pending_free_dma_bufs); static void free_buf(struct port_buffer *buf, bool can_sleep){ unsigned int i; for (i = 0; i < buf->sgpages; i++) { struct page *page = sg_page(&buf->sg[i]); if (!page) break; put_page(page); } if (!buf->dev) { kfree(buf->buf); } else if (is_rproc_enabled) { unsigned long flags; /* dma_free_coherent requires interrupts to be enabled. */ if (!can_sleep) { /* queue up dma-buffers to be freed later */ spin_lock_irqsave(&dma_bufs_lock, flags); list_add_tail(&buf->list, &pending_free_dma_bufs); spin_unlock_irqrestore(&dma_bufs_lock, flags); return; } dma_free_coherent(buf->dev, buf->size, buf->buf, buf->dma); /* Release device refcnt and allow it to be freed */ put_device(buf->dev); } kfree(buf);} static void reclaim_dma_bufs(void){ unsigned long flags; struct port_buffer *buf, *tmp; LIST_HEAD(tmp_list); if (list_empty(&pending_free_dma_bufs)) return; /* Create a copy of the pending_free_dma_bufs while holding the lock */ spin_lock_irqsave(&dma_bufs_lock, flags); list_cut_position(&tmp_list, &pending_free_dma_bufs, pending_free_dma_bufs.prev); spin_unlock_irqrestore(&dma_bufs_lock, flags); /* Release the dma buffers, without irqs enabled */ list_for_each_entry_safe(buf, tmp, &tmp_list, list) { list_del(&buf->list); free_buf(buf, true); }} static struct port_buffer *alloc_buf(struct virtio_device *vdev, size_t buf_size, int pages, gfp_t gfp){ struct port_buffer *buf; reclaim_dma_bufs(); /* * Allocate buffer and the sg list. The sg list array is allocated * directly after the port_buffer struct. */ buf = kmalloc_flex(*buf, sg, pages, gfp); if (!buf) goto fail; buf->sgpages = pages; if (pages > 0) { buf->dev = NULL; buf->buf = NULL; return buf; } if (is_rproc_serial(vdev)) { /* * Allocate DMA memory from ancestor. When a virtio * device is created by remoteproc, the DMA memory is * associated with the parent device: * virtioY => remoteprocX#vdevYbuffer. */ buf->dev = vdev->dev.parent; if (!buf->dev) goto free_buf; /* Increase device refcnt to avoid freeing it */ get_device(buf->dev); buf->buf = dma_alloc_coherent(buf->dev, buf_size, &buf->dma, gfp); } else { buf->dev = NULL; buf->buf = kmalloc(buf_size, gfp); } if (!buf->buf) goto free_buf; buf->len = 0; buf->offset = 0; buf->size = buf_size; return buf; free_buf: kfree(buf);fail: return NULL;} /* Callers should take appropriate locks */static struct port_buffer *get_inbuf(struct port *port){ struct port_buffer *buf; unsigned int len; if (port->inbuf) return port->inbuf; buf = virtqueue_get_buf(port->in_vq, &len); if (buf) { buf->len = min_t(size_t, len, buf->size); buf->offset = 0; port->stats.bytes_received += len; } return buf;} /* * Create a scatter-gather list representing our input buffer and put * it in the queue. * * Callers should take appropriate locks. */static int add_inbuf(struct virtqueue *vq, struct port_buffer *buf){ struct scatterlist sg[1]; int ret; Lines 1–500 of 2364. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2006, 2007, 2009 Rusty Russell, IBM Corporation * Copyright (C) 2009, 2010, 2011 Red Hat, Inc. * Copyright (C) 2009, 2010, 2011 Amit Shah <amit.shah@redhat.com> */ #include <linux/cdev.h> #include <linux/debugfs.h> #include <linux/completion.h> #include <linux/device.h> #include <linux/err.h> #include <linux/freezer.h> #include <linux/fs.h> #include <linux/splice.h> #include <linux/pagemap.h> #include <linux/idr.h> #include <linux/init.h> #include <linux/list.h> #include <linux/poll.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/virtio.h> #include <linux/virtio_console.h> #include <linux/wait.h> #include <linux/workqueue.h> #include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/string_choices.h> #include "../tty/hvc/hvc_console.h" #define is_rproc_enabled IS_ENABLED(CONFIG_REMOTEPROC) #define VIRTCONS_MAX_PORTS 0x8000 /* * This is a global struct for storing common data for all the devices * this driver handles. * * Mainly, it has a linked list for all the consoles in one place so * that callbacks from hvc for get_chars(), put_chars() work properly * across multiple devices and multiple ports per device. */ struct ports_driver_data { /* Used for exporting per-port information to debugfs */ struct dentry *debugfs_dir; /* List of all the devices we're handling */ struct list_head portdevs; /* All the console devices handled by this driver */ struct list_head consoles; }; static struct ports_driver_data pdrvdata; static const struct class port_class = { .name = "virtio-ports", }; static DEFINE_SPINLOCK(pdrvdata_lock); static DECLARE_COMPLETION(early_console_added); /* This struct holds information that's relevant only for console ports */ struct console { /* We'll place all consoles in a list in the pdrvdata struct */ struct list_head list; /* The hvc device associated with this console port */ struct hvc_struct *hvc; /* The size of the console */ struct winsize ws; /* * This number identifies the number that we used to register * with hvc in hvc_instantiate() and hvc_alloc(); this is the * number passed on by the hvc callbacks to us to * differentiate between the other console ports handled by * this driver */ u32 vtermno; }; static DEFINE_IDA(vtermno_ida); struct port_buffer { char *buf; /* size of the buffer in *buf above */ size_t size; /* used length of the buffer */ size_t len; /* offset in the buf from which to consume data */ size_t offset; /* DMA address of buffer */ dma_addr_t dma; /* Device we got DMA memory from */ struct device *dev; /* List of pending dma buffers to free */ struct list_head list; /* If sgpages == 0 then buf is used */ unsigned int sgpages; /* sg is used if spages > 0. sg must be the last in is struct */ struct scatterlist sg[] __counted_by(sgpages); }; /* * This is a per-device struct that stores data common to all the * ports for that device (vdev->priv). */ struct ports_device { /* Next portdev in the list, head is in the pdrvdata struct */ struct list_head list; /* * Workqueue handlers where we process deferred work after * notification */ struct work_struct control_work; struct work_struct config_work; struct list_head ports; /* To protect the list of ports */ spinlock_t ports_lock; /* To protect the vq operations for the control channel */ spinlock_t c_ivq_lock; spinlock_t c_ovq_lock; /* max. number of ports this device can hold */ u32 max_nr_ports; /* The virtio device we're associated with */ struct virtio_device *vdev; /* * A couple of virtqueues for the control channel: one for * guest->host transfers, one for host->guest transfers */ struct virtqueue *c_ivq, *c_ovq; /* * A control packet buffer for guest->host requests, protected * by c_ovq_lock. */ struct virtio_console_control cpkt; /* Array of per-port IO virtqueues */ struct virtqueue **in_vqs, **out_vqs; /* Major number for this device. Ports will be created as minors. */ int chr_major; /* * Set to true during PM freeze to block TX paths that may race * with virtqueue teardown (e.g. hvc put_chars with no_console_suspend). */ bool pm_freezing; }; struct port_stats { unsigned long bytes_sent, bytes_received, bytes_discarded; }; /* This struct holds the per-port data */ struct port { /* Next port in the list, head is in the ports_device */ struct list_head list; /* Pointer to the parent virtio_console device */ struct ports_device *portdev; /* The current buffer from which data has to be fed to readers */ struct port_buffer *inbuf; /* * To protect the operations on the in_vq associated with this * port. Has to be a spinlock because it can be called from * interrupt context (get_char()). */ spinlock_t inbuf_lock; /* Protect the operations on the out_vq. */ spinlock_t outvq_lock; /* The IO vqs for this port */ struct virtqueue *in_vq, *out_vq; /* File in the debugfs directory that exposes this port's information */ struct dentry *debugfs_file; /* * Keep count of the bytes sent, received and discarded for * this port for accounting and debugging purposes. These * counts are not reset across port open / close events. */ struct port_stats stats; /* * The entries in this struct will be valid if this port is * hooked up to an hvc console */ struct console cons; /* Each port associates with a separate char device */ struct cdev *cdev; struct device *dev; /* Reference-counting to handle port hot-unplugs and file operations */ struct kref kref; /* A waitqueue for poll() or blocking read operations */ wait_queue_head_t waitqueue; /* The 'name' of the port that we expose via sysfs properties */ char *name; /* We can notify apps of host connect / disconnect events via SIGIO */ struct fasync_struct *async_queue; /* The 'id' to identify the port with the Host */ u32 id; bool outvq_full; /* Is the host device open */ bool host_connected; /* We should allow only one process to open a port */ bool guest_connected; }; static struct port *find_port_by_vtermno(u32 vtermno) { struct port *port; struct console *cons; unsigned long flags; spin_lock_irqsave(&pdrvdata_lock, flags); list_for_each_entry(cons, &pdrvdata.consoles, list) { if (cons->vtermno == vtermno) { port = container_of(cons, struct port, cons); goto out; } } port = NULL; out: spin_unlock_irqrestore(&pdrvdata_lock, flags); return port; } static struct port *find_port_by_devt_in_portdev(struct ports_device *portdev, dev_t dev) { struct port *port; unsigned long flags; spin_lock_irqsave(&portdev->ports_lock, flags); list_for_each_entry(port, &portdev->ports, list) { if (port->cdev->dev == dev) { kref_get(&port->kref); goto out; } } port = NULL; out: spin_unlock_irqrestore(&portdev->ports_lock, flags); return port; } static struct port *find_port_by_devt(dev_t dev) { struct ports_device *portdev; struct port *port; unsigned long flags; spin_lock_irqsave(&pdrvdata_lock, flags); list_for_each_entry(portdev, &pdrvdata.portdevs, list) { port = find_port_by_devt_in_portdev(portdev, dev); if (port) goto out; } port = NULL; out: spin_unlock_irqrestore(&pdrvdata_lock, flags); return port; } static struct port *find_port_by_id(struct ports_device *portdev, u32 id) { struct port *port; unsigned long flags; spin_lock_irqsave(&portdev->ports_lock, flags); list_for_each_entry(port, &portdev->ports, list) if (port->id == id) goto out; port = NULL; out: spin_unlock_irqrestore(&portdev->ports_lock, flags); return port; } /* * Finds a port by the virtqueue and returns a pointer to struct port * with the reference count incremented. * * Callers MUST decrement it when finished. */ static struct port *find_port_by_vq(struct ports_device *portdev, struct virtqueue *vq) { struct port *port; unsigned long flags; spin_lock_irqsave(&portdev->ports_lock, flags); list_for_each_entry(port, &portdev->ports, list) if (port->in_vq == vq || port->out_vq == vq) { kref_get(&port->kref); goto out; } port = NULL; out: spin_unlock_irqrestore(&portdev->ports_lock, flags); return port; } static bool is_console_port(struct port *port) { if (port->cons.hvc) return true; return false; } static bool is_rproc_serial(const struct virtio_device *vdev) { return is_rproc_enabled && vdev->id.device == VIRTIO_ID_RPROC_SERIAL; } static inline bool use_multiport(struct ports_device *portdev) { /* * This condition can be true when put_chars is called from * early_init */ if (!portdev->vdev) return false; return __virtio_test_bit(portdev->vdev, VIRTIO_CONSOLE_F_MULTIPORT); } static DEFINE_SPINLOCK(dma_bufs_lock); static LIST_HEAD(pending_free_dma_bufs); static void free_buf(struct port_buffer *buf, bool can_sleep) { unsigned int i; for (i = 0; i < buf->sgpages; i++) { struct page *page = sg_page(&buf->sg[i]); if (!page) break; put_page(page); } if (!buf->dev) { kfree(buf->buf); } else if (is_rproc_enabled) { unsigned long flags; /* dma_free_coherent requires interrupts to be enabled. */ if (!can_sleep) { /* queue up dma-buffers to be freed later */ spin_lock_irqsave(&dma_bufs_lock, flags); list_add_tail(&buf->list, &pending_free_dma_bufs); spin_unlock_irqrestore(&dma_bufs_lock, flags); return; } dma_free_coherent(buf->dev, buf->size, buf->buf, buf->dma); /* Release device refcnt and allow it to be freed */ put_device(buf->dev); } kfree(buf); } static void reclaim_dma_bufs(void) { unsigned long flags; struct port_buffer *buf, *tmp; LIST_HEAD(tmp_list); if (list_empty(&pending_free_dma_bufs)) return; /* Create a copy of the pending_free_dma_bufs while holding the lock */ spin_lock_irqsave(&dma_bufs_lock, flags); list_cut_position(&tmp_list, &pending_free_dma_bufs, pending_free_dma_bufs.prev); spin_unlock_irqrestore(&dma_bufs_lock, flags); /* Release the dma buffers, without irqs enabled */ list_for_each_entry_safe(buf, tmp, &tmp_list, list) { list_del(&buf->list); free_buf(buf, true); } } static struct port_buffer *alloc_buf(struct virtio_device *vdev, size_t buf_size, int pages, gfp_t gfp) { struct port_buffer *buf; reclaim_dma_bufs(); /* * Allocate buffer and the sg list. The sg list array is allocated * directly after the port_buffer struct. */ buf = kmalloc_flex(*buf, sg, pages, gfp); if (!buf) goto fail; buf->sgpages = pages; if (pages > 0) { buf->dev = NULL; buf->buf = NULL; return buf; } if (is_rproc_serial(vdev)) { /* * Allocate DMA memory from ancestor. When a virtio * device is created by remoteproc, the DMA memory is * associated with the parent device: * virtioY => remoteprocX#vdevYbuffer. */ buf->dev = vdev->dev.parent; if (!buf->dev) goto free_buf; /* Increase device refcnt to avoid freeing it */ get_device(buf->dev); buf->buf = dma_alloc_coherent(buf->dev, buf_size, &buf->dma, gfp); } else { buf->dev = NULL; buf->buf = kmalloc(buf_size, gfp); } if (!buf->buf) goto free_buf; buf->len = 0; buf->offset = 0; buf->size = buf_size; return buf; free_buf: kfree(buf); fail: return NULL; } /* Callers should take appropriate locks */ static struct port_buffer *get_inbuf(struct port *port) { struct port_buffer *buf; unsigned int len; if (port->inbuf) return port->inbuf; buf = virtqueue_get_buf(port->in_vq, &len); if (buf) { buf->len = min_t(size_t, len, buf->size); buf->offset = 0; port->stats.bytes_received += len; } return buf; } /* * Create a scatter-gather list representing our input buffer and put * it in the queue. * * Callers should take appropriate locks. */ static int add_inbuf(struct virtqueue *vq, struct port_buffer *buf) { struct scatterlist sg[1]; int ret;