// SPDX-License-Identifier: GPL-2.0/* * Stream data over Thunderbolt/USB4 cable * * Copyright (C) 2026, Intel Corporation * Authors: Alan Borzeszkowski <alan.borzeszkowski@linux.intel.com> * Mika Westerberg <mika.westerberg@linux.intel.com> */ #define pr_fmt(fmt) "tbstream: " fmt #include <linux/delay.h>#include <linux/configfs.h>#include <linux/file.h>#include <linux/fs.h>#include <linux/idr.h>#include <linux/ktime.h>#include <linux/miscdevice.h>#include <linux/module.h>#include <linux/mutex.h>#include <linux/poll.h>#include <linux/sizes.h>#include <linux/thunderbolt.h>#include <linux/uaccess.h>#include <linux/uio.h>#include <linux/uuid.h>#include <linux/wait.h> /* * USB4STREAM - Stream data directly over Thunderbolt/USB4 cable * * HopIDs are configured by the user. In Linux this is done through * ConfigFS. Once that is done paths are be established the first time * the stream is opened. Typically the read side is opened first to make * sure all the data will be received. * * End-to-end flow control is mandatory on both sides. * * Data is sent to the other side as tunneled DATA packets. All the data * is owned by the user and passed as-is from the writer to the reader. * * Once the stream device is closed, a CLOSE packet is sent to the peer * so it can take the necessary action. On Linux this typically results * in EOF being returned to the reader. * * Tunneled packet types: * * +-------+---------+------------------+ * | PDF | Type | Payload size | * +-------+---------+------------------+ * | 2 | DATA | up to 4 KiB | * | 3 | CLOSE | up to 256 bytes | * +-------+---------+------------------+ * * Each stream can optionally publish configuration values under its own * XDomain property directory. The name of the directory is the name of * the stream in question and the UUID is up to the stream. For example * if the stream exposes video output then the directory name could be * "video". * * Below values are reserved and can be used by the stream: * * +----------+-----------+-------------------------+ * | Key | Type | Contents | * +----------+-----------+-------------------------+ * | inhopid | IMMEDIATE | Configured input HopID | * | outhopid | IMMEDIATE | Configured output HopID | * +----------+-----------+-------------------------+ * * It is allowed to add more stream specific properties as well if the * above are not enough. */ #define TBSTREAM_DEV_RING_SIZE 256#define TBSTREAM_DEV_MIN_RING_SIZE 32#define TBSTREAM_DEV_MAX_RING_SIZE 4096#define TBSTREAM_DEV_THROTTLING 8192#define TBSTREAM_DEV_MAX_THROTTLING 16776960 /** * enum tbstream_frame_pdf - PDF numbers for tunneled frames * @TBSTREAM_FRAME_START: PDF of the start of the frame * @TBSTREAM_DATA: PDF of the DATA frame * @TBSTREAM_CLOSE: PDF of the CLOSE frame */enum tbstream_frame_pdf { TBSTREAM_FRAME_START = 1, TBSTREAM_DATA, TBSTREAM_CLOSE,}; /** * struct tbstream_frame - Frame submitted to/from the rings * @sdev: Pointer to the stream device * @page: Page holding the packet * @offset: Offset inside @page if partial read is done * @completed: %true if the RX frame is completed * @frame: Underlying frame structure */struct tbstream_frame { struct tbstream_dev *sdev; struct page *page; unsigned int offset; bool completed; struct ring_frame frame;}; /** * struct tbstream_ring - Stream RX/TX ring structure * @ring: Pointer to the API ring * @prod: Current value of producer * @cons: Current value of consumer * @frames: Holds the ring frames */struct tbstream_ring { struct tb_ring *ring; unsigned long prod; unsigned long cons; struct tbstream_frame *frames;}; /** * struct tbstream_dev - Stream character device * @group: ConfigFS group for this device * @stream: Pointer to the stream if it is attached (%NULL otherwise) * @misc: Character device used for tunneling * @kref: Reference count * @index: Unique identifier for the character device * @in_hopid: In HopID * @out_hopid: Out HopID * @ring_size: Size of the rings * @throttling: Interrupt throttling rate in ns * @busy_poll: Instead of interrupts, busy poll the rings * @users: Number of times @cdev has been opened * @closed: CLOSE packet was received * @removed: Userspace removed the ConfigFS group underneath. * @wait: Waitqueue for open, read and write * @lock: Lock protecting this structure * @tx_ring: Transmit ring * @rx_ring: Receive ring * @list: Stream devices are linked through this */struct tbstream_dev { struct config_group group; struct tbstream *stream; struct miscdevice misc; struct kref kref; int index; int in_hopid; int out_hopid; unsigned int ring_size; unsigned int throttling; bool busy_poll; int users; bool closed; bool removed; wait_queue_head_t wait; struct mutex lock; struct tbstream_ring tx_ring; struct tbstream_ring rx_ring; struct list_head list;}; /** * struct tbstream_group - Config group for stream * @group: ConfigFS group for @stream * @stream: Stream the ConfigFS group is attached to. %NULL if there is * no stream attached. * @lock: Lock protecting this structure * @dev_list: List of stream devices * * This is the ConfigFS directory for one connection to another host. * There can be several &struct stream_dev linked through @dev_list of * this structure. Reference count managed through @group. */struct tbstream_group { struct config_group group; struct tbstream *stream; struct mutex lock; struct list_head dev_list;}; /** * struct tbstream - Stream service private data * @kref: Reference count * @svc: Pointer to the service device * @list: Streams are linked through this in @stream_list * * This represents the actual physical connection between two hosts. */struct tbstream { struct kref kref; struct tb_service *svc; struct list_head list;}; static DEFINE_IDA(tbstream_indices); /* Protects tbstream_list */static DEFINE_MUTEX(tbstream_lock);static LIST_HEAD(tbstream_list); /* Serializes tbstream_get()/put() */static DEFINE_MUTEX(tbstream_kref_lock); /* Serializes tbstream_dev_get()/put() */static DEFINE_MUTEX(tbstream_dev_kref_lock); /* Stream property directory UUID: 3a1cb984-c4d9-4469-a277-ce2fdfd11f0d */static const uuid_t tbstream_dir_uuid = UUID_INIT(0x3a1cb984, 0xc4d9, 0x4469, 0xa2, 0x77, 0xce, 0x2f, 0xdf, 0xd1, 0x1f, 0x0d); static struct tb_property_dir *tbstream_dir; static void tbstream_release(struct kref *kref){ struct tbstream *stream = container_of(kref, typeof(*stream), kref); tb_service_put(stream->svc); kfree(stream);} static void tbstream_put(struct tbstream *stream){ if (stream) { guard(mutex)(&tbstream_kref_lock); kref_put(&stream->kref, tbstream_release); }} static struct tbstream *tbstream_get(struct tbstream *stream){ if (stream) { guard(mutex)(&tbstream_kref_lock); kref_get(&stream->kref); } return stream;} static inline bool tbstream_valid(const struct tbstream *stream){ if (stream) return !tb_service_parent(stream->svc)->is_unplugged; return false;} static void tbstream_ring_free(struct tbstream_ring *ring){ struct device *dma_dev = tb_ring_dma_device(ring->ring); enum dma_data_direction dir; int i; if (ring->ring->is_tx) dir = DMA_TO_DEVICE; else dir = DMA_FROM_DEVICE; for (i = 0; i < tb_ring_size(ring->ring); i++) { struct tbstream_frame *sf = &ring->frames[i]; if (sf->frame.buffer_phy) dma_unmap_page(dma_dev, sf->frame.buffer_phy, TB_MAX_FRAME_SIZE, dir); sf->frame.buffer_phy = 0; if (sf->page) __free_page(sf->page); sf->page = NULL; } ring->prod = 0; ring->cons = 0; kfree(ring->frames);} static inline bool tbstream_ring_available(const struct tbstream_ring *ring){ return ring->prod > ring->cons;} static inline struct tb_xdomain *tbstream_dev_xdomain(struct tbstream_dev *sdev){ if (sdev->stream) return tb_service_parent(sdev->stream->svc); return NULL;} static void tbstream_dev_release(struct kref *kref){ struct tbstream_dev *sdev = container_of(kref, struct tbstream_dev, kref); if (sdev->stream) { struct tb_xdomain *xd = tbstream_dev_xdomain(sdev); if (sdev->out_hopid > 0) tb_xdomain_release_out_hopid(xd, sdev->out_hopid); if (sdev->in_hopid > 0) tb_xdomain_release_in_hopid(xd, sdev->in_hopid); tbstream_put(sdev->stream); } ida_free(&tbstream_indices, sdev->index); kfree(sdev->misc.name); kfree(sdev);} static inline void tbstream_dev_put(struct tbstream_dev *sdev){ guard(mutex)(&tbstream_dev_kref_lock); kref_put(&sdev->kref, tbstream_dev_release);} static inline struct tbstream_dev *tbstream_dev_get(struct tbstream_dev *sdev){ guard(mutex)(&tbstream_dev_kref_lock); kref_get(&sdev->kref); return sdev;} static inline struct tbstream_dev *to_tbstream_dev(struct miscdevice *misc){ return container_of(misc, struct tbstream_dev, misc);} static inline int tbstream_dev_valid(const struct tbstream_dev *sdev){ const struct tbstream *stream = sdev->stream; if (!tbstream_valid(stream)) return -ENXIO; if (sdev->in_hopid <= 0 || sdev->out_hopid <= 0) return -EINVAL; return 0;} static inline bool tbstream_dev_removed(const struct tbstream_dev *sdev){ return sdev->removed;} static inline bool tbstream_dev_closed(const struct tbstream_dev *sdev){ return sdev->closed;} static voidtbstream_dev_rx_callback(struct tb_ring *ring, struct ring_frame *frame, bool canceled){ struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame); struct tbstream_dev *sdev = sf->sdev; if (canceled) return; sf->completed = true; sdev->rx_ring.prod++; if (sf->frame.flags & RING_DESC_CRC_ERROR) pr_warn("RX CRC error\n"); else if (sf->frame.flags & RING_DESC_BUFFER_OVERRUN) pr_warn("RX buffer overrun\n"); else wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM);} static struct tbstream_frame *tbstream_dev_completed_rx(struct tbstream_dev *sdev){ struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring); struct tbstream_frame *sf; int index; index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring); sf = &sdev->rx_ring.frames[index]; if (!sf->completed) return NULL; dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy, tb_ring_frame_size(&sf->frame), DMA_FROM_DEVICE); return sf;} static int tbstream_dev_consume_rx(struct tbstream_dev *sdev){ struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring); struct tbstream_frame *sf; int index; index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring); sdev->rx_ring.cons++; sf = &sdev->rx_ring.frames[index]; sf->completed = false; sf->offset = 0; sf->frame.size = 0; dma_sync_single_for_device(dma_dev, sf->frame.buffer_phy, tb_ring_frame_size(&sf->frame), DMA_FROM_DEVICE); return tb_ring_rx(sdev->rx_ring.ring, &sf->frame);} static int tbstream_dev_alloc_rx_buffers(struct tbstream_dev *sdev){ size_t ring_size = tb_ring_size(sdev->rx_ring.ring); int i; sdev->rx_ring.frames = kzalloc_objs(struct tbstream_frame, ring_size); if (!sdev->rx_ring.frames) return -ENOMEM; for (i = 0; i < ring_size; i++) { struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring); struct tbstream_frame *sf = &sdev->rx_ring.frames[i]; dma_addr_t dma_addr; sf->page = alloc_page(GFP_KERNEL); if (!sf->page) return -ENOMEM; dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE, DMA_FROM_DEVICE); if (dma_mapping_error(dma_dev, dma_addr)) { __free_page(sf->page); sf->page = NULL; return -ENOMEM; } sf->sdev = sdev; sf->frame.callback = tbstream_dev_rx_callback; sf->frame.buffer_phy = dma_addr; tb_ring_rx(sdev->rx_ring.ring, &sf->frame); } sdev->rx_ring.cons = 0; sdev->rx_ring.prod = 0; return 0;} static voidtbstream_dev_tx_callback(struct tb_ring *ring, struct ring_frame *frame, bool canceled){ struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame); struct tbstream_dev *sdev = sf->sdev; if (canceled) return; sdev->tx_ring.prod++; if (sf->frame.eof == TBSTREAM_DATA) wake_up_interruptible_poll(&sdev->wait, EPOLLOUT | EPOLLWRNORM);} static int tbstream_dev_alloc_tx_buffers(struct tbstream_dev *sdev){ struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring); size_t ring_size = tb_ring_size(sdev->tx_ring.ring); int i; sdev->tx_ring.frames = kzalloc_objs(struct tbstream_frame, ring_size); if (!sdev->tx_ring.frames) return -ENOMEM; for (i = 0; i < ring_size; i++) { struct tbstream_frame *sf = &sdev->tx_ring.frames[i]; dma_addr_t dma_addr; sf->page = alloc_page(GFP_KERNEL); if (!sf->page) return -ENOMEM; dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dma_dev, dma_addr)) { __free_page(sf->page); sf->page = NULL; return -ENOMEM; } sf->sdev = sdev; sf->frame.callback = tbstream_dev_tx_callback; sf->frame.buffer_phy = dma_addr; sf->frame.sof = TBSTREAM_FRAME_START; } sdev->tx_ring.cons = 0; sdev->tx_ring.prod = ring_size - 1; return 0;} static struct tbstream_frame *tbstream_dev_alloc_tx(struct tbstream_dev *sdev, enum tbstream_frame_pdf pdf, struct iov_iter *from, size_t size){ struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring);Lines 1–500 of 1850. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Stream data over Thunderbolt/USB4 cable * * Copyright (C) 2026, Intel Corporation * Authors: Alan Borzeszkowski <alan.borzeszkowski@linux.intel.com> * Mika Westerberg <mika.westerberg@linux.intel.com> */ #define pr_fmt(fmt) "tbstream: " fmt #include <linux/delay.h> #include <linux/configfs.h> #include <linux/file.h> #include <linux/fs.h> #include <linux/idr.h> #include <linux/ktime.h> #include <linux/miscdevice.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/poll.h> #include <linux/sizes.h> #include <linux/thunderbolt.h> #include <linux/uaccess.h> #include <linux/uio.h> #include <linux/uuid.h> #include <linux/wait.h> /* * USB4STREAM - Stream data directly over Thunderbolt/USB4 cable * * HopIDs are configured by the user. In Linux this is done through * ConfigFS. Once that is done paths are be established the first time * the stream is opened. Typically the read side is opened first to make * sure all the data will be received. * * End-to-end flow control is mandatory on both sides. * * Data is sent to the other side as tunneled DATA packets. All the data * is owned by the user and passed as-is from the writer to the reader. * * Once the stream device is closed, a CLOSE packet is sent to the peer * so it can take the necessary action. On Linux this typically results * in EOF being returned to the reader. * * Tunneled packet types: * * +-------+---------+------------------+ * | PDF | Type | Payload size | * +-------+---------+------------------+ * | 2 | DATA | up to 4 KiB | * | 3 | CLOSE | up to 256 bytes | * +-------+---------+------------------+ * * Each stream can optionally publish configuration values under its own * XDomain property directory. The name of the directory is the name of * the stream in question and the UUID is up to the stream. For example * if the stream exposes video output then the directory name could be * "video". * * Below values are reserved and can be used by the stream: * * +----------+-----------+-------------------------+ * | Key | Type | Contents | * +----------+-----------+-------------------------+ * | inhopid | IMMEDIATE | Configured input HopID | * | outhopid | IMMEDIATE | Configured output HopID | * +----------+-----------+-------------------------+ * * It is allowed to add more stream specific properties as well if the * above are not enough. */ #define TBSTREAM_DEV_RING_SIZE 256 #define TBSTREAM_DEV_MIN_RING_SIZE 32 #define TBSTREAM_DEV_MAX_RING_SIZE 4096 #define TBSTREAM_DEV_THROTTLING 8192 #define TBSTREAM_DEV_MAX_THROTTLING 16776960 /** * enum tbstream_frame_pdf - PDF numbers for tunneled frames * @TBSTREAM_FRAME_START: PDF of the start of the frame * @TBSTREAM_DATA: PDF of the DATA frame * @TBSTREAM_CLOSE: PDF of the CLOSE frame */ enum tbstream_frame_pdf { TBSTREAM_FRAME_START = 1, TBSTREAM_DATA, TBSTREAM_CLOSE, }; /** * struct tbstream_frame - Frame submitted to/from the rings * @sdev: Pointer to the stream device * @page: Page holding the packet * @offset: Offset inside @page if partial read is done * @completed: %true if the RX frame is completed * @frame: Underlying frame structure */ struct tbstream_frame { struct tbstream_dev *sdev; struct page *page; unsigned int offset; bool completed; struct ring_frame frame; }; /** * struct tbstream_ring - Stream RX/TX ring structure * @ring: Pointer to the API ring * @prod: Current value of producer * @cons: Current value of consumer * @frames: Holds the ring frames */ struct tbstream_ring { struct tb_ring *ring; unsigned long prod; unsigned long cons; struct tbstream_frame *frames; }; /** * struct tbstream_dev - Stream character device * @group: ConfigFS group for this device * @stream: Pointer to the stream if it is attached (%NULL otherwise) * @misc: Character device used for tunneling * @kref: Reference count * @index: Unique identifier for the character device * @in_hopid: In HopID * @out_hopid: Out HopID * @ring_size: Size of the rings * @throttling: Interrupt throttling rate in ns * @busy_poll: Instead of interrupts, busy poll the rings * @users: Number of times @cdev has been opened * @closed: CLOSE packet was received * @removed: Userspace removed the ConfigFS group underneath. * @wait: Waitqueue for open, read and write * @lock: Lock protecting this structure * @tx_ring: Transmit ring * @rx_ring: Receive ring * @list: Stream devices are linked through this */ struct tbstream_dev { struct config_group group; struct tbstream *stream; struct miscdevice misc; struct kref kref; int index; int in_hopid; int out_hopid; unsigned int ring_size; unsigned int throttling; bool busy_poll; int users; bool closed; bool removed; wait_queue_head_t wait; struct mutex lock; struct tbstream_ring tx_ring; struct tbstream_ring rx_ring; struct list_head list; }; /** * struct tbstream_group - Config group for stream * @group: ConfigFS group for @stream * @stream: Stream the ConfigFS group is attached to. %NULL if there is * no stream attached. * @lock: Lock protecting this structure * @dev_list: List of stream devices * * This is the ConfigFS directory for one connection to another host. * There can be several &struct stream_dev linked through @dev_list of * this structure. Reference count managed through @group. */ struct tbstream_group { struct config_group group; struct tbstream *stream; struct mutex lock; struct list_head dev_list; }; /** * struct tbstream - Stream service private data * @kref: Reference count * @svc: Pointer to the service device * @list: Streams are linked through this in @stream_list * * This represents the actual physical connection between two hosts. */ struct tbstream { struct kref kref; struct tb_service *svc; struct list_head list; }; static DEFINE_IDA(tbstream_indices); /* Protects tbstream_list */ static DEFINE_MUTEX(tbstream_lock); static LIST_HEAD(tbstream_list); /* Serializes tbstream_get()/put() */ static DEFINE_MUTEX(tbstream_kref_lock); /* Serializes tbstream_dev_get()/put() */ static DEFINE_MUTEX(tbstream_dev_kref_lock); /* Stream property directory UUID: 3a1cb984-c4d9-4469-a277-ce2fdfd11f0d */ static const uuid_t tbstream_dir_uuid = UUID_INIT(0x3a1cb984, 0xc4d9, 0x4469, 0xa2, 0x77, 0xce, 0x2f, 0xdf, 0xd1, 0x1f, 0x0d); static struct tb_property_dir *tbstream_dir; static void tbstream_release(struct kref *kref) { struct tbstream *stream = container_of(kref, typeof(*stream), kref); tb_service_put(stream->svc); kfree(stream); } static void tbstream_put(struct tbstream *stream) { if (stream) { guard(mutex)(&tbstream_kref_lock); kref_put(&stream->kref, tbstream_release); } } static struct tbstream *tbstream_get(struct tbstream *stream) { if (stream) { guard(mutex)(&tbstream_kref_lock); kref_get(&stream->kref); } return stream; } static inline bool tbstream_valid(const struct tbstream *stream) { if (stream) return !tb_service_parent(stream->svc)->is_unplugged; return false; } static void tbstream_ring_free(struct tbstream_ring *ring) { struct device *dma_dev = tb_ring_dma_device(ring->ring); enum dma_data_direction dir; int i; if (ring->ring->is_tx) dir = DMA_TO_DEVICE; else dir = DMA_FROM_DEVICE; for (i = 0; i < tb_ring_size(ring->ring); i++) { struct tbstream_frame *sf = &ring->frames[i]; if (sf->frame.buffer_phy) dma_unmap_page(dma_dev, sf->frame.buffer_phy, TB_MAX_FRAME_SIZE, dir); sf->frame.buffer_phy = 0; if (sf->page) __free_page(sf->page); sf->page = NULL; } ring->prod = 0; ring->cons = 0; kfree(ring->frames); } static inline bool tbstream_ring_available(const struct tbstream_ring *ring) { return ring->prod > ring->cons; } static inline struct tb_xdomain *tbstream_dev_xdomain(struct tbstream_dev *sdev) { if (sdev->stream) return tb_service_parent(sdev->stream->svc); return NULL; } static void tbstream_dev_release(struct kref *kref) { struct tbstream_dev *sdev = container_of(kref, struct tbstream_dev, kref); if (sdev->stream) { struct tb_xdomain *xd = tbstream_dev_xdomain(sdev); if (sdev->out_hopid > 0) tb_xdomain_release_out_hopid(xd, sdev->out_hopid); if (sdev->in_hopid > 0) tb_xdomain_release_in_hopid(xd, sdev->in_hopid); tbstream_put(sdev->stream); } ida_free(&tbstream_indices, sdev->index); kfree(sdev->misc.name); kfree(sdev); } static inline void tbstream_dev_put(struct tbstream_dev *sdev) { guard(mutex)(&tbstream_dev_kref_lock); kref_put(&sdev->kref, tbstream_dev_release); } static inline struct tbstream_dev *tbstream_dev_get(struct tbstream_dev *sdev) { guard(mutex)(&tbstream_dev_kref_lock); kref_get(&sdev->kref); return sdev; } static inline struct tbstream_dev *to_tbstream_dev(struct miscdevice *misc) { return container_of(misc, struct tbstream_dev, misc); } static inline int tbstream_dev_valid(const struct tbstream_dev *sdev) { const struct tbstream *stream = sdev->stream; if (!tbstream_valid(stream)) return -ENXIO; if (sdev->in_hopid <= 0 || sdev->out_hopid <= 0) return -EINVAL; return 0; } static inline bool tbstream_dev_removed(const struct tbstream_dev *sdev) { return sdev->removed; } static inline bool tbstream_dev_closed(const struct tbstream_dev *sdev) { return sdev->closed; } static void tbstream_dev_rx_callback(struct tb_ring *ring, struct ring_frame *frame, bool canceled) { struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame); struct tbstream_dev *sdev = sf->sdev; if (canceled) return; sf->completed = true; sdev->rx_ring.prod++; if (sf->frame.flags & RING_DESC_CRC_ERROR) pr_warn("RX CRC error\n"); else if (sf->frame.flags & RING_DESC_BUFFER_OVERRUN) pr_warn("RX buffer overrun\n"); else wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM); } static struct tbstream_frame * tbstream_dev_completed_rx(struct tbstream_dev *sdev) { struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring); struct tbstream_frame *sf; int index; index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring); sf = &sdev->rx_ring.frames[index]; if (!sf->completed) return NULL; dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy, tb_ring_frame_size(&sf->frame), DMA_FROM_DEVICE); return sf; } static int tbstream_dev_consume_rx(struct tbstream_dev *sdev) { struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring); struct tbstream_frame *sf; int index; index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring); sdev->rx_ring.cons++; sf = &sdev->rx_ring.frames[index]; sf->completed = false; sf->offset = 0; sf->frame.size = 0; dma_sync_single_for_device(dma_dev, sf->frame.buffer_phy, tb_ring_frame_size(&sf->frame), DMA_FROM_DEVICE); return tb_ring_rx(sdev->rx_ring.ring, &sf->frame); } static int tbstream_dev_alloc_rx_buffers(struct tbstream_dev *sdev) { size_t ring_size = tb_ring_size(sdev->rx_ring.ring); int i; sdev->rx_ring.frames = kzalloc_objs(struct tbstream_frame, ring_size); if (!sdev->rx_ring.frames) return -ENOMEM; for (i = 0; i < ring_size; i++) { struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring); struct tbstream_frame *sf = &sdev->rx_ring.frames[i]; dma_addr_t dma_addr; sf->page = alloc_page(GFP_KERNEL); if (!sf->page) return -ENOMEM; dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE, DMA_FROM_DEVICE); if (dma_mapping_error(dma_dev, dma_addr)) { __free_page(sf->page); sf->page = NULL; return -ENOMEM; } sf->sdev = sdev; sf->frame.callback = tbstream_dev_rx_callback; sf->frame.buffer_phy = dma_addr; tb_ring_rx(sdev->rx_ring.ring, &sf->frame); } sdev->rx_ring.cons = 0; sdev->rx_ring.prod = 0; return 0; } static void tbstream_dev_tx_callback(struct tb_ring *ring, struct ring_frame *frame, bool canceled) { struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame); struct tbstream_dev *sdev = sf->sdev; if (canceled) return; sdev->tx_ring.prod++; if (sf->frame.eof == TBSTREAM_DATA) wake_up_interruptible_poll(&sdev->wait, EPOLLOUT | EPOLLWRNORM); } static int tbstream_dev_alloc_tx_buffers(struct tbstream_dev *sdev) { struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring); size_t ring_size = tb_ring_size(sdev->tx_ring.ring); int i; sdev->tx_ring.frames = kzalloc_objs(struct tbstream_frame, ring_size); if (!sdev->tx_ring.frames) return -ENOMEM; for (i = 0; i < ring_size; i++) { struct tbstream_frame *sf = &sdev->tx_ring.frames[i]; dma_addr_t dma_addr; sf->page = alloc_page(GFP_KERNEL); if (!sf->page) return -ENOMEM; dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dma_dev, dma_addr)) { __free_page(sf->page); sf->page = NULL; return -ENOMEM; } sf->sdev = sdev; sf->frame.callback = tbstream_dev_tx_callback; sf->frame.buffer_phy = dma_addr; sf->frame.sof = TBSTREAM_FRAME_START; } sdev->tx_ring.cons = 0; sdev->tx_ring.prod = ring_size - 1; return 0; } static struct tbstream_frame * tbstream_dev_alloc_tx(struct tbstream_dev *sdev, enum tbstream_frame_pdf pdf, struct iov_iter *from, size_t size) { struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring);