History

USB: Add support for usbfs zerocopy.

Add a new interface for userspace to preallocate memory that can be used with usbfs. This gives two primary benefits: - Zerocopy; data no longer needs to be copied between the userspace and the kernel, but can instead be read directly by the driver from userspace's buffers. This works for all kinds of transfers (even if nonsensical for control and interrupt transfers); isochronous also no longer need to memset() the buffer to zero to avoid leaking kernel data. - Once the buffers are allocated, USB transfers can no longer fail due to memory fragmentation; previously, long-running programs could run into problems finding a large enough contiguous memory chunk, especially on embedded systems or at high rates. Memory is allocated by using mmap() against the usbfs file descriptor, and similarly deallocated by munmap(). Once memory has been allocated, using it as pointers to a bulk or isochronous operation means you will automatically get zerocopy behavior. Note that this also means you cannot modify outgoing data until the transfer is complete. The same holds for data on the same cache lines as incoming data; DMA modifying them at the same time could lead to your changes being overwritten. There's a new capability USBDEVFS_CAP_MMAP that userspace can query to see if the running kernel supports this functionality, if just trying mmap() is not acceptable. Largely based on a patch by Markus Rechberger with some updates. The original patch can be found at: http://sundtek.de/support/devio_mmap_v0.4.diff

Steinar H. Gunderson committed 10 years ago

Commit f7d34b41 parent 4a6d9f82 files · +203 −25Browse files

Unified · Split

2 files
  1. drivers/usb/core/devio.c +202 −25
  2. include/uapi/linux/usbdevice_fs.h +1 −0
drivers/usb/core/devio.c+202−25
@@ -50,6 +50,7 @@ · 49 unmodified lines
50#include <linux/user_namespace.h>50#include <linux/user_namespace.h>
51#include <linux/scatterlist.h>51#include <linux/scatterlist.h>
52#include <linux/uaccess.h>52#include <linux/uaccess.h>
53#include <linux/dma-mapping.h>
53#include <asm/byteorder.h>54#include <asm/byteorder.h>
54#include <linux/moduleparam.h>55#include <linux/moduleparam.h>
55 56
@@ -69,6 +70,7 @@ struct usb_dev_state { · 13 unmodified lines
69 spinlock_t lock; /* protects the async urb lists */70 spinlock_t lock; /* protects the async urb lists */
70 struct list_head async_pending;71 struct list_head async_pending;
71 struct list_head async_completed;72 struct list_head async_completed;
73 struct list_head memory_list;
72 wait_queue_head_t wait; /* wake up if a request completed */74 wait_queue_head_t wait; /* wake up if a request completed */
73 unsigned int discsignr;75 unsigned int discsignr;
74 struct pid *disc_pid;76 struct pid *disc_pid;
@@ -79,6 +81,17 @@ struct usb_dev_state { · 4 unmodified lines
79 u32 disabled_bulk_eps;81 u32 disabled_bulk_eps;
80};82};
81 83
84struct usb_memory {
85 struct list_head memlist;
86 int vma_use_count;
87 int urb_use_count;
88 u32 size;
89 void *mem;
90 dma_addr_t dma_handle;
91 unsigned long vm_start;
92 struct usb_dev_state *ps;
93};
94
82struct async {95struct async {
83 struct list_head asynclist;96 struct list_head asynclist;
84 struct usb_dev_state *ps;97 struct usb_dev_state *ps;
@@ -89,6 +102,7 @@ struct async { · 4 unmodified lines
89 void __user *userbuffer;102 void __user *userbuffer;
90 void __user *userurb;103 void __user *userurb;
91 struct urb *urb;104 struct urb *urb;
105 struct usb_memory *usbm;
92 unsigned int mem_usage;106 unsigned int mem_usage;
93 int status;107 int status;
94 u32 secid;108 u32 secid;
@@ -162,6 +176,111 @@ static int connected(struct usb_dev_state *ps) · 67 unmodified lines
162 ps->dev->state != USB_STATE_NOTATTACHED);176 ps->dev->state != USB_STATE_NOTATTACHED);
163}177}
164 178
179static void dec_usb_memory_use_count(struct usb_memory *usbm, int *count)
180{
181 struct usb_dev_state *ps = usbm->ps;
182 unsigned long flags;
183
184 spin_lock_irqsave(&ps->lock, flags);
185 --*count;
186 if (usbm->urb_use_count == 0 && usbm->vma_use_count == 0) {
187 list_del(&usbm->memlist);
188 spin_unlock_irqrestore(&ps->lock, flags);
189
190 usb_free_coherent(ps->dev, usbm->size, usbm->mem,
191 usbm->dma_handle);
192 usbfs_decrease_memory_usage(
193 usbm->size + sizeof(struct usb_memory));
194 kfree(usbm);
195 } else {
196 spin_unlock_irqrestore(&ps->lock, flags);
197 }
198}
199
200static void usbdev_vm_open(struct vm_area_struct *vma)
201{
202 struct usb_memory *usbm = vma->vm_private_data;
203 unsigned long flags;
204
205 spin_lock_irqsave(&usbm->ps->lock, flags);
206 ++usbm->vma_use_count;
207 spin_unlock_irqrestore(&usbm->ps->lock, flags);
208}
209
210static void usbdev_vm_close(struct vm_area_struct *vma)
211{
212 struct usb_memory *usbm = vma->vm_private_data;
213
214 dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
215}
216
217struct vm_operations_struct usbdev_vm_ops = {
218 .open = usbdev_vm_open,
219 .close = usbdev_vm_close
220};
221
222static int usbdev_mmap(struct file *file, struct vm_area_struct *vma)
223{
224 struct usb_memory *usbm = NULL;
225 struct usb_dev_state *ps = file->private_data;
226 size_t size = vma->vm_end - vma->vm_start;
227 void *mem;
228 unsigned long flags;
229 dma_addr_t dma_handle;
230 int ret;
231
232 ret = usbfs_increase_memory_usage(size + sizeof(struct usb_memory));
233 if (ret)
234 goto error;
235
236 usbm = kzalloc(sizeof(struct usb_memory), GFP_KERNEL);
237 if (!usbm) {
238 ret = -ENOMEM;
239 goto error_decrease_mem;
240 }
241
242 mem = usb_alloc_coherent(ps->dev, size, GFP_USER, &dma_handle);
243 if (!mem) {
244 ret = -ENOMEM;
245 goto error_free_usbm;
246 }
247
248 memset(mem, 0, size);
249
250 usbm->mem = mem;
251 usbm->dma_handle = dma_handle;
252 usbm->size = size;
253 usbm->ps = ps;
254 usbm->vm_start = vma->vm_start;
255 usbm->vma_use_count = 1;
256 INIT_LIST_HEAD(&usbm->memlist);
257
258 if (remap_pfn_range(vma, vma->vm_start,
259 virt_to_phys(usbm->mem) >> PAGE_SHIFT,
260 size, vma->vm_page_prot) < 0) {
261 dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
262 return -EAGAIN;
263 }
264
265 vma->vm_flags |= VM_IO;
266 vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
267 vma->vm_ops = &usbdev_vm_ops;
268 vma->vm_private_data = usbm;
269
270 spin_lock_irqsave(&ps->lock, flags);
271 list_add_tail(&usbm->memlist, &ps->memory_list);
272 spin_unlock_irqrestore(&ps->lock, flags);
273
274 return 0;
275
276error_free_usbm:
277 kfree(usbm);
278error_decrease_mem:
279 usbfs_decrease_memory_usage(size + sizeof(struct usb_memory));
280error:
281 return ret;
282}
283
165static ssize_t usbdev_read(struct file *file, char __user *buf, size_t nbytes,284static ssize_t usbdev_read(struct file *file, char __user *buf, size_t nbytes,
166 loff_t *ppos)285 loff_t *ppos)
167{286{
@@ -278,8 +397,13 @@ static void free_async(struct async *as) · 110 unmodified lines
278 if (sg_page(&as->urb->sg[i]))397 if (sg_page(&as->urb->sg[i]))
279 kfree(sg_virt(&as->urb->sg[i]));398 kfree(sg_virt(&as->urb->sg[i]));
280 }399 }
400
281 kfree(as->urb->sg);401 kfree(as->urb->sg);
282 kfree(as->urb->transfer_buffer);402 if (as->usbm == NULL)
403 kfree(as->urb->transfer_buffer);
404 else
405 dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
406
283 kfree(as->urb->setup_packet);407 kfree(as->urb->setup_packet);
284 usb_free_urb(as->urb);408 usb_free_urb(as->urb);
285 usbfs_decrease_memory_usage(as->mem_usage);409 usbfs_decrease_memory_usage(as->mem_usage);
@@ -893,6 +1017,7 @@ static int usbdev_open(struct inode *inode, struct file *file) · 607 unmodified lines
893 INIT_LIST_HEAD(&ps->list);1017 INIT_LIST_HEAD(&ps->list);
894 INIT_LIST_HEAD(&ps->async_pending);1018 INIT_LIST_HEAD(&ps->async_pending);
895 INIT_LIST_HEAD(&ps->async_completed);1019 INIT_LIST_HEAD(&ps->async_completed);
1020 INIT_LIST_HEAD(&ps->memory_list);
896 init_waitqueue_head(&ps->wait);1021 init_waitqueue_head(&ps->wait);
897 ps->discsignr = 0;1022 ps->discsignr = 0;
898 ps->disc_pid = get_pid(task_pid(current));1023 ps->disc_pid = get_pid(task_pid(current));
@@ -945,6 +1070,7 @@ static int usbdev_release(struct inode *inode, struct file *file) · 46 unmodified lines
945 free_async(as);1070 free_async(as);
946 as = async_getcompleted(ps);1071 as = async_getcompleted(ps);
947 }1072 }
1073
948 kfree(ps);1074 kfree(ps);
949 return 0;1075 return 0;
950}1076}
@@ -1266,6 +1392,31 @@ static int proc_setconfig(struct usb_dev_state *ps, void __user *arg) · 315 unmodified lines
1266 return status;1392 return status;
1267}1393}
1268 1394
1395static struct usb_memory *
1396find_memory_area(struct usb_dev_state *ps, const struct usbdevfs_urb *uurb)
1397{
1398 struct usb_memory *usbm = NULL, *iter;
1399 unsigned long flags;
1400 unsigned long uurb_start = (unsigned long)uurb->buffer;
1401
1402 spin_lock_irqsave(&ps->lock, flags);
1403 list_for_each_entry(iter, &ps->memory_list, memlist) {
1404 if (uurb_start >= iter->vm_start &&
1405 uurb_start < iter->vm_start + iter->size) {
1406 if (uurb->buffer_length > iter->vm_start + iter->size -
1407 uurb_start) {
1408 usbm = ERR_PTR(-EINVAL);
1409 } else {
1410 usbm = iter;
1411 usbm->urb_use_count++;
1412 }
1413 break;
1414 }
1415 }
1416 spin_unlock_irqrestore(&ps->lock, flags);
1417 return usbm;
1418}
1419
1269static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb,1420static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb,
1270 struct usbdevfs_iso_packet_desc __user *iso_frame_desc,1421 struct usbdevfs_iso_packet_desc __user *iso_frame_desc,
1271 void __user *arg)1422 void __user *arg)
@@ -1421,6 +1572,19 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb · 149 unmodified lines
1421 goto error;1572 goto error;
1422 }1573 }
1423 1574
1575 as->usbm = find_memory_area(ps, uurb);
1576 if (IS_ERR(as->usbm)) {
1577 ret = PTR_ERR(as->usbm);
1578 as->usbm = NULL;
1579 goto error;
1580 }
1581
1582 /* do not use SG buffers when memory mapped segments
1583 * are in use
1584 */
1585 if (as->usbm)
1586 num_sgs = 0;
1587
1424 u += sizeof(struct async) + sizeof(struct urb) + uurb->buffer_length +1588 u += sizeof(struct async) + sizeof(struct urb) + uurb->buffer_length +
1425 num_sgs * sizeof(struct scatterlist);1589 num_sgs * sizeof(struct scatterlist);
1426 ret = usbfs_increase_memory_usage(u);1590 ret = usbfs_increase_memory_usage(u);
@@ -1458,29 +1622,35 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb · 31 unmodified lines
1458 totlen -= u;1622 totlen -= u;
1459 }1623 }
1460 } else if (uurb->buffer_length > 0) {1624 } else if (uurb->buffer_length > 0) {
1461 as->urb->transfer_buffer = kmalloc(uurb->buffer_length,1625 if (as->usbm) {
1462 GFP_KERNEL);1626 unsigned long uurb_start = (unsigned long)uurb->buffer;
1463 if (!as->urb->transfer_buffer) {
1464 ret = -ENOMEM;
1465 goto error;
1466 }
1467 1627
1468 if (!is_in) {1628 as->urb->transfer_buffer = as->usbm->mem +
1469 if (copy_from_user(as->urb->transfer_buffer,1629 (uurb_start - as->usbm->vm_start);
1470 uurb->buffer,1630 } else {
1471 uurb->buffer_length)) {1631 as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
1472 ret = -EFAULT;1632 GFP_KERNEL);
1633 if (!as->urb->transfer_buffer) {
1634 ret = -ENOMEM;
1473 goto error;1635 goto error;
1474 }1636 }
1475 } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {1637 if (!is_in) {
1476 /*1638 if (copy_from_user(as->urb->transfer_buffer,
1477 * Isochronous input data may end up being1639 uurb->buffer,
1478 * discontiguous if some of the packets are short.1640 uurb->buffer_length)) {
1479 * Clear the buffer so that the gaps don't leak1641 ret = -EFAULT;
1480 * kernel data to userspace.1642 goto error;
1481 */1643 }
1482 memset(as->urb->transfer_buffer, 0,1644 } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
1483 uurb->buffer_length);1645 /*
1646 * Isochronous input data may end up being
1647 * discontiguous if some of the packets are
1648 * short. Clear the buffer so that the gaps
1649 * don't leak kernel data to userspace.
1650 */
1651 memset(as->urb->transfer_buffer, 0,
1652 uurb->buffer_length);
1653 }
1484 }1654 }
1485 }1655 }
1486 as->urb->dev = ps->dev;1656 as->urb->dev = ps->dev;
@@ -1527,10 +1697,14 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb · 40 unmodified lines
1527 isopkt = NULL;1697 isopkt = NULL;
1528 as->ps = ps;1698 as->ps = ps;
1529 as->userurb = arg;1699 as->userurb = arg;
1530 if (is_in && uurb->buffer_length > 0)1700 if (as->usbm) {
1701 unsigned long uurb_start = (unsigned long)uurb->buffer;
1702
1703 as->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
1704 as->urb->transfer_dma = as->usbm->dma_handle +
1705 (uurb_start - as->usbm->vm_start);
1706 } else if (is_in && uurb->buffer_length > 0)
1531 as->userbuffer = uurb->buffer;1707 as->userbuffer = uurb->buffer;
1532 else
1533 as->userbuffer = NULL;
1534 as->signr = uurb->signr;1708 as->signr = uurb->signr;
1535 as->ifnum = ifnum;1709 as->ifnum = ifnum;
1536 as->pid = get_pid(task_pid(current));1710 as->pid = get_pid(task_pid(current));
@@ -1586,6 +1760,8 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb · 49 unmodified lines
1586 return 0;1760 return 0;
1587 1761
1588 error:1762 error:
1763 if (as && as->usbm)
1764 dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
1589 kfree(isopkt);1765 kfree(isopkt);
1590 kfree(dr);1766 kfree(dr);
1591 if (as)1767 if (as)
@@ -2039,7 +2215,7 @@ static int proc_get_capabilities(struct usb_dev_state *ps, void __user *arg) · 447 unmodified lines
2039 __u32 caps;2215 __u32 caps;
2040 2216
2041 caps = USBDEVFS_CAP_ZERO_PACKET | USBDEVFS_CAP_NO_PACKET_SIZE_LIM |2217 caps = USBDEVFS_CAP_ZERO_PACKET | USBDEVFS_CAP_NO_PACKET_SIZE_LIM |
2042 USBDEVFS_CAP_REAP_AFTER_DISCONNECT;2218 USBDEVFS_CAP_REAP_AFTER_DISCONNECT | USBDEVFS_CAP_MMAP;
2043 if (!ps->dev->bus->no_stop_on_short)2219 if (!ps->dev->bus->no_stop_on_short)
2044 caps |= USBDEVFS_CAP_BULK_CONTINUATION;2220 caps |= USBDEVFS_CAP_BULK_CONTINUATION;
2045 if (ps->dev->bus->sg_tablesize)2221 if (ps->dev->bus->sg_tablesize)
@@ -2365,6 +2541,7 @@ const struct file_operations usbdev_file_operations = { · 319 unmodified lines
2365#ifdef CONFIG_COMPAT2541#ifdef CONFIG_COMPAT
2366 .compat_ioctl = usbdev_compat_ioctl,2542 .compat_ioctl = usbdev_compat_ioctl,
2367#endif2543#endif
2544 .mmap = usbdev_mmap,
2368 .open = usbdev_open,2545 .open = usbdev_open,
2369 .release = usbdev_release,2546 .release = usbdev_release,
2370};2547};
include/uapi/linux/usbdevice_fs.h+1−0
@@ -134,6 +134,7 @@ struct usbdevfs_hub_portinfo { · 133 unmodified lines
134#define USBDEVFS_CAP_NO_PACKET_SIZE_LIM 0x04134#define USBDEVFS_CAP_NO_PACKET_SIZE_LIM 0x04
135#define USBDEVFS_CAP_BULK_SCATTER_GATHER 0x08135#define USBDEVFS_CAP_BULK_SCATTER_GATHER 0x08
136#define USBDEVFS_CAP_REAP_AFTER_DISCONNECT 0x10136#define USBDEVFS_CAP_REAP_AFTER_DISCONNECT 0x10
137#define USBDEVFS_CAP_MMAP 0x20
137 138
138/* USBDEVFS_DISCONNECT_CLAIM flags & struct */139/* USBDEVFS_DISCONNECT_CLAIM flags & struct */
139 140

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t