History

USB: Add support for usbfs zerocopy.

Add a new interface for userspace to preallocate memory that can be used with usbfs. This gives two primary benefits: - Zerocopy; data no longer needs to be copied between the userspace and the kernel, but can instead be read directly by the driver from userspace's buffers. This works for all kinds of transfers (even if nonsensical for control and interrupt transfers); isochronous also no longer need to memset() the buffer to zero to avoid leaking kernel data. - Once the buffers are allocated, USB transfers can no longer fail due to memory fragmentation; previously, long-running programs could run into problems finding a large enough contiguous memory chunk, especially on embedded systems or at high rates. Memory is allocated by using mmap() against the usbfs file descriptor, and similarly deallocated by munmap(). Once memory has been allocated, using it as pointers to a bulk or isochronous operation means you will automatically get zerocopy behavior. Note that this also means you cannot modify outgoing data until the transfer is complete. The same holds for data on the same cache lines as incoming data; DMA modifying them at the same time could lead to your changes being overwritten. There's a new capability USBDEVFS_CAP_MMAP that userspace can query to see if the running kernel supports this functionality, if just trying mmap() is not acceptable. Largely based on a patch by Markus Rechberger with some updates. The original patch can be found at: http://sundtek.de/support/devio_mmap_v0.4.diff

Steinar H. Gunderson committed 10 years ago

Commit f7d34b41 parent 4a6d9f82 files · +203 −25Browse files

Unified · Split

2 files
  1. drivers/usb/core/devio.c +202 −25
  2. include/uapi/linux/usbdevice_fs.h +1 −0
drivers/usb/core/devio.c+202−25
@@ -50,6 +50,7 @@ · 49 unmodified lines
5050 #include <linux/user_namespace.h>
5151 #include <linux/scatterlist.h>
5252 #include <linux/uaccess.h>
53+#include <linux/dma-mapping.h>
5354 #include <asm/byteorder.h>
5455 #include <linux/moduleparam.h>
5556
@@ -69,6 +70,7 @@ struct usb_dev_state { · 13 unmodified lines
6970 spinlock_t lock; /* protects the async urb lists */
7071 struct list_head async_pending;
7172 struct list_head async_completed;
73+ struct list_head memory_list;
7274 wait_queue_head_t wait; /* wake up if a request completed */
7375 unsigned int discsignr;
7476 struct pid *disc_pid;
@@ -79,6 +81,17 @@ struct usb_dev_state { · 4 unmodified lines
7981 u32 disabled_bulk_eps;
8082 };
8183
84+struct usb_memory {
85+ struct list_head memlist;
86+ int vma_use_count;
87+ int urb_use_count;
88+ u32 size;
89+ void *mem;
90+ dma_addr_t dma_handle;
91+ unsigned long vm_start;
92+ struct usb_dev_state *ps;
93+};
94+
8295 struct async {
8396 struct list_head asynclist;
8497 struct usb_dev_state *ps;
@@ -89,6 +102,7 @@ struct async { · 4 unmodified lines
89102 void __user *userbuffer;
90103 void __user *userurb;
91104 struct urb *urb;
105+ struct usb_memory *usbm;
92106 unsigned int mem_usage;
93107 int status;
94108 u32 secid;
@@ -162,6 +176,111 @@ static int connected(struct usb_dev_state *ps) · 67 unmodified lines
162176 ps->dev->state != USB_STATE_NOTATTACHED);
163177 }
164178
179+static void dec_usb_memory_use_count(struct usb_memory *usbm, int *count)
180+{
181+ struct usb_dev_state *ps = usbm->ps;
182+ unsigned long flags;
183+
184+ spin_lock_irqsave(&ps->lock, flags);
185+ --*count;
186+ if (usbm->urb_use_count == 0 && usbm->vma_use_count == 0) {
187+ list_del(&usbm->memlist);
188+ spin_unlock_irqrestore(&ps->lock, flags);
189+
190+ usb_free_coherent(ps->dev, usbm->size, usbm->mem,
191+ usbm->dma_handle);
192+ usbfs_decrease_memory_usage(
193+ usbm->size + sizeof(struct usb_memory));
194+ kfree(usbm);
195+ } else {
196+ spin_unlock_irqrestore(&ps->lock, flags);
197+ }
198+}
199+
200+static void usbdev_vm_open(struct vm_area_struct *vma)
201+{
202+ struct usb_memory *usbm = vma->vm_private_data;
203+ unsigned long flags;
204+
205+ spin_lock_irqsave(&usbm->ps->lock, flags);
206+ ++usbm->vma_use_count;
207+ spin_unlock_irqrestore(&usbm->ps->lock, flags);
208+}
209+
210+static void usbdev_vm_close(struct vm_area_struct *vma)
211+{
212+ struct usb_memory *usbm = vma->vm_private_data;
213+
214+ dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
215+}
216+
217+struct vm_operations_struct usbdev_vm_ops = {
218+ .open = usbdev_vm_open,
219+ .close = usbdev_vm_close
220+};
221+
222+static int usbdev_mmap(struct file *file, struct vm_area_struct *vma)
223+{
224+ struct usb_memory *usbm = NULL;
225+ struct usb_dev_state *ps = file->private_data;
226+ size_t size = vma->vm_end - vma->vm_start;
227+ void *mem;
228+ unsigned long flags;
229+ dma_addr_t dma_handle;
230+ int ret;
231+
232+ ret = usbfs_increase_memory_usage(size + sizeof(struct usb_memory));
233+ if (ret)
234+ goto error;
235+
236+ usbm = kzalloc(sizeof(struct usb_memory), GFP_KERNEL);
237+ if (!usbm) {
238+ ret = -ENOMEM;
239+ goto error_decrease_mem;
240+ }
241+
242+ mem = usb_alloc_coherent(ps->dev, size, GFP_USER, &dma_handle);
243+ if (!mem) {
244+ ret = -ENOMEM;
245+ goto error_free_usbm;
246+ }
247+
248+ memset(mem, 0, size);
249+
250+ usbm->mem = mem;
251+ usbm->dma_handle = dma_handle;
252+ usbm->size = size;
253+ usbm->ps = ps;
254+ usbm->vm_start = vma->vm_start;
255+ usbm->vma_use_count = 1;
256+ INIT_LIST_HEAD(&usbm->memlist);
257+
258+ if (remap_pfn_range(vma, vma->vm_start,
259+ virt_to_phys(usbm->mem) >> PAGE_SHIFT,
260+ size, vma->vm_page_prot) < 0) {
261+ dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
262+ return -EAGAIN;
263+ }
264+
265+ vma->vm_flags |= VM_IO;
266+ vma->vm_flags |= (VM_DONTEXPAND | VM_DONTDUMP);
267+ vma->vm_ops = &usbdev_vm_ops;
268+ vma->vm_private_data = usbm;
269+
270+ spin_lock_irqsave(&ps->lock, flags);
271+ list_add_tail(&usbm->memlist, &ps->memory_list);
272+ spin_unlock_irqrestore(&ps->lock, flags);
273+
274+ return 0;
275+
276+error_free_usbm:
277+ kfree(usbm);
278+error_decrease_mem:
279+ usbfs_decrease_memory_usage(size + sizeof(struct usb_memory));
280+error:
281+ return ret;
282+}
283+
165284 static ssize_t usbdev_read(struct file *file, char __user *buf, size_t nbytes,
166285 loff_t *ppos)
167286 {
@@ -278,8 +397,13 @@ static void free_async(struct async *as) · 110 unmodified lines
278397 if (sg_page(&as->urb->sg[i]))
279398 kfree(sg_virt(&as->urb->sg[i]));
280399 }
400+
281401 kfree(as->urb->sg);
282− kfree(as->urb->transfer_buffer);
402+ if (as->usbm == NULL)
403+ kfree(as->urb->transfer_buffer);
404+ else
405+ dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
406+
283407 kfree(as->urb->setup_packet);
284408 usb_free_urb(as->urb);
285409 usbfs_decrease_memory_usage(as->mem_usage);
@@ -893,6 +1017,7 @@ static int usbdev_open(struct inode *inode, struct file *file) · 607 unmodified lines
8931017 INIT_LIST_HEAD(&ps->list);
8941018 INIT_LIST_HEAD(&ps->async_pending);
8951019 INIT_LIST_HEAD(&ps->async_completed);
1020+ INIT_LIST_HEAD(&ps->memory_list);
8961021 init_waitqueue_head(&ps->wait);
8971022 ps->discsignr = 0;
8981023 ps->disc_pid = get_pid(task_pid(current));
@@ -945,6 +1070,7 @@ static int usbdev_release(struct inode *inode, struct file *file) · 46 unmodified lines
9451070 free_async(as);
9461071 as = async_getcompleted(ps);
9471072 }
1073+
9481074 kfree(ps);
9491075 return 0;
9501076 }
@@ -1266,6 +1392,31 @@ static int proc_setconfig(struct usb_dev_state *ps, void __user *arg) · 315 unmodified lines
12661392 return status;
12671393 }
12681394
1395+static struct usb_memory *
1396+find_memory_area(struct usb_dev_state *ps, const struct usbdevfs_urb *uurb)
1397+{
1398+ struct usb_memory *usbm = NULL, *iter;
1399+ unsigned long flags;
1400+ unsigned long uurb_start = (unsigned long)uurb->buffer;
1401+
1402+ spin_lock_irqsave(&ps->lock, flags);
1403+ list_for_each_entry(iter, &ps->memory_list, memlist) {
1404+ if (uurb_start >= iter->vm_start &&
1405+ uurb_start < iter->vm_start + iter->size) {
1406+ if (uurb->buffer_length > iter->vm_start + iter->size -
1407+ uurb_start) {
1408+ usbm = ERR_PTR(-EINVAL);
1409+ } else {
1410+ usbm = iter;
1411+ usbm->urb_use_count++;
1412+ }
1413+ break;
1414+ }
1415+ }
1416+ spin_unlock_irqrestore(&ps->lock, flags);
1417+ return usbm;
1418+}
1419+
12691420 static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb,
12701421 struct usbdevfs_iso_packet_desc __user *iso_frame_desc,
12711422 void __user *arg)
@@ -1421,6 +1572,19 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb · 149 unmodified lines
14211572 goto error;
14221573 }
14231574
1575+ as->usbm = find_memory_area(ps, uurb);
1576+ if (IS_ERR(as->usbm)) {
1577+ ret = PTR_ERR(as->usbm);
1578+ as->usbm = NULL;
1579+ goto error;
1580+ }
1581+
1582+ /* do not use SG buffers when memory mapped segments
1583+ * are in use
1584+ */
1585+ if (as->usbm)
1586+ num_sgs = 0;
1587+
14241588 u += sizeof(struct async) + sizeof(struct urb) + uurb->buffer_length +
14251589 num_sgs * sizeof(struct scatterlist);
14261590 ret = usbfs_increase_memory_usage(u);
@@ -1458,29 +1622,35 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb · 31 unmodified lines
14581622 totlen -= u;
14591623 }
14601624 } else if (uurb->buffer_length > 0) {
1461− as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
1462− GFP_KERNEL);
1463− if (!as->urb->transfer_buffer) {
1464− ret = -ENOMEM;
1465− goto error;
1466− }
1625+ if (as->usbm) {
1626+ unsigned long uurb_start = (unsigned long)uurb->buffer;
14671627
1468− if (!is_in) {
1469− if (copy_from_user(as->urb->transfer_buffer,
1470− uurb->buffer,
1471− uurb->buffer_length)) {
1472− ret = -EFAULT;
1628+ as->urb->transfer_buffer = as->usbm->mem +
1629+ (uurb_start - as->usbm->vm_start);
1630+ } else {
1631+ as->urb->transfer_buffer = kmalloc(uurb->buffer_length,
1632+ GFP_KERNEL);
1633+ if (!as->urb->transfer_buffer) {
1634+ ret = -ENOMEM;
14731635 goto error;
14741636 }
1475− } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
1476− /*
1477− * Isochronous input data may end up being
1478− * discontiguous if some of the packets are short.
1479− * Clear the buffer so that the gaps don't leak
1480− * kernel data to userspace.
1481− */
1482− memset(as->urb->transfer_buffer, 0,
1483− uurb->buffer_length);
1637+ if (!is_in) {
1638+ if (copy_from_user(as->urb->transfer_buffer,
1639+ uurb->buffer,
1640+ uurb->buffer_length)) {
1641+ ret = -EFAULT;
1642+ goto error;
1643+ }
1644+ } else if (uurb->type == USBDEVFS_URB_TYPE_ISO) {
1645+ /*
1646+ * Isochronous input data may end up being
1647+ * discontiguous if some of the packets are
1648+ * short. Clear the buffer so that the gaps
1649+ * don't leak kernel data to userspace.
1650+ */
1651+ memset(as->urb->transfer_buffer, 0,
1652+ uurb->buffer_length);
1653+ }
14841654 }
14851655 }
14861656 as->urb->dev = ps->dev;
@@ -1527,10 +1697,14 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb · 40 unmodified lines
15271697 isopkt = NULL;
15281698 as->ps = ps;
15291699 as->userurb = arg;
1530− if (is_in && uurb->buffer_length > 0)
1700+ if (as->usbm) {
1701+ unsigned long uurb_start = (unsigned long)uurb->buffer;
1702+
1703+ as->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
1704+ as->urb->transfer_dma = as->usbm->dma_handle +
1705+ (uurb_start - as->usbm->vm_start);
1706+ } else if (is_in && uurb->buffer_length > 0)
15311707 as->userbuffer = uurb->buffer;
1532− else
1533− as->userbuffer = NULL;
15341708 as->signr = uurb->signr;
15351709 as->ifnum = ifnum;
15361710 as->pid = get_pid(task_pid(current));
@@ -1586,6 +1760,8 @@ static int proc_do_submiturb(struct usb_dev_state *ps, struct usbdevfs_urb *uurb · 49 unmodified lines
15861760 return 0;
15871761
15881762 error:
1763+ if (as && as->usbm)
1764+ dec_usb_memory_use_count(as->usbm, &as->usbm->urb_use_count);
15891765 kfree(isopkt);
15901766 kfree(dr);
15911767 if (as)
@@ -2039,7 +2215,7 @@ static int proc_get_capabilities(struct usb_dev_state *ps, void __user *arg) · 447 unmodified lines
20392215 __u32 caps;
20402216
20412217 caps = USBDEVFS_CAP_ZERO_PACKET | USBDEVFS_CAP_NO_PACKET_SIZE_LIM |
2042− USBDEVFS_CAP_REAP_AFTER_DISCONNECT;
2218+ USBDEVFS_CAP_REAP_AFTER_DISCONNECT | USBDEVFS_CAP_MMAP;
20432219 if (!ps->dev->bus->no_stop_on_short)
20442220 caps |= USBDEVFS_CAP_BULK_CONTINUATION;
20452221 if (ps->dev->bus->sg_tablesize)
@@ -2365,6 +2541,7 @@ const struct file_operations usbdev_file_operations = { · 319 unmodified lines
23652541 #ifdef CONFIG_COMPAT
23662542 .compat_ioctl = usbdev_compat_ioctl,
23672543 #endif
2544+ .mmap = usbdev_mmap,
23682545 .open = usbdev_open,
23692546 .release = usbdev_release,
23702547 };
include/uapi/linux/usbdevice_fs.h+1−0
@@ -134,6 +134,7 @@ struct usbdevfs_hub_portinfo { · 133 unmodified lines
134134 #define USBDEVFS_CAP_NO_PACKET_SIZE_LIM 0x04
135135 #define USBDEVFS_CAP_BULK_SCATTER_GATHER 0x08
136136 #define USBDEVFS_CAP_REAP_AFTER_DISCONNECT 0x10
137+#define USBDEVFS_CAP_MMAP 0x20
137138
138139 /* USBDEVFS_DISCONNECT_CLAIM flags & struct */
139140

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t