History

vringh: update for virtio 1.0 APIs

When switching everything over to virtio 1.0 memory access APIs, I missed converting vringh. Fortunately, it's straight-forward.

Michael S. Tsirkin committed 11 years ago

Commit b9f7ac81 parent b97a8a92 files · +107 −47Browse files

Unified · Split

2 files
  1. drivers/vhost/vringh.c +74 −47
  2. include/linux/vringh.h +33 −0
drivers/vhost/vringh.c+74−47
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 #include <linux/uaccess.h>
1212 #include <linux/slab.h>
1313 #include <linux/export.h>
14+#include <uapi/linux/virtio_config.h>
1415
1516 static __printf(1,2) __cold void vringh_bad(const char *fmt, ...)
1617 {
@@ -28,13 +29,14 @@ static __printf(1,2) __cold void vringh_bad(const char *fmt, ...) · 11 unmodified lines
2829
2930 /* Returns vring->num if empty, -ve on error. */
3031 static inline int __vringh_get_head(const struct vringh *vrh,
31− int (*getu16)(u16 *val, const u16 *p),
32+ int (*getu16)(const struct vringh *vrh,
33+ u16 *val, const __virtio16 *p),
3234 u16 *last_avail_idx)
3335 {
3436 u16 avail_idx, i, head;
3537 int err;
3638
37− err = getu16(&avail_idx, &vrh->vring.avail->idx);
39+ err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);
3840 if (err) {
3941 vringh_bad("Failed to access avail idx at %p",
4042 &vrh->vring.avail->idx);
@@ -49,7 +51,7 @@ static inline int __vringh_get_head(const struct vringh *vrh, · 8 unmodified lines
4951
5052 i = *last_avail_idx & (vrh->vring.num - 1);
5153
52− err = getu16(&head, &vrh->vring.avail->ring[i]);
54+ err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);
5355 if (err) {
5456 vringh_bad("Failed to read head: idx %d address %p",
5557 *last_avail_idx, &vrh->vring.avail->ring[i]);
@@ -144,28 +146,32 @@ static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len, · 88 unmodified lines
144146 }
145147
146148 /* No reason for this code to be inline. */
147−static int move_to_indirect(int *up_next, u16 *i, void *addr,
149+static int move_to_indirect(const struct vringh *vrh,
150+ int *up_next, u16 *i, void *addr,
148151 const struct vring_desc *desc,
149152 struct vring_desc **descs, int *desc_max)
150153 {
154+ u32 len;
155+
151156 /* Indirect tables can't have indirect. */
152157 if (*up_next != -1) {
153158 vringh_bad("Multilevel indirect %u->%u", *up_next, *i);
154159 return -EINVAL;
155160 }
156161
157− if (unlikely(desc->len % sizeof(struct vring_desc))) {
162+ len = vringh32_to_cpu(vrh, desc->len);
163+ if (unlikely(len % sizeof(struct vring_desc))) {
158164 vringh_bad("Strange indirect len %u", desc->len);
159165 return -EINVAL;
160166 }
161167
162168 /* We will check this when we follow it! */
163− if (desc->flags & VRING_DESC_F_NEXT)
164− *up_next = desc->next;
169+ if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT))
170+ *up_next = vringh16_to_cpu(vrh, desc->next);
165171 else
166172 *up_next = -2;
167173 *descs = addr;
168− *desc_max = desc->len / sizeof(struct vring_desc);
174+ *desc_max = len / sizeof(struct vring_desc);
169175
170176 /* Now, start at the first indirect. */
171177 *i = 0;
@@ -287,22 +293,25 @@ __vringh_iov(struct vringh *vrh, u16 i, · 115 unmodified lines
287293 if (unlikely(err))
288294 goto fail;
289295
290− if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
296+ if (unlikely(desc.flags &
297+ cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) {
298+ u64 a = vringh64_to_cpu(vrh, desc.addr);
299+
291300 /* Make sure it's OK, and get offset. */
292− len = desc.len;
293− if (!rcheck(vrh, desc.addr, &len, &range, getrange)) {
301+ len = vringh32_to_cpu(vrh, desc.len);
302+ if (!rcheck(vrh, a, &len, &range, getrange)) {
294303 err = -EINVAL;
295304 goto fail;
296305 }
297306
298− if (unlikely(len != desc.len)) {
307+ if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
299308 slow = true;
300309 /* We need to save this range to use offset */
301310 slowrange = range;
302311 }
303312
304− addr = (void *)(long)(desc.addr + range.offset);
305− err = move_to_indirect(&up_next, &i, addr, &desc,
313+ addr = (void *)(long)(a + range.offset);
314+ err = move_to_indirect(vrh, &up_next, &i, addr, &desc,
306315 &descs, &desc_max);
307316 if (err)
308317 goto fail;
@@ -315,7 +324,7 @@ __vringh_iov(struct vringh *vrh, u16 i, · 6 unmodified lines
315324 goto fail;
316325 }
317326
318− if (desc.flags & VRING_DESC_F_WRITE)
327+ if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE))
319328 iov = wiov;
320329 else {
321330 iov = riov;
@@ -336,12 +345,14 @@ __vringh_iov(struct vringh *vrh, u16 i, · 14 unmodified lines
336345
337346 again:
338347 /* Make sure it's OK, and get offset. */
339− len = desc.len;
340− if (!rcheck(vrh, desc.addr, &len, &range, getrange)) {
348+ len = vringh32_to_cpu(vrh, desc.len);
349+ if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range,
350+ getrange)) {
341351 err = -EINVAL;
342352 goto fail;
343353 }
344− addr = (void *)(unsigned long)(desc.addr + range.offset);
354+ addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) +
355+ range.offset);
345356
346357 if (unlikely(iov->used == (iov->max_num & ~VRINGH_IOV_ALLOCATED))) {
347358 err = resize_iovec(iov, gfp);
@@ -353,14 +364,16 @@ __vringh_iov(struct vringh *vrh, u16 i, · 5 unmodified lines
353364 iov->iov[iov->used].iov_len = len;
354365 iov->used++;
355366
356− if (unlikely(len != desc.len)) {
357− desc.len -= len;
358− desc.addr += len;
367+ if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
368+ desc.len = cpu_to_vringh32(vrh,
369+ vringh32_to_cpu(vrh, desc.len) - len);
370+ desc.addr = cpu_to_vringh64(vrh,
371+ vringh64_to_cpu(vrh, desc.addr) + len);
359372 goto again;
360373 }
361374
362− if (desc.flags & VRING_DESC_F_NEXT) {
363− i = desc.next;
375+ if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) {
376+ i = vringh16_to_cpu(vrh, desc.next);
364377 } else {
365378 /* Just in case we need to finish traversing above. */
366379 if (unlikely(up_next > 0)) {
@@ -387,7 +400,8 @@ fail: · 20 unmodified lines
387400 static inline int __vringh_complete(struct vringh *vrh,
388401 const struct vring_used_elem *used,
389402 unsigned int num_used,
390− int (*putu16)(u16 *p, u16 val),
403+ int (*putu16)(const struct vringh *vrh,
404+ __virtio16 *p, u16 val),
391405 int (*putused)(struct vring_used_elem *dst,
392406 const struct vring_used_elem
393407 *src, unsigned num))
@@ -420,7 +434,7 @@ static inline int __vringh_complete(struct vringh *vrh, · 26 unmodified lines
420434 /* Make sure buffer is written before we update index. */
421435 virtio_wmb(vrh->weak_barriers);
422436
423− err = putu16(&vrh->vring.used->idx, used_idx + num_used);
437+ err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);
424438 if (err) {
425439 vringh_bad("Failed to update used index at %p",
426440 &vrh->vring.used->idx);
@@ -433,7 +447,9 @@ static inline int __vringh_complete(struct vringh *vrh, · 6 unmodified lines
433447
434448
435449 static inline int __vringh_need_notify(struct vringh *vrh,
436− int (*getu16)(u16 *val, const u16 *p))
450+ int (*getu16)(const struct vringh *vrh,
451+ u16 *val,
452+ const __virtio16 *p))
437453 {
438454 bool notify;
439455 u16 used_event;
@@ -447,7 +463,7 @@ static inline int __vringh_need_notify(struct vringh *vrh, · 7 unmodified lines
447463 /* Old-style, without event indices. */
448464 if (!vrh->event_indices) {
449465 u16 flags;
450− err = getu16(&flags, &vrh->vring.avail->flags);
466+ err = getu16(vrh, &flags, &vrh->vring.avail->flags);
451467 if (err) {
452468 vringh_bad("Failed to get flags at %p",
453469 &vrh->vring.avail->flags);
@@ -457,7 +473,7 @@ static inline int __vringh_need_notify(struct vringh *vrh, · 3 unmodified lines
457473 }
458474
459475 /* Modern: we know when other side wants to know. */
460− err = getu16(&used_event, &vring_used_event(&vrh->vring));
476+ err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));
461477 if (err) {
462478 vringh_bad("Failed to get used event idx at %p",
463479 &vring_used_event(&vrh->vring));
@@ -478,20 +494,22 @@ static inline int __vringh_need_notify(struct vringh *vrh, · 14 unmodified lines
478494 }
479495
480496 static inline bool __vringh_notify_enable(struct vringh *vrh,
481− int (*getu16)(u16 *val, const u16 *p),
482− int (*putu16)(u16 *p, u16 val))
497+ int (*getu16)(const struct vringh *vrh,
498+ u16 *val, const __virtio16 *p),
499+ int (*putu16)(const struct vringh *vrh,
500+ __virtio16 *p, u16 val))
483501 {
484502 u16 avail;
485503
486504 if (!vrh->event_indices) {
487505 /* Old-school; update flags. */
488− if (putu16(&vrh->vring.used->flags, 0) != 0) {
506+ if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {
489507 vringh_bad("Clearing used flags %p",
490508 &vrh->vring.used->flags);
491509 return true;
492510 }
493511 } else {
494− if (putu16(&vring_avail_event(&vrh->vring),
512+ if (putu16(vrh, &vring_avail_event(&vrh->vring),
495513 vrh->last_avail_idx) != 0) {
496514 vringh_bad("Updating avail event index %p",
497515 &vring_avail_event(&vrh->vring));
@@ -503,7 +521,7 @@ static inline bool __vringh_notify_enable(struct vringh *vrh, · 5 unmodified lines
503521 * sure it's written, then check again. */
504522 virtio_mb(vrh->weak_barriers);
505523
506− if (getu16(&avail, &vrh->vring.avail->idx) != 0) {
524+ if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {
507525 vringh_bad("Failed to check avail idx at %p",
508526 &vrh->vring.avail->idx);
509527 return true;
@@ -516,11 +534,13 @@ static inline bool __vringh_notify_enable(struct vringh *vrh, · 6 unmodified lines
516534 }
517535
518536 static inline void __vringh_notify_disable(struct vringh *vrh,
519− int (*putu16)(u16 *p, u16 val))
537+ int (*putu16)(const struct vringh *vrh,
538+ __virtio16 *p, u16 val))
520539 {
521540 if (!vrh->event_indices) {
522541 /* Old-school; update flags. */
523− if (putu16(&vrh->vring.used->flags, VRING_USED_F_NO_NOTIFY)) {
542+ if (putu16(vrh, &vrh->vring.used->flags,
543+ VRING_USED_F_NO_NOTIFY)) {
524544 vringh_bad("Setting used flags %p",
525545 &vrh->vring.used->flags);
526546 }
@@ -528,14 +548,18 @@ static inline void __vringh_notify_disable(struct vringh *vrh, · 1 unmodified line
528548 }
529549
530550 /* Userspace access helpers: in this case, addresses are really userspace. */
531−static inline int getu16_user(u16 *val, const u16 *p)
551+static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p)
532552 {
533− return get_user(*val, (__force u16 __user *)p);
553+ __virtio16 v = 0;
554+ int rc = get_user(v, (__force __virtio16 __user *)p);
555+ *val = vringh16_to_cpu(vrh, v);
556+ return rc;
534557 }
535558
536−static inline int putu16_user(u16 *p, u16 val)
559+static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val)
537560 {
538− return put_user(val, (__force u16 __user *)p);
561+ __virtio16 v = cpu_to_vringh16(vrh, val);
562+ return put_user(v, (__force __virtio16 __user *)p);
539563 }
540564
541565 static inline int copydesc_user(void *dst, const void *src, size_t len)
@@ -589,6 +613,7 @@ int vringh_init_user(struct vringh *vrh, u64 features, · 47 unmodified lines
589613 return -EINVAL;
590614 }
591615
616+ vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
592617 vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
593618 vrh->weak_barriers = weak_barriers;
594619 vrh->completed = 0;
@@ -729,8 +754,8 @@ int vringh_complete_user(struct vringh *vrh, u16 head, u32 len) · 134 unmodified lines
729754 {
730755 struct vring_used_elem used;
731756
732− used.id = head;
733− used.len = len;
757+ used.id = cpu_to_vringh32(vrh, head);
758+ used.len = cpu_to_vringh32(vrh, len);
734759 return __vringh_complete(vrh, &used, 1, putu16_user, putused_user);
735760 }
736761 EXPORT_SYMBOL(vringh_complete_user);
@@ -792,15 +817,16 @@ int vringh_need_notify_user(struct vringh *vrh) · 55 unmodified lines
792817 EXPORT_SYMBOL(vringh_need_notify_user);
793818
794819 /* Kernelspace access helpers. */
795−static inline int getu16_kern(u16 *val, const u16 *p)
820+static inline int getu16_kern(const struct vringh *vrh,
821+ u16 *val, const __virtio16 *p)
796822 {
797− *val = ACCESS_ONCE(*p);
823+ *val = vringh16_to_cpu(vrh, ACCESS_ONCE(*p));
798824 return 0;
799825 }
800826
801−static inline int putu16_kern(u16 *p, u16 val)
827+static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)
802828 {
803− ACCESS_ONCE(*p) = val;
829+ ACCESS_ONCE(*p) = cpu_to_vringh16(vrh, val);
804830 return 0;
805831 }
806832
@@ -848,6 +874,7 @@ int vringh_init_kern(struct vringh *vrh, u64 features, · 41 unmodified lines
848874 return -EINVAL;
849875 }
850876
877+ vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
851878 vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
852879 vrh->weak_barriers = weak_barriers;
853880 vrh->completed = 0;
@@ -962,8 +989,8 @@ int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len) · 108 unmodified lines
962989 {
963990 struct vring_used_elem used;
964991
965− used.id = head;
966− used.len = len;
992+ used.id = cpu_to_vringh32(vrh, head);
993+ used.len = cpu_to_vringh32(vrh, len);
967994
968995 return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern);
969996 }
include/linux/vringh.h+33−0
@@ -24,12 +24,16 @@ · 23 unmodified lines
2424 #ifndef _LINUX_VRINGH_H
2525 #define _LINUX_VRINGH_H
2626 #include <uapi/linux/virtio_ring.h>
27+#include <linux/virtio_byteorder.h>
2728 #include <linux/uio.h>
2829 #include <linux/slab.h>
2930 #include <asm/barrier.h>
3031
3132 /* virtio_ring with information needed for host access. */
3233 struct vringh {
34+ /* Everything is little endian */
35+ bool little_endian;
36+
3337 /* Guest publishes used event idx (note: we always do). */
3438 bool event_indices;
3539
@@ -222,4 +226,33 @@ static inline void vringh_notify(struct vringh *vrh) · 186 unmodified lines
222226 vrh->notify(vrh);
223227 }
224228
229+static inline u16 vringh16_to_cpu(const struct vringh *vrh, __virtio16 val)
230+{
231+ return __virtio16_to_cpu(vrh->little_endian, val);
232+}
233+
234+static inline __virtio16 cpu_to_vringh16(const struct vringh *vrh, u16 val)
235+{
236+ return __cpu_to_virtio16(vrh->little_endian, val);
237+}
238+
239+static inline u32 vringh32_to_cpu(const struct vringh *vrh, __virtio32 val)
240+{
241+ return __virtio32_to_cpu(vrh->little_endian, val);
242+}
243+
244+static inline __virtio32 cpu_to_vringh32(const struct vringh *vrh, u32 val)
245+{
246+ return __cpu_to_virtio32(vrh->little_endian, val);
247+}
248+
249+static inline u64 vringh64_to_cpu(const struct vringh *vrh, __virtio64 val)
250+{
251+ return __virtio64_to_cpu(vrh->little_endian, val);
252+}
253+
254+static inline __virtio64 cpu_to_vringh64(const struct vringh *vrh, u64 val)
255+{
256+ return __cpu_to_virtio64(vrh->little_endian, val);
257+}
225258 #endif /* _LINUX_VRINGH_H */