/* * Copyright (C) 2015 Red Hat, Inc. * All Rights Reserved. * * Authors: * Dave Airlie <airlied@redhat.com> * Gerd Hoffmann <kraxel@redhat.com> * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/dma-mapping.h>#include <linux/virtio.h>#include <linux/virtio_config.h>#include <linux/virtio_ring.h> #include <drm/drm_edid.h>#include <drm/drm_print.h> #include "virtgpu_drv.h"#include "virtgpu_trace.h" #define MAX_INLINE_CMD_SIZE 96#define MAX_INLINE_RESP_SIZE 24#define VBUFFER_SIZE (sizeof(struct virtio_gpu_vbuffer) \ + MAX_INLINE_CMD_SIZE \ + MAX_INLINE_RESP_SIZE) static void convert_to_hw_box(struct virtio_gpu_box *dst, const struct drm_virtgpu_3d_box *src){ dst->x = cpu_to_le32(src->x); dst->y = cpu_to_le32(src->y); dst->z = cpu_to_le32(src->z); dst->w = cpu_to_le32(src->w); dst->h = cpu_to_le32(src->h); dst->d = cpu_to_le32(src->d);} void virtio_gpu_ctrl_ack(struct virtqueue *vq){ struct drm_device *dev = vq->vdev->priv; struct virtio_gpu_device *vgdev = dev->dev_private; schedule_work(&vgdev->ctrlq.dequeue_work);} void virtio_gpu_cursor_ack(struct virtqueue *vq){ struct drm_device *dev = vq->vdev->priv; struct virtio_gpu_device *vgdev = dev->dev_private; schedule_work(&vgdev->cursorq.dequeue_work);} int virtio_gpu_alloc_vbufs(struct virtio_gpu_device *vgdev){ vgdev->vbufs = kmem_cache_create("virtio-gpu-vbufs", VBUFFER_SIZE, __alignof__(struct virtio_gpu_vbuffer), 0, NULL); if (!vgdev->vbufs) return -ENOMEM; return 0;} void virtio_gpu_free_vbufs(struct virtio_gpu_device *vgdev){ kmem_cache_destroy(vgdev->vbufs); vgdev->vbufs = NULL;} /* For drm_panic */static struct virtio_gpu_vbuffer*virtio_gpu_panic_get_vbuf(struct virtio_gpu_device *vgdev, int size){ struct virtio_gpu_vbuffer *vbuf; vbuf = kmem_cache_zalloc(vgdev->vbufs, GFP_ATOMIC); vbuf->buf = (void *)vbuf + sizeof(*vbuf); vbuf->size = size; vbuf->resp_cb = NULL; vbuf->resp_size = sizeof(struct virtio_gpu_ctrl_hdr); vbuf->resp_buf = (void *)vbuf->buf + size; return vbuf;} static struct virtio_gpu_vbuffer*virtio_gpu_get_vbuf(struct virtio_gpu_device *vgdev, int size, int resp_size, void *resp_buf, virtio_gpu_resp_cb resp_cb){ struct virtio_gpu_vbuffer *vbuf; vbuf = kmem_cache_zalloc(vgdev->vbufs, GFP_KERNEL | __GFP_NOFAIL); BUG_ON(size > MAX_INLINE_CMD_SIZE || size < sizeof(struct virtio_gpu_ctrl_hdr)); vbuf->buf = (void *)vbuf + sizeof(*vbuf); vbuf->size = size; vbuf->resp_cb = resp_cb; vbuf->resp_size = resp_size; if (resp_size <= MAX_INLINE_RESP_SIZE) vbuf->resp_buf = (void *)vbuf->buf + size; else vbuf->resp_buf = resp_buf; BUG_ON(!vbuf->resp_buf); return vbuf;} static struct virtio_gpu_ctrl_hdr *virtio_gpu_vbuf_ctrl_hdr(struct virtio_gpu_vbuffer *vbuf){ /* this assumes a vbuf contains a command that starts with a * virtio_gpu_ctrl_hdr, which is true for both ctrl and cursor * virtqueues. */ return (struct virtio_gpu_ctrl_hdr *)vbuf->buf;} static struct virtio_gpu_update_cursor*virtio_gpu_alloc_cursor(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer **vbuffer_p){ struct virtio_gpu_vbuffer *vbuf; vbuf = virtio_gpu_get_vbuf (vgdev, sizeof(struct virtio_gpu_update_cursor), 0, NULL, NULL); if (IS_ERR(vbuf)) { *vbuffer_p = NULL; return ERR_CAST(vbuf); } *vbuffer_p = vbuf; return (struct virtio_gpu_update_cursor *)vbuf->buf;} /* For drm_panic */static void *virtio_gpu_panic_alloc_cmd_resp(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer **vbuffer_p, int cmd_size){ struct virtio_gpu_vbuffer *vbuf; vbuf = virtio_gpu_panic_get_vbuf(vgdev, cmd_size); *vbuffer_p = vbuf; return (struct virtio_gpu_command *)vbuf->buf;} static void *virtio_gpu_alloc_cmd_resp(struct virtio_gpu_device *vgdev, virtio_gpu_resp_cb cb, struct virtio_gpu_vbuffer **vbuffer_p, int cmd_size, int resp_size, void *resp_buf){ struct virtio_gpu_vbuffer *vbuf; vbuf = virtio_gpu_get_vbuf(vgdev, cmd_size, resp_size, resp_buf, cb); *vbuffer_p = vbuf; return (struct virtio_gpu_command *)vbuf->buf;} static void *virtio_gpu_alloc_cmd(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer **vbuffer_p, int size){ return virtio_gpu_alloc_cmd_resp(vgdev, NULL, vbuffer_p, size, sizeof(struct virtio_gpu_ctrl_hdr), NULL);} static void *virtio_gpu_alloc_cmd_cb(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer **vbuffer_p, int size, virtio_gpu_resp_cb cb){ return virtio_gpu_alloc_cmd_resp(vgdev, cb, vbuffer_p, size, sizeof(struct virtio_gpu_ctrl_hdr), NULL);} static void free_vbuf(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf){ if (vbuf->resp_size > MAX_INLINE_RESP_SIZE) kfree(vbuf->resp_buf); kvfree(vbuf->data_buf); kmem_cache_free(vgdev->vbufs, vbuf);} void virtio_gpu_reclaim_vbufs(struct virtio_gpu_device *vgdev){ struct virtio_gpu_vbuffer *vbuf; while ((vbuf = virtqueue_detach_unused_buf(vgdev->ctrlq.vq))) { if (vbuf->objs) virtio_gpu_array_put_free(vbuf->objs); if (vbuf->resp_cb_data) virtio_gpu_cleanup_object(vbuf->resp_cb_data); free_vbuf(vgdev, vbuf); } while ((vbuf = virtqueue_detach_unused_buf(vgdev->cursorq.vq))) free_vbuf(vgdev, vbuf);} static void reclaim_vbufs(struct virtqueue *vq, struct list_head *reclaim_list){ struct virtio_gpu_vbuffer *vbuf; unsigned int len; int freed = 0; while ((vbuf = virtqueue_get_buf(vq, &len))) { list_add_tail(&vbuf->list, reclaim_list); freed++; } if (freed == 0) DRM_DEBUG("Huh? zero vbufs reclaimed");} void virtio_gpu_dequeue_ctrl_func(struct work_struct *work){ struct virtio_gpu_device *vgdev = container_of(work, struct virtio_gpu_device, ctrlq.dequeue_work); struct list_head reclaim_list; struct virtio_gpu_vbuffer *entry, *tmp; struct virtio_gpu_ctrl_hdr *resp; u64 fence_id; INIT_LIST_HEAD(&reclaim_list); spin_lock(&vgdev->ctrlq.qlock); do { virtqueue_disable_cb(vgdev->ctrlq.vq); reclaim_vbufs(vgdev->ctrlq.vq, &reclaim_list); } while (!virtqueue_enable_cb(vgdev->ctrlq.vq)); spin_unlock(&vgdev->ctrlq.qlock); list_for_each_entry(entry, &reclaim_list, list) { resp = (struct virtio_gpu_ctrl_hdr *)entry->resp_buf; trace_virtio_gpu_cmd_response(vgdev->ctrlq.vq, resp, entry->seqno); if (resp->type != cpu_to_le32(VIRTIO_GPU_RESP_OK_NODATA)) { if (le32_to_cpu(resp->type) >= VIRTIO_GPU_RESP_ERR_UNSPEC) { struct virtio_gpu_ctrl_hdr *cmd; cmd = virtio_gpu_vbuf_ctrl_hdr(entry); DRM_ERROR_RATELIMITED("response 0x%x (command 0x%x)\n", le32_to_cpu(resp->type), le32_to_cpu(cmd->type)); } else DRM_DEBUG("response 0x%x\n", le32_to_cpu(resp->type)); } if (resp->flags & cpu_to_le32(VIRTIO_GPU_FLAG_FENCE)) { fence_id = le64_to_cpu(resp->fence_id); virtio_gpu_fence_event_process(vgdev, fence_id); } if (entry->resp_cb) entry->resp_cb(vgdev, entry); } wake_up(&vgdev->ctrlq.ack_queue); list_for_each_entry_safe(entry, tmp, &reclaim_list, list) { if (entry->objs) virtio_gpu_array_put_free_delayed(vgdev, entry->objs); list_del(&entry->list); free_vbuf(vgdev, entry); }} void virtio_gpu_dequeue_cursor_func(struct work_struct *work){ struct virtio_gpu_device *vgdev = container_of(work, struct virtio_gpu_device, cursorq.dequeue_work); struct list_head reclaim_list; struct virtio_gpu_vbuffer *entry, *tmp; INIT_LIST_HEAD(&reclaim_list); spin_lock(&vgdev->cursorq.qlock); do { virtqueue_disable_cb(vgdev->cursorq.vq); reclaim_vbufs(vgdev->cursorq.vq, &reclaim_list); } while (!virtqueue_enable_cb(vgdev->cursorq.vq)); spin_unlock(&vgdev->cursorq.qlock); list_for_each_entry_safe(entry, tmp, &reclaim_list, list) { struct virtio_gpu_ctrl_hdr *resp = (struct virtio_gpu_ctrl_hdr *)entry->resp_buf; trace_virtio_gpu_cmd_response(vgdev->cursorq.vq, resp, entry->seqno); list_del(&entry->list); free_vbuf(vgdev, entry); } wake_up(&vgdev->cursorq.ack_queue);} /* Create sg_table from a vmalloc'd buffer. */static struct sg_table *vmalloc_to_sgt(char *data, uint32_t size, int *sg_ents){ int ret, s, i; struct sg_table *sgt; struct scatterlist *sg; struct page *pg; if (WARN_ON(!PAGE_ALIGNED(data))) return NULL; sgt = kmalloc_obj(*sgt); if (!sgt) return NULL; *sg_ents = DIV_ROUND_UP(size, PAGE_SIZE); ret = sg_alloc_table(sgt, *sg_ents, GFP_KERNEL); if (ret) { kfree(sgt); return NULL; } for_each_sgtable_sg(sgt, sg, i) { pg = vmalloc_to_page(data); if (!pg) { sg_free_table(sgt); kfree(sgt); return NULL; } s = min_t(int, PAGE_SIZE, size); sg_set_page(sg, pg, s, 0); size -= s; data += s; } return sgt;} /* For drm_panic */static int virtio_gpu_panic_queue_ctrl_sgs(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf, int elemcnt, struct scatterlist **sgs, int outcnt, int incnt){ struct virtqueue *vq = vgdev->ctrlq.vq; int ret; if (vgdev->has_indirect) elemcnt = 1; if (vq->num_free < elemcnt) return -ENOMEM; ret = virtqueue_add_sgs(vq, sgs, outcnt, incnt, vbuf, GFP_ATOMIC); WARN_ON(ret); vbuf->seqno = ++vgdev->ctrlq.seqno; trace_virtio_gpu_cmd_queue(vq, virtio_gpu_vbuf_ctrl_hdr(vbuf), vbuf->seqno); atomic_inc(&vgdev->pending_commands); return 0;} int virtio_gpu_wait_queue(struct virtio_gpu_queue *vgvq, unsigned int num_elem){ int ret; /* Wait up to 5 seconds for enough free slots to become available */ ret = wait_event_timeout(vgvq->ack_queue, vgvq->vq->num_free >= num_elem, 5 * HZ); if (ret == 0) return -ETIMEDOUT; return 0;} static int virtio_gpu_queue_ctrl_sgs(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf, struct virtio_gpu_fence *fence, int elemcnt, struct scatterlist **sgs, int outcnt, int incnt){ struct virtqueue *vq = vgdev->ctrlq.vq; int ret, idx; if (!drm_dev_enter(vgdev->ddev, &idx)) { if (fence && vbuf->objs) virtio_gpu_array_unlock_resv(vbuf->objs); free_vbuf(vgdev, vbuf); return -ENODEV; } if (vgdev->has_indirect) elemcnt = 1; again: spin_lock(&vgdev->ctrlq.qlock); if (vq->num_free < elemcnt) { spin_unlock(&vgdev->ctrlq.qlock); virtio_gpu_notify(vgdev); wait_event(vgdev->ctrlq.ack_queue, vq->num_free >= elemcnt || vgdev->vqs_released); /* * Set by virtio_gpu_release_vqs() to unblock * synchronize_srcu() wait in drm_dev_unplug(). */ if (vgdev->vqs_released) { if (fence && vbuf->objs) virtio_gpu_array_unlock_resv(vbuf->objs); free_vbuf(vgdev, vbuf); drm_dev_exit(idx); return -ENODEV; } goto again; } /* now that the position of the vbuf in the virtqueue is known, we can * finally set the fence id */ if (fence) { virtio_gpu_fence_emit(vgdev, virtio_gpu_vbuf_ctrl_hdr(vbuf), fence); if (vbuf->objs) { virtio_gpu_array_add_fence(vbuf->objs, &fence->f); virtio_gpu_array_unlock_resv(vbuf->objs); } } ret = virtqueue_add_sgs(vq, sgs, outcnt, incnt, vbuf, GFP_ATOMIC); WARN_ON(ret); vbuf->seqno = ++vgdev->ctrlq.seqno; trace_virtio_gpu_cmd_queue(vq, virtio_gpu_vbuf_ctrl_hdr(vbuf), vbuf->seqno); atomic_inc(&vgdev->pending_commands); spin_unlock(&vgdev->ctrlq.qlock); drm_dev_exit(idx); return 0;} /* For drm_panic */static int virtio_gpu_panic_queue_ctrl_buffer(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf){ struct scatterlist *sgs[3], vcmd, vresp; int elemcnt = 0, outcnt = 0, incnt = 0; /* set up vcmd */ sg_init_one(&vcmd, vbuf->buf, vbuf->size); elemcnt++; sgs[outcnt] = &vcmd; outcnt++; /* set up vresp */ if (vbuf->resp_size) { sg_init_one(&vresp, vbuf->resp_buf, vbuf->resp_size); elemcnt++; sgs[outcnt + incnt] = &vresp; incnt++; } return virtio_gpu_panic_queue_ctrl_sgs(vgdev, vbuf, elemcnt, sgs, outcnt, incnt);} static int virtio_gpu_queue_fenced_ctrl_buffer(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf, struct virtio_gpu_fence *fence){ struct scatterlist *sgs[3], vcmd, vout, vresp;Lines 1–500 of 1546. Load more
/* * Copyright (C) 2015 Red Hat, Inc. * All Rights Reserved. * * Authors: * Dave Airlie <airlied@redhat.com> * Gerd Hoffmann <kraxel@redhat.com> * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/dma-mapping.h> #include <linux/virtio.h> #include <linux/virtio_config.h> #include <linux/virtio_ring.h> #include <drm/drm_edid.h> #include <drm/drm_print.h> #include "virtgpu_drv.h" #include "virtgpu_trace.h" #define MAX_INLINE_CMD_SIZE 96 #define MAX_INLINE_RESP_SIZE 24 #define VBUFFER_SIZE (sizeof(struct virtio_gpu_vbuffer) \ + MAX_INLINE_CMD_SIZE \ + MAX_INLINE_RESP_SIZE) static void convert_to_hw_box(struct virtio_gpu_box *dst, const struct drm_virtgpu_3d_box *src) { dst->x = cpu_to_le32(src->x); dst->y = cpu_to_le32(src->y); dst->z = cpu_to_le32(src->z); dst->w = cpu_to_le32(src->w); dst->h = cpu_to_le32(src->h); dst->d = cpu_to_le32(src->d); } void virtio_gpu_ctrl_ack(struct virtqueue *vq) { struct drm_device *dev = vq->vdev->priv; struct virtio_gpu_device *vgdev = dev->dev_private; schedule_work(&vgdev->ctrlq.dequeue_work); } void virtio_gpu_cursor_ack(struct virtqueue *vq) { struct drm_device *dev = vq->vdev->priv; struct virtio_gpu_device *vgdev = dev->dev_private; schedule_work(&vgdev->cursorq.dequeue_work); } int virtio_gpu_alloc_vbufs(struct virtio_gpu_device *vgdev) { vgdev->vbufs = kmem_cache_create("virtio-gpu-vbufs", VBUFFER_SIZE, __alignof__(struct virtio_gpu_vbuffer), 0, NULL); if (!vgdev->vbufs) return -ENOMEM; return 0; } void virtio_gpu_free_vbufs(struct virtio_gpu_device *vgdev) { kmem_cache_destroy(vgdev->vbufs); vgdev->vbufs = NULL; } /* For drm_panic */ static struct virtio_gpu_vbuffer* virtio_gpu_panic_get_vbuf(struct virtio_gpu_device *vgdev, int size) { struct virtio_gpu_vbuffer *vbuf; vbuf = kmem_cache_zalloc(vgdev->vbufs, GFP_ATOMIC); vbuf->buf = (void *)vbuf + sizeof(*vbuf); vbuf->size = size; vbuf->resp_cb = NULL; vbuf->resp_size = sizeof(struct virtio_gpu_ctrl_hdr); vbuf->resp_buf = (void *)vbuf->buf + size; return vbuf; } static struct virtio_gpu_vbuffer* virtio_gpu_get_vbuf(struct virtio_gpu_device *vgdev, int size, int resp_size, void *resp_buf, virtio_gpu_resp_cb resp_cb) { struct virtio_gpu_vbuffer *vbuf; vbuf = kmem_cache_zalloc(vgdev->vbufs, GFP_KERNEL | __GFP_NOFAIL); BUG_ON(size > MAX_INLINE_CMD_SIZE || size < sizeof(struct virtio_gpu_ctrl_hdr)); vbuf->buf = (void *)vbuf + sizeof(*vbuf); vbuf->size = size; vbuf->resp_cb = resp_cb; vbuf->resp_size = resp_size; if (resp_size <= MAX_INLINE_RESP_SIZE) vbuf->resp_buf = (void *)vbuf->buf + size; else vbuf->resp_buf = resp_buf; BUG_ON(!vbuf->resp_buf); return vbuf; } static struct virtio_gpu_ctrl_hdr * virtio_gpu_vbuf_ctrl_hdr(struct virtio_gpu_vbuffer *vbuf) { /* this assumes a vbuf contains a command that starts with a * virtio_gpu_ctrl_hdr, which is true for both ctrl and cursor * virtqueues. */ return (struct virtio_gpu_ctrl_hdr *)vbuf->buf; } static struct virtio_gpu_update_cursor* virtio_gpu_alloc_cursor(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer **vbuffer_p) { struct virtio_gpu_vbuffer *vbuf; vbuf = virtio_gpu_get_vbuf (vgdev, sizeof(struct virtio_gpu_update_cursor), 0, NULL, NULL); if (IS_ERR(vbuf)) { *vbuffer_p = NULL; return ERR_CAST(vbuf); } *vbuffer_p = vbuf; return (struct virtio_gpu_update_cursor *)vbuf->buf; } /* For drm_panic */ static void *virtio_gpu_panic_alloc_cmd_resp(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer **vbuffer_p, int cmd_size) { struct virtio_gpu_vbuffer *vbuf; vbuf = virtio_gpu_panic_get_vbuf(vgdev, cmd_size); *vbuffer_p = vbuf; return (struct virtio_gpu_command *)vbuf->buf; } static void *virtio_gpu_alloc_cmd_resp(struct virtio_gpu_device *vgdev, virtio_gpu_resp_cb cb, struct virtio_gpu_vbuffer **vbuffer_p, int cmd_size, int resp_size, void *resp_buf) { struct virtio_gpu_vbuffer *vbuf; vbuf = virtio_gpu_get_vbuf(vgdev, cmd_size, resp_size, resp_buf, cb); *vbuffer_p = vbuf; return (struct virtio_gpu_command *)vbuf->buf; } static void *virtio_gpu_alloc_cmd(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer **vbuffer_p, int size) { return virtio_gpu_alloc_cmd_resp(vgdev, NULL, vbuffer_p, size, sizeof(struct virtio_gpu_ctrl_hdr), NULL); } static void *virtio_gpu_alloc_cmd_cb(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer **vbuffer_p, int size, virtio_gpu_resp_cb cb) { return virtio_gpu_alloc_cmd_resp(vgdev, cb, vbuffer_p, size, sizeof(struct virtio_gpu_ctrl_hdr), NULL); } static void free_vbuf(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf) { if (vbuf->resp_size > MAX_INLINE_RESP_SIZE) kfree(vbuf->resp_buf); kvfree(vbuf->data_buf); kmem_cache_free(vgdev->vbufs, vbuf); } void virtio_gpu_reclaim_vbufs(struct virtio_gpu_device *vgdev) { struct virtio_gpu_vbuffer *vbuf; while ((vbuf = virtqueue_detach_unused_buf(vgdev->ctrlq.vq))) { if (vbuf->objs) virtio_gpu_array_put_free(vbuf->objs); if (vbuf->resp_cb_data) virtio_gpu_cleanup_object(vbuf->resp_cb_data); free_vbuf(vgdev, vbuf); } while ((vbuf = virtqueue_detach_unused_buf(vgdev->cursorq.vq))) free_vbuf(vgdev, vbuf); } static void reclaim_vbufs(struct virtqueue *vq, struct list_head *reclaim_list) { struct virtio_gpu_vbuffer *vbuf; unsigned int len; int freed = 0; while ((vbuf = virtqueue_get_buf(vq, &len))) { list_add_tail(&vbuf->list, reclaim_list); freed++; } if (freed == 0) DRM_DEBUG("Huh? zero vbufs reclaimed"); } void virtio_gpu_dequeue_ctrl_func(struct work_struct *work) { struct virtio_gpu_device *vgdev = container_of(work, struct virtio_gpu_device, ctrlq.dequeue_work); struct list_head reclaim_list; struct virtio_gpu_vbuffer *entry, *tmp; struct virtio_gpu_ctrl_hdr *resp; u64 fence_id; INIT_LIST_HEAD(&reclaim_list); spin_lock(&vgdev->ctrlq.qlock); do { virtqueue_disable_cb(vgdev->ctrlq.vq); reclaim_vbufs(vgdev->ctrlq.vq, &reclaim_list); } while (!virtqueue_enable_cb(vgdev->ctrlq.vq)); spin_unlock(&vgdev->ctrlq.qlock); list_for_each_entry(entry, &reclaim_list, list) { resp = (struct virtio_gpu_ctrl_hdr *)entry->resp_buf; trace_virtio_gpu_cmd_response(vgdev->ctrlq.vq, resp, entry->seqno); if (resp->type != cpu_to_le32(VIRTIO_GPU_RESP_OK_NODATA)) { if (le32_to_cpu(resp->type) >= VIRTIO_GPU_RESP_ERR_UNSPEC) { struct virtio_gpu_ctrl_hdr *cmd; cmd = virtio_gpu_vbuf_ctrl_hdr(entry); DRM_ERROR_RATELIMITED("response 0x%x (command 0x%x)\n", le32_to_cpu(resp->type), le32_to_cpu(cmd->type)); } else DRM_DEBUG("response 0x%x\n", le32_to_cpu(resp->type)); } if (resp->flags & cpu_to_le32(VIRTIO_GPU_FLAG_FENCE)) { fence_id = le64_to_cpu(resp->fence_id); virtio_gpu_fence_event_process(vgdev, fence_id); } if (entry->resp_cb) entry->resp_cb(vgdev, entry); } wake_up(&vgdev->ctrlq.ack_queue); list_for_each_entry_safe(entry, tmp, &reclaim_list, list) { if (entry->objs) virtio_gpu_array_put_free_delayed(vgdev, entry->objs); list_del(&entry->list); free_vbuf(vgdev, entry); } } void virtio_gpu_dequeue_cursor_func(struct work_struct *work) { struct virtio_gpu_device *vgdev = container_of(work, struct virtio_gpu_device, cursorq.dequeue_work); struct list_head reclaim_list; struct virtio_gpu_vbuffer *entry, *tmp; INIT_LIST_HEAD(&reclaim_list); spin_lock(&vgdev->cursorq.qlock); do { virtqueue_disable_cb(vgdev->cursorq.vq); reclaim_vbufs(vgdev->cursorq.vq, &reclaim_list); } while (!virtqueue_enable_cb(vgdev->cursorq.vq)); spin_unlock(&vgdev->cursorq.qlock); list_for_each_entry_safe(entry, tmp, &reclaim_list, list) { struct virtio_gpu_ctrl_hdr *resp = (struct virtio_gpu_ctrl_hdr *)entry->resp_buf; trace_virtio_gpu_cmd_response(vgdev->cursorq.vq, resp, entry->seqno); list_del(&entry->list); free_vbuf(vgdev, entry); } wake_up(&vgdev->cursorq.ack_queue); } /* Create sg_table from a vmalloc'd buffer. */ static struct sg_table *vmalloc_to_sgt(char *data, uint32_t size, int *sg_ents) { int ret, s, i; struct sg_table *sgt; struct scatterlist *sg; struct page *pg; if (WARN_ON(!PAGE_ALIGNED(data))) return NULL; sgt = kmalloc_obj(*sgt); if (!sgt) return NULL; *sg_ents = DIV_ROUND_UP(size, PAGE_SIZE); ret = sg_alloc_table(sgt, *sg_ents, GFP_KERNEL); if (ret) { kfree(sgt); return NULL; } for_each_sgtable_sg(sgt, sg, i) { pg = vmalloc_to_page(data); if (!pg) { sg_free_table(sgt); kfree(sgt); return NULL; } s = min_t(int, PAGE_SIZE, size); sg_set_page(sg, pg, s, 0); size -= s; data += s; } return sgt; } /* For drm_panic */ static int virtio_gpu_panic_queue_ctrl_sgs(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf, int elemcnt, struct scatterlist **sgs, int outcnt, int incnt) { struct virtqueue *vq = vgdev->ctrlq.vq; int ret; if (vgdev->has_indirect) elemcnt = 1; if (vq->num_free < elemcnt) return -ENOMEM; ret = virtqueue_add_sgs(vq, sgs, outcnt, incnt, vbuf, GFP_ATOMIC); WARN_ON(ret); vbuf->seqno = ++vgdev->ctrlq.seqno; trace_virtio_gpu_cmd_queue(vq, virtio_gpu_vbuf_ctrl_hdr(vbuf), vbuf->seqno); atomic_inc(&vgdev->pending_commands); return 0; } int virtio_gpu_wait_queue(struct virtio_gpu_queue *vgvq, unsigned int num_elem) { int ret; /* Wait up to 5 seconds for enough free slots to become available */ ret = wait_event_timeout(vgvq->ack_queue, vgvq->vq->num_free >= num_elem, 5 * HZ); if (ret == 0) return -ETIMEDOUT; return 0; } static int virtio_gpu_queue_ctrl_sgs(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf, struct virtio_gpu_fence *fence, int elemcnt, struct scatterlist **sgs, int outcnt, int incnt) { struct virtqueue *vq = vgdev->ctrlq.vq; int ret, idx; if (!drm_dev_enter(vgdev->ddev, &idx)) { if (fence && vbuf->objs) virtio_gpu_array_unlock_resv(vbuf->objs); free_vbuf(vgdev, vbuf); return -ENODEV; } if (vgdev->has_indirect) elemcnt = 1; again: spin_lock(&vgdev->ctrlq.qlock); if (vq->num_free < elemcnt) { spin_unlock(&vgdev->ctrlq.qlock); virtio_gpu_notify(vgdev); wait_event(vgdev->ctrlq.ack_queue, vq->num_free >= elemcnt || vgdev->vqs_released); /* * Set by virtio_gpu_release_vqs() to unblock * synchronize_srcu() wait in drm_dev_unplug(). */ if (vgdev->vqs_released) { if (fence && vbuf->objs) virtio_gpu_array_unlock_resv(vbuf->objs); free_vbuf(vgdev, vbuf); drm_dev_exit(idx); return -ENODEV; } goto again; } /* now that the position of the vbuf in the virtqueue is known, we can * finally set the fence id */ if (fence) { virtio_gpu_fence_emit(vgdev, virtio_gpu_vbuf_ctrl_hdr(vbuf), fence); if (vbuf->objs) { virtio_gpu_array_add_fence(vbuf->objs, &fence->f); virtio_gpu_array_unlock_resv(vbuf->objs); } } ret = virtqueue_add_sgs(vq, sgs, outcnt, incnt, vbuf, GFP_ATOMIC); WARN_ON(ret); vbuf->seqno = ++vgdev->ctrlq.seqno; trace_virtio_gpu_cmd_queue(vq, virtio_gpu_vbuf_ctrl_hdr(vbuf), vbuf->seqno); atomic_inc(&vgdev->pending_commands); spin_unlock(&vgdev->ctrlq.qlock); drm_dev_exit(idx); return 0; } /* For drm_panic */ static int virtio_gpu_panic_queue_ctrl_buffer(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf) { struct scatterlist *sgs[3], vcmd, vresp; int elemcnt = 0, outcnt = 0, incnt = 0; /* set up vcmd */ sg_init_one(&vcmd, vbuf->buf, vbuf->size); elemcnt++; sgs[outcnt] = &vcmd; outcnt++; /* set up vresp */ if (vbuf->resp_size) { sg_init_one(&vresp, vbuf->resp_buf, vbuf->resp_size); elemcnt++; sgs[outcnt + incnt] = &vresp; incnt++; } return virtio_gpu_panic_queue_ctrl_sgs(vgdev, vbuf, elemcnt, sgs, outcnt, incnt); } static int virtio_gpu_queue_fenced_ctrl_buffer(struct virtio_gpu_device *vgdev, struct virtio_gpu_vbuffer *vbuf, struct virtio_gpu_fence *fence) { struct scatterlist *sgs[3], vcmd, vout, vresp;