| @@ -96,11 +96,17 @@ static void vdpasim_do_reset(struct vdpasim *vdpasim) · 95 unmodified lines |
| 96 | { | 96 | { |
| 97 | int i; | 97 | int i; |
| 98 | | 98 | |
| 99 | for (i = 0; i < vdpasim->dev_attr.nvqs; i++) | 99 | spin_lock(&vdpasim->iommu_lock); |
| | 100 | |
| | 101 | for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { |
| 100 | vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]); | 102 | vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]); |
| | 103 | vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0], |
| | 104 | &vdpasim->iommu_lock); |
| | 105 | } |
| | 106 | |
| | 107 | for (i = 0; i < vdpasim->dev_attr.nas; i++) |
| | 108 | vhost_iotlb_reset(&vdpasim->iommu[i]); |
| 101 | | 109 | |
| 102 | spin_lock(&vdpasim->iommu_lock); | | |
| 103 | vhost_iotlb_reset(vdpasim->iommu); | | |
| 104 | spin_unlock(&vdpasim->iommu_lock); | 110 | spin_unlock(&vdpasim->iommu_lock); |
| 105 | | 111 | |
| 106 | vdpasim->features = 0; | 112 | vdpasim->features = 0; |
| @@ -145,7 +151,7 @@ static dma_addr_t vdpasim_map_range(struct vdpasim *vdpasim, phys_addr_t paddr, · 38 unmodified lines |
| 145 | dma_addr = iova_dma_addr(&vdpasim->iova, iova); | 151 | dma_addr = iova_dma_addr(&vdpasim->iova, iova); |
| 146 | | 152 | |
| 147 | spin_lock(&vdpasim->iommu_lock); | 153 | spin_lock(&vdpasim->iommu_lock); |
| 148 | ret = vhost_iotlb_add_range(vdpasim->iommu, (u64)dma_addr, | 154 | ret = vhost_iotlb_add_range(&vdpasim->iommu[0], (u64)dma_addr, |
| 149 | (u64)dma_addr + size - 1, (u64)paddr, perm); | 155 | (u64)dma_addr + size - 1, (u64)paddr, perm); |
| 150 | spin_unlock(&vdpasim->iommu_lock); | 156 | spin_unlock(&vdpasim->iommu_lock); |
| 151 | | 157 | |
| @@ -161,7 +167,7 @@ static void vdpasim_unmap_range(struct vdpasim *vdpasim, dma_addr_t dma_addr, · 9 unmodified lines |
| 161 | size_t size) | 167 | size_t size) |
| 162 | { | 168 | { |
| 163 | spin_lock(&vdpasim->iommu_lock); | 169 | spin_lock(&vdpasim->iommu_lock); |
| 164 | vhost_iotlb_del_range(vdpasim->iommu, (u64)dma_addr, | 170 | vhost_iotlb_del_range(&vdpasim->iommu[0], (u64)dma_addr, |
| 165 | (u64)dma_addr + size - 1); | 171 | (u64)dma_addr + size - 1); |
| 166 | spin_unlock(&vdpasim->iommu_lock); | 172 | spin_unlock(&vdpasim->iommu_lock); |
| 167 | | 173 | |
| @@ -250,8 +256,9 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr) · 82 unmodified lines |
| 250 | else | 256 | else |
| 251 | ops = &vdpasim_config_ops; | 257 | ops = &vdpasim_config_ops; |
| 252 | | 258 | |
| 253 | vdpasim = vdpa_alloc_device(struct vdpasim, vdpa, NULL, ops, 1, | 259 | vdpasim = vdpa_alloc_device(struct vdpasim, vdpa, NULL, ops, |
| 254 | 1, dev_attr->name, false); | 260 | dev_attr->ngroups, dev_attr->nas, |
| | 261 | dev_attr->name, false); |
| 255 | if (IS_ERR(vdpasim)) { | 262 | if (IS_ERR(vdpasim)) { |
| 256 | ret = PTR_ERR(vdpasim); | 263 | ret = PTR_ERR(vdpasim); |
| 257 | goto err_alloc; | 264 | goto err_alloc; |
| @@ -278,16 +285,20 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr) · 20 unmodified lines |
| 278 | if (!vdpasim->vqs) | 285 | if (!vdpasim->vqs) |
| 279 | goto err_iommu; | 286 | goto err_iommu; |
| 280 | | 287 | |
| 281 | vdpasim->iommu = vhost_iotlb_alloc(max_iotlb_entries, 0); | 288 | vdpasim->iommu = kmalloc_array(vdpasim->dev_attr.nas, |
| | 289 | sizeof(*vdpasim->iommu), GFP_KERNEL); |
| 282 | if (!vdpasim->iommu) | 290 | if (!vdpasim->iommu) |
| 283 | goto err_iommu; | 291 | goto err_iommu; |
| 284 | | 292 | |
| | 293 | for (i = 0; i < vdpasim->dev_attr.nas; i++) |
| | 294 | vhost_iotlb_init(&vdpasim->iommu[i], 0, 0); |
| | 295 | |
| 285 | vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL); | 296 | vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL); |
| 286 | if (!vdpasim->buffer) | 297 | if (!vdpasim->buffer) |
| 287 | goto err_iommu; | 298 | goto err_iommu; |
| 288 | | 299 | |
| 289 | for (i = 0; i < dev_attr->nvqs; i++) | 300 | for (i = 0; i < dev_attr->nvqs; i++) |
| 290 | vringh_set_iotlb(&vdpasim->vqs[i].vring, vdpasim->iommu, | 301 | vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0], |
| 291 | &vdpasim->iommu_lock); | 302 | &vdpasim->iommu_lock); |
| 292 | | 303 | |
| 293 | ret = iova_cache_get(); | 304 | ret = iova_cache_get(); |
| @@ -401,7 +412,11 @@ static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa) · 107 unmodified lines |
| 401 | | 412 | |
| 402 | static u32 vdpasim_get_vq_group(struct vdpa_device *vdpa, u16 idx) | 413 | static u32 vdpasim_get_vq_group(struct vdpa_device *vdpa, u16 idx) |
| 403 | { | 414 | { |
| 404 | return 0; | 415 | /* RX and TX belongs to group 0, CVQ belongs to group 1 */ |
| | 416 | if (idx == 2) |
| | 417 | return 1; |
| | 418 | else |
| | 419 | return 0; |
| 405 | } | 420 | } |
| 406 | | 421 | |
| 407 | static u64 vdpasim_get_device_features(struct vdpa_device *vdpa) | 422 | static u64 vdpasim_get_device_features(struct vdpa_device *vdpa) |
| @@ -539,20 +554,53 @@ static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa) · 131 unmodified lines |
| 539 | return range; | 554 | return range; |
| 540 | } | 555 | } |
| 541 | | 556 | |
| | 557 | static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group, |
| | 558 | unsigned int asid) |
| | 559 | { |
| | 560 | struct vdpasim *vdpasim = vdpa_to_sim(vdpa); |
| | 561 | struct vhost_iotlb *iommu; |
| | 562 | int i; |
| | 563 | |
| | 564 | if (group > vdpasim->dev_attr.ngroups) |
| | 565 | return -EINVAL; |
| | 566 | |
| | 567 | if (asid > vdpasim->dev_attr.nas) |
| | 568 | return -EINVAL; |
| | 569 | |
| | 570 | iommu = &vdpasim->iommu[asid]; |
| | 571 | |
| | 572 | spin_lock(&vdpasim->lock); |
| | 573 | |
| | 574 | for (i = 0; i < vdpasim->dev_attr.nvqs; i++) |
| | 575 | if (vdpasim_get_vq_group(vdpa, i) == group) |
| | 576 | vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu, |
| | 577 | &vdpasim->iommu_lock); |
| | 578 | |
| | 579 | spin_unlock(&vdpasim->lock); |
| | 580 | |
| | 581 | return 0; |
| | 582 | } |
| | 583 | |
| 542 | static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid, | 584 | static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid, |
| 543 | struct vhost_iotlb *iotlb) | 585 | struct vhost_iotlb *iotlb) |
| 544 | { | 586 | { |
| 545 | struct vdpasim *vdpasim = vdpa_to_sim(vdpa); | 587 | struct vdpasim *vdpasim = vdpa_to_sim(vdpa); |
| 546 | struct vhost_iotlb_map *map; | 588 | struct vhost_iotlb_map *map; |
| | 589 | struct vhost_iotlb *iommu; |
| 547 | u64 start = 0ULL, last = 0ULL - 1; | 590 | u64 start = 0ULL, last = 0ULL - 1; |
| 548 | int ret; | 591 | int ret; |
| 549 | | 592 | |
| | 593 | if (asid >= vdpasim->dev_attr.nas) |
| | 594 | return -EINVAL; |
| | 595 | |
| 550 | spin_lock(&vdpasim->iommu_lock); | 596 | spin_lock(&vdpasim->iommu_lock); |
| 551 | vhost_iotlb_reset(vdpasim->iommu); | 597 | |
| | 598 | iommu = &vdpasim->iommu[asid]; |
| | 599 | vhost_iotlb_reset(iommu); |
| 552 | | 600 | |
| 553 | for (map = vhost_iotlb_itree_first(iotlb, start, last); map; | 601 | for (map = vhost_iotlb_itree_first(iotlb, start, last); map; |
| 554 | map = vhost_iotlb_itree_next(map, start, last)) { | 602 | map = vhost_iotlb_itree_next(map, start, last)) { |
| 555 | ret = vhost_iotlb_add_range(vdpasim->iommu, map->start, | 603 | ret = vhost_iotlb_add_range(iommu, map->start, |
| 556 | map->last, map->addr, map->perm); | 604 | map->last, map->addr, map->perm); |
| 557 | if (ret) | 605 | if (ret) |
| 558 | goto err; | 606 | goto err; |
| @@ -561,7 +609,7 @@ static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid, · 2 unmodified lines |
| 561 | return 0; | 609 | return 0; |
| 562 | | 610 | |
| 563 | err: | 611 | err: |
| 564 | vhost_iotlb_reset(vdpasim->iommu); | 612 | vhost_iotlb_reset(iommu); |
| 565 | spin_unlock(&vdpasim->iommu_lock); | 613 | spin_unlock(&vdpasim->iommu_lock); |
| 566 | return ret; | 614 | return ret; |
| 567 | } | 615 | } |
| @@ -573,9 +621,12 @@ static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid, · 5 unmodified lines |
| 573 | struct vdpasim *vdpasim = vdpa_to_sim(vdpa); | 621 | struct vdpasim *vdpasim = vdpa_to_sim(vdpa); |
| 574 | int ret; | 622 | int ret; |
| 575 | | 623 | |
| | 624 | if (asid >= vdpasim->dev_attr.nas) |
| | 625 | return -EINVAL; |
| | 626 | |
| 576 | spin_lock(&vdpasim->iommu_lock); | 627 | spin_lock(&vdpasim->iommu_lock); |
| 577 | ret = vhost_iotlb_add_range_ctx(vdpasim->iommu, iova, iova + size - 1, | 628 | ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova, |
| 578 | pa, perm, opaque); | 629 | iova + size - 1, pa, perm, opaque); |
| 579 | spin_unlock(&vdpasim->iommu_lock); | 630 | spin_unlock(&vdpasim->iommu_lock); |
| 580 | | 631 | |
| 581 | return ret; | 632 | return ret; |
| @@ -586,8 +637,11 @@ static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid, · 4 unmodified lines |
| 586 | { | 637 | { |
| 587 | struct vdpasim *vdpasim = vdpa_to_sim(vdpa); | 638 | struct vdpasim *vdpasim = vdpa_to_sim(vdpa); |
| 588 | | 639 | |
| | 640 | if (asid >= vdpasim->dev_attr.nas) |
| | 641 | return -EINVAL; |
| | 642 | |
| 589 | spin_lock(&vdpasim->iommu_lock); | 643 | spin_lock(&vdpasim->iommu_lock); |
| 590 | vhost_iotlb_del_range(vdpasim->iommu, iova, iova + size - 1); | 644 | vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1); |
| 591 | spin_unlock(&vdpasim->iommu_lock); | 645 | spin_unlock(&vdpasim->iommu_lock); |
| 592 | | 646 | |
| 593 | return 0; | 647 | return 0; |
| @@ -611,8 +665,7 @@ static void vdpasim_free(struct vdpa_device *vdpa) · 17 unmodified lines |
| 611 | } | 665 | } |
| 612 | | 666 | |
| 613 | kvfree(vdpasim->buffer); | 667 | kvfree(vdpasim->buffer); |
| 614 | if (vdpasim->iommu) | 668 | vhost_iotlb_free(vdpasim->iommu); |
| 615 | vhost_iotlb_free(vdpasim->iommu); | | |
| 616 | kfree(vdpasim->vqs); | 669 | kfree(vdpasim->vqs); |
| 617 | kfree(vdpasim->config); | 670 | kfree(vdpasim->config); |
| 618 | } | 671 | } |
| @@ -643,6 +696,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = { · 24 unmodified lines |
| 643 | .set_config = vdpasim_set_config, | 696 | .set_config = vdpasim_set_config, |
| 644 | .get_generation = vdpasim_get_generation, | 697 | .get_generation = vdpasim_get_generation, |
| 645 | .get_iova_range = vdpasim_get_iova_range, | 698 | .get_iova_range = vdpasim_get_iova_range, |
| | 699 | .set_group_asid = vdpasim_set_group_asid, |
| 646 | .dma_map = vdpasim_dma_map, | 700 | .dma_map = vdpasim_dma_map, |
| 647 | .dma_unmap = vdpasim_dma_unmap, | 701 | .dma_unmap = vdpasim_dma_unmap, |
| 648 | .free = vdpasim_free, | 702 | .free = vdpasim_free, |
| @@ -674,6 +728,7 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = { · 25 unmodified lines |
| 674 | .set_config = vdpasim_set_config, | 728 | .set_config = vdpasim_set_config, |
| 675 | .get_generation = vdpasim_get_generation, | 729 | .get_generation = vdpasim_get_generation, |
| 676 | .get_iova_range = vdpasim_get_iova_range, | 730 | .get_iova_range = vdpasim_get_iova_range, |
| | 731 | .set_group_asid = vdpasim_set_group_asid, |
| 677 | .set_map = vdpasim_set_map, | 732 | .set_map = vdpasim_set_map, |
| 678 | .free = vdpasim_free, | 733 | .free = vdpasim_free, |
| 679 | }; | 734 | }; |