When working with SRCU protected xarrays the xarray itself should be the SRCU 'update' point. Instead prefetch is using live as the SRCU update point and this prevents switching the locking design to use the xarray instead. To solve this the prefetch must only read from the xarray once, and hold on to the actual MR pointer for the duration of the async operation. Incrementing num_pending_prefetch delays destruction of the MR, so it is suitable. Prefetch calls directly to the pagefault_mr using the MR pointer and only does a single xarray lookup. All the testing if a MR is prefetchable or not is now done only in the prefetch code and removed from the pagefault critical path.
Unified · Split
| @@ -606,16 +606,13 @@ void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) · 605 unmodified lines | |||
| 606 | wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free)); | 606 | wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free)); |
| 607 | } | 607 | } |
| 608 | 608 | ||
| 609 | #define MLX5_PF_FLAGS_PREFETCH BIT(0) | ||
| 610 | #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) | 609 | #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) |
| 611 | static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, | 610 | static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, |
| 612 | u64 io_virt, size_t bcnt, u32 *bytes_mapped, | 611 | u32 *bytes_mapped, u32 flags) |
| 613 | u32 flags) | ||
| 614 | { | 612 | { |
| 615 | int npages = 0, current_seq, page_shift, ret, np; | 613 | int npages = 0, current_seq, page_shift, ret, np; |
| 616 | struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); | 614 | struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); |
| 617 | bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; | 615 | bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; |
| 618 | bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; | ||
| 619 | u64 access_mask; | 616 | u64 access_mask; |
| 620 | u64 start_idx, page_mask; | 617 | u64 start_idx, page_mask; |
| 621 | struct ib_umem_odp *odp; | 618 | struct ib_umem_odp *odp; |
| @@ -639,14 +636,6 @@ next_mr: · 17 unmodified lines | |||
| 639 | start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; | 636 | start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; |
| 640 | access_mask = ODP_READ_ALLOWED_BIT; | 637 | access_mask = ODP_READ_ALLOWED_BIT; |
| 641 | 638 | ||
| 642 | if (prefetch && !downgrade && !odp->umem.writable) { | ||
| 643 | /* prefetch with write-access must | ||
| 644 | * be supported by the MR | ||
| 645 | */ | ||
| 646 | ret = -EINVAL; | ||
| 647 | goto out; | ||
| 648 | } | ||
| 649 | |||
| 650 | if (odp->umem.writable && !downgrade) | 639 | if (odp->umem.writable && !downgrade) |
| 651 | access_mask |= ODP_WRITE_ALLOWED_BIT; | 640 | access_mask |= ODP_WRITE_ALLOWED_BIT; |
| 652 | 641 | ||
| @@ -681,7 +670,8 @@ next_mr: · 28 unmodified lines | |||
| 681 | 670 | ||
| 682 | if (ret < 0) { | 671 | if (ret < 0) { |
| 683 | if (ret != -EAGAIN) | 672 | if (ret != -EAGAIN) |
| 684 | mlx5_ib_err(dev, "Failed to update mkey page tables\n"); | 673 | mlx5_ib_err(mr->dev, |
| 674 | "Failed to update mkey page tables\n"); | ||
| 685 | goto out; | 675 | goto out; |
| 686 | } | 676 | } |
| 687 | 677 | ||
| @@ -700,8 +690,10 @@ next_mr: · 12 unmodified lines | |||
| 700 | io_virt += size; | 690 | io_virt += size; |
| 701 | next = odp_next(odp); | 691 | next = odp_next(odp); |
| 702 | if (unlikely(!next || ib_umem_start(next) != io_virt)) { | 692 | if (unlikely(!next || ib_umem_start(next) != io_virt)) { |
| 703 | mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n", | 693 | mlx5_ib_dbg( |
| 704 | io_virt, next); | 694 | mr->dev, |
| 695 | "next implicit leaf removed at 0x%llx. got %p\n", | ||
| 696 | io_virt, next); | ||
| 705 | return -EAGAIN; | 697 | return -EAGAIN; |
| 706 | } | 698 | } |
| 707 | odp = next; | 699 | odp = next; |
| @@ -718,7 +710,7 @@ out: · 10 unmodified lines | |||
| 718 | if (!wait_for_completion_timeout(&odp->notifier_completion, | 710 | if (!wait_for_completion_timeout(&odp->notifier_completion, |
| 719 | timeout)) { | 711 | timeout)) { |
| 720 | mlx5_ib_warn( | 712 | mlx5_ib_warn( |
| 721 | dev, | 713 | mr->dev, |
| 722 | "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n", | 714 | "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n", |
| 723 | current_seq, odp->notifiers_seq, | 715 | current_seq, odp->notifiers_seq, |
| 724 | odp->notifiers_count); | 716 | odp->notifiers_count); |
| @@ -775,10 +767,9 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, · 50 unmodified lines | |||
| 775 | struct ib_pd *pd, u32 key, | 767 | struct ib_pd *pd, u32 key, |
| 776 | u64 io_virt, size_t bcnt, | 768 | u64 io_virt, size_t bcnt, |
| 777 | u32 *bytes_committed, | 769 | u32 *bytes_committed, |
| 778 | u32 *bytes_mapped, u32 flags) | 770 | u32 *bytes_mapped) |
| 779 | { | 771 | { |
| 780 | int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; | 772 | int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; |
| 781 | bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; | ||
| 782 | struct pf_frame *head = NULL, *frame; | 773 | struct pf_frame *head = NULL, *frame; |
| 783 | struct mlx5_core_mkey *mmkey; | 774 | struct mlx5_core_mkey *mmkey; |
| 784 | struct mlx5_ib_mr *mr; | 775 | struct mlx5_ib_mr *mr; |
| @@ -800,12 +791,6 @@ next_mr: · 15 unmodified lines | |||
| 800 | goto srcu_unlock; | 791 | goto srcu_unlock; |
| 801 | } | 792 | } |
| 802 | 793 | ||
| 803 | if (prefetch && mmkey->type != MLX5_MKEY_MR) { | ||
| 804 | mlx5_ib_dbg(dev, "prefetch is allowed only for MR\n"); | ||
| 805 | ret = -EINVAL; | ||
| 806 | goto srcu_unlock; | ||
| 807 | } | ||
| 808 | |||
| 809 | switch (mmkey->type) { | 794 | switch (mmkey->type) { |
| 810 | case MLX5_MKEY_MR: | 795 | case MLX5_MKEY_MR: |
| 811 | mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); | 796 | mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); |
| @@ -815,17 +800,6 @@ next_mr: · 3 unmodified lines | |||
| 815 | goto srcu_unlock; | 800 | goto srcu_unlock; |
| 816 | } | 801 | } |
| 817 | 802 | ||
| 818 | if (prefetch) { | ||
| 819 | if (!is_odp_mr(mr) || | ||
| 820 | mr->ibmr.pd != pd) { | ||
| 821 | mlx5_ib_dbg(dev, "Invalid prefetch request: %s\n", | ||
| 822 | is_odp_mr(mr) ? "MR is not ODP" : | ||
| 823 | "PD is not of the MR"); | ||
| 824 | ret = -EINVAL; | ||
| 825 | goto srcu_unlock; | ||
| 826 | } | ||
| 827 | } | ||
| 828 | |||
| 829 | if (!is_odp_mr(mr)) { | 803 | if (!is_odp_mr(mr)) { |
| 830 | mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", | 804 | mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", |
| 831 | key); | 805 | key); |
| @@ -835,7 +809,7 @@ next_mr: · 3 unmodified lines | |||
| 835 | goto srcu_unlock; | 809 | goto srcu_unlock; |
| 836 | } | 810 | } |
| 837 | 811 | ||
| 838 | ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped, flags); | 812 | ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0); |
| 839 | if (ret < 0) | 813 | if (ret < 0) |
| 840 | goto srcu_unlock; | 814 | goto srcu_unlock; |
| 841 | 815 | ||
| @@ -1009,7 +983,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 167 unmodified lines | |||
| 1009 | ret = pagefault_single_data_segment(dev, NULL, key, | 983 | ret = pagefault_single_data_segment(dev, NULL, key, |
| 1010 | io_virt, bcnt, | 984 | io_virt, bcnt, |
| 1011 | &pfault->bytes_committed, | 985 | &pfault->bytes_committed, |
| 1012 | bytes_mapped, 0); | 986 | bytes_mapped); |
| 1013 | if (ret < 0) | 987 | if (ret < 0) |
| 1014 | break; | 988 | break; |
| 1015 | npages += ret; | 989 | npages += ret; |
| @@ -1292,8 +1266,7 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, · 276 unmodified lines | |||
| 1292 | } | 1266 | } |
| 1293 | 1267 | ||
| 1294 | ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, | 1268 | ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, |
| 1295 | &pfault->bytes_committed, NULL, | 1269 | &pfault->bytes_committed, NULL); |
| 1296 | 0); | ||
| 1297 | if (ret == -EAGAIN) { | 1270 | if (ret == -EAGAIN) { |
| 1298 | /* We're racing with an invalidation, don't prefetch */ | 1271 | /* We're racing with an invalidation, don't prefetch */ |
| 1299 | prefetch_activated = 0; | 1272 | prefetch_activated = 0; |
| @@ -1320,8 +1293,7 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, · 20 unmodified lines | |||
| 1320 | 1293 | ||
| 1321 | ret = pagefault_single_data_segment(dev, NULL, rkey, address, | 1294 | ret = pagefault_single_data_segment(dev, NULL, rkey, address, |
| 1322 | prefetch_len, | 1295 | prefetch_len, |
| 1323 | &bytes_committed, NULL, | 1296 | &bytes_committed, NULL); |
| 1324 | 0); | ||
| 1325 | if (ret < 0 && ret != -EAGAIN) { | 1297 | if (ret < 0 && ret != -EAGAIN) { |
| 1326 | mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", | 1298 | mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", |
| 1327 | ret, pfault->token, address, prefetch_len); | 1299 | ret, pfault->token, address, prefetch_len); |
| @@ -1624,114 +1596,138 @@ int mlx5_ib_odp_init(void) · 296 unmodified lines | |||
| 1624 | 1596 | ||
| 1625 | struct prefetch_mr_work { | 1597 | struct prefetch_mr_work { |
| 1626 | struct work_struct work; | 1598 | struct work_struct work; |
| 1627 | struct ib_pd *pd; | ||
| 1628 | u32 pf_flags; | 1599 | u32 pf_flags; |
| 1629 | u32 num_sge; | 1600 | u32 num_sge; |
| 1630 | struct ib_sge sg_list[0]; | 1601 | struct { |
| 1602 | u64 io_virt; | ||
| 1603 | struct mlx5_ib_mr *mr; | ||
| 1604 | size_t length; | ||
| 1605 | } frags[]; | ||
| 1631 | }; | 1606 | }; |
| 1632 | 1607 | ||
| 1633 | static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev, | 1608 | static void destroy_prefetch_work(struct prefetch_mr_work *work) |
| 1634 | struct ib_sge *sg_list, u32 num_sge, | ||
| 1635 | u32 from) | ||
| 1636 | { | 1609 | { |
| 1637 | u32 i; | 1610 | u32 i; |
| 1638 | int srcu_key; | ||
| 1639 | |||
| 1640 | srcu_key = srcu_read_lock(&dev->mr_srcu); | ||
| 1641 | 1611 | ||
| 1642 | for (i = from; i < num_sge; ++i) { | 1612 | for (i = 0; i < work->num_sge; ++i) |
| 1643 | struct mlx5_core_mkey *mmkey; | 1613 | atomic_dec(&work->frags[i].mr->num_pending_prefetch); |
| 1644 | struct mlx5_ib_mr *mr; | 1614 | kvfree(work); |
| 1645 | |||
| 1646 | mmkey = xa_load(&dev->mdev->priv.mkey_table, | ||
| 1647 | mlx5_base_mkey(sg_list[i].lkey)); | ||
| 1648 | mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); | ||
| 1649 | atomic_dec(&mr->num_pending_prefetch); | ||
| 1650 | } | ||
| 1651 | |||
| 1652 | srcu_read_unlock(&dev->mr_srcu, srcu_key); | ||
| 1653 | } | 1615 | } |
| 1654 | 1616 | ||
| 1655 | static bool num_pending_prefetch_inc(struct ib_pd *pd, | 1617 | static struct mlx5_ib_mr * |
| 1656 | struct ib_sge *sg_list, u32 num_sge) | 1618 | get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, |
| 1619 | u32 lkey) | ||
| 1657 | { | 1620 | { |
| 1658 | struct mlx5_ib_dev *dev = to_mdev(pd->device); | 1621 | struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 1659 | bool ret = true; | 1622 | struct mlx5_core_mkey *mmkey; |
| 1660 | u32 i; | 1623 | struct ib_umem_odp *odp; |
| 1624 | struct mlx5_ib_mr *mr; | ||
| 1661 | 1625 | ||
| 1662 | for (i = 0; i < num_sge; ++i) { | 1626 | lockdep_assert_held(&dev->mr_srcu); |
| 1663 | struct mlx5_core_mkey *mmkey; | ||
| 1664 | struct mlx5_ib_mr *mr; | ||
| 1665 | 1627 | ||
| 1666 | mmkey = xa_load(&dev->mdev->priv.mkey_table, | 1628 | mmkey = xa_load(&dev->mdev->priv.mkey_table, mlx5_base_mkey(lkey)); |
| 1667 | mlx5_base_mkey(sg_list[i].lkey)); | 1629 | if (!mmkey || mmkey->key != lkey || mmkey->type != MLX5_MKEY_MR) |
| 1668 | if (!mmkey || mmkey->key != sg_list[i].lkey) { | 1630 | return NULL; |
| 1669 | ret = false; | ||
| 1670 | break; | ||
| 1671 | } | ||
| 1672 | 1631 | ||
| 1673 | if (mmkey->type != MLX5_MKEY_MR) { | 1632 | mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); |
| 1674 | ret = false; | ||
| 1675 | break; | ||
| 1676 | } | ||
| 1677 | 1633 | ||
| 1678 | mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); | 1634 | if (!smp_load_acquire(&mr->live)) |
| 1635 | return NULL; | ||
| 1679 | 1636 | ||
| 1680 | if (!smp_load_acquire(&mr->live)) { | 1637 | if (mr->ibmr.pd != pd || !is_odp_mr(mr)) |
| 1681 | ret = false; | 1638 | return NULL; |
| 1682 | break; | ||
| 1683 | } | ||
| 1684 | 1639 | ||
| 1685 | if (mr->ibmr.pd != pd) { | 1640 | /* |
| 1686 | ret = false; | 1641 | * Implicit child MRs are internal and userspace should not refer to |
| 1687 | break; | 1642 | * them. |
| 1688 | } | 1643 | */ |
| 1644 | if (mr->parent) | ||
| 1645 | return NULL; | ||
| 1689 | 1646 | ||
| 1690 | atomic_inc(&mr->num_pending_prefetch); | 1647 | odp = to_ib_umem_odp(mr->umem); |
| 1691 | } | ||
| 1692 | 1648 | ||
| 1693 | if (!ret) | 1649 | /* prefetch with write-access must be supported by the MR */ |
| 1694 | num_pending_prefetch_dec(dev, sg_list, i, 0); | 1650 | if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && |
| 1651 | !odp->umem.writable) | ||
| 1652 | return NULL; | ||
| 1695 | 1653 | ||
| 1696 | return ret; | 1654 | return mr; |
| 1697 | } | 1655 | } |
| 1698 | 1656 | ||
| 1699 | static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, u32 pf_flags, | 1657 | static void mlx5_ib_prefetch_mr_work(struct work_struct *w) |
| 1700 | struct ib_sge *sg_list, u32 num_sge) | ||
| 1701 | { | 1658 | { |
| 1659 | struct prefetch_mr_work *work = | ||
| 1660 | container_of(w, struct prefetch_mr_work, work); | ||
| 1661 | u32 bytes_mapped = 0; | ||
| 1702 | u32 i; | 1662 | u32 i; |
| 1703 | int ret = 0; | 1663 | |
| 1704 | struct mlx5_ib_dev *dev = to_mdev(pd->device); | 1664 | for (i = 0; i < work->num_sge; ++i) |
| 1665 | pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, | ||
| 1666 | work->frags[i].length, &bytes_mapped, | ||
| 1667 | work->pf_flags); | ||
| 1668 | |||
| 1669 | destroy_prefetch_work(work); | ||
| 1670 | } | ||
| 1671 | |||
| 1672 | static bool init_prefetch_work(struct ib_pd *pd, | ||
| 1673 | enum ib_uverbs_advise_mr_advice advice, | ||
| 1674 | u32 pf_flags, struct prefetch_mr_work *work, | ||
| 1675 | struct ib_sge *sg_list, u32 num_sge) | ||
| 1676 | { | ||
| 1677 | u32 i; | ||
| 1678 | |||
| 1679 | INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); | ||
| 1680 | work->pf_flags = pf_flags; | ||
| 1705 | 1681 | ||
| 1706 | for (i = 0; i < num_sge; ++i) { | 1682 | for (i = 0; i < num_sge; ++i) { |
| 1707 | struct ib_sge *sg = &sg_list[i]; | 1683 | work->frags[i].io_virt = sg_list[i].addr; |
| 1708 | int bytes_committed = 0; | 1684 | work->frags[i].length = sg_list[i].length; |
| 1685 | work->frags[i].mr = | ||
| 1686 | get_prefetchable_mr(pd, advice, sg_list[i].lkey); | ||
| 1687 | if (!work->frags[i].mr) { | ||
| 1688 | work->num_sge = i - 1; | ||
| 1689 | if (i) | ||
| 1690 | destroy_prefetch_work(work); | ||
| 1691 | return false; | ||
| 1692 | } | ||
| 1709 | 1693 | ||
| 1710 | ret = pagefault_single_data_segment(dev, pd, sg->lkey, sg->addr, | 1694 | /* Keep the MR pointer will valid outside the SRCU */ |
| 1711 | sg->length, | 1695 | atomic_inc(&work->frags[i].mr->num_pending_prefetch); |
| 1712 | &bytes_committed, NULL, | ||
| 1713 | pf_flags); | ||
| 1714 | if (ret < 0) | ||
| 1715 | break; | ||
| 1716 | } | 1696 | } |
| 1717 | 1697 | work->num_sge = num_sge; | |
| 1718 | return ret < 0 ? ret : 0; | 1698 | return true; |
| 1719 | } | 1699 | } |
| 1720 | 1700 | ||
| 1721 | static void mlx5_ib_prefetch_mr_work(struct work_struct *work) | 1701 | static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, |
| 1702 | enum ib_uverbs_advise_mr_advice advice, | ||
| 1703 | u32 pf_flags, struct ib_sge *sg_list, | ||
| 1704 | u32 num_sge) | ||
| 1722 | { | 1705 | { |
| 1723 | struct prefetch_mr_work *w = | 1706 | struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 1724 | container_of(work, struct prefetch_mr_work, work); | 1707 | u32 bytes_mapped = 0; |
| 1708 | int srcu_key; | ||
| 1709 | int ret = 0; | ||
| 1710 | u32 i; | ||
| 1725 | 1711 | ||
| 1726 | if (ib_device_try_get(w->pd->device)) { | 1712 | srcu_key = srcu_read_lock(&dev->mr_srcu); |
| 1727 | mlx5_ib_prefetch_sg_list(w->pd, w->pf_flags, w->sg_list, | 1713 | for (i = 0; i < num_sge; ++i) { |
| 1728 | w->num_sge); | 1714 | struct mlx5_ib_mr *mr; |
| 1729 | ib_device_put(w->pd->device); | 1715 | |
| 1716 | mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); | ||
| 1717 | if (!mr) { | ||
| 1718 | ret = -ENOENT; | ||
| 1719 | goto out; | ||
| 1720 | } | ||
| 1721 | ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length, | ||
| 1722 | &bytes_mapped, pf_flags); | ||
| 1723 | if (ret < 0) | ||
| 1724 | goto out; | ||
| 1730 | } | 1725 | } |
| 1726 | ret = 0; | ||
| 1731 | 1727 | ||
| 1732 | num_pending_prefetch_dec(to_mdev(w->pd->device), w->sg_list, | 1728 | out: |
| 1733 | w->num_sge, 0); | 1729 | srcu_read_unlock(&dev->mr_srcu, srcu_key); |
| 1734 | kvfree(w); | 1730 | return ret; |
| 1735 | } | 1731 | } |
| 1736 | 1732 | ||
| 1737 | int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, | 1733 | int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, |
| @@ -1739,43 +1735,27 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, · 1 unmodified line | |||
| 1739 | u32 flags, struct ib_sge *sg_list, u32 num_sge) | 1735 | u32 flags, struct ib_sge *sg_list, u32 num_sge) |
| 1740 | { | 1736 | { |
| 1741 | struct mlx5_ib_dev *dev = to_mdev(pd->device); | 1737 | struct mlx5_ib_dev *dev = to_mdev(pd->device); |
| 1742 | u32 pf_flags = MLX5_PF_FLAGS_PREFETCH; | 1738 | u32 pf_flags = 0; |
| 1743 | struct prefetch_mr_work *work; | 1739 | struct prefetch_mr_work *work; |
| 1744 | bool valid_req; | ||
| 1745 | int srcu_key; | 1740 | int srcu_key; |
| 1746 | 1741 | ||
| 1747 | if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) | 1742 | if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) |
| 1748 | pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; | 1743 | pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; |
| 1749 | 1744 | ||
| 1750 | if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) | 1745 | if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) |
| 1751 | return mlx5_ib_prefetch_sg_list(pd, pf_flags, sg_list, | 1746 | return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list, |
| 1752 | num_sge); | 1747 | num_sge); |
| 1753 | 1748 | ||
| 1754 | work = kvzalloc(struct_size(work, sg_list, num_sge), GFP_KERNEL); | 1749 | work = kvzalloc(struct_size(work, frags, num_sge), GFP_KERNEL); |
| 1755 | if (!work) | 1750 | if (!work) |
| 1756 | return -ENOMEM; | 1751 | return -ENOMEM; |
| 1757 | 1752 | ||
| 1758 | memcpy(work->sg_list, sg_list, num_sge * sizeof(struct ib_sge)); | ||
| 1759 | |||
| 1760 | /* It is guaranteed that the pd when work is executed is the pd when | ||
| 1761 | * work was queued since pd can't be destroyed while it holds MRs and | ||
| 1762 | * destroying a MR leads to flushing the workquque | ||
| 1763 | */ | ||
| 1764 | work->pd = pd; | ||
| 1765 | work->pf_flags = pf_flags; | ||
| 1766 | work->num_sge = num_sge; | ||
| 1767 | |||
| 1768 | INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); | ||
| 1769 | |||
| 1770 | srcu_key = srcu_read_lock(&dev->mr_srcu); | 1753 | srcu_key = srcu_read_lock(&dev->mr_srcu); |
| 1771 | 1754 | if (!init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge)) { | |
| 1772 | valid_req = num_pending_prefetch_inc(pd, sg_list, num_sge); | 1755 | srcu_read_unlock(&dev->mr_srcu, srcu_key); |
| 1773 | if (valid_req) | 1756 | return -EINVAL; |
| 1774 | queue_work(system_unbound_wq, &work->work); | 1757 | } |
| 1775 | else | 1758 | queue_work(system_unbound_wq, &work->work); |
| 1776 | kvfree(work); | ||
| 1777 | |||
| 1778 | srcu_read_unlock(&dev->mr_srcu, srcu_key); | 1759 | srcu_read_unlock(&dev->mr_srcu, srcu_key); |
| 1779 | 1760 | return 0; | |
| 1780 | return valid_req ? 0 : -EINVAL; | ||
| 1781 | } | 1761 | } |