| @@ -783,14 +783,20 @@ static int mpi3mr_build_nvme_sgl(struct mpi3mr_ioc *mrioc, · 782 unmodified lines |
| 783 | struct mpi3mr_buf_map *drv_bufs, u8 bufcnt) | 783 | struct mpi3mr_buf_map *drv_bufs, u8 bufcnt) |
| 784 | { | 784 | { |
| 785 | struct mpi3mr_nvme_pt_sge *nvme_sgl; | 785 | struct mpi3mr_nvme_pt_sge *nvme_sgl; |
| 786 | u64 sgl_ptr; | 786 | __le64 sgl_dma; |
| 787 | u8 count; | 787 | u8 count; |
| 788 | size_t length = 0; | 788 | size_t length = 0; |
| | 789 | u16 available_sges = 0, i; |
| | 790 | u32 sge_element_size = sizeof(struct mpi3mr_nvme_pt_sge); |
| 789 | struct mpi3mr_buf_map *drv_buf_iter = drv_bufs; | 791 | struct mpi3mr_buf_map *drv_buf_iter = drv_bufs; |
| 790 | u64 sgemod_mask = ((u64)((mrioc->facts.sge_mod_mask) << | 792 | u64 sgemod_mask = ((u64)((mrioc->facts.sge_mod_mask) << |
| 791 | mrioc->facts.sge_mod_shift) << 32); | 793 | mrioc->facts.sge_mod_shift) << 32); |
| 792 | u64 sgemod_val = ((u64)(mrioc->facts.sge_mod_value) << | 794 | u64 sgemod_val = ((u64)(mrioc->facts.sge_mod_value) << |
| 793 | mrioc->facts.sge_mod_shift) << 32; | 795 | mrioc->facts.sge_mod_shift) << 32; |
| | 796 | u32 size; |
| | 797 | |
| | 798 | nvme_sgl = (struct mpi3mr_nvme_pt_sge *) |
| | 799 | ((u8 *)(nvme_encap_request->command) + MPI3MR_NVME_CMD_SGL_OFFSET); |
| 794 | | 800 | |
| 795 | /* | 801 | /* |
| 796 | * Not all commands require a data transfer. If no data, just return | 802 | * Not all commands require a data transfer. If no data, just return |
| @@ -799,27 +805,59 @@ static int mpi3mr_build_nvme_sgl(struct mpi3mr_ioc *mrioc, · 2 unmodified lines |
| 799 | for (count = 0; count < bufcnt; count++, drv_buf_iter++) { | 805 | for (count = 0; count < bufcnt; count++, drv_buf_iter++) { |
| 800 | if (drv_buf_iter->data_dir == DMA_NONE) | 806 | if (drv_buf_iter->data_dir == DMA_NONE) |
| 801 | continue; | 807 | continue; |
| 802 | sgl_ptr = (u64)drv_buf_iter->kern_buf_dma; | | |
| 803 | length = drv_buf_iter->kern_buf_len; | 808 | length = drv_buf_iter->kern_buf_len; |
| 804 | break; | 809 | break; |
| 805 | } | 810 | } |
| 806 | if (!length) | 811 | if (!length || !drv_buf_iter->num_dma_desc) |
| 807 | return 0; | 812 | return 0; |
| 808 | | 813 | |
| 809 | if (sgl_ptr & sgemod_mask) { | 814 | if (drv_buf_iter->num_dma_desc == 1) { |
| | 815 | available_sges = 1; |
| | 816 | goto build_sges; |
| | 817 | } |
| | 818 | |
| | 819 | sgl_dma = cpu_to_le64(mrioc->ioctl_chain_sge.dma_addr); |
| | 820 | if (sgl_dma & sgemod_mask) { |
| 810 | dprint_bsg_err(mrioc, | 821 | dprint_bsg_err(mrioc, |
| 811 | "%s: SGL address collides with SGE modifier\n", | 822 | "%s: SGL chain address collides with SGE modifier\n", |
| 812 | __func__); | 823 | __func__); |
| 813 | return -1; | 824 | return -1; |
| 814 | } | 825 | } |
| 815 | | 826 | |
| 816 | sgl_ptr &= ~sgemod_mask; | 827 | sgl_dma &= ~sgemod_mask; |
| 817 | sgl_ptr |= sgemod_val; | 828 | sgl_dma |= sgemod_val; |
| 818 | nvme_sgl = (struct mpi3mr_nvme_pt_sge *) | 829 | |
| 819 | ((u8 *)(nvme_encap_request->command) + MPI3MR_NVME_CMD_SGL_OFFSET); | 830 | memset(mrioc->ioctl_chain_sge.addr, 0, mrioc->ioctl_chain_sge.size); |
| | 831 | available_sges = mrioc->ioctl_chain_sge.size / sge_element_size; |
| | 832 | if (available_sges < drv_buf_iter->num_dma_desc) |
| | 833 | return -1; |
| 820 | memset(nvme_sgl, 0, sizeof(struct mpi3mr_nvme_pt_sge)); | 834 | memset(nvme_sgl, 0, sizeof(struct mpi3mr_nvme_pt_sge)); |
| 821 | nvme_sgl->base_addr = sgl_ptr; | 835 | nvme_sgl->base_addr = sgl_dma; |
| 822 | nvme_sgl->length = length; | 836 | size = drv_buf_iter->num_dma_desc * sizeof(struct mpi3mr_nvme_pt_sge); |
| | 837 | nvme_sgl->length = cpu_to_le32(size); |
| | 838 | nvme_sgl->type = MPI3MR_NVMESGL_LAST_SEGMENT; |
| | 839 | nvme_sgl = (struct mpi3mr_nvme_pt_sge *)mrioc->ioctl_chain_sge.addr; |
| | 840 | |
| | 841 | build_sges: |
| | 842 | for (i = 0; i < drv_buf_iter->num_dma_desc; i++) { |
| | 843 | sgl_dma = cpu_to_le64(drv_buf_iter->dma_desc[i].dma_addr); |
| | 844 | if (sgl_dma & sgemod_mask) { |
| | 845 | dprint_bsg_err(mrioc, |
| | 846 | "%s: SGL address collides with SGE modifier\n", |
| | 847 | __func__); |
| | 848 | return -1; |
| | 849 | } |
| | 850 | |
| | 851 | sgl_dma &= ~sgemod_mask; |
| | 852 | sgl_dma |= sgemod_val; |
| | 853 | |
| | 854 | nvme_sgl->base_addr = sgl_dma; |
| | 855 | nvme_sgl->length = cpu_to_le32(drv_buf_iter->dma_desc[i].size); |
| | 856 | nvme_sgl->type = MPI3MR_NVMESGL_DATA_SEGMENT; |
| | 857 | nvme_sgl++; |
| | 858 | available_sges--; |
| | 859 | } |
| | 860 | |
| 823 | return 0; | 861 | return 0; |
| 824 | } | 862 | } |
| 825 | | 863 | |
| @@ -847,7 +885,7 @@ static int mpi3mr_build_nvme_prp(struct mpi3mr_ioc *mrioc, · 21 unmodified lines |
| 847 | dma_addr_t prp_entry_dma, prp_page_dma, dma_addr; | 885 | dma_addr_t prp_entry_dma, prp_page_dma, dma_addr; |
| 848 | u32 offset, entry_len, dev_pgsz; | 886 | u32 offset, entry_len, dev_pgsz; |
| 849 | u32 page_mask_result, page_mask; | 887 | u32 page_mask_result, page_mask; |
| 850 | size_t length = 0; | 888 | size_t length = 0, desc_len; |
| 851 | u8 count; | 889 | u8 count; |
| 852 | struct mpi3mr_buf_map *drv_buf_iter = drv_bufs; | 890 | struct mpi3mr_buf_map *drv_buf_iter = drv_bufs; |
| 853 | u64 sgemod_mask = ((u64)((mrioc->facts.sge_mod_mask) << | 891 | u64 sgemod_mask = ((u64)((mrioc->facts.sge_mod_mask) << |
| @@ -856,6 +894,7 @@ static int mpi3mr_build_nvme_prp(struct mpi3mr_ioc *mrioc, · 2 unmodified lines |
| 856 | mrioc->facts.sge_mod_shift) << 32; | 894 | mrioc->facts.sge_mod_shift) << 32; |
| 857 | u16 dev_handle = nvme_encap_request->dev_handle; | 895 | u16 dev_handle = nvme_encap_request->dev_handle; |
| 858 | struct mpi3mr_tgt_dev *tgtdev; | 896 | struct mpi3mr_tgt_dev *tgtdev; |
| | 897 | u16 desc_count = 0; |
| 859 | | 898 | |
| 860 | tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle); | 899 | tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle); |
| 861 | if (!tgtdev) { | 900 | if (!tgtdev) { |
| @@ -874,6 +913,21 @@ static int mpi3mr_build_nvme_prp(struct mpi3mr_ioc *mrioc, · 12 unmodified lines |
| 874 | | 913 | |
| 875 | dev_pgsz = 1 << (tgtdev->dev_spec.pcie_inf.pgsz); | 914 | dev_pgsz = 1 << (tgtdev->dev_spec.pcie_inf.pgsz); |
| 876 | mpi3mr_tgtdev_put(tgtdev); | 915 | mpi3mr_tgtdev_put(tgtdev); |
| | 916 | page_mask = dev_pgsz - 1; |
| | 917 | |
| | 918 | if (dev_pgsz > MPI3MR_IOCTL_SGE_SIZE) { |
| | 919 | dprint_bsg_err(mrioc, |
| | 920 | "%s: NVMe device page size(%d) is greater than ioctl data sge size(%d) for handle 0x%04x\n", |
| | 921 | __func__, dev_pgsz, MPI3MR_IOCTL_SGE_SIZE, dev_handle); |
| | 922 | return -1; |
| | 923 | } |
| | 924 | |
| | 925 | if (MPI3MR_IOCTL_SGE_SIZE % dev_pgsz) { |
| | 926 | dprint_bsg_err(mrioc, |
| | 927 | "%s: ioctl data sge size(%d) is not a multiple of NVMe device page size(%d) for handle 0x%04x\n", |
| | 928 | __func__, MPI3MR_IOCTL_SGE_SIZE, dev_pgsz, dev_handle); |
| | 929 | return -1; |
| | 930 | } |
| 877 | | 931 | |
| 878 | /* | 932 | /* |
| 879 | * Not all commands require a data transfer. If no data, just return | 933 | * Not all commands require a data transfer. If no data, just return |
| @@ -882,14 +936,26 @@ static int mpi3mr_build_nvme_prp(struct mpi3mr_ioc *mrioc, · 2 unmodified lines |
| 882 | for (count = 0; count < bufcnt; count++, drv_buf_iter++) { | 936 | for (count = 0; count < bufcnt; count++, drv_buf_iter++) { |
| 883 | if (drv_buf_iter->data_dir == DMA_NONE) | 937 | if (drv_buf_iter->data_dir == DMA_NONE) |
| 884 | continue; | 938 | continue; |
| 885 | dma_addr = drv_buf_iter->kern_buf_dma; | | |
| 886 | length = drv_buf_iter->kern_buf_len; | 939 | length = drv_buf_iter->kern_buf_len; |
| 887 | break; | 940 | break; |
| 888 | } | 941 | } |
| 889 | | 942 | |
| 890 | if (!length) | 943 | if (!length || !drv_buf_iter->num_dma_desc) |
| 891 | return 0; | 944 | return 0; |
| 892 | | 945 | |
| | 946 | for (count = 0; count < drv_buf_iter->num_dma_desc; count++) { |
| | 947 | dma_addr = drv_buf_iter->dma_desc[count].dma_addr; |
| | 948 | if (dma_addr & page_mask) { |
| | 949 | dprint_bsg_err(mrioc, |
| | 950 | "%s:dma_addr 0x%llx is not aligned with page size 0x%x\n", |
| | 951 | __func__, dma_addr, dev_pgsz); |
| | 952 | return -1; |
| | 953 | } |
| | 954 | } |
| | 955 | |
| | 956 | dma_addr = drv_buf_iter->dma_desc[0].dma_addr; |
| | 957 | desc_len = drv_buf_iter->dma_desc[0].size; |
| | 958 | |
| 893 | mrioc->prp_sz = 0; | 959 | mrioc->prp_sz = 0; |
| 894 | mrioc->prp_list_virt = dma_alloc_coherent(&mrioc->pdev->dev, | 960 | mrioc->prp_list_virt = dma_alloc_coherent(&mrioc->pdev->dev, |
| 895 | dev_pgsz, &mrioc->prp_list_dma, GFP_KERNEL); | 961 | dev_pgsz, &mrioc->prp_list_dma, GFP_KERNEL); |
| @@ -919,7 +985,6 @@ static int mpi3mr_build_nvme_prp(struct mpi3mr_ioc *mrioc, · 23 unmodified lines |
| 919 | * Check if we are within 1 entry of a page boundary we don't | 985 | * Check if we are within 1 entry of a page boundary we don't |
| 920 | * want our first entry to be a PRP List entry. | 986 | * want our first entry to be a PRP List entry. |
| 921 | */ | 987 | */ |
| 922 | page_mask = dev_pgsz - 1; | | |
| 923 | page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask; | 988 | page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask; |
| 924 | if (!page_mask_result) { | 989 | if (!page_mask_result) { |
| 925 | dprint_bsg_err(mrioc, "%s: PRP page is not page aligned\n", | 990 | dprint_bsg_err(mrioc, "%s: PRP page is not page aligned\n", |
| @@ -1033,18 +1098,31 @@ static int mpi3mr_build_nvme_prp(struct mpi3mr_ioc *mrioc, · 107 unmodified lines |
| 1033 | prp_entry_dma += prp_size; | 1098 | prp_entry_dma += prp_size; |
| 1034 | } | 1099 | } |
| 1035 | | 1100 | |
| 1036 | /* | | |
| 1037 | * Bump the phys address of the command's data buffer by the | | |
| 1038 | * entry_len. | | |
| 1039 | */ | | |
| 1040 | dma_addr += entry_len; | | |
| 1041 | | | |
| 1042 | /* decrement length accounting for last partial page. */ | 1101 | /* decrement length accounting for last partial page. */ |
| 1043 | if (entry_len > length) | 1102 | if (entry_len >= length) { |
| 1044 | length = 0; | 1103 | length = 0; |
| 1045 | else | 1104 | } else { |
| | 1105 | if (entry_len <= desc_len) { |
| | 1106 | dma_addr += entry_len; |
| | 1107 | desc_len -= entry_len; |
| | 1108 | } |
| | 1109 | if (!desc_len) { |
| | 1110 | if ((++desc_count) >= |
| | 1111 | drv_buf_iter->num_dma_desc) { |
| | 1112 | dprint_bsg_err(mrioc, |
| | 1113 | "%s: Invalid len %ld while building PRP\n", |
| | 1114 | __func__, length); |
| | 1115 | goto err_out; |
| | 1116 | } |
| | 1117 | dma_addr = |
| | 1118 | drv_buf_iter->dma_desc[desc_count].dma_addr; |
| | 1119 | desc_len = |
| | 1120 | drv_buf_iter->dma_desc[desc_count].size; |
| | 1121 | } |
| 1046 | length -= entry_len; | 1122 | length -= entry_len; |
| | 1123 | } |
| 1047 | } | 1124 | } |
| | 1125 | |
| 1048 | return 0; | 1126 | return 0; |
| 1049 | err_out: | 1127 | err_out: |
| 1050 | if (mrioc->prp_list_virt) { | 1128 | if (mrioc->prp_list_virt) { |