| @@ -957,7 +957,7 @@ static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) · 956 unmodified lines |
| 957 | #endif /* CONFIG_DEBUG_FS */ | 957 | #endif /* CONFIG_DEBUG_FS */ |
| 958 | | 958 | |
| 959 | /** | 959 | /** |
| 960 | * swiotlb_area_find_slots() - search for slots in one IO TLB memory area | 960 | * swiotlb_search_pool_area() - search one memory area in one pool |
| 961 | * @dev: Device which maps the buffer. | 961 | * @dev: Device which maps the buffer. |
| 962 | * @pool: Memory pool to be searched. | 962 | * @pool: Memory pool to be searched. |
| 963 | * @area_index: Index of the IO TLB memory area to be searched. | 963 | * @area_index: Index of the IO TLB memory area to be searched. |
| @@ -972,7 +972,7 @@ static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) · 8 unmodified lines |
| 972 | * | 972 | * |
| 973 | * Return: Index of the first allocated slot, or -1 on error. | 973 | * Return: Index of the first allocated slot, or -1 on error. |
| 974 | */ | 974 | */ |
| 975 | static int swiotlb_area_find_slots(struct device *dev, struct io_tlb_pool *pool, | 975 | static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool, |
| 976 | int area_index, phys_addr_t orig_addr, size_t alloc_size, | 976 | int area_index, phys_addr_t orig_addr, size_t alloc_size, |
| 977 | unsigned int alloc_align_mask) | 977 | unsigned int alloc_align_mask) |
| 978 | { | 978 | { |
| @@ -1066,41 +1066,50 @@ found: · 87 unmodified lines |
| 1066 | return slot_index; | 1066 | return slot_index; |
| 1067 | } | 1067 | } |
| 1068 | | 1068 | |
| | 1069 | #ifdef CONFIG_SWIOTLB_DYNAMIC |
| | 1070 | |
| 1069 | /** | 1071 | /** |
| 1070 | * swiotlb_pool_find_slots() - search for slots in one memory pool | 1072 | * swiotlb_search_area() - search one memory area in all pools |
| 1071 | * @dev: Device which maps the buffer. | 1073 | * @dev: Device which maps the buffer. |
| 1072 | * @pool: Memory pool to be searched. | 1074 | * @start_cpu: Start CPU number. |
| | 1075 | * @cpu_offset: Offset from @start_cpu. |
| 1073 | * @orig_addr: Original (non-bounced) IO buffer address. | 1076 | * @orig_addr: Original (non-bounced) IO buffer address. |
| 1074 | * @alloc_size: Total requested size of the bounce buffer, | 1077 | * @alloc_size: Total requested size of the bounce buffer, |
| 1075 | * including initial alignment padding. | 1078 | * including initial alignment padding. |
| 1076 | * @alloc_align_mask: Required alignment of the allocated buffer. | 1079 | * @alloc_align_mask: Required alignment of the allocated buffer. |
| | 1080 | * @retpool: Used memory pool, updated on return. |
| 1077 | * | 1081 | * |
| 1078 | * Search through one memory pool to find a sequence of slots that match the | 1082 | * Search one memory area in all pools for a sequence of slots that match the |
| 1079 | * allocation constraints. | 1083 | * allocation constraints. |
| 1080 | * | 1084 | * |
| 1081 | * Return: Index of the first allocated slot, or -1 on error. | 1085 | * Return: Index of the first allocated slot, or -1 on error. |
| 1082 | */ | 1086 | */ |
| 1083 | static int swiotlb_pool_find_slots(struct device *dev, struct io_tlb_pool *pool, | 1087 | static int swiotlb_search_area(struct device *dev, int start_cpu, |
| 1084 | phys_addr_t orig_addr, size_t alloc_size, | 1088 | int cpu_offset, phys_addr_t orig_addr, size_t alloc_size, |
| 1085 | unsigned int alloc_align_mask) | 1089 | unsigned int alloc_align_mask, struct io_tlb_pool **retpool) |
| 1086 | { | 1090 | { |
| 1087 | int start = raw_smp_processor_id() & (pool->nareas - 1); | 1091 | struct io_tlb_mem *mem = dev->dma_io_tlb_mem; |
| 1088 | int i = start, index; | 1092 | struct io_tlb_pool *pool; |
| 1089 | | 1093 | int area_index; |
| 1090 | do { | 1094 | int index = -1; |
| 1091 | index = swiotlb_area_find_slots(dev, pool, i, orig_addr, | | |
| 1092 | alloc_size, alloc_align_mask); | | |
| 1093 | if (index >= 0) | | |
| 1094 | return index; | | |
| 1095 | if (++i >= pool->nareas) | | |
| 1096 | i = 0; | | |
| 1097 | } while (i != start); | | |
| 1098 | | 1095 | |
| 1099 | return -1; | 1096 | rcu_read_lock(); |
| | 1097 | list_for_each_entry_rcu(pool, &mem->pools, node) { |
| | 1098 | if (cpu_offset >= pool->nareas) |
| | 1099 | continue; |
| | 1100 | area_index = (start_cpu + cpu_offset) & (pool->nareas - 1); |
| | 1101 | index = swiotlb_search_pool_area(dev, pool, area_index, |
| | 1102 | orig_addr, alloc_size, |
| | 1103 | alloc_align_mask); |
| | 1104 | if (index >= 0) { |
| | 1105 | *retpool = pool; |
| | 1106 | break; |
| | 1107 | } |
| | 1108 | } |
| | 1109 | rcu_read_unlock(); |
| | 1110 | return index; |
| 1100 | } | 1111 | } |
| 1101 | | 1112 | |
| 1102 | #ifdef CONFIG_SWIOTLB_DYNAMIC | | |
| 1103 | | | |
| 1104 | /** | 1113 | /** |
| 1105 | * swiotlb_find_slots() - search for slots in the whole swiotlb | 1114 | * swiotlb_find_slots() - search for slots in the whole swiotlb |
| 1106 | * @dev: Device which maps the buffer. | 1115 | * @dev: Device which maps the buffer. |
| @@ -1124,18 +1133,17 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, · 17 unmodified lines |
| 1124 | unsigned long nslabs; | 1133 | unsigned long nslabs; |
| 1125 | unsigned long flags; | 1134 | unsigned long flags; |
| 1126 | u64 phys_limit; | 1135 | u64 phys_limit; |
| | 1136 | int cpu, i; |
| 1127 | int index; | 1137 | int index; |
| 1128 | | 1138 | |
| 1129 | rcu_read_lock(); | 1139 | cpu = raw_smp_processor_id(); |
| 1130 | list_for_each_entry_rcu(pool, &mem->pools, node) { | 1140 | for (i = 0; i < default_nareas; ++i) { |
| 1131 | index = swiotlb_pool_find_slots(dev, pool, orig_addr, | 1141 | index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size, |
| 1132 | alloc_size, alloc_align_mask); | 1142 | alloc_align_mask, &pool); |
| 1133 | if (index >= 0) { | 1143 | if (index >= 0) |
| 1134 | rcu_read_unlock(); | | |
| 1135 | goto found; | 1144 | goto found; |
| 1136 | } | | |
| 1137 | } | 1145 | } |
| 1138 | rcu_read_unlock(); | 1146 | |
| 1139 | if (!mem->can_grow) | 1147 | if (!mem->can_grow) |
| 1140 | return -1; | 1148 | return -1; |
| 1141 | | 1149 | |
| @@ -1148,8 +1156,8 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, · 6 unmodified lines |
| 1148 | if (!pool) | 1156 | if (!pool) |
| 1149 | return -1; | 1157 | return -1; |
| 1150 | | 1158 | |
| 1151 | index = swiotlb_pool_find_slots(dev, pool, orig_addr, | 1159 | index = swiotlb_search_pool_area(dev, pool, 0, orig_addr, |
| 1152 | alloc_size, alloc_align_mask); | 1160 | alloc_size, alloc_align_mask); |
| 1153 | if (index < 0) { | 1161 | if (index < 0) { |
| 1154 | swiotlb_dyn_free(&pool->rcu); | 1162 | swiotlb_dyn_free(&pool->rcu); |
| 1155 | return -1; | 1163 | return -1; |
| @@ -1192,9 +1200,21 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, · 36 unmodified lines |
| 1192 | size_t alloc_size, unsigned int alloc_align_mask, | 1200 | size_t alloc_size, unsigned int alloc_align_mask, |
| 1193 | struct io_tlb_pool **retpool) | 1201 | struct io_tlb_pool **retpool) |
| 1194 | { | 1202 | { |
| 1195 | *retpool = &dev->dma_io_tlb_mem->defpool; | 1203 | struct io_tlb_pool *pool; |
| 1196 | return swiotlb_pool_find_slots(dev, *retpool, | 1204 | int start, i; |
| 1197 | orig_addr, alloc_size, alloc_align_mask); | 1205 | int index; |
| | 1206 | |
| | 1207 | *retpool = pool = &dev->dma_io_tlb_mem->defpool; |
| | 1208 | i = start = raw_smp_processor_id() & (pool->nareas - 1); |
| | 1209 | do { |
| | 1210 | index = swiotlb_search_pool_area(dev, pool, i, orig_addr, |
| | 1211 | alloc_size, alloc_align_mask); |
| | 1212 | if (index >= 0) |
| | 1213 | return index; |
| | 1214 | if (++i >= pool->nareas) |
| | 1215 | i = 0; |
| | 1216 | } while (i != start); |
| | 1217 | return -1; |
| 1198 | } | 1218 | } |
| 1199 | | 1219 | |
| 1200 | #endif /* CONFIG_SWIOTLB_DYNAMIC */ | 1220 | #endif /* CONFIG_SWIOTLB_DYNAMIC */ |