History

Merge tag 'dma-mapping-6.8-2024-01-08' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping updates from Christoph Hellwig: - reduce area lock contention for non-primary IO TLB pools (Petr Tesarik) - don't store redundant offsets in the dma_ranges stuctures (Robin Murphy) - clear dev->dma_mem when freeing per-device pools (Joakim Zhang) * tag 'dma-mapping-6.8-2024-01-08' of git://git.infradead.org/users/hch/dma-mapping: dma-mapping: clear dev->dma_mem to NULL after freeing it swiotlb: reduce area lock contention for non-primary IO TLB pools dma-mapping: don't store redundant offsets

Linus Torvalds committed 2 years ago

Commit 893e2f92 parents 457e4f9 b07bc236 files · +70 −46Browse files

Showing the changes to kernel (3 of 6 files). Show all

Unified · Split

3 files
  1. kernel/dma/coherent.c +3 −1
  2. kernel/dma/direct.c +0 −1
  3. kernel/dma/swiotlb.c +55 −35
kernel/dma/coherent.c+3−1
@@ -132,8 +132,10 @@ int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, · 131 unmodified lines
132 132
133void dma_release_coherent_memory(struct device *dev)133void dma_release_coherent_memory(struct device *dev)
134{134{
135 if (dev)135 if (dev) {
136 _dma_release_coherent_memory(dev->dma_mem);136 _dma_release_coherent_memory(dev->dma_mem);
137 dev->dma_mem = NULL;
138 }
137}139}
138 140
139static void *__dma_alloc_from_coherent(struct device *dev,141static void *__dma_alloc_from_coherent(struct device *dev,
kernel/dma/direct.c+0−1
@@ -677,7 +677,6 @@ int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start, · 676 unmodified lines
677 return -ENOMEM;677 return -ENOMEM;
678 map[0].cpu_start = cpu_start;678 map[0].cpu_start = cpu_start;
679 map[0].dma_start = dma_start;679 map[0].dma_start = dma_start;
680 map[0].offset = offset;
681 map[0].size = size;680 map[0].size = size;
682 dev->dma_range_map = map;681 dev->dma_range_map = map;
683 return 0;682 return 0;
kernel/dma/swiotlb.c+55−35
@@ -957,7 +957,7 @@ static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) · 956 unmodified lines
957#endif /* CONFIG_DEBUG_FS */957#endif /* CONFIG_DEBUG_FS */
958 958
959/**959/**
960 * swiotlb_area_find_slots() - search for slots in one IO TLB memory area960 * swiotlb_search_pool_area() - search one memory area in one pool
961 * @dev: Device which maps the buffer.961 * @dev: Device which maps the buffer.
962 * @pool: Memory pool to be searched.962 * @pool: Memory pool to be searched.
963 * @area_index: Index of the IO TLB memory area to be searched.963 * @area_index: Index of the IO TLB memory area to be searched.
@@ -972,7 +972,7 @@ static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) · 8 unmodified lines
972 *972 *
973 * Return: Index of the first allocated slot, or -1 on error.973 * Return: Index of the first allocated slot, or -1 on error.
974 */974 */
975static int swiotlb_area_find_slots(struct device *dev, struct io_tlb_pool *pool,975static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
976 int area_index, phys_addr_t orig_addr, size_t alloc_size,976 int area_index, phys_addr_t orig_addr, size_t alloc_size,
977 unsigned int alloc_align_mask)977 unsigned int alloc_align_mask)
978{978{
@@ -1066,41 +1066,50 @@ found: · 87 unmodified lines
1066 return slot_index;1066 return slot_index;
1067}1067}
1068 1068
1069#ifdef CONFIG_SWIOTLB_DYNAMIC
1070
1069/**1071/**
1070 * swiotlb_pool_find_slots() - search for slots in one memory pool1072 * swiotlb_search_area() - search one memory area in all pools
1071 * @dev: Device which maps the buffer.1073 * @dev: Device which maps the buffer.
1072 * @pool: Memory pool to be searched.1074 * @start_cpu: Start CPU number.
1075 * @cpu_offset: Offset from @start_cpu.
1073 * @orig_addr: Original (non-bounced) IO buffer address.1076 * @orig_addr: Original (non-bounced) IO buffer address.
1074 * @alloc_size: Total requested size of the bounce buffer,1077 * @alloc_size: Total requested size of the bounce buffer,
1075 * including initial alignment padding.1078 * including initial alignment padding.
1076 * @alloc_align_mask: Required alignment of the allocated buffer.1079 * @alloc_align_mask: Required alignment of the allocated buffer.
1080 * @retpool: Used memory pool, updated on return.
1077 *1081 *
1078 * Search through one memory pool to find a sequence of slots that match the1082 * Search one memory area in all pools for a sequence of slots that match the
1079 * allocation constraints.1083 * allocation constraints.
1080 *1084 *
1081 * Return: Index of the first allocated slot, or -1 on error.1085 * Return: Index of the first allocated slot, or -1 on error.
1082 */1086 */
1083static int swiotlb_pool_find_slots(struct device *dev, struct io_tlb_pool *pool,1087static int swiotlb_search_area(struct device *dev, int start_cpu,
1084 phys_addr_t orig_addr, size_t alloc_size,1088 int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1085 unsigned int alloc_align_mask)1089 unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1086{1090{
1087 int start = raw_smp_processor_id() & (pool->nareas - 1);1091 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1088 int i = start, index;1092 struct io_tlb_pool *pool;
1089 1093 int area_index;
1090 do {1094 int index = -1;
1091 index = swiotlb_area_find_slots(dev, pool, i, orig_addr,
1092 alloc_size, alloc_align_mask);
1093 if (index >= 0)
1094 return index;
1095 if (++i >= pool->nareas)
1096 i = 0;
1097 } while (i != start);
1098 1095
1099 return -1;1096 rcu_read_lock();
1097 list_for_each_entry_rcu(pool, &mem->pools, node) {
1098 if (cpu_offset >= pool->nareas)
1099 continue;
1100 area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1101 index = swiotlb_search_pool_area(dev, pool, area_index,
1102 orig_addr, alloc_size,
1103 alloc_align_mask);
1104 if (index >= 0) {
1105 *retpool = pool;
1106 break;
1107 }
1108 }
1109 rcu_read_unlock();
1110 return index;
1100}1111}
1101 1112
1102#ifdef CONFIG_SWIOTLB_DYNAMIC
1103
1104/**1113/**
1105 * swiotlb_find_slots() - search for slots in the whole swiotlb1114 * swiotlb_find_slots() - search for slots in the whole swiotlb
1106 * @dev: Device which maps the buffer.1115 * @dev: Device which maps the buffer.
@@ -1124,18 +1133,17 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, · 17 unmodified lines
1124 unsigned long nslabs;1133 unsigned long nslabs;
1125 unsigned long flags;1134 unsigned long flags;
1126 u64 phys_limit;1135 u64 phys_limit;
1136 int cpu, i;
1127 int index;1137 int index;
1128 1138
1129 rcu_read_lock();1139 cpu = raw_smp_processor_id();
1130 list_for_each_entry_rcu(pool, &mem->pools, node) {1140 for (i = 0; i < default_nareas; ++i) {
1131 index = swiotlb_pool_find_slots(dev, pool, orig_addr,1141 index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,
1132 alloc_size, alloc_align_mask);1142 alloc_align_mask, &pool);
1133 if (index >= 0) {1143 if (index >= 0)
1134 rcu_read_unlock();
1135 goto found;1144 goto found;
1136 }
1137 }1145 }
1138 rcu_read_unlock();1146
1139 if (!mem->can_grow)1147 if (!mem->can_grow)
1140 return -1;1148 return -1;
1141 1149
@@ -1148,8 +1156,8 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, · 6 unmodified lines
1148 if (!pool)1156 if (!pool)
1149 return -1;1157 return -1;
1150 1158
1151 index = swiotlb_pool_find_slots(dev, pool, orig_addr,1159 index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
1152 alloc_size, alloc_align_mask);1160 alloc_size, alloc_align_mask);
1153 if (index < 0) {1161 if (index < 0) {
1154 swiotlb_dyn_free(&pool->rcu);1162 swiotlb_dyn_free(&pool->rcu);
1155 return -1;1163 return -1;
@@ -1192,9 +1200,21 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, · 36 unmodified lines
1192 size_t alloc_size, unsigned int alloc_align_mask,1200 size_t alloc_size, unsigned int alloc_align_mask,
1193 struct io_tlb_pool **retpool)1201 struct io_tlb_pool **retpool)
1194{1202{
1195 *retpool = &dev->dma_io_tlb_mem->defpool;1203 struct io_tlb_pool *pool;
1196 return swiotlb_pool_find_slots(dev, *retpool,1204 int start, i;
1197 orig_addr, alloc_size, alloc_align_mask);1205 int index;
1206
1207 *retpool = pool = &dev->dma_io_tlb_mem->defpool;
1208 i = start = raw_smp_processor_id() & (pool->nareas - 1);
1209 do {
1210 index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1211 alloc_size, alloc_align_mask);
1212 if (index >= 0)
1213 return index;
1214 if (++i >= pool->nareas)
1215 i = 0;
1216 } while (i != start);
1217 return -1;
1198}1218}
1199 1219
1200#endif /* CONFIG_SWIOTLB_DYNAMIC */1220#endif /* CONFIG_SWIOTLB_DYNAMIC */