For a long time the DMA API has been implemented inline in dma-mapping.h, but the function bodies can be quite large. Move them all out of line. This also removes all the dma_direct_* exports as those are just implementation details and should never be used by drivers directly.
Showing the changes to include/linux (2 of 4 files). Show all
Unified · Split
| @@ -87,4 +87,62 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, · 86 unmodified lines | |||
| 87 | unsigned long attrs); | 87 | unsigned long attrs); |
| 88 | int dma_direct_supported(struct device *dev, u64 mask); | 88 | int dma_direct_supported(struct device *dev, u64 mask); |
| 89 | bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr); | 89 | bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr); |
| 90 | dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, | ||
| 91 | unsigned long offset, size_t size, enum dma_data_direction dir, | ||
| 92 | unsigned long attrs); | ||
| 93 | int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, | ||
| 94 | enum dma_data_direction dir, unsigned long attrs); | ||
| 95 | dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, | ||
| 96 | size_t size, enum dma_data_direction dir, unsigned long attrs); | ||
| 97 | |||
| 98 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ | ||
| 99 | defined(CONFIG_SWIOTLB) | ||
| 100 | void dma_direct_sync_single_for_device(struct device *dev, | ||
| 101 | dma_addr_t addr, size_t size, enum dma_data_direction dir); | ||
| 102 | void dma_direct_sync_sg_for_device(struct device *dev, | ||
| 103 | struct scatterlist *sgl, int nents, enum dma_data_direction dir); | ||
| 104 | #else | ||
| 105 | static inline void dma_direct_sync_single_for_device(struct device *dev, | ||
| 106 | dma_addr_t addr, size_t size, enum dma_data_direction dir) | ||
| 107 | { | ||
| 108 | } | ||
| 109 | static inline void dma_direct_sync_sg_for_device(struct device *dev, | ||
| 110 | struct scatterlist *sgl, int nents, enum dma_data_direction dir) | ||
| 111 | { | ||
| 112 | } | ||
| 113 | #endif | ||
| 114 | |||
| 115 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ | ||
| 116 | defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \ | ||
| 117 | defined(CONFIG_SWIOTLB) | ||
| 118 | void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, | ||
| 119 | size_t size, enum dma_data_direction dir, unsigned long attrs); | ||
| 120 | void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, | ||
| 121 | int nents, enum dma_data_direction dir, unsigned long attrs); | ||
| 122 | void dma_direct_sync_single_for_cpu(struct device *dev, | ||
| 123 | dma_addr_t addr, size_t size, enum dma_data_direction dir); | ||
| 124 | void dma_direct_sync_sg_for_cpu(struct device *dev, | ||
| 125 | struct scatterlist *sgl, int nents, enum dma_data_direction dir); | ||
| 126 | #else | ||
| 127 | static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, | ||
| 128 | size_t size, enum dma_data_direction dir, unsigned long attrs) | ||
| 129 | { | ||
| 130 | } | ||
| 131 | static inline void dma_direct_unmap_sg(struct device *dev, | ||
| 132 | struct scatterlist *sgl, int nents, enum dma_data_direction dir, | ||
| 133 | unsigned long attrs) | ||
| 134 | { | ||
| 135 | } | ||
| 136 | static inline void dma_direct_sync_single_for_cpu(struct device *dev, | ||
| 137 | dma_addr_t addr, size_t size, enum dma_data_direction dir) | ||
| 138 | { | ||
| 139 | } | ||
| 140 | static inline void dma_direct_sync_sg_for_cpu(struct device *dev, | ||
| 141 | struct scatterlist *sgl, int nents, enum dma_data_direction dir) | ||
| 142 | { | ||
| 143 | } | ||
| 144 | #endif | ||
| 145 | |||
| 146 | size_t dma_direct_max_mapping_size(struct device *dev); | ||
| 147 | |||
| 90 | #endif /* _LINUX_DMA_DIRECT_H */ | 148 | #endif /* _LINUX_DMA_DIRECT_H */ |
| @@ -188,73 +188,6 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma, · 187 unmodified lines | |||
| 188 | } | 188 | } |
| 189 | #endif /* CONFIG_DMA_DECLARE_COHERENT */ | 189 | #endif /* CONFIG_DMA_DECLARE_COHERENT */ |
| 190 | 190 | ||
| 191 | static inline bool dma_is_direct(const struct dma_map_ops *ops) | ||
| 192 | { | ||
| 193 | return likely(!ops); | ||
| 194 | } | ||
| 195 | |||
| 196 | /* | ||
| 197 | * All the dma_direct_* declarations are here just for the indirect call bypass, | ||
| 198 | * and must not be used directly drivers! | ||
| 199 | */ | ||
| 200 | dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, | ||
| 201 | unsigned long offset, size_t size, enum dma_data_direction dir, | ||
| 202 | unsigned long attrs); | ||
| 203 | int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, | ||
| 204 | enum dma_data_direction dir, unsigned long attrs); | ||
| 205 | dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, | ||
| 206 | size_t size, enum dma_data_direction dir, unsigned long attrs); | ||
| 207 | |||
| 208 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ | ||
| 209 | defined(CONFIG_SWIOTLB) | ||
| 210 | void dma_direct_sync_single_for_device(struct device *dev, | ||
| 211 | dma_addr_t addr, size_t size, enum dma_data_direction dir); | ||
| 212 | void dma_direct_sync_sg_for_device(struct device *dev, | ||
| 213 | struct scatterlist *sgl, int nents, enum dma_data_direction dir); | ||
| 214 | #else | ||
| 215 | static inline void dma_direct_sync_single_for_device(struct device *dev, | ||
| 216 | dma_addr_t addr, size_t size, enum dma_data_direction dir) | ||
| 217 | { | ||
| 218 | } | ||
| 219 | static inline void dma_direct_sync_sg_for_device(struct device *dev, | ||
| 220 | struct scatterlist *sgl, int nents, enum dma_data_direction dir) | ||
| 221 | { | ||
| 222 | } | ||
| 223 | #endif | ||
| 224 | |||
| 225 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ | ||
| 226 | defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \ | ||
| 227 | defined(CONFIG_SWIOTLB) | ||
| 228 | void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, | ||
| 229 | size_t size, enum dma_data_direction dir, unsigned long attrs); | ||
| 230 | void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, | ||
| 231 | int nents, enum dma_data_direction dir, unsigned long attrs); | ||
| 232 | void dma_direct_sync_single_for_cpu(struct device *dev, | ||
| 233 | dma_addr_t addr, size_t size, enum dma_data_direction dir); | ||
| 234 | void dma_direct_sync_sg_for_cpu(struct device *dev, | ||
| 235 | struct scatterlist *sgl, int nents, enum dma_data_direction dir); | ||
| 236 | #else | ||
| 237 | static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, | ||
| 238 | size_t size, enum dma_data_direction dir, unsigned long attrs) | ||
| 239 | { | ||
| 240 | } | ||
| 241 | static inline void dma_direct_unmap_sg(struct device *dev, | ||
| 242 | struct scatterlist *sgl, int nents, enum dma_data_direction dir, | ||
| 243 | unsigned long attrs) | ||
| 244 | { | ||
| 245 | } | ||
| 246 | static inline void dma_direct_sync_single_for_cpu(struct device *dev, | ||
| 247 | dma_addr_t addr, size_t size, enum dma_data_direction dir) | ||
| 248 | { | ||
| 249 | } | ||
| 250 | static inline void dma_direct_sync_sg_for_cpu(struct device *dev, | ||
| 251 | struct scatterlist *sgl, int nents, enum dma_data_direction dir) | ||
| 252 | { | ||
| 253 | } | ||
| 254 | #endif | ||
| 255 | |||
| 256 | size_t dma_direct_max_mapping_size(struct device *dev); | ||
| 257 | |||
| 258 | #ifdef CONFIG_HAS_DMA | 191 | #ifdef CONFIG_HAS_DMA |
| 259 | #include <asm/dma-mapping.h> | 192 | #include <asm/dma-mapping.h> |
| 260 | 193 | ||
| @@ -271,164 +204,6 @@ static inline void set_dma_ops(struct device *dev, · 10 unmodified lines | |||
| 271 | dev->dma_ops = dma_ops; | 204 | dev->dma_ops = dma_ops; |
| 272 | } | 205 | } |
| 273 | 206 | ||
| 274 | static inline dma_addr_t dma_map_page_attrs(struct device *dev, | ||
| 275 | struct page *page, size_t offset, size_t size, | ||
| 276 | enum dma_data_direction dir, unsigned long attrs) | ||
| 277 | { | ||
| 278 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 279 | dma_addr_t addr; | ||
| 280 | |||
| 281 | BUG_ON(!valid_dma_direction(dir)); | ||
| 282 | if (dma_is_direct(ops)) | ||
| 283 | addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); | ||
| 284 | else | ||
| 285 | addr = ops->map_page(dev, page, offset, size, dir, attrs); | ||
| 286 | debug_dma_map_page(dev, page, offset, size, dir, addr); | ||
| 287 | |||
| 288 | return addr; | ||
| 289 | } | ||
| 290 | |||
| 291 | static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, | ||
| 292 | size_t size, enum dma_data_direction dir, unsigned long attrs) | ||
| 293 | { | ||
| 294 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 295 | |||
| 296 | BUG_ON(!valid_dma_direction(dir)); | ||
| 297 | if (dma_is_direct(ops)) | ||
| 298 | dma_direct_unmap_page(dev, addr, size, dir, attrs); | ||
| 299 | else if (ops->unmap_page) | ||
| 300 | ops->unmap_page(dev, addr, size, dir, attrs); | ||
| 301 | debug_dma_unmap_page(dev, addr, size, dir); | ||
| 302 | } | ||
| 303 | |||
| 304 | /* | ||
| 305 | * dma_maps_sg_attrs returns 0 on error and > 0 on success. | ||
| 306 | * It should never return a value < 0. | ||
| 307 | */ | ||
| 308 | static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, | ||
| 309 | int nents, enum dma_data_direction dir, | ||
| 310 | unsigned long attrs) | ||
| 311 | { | ||
| 312 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 313 | int ents; | ||
| 314 | |||
| 315 | BUG_ON(!valid_dma_direction(dir)); | ||
| 316 | if (dma_is_direct(ops)) | ||
| 317 | ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); | ||
| 318 | else | ||
| 319 | ents = ops->map_sg(dev, sg, nents, dir, attrs); | ||
| 320 | BUG_ON(ents < 0); | ||
| 321 | debug_dma_map_sg(dev, sg, nents, ents, dir); | ||
| 322 | |||
| 323 | return ents; | ||
| 324 | } | ||
| 325 | |||
| 326 | static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, | ||
| 327 | int nents, enum dma_data_direction dir, | ||
| 328 | unsigned long attrs) | ||
| 329 | { | ||
| 330 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 331 | |||
| 332 | BUG_ON(!valid_dma_direction(dir)); | ||
| 333 | debug_dma_unmap_sg(dev, sg, nents, dir); | ||
| 334 | if (dma_is_direct(ops)) | ||
| 335 | dma_direct_unmap_sg(dev, sg, nents, dir, attrs); | ||
| 336 | else if (ops->unmap_sg) | ||
| 337 | ops->unmap_sg(dev, sg, nents, dir, attrs); | ||
| 338 | } | ||
| 339 | |||
| 340 | static inline dma_addr_t dma_map_resource(struct device *dev, | ||
| 341 | phys_addr_t phys_addr, | ||
| 342 | size_t size, | ||
| 343 | enum dma_data_direction dir, | ||
| 344 | unsigned long attrs) | ||
| 345 | { | ||
| 346 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 347 | dma_addr_t addr = DMA_MAPPING_ERROR; | ||
| 348 | |||
| 349 | BUG_ON(!valid_dma_direction(dir)); | ||
| 350 | |||
| 351 | /* Don't allow RAM to be mapped */ | ||
| 352 | if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) | ||
| 353 | return DMA_MAPPING_ERROR; | ||
| 354 | |||
| 355 | if (dma_is_direct(ops)) | ||
| 356 | addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); | ||
| 357 | else if (ops->map_resource) | ||
| 358 | addr = ops->map_resource(dev, phys_addr, size, dir, attrs); | ||
| 359 | |||
| 360 | debug_dma_map_resource(dev, phys_addr, size, dir, addr); | ||
| 361 | return addr; | ||
| 362 | } | ||
| 363 | |||
| 364 | static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr, | ||
| 365 | size_t size, enum dma_data_direction dir, | ||
| 366 | unsigned long attrs) | ||
| 367 | { | ||
| 368 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 369 | |||
| 370 | BUG_ON(!valid_dma_direction(dir)); | ||
| 371 | if (!dma_is_direct(ops) && ops->unmap_resource) | ||
| 372 | ops->unmap_resource(dev, addr, size, dir, attrs); | ||
| 373 | debug_dma_unmap_resource(dev, addr, size, dir); | ||
| 374 | } | ||
| 375 | |||
| 376 | static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, | ||
| 377 | size_t size, | ||
| 378 | enum dma_data_direction dir) | ||
| 379 | { | ||
| 380 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 381 | |||
| 382 | BUG_ON(!valid_dma_direction(dir)); | ||
| 383 | if (dma_is_direct(ops)) | ||
| 384 | dma_direct_sync_single_for_cpu(dev, addr, size, dir); | ||
| 385 | else if (ops->sync_single_for_cpu) | ||
| 386 | ops->sync_single_for_cpu(dev, addr, size, dir); | ||
| 387 | debug_dma_sync_single_for_cpu(dev, addr, size, dir); | ||
| 388 | } | ||
| 389 | |||
| 390 | static inline void dma_sync_single_for_device(struct device *dev, | ||
| 391 | dma_addr_t addr, size_t size, | ||
| 392 | enum dma_data_direction dir) | ||
| 393 | { | ||
| 394 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 395 | |||
| 396 | BUG_ON(!valid_dma_direction(dir)); | ||
| 397 | if (dma_is_direct(ops)) | ||
| 398 | dma_direct_sync_single_for_device(dev, addr, size, dir); | ||
| 399 | else if (ops->sync_single_for_device) | ||
| 400 | ops->sync_single_for_device(dev, addr, size, dir); | ||
| 401 | debug_dma_sync_single_for_device(dev, addr, size, dir); | ||
| 402 | } | ||
| 403 | |||
| 404 | static inline void | ||
| 405 | dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, | ||
| 406 | int nelems, enum dma_data_direction dir) | ||
| 407 | { | ||
| 408 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 409 | |||
| 410 | BUG_ON(!valid_dma_direction(dir)); | ||
| 411 | if (dma_is_direct(ops)) | ||
| 412 | dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); | ||
| 413 | else if (ops->sync_sg_for_cpu) | ||
| 414 | ops->sync_sg_for_cpu(dev, sg, nelems, dir); | ||
| 415 | debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); | ||
| 416 | } | ||
| 417 | |||
| 418 | static inline void | ||
| 419 | dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, | ||
| 420 | int nelems, enum dma_data_direction dir) | ||
| 421 | { | ||
| 422 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 423 | |||
| 424 | BUG_ON(!valid_dma_direction(dir)); | ||
| 425 | if (dma_is_direct(ops)) | ||
| 426 | dma_direct_sync_sg_for_device(dev, sg, nelems, dir); | ||
| 427 | else if (ops->sync_sg_for_device) | ||
| 428 | ops->sync_sg_for_device(dev, sg, nelems, dir); | ||
| 429 | debug_dma_sync_sg_for_device(dev, sg, nelems, dir); | ||
| 430 | |||
| 431 | } | ||
| 432 | 207 | ||
| 433 | static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) | 208 | static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) |
| 434 | { | 209 | { |
| @@ -439,6 +214,28 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) · 4 unmodified lines | |||
| 439 | return 0; | 214 | return 0; |
| 440 | } | 215 | } |
| 441 | 216 | ||
| 217 | dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, | ||
| 218 | size_t offset, size_t size, enum dma_data_direction dir, | ||
| 219 | unsigned long attrs); | ||
| 220 | void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, | ||
| 221 | enum dma_data_direction dir, unsigned long attrs); | ||
| 222 | int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, | ||
| 223 | enum dma_data_direction dir, unsigned long attrs); | ||
| 224 | void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, | ||
| 225 | int nents, enum dma_data_direction dir, | ||
| 226 | unsigned long attrs); | ||
| 227 | dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, | ||
| 228 | size_t size, enum dma_data_direction dir, unsigned long attrs); | ||
| 229 | void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, | ||
| 230 | enum dma_data_direction dir, unsigned long attrs); | ||
| 231 | void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, | ||
| 232 | enum dma_data_direction dir); | ||
| 233 | void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, | ||
| 234 | size_t size, enum dma_data_direction dir); | ||
| 235 | void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, | ||
| 236 | int nelems, enum dma_data_direction dir); | ||
| 237 | void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, | ||
| 238 | int nelems, enum dma_data_direction dir); | ||
| 442 | void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, | 239 | void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, |
| 443 | gfp_t flag, unsigned long attrs); | 240 | gfp_t flag, unsigned long attrs); |
| 444 | void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, | 241 | void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, |