After the conversion to lock-less dma-api call the increase_address_space() function can be called without any locking. Multiple CPUs could potentially race for increasing the address space, leading to invalid domain->mode settings and invalid page-tables. This has been happening in the wild under high IO load and memory pressure. Fix the race by locking this operation. The function is called infrequently so that this does not introduce a performance regression in the dma-api path again.
Unified · Split
| @@ -1435,18 +1435,21 @@ static void free_pagetable(struct protection_domain *domain) · 1434 unmodified lines | |||
| 1435 | * another level increases the size of the address space by 9 bits to a size up | 1435 | * another level increases the size of the address space by 9 bits to a size up |
| 1436 | * to 64 bits. | 1436 | * to 64 bits. |
| 1437 | */ | 1437 | */ |
| 1438 | static bool increase_address_space(struct protection_domain *domain, | 1438 | static void increase_address_space(struct protection_domain *domain, |
| 1439 | gfp_t gfp) | 1439 | gfp_t gfp) |
| 1440 | { | 1440 | { |
| 1441 | unsigned long flags; | ||
| 1441 | u64 *pte; | 1442 | u64 *pte; |
| 1442 | 1443 | ||
| 1443 | if (domain->mode == PAGE_MODE_6_LEVEL) | 1444 | spin_lock_irqsave(&domain->lock, flags); |
| 1445 | |||
| 1446 | if (WARN_ON_ONCE(domain->mode == PAGE_MODE_6_LEVEL)) | ||
| 1444 | /* address space already 64 bit large */ | 1447 | /* address space already 64 bit large */ |
| 1445 | return false; | 1448 | goto out; |
| 1446 | 1449 | ||
| 1447 | pte = (void *)get_zeroed_page(gfp); | 1450 | pte = (void *)get_zeroed_page(gfp); |
| 1448 | if (!pte) | 1451 | if (!pte) |
| 1449 | return false; | 1452 | goto out; |
| 1450 | 1453 | ||
| 1451 | *pte = PM_LEVEL_PDE(domain->mode, | 1454 | *pte = PM_LEVEL_PDE(domain->mode, |
| 1452 | iommu_virt_to_phys(domain->pt_root)); | 1455 | iommu_virt_to_phys(domain->pt_root)); |
| @@ -1454,7 +1457,10 @@ static bool increase_address_space(struct protection_domain *domain, · 1 unmodified line | |||
| 1454 | domain->mode += 1; | 1457 | domain->mode += 1; |
| 1455 | domain->updated = true; | 1458 | domain->updated = true; |
| 1456 | 1459 | ||
| 1457 | return true; | 1460 | out: |
| 1461 | spin_unlock_irqrestore(&domain->lock, flags); | ||
| 1462 | |||
| 1463 | return; | ||
| 1458 | } | 1464 | } |
| 1459 | 1465 | ||
| 1460 | static u64 *alloc_pte(struct protection_domain *domain, | 1466 | static u64 *alloc_pte(struct protection_domain *domain, |