History

iommu/vt-d: Use put_pages_list

page->freelist is for the use of slab. We already have the ability to free a list of pages in the core mm, but it requires the use of a list_head and for the pages to be chained together through page->lru. Switch the Intel IOMMU and IOVA code over to using free_pages_list().

Matthew Wilcox (Oracle) committed 4 years ago

Commit 87f60cc1 parent ce00eec5 files · +45 −79Browse files

Unified · Split

5 files
  1. drivers/iommu/dma-iommu.c +1 −1
  2. drivers/iommu/intel/iommu.c +31 −58
  3. drivers/iommu/iova.c +9 −17
  4. include/linux/iommu.h +2 −1
  5. include/linux/iova.h +2 −2
drivers/iommu/dma-iommu.c+1−1
@@ -447,7 +447,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, · 446 unmodified lines
447 else if (gather && gather->queued)447 else if (gather && gather->queued)
448 queue_iova(iovad, iova_pfn(iovad, iova),448 queue_iova(iovad, iova_pfn(iovad, iova),
449 size >> iova_shift(iovad),449 size >> iova_shift(iovad),
450 gather->freelist);450 &gather->freelist);
451 else451 else
452 free_iova_fast(iovad, iova_pfn(iovad, iova),452 free_iova_fast(iovad, iova_pfn(iovad, iova),
453 size >> iova_shift(iovad));453 size >> iova_shift(iovad));
drivers/iommu/intel/iommu.c+31−58
@@ -1303,35 +1303,30 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain, · 1302 unmodified lines
1303 know the hardware page-walk will no longer touch them.1303 know the hardware page-walk will no longer touch them.
1304 The 'pte' argument is the *parent* PTE, pointing to the page that is to1304 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1305 be freed. */1305 be freed. */
1306static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,1306static void dma_pte_list_pagetables(struct dmar_domain *domain,
1307 int level, struct dma_pte *pte,1307 int level, struct dma_pte *pte,
1308 struct page *freelist)1308 struct list_head *freelist)
1309{1309{
1310 struct page *pg;1310 struct page *pg;
1311 1311
1312 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);1312 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1313 pg->freelist = freelist;1313 list_add_tail(&pg->lru, freelist);
1314 freelist = pg;
1315 1314
1316 if (level == 1)1315 if (level == 1)
1317 return freelist;1316 return;
1318 1317
1319 pte = page_address(pg);1318 pte = page_address(pg);
1320 do {1319 do {
1321 if (dma_pte_present(pte) && !dma_pte_superpage(pte))1320 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1322 freelist = dma_pte_list_pagetables(domain, level - 1,1321 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1323 pte, freelist);
1324 pte++;1322 pte++;
1325 } while (!first_pte_in_page(pte));1323 } while (!first_pte_in_page(pte));
1326
1327 return freelist;
1328}1324}
1329 1325
1330static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,1326static void dma_pte_clear_level(struct dmar_domain *domain, int level,
1331 struct dma_pte *pte, unsigned long pfn,1327 struct dma_pte *pte, unsigned long pfn,
1332 unsigned long start_pfn,1328 unsigned long start_pfn, unsigned long last_pfn,
1333 unsigned long last_pfn,1329 struct list_head *freelist)
1334 struct page *freelist)
1335{1330{
1336 struct dma_pte *first_pte = NULL, *last_pte = NULL;1331 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1337 1332
@@ -1350,7 +1345,7 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level, · 12 unmodified lines
1350 /* These suborbinate page tables are going away entirely. Don't1345 /* These suborbinate page tables are going away entirely. Don't
1351 bother to clear them; we're just going to *free* them. */1346 bother to clear them; we're just going to *free* them. */
1352 if (level > 1 && !dma_pte_superpage(pte))1347 if (level > 1 && !dma_pte_superpage(pte))
1353 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);1348 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1354 1349
1355 dma_clear_pte(pte);1350 dma_clear_pte(pte);
1356 if (!first_pte)1351 if (!first_pte)
@@ -1358,10 +1353,10 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level, · 1 unmodified line
1358 last_pte = pte;1353 last_pte = pte;
1359 } else if (level > 1) {1354 } else if (level > 1) {
1360 /* Recurse down into a level that isn't *entirely* obsolete */1355 /* Recurse down into a level that isn't *entirely* obsolete */
1361 freelist = dma_pte_clear_level(domain, level - 1,1356 dma_pte_clear_level(domain, level - 1,
1362 phys_to_virt(dma_pte_addr(pte)),1357 phys_to_virt(dma_pte_addr(pte)),
1363 level_pfn, start_pfn, last_pfn,1358 level_pfn, start_pfn, last_pfn,
1364 freelist);1359 freelist);
1365 }1360 }
1366next:1361next:
1367 pfn = level_pfn + level_size(level);1362 pfn = level_pfn + level_size(level);
@@ -1370,47 +1365,28 @@ next: · 2 unmodified lines
1370 if (first_pte)1365 if (first_pte)
1371 domain_flush_cache(domain, first_pte,1366 domain_flush_cache(domain, first_pte,
1372 (void *)++last_pte - (void *)first_pte);1367 (void *)++last_pte - (void *)first_pte);
1373
1374 return freelist;
1375}1368}
1376 1369
1377/* We can't just free the pages because the IOMMU may still be walking1370/* We can't just free the pages because the IOMMU may still be walking
1378 the page tables, and may have cached the intermediate levels. The1371 the page tables, and may have cached the intermediate levels. The
1379 pages can only be freed after the IOTLB flush has been done. */1372 pages can only be freed after the IOTLB flush has been done. */
1380static struct page *domain_unmap(struct dmar_domain *domain,1373static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
1381 unsigned long start_pfn,1374 unsigned long last_pfn, struct list_head *freelist)
1382 unsigned long last_pfn,
1383 struct page *freelist)
1384{1375{
1385 BUG_ON(!domain_pfn_supported(domain, start_pfn));1376 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1386 BUG_ON(!domain_pfn_supported(domain, last_pfn));1377 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1387 BUG_ON(start_pfn > last_pfn);1378 BUG_ON(start_pfn > last_pfn);
1388 1379
1389 /* we don't need lock here; nobody else touches the iova range */1380 /* we don't need lock here; nobody else touches the iova range */
1390 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),1381 dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1391 domain->pgd, 0, start_pfn, last_pfn,1382 domain->pgd, 0, start_pfn, last_pfn, freelist);
1392 freelist);
1393 1383
1394 /* free pgd */1384 /* free pgd */
1395 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {1385 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1396 struct page *pgd_page = virt_to_page(domain->pgd);1386 struct page *pgd_page = virt_to_page(domain->pgd);
1397 pgd_page->freelist = freelist;1387 list_add_tail(&pgd_page->lru, freelist);
1398 freelist = pgd_page;
1399
1400 domain->pgd = NULL;1388 domain->pgd = NULL;
1401 }1389 }
1402
1403 return freelist;
1404}
1405
1406static void dma_free_pagelist(struct page *freelist)
1407{
1408 struct page *pg;
1409
1410 while ((pg = freelist)) {
1411 freelist = pg->freelist;
1412 free_pgtable_page(page_address(pg));
1413 }
1414}1390}
1415 1391
1416/* iommu handling */1392/* iommu handling */
@@ -2095,11 +2071,10 @@ static void domain_exit(struct dmar_domain *domain) · 678 unmodified lines
2095 domain_remove_dev_info(domain);2071 domain_remove_dev_info(domain);
2096 2072
2097 if (domain->pgd) {2073 if (domain->pgd) {
2098 struct page *freelist;2074 LIST_HEAD(freelist);
2099 2075
2100 freelist = domain_unmap(domain, 0,2076 domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
2101 DOMAIN_MAX_PFN(domain->gaw), NULL);2077 put_pages_list(&freelist);
2102 dma_free_pagelist(freelist);
2103 }2078 }
2104 2079
2105 free_domain_mem(domain);2080 free_domain_mem(domain);
@@ -4192,19 +4167,17 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, · 2086 unmodified lines
4192 {4167 {
4193 struct dmar_drhd_unit *drhd;4168 struct dmar_drhd_unit *drhd;
4194 struct intel_iommu *iommu;4169 struct intel_iommu *iommu;
4195 struct page *freelist;4170 LIST_HEAD(freelist);
4196 4171
4197 freelist = domain_unmap(si_domain,4172 domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist);
4198 start_vpfn, last_vpfn,
4199 NULL);
4200 4173
4201 rcu_read_lock();4174 rcu_read_lock();
4202 for_each_active_iommu(iommu, drhd)4175 for_each_active_iommu(iommu, drhd)
4203 iommu_flush_iotlb_psi(iommu, si_domain,4176 iommu_flush_iotlb_psi(iommu, si_domain,
4204 start_vpfn, mhp->nr_pages,4177 start_vpfn, mhp->nr_pages,
4205 !freelist, 0);4178 list_empty(&freelist), 0);
4206 rcu_read_unlock();4179 rcu_read_unlock();
4207 dma_free_pagelist(freelist);4180 put_pages_list(&freelist);
4208 }4181 }
4209 break;4182 break;
4210 }4183 }
@@ -5211,8 +5184,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, · 1000 unmodified lines
5211 start_pfn = iova >> VTD_PAGE_SHIFT;5184 start_pfn = iova >> VTD_PAGE_SHIFT;
5212 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;5185 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5213 5186
5214 gather->freelist = domain_unmap(dmar_domain, start_pfn,5187 domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
5215 last_pfn, gather->freelist);
5216 5188
5217 if (dmar_domain->max_addr == iova + size)5189 if (dmar_domain->max_addr == iova + size)
5218 dmar_domain->max_addr = iova;5190 dmar_domain->max_addr = iova;
@@ -5248,9 +5220,10 @@ static void intel_iommu_tlb_sync(struct iommu_domain *domain, · 29 unmodified lines
5248 5220
5249 for_each_domain_iommu(iommu_id, dmar_domain)5221 for_each_domain_iommu(iommu_id, dmar_domain)
5250 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,5222 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5251 start_pfn, nrpages, !gather->freelist, 0);5223 start_pfn, nrpages,
5224 list_empty(&gather->freelist), 0);
5252 5225
5253 dma_free_pagelist(gather->freelist);5226 put_pages_list(&gather->freelist);
5254}5227}
5255 5228
5256static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,5229static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
drivers/iommu/iova.c+9−17
@@ -96,7 +96,7 @@ static void free_iova_flush_queue(struct iova_domain *iovad) · 95 unmodified lines
96int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain)96int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain)
97{97{
98 struct iova_fq __percpu *queue;98 struct iova_fq __percpu *queue;
99 int cpu;99 int i, cpu;
100 100
101 atomic64_set(&iovad->fq_flush_start_cnt, 0);101 atomic64_set(&iovad->fq_flush_start_cnt, 0);
102 atomic64_set(&iovad->fq_flush_finish_cnt, 0);102 atomic64_set(&iovad->fq_flush_finish_cnt, 0);
@@ -113,6 +113,9 @@ int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_dom · 10 unmodified lines
113 fq->tail = 0;113 fq->tail = 0;
114 114
115 spin_lock_init(&fq->lock);115 spin_lock_init(&fq->lock);
116
117 for (i = 0; i < IOVA_FQ_SIZE; i++)
118 INIT_LIST_HEAD(&fq->entries[i].freelist);
116 }119 }
117 120
118 iovad->fq_domain = fq_domain;121 iovad->fq_domain = fq_domain;
@@ -543,16 +546,6 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) · 424 unmodified lines
543}546}
544EXPORT_SYMBOL_GPL(free_iova_fast);547EXPORT_SYMBOL_GPL(free_iova_fast);
545 548
546static void fq_entry_dtor(struct page *freelist)
547{
548 while (freelist) {
549 unsigned long p = (unsigned long)page_address(freelist);
550
551 freelist = freelist->freelist;
552 free_page(p);
553 }
554}
555
556#define fq_ring_for_each(i, fq) \549#define fq_ring_for_each(i, fq) \
557 for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)550 for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
558 551
@@ -585,7 +578,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) · 26 unmodified lines
585 if (fq->entries[idx].counter >= counter)578 if (fq->entries[idx].counter >= counter)
586 break;579 break;
587 580
588 fq_entry_dtor(fq->entries[idx].freelist);581 put_pages_list(&fq->entries[idx].freelist);
589 free_iova_fast(iovad,582 free_iova_fast(iovad,
590 fq->entries[idx].iova_pfn,583 fq->entries[idx].iova_pfn,
591 fq->entries[idx].pages);584 fq->entries[idx].pages);
@@ -607,15 +600,14 @@ static void fq_destroy_all_entries(struct iova_domain *iovad) · 15 unmodified lines
607 600
608 /*601 /*
609 * This code runs when the iova_domain is being detroyed, so don't602 * This code runs when the iova_domain is being detroyed, so don't
610 * bother to free iovas, just call the entry_dtor on all remaining603 * bother to free iovas, just free any remaining pagetable pages.
611 * entries.
612 */604 */
613 for_each_possible_cpu(cpu) {605 for_each_possible_cpu(cpu) {
614 struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);606 struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
615 int idx;607 int idx;
616 608
617 fq_ring_for_each(idx, fq)609 fq_ring_for_each(idx, fq)
618 fq_entry_dtor(fq->entries[idx].freelist);610 put_pages_list(&fq->entries[idx].freelist);
619 }611 }
620}612}
621 613
@@ -640,7 +632,7 @@ static void fq_flush_timeout(struct timer_list *t) · 18 unmodified lines
640 632
641void queue_iova(struct iova_domain *iovad,633void queue_iova(struct iova_domain *iovad,
642 unsigned long pfn, unsigned long pages,634 unsigned long pfn, unsigned long pages,
643 struct page *freelist)635 struct list_head *freelist)
644{636{
645 struct iova_fq *fq;637 struct iova_fq *fq;
646 unsigned long flags;638 unsigned long flags;
@@ -674,8 +666,8 @@ void queue_iova(struct iova_domain *iovad, · 27 unmodified lines
674 666
675 fq->entries[idx].iova_pfn = pfn;667 fq->entries[idx].iova_pfn = pfn;
676 fq->entries[idx].pages = pages;668 fq->entries[idx].pages = pages;
677 fq->entries[idx].freelist = freelist;
678 fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);669 fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);
670 list_splice(freelist, &fq->entries[idx].freelist);
679 671
680 spin_unlock_irqrestore(&fq->lock, flags);672 spin_unlock_irqrestore(&fq->lock, flags);
681 673
include/linux/iommu.h+2−1
@@ -186,7 +186,7 @@ struct iommu_iotlb_gather { · 185 unmodified lines
186 unsigned long start;186 unsigned long start;
187 unsigned long end;187 unsigned long end;
188 size_t pgsize;188 size_t pgsize;
189 struct page *freelist;189 struct list_head freelist;
190 bool queued;190 bool queued;
191};191};
192 192
@@ -399,6 +399,7 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) · 206 unmodified lines
399{399{
400 *gather = (struct iommu_iotlb_gather) {400 *gather = (struct iommu_iotlb_gather) {
401 .start = ULONG_MAX,401 .start = ULONG_MAX,
402 .freelist = LIST_HEAD_INIT(gather->freelist),
402 };403 };
403}404}
404 405
include/linux/iova.h+2−2
@@ -46,7 +46,7 @@ struct iova_rcache { · 45 unmodified lines
46struct iova_fq_entry {46struct iova_fq_entry {
47 unsigned long iova_pfn;47 unsigned long iova_pfn;
48 unsigned long pages;48 unsigned long pages;
49 struct page *freelist;49 struct list_head freelist;
50 u64 counter; /* Flush counter when this entrie was added */50 u64 counter; /* Flush counter when this entrie was added */
51};51};
52 52
@@ -135,7 +135,7 @@ void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, · 82 unmodified lines
135 unsigned long size);135 unsigned long size);
136void queue_iova(struct iova_domain *iovad,136void queue_iova(struct iova_domain *iovad,
137 unsigned long pfn, unsigned long pages,137 unsigned long pfn, unsigned long pages,
138 struct page *freelist);138 struct list_head *freelist);
139unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,139unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
140 unsigned long limit_pfn, bool flush_rcache);140 unsigned long limit_pfn, bool flush_rcache);
141struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,141struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t