User applications running on SMP kernels have long suffered from instability and random segmentation faults. This patch improves the situation although there is more work to be done. One of the problems is the various routines in pgtable.h that update page table entries use different locking mechanisms, or no lock at all (set_pte_at). This change modifies the routines to all use the same lock pa_dbit_lock. This lock is used for dirty bit updates in the interruption code. The patch also purges the TLB entries associated with the PTE to ensure that inconsistent values are not used after the page table entry is updated. The UP and SMP code are now identical. The change also includes a minor update to the purge_tlb_entries function in cache.c to improve its efficiency.
Unified · Split
| @@ -16,6 +16,8 @@ · 15 unmodified lines | ||
| 16 | 16 | #include <asm/processor.h> |
| 17 | 17 | #include <asm/cache.h> |
| 18 | 18 | |
| 19 | +extern spinlock_t pa_dbit_lock; | |
| 20 | + | |
| 19 | 21 | /* |
| 20 | 22 | * kern_addr_valid(ADDR) tests if ADDR is pointing to valid kernel |
| 21 | 23 | * memory. For the return value to be meaningful, ADDR must be >= |
| @@ -44,8 +46,11 @@ extern void purge_tlb_entries(struct mm_struct *, unsigned long); · 22 unmodified lines | ||
| 44 | 46 | |
| 45 | 47 | #define set_pte_at(mm, addr, ptep, pteval) \ |
| 46 | 48 | do { \ |
| 49 | + unsigned long flags; \ | |
| 50 | + spin_lock_irqsave(&pa_dbit_lock, flags); \ | |
| 47 | 51 | set_pte(ptep, pteval); \ |
| 48 | 52 | purge_tlb_entries(mm, addr); \ |
| 53 | + spin_unlock_irqrestore(&pa_dbit_lock, flags); \ | |
| 49 | 54 | } while (0) |
| 50 | 55 | |
| 51 | 56 | #endif /* !__ASSEMBLY__ */ |
| @@ -435,48 +440,46 @@ extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); · 383 unmodified lines | ||
| 435 | 440 | |
| 436 | 441 | static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) |
| 437 | 442 | { |
| 438 | −#ifdef CONFIG_SMP | |
| 443 | + pte_t pte; | |
| 444 | + unsigned long flags; | |
| 445 | + | |
| 439 | 446 | if (!pte_young(*ptep)) |
| 440 | 447 | return 0; |
| 441 | − return test_and_clear_bit(xlate_pabit(_PAGE_ACCESSED_BIT), &pte_val(*ptep)); | |
| 442 | −#else | |
| 443 | − pte_t pte = *ptep; | |
| 444 | − if (!pte_young(pte)) | |
| 448 | + | |
| 449 | + spin_lock_irqsave(&pa_dbit_lock, flags); | |
| 450 | + pte = *ptep; | |
| 451 | + if (!pte_young(pte)) { | |
| 452 | + spin_unlock_irqrestore(&pa_dbit_lock, flags); | |
| 445 | 453 | return 0; |
| 446 | − set_pte_at(vma->vm_mm, addr, ptep, pte_mkold(pte)); | |
| 454 | + } | |
| 455 | + set_pte(ptep, pte_mkold(pte)); | |
| 456 | + purge_tlb_entries(vma->vm_mm, addr); | |
| 457 | + spin_unlock_irqrestore(&pa_dbit_lock, flags); | |
| 447 | 458 | return 1; |
| 448 | −#endif | |
| 449 | 459 | } |
| 450 | 460 | |
| 451 | −extern spinlock_t pa_dbit_lock; | |
| 452 | − | |
| 453 | 461 | struct mm_struct; |
| 454 | 462 | static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
| 455 | 463 | { |
| 456 | 464 | pte_t old_pte; |
| 465 | + unsigned long flags; | |
| 457 | 466 | |
| 458 | − spin_lock(&pa_dbit_lock); | |
| 467 | + spin_lock_irqsave(&pa_dbit_lock, flags); | |
| 459 | 468 | old_pte = *ptep; |
| 460 | 469 | pte_clear(mm,addr,ptep); |
| 461 | − spin_unlock(&pa_dbit_lock); | |
| 470 | + purge_tlb_entries(mm, addr); | |
| 471 | + spin_unlock_irqrestore(&pa_dbit_lock, flags); | |
| 462 | 472 | |
| 463 | 473 | return old_pte; |
| 464 | 474 | } |
| 465 | 475 | |
| 466 | 476 | static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
| 467 | 477 | { |
| 468 | −#ifdef CONFIG_SMP | |
| 469 | − unsigned long new, old; | |
| 470 | − | |
| 471 | − do { | |
| 472 | − old = pte_val(*ptep); | |
| 473 | − new = pte_val(pte_wrprotect(__pte (old))); | |
| 474 | − } while (cmpxchg((unsigned long *) ptep, old, new) != old); | |
| 478 | + unsigned long flags; | |
| 479 | + spin_lock_irqsave(&pa_dbit_lock, flags); | |
| 480 | + set_pte(ptep, pte_wrprotect(*ptep)); | |
| 475 | 481 | purge_tlb_entries(mm, addr); |
| 476 | −#else | |
| 477 | − pte_t old_pte = *ptep; | |
| 478 | − set_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); | |
| 479 | −#endif | |
| 482 | + spin_unlock_irqrestore(&pa_dbit_lock, flags); | |
| 480 | 483 | } |
| 481 | 484 | |
| 482 | 485 | #define pte_same(A,B) (pte_val(A) == pte_val(B)) |
| @@ -421,14 +421,11 @@ void purge_tlb_entries(struct mm_struct *mm, unsigned long addr) · 420 unmodified lines | ||
| 421 | 421 | /* Note: purge_tlb_entries can be called at startup with |
| 422 | 422 | no context. */ |
| 423 | 423 | |
| 424 | − /* Disable preemption while we play with %sr1. */ | |
| 425 | − preempt_disable(); | |
| 426 | − mtsp(mm->context, 1); | |
| 427 | 424 | purge_tlb_start(flags); |
| 425 | + mtsp(mm->context, 1); | |
| 428 | 426 | pdtlb(addr); |
| 429 | 427 | pitlb(addr); |
| 430 | 428 | purge_tlb_end(flags); |
| 431 | − preempt_enable(); | |
| 432 | 429 | } |
| 433 | 430 | EXPORT_SYMBOL(purge_tlb_entries); |
| 434 | 431 | |