| @@ -13,6 +13,7 @@ · 12 unmodified lines |
| 13 | #include <linux/sched.h> | 13 | #include <linux/sched.h> |
| 14 | #include <linux/smp.h> | 14 | #include <linux/smp.h> |
| 15 | #include <linux/memblock.h> | 15 | #include <linux/memblock.h> |
| | 16 | #include <linux/minmax.h> |
| 16 | #include <linux/mm.h> | 17 | #include <linux/mm.h> |
| 17 | #include <linux/hugetlb.h> | 18 | #include <linux/hugetlb.h> |
| 18 | #include <linux/export.h> | 19 | #include <linux/export.h> |
| @@ -24,6 +25,7 @@ · 5 unmodified lines |
| 24 | #include <asm/hazards.h> | 25 | #include <asm/hazards.h> |
| 25 | #include <asm/mmu_context.h> | 26 | #include <asm/mmu_context.h> |
| 26 | #include <asm/tlb.h> | 27 | #include <asm/tlb.h> |
| | 28 | #include <asm/tlbdebug.h> |
| 27 | #include <asm/tlbex.h> | 29 | #include <asm/tlbex.h> |
| 28 | #include <asm/tlbmisc.h> | 30 | #include <asm/tlbmisc.h> |
| 29 | #include <asm/setup.h> | 31 | #include <asm/setup.h> |
| @@ -511,87 +513,259 @@ static int __init set_ntlb(char *str) · 481 unmodified lines |
| 511 | __setup("ntlb=", set_ntlb); | 513 | __setup("ntlb=", set_ntlb); |
| 512 | | 514 | |
| 513 | | 515 | |
| 514 | /* Comparison function for EntryHi VPN fields. */ | 516 | /* The start bit position of VPN2 and Mask in EntryHi/PageMask registers. */ |
| 515 | static int r4k_vpn_cmp(const void *a, const void *b) | 517 | #define VPN2_SHIFT 13 |
| | 518 | |
| | 519 | /* Read full EntryHi even with CONFIG_32BIT. */ |
| | 520 | static inline unsigned long long read_c0_entryhi_native(void) |
| | 521 | { |
| | 522 | return cpu_has_64bits ? read_c0_entryhi_64() : read_c0_entryhi(); |
| | 523 | } |
| | 524 | |
| | 525 | /* Write full EntryHi even with CONFIG_32BIT. */ |
| | 526 | static inline void write_c0_entryhi_native(unsigned long long v) |
| 516 | { | 527 | { |
| 517 | long v = *(unsigned long *)a - *(unsigned long *)b; | 528 | if (cpu_has_64bits) |
| 518 | int s = sizeof(long) > sizeof(int) ? sizeof(long) * 8 - 1: 0; | 529 | write_c0_entryhi_64(v); |
| 519 | return s ? (v != 0) | v >> s : v; | 530 | else |
| | 531 | write_c0_entryhi(v); |
| 520 | } | 532 | } |
| 521 | | 533 | |
| | 534 | /* TLB entry state for uniquification. */ |
| | 535 | struct tlbent { |
| | 536 | unsigned long long wired:1; |
| | 537 | unsigned long long global:1; |
| | 538 | unsigned long long asid:10; |
| | 539 | unsigned long long vpn:51; |
| | 540 | unsigned long long pagesz:5; |
| | 541 | unsigned long long index:14; |
| | 542 | }; |
| | 543 | |
| 522 | /* | 544 | /* |
| 523 | * Initialise all TLB entries with unique values that do not clash with | 545 | * Comparison function for TLB entry sorting. Place wired entries first, |
| 524 | * what we have been handed over and what we'll be using ourselves. | 546 | * then global entries, then order by the increasing VPN/ASID and the |
| | 547 | * decreasing page size. This lets us avoid clashes with wired entries |
| | 548 | * easily and get entries for larger pages out of the way first. |
| | 549 | * |
| | 550 | * We could group bits so as to reduce the number of comparisons, but this |
| | 551 | * is seldom executed and not performance-critical, so prefer legibility. |
| 525 | */ | 552 | */ |
| 526 | static void __ref r4k_tlb_uniquify(void) | 553 | static int r4k_entry_cmp(const void *a, const void *b) |
| 527 | { | 554 | { |
| 528 | int tlbsize = current_cpu_data.tlbsize; | 555 | struct tlbent ea = *(struct tlbent *)a, eb = *(struct tlbent *)b; |
| 529 | bool use_slab = slab_is_available(); | 556 | |
| 530 | int start = num_wired_entries(); | 557 | if (ea.wired > eb.wired) |
| 531 | phys_addr_t tlb_vpn_size; | 558 | return -1; |
| 532 | unsigned long *tlb_vpns; | 559 | else if (ea.wired < eb.wired) |
| 533 | unsigned long vpn_mask; | 560 | return 1; |
| 534 | int cnt, ent, idx, i; | 561 | else if (ea.global > eb.global) |
| 535 | | 562 | return -1; |
| 536 | vpn_mask = GENMASK(cpu_vmbits - 1, 13); | 563 | else if (ea.global < eb.global) |
| 537 | vpn_mask |= IS_ENABLED(CONFIG_64BIT) ? 3ULL << 62 : 1 << 31; | 564 | return 1; |
| | 565 | else if (ea.vpn < eb.vpn) |
| | 566 | return -1; |
| | 567 | else if (ea.vpn > eb.vpn) |
| | 568 | return 1; |
| | 569 | else if (ea.asid < eb.asid) |
| | 570 | return -1; |
| | 571 | else if (ea.asid > eb.asid) |
| | 572 | return 1; |
| | 573 | else if (ea.pagesz > eb.pagesz) |
| | 574 | return -1; |
| | 575 | else if (ea.pagesz < eb.pagesz) |
| | 576 | return 1; |
| | 577 | else |
| | 578 | return 0; |
| | 579 | } |
| 538 | | 580 | |
| 539 | tlb_vpn_size = tlbsize * sizeof(*tlb_vpns); | 581 | /* |
| 540 | tlb_vpns = (use_slab ? | 582 | * Fetch all the TLB entries. Mask individual VPN values retrieved with |
| 541 | kmalloc(tlb_vpn_size, GFP_KERNEL) : | 583 | * the corresponding page mask and ignoring any 1KiB extension as we'll |
| 542 | memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns))); | 584 | * be using 4KiB pages for uniquification. |
| 543 | if (WARN_ON(!tlb_vpns)) | 585 | */ |
| 544 | return; /* Pray local_flush_tlb_all() is good enough. */ | 586 | static void __ref r4k_tlb_uniquify_read(struct tlbent *tlb_vpns, int tlbsize) |
| | 587 | { |
| | 588 | int start = num_wired_entries(); |
| | 589 | unsigned long long vpn_mask; |
| | 590 | bool global; |
| | 591 | int i; |
| 545 | | 592 | |
| 546 | htw_stop(); | 593 | vpn_mask = GENMASK(current_cpu_data.vmbits - 1, VPN2_SHIFT); |
| | 594 | vpn_mask |= cpu_has_64bits ? 3ULL << 62 : 1 << 31; |
| 547 | | 595 | |
| 548 | for (i = start, cnt = 0; i < tlbsize; i++, cnt++) { | 596 | for (i = 0; i < tlbsize; i++) { |
| 549 | unsigned long vpn; | 597 | unsigned long long entryhi, vpn, mask, asid; |
| | 598 | unsigned int pagesz; |
| 550 | | 599 | |
| 551 | write_c0_index(i); | 600 | write_c0_index(i); |
| 552 | mtc0_tlbr_hazard(); | 601 | mtc0_tlbr_hazard(); |
| 553 | tlb_read(); | 602 | tlb_read(); |
| 554 | tlb_read_hazard(); | 603 | tlb_read_hazard(); |
| 555 | vpn = read_c0_entryhi(); | | |
| 556 | vpn &= vpn_mask & PAGE_MASK; | | |
| 557 | tlb_vpns[cnt] = vpn; | | |
| 558 | | 604 | |
| 559 | /* Prevent any large pages from overlapping regular ones. */ | 605 | global = !!(read_c0_entrylo0() & ENTRYLO_G); |
| 560 | write_c0_pagemask(read_c0_pagemask() & PM_DEFAULT_MASK); | 606 | entryhi = read_c0_entryhi_native(); |
| 561 | mtc0_tlbw_hazard(); | 607 | mask = read_c0_pagemask(); |
| 562 | tlb_write_indexed(); | 608 | |
| 563 | tlbw_use_hazard(); | 609 | asid = entryhi & cpu_asid_mask(¤t_cpu_data); |
| | 610 | vpn = (entryhi & vpn_mask & ~mask) >> VPN2_SHIFT; |
| | 611 | pagesz = ilog2((mask >> VPN2_SHIFT) + 1); |
| | 612 | |
| | 613 | tlb_vpns[i].global = global; |
| | 614 | tlb_vpns[i].asid = global ? 0 : asid; |
| | 615 | tlb_vpns[i].vpn = vpn; |
| | 616 | tlb_vpns[i].pagesz = pagesz; |
| | 617 | tlb_vpns[i].wired = i < start; |
| | 618 | tlb_vpns[i].index = i; |
| 564 | } | 619 | } |
| | 620 | } |
| | 621 | |
| | 622 | /* |
| | 623 | * Write unique values to all but the wired TLB entries each, using |
| | 624 | * the 4KiB page size. This size might not be supported with R6, but |
| | 625 | * EHINV is mandatory for R6, so we won't ever be called in that case. |
| | 626 | * |
| | 627 | * A sorted table is supplied with any wired entries at the beginning, |
| | 628 | * followed by any global entries, and then finally regular entries. |
| | 629 | * We start at the VPN and ASID values of zero and only assign user |
| | 630 | * addresses, therefore guaranteeing no clash with addresses produced |
| | 631 | * by UNIQUE_ENTRYHI. We avoid any VPN values used by wired or global |
| | 632 | * entries, by increasing the VPN value beyond the span of such entry. |
| | 633 | * |
| | 634 | * When a VPN/ASID clash is found with a regular entry we increment the |
| | 635 | * ASID instead until no VPN/ASID clash has been found or the ASID space |
| | 636 | * has been exhausted, in which case we increase the VPN value beyond |
| | 637 | * the span of the largest clashing entry. |
| | 638 | * |
| | 639 | * We do not need to be concerned about FTLB or MMID configurations as |
| | 640 | * those are required to implement the EHINV feature. |
| | 641 | */ |
| | 642 | static void __ref r4k_tlb_uniquify_write(struct tlbent *tlb_vpns, int tlbsize) |
| | 643 | { |
| | 644 | unsigned long long asid, vpn, vpn_size, pagesz; |
| | 645 | int widx, gidx, idx, sidx, lidx, i; |
| 565 | | 646 | |
| 566 | sort(tlb_vpns, cnt, sizeof(tlb_vpns[0]), r4k_vpn_cmp, NULL); | 647 | vpn_size = 1ULL << (current_cpu_data.vmbits - VPN2_SHIFT); |
| | 648 | pagesz = ilog2((PM_4K >> VPN2_SHIFT) + 1); |
| 567 | | 649 | |
| 568 | write_c0_pagemask(PM_DEFAULT_MASK); | 650 | write_c0_pagemask(PM_4K); |
| 569 | write_c0_entrylo0(0); | 651 | write_c0_entrylo0(0); |
| 570 | write_c0_entrylo1(0); | 652 | write_c0_entrylo1(0); |
| 571 | | 653 | |
| 572 | idx = 0; | 654 | asid = 0; |
| 573 | ent = tlbsize; | 655 | vpn = 0; |
| 574 | for (i = start; i < tlbsize; i++) | 656 | widx = 0; |
| 575 | while (1) { | 657 | gidx = 0; |
| 576 | unsigned long entryhi, vpn; | 658 | for (sidx = 0; sidx < tlbsize && tlb_vpns[sidx].wired; sidx++) |
| | 659 | ; |
| | 660 | for (lidx = sidx; lidx < tlbsize && tlb_vpns[lidx].global; lidx++) |
| | 661 | ; |
| | 662 | idx = gidx = sidx + 1; |
| | 663 | for (i = sidx; i < tlbsize; i++) { |
| | 664 | unsigned long long entryhi, vpn_pagesz = 0; |
| 577 | | 665 | |
| 578 | entryhi = UNIQUE_ENTRYHI(ent); | 666 | while (1) { |
| 579 | vpn = entryhi & vpn_mask & PAGE_MASK; | 667 | if (WARN_ON(vpn >= vpn_size)) { |
| | 668 | dump_tlb_all(); |
| | 669 | /* Pray local_flush_tlb_all() will cope. */ |
| | 670 | return; |
| | 671 | } |
| 580 | | 672 | |
| 581 | if (idx >= cnt || vpn < tlb_vpns[idx]) { | 673 | /* VPN must be below the next wired entry. */ |
| 582 | write_c0_entryhi(entryhi); | 674 | if (widx < sidx && vpn >= tlb_vpns[widx].vpn) { |
| 583 | write_c0_index(i); | 675 | vpn = max(vpn, |
| 584 | mtc0_tlbw_hazard(); | 676 | (tlb_vpns[widx].vpn + |
| 585 | tlb_write_indexed(); | 677 | (1ULL << tlb_vpns[widx].pagesz))); |
| 586 | ent++; | 678 | asid = 0; |
| 587 | break; | 679 | widx++; |
| 588 | } else if (vpn == tlb_vpns[idx]) { | 680 | continue; |
| 589 | ent++; | 681 | } |
| 590 | } else { | 682 | /* VPN must be below the next global entry. */ |
| | 683 | if (gidx < lidx && vpn >= tlb_vpns[gidx].vpn) { |
| | 684 | vpn = max(vpn, |
| | 685 | (tlb_vpns[gidx].vpn + |
| | 686 | (1ULL << tlb_vpns[gidx].pagesz))); |
| | 687 | asid = 0; |
| | 688 | gidx++; |
| | 689 | continue; |
| | 690 | } |
| | 691 | /* Try to find a free ASID so as to conserve VPNs. */ |
| | 692 | if (idx < tlbsize && vpn == tlb_vpns[idx].vpn && |
| | 693 | asid == tlb_vpns[idx].asid) { |
| | 694 | unsigned long long idx_pagesz; |
| | 695 | |
| | 696 | idx_pagesz = tlb_vpns[idx].pagesz; |
| | 697 | vpn_pagesz = max(vpn_pagesz, idx_pagesz); |
| | 698 | do |
| | 699 | idx++; |
| | 700 | while (idx < tlbsize && |
| | 701 | vpn == tlb_vpns[idx].vpn && |
| | 702 | asid == tlb_vpns[idx].asid); |
| | 703 | asid++; |
| | 704 | if (asid > cpu_asid_mask(¤t_cpu_data)) { |
| | 705 | vpn += vpn_pagesz; |
| | 706 | asid = 0; |
| | 707 | vpn_pagesz = 0; |
| | 708 | } |
| | 709 | continue; |
| | 710 | } |
| | 711 | /* VPN mustn't be above the next regular entry. */ |
| | 712 | if (idx < tlbsize && vpn > tlb_vpns[idx].vpn) { |
| | 713 | vpn = max(vpn, |
| | 714 | (tlb_vpns[idx].vpn + |
| | 715 | (1ULL << tlb_vpns[idx].pagesz))); |
| | 716 | asid = 0; |
| 591 | idx++; | 717 | idx++; |
| | 718 | continue; |
| 592 | } | 719 | } |
| | 720 | break; |
| 593 | } | 721 | } |
| 594 | | 722 | |
| | 723 | entryhi = (vpn << VPN2_SHIFT) | asid; |
| | 724 | write_c0_entryhi_native(entryhi); |
| | 725 | write_c0_index(tlb_vpns[i].index); |
| | 726 | mtc0_tlbw_hazard(); |
| | 727 | tlb_write_indexed(); |
| | 728 | |
| | 729 | tlb_vpns[i].asid = asid; |
| | 730 | tlb_vpns[i].vpn = vpn; |
| | 731 | tlb_vpns[i].pagesz = pagesz; |
| | 732 | |
| | 733 | asid++; |
| | 734 | if (asid > cpu_asid_mask(¤t_cpu_data)) { |
| | 735 | vpn += 1ULL << pagesz; |
| | 736 | asid = 0; |
| | 737 | } |
| | 738 | } |
| | 739 | } |
| | 740 | |
| | 741 | /* |
| | 742 | * Initialise all TLB entries with unique values that do not clash with |
| | 743 | * what we have been handed over and what we'll be using ourselves. |
| | 744 | */ |
| | 745 | static void __ref r4k_tlb_uniquify(void) |
| | 746 | { |
| | 747 | int tlbsize = current_cpu_data.tlbsize; |
| | 748 | bool use_slab = slab_is_available(); |
| | 749 | phys_addr_t tlb_vpn_size; |
| | 750 | struct tlbent *tlb_vpns; |
| | 751 | |
| | 752 | tlb_vpn_size = tlbsize * sizeof(*tlb_vpns); |
| | 753 | tlb_vpns = (use_slab ? |
| | 754 | kmalloc(tlb_vpn_size, GFP_ATOMIC) : |
| | 755 | memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns))); |
| | 756 | if (WARN_ON(!tlb_vpns)) |
| | 757 | return; /* Pray local_flush_tlb_all() is good enough. */ |
| | 758 | |
| | 759 | htw_stop(); |
| | 760 | |
| | 761 | r4k_tlb_uniquify_read(tlb_vpns, tlbsize); |
| | 762 | |
| | 763 | sort(tlb_vpns, tlbsize, sizeof(*tlb_vpns), r4k_entry_cmp, NULL); |
| | 764 | |
| | 765 | r4k_tlb_uniquify_write(tlb_vpns, tlbsize); |
| | 766 | |
| | 767 | write_c0_pagemask(PM_DEFAULT_MASK); |
| | 768 | |
| 595 | tlbw_use_hazard(); | 769 | tlbw_use_hazard(); |
| 596 | htw_start(); | 770 | htw_start(); |
| 597 | flush_micro_tlb(); | 771 | flush_micro_tlb(); |
| @@ -640,7 +814,8 @@ static void r4k_tlb_configure(void) · 42 unmodified lines |
| 640 | temp_tlb_entry = current_cpu_data.tlbsize - 1; | 814 | temp_tlb_entry = current_cpu_data.tlbsize - 1; |
| 641 | | 815 | |
| 642 | /* From this point on the ARC firmware is dead. */ | 816 | /* From this point on the ARC firmware is dead. */ |
| 643 | r4k_tlb_uniquify(); | 817 | if (!cpu_has_tlbinv) |
| | 818 | r4k_tlb_uniquify(); |
| 644 | local_flush_tlb_all(); | 819 | local_flush_tlb_all(); |
| 645 | | 820 | |
| 646 | /* Did I tell you that ARC SUCKS? */ | 821 | /* Did I tell you that ARC SUCKS? */ |