| @@ -661,7 +661,7 @@ static struct its_collection *its_build_invall_cmd(struct its_node *its, · 660 unmodified lines |
| 661 | struct its_cmd_desc *desc) | 661 | struct its_cmd_desc *desc) |
| 662 | { | 662 | { |
| 663 | its_encode_cmd(cmd, GITS_CMD_INVALL); | 663 | its_encode_cmd(cmd, GITS_CMD_INVALL); |
| 664 | its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); | 664 | its_encode_collection(cmd, desc->its_invall_cmd.col->col_id); |
| 665 | | 665 | |
| 666 | its_fixup_cmd(cmd); | 666 | its_fixup_cmd(cmd); |
| 667 | | 667 | |
| @@ -2376,6 +2376,8 @@ static u64 inherit_vpe_l1_table_from_its(void) · 1708 unmodified lines |
| 2376 | continue; | 2376 | continue; |
| 2377 | | 2377 | |
| 2378 | /* We have a winner! */ | 2378 | /* We have a winner! */ |
| | 2379 | gic_data_rdist()->vpe_l1_base = its->tables[2].base; |
| | 2380 | |
| 2379 | val = GICR_VPROPBASER_4_1_VALID; | 2381 | val = GICR_VPROPBASER_4_1_VALID; |
| 2380 | if (baser & GITS_BASER_INDIRECT) | 2382 | if (baser & GITS_BASER_INDIRECT) |
| 2381 | val |= GICR_VPROPBASER_4_1_INDIRECT; | 2383 | val |= GICR_VPROPBASER_4_1_INDIRECT; |
| @@ -2413,14 +2415,12 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) · 31 unmodified lines |
| 2413 | | 2415 | |
| 2414 | for_each_possible_cpu(cpu) { | 2416 | for_each_possible_cpu(cpu) { |
| 2415 | void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; | 2417 | void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; |
| 2416 | u32 tmp; | | |
| 2417 | | 2418 | |
| 2418 | if (!base || cpu == smp_processor_id()) | 2419 | if (!base || cpu == smp_processor_id()) |
| 2419 | continue; | 2420 | continue; |
| 2420 | | 2421 | |
| 2421 | val = gic_read_typer(base + GICR_TYPER); | 2422 | val = gic_read_typer(base + GICR_TYPER); |
| 2422 | tmp = compute_common_aff(val); | 2423 | if (aff != compute_common_aff(val)) |
| 2423 | if (tmp != aff) | | |
| 2424 | continue; | 2424 | continue; |
| 2425 | | 2425 | |
| 2426 | /* | 2426 | /* |
| @@ -2429,9 +2429,10 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) · 2 unmodified lines |
| 2429 | * ours wrt CommonLPIAff. Let's use its own VPROPBASER. | 2429 | * ours wrt CommonLPIAff. Let's use its own VPROPBASER. |
| 2430 | * Make sure we don't write the Z bit in that case. | 2430 | * Make sure we don't write the Z bit in that case. |
| 2431 | */ | 2431 | */ |
| 2432 | val = gits_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); | 2432 | val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); |
| 2433 | val &= ~GICR_VPROPBASER_4_1_Z; | 2433 | val &= ~GICR_VPROPBASER_4_1_Z; |
| 2434 | | 2434 | |
| | 2435 | gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base; |
| 2435 | *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; | 2436 | *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; |
| 2436 | | 2437 | |
| 2437 | return val; | 2438 | return val; |
| @@ -2440,6 +2441,72 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) · 2 unmodified lines |
| 2440 | return 0; | 2441 | return 0; |
| 2441 | } | 2442 | } |
| 2442 | | 2443 | |
| | 2444 | static bool allocate_vpe_l2_table(int cpu, u32 id) |
| | 2445 | { |
| | 2446 | void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; |
| | 2447 | u64 val, gpsz, npg; |
| | 2448 | unsigned int psz, esz, idx; |
| | 2449 | struct page *page; |
| | 2450 | __le64 *table; |
| | 2451 | |
| | 2452 | if (!gic_rdists->has_rvpeid) |
| | 2453 | return true; |
| | 2454 | |
| | 2455 | val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); |
| | 2456 | |
| | 2457 | esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1; |
| | 2458 | gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); |
| | 2459 | npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1; |
| | 2460 | |
| | 2461 | switch (gpsz) { |
| | 2462 | default: |
| | 2463 | WARN_ON(1); |
| | 2464 | /* fall through */ |
| | 2465 | case GIC_PAGE_SIZE_4K: |
| | 2466 | psz = SZ_4K; |
| | 2467 | break; |
| | 2468 | case GIC_PAGE_SIZE_16K: |
| | 2469 | psz = SZ_16K; |
| | 2470 | break; |
| | 2471 | case GIC_PAGE_SIZE_64K: |
| | 2472 | psz = SZ_64K; |
| | 2473 | break; |
| | 2474 | } |
| | 2475 | |
| | 2476 | /* Don't allow vpe_id that exceeds single, flat table limit */ |
| | 2477 | if (!(val & GICR_VPROPBASER_4_1_INDIRECT)) |
| | 2478 | return (id < (npg * psz / (esz * SZ_8))); |
| | 2479 | |
| | 2480 | /* Compute 1st level table index & check if that exceeds table limit */ |
| | 2481 | idx = id >> ilog2(psz / (esz * SZ_8)); |
| | 2482 | if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE)) |
| | 2483 | return false; |
| | 2484 | |
| | 2485 | table = gic_data_rdist_cpu(cpu)->vpe_l1_base; |
| | 2486 | |
| | 2487 | /* Allocate memory for 2nd level table */ |
| | 2488 | if (!table[idx]) { |
| | 2489 | page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); |
| | 2490 | if (!page) |
| | 2491 | return false; |
| | 2492 | |
| | 2493 | /* Flush Lvl2 table to PoC if hw doesn't support coherency */ |
| | 2494 | if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK)) |
| | 2495 | gic_flush_dcache_to_poc(page_address(page), psz); |
| | 2496 | |
| | 2497 | table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); |
| | 2498 | |
| | 2499 | /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ |
| | 2500 | if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK)) |
| | 2501 | gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE); |
| | 2502 | |
| | 2503 | /* Ensure updated table contents are visible to RD hardware */ |
| | 2504 | dsb(sy); |
| | 2505 | } |
| | 2506 | |
| | 2507 | return true; |
| | 2508 | } |
| | 2509 | |
| 2443 | static int allocate_vpe_l1_table(void) | 2510 | static int allocate_vpe_l1_table(void) |
| 2444 | { | 2511 | { |
| 2445 | void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); | 2512 | void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); |
| @@ -2457,8 +2524,8 @@ static int allocate_vpe_l1_table(void) · 11 unmodified lines |
| 2457 | * effect of making sure no doorbell will be generated and we can | 2524 | * effect of making sure no doorbell will be generated and we can |
| 2458 | * then safely clear VPROPBASER.Valid. | 2525 | * then safely clear VPROPBASER.Valid. |
| 2459 | */ | 2526 | */ |
| 2460 | if (gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid) | 2527 | if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid) |
| 2461 | gits_write_vpendbaser(GICR_VPENDBASER_PendingLast, | 2528 | gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, |
| 2462 | vlpi_base + GICR_VPENDBASER); | 2529 | vlpi_base + GICR_VPENDBASER); |
| 2463 | | 2530 | |
| 2464 | /* | 2531 | /* |
| @@ -2481,8 +2548,8 @@ static int allocate_vpe_l1_table(void) · 16 unmodified lines |
| 2481 | | 2548 | |
| 2482 | /* First probe the page size */ | 2549 | /* First probe the page size */ |
| 2483 | val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K); | 2550 | val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K); |
| 2484 | gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); | 2551 | gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); |
| 2485 | val = gits_read_vpropbaser(vlpi_base + GICR_VPROPBASER); | 2552 | val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER); |
| 2486 | gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); | 2553 | gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); |
| 2487 | esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val); | 2554 | esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val); |
| 2488 | | 2555 | |
| @@ -2531,7 +2598,7 @@ static int allocate_vpe_l1_table(void) · 42 unmodified lines |
| 2531 | npg = 1; | 2598 | npg = 1; |
| 2532 | } | 2599 | } |
| 2533 | | 2600 | |
| 2534 | val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg); | 2601 | val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1); |
| 2535 | | 2602 | |
| 2536 | /* Right, that's the number of CPU pages we need for L1 */ | 2603 | /* Right, that's the number of CPU pages we need for L1 */ |
| 2537 | np = DIV_ROUND_UP(npg * psz, PAGE_SIZE); | 2604 | np = DIV_ROUND_UP(npg * psz, PAGE_SIZE); |
| @@ -2542,7 +2609,7 @@ static int allocate_vpe_l1_table(void) · 4 unmodified lines |
| 2542 | if (!page) | 2609 | if (!page) |
| 2543 | return -ENOMEM; | 2610 | return -ENOMEM; |
| 2544 | | 2611 | |
| 2545 | gic_data_rdist()->vpe_l1_page = page; | 2612 | gic_data_rdist()->vpe_l1_base = page_address(page); |
| 2546 | pa = virt_to_phys(page_address(page)); | 2613 | pa = virt_to_phys(page_address(page)); |
| 2547 | WARN_ON(!IS_ALIGNED(pa, psz)); | 2614 | WARN_ON(!IS_ALIGNED(pa, psz)); |
| 2548 | | 2615 | |
| @@ -2553,7 +2620,7 @@ static int allocate_vpe_l1_table(void) · 4 unmodified lines |
| 2553 | val |= GICR_VPROPBASER_4_1_VALID; | 2620 | val |= GICR_VPROPBASER_4_1_VALID; |
| 2554 | | 2621 | |
| 2555 | out: | 2622 | out: |
| 2556 | gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); | 2623 | gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); |
| 2557 | cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); | 2624 | cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); |
| 2558 | | 2625 | |
| 2559 | pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n", | 2626 | pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n", |
| @@ -2660,14 +2727,14 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) · 100 unmodified lines |
| 2660 | bool clean; | 2727 | bool clean; |
| 2661 | u64 val; | 2728 | u64 val; |
| 2662 | | 2729 | |
| 2663 | val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); | 2730 | val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); |
| 2664 | val &= ~GICR_VPENDBASER_Valid; | 2731 | val &= ~GICR_VPENDBASER_Valid; |
| 2665 | val &= ~clr; | 2732 | val &= ~clr; |
| 2666 | val |= set; | 2733 | val |= set; |
| 2667 | gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); | 2734 | gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); |
| 2668 | | 2735 | |
| 2669 | do { | 2736 | do { |
| 2670 | val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER); | 2737 | val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); |
| 2671 | clean = !(val & GICR_VPENDBASER_Dirty); | 2738 | clean = !(val & GICR_VPENDBASER_Dirty); |
| 2672 | if (!clean) { | 2739 | if (!clean) { |
| 2673 | count--; | 2740 | count--; |
| @@ -2782,7 +2849,7 @@ static void its_cpu_init_lpis(void) · 108 unmodified lines |
| 2782 | val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; | 2849 | val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; |
| 2783 | pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n", | 2850 | pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n", |
| 2784 | smp_processor_id(), val); | 2851 | smp_processor_id(), val); |
| 2785 | gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); | 2852 | gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); |
| 2786 | | 2853 | |
| 2787 | /* | 2854 | /* |
| 2788 | * Also clear Valid bit of GICR_VPENDBASER, in case some | 2855 | * Also clear Valid bit of GICR_VPENDBASER, in case some |
| @@ -2790,7 +2857,6 @@ static void its_cpu_init_lpis(void) · 1 unmodified line |
| 2790 | * corrupting memory. | 2857 | * corrupting memory. |
| 2791 | */ | 2858 | */ |
| 2792 | val = its_clear_vpend_valid(vlpi_base, 0, 0); | 2859 | val = its_clear_vpend_valid(vlpi_base, 0, 0); |
| 2793 | WARN_ON(val & GICR_VPENDBASER_Dirty); | | |
| 2794 | } | 2860 | } |
| 2795 | | 2861 | |
| 2796 | if (allocate_vpe_l1_table()) { | 2862 | if (allocate_vpe_l1_table()) { |
| @@ -2954,6 +3020,7 @@ static bool its_alloc_device_table(struct its_node *its, u32 dev_id) · 157 unmodified lines |
| 2954 | static bool its_alloc_vpe_table(u32 vpe_id) | 3020 | static bool its_alloc_vpe_table(u32 vpe_id) |
| 2955 | { | 3021 | { |
| 2956 | struct its_node *its; | 3022 | struct its_node *its; |
| | 3023 | int cpu; |
| 2957 | | 3024 | |
| 2958 | /* | 3025 | /* |
| 2959 | * Make sure the L2 tables are allocated on *all* v4 ITSs. We | 3026 | * Make sure the L2 tables are allocated on *all* v4 ITSs. We |
| @@ -2976,6 +3043,19 @@ static bool its_alloc_vpe_table(u32 vpe_id) · 16 unmodified lines |
| 2976 | return false; | 3043 | return false; |
| 2977 | } | 3044 | } |
| 2978 | | 3045 | |
| | 3046 | /* Non v4.1? No need to iterate RDs and go back early. */ |
| | 3047 | if (!gic_rdists->has_rvpeid) |
| | 3048 | return true; |
| | 3049 | |
| | 3050 | /* |
| | 3051 | * Make sure the L2 tables are allocated for all copies of |
| | 3052 | * the L1 table on *all* v4.1 RDs. |
| | 3053 | */ |
| | 3054 | for_each_possible_cpu(cpu) { |
| | 3055 | if (!allocate_vpe_l2_table(cpu, vpe_id)) |
| | 3056 | return false; |
| | 3057 | } |
| | 3058 | |
| 2979 | return true; | 3059 | return true; |
| 2980 | } | 3060 | } |
| 2981 | | 3061 | |
| @@ -3443,7 +3523,7 @@ static void its_vpe_schedule(struct its_vpe *vpe) · 461 unmodified lines |
| 3443 | val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; | 3523 | val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; |
| 3444 | val |= GICR_VPROPBASER_RaWb; | 3524 | val |= GICR_VPROPBASER_RaWb; |
| 3445 | val |= GICR_VPROPBASER_InnerShareable; | 3525 | val |= GICR_VPROPBASER_InnerShareable; |
| 3446 | gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); | 3526 | gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); |
| 3447 | | 3527 | |
| 3448 | val = virt_to_phys(page_address(vpe->vpt_page)) & | 3528 | val = virt_to_phys(page_address(vpe->vpt_page)) & |
| 3449 | GENMASK_ULL(51, 16); | 3529 | GENMASK_ULL(51, 16); |
| @@ -3461,7 +3541,7 @@ static void its_vpe_schedule(struct its_vpe *vpe) · 11 unmodified lines |
| 3461 | val |= GICR_VPENDBASER_PendingLast; | 3541 | val |= GICR_VPENDBASER_PendingLast; |
| 3462 | val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; | 3542 | val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; |
| 3463 | val |= GICR_VPENDBASER_Valid; | 3543 | val |= GICR_VPENDBASER_Valid; |
| 3464 | gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); | 3544 | gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); |
| 3465 | } | 3545 | } |
| 3466 | | 3546 | |
| 3467 | static void its_vpe_deschedule(struct its_vpe *vpe) | 3547 | static void its_vpe_deschedule(struct its_vpe *vpe) |
| @@ -3661,7 +3741,7 @@ static void its_vpe_4_1_schedule(struct its_vpe *vpe, · 193 unmodified lines |
| 3661 | val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; | 3741 | val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; |
| 3662 | val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); | 3742 | val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); |
| 3663 | | 3743 | |
| 3664 | gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); | 3744 | gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); |
| 3665 | } | 3745 | } |
| 3666 | | 3746 | |
| 3667 | static void its_vpe_4_1_deschedule(struct its_vpe *vpe, | 3747 | static void its_vpe_4_1_deschedule(struct its_vpe *vpe, |