Pull arm64 fixes from Catalin Marinas: - The alternatives patching code uses flush_icache_range() which itself uses alternatives. Change the code to use an unpatched variant of cache maintenance - Remove unnecessary ISBs from set_{pte,pmd,pud} - perf: xgene_pmu: Fix IOB SLOW PMU parser error * tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: arm64: Remove unnecessary ISBs from set_{pte,pmd,pud} arm64: Avoid flush_icache_range() in alternatives patching code drivers/perf: xgene_pmu: Fix IOB SLOW PMU parser error
Unified · Split
| @@ -28,7 +28,12 @@ typedef void (*alternative_cb_t)(struct alt_instr *alt, · 27 unmodified lines | ||
| 28 | 28 | __le32 *origptr, __le32 *updptr, int nr_inst); |
| 29 | 29 | |
| 30 | 30 | void __init apply_alternatives_all(void); |
| 31 | −void apply_alternatives(void *start, size_t length); | |
| 31 | + | |
| 32 | +#ifdef CONFIG_MODULES | |
| 33 | +void apply_alternatives_module(void *start, size_t length); | |
| 34 | +#else | |
| 35 | +static inline void apply_alternatives_module(void *start, size_t length) { } | |
| 36 | +#endif | |
| 32 | 37 | |
| 33 | 38 | #define ALTINSTR_ENTRY(feature,cb) \ |
| 34 | 39 | " .word 661b - .\n" /* label */ \ |
| @@ -224,10 +224,8 @@ static inline void set_pte(pte_t *ptep, pte_t pte) · 223 unmodified lines | ||
| 224 | 224 | * Only if the new pte is valid and kernel, otherwise TLB maintenance |
| 225 | 225 | * or update_mmu_cache() have the necessary barriers. |
| 226 | 226 | */ |
| 227 | − if (pte_valid_not_user(pte)) { | |
| 227 | + if (pte_valid_not_user(pte)) | |
| 228 | 228 | dsb(ishst); |
| 229 | − isb(); | |
| 230 | − } | |
| 231 | 229 | } |
| 232 | 230 | |
| 233 | 231 | extern void __sync_icache_dcache(pte_t pteval); |
| @@ -434,7 +432,6 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) · 200 unmodified lines | ||
| 434 | 432 | { |
| 435 | 433 | WRITE_ONCE(*pmdp, pmd); |
| 436 | 434 | dsb(ishst); |
| 437 | − isb(); | |
| 438 | 435 | } |
| 439 | 436 | |
| 440 | 437 | static inline void pmd_clear(pmd_t *pmdp) |
| @@ -485,7 +482,6 @@ static inline void set_pud(pud_t *pudp, pud_t pud) · 44 unmodified lines | ||
| 485 | 482 | { |
| 486 | 483 | WRITE_ONCE(*pudp, pud); |
| 487 | 484 | dsb(ishst); |
| 488 | − isb(); | |
| 489 | 485 | } |
| 490 | 486 | |
| 491 | 487 | static inline void pud_clear(pud_t *pudp) |
| @@ -122,7 +122,30 @@ static void patch_alternative(struct alt_instr *alt, · 121 unmodified lines | ||
| 122 | 122 | } |
| 123 | 123 | } |
| 124 | 124 | |
| 125 | −static void __apply_alternatives(void *alt_region, bool use_linear_alias) | |
| 125 | +/* | |
| 126 | + * We provide our own, private D-cache cleaning function so that we don't | |
| 127 | + * accidentally call into the cache.S code, which is patched by us at | |
| 128 | + * runtime. | |
| 129 | + */ | |
| 130 | +static void clean_dcache_range_nopatch(u64 start, u64 end) | |
| 131 | +{ | |
| 132 | + u64 cur, d_size, ctr_el0; | |
| 133 | + | |
| 134 | + ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0); | |
| 135 | + d_size = 4 << cpuid_feature_extract_unsigned_field(ctr_el0, | |
| 136 | + CTR_DMINLINE_SHIFT); | |
| 137 | + cur = start & ~(d_size - 1); | |
| 138 | + do { | |
| 139 | + /* | |
| 140 | + * We must clean+invalidate to the PoC in order to avoid | |
| 141 | + * Cortex-A53 errata 826319, 827319, 824069 and 819472 | |
| 142 | + * (this corresponds to ARM64_WORKAROUND_CLEAN_CACHE) | |
| 143 | + */ | |
| 144 | + asm volatile("dc civac, %0" : : "r" (cur) : "memory"); | |
| 145 | + } while (cur += d_size, cur < end); | |
| 146 | +} | |
| 147 | + | |
| 148 | +static void __apply_alternatives(void *alt_region, bool is_module) | |
| 126 | 149 | { |
| 127 | 150 | struct alt_instr *alt; |
| 128 | 151 | struct alt_region *region = alt_region; |
| @@ -145,7 +168,7 @@ static void __apply_alternatives(void *alt_region, bool use_linear_alias) · 16 unmodified lines | ||
| 145 | 168 | pr_info_once("patching kernel code\n"); |
| 146 | 169 | |
| 147 | 170 | origptr = ALT_ORIG_PTR(alt); |
| 148 | − updptr = use_linear_alias ? lm_alias(origptr) : origptr; | |
| 171 | + updptr = is_module ? origptr : lm_alias(origptr); | |
| 149 | 172 | nr_inst = alt->orig_len / AARCH64_INSN_SIZE; |
| 150 | 173 | |
| 151 | 174 | if (alt->cpufeature < ARM64_CB_PATCH) |
| @@ -155,8 +178,20 @@ static void __apply_alternatives(void *alt_region, bool use_linear_alias) · 3 unmodified lines | ||
| 155 | 178 | |
| 156 | 179 | alt_cb(alt, origptr, updptr, nr_inst); |
| 157 | 180 | |
| 158 | − flush_icache_range((uintptr_t)origptr, | |
| 159 | − (uintptr_t)(origptr + nr_inst)); | |
| 181 | + if (!is_module) { | |
| 182 | + clean_dcache_range_nopatch((u64)origptr, | |
| 183 | + (u64)(origptr + nr_inst)); | |
| 184 | + } | |
| 185 | + } | |
| 186 | + | |
| 187 | + /* | |
| 188 | + * The core module code takes care of cache maintenance in | |
| 189 | + * flush_module_icache(). | |
| 190 | + */ | |
| 191 | + if (!is_module) { | |
| 192 | + dsb(ish); | |
| 193 | + __flush_icache_all(); | |
| 194 | + isb(); | |
| 160 | 195 | } |
| 161 | 196 | } |
| 162 | 197 | |
| @@ -178,7 +213,7 @@ static int __apply_alternatives_multi_stop(void *unused) · 15 unmodified lines | ||
| 178 | 213 | isb(); |
| 179 | 214 | } else { |
| 180 | 215 | BUG_ON(alternatives_applied); |
| 181 | − __apply_alternatives(®ion, true); | |
| 216 | + __apply_alternatives(®ion, false); | |
| 182 | 217 | /* Barriers provided by the cache flushing */ |
| 183 | 218 | WRITE_ONCE(alternatives_applied, 1); |
| 184 | 219 | } |
| @@ -192,12 +227,14 @@ void __init apply_alternatives_all(void) · 7 unmodified lines | ||
| 192 | 227 | stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask); |
| 193 | 228 | } |
| 194 | 229 | |
| 195 | −void apply_alternatives(void *start, size_t length) | |
| 230 | +#ifdef CONFIG_MODULES | |
| 231 | +void apply_alternatives_module(void *start, size_t length) | |
| 196 | 232 | { |
| 197 | 233 | struct alt_region region = { |
| 198 | 234 | .begin = start, |
| 199 | 235 | .end = start + length, |
| 200 | 236 | }; |
| 201 | 237 | |
| 202 | − __apply_alternatives(®ion, false); | |
| 238 | + __apply_alternatives(®ion, true); | |
| 203 | 239 | } |
| 240 | +#endif | |
| @@ -448,9 +448,8 @@ int module_finalize(const Elf_Ehdr *hdr, · 447 unmodified lines | ||
| 448 | 448 | const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; |
| 449 | 449 | |
| 450 | 450 | for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { |
| 451 | − if (strcmp(".altinstructions", secstrs + s->sh_name) == 0) { | |
| 452 | − apply_alternatives((void *)s->sh_addr, s->sh_size); | |
| 453 | − } | |
| 451 | + if (strcmp(".altinstructions", secstrs + s->sh_name) == 0) | |
| 452 | + apply_alternatives_module((void *)s->sh_addr, s->sh_size); | |
| 454 | 453 | #ifdef CONFIG_ARM64_MODULE_PLTS |
| 455 | 454 | if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) && |
| 456 | 455 | !strcmp(".text.ftrace_trampoline", secstrs + s->sh_name)) |
| @@ -1463,7 +1463,7 @@ static char *xgene_pmu_dev_name(struct device *dev, u32 type, int id) · 1462 unmodified lines | ||
| 1463 | 1463 | case PMU_TYPE_IOB: |
| 1464 | 1464 | return devm_kasprintf(dev, GFP_KERNEL, "iob%d", id); |
| 1465 | 1465 | case PMU_TYPE_IOB_SLOW: |
| 1466 | − return devm_kasprintf(dev, GFP_KERNEL, "iob-slow%d", id); | |
| 1466 | + return devm_kasprintf(dev, GFP_KERNEL, "iob_slow%d", id); | |
| 1467 | 1467 | case PMU_TYPE_MCB: |
| 1468 | 1468 | return devm_kasprintf(dev, GFP_KERNEL, "mcb%d", id); |
| 1469 | 1469 | case PMU_TYPE_MC: |