History

arm64: Move some head.text functions to executable section

The head.text section is intended to be run at early bootup before any of the regular kernel mappings have been setup. Parts of head.text may be freed back into the buddy allocator due to TEXT_OFFSET so for security requirements this memory must not be executable. The suspend/resume/hotplug code path requires some of these head.S functions to run however which means they need to be executable. Support these conflicting requirements by moving the few head.text functions that need to be executable to the text section which has the appropriate page table permissions.

Laura Abbott committed 11 years ago

Commit 034edab1 parent 6ddae411 file · +209 −198Browse files

Unified · Split

arch/arm64/kernel/head.S+209−198
@@ -248,7 +248,13 @@ ENTRY(stext) · 247 unmodified lines
248248 mov x0, x22
249249 bl lookup_processor_type
250250 mov x23, x0 // x23=current cpu_table
251− cbz x23, __error_p // invalid processor (x23=0)?
251+ /*
252+ * __error_p may end up out of range for cbz if text areas are
253+ * aligned up to section sizes.
254+ */
255+ cbnz x23, 1f // invalid processor (x23=0)?
256+ b __error_p
257+1:
252258 bl __vet_fdt
253259 bl __create_page_tables // x25=TTBR0, x26=TTBR1
254260 /*
@@ -260,12 +266,213 @@ ENTRY(stext) · 5 unmodified lines
260266 */
261267 ldr x27, __switch_data // address to jump to after
262268 // MMU has been enabled
263− adr lr, __enable_mmu // return (PIC) address
269+ adrp lr, __enable_mmu // return (PIC) address
270+ add lr, lr, #:lo12:__enable_mmu
264271 ldr x12, [x23, #CPU_INFO_SETUP]
265272 add x12, x12, x28 // __virt_to_phys
266273 br x12 // initialise processor
267274 ENDPROC(stext)
268275
276+/*
277+ * Determine validity of the x21 FDT pointer.
278+ * The dtb must be 8-byte aligned and live in the first 512M of memory.
279+ */
280+__vet_fdt:
281+ tst x21, #0x7
282+ b.ne 1f
283+ cmp x21, x24
284+ b.lt 1f
285+ mov x0, #(1 << 29)
286+ add x0, x0, x24
287+ cmp x21, x0
288+ b.ge 1f
289+ ret
290+1:
291+ mov x21, #0
292+ ret
293+ENDPROC(__vet_fdt)
294+/*
295+ * Macro to create a table entry to the next page.
296+ *
297+ * tbl: page table address
298+ * virt: virtual address
299+ * shift: #imm page table shift
300+ * ptrs: #imm pointers per table page
301+ *
302+ * Preserves: virt
303+ * Corrupts: tmp1, tmp2
304+ * Returns: tbl -> next level table page address
305+ */
306+ .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2
307+ lsr \tmp1, \virt, #\shift
308+ and \tmp1, \tmp1, #\ptrs - 1 // table index
309+ add \tmp2, \tbl, #PAGE_SIZE
310+ orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type
311+ str \tmp2, [\tbl, \tmp1, lsl #3]
312+ add \tbl, \tbl, #PAGE_SIZE // next level table page
313+ .endm
314+
315+/*
316+ * Macro to populate the PGD (and possibily PUD) for the corresponding
317+ * block entry in the next level (tbl) for the given virtual address.
318+ *
319+ * Preserves: tbl, next, virt
320+ * Corrupts: tmp1, tmp2
321+ */
322+ .macro create_pgd_entry, tbl, virt, tmp1, tmp2
323+ create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2
324+#if SWAPPER_PGTABLE_LEVELS == 3
325+ create_table_entry \tbl, \virt, TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2
326+#endif
327+ .endm
328+
329+/*
330+ * Macro to populate block entries in the page table for the start..end
331+ * virtual range (inclusive).
332+ *
333+ * Preserves: tbl, flags
334+ * Corrupts: phys, start, end, pstate
335+ */
336+ .macro create_block_map, tbl, flags, phys, start, end
337+ lsr \phys, \phys, #BLOCK_SHIFT
338+ lsr \start, \start, #BLOCK_SHIFT
339+ and \start, \start, #PTRS_PER_PTE - 1 // table index
340+ orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry
341+ lsr \end, \end, #BLOCK_SHIFT
342+ and \end, \end, #PTRS_PER_PTE - 1 // table end index
343+9999: str \phys, [\tbl, \start, lsl #3] // store the entry
344+ add \start, \start, #1 // next entry
345+ add \phys, \phys, #BLOCK_SIZE // next block
346+ cmp \start, \end
347+ b.ls 9999b
348+ .endm
349+
350+/*
351+ * Setup the initial page tables. We only setup the barest amount which is
352+ * required to get the kernel running. The following sections are required:
353+ * - identity mapping to enable the MMU (low address, TTBR0)
354+ * - first few MB of the kernel linear mapping to jump to once the MMU has
355+ * been enabled, including the FDT blob (TTBR1)
356+ * - pgd entry for fixed mappings (TTBR1)
357+ */
358+__create_page_tables:
359+ pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses
360+ mov x27, lr
361+
362+ /*
363+ * Invalidate the idmap and swapper page tables to avoid potential
364+ * dirty cache lines being evicted.
365+ */
366+ mov x0, x25
367+ add x1, x26, #SWAPPER_DIR_SIZE
368+ bl __inval_cache_range
369+
370+ /*
371+ * Clear the idmap and swapper page tables.
372+ */
373+ mov x0, x25
374+ add x6, x26, #SWAPPER_DIR_SIZE
375+1: stp xzr, xzr, [x0], #16
376+ stp xzr, xzr, [x0], #16
377+ stp xzr, xzr, [x0], #16
378+ stp xzr, xzr, [x0], #16
379+ cmp x0, x6
380+ b.lo 1b
381+
382+ ldr x7, =MM_MMUFLAGS
383+
384+ /*
385+ * Create the identity mapping.
386+ */
387+ mov x0, x25 // idmap_pg_dir
388+ ldr x3, =KERNEL_START
389+ add x3, x3, x28 // __pa(KERNEL_START)
390+ create_pgd_entry x0, x3, x5, x6
391+ ldr x6, =KERNEL_END
392+ mov x5, x3 // __pa(KERNEL_START)
393+ add x6, x6, x28 // __pa(KERNEL_END)
394+ create_block_map x0, x7, x3, x5, x6
395+
396+ /*
397+ * Map the kernel image (starting with PHYS_OFFSET).
398+ */
399+ mov x0, x26 // swapper_pg_dir
400+ mov x5, #PAGE_OFFSET
401+ create_pgd_entry x0, x5, x3, x6
402+ ldr x6, =KERNEL_END
403+ mov x3, x24 // phys offset
404+ create_block_map x0, x7, x3, x5, x6
405+
406+ /*
407+ * Map the FDT blob (maximum 2MB; must be within 512MB of
408+ * PHYS_OFFSET).
409+ */
410+ mov x3, x21 // FDT phys address
411+ and x3, x3, #~((1 << 21) - 1) // 2MB aligned
412+ mov x6, #PAGE_OFFSET
413+ sub x5, x3, x24 // subtract PHYS_OFFSET
414+ tst x5, #~((1 << 29) - 1) // within 512MB?
415+ csel x21, xzr, x21, ne // zero the FDT pointer
416+ b.ne 1f
417+ add x5, x5, x6 // __va(FDT blob)
418+ add x6, x5, #1 << 21 // 2MB for the FDT blob
419+ sub x6, x6, #1 // inclusive range
420+ create_block_map x0, x7, x3, x5, x6
421+1:
422+ /*
423+ * Since the page tables have been populated with non-cacheable
424+ * accesses (MMU disabled), invalidate the idmap and swapper page
425+ * tables again to remove any speculatively loaded cache lines.
426+ */
427+ mov x0, x25
428+ add x1, x26, #SWAPPER_DIR_SIZE
429+ bl __inval_cache_range
430+
431+ mov lr, x27
432+ ret
433+ENDPROC(__create_page_tables)
434+ .ltorg
435+
436+ .align 3
437+ .type __switch_data, %object
438+__switch_data:
439+ .quad __mmap_switched
440+ .quad __bss_start // x6
441+ .quad __bss_stop // x7
442+ .quad processor_id // x4
443+ .quad __fdt_pointer // x5
444+ .quad memstart_addr // x6
445+ .quad init_thread_union + THREAD_START_SP // sp
446+
447+/*
448+ * The following fragment of code is executed with the MMU on in MMU mode, and
449+ * uses absolute addresses; this is not position independent.
450+ */
451+__mmap_switched:
452+ adr x3, __switch_data + 8
453+
454+ ldp x6, x7, [x3], #16
455+1: cmp x6, x7
456+ b.hs 2f
457+ str xzr, [x6], #8 // Clear BSS
458+ b 1b
459+2:
460+ ldp x4, x5, [x3], #16
461+ ldr x6, [x3], #8
462+ ldr x16, [x3]
463+ mov sp, x16
464+ str x22, [x4] // Save processor ID
465+ str x21, [x5] // Save FDT pointer
466+ str x24, [x6] // Save PHYS_OFFSET
467+ mov x29, #0
468+ b start_kernel
469+ENDPROC(__mmap_switched)
470+
471+/*
472+ * end early head section, begin head code that is also used for
473+ * hotplug and needs to have the same protections as the text region
474+ */
475+ .section ".text","ax"
269476 /*
270477 * If we're fortunate enough to boot at EL2, ensure that the world is
271478 * sane before dropping to EL1.
@@ -502,183 +709,6 @@ ENDPROC(__calc_phys_offset) · 230 unmodified lines
502709 1: .quad .
503710 .quad PAGE_OFFSET
504711
505−/*
506− * Macro to create a table entry to the next page.
507− *
508− * tbl: page table address
509− * virt: virtual address
510− * shift: #imm page table shift
511− * ptrs: #imm pointers per table page
512− *
513− * Preserves: virt
514− * Corrupts: tmp1, tmp2
515− * Returns: tbl -> next level table page address
516− */
517− .macro create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2
518− lsr \tmp1, \virt, #\shift
519− and \tmp1, \tmp1, #\ptrs - 1 // table index
520− add \tmp2, \tbl, #PAGE_SIZE
521− orr \tmp2, \tmp2, #PMD_TYPE_TABLE // address of next table and entry type
522− str \tmp2, [\tbl, \tmp1, lsl #3]
523− add \tbl, \tbl, #PAGE_SIZE // next level table page
524− .endm
525−
526−/*
527− * Macro to populate the PGD (and possibily PUD) for the corresponding
528− * block entry in the next level (tbl) for the given virtual address.
529− *
530− * Preserves: tbl, next, virt
531− * Corrupts: tmp1, tmp2
532− */
533− .macro create_pgd_entry, tbl, virt, tmp1, tmp2
534− create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2
535−#if SWAPPER_PGTABLE_LEVELS == 3
536− create_table_entry \tbl, \virt, TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2
537−#endif
538− .endm
539−
540−/*
541− * Macro to populate block entries in the page table for the start..end
542− * virtual range (inclusive).
543− *
544− * Preserves: tbl, flags
545− * Corrupts: phys, start, end, pstate
546− */
547− .macro create_block_map, tbl, flags, phys, start, end
548− lsr \phys, \phys, #BLOCK_SHIFT
549− lsr \start, \start, #BLOCK_SHIFT
550− and \start, \start, #PTRS_PER_PTE - 1 // table index
551− orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry
552− lsr \end, \end, #BLOCK_SHIFT
553− and \end, \end, #PTRS_PER_PTE - 1 // table end index
554−9999: str \phys, [\tbl, \start, lsl #3] // store the entry
555− add \start, \start, #1 // next entry
556− add \phys, \phys, #BLOCK_SIZE // next block
557− cmp \start, \end
558− b.ls 9999b
559− .endm
560−
561−/*
562− * Setup the initial page tables. We only setup the barest amount which is
563− * required to get the kernel running. The following sections are required:
564− * - identity mapping to enable the MMU (low address, TTBR0)
565− * - first few MB of the kernel linear mapping to jump to once the MMU has
566− * been enabled, including the FDT blob (TTBR1)
567− * - pgd entry for fixed mappings (TTBR1)
568− */
569−__create_page_tables:
570− pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses
571− mov x27, lr
572−
573− /*
574− * Invalidate the idmap and swapper page tables to avoid potential
575− * dirty cache lines being evicted.
576− */
577− mov x0, x25
578− add x1, x26, #SWAPPER_DIR_SIZE
579− bl __inval_cache_range
580−
581− /*
582− * Clear the idmap and swapper page tables.
583− */
584− mov x0, x25
585− add x6, x26, #SWAPPER_DIR_SIZE
586−1: stp xzr, xzr, [x0], #16
587− stp xzr, xzr, [x0], #16
588− stp xzr, xzr, [x0], #16
589− stp xzr, xzr, [x0], #16
590− cmp x0, x6
591− b.lo 1b
592−
593− ldr x7, =MM_MMUFLAGS
594−
595− /*
596− * Create the identity mapping.
597− */
598− mov x0, x25 // idmap_pg_dir
599− ldr x3, =KERNEL_START
600− add x3, x3, x28 // __pa(KERNEL_START)
601− create_pgd_entry x0, x3, x5, x6
602− ldr x6, =KERNEL_END
603− mov x5, x3 // __pa(KERNEL_START)
604− add x6, x6, x28 // __pa(KERNEL_END)
605− create_block_map x0, x7, x3, x5, x6
606−
607− /*
608− * Map the kernel image (starting with PHYS_OFFSET).
609− */
610− mov x0, x26 // swapper_pg_dir
611− mov x5, #PAGE_OFFSET
612− create_pgd_entry x0, x5, x3, x6
613− ldr x6, =KERNEL_END
614− mov x3, x24 // phys offset
615− create_block_map x0, x7, x3, x5, x6
616−
617− /*
618− * Map the FDT blob (maximum 2MB; must be within 512MB of
619− * PHYS_OFFSET).
620− */
621− mov x3, x21 // FDT phys address
622− and x3, x3, #~((1 << 21) - 1) // 2MB aligned
623− mov x6, #PAGE_OFFSET
624− sub x5, x3, x24 // subtract PHYS_OFFSET
625− tst x5, #~((1 << 29) - 1) // within 512MB?
626− csel x21, xzr, x21, ne // zero the FDT pointer
627− b.ne 1f
628− add x5, x5, x6 // __va(FDT blob)
629− add x6, x5, #1 << 21 // 2MB for the FDT blob
630− sub x6, x6, #1 // inclusive range
631− create_block_map x0, x7, x3, x5, x6
632−1:
633− /*
634− * Since the page tables have been populated with non-cacheable
635− * accesses (MMU disabled), invalidate the idmap and swapper page
636− * tables again to remove any speculatively loaded cache lines.
637− */
638− mov x0, x25
639− add x1, x26, #SWAPPER_DIR_SIZE
640− bl __inval_cache_range
641−
642− mov lr, x27
643− ret
644−ENDPROC(__create_page_tables)
645− .ltorg
646−
647− .align 3
648− .type __switch_data, %object
649−__switch_data:
650− .quad __mmap_switched
651− .quad __bss_start // x6
652− .quad __bss_stop // x7
653− .quad processor_id // x4
654− .quad __fdt_pointer // x5
655− .quad memstart_addr // x6
656− .quad init_thread_union + THREAD_START_SP // sp
657−
658−/*
659− * The following fragment of code is executed with the MMU on in MMU mode, and
660− * uses absolute addresses; this is not position independent.
661− */
662−__mmap_switched:
663− adr x3, __switch_data + 8
664−
665− ldp x6, x7, [x3], #16
666−1: cmp x6, x7
667− b.hs 2f
668− str xzr, [x6], #8 // Clear BSS
669− b 1b
670−2:
671− ldp x4, x5, [x3], #16
672− ldr x6, [x3], #8
673− ldr x16, [x3]
674− mov sp, x16
675− str x22, [x4] // Save processor ID
676− str x21, [x5] // Save FDT pointer
677− str x24, [x6] // Save PHYS_OFFSET
678− mov x29, #0
679− b start_kernel
680−ENDPROC(__mmap_switched)
681−
682712 /*
683713 * Exception handling. Something went wrong and we can't proceed. We ought to
684714 * tell the user, but since we don't have any guarantee that we're even
@@ -726,22 +756,3 @@ __lookup_processor_type_data: · 41 unmodified lines
726756 .quad .
727757 .quad cpu_table
728758 .size __lookup_processor_type_data, . - __lookup_processor_type_data
729−
730−/*
731− * Determine validity of the x21 FDT pointer.
732− * The dtb must be 8-byte aligned and live in the first 512M of memory.
733− */
734−__vet_fdt:
735− tst x21, #0x7
736− b.ne 1f
737− cmp x21, x24
738− b.lt 1f
739− mov x0, #(1 << 29)
740− add x0, x0, x24
741− cmp x21, x0
742− b.ge 1f
743− ret
744−1:
745− mov x21, #0
746− ret
747−ENDPROC(__vet_fdt)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t