refactor: 📦 Improve boot code structure

Rename sections to not conflict during link.
Update linker script docs to align on PAGE_SIZE.

bin/chainboot/src/boot.rs

@@ -1,7 +1,7 @@
 // Make first function small enough so that compiler doesn't try
 // to crate a huge stack frame before we have a chance to set SP.
 #[no_mangle]
-#[link_section = ".text._start"]
+#[link_section = ".text.chainboot.entry"]
 pub unsafe extern "C" fn _start() -> ! {
     use {
         core::cell::UnsafeCell,
@@ -29,7 +29,7 @@ pub unsafe extern "C" fn _start() -> ! {
 }
 
 #[no_mangle]
-#[link_section = ".text._start"]
+#[link_section = ".text.chainboot"]
 pub unsafe extern "C" fn reset() -> ! {
     use core::{
         cell::UnsafeCell,
@@ -40,8 +40,8 @@ pub unsafe extern "C" fn reset() -> ! {
     // Subsequently, this code tries to read values from not-yet-existing data locations.
     extern "Rust" {
         // Boundaries of the .bss section, provided by the linker script
-        static __bss_start: UnsafeCell<()>;
+        static __BSS_START: UnsafeCell<()>;
-        static __bss_size: UnsafeCell<()>;
+        static __BSS_SIZE_U64S: UnsafeCell<()>;
         // Load address of the kernel binary
         static __binary_nonzero_lma: UnsafeCell<()>;
         // Address to relocate to and image size
@@ -73,8 +73,10 @@ pub unsafe extern "C" fn reset() -> ! {
     // Zeroes the .bss section
     // Emulate
     // crate::stdmem::local_memset(__bss_start.get() as *mut u8, 0u8, __bss_size.get() as usize);
-    let bss =
+    let bss = core::slice::from_raw_parts_mut(
-        core::slice::from_raw_parts_mut(__bss_start.get() as *mut u8, __bss_size.get() as usize);
+        __BSS_START.get() as *mut u64,
+        __BSS_SIZE_U64S.get() as usize,
+    );
     for i in bss {
         *i = 0;
     }
@@ -91,7 +93,7 @@ pub unsafe extern "C" fn reset() -> ! {
 }
 
 #[inline(always)]
-#[link_section = ".text.boot"]
+#[link_section = ".text.chainboot"]
 unsafe fn local_memcpy(mut dest: *mut u8, mut src: *const u8, n: usize) {
     let dest_end = dest.add(n);
     while dest < dest_end {

bin/chainboot/src/link.ld

@@ -51,8 +51,8 @@ SECTIONS
 
     .text :
     {
-        KEEP(*(.text._start))
+        KEEP(*(.text.chainboot.entry))
-        *(.text.stdmem)
+        *(.text.chainboot)
     } :segment_start_code
 
     /* Align to 8 bytes, b/c relocating the binary is done in u64 chunks */
@@ -70,9 +70,7 @@ SECTIONS
     __binary_nonzero_vma = .;
     .text : AT (ADDR(.text) + SIZEOF(.text))
     {
-        *(.text._start_rust)      /* The Rust entry point */
+        *(.text*)                 /* The Rust entry point and everything else */
-        /* *(text.memcpy) -- only relevant for Rust relocator impl which is currently impossible */
-        *(.text*)                 /* Everything else */
     } :segment_code
 
     .rodata : ALIGN(8) { *(.rodata*) } :segment_code
@@ -87,12 +85,15 @@ SECTIONS
     . = ALIGN(8);
     __binary_nonzero_vma_end_exclusive = .;
 
-    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes at least */
     .bss (NOLOAD) : ALIGN(16)
     {
-        __bss_start = .;
+        __BSS_START = .;
-        *(.bss*);
+        *(.bss .bss.*)
-        . = ALIGN(16);
+        *(COMMON)
-        __bss_size = . - __bss_start;
+        . = ALIGN(PAGE_SIZE); /* Align up to page size */
+        __BSS_SIZE_U64S = (. - __BSS_START) / 8;
     } :segment_data
+
+    /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) *(.text.boot*)}
 }

bin/chainboot/src/main.rs

@@ -5,6 +5,7 @@
 #![reexport_test_harness_main = "test_main"]
 #![no_main]
 #![no_std]
+#![no_builtins]
 
 use {
     core::{hash::Hasher, panic::PanicInfo},

linker/aarch64.ld

@@ -11,7 +11,7 @@ PAGE_SIZE = 65536;
 
 /* Symbols between __BOOT_START and __BOOT_END should be dropped after init is complete.
    Symbols between __RO_START and __RO_END are the kernel code.
-   Symbols between __BSS_START and __BSS_END must be initialized to zero by r0 code in kernel.
+   Symbols between __BSS_START and __BSS_END must be initialized to zero by startup code in the kernel.
 */
 SECTIONS
 {
@@ -20,12 +20,12 @@ SECTIONS
     __BOOT_START = .;
     .text :
     {
-        KEEP(*(.text.boot.entry)) // Entry point must go first
+        KEEP(*(.text.main.entry)) // Entry point must go first
         *(.text.boot)
         //. = ALIGN(PAGE_SIZE);
         *(.data.boot)
-        . = ALIGN(PAGE_SIZE); /* Here boot code ends */
+        . = ALIGN(PAGE_SIZE); /* Here the boot code ends */
-        __BOOT_END = .; // __BOOT_END must be 4KiB aligned
+        __BOOT_END = .; // __BOOT_END must be PAGE_SIZE aligned
         __RO_START = .;
         *(.text .text.*)
     }
@@ -41,8 +41,8 @@ SECTIONS
         FILL(0x00)
     }
     . = ALIGN(PAGE_SIZE); /* Fill up to page size */
-    __RO_END = .; /* __RO_END must be 4KiB aligned */
+    __RO_END = .; /* __RO_END must be PAGE_SIZE aligned */
-    __DATA_START = .; /* __DATA_START must be 4KiB aligned */
+    __DATA_START = .; /* __DATA_START must be PAGE_SIZE aligned */
 
     .data : /* @todo align data to 4K -- it's already aligned up to __RO_END marker now */
     {
@@ -58,11 +58,11 @@ SECTIONS
         *(.bss .bss.*)
         *(COMMON)
         . = ALIGN(PAGE_SIZE); /* Align up to page size */
-        __BSS_SIZE = . - __BSS_START;
+        __BSS_END = .;
-        /* __BSS_END = .; unused */
+        __BSS_SIZE_U64S = (__BSS_END - __BSS_START) / 8;
     }
 
-    /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) }
+    /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) *(.text.chainboot*) }
 }
 
 PROVIDE(current_el0_synchronous   = default_exception_handler);

machine/src/arch/aarch64/boot.rs

@@ -19,9 +19,6 @@ use {
     tock_registers::interfaces::{Readable, Writeable},
 };
 
-// Stack placed before first executable instruction
-const STACK_START: u64 = 0x0008_0000; // Keep in sync with linker script
-
 /// Type check the user-supplied entry function.
 #[macro_export]
 macro_rules! entry {
@@ -39,71 +36,58 @@ macro_rules! entry {
     };
 }
 
-/// Reset function.
+/// Entrypoint of the processor.
 ///
-/// Initializes the bss section before calling into the user's `main()`.
+/// Parks all cores except core0 and checks if we started in EL2/EL3. If
+/// so, proceeds with setting up EL1.
+///
+/// This is invoked from the linker script, does arch-specific init
+/// and passes control to the kernel boot function reset().
+///
+/// Dissection of various RPi core boot stubs is available
+/// [here](https://leiradel.github.io/2019/01/20/Raspberry-Pi-Stubs.html).
 ///
 /// # Safety
 ///
 /// Totally unsafe! We're in the hardware land.
-/// We assume that no statics are accessed before transition to main from this function.
+/// We assume that no statics are accessed before transition to main from reset() function.
-#[link_section = ".text.boot"]
+#[no_mangle]
-unsafe fn reset() -> ! {
+#[link_section = ".text.main.entry"]
-    extern "Rust" {
+pub unsafe extern "C" fn _boot_cores() -> ! {
-        // Boundaries of the .bss section, provided by the linker script.
+    const CORE_0: u64 = 0;
-        static __BSS_START: UnsafeCell<()>;
+    const CORE_MASK: u64 = 0x3;
-        static __BSS_SIZE: UnsafeCell<()>;
+    // Can't match values with dots in match, so use intermediate consts.
-    }
+    #[cfg(qemu)]
-
+    const EL3: u64 = CurrentEL::EL::EL3.value;
-    // Zeroes the .bss section
+    const EL2: u64 = CurrentEL::EL::EL2.value;
-    // Based on https://gist.github.com/skoe/dbd3add2fc3baa600e9ebc995ddf0302 and discussions
+    const EL1: u64 = CurrentEL::EL::EL1.value;
-    // on pointer provenance in closing r0 issues (https://github.com/rust-embedded/cortex-m-rt/issues/300)
-
-    // NB: https://doc.rust-lang.org/nightly/core/ptr/index.html#provenance
-    // Importing pointers like `__BSS_START` and `__BSS_END` and performing pointer
-    // arithmetic on them directly may lead to Undefined Behavior, because the
-    // compiler may assume they come from different allocations and thus performing
-    // undesirable optimizations on them.
-    // So we use a painter-and-a-size as described in provenance section.
-
-    let bss = slice::from_raw_parts_mut(__BSS_START.get() as *mut u8, __BSS_SIZE.get() as usize);
-    for i in bss {
-        *i = 0;
-    }
-
-    // Don't cross this line with loads and stores. The initializations
-    // done above could be "invisible" to the compiler, because we write to the
-    // same memory location that is used by statics after this point.
-    // Additionally, we assume that no statics are accessed before this point.
-    atomic::compiler_fence(Ordering::SeqCst);
 
     extern "Rust" {
-        fn main() -> !;
+        // Stack top
+        // Stack placed before first executable instruction
+        static __STACK_START: UnsafeCell<()>;
+    }
+    // Set stack pointer. Used in case we started in EL1.
+    SP.set(__STACK_START.get() as u64);
+
+    shared_setup_and_enter_pre();
+
+    if CORE_0 == MPIDR_EL1.get() & CORE_MASK {
+        match CurrentEL.get() {
+            #[cfg(qemu)]
+            EL3 => setup_and_enter_el1_from_el3(),
+            EL2 => setup_and_enter_el1_from_el2(),
+            EL1 => reset(),
+            _ => endless_sleep(),
+        }
     }
 
-    main()
+    // if not core0 or not EL3/EL2/EL1, infinitely wait for events
+    endless_sleep()
 }
 
-// [ARMv6 unaligned data access restrictions](https://developer.arm.com/documentation/ddi0333/h/unaligned-and-mixed-endian-data-access-support/unaligned-access-support/armv6-unaligned-data-access-restrictions?lang=en)
-// dictates that compatibility bit U in CP15 must be set to 1 to allow Unaligned accesses while MMU is off.
-// (In addition to SCTLR_EL1.A being 0)
-// See also [CP15 C1 docs](https://developer.arm.com/documentation/ddi0290/g/system-control-coprocessor/system-control-processor-registers/c1--control-register).
-// #[link_section = ".text.boot"]
-// #[inline]
-// fn enable_armv6_unaligned_access() {
-//     unsafe {
-//         core::arch::asm!(
-//             "mrc p15, 0, {u}, c1, c0, 0",
-//             "or {u}, {u}, {CR_U}",
-//             "mcr p15, 0, {u}, c1, c0, 0",
-//             u = out(reg) _,
-//             CR_U = const 1 << 22
-//         );
-//     }
-// }
-
 #[link_section = ".text.boot"]
-#[inline]
+#[inline(always)]
 fn shared_setup_and_enter_pre() {
     // Enable timer counter registers for EL1
     CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);
@@ -136,9 +120,15 @@ fn shared_setup_and_enter_pre() {
 #[link_section = ".text.boot"]
 #[inline]
 fn shared_setup_and_enter_post() -> ! {
+    extern "Rust" {
+        // Stack top
+        static __STACK_START: UnsafeCell<()>;
+    }
     // Set up SP_EL1 (stack pointer), which will be used by EL1 once
     // we "return" to it.
-    SP_EL1.set(STACK_START);
+    unsafe {
+        SP_EL1.set(__STACK_START.get() as u64);
+    }
 
     // Use `eret` to "return" to EL1. This will result in execution of
     // `reset()` in EL1.
@@ -206,47 +196,52 @@ fn setup_and_enter_el1_from_el3() -> ! {
     shared_setup_and_enter_post()
 }
 
-/// Entrypoint of the processor.
+/// Reset function.
 ///
-/// Parks all cores except core0 and checks if we started in EL2/EL3. If
+/// Initializes the bss section before calling into the user's `main()`.
-/// so, proceeds with setting up EL1.
-///
-/// This is invoked from the linker script, does arch-specific init
-/// and passes control to the kernel boot function reset().
-///
-/// Dissection of various RPi core boot stubs is available
-/// [here](https://leiradel.github.io/2019/01/20/Raspberry-Pi-Stubs.html).
 ///
 /// # Safety
 ///
 /// Totally unsafe! We're in the hardware land.
-/// We assume that no statics are accessed before transition to main from reset() function.
+/// We assume that no statics are accessed before transition to main from this function.
-#[no_mangle]
+///
-#[link_section = ".text.boot.entry"]
+/// We are guaranteed to be in EL1 non-secure mode here.
-pub unsafe extern "C" fn _boot_cores() -> ! {
+#[link_section = ".text.boot"]
-    const CORE_0: u64 = 0;
+unsafe fn reset() -> ! {
-    const CORE_MASK: u64 = 0x3;
+    extern "Rust" {
-    // Can't match values with dots in match, so use intermediate consts.
+        // Boundaries of the .bss section, provided by the linker script.
-    #[cfg(qemu)]
+        static __BSS_START: UnsafeCell<()>;
-    const EL3: u64 = CurrentEL::EL::EL3.value;
+        static __BSS_SIZE_U64S: UnsafeCell<()>;
-    const EL2: u64 = CurrentEL::EL::EL2.value;
-    const EL1: u64 = CurrentEL::EL::EL1.value;
-
-    // Set stack pointer. Used in case we started in EL1.
-    SP.set(STACK_START);
-
-    shared_setup_and_enter_pre();
-
-    if CORE_0 == MPIDR_EL1.get() & CORE_MASK {
-        match CurrentEL.get() {
-            #[cfg(qemu)]
-            EL3 => setup_and_enter_el1_from_el3(),
-            EL2 => setup_and_enter_el1_from_el2(),
-            EL1 => reset(),
-            _ => endless_sleep(),
-        }
     }
 
-    // if not core0 or not EL3/EL2/EL1, infinitely wait for events
+    // Zeroes the .bss section
-    endless_sleep()
+    // Based on https://gist.github.com/skoe/dbd3add2fc3baa600e9ebc995ddf0302 and discussions
+    // on pointer provenance in closing r0 issues (https://github.com/rust-embedded/cortex-m-rt/issues/300)
+
+    // NB: https://doc.rust-lang.org/nightly/core/ptr/index.html#provenance
+    // Importing pointers like `__BSS_START` and `__BSS_END` and performing pointer
+    // arithmetic on them directly may lead to Undefined Behavior, because the
+    // compiler may assume they come from different allocations and thus performing
+    // undesirable optimizations on them.
+    // So we use a painter-and-a-size as described in provenance section.
+
+    let bss = slice::from_raw_parts_mut(
+        __BSS_START.get() as *mut u64,
+        __BSS_SIZE_U64S.get() as usize,
+    );
+    for i in bss {
+        *i = 0;
+    }
+
+    // Don't cross this line with loads and stores. The initializations
+    // done above could be "invisible" to the compiler, because we write to the
+    // same memory location that is used by statics after this point.
+    // Additionally, we assume that no statics are accessed before this point.
+    atomic::compiler_fence(Ordering::SeqCst);
+
+    extern "Rust" {
+        fn main() -> !;
+    }
+
+    main()
 }

nucleus/src/main.rs

@@ -124,10 +124,11 @@ pub fn kmain() -> ! {
     #[cfg(feature = "jtag")]
     machine::arch::jtag::wait_debugger();
 
+    init_exception_traps();
+
     #[cfg(not(feature = "noserial"))]
     init_uart_serial();
 
-    init_exception_traps();
     init_mmu();
 
     #[cfg(test)]