diff --git a/bin/chainboot/src/boot.rs b/bin/chainboot/src/boot.rs index 5576eaa..106660a 100644 --- a/bin/chainboot/src/boot.rs +++ b/bin/chainboot/src/boot.rs @@ -1,7 +1,7 @@ // Make first function small enough so that compiler doesn't try // to crate a huge stack frame before we have a chance to set SP. #[no_mangle] -#[link_section = ".text._start"] +#[link_section = ".text.chainboot.entry"] pub unsafe extern "C" fn _start() -> ! { use { core::cell::UnsafeCell, @@ -29,7 +29,7 @@ pub unsafe extern "C" fn _start() -> ! { } #[no_mangle] -#[link_section = ".text._start"] +#[link_section = ".text.chainboot"] pub unsafe extern "C" fn reset() -> ! { use core::{ cell::UnsafeCell, @@ -40,8 +40,8 @@ pub unsafe extern "C" fn reset() -> ! { // Subsequently, this code tries to read values from not-yet-existing data locations. extern "Rust" { // Boundaries of the .bss section, provided by the linker script - static __bss_start: UnsafeCell<()>; - static __bss_size: UnsafeCell<()>; + static __BSS_START: UnsafeCell<()>; + static __BSS_SIZE_U64S: UnsafeCell<()>; // Load address of the kernel binary static __binary_nonzero_lma: UnsafeCell<()>; // Address to relocate to and image size @@ -73,8 +73,10 @@ pub unsafe extern "C" fn reset() -> ! { // Zeroes the .bss section // Emulate // crate::stdmem::local_memset(__bss_start.get() as *mut u8, 0u8, __bss_size.get() as usize); - let bss = - core::slice::from_raw_parts_mut(__bss_start.get() as *mut u8, __bss_size.get() as usize); + let bss = core::slice::from_raw_parts_mut( + __BSS_START.get() as *mut u64, + __BSS_SIZE_U64S.get() as usize, + ); for i in bss { *i = 0; } @@ -91,7 +93,7 @@ pub unsafe extern "C" fn reset() -> ! { } #[inline(always)] -#[link_section = ".text.boot"] +#[link_section = ".text.chainboot"] unsafe fn local_memcpy(mut dest: *mut u8, mut src: *const u8, n: usize) { let dest_end = dest.add(n); while dest < dest_end { diff --git a/bin/chainboot/src/link.ld b/bin/chainboot/src/link.ld index eeac8d1..d940a7c 100644 --- a/bin/chainboot/src/link.ld +++ b/bin/chainboot/src/link.ld @@ -51,8 +51,8 @@ SECTIONS .text : { - KEEP(*(.text._start)) - *(.text.stdmem) + KEEP(*(.text.chainboot.entry)) + *(.text.chainboot) } :segment_start_code /* Align to 8 bytes, b/c relocating the binary is done in u64 chunks */ @@ -70,9 +70,7 @@ SECTIONS __binary_nonzero_vma = .; .text : AT (ADDR(.text) + SIZEOF(.text)) { - *(.text._start_rust) /* The Rust entry point */ - /* *(text.memcpy) -- only relevant for Rust relocator impl which is currently impossible */ - *(.text*) /* Everything else */ + *(.text*) /* The Rust entry point and everything else */ } :segment_code .rodata : ALIGN(8) { *(.rodata*) } :segment_code @@ -87,12 +85,15 @@ SECTIONS . = ALIGN(8); __binary_nonzero_vma_end_exclusive = .; - /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes at least */ .bss (NOLOAD) : ALIGN(16) { - __bss_start = .; - *(.bss*); - . = ALIGN(16); - __bss_size = . - __bss_start; + __BSS_START = .; + *(.bss .bss.*) + *(COMMON) + . = ALIGN(PAGE_SIZE); /* Align up to page size */ + __BSS_SIZE_U64S = (. - __BSS_START) / 8; } :segment_data + + /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) *(.text.boot*)} } diff --git a/bin/chainboot/src/main.rs b/bin/chainboot/src/main.rs index 64cb362..e794654 100644 --- a/bin/chainboot/src/main.rs +++ b/bin/chainboot/src/main.rs @@ -5,6 +5,7 @@ #![reexport_test_harness_main = "test_main"] #![no_main] #![no_std] +#![no_builtins] use { core::{hash::Hasher, panic::PanicInfo}, diff --git a/linker/aarch64.ld b/linker/aarch64.ld index 802b0d3..5d3e083 100644 --- a/linker/aarch64.ld +++ b/linker/aarch64.ld @@ -11,7 +11,7 @@ PAGE_SIZE = 65536; /* Symbols between __BOOT_START and __BOOT_END should be dropped after init is complete. Symbols between __RO_START and __RO_END are the kernel code. - Symbols between __BSS_START and __BSS_END must be initialized to zero by r0 code in kernel. + Symbols between __BSS_START and __BSS_END must be initialized to zero by startup code in the kernel. */ SECTIONS { @@ -20,12 +20,12 @@ SECTIONS __BOOT_START = .; .text : { - KEEP(*(.text.boot.entry)) // Entry point must go first + KEEP(*(.text.main.entry)) // Entry point must go first *(.text.boot) //. = ALIGN(PAGE_SIZE); *(.data.boot) - . = ALIGN(PAGE_SIZE); /* Here boot code ends */ - __BOOT_END = .; // __BOOT_END must be 4KiB aligned + . = ALIGN(PAGE_SIZE); /* Here the boot code ends */ + __BOOT_END = .; // __BOOT_END must be PAGE_SIZE aligned __RO_START = .; *(.text .text.*) } @@ -41,8 +41,8 @@ SECTIONS FILL(0x00) } . = ALIGN(PAGE_SIZE); /* Fill up to page size */ - __RO_END = .; /* __RO_END must be 4KiB aligned */ - __DATA_START = .; /* __DATA_START must be 4KiB aligned */ + __RO_END = .; /* __RO_END must be PAGE_SIZE aligned */ + __DATA_START = .; /* __DATA_START must be PAGE_SIZE aligned */ .data : /* @todo align data to 4K -- it's already aligned up to __RO_END marker now */ { @@ -58,11 +58,11 @@ SECTIONS *(.bss .bss.*) *(COMMON) . = ALIGN(PAGE_SIZE); /* Align up to page size */ - __BSS_SIZE = . - __BSS_START; - /* __BSS_END = .; unused */ + __BSS_END = .; + __BSS_SIZE_U64S = (__BSS_END - __BSS_START) / 8; } - /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) } + /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) *(.text.chainboot*) } } PROVIDE(current_el0_synchronous = default_exception_handler); diff --git a/machine/src/arch/aarch64/boot.rs b/machine/src/arch/aarch64/boot.rs index db7fc2f..ddaf295 100644 --- a/machine/src/arch/aarch64/boot.rs +++ b/machine/src/arch/aarch64/boot.rs @@ -19,9 +19,6 @@ use { tock_registers::interfaces::{Readable, Writeable}, }; -// Stack placed before first executable instruction -const STACK_START: u64 = 0x0008_0000; // Keep in sync with linker script - /// Type check the user-supplied entry function. #[macro_export] macro_rules! entry { @@ -39,71 +36,58 @@ macro_rules! entry { }; } -/// Reset function. +/// Entrypoint of the processor. /// -/// Initializes the bss section before calling into the user's `main()`. +/// Parks all cores except core0 and checks if we started in EL2/EL3. If +/// so, proceeds with setting up EL1. +/// +/// This is invoked from the linker script, does arch-specific init +/// and passes control to the kernel boot function reset(). +/// +/// Dissection of various RPi core boot stubs is available +/// [here](https://leiradel.github.io/2019/01/20/Raspberry-Pi-Stubs.html). /// /// # Safety /// /// Totally unsafe! We're in the hardware land. -/// We assume that no statics are accessed before transition to main from this function. -#[link_section = ".text.boot"] -unsafe fn reset() -> ! { - extern "Rust" { - // Boundaries of the .bss section, provided by the linker script. - static __BSS_START: UnsafeCell<()>; - static __BSS_SIZE: UnsafeCell<()>; - } - - // Zeroes the .bss section - // Based on https://gist.github.com/skoe/dbd3add2fc3baa600e9ebc995ddf0302 and discussions - // on pointer provenance in closing r0 issues (https://github.com/rust-embedded/cortex-m-rt/issues/300) - - // NB: https://doc.rust-lang.org/nightly/core/ptr/index.html#provenance - // Importing pointers like `__BSS_START` and `__BSS_END` and performing pointer - // arithmetic on them directly may lead to Undefined Behavior, because the - // compiler may assume they come from different allocations and thus performing - // undesirable optimizations on them. - // So we use a painter-and-a-size as described in provenance section. - - let bss = slice::from_raw_parts_mut(__BSS_START.get() as *mut u8, __BSS_SIZE.get() as usize); - for i in bss { - *i = 0; - } - - // Don't cross this line with loads and stores. The initializations - // done above could be "invisible" to the compiler, because we write to the - // same memory location that is used by statics after this point. - // Additionally, we assume that no statics are accessed before this point. - atomic::compiler_fence(Ordering::SeqCst); +/// We assume that no statics are accessed before transition to main from reset() function. +#[no_mangle] +#[link_section = ".text.main.entry"] +pub unsafe extern "C" fn _boot_cores() -> ! { + const CORE_0: u64 = 0; + const CORE_MASK: u64 = 0x3; + // Can't match values with dots in match, so use intermediate consts. + #[cfg(qemu)] + const EL3: u64 = CurrentEL::EL::EL3.value; + const EL2: u64 = CurrentEL::EL::EL2.value; + const EL1: u64 = CurrentEL::EL::EL1.value; extern "Rust" { - fn main() -> !; + // Stack top + // Stack placed before first executable instruction + static __STACK_START: UnsafeCell<()>; + } + // Set stack pointer. Used in case we started in EL1. + SP.set(__STACK_START.get() as u64); + + shared_setup_and_enter_pre(); + + if CORE_0 == MPIDR_EL1.get() & CORE_MASK { + match CurrentEL.get() { + #[cfg(qemu)] + EL3 => setup_and_enter_el1_from_el3(), + EL2 => setup_and_enter_el1_from_el2(), + EL1 => reset(), + _ => endless_sleep(), + } } - main() + // if not core0 or not EL3/EL2/EL1, infinitely wait for events + endless_sleep() } -// [ARMv6 unaligned data access restrictions](https://developer.arm.com/documentation/ddi0333/h/unaligned-and-mixed-endian-data-access-support/unaligned-access-support/armv6-unaligned-data-access-restrictions?lang=en) -// dictates that compatibility bit U in CP15 must be set to 1 to allow Unaligned accesses while MMU is off. -// (In addition to SCTLR_EL1.A being 0) -// See also [CP15 C1 docs](https://developer.arm.com/documentation/ddi0290/g/system-control-coprocessor/system-control-processor-registers/c1--control-register). -// #[link_section = ".text.boot"] -// #[inline] -// fn enable_armv6_unaligned_access() { -// unsafe { -// core::arch::asm!( -// "mrc p15, 0, {u}, c1, c0, 0", -// "or {u}, {u}, {CR_U}", -// "mcr p15, 0, {u}, c1, c0, 0", -// u = out(reg) _, -// CR_U = const 1 << 22 -// ); -// } -// } - #[link_section = ".text.boot"] -#[inline] +#[inline(always)] fn shared_setup_and_enter_pre() { // Enable timer counter registers for EL1 CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -136,9 +120,15 @@ fn shared_setup_and_enter_pre() { #[link_section = ".text.boot"] #[inline] fn shared_setup_and_enter_post() -> ! { + extern "Rust" { + // Stack top + static __STACK_START: UnsafeCell<()>; + } // Set up SP_EL1 (stack pointer), which will be used by EL1 once // we "return" to it. - SP_EL1.set(STACK_START); + unsafe { + SP_EL1.set(__STACK_START.get() as u64); + } // Use `eret` to "return" to EL1. This will result in execution of // `reset()` in EL1. @@ -206,47 +196,52 @@ fn setup_and_enter_el1_from_el3() -> ! { shared_setup_and_enter_post() } -/// Entrypoint of the processor. +/// Reset function. /// -/// Parks all cores except core0 and checks if we started in EL2/EL3. If -/// so, proceeds with setting up EL1. -/// -/// This is invoked from the linker script, does arch-specific init -/// and passes control to the kernel boot function reset(). -/// -/// Dissection of various RPi core boot stubs is available -/// [here](https://leiradel.github.io/2019/01/20/Raspberry-Pi-Stubs.html). +/// Initializes the bss section before calling into the user's `main()`. /// /// # Safety /// /// Totally unsafe! We're in the hardware land. -/// We assume that no statics are accessed before transition to main from reset() function. -#[no_mangle] -#[link_section = ".text.boot.entry"] -pub unsafe extern "C" fn _boot_cores() -> ! { - const CORE_0: u64 = 0; - const CORE_MASK: u64 = 0x3; - // Can't match values with dots in match, so use intermediate consts. - #[cfg(qemu)] - const EL3: u64 = CurrentEL::EL::EL3.value; - const EL2: u64 = CurrentEL::EL::EL2.value; - const EL1: u64 = CurrentEL::EL::EL1.value; - - // Set stack pointer. Used in case we started in EL1. - SP.set(STACK_START); - - shared_setup_and_enter_pre(); - - if CORE_0 == MPIDR_EL1.get() & CORE_MASK { - match CurrentEL.get() { - #[cfg(qemu)] - EL3 => setup_and_enter_el1_from_el3(), - EL2 => setup_and_enter_el1_from_el2(), - EL1 => reset(), - _ => endless_sleep(), - } +/// We assume that no statics are accessed before transition to main from this function. +/// +/// We are guaranteed to be in EL1 non-secure mode here. +#[link_section = ".text.boot"] +unsafe fn reset() -> ! { + extern "Rust" { + // Boundaries of the .bss section, provided by the linker script. + static __BSS_START: UnsafeCell<()>; + static __BSS_SIZE_U64S: UnsafeCell<()>; } - // if not core0 or not EL3/EL2/EL1, infinitely wait for events - endless_sleep() + // Zeroes the .bss section + // Based on https://gist.github.com/skoe/dbd3add2fc3baa600e9ebc995ddf0302 and discussions + // on pointer provenance in closing r0 issues (https://github.com/rust-embedded/cortex-m-rt/issues/300) + + // NB: https://doc.rust-lang.org/nightly/core/ptr/index.html#provenance + // Importing pointers like `__BSS_START` and `__BSS_END` and performing pointer + // arithmetic on them directly may lead to Undefined Behavior, because the + // compiler may assume they come from different allocations and thus performing + // undesirable optimizations on them. + // So we use a painter-and-a-size as described in provenance section. + + let bss = slice::from_raw_parts_mut( + __BSS_START.get() as *mut u64, + __BSS_SIZE_U64S.get() as usize, + ); + for i in bss { + *i = 0; + } + + // Don't cross this line with loads and stores. The initializations + // done above could be "invisible" to the compiler, because we write to the + // same memory location that is used by statics after this point. + // Additionally, we assume that no statics are accessed before this point. + atomic::compiler_fence(Ordering::SeqCst); + + extern "Rust" { + fn main() -> !; + } + + main() } diff --git a/nucleus/src/main.rs b/nucleus/src/main.rs index c9fedef..a1f9c04 100644 --- a/nucleus/src/main.rs +++ b/nucleus/src/main.rs @@ -124,10 +124,11 @@ pub fn kmain() -> ! { #[cfg(feature = "jtag")] machine::arch::jtag::wait_debugger(); + init_exception_traps(); + #[cfg(not(feature = "noserial"))] init_uart_serial(); - init_exception_traps(); init_mmu(); #[cfg(test)]