diff --git a/machine/src/arch/aarch64/memory/addr/mod.rs b/machine/src/arch/aarch64/memory/addr/mod.rs index 665f2ff..ef1ecda 100644 --- a/machine/src/arch/aarch64/memory/addr/mod.rs +++ b/machine/src/arch/aarch64/memory/addr/mod.rs @@ -4,7 +4,5 @@ */ mod asid; -mod phys_addr; -mod virt_addr; -pub use {asid::*, phys_addr::*, virt_addr::*}; +pub use asid::*; diff --git a/machine/src/arch/aarch64/memory/mmu/mod.rs b/machine/src/arch/aarch64/memory/mmu/mod.rs index c57e45d..b796bc3 100644 --- a/machine/src/arch/aarch64/memory/mmu/mod.rs +++ b/machine/src/arch/aarch64/memory/mmu/mod.rs @@ -13,9 +13,9 @@ use { crate::{ - memory::mmu::{ - interface, interface::MMU, translation_table::KernelTranslationTable, AddressSpace, - MMUEnableError, TranslationGranule, + memory::{ + mmu::{interface, interface::MMU, AddressSpace, MMUEnableError, TranslationGranule}, + Address, Physical, }, platform, println, }, @@ -58,13 +58,6 @@ pub mod mair { // Global instances //-------------------------------------------------------------------------------------------------- -/// The kernel translation tables. -/// -/// # Safety -/// -/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". -static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new(); - static MMU: MemoryManagementUnit = MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- @@ -75,7 +68,7 @@ impl AddressSpace { /// Checks for architectural restrictions. pub const fn arch_address_space_size_sanity_checks() { // Size must be at least one full 512 MiB table. - assert!((AS_SIZE % Granule512MiB::SIZE) == 0); + assert!((AS_SIZE % Granule512MiB::SIZE) == 0); // assert!() is const-friendly // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8 // version. @@ -102,7 +95,7 @@ impl MemoryManagementUnit { /// Configure various settings of stage 1 of the EL1 translation regime. fn configure_translation_control(&self) { - let t0sz = (64 - platform::memory::mmu::KernelAddrSpace::SIZE_SHIFT) as u64; + let t0sz = (64 - platform::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64; TCR_EL1.write( TCR_EL1::TBI0::Used @@ -124,7 +117,7 @@ impl MemoryManagementUnit { //-------------------------------------------------------------------------------------------------- /// Return a reference to the MMU instance. -pub fn mmu() -> &'static impl MMU { +pub fn mmu() -> &'static impl interface::MMU { &MMU } @@ -133,7 +126,10 @@ pub fn mmu() -> &'static impl MMU { //------------------------------------------------------------------------------ impl interface::MMU for MemoryManagementUnit { - unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError> { + unsafe fn enable_mmu_and_caching( + &self, + phys_tables_base_addr: Address, + ) -> Result<(), MMUEnableError> { if unlikely(self.is_enabled()) { return Err(MMUEnableError::AlreadyEnabled); } @@ -141,20 +137,20 @@ impl interface::MMU for MemoryManagementUnit { // Fail early if translation granule is not supported. if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) { return Err(MMUEnableError::Other { - err: "Translation granule not supported in HW", + err: "Translation granule not supported by hardware", }); } // Prepare the memory attribute indirection register. self.set_up_mair(); - // Populate translation tables. - KERNEL_TABLES - .populate_translation_table_entries() - .map_err(|err| MMUEnableError::Other { err })?; + // // Populate translation tables. + // KERNEL_TABLES + // .populate_translation_table_entries() + // .map_err(|err| MMUEnableError::Other { err })?; // Set the "Translation Table Base Register". - TTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address()); + TTBR0_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64); self.configure_translation_control(); diff --git a/machine/src/arch/aarch64/memory/mmu/translation_table.rs b/machine/src/arch/aarch64/memory/mmu/translation_table.rs index 60d27ce..ace68db 100644 --- a/machine/src/arch/aarch64/memory/mmu/translation_table.rs +++ b/machine/src/arch/aarch64/memory/mmu/translation_table.rs @@ -1,7 +1,11 @@ use { super::{mair, Granule512MiB, Granule64KiB}, crate::{ - memory::mmu::{AccessPermissions, AttributeFields, MemAttributes}, + memory::{ + self, + mmu::{AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress}, + Address, Physical, Virtual, + }, platform, }, core::convert, @@ -18,7 +22,9 @@ use { register_bitfields! { u64, - // AArch64 Reference Manual page 2150, D5-2445 + + /// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. + /// AArch64 Reference Manual page 2150, D5-2445 STAGE1_TABLE_DESCRIPTOR [ /// Physical address of the next descriptor. NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] @@ -38,9 +44,11 @@ register_bitfields! { register_bitfields! { u64, - // AArch64 Reference Manual page 2150, D5-2445 + + /// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. + /// AArch64 Reference Manual page 2150, D5-2445 STAGE1_PAGE_DESCRIPTOR [ - // User execute-never + /// Unprivileged execute-never. UXN OFFSET(54) NUMBITS(1) [ Execute = 0, NeverExecute = 1 @@ -53,8 +61,8 @@ register_bitfields! { ], /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). - LVL2_OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - LVL2_OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + OUTPUT_ADDR_4KiB OFFSET(21) NUMBITS(27) [], // [47:21] /// Access flag AF OFFSET(10) NUMBITS(1) [ @@ -110,11 +118,12 @@ struct PageDescriptor { } trait BaseAddr { + fn phys_start_addr(&self) -> Address; fn base_addr_u64(&self) -> u64; fn base_addr_usize(&self) -> usize; } -const NUM_LVL2_TABLES: usize = platform::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT; +// const NUM_LVL2_TABLES: usize = platform::memory::mmu::KernelAddrSpace::SIZE >> Granule512MiB::SHIFT; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -130,17 +139,24 @@ pub struct FixedSizeTranslationTable { /// Table descriptors, covering 512 MiB windows. lvl2: [TableDescriptor; NUM_TABLES], + + /// Have the tables been initialized? + initialized: bool, } -/// A translation table type for the kernel space. -pub type KernelTranslationTable = FixedSizeTranslationTable; +// /// A translation table type for the kernel space. +// pub type KernelTranslationTable = FixedSizeTranslationTable; //-------------------------------------------------------------------------------------------------- // Private Implementations //-------------------------------------------------------------------------------------------------- -// The binary is still identity mapped, so we don't need to convert here. impl BaseAddr for [T; N] { + // The binary is still identity mapped, so we don't need to convert here. + fn phys_start_addr(&self) -> Address { + Address::new(self as *const _ as usize) + } + fn base_addr_u64(&self) -> u64 { self as *const T as u64 } @@ -159,10 +175,10 @@ impl TableDescriptor { } /// Create an instance pointing to the supplied address. - pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: usize) -> Self { + pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address) -> Self { let val = InMemoryRegister::::new(0); - let shifted = phys_next_lvl_table_addr >> Granule64KiB::SHIFT; + let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT; val.write( STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64) + STAGE1_TABLE_DESCRIPTOR::TYPE::Table @@ -182,12 +198,15 @@ impl PageDescriptor { } /// Create an instance. - pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self { + pub fn from_output_page_addr( + phys_output_page_addr: PageAddress, + attribute_fields: &AttributeFields, + ) -> Self { let val = InMemoryRegister::::new(0); - let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; + let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT; val.write( - STAGE1_PAGE_DESCRIPTOR::LVL2_OUTPUT_ADDR_64KiB.val(shifted) + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64) + STAGE1_PAGE_DESCRIPTOR::AF::Accessed + STAGE1_PAGE_DESCRIPTOR::TYPE::Page + STAGE1_PAGE_DESCRIPTOR::VALID::True @@ -196,6 +215,12 @@ impl PageDescriptor { Self { value: val.get() } } + + /// Returns the valid bit. + fn is_valid(&self) -> bool { + InMemoryRegister::::new(self.value) + .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) + } } /// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. @@ -243,43 +268,174 @@ impl convert::From // Public Code //-------------------------------------------------------------------------------------------------- +impl memory::mmu::AssociatedTranslationTable + for memory::mmu::AddressSpace +where + [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized, +{ + type TableStartFromBottom = FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }>; +} + impl FixedSizeTranslationTable { /// Create an instance. + #[allow(clippy::assertions_on_constants)] pub const fn new() -> Self { + assert!(platform::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE); // assert! is const-fn-friendly + // Can't have a zero-sized address space. assert!(NUM_TABLES > 0); Self { lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + initialized: false, } } + /// Helper to calculate the lvl2 and lvl3 indices from an address. + #[inline(always)] + fn lvl2_lvl3_index_from_page_addr( + &self, + virt_page_addr: PageAddress, + ) -> Result<(usize, usize), &'static str> { + let addr = virt_page_addr.into_inner().as_usize(); + let lvl2_index = addr >> Granule512MiB::SHIFT; + let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; + + if lvl2_index > (NUM_TABLES - 1) { + return Err("Virtual page is out of bounds of translation table"); + } + + Ok((lvl2_index, lvl3_index)) + } + + /// Sets the PageDescriptor corresponding to the supplied page address. + /// + /// Doesn't allow overriding an already valid page. + #[inline(always)] + fn set_page_descriptor_from_page_addr( + &mut self, + virt_page_addr: PageAddress, + new_desc: &PageDescriptor, + ) -> Result<(), &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?; + let desc = &mut self.lvl3[lvl2_index][lvl3_index]; + + if desc.is_valid() { + return Err("Virtual page is already mapped"); + } + + *desc = *new_desc; + Ok(()) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl memory::mmu::translation_table::interface::TranslationTable + for FixedSizeTranslationTable +{ /// Iterates over all static translation table entries and fills them at once. /// /// # Safety /// /// - Modifies a `static mut`. Ensure it only happens from here. - pub unsafe fn populate_translation_table_entries(&mut self) -> Result<(), &'static str> { - for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { - *l2_entry = - TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); + // pub unsafe fn populate_translation_table_entries(&mut self) -> Result<(), &'static str> { + // for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { + // *l2_entry = + // TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); + // + // for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { + // let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); + // + // let (phys_output_addr, attribute_fields) = + // platform::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; + // + // *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields); + // } + // } + // + // Ok(()) + // } + fn init(&mut self) { + if self.initialized { + return; + } - for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { - let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); + // Populate the l2 entries. + for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { + let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr(); - let (phys_output_addr, attribute_fields) = - platform::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; + let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr); + *lvl2_entry = new_desc; + } - *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields); - } + self.initialized = true; + } + + fn phys_base_address(&self) -> Address { + self.lvl2.phys_start_addr() + } + + unsafe fn map_at( + &mut self, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + assert!(self.initialized, "Translation tables not initialized"); + + if virt_region.size() != phys_region.size() { + return Err("Tried to map memory regions with different sizes"); + } + + if phys_region.end_exclusive_page_addr() + > platform::memory::phys_addr_space_end_exclusive_addr() + { + return Err("Tried to map outside of physical address space"); + } + + #[allow(clippy::useless_conversion)] + let iter = phys_region.into_iter().zip(virt_region.into_iter()); + for (phys_page_addr, virt_page_addr) in iter { + let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr); + let virt_page = virt_page_addr; + + self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?; } Ok(()) } +} - /// The translation table's base address to be used for programming the MMU. - pub fn phys_base_address(&self) -> u64 { - self.lvl2.base_addr_u64() +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +pub type MinSizeTranslationTable = FixedSizeTranslationTable<1>; + +#[cfg(test)] +mod tests { + use super::*; + + /// Check if the size of `struct TableDescriptor` is as expected. + #[test_case] + fn size_of_tabledescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } + + /// Check if the size of `struct PageDescriptor` is as expected. + #[test_case] + fn size_of_pagedescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); } } diff --git a/machine/src/arch/aarch64/memory/mod.rs b/machine/src/arch/aarch64/memory/mod.rs index 316048d..8d551f9 100644 --- a/machine/src/arch/aarch64/memory/mod.rs +++ b/machine/src/arch/aarch64/memory/mod.rs @@ -8,10 +8,10 @@ mod addr; pub mod mmu; -pub use addr::{PhysAddr, VirtAddr}; +// pub use addr::{PhysAddr, VirtAddr}; // aarch64 granules and page sizes howto: // https://stackoverflow.com/questions/34269185/simultaneous-existence-of-different-sized-pages-on-aarch64 /// Default page size used by the kernel. -pub const PAGE_SIZE: usize = 4096; +pub const PAGE_SIZE: usize = 65536; diff --git a/machine/src/lib.rs b/machine/src/lib.rs index 4d7afe1..c4b99ff 100644 --- a/machine/src/lib.rs +++ b/machine/src/lib.rs @@ -1,19 +1,27 @@ #![no_std] #![no_main] #![allow(stable_features)] +#![allow(incomplete_features)] #![feature(asm_const)] +#![feature(const_option)] +#![feature(core_intrinsics)] +#![feature(format_args_nl)] +#![feature(generic_const_exprs)] +#![feature(int_roundings)] +#![feature(is_sorted)] +#![feature(linkage)] +#![feature(nonzero_min_max)] +#![feature(panic_info_message)] +#![feature(step_trait)] +#![feature(trait_alias)] +#![feature(unchecked_math)] #![feature(decl_macro)] #![feature(ptr_internals)] #![feature(allocator_api)] -#![feature(format_args_nl)] -#![feature(core_intrinsics)] -#![feature(const_option)] #![feature(strict_provenance)] #![feature(stmt_expr_attributes)] #![feature(slice_ptr_get)] -#![feature(panic_info_message)] #![feature(nonnull_slice_from_raw_parts)] // stabilised in 1.71 nightly -#![feature(unchecked_math)] #![feature(custom_test_frameworks)] #![test_runner(crate::tests::test_runner)] #![reexport_test_harness_main = "test_main"] @@ -91,8 +99,21 @@ mod lib_tests { #[no_mangle] pub unsafe fn main() -> ! { exception::handling_init(); + + let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() { + Err(string) => panic!("Error mapping kernel binary: {}", string), + Ok(addr) => addr, + }; + + if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) { + panic!("Enabling MMU failed: {}", e); + } + + memory::mmu::post_enable_init(); platform::drivers::qemu_bring_up_console(); + test_main(); + qemu::semihosting::exit_success() } } diff --git a/machine/src/memory/mmu/mapping_record.rs b/machine/src/memory/mmu/mapping_record.rs new file mode 100644 index 0000000..48a0021 --- /dev/null +++ b/machine/src/memory/mmu/mapping_record.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! A record of mapped pages. + +use { + super::{ + types::{AccessPermissions, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion}, + Address, Physical, Virtual, + }, + crate::{ + info, mm, platform, + synchronization::{self, InitStateLock}, + warn, + }, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Type describing a virtual memory mapping. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +struct MappingRecordEntry { + pub users: [Option<&'static str>; 5], + pub phys_start_addr: Address, + pub virt_start_addr: Address, + pub num_pages: usize, + pub attribute_fields: AttributeFields, +} + +struct MappingRecord { + inner: [Option; 12], +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static KERNEL_MAPPING_RECORD: InitStateLock = + InitStateLock::new(MappingRecord::new()); + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl MappingRecordEntry { + pub fn new( + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, + ) -> Self { + Self { + users: [Some(name), None, None, None, None], + phys_start_addr: phys_region.start_addr(), + virt_start_addr: virt_region.start_addr(), + num_pages: phys_region.num_pages(), + attribute_fields: *attr, + } + } + + fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> { + if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) { + return Ok(x); + }; + + Err("Storage for user info exhausted") + } + + pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> { + let x = self.find_next_free_user()?; + *x = Some(user); + Ok(()) + } +} + +impl MappingRecord { + pub const fn new() -> Self { + Self { inner: [None; 12] } + } + + fn size(&self) -> usize { + self.inner.iter().filter(|x| x.is_some()).count() + } + + fn sort(&mut self) { + let upper_bound_exclusive = self.size(); + let entries = &mut self.inner[0..upper_bound_exclusive]; + + if !entries.is_sorted_by_key(|item| item.unwrap().virt_start_addr) { + entries.sort_unstable_by_key(|item| item.unwrap().virt_start_addr) + } + } + + fn find_next_free(&mut self) -> Result<&mut Option, &'static str> { + if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) { + return Ok(x); + } + + Err("Storage for mapping info exhausted") + } + + fn find_duplicate( + &mut self, + phys_region: &MemoryRegion, + ) -> Option<&mut MappingRecordEntry> { + self.inner + .iter_mut() + .filter_map(|x| x.as_mut()) + .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device) + .find(|x| { + if x.phys_start_addr != phys_region.start_addr() { + return false; + } + + if x.num_pages != phys_region.num_pages() { + return false; + } + + true + }) + } + + pub fn add( + &mut self, + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + let x = self.find_next_free()?; + + *x = Some(MappingRecordEntry::new( + name, + virt_region, + phys_region, + attr, + )); + + self.sort(); + + Ok(()) + } + + pub fn print(&self) { + info!(" -------------------------------------------------------------------------------------------------------------------------------------------"); + info!( + " {:^44} {:^30} {:^7} {:^9} {:^35}", + "Virtual", "Physical", "Size", "Attr", "Entity" + ); + info!(" -------------------------------------------------------------------------------------------------------------------------------------------"); + + for i in self.inner.iter().flatten() { + let size = i.num_pages * platform::memory::mmu::KernelGranule::SIZE; + let virt_start = i.virt_start_addr; + let virt_end_inclusive = virt_start + (size - 1); + let phys_start = i.phys_start_addr; + let phys_end_inclusive = phys_start + (size - 1); + + let (size, unit) = mm::size_human_readable_ceil(size); + + let attr = match i.attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::NonCacheableDRAM => "NC", + MemAttributes::Device => "Dev", + }; + + let acc_p = match i.attribute_fields.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; + + let xn = if i.attribute_fields.execute_never { + "XN" + } else { + "X" + }; + + info!( + " {}..{} --> {}..{} | {:>3} {} | {:<3} {} {:<2} | {}", + virt_start, + virt_end_inclusive, + phys_start, + phys_end_inclusive, + size, + unit, + attr, + acc_p, + xn, + i.users[0].unwrap() + ); + + for k in i.users[1..].iter() { + if let Some(additional_user) = *k { + info!( + " | {}", + additional_user + ); + } + } + } + + info!(" -------------------------------------------------------------------------------------------------------------------------------------------"); + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use synchronization::interface::ReadWriteEx; + +/// Add an entry to the mapping info record. +pub fn kernel_add( + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, +) -> Result<(), &'static str> { + KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr)) +} + +pub fn kernel_find_and_insert_mmio_duplicate( + mmio_descriptor: &MMIODescriptor, + new_user: &'static str, +) -> Option> { + let phys_region: MemoryRegion = (*mmio_descriptor).into(); + + KERNEL_MAPPING_RECORD.write(|mr| { + let dup = mr.find_duplicate(&phys_region)?; + + if let Err(x) = dup.add_user(new_user) { + warn!("{}", x); + } + + Some(dup.virt_start_addr) + }) +} + +/// Human-readable print of all recorded kernel mappings. +pub fn kernel_print() { + KERNEL_MAPPING_RECORD.read(|mr| mr.print()); +} diff --git a/machine/src/memory/mmu/mod.rs b/machine/src/memory/mmu/mod.rs index 0de20ba..8fa8c6a 100644 --- a/machine/src/memory/mmu/mod.rs +++ b/machine/src/memory/mmu/mod.rs @@ -1,7 +1,11 @@ use { - crate::println, + crate::{ + memory::{Address, Physical, Virtual}, + platform, println, synchronization, warn, + }, core::{ fmt::{self, Formatter}, + num::NonZeroUsize, ops::RangeInclusive, }, snafu::Snafu, @@ -10,12 +14,17 @@ use { #[cfg(target_arch = "aarch64")] use crate::arch::aarch64::memory::mmu as arch_mmu; -pub mod translation_table; +mod mapping_record; +mod page_alloc; +pub(crate) mod translation_table; +mod types; + +pub use types::*; //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- -pub use arch_mmu::mmu; +// pub use arch_mmu::mmu; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -37,13 +46,15 @@ pub mod interface { /// MMU functions. pub trait MMU { - /// Called by the kernel during early init. Supposed to take the translation tables from the - /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU. + /// Turns on the MMU for the first time and enables data and instruction caching. /// /// # Safety /// - /// - Changes the HW's global state. - unsafe fn enable_mmu_and_caching(&self) -> Result<(), MMUEnableError>; + /// - Changes the hardware's global state. + unsafe fn enable_mmu_and_caching( + &self, + phys_tables_base_addr: Address, + ) -> Result<(), MMUEnableError>; /// Returns true if the MMU is enabled, false otherwise. fn is_enabled(&self) -> bool; @@ -58,80 +69,64 @@ pub struct TranslationGranule; /// Describes properties of an address space. pub struct AddressSpace; -/// Architecture agnostic memory attributes. -#[derive(Copy, Clone)] -pub enum MemAttributes { - /// Regular memory - CacheableDRAM, - /// Memory without caching - NonCacheableDRAM, - /// Device memory - Device, -} - -/// Architecture agnostic memory region access permissions. -#[derive(Copy, Clone)] -pub enum AccessPermissions { - /// Read-only access - ReadOnly, - /// Read-write access - ReadWrite, -} - -// Architecture agnostic memory region translation types. -#[allow(dead_code)] -#[derive(Copy, Clone)] -pub enum Translation { - /// One-to-one address mapping - Identity, - /// Mapping with a specified offset - Offset(usize), -} - -/// Summary structure of memory region properties. -#[derive(Copy, Clone)] -pub struct AttributeFields { - /// Attributes - pub mem_attributes: MemAttributes, - /// Permissions - pub acc_perms: AccessPermissions, - /// Disable executable code in this region - pub execute_never: bool, -} - -/// Types used for compiling the virtual memory layout of the kernel using address ranges. -/// -/// Memory region descriptor. -/// -/// Used to construct iterable kernel memory ranges. -pub struct TranslationDescriptor { - /// Name of the region - pub name: &'static str, - /// Virtual memory range - pub virtual_range: fn() -> RangeInclusive, - /// Mapping translation - pub physical_range_translation: Translation, - /// Attributes - pub attribute_fields: AttributeFields, -} - -/// Type for expressing the kernel's virtual memory layout. -pub struct KernelVirtualLayout { - /// The last (inclusive) address of the address space. - max_virt_addr_inclusive: usize, - - /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions. - inner: [TranslationDescriptor; NUM_SPECIAL_RANGES], +/// Intended to be implemented for [`AddressSpace`]. +pub trait AssociatedTranslationTable { + /// A translation table whose address range is: + /// + /// [AS_SIZE - 1, 0] + type TableStartFromBottom; } //-------------------------------------------------------------------------------------------------- -// Public Implementations +// Private Code +//-------------------------------------------------------------------------------------------------- +use { + interface::MMU, synchronization::interface::*, translation_table::interface::TranslationTable, +}; + +/// Query the platform for the reserved virtual addresses for MMIO remapping +/// and initialize the kernel's MMIO VA allocator with it. +fn kernel_init_mmio_va_allocator() { + let region = platform::memory::mmu::virt_mmio_remap_region(); + + page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.init(region)); +} + +/// Map a region in the kernel's translation tables. +/// +/// No input checks done, input is passed through to the architectural implementation. +/// +/// # Safety +/// +/// - See `map_at()`. +/// - Does not prevent aliasing. +unsafe fn kernel_map_at_unchecked( + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, +) -> Result<(), &'static str> { + platform::memory::mmu::kernel_translation_tables() + .write(|tables| tables.map_at(virt_region, phys_region, attr))?; + + if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) { + warn!("{}", x); + } + + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Public Code //-------------------------------------------------------------------------------------------------- impl TranslationGranule { /// The granule's size. pub const SIZE: usize = Self::size_checked(); + /// The granule's mask. + pub const MASK: usize = Self::SIZE - 1; + /// The granule's shift, aka log2(size). pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; @@ -159,110 +154,158 @@ impl AddressSpace { } } -impl Default for AttributeFields { - fn default() -> AttributeFields { - AttributeFields { +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Raw mapping of a virtual to physical region in the kernel translation tables. +/// +/// Prevents mapping into the MMIO range of the tables. +/// +/// # Safety +/// +/// - See `kernel_map_at_unchecked()`. +/// - Does not prevent aliasing. Currently, the callers must be trusted. +pub unsafe fn kernel_map_at( + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, +) -> Result<(), &'static str> { + if platform::memory::mmu::virt_mmio_remap_region().overlaps(virt_region) { + return Err("Attempt to manually map into MMIO region"); + } + + kernel_map_at_unchecked(name, virt_region, phys_region, attr)?; + + Ok(()) +} + +/// MMIO remapping in the kernel translation tables. +/// +/// Typically used by device drivers. +/// +/// # Safety +/// +/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part. +pub unsafe fn kernel_map_mmio( + name: &'static str, + mmio_descriptor: &MMIODescriptor, +) -> Result, &'static str> { + let phys_region = MemoryRegion::from(*mmio_descriptor); + let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page(); + + // Check if an identical region has been mapped for another driver. If so, reuse it. + let virt_addr = if let Some(addr) = + mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name) + { + addr + // Otherwise, allocate a new region and map it. + } else { + let num_pages = match NonZeroUsize::new(phys_region.num_pages()) { + None => return Err("Requested 0 pages"), + Some(x) => x, + }; + + let virt_region = + page_alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?; + + kernel_map_at_unchecked( + name, + &virt_region, + &phys_region, + &AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; + + virt_region.start_addr() + }; + + Ok(virt_addr + offset_into_start_page) +} + +/// Map the kernel's binary. Returns the translation table's base address. +/// +/// # Safety +/// +/// - See [`bsp::memory::mmu::kernel_map_binary()`]. +pub unsafe fn kernel_map_binary() -> Result, &'static str> { + let phys_kernel_tables_base_addr = + platform::memory::mmu::kernel_translation_tables().write(|tables| { + tables.init(); + tables.phys_base_address() + }); + + platform::memory::mmu::kernel_map_binary()?; + + Ok(phys_kernel_tables_base_addr) +} + +/// Enable the MMU and data + instruction caching. +/// +/// # Safety +/// +/// - Crucial function during kernel init. Changes the the complete memory view of the processor. +#[inline] +pub unsafe fn enable_mmu_and_caching( + phys_tables_base_addr: Address, +) -> Result<(), MMUEnableError> { + arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr) +} + +/// Finish initialization of the MMU subsystem. +#[inline] +pub fn post_enable_init() { + kernel_init_mmio_va_allocator(); +} + +/// Human-readable print of all recorded kernel mappings. +#[inline] +pub fn kernel_print_mappings() { + mapping_record::kernel_print() +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use { + super::*, + crate::memory::mmu::types::{ + AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress, + }, + core::num::NonZeroUsize, + }; + + /// Check that you cannot map into the MMIO VA range from kernel_map_at(). + #[test_case] + fn no_manual_mmio_map() { + let phys_start_page_addr: PageAddress = PageAddress::from(0); + let phys_end_exclusive_page_addr: PageAddress = + phys_start_page_addr.checked_offset(5).unwrap(); + let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr); + + let num_pages = NonZeroUsize::new(phys_region.num_pages()).unwrap(); + let virt_region = page_alloc::kernel_mmio_va_allocator() + .lock(|allocator| allocator.alloc(num_pages)) + .unwrap(); + + let attr = AttributeFields { mem_attributes: MemAttributes::CacheableDRAM, acc_perms: AccessPermissions::ReadWrite, execute_never: true, - } - } -} - -/// Human-readable output of AttributeFields -impl fmt::Display for AttributeFields { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - let attr = match self.mem_attributes { - MemAttributes::CacheableDRAM => "C", - MemAttributes::NonCacheableDRAM => "NC", - MemAttributes::Device => "Dev", }; - let acc_p = match self.acc_perms { - AccessPermissions::ReadOnly => "RO", - AccessPermissions::ReadWrite => "RW", + unsafe { + assert_eq!( + kernel_map_at("test", &virt_region, &phys_region, &attr), + Err("Attempt to manually map into MMIO region") + ) }; - - let xn = if self.execute_never { "PXN" } else { "PX" }; - - write!(f, "{: <3} {} {: <3}", attr, acc_p, xn) - } -} - -/// Human-readable output of a Descriptor. -impl fmt::Display for TranslationDescriptor { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Call the function to which self.range points, and dereference the - // result, which causes Rust to copy the value. - let start = *(self.virtual_range)().start(); - let end = *(self.virtual_range)().end(); - let size = end - start + 1; - - // log2(1024) - const KIB_SHIFT: u32 = 10; - - // log2(1024 * 1024) - const MIB_SHIFT: u32 = 20; - - let (size, unit) = if (size >> MIB_SHIFT) > 0 { - (size >> MIB_SHIFT, "MiB") - } else if (size >> KIB_SHIFT) > 0 { - (size >> KIB_SHIFT, "KiB") - } else { - (size, "Byte") - }; - - write!( - f, - " {:#010x} - {:#010x} | {: >3} {} | {} | {}", - start, end, size, unit, self.attribute_fields, self.name - ) - } -} - -impl KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> { - /// Create a new instance. - pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self { - Self { - max_virt_addr_inclusive: max, - inner: layout, - } - } - - /// For a given virtual address, find and return the output address and - /// corresponding attributes. - /// - /// If the address is not found in `inner`, return an identity mapped default for normal - /// cacheable DRAM. - pub fn virt_addr_properties( - &self, - virt_addr: usize, - ) -> Result<(usize, AttributeFields), &'static str> { - if virt_addr > self.max_virt_addr_inclusive { - return Err("Address out of range"); - } - - for i in self.inner.iter() { - if (i.virtual_range)().contains(&virt_addr) { - let output_addr = match i.physical_range_translation { - Translation::Identity => virt_addr, - Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), - }; - - return Ok((output_addr, i.attribute_fields)); - } - } - - Ok((virt_addr, AttributeFields::default())) - } - - /// Print the kernel memory layout. - pub fn print_layout(&self) { - println!("[i] Kernel memory layout:"); //info! - - for i in self.inner.iter() { - // for i in KERNEL_VIRTUAL_LAYOUT.iter() { - println!("{}", i); //info! - } } } diff --git a/machine/src/memory/mmu/page_alloc.rs b/machine/src/memory/mmu/page_alloc.rs new file mode 100644 index 0000000..b536b4e --- /dev/null +++ b/machine/src/memory/mmu/page_alloc.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021-2022 Andre Richter + +//! Page allocation. + +use { + super::MemoryRegion, + crate::{ + memory::{AddressType, Virtual}, + synchronization::IRQSafeNullLock, + warn, + }, + core::num::NonZeroUsize, +}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// A page allocator that can be lazyily initialized. +pub struct PageAllocator { + pool: Option>, +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock> = + IRQSafeNullLock::new(PageAllocator::new()); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the kernel's MMIO virtual address allocator. +pub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock> { + &KERNEL_MMIO_VA_ALLOCATOR +} + +impl PageAllocator { + /// Create an instance. + pub const fn new() -> Self { + Self { pool: None } + } + + /// Initialize the allocator. + pub fn init(&mut self, pool: MemoryRegion) { + if self.pool.is_some() { + warn!("Already initialized"); + return; + } + + self.pool = Some(pool); + } + + /// Allocate a number of pages. + pub fn alloc( + &mut self, + num_requested_pages: NonZeroUsize, + ) -> Result, &'static str> { + if self.pool.is_none() { + return Err("Allocator not initialized"); + } + + self.pool + .as_mut() + .unwrap() + .take_first_n_pages(num_requested_pages) + } +} diff --git a/machine/src/memory/mmu/translation_table.rs b/machine/src/memory/mmu/translation_table.rs index 1e71714..fc14c59 100644 --- a/machine/src/memory/mmu/translation_table.rs +++ b/machine/src/memory/mmu/translation_table.rs @@ -3,7 +3,94 @@ #[cfg(target_arch = "aarch64")] use crate::arch::aarch64::memory::mmu::translation_table as arch_translation_table; +use { + super::{AttributeFields, MemoryRegion}, + crate::memory::{Address, Physical, Virtual}, +}; + //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- -pub use arch_translation_table::KernelTranslationTable; +#[cfg(target_arch = "aarch64")] +pub use arch_translation_table::FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Translation table interfaces. +pub mod interface { + use super::*; + + /// Translation table operations. + pub trait TranslationTable { + /// Anything that needs to run before any of the other provided functions can be used. + /// + /// # Safety + /// + /// - Implementor must ensure that this function can run only once or is harmless if invoked + /// multiple times. + fn init(&mut self); + + /// The translation table's base address to be used for programming the MMU. + fn phys_base_address(&self) -> Address; + + /// Map the given virtual memory region to the given physical memory region. + /// + /// # Safety + /// + /// - Using wrong attributes can cause multiple issues of different nature in the system. + /// - It is not required that the architectural implementation prevents aliasing. That is, + /// mapping to the same physical memory using multiple virtual addresses, which would + /// break Rust's ownership assumptions. This should be protected against in the kernel's + /// generic MMU code. + unsafe fn map_at( + &mut self, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, + ) -> Result<(), &'static str>; + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use { + super::*, + crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress}, + arch_translation_table::MinSizeTranslationTable, + interface::TranslationTable, + }; + + /// Sanity checks for the TranslationTable implementation. + #[test_case] + fn translation_table_implementation_sanity() { + // This will occupy a lot of space on the stack. + let mut tables = MinSizeTranslationTable::new(); + + tables.init(); + + let virt_start_page_addr: PageAddress = PageAddress::from(0); + let virt_end_exclusive_page_addr: PageAddress = + virt_start_page_addr.checked_offset(5).unwrap(); + + let phys_start_page_addr: PageAddress = PageAddress::from(0); + let phys_end_exclusive_page_addr: PageAddress = + phys_start_page_addr.checked_offset(5).unwrap(); + + let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr); + let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr); + + let attr = AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }; + + unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) }; + } +} diff --git a/machine/src/memory/mmu/types.rs b/machine/src/memory/mmu/types.rs new file mode 100644 index 0000000..6e72afe --- /dev/null +++ b/machine/src/memory/mmu/types.rs @@ -0,0 +1,402 @@ +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +use { + crate::{ + memory::{Address, AddressType, Physical}, + mm, + platform::{self, memory::mmu::KernelGranule}, + }, + core::{ + fmt::{self, Formatter}, + iter::Step, + num::NonZeroUsize, + ops::Range, + }, +}; + +/// A wrapper type around [Address] that ensures page alignment. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub struct PageAddress { + inner: Address, +} + +/// A type that describes a region of memory in quantities of pages. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub struct MemoryRegion { + start: PageAddress, + end_exclusive: PageAddress, +} + +/// Architecture agnostic memory attributes. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub enum MemAttributes { + /// Regular memory + CacheableDRAM, + /// Memory without caching + NonCacheableDRAM, + /// Device memory + Device, +} + +/// Architecture agnostic memory region access permissions. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub enum AccessPermissions { + /// Read-only access + ReadOnly, + /// Read-write access + ReadWrite, +} + +/// Summary structure of memory region properties. +#[derive(Copy, Clone, Debug, Eq, PartialOrd, PartialEq)] +pub struct AttributeFields { + /// Attributes + pub mem_attributes: MemAttributes, + /// Permissions + pub acc_perms: AccessPermissions, + /// Disable executable code in this region + pub execute_never: bool, +} + +/// An MMIO descriptor for use in device drivers. +#[derive(Copy, Clone)] +pub struct MMIODescriptor { + start_addr: Address, + end_addr_exclusive: Address, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +//------------------------------------------------------------------------------ +// PageAddress +//------------------------------------------------------------------------------ +impl PageAddress { + /// Unwraps the value. + pub fn into_inner(self) -> Address { + self.inner + } + + /// Calculates the offset from the page address. + /// + /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 * + /// page_size`. + pub fn checked_offset(self, count: isize) -> Option { + if count == 0 { + return Some(self); + } + + let delta = count.unsigned_abs().checked_mul(KernelGranule::SIZE)?; + let result = if count.is_positive() { + self.inner.as_usize().checked_add(delta)? + } else { + self.inner.as_usize().checked_sub(delta)? + }; + + Some(Self { + inner: Address::new(result), + }) + } +} + +impl From for PageAddress { + fn from(addr: usize) -> Self { + assert!( + mm::is_aligned(addr, KernelGranule::SIZE), + "Input usize not page aligned" + ); + + Self { + inner: Address::new(addr), + } + } +} + +impl From> for PageAddress { + fn from(addr: Address) -> Self { + assert!(addr.is_page_aligned(), "Input Address not page aligned"); + + Self { inner: addr } + } +} + +impl Step for PageAddress { + fn steps_between(start: &Self, end: &Self) -> Option { + if start > end { + return None; + } + + // Since start <= end, do unchecked arithmetic. + Some((end.inner.as_usize() - start.inner.as_usize()) >> KernelGranule::SHIFT) + } + + fn forward_checked(start: Self, count: usize) -> Option { + start.checked_offset(count as isize) + } + + fn backward_checked(start: Self, count: usize) -> Option { + start.checked_offset(-(count as isize)) + } +} + +//------------------------------------------------------------------------------ +// MemoryRegion +//------------------------------------------------------------------------------ +impl MemoryRegion { + /// Create an instance. + pub fn new(start: PageAddress, end_exclusive: PageAddress) -> Self { + assert!(start <= end_exclusive); + + Self { + start, + end_exclusive, + } + } + + fn as_range(&self) -> Range> { + self.into_iter() + } + + /// Returns the start page address. + pub fn start_page_addr(&self) -> PageAddress { + self.start + } + + /// Returns the start address. + pub fn start_addr(&self) -> Address { + self.start.into_inner() + } + + /// Returns the exclusive end page address. + pub fn end_exclusive_page_addr(&self) -> PageAddress { + self.end_exclusive + } + + /// Returns the exclusive end page address. + pub fn end_inclusive_page_addr(&self) -> PageAddress { + self.end_exclusive.checked_offset(-1).unwrap() + } + + /// Checks if self contains an address. + pub fn contains(&self, addr: Address) -> bool { + let page_addr = PageAddress::from(addr.align_down_page()); + self.as_range().contains(&page_addr) + } + + /// Checks if there is an overlap with another memory region. + pub fn overlaps(&self, other_region: &Self) -> bool { + let self_range = self.as_range(); + + self_range.contains(&other_region.start_page_addr()) + || self_range.contains(&other_region.end_inclusive_page_addr()) + } + + /// Returns the number of pages contained in this region. + pub fn num_pages(&self) -> usize { + PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap() + } + + /// Returns the size in bytes of this region. + pub fn size(&self) -> usize { + // Invariant: start <= end_exclusive, so do unchecked arithmetic. + let end_exclusive = self.end_exclusive.into_inner().as_usize(); + let start = self.start.into_inner().as_usize(); + + end_exclusive - start + } + + /// Splits the MemoryRegion like: + /// + /// -------------------------------------------------------------------------------- + /// | | | | | | | | | | | | | | | | | | | + /// -------------------------------------------------------------------------------- + /// ^ ^ ^ + /// | | | + /// left_start left_end_exclusive | + /// | + /// ^ | + /// | | + /// right_start right_end_exclusive + /// + /// Left region is returned to the caller. Right region is the new region for this struct. + pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result { + let count: usize = num_pages.into(); + + let left_end_exclusive = self.start.checked_offset(count as isize); + let left_end_exclusive = match left_end_exclusive { + None => return Err("Overflow while calculating left_end_exclusive"), + Some(x) => x, + }; + + if left_end_exclusive > self.end_exclusive { + return Err("Not enough free pages"); + } + + let allocation = Self { + start: self.start, + end_exclusive: left_end_exclusive, + }; + self.start = left_end_exclusive; + + Ok(allocation) + } +} + +impl IntoIterator for MemoryRegion { + type Item = PageAddress; + type IntoIter = Range; + + fn into_iter(self) -> Self::IntoIter { + Range { + start: self.start, + end: self.end_exclusive, + } + } +} + +impl From for MemoryRegion { + fn from(desc: MMIODescriptor) -> Self { + let start = PageAddress::from(desc.start_addr.align_down_page()); + let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page()); + + Self { + start, + end_exclusive, + } + } +} + +//------------------------------------------------------------------------------ +// MMIODescriptor +//------------------------------------------------------------------------------ + +impl MMIODescriptor { + /// Create an instance. + pub const fn new(start_addr: Address, size: usize) -> Self { + assert!(size > 0); + let end_addr_exclusive = Address::new(start_addr.as_usize() + size); + + Self { + start_addr, + end_addr_exclusive, + } + } + + /// Return the start address. + pub const fn start_addr(&self) -> Address { + self.start_addr + } + + /// Return the exclusive end address. + pub fn end_addr_exclusive(&self) -> Address { + self.end_addr_exclusive + } +} + +//------------------------------------------------------------------------------ +// AttributeFields +//------------------------------------------------------------------------------ + +impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + } + } +} + +/// Human-readable output of AttributeFields +impl fmt::Display for AttributeFields { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let attr = match self.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::NonCacheableDRAM => "NC", + MemAttributes::Device => "Dev", + }; + + let acc_p = match self.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; + + let xn = if self.execute_never { "PXN" } else { "PX" }; + + write!(f, "{: <3} {} {: <3}", attr, acc_p, xn) + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use {super::*, crate::memory::Virtual}; + + /// Sanity of [PageAddress] methods. + #[test_case] + fn pageaddress_type_method_sanity() { + let page_addr: PageAddress = PageAddress::from(KernelGranule::SIZE * 2); + + assert_eq!( + page_addr.checked_offset(-2), + Some(PageAddress::::from(0)) + ); + + assert_eq!( + page_addr.checked_offset(2), + Some(PageAddress::::from(KernelGranule::SIZE * 4)) + ); + + assert_eq!( + PageAddress::::from(0).checked_offset(0), + Some(PageAddress::::from(0)) + ); + assert_eq!(PageAddress::::from(0).checked_offset(-1), None); + + let max_page_addr = Address::::new(usize::MAX).align_down_page(); + assert_eq!( + PageAddress::::from(max_page_addr).checked_offset(1), + None + ); + + let zero = PageAddress::::from(0); + let three = PageAddress::::from(KernelGranule::SIZE * 3); + assert_eq!(PageAddress::steps_between(&zero, &three), Some(3)); + } + + /// Sanity of [MemoryRegion] methods. + #[test_case] + fn memoryregion_type_method_sanity() { + let zero = PageAddress::::from(0); + let zero_region = MemoryRegion::new(zero, zero); + assert_eq!(zero_region.num_pages(), 0); + assert_eq!(zero_region.size(), 0); + + let one = PageAddress::::from(KernelGranule::SIZE); + let one_region = MemoryRegion::new(zero, one); + assert_eq!(one_region.num_pages(), 1); + assert_eq!(one_region.size(), KernelGranule::SIZE); + + let three = PageAddress::::from(KernelGranule::SIZE * 3); + let mut three_region = MemoryRegion::new(zero, three); + assert!(three_region.contains(zero.into_inner())); + assert!(!three_region.contains(three.into_inner())); + assert!(three_region.overlaps(&one_region)); + + let allocation = three_region + .take_first_n_pages(NonZeroUsize::new(2).unwrap()) + .unwrap(); + assert_eq!(allocation.num_pages(), 2); + assert_eq!(three_region.num_pages(), 1); + + for (i, alloc) in allocation.into_iter().enumerate() { + assert_eq!(alloc.into_inner().as_usize(), i * KernelGranule::SIZE); + } + } +} diff --git a/machine/src/memory/mmu/unused.rs b/machine/src/memory/mmu/unused.rs new file mode 100644 index 0000000..897c079 --- /dev/null +++ b/machine/src/memory/mmu/unused.rs @@ -0,0 +1,124 @@ +//-------------------------------------------------------------------------------------------------- +// Laterrrr +//-------------------------------------------------------------------------------------------------- + +/// Architecture agnostic memory region translation types. +#[allow(dead_code)] +#[derive(Copy, Clone)] +pub enum Translation { + /// One-to-one address mapping + Identity, + /// Mapping with a specified offset + Offset(usize), +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Types used for compiling the virtual memory layout of the kernel using address ranges. +/// +/// Memory region descriptor. +/// +/// Used to construct iterable kernel memory ranges. +pub struct TranslationDescriptor { + /// Name of the region + pub name: &'static str, + /// Virtual memory range + pub virtual_range: fn() -> RangeInclusive, + /// Mapping translation + pub physical_range_translation: Translation, + /// Attributes + pub attribute_fields: AttributeFields, +} + +/// Type for expressing the kernel's virtual memory layout. +pub struct KernelVirtualLayout { + /// The last (inclusive) address of the address space. + max_virt_addr_inclusive: usize, + + /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions. + inner: [TranslationDescriptor; NUM_SPECIAL_RANGES], +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Human-readable output of a Descriptor. +impl fmt::Display for TranslationDescriptor { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Call the function to which self.range points, and dereference the + // result, which causes Rust to copy the value. + let start = *(self.virtual_range)().start(); + let end = *(self.virtual_range)().end(); + let size = end - start + 1; + + // log2(1024) + const KIB_SHIFT: u32 = 10; + + // log2(1024 * 1024) + const MIB_SHIFT: u32 = 20; + + let (size, unit) = if (size >> MIB_SHIFT) > 0 { + (size >> MIB_SHIFT, "MiB") + } else if (size >> KIB_SHIFT) > 0 { + (size >> KIB_SHIFT, "KiB") + } else { + (size, "Byte") + }; + + write!( + f, + " {:#010x} - {:#010x} | {: >3} {} | {} | {}", + start, end, size, unit, self.attribute_fields, self.name + ) + } +} + +impl KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> { + /// Create a new instance. + pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self { + Self { + max_virt_addr_inclusive: max, + inner: layout, + } + } + + /// For a given virtual address, find and return the output address and + /// corresponding attributes. + /// + /// If the address is not found in `inner`, return an identity mapped default for normal + /// cacheable DRAM. + pub fn virt_addr_properties( + &self, + virt_addr: usize, + ) -> Result<(usize, AttributeFields), &'static str> { + if virt_addr > self.max_virt_addr_inclusive { + return Err("Address out of range"); + } + + for i in self.inner.iter() { + if (i.virtual_range)().contains(&virt_addr) { + let output_addr = match i.physical_range_translation { + Translation::Identity => virt_addr, + Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), + }; + + return Ok((output_addr, i.attribute_fields)); + } + } + + Ok((virt_addr, AttributeFields::default())) + } + + /// Print the kernel memory layout. + pub fn print_layout(&self) { + println!("[i] Kernel memory layout:"); //info! + + for i in self.inner.iter() { + // for i in KERNEL_VIRTUAL_LAYOUT.iter() { + println!("{}", i); //info! + } + } +} diff --git a/machine/src/memory/mod.rs b/machine/src/memory/mod.rs index b414d49..f19eb6c 100644 --- a/machine/src/memory/mod.rs +++ b/machine/src/memory/mod.rs @@ -1 +1,168 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Memory Management. + +use { + crate::{mm, platform}, + core::{ + fmt, + marker::PhantomData, + ops::{Add, Sub}, + }, +}; + pub mod mmu; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Metadata trait for marking the type of an address. +pub trait AddressType: Copy + Clone + PartialOrd + PartialEq + Ord + Eq {} + +/// Zero-sized type to mark a physical address. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)] +pub enum Physical {} + +/// Zero-sized type to mark a virtual address. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)] +pub enum Virtual {} + +/// Generic address type. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq)] +pub struct Address { + value: usize, + _address_type: PhantomData ATYPE>, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl AddressType for Physical {} +impl AddressType for Virtual {} + +impl Address { + /// Create an instance. + pub const fn new(value: usize) -> Self { + Self { + value, + _address_type: PhantomData, + } + } + + /// Convert to usize. + pub const fn as_usize(self) -> usize { + self.value + } + + /// Align down to page size. + #[must_use] + pub const fn align_down_page(self) -> Self { + let aligned = mm::align_down(self.value, platform::memory::mmu::KernelGranule::SIZE); + + Self::new(aligned) + } + + /// Align up to page size. + #[must_use] + pub const fn align_up_page(self) -> Self { + let aligned = mm::align_up(self.value, platform::memory::mmu::KernelGranule::SIZE); + + Self::new(aligned) + } + + /// Checks if the address is page aligned. + pub const fn is_page_aligned(&self) -> bool { + mm::is_aligned(self.value, platform::memory::mmu::KernelGranule::SIZE) + } + + /// Return the address' offset into the corresponding page. + pub const fn offset_into_page(&self) -> usize { + self.value & platform::memory::mmu::KernelGranule::MASK + } +} + +impl Add for Address { + type Output = Self; + + #[inline(always)] + fn add(self, rhs: usize) -> Self::Output { + match self.value.checked_add(rhs) { + None => panic!("Overflow on Address::add"), + Some(x) => Self::new(x), + } + } +} + +impl Sub> for Address { + type Output = Self; + + #[inline(always)] + fn sub(self, rhs: Address) -> Self::Output { + match self.value.checked_sub(rhs.value) { + None => panic!("Overflow on Address::sub"), + Some(x) => Self::new(x), + } + } +} + +impl fmt::Display for Address { + // Don't expect to see physical addresses greater than 40 bit. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let q3: u8 = ((self.value >> 32) & 0xff) as u8; + let q2: u16 = ((self.value >> 16) & 0xffff) as u16; + let q1: u16 = (self.value & 0xffff) as u16; + + write!(f, "0x")?; + write!(f, "{:02x}_", q3)?; + write!(f, "{:04x}_", q2)?; + write!(f, "{:04x}", q1) + } +} + +impl fmt::Display for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let q4: u16 = ((self.value >> 48) & 0xffff) as u16; + let q3: u16 = ((self.value >> 32) & 0xffff) as u16; + let q2: u16 = ((self.value >> 16) & 0xffff) as u16; + let q1: u16 = (self.value & 0xffff) as u16; + + write!(f, "0x")?; + write!(f, "{:04x}_", q4)?; + write!(f, "{:04x}_", q3)?; + write!(f, "{:04x}_", q2)?; + write!(f, "{:04x}", q1) + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + /// Sanity of [Address] methods. + #[test_case] + fn address_type_method_sanity() { + let addr = Address::::new(platform::memory::mmu::KernelGranule::SIZE + 100); + + assert_eq!( + addr.align_down_page().as_usize(), + platform::memory::mmu::KernelGranule::SIZE + ); + + assert_eq!( + addr.align_up_page().as_usize(), + platform::memory::mmu::KernelGranule::SIZE * 2 + ); + + assert!(!addr.is_page_aligned()); + + assert_eq!(addr.offset_into_page(), 100); + } +} diff --git a/machine/src/arch/aarch64/memory/addr/phys_addr.rs b/machine/src/memory/phys_addr.rs similarity index 100% rename from machine/src/arch/aarch64/memory/addr/phys_addr.rs rename to machine/src/memory/phys_addr.rs diff --git a/machine/src/arch/aarch64/memory/addr/virt_addr.rs b/machine/src/memory/virt_addr.rs similarity index 100% rename from machine/src/arch/aarch64/memory/addr/virt_addr.rs rename to machine/src/memory/virt_addr.rs diff --git a/machine/src/mm/mod.rs b/machine/src/mm/mod.rs index c6899a5..796b1f7 100644 --- a/machine/src/mm/mod.rs +++ b/machine/src/mm/mod.rs @@ -3,29 +3,62 @@ * Copyright (c) Berkus Decker */ -pub mod bump_allocator; +mod bump_allocator; pub use bump_allocator::BumpAllocator; /// Align address downwards. /// /// Returns the greatest x with alignment `align` so that x <= addr. /// The alignment must be a power of 2. -pub fn align_down(addr: u64, align: u64) -> u64 { - assert!(align.is_power_of_two(), "`align` must be a power of two"); - addr & !(align - 1) +#[inline(always)] +pub const fn align_down(addr: usize, alignment: usize) -> usize { + assert!( + alignment.is_power_of_two(), + "`alignment` must be a power of two" + ); + addr & !(alignment - 1) } /// Align address upwards. /// /// Returns the smallest x with alignment `align` so that x >= addr. /// The alignment must be a power of 2. -pub fn align_up(addr: u64, align: u64) -> u64 { - assert!(align.is_power_of_two(), "`align` must be a power of two"); - let align_mask = align - 1; - if addr & align_mask == 0 { - addr // already aligned +#[inline(always)] +pub const fn align_up(value: usize, alignment: usize) -> usize { + assert!( + alignment.is_power_of_two(), + "`alignment` must be a power of two" + ); + + (value + alignment - 1) & !(alignment - 1) +} + +/// Check if a value is aligned to a given alignment. +/// The alignment must be a power of 2. +#[inline(always)] +pub const fn is_aligned(value: usize, alignment: usize) -> bool { + assert!( + alignment.is_power_of_two(), + "`alignment` must be a power of two" + ); + + (value & (alignment - 1)) == 0 +} + +/// Convert a size into human readable format. +pub const fn size_human_readable_ceil(size: usize) -> (usize, &'static str) { + const KIB: usize = 1024; + const MIB: usize = 1024 * 1024; + const GIB: usize = 1024 * 1024 * 1024; + + if (size / GIB) > 0 { + (size.div_ceil(GIB), "GiB") + } else if (size / MIB) > 0 { + (size.div_ceil(MIB), "MiB") + } else if (size / KIB) > 0 { + (size.div_ceil(KIB), "KiB") } else { - (addr | align_mask) + 1 + (size, "Byte") } } diff --git a/machine/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs b/machine/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs index ae77934..b2819d7 100644 --- a/machine/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs +++ b/machine/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/mod.rs @@ -10,6 +10,7 @@ use { crate::{ drivers, exception::{self, asynchronous::IRQHandlerDescriptor}, + memory::{Address, Virtual}, platform::device_driver::common::BoundedUsize, }, core::fmt, @@ -93,7 +94,7 @@ impl InterruptController { /// # Safety /// /// - The user must ensure to provide a correct MMIO start address. - pub const unsafe fn new(periph_mmio_start_addr: usize) -> Self { + pub const unsafe fn new(periph_mmio_start_addr: Address) -> Self { Self { periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr), } diff --git a/machine/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs b/machine/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs index 72b7687..05bf930 100644 --- a/machine/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs +++ b/machine/src/platform/raspberrypi/device_driver/bcm/interrupt_controller/peripheral_ic.rs @@ -81,7 +81,7 @@ impl PeripheralIC { /// # Safety /// /// - The user must ensure to provide a correct MMIO start address. - pub const unsafe fn new(mmio_start_addr: usize) -> Self { + pub const unsafe fn new(mmio_start_addr: Address) -> Self { Self { wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)), ro_registers: ReadOnlyRegisters::new(mmio_start_addr), @@ -101,7 +101,10 @@ impl PeripheralIC { //------------------------------------------------------------------------------ // OS Interface Code //------------------------------------------------------------------------------ -use synchronization::interface::{Mutex, ReadWriteEx}; +use { + crate::memory::{Address, Virtual}, + synchronization::interface::{Mutex, ReadWriteEx}, +}; impl exception::asynchronous::interface::IRQManager for PeripheralIC { type IRQNumberType = PeripheralIRQ; diff --git a/machine/src/platform/raspberrypi/device_driver/bcm/mini_uart.rs b/machine/src/platform/raspberrypi/device_driver/bcm/mini_uart.rs index 7df3228..459d6a4 100644 --- a/machine/src/platform/raspberrypi/device_driver/bcm/mini_uart.rs +++ b/machine/src/platform/raspberrypi/device_driver/bcm/mini_uart.rs @@ -12,6 +12,7 @@ use { console::interface, devices::serial::SerialOps, exception::asynchronous::IRQNumber, + memory::{Address, Virtual}, platform::{ device_driver::{common::MMIODerefWrapper, gpio}, BcmHost, @@ -195,9 +196,9 @@ impl MiniUart { /// # Safety /// /// - The user must ensure to provide a correct MMIO start address. - pub const unsafe fn new(base_addr: usize) -> Self { + pub const unsafe fn new(mmio_base_addr: Address) -> Self { Self { - inner: IRQSafeNullLock::new(MiniUartInner::new(base_addr)), + inner: IRQSafeNullLock::new(MiniUartInner::new(mmio_base_addr)), } } @@ -224,9 +225,9 @@ impl MiniUartInner { /// # Safety /// /// - The user must ensure to provide a correct MMIO start address. - pub const unsafe fn new(base_addr: usize) -> Self { + pub const unsafe fn new(mmio_base_addr: Address) -> Self { Self { - registers: Registers::new(base_addr), + registers: Registers::new(mmio_base_addr), } } diff --git a/machine/src/platform/raspberrypi/device_driver/bcm/pl011_uart.rs b/machine/src/platform/raspberrypi/device_driver/bcm/pl011_uart.rs index 5f97418..43ab3b3 100644 --- a/machine/src/platform/raspberrypi/device_driver/bcm/pl011_uart.rs +++ b/machine/src/platform/raspberrypi/device_driver/bcm/pl011_uart.rs @@ -14,6 +14,7 @@ use { cpu::loop_while, devices::serial::SerialOps, exception, + memory::{Address, Virtual}, platform::{ device_driver::{common::MMIODerefWrapper, gpio, IRQNumber}, mailbox::{self, Mailbox, MailboxOps}, @@ -282,9 +283,6 @@ pub struct RateDivisors { fractional_baud_rate_divisor: u32, } -// [temporary] Used in mmu.rs to set up local paging -pub const UART0_BASE: usize = BcmHost::get_peripheral_address() + 0x20_1000; - //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- @@ -329,9 +327,9 @@ impl PL011Uart { /// # Safety /// /// - The user must ensure to provide a correct MMIO start address. - pub const unsafe fn new(base_addr: usize) -> Self { + pub const unsafe fn new(mmio_base_addr: Address) -> Self { Self { - inner: IRQSafeNullLock::new(PL011UartInner::new(base_addr)), + inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_base_addr)), } } @@ -362,9 +360,9 @@ impl PL011UartInner { /// # Safety /// /// - The user must ensure to provide a correct MMIO start address. - pub const unsafe fn new(base_addr: usize) -> Self { + pub const unsafe fn new(mmio_base_addr: Address) -> Self { Self { - registers: Registers::new(base_addr), + registers: Registers::new(mmio_base_addr), } } diff --git a/machine/src/platform/raspberrypi/device_driver/common.rs b/machine/src/platform/raspberrypi/device_driver/common.rs index 3cbca19..f242a6a 100644 --- a/machine/src/platform/raspberrypi/device_driver/common.rs +++ b/machine/src/platform/raspberrypi/device_driver/common.rs @@ -4,14 +4,17 @@ //! Common device driver code. -use core::{fmt, marker::PhantomData, ops}; +use { + crate::memory::{Address, Virtual}, + core::{fmt, marker::PhantomData, ops}, +}; //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- pub struct MMIODerefWrapper { - pub base_addr: usize, // @todo unmake public, GPIO::Pin uses it + pub base_addr: Address, // @todo unmake public, GPIO::Pin uses it phantom: PhantomData T>, } @@ -25,7 +28,7 @@ pub struct BoundedUsize(usize); impl MMIODerefWrapper { /// Create an instance. - pub const fn new(base_addr: usize) -> Self { + pub const fn new(base_addr: Address) -> Self { Self { base_addr, phantom: PhantomData, @@ -47,7 +50,7 @@ impl ops::Deref for MMIODerefWrapper { type Target = T; fn deref(&self) -> &Self::Target { - unsafe { &*(self.base_addr as *const _) } + unsafe { &*(self.base_addr.as_usize() as *const _) } } } diff --git a/machine/src/platform/raspberrypi/drivers.rs b/machine/src/platform/raspberrypi/drivers.rs index 19b0d08..beed598 100644 --- a/machine/src/platform/raspberrypi/drivers.rs +++ b/machine/src/platform/raspberrypi/drivers.rs @@ -3,9 +3,13 @@ use { crate::{ console, drivers, exception::{self as generic_exception}, + memory::{self, mmu::MMIODescriptor}, platform::{device_driver, memory::map::mmio}, }, - core::sync::atomic::{AtomicBool, Ordering}, + core::{ + mem::MaybeUninit, + sync::atomic::{AtomicBool, Ordering}, + }, }; //-------------------------------------------------------------------------------------------------- @@ -34,9 +38,9 @@ pub unsafe fn init() -> Result<(), &'static str> { return Err("Init already done"); } - driver_gpio()?; #[cfg(not(feature = "noserial"))] driver_uart()?; + driver_gpio()?; driver_interrupt_controller()?; INIT_DONE.store(true, Ordering::Relaxed); @@ -47,66 +51,107 @@ pub unsafe fn init() -> Result<(), &'static str> { /// than on real hardware due to QEMU's abstractions. #[cfg(test)] pub fn qemu_bring_up_console() { - console::register_console(&PL011_UART); + unsafe { + instantiate_uart().unwrap_or_else(|_| crate::qemu::semihosting::exit_failure()); + console::register_console(PL011_UART.assume_init_ref()); + }; } //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- -// static MINI_UART: device_driver::MiniUart = -// unsafe { device_driver::MiniUart::new(device_driver::UART1_BASE) }; -static PL011_UART: device_driver::PL011Uart = - unsafe { device_driver::PL011Uart::new(device_driver::UART0_BASE) }; -static GPIO: device_driver::GPIO = unsafe { device_driver::GPIO::new(device_driver::GPIO_BASE) }; +static mut PL011_UART: MaybeUninit = MaybeUninit::uninit(); +static mut GPIO: MaybeUninit = MaybeUninit::uninit(); #[cfg(feature = "rpi3")] -static INTERRUPT_CONTROLLER: device_driver::InterruptController = - unsafe { device_driver::InterruptController::new(mmio::PERIPHERAL_IC_START) }; +static mut INTERRUPT_CONTROLLER: MaybeUninit = + MaybeUninit::uninit(); #[cfg(feature = "rpi4")] -static INTERRUPT_CONTROLLER: device_driver::GICv2 = - unsafe { device_driver::GICv2::new(mmio::GICD_START, mmio::GICC_START) }; +static mut INTERRUPT_CONTROLLER: MaybeUninit = MaybeUninit::uninit(); //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -/// This must be called only after successful init of the Mini UART driver. -// fn post_init_mini_uart() -> Result<(), &'static str> { -// console::register_console(&MINI_UART); -// crate::info!("[0] MiniUART is live!"); -// Ok(()) -// } +/// This must be called only after successful init of the memory subsystem. +unsafe fn instantiate_uart() -> Result<(), &'static str> { + let mmio_descriptor = MMIODescriptor::new(mmio::PL011_UART_BASE, mmio::PL011_UART_SIZE); + let virt_addr = + memory::mmu::kernel_map_mmio(device_driver::PL011Uart::COMPATIBLE, &mmio_descriptor)?; + + PL011_UART.write(device_driver::PL011Uart::new(virt_addr)); + + Ok(()) +} /// This must be called only after successful init of the PL011 UART driver. -fn post_init_pl011_uart() -> Result<(), &'static str> { - console::register_console(&PL011_UART); +unsafe fn post_init_pl011_uart() -> Result<(), &'static str> { + console::register_console(PL011_UART.assume_init_ref()); crate::info!("[0] UART0 is live!"); Ok(()) } -// This must be called only after successful init of the GPIO driver. -fn post_init_gpio() -> Result<(), &'static str> { - // device_driver::MiniUart::prepare_gpio(&GPIO); - device_driver::PL011Uart::prepare_gpio(&GPIO); +/// This must be called only after successful init of the memory subsystem. +unsafe fn instantiate_gpio() -> Result<(), &'static str> { + let mmio_descriptor = MMIODescriptor::new(mmio::GPIO_BASE, mmio::GPIO_SIZE); + let virt_addr = + memory::mmu::kernel_map_mmio(device_driver::GPIO::COMPATIBLE, &mmio_descriptor)?; + + GPIO.write(device_driver::GPIO::new(virt_addr)); + + Ok(()) +} + +/// This must be called only after successful init of the GPIO driver. +unsafe fn post_init_gpio() -> Result<(), &'static str> { + device_driver::PL011Uart::prepare_gpio(GPIO.assume_init_ref()); + Ok(()) +} + +/// This must be called only after successful init of the memory subsystem. +#[cfg(feature = "rpi3")] +unsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> { + let periph_mmio_descriptor = + MMIODescriptor::new(mmio::PERIPHERAL_IC_BASE, mmio::PERIPHERAL_IC_SIZE); + let periph_virt_addr = memory::mmu::kernel_map_mmio( + device_driver::InterruptController::COMPATIBLE, + &periph_mmio_descriptor, + )?; + + INTERRUPT_CONTROLLER.write(device_driver::InterruptController::new(periph_virt_addr)); + + Ok(()) +} + +/// This must be called only after successful init of the memory subsystem. +#[cfg(feature = "rpi4")] +unsafe fn instantiate_interrupt_controller() -> Result<(), &'static str> { + let gicd_mmio_descriptor = MMIODescriptor::new(mmio::GICD_BASE, mmio::GICD_SIZE); + let gicd_virt_addr = memory::mmu::kernel_map_mmio("GICv2 GICD", &gicd_mmio_descriptor)?; + + let gicc_mmio_descriptor = MMIODescriptor::new(mmio::GICC_BASE, mmio::GICC_SIZE); + let gicc_virt_addr = memory::mmu::kernel_map_mmio("GICV2 GICC", &gicc_mmio_descriptor)?; + + INTERRUPT_CONTROLLER.write(device_driver::GICv2::new(gicd_virt_addr, gicc_virt_addr)); + Ok(()) } /// This must be called only after successful init of the interrupt controller driver. -fn post_init_interrupt_controller() -> Result<(), &'static str> { - generic_exception::asynchronous::register_irq_manager(&INTERRUPT_CONTROLLER); +unsafe fn post_init_interrupt_controller() -> Result<(), &'static str> { + generic_exception::asynchronous::register_irq_manager(INTERRUPT_CONTROLLER.assume_init_ref()); Ok(()) } -fn driver_uart() -> Result<(), &'static str> { - // let uart_descriptor = - // drivers::DeviceDriverDescriptor::new(&MINI_UART, Some(post_init_mini_uart)); - // drivers::driver_manager().register_driver(uart_descriptor); +/// Function needs to ensure that driver registration happens only after correct instantiation. +unsafe fn driver_uart() -> Result<(), &'static str> { + instantiate_uart()?; let uart_descriptor = drivers::DeviceDriverDescriptor::new( - &PL011_UART, + PL011_UART.assume_init_ref(), Some(post_init_pl011_uart), Some(exception::asynchronous::irq_map::PL011_UART), ); @@ -115,16 +160,23 @@ fn driver_uart() -> Result<(), &'static str> { Ok(()) } -fn driver_gpio() -> Result<(), &'static str> { - let gpio_descriptor = drivers::DeviceDriverDescriptor::new(&GPIO, Some(post_init_gpio), None); +/// Function needs to ensure that driver registration happens only after correct instantiation. +unsafe fn driver_gpio() -> Result<(), &'static str> { + instantiate_gpio()?; + + let gpio_descriptor = + drivers::DeviceDriverDescriptor::new(GPIO.assume_init_ref(), Some(post_init_gpio), None); drivers::driver_manager().register_driver(gpio_descriptor); Ok(()) } -fn driver_interrupt_controller() -> Result<(), &'static str> { +/// Function needs to ensure that driver registration happens only after correct instantiation. +unsafe fn driver_interrupt_controller() -> Result<(), &'static str> { + instantiate_interrupt_controller()?; + let interrupt_controller_descriptor = drivers::DeviceDriverDescriptor::new( - &INTERRUPT_CONTROLLER, + INTERRUPT_CONTROLLER.assume_init_ref(), Some(post_init_interrupt_controller), None, ); diff --git a/machine/src/platform/raspberrypi/memory/mmu.rs b/machine/src/platform/raspberrypi/memory/mmu.rs index 55c6961..3764516 100644 --- a/machine/src/platform/raspberrypi/memory/mmu.rs +++ b/machine/src/platform/raspberrypi/memory/mmu.rs @@ -1,128 +1,336 @@ -use {super::map as memory_map, crate::memory::mmu::*, core::ops::RangeInclusive}; +//! Platform memory management unit. + +use crate::{ + memory::{ + mmu::{ + self as generic_mmu, AccessPermissions, AddressSpace, AssociatedTranslationTable, + AttributeFields, MemAttributes, MemoryRegion, PageAddress, TranslationGranule, + }, + Physical, Virtual, + }, + synchronization::InitStateLock, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +type KernelTranslationTable = + ::TableStartFromBottom; //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- -/// The kernel's address space defined by this BSP. -pub type KernelAddrSpace = AddressSpace<{ memory_map::END_INCLUSIVE + 1 }>; +/// The translation granule chosen by this platform. This will be used everywhere else +/// in the kernel to derive respective data structures and their sizes. +/// For example, the `crate::memory::mmu::Page`. +pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; -const NUM_MEM_RANGES: usize = 6; +/// The kernel's virtual address space defined by this platform. +pub type KernelVirtAddrSpace = AddressSpace<{ 1024 * 1024 * 1024 }>; -/// The virtual memory layout that is agnostic of the paging granularity that the -/// hardware MMU will use. +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +/// The kernel translation tables. /// -/// Contains only special ranges, aka anything that is _not_ normal cacheable -/// DRAM. -pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( - memory_map::END_INCLUSIVE, - [ - TranslationDescriptor { - name: "Boot code and data", - virtual_range: boot_range_inclusive, - physical_range_translation: Translation::Identity, - attribute_fields: AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadOnly, - execute_never: false, - }, - }, - TranslationDescriptor { - name: "Kernel code and RO data", - virtual_range: code_range_inclusive, - physical_range_translation: Translation::Identity, - attribute_fields: AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadOnly, - execute_never: false, - }, - }, - TranslationDescriptor { - name: "Remapped Device MMIO", - virtual_range: remapped_mmio_range_inclusive, - physical_range_translation: Translation::Offset( - memory_map::mmio::MMIO_BASE + 0x20_0000, - ), - attribute_fields: AttributeFields { - mem_attributes: MemAttributes::Device, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, - }, - TranslationDescriptor { - name: "Device MMIO", - virtual_range: mmio_range_inclusive, - physical_range_translation: Translation::Identity, - attribute_fields: AttributeFields { - mem_attributes: MemAttributes::Device, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, - }, - TranslationDescriptor { - name: "DMA heap pool", - virtual_range: dma_range_inclusive, - physical_range_translation: Translation::Identity, - attribute_fields: AttributeFields { - mem_attributes: MemAttributes::NonCacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, - }, - TranslationDescriptor { - name: "Framebuffer area (static for now)", - virtual_range: || { - RangeInclusive::new( - memory_map::phys::VIDEOMEM_BASE, - memory_map::mmio::MMIO_BASE - 1, - ) - }, - physical_range_translation: Translation::Identity, - attribute_fields: AttributeFields { - mem_attributes: MemAttributes::Device, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, - }, - ], -); +/// It is mandatory that InitStateLock is transparent. +/// That is, `size_of(InitStateLock) == size_of(KernelTranslationTable)`. +/// There is a unit tests that checks this property. +static KERNEL_TABLES: InitStateLock = + InitStateLock::new(KernelTranslationTable::new()); //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -fn boot_range_inclusive() -> RangeInclusive { - RangeInclusive::new(super::boot_start(), super::boot_end_exclusive() - 1) +/// Helper function for calculating the number of pages the given parameter spans. +const fn size_to_num_pages(size: usize) -> usize { + assert!(size > 0); + assert!(size % KernelGranule::SIZE == 0); // assert! is const-fn-friendly + + size >> KernelGranule::SHIFT } -fn code_range_inclusive() -> RangeInclusive { - // Notice the subtraction to turn the exclusive end into an inclusive end. - #[allow(clippy::range_minus_one)] - RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1) +/// The code pages of the kernel binary. +fn virt_code_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::code_size()); + + let start_page_addr = super::virt_code_start(); + let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); + + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) } -fn remapped_mmio_range_inclusive() -> RangeInclusive { - // The last 64 KiB slot in the first 512 MiB - RangeInclusive::new(0x1FFF_0000, 0x1FFF_FFFF) +/// The data pages of the kernel binary. +fn virt_data_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::data_size()); + + let start_page_addr = super::virt_data_start(); + let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); + + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) } -fn mmio_range_inclusive() -> RangeInclusive { - RangeInclusive::new(memory_map::mmio::MMIO_BASE, memory_map::mmio::MMIO_END) - // RangeInclusive::new(map::phys::VIDEOMEM_BASE, map::mmio::MMIO_END), +/// The boot core stack pages. +fn virt_boot_core_stack_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::boot_core_stack_size()); + + let start_page_addr = super::virt_boot_core_stack_start(); + let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); + + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) } -fn dma_range_inclusive() -> RangeInclusive { - RangeInclusive::new( - memory_map::virt::DMA_HEAP_START, - memory_map::virt::DMA_HEAP_END, +// The binary is still identity mapped, so use this trivial conversion function for mapping below. + +fn kernel_virt_to_phys_region(virt_region: MemoryRegion) -> MemoryRegion { + MemoryRegion::new( + PageAddress::from(virt_region.start_page_addr().into_inner().as_usize()), + PageAddress::from( + virt_region + .end_exclusive_page_addr() + .into_inner() + .as_usize(), + ), ) } +//-------------------------------------------------------------------------------------------------- +// Subsumed by the kernel_map_binary() function +//-------------------------------------------------------------------------------------------------- + +// pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( +// memory_map::END_INCLUSIVE, +// [ +// TranslationDescriptor { +// name: "Remapped Device MMIO", +// virtual_range: remapped_mmio_range_inclusive, +// physical_range_translation: Translation::Offset( +// memory_map::mmio::MMIO_BASE + 0x20_0000, +// ), +// attribute_fields: AttributeFields { +// mem_attributes: MemAttributes::Device, +// acc_perms: AccessPermissions::ReadWrite, +// execute_never: true, +// }, +// }, +// TranslationDescriptor { +// name: "Device MMIO", +// virtual_range: mmio_range_inclusive, +// physical_range_translation: Translation::Identity, +// attribute_fields: AttributeFields { +// mem_attributes: MemAttributes::Device, +// acc_perms: AccessPermissions::ReadWrite, +// execute_never: true, +// }, +// }, +// TranslationDescriptor { +// name: "DMA heap pool", +// virtual_range: dma_range_inclusive, +// physical_range_translation: Translation::Identity, +// attribute_fields: AttributeFields { +// mem_attributes: MemAttributes::NonCacheableDRAM, +// acc_perms: AccessPermissions::ReadWrite, +// execute_never: true, +// }, +// }, +// TranslationDescriptor { +// name: "Framebuffer area (static for now)", +// virtual_range: || { +// RangeInclusive::new( +// memory_map::phys::VIDEOMEM_BASE, +// memory_map::mmio::MMIO_BASE - 1, +// ) +// }, +// physical_range_translation: Translation::Identity, +// attribute_fields: AttributeFields { +// mem_attributes: MemAttributes::Device, +// acc_perms: AccessPermissions::ReadWrite, +// execute_never: true, +// }, +// }, +// ], +// ); + //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// Return a reference to the virtual memory layout. -pub fn virt_mem_layout() -> &'static KernelVirtualLayout { - &LAYOUT +/// Return a reference to the kernel's translation tables. +pub fn kernel_translation_tables() -> &'static InitStateLock { + &KERNEL_TABLES } + +/// The MMIO remap pages. +pub fn virt_mmio_remap_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::mmio_remap_size()); + + let start_page_addr = super::virt_mmio_remap_start(); + let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); + + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) +} + +/// Map the kernel binary. +/// +/// # Safety +/// +/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking. +pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { + generic_mmu::kernel_map_at( + "Kernel boot-core stack", + &virt_boot_core_stack_region(), + &kernel_virt_to_phys_region(virt_boot_core_stack_region()), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; + + // TranslationDescriptor { + // name: "Boot code and data", + // virtual_range: boot_range_inclusive, + // physical_range_translation: Translation::Identity, + // attribute_fields: AttributeFields { + // mem_attributes: MemAttributes::CacheableDRAM, + // acc_perms: AccessPermissions::ReadOnly, + // execute_never: false, + // }, + // }, + + // TranslationDescriptor { + // name: "Kernel code and RO data", + // virtual_range: code_range_inclusive, + // physical_range_translation: Translation::Identity, + // attribute_fields: AttributeFields { + // mem_attributes: MemAttributes::CacheableDRAM, + // acc_perms: AccessPermissions::ReadOnly, + // execute_never: false, + // }, + // }, + + generic_mmu::kernel_map_at( + "Kernel code and RO data", + &virt_code_region(), + &kernel_virt_to_phys_region(virt_code_region()), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + )?; + + generic_mmu::kernel_map_at( + "Kernel data and bss", + &virt_data_region(), + &kernel_virt_to_phys_region(virt_data_region()), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; + + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use { + super::*, + core::{cell::UnsafeCell, ops::Range}, + }; + + /// Check alignment of the kernel's virtual memory layout sections. + #[test_case] + fn virt_mem_layout_sections_are_64KiB_aligned() { + for i in [ + virt_boot_core_stack_region, + virt_code_region, + virt_data_region, + ] + .iter() + { + let start = i().start_page_addr().into_inner(); + let end_exclusive = i().end_exclusive_page_addr().into_inner(); + + assert!(start.is_page_aligned()); + assert!(end_exclusive.is_page_aligned()); + assert!(end_exclusive >= start); + } + } + + /// Ensure the kernel's virtual memory layout is free of overlaps. + #[test_case] + fn virt_mem_layout_has_no_overlaps() { + let layout = [ + virt_boot_core_stack_region(), + virt_code_region(), + virt_data_region(), + ]; + + for (i, first_range) in layout.iter().enumerate() { + for second_range in layout.iter().skip(i + 1) { + assert!(!first_range.overlaps(second_range)) + } + } + } + + /// Check if KERNEL_TABLES is in .bss. + #[test_case] + fn kernel_tables_in_bss() { + extern "Rust" { + static __bss_start: UnsafeCell; + static __bss_end_exclusive: UnsafeCell; + } + + let bss_range = unsafe { + Range { + start: __bss_start.get(), + end: __bss_end_exclusive.get(), + } + }; + let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; + + assert!(bss_range.contains(&kernel_tables_addr)); + } +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +// fn boot_range_inclusive() -> RangeInclusive { +// RangeInclusive::new(super::boot_start(), super::boot_end_exclusive() - 1) +// } +// +// fn code_range_inclusive() -> RangeInclusive { +// // Notice the subtraction to turn the exclusive end into an inclusive end. +// #[allow(clippy::range_minus_one)] +// RangeInclusive::new(super::code_start(), super::code_end_exclusive() - 1) +// } +// +// fn remapped_mmio_range_inclusive() -> RangeInclusive { +// // The last 64 KiB slot in the first 512 MiB +// RangeInclusive::new(0x1FFF_0000, 0x1FFF_FFFF) +// } +// +// fn mmio_range_inclusive() -> RangeInclusive { +// RangeInclusive::new(memory_map::mmio::MMIO_BASE, memory_map::mmio::MMIO_END) +// // RangeInclusive::new(map::phys::VIDEOMEM_BASE, map::mmio::MMIO_END), +// } +// +// fn dma_range_inclusive() -> RangeInclusive { +// RangeInclusive::new( +// memory_map::virt::DMA_HEAP_START, +// memory_map::virt::DMA_HEAP_END, +// ) +// } diff --git a/machine/src/platform/raspberrypi/memory/mod.rs b/machine/src/platform/raspberrypi/memory/mod.rs index 3f87898..327ddcf 100644 --- a/machine/src/platform/raspberrypi/memory/mod.rs +++ b/machine/src/platform/raspberrypi/memory/mod.rs @@ -1,11 +1,72 @@ -use core::cell::UnsafeCell; - +//! Platform memory Management. +//! +//! The physical memory layout. +//! +//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used +//! as the boot core's stack. +//! +//! +---------------------------------------+ +//! | | boot_core_stack_start @ 0x0 +//! | | ^ +//! | Boot-core Stack | | stack +//! | | | growth +//! | | | direction +//! +---------------------------------------+ +//! | | code_start @ 0x8_0000 == boot_core_stack_end_exclusive +//! | .text | +//! | .rodata | +//! | .got | +//! | | +//! +---------------------------------------+ +//! | | data_start == code_end_exclusive +//! | .data | +//! | .bss | +//! | | +//! +---------------------------------------+ +//! | | data_end_exclusive +//! | | +//! +//! +//! +//! +//! +//! The virtual memory layout is as follows: +//! +//! +---------------------------------------+ +//! | | boot_core_stack_start @ 0x0 +//! | | ^ +//! | Boot-core Stack | | stack +//! | | | growth +//! | | | direction +//! +---------------------------------------+ +//! | | code_start @ 0x8_0000 == boot_core_stack_end_exclusive +//! | .text | +//! | .rodata | +//! | .got | +//! | | +//! +---------------------------------------+ +//! | | data_start == code_end_exclusive +//! | .data | +//! | .bss | +//! | | +//! +---------------------------------------+ +//! | | mmio_remap_start == data_end_exclusive +//! | VA region for MMIO remapping | +//! | | +//! +---------------------------------------+ +//! | | mmio_remap_end_exclusive +//! | | pub mod mmu; //-------------------------------------------------------------------------------------------------- // Private Definitions //-------------------------------------------------------------------------------------------------- +use { + crate::memory::{mmu::PageAddress, Address, Physical, Virtual}, + core::cell::UnsafeCell, +}; + // Symbols from the linker script. extern "Rust" { // Boot code. @@ -38,15 +99,32 @@ extern "Rust" { static __RO_END: UnsafeCell<()>; } +// Symbols from the linker script. +// extern "Rust" { +// static __code_start: UnsafeCell<()>; // __RO_START +// static __code_end_exclusive: UnsafeCell<()>; // __RO_END +// +// static __data_start: UnsafeCell<()>; +// static __data_end_exclusive: UnsafeCell<()>; +// +// static __mmio_remap_start: UnsafeCell<()>; +// static __mmio_remap_end_exclusive: UnsafeCell<()>; +// +// static __boot_core_stack_start: UnsafeCell<()>; +// static __boot_core_stack_end_exclusive: UnsafeCell<()>; +// } + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- -/// System memory map. +/// The board's physical memory map. /// This is a fixed memory map for Raspberry Pi, -/// @todo we need to infer the memory map from the provided DTB. +/// @todo we need to infer the memory map from the provided DTB instead. #[rustfmt::skip] -pub mod map { // @todo only pub(super) for proper isolation! +pub(super) mod map { + use super::*; + /// Beginning of memory. pub const START: usize = 0x0000_0000; /// End of memory - 8Gb RPi4 @@ -63,49 +141,75 @@ pub mod map { // @todo only pub(super) for proper isolation! pub const UART_OFFSET: usize = 0x0020_1000; pub const MINIUART_OFFSET: usize = 0x0021_5000; - /// Memory-mapped devices. + /// Physical devices. #[cfg(feature = "rpi3")] pub mod mmio { use super::*; /// Base address of MMIO register range. pub const MMIO_BASE: usize = 0x3F00_0000; - /// Base address of ARM<->VC mailbox area. - pub const VIDEOCORE_MBOX_BASE: usize = MMIO_BASE + VIDEOCORE_MBOX_OFFSET; - /// Base address of GPIO registers. - pub const GPIO_BASE: usize = MMIO_BASE + GPIO_OFFSET; - /// Base address of regular UART. - pub const PL011_UART_BASE: usize = MMIO_BASE + UART_OFFSET; - /// Base address of MiniUART. - pub const MINI_UART_BASE: usize = MMIO_BASE + MINIUART_OFFSET; + /// Interrupt controller - pub const PERIPHERAL_IC_START: usize = MMIO_BASE + 0x0000_B200; - /// End of MMIO memory. - pub const MMIO_END: usize = super::END_INCLUSIVE; + pub const PERIPHERAL_IC_BASE: Address = Address::new(MMIO_BASE + 0x0000_B200); + pub const PERIPHERAL_IC_SIZE: usize = 0x24; + + /// Base address of ARM<->VC mailbox area. + pub const VIDEOCORE_MBOX_BASE: Address = Address::new(MMIO_BASE + VIDEOCORE_MBOX_OFFSET); + + /// Base address of GPIO registers. + pub const GPIO_BASE: Address = Address::new(MMIO_BASE + GPIO_OFFSET); + pub const GPIO_SIZE: usize = 0xA0; + + pub const PL011_UART_BASE: Address = Address::new(MMIO_BASE + UART_OFFSET); + pub const PL011_UART_SIZE: usize = 0x48; + + /// Base address of MiniUART. + pub const MINI_UART_BASE: Address = Address::new(MMIO_BASE + MINIUART_OFFSET); + + /// End of MMIO memory region. + pub const END: Address = Address::new(0x4001_0000); } - /// Memory-mapped devices. + /// Physical devices. #[cfg(feature = "rpi4")] pub mod mmio { use super::*; /// Base address of MMIO register range. - pub const MMIO_BASE: usize = 0xFE00_0000; + pub const MMIO_BASE: usize = 0xFE00_0000; + + /// Base address of GPIO registers. + pub const GPIO_BASE: Address = Address::new(MMIO_BASE + GPIO_OFFSET); + pub const GPIO_SIZE: usize = 0xA0; + + /// Base address of regular UART. + pub const PL011_UART_BASE: Address = Address::new(MMIO_BASE + UART_OFFSET); + pub const PL011_UART_SIZE: usize = 0x48; + + /// Base address of MiniUART. + pub const MINI_UART_BASE: Address = Address::new(MMIO_BASE + MINIUART_OFFSET); + + /// Interrupt controller + pub const GICD_BASE: Address = Address::new(0xFF84_1000); + pub const GICD_SIZE: usize = 0x824; + + pub const GICC_BASE: Address = Address::new(0xFF84_2000); + pub const GICC_SIZE: usize = 0x14; + /// Base address of ARM<->VC mailbox area. pub const VIDEOCORE_MBOX_BASE: usize = MMIO_BASE + VIDEOCORE_MBOX_OFFSET; - /// Base address of GPIO registers. - pub const GPIO_BASE: usize = MMIO_BASE + GPIO_OFFSET; - /// Base address of regular UART. - pub const PL011_UART_BASE: usize = MMIO_BASE + UART_OFFSET; - /// Base address of MiniUART. - pub const MINI_UART_BASE: usize = MMIO_BASE + MINIUART_OFFSET; - /// Interrupt controller - pub const GICD_START: usize = 0xFF84_1000; - pub const GICC_START: usize = 0xFF84_2000; - /// End of MMIO memory. - pub const MMIO_END: usize = super::END_INCLUSIVE; + + /// End of MMIO memory region. + pub const END: Address = Address::new(0xFF85_0000); } + /// End address of mapped memory. + pub const END: Address = mmio::END; + + //---- + // Unused? + //---- + /// Virtual (mapped) addresses. pub mod virt { /// Start (top) of kernel stack. @@ -153,11 +257,91 @@ fn code_start() -> usize { unsafe { __RO_START.get() as usize } } -/// Exclusive end page address of the code segment. +/// Start page address of the code segment. +/// /// # Safety /// /// - Value is provided by the linker script and must be trusted as-is. #[inline(always)] -fn code_end_exclusive() -> usize { - unsafe { __RO_END.get() as usize } +fn virt_code_start() -> PageAddress { + PageAddress::from(unsafe { __RO_START.get() as usize }) +} + +/// Size of the code segment. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn code_size() -> usize { + unsafe { (__RO_END.get() as usize) - (__RO_START.get() as usize) } +} + +/// Exclusive end page address of the code segment. +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +// #[inline(always)] +// fn code_end_exclusive() -> usize { +// unsafe { __RO_END.get() as usize } +// } + +/// Start page address of the data segment. +#[inline(always)] +fn virt_data_start() -> PageAddress { + PageAddress::from(unsafe { __data_start.get() as usize }) +} + +/// Size of the data segment. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn data_size() -> usize { + unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) } +} + +/// Start page address of the MMIO remap reservation. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn virt_mmio_remap_start() -> PageAddress { + PageAddress::from(unsafe { __mmio_remap_start.get() as usize }) +} + +/// Size of the MMIO remap reservation. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn mmio_remap_size() -> usize { + unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) } +} + +/// Start page address of the boot core's stack. +#[inline(always)] +fn virt_boot_core_stack_start() -> PageAddress { + PageAddress::from(unsafe { __boot_core_stack_start.get() as usize }) +} + +/// Size of the boot core's stack. +#[inline(always)] +fn boot_core_stack_size() -> usize { + unsafe { + (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Exclusive end address of the physical address space. +#[inline(always)] +pub fn phys_addr_space_end_exclusive_addr() -> PageAddress { + PageAddress::from(map::END) } diff --git a/machine/src/platform/raspberrypi/power.rs b/machine/src/platform/raspberrypi/power.rs index 81e8560..05f4fe8 100644 --- a/machine/src/platform/raspberrypi/power.rs +++ b/machine/src/platform/raspberrypi/power.rs @@ -11,7 +11,10 @@ use { mailbox::{channel, Mailbox, MailboxOps}, BcmHost, }, - crate::platform::device_driver::common::MMIODerefWrapper, + crate::{ + memory::{Address, Virtual}, + platform::device_driver::common::MMIODerefWrapper, + }, snafu::Snafu, tock_registers::{ interfaces::{Readable, Writeable}, @@ -74,9 +77,9 @@ impl Power { /// # Safety /// /// Unsafe, duh! - pub const unsafe fn new(base_addr: usize) -> Power { + pub const unsafe fn new(mmio_base_addr: Address) -> Power { Power { - registers: Registers::new(base_addr), + registers: Registers::new(mmio_base_addr), } } diff --git a/nucleus/src/main.rs b/nucleus/src/main.rs index 45b018c..597e5f8 100644 --- a/nucleus/src/main.rs +++ b/nucleus/src/main.rs @@ -36,17 +36,7 @@ use machine::devices::serial::SerialOps; use { cfg_if::cfg_if, core::{cell::UnsafeCell, time::Duration}, - machine::{ - arch, - console::console, - entry, exception, info, memory, - platform::raspberrypi::{ - display::{Color, DrawError}, - mailbox::{channel, Mailbox, MailboxOps}, - vc::VC, - }, - println, time, warn, - }, + machine::{arch, console::console, entry, exception, info, memory, println, time, warn}, }; entry!(kernel_init); @@ -65,17 +55,19 @@ pub unsafe fn kernel_init() -> ! { #[cfg(feature = "jtag")] machine::debug::jtag::wait_debugger(); - // init_exception_traps(); // @todo - // - // init_mmu(); // @todo exception::handling_init(); - use machine::memory::mmu::interface::MMU; + let phys_kernel_tables_base_addr = match memory::mmu::kernel_map_binary() { + Err(string) => panic!("Error mapping kernel binary: {}", string), + Ok(addr) => addr, + }; - if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() { - panic!("MMU: {}", string); + if let Err(e) = memory::mmu::enable_mmu_and_caching(phys_kernel_tables_base_addr) { + panic!("Enabling MMU failed: {}", e); } + memory::mmu::post_enable_init(); + if let Err(x) = machine::platform::drivers::init() { panic!("Error initializing platform drivers: {}", x); } @@ -95,10 +87,8 @@ pub unsafe fn kernel_init() -> ! { /// Safe kernel code. // #[inline] +#[cfg(not(test))] pub fn kernel_main() -> ! { - #[cfg(test)] - test_main(); - // info!("{}", libkernel::version()); // info!("Booting on: {}", bsp::board_name()); @@ -109,8 +99,8 @@ pub fn kernel_main() -> ! { ); info!("Booting on: {}", machine::platform::BcmHost::board_name()); - info!("MMU online. Special regions:"); - machine::platform::memory::mmu::virt_mem_layout().print_layout(); + // info!("MMU online. Special regions:"); + // machine::platform::memory::mmu::virt_mem_layout().print_layout(); let (_, privilege_level) = exception::current_privilege_level(); info!("Current privilege level: {}", privilege_level); @@ -148,8 +138,8 @@ fn panicked(info: &PanicInfo) -> ! { } fn print_mmu_state_and_features() { - use machine::memory::mmu::interface::MMU; - memory::mmu::mmu().print_features(); + // use machine::memory::mmu::interface::MMU; + // memory::mmu::mmu().print_features(); } //------------------------------------------------------------ @@ -162,11 +152,11 @@ fn command_prompt() { match machine::console::command_prompt(&mut buf) { // b"mmu" => init_mmu(), b"feats" => print_mmu_state_and_features(), - b"disp" => check_display_init(), + // b"disp" => check_display_init(), b"trap" => check_data_abort_trap(), - b"map" => machine::platform::memory::mmu::virt_mem_layout().print_layout(), - b"led on" => set_led(true), - b"led off" => set_led(false), + // b"map" => machine::platform::memory::mmu::virt_mem_layout().print_layout(), + // b"led on" => set_led(true), + // b"led off" => set_led(false), b"help" => print_help(), b"end" => break 'cmd_loop, x => warn!("[!] Unknown command {:?}, try 'help'", x), @@ -180,26 +170,26 @@ fn print_help() { println!(" feats - print MMU state and supported features"); #[cfg(not(feature = "noserial"))] println!(" uart - try to reinitialize UART serial"); - println!(" disp - try to init VC framebuffer and draw some text"); + // println!(" disp - try to init VC framebuffer and draw some text"); println!(" trap - trigger and recover from a data abort exception"); println!(" map - show kernel memory layout"); - println!(" led [on|off] - change RPi LED status"); + // println!(" led [on|off] - change RPi LED status"); println!(" end - leave console and reset board"); } -fn set_led(enable: bool) { - let mut mbox = Mailbox::<8>::default(); - let index = mbox.request(); - let index = mbox.set_led_on(index, enable); - let mbox = mbox.end(index); - - mbox.call(channel::PropertyTagsArmToVc) - .map_err(|e| { - warn!("Mailbox call returned error {}", e); - warn!("Mailbox contents: {:?}", mbox); - }) - .ok(); -} +// fn set_led(enable: bool) { +// let mut mbox = Mailbox::<8>::default(); +// let index = mbox.request(); +// let index = mbox.set_led_on(index, enable); +// let mbox = mbox.end(index); +// +// mbox.call(channel::PropertyTagsArmToVc) +// .map_err(|e| { +// warn!("Mailbox call returned error {}", e); +// warn!("Mailbox contents: {:?}", mbox); +// }) +// .ok(); +// } fn reboot() -> ! { cfg_if! { @@ -207,47 +197,48 @@ fn reboot() -> ! { info!("Bye, shutting down QEMU"); machine::qemu::semihosting::exit_success() } else { - use machine::platform::raspberrypi::power::Power; + // use machine::platform::raspberrypi::power::Power; info!("Bye, going to reset now"); - Power::default().reset() + // Power::default().reset() + machine::cpu::endless_sleep() } } } -fn check_display_init() { - display_graphics() - .map_err(|e| { - warn!("Error in display: {}", e); - }) - .ok(); -} - -fn display_graphics() -> Result<(), DrawError> { - if let Ok(mut display) = VC::init_fb(800, 600, 32) { - info!("Display created"); - - display.clear(Color::black()); - info!("Display cleared"); - - display.rect(10, 10, 250, 250, Color::rgb(32, 96, 64)); - display.draw_text(50, 50, "Hello there!", Color::rgb(128, 192, 255))?; - - let mut buf = [0u8; 64]; - let s = machine::write_to::show(&mut buf, format_args!("Display width {}", display.width)); - - if s.is_err() { - display.draw_text(50, 150, "Error displaying", Color::red())? - } else { - display.draw_text(50, 150, s.unwrap(), Color::white())? - } - - display.draw_text(150, 50, "RED", Color::red())?; - display.draw_text(160, 60, "GREEN", Color::green())?; - display.draw_text(170, 70, "BLUE", Color::blue())?; - } - Ok(()) -} +// fn check_display_init() { +// display_graphics() +// .map_err(|e| { +// warn!("Error in display: {}", e); +// }) +// .ok(); +// } +// +// fn display_graphics() -> Result<(), DrawError> { +// if let Ok(mut display) = VC::init_fb(800, 600, 32) { +// info!("Display created"); +// +// display.clear(Color::black()); +// info!("Display cleared"); +// +// display.rect(10, 10, 250, 250, Color::rgb(32, 96, 64)); +// display.draw_text(50, 50, "Hello there!", Color::rgb(128, 192, 255))?; +// +// let mut buf = [0u8; 64]; +// let s = machine::write_to::show(&mut buf, format_args!("Display width {}", display.width)); +// +// if s.is_err() { +// display.draw_text(50, 150, "Error displaying", Color::red())? +// } else { +// display.draw_text(50, 150, s.unwrap(), Color::white())? +// } +// +// display.draw_text(150, 50, "RED", Color::red())?; +// display.draw_text(160, 60, "GREEN", Color::green())?; +// display.draw_text(170, 70, "BLUE", Color::blue())?; +// } +// Ok(()) +// } fn check_data_abort_trap() { // Cause an exception by accessing a virtual address for which no @@ -261,6 +252,11 @@ fn check_data_abort_trap() { info!("[i] Whoa! We recovered from an exception."); } +#[cfg(test)] +pub fn kernel_main() -> ! { + test_main() +} + #[cfg(test)] mod main_tests { use {super::*, core::panic::PanicInfo};