From 7f666000ceb65cce8134cd0f0e0b4cf066523daf Mon Sep 17 00:00:00 2001 From: Andre Richter Date: Fri, 2 Jul 2021 23:04:18 +0200 Subject: [PATCH] Init DRAM in assembly instead of Rust See https://github.com/rust-embedded/cortex-m-rt/issues/300 --- 02_runtime_init/README.md | 200 ++++-------------- 02_runtime_init/src/_arch/aarch64/cpu/boot.rs | 8 +- 02_runtime_init/src/_arch/aarch64/cpu/boot.s | 21 +- 02_runtime_init/src/bsp/raspberrypi.rs | 1 - 02_runtime_init/src/bsp/raspberrypi/link.ld | 10 +- 02_runtime_init/src/bsp/raspberrypi/memory.rs | 37 ---- 02_runtime_init/src/main.rs | 6 +- 02_runtime_init/src/memory.rs | 30 --- 02_runtime_init/src/runtime_init.rs | 37 ---- 03_hacky_hello_world/README.md | 14 +- .../src/_arch/aarch64/cpu/boot.rs | 8 +- .../src/_arch/aarch64/cpu/boot.s | 21 +- 03_hacky_hello_world/src/bsp/raspberrypi.rs | 1 - .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 37 ---- 03_hacky_hello_world/src/main.rs | 6 +- 03_hacky_hello_world/src/memory.rs | 30 --- 03_hacky_hello_world/src/runtime_init.rs | 37 ---- 04_safe_globals/README.md | 8 +- 04_safe_globals/src/_arch/aarch64/cpu/boot.rs | 8 +- 04_safe_globals/src/_arch/aarch64/cpu/boot.s | 21 +- 04_safe_globals/src/bsp/raspberrypi.rs | 1 - 04_safe_globals/src/bsp/raspberrypi/link.ld | 10 +- 04_safe_globals/src/bsp/raspberrypi/memory.rs | 37 ---- 04_safe_globals/src/main.rs | 6 +- 04_safe_globals/src/memory.rs | 30 --- 04_safe_globals/src/runtime_init.rs | 37 ---- 05_drivers_gpio_uart/README.md | 33 ++- .../src/_arch/aarch64/cpu/boot.rs | 8 +- .../src/_arch/aarch64/cpu/boot.s | 21 +- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 32 --- 05_drivers_gpio_uart/src/main.rs | 6 +- 05_drivers_gpio_uart/src/memory.rs | 30 --- 05_drivers_gpio_uart/src/runtime_init.rs | 37 ---- 06_uart_chainloader/README.md | 78 +++---- 06_uart_chainloader/demo_payload_rpi3.img | Bin 6920 -> 6696 bytes 06_uart_chainloader/demo_payload_rpi4.img | Bin 6760 -> 6552 bytes .../src/_arch/aarch64/cpu/boot.rs | 8 +- .../src/_arch/aarch64/cpu/boot.s | 24 ++- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 28 --- 06_uart_chainloader/src/main.rs | 6 +- 06_uart_chainloader/src/memory.rs | 30 --- 06_uart_chainloader/src/runtime_init.rs | 37 ---- 07_timestamps/README.md | 82 +++---- 07_timestamps/src/_arch/aarch64/cpu/boot.rs | 8 +- 07_timestamps/src/_arch/aarch64/cpu/boot.s | 21 +- 07_timestamps/src/bsp/raspberrypi/link.ld | 10 +- 07_timestamps/src/bsp/raspberrypi/memory.rs | 32 --- 07_timestamps/src/main.rs | 6 +- 07_timestamps/src/memory.rs | 30 --- 07_timestamps/src/runtime_init.rs | 37 ---- .../src/_arch/aarch64/cpu/boot.rs | 8 +- 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s | 21 +- 08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 32 --- 08_hw_debug_JTAG/src/main.rs | 6 +- 08_hw_debug_JTAG/src/memory.rs | 30 --- 08_hw_debug_JTAG/src/runtime_init.rs | 37 ---- 09_privilege_level/README.md | 57 +++-- .../src/_arch/aarch64/cpu/boot.rs | 10 +- .../src/_arch/aarch64/cpu/boot.s | 23 +- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 32 --- 09_privilege_level/src/main.rs | 6 +- 09_privilege_level/src/memory.rs | 30 --- 09_privilege_level/src/runtime_init.rs | 37 ---- .../README.md | 72 ++++--- .../src/_arch/aarch64/cpu/boot.rs | 10 +- .../src/_arch/aarch64/cpu/boot.s | 23 +- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 25 +-- .../src/main.rs | 5 +- .../src/memory.rs | 25 --- .../src/runtime_init.rs | 37 ---- 11_exceptions_part1_groundwork/README.md | 4 +- .../src/_arch/aarch64/cpu/boot.rs | 10 +- .../src/_arch/aarch64/cpu/boot.s | 23 +- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 25 +-- 11_exceptions_part1_groundwork/src/main.rs | 5 +- 11_exceptions_part1_groundwork/src/memory.rs | 25 --- .../src/runtime_init.rs | 37 ---- 12_integrated_testing/README.md | 125 ++++------- .../src/_arch/aarch64/cpu/boot.rs | 10 +- .../src/_arch/aarch64/cpu/boot.s | 23 +- .../src/_arch/aarch64/memory/mmu.rs | 13 +- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 25 +-- 12_integrated_testing/src/lib.rs | 12 +- 12_integrated_testing/src/memory.rs | 62 ------ 12_integrated_testing/src/runtime_init.rs | 40 ---- 13_exceptions_part2_peripheral_IRQs/README.md | 8 +- .../src/_arch/aarch64/cpu/boot.rs | 10 +- .../src/_arch/aarch64/cpu/boot.s | 23 +- .../src/_arch/aarch64/memory/mmu.rs | 13 +- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 25 +-- .../src/lib.rs | 12 +- .../src/memory.rs | 62 ------ .../src/runtime_init.rs | 40 ---- 14_virtual_mem_part2_mmio_remap/README.md | 148 +++++++------ .../src/_arch/aarch64/cpu/boot.rs | 10 +- .../src/_arch/aarch64/cpu/boot.s | 23 +- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 24 +-- .../src/bsp/raspberrypi/memory/mmu.rs | 13 +- 14_virtual_mem_part2_mmio_remap/src/lib.rs | 12 +- 14_virtual_mem_part2_mmio_remap/src/memory.rs | 58 +---- .../src/runtime_init.rs | 40 ---- .../README.md | 38 ++-- .../src/_arch/aarch64/cpu/boot.rs | 11 +- .../src/_arch/aarch64/cpu/boot.s | 23 +- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 24 +-- .../src/lib.rs | 12 +- .../src/memory.rs | 58 +---- .../src/runtime_init.rs | 40 ---- .../README.md | 65 +++--- .../src/_arch/aarch64/cpu/boot.rs | 15 +- .../src/_arch/aarch64/cpu/boot.s | 25 ++- .../src/bsp/raspberrypi/link.ld | 10 +- .../src/bsp/raspberrypi/memory.rs | 24 +-- .../src/lib.rs | 7 +- .../src/memory.rs | 58 +---- .../src/runtime_init.rs | 41 ---- X1_JTAG_boot/jtag_boot_rpi3.img | Bin 8032 -> 7856 bytes X1_JTAG_boot/jtag_boot_rpi4.img | Bin 6736 -> 6584 bytes X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs | 8 +- X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s | 21 +- X1_JTAG_boot/src/bsp/raspberrypi/link.ld | 10 +- X1_JTAG_boot/src/bsp/raspberrypi/memory.rs | 32 --- X1_JTAG_boot/src/main.rs | 6 +- X1_JTAG_boot/src/memory.rs | 30 --- X1_JTAG_boot/src/runtime_init.rs | 37 ---- 136 files changed, 872 insertions(+), 2585 deletions(-) delete mode 100644 02_runtime_init/src/bsp/raspberrypi/memory.rs delete mode 100644 02_runtime_init/src/memory.rs delete mode 100644 02_runtime_init/src/runtime_init.rs delete mode 100644 03_hacky_hello_world/src/bsp/raspberrypi/memory.rs delete mode 100644 03_hacky_hello_world/src/memory.rs delete mode 100644 03_hacky_hello_world/src/runtime_init.rs delete mode 100644 04_safe_globals/src/bsp/raspberrypi/memory.rs delete mode 100644 04_safe_globals/src/memory.rs delete mode 100644 04_safe_globals/src/runtime_init.rs delete mode 100644 05_drivers_gpio_uart/src/memory.rs delete mode 100644 05_drivers_gpio_uart/src/runtime_init.rs delete mode 100644 06_uart_chainloader/src/memory.rs delete mode 100644 06_uart_chainloader/src/runtime_init.rs delete mode 100644 07_timestamps/src/memory.rs delete mode 100644 07_timestamps/src/runtime_init.rs delete mode 100644 08_hw_debug_JTAG/src/memory.rs delete mode 100644 08_hw_debug_JTAG/src/runtime_init.rs delete mode 100644 09_privilege_level/src/memory.rs delete mode 100644 09_privilege_level/src/runtime_init.rs delete mode 100644 10_virtual_mem_part1_identity_mapping/src/runtime_init.rs delete mode 100644 11_exceptions_part1_groundwork/src/runtime_init.rs delete mode 100644 12_integrated_testing/src/runtime_init.rs delete mode 100644 13_exceptions_part2_peripheral_IRQs/src/runtime_init.rs delete mode 100644 14_virtual_mem_part2_mmio_remap/src/runtime_init.rs delete mode 100644 15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs delete mode 100644 16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs delete mode 100644 X1_JTAG_boot/src/memory.rs delete mode 100644 X1_JTAG_boot/src/runtime_init.rs diff --git a/02_runtime_init/README.md b/02_runtime_init/README.md index 5cba10fb3..3dc0b3d36 100644 --- a/02_runtime_init/README.md +++ b/02_runtime_init/README.md @@ -2,8 +2,9 @@ ## tl;dr -- We extend `boot.s` to call into Rust code for the first time. There, we zero the [bss] section - before execution is halted with a call to `panic()`. +- We extend `boot.s` to call into Rust code for the first time. Before the jump + to Rust happens, a bit of runtime init work is done. +- The Rust code being called just halts execution with a call to `panic!()`. - Check out `make qemu` again to see the additional code run. ## Notable additions @@ -12,11 +13,11 @@ - New sections: `.rodata`, `.got`, `.data`, `.bss`. - A dedicated place for linking boot-time arguments that need to be read by `_start()`. - `_start()` in `_arch/__arch_name__/cpu/boot.s`: - 1. Halt core if core != core0. - 1. Set up the `stack pointer`. - 1. Jump to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`. -- `runtime_init()` in `runtime_init.rs`: - - Zeros the `.bss` section. + 1. Halts core if core != core0. + 1. Initializes the `DRAM` by zeroing the [bss] section. + 1. Sets up the `stack pointer`. + 1. Jumps to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`. +- `_start_rust()`: - Calls `kernel_init()`, which calls `panic!()`, which eventually halts core0 as well. - The library now uses the [cortex-a] crate, which provides zero-overhead abstractions and wraps `unsafe` parts when dealing with the CPU's resources. @@ -64,12 +65,8 @@ diff -uNr 01_wait_forever/Makefile 02_runtime_init/Makefile diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arch/aarch64/cpu/boot.rs --- 01_wait_forever/src/_arch/aarch64/cpu/boot.rs +++ 02_runtime_init/src/_arch/aarch64/cpu/boot.rs -@@ -11,5 +11,23 @@ - //! - //! crate::cpu::boot::arch_boot +@@ -13,3 +13,15 @@ -+use crate::runtime_init; -+ // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); + @@ -80,13 +77,9 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arc +/// The Rust entry of the `kernel` binary. +/// +/// The function is called from the assembly `_start` function. -+/// -+/// # Safety -+/// -+/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +#[no_mangle] +pub unsafe fn _start_rust() -> ! { -+ runtime_init::runtime_init() ++ crate::kernel_init() +} diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch/aarch64/cpu/boot.s @@ -117,7 +110,7 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch // Public Code //-------------------------------------------------------------------------------------------------- .section .text._start -@@ -11,6 +29,22 @@ +@@ -11,9 +29,38 @@ // fn _start() //------------------------------------------------------------------------------ _start: @@ -126,10 +119,22 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 -+ b.ne 1f ++ b.ne parking_loop ++ ++ // If execution reaches here, it is the boot core. ++ ++ // Initialize DRAM. ++ ADR_REL x0, __bss_start ++ ADR_REL x1, __bss_end_exclusive + -+ // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. ++bss_init_loop: ++ cmp x0, x1 ++ b.eq prepare_rust ++ stp xzr, xzr, [x0], #16 ++ b bss_init_loop + ++ // Prepare the jump to Rust code. ++prepare_rust: + // Set the stack pointer. + ADR_REL x0, __boot_core_stack_end_exclusive + mov sp, x0 @@ -138,8 +143,14 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch + b _start_rust + // Infinitely wait for events (aka "park the core"). - 1: wfe - b 1b +-1: wfe +- b 1b ++parking_loop: ++ wfe ++ b parking_loop + + .size _start, . - _start + .type _start, function diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aarch64/cpu.rs --- 01_wait_forever/src/_arch/aarch64/cpu.rs @@ -194,7 +205,7 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/cpu.rs 02_runtime_init/src/bsp/ras diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/raspberrypi/link.ld --- 01_wait_forever/src/bsp/raspberrypi/link.ld +++ 02_runtime_init/src/bsp/raspberrypi/link.ld -@@ -11,17 +11,45 @@ +@@ -11,17 +11,43 @@ PHDRS { segment_rx PT_LOAD FLAGS(5); /* 5 == RX */ @@ -230,70 +241,25 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/ra + ***********************************************************************************************/ + .data : { *(.data*) } :segment_rw + -+ /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -+ .bss : ALIGN(8) ++ /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ ++ .bss : ALIGN(16) + { + __bss_start = .; + *(.bss*); -+ . = ALIGN(8); -+ -+ . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ -+ __bss_end_inclusive = . - 8; ++ . = ALIGN(16); ++ __bss_end_exclusive = .; + } :NONE } -diff -uNr 01_wait_forever/src/bsp/raspberrypi/memory.rs 02_runtime_init/src/bsp/raspberrypi/memory.rs ---- 01_wait_forever/src/bsp/raspberrypi/memory.rs -+++ 02_runtime_init/src/bsp/raspberrypi/memory.rs -@@ -0,0 +1,37 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! BSP Memory Management. -+ -+use core::{cell::UnsafeCell, ops::RangeInclusive}; -+ -+//-------------------------------------------------------------------------------------------------- -+// Private Definitions -+//-------------------------------------------------------------------------------------------------- -+ -+// Symbols from the linker script. -+extern "Rust" { -+ static __bss_start: UnsafeCell; -+ static __bss_end_inclusive: UnsafeCell; -+} -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Return the inclusive range spanning the .bss section. -+/// -+/// # Safety -+/// -+/// - Values are provided by the linker script and must be trusted as-is. -+/// - The linker-provided addresses must be u64 aligned. -+pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { -+ let range; -+ unsafe { -+ range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); -+ } -+ assert!(!range.is_empty()); -+ -+ range -+} - diff -uNr 01_wait_forever/src/bsp/raspberrypi.rs 02_runtime_init/src/bsp/raspberrypi.rs --- 01_wait_forever/src/bsp/raspberrypi.rs +++ 02_runtime_init/src/bsp/raspberrypi.rs -@@ -4,4 +4,5 @@ +@@ -4,4 +4,4 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. -// Coming soon. +pub mod cpu; -+pub mod memory; diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs --- 01_wait_forever/src/cpu.rs @@ -316,24 +282,19 @@ diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs --- 01_wait_forever/src/main.rs +++ 02_runtime_init/src/main.rs -@@ -102,14 +102,25 @@ +@@ -102,8 +102,8 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -+//! -+//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html ++//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. -#![feature(asm)] #![feature(global_asm)] #![no_main] #![no_std] - - mod bsp; +@@ -112,4 +112,11 @@ mod cpu; -+mod memory; mod panic_wait; -+mod runtime_init; -// Kernel code coming next tutorial. +/// Early init code. @@ -345,41 +306,6 @@ diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs + panic!() +} -diff -uNr 01_wait_forever/src/memory.rs 02_runtime_init/src/memory.rs ---- 01_wait_forever/src/memory.rs -+++ 02_runtime_init/src/memory.rs -@@ -0,0 +1,30 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! Memory Management. -+ -+use core::ops::RangeInclusive; -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Zero out an inclusive memory range. -+/// -+/// # Safety -+/// -+/// - `range.start` and `range.end` must be valid. -+/// - `range.start` and `range.end` must be `T` aligned. -+pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -+where -+ T: From, -+{ -+ let mut ptr = *range.start(); -+ let end_inclusive = *range.end(); -+ -+ while ptr <= end_inclusive { -+ core::ptr::write_volatile(ptr, T::from(0)); -+ ptr = ptr.offset(1); -+ } -+} - diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs --- 01_wait_forever/src/panic_wait.rs +++ 02_runtime_init/src/panic_wait.rs @@ -396,46 +322,4 @@ diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs + cpu::wait_forever() } -diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.rs ---- 01_wait_forever/src/runtime_init.rs -+++ 02_runtime_init/src/runtime_init.rs -@@ -0,0 +1,37 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! Rust runtime initialization code. -+ -+use crate::{bsp, memory}; -+ -+//-------------------------------------------------------------------------------------------------- -+// Private Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Zero out the .bss section. -+/// -+/// # Safety -+/// -+/// - Must only be called pre `kernel_init()`. -+#[inline(always)] -+unsafe fn zero_bss() { -+ memory::zero_volatile(bsp::memory::bss_range_inclusive()); -+} -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -+/// init code. -+/// -+/// # Safety -+/// -+/// - Only a single core must be active and running this function. -+pub unsafe fn runtime_init() -> ! { -+ zero_bss(); -+ -+ crate::kernel_init() -+} - ``` diff --git a/02_runtime_init/src/_arch/aarch64/cpu/boot.rs b/02_runtime_init/src/_arch/aarch64/cpu/boot.rs index c85bb94b8..7513df074 100644 --- a/02_runtime_init/src/_arch/aarch64/cpu/boot.rs +++ b/02_runtime_init/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/02_runtime_init/src/_arch/aarch64/cpu/boot.s b/02_runtime_init/src/_arch/aarch64/cpu/boot.s index bfa94abf3..f4162c877 100644 --- a/02_runtime_init/src/_arch/aarch64/cpu/boot.s +++ b/02_runtime_init/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/02_runtime_init/src/bsp/raspberrypi.rs b/02_runtime_init/src/bsp/raspberrypi.rs index 10535d7b8..8fed4b83c 100644 --- a/02_runtime_init/src/bsp/raspberrypi.rs +++ b/02_runtime_init/src/bsp/raspberrypi.rs @@ -5,4 +5,3 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. pub mod cpu; -pub mod memory; diff --git a/02_runtime_init/src/bsp/raspberrypi/link.ld b/02_runtime_init/src/bsp/raspberrypi/link.ld index 97ea6d69c..cf63a4a64 100644 --- a/02_runtime_init/src/bsp/raspberrypi/link.ld +++ b/02_runtime_init/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/02_runtime_init/src/bsp/raspberrypi/memory.rs b/02_runtime_init/src/bsp/raspberrypi/memory.rs deleted file mode 100644 index 5c6b1ea5b..000000000 --- a/02_runtime_init/src/bsp/raspberrypi/memory.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! BSP Memory Management. - -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/02_runtime_init/src/main.rs b/02_runtime_init/src/main.rs index faad6b091..cf15402fd 100644 --- a/02_runtime_init/src/main.rs +++ b/02_runtime_init/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![feature(global_asm)] #![no_main] @@ -112,9 +110,7 @@ mod bsp; mod cpu; -mod memory; mod panic_wait; -mod runtime_init; /// Early init code. /// diff --git a/02_runtime_init/src/memory.rs b/02_runtime_init/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/02_runtime_init/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/02_runtime_init/src/runtime_init.rs b/02_runtime_init/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/02_runtime_init/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/03_hacky_hello_world/README.md b/03_hacky_hello_world/README.md index 6958b8bc5..01bbf54cb 100644 --- a/03_hacky_hello_world/README.md +++ b/03_hacky_hello_world/README.md @@ -117,13 +117,12 @@ diff -uNr 02_runtime_init/src/bsp/raspberrypi/console.rs 03_hacky_hello_world/sr diff -uNr 02_runtime_init/src/bsp/raspberrypi.rs 03_hacky_hello_world/src/bsp/raspberrypi.rs --- 02_runtime_init/src/bsp/raspberrypi.rs +++ 03_hacky_hello_world/src/bsp/raspberrypi.rs -@@ -4,5 +4,6 @@ +@@ -4,4 +4,5 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. +pub mod console; pub mod cpu; - pub mod memory; diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs --- 02_runtime_init/src/console.rs @@ -152,9 +151,9 @@ diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs --- 02_runtime_init/src/main.rs +++ 03_hacky_hello_world/src/main.rs -@@ -106,14 +106,18 @@ - //! - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -104,13 +104,17 @@ + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. +#![feature(format_args_nl)] #![feature(global_asm)] @@ -165,13 +164,12 @@ diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs mod bsp; +mod console; mod cpu; - mod memory; mod panic_wait; +mod print; - mod runtime_init; /// Early init code. -@@ -122,5 +126,7 @@ + /// +@@ -118,5 +122,7 @@ /// /// - Only a single core must be active and running this function. unsafe fn kernel_init() -> ! { diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs index c85bb94b8..7513df074 100644 --- a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s index bfa94abf3..f4162c877 100644 --- a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/03_hacky_hello_world/src/bsp/raspberrypi.rs b/03_hacky_hello_world/src/bsp/raspberrypi.rs index fdb7551a1..73863b174 100644 --- a/03_hacky_hello_world/src/bsp/raspberrypi.rs +++ b/03_hacky_hello_world/src/bsp/raspberrypi.rs @@ -6,4 +6,3 @@ pub mod console; pub mod cpu; -pub mod memory; diff --git a/03_hacky_hello_world/src/bsp/raspberrypi/link.ld b/03_hacky_hello_world/src/bsp/raspberrypi/link.ld index 97ea6d69c..cf63a4a64 100644 --- a/03_hacky_hello_world/src/bsp/raspberrypi/link.ld +++ b/03_hacky_hello_world/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/03_hacky_hello_world/src/bsp/raspberrypi/memory.rs b/03_hacky_hello_world/src/bsp/raspberrypi/memory.rs deleted file mode 100644 index 5c6b1ea5b..000000000 --- a/03_hacky_hello_world/src/bsp/raspberrypi/memory.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! BSP Memory Management. - -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/03_hacky_hello_world/src/main.rs b/03_hacky_hello_world/src/main.rs index bc290ddde..de9a05768 100644 --- a/03_hacky_hello_world/src/main.rs +++ b/03_hacky_hello_world/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![feature(format_args_nl)] #![feature(global_asm)] @@ -115,10 +113,8 @@ mod bsp; mod console; mod cpu; -mod memory; mod panic_wait; mod print; -mod runtime_init; /// Early init code. /// diff --git a/03_hacky_hello_world/src/memory.rs b/03_hacky_hello_world/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/03_hacky_hello_world/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/03_hacky_hello_world/src/runtime_init.rs b/03_hacky_hello_world/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/03_hacky_hello_world/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/04_safe_globals/README.md b/04_safe_globals/README.md index 272edfce6..6b7f52c4e 100644 --- a/04_safe_globals/README.md +++ b/04_safe_globals/README.md @@ -224,7 +224,7 @@ diff -uNr 03_hacky_hello_world/src/console.rs 04_safe_globals/src/console.rs diff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs --- 03_hacky_hello_world/src/main.rs +++ 04_safe_globals/src/main.rs -@@ -109,6 +109,7 @@ +@@ -107,6 +107,7 @@ #![feature(format_args_nl)] #![feature(global_asm)] #![feature(panic_info_message)] @@ -232,15 +232,15 @@ diff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs #![no_main] #![no_std] -@@ -119,6 +120,7 @@ +@@ -115,6 +116,7 @@ + mod cpu; mod panic_wait; mod print; - mod runtime_init; +mod synchronization; /// Early init code. /// -@@ -126,7 +128,15 @@ +@@ -122,7 +124,15 @@ /// /// - Only a single core must be active and running this function. unsafe fn kernel_init() -> ! { diff --git a/04_safe_globals/src/_arch/aarch64/cpu/boot.rs b/04_safe_globals/src/_arch/aarch64/cpu/boot.rs index c85bb94b8..7513df074 100644 --- a/04_safe_globals/src/_arch/aarch64/cpu/boot.rs +++ b/04_safe_globals/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/04_safe_globals/src/_arch/aarch64/cpu/boot.s b/04_safe_globals/src/_arch/aarch64/cpu/boot.s index bfa94abf3..f4162c877 100644 --- a/04_safe_globals/src/_arch/aarch64/cpu/boot.s +++ b/04_safe_globals/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/04_safe_globals/src/bsp/raspberrypi.rs b/04_safe_globals/src/bsp/raspberrypi.rs index fdb7551a1..73863b174 100644 --- a/04_safe_globals/src/bsp/raspberrypi.rs +++ b/04_safe_globals/src/bsp/raspberrypi.rs @@ -6,4 +6,3 @@ pub mod console; pub mod cpu; -pub mod memory; diff --git a/04_safe_globals/src/bsp/raspberrypi/link.ld b/04_safe_globals/src/bsp/raspberrypi/link.ld index 97ea6d69c..cf63a4a64 100644 --- a/04_safe_globals/src/bsp/raspberrypi/link.ld +++ b/04_safe_globals/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/04_safe_globals/src/bsp/raspberrypi/memory.rs b/04_safe_globals/src/bsp/raspberrypi/memory.rs deleted file mode 100644 index 5c6b1ea5b..000000000 --- a/04_safe_globals/src/bsp/raspberrypi/memory.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! BSP Memory Management. - -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/04_safe_globals/src/main.rs b/04_safe_globals/src/main.rs index 48687ace2..82262ea17 100644 --- a/04_safe_globals/src/main.rs +++ b/04_safe_globals/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![feature(format_args_nl)] #![feature(global_asm)] @@ -116,10 +114,8 @@ mod bsp; mod console; mod cpu; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; /// Early init code. diff --git a/04_safe_globals/src/memory.rs b/04_safe_globals/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/04_safe_globals/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/04_safe_globals/src/runtime_init.rs b/04_safe_globals/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/04_safe_globals/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/05_drivers_gpio_uart/README.md b/05_drivers_gpio_uart/README.md index b5895cd77..4125583e3 100644 --- a/05_drivers_gpio_uart/README.md +++ b/05_drivers_gpio_uart/README.md @@ -1120,10 +1120,14 @@ diff -uNr 04_safe_globals/src/bsp/raspberrypi/driver.rs 05_drivers_gpio_uart/src diff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs --- 04_safe_globals/src/bsp/raspberrypi/memory.rs +++ 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs -@@ -17,6 +17,38 @@ - } - - //-------------------------------------------------------------------------------------------------- +@@ -0,0 +1,37 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! BSP Memory Management. ++ ++//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + @@ -1154,21 +1158,16 @@ diff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src + pub const PL011_UART_START: usize = START + UART_OFFSET; + } +} -+ -+//-------------------------------------------------------------------------------------------------- - // Public Code - //-------------------------------------------------------------------------------------------------- - diff -uNr 04_safe_globals/src/bsp/raspberrypi.rs 05_drivers_gpio_uart/src/bsp/raspberrypi.rs --- 04_safe_globals/src/bsp/raspberrypi.rs +++ 05_drivers_gpio_uart/src/bsp/raspberrypi.rs -@@ -6,4 +6,33 @@ +@@ -6,3 +6,33 @@ pub mod console; pub mod cpu; +pub mod driver; - pub mod memory; ++pub mod memory; + +//-------------------------------------------------------------------------------------------------- +// Global instances @@ -1321,24 +1320,24 @@ diff -uNr 04_safe_globals/src/driver.rs 05_drivers_gpio_uart/src/driver.rs diff -uNr 04_safe_globals/src/main.rs 05_drivers_gpio_uart/src/main.rs --- 04_safe_globals/src/main.rs +++ 05_drivers_gpio_uart/src/main.rs -@@ -106,6 +106,8 @@ - //! - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -104,6 +104,8 @@ + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. +#![allow(clippy::upper_case_acronyms)] +#![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] #![feature(global_asm)] #![feature(panic_info_message)] -@@ -116,6 +118,7 @@ +@@ -114,6 +116,7 @@ mod bsp; mod console; mod cpu; +mod driver; - mod memory; mod panic_wait; mod print; -@@ -127,16 +130,54 @@ + mod synchronization; +@@ -123,16 +126,54 @@ /// # Safety /// /// - Only a single core must be active and running this function. diff --git a/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs b/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs index c85bb94b8..7513df074 100644 --- a/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs +++ b/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s b/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s index bfa94abf3..f4162c877 100644 --- a/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s +++ b/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld b/05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld index 97ea6d69c..cf63a4a64 100644 --- a/05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld +++ b/05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs b/05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs index c6d65e36b..ef397a6f9 100644 --- a/05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs +++ b/05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/05_drivers_gpio_uart/src/main.rs b/05_drivers_gpio_uart/src/main.rs index d72919118..0261a136e 100644 --- a/05_drivers_gpio_uart/src/main.rs +++ b/05_drivers_gpio_uart/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -119,10 +117,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; /// Early init code. diff --git a/05_drivers_gpio_uart/src/memory.rs b/05_drivers_gpio_uart/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/05_drivers_gpio_uart/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/05_drivers_gpio_uart/src/runtime_init.rs b/05_drivers_gpio_uart/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/05_drivers_gpio_uart/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/06_uart_chainloader/README.md b/06_uart_chainloader/README.md index 44fc10282..18741f043 100644 --- a/06_uart_chainloader/README.md +++ b/06_uart_chainloader/README.md @@ -232,26 +232,36 @@ diff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/ .equ _core_id_mask, 0b11 //-------------------------------------------------------------------------------------------------- -@@ -34,20 +45,31 @@ - and x1, x1, _core_id_mask - ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs - cmp x1, x2 -- b.ne 1f -+ b.ne 2f -+ -+ // If execution reaches here, it is the boot core. +@@ -39,23 +50,35 @@ + // If execution reaches here, it is the boot core. + + // Initialize DRAM. +- ADR_REL x0, __bss_start +- ADR_REL x1, __bss_end_exclusive ++ ADR_ABS x0, __bss_start ++ ADR_ABS x1, __bss_end_exclusive + + bss_init_loop: + cmp x0, x1 +- b.eq prepare_rust ++ b.eq relocate_binary + stp xzr, xzr, [x0], #16 + b bss_init_loop -- // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // Next, relocate the binary. ++relocate_binary: + ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to. + ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to. + ADR_ABS x2, __binary_nonzero_end_exclusive + -+1: ldr x3, [x0], #8 ++copy_loop: ++ ldr x3, [x0], #8 + str x3, [x1], #8 + cmp x1, x2 -+ b.lo 1b - ++ b.lo copy_loop ++ + // Prepare the jump to Rust code. +-prepare_rust: // Set the stack pointer. - ADR_REL x0, __boot_core_stack_end_exclusive + ADR_ABS x0, __boot_core_stack_end_exclusive @@ -264,13 +274,7 @@ diff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/ + br x1 // Infinitely wait for events (aka "park the core"). --1: wfe -- b 1b -+2: wfe -+ b 2b - - .size _start, . - _start - .type _start, function + parking_loop: diff -uNr 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs --- 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs @@ -350,7 +354,7 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 06_uart_chainloader/s .text : { KEEP(*(.text._start)) -@@ -42,8 +44,12 @@ +@@ -42,6 +44,10 @@ ***********************************************************************************************/ .data : { *(.data*) } :segment_rw @@ -358,17 +362,14 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 06_uart_chainloader/s + . = ALIGN(8); + __binary_nonzero_end_exclusive = .; + - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -- .bss : ALIGN(8) -+ .bss : + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { - __bss_start = .; - *(.bss*); diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader/src/bsp/raspberrypi/memory.rs --- 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs +++ 06_uart_chainloader/src/bsp/raspberrypi/memory.rs -@@ -23,9 +23,10 @@ +@@ -11,9 +11,10 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -381,34 +382,33 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader /// Physical devices. #[cfg(feature = "bsp_rpi3")] -@@ -52,7 +53,13 @@ - // Public Code - //-------------------------------------------------------------------------------------------------- - --/// Return the inclusive range spanning the .bss section. +@@ -35,3 +36,13 @@ + pub const PL011_UART_START: usize = START + UART_OFFSET; + } + } ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ +/// The address on which the Raspberry firmware loads every binary by default. +#[inline(always)] +pub fn board_default_load_addr() -> *const u64 { + map::BOARD_DEFAULT_LOAD_ADDRESS as _ +} -+ -+/// Return the inclusive range spanning the relocated .bss section. - /// - /// # Safety - /// diff -uNr 05_drivers_gpio_uart/src/main.rs 06_uart_chainloader/src/main.rs --- 05_drivers_gpio_uart/src/main.rs +++ 06_uart_chainloader/src/main.rs -@@ -107,6 +107,7 @@ - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -105,6 +105,7 @@ + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] +#![feature(asm)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] #![feature(global_asm)] -@@ -146,38 +147,56 @@ +@@ -142,38 +143,56 @@ kernel_main() } diff --git a/06_uart_chainloader/demo_payload_rpi3.img b/06_uart_chainloader/demo_payload_rpi3.img index a8d7f680ef624efc1753bb890807835d880967e8..4e640c636357f92152d3c180e30879ff3abcb130 100755 GIT binary patch delta 1586 zcmZ8hZ%kWN6hHU97TOL+|Cg3QTMF136aEM*+l)NA;1<-VjOkG0S}<{mA5iwi55^Xk z2opZ^dReB%m=Qm~YfKy#lVPHPEiwAZEGU`ZUcnCq5+kC+s_;Db^^zEF@_OI-{m%K_ zbIv`l6UXN|I%c5jr?5$RKOw<|ZjiQS2n+zgO<0Ojf_oVEAdsyjz(DY=vAI26_fdDF zv>?x)bV{DDOfb#ev~{52B@bPbfsW);QX`QvUN$bjz&xf_smGT;W9lPPDw1dBUx_;v zp|NEzEk5{-s#OwI!=RP?waQjAS*rf(6+l*tfNH@r3P*^#bsFeY z+X-q(1=#x0K1mC(mTLb+H_|(DFgHEaOw;ecHke>wLUK()lcGi)N(o5!>g5E`kghLq zT&qOMrnDM+Kq+amB^N^8iW%P(A%%enB0kviVUZ1H)cimYY%7`>c`())M> zc8r~}$7=WX_3N4wI7km=ujE|LubVvKKC=~B#%NTwj;}(ryHVFlij(Z3rc?iOkYB5SJ!{j#wGP?15obAb1Pi| zx+~e~-=6}VP6Um^&Y~|r|BR!qDigy!b(dOta^_hX%9PBw~iwCXO>cBaLNV*f= zc!Ep@svPViXZ&O`LK912I;=F%0-m#ry8JHnJLK;EBT(8#djj*Z{4zV5se}a^p0j~D z^-&kgyFPK^ut)u{$WuRSq}{}Z-C-%X&)#*<{+~693P)JC$6JMiU&K1r?f@;al*c{D zlL$4^;Tw&#fZ&@V3H$=H1P1|>X1XL_T-9*43;6JN0HyTOzV5R$Dk?T8$LF^oTXPwX zVcdJBOuyC4es`XF*xXwRUW3Kphp@;aZv!k8Wyr{=J+g1;B&FK07=aQ`8@x`(d zD&Tqr%KHUjCr&>VdgjU2wzkmuo__p)m|$VAk4&)>-X8|<)s%gWK>YFZEO1H2FF^0; z=*TF(%L4Zb{IK6&-g4X#!H=_)uOYNNBYrki^EOGcXZ=2#y}F#;Tn!ub zAD71bEa!i$%(WkIuZs#CLXhtYK1qRB3mk>p1CNI-fwJT+LED9`3EQ~n_V3(fuX|!o WiVLq1Vr3TKW4ZkHnatJ#&Hn)+exp4A delta 1879 zcmZuxZEO=|9RELe*Y!4#wcWb5gRNKh)`NsG+1Nzmj^T`%7;w(07{xKCEIJr1s7naf z#+ZOErVlqICPX*=Ft4A)L1fVwR)S`JA@~9xOiVld&~5Pr7Z(p?=l{7rGiLEgo<8?~ z|NYS!yeMHhubr5)>9fXB(fF9ahD`cdzdwqicM+rs=z{lC2fJPDlIx7M_ zHklM^iJ%3I332aMkz6fd3U1NNUjnK^qPr_sPqV0)6NR*`6Eoic=DaSLKLCs>pnP{q zY7$6ja;wxugk1fk?)sIKqluaf(tN`gX4XwiW^PX=-CJCwZ}OA18A7fYN7vn3O`7Oh zfK;{yOc|r*)(~@HC(x1P-BgZjFs`iY62>+d>q|pApcoQ0lq7)O#K4OpN$UbEfCN2q zt!9>M2-d4vSy4_ar2Sf6gmhLcPv@8Ib9pzIi92;uzC*JBWyR<#{~Dp{bOoJ9WLbBS z$*L03s%RF&=NlyRK}>wU!DTXCb$Y{kYGcN8HN>0~37xCK>fE+Zv+Ofqd|6u8obqEH z@{VeHK2t%_C{>16ovx zN9r9L)*jZTS&X-IU2v+-Kr!F;w=PpnhH2iG%qa2}ClF67SeWeTga=xfP6b{To5@w8 z&}*~xPRm&u@3NLU7&yjA|sSOz7~`V{iAVp;kYS|yFAy+HvLUhzi46U@B~tp|!Ui$PO&C{Bu9 z+{vfjx@X=&9jypjnfzba)5`xrHf`NyF6l+)NwgW03Z*L^?zC75H-@ySc~Dy0i8fV| z?6kUEKueOwc3-0(sTZrjH-a3!1r9$%-mF}kG>-dPBiVY|q1Dm8i*-2Ug#L+sS!C-vPUMAR0zEjbDFH8JRoaUl zIuaXc)!zlLP6e3m>+l!TpRw(pUwB0O61lp(6|BohpTO$KKTpIGZa9Z4nR3rz=eou# zjNg3k`EWR*A(&#mSS!?I)Of!vA}roEGG!C`f22_+9mfT(0Xpvn%hn|vW09>}O2#eC zV=ZSd0sS1uS+nfs7389W9_upJK>L&^&7saNj*{sR996ieQdWhm7*6N#vTp#*YT2@E z2iQ{LoZ%A6C@fC*ATI-OjxDBIuC;YYaHH4{ZR40eGt`X3{(w3pATT`lL*Z@XbjQwz zl|8$*G&L!Y?dZnxo-xk)17ysY@n3oJLa3nY7~;>#K!L^vIHxlfI~2pOIOpSLIA^uq z0WM#|4{DjN0c!Gbpz=YF$_zRYQf%V#)^t#Quo%7Q%(wR6sMxrfuM<>OqAyU5O? z@D^v_ABuBU8@$TpD`^dJx`^w)%;}YK?5;Ha2po^(IWFMU`<>HKPCNbZj`$1c&+u0> za-o3tm{{PB@E3K1fs)$N0{2A7I2GI`^i~>Q2k#?}?5-8r9v;QZ!Le6km-9VIy#5{dO5e4&5yKzv{S;LwrbsOlJ8;O14_yaQj>uz0Mm vaesWMG1eO&+TS0G4G#@84)zZoiVbg$#ZfM=FF;;-0hD&fg~xe~K&AH|E4Bxf diff --git a/06_uart_chainloader/demo_payload_rpi4.img b/06_uart_chainloader/demo_payload_rpi4.img index 2d181336afb87cade1e96e18297d2750aef5ff8e..7e017a42beb30832cd2c7beb0d8e180310445b27 100755 GIT binary patch delta 1403 zcmZ8gU1%It6h8OPZZb_{CbK)UGrQ(zW)pXtDw@`mv_;JBCTfipO01{|bwP`Y50(_A zqI7qq5JW9|dn?rkwIMz@c~aLt6ft&FeTWZMt+o$&=wyWwT@XtoH9|94rNZ|C_Fg=v_4JXAOL&@#S|kbZIr!0>Un^Jy>FC0z2$iZ z)B5Qb65IPddA<@RX`+7^g42O=l)1kFt4Y9K;aS>8l*(bM5t`BX>(U{9TWF9Hy)qIS zC)BnpZ-@URnr*XzlE#+PbB^0e92ZS5SKe(M43mqMx%PQN-s7c?`v~K69Y%|~hsn0d z7dyu3S(B%gBu)17N0eNos-lOGvy9el81;D7}oELRikMJP@*oO^|^vx*n&L%iD8;$X&Q zyBKAE&j54rt4V~duX|_Q2<-B0nZ*`J!xo&?tF&=XSuWkqo?e-eoQKiL?;bk69Co%f zE*OTjro*6wgw@emu@iQ!NFo^0#RgC}hK{ZzZ9ii_sKQ1t{8DEsR=b0_k~VuC;IunM zotvKb*Ug&3x2g$ByLnpep~WbFNbSa=9#-#a0IsHJwu^riP>X6JQ4Oxz$oqd+D3MrC zb<8i%_IT#}A7BlSkj(kEg|W5qFxGz|#78508}J>3?FgqLcbk^YD8`*D_A&|OvtY3$ zEZ9Fh8;u$_FjHB2b3*?1n@Ot$zTAQucTTJPrRZ^5P4arQryUFJ&Jb3+2y8powG)LU z%P+=qS_&$QgX9%(1h7jRh3vzwpusx!|Lnis5Ds=3nt`}>5J*1BmQd#;hR~Q1fXv!! z>z;w#oRvcv97uWgBgDQf^fQ7Ty&yevg`0YNtKZVE$M|BToA>KoVN`-V&+5j&5sDnH z=OB=~1 zQE`rQ;IGYkF9?3C;HP*sc2D7|5qR^1MCRvB_&b3^!fHWa?1jIsiRiGn{yHc4PJuV` z(}LFm{xLWy4)BfnYqP;I!8g-7B5;e)e^uaSIj)&}-Z)P3{BzUTA}c|5CjZGCrZ-I9 z7Qa8Bb~Y(jB%lg^4Vl63M{%x*bM$*E{-Av<9(dd)WKqZiLi&B)S=0ipvXFVvZ&Nr6 PSbT|6u=_;fxkTzek9%Fk delta 1731 zcmZ8hUrbw782`?_Ewl{Rmj17_w*2erl=9D1h>=pnY(x{R82?5nvpS3nY)K3+DKA78=kyIa0gD^~Vz9`v56JJ^ulRA>QWXVR+asBSSr#^6!+wXjTe&6?< z-*-+gw--BX%kbo{0bTTy7=c&1LD*de7^EEy!itW*_7kDGMiM0UH61JxfRzB)Y5peA z9TDg(O9=I(Gj~p~5n+;7s*x?hUF_oskjL=1k>^UL<7xZctI4ha)wC7AV{|| zg7gQ#q96P>*q+!b%s6t#b^Sz0G~_iq#aXm{TVe zF`v?Be~*Vu>8xK=E27R@IpqJGO3kFh)dm>q}YC0C8 zcb!ne$_s{`T4pl)1hBKP+2h!%r`O1ITsi8!-vp&MM0(kf7LhS~>SnwuIS%w}k2I)F z$r-(LFH<2Q&kM*i`^MaqsS6=@wya2CF@Ii=?#uF>`)y{r{~^%*A&^R+11(`Y7wUl8 z5Vkn$qs8l@bQ*at$~sB@TQ8kZ_7N!Yw0wMot{upi@>LLd9sU1Up_CV`*|~qwhIQ23 zn3oM(yvms}mL_4t%f7Jq-c>FDR^1T^{S-4qD9p!uxTZ@{=TWUjHC@x38E&8gB4g`yR780y zT95aN&^^qm3{EAwv9Dy*M^``6O~+5#ZS6)qtQsL3H~tB$>*CP18so(n#D@ZXrEX%_ zH4+#E)L}TN<%eT5f%wOf4<&CoHqw0#_7uQ1Ssr}Aiz-RwO|o^+4y7AtGl;Ws!}s5% z{1xz}0zAH9WjAZiK1Q18No5v65!t~zSg*a0Vo+ZmIWkM=&L+@@*vHnnOaLDN2dGWs zqfmi(PyQb!CQ$$dytzyJa))*ZPLjV-BWbCT7BJ?0-Y-D3^u~eg#BpQ|x+p#_Lv{vr z6&npwf;U$sDla~gE#Sw#wfRYu6>QGXfPkIgdImJ<;(Dz2dH=w3Ev>EomwMtzwEz67%dN$#K>Yd3q3JkCb@ioE<0;%-=XQJ(>N*_vBljQS{);T(2xlTLEt{TO z+YafCT-R}bi0e2RbzR}zA*1TDbHADEhy8^6@wqAMMY!$M8o<82}?G96dkY{YpsaKfXd9p?20xPF1_>O3q4yB6i*GRj_XJ1vu1->zRyY|J-kTz$_$_g}iQBQqtFC7=iF7)dnj9TJ<9|Cb`d;#_ zE`K1EIP*@jc{Dwg9GjS&Iu=p^F5bJB!(%9wd8JdQ!XxR4a4M0W7)hp5Qxl`%vE ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/06_uart_chainloader/src/_arch/aarch64/cpu/boot.s b/06_uart_chainloader/src/_arch/aarch64/cpu/boot.s index 10aebb343..c4ae4219e 100644 --- a/06_uart_chainloader/src/_arch/aarch64/cpu/boot.s +++ b/06_uart_chainloader/src/_arch/aarch64/cpu/boot.s @@ -45,20 +45,33 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 2f + b.ne parking_loop // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_ABS x0, __bss_start + ADR_ABS x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq relocate_binary + stp xzr, xzr, [x0], #16 + b bss_init_loop + // Next, relocate the binary. +relocate_binary: ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to. ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to. ADR_ABS x2, __binary_nonzero_end_exclusive -1: ldr x3, [x0], #8 +copy_loop: + ldr x3, [x0], #8 str x3, [x1], #8 cmp x1, x2 - b.lo 1b + b.lo copy_loop + // Prepare the jump to Rust code. // Set the stack pointer. ADR_ABS x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -68,8 +81,9 @@ _start: br x1 // Infinitely wait for events (aka "park the core"). -2: wfe - b 2b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/06_uart_chainloader/src/bsp/raspberrypi/link.ld b/06_uart_chainloader/src/bsp/raspberrypi/link.ld index 324172b19..40added4a 100644 --- a/06_uart_chainloader/src/bsp/raspberrypi/link.ld +++ b/06_uart_chainloader/src/bsp/raspberrypi/link.ld @@ -48,14 +48,12 @@ SECTIONS . = ALIGN(8); __binary_nonzero_end_exclusive = .; - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/06_uart_chainloader/src/bsp/raspberrypi/memory.rs b/06_uart_chainloader/src/bsp/raspberrypi/memory.rs index 9a66cc8ac..852b89050 100644 --- a/06_uart_chainloader/src/bsp/raspberrypi/memory.rs +++ b/06_uart_chainloader/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -58,19 +46,3 @@ pub(super) mod map { pub fn board_default_load_addr() -> *const u64 { map::BOARD_DEFAULT_LOAD_ADDRESS as _ } - -/// Return the inclusive range spanning the relocated .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/06_uart_chainloader/src/main.rs b/06_uart_chainloader/src/main.rs index aed10f7a8..5c45cb5f0 100644 --- a/06_uart_chainloader/src/main.rs +++ b/06_uart_chainloader/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(asm)] @@ -120,10 +118,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; /// Early init code. diff --git a/06_uart_chainloader/src/memory.rs b/06_uart_chainloader/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/06_uart_chainloader/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/06_uart_chainloader/src/runtime_init.rs b/06_uart_chainloader/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/06_uart_chainloader/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/07_timestamps/README.md b/07_timestamps/README.md index 88b829086..25d08f847 100644 --- a/07_timestamps/README.md +++ b/07_timestamps/README.md @@ -154,26 +154,36 @@ diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_ar .equ _core_id_mask, 0b11 //-------------------------------------------------------------------------------------------------- -@@ -45,31 +34,20 @@ - and x1, x1, _core_id_mask - ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs - cmp x1, x2 -- b.ne 2f -- -- // If execution reaches here, it is the boot core. -+ b.ne 1f +@@ -50,35 +39,23 @@ + // If execution reaches here, it is the boot core. + + // Initialize DRAM. +- ADR_ABS x0, __bss_start +- ADR_ABS x1, __bss_end_exclusive ++ ADR_REL x0, __bss_start ++ ADR_REL x1, __bss_end_exclusive + + bss_init_loop: + cmp x0, x1 +- b.eq relocate_binary ++ b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop - // Next, relocate the binary. +-relocate_binary: - ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to. - ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to. - ADR_ABS x2, __binary_nonzero_end_exclusive - --1: ldr x3, [x0], #8 +-copy_loop: +- ldr x3, [x0], #8 - str x3, [x1], #8 - cmp x1, x2 -- b.lo 1b -+ // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. - +- b.lo copy_loop +- + // Prepare the jump to Rust code. ++prepare_rust: // Set the stack pointer. - ADR_ABS x0, __boot_core_stack_end_exclusive + ADR_REL x0, __boot_core_stack_end_exclusive @@ -186,13 +196,7 @@ diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_ar + b _start_rust // Infinitely wait for events (aka "park the core"). --2: wfe -- b 2b -+1: wfe -+ b 1b - - .size _start, . - _start - .type _start, function + parking_loop: diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu.rs 07_timestamps/src/_arch/aarch64/cpu.rs --- 06_uart_chainloader/src/_arch/aarch64/cpu.rs @@ -438,7 +442,7 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/link.ld 07_timestamps/src/bsp/ .text : { KEEP(*(.text._start)) -@@ -44,12 +42,8 @@ +@@ -44,10 +42,6 @@ ***********************************************************************************************/ .data : { *(.data*) } :segment_rw @@ -446,17 +450,14 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/link.ld 07_timestamps/src/bsp/ - . = ALIGN(8); - __binary_nonzero_end_exclusive = .; - - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -- .bss : -+ .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { - __bss_start = .; - *(.bss*); diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bsp/raspberrypi/memory.rs --- 06_uart_chainloader/src/bsp/raspberrypi/memory.rs +++ 07_timestamps/src/bsp/raspberrypi/memory.rs -@@ -23,10 +23,9 @@ +@@ -11,10 +11,9 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -469,21 +470,20 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bs /// Physical devices. #[cfg(feature = "bsp_rpi3")] -@@ -53,13 +52,7 @@ - // Public Code - //-------------------------------------------------------------------------------------------------- - +@@ -36,13 +35,3 @@ + pub const PL011_UART_START: usize = START + UART_OFFSET; + } + } +- +-//-------------------------------------------------------------------------------------------------- +-// Public Code +-//-------------------------------------------------------------------------------------------------- +- -/// The address on which the Raspberry firmware loads every binary by default. -#[inline(always)] -pub fn board_default_load_addr() -> *const u64 { - map::BOARD_DEFAULT_LOAD_ADDRESS as _ -} -- --/// Return the inclusive range spanning the relocated .bss section. -+/// Return the inclusive range spanning the .bss section. - /// - /// # Safety - /// diff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs --- 06_uart_chainloader/src/cpu.rs @@ -499,23 +499,23 @@ diff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs diff -uNr 06_uart_chainloader/src/main.rs 07_timestamps/src/main.rs --- 06_uart_chainloader/src/main.rs +++ 07_timestamps/src/main.rs -@@ -107,7 +107,6 @@ - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -105,7 +105,6 @@ + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] -#![feature(asm)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] #![feature(global_asm)] -@@ -125,6 +124,7 @@ +@@ -121,6 +120,7 @@ + mod panic_wait; mod print; - mod runtime_init; mod synchronization; +mod time; /// Early init code. /// -@@ -147,56 +147,38 @@ +@@ -143,56 +143,38 @@ kernel_main() } diff --git a/07_timestamps/src/_arch/aarch64/cpu/boot.rs b/07_timestamps/src/_arch/aarch64/cpu/boot.rs index c85bb94b8..7513df074 100644 --- a/07_timestamps/src/_arch/aarch64/cpu/boot.rs +++ b/07_timestamps/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/07_timestamps/src/_arch/aarch64/cpu/boot.s b/07_timestamps/src/_arch/aarch64/cpu/boot.s index bfa94abf3..f4162c877 100644 --- a/07_timestamps/src/_arch/aarch64/cpu/boot.s +++ b/07_timestamps/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/07_timestamps/src/bsp/raspberrypi/link.ld b/07_timestamps/src/bsp/raspberrypi/link.ld index 97ea6d69c..cf63a4a64 100644 --- a/07_timestamps/src/bsp/raspberrypi/link.ld +++ b/07_timestamps/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/07_timestamps/src/bsp/raspberrypi/memory.rs b/07_timestamps/src/bsp/raspberrypi/memory.rs index c6d65e36b..ef397a6f9 100644 --- a/07_timestamps/src/bsp/raspberrypi/memory.rs +++ b/07_timestamps/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/07_timestamps/src/main.rs b/07_timestamps/src/main.rs index 54b69275c..1e6bf16a4 100644 --- a/07_timestamps/src/main.rs +++ b/07_timestamps/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -119,10 +117,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/07_timestamps/src/memory.rs b/07_timestamps/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/07_timestamps/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/07_timestamps/src/runtime_init.rs b/07_timestamps/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/07_timestamps/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs b/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs index c85bb94b8..7513df074 100644 --- a/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs +++ b/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s b/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s index bfa94abf3..f4162c877 100644 --- a/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s +++ b/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld b/08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld index 97ea6d69c..cf63a4a64 100644 --- a/08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld +++ b/08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs b/08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs index c6d65e36b..ef397a6f9 100644 --- a/08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs +++ b/08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/08_hw_debug_JTAG/src/main.rs b/08_hw_debug_JTAG/src/main.rs index 54b69275c..1e6bf16a4 100644 --- a/08_hw_debug_JTAG/src/main.rs +++ b/08_hw_debug_JTAG/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -119,10 +117,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/08_hw_debug_JTAG/src/memory.rs b/08_hw_debug_JTAG/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/08_hw_debug_JTAG/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/08_hw_debug_JTAG/src/runtime_init.rs b/08_hw_debug_JTAG/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/08_hw_debug_JTAG/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/09_privilege_level/README.md b/09_privilege_level/README.md index 869c39600..7b4c0d3c0 100644 --- a/09_privilege_level/README.md +++ b/09_privilege_level/README.md @@ -62,7 +62,7 @@ Afterwards, we continue with preparing the `EL2` -> `EL1` transition by calling pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } ``` @@ -125,19 +125,16 @@ SPSR_EL2.write( + SPSR_EL2::M::EL1h, ); -// Second, let the link register point to runtime_init(). -ELR_EL2.set(runtime_init::runtime_init as *const () as u64); +// Second, let the link register point to kernel_init(). +ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); ``` -As you can see, we are populating `ELR_EL2` with the address of the [runtime_init()] function that -we earlier used to call directly from the entrypoint. Finally, we set the stack pointer for -`SP_EL1`. - -[runtime_init()]: src/runtime_init.rs +As you can see, we are populating `ELR_EL2` with the address of the `kernel_init()` function that we +earlier used to call directly from the entrypoint. Finally, we set the stack pointer for `SP_EL1`. You might have noticed that the stack's address was supplied as a function argument. As you might remember, in `_start()` in `boot.s`, we are already setting up the stack for `EL2`. Since there @@ -151,7 +148,7 @@ Lastly, back in `_start_rust()` a call to `ERET` is made: pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } ``` @@ -214,12 +211,12 @@ diff -uNr 08_hw_debug_JTAG/Cargo.toml 09_privilege_level/Cargo.toml diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/_arch/aarch64/cpu/boot.rs --- 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs +++ 09_privilege_level/src/_arch/aarch64/cpu/boot.rs -@@ -12,11 +12,53 @@ +@@ -11,17 +11,67 @@ + //! //! crate::cpu::boot::arch_boot - use crate::runtime_init; +use cortex_a::{asm, regs::*}; - ++ // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -256,8 +253,8 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/ + + SPSR_EL2::M::EL1h, + ); + -+ // Second, let the link register point to runtime_init(). -+ ELR_EL2.set(runtime_init::runtime_init as *const () as u64); ++ // Second, let the link register point to kernel_init(). ++ ELR_EL2.set(crate::kernel_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. @@ -268,18 +265,20 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/ // Public Code //-------------------------------------------------------------------------------------------------- -@@ -27,7 +69,11 @@ - /// # Safety + /// The Rust entry of the `kernel` binary. /// - /// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -+/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. + /// The function is called from the assembly `_start` function. ++/// ++/// # Safety ++/// ++/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] -pub unsafe fn _start_rust() -> ! { -- runtime_init::runtime_init() +- crate::kernel_init() +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + -+ // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. ++ // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. + asm::eret() } @@ -301,15 +300,15 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s 09_privilege_level/src/_ + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 -+ b.ne 1f ++ b.ne parking_loop + // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask -@@ -38,11 +44,11 @@ - - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. +@@ -50,11 +56,11 @@ + // Prepare the jump to Rust code. + prepare_rust: - // Set the stack pointer. + // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive @@ -499,15 +498,15 @@ diff -uNr 08_hw_debug_JTAG/src/exception.rs 09_privilege_level/src/exception.rs diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs --- 08_hw_debug_JTAG/src/main.rs +++ 09_privilege_level/src/main.rs -@@ -119,6 +119,7 @@ +@@ -117,6 +117,7 @@ mod console; mod cpu; mod driver; +mod exception; - mod memory; mod panic_wait; mod print; -@@ -149,6 +150,8 @@ + mod synchronization; +@@ -145,6 +146,8 @@ /// The main function running after the early init. fn kernel_main() -> ! { @@ -516,7 +515,7 @@ diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs use core::time::Duration; use driver::interface::DriverManager; use time::interface::TimeManager; -@@ -160,6 +163,12 @@ +@@ -156,6 +159,12 @@ ); info!("Booting on: {}", bsp::board_name()); @@ -529,7 +528,7 @@ diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs info!( "Architectural timer resolution: {} ns", time::time_manager().resolution().as_nanos() -@@ -174,11 +183,15 @@ +@@ -170,11 +179,15 @@ info!(" {}. {}", i + 1, driver.compatible()); } diff --git a/09_privilege_level/src/_arch/aarch64/cpu/boot.rs b/09_privilege_level/src/_arch/aarch64/cpu/boot.rs index 02798fdc0..b743418e6 100644 --- a/09_privilege_level/src/_arch/aarch64/cpu/boot.rs +++ b/09_privilege_level/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/09_privilege_level/src/_arch/aarch64/cpu/boot.s b/09_privilege_level/src/_arch/aarch64/cpu/boot.s index 5696220d0..d1666919a 100644 --- a/09_privilege_level/src/_arch/aarch64/cpu/boot.s +++ b/09_privilege_level/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/09_privilege_level/src/bsp/raspberrypi/link.ld b/09_privilege_level/src/bsp/raspberrypi/link.ld index 97ea6d69c..cf63a4a64 100644 --- a/09_privilege_level/src/bsp/raspberrypi/link.ld +++ b/09_privilege_level/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/09_privilege_level/src/bsp/raspberrypi/memory.rs b/09_privilege_level/src/bsp/raspberrypi/memory.rs index c6d65e36b..ef397a6f9 100644 --- a/09_privilege_level/src/bsp/raspberrypi/memory.rs +++ b/09_privilege_level/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/09_privilege_level/src/main.rs b/09_privilege_level/src/main.rs index 5cede7f98..74902def1 100644 --- a/09_privilege_level/src/main.rs +++ b/09_privilege_level/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -120,10 +118,8 @@ mod console; mod cpu; mod driver; mod exception; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/09_privilege_level/src/memory.rs b/09_privilege_level/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/09_privilege_level/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/09_privilege_level/src/runtime_init.rs b/09_privilege_level/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/09_privilege_level/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/10_virtual_mem_part1_identity_mapping/README.md b/10_virtual_mem_part1_identity_mapping/README.md index 75ff9c05c..f573de03a 100644 --- a/10_virtual_mem_part1_identity_mapping/README.md +++ b/10_virtual_mem_part1_identity_mapping/README.md @@ -939,26 +939,28 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 10_virtual_mem_pa diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs --- 09_privilege_level/src/bsp/raspberrypi/memory.rs +++ 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs -@@ -4,6 +4,8 @@ +@@ -4,6 +4,20 @@ //! BSP Memory Management. +pub mod mmu; + - use core::{cell::UnsafeCell, ops::RangeInclusive}; - - //-------------------------------------------------------------------------------------------------- -@@ -12,6 +14,9 @@ - - // Symbols from the linker script. - extern "Rust" { ++use core::cell::UnsafeCell; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++// Symbols from the linker script. ++extern "Rust" { + static __rx_start: UnsafeCell<()>; + static __rx_end_exclusive: UnsafeCell<()>; ++} + - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; - } -@@ -23,6 +28,20 @@ + //-------------------------------------------------------------------------------------------------- + // Public Definitions + //-------------------------------------------------------------------------------------------------- +@@ -11,6 +25,20 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -979,7 +981,7 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_ pub const GPIO_OFFSET: usize = 0x0020_0000; pub const UART_OFFSET: usize = 0x0020_1000; -@@ -35,6 +54,7 @@ +@@ -23,6 +51,7 @@ pub const START: usize = 0x3F00_0000; pub const GPIO_START: usize = START + GPIO_OFFSET; pub const PL011_UART_START: usize = START + UART_OFFSET; @@ -987,15 +989,15 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_ } /// Physical devices. -@@ -45,10 +65,35 @@ +@@ -33,5 +62,30 @@ pub const START: usize = 0xFE00_0000; pub const GPIO_START: usize = START + GPIO_OFFSET; pub const PL011_UART_START: usize = START + UART_OFFSET; + pub const END_INCLUSIVE: usize = 0xFF84_FFFF; } } - - //-------------------------------------------------------------------------------------------------- ++ ++//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + @@ -1018,11 +1020,6 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_ +fn rx_end_exclusive() -> usize { + unsafe { __rx_end_exclusive.get() as usize } +} -+ -+//-------------------------------------------------------------------------------------------------- - // Public Code - //-------------------------------------------------------------------------------------------------- - diff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/src/bsp.rs --- 09_privilege_level/src/bsp.rs @@ -1040,8 +1037,8 @@ diff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/sr diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/src/main.rs --- 09_privilege_level/src/main.rs +++ 10_virtual_mem_part1_identity_mapping/src/main.rs -@@ -107,7 +107,11 @@ - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -105,7 +105,11 @@ + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] +#![allow(incomplete_features)] @@ -1052,7 +1049,15 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s #![feature(format_args_nl)] #![feature(global_asm)] #![feature(panic_info_message)] -@@ -132,9 +136,17 @@ +@@ -118,6 +122,7 @@ + mod cpu; + mod driver; + mod exception; ++mod memory; + mod panic_wait; + mod print; + mod synchronization; +@@ -128,9 +133,17 @@ /// # Safety /// /// - Only a single core must be active and running this function. @@ -1071,7 +1076,7 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s for i in bsp::driver::driver_manager().all_device_drivers().iter() { if let Err(x) = i.init() { -@@ -163,6 +175,9 @@ +@@ -159,6 +172,9 @@ ); info!("Booting on: {}", bsp::board_name()); @@ -1081,7 +1086,7 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s let (_, privilege_level) = exception::current_privilege_level(); info!("Current privilege level: {}", privilege_level); -@@ -186,6 +201,13 @@ +@@ -182,6 +198,13 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); @@ -1387,14 +1392,13 @@ diff -uNr 09_privilege_level/src/memory/mmu.rs 10_virtual_mem_part1_identity_map diff -uNr 09_privilege_level/src/memory.rs 10_virtual_mem_part1_identity_mapping/src/memory.rs --- 09_privilege_level/src/memory.rs +++ 10_virtual_mem_part1_identity_mapping/src/memory.rs -@@ -4,6 +4,8 @@ - - //! Memory Management. - -+pub mod mmu; +@@ -0,0 +1,7 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter + - use core::ops::RangeInclusive; - - //-------------------------------------------------------------------------------------------------- ++//! Memory Management. ++ ++pub mod mmu; ``` diff --git a/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs b/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs index 02798fdc0..b743418e6 100644 --- a/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs +++ b/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s b/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s index 5696220d0..d1666919a 100644 --- a/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s +++ b/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld b/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld index bda0da5e9..485ba49b0 100644 --- a/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld +++ b/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld @@ -46,14 +46,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs b/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs index 81233775d..48b1d603a 100644 --- a/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs +++ b/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs @@ -6,7 +6,7 @@ pub mod mmu; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -92,23 +89,3 @@ fn rx_start() -> usize { fn rx_end_exclusive() -> usize { unsafe { __rx_end_exclusive.get() as usize } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/10_virtual_mem_part1_identity_mapping/src/main.rs b/10_virtual_mem_part1_identity_mapping/src/main.rs index 35f41c0e7..3a1958d6a 100644 --- a/10_virtual_mem_part1_identity_mapping/src/main.rs +++ b/10_virtual_mem_part1_identity_mapping/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -127,7 +125,6 @@ mod exception; mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/10_virtual_mem_part1_identity_mapping/src/memory.rs b/10_virtual_mem_part1_identity_mapping/src/memory.rs index e8cc752f5..7959c555e 100644 --- a/10_virtual_mem_part1_identity_mapping/src/memory.rs +++ b/10_virtual_mem_part1_identity_mapping/src/memory.rs @@ -5,28 +5,3 @@ //! Memory Management. pub mod mmu; - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/10_virtual_mem_part1_identity_mapping/src/runtime_init.rs b/10_virtual_mem_part1_identity_mapping/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/10_virtual_mem_part1_identity_mapping/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/11_exceptions_part1_groundwork/README.md b/11_exceptions_part1_groundwork/README.md index f9c70824b..77a45ec6d 100644 --- a/11_exceptions_part1_groundwork/README.md +++ b/11_exceptions_part1_groundwork/README.md @@ -967,7 +967,7 @@ diff -uNr 10_virtual_mem_part1_identity_mapping/src/exception.rs 11_exceptions_p diff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_groundwork/src/main.rs --- 10_virtual_mem_part1_identity_mapping/src/main.rs +++ 11_exceptions_part1_groundwork/src/main.rs -@@ -144,6 +144,8 @@ +@@ -141,6 +141,8 @@ use driver::interface::DriverManager; use memory::mmu::interface::MMU; @@ -976,7 +976,7 @@ diff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_ if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() { panic!("MMU: {}", string); } -@@ -201,13 +203,28 @@ +@@ -198,13 +200,28 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); diff --git a/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs b/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs index 02798fdc0..b743418e6 100644 --- a/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs +++ b/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s b/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s index 5696220d0..d1666919a 100644 --- a/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s +++ b/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld b/11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld index bda0da5e9..485ba49b0 100644 --- a/11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld +++ b/11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld @@ -46,14 +46,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs b/11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs index 81233775d..48b1d603a 100644 --- a/11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs +++ b/11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs @@ -6,7 +6,7 @@ pub mod mmu; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -92,23 +89,3 @@ fn rx_start() -> usize { fn rx_end_exclusive() -> usize { unsafe { __rx_end_exclusive.get() as usize } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/11_exceptions_part1_groundwork/src/main.rs b/11_exceptions_part1_groundwork/src/main.rs index 5d00393c4..cd9d0786e 100644 --- a/11_exceptions_part1_groundwork/src/main.rs +++ b/11_exceptions_part1_groundwork/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -127,7 +125,6 @@ mod exception; mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/11_exceptions_part1_groundwork/src/memory.rs b/11_exceptions_part1_groundwork/src/memory.rs index e8cc752f5..7959c555e 100644 --- a/11_exceptions_part1_groundwork/src/memory.rs +++ b/11_exceptions_part1_groundwork/src/memory.rs @@ -5,28 +5,3 @@ //! Memory Management. pub mod mmu; - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/11_exceptions_part1_groundwork/src/runtime_init.rs b/11_exceptions_part1_groundwork/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/11_exceptions_part1_groundwork/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/12_integrated_testing/README.md b/12_integrated_testing/README.md index cff8587b9..d252d0c23 100644 --- a/12_integrated_testing/README.md +++ b/12_integrated_testing/README.md @@ -215,25 +215,24 @@ kernel boot: | | Function | File | | - | - | - | -| 1. | `_start()` | `lib.rs` | -| 2. | (some more aarch64 code) | `lib.rs` | -| 3. | `runtime_init()` | `lib.rs` | -| 4. | `kernel_init()` | `main.rs` | -| 5. | `kernel_main()` | `main.rs` | +| 1. | `_start()` | The library's `boot.s` | +| 2. | (some more aarch64 code) | The library's `boot.rs` | +| 3. | `kernel_init()` | `main.rs` | +| 4. | `kernel_main()` | `main.rs` | A function named `main` is never called. Hence, the `main()` function generated by `cargo test` -would be silently dropped, and therefore the tests would never be executed. As you can see, -`runtime_init()` is the last function residing in our carved-out `lib.rs`, and it calls into -`kernel_init()`. So in order to get the tests to execute, we add a test-environment version of -`kernel_init()` to `lib.rs` as well (conditional compilation ensures it is only present when the -test flag is set), and call the `cargo test` generated `main()` function from there. +would be silently dropped, and therefore the tests would never be executed. As you can see, the +first function getting called in our carved-out `main.rs` is `kernel_init()`. So in order to get the +tests to execute, we add a test-environment version of `kernel_init()` to `lib.rs` as well +(conditional compilation ensures it is only present when the test flag is set), and call the `cargo +test` generated `main()` function from there. This is where `#![reexport_test_harness_main = "test_main"]` finally comes into picture. It declares the name of the generated main function so that we can manually call it. Here is the final implementation in `lib.rs`: ```rust -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { @@ -1073,7 +1072,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translatio diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 12_integrated_testing/src/_arch/aarch64/memory/mmu.rs --- 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs +++ 12_integrated_testing/src/_arch/aarch64/memory/mmu.rs -@@ -162,3 +162,22 @@ +@@ -162,3 +162,33 @@ SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) } } @@ -1085,12 +1084,23 @@ diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 12_inte +#[cfg(test)] +mod tests { + use super::*; ++ use core::{cell::UnsafeCell, ops::Range}; + use test_macros::kernel_test; + + /// Check if KERNEL_TABLES is in .bss. + #[kernel_test] + fn kernel_tables_in_bss() { -+ let bss_range = bsp::memory::bss_range_inclusive(); ++ extern "Rust" { ++ static __bss_start: UnsafeCell; ++ static __bss_end_exclusive: UnsafeCell; ++ } ++ ++ let bss_range = unsafe { ++ Range { ++ start: __bss_start.get(), ++ end: __bss_end_exclusive.get(), ++ } ++ }; + let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 }; + + assert!(bss_range.contains(&kernel_tables_addr)); @@ -1207,7 +1217,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/exception.rs 12_integrated_testing/ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/lib.rs --- 11_exceptions_part1_groundwork/src/lib.rs +++ 12_integrated_testing/src/lib.rs -@@ -0,0 +1,184 @@ +@@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter @@ -1314,9 +1324,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li +//! +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -+//! -+//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html ++//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. + +#![allow(clippy::upper_case_acronyms)] +#![allow(incomplete_features)] @@ -1337,7 +1345,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li +#![test_runner(crate::test_runner)] + +mod panic_wait; -+mod runtime_init; +mod synchronization; + +pub mod bsp; @@ -1362,6 +1369,11 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li + ) +} + ++#[cfg(not(test))] ++extern "Rust" { ++ fn kernel_init() -> !; ++} ++ +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- @@ -1381,7 +1393,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li + } +} + -+/// The `kernel_init()` for unit tests. Called from `runtime_init()`. ++/// The `kernel_init()` for unit tests. +#[cfg(test)] +#[no_mangle] +unsafe fn kernel_init() -> ! { @@ -1396,7 +1408,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/main.rs --- 11_exceptions_part1_groundwork/src/main.rs +++ 12_integrated_testing/src/main.rs -@@ -6,130 +6,12 @@ +@@ -6,127 +6,12 @@ #![doc(html_logo_url = "https://git.io/JeGIp")] //! The `kernel` binary. @@ -1496,9 +1508,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m -//! -//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. --//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. --//! --//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +-//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. - -#![allow(clippy::upper_case_acronyms)] -#![allow(incomplete_features)] @@ -1522,14 +1532,13 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m -mod memory; -mod panic_wait; -mod print; --mod runtime_init; -mod synchronization; -mod time; +use libkernel::{bsp, console, driver, exception, info, memory, time}; /// Early init code. /// -@@ -140,6 +22,7 @@ +@@ -137,6 +22,7 @@ /// - MMU + Data caching must be activated at the earliest. Without it, any atomic operations, /// e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ /// NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs. @@ -1537,7 +1546,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m unsafe fn kernel_init() -> ! { use driver::interface::DriverManager; use memory::mmu::interface::MMU; -@@ -166,15 +49,9 @@ +@@ -163,15 +49,9 @@ fn kernel_main() -> ! { use bsp::console::console; use console::interface::All; @@ -1554,7 +1563,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m info!("Booting on: {}", bsp::board_name()); info!("MMU online. Special regions:"); -@@ -200,31 +77,6 @@ +@@ -197,31 +77,6 @@ info!(" {}. {}", i + 1, driver.compatible()); } @@ -1609,51 +1618,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/memory/mmu.rs 12_integrated_testing + } } -diff -uNr 11_exceptions_part1_groundwork/src/memory.rs 12_integrated_testing/src/memory.rs ---- 11_exceptions_part1_groundwork/src/memory.rs -+++ 12_integrated_testing/src/memory.rs -@@ -30,3 +30,40 @@ - ptr = ptr.offset(1); - } - } -+ -+//-------------------------------------------------------------------------------------------------- -+// Testing -+//-------------------------------------------------------------------------------------------------- -+ -+#[cfg(test)] -+mod tests { -+ use super::*; -+ use test_macros::kernel_test; -+ -+ /// Check `zero_volatile()`. -+ #[kernel_test] -+ fn zero_volatile_works() { -+ let mut x: [usize; 3] = [10, 11, 12]; -+ let x_range = x.as_mut_ptr_range(); -+ let x_range_inclusive = -+ RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); -+ -+ unsafe { zero_volatile(x_range_inclusive) }; -+ -+ assert_eq!(x, [0, 0, 0]); -+ } -+ -+ /// Check `bss` section layout. -+ #[kernel_test] -+ fn bss_section_is_sane() { -+ use crate::bsp::memory::bss_range_inclusive; -+ use core::mem; -+ -+ let start = *bss_range_inclusive().start() as usize; -+ let end = *bss_range_inclusive().end() as usize; -+ -+ assert_eq!(start modulo mem::size_of::(), 0); -+ assert_eq!(end modulo mem::size_of::(), 0); -+ assert!(end >= start); -+ } -+} - diff -uNr 11_exceptions_part1_groundwork/src/panic_wait.rs 12_integrated_testing/src/panic_wait.rs --- 11_exceptions_part1_groundwork/src/panic_wait.rs +++ 12_integrated_testing/src/panic_wait.rs @@ -1689,23 +1653,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/panic_wait.rs 12_integrated_testing + _panic_exit() } -diff -uNr 11_exceptions_part1_groundwork/src/runtime_init.rs 12_integrated_testing/src/runtime_init.rs ---- 11_exceptions_part1_groundwork/src/runtime_init.rs -+++ 12_integrated_testing/src/runtime_init.rs -@@ -31,7 +31,10 @@ - /// - /// - Only a single core must be active and running this function. - pub unsafe fn runtime_init() -> ! { -- zero_bss(); -+ extern "Rust" { -+ fn kernel_init() -> !; -+ } - -- crate::kernel_init() -+ zero_bss(); -+ kernel_init() - } - diff -uNr 11_exceptions_part1_groundwork/test-macros/Cargo.toml 12_integrated_testing/test-macros/Cargo.toml --- 11_exceptions_part1_groundwork/test-macros/Cargo.toml +++ 12_integrated_testing/test-macros/Cargo.toml diff --git a/12_integrated_testing/src/_arch/aarch64/cpu/boot.rs b/12_integrated_testing/src/_arch/aarch64/cpu/boot.rs index 02798fdc0..b743418e6 100644 --- a/12_integrated_testing/src/_arch/aarch64/cpu/boot.rs +++ b/12_integrated_testing/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/12_integrated_testing/src/_arch/aarch64/cpu/boot.s b/12_integrated_testing/src/_arch/aarch64/cpu/boot.s index 5696220d0..d1666919a 100644 --- a/12_integrated_testing/src/_arch/aarch64/cpu/boot.s +++ b/12_integrated_testing/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/12_integrated_testing/src/_arch/aarch64/memory/mmu.rs b/12_integrated_testing/src/_arch/aarch64/memory/mmu.rs index 29e8125d7..a706290bd 100644 --- a/12_integrated_testing/src/_arch/aarch64/memory/mmu.rs +++ b/12_integrated_testing/src/_arch/aarch64/memory/mmu.rs @@ -170,12 +170,23 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { #[cfg(test)] mod tests { use super::*; + use core::{cell::UnsafeCell, ops::Range}; use test_macros::kernel_test; /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { - let bss_range = bsp::memory::bss_range_inclusive(); + extern "Rust" { + static __bss_start: UnsafeCell; + static __bss_end_exclusive: UnsafeCell; + } + + let bss_range = unsafe { + Range { + start: __bss_start.get(), + end: __bss_end_exclusive.get(), + } + }; let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 }; assert!(bss_range.contains(&kernel_tables_addr)); diff --git a/12_integrated_testing/src/bsp/raspberrypi/link.ld b/12_integrated_testing/src/bsp/raspberrypi/link.ld index bda0da5e9..485ba49b0 100644 --- a/12_integrated_testing/src/bsp/raspberrypi/link.ld +++ b/12_integrated_testing/src/bsp/raspberrypi/link.ld @@ -46,14 +46,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/12_integrated_testing/src/bsp/raspberrypi/memory.rs b/12_integrated_testing/src/bsp/raspberrypi/memory.rs index 81233775d..48b1d603a 100644 --- a/12_integrated_testing/src/bsp/raspberrypi/memory.rs +++ b/12_integrated_testing/src/bsp/raspberrypi/memory.rs @@ -6,7 +6,7 @@ pub mod mmu; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -92,23 +89,3 @@ fn rx_start() -> usize { fn rx_end_exclusive() -> usize { unsafe { __rx_end_exclusive.get() as usize } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/12_integrated_testing/src/lib.rs b/12_integrated_testing/src/lib.rs index 1e315b4da..9890351f9 100644 --- a/12_integrated_testing/src/lib.rs +++ b/12_integrated_testing/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -127,7 +125,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -152,6 +149,11 @@ pub fn version() -> &'static str { ) } +#[cfg(not(test))] +extern "Rust" { + fn kernel_init() -> !; +} + //-------------------------------------------------------------------------------------------------- // Testing //-------------------------------------------------------------------------------------------------- @@ -171,7 +173,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/12_integrated_testing/src/memory.rs b/12_integrated_testing/src/memory.rs index 1ef0285a6..7959c555e 100644 --- a/12_integrated_testing/src/memory.rs +++ b/12_integrated_testing/src/memory.rs @@ -5,65 +5,3 @@ //! Memory Management. pub mod mmu; - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/12_integrated_testing/src/runtime_init.rs b/12_integrated_testing/src/runtime_init.rs deleted file mode 100644 index 0a1c685cd..000000000 --- a/12_integrated_testing/src/runtime_init.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/13_exceptions_part2_peripheral_IRQs/README.md b/13_exceptions_part2_peripheral_IRQs/README.md index 2a44ecdac..1174ca9e4 100644 --- a/13_exceptions_part2_peripheral_IRQs/README.md +++ b/13_exceptions_part2_peripheral_IRQs/README.md @@ -2104,7 +2104,7 @@ diff -uNr 12_integrated_testing/src/bsp/raspberrypi/exception.rs 13_exceptions_p diff -uNr 12_integrated_testing/src/bsp/raspberrypi/memory.rs 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs --- 12_integrated_testing/src/bsp/raspberrypi/memory.rs +++ 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs -@@ -51,10 +51,12 @@ +@@ -48,10 +48,12 @@ pub mod mmio { use super::*; @@ -2121,7 +2121,7 @@ diff -uNr 12_integrated_testing/src/bsp/raspberrypi/memory.rs 13_exceptions_part } /// Physical devices. -@@ -65,6 +67,8 @@ +@@ -62,6 +64,8 @@ pub const START: usize = 0xFE00_0000; pub const GPIO_START: usize = START + GPIO_OFFSET; pub const PL011_UART_START: usize = START + UART_OFFSET; @@ -2376,7 +2376,7 @@ diff -uNr 12_integrated_testing/src/exception/asynchronous.rs 13_exceptions_part diff -uNr 12_integrated_testing/src/lib.rs 13_exceptions_part2_peripheral_IRQs/src/lib.rs --- 12_integrated_testing/src/lib.rs +++ 13_exceptions_part2_peripheral_IRQs/src/lib.rs -@@ -110,6 +110,7 @@ +@@ -108,6 +108,7 @@ #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -2384,7 +2384,7 @@ diff -uNr 12_integrated_testing/src/lib.rs 13_exceptions_part2_peripheral_IRQs/s #![feature(const_fn_fn_ptr_basics)] #![feature(const_generics)] #![feature(const_panic)] -@@ -137,6 +138,7 @@ +@@ -134,6 +135,7 @@ pub mod exception; pub mod memory; pub mod print; diff --git a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs index 02798fdc0..b743418e6 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s index 5696220d0..d1666919a 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s +++ b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs index 29e8125d7..a706290bd 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs @@ -170,12 +170,23 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { #[cfg(test)] mod tests { use super::*; + use core::{cell::UnsafeCell, ops::Range}; use test_macros::kernel_test; /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { - let bss_range = bsp::memory::bss_range_inclusive(); + extern "Rust" { + static __bss_start: UnsafeCell; + static __bss_end_exclusive: UnsafeCell; + } + + let bss_range = unsafe { + Range { + start: __bss_start.get(), + end: __bss_end_exclusive.get(), + } + }; let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 }; assert!(bss_range.contains(&kernel_tables_addr)); diff --git a/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld b/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld index bda0da5e9..485ba49b0 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld +++ b/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld @@ -46,14 +46,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs b/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs index 8a58a17eb..ede648f67 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs @@ -6,7 +6,7 @@ pub mod mmu; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -96,23 +93,3 @@ fn rx_start() -> usize { fn rx_end_exclusive() -> usize { unsafe { __rx_end_exclusive.get() as usize } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/13_exceptions_part2_peripheral_IRQs/src/lib.rs b/13_exceptions_part2_peripheral_IRQs/src/lib.rs index 9a787e60e..ca761361d 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/lib.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -128,7 +126,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -154,6 +151,11 @@ pub fn version() -> &'static str { ) } +#[cfg(not(test))] +extern "Rust" { + fn kernel_init() -> !; +} + //-------------------------------------------------------------------------------------------------- // Testing //-------------------------------------------------------------------------------------------------- @@ -173,7 +175,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/13_exceptions_part2_peripheral_IRQs/src/memory.rs b/13_exceptions_part2_peripheral_IRQs/src/memory.rs index 1ef0285a6..7959c555e 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/memory.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/memory.rs @@ -5,65 +5,3 @@ //! Memory Management. pub mod mmu; - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/13_exceptions_part2_peripheral_IRQs/src/runtime_init.rs b/13_exceptions_part2_peripheral_IRQs/src/runtime_init.rs deleted file mode 100644 index 0a1c685cd..000000000 --- a/13_exceptions_part2_peripheral_IRQs/src/runtime_init.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/14_virtual_mem_part2_mmio_remap/README.md b/14_virtual_mem_part2_mmio_remap/README.md index 85aa5138a..73ef83a3a 100644 --- a/14_virtual_mem_part2_mmio_remap/README.md +++ b/14_virtual_mem_part2_mmio_remap/README.md @@ -806,7 +806,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 14 self.configure_translation_control(); -@@ -162,22 +153,3 @@ +@@ -162,33 +153,3 @@ SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) } } @@ -818,12 +818,23 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 14 -#[cfg(test)] -mod tests { - use super::*; +- use core::{cell::UnsafeCell, ops::Range}; - use test_macros::kernel_test; - - /// Check if KERNEL_TABLES is in .bss. - #[kernel_test] - fn kernel_tables_in_bss() { -- let bss_range = bsp::memory::bss_range_inclusive(); +- extern "Rust" { +- static __bss_start: UnsafeCell; +- static __bss_end_exclusive: UnsafeCell; +- } +- +- let bss_range = unsafe { +- Range { +- start: __bss_start.get(), +- end: __bss_end_exclusive.get(), +- } +- }; - let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 }; - - assert!(bss_range.contains(&kernel_tables_addr)); @@ -1489,10 +1500,10 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 14_vir + __rw_start = .; .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -@@ -56,4 +52,23 @@ - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ +@@ -54,4 +50,23 @@ + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE + + . = ALIGN(64K); /* Align to page boundary */ @@ -1554,16 +1565,16 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to +/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. +pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; -+ + +-const NUM_MEM_RANGES: usize = 2; +/// The kernel's virtual address space defined by this BSP. +pub type KernelVirtAddrSpace = AddressSpace<{ 8 * 1024 * 1024 * 1024 }>; --const NUM_MEM_RANGES: usize = 2; +-/// The virtual memory layout. +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- - --/// The virtual memory layout. ++ +/// The kernel translation tables. /// -/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. @@ -1642,15 +1653,15 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +/// The Read+Execute (RX) pages of the kernel binary. +fn phys_rx_page_desc() -> PageSliceDescriptor { + virt_rx_page_desc().into() ++} ++ ++/// The Read+Write (RW) pages of the kernel binary. ++fn phys_rw_page_desc() -> PageSliceDescriptor { ++ virt_rw_page_desc().into() } -fn mmio_range_inclusive() -> RangeInclusive { - RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE) -+/// The Read+Write (RW) pages of the kernel binary. -+fn phys_rw_page_desc() -> PageSliceDescriptor { -+ virt_rw_page_desc().into() -+} -+ +/// The boot core's stack. +fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor { + virt_boot_core_stack_page_desc().into() @@ -1726,15 +1737,17 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs } //-------------------------------------------------------------------------------------------------- -@@ -82,14 +176,18 @@ +@@ -77,19 +171,24 @@ + #[cfg(test)] + mod tests { + use super::*; ++ use core::{cell::UnsafeCell, ops::Range}; + use test_macros::kernel_test; + /// Check alignment of the kernel's virtual memory layout sections. #[kernel_test] fn virt_mem_layout_sections_are_64KiB_aligned() { - const SIXTYFOUR_KIB: usize = 65536; -- -- for i in LAYOUT.inner().iter() { -- let start: usize = *(i.virtual_range)().start(); -- let end: usize = *(i.virtual_range)().end() + 1; + for i in [ + virt_rx_page_desc, + virt_rw_page_desc, @@ -1745,6 +1758,10 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + let start: usize = i().start_addr().into_usize(); + let end: usize = i().end_addr().into_usize(); +- for i in LAYOUT.inner().iter() { +- let start: usize = *(i.virtual_range)().start(); +- let end: usize = *(i.virtual_range)().end() + 1; +- - assert_eq!(start modulo SIXTYFOUR_KIB, 0); - assert_eq!(end modulo SIXTYFOUR_KIB, 0); + assert_eq!(start modulo KernelGranule::SIZE, 0); @@ -1752,7 +1769,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs assert!(end >= start); } } -@@ -97,18 +195,28 @@ +@@ -97,18 +196,38 @@ /// Ensure the kernel's virtual memory layout is free of overlaps. #[kernel_test] fn virt_mem_layout_has_no_overlaps() { @@ -1786,7 +1803,17 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + /// Check if KERNEL_TABLES is in .bss. + #[kernel_test] + fn kernel_tables_in_bss() { -+ let bss_range = super::super::bss_range_inclusive(); ++ extern "Rust" { ++ static __bss_start: UnsafeCell; ++ static __bss_end_exclusive: UnsafeCell; ++ } ++ ++ let bss_range = unsafe { ++ Range { ++ start: __bss_start.get(), ++ end: __bss_end_exclusive.get(), ++ } ++ }; + let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; + + assert!(bss_range.contains(&kernel_tables_addr)); @@ -1834,16 +1861,15 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v pub mod mmu; +use crate::memory::{Address, Physical, Virtual}; - use core::{cell::UnsafeCell, ops::RangeInclusive}; + use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- -@@ -17,8 +48,16 @@ +@@ -16,6 +47,15 @@ + extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - ++ + static __rw_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; + static __rw_end_exclusive: UnsafeCell<()>; + + static __boot_core_stack_start: UnsafeCell<()>; @@ -1854,7 +1880,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v } //-------------------------------------------------------------------------------------------------- -@@ -28,35 +67,26 @@ +@@ -25,35 +65,26 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -1904,7 +1930,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v } /// Physical devices. -@@ -64,13 +94,22 @@ +@@ -61,13 +92,22 @@ pub mod mmio { use super::*; @@ -1933,7 +1959,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v } //-------------------------------------------------------------------------------------------------- -@@ -83,18 +122,69 @@ +@@ -80,16 +120,67 @@ /// /// - Value is provided by the linker script and must be trusted as-is. #[inline(always)] @@ -1941,21 +1967,19 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v - unsafe { __rx_start.get() as usize } +fn virt_rx_start() -> Address { + Address::new(unsafe { __rx_start.get() as usize }) - } - --/// Exclusive end address of the Read+Execute (RX) range. ++} ++ +/// Size of the Read+Execute (RX) range. - /// - /// # Safety - /// - /// - Value is provided by the linker script and must be trusted as-is. - #[inline(always)] --fn rx_end_exclusive() -> usize { -- unsafe { __rx_end_exclusive.get() as usize } ++/// ++/// # Safety ++/// ++/// - Value is provided by the linker script and must be trusted as-is. ++#[inline(always)] +fn rx_size() -> usize { + unsafe { (__rx_end_exclusive.get() as usize) - (__rx_start.get() as usize) } -+} -+ + } + +-/// Exclusive end address of the Read+Execute (RX) range. +/// Start address of the Read+Write (RW) range. +#[inline(always)] +fn virt_rw_start() -> Address { @@ -1963,11 +1987,13 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v +} + +/// Size of the Read+Write (RW) range. -+/// -+/// # Safety -+/// -+/// - Value is provided by the linker script and must be trusted as-is. -+#[inline(always)] + /// + /// # Safety + /// + /// - Value is provided by the linker script and must be trusted as-is. + #[inline(always)] +-fn rx_end_exclusive() -> usize { +- unsafe { __rx_end_exclusive.get() as usize } +fn rw_size() -> usize { + unsafe { (__rw_end_exclusive.get() as usize) - (__rw_start.get() as usize) } +} @@ -2007,8 +2033,6 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v + map::END } - //-------------------------------------------------------------------------------------------------- - diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi.rs --- 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs +++ 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi.rs @@ -2131,7 +2155,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/driver.rs 14_virtual_mem_part2 diff -uNr 13_exceptions_part2_peripheral_IRQs/src/lib.rs 14_virtual_mem_part2_mmio_remap/src/lib.rs --- 13_exceptions_part2_peripheral_IRQs/src/lib.rs +++ 14_virtual_mem_part2_mmio_remap/src/lib.rs -@@ -111,6 +111,8 @@ +@@ -109,6 +109,8 @@ #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] #![feature(asm)] @@ -2140,7 +2164,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/lib.rs 14_virtual_mem_part2_mm #![feature(const_fn_fn_ptr_basics)] #![feature(const_generics)] #![feature(const_panic)] -@@ -132,6 +134,7 @@ +@@ -129,6 +131,7 @@ mod synchronization; pub mod bsp; @@ -3131,16 +3155,16 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 14_virtual_mem_p diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory.rs 14_virtual_mem_part2_mmio_remap/src/memory.rs --- 13_exceptions_part2_peripheral_IRQs/src/memory.rs +++ 14_virtual_mem_part2_mmio_remap/src/memory.rs -@@ -6,12 +6,136 @@ +@@ -5,3 +5,133 @@ + //! Memory Management. pub mod mmu; - --use core::ops::RangeInclusive; ++ +use crate::common; +use core::{ + fmt, + marker::PhantomData, -+ ops::{AddAssign, RangeInclusive, SubAssign}, ++ ops::{AddAssign, SubAssign}, +}; + +//-------------------------------------------------------------------------------------------------- @@ -3164,11 +3188,11 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory.rs 14_virtual_mem_part2 + value: usize, + _address_type: PhantomData ATYPE>, +} - - //-------------------------------------------------------------------------------------------------- - // Public Code - //-------------------------------------------------------------------------------------------------- - ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ +impl AddressType for Physical {} +impl AddressType for Virtual {} + @@ -3265,10 +3289,6 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory.rs 14_virtual_mem_part2 + write!(f, "{:04x}", q1) + } +} -+ - /// Zero out an inclusive memory range. - /// - /// # Safety diff -uNr 13_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs 14_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs --- 13_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs diff --git a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs index 02798fdc0..b743418e6 100644 --- a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs +++ b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s index 5696220d0..d1666919a 100644 --- a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s +++ b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld index 64545edf2..698ad85da 100644 --- a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld +++ b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld @@ -42,15 +42,13 @@ SECTIONS __rw_start = .; .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE . = ALIGN(64K); /* Align to page boundary */ diff --git a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs index 7f571e080..e51b00fa1 100644 --- a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs +++ b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs @@ -37,7 +37,7 @@ pub mod mmu; use crate::memory::{Address, Physical, Virtual}; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -49,8 +49,6 @@ extern "Rust" { static __rx_end_exclusive: UnsafeCell<()>; static __rw_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; static __rw_end_exclusive: UnsafeCell<()>; static __boot_core_stack_start: UnsafeCell<()>; @@ -186,23 +184,3 @@ fn boot_core_stack_guard_page_size() -> usize { fn phys_addr_space_end() -> Address { map::END } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs index 88fc80bd5..813fa0b4d 100644 --- a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs +++ b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs @@ -171,6 +171,7 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { #[cfg(test)] mod tests { use super::*; + use core::{cell::UnsafeCell, ops::Range}; use test_macros::kernel_test; /// Check alignment of the kernel's virtual memory layout sections. @@ -214,7 +215,17 @@ mod tests { /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { - let bss_range = super::super::bss_range_inclusive(); + extern "Rust" { + static __bss_start: UnsafeCell; + static __bss_end_exclusive: UnsafeCell; + } + + let bss_range = unsafe { + Range { + start: __bss_start.get(), + end: __bss_end_exclusive.get(), + } + }; let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; assert!(bss_range.contains(&kernel_tables_addr)); diff --git a/14_virtual_mem_part2_mmio_remap/src/lib.rs b/14_virtual_mem_part2_mmio_remap/src/lib.rs index f3ef29513..1b2cb367a 100644 --- a/14_virtual_mem_part2_mmio_remap/src/lib.rs +++ b/14_virtual_mem_part2_mmio_remap/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -130,7 +128,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -157,6 +154,11 @@ pub fn version() -> &'static str { ) } +#[cfg(not(test))] +extern "Rust" { + fn kernel_init() -> !; +} + //-------------------------------------------------------------------------------------------------- // Testing //-------------------------------------------------------------------------------------------------- @@ -176,7 +178,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/14_virtual_mem_part2_mmio_remap/src/memory.rs b/14_virtual_mem_part2_mmio_remap/src/memory.rs index 515731eb7..3de1d6077 100644 --- a/14_virtual_mem_part2_mmio_remap/src/memory.rs +++ b/14_virtual_mem_part2_mmio_remap/src/memory.rs @@ -10,7 +10,7 @@ use crate::common; use core::{ fmt, marker::PhantomData, - ops::{AddAssign, RangeInclusive, SubAssign}, + ops::{AddAssign, SubAssign}, }; //-------------------------------------------------------------------------------------------------- @@ -135,59 +135,3 @@ impl fmt::Display for Address { write!(f, "{:04x}", q1) } } - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/14_virtual_mem_part2_mmio_remap/src/runtime_init.rs b/14_virtual_mem_part2_mmio_remap/src/runtime_init.rs deleted file mode 100644 index 0a1c685cd..000000000 --- a/14_virtual_mem_part2_mmio_remap/src/runtime_init.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/15_virtual_mem_part3_precomputed_tables/README.md b/15_virtual_mem_part3_precomputed_tables/README.md index bb9a50bb6..b06b222df 100644 --- a/15_virtual_mem_part3_precomputed_tables/README.md +++ b/15_virtual_mem_part3_precomputed_tables/README.md @@ -530,7 +530,7 @@ pub unsafe extern "C" fn _start_rust( cpu::wait_forever(); } - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } ``` @@ -796,19 +796,18 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/Makefile 15_virtual_mem_part3_precompu diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs --- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs +++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs -@@ -11,7 +11,8 @@ +@@ -11,6 +11,8 @@ //! //! crate::cpu::boot::arch_boot --use crate::runtime_init; -+use crate::{cpu, memory, memory::Address, runtime_init}; ++use crate::{cpu, memory, memory::Address}; +use core::intrinsics::unlikely; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. -@@ -71,9 +72,18 @@ - /// - The `bss` section is not initialized yet. The code must not use or reference it in any way. - /// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +@@ -69,9 +71,18 @@ + /// + /// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] -pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { +pub unsafe extern "C" fn _start_rust( @@ -823,17 +822,17 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs 15_virtu + cpu::wait_forever(); + } + - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s --- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s +++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s -@@ -44,11 +44,14 @@ - - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. +@@ -56,11 +56,14 @@ + // Prepare the jump to Rust code. + prepare_rust: + // Load the base address of the kernel's translation tables. + ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs + @@ -1204,7 +1203,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v "Kernel boot-core stack", &virt_boot_core_stack_page_desc(), &phys_boot_core_stack_page_desc(), -@@ -159,64 +183,5 @@ +@@ -159,75 +183,5 @@ acc_perms: AccessPermissions::ReadWrite, execute_never: true, }, @@ -1220,6 +1219,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v -#[cfg(test)] -mod tests { - use super::*; +- use core::{cell::UnsafeCell, ops::Range}; - use test_macros::kernel_test; - - /// Check alignment of the kernel's virtual memory layout sections. @@ -1263,7 +1263,17 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v - /// Check if KERNEL_TABLES is in .bss. - #[kernel_test] - fn kernel_tables_in_bss() { -- let bss_range = super::super::bss_range_inclusive(); +- extern "Rust" { +- static __bss_start: UnsafeCell; +- static __bss_end_exclusive: UnsafeCell; +- } +- +- let bss_range = unsafe { +- Range { +- start: __bss_start.get(), +- end: __bss_end_exclusive.get(), +- } +- }; - let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; - - assert!(bss_range.contains(&kernel_tables_addr)); @@ -1511,7 +1521,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory.rs 15_virtual_mem_part3_pre + convert::TryFrom, fmt, marker::PhantomData, - ops::{AddAssign, RangeInclusive, SubAssign}, + ops::{AddAssign, SubAssign}, @@ -67,6 +68,14 @@ } } diff --git a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs index 5e0ecfc6c..f275c792d 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,7 @@ //! //! crate::cpu::boot::arch_boot -use crate::{cpu, memory, memory::Address, runtime_init}; +use crate::{cpu, memory, memory::Address}; use core::intrinsics::unlikely; use cortex_a::{asm, regs::*}; @@ -51,8 +51,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -69,8 +69,7 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust( phys_kernel_tables_base_addr: u64, @@ -84,6 +83,6 @@ pub unsafe extern "C" fn _start_rust( cpu::wait_forever(); } - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s index 286462616..9e7fc619a 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s +++ b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Load the base address of the kernel's translation tables. ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs @@ -55,8 +67,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld index 7ad75a40c..aa127cc01 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld +++ b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld @@ -45,15 +45,13 @@ SECTIONS __rw_start = .; .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE . = ALIGN(64K); /* Align to page boundary */ diff --git a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory.rs b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory.rs index 7f571e080..e51b00fa1 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory.rs @@ -37,7 +37,7 @@ pub mod mmu; use crate::memory::{Address, Physical, Virtual}; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -49,8 +49,6 @@ extern "Rust" { static __rx_end_exclusive: UnsafeCell<()>; static __rw_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; static __rw_end_exclusive: UnsafeCell<()>; static __boot_core_stack_start: UnsafeCell<()>; @@ -186,23 +184,3 @@ fn boot_core_stack_guard_page_size() -> usize { fn phys_addr_space_end() -> Address { map::END } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/15_virtual_mem_part3_precomputed_tables/src/lib.rs b/15_virtual_mem_part3_precomputed_tables/src/lib.rs index f3ef29513..1b2cb367a 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/lib.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -130,7 +128,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -157,6 +154,11 @@ pub fn version() -> &'static str { ) } +#[cfg(not(test))] +extern "Rust" { + fn kernel_init() -> !; +} + //-------------------------------------------------------------------------------------------------- // Testing //-------------------------------------------------------------------------------------------------- @@ -176,7 +178,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/15_virtual_mem_part3_precomputed_tables/src/memory.rs b/15_virtual_mem_part3_precomputed_tables/src/memory.rs index 87ffa81f5..4ac969040 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/memory.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/memory.rs @@ -11,7 +11,7 @@ use core::{ convert::TryFrom, fmt, marker::PhantomData, - ops::{AddAssign, RangeInclusive, SubAssign}, + ops::{AddAssign, SubAssign}, }; //-------------------------------------------------------------------------------------------------- @@ -144,59 +144,3 @@ impl fmt::Display for Address { write!(f, "{:04x}", q1) } } - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs b/15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs deleted file mode 100644 index 0a1c685cd..000000000 --- a/15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/16_virtual_mem_part4_higher_half_kernel/README.md b/16_virtual_mem_part4_higher_half_kernel/README.md index 17d71e2b9..0a4381841 100644 --- a/16_virtual_mem_part4_higher_half_kernel/README.md +++ b/16_virtual_mem_part4_higher_half_kernel/README.md @@ -99,7 +99,7 @@ by the time the CPU enters `EL1`, virtual memory will be active, and the CPU mus new higher-half `virtual addresses` for everything it does. Specifically, this means the address from which the CPU should execute upon entering `EL1` (function -`runtime_init()`) must be a valid _virtual address_, same as the stack pointer's address. Both of +`kernel_init()`) must be a valid _virtual address_, same as the stack pointer's address. Both of them are programmed in function `fn prepare_el2_to_el1_transition(...)`, so we must ensure now that _link-time_ addresses are used here. For this reason, retrieval of these addresses happens in `assembly` in `boot.s`, where we can explicitly enforce generation of **absolute** addresses: @@ -108,7 +108,7 @@ _link-time_ addresses are used here. For this reason, retrieval of these address // Load the _absolute_ addresses of the following symbols. Since the kernel is linked at // the top of the 64 bit address space, these are effectively virtual addresses. ADR_ABS x1, __boot_core_stack_end_exclusive -ADR_ABS x2, runtime_init +ADR_ABS x2, kernel_init ``` Both values are forwarded to the Rust entry point function `_start_rust()`, which in turn forwards @@ -246,15 +246,6 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/Cargo.toml 16_virtual_mem_part diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs 16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs --- 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs +++ 16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs -@@ -11,7 +11,7 @@ - //! - //! crate::cpu::boot::arch_boot - --use crate::{cpu, memory, memory::Address, runtime_init}; -+use crate::{cpu, memory, memory::Address}; - use core::intrinsics::unlikely; - use cortex_a::{asm, regs::*}; - @@ -29,7 +29,10 @@ /// - The `bss` section is not initialized yet. The code must not use or reference it in any way. /// - The HW state of EL1 must be prepared in a sound way. @@ -262,7 +253,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs -unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { +unsafe fn prepare_el2_to_el1_transition( + virt_boot_core_stack_end_exclusive_addr: u64, -+ virt_runtime_init_addr: u64, ++ virt_kernel_init_addr: u64, +) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -270,9 +261,9 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs @@ -52,11 +55,11 @@ ); - // Second, let the link register point to runtime_init(). -- ELR_EL2.set(runtime_init::runtime_init as *const () as u64); -+ ELR_EL2.set(virt_runtime_init_addr); + // Second, let the link register point to kernel_init(). +- ELR_EL2.set(crate::kernel_init as *const () as u64); ++ ELR_EL2.set(virt_kernel_init_addr); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -281,29 +272,29 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs } //-------------------------------------------------------------------------------------------------- -@@ -74,9 +77,13 @@ +@@ -73,9 +76,13 @@ #[no_mangle] pub unsafe extern "C" fn _start_rust( phys_kernel_tables_base_addr: u64, - phys_boot_core_stack_end_exclusive_addr: u64, + virt_boot_core_stack_end_exclusive_addr: u64, -+ virt_runtime_init_addr: u64, ++ virt_kernel_init_addr: u64, ) -> ! { - prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + prepare_el2_to_el1_transition( + virt_boot_core_stack_end_exclusive_addr, -+ virt_runtime_init_addr, ++ virt_kernel_init_addr, + ); // Turn on the MMU for EL1. let addr = Address::new(phys_kernel_tables_base_addr as usize); -@@ -84,6 +91,7 @@ +@@ -83,6 +90,7 @@ cpu::wait_forever(); } -- // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. +- // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. + // Use `eret` to "return" to EL1. Since virtual memory will already be enabled, this results in -+ // execution of runtime_init() in EL1 from its _virtual address_. ++ // execution of kernel_init() in EL1 from its _virtual address_. asm::eret() } @@ -329,7 +320,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s 1 .equ _EL2, 0x8 .equ _core_id_mask, 0b11 -@@ -47,11 +59,23 @@ +@@ -59,11 +71,23 @@ // Load the base address of the kernel's translation tables. ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs @@ -339,7 +330,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s 1 + // Load the _absolute_ addresses of the following symbols. Since the kernel is linked at + // the top of the 64 bit address space, these are effectively virtual addresses. + ADR_ABS x1, __boot_core_stack_end_exclusive -+ ADR_ABS x2, runtime_init ++ ADR_ABS x2, kernel_init + + // Load the PC-relative address of the stack and set the stack pointer. + // @@ -590,6 +581,22 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu //-------------------------------------------------------------------------------------------------- // Public Definitions +diff -uNr 15_virtual_mem_part3_precomputed_tables/src/lib.rs 16_virtual_mem_part4_higher_half_kernel/src/lib.rs +--- 15_virtual_mem_part3_precomputed_tables/src/lib.rs ++++ 16_virtual_mem_part4_higher_half_kernel/src/lib.rs +@@ -154,11 +154,6 @@ + ) + } + +-#[cfg(not(test))] +-extern "Rust" { +- fn kernel_init() -> !; +-} +- + //-------------------------------------------------------------------------------------------------- + // Testing + //-------------------------------------------------------------------------------------------------- + diff -uNr 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs --- 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs +++ 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs @@ -606,18 +613,6 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs 16_virtual_m type TableStartFromBottom; } -diff -uNr 15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs 16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs ---- 15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs -+++ 16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs -@@ -30,6 +30,7 @@ - /// # Safety - /// - /// - Only a single core must be active and running this function. -+#[no_mangle] - pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - diff -uNr 15_virtual_mem_part3_precomputed_tables/tests/02_exception_sync_page_fault.rs 16_virtual_mem_part4_higher_half_kernel/tests/02_exception_sync_page_fault.rs --- 15_virtual_mem_part3_precomputed_tables/tests/02_exception_sync_page_fault.rs +++ 16_virtual_mem_part4_higher_half_kernel/tests/02_exception_sync_page_fault.rs diff --git a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs index 4f0ef0b4d..fe8e90600 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs @@ -31,7 +31,7 @@ global_asm!(include_str!("boot.s")); #[inline(always)] unsafe fn prepare_el2_to_el1_transition( virt_boot_core_stack_end_exclusive_addr: u64, - virt_runtime_init_addr: u64, + virt_kernel_init_addr: u64, ) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -54,8 +54,8 @@ unsafe fn prepare_el2_to_el1_transition( + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(virt_runtime_init_addr); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(virt_kernel_init_addr); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -72,17 +72,16 @@ unsafe fn prepare_el2_to_el1_transition( /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust( phys_kernel_tables_base_addr: u64, virt_boot_core_stack_end_exclusive_addr: u64, - virt_runtime_init_addr: u64, + virt_kernel_init_addr: u64, ) -> ! { prepare_el2_to_el1_transition( virt_boot_core_stack_end_exclusive_addr, - virt_runtime_init_addr, + virt_kernel_init_addr, ); // Turn on the MMU for EL1. @@ -92,6 +91,6 @@ pub unsafe extern "C" fn _start_rust( } // Use `eret` to "return" to EL1. Since virtual memory will already be enabled, this results in - // execution of runtime_init() in EL1 from its _virtual address_. + // execution of kernel_init() in EL1 from its _virtual address_. asm::eret() } diff --git a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.s b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.s index 638259d47..32a5c3ff6 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.s +++ b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.s @@ -45,24 +45,36 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Load the base address of the kernel's translation tables. ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs // Load the _absolute_ addresses of the following symbols. Since the kernel is linked at // the top of the 64 bit address space, these are effectively virtual addresses. ADR_ABS x1, __boot_core_stack_end_exclusive - ADR_ABS x2, runtime_init + ADR_ABS x2, kernel_init // Load the PC-relative address of the stack and set the stack pointer. // @@ -79,8 +91,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/link.ld b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/link.ld index 962638d00..a3179f1a3 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/link.ld +++ b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/link.ld @@ -55,15 +55,13 @@ SECTIONS __rw_start = .; .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE . = ALIGN(64K); /* Align to page boundary */ diff --git a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory.rs b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory.rs index 7f571e080..e51b00fa1 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory.rs @@ -37,7 +37,7 @@ pub mod mmu; use crate::memory::{Address, Physical, Virtual}; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -49,8 +49,6 @@ extern "Rust" { static __rx_end_exclusive: UnsafeCell<()>; static __rw_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; static __rw_end_exclusive: UnsafeCell<()>; static __boot_core_stack_start: UnsafeCell<()>; @@ -186,23 +184,3 @@ fn boot_core_stack_guard_page_size() -> usize { fn phys_addr_space_end() -> Address { map::END } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/16_virtual_mem_part4_higher_half_kernel/src/lib.rs b/16_virtual_mem_part4_higher_half_kernel/src/lib.rs index f3ef29513..3e3e08d8f 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/lib.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -130,7 +128,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -176,7 +173,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/16_virtual_mem_part4_higher_half_kernel/src/memory.rs b/16_virtual_mem_part4_higher_half_kernel/src/memory.rs index 87ffa81f5..4ac969040 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/memory.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/memory.rs @@ -11,7 +11,7 @@ use core::{ convert::TryFrom, fmt, marker::PhantomData, - ops::{AddAssign, RangeInclusive, SubAssign}, + ops::{AddAssign, SubAssign}, }; //-------------------------------------------------------------------------------------------------- @@ -144,59 +144,3 @@ impl fmt::Display for Address { write!(f, "{:04x}", q1) } } - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs b/16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs deleted file mode 100644 index 2c0a62e67..000000000 --- a/16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -#[no_mangle] -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/X1_JTAG_boot/jtag_boot_rpi3.img b/X1_JTAG_boot/jtag_boot_rpi3.img index 96d8bfb1817aef0dfb5458871a1ea93582acea84..9122433173b872ffac01c7ef68ea4d3f32a5243e 100755 GIT binary patch delta 1571 zcmb7ET})eL7=FKV3bY7Y3;h8K{gIzii0itoP}zVME|9W`DH!~T0cD%bc4N3(;!@^? zVai26K6I0rj5?z`IO0q-+L_EUVuHHN7&g7}qOe5Uq72l;RBey%={Y*17wVUMeZQaQ zectzZzxOoRb-8n26uh_WD&I^G2aj|Ew-F^U3;>rQ(Lo7XJ6bD{jWK{>>$9gW=0wp% z^$p}JdjDtSLac~5h~aP%%u3kgZJ>6r(RChZg!#F8G7vk$RTE-kVg3n^8X;vkC8*lp zNS*v-FtZG_6()nzQ4h5z8fg+;#V`?44qnK(K=_Ukm6OR{(8tWGDrlsmdwj;B+LYI7hrvAVeCkRBkCc^cv`V1<5ovqZq+ zvfCAVO6NVa$F_@(&+lT^%17( z*7x*b{0!DCSyC)W%=UzfCVK(vsS+N*2pLC_Am!!syYLM-Aze>|G*4){o)<=x5UT*Q zY(?MSq{wX6tjraBuzZ^+_xn~5j$Jt&43xb#aRX(y8!X(?ZZ=}*GA_Ak05zt~kPW=I z!<@!UBt35#s{3x{#53~%o>3crOT-3y<5yV_He!Us$>W5(qUjoDbtQinL&#(hydGvX z){!9VG*)rM!bXhum0v}W8qNl%UWWLGj2aK^wb4)w#3i}ucKI2c{&tkB5|S5Wsi}IP zPP|WHIo6kfKF@Tf?(!G^!Zal@g=MB@i3xi|E`o<9(OY7ZiiQmzQDf7l!ZOnk%1mU% zegBX()<(yZZFCKzb^a;Km%Wh_P3i`pu+Ec(B(Gjr<24Hh^P(v~_>rK-4>EFEH5shz z@5_dYeeq_4es8E4;$f8fpptB84&np2md?Vs&JQkKl$zukz`H3P7MyyYw?_rr8UuKv zqBtzwVRD5r#{5+#=QA9Nz;PI4>s9tWy@zZM9o+49+aC4@@egB#ovJdC>+F@P&vl_X z8C!6~A16Ce-A<<1JJs#(X0v>UFIpI81&jSy7=QOY!m4Ym{{QRMszlK$tovgpY6i)@ zV~Pmn6#fYMwN_3lVP=ojHn}Tn<*iY{Ih8$jVqvJ2{XNPamDRf3Rf>}Ts;rjj>V76V zrm=Jf{w*=E$QkLBh?4c;+)`rhRL;dZHe>0sM-;eEv1b(r9U&PO~Yapn7TvV9pY_h}>NF|MbpN2+iNJuiKe&PIJE z>9ta?N-M*KL@}05M^x1~gMO=Oe%z`eRm!`=5XgYTSWI>}Zg=NQ5!v-g$PbN}Y+K~) zrW}Q`_NrDK-SCyfnOS`%C6MF5k)M79iKWO|(T+ZiPG zH*Uy)3}5Zne99`o)+g<8#*S5lT4c64lSX}_MTX|QZ=t!995_Q zxXW?15ab{q6jv_!a=!?LeyUNhYQQ~U^y7mm7P=H`j{%W3p|F0rjOEie_T*5#TC2x7 zg!4++U@B&V3?{);hRqgJoPn@ut-9(7QF@bdaX9Q9O1cdt`2HATFMr(UbR2%q{RNaa zSPJ?>NzL6$BRd<2>^?x`5kA_z>Xo+M6LdpeQrnwL98i#80tz`kI+75Ci1<$SZ(!(V zOcYR{?<^5);F#PTw8+>G_qZV&Mvv7SnEM>Z>HeuAFH)CAo$_btpwBVs-xt*42^jY{ z(Zf7Cq)AFhgL@=LGlNihJ{R>jx7+~;!fZ6qWzD0J17PRNb{)#~I#p^&Yli7<4K(u7LDbtKa9gbeF~ z=u9i(R>d>?f#ESLo}urA&iBkGc|n!FOm~QmNzL0-F}=6)k$?R~yVx z@jgntpGv0dNhMyF$J6BQY`uEL(Clh`uIbrAgVF7}a;Yi1)l=Wp-qG1*GKUE&QQahw zZih7jrpMiAx#;PzxLux(i%o8KS4XR*y{Y}GyDQu60egeUJ6I7il!l*Bg;#nrIr?`P Cl<*G# diff --git a/X1_JTAG_boot/jtag_boot_rpi4.img b/X1_JTAG_boot/jtag_boot_rpi4.img index 76531e6ff99de9580fcf1d1b05c8eed4404543ef..25e8aedfdf849d61eb75103719199b31f4b9c2d0 100755 GIT binary patch delta 1053 zcmZ8fT}TvB6h3!m)k)2Eb#+HecgB@f6Y?MWqt>-R${-h|qz5+=DTIpcp+L(u3@Qxv zb|a-8T8p3&qGWprD$_LRB`V75aaRcz|0?RmN~g0kw}-rN=ezfu@BG~F%-fpB)ipz~ zc}lf-u6hMHxD$lAAp*?+@DQMw5=0fz31rR>(CqB)y|=9E4r(tV?vX7iLiV5qt{JRg z7LCpTofd&!je zSkz|AL7QjGVhJH^G2=RWYI$ZyT4}GH#*0U_XbRC1bT7M-)GVkdB>143n1i0R@x~FIlUtoy6Fv&EY27>0C9@(#{UwT=)TW zp<2*7mpk6$YeW6LZn_rneqA2>ZQCT&t!HJKg&l@U1XUvR!iqi+2B>XQ#3>y$J@CgY zfZB&ZQK*3E9V?(B;t`LN#xSN+H#zzQdACvxOKEtQ#PdFB@_9f>hDU3VTQ+24+p^9H z<0^ZSRgj*jcn2#^0NusDWcBsLoU|0BPe`4RMM7=$9(;CBl7=G+sfhp=4iW?8M2o-l zn*UmgDrQT5mBx@G@Rk5POKe4V;M0aBiw9>U4QJxBj973{LV~_AvF#&^+tX8w!moF+ z2buZor+q^blEBYw*^bga0e!gI!Nf*>_tx)I_BU2kRw^xZKKwr?SyQ%yykf_)-?U1y z$u$D;b1BEXqk_iT9SEEV;OP{%=WzRK-cjyPS$IpEb#j)X!EkHtt{n*)5F7?1sF<^i}=yg?aZjo-ctO+#zmX$L+UdR+xL7NQ~xI qlyKW%u9?4qW0>Pfj)taHLA%KcarAR-WuC?KVnxwBg+;b8&;1wdoEY2y delta 1280 zcmZ8fO>7%g5T3VdC+mh1+p)88q3N#E#IaKHgM(;6A{&=Tw18ABgQ#*y-Nr#to5Wd{ zLk`F$qKKj(03lGjj)Dm^q3r3vYU6Td+m*Ho3|R%0+Lv(x@ArYQu?Uc)y*tU*=ML5i!Lu%yBLLs2T7a5Yfc`B4 z{dDO=K_LPJ*}Rx)-GW}TRy4|MK%|YF16Hz5mC|WsUlp|nVXD_(g$tr8EKgDji zE(Q=1?GMs&)}MWYtS&Xz50dSZJ@kfU z-7#ChkN{>&b;Q`%3v}l09x9dCDes%Yp%A;`4GYsdSgkMaR~xW^%P3=V>~VJg$00W8 zYxcr6YGRo5$eLF``6J0r`i6tok^7}IwJyU9@ zTTz}R&#?}Fr$C-(gZ^;NW`!Utgr2Ba^BRGdh7@t#vMd=84J)lwKpEHCs4C_Y@BQNx zwS5a=?o+9Vm?JAHaz^Ngg=nl4kfXx_c7o@1aO5to z$M?Ri9C)j{r$>3WKY^^yF*PWY(`--h=YxeN2Wto7XSvbQap3kD(Dex&|IToGBey?| zJI(!d+&{xoja@wj*~x~hVV`ffj+ENx>FVFI)yCUu_cq=W?`=2N+qrJP*7sMyL6y1G z*knldWkkoC=Tc+mWO+-$>sUW)V!z34!aE`MP<{u yr5kTnKhkwLo9WV1+05azu1{q~yT;OE6Z%xAo<;uw9`{%RC>1+}*O9Cc^8O2m9AoDI diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs index c85bb94b8..7513df074 100644 --- a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs +++ b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s index bfa94abf3..f4162c877 100644 --- a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s +++ b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/X1_JTAG_boot/src/bsp/raspberrypi/link.ld b/X1_JTAG_boot/src/bsp/raspberrypi/link.ld index 97ea6d69c..cf63a4a64 100644 --- a/X1_JTAG_boot/src/bsp/raspberrypi/link.ld +++ b/X1_JTAG_boot/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs b/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs index c6d65e36b..ef397a6f9 100644 --- a/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs +++ b/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/X1_JTAG_boot/src/main.rs b/X1_JTAG_boot/src/main.rs index 0dc87fa16..23221621b 100644 --- a/X1_JTAG_boot/src/main.rs +++ b/X1_JTAG_boot/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -119,10 +117,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/X1_JTAG_boot/src/memory.rs b/X1_JTAG_boot/src/memory.rs deleted file mode 100644 index 1f79e0c95..000000000 --- a/X1_JTAG_boot/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/X1_JTAG_boot/src/runtime_init.rs b/X1_JTAG_boot/src/runtime_init.rs deleted file mode 100644 index ee0946863..000000000 --- a/X1_JTAG_boot/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -}