#![no_std] #![no_main] use core::arch::{asm, global_asm}; use core::mem::{offset_of, size_of}; use core::panic::PanicInfo; use core::ptr::{read_volatile, write_volatile}; use core::sync::atomic::{AtomicUsize, Ordering}; use turkos_bootinfo::{BOOT_INFO_MAP_CAPACITY, BOOT_INFO_PAGE_BYTES}; use turkos_kernel::bootinfo::{self, BootInfoV1}; #[cfg(feature = "irq-service-test")] mod irq_service; #[cfg(feature = "p12-journal-test")] mod journal_probe; #[cfg(feature = "user-three-test")] mod process_probe; #[cfg(feature = "irq-spurious-vector-test")] mod spurious_vector; #[cfg(feature = "user-three-test")] global_asm!(include_str!("process_probe.S")); #[cfg(feature = "heap-test")] use turkos_kernel::heap; use turkos_kernel::serial_tx::SerialLineLock; #[cfg(feature = "smp-topology-test")] use turkos_kernel::smp::{self, StackSpan}; #[cfg(any(feature = "elf-hello-test", feature = "turkfs-init-test"))] mod elf_hello_probe; #[cfg(feature = "elf-ident-test")] mod elf_ident_probe; #[cfg(feature = "elf-layout-test")] mod elf_layout_probe; #[cfg(feature = "elf-load-test")] mod elf_load_probe; #[cfg(feature = "elf-sequence-test")] mod elf_sequence_probe; #[cfg(feature = "elf-stack-test")] mod elf_stack_probe; #[cfg(feature = "ipc-basic-test")] mod ipc_basic_probe; #[cfg(feature = "ipc-multi-test")] mod ipc_multi_probe; #[cfg(feature = "ipc-timeline-test")] mod ipc_timeline_probe; #[cfg(feature = "ipc-wait-test")] mod ipc_wait_probe; #[cfg(feature = "smp-allocator-test")] mod smp_allocator_probe; #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] use turkos_kernel::irq; #[cfg(any( feature = "task-state-test", feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "ipc-wait-test", feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] use turkos_kernel::task; #[cfg(any( feature = "task-state-test", feature = "task-switch-test", feature = "task-lifecycle-test" ))] use turkos_kernel::task::Scheduler; #[cfg(any(feature = "task-preempt-test", feature = "task-sleep-test"))] mod preempt; #[cfg(feature = "serial-arguments-test")] mod serial_arguments_probe; #[cfg(feature = "serial-cancel-test")] mod serial_cancel_probe; #[cfg(feature = "serial-console-test")] mod serial_console_probe; #[cfg(feature = "serial-echo-test")] mod serial_echo_probe; #[cfg(feature = "serial-edit-test")] mod serial_edit_probe; #[cfg(feature = "serial-limit-test")] mod serial_limit_probe; #[cfg(any( feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] mod serial_line_lock_probe; #[cfg(feature = "serial-line-test")] mod serial_line_probe; #[cfg(feature = "serial-range-test")] mod serial_range_probe; #[cfg(feature = "serial-rx-test")] mod serial_rx_probe; #[cfg(feature = "serial-unknown-test")] mod serial_unknown_probe; #[cfg(feature = "serial-utf8-test")] mod serial_utf8_probe; #[cfg(feature = "task-cancel-test")] mod task_cancel_probe; #[cfg(feature = "task-guard-test")] mod task_guard; #[cfg(feature = "turkfs-cat-test")] mod turkfs_cat_probe; #[cfg(feature = "turkfs-ls-test")] mod turkfs_ls_probe; #[cfg(any( feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test" ))] mod user_dual; #[cfg(any( feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test" ))] mod user_first; #[cfg(any(feature = "user-handle-test", feature = "user-handle-transfer-test"))] mod user_handle; #[cfg(feature = "user-handle-death-test")] mod user_handle_death; #[cfg(any( feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test" ))] mod user_syscall; #[cfg(any( feature = "user-task-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test" ))] mod user_task; #[cfg(feature = "user-wait-test")] mod user_wait; #[cfg(feature = "turkfs-init-test")] mod virtio_init_probe; #[cfg(feature = "virtio-magic-test")] mod virtio_magic_probe; #[cfg(feature = "virtio-modern-features-test")] mod virtio_modern_probe; #[cfg(any( feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test" ))] mod virtio_net_arp_reply_probe; #[cfg(feature = "virtio-net-id-test")] mod virtio_net_id_probe; #[cfg(feature = "virtio-net-mac-test")] mod virtio_net_mac_probe; #[cfg(feature = "virtio-net-pool-test")] mod virtio_net_pool_probe; #[cfg(all(feature = "virtio-net-rx-test", not(feature = "virtio-net-pool-test")))] mod virtio_net_rx_probe; #[cfg(feature = "virtio-net-tx-test")] mod virtio_net_tx_probe; #[cfg(all( feature = "virtio-modern-queue-test", not(feature = "turkfs-init-test"), not(feature = "turkfs-ls-test"), not(feature = "turkfs-cat-test"), not(feature = "p12-journal-test") ))] mod virtio_queue_probe; #[cfg(any(feature = "turkfs-ls-test", feature = "turkfs-cat-test"))] mod virtio_ro_turkfs; #[cfg(feature = "task-state-test")] fn task_state_smoke_test() -> bool { let mut scheduler = Scheduler::new(); if scheduler.run_next() != Ok(None) { return false; } write_line("TURKOS/TASK-IDLE: PASS"); let (Ok(a), Ok(b)) = (scheduler.spawn(), scheduler.spawn()) else { return false; }; if a == b || a.generation == b.generation || scheduler.state(a) != Ok(task::State::Ready) || scheduler.state(b) != Ok(task::State::Ready) { return false; } write_str("TURKOS/TASK-IDS: A-SLOT=0x"); write_hex(a.slot as usize); write_str(" A-GEN=0x"); write_hex(a.generation as usize); write_str(" B-SLOT=0x"); write_hex(b.slot as usize); write_str(" B-GEN=0x"); write_hex(b.generation as usize); write_str("\r\n"); write_line("TURKOS/TASK-STATE: PASS"); if scheduler.run_next() != Ok(Some(a)) || scheduler.yield_current(false) != Ok(Some(b)) || scheduler.yield_current(false) != Ok(Some(a)) || scheduler.ready_len() != 1 { return false; } write_line("TURKOS/TASK-QUEUE: PASS"); if scheduler.yield_current(true) != Err(task::Error::InInterrupt) || scheduler.running() != Some(a) { return false; } write_line("TURKOS/TASK-IRQ-YIELD: REJECTED"); if scheduler.sleep_current(10, 5) != Ok(Some(b)) || scheduler.state(a) != Ok(task::State::Sleeping { until: 10 }) || scheduler.wake_due(9) != Ok(0) { return false; } write_line("TURKOS/TASK-SLEEP-BEFORE: PASS"); if scheduler.wake_due(10) != Ok(1) || scheduler.wake_due(10) != Ok(0) || scheduler.yield_current(false) != Ok(Some(a)) { return false; } write_line("TURKOS/TASK-WAKE-AT: PASS"); if scheduler.exit_current(42) != Ok(Some(b)) || scheduler.state(a) != Ok(task::State::Exited { code: 42 }) || scheduler.reap(a) != Ok(42) { return false; } write_line("TURKOS/TASK-EXIT-CODE: 0x000000000000002a"); let Ok(reused) = scheduler.spawn() else { return false; }; if reused.slot != a.slot || reused.generation <= a.generation || scheduler.state(a) != Err(task::Error::InvalidTask) { return false; } write_str("TURKOS/TASK-REUSE: OLD-GEN=0x"); write_hex(a.generation as usize); write_str(" NEW-GEN=0x"); write_hex(reused.generation as usize); write_str("\r\n"); if scheduler.spawn().is_err() || scheduler.spawn().is_err() || scheduler.spawn() != Err(task::Error::Capacity) { return false; } write_line("TURKOS/TASK-CAPACITY: REJECTED"); if Scheduler::deadline(10, 4) != Ok(14) || Scheduler::deadline(u64::MAX, 1) != Err(task::Error::InvalidDeadline) || Scheduler::deadline(10, 0) != Err(task::Error::InvalidDeadline) { return false; } write_line("TURKOS/TASK-DEADLINE-OVERFLOW: REJECTED"); write_line("TURKOS/TASK-MODEL-ONLY: NO-CONTEXT-SWITCH"); true } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_switch_error(reason: &str) -> ! { write_str("TURKOS/TASK-SWITCH-ERROR: "); write_line(reason); park() } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_context_ptr(slot: usize) -> *mut task::CpuContext { // SAFETY: the only supported slots are A=0 and B=1, both owned by the // single masked-IRQ task-switch experiment. unsafe { core::ptr::addr_of_mut!(TASK_CONTEXTS) .cast::() .add(slot) } } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_step(name: &str, counter: *mut usize) { let mut sp: usize; unsafe { asm!("mov {sp}, sp", sp = out(reg) sp, options(nomem, nostack, preserves_flags)) }; // SAFETY: this test is single-CPU, IRQ-masked, and only the named task // writes its counter; no exclusive/atomic RMW is needed on QEMU/HVF. let index = unsafe { read_volatile(counter) + 1 }; unsafe { write_volatile(counter, index) }; write_str("TURKOS/TASK-STEP: ID="); write_str(name); write_str(" INDEX=0x"); write_hex(index); write_str(" SP=0x"); write_hex(sp); write_str("\r\n"); } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] #[inline(never)] fn task_yield() { // SAFETY: only one EL1 CPU runs this isolated image with IRQ masked. The // scheduler borrow ends before the assembly switches to another stack. let (old, new) = unsafe { let scheduler = &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER); let old = scheduler .running() .unwrap_or_else(|| task_switch_error("no running task")); let new = scheduler .yield_current(false) .unwrap_or_else(|_| task_switch_error("yield rejected")) .unwrap_or_else(|| task_switch_error("no ready task")); (old.slot as usize, new.slot as usize) }; if old == new { return; } // SAFETY: each context belongs to a distinct task stack; assembly saves // AAPCS64 callee registers, SIMD q8..q15 and SP before loading the next. unsafe { __turkos_swap_task_context(task_context_ptr(old), task_context_ptr(new)) }; } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_exit(code: i32) -> ! { let (old, next) = unsafe { let scheduler = &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER); let old = scheduler .running() .unwrap_or_else(|| task_switch_error("exit without task")); let next = scheduler .exit_current(code) .unwrap_or_else(|_| task_switch_error("exit rejected")); (old.slot as usize, next.map(|id| id.slot as usize)) }; let target = match next { Some(slot) => task_context_ptr(slot).cast_const(), None => core::ptr::addr_of!(TASK_BOOT_CONTEXT), }; unsafe { __turkos_swap_task_context(task_context_ptr(old), target) }; task_switch_error("exited task resumed") } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] extern "C" fn task_entry_a() -> ! { for _ in 0..4 { task_step("A", core::ptr::addr_of_mut!(TASK_A_COUNT)); task_yield(); } task_exit(0x0a) } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] extern "C" fn task_entry_b() -> ! { for _ in 0..4 { task_step("B", core::ptr::addr_of_mut!(TASK_B_COUNT)); if unsafe { read_volatile(core::ptr::addr_of!(TASK_B_COUNT)) } == 4 { write_line("TURKOS/TASK-WAIT: WAITER=B TARGET=A"); } task_yield(); } // The fourth yield leaves B ready while A exits. B then resumes on its // own stack and reads the retained exit state through A's original ID. let a = unsafe { *core::ptr::addr_of!(TASK_A_ID) } .unwrap_or_else(|| task_switch_error("missing A ID")); if unsafe { (&*core::ptr::addr_of!(TASK_SCHEDULER)).state(a) } != Ok(task::State::Exited { code: 0x0a }) { task_switch_error("waiter could not read A exit"); } write_line("TURKOS/TASK-WAIT-EXIT: WAITER=B TARGET=A CODE=0x000000000000000a"); task_exit(0x0b) } #[cfg(feature = "task-lifecycle-test")] extern "C" fn task_entry_c() -> ! { let mut sp: usize; unsafe { asm!("mov {value}, sp", value = out(reg) sp, options(nomem, nostack, preserves_flags)) }; let a_start = unsafe { core::ptr::addr_of!(TASK_A_STACK.0).cast::() as usize }; let current = unsafe { (&*core::ptr::addr_of!(TASK_SCHEDULER)).running() } .unwrap_or_else(|| task_switch_error("C has no running ID")); if current.slot != 0 || unsafe { (&*core::ptr::addr_of!(TASK_A_LEASE)).owner() } != Some(current) || sp < a_start + 16 || sp >= a_start + 16 * 1024 || sp & 15 != 0 || unsafe { read_volatile(a_start as *const u64) } != TASK_C_CANARY { task_switch_error("C stack ownership or bounds"); } unsafe { write_volatile(core::ptr::addr_of_mut!(TASK_C_COUNT), 1) }; write_str("TURKOS/TASK-LIFECYCLE-C: GEN=0x"); write_hex(current.generation as usize); write_str(" SP=0x"); write_hex(sp); write_str(" COUNT=0x"); write_hex(1); write_str("\r\n"); task_exit(0x0c) } #[cfg(feature = "task-lifecycle-test")] fn task_lifecycle_after_ab( a: task::TaskId, b: task::TaskId, a_start: usize, a_top: usize, b_start: usize, ) { let c = { let scheduler = unsafe { &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER) }; if scheduler.running().is_some() || scheduler.ready_len() != 0 || scheduler.run_next() != Ok(None) || scheduler.state(a) != Ok(task::State::Exited { code: 0x0a }) || scheduler.state(b) != Ok(task::State::Exited { code: 0x0b }) || unsafe { (&*core::ptr::addr_of!(TASK_A_LEASE)).owner() } != Some(a) || unsafe { (&*core::ptr::addr_of!(TASK_B_LEASE)).owner() } != Some(b) { task_switch_error("exited task was ready or stack owner changed"); } write_str("TURKOS/TASK-LIFECYCLE-RETIRED: A-GEN=0x"); write_hex(a.generation as usize); write_str(" B-GEN=0x"); write_hex(b.generation as usize); write_str(" READY=0x"); write_hex(scheduler.ready_len()); write_str("\r\n"); if scheduler.reap(a) != Ok(0x0a) || scheduler.reap(b) != Ok(0x0b) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).release(a) } != Ok(()) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_B_LEASE)).release(b) } != Ok(()) || scheduler.state(a) != Err(task::Error::InvalidTask) || scheduler.state(b) != Err(task::Error::InvalidTask) { task_switch_error("reap or lease release"); } unsafe { task_context_ptr(0).write(task::CpuContext::EMPTY); task_context_ptr(1).write(task::CpuContext::EMPTY); core::ptr::write_bytes(a_start as *mut u8, 0xdd, 16 * 1024); core::ptr::write_bytes(b_start as *mut u8, 0xdd, 16 * 1024); write_volatile(a_start as *mut u64, TASK_C_CANARY); write_volatile(b_start as *mut u64, TASK_B_POISON); } if unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).claim(a) } != Err(task::Error::InvalidTask) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_B_LEASE)).claim(b) } != Err(task::Error::InvalidTask) { task_switch_error("old generation reclaimed freed stack"); } write_str("TURKOS/TASK-LIFECYCLE-RELEASED: A=0x"); write_hex(a_start); write_str(" B=0x"); write_hex(b_start); write_str(" B-POISON=0x"); write_hex(unsafe { read_volatile(b_start as *const u64) } as usize); write_str("\r\n"); let c = scheduler .spawn() .unwrap_or_else(|_| task_switch_error("spawn C after reap")); if c.slot != a.slot || c.generation <= b.generation || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).claim(c) } != Ok(()) || unsafe { (&*core::ptr::addr_of!(TASK_B_LEASE)).owner() }.is_some() { task_switch_error("new generation stack claim"); } let c_entry = task_entry_c as *const () as usize; let context = task::CpuContext::initial(a_top as u64, c_entry as u64) .unwrap_or_else(|_| task_switch_error("C first context")); unsafe { task_context_ptr(0).write(context) }; write_str("TURKOS/TASK-LIFECYCLE-REUSE: OLD-GEN=0x"); write_hex(a.generation as usize); write_str(" NEW-GEN=0x"); write_hex(c.generation as usize); write_str(" STACK=0x"); write_hex(a_start); write_str("\r\n"); if scheduler.run_next() != Ok(Some(c)) { task_switch_error("C first run"); } c }; unsafe { __turkos_swap_task_context( core::ptr::addr_of_mut!(TASK_BOOT_CONTEXT), task_context_ptr(0), ); } let scheduler = unsafe { &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER) }; if unsafe { read_volatile(core::ptr::addr_of!(TASK_C_COUNT)) } != 1 || scheduler.running().is_some() || scheduler.run_next() != Ok(None) || scheduler.state(c) != Ok(task::State::Exited { code: 0x0c }) || scheduler.state(a) != Err(task::Error::InvalidTask) || scheduler.state(b) != Err(task::Error::InvalidTask) || unsafe { (&*core::ptr::addr_of!(TASK_A_LEASE)).owner() } != Some(c) || unsafe { (&*core::ptr::addr_of!(TASK_B_LEASE)).owner() }.is_some() || unsafe { read_volatile(a_start as *const u64) } != TASK_C_CANARY || unsafe { read_volatile(b_start as *const u64) } != TASK_B_POISON { task_switch_error("C exit or retired stack changed"); } write_str( "TURKOS/TASK-LIFECYCLE-EXIT: C=0x000000000000000c OLD-A=REJECTED OLD-B=REJECTED C-GUARD=0x", ); write_hex(unsafe { read_volatile(a_start as *const u64) } as usize); write_str(" B-POISON=0x"); write_hex(unsafe { read_volatile(b_start as *const u64) } as usize); write_str("\r\n"); if scheduler.reap(c) != Ok(0x0c) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).release(c) } != Ok(()) || scheduler.state(c) != Err(task::Error::InvalidTask) { task_switch_error("C final reap"); } unsafe { task_context_ptr(0).write(task::CpuContext::EMPTY) }; write_line("TURKOS/TASK-LIFECYCLE: PASS"); } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_switch_smoke_test() -> ! { // SAFETY: the two dedicated, page-aligned BSS stacks are exclusive to // this image. The low canaries remain outside every downward stack frame. let a_start = unsafe { core::ptr::addr_of_mut!(TASK_A_STACK.0).cast::() as usize }; let b_start = unsafe { core::ptr::addr_of_mut!(TASK_B_STACK.0).cast::() as usize }; let a_top = a_start + 16 * 1024; let b_top = b_start + 16 * 1024; if a_start & 4095 != 0 || b_start & 4095 != 0 || (a_start < b_top && b_start < a_top) { task_switch_error("stack bounds"); } unsafe { write_volatile(a_start as *mut u64, TASK_A_CANARY); write_volatile(b_start as *mut u64, TASK_B_CANARY); } for (name, start, top) in [("A", a_start, a_top), ("B", b_start, b_top)] { write_str("TURKOS/TASK-STACK: ID="); write_str(name); write_str(" START=0x"); write_hex(start); write_str(" TOP=0x"); write_hex(top); write_str("\r\n"); } let (a, b) = unsafe { let scheduler = &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER); let a = scheduler .spawn() .unwrap_or_else(|_| task_switch_error("spawn A")); let b = scheduler .spawn() .unwrap_or_else(|_| task_switch_error("spawn B")); (a, b) }; if a.slot != 0 || b.slot != 1 { task_switch_error("task ID order"); } #[cfg(feature = "task-lifecycle-test")] if unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).claim(a) } != Ok(()) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_B_LEASE)).claim(b) } != Ok(()) { task_switch_error("initial stack leases"); } unsafe { core::ptr::addr_of_mut!(TASK_A_ID).write(Some(a)) }; let a_entry = task_entry_a as *const () as usize; let b_entry = task_entry_b as *const () as usize; let a_context = task::CpuContext::initial(a_top as u64, a_entry as u64) .unwrap_or_else(|_| task_switch_error("first frame A")); let b_context = task::CpuContext::initial(b_top as u64, b_entry as u64) .unwrap_or_else(|_| task_switch_error("first frame B")); unsafe { task_context_ptr(0).write(a_context); task_context_ptr(1).write(b_context); } write_str("TURKOS/TASK-ENTRY: A=0x"); write_hex(a_entry); write_str(" B=0x"); write_hex(b_entry); write_str("\r\n"); let first = unsafe { (&mut *core::ptr::addr_of_mut!(TASK_SCHEDULER)).run_next() }; if first != Ok(Some(a)) { task_switch_error("first run"); } unsafe { __turkos_swap_task_context( core::ptr::addr_of_mut!(TASK_BOOT_CONTEXT), task_context_ptr(0), ) }; // Both task exits must have returned here without ever running an exited // context or changing either low-end stack canary. let scheduler = unsafe { &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER) }; if unsafe { read_volatile(core::ptr::addr_of!(TASK_A_COUNT)) } != 4 || unsafe { read_volatile(core::ptr::addr_of!(TASK_B_COUNT)) } != 4 || scheduler.running().is_some() || scheduler.run_next() != Ok(None) || scheduler.state(a) != Ok(task::State::Exited { code: 0x0a }) || scheduler.state(b) != Ok(task::State::Exited { code: 0x0b }) || unsafe { read_volatile(a_start as *const u64) } != TASK_A_CANARY || unsafe { read_volatile(b_start as *const u64) } != TASK_B_CANARY { task_switch_error("final counters/state/guards"); } write_str("TURKOS/TASK-STACK-GUARDS: A=0x"); write_hex(TASK_A_CANARY as usize); write_str(" B=0x"); write_hex(TASK_B_CANARY as usize); write_str("\r\n"); write_line("TURKOS/TASK-EXIT: A=0x000000000000000a B=0x000000000000000b"); write_line("TURKOS/TASK-COUNTS: A=0x0000000000000004 B=0x0000000000000004"); #[cfg(feature = "task-lifecycle-test")] task_lifecycle_after_ab(a, b, a_start, a_top, b_start); write_line("TURKOS/TASK-SWITCH: PASS"); write_line("TURKOS/TASK-IDLE: EMPTY-QUEUE-WFE"); park() } #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] use turkos_kernel::mmu; use turkos_kernel::{ dtb, exception::{self, ExceptionFrame}, memory, }; global_asm!(include_str!("boot.S")); #[cfg(feature = "smp-boot-protocol-test")] global_asm!(include_str!("smp_boot.S"), vector_probe = const cfg!(feature = "smp-vector-test") as usize, timer_probe = const cfg!(feature = "smp-timer-test") as usize); #[cfg(feature = "smp-allocator-test")] global_asm!(include_str!("smp_allocator_boot.S")); #[cfg(feature = "irq-gp-test")] global_asm!(include_str!("gp_probe.S")); #[cfg(feature = "mmu-tlb-test")] global_asm!(include_str!("tlb_probe.S")); #[cfg(any( feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "ipc-wait-test", feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] global_asm!( include_str!("task_switch.S"), sp_offset = const offset_of!(task::CpuContext, sp), callee_offset = const offset_of!(task::CpuContext, callee), vector_offset = const offset_of!(task::CpuContext, vector), ); global_asm!( include_str!("vectors.S"), frame_size = const size_of::(), x3_offset = const offset_of!(ExceptionFrame, registers) + 3 * size_of::(), sp_offset = const offset_of!(ExceptionFrame, interrupted_sp), canary_offset = const offset_of!(ExceptionFrame, canary), canary_value = const exception::FRAME_CANARY, return_test = const cfg!(feature = "irq-return-test") as usize, guard_test = const cfg!(feature = "task-guard-test") as usize, user_test = const (cfg!(feature = "user-three-test") || cfg!(feature = "user-first-test") || cfg!(feature = "user-code-write-test") || cfg!(feature = "user-data-exec-test") || cfg!(feature = "user-stack-overflow-test") || cfg!(feature = "user-align-test") || cfg!(feature = "user-dual-test") || cfg!(feature = "user-dual-fault-test") || cfg!(feature = "user-dual-stack-fault-test") || cfg!(feature = "user-peer-read-test") || cfg!(feature = "user-mmio-read-test") || cfg!(feature = "user-asid-test") || cfg!(feature = "elf-load-test") || cfg!(feature = "elf-stack-test")) as usize, user_syscall = const (cfg!(feature = "user-syscall-test") || cfg!(feature = "user-copy-test") || cfg!(feature = "ipc-user-test") || cfg!(feature = "user-task-test") || cfg!(feature = "user-clock-test") || cfg!(feature = "user-handle-test") || cfg!(feature = "user-handle-transfer-test") || cfg!(feature = "user-handle-death-test") || cfg!(feature = "user-wait-test") || cfg!(feature = "user-mmio-request-test") || cfg!(feature = "elf-hello-test") || cfg!(feature = "elf-sequence-test") || cfg!(feature = "turkfs-init-test")) as usize, el0_irq = const cfg!(feature = "user-el0-irq-test") as usize, sgi_test = const (cfg!(feature = "irq-sgi-test") || cfg!(feature = "irq-timer-test") || cfg!(feature = "irq-unknown-test") || cfg!(feature = "task-preempt-test") || cfg!(feature = "task-sleep-test") || cfg!(feature = "user-copy-irq-test")) as usize, ); #[cfg(feature = "task-guard-test")] global_asm!(include_str!("task_guard.S")); #[cfg(any( feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test", feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test", feature = "elf-load-test", feature = "elf-stack-test" ))] global_asm!(include_str!("user_first.S"), write_fault = const cfg!(feature = "user-code-write-test") as usize, data_exec = const cfg!(feature = "user-data-exec-test") as usize, stack_overflow = const cfg!(feature = "user-stack-overflow-test") as usize, unaligned = const cfg!(feature = "user-align-test") as usize, dual_space = const (cfg!(feature = "user-dual-test") || cfg!(feature = "user-dual-fault-test") || cfg!(feature = "user-dual-stack-fault-test") || cfg!(feature = "user-peer-read-test") || cfg!(feature = "user-mmio-read-test") || cfg!(feature = "user-asid-test")) as usize, dual_fault = const (cfg!(feature = "user-dual-fault-test") || cfg!(feature = "user-peer-read-test") || cfg!(feature = "user-mmio-read-test")) as usize, dual_stack_fault = const cfg!(feature = "user-dual-stack-fault-test") as usize, elf_load = const cfg!(feature = "elf-load-test") as usize, elf_stack = const cfg!(feature = "elf-stack-test") as usize); #[cfg(any( feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] global_asm!(include_str!("elf_hello.S")); #[cfg(any( feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test" ))] global_asm!(include_str!("user_syscall.S"), copy_test = const cfg!(feature = "user-copy-test") as usize, clock_test = const cfg!(feature = "user-clock-test") as usize, mmio_test = const cfg!(feature = "user-mmio-request-test") as usize, ipc_test = const cfg!(feature = "ipc-user-test") as usize); #[cfg(feature = "user-el0-irq-test")] global_asm!(include_str!("user_el0_irq.S")); #[cfg(feature = "user-task-test")] global_asm!(include_str!("user_task.S")); #[cfg(feature = "user-handle-test")] global_asm!(include_str!("user_handle.S")); #[cfg(feature = "user-handle-transfer-test")] global_asm!(include_str!("user_handle_transfer.S")); #[cfg(feature = "user-handle-death-test")] global_asm!(include_str!("user_handle_death.S")); #[cfg(feature = "user-wait-test")] global_asm!(include_str!("user_wait.S")); #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] global_asm!( include_str!("return.S"), ctx_size = const size_of::(), sp_offset = const offset_of!(irq::IrqContext, interrupted_sp), sp_el0_offset = const offset_of!(irq::IrqContext, sp_el0), elr_offset = const offset_of!(irq::IrqContext, elr), spsr_offset = const offset_of!(irq::IrqContext, spsr), q_offset = const offset_of!(irq::IrqContext, q), fpcr_offset = const offset_of!(irq::IrqContext, fpcr), fpsr_offset = const offset_of!(irq::IrqContext, fpsr), slot_offset = const offset_of!(irq::IrqContext, vector_slot), nested_irq = const cfg!(feature = "user-copy-irq-test") as usize, ); const _: () = { let tests = cfg!(feature = "p12-journal-test") as u8 + cfg!(feature = "panic-test") as u8 + cfg!(feature = "exception-test") as u8 + cfg!(feature = "fault-test") as u8 + cfg!(feature = "frame-stress-test") as u8 + cfg!(feature = "mmu-test") as u8 + cfg!(feature = "mmu-no-map-test") as u8 + cfg!(feature = "mmu-write-test") as u8 + cfg!(feature = "mmu-nx-test") as u8 + cfg!(feature = "mmu-rodata-test") as u8 + cfg!(feature = "mmu-rodata-exec-test") as u8 + cfg!(feature = "mmu-stack-exec-test") as u8 + cfg!(feature = "mmu-mmio-exec-test") as u8 + cfg!(feature = "mmu-panic-test") as u8 + cfg!(feature = "mmu-brk-test") as u8 + cfg!(feature = "mmu-udf-test") as u8 + cfg!(feature = "mmu-guard-test") as u8 + cfg!(feature = "mmu-normal-test") as u8 + cfg!(feature = "mmu-tlb-test") as u8 + cfg!(feature = "el0-fault-test") as u8 + cfg!(feature = "nested-fault-test") as u8 + cfg!(feature = "irq-return-test") as u8 + (cfg!(feature = "irq-sgi-test") && !cfg!(feature = "smp-irq-lock-test")) as u8 + cfg!(feature = "irq-timer-test") as u8 + cfg!(feature = "irq-unknown-test") as u8 + cfg!(feature = "heap-test") as u8 + cfg!(feature = "task-state-test") as u8 + cfg!(feature = "task-switch-test") as u8 + cfg!(feature = "task-lifecycle-test") as u8 + cfg!(feature = "task-guard-test") as u8 + cfg!(feature = "user-first-test") as u8 + (cfg!(feature = "user-three-test") && !cfg!(feature = "process-exit-irq-test")) as u8 + cfg!(feature = "user-code-write-test") as u8 + cfg!(feature = "user-data-exec-test") as u8 + cfg!(feature = "user-stack-overflow-test") as u8 + cfg!(feature = "user-align-test") as u8 + cfg!(feature = "user-dual-test") as u8 + cfg!(feature = "user-dual-fault-test") as u8 + cfg!(feature = "user-dual-stack-fault-test") as u8 + cfg!(feature = "user-peer-read-test") as u8 + cfg!(feature = "user-mmio-read-test") as u8 + cfg!(feature = "user-asid-test") as u8 + cfg!(feature = "user-syscall-test") as u8 + cfg!(feature = "user-copy-test") as u8 + cfg!(feature = "ipc-user-test") as u8 + cfg!(feature = "user-copy-irq-test") as u8 + cfg!(feature = "elf-ident-test") as u8 + cfg!(feature = "elf-layout-test") as u8 + cfg!(feature = "elf-load-test") as u8 + cfg!(feature = "elf-stack-test") as u8 + cfg!(feature = "elf-hello-test") as u8 + cfg!(feature = "turkfs-init-test") as u8 + cfg!(feature = "serial-rx-test") as u8 + cfg!(feature = "serial-line-test") as u8 + cfg!(feature = "serial-edit-test") as u8 + cfg!(feature = "serial-limit-test") as u8 + cfg!(feature = "serial-console-test") as u8 + cfg!(feature = "serial-unknown-test") as u8 + cfg!(feature = "serial-utf8-test") as u8 + cfg!(feature = "serial-arguments-test") as u8 + cfg!(feature = "serial-range-test") as u8 + cfg!(feature = "serial-cancel-test") as u8 + cfg!(feature = "serial-line-lock-test") as u8 + cfg!(feature = "serial-line-unlocked-test") as u8 + cfg!(feature = "virtio-net-id-test") as u8 + cfg!(feature = "virtio-net-mac-test") as u8 + (cfg!(feature = "virtio-net-rx-test") && !cfg!(feature = "virtio-net-rx-completion-test") && !cfg!(feature = "virtio-net-pool-test")) as u8 + (cfg!(feature = "virtio-net-rx-completion-test") && !cfg!(feature = "virtio-net-broadcast-test") && !cfg!(feature = "virtio-net-unicast-test")) as u8 + (cfg!(feature = "virtio-net-broadcast-test") && !cfg!(feature = "virtio-net-arp-request-test")) as u8 + cfg!(feature = "virtio-net-arp-request-test") as u8 + cfg!(feature = "virtio-net-arp-reply-test") as u8 + cfg!(feature = "virtio-net-icmp-echo-test") as u8 + cfg!(feature = "virtio-net-reset-traffic-test") as u8 + (cfg!(feature = "virtio-net-unicast-test") && !cfg!(feature = "virtio-net-ipv4-header-test")) as u8 + (cfg!(feature = "virtio-net-ipv4-header-test") && !cfg!(feature = "virtio-net-ipv4-checksum-test")) as u8 + cfg!(feature = "virtio-net-ipv4-checksum-test") as u8 + cfg!(feature = "virtio-net-pool-test") as u8 + (cfg!(feature = "virtio-net-tx-test") && !cfg!(feature = "virtio-net-size-test") && !cfg!(feature = "virtio-net-tx-completion-test")) as u8 + cfg!(feature = "virtio-net-tx-completion-test") as u8 + (cfg!(feature = "virtio-net-size-test") && !cfg!(feature = "virtio-net-header-test")) as u8 + (cfg!(feature = "virtio-net-header-test") && !cfg!(feature = "virtio-net-type-test")) as u8 + cfg!(feature = "virtio-net-type-test") as u8 + cfg!(feature = "serial-echo-test") as u8 + cfg!(feature = "turkfs-ls-test") as u8 + cfg!(feature = "turkfs-cat-test") as u8 + cfg!(feature = "elf-sequence-test") as u8 + cfg!(feature = "ipc-basic-test") as u8 + cfg!(feature = "ipc-wait-test") as u8 + cfg!(feature = "task-cancel-test") as u8 + cfg!(feature = "ipc-multi-test") as u8 + cfg!(feature = "ipc-timeline-test") as u8 + cfg!(feature = "user-clock-test") as u8 + cfg!(feature = "user-mmio-request-test") as u8 + cfg!(feature = "user-handle-test") as u8 + cfg!(feature = "user-handle-transfer-test") as u8 + cfg!(feature = "user-handle-death-test") as u8 + cfg!(feature = "user-wait-test") as u8 + cfg!(feature = "user-task-test") as u8 + cfg!(feature = "task-preempt-test") as u8 + cfg!(feature = "task-sleep-test") as u8 + (cfg!(feature = "smp-boot-protocol-test") && !cfg!(feature = "smp-topology-test")) as u8 + (cfg!(feature = "smp-topology-test") && !cfg!(feature = "smp-vector-test")) as u8 + (cfg!(feature = "smp-vector-test") && !cfg!(feature = "smp-timer-test")) as u8 + cfg!(feature = "smp-timer-test") as u8 + (cfg!(feature = "smp-allocator-test") && !cfg!(feature = "smp-ready-test")) as u8 + (cfg!(feature = "smp-ready-test") && !cfg!(feature = "smp-serial-test")) as u8 + (cfg!(feature = "smp-serial-test") && !cfg!(feature = "smp-owner-test")) as u8 + (cfg!(feature = "smp-owner-test") && !cfg!(feature = "smp-irq-lock-test")) as u8 + (cfg!(feature = "smp-irq-lock-test") && !cfg!(feature = "smp-order-test")) as u8 + (cfg!(feature = "smp-order-test") && !cfg!(feature = "smp-fair-test")) as u8 + (cfg!(feature = "smp-fair-test") && !cfg!(feature = "smp-wait-test")) as u8 + (cfg!(feature = "smp-wait-test") && !cfg!(feature = "smp-time-test")) as u8 + (cfg!(feature = "smp-time-test") && !cfg!(feature = "smp-stats-test")) as u8 + (cfg!(feature = "smp-stats-test") && !cfg!(feature = "smp-wait-class-test")) as u8 + (cfg!(feature = "smp-wait-class-test") && !cfg!(feature = "smp-hold-test")) as u8 + (cfg!(feature = "smp-hold-test") && !cfg!(feature = "smp-counter-baseline-test")) as u8 + (cfg!(feature = "smp-counter-baseline-test") && !cfg!(feature = "smp-classified-admission-test")) as u8 + (cfg!(feature = "smp-classified-admission-test") && !cfg!(feature = "smp-race-trace-test")) as u8 + cfg!(feature = "smp-race-trace-test") as u8 - cfg!(feature = "user-copy-irq-test") as u8; assert!( tests <= 1, "test images must be built with at most one test feature" ); }; unsafe extern "C" { fn __vectors_start(); static _image_start: u8; #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static _text_start: u8; static _image_end: u8; #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static __irq_guard_page_start: u8; #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static _text_end: u8; #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static _rodata_end: u8; #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] static mut __irq_stack_guard: u64; #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] static __irq_stack_start: u8; #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] static __irq_stack_top: u8; #[cfg(any( feature = "irq-sgi-test", feature = "task-guard-test", feature = "smp-topology-test" ))] static mut __stack_start: u64; #[cfg(any( feature = "irq-sgi-test", feature = "task-guard-test", feature = "smp-topology-test" ))] static __stack_top: u8; #[cfg(feature = "irq-gp-test")] fn __gp_probe_run(command: usize, output: *mut u64); #[cfg(feature = "smp-boot-protocol-test")] fn __secondary_entry(); #[cfg(feature = "smp-boot-protocol-test")] static __secondary_stack_start: u8; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_stack_top: u8; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_ready: u64; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_mpidr: u64; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_context: u64; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_sp: u64; #[cfg(feature = "smp-vector-test")] fn __smp_vectors_start(); #[cfg(feature = "smp-vector-test")] fn __secondary_brk_pc(); #[cfg(feature = "smp-vector-test")] static mut __secondary_vector_command: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_seen: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_esr: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_elr: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_mpidr: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_sp: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_vbar: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_freq: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_start: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_deadline0: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_tick0: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_deadline1: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_tick1: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_mpidr: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_seen: u64; } #[cfg(feature = "smp-boot-protocol-test")] fn psci_hvc(fid: usize, target: usize, entry: usize, context: usize) -> usize { let result; // SAFETY: this probe runs only on QEMU virt with a DTB-confirmed `hvc` // PSCI conduit. The ABI returns x0 and does not use the guest stack. unsafe { asm!( "hvc #0", inlateout("x0") fid => result, in("x1") target, in("x2") entry, in("x3") context, options(nostack) ); } result } #[cfg(feature = "smp-boot-protocol-test")] fn smp_boot_protocol_test() -> ! { const TARGET: usize = 1; const CONTEXT: usize = 0x461; const PSCI_CPU_ON: usize = 0xc400_0003; let image_start = core::ptr::addr_of!(_image_start) as usize; let image_end = core::ptr::addr_of!(_image_end) as usize; let entry = __secondary_entry as *const () as usize; let stack_start = core::ptr::addr_of!(__secondary_stack_start) as usize; let stack_top = core::ptr::addr_of!(__secondary_stack_top) as usize; if entry < image_start || entry >= image_end || entry & 3 != 0 || stack_top - stack_start != 4096 { write_line("TURKOS/SMP-ERROR: entry or stack bounds"); park(); } let version = psci_hvc(0x8400_0000, 0, 0, 0); let result = psci_hvc(PSCI_CPU_ON, TARGET, entry, CONTEXT); let mut ready = 0; for _ in 0..100_000_000 { // SAFETY: only the secondary writes the one-word publication marker. ready = unsafe { read_volatile(core::ptr::addr_of!(__secondary_ready)) }; if ready == 1 { // SAFETY: the secondary publishes MPIDR/context/SP before READY. unsafe { asm!("dsb sy", options(nostack, preserves_flags)) }; break; } core::hint::spin_loop(); } let duplicate = psci_hvc(PSCI_CPU_ON, TARGET, entry, CONTEXT); let invalid = psci_hvc(PSCI_CPU_ON, 2, entry, CONTEXT); let mpidr = unsafe { read_volatile(core::ptr::addr_of!(__secondary_mpidr)) }; let context = unsafe { read_volatile(core::ptr::addr_of!(__secondary_context)) }; let secondary_sp = unsafe { read_volatile(core::ptr::addr_of!(__secondary_sp)) }; write_str("TURKOS/SMP-PSCI: VERSION=0x"); write_hex(version); write_str(" ON=0x"); write_hex(result); write_str(" DUP=0x"); write_hex(duplicate); write_str(" BAD=0x"); write_hex(invalid); write_str("\r\nTURKOS/SMP-SECONDARY: ENTRY=0x"); write_hex(entry); write_str(" MPIDR=0x"); write_hex(mpidr as usize); write_str(" CONTEXT=0x"); write_hex(context as usize); write_str(" SP=0x"); write_hex(secondary_sp as usize); write_str(" STACK=0x"); write_hex(stack_start); write_str("..0x"); write_hex(stack_top); write_str(" READY=0x"); write_hex(ready as usize); write_str("\r\n"); if version >= 0x0001_0000 && result == 0 && duplicate == usize::MAX - 3 && invalid == usize::MAX - 1 && ready == 1 && mpidr & 0xff == TARGET as u64 && context == CONTEXT as u64 && secondary_sp == stack_top as u64 { write_line("TURKOS/SMP-PROTOCOL: PASS"); } else { write_line("TURKOS/SMP-ERROR: PSCI or secondary contract"); park(); } #[cfg(feature = "smp-topology-test")] smp_topology_test(mpidr, secondary_sp, stack_start, stack_top); #[cfg(feature = "smp-vector-test")] smp_vector_test(mpidr, stack_start, stack_top); #[cfg(feature = "smp-timer-test")] smp_timer_test(mpidr); park(); } #[cfg(feature = "smp-vector-test")] fn smp_vector_test(secondary_mpidr: u64, stack_start: usize, stack_top: usize) { let primary_vbar: usize; // SAFETY: VBAR_EL1 is a per-CPU register; this read is on the boot CPU. unsafe { asm!("mrs {value}, vbar_el1", value = out(reg) primary_vbar, options(nomem, nostack, preserves_flags)) }; let expected_primary = __vectors_start as *const () as usize; let expected_secondary = __smp_vectors_start as *const () as usize; let brk_pc = __secondary_brk_pc as *const () as usize; if primary_vbar != expected_primary || expected_primary & 2047 != 0 || expected_secondary & 2047 != 0 || expected_primary == expected_secondary { write_line("TURKOS/SMP-ERROR: vector table identity or alignment"); park(); } // SAFETY: only the primary writes the command; the secondary reads it // after the full-system barrier and event. The secondary never returns. unsafe { write_volatile(core::ptr::addr_of_mut!(__secondary_vector_command), 1); asm!("dsb sy", "sev", options(nostack, preserves_flags)); } let mut seen = 0; for _ in 0..100_000_000 { // SAFETY: the secondary publishes all fault fields before SEEN. seen = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_seen)) }; if seen != 0 { unsafe { asm!("dsb sy", options(nostack, preserves_flags)) }; break; } core::hint::spin_loop(); } let esr = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_esr)) }; let elr = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_elr)) }; let mpidr = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_mpidr)) }; let sp = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_sp)) }; let vbar = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_vbar)) }; let primary_vbar_after: usize; unsafe { asm!("mrs {value}, vbar_el1", value = out(reg) primary_vbar_after, options(nomem, nostack, preserves_flags)) }; write_str("TURKOS/SMP-VECTOR: PRIMARY=0x"); write_hex(primary_vbar); write_str(" SECONDARY=0x"); write_hex(expected_secondary); write_str(" BRK-PC=0x"); write_hex(brk_pc); write_str(" PRIMARY-AFTER=0x"); write_hex(primary_vbar_after); write_str("\r\nTURKOS/SMP-FAULT: CPU=0x"); write_hex(mpidr as usize); write_str(" ESR=0x"); write_hex(esr as usize); write_str(" ELR=0x"); write_hex(elr as usize); write_str(" SP=0x"); write_hex(sp as usize); write_str(" VBAR=0x"); write_hex(vbar as usize); write_str(" SEEN=0x"); write_hex(seen as usize); write_str("\r\n"); if smp::validate_secondary_brk( smp::VectorFault { esr, elr, mpidr, sp, vbar, seen, }, secondary_mpidr, brk_pc as u64, expected_secondary as u64, StackSpan { start: stack_start as u64, end: stack_top as u64, sp, }, ) .is_ok() && primary_vbar_after == primary_vbar { write_line("TURKOS/SMP-VECTOR: PASS"); } else { write_line("TURKOS/SMP-ERROR: secondary exception source or vector state"); park(); } } #[cfg(feature = "smp-timer-test")] fn smp_timer_test(secondary_mpidr: u64) { use turkos_kernel::smp::TimerSample; let primary_mpidr: u64; let frequency: u64; let start: u64; // SAFETY: all physical timer registers below are accessed only on the // primary CPU; secondary uses the same banked registers in assembly. unsafe { asm!("mrs {value}, mpidr_el1", value = out(reg) primary_mpidr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, cntfrq_el0", value = out(reg) frequency, options(nomem, nostack, preserves_flags)); } let period = frequency / 100; if period == 0 { write_line("TURKOS/SMP-ERROR: timer frequency"); park(); } // SAFETY: the secondary reads command 2 after dsb/sev. It is already // parked after its controlled BRK and has a private stack/vector table. unsafe { write_volatile(core::ptr::addr_of_mut!(__secondary_vector_command), 2); asm!("dsb sy", "sev", options(nostack, preserves_flags)); asm!("mrs {value}, cntpct_el0", value = out(reg) start, options(nomem, nostack, preserves_flags)); } let Some(deadline0) = start.checked_add(period) else { write_line("TURKOS/SMP-ERROR: timer deadline overflow"); park(); }; unsafe { asm!("msr cntp_cval_el0, {deadline}", "msr cntp_ctl_el0, {enable}", "isb", deadline = in(reg) deadline0, enable = in(reg) 1u64, options(nomem, nostack)) }; let mut status = 0u64; for _ in 0..100_000_000 { unsafe { asm!("mrs {value}, cntp_ctl_el0", value = out(reg) status, options(nomem, nostack, preserves_flags)) }; if status & 4 != 0 { break; } core::hint::spin_loop(); } if status & 4 == 0 { write_line("TURKOS/SMP-ERROR: primary timer first expiry"); park(); } let tick0: u64; unsafe { asm!("mrs {value}, cntpct_el0", value = out(reg) tick0, options(nomem, nostack, preserves_flags)) }; let Some(deadline1) = tick0.checked_add(period) else { write_line("TURKOS/SMP-ERROR: timer second deadline overflow"); park(); }; unsafe { asm!("msr cntp_cval_el0, {deadline}", "isb", deadline = in(reg) deadline1, options(nomem, nostack)) }; status = 0; for _ in 0..100_000_000 { unsafe { asm!("mrs {value}, cntp_ctl_el0", value = out(reg) status, options(nomem, nostack, preserves_flags)) }; if status & 4 != 0 { break; } core::hint::spin_loop(); } let tick1: u64; unsafe { asm!("mrs {value}, cntpct_el0", value = out(reg) tick1, options(nomem, nostack, preserves_flags)); asm!("msr cntp_ctl_el0, xzr", "isb", options(nomem, nostack)); } if status & 4 == 0 { write_line("TURKOS/SMP-ERROR: primary timer second expiry"); park(); } let primary = TimerSample { mpidr: primary_mpidr, frequency, start, deadline0, tick0, deadline1, tick1, seen: 1, }; let mut seen = 0; for _ in 0..100_000_000 { seen = unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_seen)) }; if seen != 0 { unsafe { asm!("dsb sy", options(nostack, preserves_flags)) }; break; } core::hint::spin_loop(); } let secondary = TimerSample { mpidr: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_mpidr)) }, frequency: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_freq)) }, start: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_start)) }, deadline0: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_deadline0)) }, tick0: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_tick0)) }, deadline1: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_deadline1)) }, tick1: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_tick1)) }, seen, }; for (label, sample) in [ ("TURKOS/SMP-TIMER0: CPU=0x", primary), ("TURKOS/SMP-TIMER1: CPU=0x", secondary), ] { write_str(label); write_hex(sample.mpidr as usize); write_str(" FREQ=0x"); write_hex(sample.frequency as usize); write_str(" START=0x"); write_hex(sample.start as usize); write_str(" D0=0x"); write_hex(sample.deadline0 as usize); write_str(" T0=0x"); write_hex(sample.tick0 as usize); write_str(" D1=0x"); write_hex(sample.deadline1 as usize); write_str(" T1=0x"); write_hex(sample.tick1 as usize); write_str(" SEEN=0x"); write_hex(sample.seen as usize); write_str("\r\n"); } if secondary_mpidr == secondary.mpidr && smp::validate_timer_pair(primary, secondary).is_ok() { write_line("TURKOS/SMP-TIMER: PASS"); } else { write_line("TURKOS/SMP-ERROR: per-CPU timer sample"); park(); } } #[cfg(feature = "smp-topology-test")] fn smp_topology_test( secondary_mpidr: u64, secondary_sp: u64, secondary_start: usize, secondary_top: usize, ) { let primary_mpidr: u64; let primary_sp: usize; // SAFETY: read-only per-PE identity and the current AArch64 stack pointer. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) primary_mpidr, options(nomem, nostack, preserves_flags)); asm!("mov {value}, sp", value = out(reg) primary_sp, options(nomem, nostack, preserves_flags)); } let primary_start = core::ptr::addr_of!(__stack_start) as usize; let primary_top = core::ptr::addr_of!(__stack_top) as usize; let ids = smp::distinct_pair(primary_mpidr, secondary_mpidr); write_str("TURKOS/SMP-CPU-IDS: PRIMARY=0x"); write_hex(primary_mpidr as usize); write_str(" SECONDARY=0x"); write_hex(secondary_mpidr as usize); write_str(" AFF0=0x"); write_hex(smp::affinity(primary_mpidr) as usize); write_str(" AFF1=0x"); write_hex(smp::affinity(secondary_mpidr) as usize); write_str("\r\n"); if ids.is_ok() && smp::affinity(primary_mpidr) == 0 && smp::affinity(secondary_mpidr) == 1 { write_line("TURKOS/SMP-CPU-IDS: PASS"); } else { write_line("TURKOS/SMP-ERROR: duplicate or unexpected CPU identity"); park(); } let primary = StackSpan { start: primary_start as u64, end: primary_top as u64, sp: primary_sp as u64, }; let secondary = StackSpan { start: secondary_start as u64, end: secondary_top as u64, sp: secondary_sp, }; let stacks = smp::disjoint_stacks(primary, secondary); write_str("TURKOS/SMP-STACKS: PRIMARY=0x"); write_hex(primary_start); write_str("..0x"); write_hex(primary_top); write_str(" SP0=0x"); write_hex(primary_sp); write_str(" SECONDARY=0x"); write_hex(secondary_start); write_str("..0x"); write_hex(secondary_top); write_str(" SP1=0x"); write_hex(secondary_sp as usize); write_str("\r\n"); if stacks.is_ok() && primary_top - primary_start == 16 * 1024 && secondary_top - secondary_start == 4096 { write_line("TURKOS/SMP-STACKS: PASS"); } else { write_line("TURKOS/SMP-ERROR: CPU stacks overlap or have invalid bounds"); park(); } } #[cfg(feature = "mmu-tlb-test")] unsafe extern "C" { fn __turkos_replace_l3_probe(slot: *mut u64, va_page: u64, descriptor: u64); } #[cfg(any( feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] unsafe extern "C" { fn __turkos_swap_task_context(old: *mut task::CpuContext, new: *const task::CpuContext); } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] #[repr(align(4096))] struct TaskStack([u8; 16 * 1024]); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_A_STACK: TaskStack = TaskStack([0; 16 * 1024]); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_B_STACK: TaskStack = TaskStack([0; 16 * 1024]); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_CONTEXTS: [task::CpuContext; 2] = [task::CpuContext::EMPTY; 2]; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_BOOT_CONTEXT: task::CpuContext = task::CpuContext::EMPTY; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_SCHEDULER: Scheduler = Scheduler::new(); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_A_ID: Option = None; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_A_COUNT: usize = 0; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_B_COUNT: usize = 0; #[cfg(feature = "task-lifecycle-test")] static mut TASK_C_COUNT: usize = 0; #[cfg(feature = "task-lifecycle-test")] static mut TASK_A_LEASE: task::StackLease = task::StackLease::new(); #[cfg(feature = "task-lifecycle-test")] static mut TASK_B_LEASE: task::StackLease = task::StackLease::new(); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] const TASK_A_CANARY: u64 = 0x5455_524b_5441_534b; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] const TASK_B_CANARY: u64 = 0x5455_524b_5442_534b; #[cfg(feature = "task-lifecycle-test")] const TASK_C_CANARY: u64 = 0x5455_524b_5443_534b; #[cfg(feature = "task-lifecycle-test")] const TASK_B_POISON: u64 = 0xdddd_dddd_dddd_dddd; struct BootData { hardware: dtb::Hardware, interrupts: dtb::InterruptHardware, reservations: dtb::Reservations, static_reserved: dtb::Reservations, #[cfg(any( feature = "virtio-discover-test", feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-tx-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "user-mmio-read-test" ))] virtio_mmio: dtb::VirtioMmioNodes, #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] dtb_address: usize, dtb_size: usize, } // Bootstrap address is specific to QEMU virt. The DTB replaces it after discovery. static UART_BASE: AtomicUsize = AtomicUsize::new(0x0900_0000); static SERIAL_LINE_LOCK: SerialLineLock = SerialLineLock::new(); const UART_TX_FULL: u32 = 1 << 5; // An AArch64 BRK instruction in writable data: if PXN were ineffective, the // branch would execute BRK rather than produce the expected instruction abort. #[cfg(feature = "mmu-nx-test")] #[used] #[unsafe(link_section = ".data.mmu_nx_probe")] static mut NX_PROBE: u32 = 0xd420_0000; #[cfg(feature = "mmu-rodata-test")] #[used] #[unsafe(link_section = ".rodata.mmu_rodata_probe")] static RODATA_PROBE: u64 = 0x5455_524b_4f53_524f; #[cfg(feature = "mmu-rodata-exec-test")] #[used] #[unsafe(link_section = ".rodata.mmu_rodata_exec_probe")] static RODATA_EXEC_PROBE: u32 = 0xd420_0000; // A nonzero initialized value forces this canary into writable .data. #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static mut MMU_DATA_CANARY: u64 = 0x5455_524b_4f53_4441; #[unsafe(no_mangle)] #[cfg_attr(feature = "smp-allocator-test", allow(unreachable_code))] #[cfg_attr( any(feature = "fiq-label-test", feature = "serror-label-test"), allow(unreachable_code) )] pub extern "C" fn kernel_main(dtb_address: usize, boot_info_address: usize) -> ! { let level = install_vectors(); write_line("TURKOS/BOOT0: ARM64 kernel entered"); write_str("TURKOS/EL: "); write_byte(b'0' + level as u8); write_str("\r\n"); write_str("TURKOS/DTB: 0x"); write_hex(dtb_address); write_str("\r\n"); let boot = match read_boot_data(dtb_address) { Ok(boot) => boot, Err(_) => { write_line("TURKOS/DTB-ERROR"); park(); } }; let hardware = &boot.hardware; UART_BASE.store(hardware.uart_base as usize, Ordering::Relaxed); write_str("TURKOS/UART: 0x"); write_hex(hardware.uart_base as usize); write_str("\r\nTURKOS/RAM: 0x"); write_hex(hardware.ram_base as usize); write_str("+0x"); write_hex(hardware.ram_size as usize); write_str("\r\n"); let uefi_info = match read_uefi_boot_info(boot_info_address, &boot, dtb_address) { Ok(info) => info, Err(()) => { write_line("TURKOS/BOOTINFO-ERROR"); park(); } }; write_str("TURKOS/GICD: 0x"); write_hex(boot.interrupts.gic_distributor as usize); write_str("\r\nTURKOS/GICR: 0x"); write_hex(boot.interrupts.gic_redistributor as usize); write_str("\r\nTURKOS/TIMER-PPI: 0x"); write_hex(boot.interrupts.nonsecure_phys_timer_ppi as usize); write_str("\r\nTURKOS/IRQ-MASK: ON\r\n"); #[cfg(any(feature = "fiq-label-test", feature = "serror-label-test"))] { // These fault-label images stop before the normal allocator path. let _ = &uefi_info; let slot = if cfg!(feature = "fiq-label-test") { 6 } else { 7 }; let interrupted_sp: usize; unsafe { asm!("mov {value}, sp", value = out(reg) interrupted_sp, options(nomem, nostack, preserves_flags)) }; let synthetic = ExceptionFrame { registers: [0; 31], interrupted_sp, canary: exception::FRAME_CANARY, }; write_str("TURKOS/ASYNC-LABEL-PROBE: SYNTHETIC SLOT=0x"); write_hex_byte(slot as u8); write_str("\r\n"); // A stale synchronous data-abort ESR is deliberate: an asynchronous // slot must never claim its old FAR. This calls the fatal reporter, // not the hardware FIQ/SError vector entry. kernel_exception(1, (0x25 << 26) | 0x07, 0, slot, &synthetic, 0x3c0); } write_str("TURKOS/STATIC-RESERVATIONS: 0x"); write_hex(boot.static_reserved.as_slice().len()); write_str("\r\n"); if cfg!(feature = "smp-boot-protocol-test") { #[cfg(feature = "smp-boot-protocol-test")] smp_boot_protocol_test(); } let free = match plan_free_memory(&boot, dtb_address, uefi_info.as_ref()) { Ok(free) => free, Err(_) => { write_line("TURKOS/MEM-ERROR"); park(); } }; if uefi_info.is_some() { write_line("TURKOS/EFI-FREEMAP: CONVENTIONAL-ONLY PASS"); } #[cfg(feature = "uefi-ownership-test")] if let Some((address, _)) = uefi_info.as_ref() { uefi_owned_endpoint_probe(free.clone(), *address); } #[cfg(feature = "smp-allocator-test")] smp_allocator_probe::run(free, &boot, level); write_str("TURKOS/FREE-RANGES: 0x"); write_hex(free.regions().len()); write_str("\r\n"); for (index, region) in free.regions().iter().enumerate() { write_str("TURKOS/FREE-RANGE: INDEX=0x"); write_hex(index); write_str(" START=0x"); write_hex(region.start as usize); write_str(" END=0x"); write_hex(region.end as usize); write_str("\r\n"); } write_str("TURKOS/FREE-PAGES: 0x"); write_hex(free.total_pages() as usize); write_str("\r\n"); #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] let mmu_free = free.clone(); if !frame_smoke_test(free) { write_line("TURKOS/FRAME-ERROR"); park(); } write_line("TURKOS/FRAME-TEST: PASS"); #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] mmu_smoke_test(mmu_free, &boot, level); #[cfg(feature = "panic-test")] { panic!("intentional smoke-test panic"); } #[cfg(feature = "exception-test")] { #[cfg(feature = "log-secret-test")] { const CANARY: &[u8; 32] = b"K453-PRIVATE-CANARY-9a7F3d2E4b6C"; let secret = core::ptr::addr_of_mut!(LOG_SECRET_PAGE).cast::(); for (index, byte) in CANARY.iter().copied().enumerate() { // SAFETY: this isolated test image owns its 4 KiB BSS page. unsafe { write_volatile(secret.add(index), byte) }; } for (index, byte) in CANARY.iter().copied().enumerate() { if unsafe { read_volatile(secret.add(index)) } != byte { write_line("TURKOS/LOG-SECRET-ERROR: seed mismatch"); park(); } } write_line("TURKOS/LOG-SECRET: SEEDED BYTES=32"); // Deliberately place canary fragments in saved GP registers. unsafe { asm!( "brk #0x42", in("x0") u64::from_be_bytes(*b"K453-PRI") as usize, in("x3") u64::from_be_bytes(*b"VATE-CAN") as usize, in("x30") u64::from_be_bytes(*b"ARY-9a7F") as usize, options(nomem, nostack, noreturn) ) } } #[cfg(not(feature = "log-secret-test"))] { // SAFETY: BRK enters the fatal vector and never returns. The three // markers identify the interrupted values, including the link register. unsafe { asm!( "brk #0x42", in("x0") 0x544b_4252_4b5f_5830usize, in("x3") 0x544b_4252_4b5f_5833usize, in("x30") 0x544b_4252_4b5f_4c52usize, options(nomem, nostack, noreturn) ) } } } #[cfg(feature = "fault-test")] { // SAFETY: UDF enters the same fatal vector and never returns. unsafe { asm!( "udf #0", in("x0") 0x544b_5544_465f_5830usize, in("x3") 0x544b_5544_465f_5833usize, in("x30") 0x544b_5544_465f_4c52usize, options(nomem, nostack, noreturn) ) } } #[cfg(feature = "frame-stress-test")] park() } fn install_vectors() -> usize { let base = __vectors_start as *const () as usize; let current_el: usize; // SAFETY: CurrentEL is readable in privileged AArch64 modes. unsafe { asm!("mrs {level}, CurrentEL", level = out(reg) current_el, options(nomem, nostack, preserves_flags)) } let level = (current_el >> 2) & 3; // SAFETY: QEMU's direct Image boot enters at EL1 or EL2; the table is // linker-aligned to 2 KiB and exceptions remain masked during setup. unsafe { match level { 1 => asm!("msr VBAR_EL1, {base}", "isb", base = in(reg) base, options(nomem, nostack)), 2 => asm!("msr VBAR_EL2, {base}", "isb", base = in(reg) base, options(nomem, nostack)), _ => { write_line("TURKOS/EL-ERROR"); park() } } } level } // The fatal vector uses 8-byte AArch64 register slots in a 272-byte frame. // Keep these assertions beside the Rust view; assembly receives the three // variable offsets from this layout through global_asm! above. const _: () = { assert!(size_of::() == 8); assert!(size_of::() == 272); assert!(offset_of!(ExceptionFrame, registers) == 0); assert!(offset_of!(ExceptionFrame, interrupted_sp) == 248); assert!(offset_of!(ExceptionFrame, canary) == 256); }; // One CPU, IRQ masked, and a non-returning fatal path: volatile state avoids // depending on an exclusive-monitor retry inside a nested exception. After // the second entry, further entries park without touching the UART again. static mut FATAL_ENTRIES: usize = 0; #[cfg(feature = "log-secret-test")] static mut LOG_SECRET_PAGE: [u8; 4096] = [0; 4096]; #[unsafe(no_mangle)] pub extern "C" fn kernel_exception( level: usize, esr: usize, elr: usize, slot: usize, frame: *const ExceptionFrame, spsr: usize, ) -> ! { let depth = core::ptr::addr_of_mut!(FATAL_ENTRIES); // SAFETY: only the boot CPU runs this fatal path, IRQ remains masked, and // the volatile store completes before any deliberate nested BRK. let prior = unsafe { read_volatile(depth) }; unsafe { write_volatile(depth, prior.saturating_add(1)) }; match exception::fatal_reentry(prior) { exception::FatalReentry::First => {} exception::FatalReentry::StopOnce => { write_line("TURKOS/EXCEPTION-NESTED: STOP"); park(); } exception::FatalReentry::ParkSilently => park(), } #[cfg(feature = "nested-fault-test")] { write_line("TURKOS/EXCEPTION-FIRST: ENTER"); // The first deliberate BRK in mmu_smoke_test reached this handler. // Trigger a second BRK *inside* it to exercise the reentry guard. unsafe { asm!("brk #0x159") }; write_line("TURKOS/EXCEPTION-NESTED: RETURNED"); } // SAFETY: the vector entry placed a complete, aligned frame on the // current exception stack before passing its address in x4. let frame = unsafe { &*frame }; if !frame.canary_valid() { write_line("TURKOS/EXCEPTION-FRAME: CORRUPT"); park(); } write_str("TURKOS/EXCEPTION: EL"); write_byte(b'0' + level as u8); write_str(" SLOT=0x"); write_hex_byte(slot as u8); write_str(" EC=0x"); let ec = ((esr >> 26) & 0x3f) as u8; write_hex_byte(ec); write_str(" ESR=0x"); write_hex(esr); write_str(" ELR=0x"); write_hex(elr); write_str(" KIND="); write_str(exception::kind_name(slot)); if let Some(kind) = exception::unexpected_el1_async(slot) { write_str(" UNEXPECTED-EL1-ASYNC="); write_str(kind); write_str(" FATAL=YES"); } write_str(" SP=0x"); write_hex(frame.interrupted_sp); write_str(" SPSR=0x"); write_hex(spsr); // Only four explicitly diagnostic fixtures may reveal saved GP values. // Ordinary fatal paths omit them; they may contain user or kernel secrets. #[cfg(all( not(feature = "log-secret-test"), any( feature = "exception-test", feature = "fault-test", feature = "mmu-brk-test", feature = "mmu-udf-test" ) ))] { write_str(" X0=0x"); write_hex(frame.registers[0]); write_str(" X3=0x"); write_hex(frame.registers[3]); write_str(" X30=0x"); write_hex(frame.registers[30]); } #[cfg(not(all( not(feature = "log-secret-test"), any( feature = "exception-test", feature = "fault-test", feature = "mmu-brk-test", feature = "mmu-udf-test" ) )))] write_str(" REGISTERS=REDACTED"); write_str(" EC-NAME="); write_str(exception::class_name(ec)); if exception::far_is_valid_for_slot(slot, esr as u64) { write_str(" FAR-VALID=YES FAR=0x"); let far: usize; // SAFETY: this ESR class defines FAR for the current privileged level. unsafe { match level { 1 => { asm!("mrs {far}, FAR_EL1", far = out(reg) far, options(nomem, nostack, preserves_flags)) } 2 => { asm!("mrs {far}, FAR_EL2", far = out(reg) far, options(nomem, nostack, preserves_flags)) } _ => park(), } } write_hex(far); } else { write_str(" FAR-VALID=NO FAR=NA"); } write_str(" FRAME-CANARY=PASS\r\n"); write_str("TURKOS/EXCEPTION-ORIGIN: "); write_str(exception::origin_name(slot)); write_str("\r\n"); park() } #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test" ))] const IRQ_GUARD: u64 = 0x5455_524b_4952_5147; #[cfg(feature = "irq-sgi-test")] const MAIN_STACK_GUARD: u64 = 0x5455_524b_4d41_494e; #[cfg(feature = "irq-sgi-test")] static mut IRQ_SGI_SEEN: usize = 0; #[cfg(feature = "irq-gp-test")] static mut GP_PROBE_ACTIVE: usize = 0; #[cfg(feature = "irq-gp-test")] #[unsafe(no_mangle)] static mut GP_IRQ_WAKE: usize = 0; #[cfg(feature = "irq-gp-test")] static mut GP_RETURNED: [u64; 31] = [0; 31]; #[cfg(feature = "irq-gp-test")] #[unsafe(no_mangle)] static mut GP_RETURNED_SP_EL0: u64 = 0; #[cfg(feature = "irq-gp-test")] fn gp_register_mask(registers: &[u64; 31]) -> u64 { let mut mask = 0; for (index, value) in registers.iter().enumerate() { if *value == 0x6000 + index as u64 { mask |= 1u64 << index; } } mask } #[cfg(feature = "irq-gp-test")] fn gp_return_probe( frame: *const irq::IrqContext, stack_start: usize, stack_top: usize, current_sp: usize, ) -> usize { // SAFETY: the vector placed an 816-byte frame at the top of this IRQ // stack, and this feature runs only on one CPU with a single pending SGI. let context = unsafe { &*frame }; let mask = gp_register_mask(&context.x); let valid = frame as usize == stack_top - size_of::() && irq::handler_sp_in_stack(stack_start, frame as usize, current_sp) && irq::main_stack_intact( core::ptr::addr_of!(__stack_start) as usize, core::ptr::addr_of!(__stack_top) as usize, stack_start, stack_top, context.interrupted_sp as usize, unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) }, MAIN_STACK_GUARD, ) && context.vector_slot == 5 && context.spsr & 15 == 5 && context.sp_el0 == 0x7a5a && mask == 0x7fff_ffff && unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) } == IRQ_GUARD && unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) } == MAIN_STACK_GUARD; let iar: usize; // SAFETY: the isolated guest enables SGI 1 and owns the GIC CPU interface. unsafe { asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags)) }; if !valid || irq::classify_iar(iar as u64, 1) != irq::IarClass::Expected { write_line("TURKOS/IRQ-GP-ERROR: frame or source"); return 0; } // SAFETY: EOIR and, when required, DIR receive precisely the IAR value. unsafe { asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } let previous = unsafe { read_volatile(core::ptr::addr_of!(IRQ_SGI_SEEN)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_SGI_SEEN), previous + 1) }; if previous != 0 { write_line("TURKOS/IRQ-GP-ERROR: repeated interrupt"); return 0; } write_str("TURKOS/IRQ-GP-IAR: 0x"); write_hex(iar); write_str("\r\nTURKOS/IRQ-GP-EOI: 0x"); write_hex(iar); write_str("\r\nTURKOS/IRQ-GP-SAVED-MASK: 0x"); write_hex(mask as usize); write_str("\r\nTURKOS/IRQ-GP-SAVED-SP_EL0: 0x"); write_hex(context.sp_el0 as usize); write_str("\r\n"); unsafe { write_volatile(core::ptr::addr_of_mut!(GP_IRQ_WAKE), 1) }; // WFE in gp_probe.S must wake even if the SGI arrived before WFE. unsafe { asm!("sev", options(nomem, nostack, preserves_flags)) }; 1 } #[cfg(any(feature = "irq-timer-test", feature = "irq-unknown-test"))] static mut IRQ_TIMER_SEEN: usize = 0; #[cfg(any(feature = "irq-timer-test", feature = "irq-unknown-test"))] static mut IRQ_TIMER_PERIOD: usize = 0; #[cfg(any(feature = "irq-timer-test", feature = "irq-unknown-test"))] static mut IRQ_TIMER_DEADLINE: u64 = 0; #[cfg(feature = "irq-timer-test")] #[cfg_attr(feature = "irq-service-test", allow(dead_code))] static mut IRQ_TIMER_MAIN_PROGRESS: usize = 0; #[cfg(feature = "irq-timer-storm-test")] static mut IRQ_STORM_LAST_PROGRESS: usize = 0; #[cfg(feature = "irq-timer-storm-test")] static mut IRQ_STORM_INTERLEAVED: usize = 0; #[cfg(feature = "irq-timer-storm-test")] static mut IRQ_STORM_SKIPPED: usize = 0; #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] static mut IRQ_UNKNOWN_GICR: usize = 0; #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] static mut IRQ_UNKNOWN_HANDLED: usize = 0; #[cfg(any(feature = "task-preempt-test", feature = "task-sleep-test"))] #[unsafe(no_mangle)] pub extern "C" fn kernel_return_probe(frame: *const irq::IrqContext) -> usize { preempt::return_probe(frame) } #[cfg(all( not(any(feature = "task-preempt-test", feature = "task-sleep-test")), any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test" ) ))] #[cfg_attr(feature = "smp-irq-lock-test", allow(unreachable_code))] #[cfg_attr(feature = "irq-spurious-vector-test", allow(unreachable_code))] #[unsafe(no_mangle)] pub extern "C" fn kernel_return_probe(frame: *const irq::IrqContext) -> usize { let stack_start = core::ptr::addr_of!(__irq_stack_start) as usize; let stack_top = core::ptr::addr_of!(__irq_stack_top) as usize; let mut current_sp: usize; #[cfg(feature = "irq-return-test")] let esr: usize; // SAFETY: this handler is called only by an isolated EL1 test vector. unsafe { asm!("mov {value}, sp", value = out(reg) current_sp, options(nomem, nostack, preserves_flags)); #[cfg(feature = "irq-return-test")] asm!("mrs {value}, ESR_EL1", value = out(reg) esr, options(nomem, nostack, preserves_flags)); } #[cfg(feature = "smp-irq-lock-test")] return smp_allocator_probe::irq_lock_probe(frame, stack_start, stack_top, current_sp); #[cfg(feature = "irq-service-test")] if irq_service::active() { return irq_service::handle(frame, stack_start, stack_top, current_sp); } #[cfg(feature = "irq-spurious-vector-test")] return spurious_vector::handle(frame, stack_start, stack_top, current_sp); #[cfg(feature = "process-exit-irq-test")] if process_probe::exit_irq_active() { return process_probe::handle_exit_irq(frame, stack_start, stack_top, current_sp); } let frame_address = frame as usize; #[cfg(feature = "irq-gp-test")] if unsafe { read_volatile(core::ptr::addr_of!(GP_PROBE_ACTIVE)) } != 0 { return gp_return_probe(frame, stack_start, stack_top, current_sp); } // SAFETY: the vector wrote the complete frame at the top of the dedicated // stack before calling Rust; the low guard was initialized by the probe. // A leaf Rust handler may keep SP exactly at the frame base; the first // local stack allocation, if any, moves it below. Both states are valid. #[allow(unused_mut)] let mut valid = unsafe { frame_address == stack_top - size_of::() && irq::handler_sp_in_stack(stack_start, frame_address, current_sp) && read_volatile(core::ptr::addr_of!(__irq_stack_guard)) == IRQ_GUARD && (*frame).interrupted_sp & 15 == 0 && ((*frame).interrupted_sp < stack_start as u64 || (*frame).interrupted_sp >= stack_top as u64) && (*frame).elr != 0 && ((*frame).spsr & 0xf) == 5 && irq::return_slot_matches((*frame).vector_slot, !cfg!(feature = "irq-return-test")) && (*frame).x[0] == 0x5455_524b_4952_5130 && (*frame).x[3] == 0x5455_524b_4952_5133 && (*frame).x[16] == 0x5455_524b_4952_5116 && (*frame).x[17] == 0x5455_524b_4952_5117 && (*frame).x[30] == 0x5455_524b_4952_514c && (*frame).q[0] == [0x5a5a_5a5a_5a5a_5a5a; 2] && (*frame).q[31] == [0xa5a5_a5a5_a5a5_a5a5; 2] }; #[cfg(feature = "irq-sgi-test")] { let main_start = core::ptr::addr_of!(__stack_start) as usize; let main_top = core::ptr::addr_of!(__stack_top) as usize; let main_guard = unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) }; valid &= irq::main_stack_intact( main_start, main_top, stack_start, stack_top, unsafe { (*frame).interrupted_sp as usize }, main_guard, MAIN_STACK_GUARD, ); } #[cfg(feature = "irq-return-test")] { valid &= ((esr >> 26) & 0x3f) == 0x15 && (esr & 0xffff) == 0x42; if valid { write_str("TURKOS/SVC-VECTOR-SLOT: 0x"); write_hex_byte(unsafe { (*frame).vector_slot as u8 }); write_str("\r\n"); write_str("TURKOS/SVC-RETURN-ESR: 0x"); write_hex(esr); write_str("\r\n"); write_line("TURKOS/SVC-RETURN-HANDLER: PASS"); return 1; } write_str("TURKOS/SVC-RETURN-ERROR: context esr=0x"); write_hex(esr); write_str(" frame=0x"); write_hex(frame_address); write_str(" stack=0x"); write_hex(current_sp); write_str(" guard=0x"); write_hex(unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) as usize }); write_str(" old=0x"); write_hex(unsafe { (*frame).interrupted_sp as usize }); write_str(" spsr=0x"); write_hex(unsafe { (*frame).spsr as usize }); write_str(" x0=0x"); write_hex(unsafe { (*frame).x[0] as usize }); write_str(" x3=0x"); write_hex(unsafe { (*frame).x[3] as usize }); write_str(" x16=0x"); write_hex(unsafe { (*frame).x[16] as usize }); write_str(" x17=0x"); write_hex(unsafe { (*frame).x[17] as usize }); write_str(" x30=0x"); write_hex(unsafe { (*frame).x[30] as usize }); write_str(" q0=0x"); write_hex(unsafe { (*frame).q[0][0] as usize }); write_str(" q31=0x"); write_hex(unsafe { (*frame).q[31][0] as usize }); write_str("\r\n"); return 0; } #[cfg(feature = "irq-sgi-test")] { let iar: usize; // SAFETY: setup enabled the GICv3 system-register interface before // unmasking IRQ. Reading IAR acknowledges one Group 1 interrupt. unsafe { asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags)); } valid &= (iar & 0x00ff_ffff) == 1; valid &= irq::nzcv_matches(unsafe { (*frame).spsr }, irq::IRQ_PROBE_NZCV); if !valid { write_str("TURKOS/IRQ-SGI-ERROR: iar=0x"); write_hex(iar); write_str(" frame=0x"); write_hex(frame_address); write_str(" stack=0x"); write_hex(current_sp); write_str(" old=0x"); write_hex(unsafe { (*frame).interrupted_sp as usize }); write_str(" spsr=0x"); write_hex(unsafe { (*frame).spsr as usize }); write_str(" x0=0x"); write_hex(unsafe { (*frame).x[0] as usize }); write_str(" x3=0x"); write_hex(unsafe { (*frame).x[3] as usize }); write_str(" x16=0x"); write_hex(unsafe { (*frame).x[16] as usize }); write_str(" x17=0x"); write_hex(unsafe { (*frame).x[17] as usize }); write_str(" x30=0x"); write_hex(unsafe { (*frame).x[30] as usize }); write_str(" q0=0x"); write_hex(unsafe { (*frame).q[0][0] as usize }); write_str(" q31=0x"); write_hex(unsafe { (*frame).q[31][0] as usize }); write_str("\r\n"); return 0; } // SAFETY: the acknowledged SGI is edge-triggered. EOIR drops its // priority and deactivates it in EOImode 0; mode 1 needs DIR as well. unsafe { asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } // SAFETY: this dedicated guest has one CPU. IRQ is masked on entry, // so a plain volatile count avoids an exclusive/atomic access while // still detecting an unexpected second interrupt. let previous = unsafe { read_volatile(core::ptr::addr_of!(IRQ_SGI_SEEN)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_SGI_SEEN), previous + 1) }; if previous != 0 { write_line("TURKOS/IRQ-SGI-ERROR: repeated interrupt"); return 0; } write_str("TURKOS/IRQ-SGI-IAR: 0x"); write_hex(iar); write_str("\r\n"); write_str("TURKOS/IRQ-SGI-EOI: 0x"); write_hex(iar); write_str("\r\n"); if cfg!(feature = "irq-guard-test") { let guard_page = core::ptr::addr_of!(__irq_guard_page_start) as usize; let target = guard_page + memory::PAGE_SIZE as usize - 1; write_str("TURKOS/IRQ-GUARD-PAGE: 0x"); write_hex(guard_page); write_str("\r\nTURKOS/IRQ-GUARD-STACK: 0x"); write_hex(stack_start); write_str("-0x"); write_hex(stack_top); write_str(" HANDLER-SP=0x"); write_hex(current_sp); write_str("\r\nTURKOS/IRQ-GUARD-TARGET: 0x"); write_hex(target); write_str("\r\n"); write_line("TURKOS/IRQ-GUARD-PROBE: WRITE-LOWER-PAGE"); // SAFETY: this isolated guest intentionally writes the final byte // of the unmapped page below its IRQ stack. The fatal same-EL // vector must report FAR and park; returning is a test failure. unsafe { write_volatile(target as *mut u8, 0xa5) }; write_line("TURKOS/IRQ-GUARD-ERROR: write unexpectedly returned"); return 0; } write_str("TURKOS/IRQ-VECTOR-SLOT: 0x"); write_hex_byte(unsafe { (*frame).vector_slot as u8 }); write_str("\r\n"); write_str("TURKOS/IRQ-SGI-SPSR: 0x"); write_hex(unsafe { (*frame).spsr as usize }); write_str("\r\n"); write_str("TURKOS/IRQ-STACK-BOUNDS: MAIN=0x"); write_hex(core::ptr::addr_of!(__stack_start) as usize); write_str("-0x"); write_hex(core::ptr::addr_of!(__stack_top) as usize); write_str(" IRQ=0x"); write_hex(stack_start); write_str("-0x"); write_hex(stack_top); write_str(" OLD-SP=0x"); write_hex(unsafe { (*frame).interrupted_sp as usize }); write_str(" HANDLER-SP=0x"); write_hex(current_sp); write_str(" GUARD=0x"); write_hex(unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) as usize }); write_str("\r\n"); write_line("TURKOS/IRQ-SGI-HANDLER: PASS"); return 1; } #[cfg(any(feature = "irq-timer-test", feature = "irq-unknown-test"))] { let iar: usize; // SAFETY: this image enabled the Group 1 CPU interface for PPI 30; // the negative image additionally enables SGI 2 to test rejection. unsafe { asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags)) }; match irq::classify_iar(iar as u64, 30) { irq::IarClass::Unexpected(intid) => { // The isolated K457 image owns SGI 2. Disable the exact // redistributor source, wait for RWP, and verify enable // readback before allowing ERET. Other images stay fatal. #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] let masked = if intid == 2 { let base = unsafe { read_volatile(core::ptr::addr_of!(IRQ_UNKNOWN_GICR)) }; if base == 0 { false } else { let bit = irq::sgi_disable_bit(intid).unwrap_or(0); let control = base as *const u32; let enabled = (base + 0x1_0100) as *const u32; let disable = (base + 0x1_0180) as *mut u32; // SAFETY: the validated 128 KiB GICR Device window // was mapped before IRQ unmasking in this image. unsafe { write_volatile(disable, bit); asm!("dsb sy", options(nostack, preserves_flags)); let mut ready = false; for _ in 0..1_000_000 { if read_volatile(control) & 8 == 0 { ready = true; break; } } ready && read_volatile(enabled) & bit == 0 } } } else { false }; #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] if masked { write_line("TURKOS/IRQ-UNKNOWN-ORDER: MASK-READBACK=OFF BEFORE-EOI"); } // SAFETY: an unexpected acknowledged source must be EOI'd. // Stop the timer so an unrelated PPI cannot storm meanwhile. unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)); asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } write_str("TURKOS/IRQ-UNKNOWN-EOI: 0x"); write_hex(intid as usize); write_str("\r\n"); write_line("TURKOS/IRQ-UNKNOWN: REJECTED"); #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] if masked { // A repeat delivery must become count=2, never be hidden // by overwriting the first observation with a constant. unsafe { let counter = core::ptr::addr_of_mut!(IRQ_UNKNOWN_HANDLED); write_volatile(counter, read_volatile(counter).saturating_add(1)); } return 1; } #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] write_line("TURKOS/IRQ-WRONG-MASK: FAIL"); return 0; } irq::IarClass::Special(intid) => { // Special IAR values did not acknowledge an interrupt: no EOI. unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; write_str("TURKOS/IRQ-SPECIAL-NO-EOI: 0x"); write_hex(intid as usize); write_str("\r\n"); return 0; } irq::IarClass::Expected => {} } if !valid { // SAFETY: disabling the timer prevents a repeated level signal. unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; write_str("TURKOS/IRQ-TIMER-ERROR: context/intid=0x"); write_hex(iar & 0x00ff_ffff); write_str(" frame=0x"); write_hex(frame_address); write_str(" stack=0x"); write_hex(current_sp); write_str(" guard=0x"); write_hex(unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) as usize }); write_str(" old=0x"); write_hex(unsafe { (*frame).interrupted_sp as usize }); write_str(" spsr=0x"); write_hex(unsafe { (*frame).spsr as usize }); write_str(" x0=0x"); write_hex(unsafe { (*frame).x[0] as usize }); write_str(" x3=0x"); write_hex(unsafe { (*frame).x[3] as usize }); write_str(" x16=0x"); write_hex(unsafe { (*frame).x[16] as usize }); write_str(" x17=0x"); write_hex(unsafe { (*frame).x[17] as usize }); write_str(" x30=0x"); write_hex(unsafe { (*frame).x[30] as usize }); write_str(" q0=0x"); write_hex(unsafe { (*frame).q[0][0] as usize }); write_str(" q31=0x"); write_hex(unsafe { (*frame).q[31][0] as usize }); write_str("\r\n"); return 0; } // SAFETY: one PE owns this isolated counter and IRQ is masked on entry. let previous = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_SEEN)) }; if previous >= 100 { unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; write_line("TURKOS/IRQ-TIMER-ERROR: extra tick"); return 0; } let count = previous + 1; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_SEEN), count) }; #[cfg(feature = "irq-timer-storm-test")] { let progress = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_MAIN_PROGRESS)) }; let last = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_LAST_PROGRESS)) }; if count > 1 && progress > last { let samples = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_INTERLEAVED)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_STORM_INTERLEAVED), samples + 1) }; } unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_STORM_LAST_PROGRESS), progress) }; } #[cfg(feature = "irq-timer-late-test")] if count == 50 { // Deliberately hold IRQ 50 beyond three 10 ms periods while IRQ // remains masked on entry. The normal timer image never does this. let period = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_PERIOD)) } as u64; let deadline = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_DEADLINE)) }; if let Some(target) = period .checked_mul(3) .and_then(|delay| deadline.checked_add(delay)) { loop { let now: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) now, options(nomem, nostack, preserves_flags)) }; if now >= target { break; } core::hint::spin_loop(); } } } // Read the counter before UART output. Advance the absolute compare // value before EOI so a late handler cannot immediately storm. let counter: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) counter, options(nomem, nostack, preserves_flags)) }; let mut deadline_ok = true; let mut skipped = 0; // Clear the level source before EOI. The hundredth tick stops it. if count < 100 { let period = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_PERIOD)) } as u64; let previous_deadline = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_DEADLINE)) }; if let Some((next, missed)) = irq::next_timer_deadline(previous_deadline, period, counter) { skipped = missed; #[cfg(feature = "irq-timer-storm-test")] { let before = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_SKIPPED)) }; unsafe { write_volatile( core::ptr::addr_of_mut!(IRQ_STORM_SKIPPED), before.saturating_add(missed as usize), ) }; } unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_DEADLINE), next); asm!("msr CNTP_CVAL_EL0, {value}", "isb", value = in(reg) next, options(nomem, nostack)); } } else { deadline_ok = false; unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; } } else { unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; } // SAFETY: the acknowledged PPI is valid; EOImode 1 also needs DIR. unsafe { asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } if !deadline_ok { write_line("TURKOS/IRQ-TIMER-ERROR: deadline overflow"); return 0; } // K458: serial diagnostics can take longer than a 100 us period. // Keep the timer level deasserted while printing, then coalesce any // elapsed deadlines just before ERET so main gets a real time slice. #[cfg(feature = "irq-timer-storm-test")] if count < 100 { unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; } write_str("TURKOS/IRQ-TIMER-IAR: 0x"); write_hex(iar); write_str("\r\n"); write_str("TURKOS/IRQ-TIMER-EOI: 0x"); write_hex(iar); write_str("\r\n"); if skipped != 0 { write_str("TURKOS/IRQ-TIMER-LATE: 0x"); write_hex(skipped as usize); write_str("\r\n"); } write_str("TURKOS/IRQ-TIMER-COUNTER: 0x"); write_hex(counter as usize); write_str("\r\nTURKOS/IRQ-TIMER-TICK: 0x"); write_hex(count); write_str("\r\n"); #[cfg(feature = "irq-timer-storm-test")] if count < 100 { let now_after_log: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) now_after_log, options(nomem, nostack, preserves_flags)) }; let previous_deadline = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_DEADLINE)) }; let period = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_PERIOD)) } as u64; let Some((next, missed)) = irq::next_timer_deadline(previous_deadline, period, now_after_log) else { write_line("TURKOS/IRQ-TIMER-ERROR: storm deadline overflow"); return 0; }; let before = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_SKIPPED)) }; unsafe { write_volatile( core::ptr::addr_of_mut!(IRQ_STORM_SKIPPED), before.saturating_add(missed as usize), ); write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_DEADLINE), next); asm!("msr CNTP_CVAL_EL0, {value}", "msr CNTP_CTL_EL0, {enable}", "isb", value = in(reg) next, enable = in(reg) 1usize, options(nomem, nostack)); } } return 1; } } #[cfg(feature = "irq-return-test")] fn irq_return_smoke_test() { // SAFETY: only the dedicated image writes this private guard before SVC. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; let x0: usize; let x3: usize; let x16: usize; let x17: usize; let x30: usize; let q0_low: usize; let q31_low: usize; // SAFETY: this image alone routes EL1h synchronous exceptions to the // returning vector. The handler restores every GP/SIMD register and ERET // resumes at the instruction after this deliberate SVC. unsafe { asm!( "movi v0.16b, #0x5a", "movi v31.16b, #0xa5", "svc #0x42", "umov {q0_low}, v0.d[0]", "umov {q31_low}, v31.d[0]", q0_low = lateout(reg) q0_low, q31_low = lateout(reg) q31_low, inout("x0") 0x5455_524b_4952_5130usize => x0, inout("x3") 0x5455_524b_4952_5133usize => x3, inout("x16") 0x5455_524b_4952_5116usize => x16, inout("x17") 0x5455_524b_4952_5117usize => x17, inout("x30") 0x5455_524b_4952_514cusize => x30, out("v0") _, out("v31") _, ); } let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) }; if x0 == 0x5455_524b_4952_5130 && x3 == 0x5455_524b_4952_5133 && x16 == 0x5455_524b_4952_5116 && x17 == 0x5455_524b_4952_5117 && x30 == 0x5455_524b_4952_514c && q0_low == 0x5a5a_5a5a_5a5a_5a5a && q31_low == 0xa5a5_a5a5_a5a5_a5a5 && guard == IRQ_GUARD { write_line("TURKOS/SVC-RETURN: PASS"); } else { write_line("TURKOS/SVC-RETURN-ERROR: restored registers"); } } #[cfg(any( feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-copy-irq-test", feature = "user-el0-irq-test" ))] fn prepare_test_interrupt(boot: &BootData, intid: usize) -> bool { if intid != 1 && intid != 2 && intid != 30 { return false; } let gicd = boot.interrupts.gic_distributor; let gicr = boot.interrupts.gic_redistributor; if gicd == 0 || gicr == 0 || gicd & 0xffff != 0 || gicr & 0xffff != 0 { return false; } // SAFETY: DTB discovery established the distributor/redistributor sizes. // This isolated single-CPU experiment configures one SGI or PPI before // translation is enabled, so these physical MMIO addresses are reachable. unsafe { let register = |base: u64, offset: usize| (base as usize + offset) as *mut u32; let distributor_id = read_volatile(register(gicd, 0xffe8)); let redistributor_id = read_volatile(register(gicr, 0xffe8)); let redistributor_type = read_volatile((gicr as usize + 0x8) as *const u64); let mpidr: u64; asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)); write_str("TURKOS/GICD-PIDR2: 0x"); write_hex(distributor_id as usize); write_str("\r\nTURKOS/GICR-PIDR2: 0x"); write_hex(redistributor_id as usize); write_str("\r\nTURKOS/GICR-TYPER: 0x"); write_hex(redistributor_type as usize); write_str("\r\nTURKOS/GIC-MPIDR: 0x"); write_hex(mpidr as usize); write_str("\r\n"); if irq::gic_arch_revision(distributor_id).is_none() || irq::gic_arch_revision(redistributor_id).is_none() || !irq::redistributor_matches_mpidr(redistributor_type, mpidr) { write_line("TURKOS/GIC-IDENTITY: FAIL"); return false; } write_line("TURKOS/GIC-IDENTITY: PASS"); let waker = register(gicr, 0x14); write_volatile(waker, read_volatile(waker) & !2); asm!("dsb sy", options(nostack, preserves_flags)); let mut awake = false; for _ in 0..1_000_000 { if read_volatile(waker) & 4 == 0 { awake = true; break; } } if !awake { return false; } let distributor = register(gicd, 0); write_volatile(distributor, read_volatile(distributor) | 0x22); // ARE_NS, Group 1 NS let mut ready = false; for _ in 0..1_000_000 { if read_volatile(distributor) & (1 << 31) == 0 { ready = true; break; } } if !ready { return false; } let group = register(gicr, 0x1_0080); let bit = 1u32 << intid; write_volatile(group, read_volatile(group) | bit); write_volatile((gicr as usize + 0x1_0400 + intid) as *mut u8, 0x80); write_volatile(register(gicr, 0x1_0100), bit); // GICR_ISENABLER0 asm!("dsb sy", options(nostack, preserves_flags)); let mut redistributor_ready = false; for _ in 0..1_000_000 { if read_volatile(register(gicr, 0)) & 8 == 0 { redistributor_ready = true; break; } } if !redistributor_ready { return false; } let sre: usize; asm!("mrs {value}, ICC_SRE_EL1", value = out(reg) sre, options(nomem, nostack, preserves_flags)); asm!("msr ICC_SRE_EL1, {value}", "isb", value = in(reg) (sre | 1), options(nomem, nostack)); let sre_after: usize; asm!("mrs {value}, ICC_SRE_EL1", value = out(reg) sre_after, options(nomem, nostack, preserves_flags)); if sre_after & 1 == 0 { return false; } asm!( "msr ICC_PMR_EL1, {priority}", "msr ICC_BPR1_EL1, xzr", "msr ICC_IGRPEN1_EL1, {enable}", "isb", priority = in(reg) 0xffusize, enable = in(reg) 1usize, options(nomem, nostack) ); } true } #[cfg(feature = "irq-sgi-test")] fn irq_sgi_smoke_test() { // SAFETY: only this test image owns the guard and unmasks IRQ inside the // bounded inline-assembly interval below. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; unsafe { write_volatile(core::ptr::addr_of_mut!(__stack_start), MAIN_STACK_GUARD) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_SGI_SEEN), 0) }; let mpidr: u64; // SAFETY: MPIDR_EL1 identifies the only running PE in this test image. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)) }; let Some(sgi_command) = irq::sgi1r_for_mpidr(mpidr, 1) else { write_line("TURKOS/IRQ-SGI-ERROR: target affinity"); return; }; write_line("TURKOS/IRQ-SGI: ARMED"); if cfg!(feature = "irq-gp-test") { irq_gp_smoke_test(sgi_command); return; } let x0: usize; let x3: usize; let x16: usize; let x17: usize; let x30: usize; let q0_low: usize; let q31_low: usize; let observed: usize; let returned_nzcv: usize; // SAFETY: GICv3 SGI 1 has been configured for this CPU. The IRQ vector // acknowledges/EOIs it, sets the flag, restores context and ERETs. The // timeout always remasks IRQ before returning to Rust. unsafe { asm!( "movi v0.16b, #0x5a", "movi v31.16b, #0xa5", "msr nzcv, {expected_nzcv}", "msr daifclr, #2", "msr ICC_SGI1R_EL1, {sgi}", "isb", "2:", "ldr {observed}, [{flag}]", "cbnz {observed}, 3f", "sub {budget}, {budget}, #1", "cbnz {budget}, 2b", "3:", "msr daifset, #2", "mrs {returned_nzcv}, nzcv", "umov {q0_low}, v0.d[0]", "umov {q31_low}, v31.d[0]", expected_nzcv = in(reg) irq::IRQ_PROBE_NZCV as usize, sgi = in(reg) sgi_command, flag = in(reg) core::ptr::addr_of!(IRQ_SGI_SEEN) as usize, budget = inout(reg) 50_000_000usize => _, observed = out(reg) observed, returned_nzcv = lateout(reg) returned_nzcv, q0_low = lateout(reg) q0_low, q31_low = lateout(reg) q31_low, inout("x0") 0x5455_524b_4952_5130usize => x0, inout("x3") 0x5455_524b_4952_5133usize => x3, inout("x16") 0x5455_524b_4952_5116usize => x16, inout("x17") 0x5455_524b_4952_5117usize => x17, inout("x30") 0x5455_524b_4952_514cusize => x30, out("v0") _, out("v31") _, ); } let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) }; let main_guard = unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) }; let count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_SGI_SEEN)) }; if observed == 1 && count == 1 && x0 == 0x5455_524b_4952_5130 && x3 == 0x5455_524b_4952_5133 && x16 == 0x5455_524b_4952_5116 && x17 == 0x5455_524b_4952_5117 && x30 == 0x5455_524b_4952_514c && q0_low == 0x5a5a_5a5a_5a5a_5a5a && q31_low == 0xa5a5_a5a5_a5a5_a5a5 && irq::nzcv_matches(returned_nzcv as u64, irq::IRQ_PROBE_NZCV) && guard == IRQ_GUARD && main_guard == MAIN_STACK_GUARD { write_line("TURKOS/IRQ-SGI-NZCV: PASS"); write_str("TURKOS/IRQ-MAIN-STACK-GUARD: 0x"); write_hex(main_guard as usize); write_str("\r\n"); write_line("TURKOS/IRQ-MAIN-STACK: PASS"); write_str("TURKOS/IRQ-SGI-COUNT: 0x"); write_hex(count); write_str("\r\n"); write_line("TURKOS/IRQ-SGI: PASS"); } else { write_line("TURKOS/IRQ-SGI-ERROR: timeout or restored registers"); } } #[cfg(feature = "irq-sgi-test")] fn irq_gp_smoke_test(sgi_command: u64) { #[cfg(feature = "irq-gp-test")] { let caller_sp_el0: usize; unsafe { asm!("mrs {value}, SP_EL0", value = out(reg) caller_sp_el0, options(nomem, nostack, preserves_flags)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(GP_IRQ_WAKE), 0) }; unsafe { write_volatile(core::ptr::addr_of_mut!(GP_PROBE_ACTIVE), 1) }; // SAFETY: the isolated assembly routine saves its caller's callee- // saved registers, owns IRQ unmasking, and copies all 31 return values // before restoring the Rust ABI state. unsafe { __gp_probe_run( sgi_command as usize, core::ptr::addr_of_mut!(GP_RETURNED).cast(), ) }; unsafe { write_volatile(core::ptr::addr_of_mut!(GP_PROBE_ACTIVE), 0) }; let returned = unsafe { read_volatile(core::ptr::addr_of!(GP_RETURNED)) }; let returned_sp_el0 = unsafe { read_volatile(core::ptr::addr_of!(GP_RETURNED_SP_EL0)) }; let after_sp_el0: usize; unsafe { asm!("mrs {value}, SP_EL0", value = out(reg) after_sp_el0, options(nomem, nostack, preserves_flags)) }; let mask = gp_register_mask(&returned); let count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_SGI_SEEN)) }; write_str("TURKOS/IRQ-GP-RESTORED-MASK: 0x"); write_hex(mask as usize); write_str("\r\nTURKOS/IRQ-GP-RESTORED-SP_EL0: 0x"); write_hex(returned_sp_el0 as usize); write_str("\r\nTURKOS/IRQ-GP-CALLER-SP_EL0: "); write_str(if after_sp_el0 == caller_sp_el0 { "PASS" } else { "FAIL" }); write_str("\r\nTURKOS/IRQ-GP-COUNT: 0x"); write_hex(count); write_str("\r\n"); if count == 1 && mask == 0x7fff_ffff && returned_sp_el0 == 0x7a5a && after_sp_el0 == caller_sp_el0 { write_line("TURKOS/IRQ-GP: PASS"); } else { write_line("TURKOS/IRQ-GP-ERROR: restored registers"); } } #[cfg(not(feature = "irq-gp-test"))] { let _ = sgi_command; } } #[cfg(feature = "irq-timer-test")] #[cfg_attr(feature = "irq-service-test", allow(dead_code))] fn irq_timer_smoke_test(gicr: u64) { // SAFETY: the dedicated image alone enables PPI 30 and uses this stack. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_SEEN), 0) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_MAIN_PROGRESS), 0) }; let frequency: usize; unsafe { asm!("mrs {value}, CNTFRQ_EL0", value = out(reg) frequency, options(nomem, nostack, preserves_flags)) }; let period_ns = if cfg!(feature = "irq-timer-storm-test") { 100_000 } else { 10_000_000 }; let Some(period_ticks) = irq::timer_ticks_for_ns(frequency as u64, period_ns) else { write_line("TURKOS/IRQ-TIMER-ERROR: frequency"); return; }; let period = period_ticks as usize; // The supported guest is AArch64. write_str("TURKOS/IRQ-TIMER-FREQ: 0x"); write_hex(frequency); write_str("\r\nTURKOS/IRQ-TIMER: ARMED\r\n"); #[cfg(feature = "irq-timer-storm-test")] { write_str("TURKOS/IRQ-STORM-PERIOD-NS: 0x"); write_hex(period_ns as usize); write_str("\r\n"); } let now: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) now, options(nomem, nostack, preserves_flags)) }; let Some((first_deadline, _)) = irq::next_timer_deadline(now, period_ticks, now) else { write_line("TURKOS/IRQ-TIMER-ERROR: first deadline overflow"); return; }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_PERIOD), period) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_DEADLINE), first_deadline) }; let (x0, x3, x16, x17, x30, q0_low, q31_low, observed, progress): ( usize, usize, usize, usize, usize, usize, usize, usize, usize, ); // SAFETY: the handler advances absolute deadlines, then disables the timer. The bounded // loop always remasks IRQ and disables the timer before Rust resumes. unsafe { asm!( "movi v0.16b, #0x5a", "movi v31.16b, #0xa5", "msr CNTP_CVAL_EL0, {deadline}", "msr CNTP_CTL_EL0, {enable}", "isb", "msr daifclr, #2", "2:", "add {progress}, {progress}, #1", "str {progress}, [{main_flag}]", "ldr {observed}, [{flag}]", "cmp {observed}, #100", "b.ge 3f", "subs {budget}, {budget}, #1", "b.ne 2b", "3:", "msr daifset, #2", "msr CNTP_CTL_EL0, xzr", "isb", "umov {q0_low}, v0.d[0]", "umov {q31_low}, v31.d[0]", deadline = in(reg) first_deadline, enable = in(reg) 1usize, flag = in(reg) core::ptr::addr_of!(IRQ_TIMER_SEEN) as usize, main_flag = in(reg) core::ptr::addr_of_mut!(IRQ_TIMER_MAIN_PROGRESS) as usize, budget = inout(reg) 10_000_000_000usize => _, progress = inout(reg) 0usize => progress, observed = out(reg) observed, q0_low = lateout(reg) q0_low, q31_low = lateout(reg) q31_low, inout("x0") 0x5455_524b_4952_5130usize => x0, inout("x3") 0x5455_524b_4952_5133usize => x3, inout("x16") 0x5455_524b_4952_5116usize => x16, inout("x17") 0x5455_524b_4952_5117usize => x17, inout("x30") 0x5455_524b_4952_514cusize => x30, out("v0") _, out("v31") _, ); } let count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_SEEN)) }; let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) }; let ppi_enabled_before: bool; let mut ppi_disabled_after = false; // SAFETY: only the timer image maps the validated 128 KiB GICR Device // window. IRQ is masked and CNTP_CTL_EL0 is already clear here. unsafe { let isenabler = (gicr as usize + 0x1_0100) as *const u32; let icenabler = (gicr as usize + 0x1_0180) as *mut u32; let control = gicr as *const u32; ppi_enabled_before = read_volatile(isenabler) & (1 << 30) != 0; write_volatile(icenabler, 1 << 30); asm!("dsb sy", options(nostack, preserves_flags)); for _ in 0..1_000_000 { if read_volatile(control) & 8 == 0 { ppi_disabled_after = read_volatile(isenabler) & (1 << 30) == 0; break; } } } write_str("TURKOS/IRQ-TIMER-PPI-ENABLED-BEFORE: 0x"); write_hex(ppi_enabled_before as usize); write_str("\r\n"); if ppi_enabled_before && ppi_disabled_after { write_line("TURKOS/IRQ-TIMER-PPI-MASK: PASS"); } else { write_line("TURKOS/IRQ-TIMER-PPI-MASK: FAIL"); } if observed == 100 && count == 100 && ppi_enabled_before && ppi_disabled_after && progress > 100 && x0 == 0x5455_524b_4952_5130 && x3 == 0x5455_524b_4952_5133 && x16 == 0x5455_524b_4952_5116 && x17 == 0x5455_524b_4952_5117 && x30 == 0x5455_524b_4952_514c && q0_low == 0x5a5a_5a5a_5a5a_5a5a && q31_low == 0xa5a5_a5a5_a5a5_a5a5 && guard == IRQ_GUARD { write_str("TURKOS/IRQ-TIMER-FIRST-DEADLINE: 0x"); write_hex(first_deadline as usize); write_str("\r\n"); write_str("TURKOS/IRQ-TIMER-COUNT: 0x"); write_hex(count); write_str("\r\nTURKOS/IRQ-TIMER-PROGRESS: 0x"); write_hex(progress); write_str("\r\n"); write_line("TURKOS/IRQ-TIMER: PASS"); #[cfg(feature = "irq-timer-storm-test")] { let interleaved = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_INTERLEAVED)) }; let skipped = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_SKIPPED)) }; write_str("TURKOS/IRQ-STORM-INTERLEAVED: 0x"); write_hex(interleaved); write_str("\r\nTURKOS/IRQ-STORM-SKIPPED: 0x"); write_hex(skipped); write_str("\r\n"); write_line( if irq::storm_main_progress_ok(interleaved, count, progress) { "TURKOS/IRQ-STORM: PASS" } else { "TURKOS/IRQ-STORM: FAIL" }, ); } } else { write_line("TURKOS/IRQ-TIMER-ERROR: timeout or restored registers"); } } #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] fn irq_unknown_smoke_test(gicr: u64) -> ! { // SAFETY: only this negative test image enables SGI 2 alongside PPI 30. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; let mpidr: u64; unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)) }; let Some(command) = irq::sgi1r_for_mpidr(mpidr, 2) else { write_line("TURKOS/IRQ-UNKNOWN-ERROR: target affinity"); park(); }; let enabled_before = unsafe { read_volatile((gicr as usize + 0x1_0100) as *const u32) & 4 != 0 }; write_str("TURKOS/IRQ-WRONG-MASK-ENABLED-BEFORE: 0x"); write_hex(enabled_before as usize); write_str("\r\n"); if !enabled_before { write_line("TURKOS/IRQ-WRONG-MASK: FAIL"); park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_UNKNOWN_GICR), gicr as usize) }; write_line("TURKOS/IRQ-UNKNOWN: ARMED"); // The first SGI reaches the real vector; the second is deliberately sent // while interrupts are still unmasked to prove the source stays disabled. unsafe { asm!("msr daifclr, #2", "msr ICC_SGI1R_EL1, {command}", "isb", command = in(reg) command, options(nostack)); } let mut progress = 0usize; for _ in 0..10_000_000 { progress += 1; if unsafe { read_volatile(core::ptr::addr_of!(IRQ_UNKNOWN_HANDLED)) } != 0 { break; } core::hint::spin_loop(); } let first_count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_UNKNOWN_HANDLED)) }; let masked_after = unsafe { read_volatile((gicr as usize + 0x1_0100) as *const u32) & 4 == 0 }; if first_count == 1 && masked_after { unsafe { asm!("msr ICC_SGI1R_EL1, {command}", "isb", command = in(reg) command, options(nostack)) }; write_line("TURKOS/IRQ-WRONG-MASK-SECOND-SGI: SENT"); for _ in 0..500_000 { progress += 1; core::hint::spin_loop(); } } unsafe { asm!("msr daifset, #2", "isb", options(nostack)) }; let final_count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_UNKNOWN_HANDLED)) }; let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) }; write_str("TURKOS/IRQ-WRONG-MASK-MASKED-AFTER: 0x"); write_hex(masked_after as usize); write_str("\r\nTURKOS/IRQ-WRONG-MASK-COUNT: 0x"); write_hex(final_count); write_str("\r\nTURKOS/IRQ-WRONG-MASK-PROGRESS: 0x"); write_hex(progress); write_str("\r\n"); if first_count == 1 && final_count == 1 && masked_after && progress > 500_000 && guard == IRQ_GUARD { write_line("TURKOS/IRQ-WRONG-MASK: PASS"); } else { write_line("TURKOS/IRQ-WRONG-MASK: FAIL"); } park() } #[cfg(feature = "irq-spurious-test")] fn irq_spurious_smoke_test() -> ! { // The GIC CPU interface is enabled but no SGI was sent and the physical // timer remains disabled. Polling the IAR is permitted at EL1; 1023 did // not acknowledge an interrupt, so neither read is followed by an EOI. let first: u64; let second: u64; unsafe { asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) first, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) second, options(nomem, nostack, preserves_flags)); } write_str("TURKOS/IRQ-SPURIOUS-FIRST: 0x"); write_hex(first as usize); write_str("\r\nTURKOS/IRQ-SPURIOUS-SECOND: 0x"); write_hex(second as usize); write_str("\r\n"); if irq::classify_iar(first, 30) == irq::IarClass::Special(1023) && irq::classify_iar(second, 30) == irq::IarClass::Special(1023) { write_line("TURKOS/IRQ-SPURIOUS-NO-EOI: PASS"); } else { write_line("TURKOS/IRQ-SPURIOUS-ERROR: unexpected IAR"); } park() } #[cfg(feature = "mmu-tlb-test")] fn mmu_tlb_probe(allocator: &mut memory::FrameAllocator, tables: &mmu::Tables) -> ! { const A: u64 = 0x5455_524b_544c_4241; const B: u64 = 0x5455_524b_544c_4242; let before_pages = allocator.remaining_pages(); let Some(first) = allocator.allocate() else { write_line("TURKOS/TLB-ERROR: first frame unavailable"); park(); }; let Some(second) = allocator.allocate() else { write_line("TURKOS/TLB-ERROR: second frame unavailable"); park(); }; if first == second { write_line("TURKOS/TLB-ERROR: duplicate frames"); park(); } // SAFETY: both pages are exclusively allocated, writable and initially // identity-mapped. The first words are restored before ownership returns. let (saved_first, saved_second) = unsafe { let old_first = read_volatile(first as *const u64); let old_second = read_volatile(second as *const u64); write_volatile(first as *mut u64, A); write_volatile(second as *mut u64, B); (old_first, old_second) }; // SAFETY: the allocated pages are data frames. The table helper rejects // any frame outside the first 2 MiB or a descriptor not matching A. let Ok((slot, second_descriptor)) = (unsafe { tables.probe_replacement(first, first, second) }) else { write_line("TURKOS/TLB-ERROR: unsafe first replacement"); park(); }; write_str("TURKOS/TLB-VA: 0x"); write_hex(first as usize); write_str("\r\nTURKOS/TLB-PA-B: 0x"); write_hex(second as usize); write_str("\r\nTURKOS/TLB-BEFORE: 0x"); let initial = unsafe { read_volatile(first as *const u64) }; write_hex(initial as usize); write_str("\r\n"); if initial != A || unsafe { read_volatile(second as *const u64) } != B { write_line("TURKOS/TLB-ERROR: initial physical canaries"); park(); } // SAFETY: the assembly performs break-before-make with a local address // operand and inner-shareable TLBI; all pages and the L3 slot remain owned. unsafe { __turkos_replace_l3_probe(slot, first >> 12, second_descriptor) }; let after = unsafe { read_volatile(first as *const u64) }; write_str("TURKOS/TLB-AFTER: 0x"); write_hex(after as usize); write_str("\r\n"); if after != B || unsafe { read_volatile(slot) } != second_descriptor { write_line("TURKOS/TLB-ERROR: old translation remained"); park(); } let Ok((slot, first_descriptor)) = (unsafe { tables.probe_replacement(first, second, first) }) else { write_line("TURKOS/TLB-ERROR: unsafe restore replacement"); park(); }; unsafe { __turkos_replace_l3_probe(slot, first >> 12, first_descriptor) }; let restored = unsafe { read_volatile(first as *const u64) }; write_str("TURKOS/TLB-RESTORED: 0x"); write_hex(restored as usize); write_str("\r\n"); if restored != A || unsafe { read_volatile(slot) } != first_descriptor { write_line("TURKOS/TLB-ERROR: identity restore failed"); park(); } unsafe { write_volatile(first as *mut u64, saved_first); write_volatile(second as *mut u64, saved_second); } if allocator.release(second).is_err() || allocator.release(first).is_err() { write_line("TURKOS/TLB-ERROR: frame release"); park(); } let after_pages = allocator.remaining_pages(); write_str("TURKOS/TLB-FRAME-ACCOUNT: BEFORE=0x"); write_hex(before_pages as usize); write_str(" AFTER=0x"); write_hex(after_pages as usize); write_str("\r\n"); if before_pages != after_pages { write_line("TURKOS/TLB-ERROR: frame ownership drift"); park(); } write_line("TURKOS/TLB-BBM: PASS"); park() } #[cfg(feature = "heap-test")] fn write_heap_allocation(kind: &str, handle: heap::Handle, size: usize, align: usize) { write_str("TURKOS/HEAP-ALLOC: KIND="); write_str(kind); write_str(" ADDR=0x"); write_hex(handle.address); write_str(" SIZE=0x"); write_hex(size); write_str(" ALIGN=0x"); write_hex(align); write_str("\r\n"); } #[cfg(feature = "heap-test")] fn heap_object_smoke_test() -> bool { // SAFETY: HeapImage alone owns the four mapped writable pages; this // experiment is single-CPU and never hands pointers to another context. // Metadata lives in BSS to keep the 16 KiB early boot stack bounded. static mut HEAP_STORAGE: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); let storage = core::ptr::addr_of_mut!(HEAP_STORAGE).cast::(); let Ok(()) = (unsafe { heap::Heap::init_in_place(storage, mmu::HEAP_WINDOW_START as usize) }) else { return false; }; let heap = unsafe { &mut *storage }; let Ok(small) = heap.allocate(16, 16) else { return false; }; let Ok(aligned) = heap.allocate(32, 64) else { return false; }; let Ok(page) = heap.allocate(4096, 4096) else { return false; }; let Ok(crossing) = heap.allocate(5000, 8) else { return false; }; if small.address % 16 != 0 || aligned.address % 64 != 0 || page.address % 4096 != 0 || crossing.address / 4096 == (crossing.address + 4999) / 4096 { return false; } for (kind, handle, size, align) in [ ("SMALL16", small, 16, 16), ("ALIGN64", aligned, 32, 64), ("PAGE", page, 4096, 4096), ("CROSS", crossing, 5000, 8), ] { // SAFETY: first and last requested bytes belong to this allocation. unsafe { write_volatile(handle.address as *mut u8, 0x5a); write_volatile((handle.address + size - 1) as *mut u8, 0xa5); if read_volatile(handle.address as *const u8) != 0x5a || read_volatile((handle.address + size - 1) as *const u8) != 0xa5 { return false; } } write_heap_allocation(kind, handle, size, align); } if heap.allocate(0, 8) != Err(heap::Error::Invalid) { return false; } write_line("TURKOS/HEAP-ZERO: PASS"); if heap.allocate(usize::MAX, 8) != Err(heap::Error::Overflow) || heap.allocate(1, 3) != Err(heap::Error::Invalid) { return false; } write_line("TURKOS/HEAP-OVERFLOW: PASS"); if heap.allocate(heap::CAPACITY - 8, 1) != Err(heap::Error::OutOfMemory) { return false; } write_line("TURKOS/HEAP-OOM: PASS"); let trailer = small.address + 16; // SAFETY: the eight-byte trailer was written by Heap::allocate; altering // one bit must make release fail without changing ownership. let previous = unsafe { core::ptr::read_unaligned(trailer as *const u64) }; unsafe { core::ptr::write_unaligned(trailer as *mut u64, previous ^ 1) }; if heap.release(small) != Err(heap::Error::CorruptedCanary) || heap.active_count() != 4 { return false; } unsafe { core::ptr::write_unaligned(trailer as *mut u64, previous) }; write_line("TURKOS/HEAP-CANARY: PASS"); if heap.release(heap::Handle { address: small.address - 8, ..small }) != Err(heap::Error::Invalid) { return false; } write_line("TURKOS/HEAP-FOREIGN: PASS"); for handle in [crossing, page, aligned, small] { if heap.release(handle).is_err() { return false; } } write_line("TURKOS/HEAP-REVERSE-FREE: PASS"); if heap.release(small) != Err(heap::Error::DoubleFree) { return false; } write_line("TURKOS/HEAP-DOUBLE-FREE: PASS"); if heap.active_count() != 0 || heap.free_bytes() != heap::CAPACITY || heap.largest_free() != heap::CAPACITY { return false; } write_line("TURKOS/HEAP-COALESCE: PASS"); let Ok(full) = heap.allocate(heap::CAPACITY - 8, 1) else { return false; }; if heap.allocate(1, 1) != Err(heap::Error::OutOfMemory) || heap.release(full).is_err() { return false; } write_line("TURKOS/HEAP-OOM-RECOVERY: PASS"); let mut small_handles: [Option; 32] = [None; 32]; for index in 0..small_handles.len() { let Ok(handle) = heap.allocate(32, 16) else { return false; }; if small_handles[..index] .iter() .flatten() .any(|prior| prior.address == handle.address) { return false; } small_handles[index] = Some(handle); } for handle in small_handles.into_iter().rev().flatten() { if heap.release(handle).is_err() { return false; } } write_line("TURKOS/HEAP-SMALL-MANY: PASS"); write_str("TURKOS/HEAP-ACCOUNT: USED=0x"); write_hex(heap.used_bytes()); write_str(" FREE=0x"); write_hex(heap.free_bytes()); write_str(" LARGEST=0x"); write_hex(heap.largest_free()); write_str(" ACTIVE=0x"); write_hex(heap.active_count()); write_str("\r\n"); heap.used_bytes() == 0 && heap.free_bytes() == heap::CAPACITY && heap.largest_free() == heap::CAPACITY && heap.active_count() == 0 } #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] fn mmu_smoke_test(free: memory::FreeMap, boot: &BootData, level: usize) -> ! { if level != 1 { write_line("TURKOS/MMU-ERROR: EL1 required"); park(); } let mut allocator = memory::FrameAllocator::new(free); let hardware = &boot.hardware; let layout = mmu::Layout { ram_base: hardware.ram_base, ram_size: hardware.ram_size, uart_base: hardware.uart_base, image_start: core::ptr::addr_of!(_image_start) as u64, text_start: core::ptr::addr_of!(_text_start) as u64, text_end: core::ptr::addr_of!(_text_end) as u64, rodata_end: core::ptr::addr_of!(_rodata_end) as u64, image_end: core::ptr::addr_of!(_image_end) as u64, irq_guard_page: Some(core::ptr::addr_of!(__irq_guard_page_start) as u64), heap_start: if cfg!(feature = "heap-test") { Some(mmu::HEAP_WINDOW_START) } else { None }, }; #[cfg(any( feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-copy-irq-test", feature = "user-el0-irq-test" ))] { #[cfg(feature = "irq-sgi-test")] let intid = 1; #[cfg(feature = "user-copy-irq-test")] let intid = 1; #[cfg(feature = "user-el0-irq-test")] let intid = boot.interrupts.nonsecure_phys_timer_ppi as usize; #[cfg(feature = "irq-timer-test")] let intid = boot.interrupts.nonsecure_phys_timer_ppi as usize; #[cfg(feature = "irq-unknown-test")] let intid = 2; #[cfg(any(feature = "task-preempt-test", feature = "task-sleep-test"))] let intid = boot.interrupts.nonsecure_phys_timer_ppi as usize; if !prepare_test_interrupt(boot, intid) { write_line("TURKOS/IRQ-ERROR: GIC setup"); park(); } #[cfg(feature = "user-el0-irq-test")] user_syscall::set_el0_irq_intid(intid); #[cfg(feature = "irq-unknown-test")] if !prepare_test_interrupt(boot, 30) { write_line("TURKOS/IRQ-ERROR: PPI setup"); park(); } } // SAFETY: FreeMap excludes the kernel, DTB, and reserved memory. This // dedicated guest keeps every table frame allocated until it is stopped. let free_before_tables = allocator.remaining_pages(); write_str("TURKOS/MMU-LINKER: TEXT-START=0x"); write_hex(layout.text_start as usize); write_str(" TEXT-END=0x"); write_hex(layout.text_end as usize); write_str(" RODATA-END=0x"); write_hex(layout.rodata_end as usize); write_str("\r\n"); #[allow(unused_mut)] let mut tables = match unsafe { mmu::build_identity_tables(&mut allocator, layout, boot.static_reserved.as_slice()) } { Ok(tables) => tables, Err(_) => { let sctlr_after_failure: u64; // SAFETY: SCTLR_EL1 is readable at EL1 even when table planning fails. unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr_after_failure, options(nomem, nostack, preserves_flags)) }; if sctlr_after_failure & 1 == 0 && allocator.remaining_pages() == free_before_tables { write_line("TURKOS/MMU-BUILD-FAIL-PRE-TTBR: PASS"); } write_line("TURKOS/MMU-ERROR: table build"); park() } }; #[cfg(any( feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-mmio-read-test", feature = "user-el0-irq-test" ))] { let gicr = boot.interrupts.gic_redistributor; #[cfg(feature = "user-mmio-read-test")] if gicr != 0x080a_0000 { write_line("TURKOS/USER-MMIO-AUDIT-ERROR: unexpected DTB GICR window"); park(); } // SAFETY: DTB validation established a 128 KiB redistributor window; // this isolated timer image maps it before installing TTBR0. if unsafe { tables.map_device_window(&mut allocator, gicr, 0x2_0000, layout.uart_base) } .is_err() { write_line("TURKOS/MMU-ERROR: GICR window"); park(); } write_str("TURKOS/MMU-GICR-WINDOW: 0x"); write_hex(gicr as usize); write_str("+0x"); write_hex(0x2_0000); write_str("\r\n"); } #[cfg(any( feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-tx-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "user-mmio-read-test" ))] { let window = dtb::virtio_page_window( &boot.virtio_mmio, boot.hardware.ram_base, boot.hardware.ram_size, ) .unwrap_or_else(|_| { write_line("TURKOS/VIRTIO-MAGIC-ERROR: invalid DTB page window"); park() }); #[cfg(feature = "user-mmio-read-test")] if window.start != 0x0a00_0000 || window.end != 0x0a00_4000 { write_line("TURKOS/USER-MMIO-AUDIT-ERROR: unexpected DTB virtio window"); park(); } // SAFETY: the DTB slots passed the RAM-disjoint page-window check; // this isolated image owns the added table until it stops. if unsafe { tables.map_device_window( &mut allocator, window.start, window.end - window.start, layout.uart_base, ) } .is_err() { write_line("TURKOS/VIRTIO-MAGIC-ERROR: MMU Device mapping"); park() } write_str("TURKOS/MMU-VIRTIO-WINDOW: 0x"); write_hex(window.start as usize); write_str("+0x"); write_hex((window.end - window.start) as usize); write_str("\r\n"); } let free_after_tables = allocator.remaining_pages(); let tagged_table_frames = tables.tagged_frames().count(); let table_tags_valid = tagged_table_frames == tables.frame_count() && tables.tagged_frames().all(|tag| { tag.address != 0 && tag.address % memory::PAGE_SIZE == 0 && tables .tagged_frames() .filter(|other| other.address == tag.address) .count() == 1 }); if !table_tags_valid || free_before_tables.checked_sub(free_after_tables) != Some(tables.frame_count() as u64) { write_line("TURKOS/MMU-ERROR: table ownership count"); park(); } write_str("TURKOS/MMU-TABLE-ACCOUNT: BEFORE=0x"); write_hex(free_before_tables as usize); write_str(" AFTER=0x"); write_hex(free_after_tables as usize); write_str(" COUNT=0x"); write_hex(tables.frame_count()); write_str("\r\n"); #[cfg(feature = "heap-test")] let heap_pages = { if !tables.heap_window_unmapped() { write_line("TURKOS/HEAP-ERROR: virtual window was mapped"); park(); } write_str("TURKOS/HEAP-WINDOW: START=0x"); write_hex(mmu::HEAP_WINDOW_START as usize); write_str(" END=0x"); write_hex( (mmu::HEAP_WINDOW_START + mmu::HEAP_WINDOW_PAGES as u64 * memory::PAGE_SIZE) as usize, ); write_str("\r\nTURKOS/HEAP-UNMAPPED-BEFORE: PASS\r\n"); let before = allocator.remaining_pages(); // SAFETY: the free map reserved the heap's identity range before // building tables, and TTBR0 has not yet been installed. let pages = match unsafe { tables.map_heap_prefix(&mut allocator) } { Ok(pages) => pages, Err(_) => { write_line("TURKOS/HEAP-ERROR: map prefix"); park(); } }; if !tables.heap_tail_unmapped() || before.checked_sub(allocator.remaining_pages()) != Some(pages.count() as u64) { write_line("TURKOS/HEAP-ERROR: map ownership"); park(); } write_str("TURKOS/HEAP-FRAME-ACCOUNT: BEFORE=0x"); write_hex(before as usize); write_str(" AFTER=0x"); write_hex(allocator.remaining_pages() as usize); write_str(" COUNT=0x"); write_hex(pages.count()); write_str("\r\nTURKOS/HEAP-TAIL-UNMAPPED: PASS\r\n"); for (index, frame) in pages.frames.iter().copied().enumerate() { write_str("TURKOS/HEAP-MAP: INDEX=0x"); write_hex(index); write_str(" VA=0x"); write_hex((mmu::HEAP_WINDOW_START + index as u64 * memory::PAGE_SIZE) as usize); write_str(" PA=0x"); write_hex(frame as usize); write_str("\r\n"); } pages }; let sctlr_before: u64; let mmfr0: u64; // SAFETY: these privileged system registers are readable at EL1. unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr_before, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ID_AA64MMFR0_EL1", value = out(reg) mmfr0, options(nomem, nostack, preserves_flags)); } if sctlr_before & 1 != 0 || (mmfr0 & 0xf) == 7 { write_line("TURKOS/MMU-ERROR: boot register state"); park(); } let Some(tcr) = mmu::tcr_el1((mmfr0 & 0xf).min(5)) else { write_line("TURKOS/MMU-ERROR: physical address width"); park() }; write_str("TURKOS/MMU-ROOT: 0x"); write_hex(tables.root as usize); write_str("\r\nTURKOS/MMU-TABLES: 0x"); write_hex(tables.frame_count()); write_str("\r\nTURKOS/MMU-BLOCKS: 0x"); write_hex(tables.block_count()); write_str("\r\n"); write_line("TURKOS/MMU-PRE-ENABLE: PASS"); // SAFETY: the root is a 4 KiB-aligned L1 table. Identity mapping covers // the current PC, stack, VBAR, UART, and all table pages. IRQ remains masked. unsafe { asm!( "dsb sy", "msr MAIR_EL1, {mair}", "msr TCR_EL1, {tcr}", "msr TTBR0_EL1, {root}", "isb", "tlbi vmalle1", "dsb sy", "isb", "msr SCTLR_EL1, {sctlr}", "isb", mair = in(reg) mmu::MAIR_EL1, tcr = in(reg) tcr, root = in(reg) tables.root, sctlr = in(reg) (sctlr_before | 1), options(nostack) ); } let sctlr_after: u64; // SAFETY: SCTLR_EL1 remains readable after enabling translation. unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr_after, options(nomem, nostack, preserves_flags)) }; if sctlr_after & 1 == 0 { write_line("TURKOS/MMU-ERROR: SCTLR.M is clear"); park(); } write_line("TURKOS/MMU: ON"); let actual_ttbr: u64; let actual_mair: u64; let actual_tcr: u64; let actual_vbar: u64; let actual_sp: u64; // SAFETY: all registers are readable in the EL1-only smoke-test context. unsafe { asm!("mrs {value}, TTBR0_EL1", value = out(reg) actual_ttbr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, MAIR_EL1", value = out(reg) actual_mair, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, TCR_EL1", value = out(reg) actual_tcr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, VBAR_EL1", value = out(reg) actual_vbar, options(nomem, nostack, preserves_flags)); asm!("mov {value}, sp", value = out(reg) actual_sp, options(nomem, nostack, preserves_flags)); } let vector_address = __vectors_start as *const () as u64; if actual_ttbr != tables.root || actual_ttbr & (memory::PAGE_SIZE - 1) != 0 || actual_mair != mmu::MAIR_EL1 || actual_tcr != tcr || actual_vbar != vector_address || actual_vbar < layout.text_start || actual_vbar >= layout.text_end || actual_sp < hardware.ram_base || actual_sp > hardware.ram_base + hardware.ram_size || actual_sp & 15 != 0 { write_line("TURKOS/MMU-ERROR: post-enable registers"); park(); } write_str("TURKOS/MMU-TTBR0: 0x"); write_hex(actual_ttbr as usize); write_str("\r\nTURKOS/MMU-MAIR: 0x"); write_hex(actual_mair as usize); write_str("\r\nTURKOS/MMU-TCR: 0x"); write_hex(actual_tcr as usize); write_str("\r\nTURKOS/MMU-VBAR: 0x"); write_hex(actual_vbar as usize); write_str("\r\nTURKOS/MMU-SP: 0x"); write_hex(actual_sp as usize); write_str("\r\n"); let dtb_bytes = boot.dtb_address as *const u8; // SAFETY: read_boot_data already validated this FDT header and the DTB is // still identity-mapped RAM while the same guest is running. let dtb_magic = unsafe { u32::from_be_bytes([ read_volatile(dtb_bytes), read_volatile(dtb_bytes.add(1)), read_volatile(dtb_bytes.add(2)), read_volatile(dtb_bytes.add(3)), ]) }; if dtb_magic != 0xd00d_feed { write_line("TURKOS/MMU-ERROR: DTB lost after enable"); park(); } write_line("TURKOS/MMU-DTB: PASS"); let free_before_probe = allocator.remaining_pages(); let mut probe_frames = [0u64; 16]; let mut probe_saved = [0u64; 16]; for index in 0..probe_frames.len() { let Some(frame) = allocator.allocate() else { write_line("TURKOS/MMU-ERROR: post-enable frame unavailable"); park(); }; probe_frames[index] = frame; // SAFETY: each frame is exclusively owned, writable, and identity-mapped. unsafe { probe_saved[index] = read_volatile(frame as *const u64); let marker = 0x5455_524b_4f53_0000u64 | index as u64; write_volatile(frame as *mut u64, marker); if read_volatile(frame as *const u64) != marker { write_line("TURKOS/MMU-ERROR: post-enable frame mismatch"); park(); } } } for index in (0..probe_frames.len()).rev() { // SAFETY: the frame is still owned and its original first word is restored. unsafe { write_volatile(probe_frames[index] as *mut u64, probe_saved[index]) }; if allocator.release(probe_frames[index]).is_err() { write_line("TURKOS/MMU-ERROR: post-enable frame release"); park(); } } if allocator.remaining_pages() != free_before_probe { write_line("TURKOS/MMU-ERROR: post-enable frame ownership"); park(); } write_str("TURKOS/MMU-FRAME-COUNT: 0x"); write_hex(probe_frames.len()); write_str("\r\n"); write_line("TURKOS/MMU-FRAME: PASS"); let data_address = core::ptr::addr_of_mut!(MMU_DATA_CANARY) as u64; if data_address < layout.rodata_end || data_address >= layout.image_end { write_line("TURKOS/MMU-ERROR: data canary outside data"); park(); } // SAFETY: this initialized static belongs to the kernel's writable data // section, and this single-CPU smoke test restores its original value. unsafe { let previous = read_volatile(data_address as *const u64); write_volatile(data_address as *mut u64, previous ^ u64::MAX); if read_volatile(data_address as *const u64) != previous ^ u64::MAX { write_line("TURKOS/MMU-ERROR: data canary mismatch"); park(); } write_volatile(data_address as *mut u64, previous); } write_line("TURKOS/MMU-DATA-WRITE: PASS"); #[cfg(feature = "heap-test")] { for (index, frame) in heap_pages.frames.iter().copied().enumerate() { let address = mmu::HEAP_WINDOW_START + index as u64 * memory::PAGE_SIZE; let marker = 0x5455_524b_4845_4150u64 ^ index as u64; // SAFETY: the heap VA is backed by this exclusively owned frame; // both aliases are readable in the isolated early-boot test. unsafe { write_volatile(address as *mut u64, marker); if read_volatile(address as *const u64) != marker || read_volatile(frame as *const u64) != marker { write_line("TURKOS/HEAP-ERROR: VA/PA mismatch"); park(); } } } write_line("TURKOS/HEAP-VA-PA: PASS"); if !heap_object_smoke_test() { write_line("TURKOS/HEAP-ERROR: object contract"); park(); } let before_return = allocator.remaining_pages(); for (index, frame) in heap_pages.frames.iter().copied().enumerate().rev() { let address = mmu::HEAP_WINDOW_START + index as u64 * memory::PAGE_SIZE; // SAFETY: the object test proved no active handles and complete // coalescing. This CPU is the only user of these VAs. The valid // descriptor is cleared before targeted TLBI and frame reuse. if unsafe { tables.detach_heap_page(index, frame) }.is_err() { write_line("TURKOS/HEAP-ERROR: detach descriptor"); park(); } unsafe { asm!( "dsb ishst", "tlbi vaae1is, {page}", "dsb ish", "isb", page = in(reg) address >> 12, options(nostack) ); } if allocator.release(frame).is_err() { write_line("TURKOS/HEAP-ERROR: return frame"); park(); } write_str("TURKOS/HEAP-RETURN: INDEX=0x"); write_hex(index); write_str(" VA=0x"); write_hex(address as usize); write_str(" PA=0x"); write_hex(frame as usize); write_str("\r\n"); } if !tables.heap_window_unmapped() { write_line("TURKOS/HEAP-ERROR: descriptor remained"); park(); } write_line("TURKOS/HEAP-UNMAPPED-AFTER: PASS"); let after_return = allocator.remaining_pages(); write_str("TURKOS/HEAP-FRAME-RETURN: BEFORE=0x"); write_hex(before_return as usize); write_str(" AFTER=0x"); write_hex(after_return as usize); write_str(" COUNT=0x"); write_hex(heap_pages.count()); write_str("\r\n"); if after_return != before_return + heap_pages.count() as u64 { write_line("TURKOS/HEAP-ERROR: return accounting"); park(); } write_line("TURKOS/HEAP: PASS"); #[cfg(feature = "heap-panic-test")] { write_line("TURKOS/HEAP-PANIC-READY: ALL-PAGES-UNMAPPED"); panic!("heap unavailable by design"); } #[cfg(not(feature = "heap-panic-test"))] park(); } #[cfg(feature = "mmu-tlb-test")] mmu_tlb_probe(&mut allocator, &tables); #[cfg(feature = "task-state-test")] { if task_state_smoke_test() { write_line("TURKOS/TASK-MODEL: PASS"); } else { write_line("TURKOS/TASK-ERROR: state contract"); } park(); } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] task_switch_smoke_test(); #[cfg(feature = "task-guard-test")] task_guard::smoke_test(&tables, layout); #[cfg(feature = "user-three-test")] process_probe::run(&mut allocator, &tables, layout); #[cfg(any( feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test" ))] user_first::smoke_test(&mut allocator, &tables, layout); #[cfg(any( feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test" ))] user_dual::smoke_test(&mut allocator, &tables, layout); #[cfg(any( feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test" ))] user_syscall::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "elf-ident-test")] elf_ident_probe::smoke_test(); #[cfg(feature = "elf-layout-test")] elf_layout_probe::smoke_test(); #[cfg(feature = "elf-load-test")] elf_load_probe::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "elf-stack-test")] elf_stack_probe::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "elf-hello-test")] elf_hello_probe::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "elf-sequence-test")] elf_sequence_probe::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "ipc-basic-test")] ipc_basic_probe::smoke_test(); #[cfg(feature = "ipc-wait-test")] ipc_wait_probe::smoke_test(); #[cfg(feature = "task-cancel-test")] task_cancel_probe::smoke_test(); #[cfg(feature = "ipc-multi-test")] ipc_multi_probe::smoke_test(); #[cfg(feature = "ipc-timeline-test")] ipc_timeline_probe::smoke_test(); #[cfg(feature = "virtio-discover-test")] virtio_discovery_smoke_test(&boot); #[cfg(feature = "virtio-magic-test")] virtio_magic_probe::smoke_test(&boot); #[cfg(feature = "virtio-net-id-test")] virtio_net_id_probe::smoke_test(&boot); #[cfg(feature = "virtio-net-mac-test")] virtio_net_mac_probe::smoke_test(&boot); #[cfg(all(feature = "virtio-net-rx-test", not(feature = "virtio-net-pool-test")))] virtio_net_rx_probe::smoke_test(&boot, &mut allocator); #[cfg(any( feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test" ))] virtio_net_arp_reply_probe::smoke_test(&boot, &mut allocator); #[cfg(feature = "virtio-net-pool-test")] virtio_net_pool_probe::smoke_test(&boot, &mut allocator); #[cfg(all( feature = "virtio-net-tx-test", not(feature = "virtio-net-tx-completion-test") ))] virtio_net_tx_probe::smoke_test(&boot, &mut allocator); #[cfg(feature = "virtio-net-tx-completion-test")] virtio_net_tx_probe::completion_smoke_test(&boot, &mut allocator); #[cfg(feature = "virtio-modern-features-test")] virtio_modern_probe::smoke_test(&boot, &mut allocator); #[cfg(feature = "p12-journal-test")] journal_probe::run(&boot, &mut allocator); #[cfg(feature = "turkfs-init-test")] virtio_init_probe::smoke_test(&boot, &mut allocator, &tables, layout); #[cfg(feature = "serial-rx-test")] serial_rx_probe::smoke_test(); #[cfg(feature = "serial-line-test")] serial_line_probe::smoke_test(); #[cfg(feature = "serial-edit-test")] serial_edit_probe::smoke_test(); #[cfg(feature = "serial-limit-test")] serial_limit_probe::smoke_test(); #[cfg(feature = "serial-console-test")] serial_console_probe::smoke_test(); #[cfg(feature = "serial-unknown-test")] serial_unknown_probe::smoke_test(); #[cfg(feature = "serial-utf8-test")] serial_utf8_probe::smoke_test(); #[cfg(feature = "serial-arguments-test")] serial_arguments_probe::smoke_test(); #[cfg(feature = "serial-range-test")] serial_range_probe::smoke_test(); #[cfg(feature = "serial-cancel-test")] serial_cancel_probe::smoke_test(); #[cfg(any( feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] serial_line_lock_probe::smoke_test(); #[cfg(feature = "serial-echo-test")] serial_echo_probe::smoke_test(); #[cfg(feature = "turkfs-ls-test")] turkfs_ls_probe::smoke_test(&boot, &mut allocator); #[cfg(feature = "turkfs-cat-test")] turkfs_cat_probe::smoke_test(&boot, &mut allocator); #[cfg(all( feature = "virtio-modern-queue-test", not(feature = "turkfs-init-test"), not(feature = "turkfs-ls-test"), not(feature = "turkfs-cat-test"), not(feature = "p12-journal-test") ))] virtio_queue_probe::smoke_test(&boot, &mut allocator); #[cfg(feature = "user-task-test")] user_task::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "user-handle-test")] user_task::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "user-handle-transfer-test")] user_task::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "user-handle-death-test")] user_task::smoke_test(&mut allocator, &tables, layout); #[cfg(feature = "user-wait-test")] user_task::smoke_test(&mut allocator, &tables, layout); #[cfg(any(feature = "task-preempt-test", feature = "task-sleep-test"))] preempt::smoke_test(); #[cfg(feature = "mmu-no-map-test")] { let Some(region) = boot.static_reserved.as_slice().iter().find(|region| { region.no_map && ((region.start == hardware.ram_base && region.size == 4 * memory::PAGE_SIZE) || (region.start == hardware.ram_base + (1 << 21) + memory::PAGE_SIZE && region.size == memory::PAGE_SIZE) || (region.start == hardware.ram_base + (1 << 21) && region.size == (1 << 21))) }) else { write_line("TURKOS/MMU-ERROR: no-map fixture missing"); park(); }; let first_hole = region.start == hardware.ram_base && region.size == 4 * memory::PAGE_SIZE; let distant_hole = region.start == hardware.ram_base + (1 << 21) + memory::PAGE_SIZE && region.size == memory::PAGE_SIZE; let full_block_hole = region.start == hardware.ram_base + (1 << 21) && region.size == (1 << 21); if !first_hole && !distant_hole && !full_block_hole { write_line("TURKOS/MMU-ERROR: unexpected no-map fixture"); park(); } let adjacent = (region.start + region.size + memory::PAGE_SIZE - 1) & !(memory::PAGE_SIZE - 1); if distant_hole || full_block_hole { // SAFETY: the page immediately before the isolated distant hole // remains in the same split L2 block and must retain a descriptor. unsafe { read_volatile((region.start - memory::PAGE_SIZE) as *const u64) }; write_line("TURKOS/MMU-PREVIOUS: PASS"); } // SAFETY: both dedicated fixtures leave the next RAM page mapped. The // read proves the hole is local even after an L2 block is split. unsafe { read_volatile(adjacent as *const u64) }; write_line("TURKOS/MMU-ADJACENT: PASS"); write_line("TURKOS/MMU-PROBE: NO-MAP"); // SAFETY: the read intentionally traps on a reserved, unmapped page. unsafe { read_volatile(region.start as *const u64) }; write_line("TURKOS/MMU-ERROR: no-map read returned"); park() } #[cfg(feature = "mmu-write-test")] { write_line("TURKOS/MMU-PROBE: TEXT-WRITE"); // SAFETY: this intentionally triggers an EL1 write-permission fault. unsafe { write_volatile(core::ptr::addr_of!(_image_start) as *mut u64, 0) }; write_line("TURKOS/MMU-ERROR: text write returned"); park() } #[cfg(feature = "mmu-nx-test")] { let target = core::ptr::addr_of_mut!(NX_PROBE) as u64; if target < layout.rodata_end || target >= layout.image_end || target & 3 != 0 { write_line("TURKOS/MMU-ERROR: NX probe outside data"); park(); } write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: DATA-EXEC"); // SAFETY: the dedicated test intentionally branches to an XN data // page. The fatal vector reports the instruction-abort frame. unsafe { asm!("br {target}", target = in(reg) target, options(noreturn)) } } #[cfg(feature = "mmu-rodata-test")] { let target = core::ptr::addr_of!(RODATA_PROBE) as u64; if target < layout.text_end || target >= layout.rodata_end { write_line("TURKOS/MMU-ERROR: rodata probe outside rodata"); park(); } write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: RODATA-WRITE"); // SAFETY: the dedicated test intentionally writes to a read-only // page. The fatal vector reports the data-abort frame. unsafe { write_volatile(target as *mut u64, 0) }; write_line("TURKOS/MMU-ERROR: rodata write returned"); park() } #[cfg(feature = "mmu-rodata-exec-test")] { let target = core::ptr::addr_of!(RODATA_EXEC_PROBE) as u64; if target < layout.text_end || target >= layout.rodata_end || target & 3 != 0 { write_line("TURKOS/MMU-ERROR: rodata exec probe outside rodata"); park(); } write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: RODATA-EXEC"); // SAFETY: this dedicated image intentionally branches to an XN rodata page. unsafe { asm!("br {target}", target = in(reg) target, options(noreturn)) } } #[cfg(feature = "mmu-stack-exec-test")] { let target: u64; // SAFETY: reading SP into a register does not modify guest state. unsafe { asm!("mov {target}, sp", target = out(reg) target, options(nomem, nostack, preserves_flags)) }; if target < hardware.ram_base || target >= hardware.ram_base + hardware.ram_size { write_line("TURKOS/MMU-ERROR: stack probe outside RAM"); park(); } write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: STACK-EXEC"); // SAFETY: this dedicated image intentionally branches to an XN stack page. unsafe { asm!("br {target}", target = in(reg) target, options(noreturn)) } } #[cfg(feature = "mmu-mmio-exec-test")] { let target = hardware.uart_base; write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: UART-EXEC"); // SAFETY: this dedicated image intentionally attempts instruction fetch // from the mapped, privileged-XN UART Device page. unsafe { asm!("br {target}", target = in(reg) target, options(noreturn)) } } #[cfg(feature = "mmu-test")] { // 0x2000_0000 is outside the one mapped PL011 page and installed RAM. // SAFETY: the read intentionally triggers an EL1 translation fault. unsafe { read_volatile(0x2000_0000 as *const u64) }; write_line("TURKOS/MMU-ERROR: unmapped read returned"); park() } #[cfg(feature = "mmu-panic-test")] { write_line("TURKOS/MMU-PROBE: PANIC"); panic!("intentional post-MMU panic"); } #[cfg(feature = "mmu-brk-test")] { write_line("TURKOS/MMU-PROBE: BRK"); // SAFETY: this dedicated image intentionally enters the fatal EL1 // synchronous vector after identity translation is enabled. unsafe { asm!( "brk #0x42", in("x0") 0x544b_4252_4b5f_5830usize, in("x3") 0x544b_4252_4b5f_5833usize, in("x30") 0x544b_4252_4b5f_4c52usize, options(nomem, nostack, noreturn) ) } } #[cfg(feature = "mmu-udf-test")] { write_line("TURKOS/MMU-PROBE: UDF"); // SAFETY: this dedicated image intentionally enters the fatal EL1 // synchronous vector with an undefined instruction after MMU enable. unsafe { asm!( "udf #0", in("x0") 0x544b_5544_465f_5830usize, in("x3") 0x544b_5544_465f_5833usize, in("x30") 0x544b_5544_465f_4c52usize, options(nomem, nostack, noreturn) ) } } #[cfg(feature = "mmu-guard-test")] { let target = (layout.image_end + memory::PAGE_SIZE - 1) & !(memory::PAGE_SIZE - 1); write_str("TURKOS/MMU-IMAGE-END: 0x"); write_hex(layout.image_end as usize); write_str("\r\n"); write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: IMAGE-GUARD"); // SAFETY: the first page beyond the image is intentionally unmapped. unsafe { read_volatile(target as *const u64) }; write_line("TURKOS/MMU-ERROR: guard read returned"); park() } #[cfg(feature = "irq-return-test")] { irq_return_smoke_test(); park() } #[cfg(feature = "el0-fault-test")] { let target = layout.text_start as usize; write_str("TURKOS/EL0-FAULT-TARGET: 0x"); write_hex(target); write_str("\r\n"); write_line("TURKOS/EL0-ENTRY: ATTEMPT"); // This image deliberately branches to EL1-only text from EL0. No user // instruction runs and SP_EL0 is never dereferenced; it is not a user // task or a usable EL0 stack. DAIF stays masked during the probe. unsafe { asm!( "msr SP_EL0, {stack}", "msr ELR_EL1, {target}", "msr SPSR_EL1, {spsr}", "isb", "eret", stack = in(reg) 0usize, target = in(reg) target, spsr = in(reg) 0x3c0usize, options(noreturn) ); } } #[cfg(feature = "nested-fault-test")] { write_line("TURKOS/MMU-PROBE: NESTED-FATAL"); unsafe { asm!("brk #0x158") }; write_line("TURKOS/EXCEPTION-FIRST: RETURNED"); park(); } #[cfg(feature = "irq-sgi-test")] { irq_sgi_smoke_test(); park() } #[cfg(all(feature = "irq-timer-test", not(feature = "process-exit-irq-test")))] { #[cfg(feature = "irq-service-test")] irq_service::run(&allocator, boot.interrupts.gic_redistributor); #[cfg(not(feature = "irq-service-test"))] { irq_timer_smoke_test(boot.interrupts.gic_redistributor); park() } } #[cfg(feature = "irq-spurious-test")] irq_spurious_smoke_test(); #[cfg(feature = "irq-spurious-vector-test")] spurious_vector::run(boot.interrupts.gic_redistributor); #[cfg(all( feature = "irq-unknown-test", not(feature = "irq-spurious-test"), not(feature = "irq-spurious-vector-test") ))] irq_unknown_smoke_test(boot.interrupts.gic_redistributor); #[cfg(not(any( feature = "mmu-test", feature = "user-three-test", feature = "mmu-no-map-test", feature = "mmu-write-test", feature = "mmu-nx-test", feature = "mmu-rodata-test", feature = "mmu-rodata-exec-test", feature = "mmu-stack-exec-test", feature = "mmu-mmio-exec-test", feature = "mmu-panic-test", feature = "mmu-brk-test", feature = "mmu-udf-test", feature = "mmu-guard-test", feature = "mmu-tlb-test", feature = "el0-fault-test", feature = "nested-fault-test", feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "heap-test", feature = "task-state-test", feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-guard-test", feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test", feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "elf-ident-test", feature = "elf-layout-test", feature = "elf-load-test", feature = "elf-stack-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test", feature = "serial-rx-test", feature = "serial-line-test", feature = "serial-edit-test", feature = "serial-limit-test", feature = "serial-console-test", feature = "serial-echo-test", feature = "turkfs-ls-test", feature = "turkfs-cat-test", feature = "ipc-basic-test", feature = "ipc-wait-test", feature = "task-cancel-test", feature = "ipc-multi-test", feature = "ipc-timeline-test", feature = "virtio-discover-test", feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-tx-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "task-preempt-test", feature = "task-sleep-test" )))] { // Measure the live mask, not only the early boot's informational label. // Acceptance probes may unmask IRQ in their own feature-gated images; // the ordinary Image must arrive here with IRQ still masked. let daif: u64; unsafe { asm!("mrs {value}, DAIF", value = out(reg) daif, options(nomem, nostack)) }; write_str("TURKOS/IRQ-DEFAULT-DAIF: 0x"); write_hex(daif as usize); write_str("\r\n"); write_line("TURKOS/MMU-NORMAL: PASS"); park() } } fn read_boot_data(address: usize) -> Result { if address == 0 || address & 7 != 0 { return Err(dtb::Error::Invalid); } // SAFETY: the trusted QEMU boot contract passes an identity-mapped, readable // DTB pointer in x0. Check its 40-byte header before bounding the full view. let header = unsafe { core::slice::from_raw_parts(address as *const u8, 40) }; let size = dtb::total_size(header)?; // SAFETY: QEMU supplies the entire blob at this address; size is capped at 2 MiB. let blob = unsafe { core::slice::from_raw_parts(address as *const u8, size) }; Ok(BootData { hardware: dtb::discover(blob)?, interrupts: dtb::discover_interrupts(blob)?, reservations: dtb::reservations(blob)?, static_reserved: dtb::static_reserved(blob)?, #[cfg(any( feature = "virtio-discover-test", feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-tx-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "user-mmio-read-test" ))] virtio_mmio: dtb::discover_virtio_mmio(blob)?, #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] dtb_address: address, dtb_size: size, }) } fn read_uefi_boot_info( address: usize, boot: &BootData, dtb_address: usize, ) -> Result, ()> { // QEMU direct -kernel keeps x1=0. Only the UEFI loader supplies BootInfo. if address == 0 { return Ok(None); } bootinfo::validate_uefi_boot_info_pointer(address as u64).map_err(|_| ())?; let ram_start = boot.hardware.ram_base; let ram_end = ram_start.checked_add(boot.hardware.ram_size).ok_or(())?; let header_end = (address as u64) .checked_add(core::mem::size_of::() as u64) .ok_or(())?; if (address as u64) < ram_start || header_end > ram_end { return Err(()); } // SAFETY: the DTB-proven RAM extent contains the entire V1 header. This // remains a trusted UEFI/QEMU boot contract, not arbitrary physical DMA. let info = unsafe { core::ptr::read_unaligned(address as *const BootInfoV1) }; let kernel_start = core::ptr::addr_of!(_image_start) as u64; let kernel_bytes = (core::ptr::addr_of!(_image_end) as u64) .checked_sub(kernel_start) .ok_or(())?; info.validate_handoff( address as u64, ram_start, boot.hardware.ram_size, kernel_start, kernel_bytes, dtb_address as u64, boot.dtb_size as u64, ) .map_err(|_| ())?; // SAFETY: validate_handoff bounded the complete transferred map to a // fixed, page-aligned 64 KiB RAM allocation disjoint from Image and DTB. let map = unsafe { core::slice::from_raw_parts( info.memory_map_ptr as *const u8, info.memory_map_bytes as usize, ) }; let summary = bootinfo::inspect_uefi_memory_map(&info, address as u64, map).map_err(|_| ())?; write_str("TURKOS/BOOTINFO: phys=0x"); write_hex(address); write_str(" map=0x"); write_hex(info.memory_map_ptr as usize); write_str(" bytes=0x"); write_hex(info.memory_map_bytes as usize); write_str(" descriptors=0x"); write_hex(summary.descriptors); write_str(" loader-data=0x"); write_hex(summary.loader_data_descriptors); write_str(" PASS\r\n"); for raw in map.chunks_exact(info.descriptor_bytes as usize) { let descriptor = bootinfo::parse_memory_descriptor(raw).map_err(|_| ())?; write_str("TURKOS/EFI-DESCRIPTOR: TYPE=0x"); write_hex(descriptor.kind as usize); write_str(" START=0x"); write_hex(descriptor.start as usize); write_str(" END=0x"); write_hex(descriptor.end as usize); write_str("\r\n"); } Ok(Some((address as u64, info))) } #[cfg(feature = "virtio-discover-test")] fn virtio_discovery_smoke_test(boot: &BootData) -> ! { let nodes = boot.virtio_mmio.as_slice(); if nodes.len() != 32 { write_line("TURKOS/VIRTIO-DISCOVER-ERROR: slot count"); park(); } let mut min = u64::MAX; let mut max = 0u64; for node in nodes { if node.size != 0x200 { write_line("TURKOS/VIRTIO-DISCOVER-ERROR: slot size"); park(); } min = min.min(node.base); max = max.max(node.base); } if min != 0x0a00_0000 || max != 0x0a00_3e00 { write_line("TURKOS/VIRTIO-DISCOVER-ERROR: slot span"); park(); } write_str("TURKOS/VIRTIO-SLOTS: COUNT=0x"); write_hex(nodes.len()); write_str(" MIN=0x"); write_hex(min as usize); write_str(" MAX=0x"); write_hex(max as usize); write_str(" SIZE=0x"); write_hex(nodes[0].size as usize); write_str("\r\n"); write_line("TURKOS/VIRTIO-DISCOVER: PASS"); let window = dtb::virtio_page_window( &boot.virtio_mmio, boot.hardware.ram_base, boot.hardware.ram_size, ) .unwrap_or_else(|_| { write_line("TURKOS/VIRTIO-RANGE-ERROR: page window"); park() }); write_str("TURKOS/VIRTIO-PAGE-WINDOW: START=0x"); write_hex(window.start as usize); write_str(" END=0x"); write_hex(window.end as usize); write_str(" PAGES=0x"); write_hex(window.pages); write_str("\r\n"); write_line("TURKOS/VIRTIO-RANGE: PASS"); park() } fn plan_free_memory( boot: &BootData, dtb_address: usize, uefi_info: Option<&(u64, BootInfoV1)>, ) -> Result { let mut free = if let Some((_, info)) = uefi_info { // read_uefi_boot_info proved the map's complete span is inside RAM, // validated every descriptor and checked both owned allocations. let raw = unsafe { core::slice::from_raw_parts( info.memory_map_ptr as *const u8, info.memory_map_bytes as usize, ) }; memory::FreeMap::from_uefi_conventional( boot.hardware.ram_base, boot.hardware.ram_size, raw, info.descriptor_bytes as usize, )? } else { memory::FreeMap::from_ram(boot.hardware.ram_base, boot.hardware.ram_size)? }; let kernel_start = core::ptr::addr_of!(_image_start) as u64; let kernel_end = core::ptr::addr_of!(_image_end) as u64; let kernel_size = kernel_end .checked_sub(kernel_start) .ok_or(memory::Error::Invalid)?; free.reserve(kernel_start, kernel_size)?; // The first whole page beyond the image is never given to the allocator. // The MMU leaves the same page unmapped to expose image-end overruns. let guard_start = kernel_end .checked_add(memory::PAGE_SIZE - 1) .ok_or(memory::Error::Invalid)? & !(memory::PAGE_SIZE - 1); free.reserve(guard_start, memory::PAGE_SIZE)?; free.reserve(dtb_address as u64, boot.dtb_size as u64)?; if let Some((address, info)) = uefi_info { free.reserve(*address, BOOT_INFO_PAGE_BYTES)?; free.reserve(info.memory_map_ptr, BOOT_INFO_MAP_CAPACITY)?; } for region in boot.reservations.as_slice() { free.reserve(region.start, region.size)?; } for region in boot.static_reserved.as_slice() { free.reserve(region.start, region.size)?; } #[cfg(feature = "heap-test")] free.reserve( mmu::HEAP_WINDOW_START, mmu::HEAP_WINDOW_PAGES as u64 * memory::PAGE_SIZE, )?; Ok(free) } #[cfg(feature = "uefi-ownership-test")] fn uefi_owned_endpoint_probe(free: memory::FreeMap, foreign: u64) { let first = free.regions()[0].start; let last = free.regions().last().unwrap().end - memory::PAGE_SIZE; let before = free.total_pages(); let mut allocator = memory::FrameAllocator::new(free); for (role, address) in [("FIRST", first), ("LAST", last)] { if allocator.allocate_at(address) != Ok(address) { write_line("TURKOS/OWNED-PROBE-ERROR: endpoint allocation"); park(); } // SAFETY: allocate_at admitted the entire frame under the validated // firmware/DTB FreeMap, disjoint from Image, DTB, payload and map. // This pre-MMU experiment owns it exclusively and restores both words. let start = address as *mut u64; let end = (address + memory::PAGE_SIZE - 8) as *mut u64; let canary = 0x4f57_4e45_4450_4147u64 ^ address; let good = unsafe { let old_start = read_volatile(start); let old_end = read_volatile(end); write_volatile(start, canary); write_volatile(end, !canary); let good = read_volatile(start) == canary && read_volatile(end) == !canary; write_volatile(start, old_start); write_volatile(end, old_end); good }; if !good { write_line("TURKOS/OWNED-PROBE-ERROR: canary"); park(); } write_str("TURKOS/OWNED-PAGE: ROLE="); write_str(role); write_str(" ADDR=0x"); write_hex(address as usize); write_str(" PASS\r\n"); } if allocator.allocate_at(foreign) != Err(memory::FrameRequestError::OutsideAuthority) || allocator.remaining_pages() + 2 != before || allocator.release_many(&[first, last]).is_err() || allocator.remaining_pages() != before || allocator.release(first).is_ok() { write_line("TURKOS/OWNED-PROBE-ERROR: foreign/accounting"); park(); } write_str("TURKOS/OWNED-FOREIGN: ADDR=0x"); write_hex(foreign as usize); write_str(" REJECTED\r\n"); write_line("TURKOS/OWNED-PROBE: ENDPOINTS+FOREIGN+RESTORE PASS"); } fn frame_smoke_test(free: memory::FreeMap) -> bool { #[cfg(feature = "frame-stress-test")] const TEST_PAGES: usize = 128; #[cfg(not(feature = "frame-stress-test"))] const TEST_PAGES: usize = 16; let initial_pages = free.total_pages(); let mut allocator = memory::FrameAllocator::new(free); let mut frames = [0; TEST_PAGES]; for index in 0..TEST_PAGES { let Some(address) = allocator.allocate() else { return false; }; if frames[..index].contains(&address) { return false; } frames[index] = address; // SAFETY: finalized free RAM excludes image, DTB and both reservation // sources. Direct QEMU Image boot uses an identity-mapped RAM view. unsafe { let first = address as *mut u64; let last = (address + memory::PAGE_SIZE - core::mem::size_of::() as u64) as *mut u64; let before_first = read_volatile(first); let before_last = read_volatile(last); let first_canary = 0x5455_524b_4f53_4652 ^ address; let last_canary = !first_canary; write_volatile(first, first_canary); write_volatile(last, last_canary); let passed = read_volatile(first) == first_canary && read_volatile(last) == last_canary; write_volatile(first, before_first); write_volatile(last, before_last); if !passed { write_str("TURKOS/FRAME-ERROR-ADDRESS: 0x"); write_hex(address as usize); write_str("\r\n"); return false; } } } let address = frames[0]; write_str("TURKOS/FRAME-ADDRESS: 0x"); write_hex(address as usize); write_str("\r\nTURKOS/FRAME-COUNT: 0x"); write_hex(TEST_PAGES); write_str("\r\nTURKOS/FRAME-EDGE: PASS\r\n"); for frame in frames.into_iter().rev() { if allocator.release(frame).is_err() { return false; } } if allocator.remaining_pages() != initial_pages { return false; } let before_duplicate = allocator.remaining_pages(); if allocator.release(address) != Err(memory::Error::Invalid) || allocator.remaining_pages() != before_duplicate { return false; } let mut reclaimed = [0; TEST_PAGES]; for index in 0..TEST_PAGES { let Some(frame) = allocator.allocate() else { return false; }; if frame != frames[index] || reclaimed[..index].contains(&frame) { return false; } reclaimed[index] = frame; } for frame in reclaimed.into_iter().rev() { if allocator.release(frame).is_err() { return false; } } if allocator.remaining_pages() != initial_pages { return false; } write_str("TURKOS/FRAME-DOUBLE-FREE: ADDRESS=0x"); write_hex(address as usize); write_str(" BEFORE=0x"); write_hex(before_duplicate as usize); write_str(" AFTER=0x"); write_hex(allocator.remaining_pages() as usize); write_str(" RECLAIMED=0x"); write_hex(TEST_PAGES); write_str(" RESULT=PASS\r\n"); write_str("TURKOS/FRAME-ACCOUNT: BEFORE=0x"); write_hex(initial_pages as usize); write_str(" AFTER=0x"); write_hex(allocator.remaining_pages() as usize); write_str("\r\n"); true } fn write_byte(byte: u8) { let base = UART_BASE.load(Ordering::Relaxed); // SAFETY: before discovery QEMU virt maps PL011 at the bootstrap address; // afterwards the DTB `reg` value identifies the same MMIO device. unsafe { while read_volatile((base + 0x18) as *const u32) & UART_TX_FULL != 0 {} write_volatile(base as *mut u32, byte as u32); } } fn write_str(value: &str) { for byte in value.bytes() { write_byte(byte); } } fn write_line(value: &str) { // Before SCTLR_EL1.M is set, this boot CPU has no competing tasks and // QEMU/HVF can reject exclusive accesses to the untranslated memory type. // The lock becomes usable only after RAM is mapped as Normal memory. let sctlr: usize; unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr, options(nomem, nostack, preserves_flags)) }; if sctlr & 1 == 0 { write_str(value); write_str("\r\n"); return; } // A one-CPU timer IRQ cannot enter another writer while this task owns // the line lock. Preserve the caller's previous I bit, including boot // paths that already run with IRQ masked. let daif: usize; unsafe { asm!( "mrs {saved}, daif", "msr daifset, #2", "isb", saved = out(reg) daif, options(nostack, preserves_flags) ) }; let guard = SERIAL_LINE_LOCK.lock(); write_str(value); write_str("\r\n"); drop(guard); if daif & (1 << 7) == 0 { unsafe { asm!("msr daifclr, #2", "isb", options(nostack, preserves_flags)) }; } } fn write_hex(value: usize) { for shift in (0..16).rev() { let nibble = ((value >> (shift * 4)) & 0xf) as u8; write_byte(if nibble < 10 { b'0' + nibble } else { b'a' + nibble - 10 }); } } fn write_hex_byte(value: u8) { for shift in [4, 0] { let nibble = (value >> shift) & 0xf; write_byte(if nibble < 10 { b'0' + nibble } else { b'a' + nibble - 10 }); } } fn park() -> ! { loop { // SAFETY: wfe only waits for an event and does not access memory. unsafe { asm!("wfe", options(nomem, nostack, preserves_flags)) } } } #[panic_handler] fn panic(_info: &PanicInfo) -> ! { write_line("TURKOS/PANIC"); park() } // K400 gate in this crate root too: `cargo rustc --bin turkos-kernel -- --cfg // 'feature="production-write"'` reaches only this binary crate, never lib.rs. // Kept last so default-build line numbers (panic locations) do not move. #[cfg(feature = "production-write")] compile_error!( "K400 production writes are blocked: journal/replay, orphan recovery, failure matrix and release evidence are not complete" );