//! K466–K468: shared frames, ready tokens, then complete UART lines on two EL1 CPUs. //! IRQs remain masked; these are not production SMP allocator/task dispatch paths. use super::*; #[cfg(feature = "smp-race-trace-test")] use core::sync::atomic::AtomicU8; use core::sync::atomic::{AtomicUsize, Ordering}; #[cfg(feature = "smp-irq-lock-test")] use turkos_kernel::serial_tx::IrqLockError; #[cfg(feature = "smp-serial-test")] use turkos_kernel::serial_tx::SmpSerialLineLock; #[cfg(feature = "smp-classified-admission-test")] use turkos_kernel::smp::ClassifiedCounterStats; #[cfg(feature = "smp-stats-test")] use turkos_kernel::smp::CounterStats; #[cfg(feature = "smp-order-test")] use turkos_kernel::smp::SmpMessageLane; #[cfg(feature = "smp-fair-test")] use turkos_kernel::smp::SmpTicketLock; #[cfg(feature = "smp-wait-class-test")] use turkos_kernel::smp::WaitClassStats; #[cfg(feature = "smp-time-test")] use turkos_kernel::smp::positive_counter_delta; #[cfg(feature = "smp-race-trace-test")] use turkos_kernel::smp::summarize_two_cpu_trace; #[cfg(feature = "smp-ready-test")] use turkos_kernel::task::SmpReadyQueue; const PER_CPU: usize = 64; const CANARY: u64 = 0x4b34_3636_414c_4c4f; static SHARED: memory::LockedFrameAllocator = memory::LockedFrameAllocator::new(); static START: AtomicUsize = AtomicUsize::new(0); static SECONDARY_STATE: AtomicUsize = AtomicUsize::new(0); static SECONDARY_MPIDR: AtomicUsize = AtomicUsize::new(0); static SECONDARY_SCTLR: AtomicUsize = AtomicUsize::new(0); static SECONDARY_TTBR: AtomicUsize = AtomicUsize::new(0); static mut PRIMARY_FRAMES: [u64; PER_CPU] = [0; PER_CPU]; static mut SECONDARY_FRAMES: [u64; PER_CPU] = [0; PER_CPU]; #[cfg(feature = "smp-ready-test")] static READY: SmpReadyQueue = SmpReadyQueue::new(); #[cfg(feature = "smp-ready-test")] static READY_GO: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-ready-test")] static READY_DONE: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-ready-test")] static READY_CPU0: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-ready-test")] static READY_CPU1: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-ready-test")] static READY_ACTIVE: [AtomicUsize; 4] = [ AtomicUsize::new(0), AtomicUsize::new(0), AtomicUsize::new(0), AtomicUsize::new(0), ]; #[cfg(feature = "smp-serial-test")] static SERIAL_LOCK: SmpSerialLineLock = SmpSerialLineLock::new(); #[cfg(feature = "smp-serial-test")] static SERIAL_GO: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-serial-test")] static SERIAL_DONE: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-owner-test")] static OWNER_GO: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-owner-test")] static OWNER_DONE: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-owner-test")] static OWNER_MPIDR: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-owner-test")] static OWNER_HELD: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-irq-lock-test")] static IRQ_DONE: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-irq-lock-test")] static IRQ_IAR: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-irq-lock-test")] static IRQ_OWNER: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-irq-lock-test")] static IRQ_SPSR: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-irq-lock-test")] static IRQ_MPIDR: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-order-test")] static ORDER_GO: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-order-test")] static ORDER_DONE: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-order-test")] static ORDER_ACK: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-order-test")] static ORDER_P2S: SmpMessageLane = SmpMessageLane::new(); #[cfg(feature = "smp-order-test")] static ORDER_S2P: SmpMessageLane = SmpMessageLane::new(); #[cfg(feature = "smp-order-test")] static ORDER_SUM_S: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); #[cfg(feature = "smp-fair-test")] static FAIR_LOCK: SmpTicketLock = SmpTicketLock::new(); #[cfg(feature = "smp-fair-test")] static FAIR_GO: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-fair-test")] static FAIR_DONE: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-fair-test")] static FAIR_TOTAL: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-fair-test")] static FAIR_CPU0: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-fair-test")] static FAIR_CPU1: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-wait-test")] static FAIR_CPU1_FIRST_WAIT: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-time-test")] static TIME_FREQUENCY: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-time-test")] static TIME_START: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-time-test")] static TIME_END: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-stats-test")] static STATS1_N: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-stats-test")] static STATS1_MIN: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-stats-test")] static STATS1_MAX: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-stats-test")] static STATS1_SUM: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-wait-class-test")] static CLASS1_ZERO: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-wait-class-test")] static CLASS1_CONTENDED: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-wait-class-test")] static CLASS1_TOTAL: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-wait-class-test")] static CLASS1_MAX: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-hold-test")] static HOLD1_N: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-hold-test")] static HOLD1_MIN: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-hold-test")] static HOLD1_MAX: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-hold-test")] static HOLD1_SUM: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-hold-test")] static HOLD0_FIRST: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-counter-baseline-test")] static BASE1_N: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-counter-baseline-test")] static BASE1_MIN: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-counter-baseline-test")] static BASE1_MAX: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-counter-baseline-test")] static BASE1_SUM: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-classified-admission-test")] static PART1_Z_N: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-classified-admission-test")] static PART1_Z_SUM: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-classified-admission-test")] static PART1_Z_MAX: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-classified-admission-test")] static PART1_C_N: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-classified-admission-test")] static PART1_C_SUM: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-classified-admission-test")] static PART1_C_MAX: AtomicUsize = AtomicUsize::new(0); #[cfg(feature = "smp-race-trace-test")] static FAIR_TRACE: [AtomicU8; 256] = [const { AtomicU8::new(0xff) }; 256]; #[cfg(feature = "smp-race-trace-test")] fn trace_ticket_owner(ticket: usize, owner: u8) -> bool { ticket < FAIR_TRACE.len() && owner <= 1 && FAIR_TRACE[ticket] .compare_exchange(0xff, owner, Ordering::Relaxed, Ordering::Relaxed) .is_ok() } #[cfg(feature = "smp-time-test")] fn physical_counter() -> usize { let value: usize; // SAFETY: the isolated EL1 QEMU virt experiment reads the architectural physical counter. unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) value, options(nomem, nostack, preserves_flags)); } value } #[cfg(feature = "smp-time-test")] fn counter_frequency() -> usize { let value: usize; // SAFETY: CNTFRQ_EL0 is the architectural counter frequency visible to EL1. unsafe { asm!("mrs {value}, CNTFRQ_EL0", value = out(reg) value, options(nomem, nostack, preserves_flags)); } value } unsafe extern "C" { fn __smp_allocator_secondary_entry(); static mut __smp_allocator_root: u64; static mut __smp_allocator_tcr: u64; static mut __smp_allocator_mair: u64; static __smp_allocator_fault_esr: u64; static __smp_allocator_fault_elr: u64; static __smp_allocator_fault_far: u64; static __smp_allocator_fault_mpidr: u64; static __smp_allocator_fault_seen: u64; } fn psci_hvc(fid: usize, target: usize, entry: usize, context: usize) -> usize { let result; // SAFETY: isolated QEMU virt experiment uses the DTB-confirmed HVC conduit. unsafe { asm!("hvc #0", inlateout("x0") fid => result, in("x1") target, in("x2") entry, in("x3") context, options(nostack)); } result } fn fail(reason: &str) -> ! { write_str("TURKOS/SMP-ALLOC-ERROR: "); write_line(reason); park() } fn wait_secondary(expected: usize) { for _ in 0..100_000_000 { if SECONDARY_STATE.load(Ordering::Acquire) == expected { return; } if unsafe { read_volatile(core::ptr::addr_of!(__smp_allocator_fault_seen)) } != 0 { write_str("TURKOS/SMP-ALLOC-FAULT: ESR=0x"); write_hex( unsafe { read_volatile(core::ptr::addr_of!(__smp_allocator_fault_esr)) } as usize, ); write_str(" ELR=0x"); write_hex( unsafe { read_volatile(core::ptr::addr_of!(__smp_allocator_fault_elr)) } as usize, ); write_str(" FAR=0x"); write_hex( unsafe { read_volatile(core::ptr::addr_of!(__smp_allocator_fault_far)) } as usize, ); write_str(" MPIDR=0x"); write_hex( unsafe { read_volatile(core::ptr::addr_of!(__smp_allocator_fault_mpidr)) } as usize, ); write_str("\r\n"); fail("secondary exception"); } if SECONDARY_STATE.load(Ordering::Acquire) == 3 { fail("secondary allocation"); } core::hint::spin_loop(); } fail("secondary timeout"); } #[unsafe(no_mangle)] pub extern "C" fn __smp_allocator_secondary_rust() -> ! { let mpidr: u64; let sctlr: u64; let ttbr: u64; // SAFETY: called on the isolated secondary after assembly enabled EL1 MMU. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, TTBR0_EL1", value = out(reg) ttbr, options(nomem, nostack, preserves_flags)); } SECONDARY_MPIDR.store(mpidr as usize, Ordering::Relaxed); SECONDARY_SCTLR.store(sctlr as usize, Ordering::Relaxed); SECONDARY_TTBR.store(ttbr as usize, Ordering::Relaxed); SECONDARY_STATE.store(1, Ordering::Release); while START.load(Ordering::Acquire) == 0 { core::hint::spin_loop(); } for index in 0..PER_CPU { let Some(Some(address)) = SHARED.with(|allocator| { for _ in 0..4096 { core::hint::spin_loop(); } allocator.allocate() }) else { SECONDARY_STATE.store(3, Ordering::Release); loop { unsafe { asm!("wfe", options(nomem, nostack)) }; } }; // SAFETY: successful allocation transfers this mapped, unique frame to CPU1. unsafe { core::ptr::addr_of_mut!(SECONDARY_FRAMES) .cast::() .add(index) .write(address); write_volatile(address as *mut u64, CANARY ^ address ^ 1); } } SECONDARY_STATE.store(2, Ordering::Release); #[cfg(feature = "smp-ready-test")] { while READY_GO.load(Ordering::Acquire) == 0 { core::hint::spin_loop(); } READY_DONE.store(if run_ready_work(1) { 1 } else { 2 }, Ordering::Release); } #[cfg(feature = "smp-serial-test")] { while SERIAL_GO.load(Ordering::Acquire) == 0 { core::hint::spin_loop(); } run_serial_lines(1); SERIAL_DONE.store(1, Ordering::Release); } #[cfg(feature = "smp-owner-test")] { while OWNER_GO.load(Ordering::Acquire) == 0 { core::hint::spin_loop(); } let mpidr: u64; // SAFETY: this function runs on the PSCI-started secondary EL1 CPU. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)); } OWNER_MPIDR.store(mpidr as usize, Ordering::Relaxed); let rejected = !SERIAL_LOCK.try_unlock((mpidr & 0xff) as usize); OWNER_HELD.store(SERIAL_LOCK.owner(), Ordering::Relaxed); OWNER_DONE.store(rejected as usize + 1, Ordering::Release); } #[cfg(feature = "smp-order-test")] { while ORDER_GO.load(Ordering::Acquire) == 0 { core::hint::spin_loop(); } ORDER_DONE.store(if order_secondary() { 1 } else { 2 }, Ordering::Release); } #[cfg(feature = "smp-fair-test")] { while FAIR_GO.load(Ordering::Acquire) == 0 { core::hint::spin_loop(); } #[cfg(feature = "smp-stats-test")] let mut cpu1_stats = CounterStats::new(); #[cfg(feature = "smp-wait-class-test")] let mut cpu1_classes = WaitClassStats::new(); #[cfg(feature = "smp-hold-test")] let mut cpu1_hold = CounterStats::new(); #[cfg(feature = "smp-classified-admission-test")] let mut cpu1_partition = ClassifiedCounterStats::new(); for _index in 0..128 { #[cfg(feature = "smp-time-test")] let counter_before = if cfg!(feature = "smp-stats-test") || _index == 0 { physical_counter() } else { 0 }; let guard = FAIR_LOCK.lock(); #[cfg(feature = "smp-time-test")] let counter_after = physical_counter(); #[cfg(feature = "smp-hold-test")] let hold_start = physical_counter(); #[cfg(feature = "smp-wait-class-test")] if cpu1_classes.record(guard.wait_spins()).is_err() { FAIR_DONE.store(2, Ordering::Release); break; } #[cfg(feature = "smp-time-test")] if _index == 0 { TIME_FREQUENCY.store(counter_frequency(), Ordering::Relaxed); TIME_START.store(counter_before, Ordering::Relaxed); TIME_END.store(counter_after, Ordering::Relaxed); } #[cfg(feature = "smp-stats-test")] if cpu1_stats .record(counter_before as u64, counter_after as u64) .is_err() { FAIR_DONE.store(2, Ordering::Release); break; } #[cfg(feature = "smp-classified-admission-test")] if cpu1_partition .record( counter_before as u64, counter_after as u64, guard.wait_spins(), ) .is_err() { FAIR_DONE.store(2, Ordering::Release); break; } let ticket = guard.ticket(); #[cfg(feature = "smp-wait-test")] if _index == 0 { FAIR_CPU1_FIRST_WAIT.store(guard.wait_spins(), Ordering::Relaxed); } if FAIR_TOTAL.load(Ordering::Relaxed) != ticket { FAIR_DONE.store(2, Ordering::Release); break; } #[cfg(feature = "smp-race-trace-test")] if !trace_ticket_owner(ticket, 1) { FAIR_DONE.store(2, Ordering::Release); break; } FAIR_TOTAL.store(ticket + 1, Ordering::Relaxed); FAIR_CPU1.fetch_add(1, Ordering::Relaxed); #[cfg(feature = "smp-hold-test")] if cpu1_hold .record(hold_start as u64, physical_counter() as u64) .is_err() { FAIR_DONE.store(2, Ordering::Release); break; } } #[cfg(feature = "smp-stats-test")] { STATS1_N.store(cpu1_stats.samples, Ordering::Relaxed); STATS1_MIN.store(cpu1_stats.minimum as usize, Ordering::Relaxed); STATS1_MAX.store(cpu1_stats.maximum as usize, Ordering::Relaxed); STATS1_SUM.store(cpu1_stats.total as usize, Ordering::Relaxed); } #[cfg(feature = "smp-wait-class-test")] { CLASS1_ZERO.store(cpu1_classes.zero, Ordering::Relaxed); CLASS1_CONTENDED.store(cpu1_classes.contended, Ordering::Relaxed); CLASS1_TOTAL.store(cpu1_classes.total_spins, Ordering::Relaxed); CLASS1_MAX.store(cpu1_classes.maximum_spins, Ordering::Relaxed); } #[cfg(feature = "smp-hold-test")] { HOLD1_N.store(cpu1_hold.samples, Ordering::Relaxed); HOLD1_MIN.store(cpu1_hold.minimum as usize, Ordering::Relaxed); HOLD1_MAX.store(cpu1_hold.maximum as usize, Ordering::Relaxed); HOLD1_SUM.store(cpu1_hold.total as usize, Ordering::Relaxed); } #[cfg(feature = "smp-counter-baseline-test")] { let mut baseline = CounterStats::new(); for _ in 0..128 { let first = physical_counter(); let second = physical_counter(); if baseline.record(first as u64, second as u64).is_err() { FAIR_DONE.store(2, Ordering::Release); break; } } BASE1_N.store(baseline.samples, Ordering::Relaxed); BASE1_MIN.store(baseline.minimum as usize, Ordering::Relaxed); BASE1_MAX.store(baseline.maximum as usize, Ordering::Relaxed); BASE1_SUM.store(baseline.total as usize, Ordering::Relaxed); } #[cfg(feature = "smp-classified-admission-test")] { PART1_Z_N.store(cpu1_partition.zero_spin.samples, Ordering::Relaxed); PART1_Z_SUM.store(cpu1_partition.zero_spin.total as usize, Ordering::Relaxed); PART1_Z_MAX.store(cpu1_partition.zero_spin.maximum as usize, Ordering::Relaxed); PART1_C_N.store(cpu1_partition.contended.samples, Ordering::Relaxed); PART1_C_SUM.store(cpu1_partition.contended.total as usize, Ordering::Relaxed); PART1_C_MAX.store(cpu1_partition.contended.maximum as usize, Ordering::Relaxed); } if FAIR_DONE.load(Ordering::Relaxed) != 2 { FAIR_DONE.store(1, Ordering::Release); } } loop { unsafe { asm!("wfe", options(nomem, nostack)) }; } } pub fn run(free: memory::FreeMap, boot: &BootData, level: usize) -> ! { if level != 1 { fail("EL1 required"); } let mut allocator = memory::FrameAllocator::new(free.clone()); let layout = mmu::Layout { ram_base: boot.hardware.ram_base, ram_size: boot.hardware.ram_size, uart_base: boot.hardware.uart_base, image_start: core::ptr::addr_of!(_image_start) as u64, text_start: core::ptr::addr_of!(_text_start) as u64, text_end: core::ptr::addr_of!(_text_end) as u64, rodata_end: core::ptr::addr_of!(_rodata_end) as u64, image_end: core::ptr::addr_of!(_image_end) as u64, irq_guard_page: Some(core::ptr::addr_of!(__irq_guard_page_start) as u64), heap_start: None, }; #[cfg(feature = "smp-irq-lock-test")] if !prepare_test_interrupt(boot, 1) { fail("SGI1 GIC setup before MMU"); } // SAFETY: all table frames come from the finalized free map and remain owned. let tables = unsafe { mmu::build_identity_tables(&mut allocator, layout, boot.static_reserved.as_slice()) } .unwrap_or_else(|_| fail("identity table build")); if !tables.normal_shareable_identity(core::ptr::addr_of!(SHARED) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(START) as u64) { fail("shared atomic state is not Normal Inner-Shareable writable RAM"); } let initial = allocator.remaining_pages(); if initial < (2 * PER_CPU) as u64 { fail("insufficient pages"); } let before: u64; let mmfr0: u64; unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) before, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ID_AA64MMFR0_EL1", value = out(reg) mmfr0, options(nomem, nostack, preserves_flags)); } if before & 1 != 0 || (mmfr0 & 0xf) == 7 { fail("boot MMU state"); } let tcr = mmu::tcr_el1((mmfr0 & 0xf).min(5)).unwrap_or_else(|| fail("TCR width")); // SAFETY: no secondary is online yet, so exclusive initialization is valid. unsafe { SHARED.initialize_single_cpu(allocator); } unsafe { write_volatile(core::ptr::addr_of_mut!(__smp_allocator_root), tables.root); write_volatile(core::ptr::addr_of_mut!(__smp_allocator_tcr), tcr); write_volatile(core::ptr::addr_of_mut!(__smp_allocator_mair), mmu::MAIR_EL1); asm!("dsb sy", "msr MAIR_EL1, {mair}", "msr TCR_EL1, {tcr}", "msr TTBR0_EL1, {root}", "isb", "tlbi vmalle1", "dsb sy", "isb", "msr SCTLR_EL1, {sctlr}", "isb", mair = in(reg) mmu::MAIR_EL1, tcr = in(reg) tcr, root = in(reg) tables.root, sctlr = in(reg) (before | 0x1005), options(nostack)); } let primary_sctlr: u64; let primary_ttbr: u64; let primary_mpidr: u64; unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) primary_sctlr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, TTBR0_EL1", value = out(reg) primary_ttbr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, MPIDR_EL1", value = out(reg) primary_mpidr, options(nomem, nostack, preserves_flags)); } if primary_sctlr & 0x1005 != 0x1005 || primary_ttbr != tables.root || primary_mpidr & 0xff != 0 { fail("primary MMU contract"); } write_str("TURKOS/SMP-ALLOC-MMU: ROOT=0x"); write_hex(tables.root as usize); write_str(" SCTLR0=0x"); write_hex(primary_sctlr as usize); write_str(" MAIR=0x"); write_hex(mmu::MAIR_EL1 as usize); write_str(" TABLES=0x"); write_hex(tables.frame_count()); write_str("\r\n"); let entry = __smp_allocator_secondary_entry as *const () as usize; if entry < layout.image_start as usize || entry >= layout.image_end as usize || entry & 3 != 0 { fail("secondary entry bounds"); } let version = psci_hvc(0x8400_0000, 0, 0, 0); let on = psci_hvc(0xc400_0003, 1, entry, 0x466); write_str("TURKOS/SMP-ALLOC-PSCI: VERSION=0x"); write_hex(version); write_str(" ON=0x"); write_hex(on); write_str(" ENTRY=0x"); write_hex(entry); write_str("\r\n"); if on != 0 { fail("PSCI CPU_ON"); } wait_secondary(1); let sctlr1 = SECONDARY_SCTLR.load(Ordering::Acquire); let ttbr1 = SECONDARY_TTBR.load(Ordering::Acquire); let mpidr1 = SECONDARY_MPIDR.load(Ordering::Acquire); if sctlr1 & 0x1005 != 0x1005 || ttbr1 as u64 != tables.root || mpidr1 & 0xff != 1 { fail("secondary MMU or CPU identity"); } write_str("TURKOS/SMP-ALLOC-CPU1: MPIDR=0x"); write_hex(mpidr1); write_str(" SCTLR=0x"); write_hex(sctlr1); write_str(" TTBR=0x"); write_hex(ttbr1); write_str("\r\n"); START.store(1, Ordering::Release); for index in 0..PER_CPU { let Some(Some(address)) = SHARED.with(|owner| { for _ in 0..4096 { core::hint::spin_loop(); } owner.allocate() }) else { fail("primary allocation"); }; unsafe { core::ptr::addr_of_mut!(PRIMARY_FRAMES) .cast::() .add(index) .write(address); write_volatile(address as *mut u64, CANARY ^ address); } } wait_secondary(2); let remaining = SHARED .with(|owner| owner.remaining_pages()) .unwrap_or_else(|| fail("owner not ready")); if remaining != initial - (2 * PER_CPU) as u64 { fail("allocation accounting"); } let mut all = [0u64; 2 * PER_CPU]; for index in 0..PER_CPU { all[index] = unsafe { core::ptr::addr_of!(PRIMARY_FRAMES) .cast::() .add(index) .read() }; all[PER_CPU + index] = unsafe { core::ptr::addr_of!(SECONDARY_FRAMES) .cast::() .add(index) .read() }; } for (index, &address) in all.iter().enumerate() { if address & (memory::PAGE_SIZE - 1) != 0 || !free .regions() .iter() .any(|r| r.start <= address && address + memory::PAGE_SIZE <= r.end) || tables.owns_table_frame(address) || !tables.normal_shareable_identity(address) || all[..index].contains(&address) || unsafe { read_volatile(address as *const u64) } != CANARY ^ address ^ (index >= PER_CPU) as u64 { fail("frame ownership, uniqueness or canary"); } write_str("TURKOS/SMP-ALLOC-FRAME: CPU=0x"); write_hex((index >= PER_CPU) as usize); write_str(" INDEX=0x"); write_hex(index % PER_CPU); write_str(" PA=0x"); write_hex(address as usize); write_str("\r\n"); } let contentions = SHARED.contention_count(); if contentions == 0 { fail("no overlapping lock attempts"); } write_line( "TURKOS/SMP-ALLOC-ATTR: 128 frames and shared atomics are Normal Inner-Shareable writable identity RAM", ); let released = SHARED .with(|owner| owner.release_many(&all)) .unwrap_or_else(|| fail("release owner")); if released.is_err() || SHARED.with(|owner| owner.remaining_pages()) != Some(initial) { fail("release accounting"); } write_str("TURKOS/SMP-ALLOC-ACCOUNT: BEFORE=0x"); write_hex(initial as usize); write_str(" ALLOCATED=0x"); write_hex(2 * PER_CPU); write_str(" MID=0x"); write_hex(remaining as usize); write_str(" AFTER=0x"); write_hex(initial as usize); write_str(" CONTENTIONS=0x"); write_hex(contentions as usize); write_str("\r\n"); write_line("TURKOS/SMP-ALLOC: PASS"); #[cfg(feature = "smp-ready-test")] ready_primary(&tables); #[cfg(feature = "smp-serial-test")] serial_primary(&tables); #[cfg(feature = "smp-owner-test")] owner_primary(&tables); #[cfg(feature = "smp-irq-lock-test")] irq_primary(&tables); #[cfg(feature = "smp-order-test")] order_primary(&tables); #[cfg(feature = "smp-fair-test")] fair_primary(&tables); park() } #[cfg(feature = "smp-ready-test")] fn run_ready_work(cpu: usize) -> bool { for _ in 0..128 { let id = match READY.claim(cpu) { Ok(Some(id)) => id, _ => return false, }; let slot = id.slot as usize; if slot >= READY_ACTIVE.len() || READY_ACTIVE[slot] .compare_exchange(0, cpu + 1, Ordering::AcqRel, Ordering::Relaxed) .is_err() { return false; } for _ in 0..4096 { core::hint::spin_loop(); } if READY_ACTIVE[slot] .compare_exchange(cpu + 1, 0, Ordering::Release, Ordering::Relaxed) .is_err() || READY.finish(cpu, id).is_err() { return false; } if cpu == 0 { READY_CPU0.fetch_add(1, Ordering::Relaxed); } else { READY_CPU1.fetch_add(1, Ordering::Relaxed); } } true } #[cfg(feature = "smp-ready-test")] fn ready_primary(tables: &mmu::Tables) { if !tables.normal_shareable_identity(core::ptr::addr_of!(READY) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(READY_ACTIVE) as u64) { fail("ready queue is not Normal Inner-Shareable writable RAM"); } READY_GO.store(1, Ordering::Release); if !run_ready_work(0) { fail("primary ready claim/work/finish"); } let mut done = 0; for _ in 0..100_000_000 { done = READY_DONE.load(Ordering::Acquire); if done != 0 { break; } core::hint::spin_loop(); } if done != 1 { fail("secondary ready claim/work/finish or timeout"); } let snapshot = READY.snapshot(); if snapshot.ready_len != 4 || snapshot.running != [None, None] || snapshot.claims != 256 || snapshot.finishes != 256 || snapshot.contentions == 0 || snapshot.per_task.iter().any(|count| *count == 0) || snapshot.per_task.iter().sum::() != 256 || READY_CPU0.load(Ordering::Acquire) != 128 || READY_CPU1.load(Ordering::Acquire) != 128 || READY_ACTIVE .iter() .any(|flag| flag.load(Ordering::Acquire) != 0) { fail("ready ownership or accounting"); } for (slot, count) in snapshot.per_task.iter().enumerate() { write_str("TURKOS/SMP-READY-TASK: SLOT=0x"); write_hex(slot); write_str(" RUNS=0x"); write_hex(*count as usize); write_str("\r\n"); } write_str("TURKOS/SMP-READY-ACCOUNT: CLAIMS=0x"); write_hex(snapshot.claims as usize); write_str(" FINISHES=0x"); write_hex(snapshot.finishes as usize); write_str(" CPU0=0x"); write_hex(READY_CPU0.load(Ordering::Acquire)); write_str(" CPU1=0x"); write_hex(READY_CPU1.load(Ordering::Acquire)); write_str(" READY=0x"); write_hex(snapshot.ready_len); write_str(" BUSY=0x"); write_hex(snapshot.contentions as usize); write_str("\r\n"); write_line("TURKOS/SMP-READY: PASS"); } #[cfg(feature = "smp-serial-test")] fn run_serial_lines(cpu: usize) { for index in 0..32 { let _guard = SERIAL_LOCK.lock(cpu); write_str("TURKOS/SMP-SERIAL-LINE: CPU=0x"); write_hex(cpu); write_str(" INDEX=0x"); write_hex(index); write_str(" PAYLOAD="); for _ in 0..96 { write_byte(if cpu == 0 { b'A' } else { b'B' }); for _ in 0..128 { core::hint::spin_loop(); } } write_str("\r\n"); } } #[cfg(feature = "smp-serial-test")] fn serial_primary(tables: &mmu::Tables) { if !tables.normal_shareable_identity(core::ptr::addr_of!(SERIAL_LOCK) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(SERIAL_GO) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(SERIAL_DONE) as u64) { fail("serial lock is not Normal Inner-Shareable writable RAM"); } write_line("TURKOS/SMP-SERIAL-BEGIN"); SERIAL_GO.store(1, Ordering::Release); run_serial_lines(0); let mut done = 0; for _ in 0..100_000_000 { done = SERIAL_DONE.load(Ordering::Acquire); if done == 1 { break; } core::hint::spin_loop(); } if done != 1 || SERIAL_LOCK.owner() != 0 || SERIAL_LOCK.contention_count() == 0 { fail("serial line completion, owner or contention"); } write_str("TURKOS/SMP-SERIAL-ACCOUNT: CPU0=0x"); write_hex(32); write_str(" CPU1=0x"); write_hex(32); write_str(" LINES=0x"); write_hex(64); write_str(" BUSY=0x"); write_hex(SERIAL_LOCK.contention_count() as usize); write_str(" OWNER=0x"); write_hex(SERIAL_LOCK.owner()); write_str("\r\n"); write_line("TURKOS/SMP-SERIAL: PASS"); } #[cfg(feature = "smp-owner-test")] fn owner_primary(tables: &mmu::Tables) { if !tables.normal_shareable_identity(core::ptr::addr_of!(OWNER_GO) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(OWNER_DONE) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(OWNER_MPIDR) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(OWNER_HELD) as u64) { fail("owner handshake is not Normal Inner-Shareable writable RAM"); } let primary_mpidr: u64; // SAFETY: current EL1 CPU reads its own architectural identity. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) primary_mpidr, options(nomem, nostack, preserves_flags)); } if primary_mpidr & 0xff != 0 || SERIAL_LOCK.rejected_unlock_count() != 0 { fail("owner precondition"); } let guard = SERIAL_LOCK.lock((primary_mpidr & 0xff) as usize); OWNER_GO.store(1, Ordering::Release); let mut done = 0; for _ in 0..100_000_000 { done = OWNER_DONE.load(Ordering::Acquire); if done != 0 { break; } core::hint::spin_loop(); } if done != 2 || OWNER_MPIDR.load(Ordering::Acquire) & 0xff != 1 || OWNER_HELD.load(Ordering::Acquire) != 1 || SERIAL_LOCK.owner() != 1 || SERIAL_LOCK.rejected_unlock_count() != 1 { fail("wrong CPU unlock changed owner or was not rejected"); } drop(guard); if SERIAL_LOCK.owner() != 0 || SERIAL_LOCK.rejected_unlock_count() != 1 { fail("right CPU release"); } write_str("TURKOS/SMP-OWNER-ACCOUNT: CPU0=0x"); write_hex(primary_mpidr as usize); write_str(" CPU1=0x"); write_hex(OWNER_MPIDR.load(Ordering::Acquire)); write_str(" WRONG=0x"); write_hex(done - 1); write_str(" HELD=0x"); write_hex(OWNER_HELD.load(Ordering::Acquire)); write_str(" REJECTS=0x"); write_hex(SERIAL_LOCK.rejected_unlock_count() as usize); write_str(" RELEASED=0x"); write_hex(SERIAL_LOCK.owner()); write_str("\r\n"); write_line("TURKOS/SMP-OWNER: PASS"); } #[cfg(feature = "smp-irq-lock-test")] pub fn irq_lock_probe( frame: *const irq::IrqContext, stack_start: usize, stack_top: usize, current_sp: usize, ) -> usize { // SAFETY: return.S saved the full EL1h IRQ context on its dedicated stack. let context = unsafe { &*frame }; let mpidr: u64; let iar: usize; // SAFETY: the primary GIC CPU interface was armed for one self-target SGI1. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags)); } let valid = frame as usize == stack_top - size_of::() && irq::handler_sp_in_stack(stack_start, frame as usize, current_sp) && context.vector_slot == 5 && context.spsr & 0x8f == 5 && context.interrupted_sp & 15 == 0 && (context.interrupted_sp < stack_start as u64 || context.interrupted_sp >= stack_top as u64) && unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) } == IRQ_GUARD && mpidr & 0xff == 0 && iar & 0x00ff_ffff == 1 && SERIAL_LOCK.owner() == 1 && SERIAL_LOCK.try_lock_from_irq((mpidr & 0xff) as usize) == Err(IrqLockError::InterruptContext) && SERIAL_LOCK.owner() == 1; if !valid { IRQ_DONE.store(2, Ordering::Release); return 0; } // SAFETY: this handler owns the acknowledged SGI1; EOIR/DIR receive the IAR value. unsafe { asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } IRQ_IAR.store(iar, Ordering::Relaxed); IRQ_OWNER.store(SERIAL_LOCK.owner(), Ordering::Relaxed); IRQ_SPSR.store(context.spsr as usize, Ordering::Relaxed); IRQ_MPIDR.store(mpidr as usize, Ordering::Relaxed); IRQ_DONE.store(1, Ordering::Release); 1 } #[cfg(feature = "smp-irq-lock-test")] fn irq_primary(tables: &mmu::Tables) { if !tables.normal_shareable_identity(core::ptr::addr_of!(IRQ_DONE) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(IRQ_IAR) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(SERIAL_LOCK) as u64) { fail("IRQ lock observation is not Normal Inner-Shareable RAM"); } // SAFETY: only the primary owns this dedicated IRQ stack guard. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; let mpidr: u64; // SAFETY: current EL1 CPU identity is used for the SGI1 self-target command. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)); } let Some(command) = irq::sgi1r_for_mpidr(mpidr, 1) else { fail("SGI1 target affinity"); }; if SERIAL_LOCK.owner() != 0 || SERIAL_LOCK.irq_denial_count() != 0 { fail("IRQ lock precondition"); } let guard = SERIAL_LOCK.lock((mpidr & 0xff) as usize); let observed: usize; // SAFETY: SGI1 is armed. IRQ is unmasked only in this bounded assembly // interval and is remasked before Rust continues, even on timeout. unsafe { asm!( "msr daifclr, #2", "msr ICC_SGI1R_EL1, {command}", "isb", "2:", "ldr {observed}, [{flag}]", "cbnz {observed}, 3f", "sub {budget}, {budget}, #1", "cbnz {budget}, 2b", "3:", "msr daifset, #2", command = in(reg) command, flag = in(reg) core::ptr::addr_of!(IRQ_DONE) as usize, budget = inout(reg) 50_000_000usize => _, observed = out(reg) observed, options(nostack) ); } if observed != 1 || IRQ_DONE.load(Ordering::Acquire) != 1 || IRQ_IAR.load(Ordering::Acquire) & 0x00ff_ffff != 1 || IRQ_OWNER.load(Ordering::Acquire) != 1 || IRQ_MPIDR.load(Ordering::Acquire) & 0xff != 0 || IRQ_SPSR.load(Ordering::Acquire) & 0x8f != 5 || SERIAL_LOCK.owner() != 1 || SERIAL_LOCK.irq_denial_count() != 1 { fail("SGI1 IRQ lock denial or ERET progress"); } drop(guard); if SERIAL_LOCK.owner() != 0 { fail("IRQ owner release"); } write_str("TURKOS/SMP-IRQ-LOCK-ACCOUNT: IAR=0x"); write_hex(IRQ_IAR.load(Ordering::Acquire)); write_str(" MPIDR=0x"); write_hex(IRQ_MPIDR.load(Ordering::Acquire)); write_str(" SPSR=0x"); write_hex(IRQ_SPSR.load(Ordering::Acquire)); write_str(" DENIED=0x"); write_hex(SERIAL_LOCK.irq_denial_count() as usize); write_str(" HELD=0x"); write_hex(IRQ_OWNER.load(Ordering::Acquire)); write_str(" RELEASED=0x"); write_hex(SERIAL_LOCK.owner()); write_str("\r\n"); write_line("TURKOS/SMP-IRQ-LOCK: PASS"); } #[cfg(feature = "smp-order-test")] fn order_payload(sequence: usize, cpu: usize) -> u64 { 0x4b34_3731_0000_0000 ^ ((sequence as u64) << 16) ^ cpu as u64 } #[cfg(feature = "smp-order-test")] fn order_secondary() -> bool { let mut sum = 0u64; for sequence in 1..=256 { let mut seen = None; for _ in 0..10_000_000 { seen = ORDER_P2S.try_receive(sequence); if seen.is_some() { break; } core::hint::spin_loop(); } let Some(value) = seen else { return false; }; if value != order_payload(sequence, 0) { return false; } sum = sum.wrapping_add(value); ORDER_S2P.publish(sequence, order_payload(sequence, 1)); let mut acknowledged = false; for _ in 0..10_000_000 { if ORDER_ACK.load(Ordering::Acquire) == sequence { acknowledged = true; break; } core::hint::spin_loop(); } if !acknowledged { return false; } } ORDER_SUM_S.store(sum, Ordering::Relaxed); true } #[cfg(feature = "smp-order-test")] fn order_primary(tables: &mmu::Tables) { if !tables.normal_shareable_identity(core::ptr::addr_of!(ORDER_P2S) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(ORDER_S2P) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(ORDER_ACK) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(ORDER_DONE) as u64) { fail("order lanes are not Normal Inner-Shareable writable RAM"); } ORDER_GO.store(1, Ordering::Release); let mut sum_primary = 0u64; for sequence in 1..=256 { ORDER_P2S.publish(sequence, order_payload(sequence, 0)); let mut seen = None; for _ in 0..10_000_000 { seen = ORDER_S2P.try_receive(sequence); if seen.is_some() { break; } if ORDER_DONE.load(Ordering::Acquire) == 2 { fail("secondary order mismatch"); } core::hint::spin_loop(); } let Some(value) = seen else { fail("secondary order timeout"); }; if value != order_payload(sequence, 1) { fail("secondary published stale payload"); } sum_primary = sum_primary.wrapping_add(value); ORDER_ACK.store(sequence, Ordering::Release); } let mut done = 0; for _ in 0..10_000_000 { done = ORDER_DONE.load(Ordering::Acquire); if done != 0 { break; } core::hint::spin_loop(); } let expected_primary = (1..=256).fold(0u64, |sum, seq| sum.wrapping_add(order_payload(seq, 1))); let expected_secondary = (1..=256).fold(0u64, |sum, seq| sum.wrapping_add(order_payload(seq, 0))); let secondary_sum = ORDER_SUM_S.load(Ordering::Acquire); if done != 1 || sum_primary != expected_primary || secondary_sum != expected_secondary || ORDER_P2S.published_sequence() != 256 || ORDER_S2P.published_sequence() != 256 || ORDER_ACK.load(Ordering::Acquire) != 256 { fail("release/acquire payload visibility or accounting"); } write_str("TURKOS/SMP-ORDER-ACCOUNT: CPU0=0x"); write_hex(256); write_str(" CPU1=0x"); write_hex(256); write_str(" SUM0=0x"); write_hex(sum_primary as usize); write_str(" SUM1=0x"); write_hex(secondary_sum as usize); write_str(" PSEQ=0x"); write_hex(ORDER_P2S.published_sequence()); write_str(" SSEQ=0x"); write_hex(ORDER_S2P.published_sequence()); write_str(" ACK=0x"); write_hex(ORDER_ACK.load(Ordering::Acquire)); write_str("\r\n"); write_line("TURKOS/SMP-ORDER: PASS"); } #[cfg(feature = "smp-fair-test")] fn fair_primary(tables: &mmu::Tables) { if !tables.normal_shareable_identity(core::ptr::addr_of!(FAIR_LOCK) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(FAIR_TOTAL) as u64) || !tables.normal_shareable_identity(core::ptr::addr_of!(FAIR_CPU1) as u64) { fail("ticket lock is not Normal Inner-Shareable writable RAM"); } #[cfg(feature = "smp-race-trace-test")] if !tables.normal_shareable_identity(core::ptr::addr_of!(FAIR_TRACE) as u64) { fail("ticket trace is not Normal Inner-Shareable writable RAM"); } #[cfg(feature = "smp-stats-test")] let mut cpu0_stats = CounterStats::new(); #[cfg(feature = "smp-wait-class-test")] let mut cpu0_classes = WaitClassStats::new(); #[cfg(feature = "smp-hold-test")] let mut cpu0_hold = CounterStats::new(); #[cfg(feature = "smp-classified-admission-test")] let mut cpu0_partition = ClassifiedCounterStats::new(); #[cfg(feature = "smp-stats-test")] let first_start = physical_counter(); let first = FAIR_LOCK.lock(); #[cfg(feature = "smp-stats-test")] let first_end = physical_counter(); #[cfg(feature = "smp-hold-test")] let first_hold_start = physical_counter(); #[cfg(feature = "smp-wait-class-test")] if cpu0_classes.record(first.wait_spins()).is_err() { fail("CPU0 first wait class"); } #[cfg(feature = "smp-stats-test")] if cpu0_stats .record(first_start as u64, first_end as u64) .is_err() { fail("CPU0 first counter stats"); } #[cfg(feature = "smp-classified-admission-test")] if cpu0_partition .record(first_start as u64, first_end as u64, first.wait_spins()) .is_err() { fail("CPU0 first partition stats"); } if first.ticket() != 0 { fail("first ticket not zero"); } FAIR_GO.store(1, Ordering::Release); for _ in 0..100_000_000 { if FAIR_LOCK.next_ticket() >= 2 { break; } core::hint::spin_loop(); } if FAIR_LOCK.next_ticket() != 2 || FAIR_LOCK.serving() != 0 { fail("second CPU did not queue behind owner"); } #[cfg(feature = "smp-wait-test")] { let mut witnessed = false; for _ in 0..100_000_000 { if FAIR_LOCK.wait_events() > 0 { witnessed = true; break; } core::hint::spin_loop(); } if !witnessed || FAIR_LOCK.serving() != 0 { fail("CPU1 did not actually wait behind ticket0"); } } if FAIR_TOTAL.load(Ordering::Relaxed) != first.ticket() { fail("first FIFO ticket mismatch"); } #[cfg(feature = "smp-race-trace-test")] if !trace_ticket_owner(first.ticket(), 0) { fail("first ticket trace slot was not empty"); } FAIR_TOTAL.store(first.ticket() + 1, Ordering::Relaxed); FAIR_CPU0.fetch_add(1, Ordering::Relaxed); #[cfg(feature = "smp-hold-test")] { let first_hold_end = physical_counter(); let Some(first_delta) = positive_counter_delta(first_hold_start as u64, first_hold_end as u64) else { fail("CPU0 first hold did not advance counter"); }; HOLD0_FIRST.store(first_delta as usize, Ordering::Relaxed); if cpu0_hold .record(first_hold_start as u64, first_hold_end as u64) .is_err() { fail("CPU0 first hold stats"); } } drop(first); for _ in 1..128 { #[cfg(feature = "smp-stats-test")] let counter_before = physical_counter(); let guard = FAIR_LOCK.lock(); #[cfg(feature = "smp-stats-test")] let counter_after = physical_counter(); #[cfg(feature = "smp-hold-test")] let hold_start = physical_counter(); #[cfg(feature = "smp-wait-class-test")] if cpu0_classes.record(guard.wait_spins()).is_err() { fail("CPU0 wait class"); } #[cfg(feature = "smp-stats-test")] if cpu0_stats .record(counter_before as u64, counter_after as u64) .is_err() { fail("CPU0 counter stats"); } #[cfg(feature = "smp-classified-admission-test")] if cpu0_partition .record( counter_before as u64, counter_after as u64, guard.wait_spins(), ) .is_err() { fail("CPU0 partition stats"); } let ticket = guard.ticket(); if FAIR_TOTAL.load(Ordering::Relaxed) != ticket { fail("ticket order inversion"); } #[cfg(feature = "smp-race-trace-test")] if !trace_ticket_owner(ticket, 0) { fail("CPU0 ticket trace slot was not empty"); } FAIR_TOTAL.store(ticket + 1, Ordering::Relaxed); FAIR_CPU0.fetch_add(1, Ordering::Relaxed); #[cfg(feature = "smp-hold-test")] if cpu0_hold .record(hold_start as u64, physical_counter() as u64) .is_err() { fail("CPU0 hold stats"); } } let mut done = 0; for _ in 0..100_000_000 { done = FAIR_DONE.load(Ordering::Acquire); if done != 0 { break; } core::hint::spin_loop(); } if done != 1 || FAIR_CPU0.load(Ordering::Acquire) != 128 || FAIR_CPU1.load(Ordering::Acquire) != 128 || FAIR_TOTAL.load(Ordering::Acquire) != 256 || FAIR_LOCK.next_ticket() != 256 || FAIR_LOCK.serving() != 256 { fail("ticket FIFO or per-CPU finite progress"); } write_str("TURKOS/SMP-FAIR-ACCOUNT: CPU0=0x"); write_hex(FAIR_CPU0.load(Ordering::Acquire)); write_str(" CPU1=0x"); write_hex(FAIR_CPU1.load(Ordering::Acquire)); write_str(" ISSUED=0x"); write_hex(FAIR_LOCK.next_ticket()); write_str(" SERVED=0x"); write_hex(FAIR_LOCK.serving()); write_str(" TOTAL=0x"); write_hex(FAIR_TOTAL.load(Ordering::Acquire)); write_str("\r\n"); write_line("TURKOS/SMP-FAIR: PASS"); #[cfg(feature = "smp-wait-test")] { let first_wait = FAIR_CPU1_FIRST_WAIT.load(Ordering::Acquire); let max_wait = FAIR_LOCK.max_wait_spins(); let events = FAIR_LOCK.wait_events(); if first_wait == 0 || max_wait < first_wait || events == 0 || events > 255 { fail("ticket wait witness accounting"); } write_str("TURKOS/SMP-WAIT-ACCOUNT: FIRST1=0x"); write_hex(first_wait); write_str(" MAX=0x"); write_hex(max_wait); write_str(" EVENTS=0x"); write_hex(events); write_str("\r\n"); write_line("TURKOS/SMP-WAIT: PASS"); } #[cfg(feature = "smp-time-test")] { let frequency = TIME_FREQUENCY.load(Ordering::Acquire); let start = TIME_START.load(Ordering::Acquire) as u64; let end = TIME_END.load(Ordering::Acquire) as u64; let Some(delta) = positive_counter_delta(start, end) else { fail("counter interval is zero or reversed"); }; if frequency != 24_000_000 || delta > usize::MAX as u64 { fail("counter frequency or interval"); } write_str("TURKOS/SMP-TIME-ACCOUNT: FREQ=0x"); write_hex(frequency); write_str(" START=0x"); write_hex(start as usize); write_str(" END=0x"); write_hex(end as usize); write_str(" DELTA=0x"); write_hex(delta as usize); write_str("\r\n"); write_line("TURKOS/SMP-TIME: PASS"); } #[cfg(feature = "smp-stats-test")] { let n1 = STATS1_N.load(Ordering::Acquire); let min1 = STATS1_MIN.load(Ordering::Acquire); let max1 = STATS1_MAX.load(Ordering::Acquire); let sum1 = STATS1_SUM.load(Ordering::Acquire); let first_delta = (TIME_END.load(Ordering::Acquire) - TIME_START.load(Ordering::Acquire)) as usize; if cpu0_stats.samples != 128 || n1 != 128 || cpu0_stats.minimum > cpu0_stats.maximum || cpu0_stats.total < cpu0_stats.maximum || min1 > max1 || sum1 < max1 || max1 < first_delta { fail("per-CPU counter distribution"); } write_str("TURKOS/SMP-STATS-ACCOUNT: N0=0x"); write_hex(cpu0_stats.samples); write_str(" MIN0=0x"); write_hex(cpu0_stats.minimum as usize); write_str(" MAX0=0x"); write_hex(cpu0_stats.maximum as usize); write_str(" SUM0=0x"); write_hex(cpu0_stats.total as usize); write_str(" N1=0x"); write_hex(n1); write_str(" MIN1=0x"); write_hex(min1); write_str(" MAX1=0x"); write_hex(max1); write_str(" SUM1=0x"); write_hex(sum1); write_str("\r\n"); write_line("TURKOS/SMP-STATS: PASS"); } #[cfg(feature = "smp-wait-class-test")] { let zero1 = CLASS1_ZERO.load(Ordering::Acquire); let contended1 = CLASS1_CONTENDED.load(Ordering::Acquire); let total1 = CLASS1_TOTAL.load(Ordering::Acquire); let max1 = CLASS1_MAX.load(Ordering::Acquire); if cpu0_classes.zero.checked_add(cpu0_classes.contended) != Some(128) || zero1.checked_add(contended1) != Some(128) || contended1 == 0 || cpu0_classes.contended.checked_add(contended1) != Some(FAIR_LOCK.wait_events()) || cpu0_classes.maximum_spins.max(max1) != FAIR_LOCK.max_wait_spins() || cpu0_classes.total_spins < cpu0_classes.maximum_spins || total1 < max1 { fail("per-CPU wait class accounting"); } write_str("TURKOS/SMP-CLASS-ACCOUNT: Z0=0x"); write_hex(cpu0_classes.zero); write_str(" C0=0x"); write_hex(cpu0_classes.contended); write_str(" SUM0=0x"); write_hex(cpu0_classes.total_spins); write_str(" MAX0=0x"); write_hex(cpu0_classes.maximum_spins); write_str(" Z1=0x"); write_hex(zero1); write_str(" C1=0x"); write_hex(contended1); write_str(" SUM1=0x"); write_hex(total1); write_str(" MAX1=0x"); write_hex(max1); write_str("\r\n"); write_line("TURKOS/SMP-CLASS: PASS"); } #[cfg(feature = "smp-hold-test")] { let n1 = HOLD1_N.load(Ordering::Acquire); let min1 = HOLD1_MIN.load(Ordering::Acquire); let max1 = HOLD1_MAX.load(Ordering::Acquire); let sum1 = HOLD1_SUM.load(Ordering::Acquire); let first0 = HOLD0_FIRST.load(Ordering::Acquire); if cpu0_hold.samples != 128 || n1 != 128 || first0 == 0 || first0 as u64 > cpu0_hold.maximum || cpu0_hold.minimum > cpu0_hold.maximum || cpu0_hold.maximum > cpu0_hold.total || min1 > max1 || max1 > sum1 { fail("per-CPU critical-section hold counter distribution"); } write_str("TURKOS/SMP-HOLD-ACCOUNT: N0=0x"); write_hex(cpu0_hold.samples); write_str(" MIN0=0x"); write_hex(cpu0_hold.minimum as usize); write_str(" MAX0=0x"); write_hex(cpu0_hold.maximum as usize); write_str(" SUM0=0x"); write_hex(cpu0_hold.total as usize); write_str(" N1=0x"); write_hex(n1); write_str(" MIN1=0x"); write_hex(min1); write_str(" MAX1=0x"); write_hex(max1); write_str(" SUM1=0x"); write_hex(sum1); write_str(" FIRST0=0x"); write_hex(first0); write_str("\r\n"); write_line("TURKOS/SMP-HOLD: PASS"); } #[cfg(feature = "smp-counter-baseline-test")] { let mut baseline0 = CounterStats::new(); for _ in 0..128 { let first = physical_counter(); let second = physical_counter(); if baseline0.record(first as u64, second as u64).is_err() { fail("CPU0 back-to-back counter baseline"); } } let n1 = BASE1_N.load(Ordering::Acquire); let min1 = BASE1_MIN.load(Ordering::Acquire); let max1 = BASE1_MAX.load(Ordering::Acquire); let sum1 = BASE1_SUM.load(Ordering::Acquire); if baseline0.samples != 128 || n1 != 128 || baseline0.minimum > baseline0.maximum || baseline0.maximum > baseline0.total || min1 > max1 || max1 > sum1 { fail("per-CPU physical counter baseline distribution"); } write_str("TURKOS/SMP-BASELINE-ACCOUNT: N0=0x"); write_hex(baseline0.samples); write_str(" MIN0=0x"); write_hex(baseline0.minimum as usize); write_str(" MAX0=0x"); write_hex(baseline0.maximum as usize); write_str(" SUM0=0x"); write_hex(baseline0.total as usize); write_str(" N1=0x"); write_hex(n1); write_str(" MIN1=0x"); write_hex(min1); write_str(" MAX1=0x"); write_hex(max1); write_str(" SUM1=0x"); write_hex(sum1); write_str("\r\n"); write_line("TURKOS/SMP-BASELINE: PASS"); } #[cfg(feature = "smp-classified-admission-test")] { let z1n = PART1_Z_N.load(Ordering::Acquire); let z1sum = PART1_Z_SUM.load(Ordering::Acquire); let z1max = PART1_Z_MAX.load(Ordering::Acquire); let c1n = PART1_C_N.load(Ordering::Acquire); let c1sum = PART1_C_SUM.load(Ordering::Acquire); let c1max = PART1_C_MAX.load(Ordering::Acquire); if cpu0_partition.zero_spin.samples != cpu0_classes.zero || cpu0_partition.contended.samples != cpu0_classes.contended || z1n != CLASS1_ZERO.load(Ordering::Acquire) || c1n != CLASS1_CONTENDED.load(Ordering::Acquire) || c1n == 0 || cpu0_partition .zero_spin .total .checked_add(cpu0_partition.contended.total) != Some(cpu0_stats.total) || z1sum.checked_add(c1sum) != Some(STATS1_SUM.load(Ordering::Acquire)) || cpu0_partition .zero_spin .maximum .max(cpu0_partition.contended.maximum) != cpu0_stats.maximum || z1max.max(c1max) != STATS1_MAX.load(Ordering::Acquire) { fail("classified admission does not partition raw intervals"); } write_str("TURKOS/SMP-PART-ACCOUNT: Z0N=0x"); write_hex(cpu0_partition.zero_spin.samples); write_str(" Z0SUM=0x"); write_hex(cpu0_partition.zero_spin.total as usize); write_str(" Z0MAX=0x"); write_hex(cpu0_partition.zero_spin.maximum as usize); write_str(" C0N=0x"); write_hex(cpu0_partition.contended.samples); write_str(" C0SUM=0x"); write_hex(cpu0_partition.contended.total as usize); write_str(" C0MAX=0x"); write_hex(cpu0_partition.contended.maximum as usize); write_str(" Z1N=0x"); write_hex(z1n); write_str(" Z1SUM=0x"); write_hex(z1sum); write_str(" Z1MAX=0x"); write_hex(z1max); write_str(" C1N=0x"); write_hex(c1n); write_str(" C1SUM=0x"); write_hex(c1sum); write_str(" C1MAX=0x"); write_hex(c1max); write_str("\r\n"); write_line("TURKOS/SMP-PART: PASS"); } #[cfg(feature = "smp-race-trace-test")] { let mut owners = [0xffu8; 256]; for (ticket, owner) in owners.iter_mut().enumerate() { *owner = FAIR_TRACE[ticket].load(Ordering::Acquire); } let Ok(summary) = summarize_two_cpu_trace(&owners) else { fail("ticket owner trace is incomplete or corrupt"); }; if summary.cpu0 != FAIR_CPU0.load(Ordering::Acquire) || summary.cpu1 != FAIR_CPU1.load(Ordering::Acquire) { fail("ticket owner trace does not match fair counts"); } let digits = b"0123456789abcdef"; let mut hex = [0u8; 64]; for (group, digit) in hex.iter_mut().enumerate() { let mut nibble = 0usize; for bit in 0..4 { nibble = (nibble << 1) | owners[group * 4 + bit] as usize; } *digit = digits[nibble]; } write_str("TURKOS/SMP-TRACE-ACCOUNT: CPU0=0x"); write_hex(summary.cpu0); write_str(" CPU1=0x"); write_hex(summary.cpu1); write_str(" SWITCH=0x"); write_hex(summary.switches); write_str(" MAXRUN=0x"); write_hex(summary.longest_run); write_str("\r\nTURKOS/SMP-TRACE-BITS: "); write_str(core::str::from_utf8(&hex).unwrap()); write_str("\r\n"); write_line("TURKOS/SMP-TRACE: PASS"); } }