//! First ELF-backed EL0 program to issue the existing versioned SERIAL_WRITE //! ABI, receive its byte count, then make the existing EXIT call. use super::*; use turkos_kernel::{elf, init_exit, program_hash, syscall, user_copy}; #[cfg(feature = "turkfs-init-test")] use turkos_kernel::{ process::{ProcessId, ProcessTable, State}, process_image, }; const GUARD: u64 = 0x454c_4648_454c_4c4f; static mut ROOT: u64 = 0; static mut CODE_FRAME: u64 = 0; static mut DATA_FRAME: u64 = 0; static mut STACK_FRAME: u64 = 0; static mut INITIAL_SP: u64 = 0; static mut CALLS: usize = 0; #[cfg(feature = "turkfs-init-test")] static mut INIT_PROCESSES: ProcessTable = ProcessTable::new(); #[cfg(feature = "turkfs-init-test")] static mut INIT_OWNER: Option = None; #[cfg(feature = "turkfs-init-test")] static mut INIT_SPACE: Option = None; #[cfg(feature = "turkfs-init-test")] static mut INIT_ALLOCATOR: *mut memory::FrameAllocator = core::ptr::null_mut(); #[cfg(feature = "turkfs-init-test")] static mut INIT_KERNEL_ROOT: u64 = 0; #[cfg(feature = "turkfs-init-test")] static mut INIT_FREE_BEFORE: u64 = 0; #[cfg(feature = "turkfs-init-test")] #[repr(align(4096))] struct InitAcceptanceStack([u8; 16 * 1024]); #[cfg(feature = "turkfs-init-test")] static mut INIT_ACCEPTANCE_STACK: InitAcceptanceStack = InitAcceptanceStack([0; 16 * 1024]); #[cfg(feature = "turkfs-init-test")] static mut INIT_OWNED_ALLOCATOR: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); #[cfg(feature = "turkfs-init-test")] static mut INIT_OWNED_TABLES: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); #[cfg(feature = "turkfs-init-test")] static mut INIT_OWNED_RAM: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); #[cfg(feature = "turkfs-init-test")] static mut INIT_OWNED_FILE: [u8; 512] = [0; 512]; unsafe extern "C" { static __elf_hello_blob_start: u8; static __elf_hello_blob_end: u8; } fn fail(reason: &str) -> ! { write_str("TURKOS/ELF-HELLO-ERROR: "); write_line(reason); park() } #[cfg(feature = "elf-hello-test")] fn fixture() -> [u8; 512] { let mut file = [0u8; 512]; file[..20].copy_from_slice(&[ 0x7f, b'E', b'L', b'F', 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0xb7, 0, ]); file[20..24].copy_from_slice(&1u32.to_le_bytes()); file[24..32].copy_from_slice(&mmu::USER_CODE_VA.to_le_bytes()); file[32..40].copy_from_slice(&64u64.to_le_bytes()); file[52..54].copy_from_slice(&64u16.to_le_bytes()); file[54..56].copy_from_slice(&56u16.to_le_bytes()); file[56..58].copy_from_slice(&2u16.to_le_bytes()); file[64..68].copy_from_slice(&1u32.to_le_bytes()); file[68..72].copy_from_slice(&5u32.to_le_bytes()); file[72..80].copy_from_slice(&320u64.to_le_bytes()); file[80..88].copy_from_slice(&mmu::USER_CODE_VA.to_le_bytes()); file[96..104].copy_from_slice(&56u64.to_le_bytes()); file[104..112].copy_from_slice(&64u64.to_le_bytes()); file[112..120].copy_from_slice(&1u64.to_le_bytes()); file[120..124].copy_from_slice(&1u32.to_le_bytes()); file[124..128].copy_from_slice(&6u32.to_le_bytes()); file[128..136].copy_from_slice(&448u64.to_le_bytes()); file[136..144].copy_from_slice(&mmu::USER_DATA_VA.to_le_bytes()); file[152..160].copy_from_slice(&5u64.to_le_bytes()); file[160..168].copy_from_slice(&8u64.to_le_bytes()); file[168..176].copy_from_slice(&1u64.to_le_bytes()); file[448..453].copy_from_slice(b"HELLO"); let start = core::ptr::addr_of!(__elf_hello_blob_start) as usize; let end = core::ptr::addr_of!(__elf_hello_blob_end) as usize; if end.checked_sub(start) != Some(56) { fail("ELF hello instruction length"); } unsafe { core::ptr::copy_nonoverlapping(start as *const u8, file[320..376].as_mut_ptr(), 56) }; file } #[cfg(feature = "elf-hello-test")] pub(super) fn smoke_test( allocator: &mut memory::FrameAllocator, tables: &mmu::Tables, ram: mmu::Layout, ) -> ! { let file = fixture(); smoke_test_with_file(allocator, tables, ram, &file) } pub(crate) fn smoke_test_with_file( allocator: &mut memory::FrameAllocator, tables: &mmu::Tables, ram: mmu::Layout, file: &[u8; 512], ) -> ! { #[cfg(feature = "turkfs-init-test")] unsafe { // The four GETs have reset DMA and this path never returns. Transfer // owners out of kernel_main's large stack before starting ELF/argv // transactions on a distinct bounded EL1 acceptance stack. (*core::ptr::addr_of_mut!(INIT_OWNED_ALLOCATOR)).write(core::ptr::read(allocator)); (*core::ptr::addr_of_mut!(INIT_OWNED_TABLES)).write(core::ptr::read(tables)); (*core::ptr::addr_of_mut!(INIT_OWNED_RAM)).write(ram); core::ptr::addr_of_mut!(INIT_OWNED_FILE).write(*file); let bottom = core::ptr::addr_of_mut!(INIT_ACCEPTANCE_STACK.0) as usize; write_volatile(bottom as *mut u64, GUARD); asm!("mov sp, {stack}", "b kernel_init_owned_entry", stack = in(reg) (bottom + 16 * 1024), options(noreturn)); } #[cfg(not(feature = "turkfs-init-test"))] smoke_on_owned_stack(allocator, tables, ram, file) } #[cfg(feature = "turkfs-init-test")] #[unsafe(no_mangle)] pub extern "C" fn kernel_init_owned_entry() -> ! { let allocator = unsafe { &mut *(*core::ptr::addr_of_mut!(INIT_OWNED_ALLOCATOR)).as_mut_ptr() }; let tables = unsafe { &*(*core::ptr::addr_of!(INIT_OWNED_TABLES)).as_ptr() }; let ram = unsafe { (*core::ptr::addr_of!(INIT_OWNED_RAM)).assume_init_read() }; let file = unsafe { &*core::ptr::addr_of!(INIT_OWNED_FILE) }; smoke_on_owned_stack(allocator, tables, ram, file) } fn smoke_on_owned_stack( allocator: &mut memory::FrameAllocator, tables: &mmu::Tables, ram: mmu::Layout, file: &[u8; 512], ) -> ! { #[cfg(feature = "turkfs-init-test")] let (user, loaded, initial) = { let image = spawn_init(allocator, tables, ram, file); (image.space, image.loaded, image.initial) }; #[cfg(not(feature = "turkfs-init-test"))] let (user, loaded, initial) = { let user = unsafe { tables.build_user_space(allocator, ram.ram_base, ram.ram_size) } .unwrap_or_else(|_| fail("private ELF hello root")); if !user.contract_valid(tables.root) { fail("ELF hello descriptors"); } let stack = unsafe { core::slice::from_raw_parts_mut( user.stack_frame() as *mut u8, memory::PAGE_SIZE as usize, ) }; let initial = elf::prepare_user_stack(stack, &[b"hello"]) .unwrap_or_else(|_| fail("initial hello stack")); let loaded = unsafe { elf::load_into_user_space(file, &user, tables.root) } .unwrap_or_else(|_| fail("ELF hello install")); (user, loaded, initial) }; if loaded.entry != mmu::USER_CODE_VA || loaded.code_bytes != 56 || loaded.data_bytes != 5 || unsafe { read_volatile(user.data_frame() as *const u32) } != if cfg!(feature = "turkfs-init-test") { 0x5449_4e49 } else { 0x4c4c_4548 } { fail("ELF hello bytes"); } let code = unsafe { core::slice::from_raw_parts(user.code_frame() as *const u8, loaded.code_bytes) }; let data = unsafe { core::slice::from_raw_parts(user.data_frame() as *const u8, loaded.data_bytes) }; let installed_digest = program_hash::digest(loaded.entry, code, data); let source_digest = program_hash::digest(loaded.entry, &file[320..376], &file[448..453]); if installed_digest != source_digest { fail("ELF installed program hash differs from file bytes"); } let hash_hex = program_hash::hex_lower(&installed_digest); write_str("TURKOS/ELF-HELLO-HASH: SHA256="); write_str(core::str::from_utf8(&hash_hex).unwrap_or_else(|_| fail("hash hex encoding"))); write_str(" ENTRY=0x"); write_hex(loaded.entry as usize); write_str(" CODE=0x"); write_hex(loaded.code_bytes); write_str(" DATA=0x"); write_hex(loaded.data_bytes); write_str("\r\n"); let ctr: usize; unsafe { asm!("mrs {value}, CTR_EL0", value = out(reg) ctr, options(nomem, nostack, preserves_flags)) }; let dline = 4usize << ((ctr >> 16) & 15); for offset in (0..loaded.code_bytes).step_by(dline) { unsafe { asm!("dc cvau, {addr}", addr = in(reg) (user.code_frame() as usize + offset), options(nostack)) }; } unsafe { asm!("dsb ish", "ic iallu", "dsb ish", "isb", options(nostack)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(ROOT), user.root); write_volatile(core::ptr::addr_of_mut!(CODE_FRAME), user.code_frame()); write_volatile(core::ptr::addr_of_mut!(DATA_FRAME), user.data_frame()); write_volatile(core::ptr::addr_of_mut!(STACK_FRAME), user.stack_frame()); write_volatile(core::ptr::addr_of_mut!(INITIAL_SP), initial.sp); write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), GUARD); asm!( "dsb ishst", "msr TTBR0_EL1, {root}", "isb", "tlbi vmalle1is", "dsb ish", "isb", root = in(reg) user.root, options(nostack) ); } #[cfg(feature = "turkfs-init-test")] unsafe { INIT_SPACE = Some(user) }; write_str("TURKOS/ELF-HELLO-INSTALL: ENTRY=0x"); write_hex(loaded.entry as usize); write_str(" CODE=0x"); write_hex(loaded.code_bytes); write_str(" DATA=0x"); write_hex(loaded.data_bytes); write_str(" SP=0x"); write_hex(initial.sp as usize); write_str("\r\n"); write_line("TURKOS/ELF-HELLO-EL0: ATTEMPT"); unsafe { asm!( "msr SP_EL0, {stack}", "msr ELR_EL1, {entry}", "msr SPSR_EL1, {mode}", "isb", "eret", stack = in(reg) initial.sp, entry = in(reg) loaded.entry, mode = in(reg) 0x3c0usize, options(noreturn) ) } } #[cfg(feature = "turkfs-init-test")] fn spawn_init( allocator: &mut memory::FrameAllocator, tables: &mmu::Tables, ram: mmu::Layout, file: &[u8; 512], ) -> process_image::ProcessImage { let processes = unsafe { &mut *core::ptr::addr_of_mut!(INIT_PROCESSES) }; let before = processes.clone(); let free_before = allocator.remaining_pages(); let mut regions = [memory::Region::default(); 2 * dtb::MAX_RESERVATIONS + 5]; let free = allocator .free_regions() .unwrap_or_else(|_| fail("INIT free audit")); let count = free.len(); regions[..count].copy_from_slice(free); // Capture the six frames a fresh private space will acquire. They are // returned before each transaction, never installed in a TTBR here. let mut frames = [0u64; 6]; for frame in &mut frames { *frame = allocator .allocate() .unwrap_or_else(|| fail("INIT rollback fixture frames")); } allocator .release_many(&frames) .unwrap_or_else(|_| fail("INIT fixture return")); for case in 0..4 { for frame in frames { unsafe { core::ptr::write_bytes(frame as *mut u8, 0xa5, 4096) }; } let mut damaged = *file; let oversized = [b'x'; elf::MAX_ARGUMENT_BYTES + 1]; let (name, expected, arguments): (&str, elf::Error, &[&[u8]]) = match case { 0 => { damaged[0] = 0; ("BAD-MAGIC", elf::Error::BadMagic, &[b"init"]) } 1 => { damaged[68..72].copy_from_slice(&7u32.to_le_bytes()); ("WX", elf::Error::WritableExecutable, &[b"init"]) } 2 => { damaged[24..32].copy_from_slice(&(mmu::USER_CODE_VA + 1).to_le_bytes()); ("ENTRY", elf::Error::BadEntry, &[b"init"]) } _ => ("ARGV-AFTER-LOAD", elf::Error::BadArguments, &[&oversized]), }; let result = unsafe { process_image::spawn( processes, allocator, tables, ram.ram_base, ram.ram_size, &damaged, arguments, ) }; if !matches!(result, Err(process_image::Error::Elf(error)) if error == expected) || *processes != before || allocator.remaining_pages() != free_before || allocator .free_regions() .unwrap_or_else(|_| fail("INIT rollback free integrity")) != ®ions[..count] || frames.iter().any(|frame| { unsafe { core::slice::from_raw_parts(*frame as *const u8, 4096) } .iter() .any(|byte| *byte != 0) }) { fail("INIT transaction rollback identity/regions/erasure"); } write_str("TURKOS/INIT-ROLLBACK: CASE="); write_str(name); write_line(" ACQUIRED=6 IDENTITY=UNCHANGED REGIONS=UNCHANGED ZEROED=YES PASS"); } let image = unsafe { process_image::spawn( processes, allocator, tables, ram.ram_base, ram.ram_size, file, &[b"init"], ) } .unwrap_or_else(|_| fail("owned INIT spawn")); if image.identity.slot != 0 || image.identity.generation != 1 || allocator.remaining_pages() + 6 != free_before { fail("INIT owner generation or frames"); } processes .run(image.identity) .unwrap_or_else(|_| fail("INIT running owner")); unsafe { INIT_OWNER = Some(image.identity); INIT_ALLOCATOR = allocator as *mut _; INIT_KERNEL_ROOT = tables.root; INIT_FREE_BEFORE = free_before; } write_str("TURKOS/INIT-PROCESS: SLOT=0 GEN=1 STATE=RUNNING ROOT=0x"); write_hex(image.space.root as usize); write_line(" OWNED-FRAMES=6 PASS"); image } #[cfg(feature = "turkfs-init-test")] fn retire_init(code: u64) { let root = unsafe { INIT_KERNEL_ROOT }; // This acceptance image is single-CPU and IRQ-masked. Disk DMA was // reset/returned before spawn. Never scrub the active TTBR user root. unsafe { asm!("dsb ish", "msr TTBR0_EL1, {root}", "isb", "tlbi vmalle1is", "dsb ish", "isb", root = in(reg) root, options(nostack)) }; let processes = unsafe { &mut *core::ptr::addr_of_mut!(INIT_PROCESSES) }; let owner = unsafe { INIT_OWNER }.unwrap_or_else(|| fail("INIT missing owner")); if processes.state(owner) != Ok(State::Running) { fail("INIT owner not running"); } processes .exit(owner, code as i32) .unwrap_or_else(|_| fail("INIT exit transition")); let user = unsafe { (&mut *core::ptr::addr_of_mut!(INIT_SPACE)).take() } .unwrap_or_else(|| fail("INIT duplicate retirement")); let data = user.data_frame(); let allocator = unsafe { &mut *INIT_ALLOCATOR }; unsafe { user.release_after_tlbi(allocator, root) } .unwrap_or_else(|_| fail("INIT page retirement")); if allocator.remaining_pages() != unsafe { INIT_FREE_BEFORE } || unsafe { read_volatile(data as *const u64) } != 0 || allocator.release(data).is_ok() || processes.reap(owner) != Ok(code as i32) || processes.state(owner).is_ok() || unsafe { read_volatile(core::ptr::addr_of!(INIT_ACCEPTANCE_STACK) as *const u64) } != GUARD { fail("INIT page leak/double return/stale process"); } write_line( "TURKOS/INIT-REAP: FRAMES=6 RESTORED ZEROED DOUBLE-RETURN=REJECTED OLD-ID=REJECTED PASS", ); } #[unsafe(no_mangle)] pub extern "C" fn kernel_return_probe(frame: *mut irq::IrqContext) -> usize { if frame.is_null() { fail("null ELF syscall frame"); } let frame = unsafe { &mut *frame }; let esr: u64; let root: u64; unsafe { asm!("mrs {value}, ESR_EL1", value = out(reg) esr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, TTBR0_EL1", value = out(reg) root, options(nomem, nostack, preserves_flags)); } let count = unsafe { read_volatile(core::ptr::addr_of!(CALLS)) }; if count > 1 || frame.vector_slot != 8 || (esr >> 26) & 0x3f != 0x15 || esr & 0xffff != 0x42 || frame.spsr & 15 != 0 || frame.sp_el0 != unsafe { read_volatile(core::ptr::addr_of!(INITIAL_SP)) } || root != unsafe { read_volatile(core::ptr::addr_of!(ROOT)) } || unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) } != GUARD || frame.x[7] != syscall::VERSION || frame.elr != mmu::USER_CODE_VA + if count == 0 { 24 } else { 48 } { fail("ELF versioned syscall frame"); } if count == 0 { if frame.x[8] != syscall::SERIAL_WRITE || frame.x[0] != mmu::USER_DATA_VA || frame.x[1] != 5 { fail("ELF SERIAL_WRITE arguments"); } let pages = user_copy::UserPages { code_frame: unsafe { read_volatile(core::ptr::addr_of!(CODE_FRAME)) }, data_frame: unsafe { read_volatile(core::ptr::addr_of!(DATA_FRAME)) }, stack_frame: unsafe { read_volatile(core::ptr::addr_of!(STACK_FRAME)) }, }; let mut scratch = [0u8; 5]; if unsafe { pages.copy_from(frame.x[0], &mut scratch) } != Ok(5) { fail("ELF user buffer copy"); } let text = user_copy::serial_text(&scratch).unwrap_or_else(|_| fail("ELF serial text")); if text != if cfg!(feature = "turkfs-init-test") { "INIT!" } else { "HELLO" } { fail("ELF hello payload"); } write_str("TURKOS/ELF-HELLO-CALL: INDEX=0x"); write_hex(count); write_str(" VERSION=0x"); write_hex(frame.x[7] as usize); write_str(" NUMBER=0x"); write_hex(frame.x[8] as usize); write_str(" PTR=0x"); write_hex(frame.x[0] as usize); write_str(" LEN=0x"); write_hex(frame.x[1] as usize); write_str(" ESR=0x"); write_hex(esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_str("\r\nTURKOS/ELF-HELLO-TEXT: "); write_line(text); frame.x[0] = 5; unsafe { write_volatile(core::ptr::addr_of_mut!(CALLS), 1) }; write_str("TURKOS/ELF-HELLO-RET: X0=0x"); write_hex(frame.x[0] as usize); write_str(" NEXT=0x"); write_hex(frame.elr as usize); write_str("\r\n"); 1 } else { if frame.x[8] != syscall::EXIT || (!cfg!(feature = "turkfs-init-test") && frame.x[0] != 0) { fail("ELF hello exit status"); } write_str("TURKOS/ELF-HELLO-EXIT: INDEX=0x"); write_hex(count); write_str(" NUMBER=0x"); write_hex(frame.x[8] as usize); write_str(" CODE=0x"); write_hex(frame.x[0] as usize); write_str(" ESR=0x"); write_hex(esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_str("\r\n"); if cfg!(feature = "turkfs-init-test") { #[cfg(feature = "turkfs-init-test")] retire_init(frame.x[0]); write_str("TURKOS/INIT-EXIT-POLICY: CODE=0x"); write_hex(frame.x[0] as usize); match init_exit::classify(frame.x[0]) { init_exit::ExitPolicy::Halt => { write_line(" ACTION=HALT WAIT=WFE"); write_line("TURKOS/INIT-EL0: PASS"); } init_exit::ExitPolicy::FatalHalt => { write_line(" ACTION=FATAL-HALT WAIT=WFE"); write_line("TURKOS/INIT-EL0: FAIL"); } } } else { write_line("TURKOS/ELF-HELLO: PASS"); } park() } }