// Mode 22 executes the trusted K1165 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fifth SVC. static mut OS_K1165_EL0_ERROR: Option<&'static str> = None; static mut OS_K1165_RIGHTS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_KERNEL_FRAME: u64 = 0; struct OsK1165El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, rights: *mut turkos_kernel::runtime::process_rights::Registry, } #[inline(never)] fn os_k1165_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1165_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1165_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1165_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1165_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1165_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1165_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1165-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1165-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1165El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1165-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1165-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1165-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [460u64, 460, 460, 356, 380] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1165-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_PRINCIPAL_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::PrincipalRequest(0)) || episode.output_bytes != 16 { park(); } let rights = unsafe { &*episode.rights }; let before_rights = rights.snapshot(); let before_table = table.snapshot(); if rights.request_user_change(episode.pid, 0) != Err(os_k1165_rights::Error::Access) || rights.snapshot() != before_rights || table.snapshot() != before_table { park(); } frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 4; write_line("TURKOS/OS/K1165-TRACE: PRINCIPAL-ZERO-DENIED"); write_line("TURKOS/OS/K1165-TRACE: RETURN"); return 0; } if episode.calls == 4 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } if unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 4 || unsafe { read_volatile((launch.data_frame + 40) as *const u64) } != 1 { park(); } episode.calls = 5; write_line("TURKOS/OS/K1165-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1165-A\n" } else { b"K1165-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1165-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1165_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1165-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } use turkos_kernel::runtime::process_guest as guest; let mask = 0x0000_ffff_ffff_f000u64; let l1 = unsafe { read_volatile((launch.user_root as *const u64) .add(((guest::CODE >> 30) & 511) as usize)) }; if l1 & 3 != 3 { return Err("entry-l1"); } let l2 = unsafe { read_volatile(((l1 & mask) as *const u64) .add(((guest::CODE >> 21) & 511) as usize)) }; if l2 & 3 != 3 { return Err("entry-l2"); } let l3 = l2 & mask; let leaf = |va: u64| unsafe { read_volatile((l3 as *const u64) .add(((va >> 12) & 511) as usize)) }; let code = leaf(guest::CODE); let stack = leaf(guest::STACK); let access = 3u64 | (1 << 6) | (1 << 10) | (1 << 11) | (1 << 53); if code & mask != launch.code_frame || code & access != access || code & (1 << 7) == 0 || code & (1 << 54) != 0 || stack & mask != launch.stack_frame || stack & access != access || stack & (1 << 7) != 0 || stack & (1 << 54) == 0 || leaf(guest::DATA + guest::PAGE) != 0 || launch.user_pc != guest::CODE || launch.user_sp != guest::STACK_TOP - 512 || launch.user_spsr != 0x3c0 { return Err("entry-rx-stack-rw-spsr"); } write_line("TURKOS/OS/K1165-TRACE: ENTRY-RX-STACK-RW-SPSR-EL0"); let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1165El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar, rights: unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_RIGHTS)) } as *mut _}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1165_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1165_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_ACTIVE), (&mut episode as *mut OsK1165El0Episode) as u64); } unsafe extern "C" { fn __os_k1165_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1165_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1165-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1165_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_KERNEL_FRAME), 0); } if episode.calls != 5 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1165_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1166, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, OS_K1165_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1165-TRACE: TRUNCATED-HEADER-REJECTED"); let long_a = [b'A'; 63]; let long_b = [b'B'; 63]; let long_c = [b'C'; 63]; let long_d = [b'D'; 64]; let max_argv: [&[u8]; 2] = [&long_a, &long_b]; let max_env: [&[u8]; 2] = [&long_c, &long_c]; let over_env: [&[u8]; 2] = [&long_c, &long_d]; let bound_image = unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, OS_K1165_USER_ELF.len()) }.map_err(|_| "argv-bound-image")?; let bound = Recipe::new(bound_image, &max_argv, &max_env) .map_err(|_| "argv-exact-limit")?; if bound.start().argc != 2 || bound.start().envc != 2 { return Err("argv-exact-count"); } let over_image = unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, OS_K1165_USER_ELF.len()) }.map_err(|_| "argv-over-image")?; if !matches!(Recipe::new(over_image, &max_argv, &over_env), Err(ImageError::Arguments)) { return Err("argv-over-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "argv-after-gate")? .map_err(|_| "argv-after-inventory")? != before { return Err("argv-resource-change"); } write_line("TURKOS/OS/K1165-TRACE: TEXT-LIMIT-PLUS-ONE-REJECTED"); let bad_image = unsafe { TrustedImage::from_kernel(&OS_K1165_BAD_ENTRY_ELF, OS_K1165_BAD_ENTRY_ELF.len()) }.map_err(|_| "bad-entry-trust")?; if !matches!(Recipe::new(bad_image, &[b"init"], &[]), Err(ImageError::Entry)) { return Err("kernel-entry-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "entry-after-gate")? .map_err(|_| "entry-after-inventory")? != before { return Err("entry-resource-change"); } write_line("TURKOS/OS/K1165-TRACE: KERNEL-ENTRY-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, OS_K1165_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let argv: [&[u8]; 2] = [b"init", b"alpha"]; let env: [&[u8]; 2] = [b"MODE=1", b"LANG=tr"]; let recipe = Recipe::new(trusted, &argv, &env) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; for stage in 0u8..11 { let fail_ptr = os_k1160_install_fail_table(&gate, &authority, &store, &image, config, stage)?; let fail_table = unsafe { &mut *fail_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let ticket = fail_table.begin(request).map_err(|_| "unwind-begin")?; let (error, ticket) = fail_table.prepare(ticket).err().ok_or("unwind-accepted")?; if error != process::Error::Resource || fail_table.snapshot().published != 0 || fail_table.snapshot().quarantined != 0 { return Err("unwind-failure-class"); } fail_table.abort(ticket).map_err(|_| "unwind-abort")?; if fail_table.snapshot() != process::Snapshot::default() { return Err("unwind-table-state"); } let backend = os_k1105_finish_table(fail_ptr)?; if backend.inventory().map_err(|_| "unwind-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("unwind-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "unwind-after-gate")? .map_err(|_| "unwind-after-inventory")? != before { return Err("unwind-physical-leak"); } write_str("TURKOS/OS/K1165-TRACE: UNWIND-STAGE="); write_hex(stage as usize); write_line(" PASS"); } let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1165-TRACE: PUBLISHED"); let mut identities = os_k1165_rights::Registry::::new(); let first = identities.bind_kernel(pid).map_err(|_| "rights-first-bind")?; if first.id() != 1 || identities.principal(pid) != Ok(first) { return Err("rights-first-principal"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_RIGHTS), (&mut identities as *mut _) as u64); } write_line("TURKOS/OS/K1165-TRACE: FIRST-PRINCIPAL-KERNEL-ASSIGNED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1165_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1165_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_EL0_ERROR)) } .unwrap_or("el0-process-body")); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_RIGHTS), 0); } if identities.principal(pid) != Ok(first) { return Err("rights-first-changed"); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) { return Err("el0-reap"); } identities.retire_kernel(pid).map_err(|_| "rights-first-retire")?; let second_ticket = table.begin(request).map_err(|_| "rights-second-begin")?; let second_ticket = table.prepare(second_ticket).map_err(|_| "rights-second-prepare")?; let second_pid = table.publish(second_ticket).map_err(|_| "rights-second-publish")?; let second = identities.bind_kernel(second_pid).map_err(|_| "rights-second-bind")?; if second.id() != 2 || second == first || identities.principal(second_pid) != Ok(second) || identities.principal(pid) != Err(os_k1165_rights::Error::Unknown) { return Err("rights-second-principal"); } write_line("TURKOS/OS/K1165-TRACE: SECOND-PRINCIPAL-KERNEL-ASSIGNED"); table.terminate_child(second_pid, 0).map_err(|_| "rights-second-terminate")?; if table.wait_child(second_pid) != Ok(0) { return Err("rights-second-reap"); } identities.retire_kernel(second_pid).map_err(|_| "rights-second-retire")?; if table.snapshot() != process::Snapshot::default() || identities.snapshot() != (3, [None, None]) { return Err("rights-final-state"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1165-EL0: PRINCIPALS=2 ZERO=EACCES FIVE-SVC=OK ERET=4 OWNER=RELEASED"); write_line("TURKOS/OS/K1165: PASS"); Ok(()) }