//! Live two-owner EL0 descriptor experiment over the shared task launcher. //! The owner comes from Scheduler::running, never from user-provided x0. use super::*; use turkos_kernel::handle::{self, Resource, Table}; use turkos_kernel::syscall::{self, Call, Outcome}; use turkos_kernel::task::State; #[cfg(feature = "user-handle-transfer-test")] use turkos_kernel::task::{Scheduler, TaskId}; #[cfg(feature = "user-handle-transfer-test")] use turkos_kernel::{ipc::ChannelTable, user_copy::UserPages}; static mut HANDLES: Table = Table::new(); static mut CALLS: usize = 0; #[cfg(feature = "user-handle-transfer-test")] static mut CHANNELS: ChannelTable = ChannelTable::new(); #[cfg(feature = "user-handle-transfer-test")] static mut CHANNEL_ID: u64 = 0; #[cfg(feature = "user-handle-transfer-test")] pub(super) fn init_channel(producer: TaskId, consumer: TaskId) { let channels = unsafe { &mut *core::ptr::addr_of_mut!(CHANNELS) }; let id = channels .create(producer, consumer) .unwrap_or_else(|_| fail("channel create")); if id != 1 || channels.live_channels() != 1 { fail("channel ID or count"); } unsafe { write_volatile(core::ptr::addr_of_mut!(CHANNEL_ID), id) }; } #[cfg(feature = "user-handle-transfer-test")] fn pages_for_task(index: usize) -> UserPages { let spaces = unsafe { &*core::ptr::addr_of!(user_task::SPACES) }; let space = spaces[index] .as_ref() .unwrap_or_else(|| fail("private pages missing")); UserPages { code_frame: space.code_frame(), data_frame: space.data_frame(), stack_frame: space.stack_frame(), } } fn fail(reason: &str) -> ! { write_str("TURKOS/USER-HANDLE-ERROR: "); write_line(reason); park() } fn task_token(index: usize) -> u64 { if index == 0 { user_task::TOKEN_A } else { user_task::TOKEN_B } } #[cfg(feature = "user-handle-test")] fn expected(index: usize) -> (usize, u64, Option, Option, usize) { use syscall::{EXIT, HANDLE_CLOSE, HANDLE_OPEN, HANDLE_QUERY, YIELD}; match index { 0 => (0, HANDLE_OPEN, Some(1), Some(1), 1), 1 => (0, YIELD, None, Some(0), 1), 2 => (1, HANDLE_QUERY, Some(1), Some(syscall::EBADF), 1), 3 => (1, HANDLE_OPEN, Some(2), Some(2), 2), 4 => (1, HANDLE_OPEN, Some(1), Some(syscall::EMFILE), 2), 5 => (1, YIELD, None, Some(0), 2), 6 => (0, HANDLE_QUERY, Some(1), Some(1), 2), 7 => (0, HANDLE_QUERY, Some(2), Some(syscall::EBADF), 2), 8 => (0, HANDLE_CLOSE, Some(1), Some(0), 1), 9 => (0, EXIT, Some(user_task::TOKEN_A), None, 1), 10 => (1, HANDLE_QUERY, Some(1), Some(syscall::EBADF), 1), 11 => (1, HANDLE_QUERY, Some(2), Some(2), 1), 12 => (1, HANDLE_CLOSE, Some(2), Some(0), 0), 13 => (1, EXIT, Some(user_task::TOKEN_B), None, 0), _ => fail("extra syscall"), } } #[cfg(feature = "user-handle-transfer-test")] fn expected(index: usize) -> (usize, u64, Option, Option, usize) { use syscall::{ EXIT, HANDLE_CLOSE, HANDLE_OPEN, HANDLE_QUERY, HANDLE_TRANSFER, IPC_RECEIVE, IPC_SEND, YIELD, }; match index { 0 => (0, HANDLE_OPEN, Some(3), Some(1), 1), 1 => (0, IPC_SEND, Some(1), Some(4), 1), 2 => (0, YIELD, None, Some(0), 1), 3 => (1, HANDLE_QUERY, Some(1), Some(syscall::EBADF), 1), 4 => (1, HANDLE_OPEN, Some(2), Some(2), 2), 5 => (1, HANDLE_TRANSFER, Some(1), Some(syscall::EBADF), 2), 6 => (1, YIELD, None, Some(0), 2), 7 => (0, HANDLE_TRANSFER, Some(1), Some(syscall::EINVAL), 2), 8 => (0, HANDLE_TRANSFER, Some(1), Some(3), 2), 9 => (0, HANDLE_QUERY, Some(1), Some(syscall::EBADF), 2), 10 => (0, HANDLE_QUERY, Some(3), Some(syscall::EBADF), 2), 11 => (0, EXIT, Some(user_task::TOKEN_A), None, 2), 12 => (1, HANDLE_QUERY, Some(3), Some(3), 2), 13 => (1, HANDLE_QUERY, Some(1), Some(syscall::EBADF), 2), 14 => (1, IPC_RECEIVE, Some(3), Some(4), 2), 15 => (1, HANDLE_QUERY, Some(2), Some(2), 2), 16 => (1, HANDLE_CLOSE, Some(3), Some(0), 1), 17 => (1, HANDLE_CLOSE, Some(2), Some(0), 0), 18 => (1, EXIT, Some(user_task::TOKEN_B), None, 0), _ => fail("extra transfer syscall"), } } #[cfg(feature = "user-handle-transfer-test")] fn transfer_result( table: &mut Table, owner: TaskId, recipient_token: u64, handle: u64, ids: [Option; 2], scheduler: &Scheduler, ) -> i64 { let target = if recipient_token == user_task::TOKEN_A { 0 } else if recipient_token == user_task::TOKEN_B { 1 } else { return syscall::EINVAL; }; let recipient = ids[target].unwrap_or_else(|| fail("transfer recipient ID missing")); if scheduler.state(recipient) != Ok(State::Ready) { return syscall::EINVAL; } match table.transfer(owner, recipient, handle) { Ok(new_handle) => new_handle as i64, Err(handle::Error::InvalidHandle) => syscall::EBADF, Err(handle::Error::SameOwner) => syscall::EINVAL, Err(handle::Error::IdExhausted) => syscall::EMFILE, Err(handle::Error::Full) => fail("transfer cannot fill another slot"), } } fn handle_result(table: &mut Table, owner: turkos_kernel::task::TaskId, outcome: Outcome) -> i64 { match outcome { Outcome::HandleOpen(kind) => { let Some(resource) = Resource::from_number(kind) else { return syscall::EINVAL; }; match table.open(owner, resource) { Ok(handle) => handle as i64, Err(handle::Error::Full | handle::Error::IdExhausted) => syscall::EMFILE, Err(_) => fail("open returned invalid-handle error"), } } Outcome::HandleQuery(handle) => table .query(owner, handle) .map(|resource| resource.number() as i64) .unwrap_or(syscall::EBADF), Outcome::HandleClose(handle) => table .close(owner, handle) .map(|()| 0) .unwrap_or(syscall::EBADF), _ => fail("non-handle syscall in handle_result"), } } #[unsafe(no_mangle)] pub extern "C" fn kernel_return_probe(frame: *mut irq::IrqContext) -> usize { if frame.is_null() { fail("null context"); } let frame = unsafe { &mut *frame }; let esr: u64; let root: u64; unsafe { asm!("mrs {value}, ESR_EL1", value = out(reg) esr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, TTBR0_EL1", value = out(reg) root, options(nomem, nostack, preserves_flags)); } let count = unsafe { read_volatile(core::ptr::addr_of!(CALLS)) }; let (current, number, argument, expected_return, expected_len) = expected(count); let ids = unsafe { read_volatile(core::ptr::addr_of!(user_task::IDS)) }; let roots = unsafe { read_volatile(core::ptr::addr_of!(user_task::ROOTS)) }; let physical = unsafe { read_volatile(core::ptr::addr_of!(user_task::FRAMES)) }; let owner = ids[current].unwrap_or_else(|| fail("missing task ID")); let scheduler = unsafe { &mut *core::ptr::addr_of_mut!(user_task::SCHEDULER) }; if unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) } != user_task::GUARD || scheduler.running() != Some(owner) || esr != 0x5600_0042 || frame.vector_slot != 8 || frame.spsr & 0xf != 0 || frame.sp_el0 != mmu::USER_STACK_TOP - 16 || frame.elr < mmu::USER_CODE_VA || frame.elr >= mmu::USER_CODE_VA + memory::PAGE_SIZE || frame.elr & 3 != 0 || root != roots[current + 1] || frame.x[19] != task_token(current) || frame.x[20] != mmu::USER_DATA_VA || frame.x[21] != 0x55 || (current == 0 && count >= 1 && frame.x[22] != 1) || (current == 1 && count >= if cfg!(feature = "user-handle-transfer-test") { 5 } else { 4 } && frame.x[22] != 2) || (cfg!(feature = "user-handle-transfer-test") && current == 0 && count >= 9 && frame.x[23] != 3) || user_task::value(physical[current][0]) != task_token(current) || user_task::value(physical[current][1] + memory::PAGE_SIZE - 16) != task_token(current) { fail("EL0 frame, owner, or private page"); } let call = Call { version: frame.x[7], number: frame.x[8], args: frame.x[..6].try_into().unwrap(), }; if call.version != syscall::VERSION || call.number != number || argument.is_some_and(|arg| call.args[0] != arg) || (cfg!(feature = "user-handle-transfer-test") && call.number == syscall::HANDLE_TRANSFER && call.args[1] != match count { 5 => user_task::TOKEN_A, 7 => 0, 8 => user_task::TOKEN_B, _ => fail("unexpected transfer index"), }) || (cfg!(feature = "user-handle-transfer-test") && matches!(call.number, syscall::IPC_SEND | syscall::IPC_RECEIVE) && (call.args[1] != mmu::USER_DATA_VA + 16 || call.args[2] != 4)) { fail("call order or argument"); } let outcome = syscall::dispatch(call); let table = unsafe { &mut *core::ptr::addr_of_mut!(HANDLES) }; let result = if cfg!(feature = "user-handle-transfer-test") && call.number == syscall::HANDLE_OPEN && call.args[0] == 3 { #[cfg(feature = "user-handle-transfer-test")] { let id = unsafe { read_volatile(core::ptr::addr_of!(CHANNEL_ID)) }; if id != 1 { fail("missing channel for handle open"); } Some( table .open(owner, Resource::Channel(id)) .unwrap_or_else(|_| fail("channel handle open")) as i64, ) } #[cfg(not(feature = "user-handle-transfer-test"))] { fail("channel handle open disabled") } } else if cfg!(feature = "user-handle-transfer-test") && matches!(call.number, syscall::IPC_SEND | syscall::IPC_RECEIVE) { #[cfg(feature = "user-handle-transfer-test")] { let id = match table.query(owner, call.args[0]) { Ok(Resource::Channel(id)) => id, _ => fail("IPC capability not owned"), }; let channels = unsafe { &mut *core::ptr::addr_of_mut!(CHANNELS) }; let pages = pages_for_task(current); let value = if call.number == syscall::IPC_SEND { let mut bytes = [0u8; 4]; if unsafe { pages.copy_from(call.args[1], &mut bytes) } != Ok(4) || &bytes != b"ABCD" { fail("IPC user source copy"); } channels .send(id, owner, &bytes) .unwrap_or_else(|_| fail("IPC authorized send")); 4 } else { if pages.resolve(call.args[1], 4, true).is_err() { fail("IPC user destination range"); } let message = channels .receive(id, owner) .unwrap_or_else(|_| fail("IPC authorized receive")); if message.bytes() != b"ABCD" || unsafe { pages.copy_to(call.args[1], message.bytes()) } != Ok(4) { fail("IPC user destination copy"); } 4 }; write_str("TURKOS/USER-HANDLE-IPC: INDEX=0x"); write_hex(count); write_str(" ID="); write_str(if current == 0 { "A" } else { "B" }); write_str(" HANDLE=0x"); write_hex(call.args[0] as usize); write_str(" QUEUED=0x"); write_hex( channels .queued(id) .unwrap_or_else(|_| fail("IPC queue missing")), ); write_str(" DATA=ABCD\r\n"); Some(value) } #[cfg(not(feature = "user-handle-transfer-test"))] { fail("IPC handle gate disabled") } } else if call.number == syscall::HANDLE_TRANSFER && cfg!(feature = "user-handle-transfer-test") { #[cfg(feature = "user-handle-transfer-test")] { Some(transfer_result( table, owner, call.args[1], call.args[0], ids, scheduler, )) } #[cfg(not(feature = "user-handle-transfer-test"))] { fail("transfer not enabled") } } else { match outcome { Outcome::HandleOpen(_) | Outcome::HandleQuery(_) | Outcome::HandleClose(_) => { Some(handle_result(table, owner, outcome)) } Outcome::Yield => Some(0), Outcome::Exit(code) if code == task_token(current) => None, _ => fail("dispatch or exit code"), } }; if result != expected_return || table.len() != expected_len { fail("handle result or table ownership"); } #[cfg(feature = "user-handle-transfer-test")] if call.number == syscall::HANDLE_TRANSFER { write_str("TURKOS/USER-HANDLE-TRANSFER: INDEX=0x"); write_hex(count); write_str(" FROM="); write_str(if current == 0 { "A" } else { "B" }); write_str(" TARGET=0x"); write_hex(call.args[1] as usize); write_str(" OLD=0x"); write_hex(call.args[0] as usize); write_str(" RESULT=0x"); write_hex(result.unwrap() as usize); write_str(" TABLE=0x"); write_hex(table.len()); write_str("\r\n"); } write_str("TURKOS/USER-HANDLE-CALL: INDEX=0x"); write_hex(count); write_str(" ID="); write_str(if current == 0 { "A" } else { "B" }); write_str(" NUMBER=0x"); write_hex(number as usize); write_str(" ARG=0x"); write_hex(call.args[0] as usize); write_str(" RESULT=0x"); write_hex(result.unwrap_or(task_token(current) as i64) as usize); write_str(" TABLE=0x"); write_hex(table.len()); write_str(" ROOT=0x"); write_hex(root as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_str("\r\n"); unsafe { write_volatile(core::ptr::addr_of_mut!(CALLS), count + 1) }; if number != syscall::YIELD && number != syscall::EXIT { frame.x[0] = result.unwrap() as u64; write_str("TURKOS/USER-HANDLE-RET: INDEX=0x"); write_hex(count); write_str(" X0=0x"); write_hex(frame.x[0] as usize); write_str(" NEXT=0x"); write_hex(frame.elr as usize); write_str("\r\n"); return 1; } let next = if number == syscall::YIELD { frame.x[0] = 0; unsafe { core::ptr::write( core::ptr::addr_of_mut!(user_task::SAVED) .cast::() .add(current), *frame, ) }; scheduler .yield_current(false) .unwrap_or_else(|_| fail("yield transition")) } else { scheduler .exit_current(task_token(current) as i32) .unwrap_or_else(|_| fail("exit transition")) }; let next_index = if cfg!(feature = "user-handle-transfer-test") { match count { 2 | 11 => Some(1), 6 => Some(0), 18 => None, _ => fail("unexpected transfer scheduler call"), } } else { match count { 1 | 9 => Some(1), 5 => Some(0), 13 => None, _ => fail("unexpected scheduler call"), } }; if next != next_index.map(|index| ids[index].unwrap()) || scheduler.ready_len() != if count < if cfg!(feature = "user-handle-transfer-test") { 11 } else { 9 } { 1 } else { 0 } { fail("scheduler selected wrong owner"); } write_str("TURKOS/USER-HANDLE-SWITCH: INDEX=0x"); write_hex(count); write_str(" FROM="); write_str(if current == 0 { "A" } else { "B" }); write_str(" TO="); write_str(match next_index { Some(0) => "A", Some(1) => "B", _ => "NONE", }); write_str(" READY=0x"); write_hex(scheduler.ready_len()); write_str("\r\n"); if let Some(next_index) = next_index { if count == if cfg!(feature = "user-handle-transfer-test") { 2 } else { 1 } { let mut fresh = irq::IrqContext::EMPTY; fresh.x[0] = user_task::TOKEN_B; fresh.interrupted_sp = frame.interrupted_sp; fresh.sp_el0 = mmu::USER_STACK_TOP; fresh.elr = mmu::USER_CODE_VA; fresh.spsr = user_task::EL0T_MASKED; fresh.vector_slot = 8; *frame = fresh; } else { *frame = unsafe { core::ptr::read( core::ptr::addr_of!(user_task::SAVED) .cast::() .add(next_index), ) }; } user_task::switch_root(roots[next_index + 1]); return 1; } if scheduler.running().is_some() || scheduler.run_next() != Ok(None) || scheduler.state(ids[0].unwrap()) != Ok(State::Exited { code: user_task::TOKEN_A as i32, }) || scheduler.state(ids[1].unwrap()) != Ok(State::Exited { code: user_task::TOKEN_B as i32, }) || table.len() != 0 || user_task::value(physical[0][0]) != user_task::TOKEN_A || user_task::value(physical[1][0]) != user_task::TOKEN_B || user_task::value(physical[0][1] + memory::PAGE_SIZE - 16) != user_task::TOKEN_A || user_task::value(physical[1][1] + memory::PAGE_SIZE - 16) != user_task::TOKEN_B { fail("exited owner or open handle after final exit"); } #[cfg(feature = "user-handle-transfer-test")] { let channels = unsafe { &mut *core::ptr::addr_of_mut!(CHANNELS) }; let id = unsafe { read_volatile(core::ptr::addr_of!(CHANNEL_ID)) }; if channels.queued(id) != Ok(0) || channels.destroy(id, ids[0].unwrap()) != Ok(()) || channels.live_channels() != 0 { fail("transferred channel cleanup"); } write_line("TURKOS/USER-HANDLE-CHANNEL-CLOSE: ID=0x1 LIVE=0x0"); } user_task::switch_root(roots[0]); let allocator_ptr = unsafe { read_volatile(core::ptr::addr_of!(user_task::ALLOCATOR)) }; if allocator_ptr.is_null() { fail("allocator missing"); } let allocator = unsafe { &mut *allocator_ptr }; let before = allocator.remaining_pages(); for index in 0..2 { let space = unsafe { core::ptr::replace( core::ptr::addr_of_mut!(user_task::SPACES) .cast::>() .add(index), None, ) } .unwrap_or_else(|| fail("owner space missing")); unsafe { space.release_after_tlbi(allocator, roots[0]) } .unwrap_or_else(|_| fail("owner frame release")); } let after = allocator.remaining_pages(); if after != before + 12 || after != unsafe { read_volatile(core::ptr::addr_of!(user_task::INITIAL_FREE)) } || unsafe { read_volatile((roots[0] as *const u64).add(2)) } != 0 { fail("reclaim accounting"); } write_str("TURKOS/USER-HANDLE-RECLAIM: KERNEL=0x"); write_hex(roots[0] as usize); write_str(" RELEASED=0x"); write_hex((after - before) as usize); write_str(" OPEN=0x"); write_hex(table.len()); write_str("\r\n"); write_line(if cfg!(feature = "user-handle-transfer-test") { "TURKOS/USER-HANDLE-TRANSFER: PASS" } else { "TURKOS/USER-HANDLE: PASS" }); 0 }