use turkos_os_exceptions::exceptions::{ Controller, EntryToken, Frame, Kind, Origin, ReturnAuthority, Stack, User, UserReturn, authorize_user_return, }; pub const ABI_VERSION: u64 = 1; pub const CALL_VERSION: u64 = 0; pub const CALL_ARGUMENTS: u64 = 1; pub const CALL_WRITE: u64 = 2; pub const CALL_PRINCIPAL_REQUEST: u64 = 3; pub const CALL_INHERIT_REQUEST: u64 = 4; pub const CALL_INSPECT_REQUEST: u64 = 5; pub const CALL_QUOTA_REQUEST: u64 = 6; pub const CALL_EXIT: u64 = 7; pub const CALL_ROLE: u64 = 8; pub const CALL_RUN_CHILD: u64 = 9; pub const CALL_WAIT_AGAIN: u64 = 10; pub const MAX_WRITE: usize = 256; pub const SVC_ESR: u64 = (0x15 << 26) | (1 << 25); #[repr(u16)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Errno { NoEnt = 2, Io = 5, Again = 11, BadFd = 9, Child = 10, NoMem = 12, Access = 13, Fault = 14, Pipe = 32, Invalid = 22, NoSys = 38, Overflow = 75, MsgSize = 90, } impl Errno { fn from_code(code: u64) -> Option { Some(match code { 2 => Self::NoEnt, 5 => Self::Io, 11 => Self::Again, 9 => Self::BadFd, 10 => Self::Child, 12 => Self::NoMem, 13 => Self::Access, 14 => Self::Fault, 32 => Self::Pipe, 22 => Self::Invalid, 38 => Self::NoSys, 75 => Self::Overflow, 90 => Self::MsgSize, _ => return None, }) } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DecodeError { NegativeSuccess, UnknownError, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Success(u64); impl Success { pub fn new(value: u64) -> Result { if value > i64::MAX as u64 { return Err(DecodeError::NegativeSuccess); } Ok(Self(value)) } pub const fn value(self) -> u64 { self.0 } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum AbiResult { Ok(Success), Err(Errno), } impl AbiResult { pub const fn encode(self) -> u64 { match self { Self::Ok(value) => value.0, Self::Err(error) => (-(error as i64)) as u64, } } pub fn decode(register: u64) -> Result { if register <= i64::MAX as u64 { return Ok(Self::Ok(Success(register))); } let code = register.wrapping_neg(); if !(1..=4095).contains(&code) { return Err(DecodeError::NegativeSuccess); } Errno::from_code(code) .map(Self::Err) .ok_or(DecodeError::UnknownError) } pub fn length(self) -> Result { match self { Self::Ok(n) => usize::try_from(n.0).map_err(|_| Errno::Overflow), Self::Err(e) => Err(e), } } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Width { U64, U32, I32, } pub const ARGUMENT_WIDTHS: [Width; 6] = [ Width::U64, Width::U32, Width::I32, Width::U64, Width::U32, Width::I32, ]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Arguments([u64; 6]); impl Arguments { pub fn normalize(raw: [u64; 6], widths: [Width; 6]) -> Self { Self(core::array::from_fn(|i| match widths[i] { Width::U64 => raw[i], Width::U32 => raw[i] as u32 as u64, Width::I32 => raw[i] as i32 as i64 as u64, })) } pub const fn values(self) -> [u64; 6] { self.0 } /// A pure ABI V1 diagnostic service consumes all six normalized arguments. /// Its success is kept outside the reserved negative-error range. pub fn checksum(self) -> Success { let mut value = 0u64; for (i, word) in self.0.iter().enumerate() { value ^= word.rotate_left(i as u32 * 9); } Success(value & i64::MAX as u64) } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Write { fd: u32, address: u64, bytes: usize, } impl Write { pub const fn fd(self) -> u32 { self.fd } pub const fn address(self) -> u64 { self.address } pub const fn bytes(self) -> usize { self.bytes } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Call { Version, Arguments(Arguments), Write(Write), PrincipalRequest(u64), InheritRequest(u64), InspectRequest(u64), QuotaRequest(u64), Exit(u64), Role, RunChild(u64), WaitAgain(u64), } pub fn decode_call(frame: &Frame) -> Result { if frame.x[7] != ABI_VERSION { return Err(Errno::NoSys); } match frame.x[8] { CALL_VERSION => Ok(Call::Version), CALL_ARGUMENTS => { let raw = core::array::from_fn(|i| frame.x[i]); let normalized = Arguments::normalize(raw, ARGUMENT_WIDTHS); if raw != normalized.values() { return Err(Errno::Invalid); } Ok(Call::Arguments(normalized)) } CALL_WRITE => { let bytes = usize::try_from(frame.x[2]).map_err(|_| Errno::Overflow)?; if bytes > MAX_WRITE { return Err(Errno::Invalid); } frame.x[1].checked_add(frame.x[2]).ok_or(Errno::Fault)?; Ok(Call::Write(Write { fd: frame.x[0] as u32, address: frame.x[1], bytes, })) } CALL_PRINCIPAL_REQUEST => Ok(Call::PrincipalRequest(frame.x[0])), CALL_INHERIT_REQUEST => Ok(Call::InheritRequest(frame.x[0])), CALL_INSPECT_REQUEST => Ok(Call::InspectRequest(frame.x[0])), CALL_QUOTA_REQUEST => Ok(Call::QuotaRequest(frame.x[0])), CALL_EXIT => Ok(Call::Exit(frame.x[0])), CALL_ROLE => Ok(Call::Role), CALL_RUN_CHILD => Ok(Call::RunChild(frame.x[0])), CALL_WAIT_AGAIN => Ok(Call::WaitAgain(frame.x[0])), _ => Err(Errno::NoSys), } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Execution { pub cpu: u32, pub current_el: u64, pub irq_masked: bool, pub in_irq: bool, pub kernel_stack: Stack, pub entry_sequence: u64, pub user: User, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { Configuration, Context, Ingress, Replay, Exhausted, Busy, Return, Backend, Result, Quarantined, } /// The exclusive adapter owns the actual current process/address-space and /// descriptor/copy services. A frame snapshot alone is never return authority. /// /// # Safety /// `execution` must observe the real CPU, masked SVC context, retained process, /// stack and a nonrepeating actual ingress sequence. `live_frame` binds every /// byte of the provided frame/address to that current CPU entry before any /// owner access. ReturnAuthority must validate retained handles, current root, /// RX continuation and RW stack. Preparation checks descriptor authority and /// copies the WHOLE range through retained Normal RAM pins without a user VA /// dereference or external output. Its errors have no effects or lost owners. /// A Prepared exclusively retains the copied bytes and write authority. /// `commit_write` may publish only those bytes; each error precedes any output. /// `release_write` consumes the actual Prepared only on success, returning it /// on failure. No callback may fabricate successful owner retirement. pub unsafe trait Backend: ReturnAuthority { type Prepared; fn execution(&self) -> Execution; fn live_frame(&self, frame: &Frame, address: u64) -> bool; fn prepare_write(&mut self, user: User, request: Write) -> Result; fn commit_write(&mut self, prepared: &mut Self::Prepared) -> Result; fn release_write(&mut self, prepared: Self::Prepared) -> Result<(), Self::Prepared>; } pub struct Outcome { result: AbiResult, return_to: UserReturn, } impl Outcome { pub const fn result(&self) -> AbiResult { self.result } pub fn into_return(self) -> UserReturn { self.return_to } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Observation { pub identity: u64, pub last_entry: u64, pub completed: u64, pub ingress_depth: usize, pub retained_prepared: bool, pub quarantined: bool, } pub struct Dispatcher { identity: u64, cpu: u32, stack: Stack, backend: Option, controller: Controller<1>, entry: Option, prepared: Option, last_entry: u64, completed: u64, quarantined: bool, } impl Drop for Dispatcher { fn drop(&mut self) { if let Some(p) = self.prepared.take() { core::mem::forget(p); } if let Some(b) = self.backend.take() { core::mem::forget(b); } } } #[cfg(not(target_os = "none"))] fn identity() -> Result { use core::sync::atomic::{AtomicU64, Ordering}; static NEXT: AtomicU64 = AtomicU64::new(0); NEXT.try_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_add(1)) .map(|n| n + 1) .map_err(|_| Error::Exhausted) } #[cfg(target_os = "none")] fn identity() -> Result { unsafe extern "C" { fn __os_k1140_syscall_identity() -> u64; } let id = unsafe { __os_k1140_syscall_identity() }; if id == 0 { Err(Error::Exhausted) } else { Ok(id) } } impl Dispatcher { pub fn new(cpu: u32, stack: Stack, backend: B) -> Result { let e = backend.execution(); if e.cpu != cpu || e.current_el != 4 || e.in_irq || !e.irq_masked || e.kernel_stack != stack { return Err((Error::Context, backend)); } let identity = match identity() { Ok(id) => id, Err(e) => return Err((e, backend)), }; let controller = match Controller::new(identity, stack) { Ok(c) => c, Err(_) => return Err((Error::Configuration, backend)), }; Ok(Self { identity, cpu, stack, backend: Some(backend), controller, entry: None, prepared: None, last_entry: 0, completed: 0, quarantined: false, }) } fn backend(&self) -> &B { self.backend.as_ref().unwrap() } fn gate(&self) -> Result { let e = self.backend().execution(); if e.cpu != self.cpu || e.current_el != 4 || e.in_irq || !e.irq_masked || e.kernel_stack != self.stack { return Err(Error::Context); } Ok(e) } fn retain(&mut self, error: Error) -> Result { self.quarantined = true; Err(error) } fn unchanged(&mut self, before: Execution) -> Result<(), Error> { if self.backend().execution() != before { return self.retain(Error::Context); } Ok(()) } pub fn observation(&self) -> Observation { Observation { identity: self.identity, last_entry: self.last_entry, completed: self.completed, ingress_depth: self.controller.snapshot().depth, retained_prepared: self.prepared.is_some(), quarantined: self.quarantined, } } pub fn dispatch(&mut self, frame: &mut Frame, address: u64) -> Result { let before = self.gate()?; if self.quarantined { return Err(Error::Quarantined); } if self.entry.is_some() || self.prepared.is_some() { return Err(Error::Busy); } if before.entry_sequence == 0 || before.entry_sequence <= self.last_entry { return Err(Error::Replay); } let completed = self.completed.checked_add(1).ok_or(Error::Exhausted)?; let vector = frame.validate_header().map_err(|_| Error::Ingress)?; if vector.origin() != Origin::LowerAarch64 || vector.kind() != Kind::Sync || frame.esr != SVC_ESR { return Err(Error::Ingress); } if !self.backend().live_frame(frame, address) { return Err(Error::Ingress); } // Before any service: real source-bound return root, code and stack. authorize_user_return(frame, address, self.stack, before.user, self.backend()) .map_err(|_| Error::Return)?; self.unchanged(before)?; let entry = self .controller .enter(frame, address) .map_err(|_| Error::Ingress)?; self.entry = Some(entry); self.last_entry = before.entry_sequence; let result = match decode_call(frame) { Err(error) => AbiResult::Err(error), Ok(Call::Version) => AbiResult::Ok(Success(ABI_VERSION)), Ok(Call::Arguments(arguments)) => AbiResult::Ok(arguments.checksum()), Ok(Call::Write(request)) => self.write(before, request, frame, address)?, Ok(Call::PrincipalRequest(_)) => AbiResult::Err(Errno::Access), Ok(Call::InheritRequest(_)) => AbiResult::Err(Errno::Access), Ok(Call::InspectRequest(_)) => AbiResult::Err(Errno::Access), Ok(Call::QuotaRequest(_)) => AbiResult::Err(Errno::Again), Ok(Call::Exit(_)) => AbiResult::Err(Errno::NoSys), Ok(Call::Role) => AbiResult::Err(Errno::NoSys), Ok(Call::RunChild(_)) => AbiResult::Err(Errno::NoSys), Ok(Call::WaitAgain(_)) => AbiResult::Err(Errno::NoSys), }; self.unchanged(before)?; let return_to = match authorize_user_return(frame, address, self.stack, before.user, self.backend()) { Ok(r) => r, Err(_) => return self.retain(Error::Return), }; self.unchanged(before)?; if self.controller.leave(self.entry.as_mut().unwrap()).is_err() { return self.retain(Error::Return); } self.entry = None; self.completed = completed; // No recoverable failure follows the sole public result write. frame.x[0] = result.encode(); Ok(Outcome { result, return_to }) } fn write( &mut self, before: Execution, request: Write, frame: &Frame, address: u64, ) -> Result { let prepared = self .backend .as_mut() .unwrap() .prepare_write(before.user, request); match prepared { Ok(p) => self.prepared = Some(p), Err(error) => { self.unchanged(before)?; return Ok(AbiResult::Err(error)); } } self.unchanged(before)?; if authorize_user_return(frame, address, self.stack, before.user, self.backend()).is_err() { return self.retain(Error::Return); } self.unchanged(before)?; let commit = self .backend .as_mut() .unwrap() .commit_write(self.prepared.as_mut().unwrap()); self.unchanged(before)?; let result = match commit { Ok(bytes) if bytes == request.bytes as u64 => AbiResult::Ok(Success(bytes)), Ok(_) => return self.retain(Error::Result), Err(error) => AbiResult::Err(error), }; if authorize_user_return(frame, address, self.stack, before.user, self.backend()).is_err() { return self.retain(Error::Return); } self.unchanged(before)?; let p = self.prepared.take().unwrap(); if let Err(p) = self.backend.as_mut().unwrap().release_write(p) { self.prepared = Some(p); return self.retain(Error::Backend); } self.unchanged(before)?; Ok(result) } // Failure returns the complete actual owner; allocating a box is not an // available operation on this no_std resource retirement boundary. #[allow(clippy::result_large_err)] pub fn into_backend(mut self) -> Result { if self.gate().is_err() || self.quarantined || self.entry.is_some() || self.prepared.is_some() || self.controller.snapshot().depth != 0 { return Err(self); } Ok(self.backend.take().unwrap()) } #[cfg(all(feature = "host-fixture", not(target_os = "none")))] pub fn fixture_backend(&mut self) -> &mut B { self.backend.as_mut().unwrap() } #[cfg(all(feature = "host-fixture", not(target_os = "none")))] pub fn fixture_prepared(&self) -> Option<&B::Prepared> { self.prepared.as_ref() } } #[cfg(test)] mod private_tests { use super::*; use turkos_os_exceptions::exceptions::{Error as ReturnError, FRAME_BYTES}; const LOW: u64 = 0x4000_0000; const HIGH: u64 = LOW + 65536; const ADDRESS: u64 = HIGH - 16 - FRAME_BYTES as u64; struct Fixture { e: Execution, f: Frame, } impl ReturnAuthority for Fixture { fn owner_root(&self, u: User) -> Result<(), ReturnError> { if u == self.e.user { Ok(()) } else { Err(ReturnError::Owner) } } fn executable(&self, _: User, entry: u64) -> Result<(), ReturnError> { if entry == self.f.elr { Ok(()) } else { Err(ReturnError::Entry) } } fn writable_stack(&self, _: User, sp: u64) -> Result<(), ReturnError> { if sp == self.f.sp_el0 { Ok(()) } else { Err(ReturnError::Stack) } } } // SAFETY: pure host values for counter/retained-controller boundary tests; // all write services panic so they cannot fabricate resource or IO proof. unsafe impl Backend for Fixture { type Prepared = (); fn execution(&self) -> Execution { self.e } fn live_frame(&self, f: &Frame, address: u64) -> bool { f == &self.f && address == ADDRESS } fn prepare_write(&mut self, _: User, _: Write) -> Result<(), Errno> { panic!("unexpected service") } fn commit_write(&mut self, _: &mut ()) -> Result { panic!("unexpected service") } fn release_write(&mut self, _: ()) -> Result<(), ()> { panic!("unexpected service") } } fn dispatcher() -> Dispatcher { let stack = Stack::new(LOW, HIGH).unwrap(); let mut f = Frame { vector_slot: 8, interrupted_sp: ADDRESS + FRAME_BYTES as u64, sp_el0: 0x8000_4000, elr: 0x8000_0040, spsr: 0x3c0, esr: SVC_ESR, ..Frame::EMPTY }; f.x[7] = 1; let e = Execution { cpu: 0, current_el: 4, irq_masked: true, in_irq: false, kernel_stack: stack, entry_sequence: 1, user: User { owner: 7, generation: 3, root: 0x400b_0000, asid: 9, }, }; Dispatcher::new(0, stack, Fixture { e, f }).unwrap_or_else(|_| panic!("fixture")) } #[test] fn completed_counter_exhaustion_precedes_ingress_or_result_mutation() { let mut d = dispatcher(); d.completed = u64::MAX; let mut f = d.backend().f; let before = f; let o = d.observation(); assert!(matches!(d.dispatch(&mut f, ADDRESS), Err(Error::Exhausted))); assert_eq!(f, before); assert_eq!(d.observation(), o); } #[test] fn maximum_actual_entry_sequence_succeeds_once_then_wrap_or_replay_refuses() { let mut d = dispatcher(); d.last_entry = u64::MAX - 1; d.backend.as_mut().unwrap().e.entry_sequence = u64::MAX; let mut f = d.backend().f; assert_eq!( d.dispatch(&mut f, ADDRESS).unwrap().result(), AbiResult::Ok(Success(1)) ); let before = d.observation(); assert!(matches!(d.dispatch(&mut f, ADDRESS), Err(Error::Replay))); d.backend.as_mut().unwrap().e.entry_sequence = 0; assert!(matches!(d.dispatch(&mut f, ADDRESS), Err(Error::Replay))); assert_eq!(d.observation(), before); } #[test] fn retained_actual_controller_entry_blocks_new_ingress_and_backend_extraction() { let mut d = dispatcher(); let mut f = d.backend().f; let token = d.controller.enter(&f, ADDRESS).unwrap(); d.entry = Some(token); let before = d.observation(); assert!(matches!(d.dispatch(&mut f, ADDRESS), Err(Error::Busy))); assert_eq!(d.observation(), before); assert!(d.into_backend().is_err()); } #[test] fn hidden_live_controller_entry_cannot_be_released_by_cleared_lookup() { let mut d = dispatcher(); let f = d.backend().f; let _token = d.controller.enter(&f, ADDRESS).unwrap(); assert!(d.entry.is_none()); assert_eq!(d.observation().ingress_depth, 1); assert!(d.into_backend().is_err()); } }