use core::{array, cell::Cell, mem}; use crate::runtime::address_spaces as spaces; use crate::runtime::frame_metadata as metadata; use crate::runtime::process_table as processes; use crate::runtime::process_threads as threads; pub const PAGE_BYTES: usize = 4096; pub const CODE_VA: u64 = crate::mmu::USER_CODE_VA; pub const DATA_VA: u64 = crate::mmu::USER_DATA_VA; pub const STACK_VA: u64 = crate::mmu::USER_STACK_VA; pub const STACK_TOP: u64 = crate::mmu::USER_STACK_TOP; pub const MAX_ARGUMENTS: usize = 4; pub const MAX_ENVIRONMENT: usize = 4; pub const MAX_STRING_BYTES: usize = 64; pub const MAX_TEXT_BYTES: usize = 256; pub const STACK_AREA: usize = 512; pub const EL0_SPSR: u64 = 0x3c0; pub const RESOURCE_COUNT: usize = 6; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { Image, Length, Arguments, Entry, Context, Resource, Capacity, Backend, Process(processes::Error), } /// Immutable kernel input authority. No safe API accepts a user trust flag. pub struct TrustedImage<'a> { bytes: &'a [u8], } impl<'a> TrustedImage<'a> { /// # Safety /// The prefix is initialized, immutable kernel-owned embedded Image /// storage for the whole `'a` lifetime. trusted_bytes is derived from the /// actual linker/loader extent, never from user input or ELF header fields. pub unsafe fn from_kernel(bytes: &'a [u8], trusted_bytes: usize) -> Result { let bytes = bytes.get(..trusted_bytes).ok_or(Error::Length)?; if bytes.is_empty() { return Err(Error::Length); } Ok(Self { bytes }) } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Start { pub pc: u64, pub sp: u64, pub spsr: u64, pub argc: u64, pub argv: u64, pub envc: u64, pub envp: u64, } pub struct Recipe<'a> { image: TrustedImage<'a>, layout: crate::runtime::process_elf::Layout, stack: [u8; STACK_AREA], start: Start, } impl<'a> Recipe<'a> { /// Complete ELF and text preflight precedes reservation and acquisition. /// Bytes are copied unchanged; NUL, lengths and checked pointer bounds are /// enforced without UTF conversion or normalization. pub fn new(image: TrustedImage<'a>, argv: &[&[u8]], env: &[&[u8]]) -> Result { let layout = crate::runtime::process_elf::parse_layout(image.bytes).map_err(|_| Error::Image)?; let mut code = None; let mut data = None; for segment in layout.segments.iter().take(layout.count).flatten().copied() { if (CODE_VA..CODE_VA + PAGE_BYTES as u64).contains(&segment.virtual_address) { if segment.flags != 5 || code.replace(segment).is_some() { return Err(Error::Image); } } else if (DATA_VA..DATA_VA + PAGE_BYTES as u64).contains(&segment.virtual_address) { if segment.flags != 6 || data.replace(segment).is_some() { return Err(Error::Image); } } else { return Err(Error::Image); } } let code = code.ok_or(Error::Image)?; if layout.entry & 3 != 0 || layout.entry < code.virtual_address || layout.entry.checked_add(4).is_none_or(|end| { code.virtual_address .checked_add(code.file_size) .is_none_or(|limit| end > limit) }) { return Err(Error::Entry); } if argv.len() > MAX_ARGUMENTS || env.len() > MAX_ENVIRONMENT { return Err(Error::Arguments); } let mut text_bytes = 0usize; for text in argv.iter().chain(env.iter()) { if text.len() > MAX_STRING_BYTES || text.contains(&0) { return Err(Error::Arguments); } text_bytes = text_bytes .checked_add(text.len() + 1) .ok_or(Error::Arguments)?; } if text_bytes > MAX_TEXT_BYTES { return Err(Error::Arguments); } let sp = STACK_TOP - STACK_AREA as u64; let argv_address = sp + 32; let envp_address = argv_address + (MAX_ARGUMENTS as u64 + 1) * 8; let mut stack = [0u8; STACK_AREA]; for (index, word) in [ argv.len() as u64, argv_address, env.len() as u64, envp_address, ] .into_iter() .enumerate() { stack[index * 8..index * 8 + 8].copy_from_slice(&word.to_le_bytes()); } let mut cursor = 128usize; for (words, list) in [(32usize, argv), (72usize, env)] { for (index, text) in list.iter().enumerate() { let pointer = sp.checked_add(cursor as u64).ok_or(Error::Arguments)?; let end = pointer .checked_add(text.len() as u64 + 1) .ok_or(Error::Arguments)?; if pointer < STACK_VA || end > STACK_TOP { return Err(Error::Arguments); } let at = words + index * 8; stack[at..at + 8].copy_from_slice(&pointer.to_le_bytes()); stack[cursor..cursor + text.len()].copy_from_slice(text); cursor += text.len() + 1; } } Ok(Self { image, layout, stack, start: Start { pc: layout.entry, sp, spsr: EL0_SPSR, argc: argv.len() as u64, argv: argv_address, envc: env.len() as u64, envp: envp_address, }, }) } pub fn start(&self) -> Start { self.start } pub fn trusted_bytes(&self) -> usize { self.image.bytes.len() } pub fn expected_byte(&self, kind: Kind, offset: usize) -> Result { if offset >= PAGE_BYTES { return Err(Error::Resource); } if kind == Kind::Stack { return Ok(if offset >= PAGE_BYTES - STACK_AREA { self.stack[offset - (PAGE_BYTES - STACK_AREA)] } else { 0 }); } let base = match kind { Kind::Code => CODE_VA, Kind::Data => DATA_VA, _ => return Err(Error::Resource), }; for segment in self .layout .segments .iter() .take(self.layout.count) .flatten() { if segment.virtual_address >= base && segment.virtual_address < base + PAGE_BYTES as u64 { let start = (segment.virtual_address - base) as usize; if offset >= start && offset - start < segment.file_size as usize { return Ok(self.image.bytes[segment.file_offset as usize + offset - start]); } } } Ok(0) } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(usize)] pub enum Kind { Root, Code, Data, Stack, Handles, Thread, } pub const KINDS: [Kind; RESOURCE_COUNT] = [ Kind::Root, Kind::Code, Kind::Data, Kind::Stack, Kind::Handles, Kind::Thread, ]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct TokenObservation { pub kind: Kind, pub owner: u64, pub generation: u64, pub address: u64, pub pages: u64, pub space: Option, pub frame: Option, pub thread: Option, pub state: Option, pub active: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct EntryObservation { pub start: Start, pub code_permissions: u8, pub stack_permissions: u8, pub code_user: bool, pub stack_user: bool, pub published: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Execution { pub process: processes::Execution, /// Actual active TTBR root identity, including its active ASID context. pub root: u64, /// Actual retained kernel stack/guard owner; zero means unproved context. pub kernel_stack_owner: u64, } pub struct AcquireFailure { pub failure: processes::Failure, pub acquired: Option, } pub struct ResourceBag { process: processes::ProcessId, owner: u64, tokens: [Option; RESOURCE_COUNT], installed: bool, } impl ResourceBag { pub fn process(&self) -> processes::ProcessId { self.process } pub fn owner(&self) -> u64 { self.owner } pub fn token(&self, kind: Kind) -> Option<&R::Token> { self.tokens[kind as usize].as_ref() } } impl Drop for ResourceBag { fn drop(&mut self) { for token in &mut self.tokens { if let Some(token) = token.take() { mem::forget(token); } } } } /// # Safety /// Tokens exclusively retain the actual frozen Root/Page/HandleTable/Thread /// owners, are not cloned and never silently free on Drop. Observations read /// those owners; entry/page verification reads actual private storage and /// mappings. Before callbacks do not mutate any resource. After acquisition /// returns the actual acquired token, or retains every uncertain acquisition /// inside this exclusively owned backend. Publication only enqueues the /// fully prepared thread; it never executes ERET or switches a Rust stack. /// detach completes root removal and synchronous TLBI before any token can /// be released. Release failures return the same token. Every backend must /// gate CPU/IRQ and revalidate generation/custody after actual retirement. /// Execution reads the actual closed root and retained stack authority. An /// unknown root or stack cannot be presented as a valid nonzero stack owner. pub unsafe trait ResourceBackend: Sized { type Token; fn execution(&self) -> Execution; fn owner(&mut self, process: processes::ProcessId) -> Result; fn acquire( &mut self, kind: Kind, process: processes::ProcessId, owner: u64, recipe: &Recipe<'_>, ) -> Result>; fn observe_token(&self, token: &Self::Token) -> Result; fn install( &mut self, bag: &ResourceBag, recipe: &Recipe<'_>, ) -> Result<(), processes::Failure>; fn observe_entry(&self, bag: &ResourceBag) -> Result; fn read_page(&self, token: &Self::Token, offset: usize) -> Result; fn publish(&mut self, bag: &ResourceBag) -> Result<(), processes::Failure>; fn exit(&mut self, bag: &ResourceBag, code: i32) -> Result<(), processes::Failure>; fn detach(&mut self, bag: &ResourceBag) -> Result<(), processes::Failure>; fn release(&mut self, token: Self::Token) -> Result<(), (processes::Error, Self::Token)>; } struct Adapter<'a, R: ResourceBackend> { recipe: Recipe<'a>, resources: R, cpu: u16, faulted: Cell, } impl Adapter<'_, R> { fn gate(&self) -> Result { if self.faulted.get() { return Err(processes::Error::Quarantined); } let e = self.resources.execution(); if e.process.cpu != self.cpu || e.process.in_irq || !e.process.irq_masked || e.kernel_stack_owner == 0 || e.root == 0 || e.root & 4095 != 0 { return Err(processes::Error::Context); } Ok(e) } fn stable(&self, before: Execution) -> Result<(), processes::Error> { if self.faulted.get() { return Err(processes::Error::Quarantined); } if self.resources.execution() != before { self.faulted.set(true); return Err(processes::Error::Context); } Ok(()) } fn callback_failure(&self, mut f: processes::Failure, before: Execution) -> processes::Failure { if let Err(e) = self.stable(before) { f.error = e; f.boundary = processes::Boundary::After; } f } fn observation( &self, bag: &ResourceBag, complete: bool, ) -> Result { let before = self.gate()?; let mut spaces = [None; processes::MAX_SPACES]; let mut handles = [None; processes::MAX_HANDLES]; let mut thread = None; let mut state = None; let mut pages = 0u64; let mut observations: [Option; RESOURCE_COUNT] = [None; RESOURCE_COUNT]; for kind in KINDS { let Some(token) = bag.token(kind) else { if complete { return Err(processes::Error::Preparation); } else { continue; } }; let result = self.resources.observe_token(token); self.stable(before)?; let o = result?; if o.kind != kind || o.owner != bag.owner || o.generation == 0 || o.address == 0 || o.address & 4095 != 0 || o.pages == 0 || o.address .checked_add( o.pages .checked_mul(4096) .ok_or(processes::Error::Resource)?, ) .is_none() { return Err(processes::Error::Resource); } for previous in observations.iter().flatten() { if o.address < previous.address + previous.pages * 4096 && previous.address < o.address + o.pages * 4096 { return Err(processes::Error::Resource); } } match kind { Kind::Root => { let h = o.space.ok_or(processes::Error::Resource)?; if h.owner() != bag.owner || h.generation() == 0 || o.frame.is_some() || o.thread.is_some() { return Err(processes::Error::Resource); } spaces[0] = Some(h); if !complete && o.active { return Err(processes::Error::Resource); } } Kind::Code | Kind::Data | Kind::Stack | Kind::Handles => { let h = o.frame.ok_or(processes::Error::Resource)?; if h.generation() != o.generation || o.pages != 1 || o.space.is_some() || o.thread.is_some() || o.state.is_some() || o.active { return Err(processes::Error::Resource); } handles[kind as usize - 1] = Some(h); } Kind::Thread => { let id = o.thread.ok_or(processes::Error::Resource)?; if id.generation() != o.generation || o.frame.is_some() || o.space.is_some() { return Err(processes::Error::Resource); } thread = Some(id); state = o.state; if !complete && (o.state != Some(threads::State::Ready) || o.active) { return Err(processes::Error::Resource); } } } pages = pages .checked_add(o.pages) .ok_or(processes::Error::Resource)?; observations[kind as usize] = Some(o); } let entry = if complete { if !bag.installed { return Err(processes::Error::Preparation); } let result = self.resources.observe_entry(bag); self.stable(before)?; let e = result?; if e.start != self.recipe.start || e.start.spsr != EL0_SPSR || e.start.sp & 15 != 0 || e.start.pc & 3 != 0 || e.code_permissions != 5 || e.stack_permissions != 6 || !e.code_user || !e.stack_user { return Err(processes::Error::Preparation); } for kind in [Kind::Code, Kind::Data, Kind::Stack] { for offset in 0..PAGE_BYTES { let result = self.resources.read_page(bag.token(kind).unwrap(), offset); self.stable(before)?; if result? != self .recipe .expected_byte(kind, offset) .map_err(|_| processes::Error::Preparation)? { return Err(processes::Error::Preparation); } } } Some(e) } else { None }; Ok(processes::BagObservation { process: bag.process, resource_owner: bag.owner, thread: thread.ok_or(processes::Error::Preparation)?, state: state.ok_or(processes::Error::Preparation)?, spaces, handles, pages, image_ready: complete, stack_ready: complete, published: entry.is_some_and(|e| e.published), }) } } unsafe impl<'a, R: ResourceBackend> processes::Backend for Adapter<'a, R> { type Bag = ResourceBag; fn execution(&self) -> processes::Execution { self.resources.execution().process } fn prepare( &mut self, process: processes::ProcessId, spawn: processes::Spawn, ) -> Result> { let fail = |failure, partial| processes::BuildFailure { failure, partial }; let before = self.gate().map_err(|e| { fail( processes::Failure { error: e, boundary: processes::Boundary::Before, }, None, ) })?; if spawn != (processes::Spawn { handles: 4, code_entry: self.recipe.start.pc, user_stack_top: STACK_TOP, }) { return Err(fail( processes::Failure { error: processes::Error::Configuration, boundary: processes::Boundary::Before, }, None, )); } let owner = match self.resources.owner(process) { Ok(owner) => owner, Err(f) => return Err(fail(self.callback_failure(f, before), None)), }; let mut bag = ResourceBag { process, owner, tokens: array::from_fn(|_| None), installed: false, }; let owner_stable = self.stable(before); if owner == 0 || owner_stable.is_err() { return Err(fail( processes::Failure { error: processes::Error::Context, boundary: processes::Boundary::After, }, Some(bag), )); } for kind in KINDS { match self.resources.acquire(kind, process, owner, &self.recipe) { Ok(token) => bag.tokens[kind as usize] = Some(token), Err(f) => { bag.tokens[kind as usize] = f.acquired; return Err(fail(self.callback_failure(f.failure, before), Some(bag))); } } if self.stable(before).is_err() { return Err(fail( processes::Failure { error: processes::Error::Context, boundary: processes::Boundary::After, }, Some(bag), )); } } if self.observation(&bag, false).is_err() { return Err(fail( processes::Failure { error: processes::Error::Resource, boundary: processes::Boundary::After, }, Some(bag), )); } if let Err(f) = self.resources.install(&bag, &self.recipe) { return Err(fail(self.callback_failure(f, before), Some(bag))); } bag.installed = true; if self.stable(before).is_err() || self.observation(&bag, true).is_err() { return Err(fail( processes::Failure { error: processes::Error::Resource, boundary: processes::Boundary::After, }, Some(bag), )); } Ok(bag) } fn observe(&self, bag: &Self::Bag) -> Result { self.observation(bag, true) } fn publish(&mut self, bag: &mut Self::Bag) -> Result<(), processes::Failure> { let before = self.gate().map_err(|error| processes::Failure { error, boundary: processes::Boundary::Before, })?; match self.resources.publish(bag) { Ok(()) => self.stable(before).map_err(|error| processes::Failure { error, boundary: processes::Boundary::After, }), Err(f) => Err(self.callback_failure(f, before)), } } fn exit(&mut self, bag: &mut Self::Bag, code: i32) -> Result<(), processes::Failure> { let before = self.gate().map_err(|error| processes::Failure { error, boundary: processes::Boundary::Before, })?; match self.resources.exit(bag, code) { Ok(()) => { let mut expected = before; if expected.process.current == Some(bag.process) { expected.process.current = None; } self.stable(expected).map_err(|error| processes::Failure { error, boundary: processes::Boundary::After, }) } Err(f) => Err(self.callback_failure(f, before)), } } fn release(&mut self, mut bag: Self::Bag) -> Result<(), (processes::Error, Self::Bag)> { let before = match self.gate() { Ok(e) => e, Err(e) => return Err((e, bag)), }; if let Err(f) = self.resources.detach(&bag) { return Err((self.callback_failure(f, before).error, bag)); } if self.stable(before).is_err() { return Err((processes::Error::Context, bag)); } for kind in KINDS.into_iter().rev() { if self.stable(before).is_err() { return Err((processes::Error::Context, bag)); } if let Some(token) = bag.tokens[kind as usize].take() { match self.resources.release(token) { Ok(()) => (), Err((e, token)) => { bag.tokens[kind as usize] = Some(token); if let Err(context) = self.stable(before) { return Err((context, bag)); } return Err((e, bag)); } } if self.stable(before).is_err() { return Err((processes::Error::Context, bag)); } } } Ok(()) } } pub struct Ticket { reservation: processes::Reservation, } pub struct InitTable<'a, R: ResourceBackend> { table: processes::Table, 1>, spawn: processes::Spawn, } impl<'a, R: ResourceBackend> InitTable<'a, R> { #[cfg(all(feature = "host-fixture", not(target_os = "none")))] pub fn fixture_observe(&mut self, observe: impl FnOnce(&R) -> T) -> T { observe(&self.table.process_test_backend().resources) } #[cfg(all(feature = "host-fixture", not(target_os = "none")))] pub fn fixture_bag(&self) -> Option<&ResourceBag> { self.table.process_test_bag(0) } // Preserve every bounded input and owned backend on constructor failure. #[allow(clippy::result_large_err)] pub fn new(recipe: Recipe<'a>, resources: R) -> Result, R)> { let e = resources.execution(); if e.kernel_stack_owner == 0 || e.root == 0 || e.root & 4095 != 0 || !e.process.kernel_bootstrap { return Err((Error::Context, recipe, resources)); } let spawn = processes::Spawn { handles: 4, code_entry: recipe.start.pc, user_stack_top: STACK_TOP, }; let adapter = Adapter { recipe, resources, cpu: e.process.cpu, faulted: Cell::new(false), }; match processes::Table::new(adapter) { Ok(table) => Ok(Self { table, spawn }), Err((e, a)) => Err((Error::Process(e), a.recipe, a.resources)), } } pub fn begin(&mut self) -> Result { self.table .begin(self.spawn) .map(|reservation| Ticket { reservation }) .map_err(Error::Process) } pub fn prepare(&mut self, t: Ticket) -> Result { self.table .prepare(t.reservation) .map(|reservation| Ticket { reservation }) .map_err(|(e, reservation)| (Error::Process(e), Ticket { reservation })) } pub fn abort(&mut self, t: Ticket) -> Result<(), (Error, Ticket)> { self.table .abort(t.reservation) .map_err(|(e, reservation)| (Error::Process(e), Ticket { reservation })) } pub fn publish(&mut self, t: Ticket) -> Result { self.table .publish(t.reservation) .map_err(|(e, reservation)| (Error::Process(e), Ticket { reservation })) } pub fn record(&self, id: processes::ProcessId) -> Result { self.table.record(id).map_err(Error::Process) } pub fn snapshot(&self) -> processes::Snapshot { self.table.snapshot() } pub fn published(&self) -> impl Iterator + '_ { self.table.published() } pub fn exit_current(&mut self, code: i32) -> Result<(), Error> { self.table.exit_current(code).map_err(Error::Process) } pub fn terminate(&mut self, id: processes::ProcessId, code: i32) -> Result<(), Error> { self.table.terminate_child(id, code).map_err(Error::Process) } pub fn reap(&mut self, id: processes::ProcessId) -> Result { self.table.wait_child(id).map_err(Error::Process) } #[allow(clippy::result_large_err)] pub fn into_resources(self) -> Result { let Self { table, spawn } = self; match table.into_backend() { Ok(a) => Ok(a.resources), Err((e, table)) => Err((Error::Process(e), Self { table, spawn })), } } }