//! Single CPU context custody; actual assembly and WFI remain trusted backend //! operations. No raw register image, observation or copied handle owns a stack. use core::marker::PhantomData; use core::mem::{align_of, offset_of, size_of}; use turkos_os_exceptions::exceptions::{self as exception, Frame}; use turkos_os_threads::threads::{self as thread, StackObservation, ThreadId, ThreadRecord}; pub const CONTEXT_CANARY: u64 = 0x4b31303430435458; const IRQ: u64 = 1 << 7; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { Configuration, Context, Capacity, Exhausted, Stale, State, Stack, Guard, Canary, Entry, Abi, Capture, Simd, Pending, Backend, Quarantined, Active, Wake, } /// Inert lookup identity, independently checked against a retained private slot. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Handle { bank: u64, slot: usize, generation: u64, } impl Handle { pub fn bank(self) -> u64 { self.bank } pub fn slot(self) -> usize { self.slot } pub fn generation(self) -> u64 { self.generation } } /// Full eager SIMD context, even though a cooperative ABI preserves fewer lanes. #[repr(C, align(16))] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Cooperative { pub sp: u64, pub x19_x30: [u64; 12], pub padding: u64, pub q: [[u64; 2]; 32], pub fpcr: u32, pub fpsr: u32, pub canary: u64, } impl Cooperative { pub const EMPTY: Self = Self { sp: 0, x19_x30: [0; 12], padding: 0, q: [[0; 2]; 32], fpcr: 0, fpsr: 0, canary: CONTEXT_CANARY, }; } pub const COOP_SP_OFFSET: usize = offset_of!(Cooperative, sp); pub const COOP_X_OFFSET: usize = offset_of!(Cooperative, x19_x30); pub const COOP_PADDING_OFFSET: usize = offset_of!(Cooperative, padding); pub const COOP_Q_OFFSET: usize = offset_of!(Cooperative, q); pub const COOP_FPCR_OFFSET: usize = offset_of!(Cooperative, fpcr); pub const COOP_FPSR_OFFSET: usize = offset_of!(Cooperative, fpsr); pub const COOP_CANARY_OFFSET: usize = offset_of!(Cooperative, canary); pub const COOP_BYTES: usize = size_of::(); const _: () = { assert!(COOP_SP_OFFSET == 0); assert!(COOP_X_OFFSET == 8); assert!(COOP_PADDING_OFFSET == 104); assert!(COOP_Q_OFFSET == 112); assert!(COOP_FPCR_OFFSET == 624); assert!(COOP_FPSR_OFFSET == 628); assert!(COOP_CANARY_OFFSET == 632); assert!(COOP_BYTES == 640); assert!(align_of::() == 16); }; /// Assembly must report exactly these offsets; a byte count alone is inadequate. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Abi { pub offsets: [usize; 7], pub bytes: usize, pub alignment: usize, } impl Abi { pub const COOPERATIVE: Self = Self { offsets: [0, 8, 104, 112, 624, 628, 632], bytes: 640, alignment: 16, }; } // Fixed caller-owned storage; this no_alloc service deliberately retains the full frame. #[allow(clippy::large_enum_variant)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Image { Initial { entry: u64, argument: u64, sp: u64, return_pc: u64, }, Cooperative(Cooperative), Timer(Frame), } impl Image { pub fn sp(self) -> u64 { match self { Self::Initial { sp, .. } => sp, Self::Cooperative(c) => c.sp, Self::Timer(f) => f.interrupted_sp, } } pub fn pc(self) -> u64 { match self { Self::Initial { entry, .. } => entry, Self::Cooperative(c) => c.x19_x30[11], Self::Timer(f) => f.elr, } } pub fn simd(self) -> ([[u64; 2]; 32], u32, u32) { match self { Self::Initial { .. } => ([[0; 2]; 32], 0, 0), Self::Cooperative(c) => (c.q, c.fpcr, c.fpsr), Self::Timer(f) => (f.q, f.fpcr, f.fpsr), } } } /// Raw CPU data. The unsafe backend must separately bind every byte and the /// episode to the current live assembly handoff in this retained thread. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Capture { pub image: Image, pub address: u64, pub abi: Abi, /// An inert observation of a nonzero, never-reused capture episode. This /// value alone does not authorize a capture or establish its freshness. pub episode: u64, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Execution { pub cpu: u32, pub daif: u64, pub in_irq: bool, pub sp: u64, pub root: u64, pub thread: Option, pub fp_enabled: bool, pub critical_depth: u32, pub safe_point: bool, pub tco: u64, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Registers { pub sp: u64, pub x19_x30: [u64; 12], pub q: [[u64; 2]; 32], pub fpcr: u32, pub fpsr: u32, pub pc: u64, pub argument: u64, pub timer: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FailurePhase { BeforeSwitch, AfterSwitch, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Disposition { Ready, Sleeping, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Failure { pub error: Error, pub phase: FailurePhase, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct IdleReceipt { /// The value actually checked atomically at the final mask/WFI handoff. pub checked_epoch: u64, pub work_at_handoff: bool, pub executed_wfi: bool, pub wake_epoch: u64, } /// # Safety /// Backend exclusively owns a frozen threads Table and actual stack/table /// authority. Binding values are nonclone pins for that live ThreadId/stack; /// dropping one never releases pages. bind/release failures return the exact /// resource. capture_matches verifies the current live assembly handoff/IRQ /// episode for this exact live ThreadId, root and retained stack, as well as /// every actual byte at the supplied stack address. Intact bytes from an older /// call on the same live stack must fail even if their address and saved SP are /// valid. Episodes are nonzero and never reused; exhaustion refuses capture. /// Rust callbacks may run below the saved assembly frame, so matching the /// current callback SP to the saved SP is not a freshness check. No caller- /// provided boolean, episode number or raw image manufactures this authority. /// Prepared owns all resources needed by a switch. Preparation failures leave /// the complete resource inventory and thread/register state unchanged; a /// backend whose rollback can fail must retain acquisitions in a successful /// Prepared and report uncertainty through commit. commit reports BeforeSwitch only when no /// hardware, table state or visible resource changed; after any possible change /// it must report AfterSwitch. Observations are real and taken before callback /// code can clobber registers. finish/abort failure returns the same Prepared. /// All operations are bounded and keep IRQs masked while core metadata is /// borrowed. Direct timer commit closes IRQ ownership before changing frozen /// thread state. Eager SIMD/FP load/store includes all32 q registers and FPCR/ /// FPSR. idle_commit performs its epoch/work check atomically at the final /// mask/WFI boundary; an earlier observation is never a substitute. Prepared /// idle resources retain every wake token through failure and spurious wake. pub unsafe trait Backend { type Binding; type Prepared; type IdlePrepared; fn execution(&self) -> Execution; fn record(&self, binding: &Self::Binding) -> Result; fn stack(&self, binding: &Self::Binding) -> Result; fn root(&self, binding: &Self::Binding) -> Result; fn executable(&self, pc: u64) -> bool; fn initial_return_pc(&self, binding: &Self::Binding) -> Result; fn capture_matches(&self, binding: &Self::Binding, capture: &Capture) -> bool; fn registers(&self) -> Registers; fn prepare( &mut self, source: Option<&Self::Binding>, target: &Self::Binding, image: &Image, disposition: Disposition, ) -> Result; fn commit( &mut self, prepared: &mut Self::Prepared, source: Option<&Self::Binding>, target: &Self::Binding, image: &Image, disposition: Disposition, ) -> Result<(), Failure>; fn finish(&mut self, prepared: Self::Prepared) -> Result<(), (Error, Self::Prepared)>; fn abort(&mut self, prepared: Self::Prepared) -> Result<(), (Error, Self::Prepared)>; fn release(&mut self, binding: Self::Binding) -> Result<(), (Error, Self::Binding)>; fn ready_empty(&self) -> bool; fn wake_epoch(&self) -> u64; fn idle_prepare(&mut self, epoch: u64) -> Result; fn idle_commit( &mut self, prepared: &mut Self::IdlePrepared, epoch: u64, ) -> Result; fn idle_finish( &mut self, prepared: Self::IdlePrepared, ) -> Result<(), (Error, Self::IdlePrepared)>; fn idle_abort( &mut self, prepared: Self::IdlePrepared, ) -> Result<(), (Error, Self::IdlePrepared)>; } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum State { Vacant, Ready, Active, Quarantined, } struct Slot { generation: u64, last_capture: u64, binding: Option, thread: Option, root: u64, image: Image, state: State, } impl Slot { fn empty() -> Self { Self { generation: 0, last_capture: 0, binding: None, thread: None, root: 0, image: Image::Initial { entry: 0, argument: 0, sp: 0, return_pc: 0, }, state: State::Vacant, } } } struct Deferred { source: Handle, target: Handle, thread: ThreadId, epoch: u64, capture: Capture, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct DeferredRecord { pub source: Handle, pub target: Handle, pub thread: ThreadId, pub epoch: u64, pub capture: Capture, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Record { pub handle: Handle, pub thread: ThreadId, pub state: State, pub root: u64, pub image: Image, pub last_capture: u64, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Snapshot { pub retained: usize, pub retained_bindings: usize, pub active: usize, pub ready: usize, pub quarantined: usize, pub retired: usize, pub deferred_epoch: Option, pub faulted: bool, pub retained_switch: bool, pub retained_idle: bool, pub retained_capture: bool, } pub struct ContextBank { backend: B, identity: u64, cpu: u32, slots: [Slot; N], active: Option, epoch: u64, deferred: Option, last_deferred: Option, faulted: bool, retained_switch: Option, retained_idle: Option, uncertain_capture: Option, _local: PhantomData<*mut ()>, } #[cfg(not(target_os = "none"))] fn identity() -> Result { use core::sync::atomic::{AtomicU64, Ordering}; static NEXT: AtomicU64 = AtomicU64::new(1); NEXT.try_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_add(1)) .map_err(|_| Error::Exhausted) } #[cfg(target_os = "none")] fn identity() -> Result { unsafe extern "C" { fn __os_k1040_context_identity() -> u64; } let n = unsafe { __os_k1040_context_identity() }; if n == 0 { Err(Error::Exhausted) } else { Ok(n) } } impl ContextBank { pub fn new(backend: B) -> Result { let e = backend.execution(); if N == 0 || N > 32 { return Err((Error::Configuration, backend)); } if e.daif & IRQ == 0 || e.daif & !0x3c0 != 0 || e.in_irq || !e.fp_enabled { return Err((Error::Context, backend)); } let id = match identity() { Ok(v) => v, Err(e) => return Err((e, backend)), }; Ok(Self { backend, identity: id, cpu: e.cpu, slots: core::array::from_fn(|_| Slot::empty()), active: None, epoch: 0, deferred: None, last_deferred: None, faulted: false, retained_switch: None, retained_idle: None, uncertain_capture: None, _local: PhantomData, }) } fn execution(&self, irq: bool) -> Result { if self.faulted { return Err(Error::Quarantined); } let e = self.backend.execution(); if e.cpu != self.cpu || e.daif & IRQ == 0 || e.daif & !0x3c0 != 0 || (!irq && e.in_irq) || !e.fp_enabled { Err(Error::Context) } else { Ok(e) } } fn index(&self, h: Handle) -> Result { if h.bank != self.identity || h.slot >= N { return Err(Error::Stale); } let s = &self.slots[h.slot]; if s.state == State::Vacant || s.generation != h.generation { Err(Error::Stale) } else { Ok(h.slot) } } fn handle(&self, i: usize) -> Handle { Handle { bank: self.identity, slot: i, generation: self.slots[i].generation, } } fn binding(&self, i: usize) -> Result<&B::Binding, Error> { self.slots[i].binding.as_ref().ok_or(Error::State) } fn validate(&self, i: usize) -> Result { let s = &self.slots[i]; if s.state == State::Quarantined { return Err(Error::Quarantined); } let b = self.binding(i)?; let r = self.backend.record(b)?; if Some(r.id) != s.thread { return Err(Error::Stale); } if self.backend.root(b)? != s.root { return Err(Error::Stack); } Self::stack_observation(self.backend.stack(b)?, r)?; Ok(r) } fn stack_observation(o: StackObservation, r: ThreadRecord) -> Result<(), Error> { let l = r.layout; if o.layout != l || l.base < 4096 || !l.base.is_multiple_of(4096) || l.physical == 0 || !l.physical.is_multiple_of(4096) || l.bytes < 16384 || l.bytes > 65536 || !l.bytes.is_multiple_of(4096) || l.base.checked_add(l.bytes).is_none() || l.physical.checked_add(l.bytes).is_none() { return Err(Error::Stack); } if o.guard_descriptor != 0 { return Err(Error::Guard); } if o.low_canary != thread::LOW_CANARY || o.high_canary != thread::HIGH_CANARY { return Err(Error::Canary); } Ok(()) } fn usable(r: ThreadRecord, sp: u64) -> bool { sp.is_multiple_of(16) && sp >= r.layout.usable_bottom() && sp <= r.layout.usable_top() } fn validate_image(&self, r: ThreadRecord, image: Image) -> Result<(), Error> { if !Self::usable(r, image.sp()) { return Err(Error::Stack); } if image.pc() == 0 || !image.pc().is_multiple_of(4) || !self.backend.executable(image.pc()) { return Err(Error::Entry); } match image { Image::Initial { entry, argument, sp, return_pc, } => { if entry != r.entry || argument != r.argument || sp != r.layout.usable_top() || return_pc == 0 || !return_pc.is_multiple_of(4) || !self.backend.executable(return_pc) { return Err(Error::Capture); } } Image::Cooperative(c) => { if c.canary != CONTEXT_CANARY || c.padding != 0 { return Err(Error::Canary); } } Image::Timer(_) => {} } Ok(()) } fn current(&self, h: Handle, irq: bool) -> Result<(usize, ThreadRecord, Execution), Error> { let e = self.execution(irq)?; let i = self.index(h)?; if self.active != Some(h) || self.slots[i].state != State::Active { return Err(Error::State); } let r = self.validate(i)?; if r.state != thread::State::Running || e.thread != Some(r.id) || e.root != self.slots[i].root || !Self::usable(r, e.sp) { return Err(Error::State); } Ok((i, r, e)) } fn capture( &self, i: usize, r: ThreadRecord, c: Capture, timer: bool, e: Execution, ) -> Result<(), Error> { if c.episode == 0 || c.episode <= self.slots[i].last_capture { return Err(Error::Capture); } self.validate_image(r, c.image)?; let bytes = if timer { exception::FRAME_BYTES } else { COOP_BYTES }; if !c.address.is_multiple_of(16) || c.address < r.layout.usable_bottom() || c.address .checked_add(bytes as u64) .is_none_or(|end| end > r.layout.usable_top()) { return Err(Error::Stack); } match c.image { Image::Cooperative(f) if !timer => { if c.abi != Abi::COOPERATIVE { return Err(Error::Abi); } if f.sp != c.address + COOP_BYTES as u64 { return Err(Error::Capture); } } Image::Timer(f) if timer => { let stack = exception::Stack::new(r.layout.base, r.layout.base + r.layout.bytes) .map_err(|_| Error::Stack)?; exception::validate_ingress(&f, c.address, stack).map_err(|_| Error::Capture)?; if f.vector_slot != 5 || f.spsr & 31 != 5 || f.spsr & IRQ != 0 || f.spsr & !((15 << 28) | 0x345 | (1 << 25)) != 0 || f.spsr & (1 << 25) != e.tco || e.tco & !(1 << 25) != 0 || f.interrupted_sp != c.address + exception::FRAME_BYTES as u64 { return Err(Error::Capture); } } _ => return Err(Error::Capture), } if !self.backend.capture_matches(self.binding(i)?, &c) { return Err(Error::Capture); } Ok(()) } /// The caller supplies an actual nonclone pin minted by the exclusive backend. /// A rejected attach returns exactly that resource; no generation is consumed. pub fn attach(&mut self, binding: B::Binding) -> Result { let result = (|| { self.execution(false)?; let i = self .slots .iter() .position(|s| s.state == State::Vacant && s.generation != u64::MAX) .ok_or(Error::Capacity)?; let generation = self.slots[i] .generation .checked_add(1) .ok_or(Error::Exhausted)?; let r = self.backend.record(&binding)?; if r.state != thread::State::Ready { return Err(Error::State); } if self .slots .iter() .any(|s| s.thread == Some(r.id) && s.state != State::Vacant) { return Err(Error::State); } Self::stack_observation(self.backend.stack(&binding)?, r)?; let root = self.backend.root(&binding)?; if root == 0 || !root.is_multiple_of(4096) || root >> 48 != 0 { return Err(Error::Stack); } let image = Image::Initial { entry: r.entry, argument: r.argument, sp: r.layout.usable_top(), return_pc: self.backend.initial_return_pc(&binding)?, }; self.validate_image(r, image)?; Ok((i, generation, r.id, root, image)) })(); match result { Ok((i, generation, id, root, image)) => { self.slots[i] = Slot { generation, last_capture: 0, binding: Some(binding), thread: Some(id), root, image, state: State::Ready, }; Ok(self.handle(i)) } Err(e) => Err((e, binding)), } } fn target(&self, h: Handle) -> Result { let i = self.index(h)?; if self.slots[i].state != State::Ready { return Err(Error::State); } let r = self.validate(i)?; if r.state != thread::State::Ready { return Err(Error::State); } self.validate_image(r, self.slots[i].image)?; Ok(i) } fn dispatch( &mut self, source: Option, target: usize, capture: Option, deferred: bool, disposition: Disposition, ) -> Result, Error> { let image = self.slots[target].image; let epoch = self.epoch.checked_add(1).ok_or(Error::Exhausted)?; let prepared = self.backend.prepare( source.and_then(|i| self.slots[i].binding.as_ref()), self.slots[target].binding.as_ref().ok_or(Error::State)?, &image, disposition, )?; Ok(Dispatch { bank: self, source, target, capture, image, epoch, prepared: Some(prepared), consume_deferred: deferred, disposition, }) } pub fn start(&mut self, target: Handle) -> Result, Error> { let e = self.execution(false)?; if self.active.is_some() || self.deferred.is_some() || e.thread.is_some() { return Err(Error::State); } let i = self.target(target)?; self.dispatch(None, i, None, false, Disposition::Ready) } pub fn cooperate( &mut self, source: Handle, target: Handle, capture: Capture, ) -> Result, Error> { let (i, r, e) = self.current(source, false)?; if e.critical_depth != 0 || !e.safe_point || self.deferred.is_some() { return Err(Error::Context); } if source == target { return Err(Error::State); } self.capture(i, r, capture, false, e)?; let t = self.target(target)?; self.dispatch(Some(i), t, Some(capture), false, Disposition::Ready) } pub fn sleep_to( &mut self, source: Handle, target: Handle, capture: Capture, ) -> Result, Error> { let (i, r, e) = self.current(source, false)?; if e.critical_depth != 0 || !e.safe_point || self.deferred.is_some() { return Err(Error::Context); } if r.wait_epoch == u64::MAX { return Err(Error::Exhausted); } if source == target { return Err(Error::State); } self.capture(i, r, capture, false, e)?; let t = self.target(target)?; self.dispatch(Some(i), t, Some(capture), false, Disposition::Sleeping) } pub fn timer( &mut self, source: Handle, target: Handle, capture: Capture, ) -> Result, Error> { let (i, r, e) = self.current(source, true)?; if !e.in_irq || e.critical_depth != 0 || !e.safe_point || self.deferred.is_some() { return Err(Error::Context); } if source == target { return Err(Error::State); } self.capture(i, r, capture, true, e)?; let t = self.target(target)?; self.dispatch(Some(i), t, Some(capture), false, Disposition::Ready) } pub fn defer_timer( &mut self, source: Handle, target: Handle, capture: Capture, ) -> Result { let (i, r, e) = self.current(source, true)?; if !e.in_irq || e.critical_depth == 0 && e.safe_point { return Err(Error::Context); } if self.deferred.is_some() { return Err(Error::Pending); } if source == target { return Err(Error::State); } self.capture(i, r, capture, true, e)?; self.target(target)?; let epoch = self.epoch.checked_add(1).ok_or(Error::Exhausted)?; self.epoch = epoch; self.deferred = Some(Deferred { source, target, thread: r.id, epoch, capture, }); Ok(epoch) } /// Original IRQ data remains an episode witness. Only this fresh checked /// cooperative capture becomes the later source return context. pub fn service_deferred(&mut self, fresh: Capture) -> Result, Error> { let d = self.deferred.as_ref().ok_or(Error::Pending)?; let (source, target, id) = (d.source, d.target, d.thread); let (i, r, e) = self.current(source, false)?; if r.id != id || e.critical_depth != 0 || !e.safe_point { return Err(Error::Context); } self.capture(i, r, fresh, false, e)?; let t = self.target(target)?; self.dispatch(Some(i), t, Some(fresh), true, Disposition::Ready) } pub fn deferred_record(&self) -> Option { self.deferred.as_ref().map(|d| DeferredRecord { source: d.source, target: d.target, thread: d.thread, epoch: d.epoch, capture: d.capture, }) } pub fn last_deferred_record(&self) -> Option { self.last_deferred } pub fn record(&self, h: Handle) -> Result { let i = self.index(h)?; let s = &self.slots[i]; Ok(Record { handle: h, thread: s.thread.ok_or(Error::State)?, state: s.state, root: s.root, image: s.image, last_capture: s.last_capture, }) } pub fn snapshot(&self) -> Snapshot { let mut r = Snapshot { faulted: self.faulted, deferred_epoch: self.deferred.as_ref().map(|d| d.epoch), retained_switch: self.retained_switch.is_some(), retained_idle: self.retained_idle.is_some(), retained_capture: self.uncertain_capture.is_some(), ..Snapshot::default() }; for s in &self.slots { r.retained_bindings += usize::from(s.binding.is_some()); if s.state == State::Vacant { r.retired += usize::from(s.generation == u64::MAX); } else { r.retained += 1; match s.state { State::Ready => r.ready += 1, State::Active => r.active += 1, State::Quarantined => r.quarantined += 1, State::Vacant => {} } } } r } pub fn retire(&mut self, h: Handle) -> Result<(), Error> { let e = self.execution(false)?; let i = self.index(h)?; if self.active == Some(h) || self .deferred .as_ref() .is_some_and(|d| d.source == h || d.target == h) { return Err(Error::Active); } let r = self.validate(i)?; if r.state != thread::State::Dead || e.thread == Some(r.id) || e.sp >= r.layout.guard() && e.sp <= r.layout.base + r.layout.bytes { return Err(Error::Active); } let b = self.slots[i].binding.take().ok_or(Error::State)?; if let Err((_e, b)) = self.backend.release(b) { self.slots[i].binding = Some(b); self.slots[i].state = State::Quarantined; return Err(Error::Backend); } let generation = self.slots[i].generation; self.slots[i] = Slot::empty(); self.slots[i].generation = generation; Ok(()) } pub fn quarantined_capture(&self) -> Option { self.uncertain_capture } pub fn with_retained_idle_observation( &self, f: impl FnOnce(Option<&B::IdlePrepared>) -> R, ) -> R { f(self.retained_idle.as_ref()) } pub fn with_backend_observation(&self, f: impl FnOnce(&B) -> R) -> R { f(&self.backend) } // Returning the actual retained bank is required on refusal; no heap boxing. #[allow(clippy::result_large_err)] pub fn into_backend(self) -> Result { if self.faulted || self.slots.iter().any(|s| s.state != State::Vacant) || self.deferred.is_some() { return Err((Error::Active, self)); } if let Err(e) = self.execution(false) { return Err((e, self)); } Ok(self.backend) } fn quarantine_switch( &mut self, source: Option, target: usize, prepared: B::Prepared, capture: Option, ) { if let Some(i) = source { self.slots[i].state = State::Quarantined; } self.slots[target].state = State::Quarantined; self.retained_switch = Some(prepared); self.uncertain_capture = capture; self.faulted = true; } pub fn idle(&mut self) -> Result, Error> { let execution = self.execution(false)?; if execution.critical_depth != 0 || !execution.safe_point { return Err(Error::Context); } if let Some(h) = self.active { self.current(h, false)?; } for i in 0..N { if matches!(self.slots[i].state, State::Ready | State::Active) && self.validate(i)?.state == thread::State::Ready { return Err(Error::Pending); } } if self.deferred.is_some() || !self.backend.ready_empty() { return Err(Error::Pending); } let epoch = self.backend.wake_epoch(); if epoch == u64::MAX { return Err(Error::Exhausted); } let prepared = self.backend.idle_prepare(epoch)?; Ok(Idle { bank: self, epoch, prepared: Some(prepared), }) } } /// A retryable pre-switch error returns exactly the dispatch and private staged /// resources. An uncertain post-switch error leaves them retained in the bank. pub enum DispatchError { Before(Error, T), Quarantined(Error), } pub struct Dispatch<'a, B: Backend, const N: usize> { bank: &'a mut ContextBank, source: Option, target: usize, capture: Option, image: Image, epoch: u64, prepared: Option, consume_deferred: bool, disposition: Disposition, } impl Dispatch<'_, B, N> { pub fn image(&self) -> Image { self.image } pub fn with_prepared_observation(&self, f: impl FnOnce(&B::Prepared) -> R) -> R { f(self.prepared.as_ref().unwrap()) } pub fn source_capture(&self) -> Option { self.capture } fn before(&self) -> Result<(), Error> { let timer = self .capture .is_some_and(|c| matches!(c.image, Image::Timer(_))); if let Some(i) = self.source { let (_, r, e) = self.bank.current(self.bank.handle(i), timer)?; if e.in_irq != timer || e.critical_depth != 0 || !e.safe_point { return Err(Error::Context); } self.bank .capture(i, r, self.capture.ok_or(Error::Capture)?, timer, e)?; } else { let e = self.bank.execution(false)?; if self.bank.active.is_some() || e.thread.is_some() || e.critical_depth != 0 || !e.safe_point { return Err(Error::Context); } } self.bank.target(self.bank.handle(self.target))?; if self.bank.slots[self.target].image != self.image { return Err(Error::Capture); } Ok(()) } fn completed(&self) -> Result<(), Error> { let e = self.bank.execution(false)?; let r = self.bank.validate(self.target)?; if r.state != thread::State::Running || e.thread != Some(r.id) || e.root != self.bank.slots[self.target].root || !ContextBank::::usable(r, e.sp) { return Err(Error::State); } if let Some(i) = self.source && self.bank.validate(i)?.state != match self.disposition { Disposition::Ready => thread::State::Ready, Disposition::Sleeping => thread::State::Sleeping, } { return Err(Error::State); } let actual = self.bank.backend.registers(); let (q, fpcr, fpsr) = self.image.simd(); if actual.sp != self.image.sp() || actual.pc != self.image.pc() || actual.q != q || actual.fpcr != fpcr || actual.fpsr != fpsr { return Err(Error::Simd); } match self.image { Image::Initial { argument, return_pc, .. } => { let mut expected = [0; 12]; expected[11] = return_pc; if actual.x19_x30 != expected || actual.argument != argument || actual.timer.is_some() { return Err(Error::Capture); } } Image::Cooperative(c) => { if actual.x19_x30 != c.x19_x30 || actual.timer.is_some() { return Err(Error::Capture); } } Image::Timer(f) => { if actual.x19_x30 != f.x[19..31] || actual.timer != Some(f) { return Err(Error::Capture); } } } Ok(()) } // The same nonclone transaction, including its capture, remains retryable. #[allow(clippy::result_large_err)] pub fn complete(mut self) -> Result<(), DispatchError> { if let Err(e) = self.before() { return Err(DispatchError::Before(e, self)); } let source = self .source .and_then(|i| self.bank.slots[i].binding.as_ref()); let target = self.bank.slots[self.target].binding.as_ref().unwrap(); let prepared = self.prepared.as_mut().unwrap(); if let Err(f) = self.bank .backend .commit(prepared, source, target, &self.image, self.disposition) { return match f.phase { FailurePhase::BeforeSwitch => Err(DispatchError::Before(f.error, self)), FailurePhase::AfterSwitch => { let p = self.prepared.take().unwrap(); self.bank .quarantine_switch(self.source, self.target, p, self.capture); Err(DispatchError::Quarantined(f.error)) } }; } if let Err(e) = self.completed() { let p = self.prepared.take().unwrap(); self.bank .quarantine_switch(self.source, self.target, p, self.capture); return Err(DispatchError::Quarantined(e)); } let prepared = self.prepared.take().unwrap(); if let Err((e, p)) = self.bank.backend.finish(prepared) { self.bank .quarantine_switch(self.source, self.target, p, self.capture); return Err(DispatchError::Quarantined(e)); } if let Some(i) = self.source { self.bank.slots[i].image = self.capture.unwrap().image; self.bank.slots[i].last_capture = self.capture.unwrap().episode; self.bank.slots[i].state = State::Ready; } self.bank.slots[self.target].state = State::Active; self.bank.active = Some(self.bank.handle(self.target)); self.bank.epoch = self.epoch; if self.consume_deferred { self.bank.last_deferred = self.bank.deferred_record(); self.bank.deferred = None; } Ok(()) } pub fn abort(mut self) -> Result<(), Error> { let prepared = self.prepared.take().unwrap(); if let Err((e, p)) = self.bank.backend.abort(prepared) { self.bank .quarantine_switch(self.source, self.target, p, self.capture); return Err(e); } Ok(()) } } impl Drop for Dispatch<'_, B, N> { fn drop(&mut self) { if let Some(p) = self.prepared.take() && let Err((_e, p)) = self.bank.backend.abort(p) { self.bank .quarantine_switch(self.source, self.target, p, self.capture); } } } pub struct Idle<'a, B: Backend, const N: usize> { bank: &'a mut ContextBank, epoch: u64, prepared: Option, } impl Idle<'_, B, N> { pub fn with_prepared_observation(&self, f: impl FnOnce(&B::IdlePrepared) -> R) -> R { f(self.prepared.as_ref().unwrap()) } pub fn epoch(&self) -> u64 { self.epoch } pub fn complete(mut self) -> Result> { let valid = self .bank .execution(false) .and_then(|e| { if e.critical_depth != 0 || !e.safe_point { return Err(Error::Context); } Ok(e) }) .and_then(|_| match self.bank.active { Some(h) => self.bank.current(h, false).map(|_| ()), None => Ok(()), }); if let Err(e) = valid { return Err(DispatchError::Before(e, self)); } let result = self .bank .backend .idle_commit(self.prepared.as_mut().unwrap(), self.epoch); let receipt = match result { Ok(r) => r, Err(f) => { return match f.phase { FailurePhase::BeforeSwitch => Err(DispatchError::Before(f.error, self)), FailurePhase::AfterSwitch => { self.bank.retained_idle = self.prepared.take(); self.bank.faulted = true; Err(DispatchError::Quarantined(f.error)) } }; } }; if receipt.checked_epoch < self.epoch || receipt.wake_epoch < receipt.checked_epoch || receipt.executed_wfi && receipt.work_at_handoff || receipt.executed_wfi && receipt.checked_epoch != self.epoch { self.bank.retained_idle = self.prepared.take(); self.bank.faulted = true; return Err(DispatchError::Quarantined(Error::Wake)); } let p = self.prepared.take().unwrap(); if let Err((e, p)) = self.bank.backend.idle_finish(p) { self.bank.retained_idle = Some(p); self.bank.faulted = true; return Err(DispatchError::Quarantined(e)); } Ok(receipt) } pub fn abort(mut self) -> Result<(), Error> { let p = self.prepared.take().unwrap(); if let Err((e, p)) = self.bank.backend.idle_abort(p) { self.bank.retained_idle = Some(p); self.bank.faulted = true; return Err(e); } Ok(()) } } impl Drop for Idle<'_, B, N> { fn drop(&mut self) { if let Some(p) = self.prepared.take() && let Err((_e, p)) = self.bank.backend.idle_abort(p) { self.bank.retained_idle = Some(p); self.bank.faulted = true; } } } #[cfg(test)] mod exhaustion_tests { use super::*; use crate::test_support::{backend, capture, finish, setup}; #[test] fn retired_slot_is_skipped_and_all_retired_returns_same_binding() { let (b, mut pins, h) = backend(2); let mut bank = ContextBank::<_, 4>::new(b).ok().unwrap(); bank.slots[0].generation = u64::MAX; let first = bank.attach(pins.remove(0)).ok().unwrap(); assert_eq!(first.slot(), 1); for i in [0, 2, 3] { bank.slots[i].generation = u64::MAX; } let binding = pins.remove(0); let id = binding.id; let binding = match bank.attach(binding) { Err((Error::Capacity, b)) => b, _ => panic!(), }; assert_eq!(binding.id, id); assert_eq!(bank.snapshot().retired, 3); assert_eq!(h.borrow().prepares, 0); assert_eq!(h.borrow().pins, 2); } #[test] fn dispatch_epoch_exhaustion_preserves_active_context_and_resources() { let (mut bank, [a, b], h) = setup(); finish(bank.start(a).unwrap()); bank.epoch = u64::MAX; let c = capture(&bank, a, &h, 17, false); let before = bank.snapshot(); let image = bank.record(a).unwrap().image; assert!(matches!(bank.cooperate(a, b, c), Err(Error::Exhausted))); assert_eq!(bank.snapshot(), before); assert_eq!(bank.record(a).unwrap().image, image); assert_eq!(h.borrow().prepares, 1); assert_eq!(h.borrow().pins, 2); } #[test] fn deferred_epoch_exhaustion_does_not_publish_episode() { let (mut bank, [a, b], h) = setup(); finish(bank.start(a).unwrap()); bank.epoch = u64::MAX; let c = capture(&bank, a, &h, 17, true); h.borrow_mut().critical = 1; let before = bank.snapshot(); assert_eq!(bank.defer_timer(a, b, c), Err(Error::Exhausted)); assert_eq!(bank.snapshot(), before); assert!(bank.deferred_record().is_none()); assert_eq!(h.borrow().pins, 2); } #[test] fn idle_wake_epoch_exhaustion_retains_sleep_episode() { let (mut bank, _, h) = crate::test_support::setup_idle(); h.borrow_mut().wake_epoch = u64::MAX; let before = bank.snapshot(); assert!(matches!(bank.idle(), Err(Error::Exhausted))); assert_eq!(bank.snapshot(), before); assert!(bank.with_backend_observation(|b| b.sleep_wait.is_some())); assert_eq!(h.borrow().pins, 2); } #[test] fn forged_private_generation_and_lost_deferred_thread_identity_refuse() { let (mut bank, [a, b], h) = setup(); finish(bank.start(a).unwrap()); let c = capture(&bank, a, &h, 17, false); let forged = Handle { generation: a.generation + 1, ..a }; assert!(matches!(bank.cooperate(forged, b, c), Err(Error::Stale))); let c = capture(&bank, a, &h, 17, true); h.borrow_mut().critical = 1; bank.defer_timer(a, b, c).unwrap(); let original = bank.deferred_record(); h.borrow_mut().critical = 0; let fresh = capture(&bank, a, &h, 43, false); bank.slots[a.slot].thread = bank.slots[b.slot].thread; assert!(matches!(bank.service_deferred(fresh), Err(Error::Stale))); assert_eq!(bank.deferred_record(), original); assert_eq!(h.borrow().pins, 2); } } // Appended only to the exact frozen Context source in an isolated generated // module. Ordinary ContextBank operations are blocked while an owned handoff // is pending; no Rust bank loan spans assembly continuation execution. /// # Safety /// Departure and arrival observe actual source/target CPU execution and frozen /// Table ownership. They never fabricate SP, register bytes or landing episode; /// callbacks keep IRQs masked and perform no allocation or implicit release. pub unsafe trait HandoffBackend: Backend { /// # Safety /// Runs on the real source stack before assembly, while IRQs are masked. /// It publishes only the actual frozen Table yield/sleep (or no source for /// start); failure before any change preserves all state. Any possible /// change reports AfterSwitch, even if the stack branch has not occurred. fn depart( &mut self, prepared: &mut Self::Prepared, source: Option<&Self::Binding>, nonce: u64, disposition: Disposition, ) -> Result<(), Failure>; /// # Safety /// Invoked only after actual target landing. It verifies the live assembly /// landing nonce, retained target generation/root/guard and actual SP, /// activates the frozen Table on that physical stack and preserves the /// actual full register observation recorded before callbacks clobber it. /// Any failure retains staged ownership and reports uncertainty. fn arrive( &mut self, prepared: &mut Self::Prepared, target: &Self::Binding, image: &Image, nonce: u64, ) -> Result<(), Error>; } #[cfg(test)] mod handoff_exhaustion_tests { use super::*; use crate::test_support as saved; unsafe impl HandoffBackend for saved::Backend { fn depart( &mut self, _: &mut Self::Prepared, _: Option<&Self::Binding>, _: u64, _: Disposition, ) -> Result<(), Failure> { Err(Failure { error: Error::Backend, phase: FailurePhase::BeforeSwitch, }) } fn arrive( &mut self, _: &mut Self::Prepared, _: &Self::Binding, _: &Image, _: u64, ) -> Result<(), Error> { Err(Error::Backend) } } fn setup() -> ( AsyncBank, Handle, std::rc::Rc>, ) { let (backend, mut pins, hardware) = saved::backend(2); let mut bank = ContextBank::<_, 4>::new(backend).ok().unwrap(); let a = bank.attach(pins.remove(0)).ok().unwrap(); bank.attach(pins.remove(0)).ok().unwrap(); (AsyncBank::new(bank).ok().unwrap(), a, hardware) } #[test] fn handoff_sequence_max_refuses_without_preparing_or_releasing() { let (mut bank, a, hardware) = setup(); bank.sequence = u64::MAX; let before = bank.bank.snapshot(); assert!(matches!(bank.start(a), Err(Error::Exhausted))); assert_eq!(bank.bank.snapshot(), before); assert!(bank.pending.is_none()); assert_eq!(hardware.borrow().prepares, 0); assert_eq!(hardware.borrow().pins, 2); assert!(bank.quiescent().is_ok()); } #[test] fn foreign_nonce_or_changed_generation_never_departs_actual_prepared() { for field in 0..3 { let (mut bank, a, hardware) = setup(); let mut token = bank.start(a).unwrap(); let prepared = bank.bank.retained_switch.as_ref().unwrap().nonce; match field { 0 => token.bank += 1, 1 => token.nonce += 1, _ => bank.bank.slots[a.slot].generation += 1, } assert!(matches!( bank.depart(token), Err(HandoffFailure::Before(Error::Stale, _)) )); assert_eq!(bank.bank.retained_switch.as_ref().unwrap().nonce, prepared); assert_eq!(bank.stage(), Some(HandoffStage::Staged)); assert!(bank.quiescent().is_err()); assert_eq!(hardware.borrow().pins, 2); assert_eq!(hardware.borrow().aborts, 0); } } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum HandoffStage { Staged, Departed, } /// Nonclone lookup authority for one sealed handoff. Every actual binding and /// Prepared remains in the bank; forgetting this token leaves them retained. pub struct OwnedHandoff { bank: u64, nonce: u64, } impl OwnedHandoff { pub fn nonce(&self) -> u64 { self.nonce } } pub enum HandoffFailure { Before(Error, OwnedHandoff), Quarantined(Error), } #[derive(Clone, Copy)] struct HandoffRecord { nonce: u64, stage: HandoffStage, source: Option, target: usize, source_generation: Option, target_generation: u64, capture: Option, image: Image, epoch: u64, old_epoch: u64, consume_deferred: bool, disposition: Disposition, } pub struct AsyncBank { bank: ContextBank, sequence: u64, pending: Option, quarantined: bool, } impl AsyncBank { #[allow(clippy::result_large_err)] pub fn new(bank: ContextBank) -> Result)> { if bank.faulted || bank.retained_switch.is_some() || bank.retained_idle.is_some() { return Err((Error::Quarantined, bank)); } Ok(Self { bank, sequence: 0, pending: None, quarantined: false, }) } pub fn bank(&self) -> &ContextBank { &self.bank } pub fn stage(&self) -> Option { self.pending.map(|p| p.stage) } pub fn quarantined(&self) -> bool { self.quarantined } /// No mutable escape is possible during any part of the staged interval. pub fn quiescent(&mut self) -> Result<&mut ContextBank, Error> { if self.pending.is_some() || self.quarantined || self.bank.faulted { return Err(Error::Pending); } Ok(&mut self.bank) } fn seal(dispatch: Dispatch<'_, B, N>, nonce: u64) -> OwnedHandoff { let mut d = dispatch; let bank_id = d.bank.identity; let record = HandoffRecord { nonce, stage: HandoffStage::Staged, source: d.source, target: d.target, source_generation: d.source.map(|i| d.bank.slots[i].generation), target_generation: d.bank.slots[d.target].generation, capture: d.capture, image: d.image, epoch: d.epoch, old_epoch: d.bank.epoch, consume_deferred: d.consume_deferred, disposition: d.disposition, }; // The wrapper receives this record separately. Prepared and capture // stay physically owned inside the existing frozen bank fields. d.bank.retained_switch = d.prepared.take(); d.bank.uncertain_capture = record.capture; d.bank.faulted = true; OwnedHandoff { bank: bank_id, nonce, } } fn prepare_with( &mut self, make: impl FnOnce(&mut ContextBank) -> Result, Error>, ) -> Result { if self.pending.is_some() || self.quarantined || self.bank.faulted { return Err(Error::Pending); } let nonce = self.sequence.checked_add(1).ok_or(Error::Exhausted)?; let d = make(&mut self.bank)?; // The frozen prepare already validated source capture and target. // Departure repeats before() while retaining the same staged token on // failure, including if a real capture changes after preparation. let record = HandoffRecord { nonce, stage: HandoffStage::Staged, source: d.source, target: d.target, source_generation: d.source.map(|i| d.bank.slots[i].generation), target_generation: d.bank.slots[d.target].generation, capture: d.capture, image: d.image, epoch: d.epoch, old_epoch: d.bank.epoch, consume_deferred: d.consume_deferred, disposition: d.disposition, }; let token = Self::seal(d, nonce); self.sequence = nonce; self.pending = Some(record); Ok(token) } pub fn start(&mut self, target: Handle) -> Result { self.prepare_with(|bank| bank.start(target)) } pub fn cooperate( &mut self, source: Handle, target: Handle, capture: Capture, ) -> Result { self.prepare_with(|bank| bank.cooperate(source, target, capture)) } pub fn sleep_to( &mut self, source: Handle, target: Handle, capture: Capture, ) -> Result { self.prepare_with(|bank| bank.sleep_to(source, target, capture)) } /// Safe-point deferred timer scope only: preserve the original848 IRQ /// witness, and prepare with a fresh640 capture after intervening work. pub fn service_deferred(&mut self, capture: Capture) -> Result { self.prepare_with(|bank| bank.service_deferred(capture)) } pub fn target_image(&self, token: &OwnedHandoff) -> Result { Ok(self.validate_token(token)?.image) } pub fn with_prepared_observation(&self, f: impl FnOnce(Option<&B::Prepared>) -> R) -> R { f(self.bank.retained_switch.as_ref()) } fn validate_token(&self, t: &OwnedHandoff) -> Result { let p = self.pending.ok_or(Error::Pending)?; if self.quarantined || !self.bank.faulted || self.bank.retained_switch.is_none() { return Err(Error::Quarantined); } if t.bank != self.bank.identity || t.nonce != p.nonce || self.bank.epoch != p.old_epoch || self.bank.slots[p.target].generation != p.target_generation || p.source .is_some_and(|i| Some(self.bank.slots[i].generation) != p.source_generation) { return Err(Error::Stale); } Ok(p) } fn before_departure(&mut self, p: HandoffRecord) -> Result<(), Error> { self.bank.faulted = false; let result = { let d = Dispatch { bank: &mut self.bank, source: p.source, target: p.target, capture: p.capture, image: p.image, epoch: p.epoch, prepared: None, consume_deferred: p.consume_deferred, disposition: p.disposition, }; d.before() }; self.bank.faulted = true; result } fn quarantine(&mut self, p: HandoffRecord) { if let Some(i) = p.source { self.bank.slots[i].state = State::Quarantined; } self.bank.slots[p.target].state = State::Quarantined; self.bank.faulted = true; self.quarantined = true; } #[allow(clippy::result_large_err)] pub fn depart(&mut self, token: OwnedHandoff) -> Result { let p = match self.validate_token(&token) { Ok(p) => p, Err(e) => return Err(HandoffFailure::Before(e, token)), }; if p.stage != HandoffStage::Staged { return Err(HandoffFailure::Before(Error::State, token)); } if let Err(e) = self.before_departure(p) { return Err(HandoffFailure::Before(e, token)); } let result = self.bank.backend.depart( self.bank.retained_switch.as_mut().unwrap(), p.source.and_then(|i| self.bank.slots[i].binding.as_ref()), p.nonce, p.disposition, ); if let Err(f) = result { return match f.phase { FailurePhase::BeforeSwitch => Err(HandoffFailure::Before(f.error, token)), FailurePhase::AfterSwitch => { self.quarantine(p); Err(HandoffFailure::Quarantined(f.error)) } }; } self.pending.as_mut().unwrap().stage = HandoffStage::Departed; Ok(token) } pub fn landed(&mut self, token: OwnedHandoff) -> Result<(), HandoffFailure> { let p = match self.validate_token(&token) { Ok(p) => p, Err(e) => return Err(HandoffFailure::Before(e, token)), }; if p.stage != HandoffStage::Departed { return Err(HandoffFailure::Before(Error::State, token)); } let result = self.bank.backend.arrive( self.bank.retained_switch.as_mut().unwrap(), self.bank.slots[p.target].binding.as_ref().unwrap(), &p.image, p.nonce, ); if let Err(e) = result { self.quarantine(p); return Err(HandoffFailure::Quarantined(e)); } self.bank.faulted = false; let checked = { let d = Dispatch { bank: &mut self.bank, source: p.source, target: p.target, capture: p.capture, image: p.image, epoch: p.epoch, prepared: None, consume_deferred: p.consume_deferred, disposition: p.disposition, }; d.completed() }; self.bank.faulted = true; if let Err(e) = checked { self.quarantine(p); return Err(HandoffFailure::Quarantined(e)); } let prepared = self.bank.retained_switch.take().unwrap(); if let Err((e, prepared)) = self.bank.backend.finish(prepared) { self.bank.retained_switch = Some(prepared); self.quarantine(p); return Err(HandoffFailure::Quarantined(e)); } if let Some(i) = p.source { let c = p.capture.unwrap(); self.bank.slots[i].image = c.image; self.bank.slots[i].last_capture = c.episode; self.bank.slots[i].state = State::Ready; } self.bank.slots[p.target].state = State::Active; self.bank.active = Some(self.bank.handle(p.target)); self.bank.epoch = p.epoch; if p.consume_deferred { self.bank.last_deferred = self.bank.deferred_record(); self.bank.deferred = None; } self.bank.uncertain_capture = None; self.bank.faulted = false; self.pending = None; Ok(()) } /// A staged transaction can abort before any Table/CPU change. After /// departure it must remain retained; no rollback or implicit free occurs. pub fn abort(&mut self, token: OwnedHandoff) -> Result<(), HandoffFailure> { let p = match self.validate_token(&token) { Ok(p) => p, Err(e) => return Err(HandoffFailure::Before(e, token)), }; if p.stage != HandoffStage::Staged { return Err(HandoffFailure::Before(Error::State, token)); } let prepared = self.bank.retained_switch.take().unwrap(); if let Err((e, prepared)) = self.bank.backend.abort(prepared) { self.bank.retained_switch = Some(prepared); self.quarantine(p); return Err(HandoffFailure::Quarantined(e)); } self.bank.uncertain_capture = None; self.bank.faulted = false; self.pending = None; Ok(()) } } // Separate source-bound extension after the frozen Context and AsyncBank bytes. // Returning to the inherited bootstrap stack owns a transaction independently // of ordinary two-thread dispatch, and never invents a bootstrap ThreadId. /// # Safety /// ReturnPrepared retains the actual source binding/capture and exact bootstrap /// stack/root/frame authority. Preparation and abort do not change execution. /// leave_return executes the frozen Table exit on the live source stack with /// IRQs masked. BeforeSwitch means every resource/Table/CPU byte is unchanged; /// any possible change is AfterSwitch. returned observes the actual bootstrap /// branch, SP/root/guard/canaries and full GP/SIMD/FP snapshot saved before Rust /// clobbers it, bound to the retained bootstrap640 bytes and this transaction. pub unsafe trait CompletionBackend: HandoffBackend { type ReturnPrepared; fn prepare_return( &mut self, source: &Self::Binding, capture: &Capture, ) -> Result; fn leave_return( &mut self, prepared: &mut Self::ReturnPrepared, source: &Self::Binding, ) -> Result<(), Failure>; fn returned(&self, prepared: &Self::ReturnPrepared) -> Result<(), Error>; /// Inert lookup of the original retained bootstrap root authority. Source /// bindings keep their distinct private roots through retirement. fn return_root(&self, prepared: &Self::ReturnPrepared) -> Result; fn finish_return( &mut self, prepared: Self::ReturnPrepared, ) -> Result<(), (Error, Self::ReturnPrepared)>; fn abort_return( &mut self, prepared: Self::ReturnPrepared, ) -> Result<(), (Error, Self::ReturnPrepared)>; } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ReturnStage { Staged, Departed, } pub struct OwnedReturn { bank: u64, nonce: u64, } impl OwnedReturn { pub fn nonce(&self) -> u64 { self.nonce } } pub enum ReturnFailure { Before(Error, OwnedReturn), Quarantined(Error), } struct ReturnTransaction

{ nonce: u64, source: Handle, capture: Capture, stage: ReturnStage, old_epoch: u64, next_epoch: u64, prepared: P, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ReturnObservation { pub nonce: u64, pub source: Handle, pub capture: Capture, pub stage: ReturnStage, } pub struct CompletionBank { handoff: AsyncBank, sequence: u64, pending: Option>, quarantined: bool, } impl CompletionBank { #[allow(clippy::result_large_err)] pub fn new(handoff: AsyncBank) -> Result)> { if handoff.pending.is_some() || handoff.quarantined || handoff.bank.faulted { return Err((Error::Pending, handoff)); } Ok(Self { handoff, sequence: 0, pending: None, quarantined: false, }) } pub fn handoff(&self) -> &AsyncBank { &self.handoff } pub fn quiescent(&mut self) -> Result<&mut AsyncBank, Error> { if self.pending.is_some() || self.quarantined { return Err(Error::Pending); } Ok(&mut self.handoff) } pub fn observation(&self) -> Option { self.pending.as_ref().map(|p| ReturnObservation { nonce: p.nonce, source: p.source, capture: p.capture, stage: p.stage, }) } pub fn with_return_observation(&self, f: impl FnOnce(Option<&B::ReturnPrepared>) -> R) -> R { f(self.pending.as_ref().map(|p| &p.prepared)) } pub fn quarantined(&self) -> bool { self.quarantined } pub fn prepare_return( &mut self, source: Handle, capture: Capture, ) -> Result { if self.pending.is_some() || self.quarantined || self.handoff.pending.is_some() || self.handoff.quarantined { return Err(Error::Pending); } let nonce = self.sequence.checked_add(1).ok_or(Error::Exhausted)?; let bank = &mut self.handoff.bank; let (i, r, e) = bank.current(source, false)?; if e.critical_depth != 0 || !e.safe_point || bank.deferred.is_some() { return Err(Error::Context); } bank.capture(i, r, capture, false, e)?; let next_epoch = bank.epoch.checked_add(1).ok_or(Error::Exhausted)?; let prepared = bank.backend.prepare_return( bank.slots[i].binding.as_ref().ok_or(Error::State)?, &capture, )?; self.pending = Some(ReturnTransaction { nonce, source, capture, stage: ReturnStage::Staged, old_epoch: bank.epoch, next_epoch, prepared, }); bank.uncertain_capture = Some(capture); bank.faulted = true; self.sequence = nonce; Ok(OwnedReturn { bank: bank.identity, nonce, }) } fn validate(&self, token: &OwnedReturn) -> Result<(), Error> { let p = self.pending.as_ref().ok_or(Error::Pending)?; let bank = &self.handoff.bank; if self.quarantined || !bank.faulted { return Err(Error::Quarantined); } if token.bank != bank.identity || token.nonce != p.nonce || bank.epoch != p.old_epoch || bank.active != Some(p.source) || bank.slots[p.source.slot].generation != p.source.generation || bank.uncertain_capture != Some(p.capture) { return Err(Error::Stale); } Ok(()) } fn before(&mut self) -> Result<(), Error> { let p = self.pending.as_ref().ok_or(Error::Pending)?; let bank = &mut self.handoff.bank; bank.faulted = false; let result = (|| { let (i, r, e) = bank.current(p.source, false)?; if e.critical_depth != 0 || !e.safe_point || bank.deferred.is_some() { return Err(Error::Context); } bank.capture(i, r, p.capture, false, e) })(); bank.faulted = true; result } fn quarantine(&mut self) { let p = self.pending.as_ref().unwrap(); self.handoff.bank.slots[p.source.slot].state = State::Quarantined; self.handoff.bank.faulted = true; self.quarantined = true; } pub fn depart_return(&mut self, token: OwnedReturn) -> Result { if let Err(e) = self.validate(&token) { return Err(ReturnFailure::Before(e, token)); } if self.pending.as_ref().unwrap().stage != ReturnStage::Staged { return Err(ReturnFailure::Before(Error::State, token)); } if let Err(e) = self.before() { return Err(ReturnFailure::Before(e, token)); } let p = self.pending.as_mut().unwrap(); let bank = &mut self.handoff.bank; let result = bank.backend.leave_return( &mut p.prepared, bank.slots[p.source.slot].binding.as_ref().unwrap(), ); if let Err(f) = result { return match f.phase { FailurePhase::BeforeSwitch => Err(ReturnFailure::Before(f.error, token)), FailurePhase::AfterSwitch => { self.quarantine(); Err(ReturnFailure::Quarantined(f.error)) } }; } self.pending.as_mut().unwrap().stage = ReturnStage::Departed; Ok(token) } pub fn return_landed(&mut self, token: OwnedReturn) -> Result<(), ReturnFailure> { if let Err(e) = self.validate(&token) { return Err(ReturnFailure::Before(e, token)); } let p = self.pending.as_ref().unwrap(); if p.stage != ReturnStage::Departed { return Err(ReturnFailure::Before(Error::State, token)); } let bank = &mut self.handoff.bank; let checked = (|| { bank.backend.returned(&p.prepared)?; bank.faulted = false; let e = bank.execution(false); bank.faulted = true; let e = e?; let r = bank.validate(p.source.slot)?; if r.state != thread::State::Dead || e.thread.is_some() || e.root != bank.backend.return_root(&p.prepared)? || e.sp >= r.layout.guard() && e.sp <= r.layout.base + r.layout.bytes { return Err(Error::Active); } Ok(()) })(); if let Err(e) = checked { self.quarantine(); return Err(ReturnFailure::Quarantined(e)); } let p = self.pending.take().unwrap(); if let Err((e, prepared)) = self.handoff.bank.backend.finish_return(p.prepared) { self.pending = Some(ReturnTransaction { prepared, ..p }); self.quarantine(); return Err(ReturnFailure::Quarantined(e)); } let bank = &mut self.handoff.bank; bank.slots[p.source.slot].image = p.capture.image; bank.slots[p.source.slot].last_capture = p.capture.episode; bank.slots[p.source.slot].state = State::Ready; bank.active = None; bank.epoch = p.next_epoch; bank.uncertain_capture = None; bank.faulted = false; Ok(()) } pub fn abort_return(&mut self, token: OwnedReturn) -> Result<(), ReturnFailure> { if let Err(e) = self.validate(&token) { return Err(ReturnFailure::Before(e, token)); } if self.pending.as_ref().unwrap().stage != ReturnStage::Staged { return Err(ReturnFailure::Before(Error::State, token)); } let p = self.pending.take().unwrap(); if let Err((e, prepared)) = self.handoff.bank.backend.abort_return(p.prepared) { self.pending = Some(ReturnTransaction { prepared, ..p }); self.quarantine(); return Err(ReturnFailure::Quarantined(e)); } self.handoff.bank.uncertain_capture = None; self.handoff.bank.faulted = false; Ok(()) } #[allow(clippy::result_large_err)] pub fn into_handoff(self) -> Result, (Error, Self)> { if self.pending.is_some() || self.quarantined { return Err((Error::Pending, self)); } Ok(self.handoff) } } impl AsyncBank { #[allow(clippy::result_large_err)] pub fn os_k1040_into_backend(self) -> Result { if self.pending.is_some() || self.quarantined { return Err((Error::Pending, self)); } match self.bank.into_backend() { Ok(backend) => Ok(backend), Err((error, bank)) => Err(( error, Self { bank, sequence: self.sequence, pending: self.pending, quarantined: self.quarantined, }, )), } } } #[cfg(test)] mod completion_tests { use super::*; use crate::test_support as saved; const SOURCE_ROOT: u64 = 0x43000000; const BOOTSTRAP_ROOT: u64 = 0x44000000; struct Device { inner: saved::Backend, root: core::cell::Cell, } impl core::ops::Deref for Device { type Target = saved::Backend; fn deref(&self) -> &Self::Target { &self.inner } } impl core::ops::DerefMut for Device { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } } unsafe impl Backend for Device { type Binding = saved::Binding; type Prepared = saved::Prepared; type IdlePrepared = saved::IdlePrepared; fn execution(&self) -> Execution { Execution { root: self.root.get(), ..self.inner.execution() } } fn record(&self, b: &Self::Binding) -> Result { self.inner.record(b) } fn stack(&self, b: &Self::Binding) -> Result { self.inner.stack(b) } fn root(&self, _: &Self::Binding) -> Result { Ok(SOURCE_ROOT) } fn executable(&self, pc: u64) -> bool { self.inner.executable(pc) } fn initial_return_pc(&self, b: &Self::Binding) -> Result { self.inner.initial_return_pc(b) } fn capture_matches(&self, b: &Self::Binding, c: &Capture) -> bool { self.inner.capture_matches(b, c) } fn registers(&self) -> Registers { self.inner.registers() } fn prepare( &mut self, s: Option<&Self::Binding>, t: &Self::Binding, i: &Image, d: Disposition, ) -> Result { self.inner.prepare(s, t, i, d) } fn commit( &mut self, p: &mut Self::Prepared, s: Option<&Self::Binding>, t: &Self::Binding, i: &Image, d: Disposition, ) -> Result<(), Failure> { self.inner.commit(p, s, t, i, d) } fn finish(&mut self, p: Self::Prepared) -> Result<(), (Error, Self::Prepared)> { self.inner.finish(p) } fn abort(&mut self, p: Self::Prepared) -> Result<(), (Error, Self::Prepared)> { self.inner.abort(p) } fn release(&mut self, p: Self::Binding) -> Result<(), (Error, Self::Binding)> { self.inner.release(p) } fn ready_empty(&self) -> bool { self.inner.ready_empty() } fn wake_epoch(&self) -> u64 { self.inner.wake_epoch() } fn idle_prepare(&mut self, e: u64) -> Result { self.inner.idle_prepare(e) } fn idle_commit( &mut self, p: &mut Self::IdlePrepared, e: u64, ) -> Result { self.inner.idle_commit(p, e) } fn idle_finish( &mut self, p: Self::IdlePrepared, ) -> Result<(), (Error, Self::IdlePrepared)> { self.inner.idle_finish(p) } fn idle_abort(&mut self, p: Self::IdlePrepared) -> Result<(), (Error, Self::IdlePrepared)> { self.inner.idle_abort(p) } } unsafe impl HandoffBackend for Device { fn depart( &mut self, p: &mut Self::Prepared, s: Option<&Self::Binding>, n: u64, d: Disposition, ) -> Result<(), Failure> { self.inner.depart(p, s, n, d) } fn arrive( &mut self, p: &mut Self::Prepared, t: &Self::Binding, i: &Image, n: u64, ) -> Result<(), Error> { self.inner.arrive(p, t, i, n) } } pub(crate) struct ReturnPrepared { nonce: u64, source: thread::ThreadId, expected: Registers, bootstrap_root: u64, } fn bootstrap() -> Registers { Registers { sp: 0x90004000, x19_x30: core::array::from_fn(|i| 0x1040 + i as u64), q: core::array::from_fn(|i| [0x104000 + i as u64, 0x204000 + i as u64]), fpcr: 0x400000, fpsr: 0x8000000, pc: saved::ENTRY + 0x900, argument: 0, timer: None, } } unsafe impl CompletionBackend for Device { type ReturnPrepared = ReturnPrepared; fn prepare_return( &mut self, source: &Self::Binding, _: &Capture, ) -> Result { let mut h = self.hardware.borrow_mut(); if h.prepare_fail { return Err(Error::Backend); } h.prepares += 1; Ok(ReturnPrepared { nonce: h.prepares as u64, source: source.id, expected: bootstrap(), bootstrap_root: BOOTSTRAP_ROOT, }) } fn leave_return( &mut self, p: &mut ReturnPrepared, source: &Self::Binding, ) -> Result<(), Failure> { let fail = self.hardware.borrow().commit_fail; if fail == Some(FailurePhase::BeforeSwitch) { return Err(Failure { error: Error::Backend, phase: FailurePhase::BeforeSwitch, }); } assert_eq!(p.source, source.id); if self.hardware.borrow().bad_registers != 9 { self.table.borrow_mut().exit_current(source.id).unwrap(); } self.hardware.borrow_mut().current = None; if fail == Some(FailurePhase::AfterSwitch) { return Err(Failure { error: Error::Backend, phase: FailurePhase::AfterSwitch, }); } Ok(()) } fn returned(&self, p: &ReturnPrepared) -> Result<(), Error> { let h = self.hardware.borrow(); if h.current.is_some() || h.sp >= p.expected.sp || h.sp < p.expected.sp - 1024 || self.registers != p.expected { return Err(Error::Capture); } Ok(()) } fn return_root(&self, p: &ReturnPrepared) -> Result { Ok(p.bootstrap_root) } fn finish_return(&mut self, p: ReturnPrepared) -> Result<(), (Error, ReturnPrepared)> { let mut h = self.hardware.borrow_mut(); if h.finish_fail { return Err((Error::Backend, p)); } h.finishes += 1; Ok(()) } fn abort_return(&mut self, p: ReturnPrepared) -> Result<(), (Error, ReturnPrepared)> { let mut h = self.hardware.borrow_mut(); if h.abort_fail { return Err((Error::Backend, p)); } h.aborts += 1; Ok(()) } } type TestBank = CompletionBank; type Hardware = std::rc::Rc>; fn setup() -> (TestBank, [Handle; 2], Hardware, Capture) { let (mut bank, handles, h) = saved::setup(); saved::finish(bank.start(handles[0]).unwrap()); let c = saved::capture(&bank, handles[0], &h, 43, false); // Move every original private token into a model with distinct closed // source/bootstrap roots; no resource is reconstructed or released. let mut slots = bank.slots; for slot in &mut slots { if slot.binding.is_some() { slot.root = SOURCE_ROOT; } } let bank = ContextBank { backend: Device { inner: bank.backend, root: core::cell::Cell::new(SOURCE_ROOT), }, identity: bank.identity, cpu: bank.cpu, slots, active: bank.active, epoch: bank.epoch, deferred: bank.deferred, last_deferred: bank.last_deferred, faulted: bank.faulted, retained_switch: bank.retained_switch, retained_idle: bank.retained_idle, uncertain_capture: bank.uncertain_capture, _local: bank._local, }; ( CompletionBank::new(AsyncBank::new(bank).ok().unwrap()) .ok() .unwrap(), handles, h, c, ) } // Explicit host hardware model. This stores the modeled CPU observation // before the bootstrap callback; it is never an assembly execution claim. fn physical_step(bank: &mut TestBank) { let d = &mut bank.handoff.bank.backend; d.root.set(BOOTSTRAP_ROOT); let mut h = d.inner.hardware.borrow_mut(); h.sp = bootstrap().sp - 64; h.current = None; d.inner.registers = bootstrap(); } fn land(bank: &mut TestBank, token: OwnedReturn) { let token = bank.depart_return(token).ok().unwrap(); physical_step(bank); assert!(bank.return_landed(token).is_ok()); } #[test] fn actual_modeled_bootstrap_landing_precedes_publication_and_pin_release() { let (mut bank, [a, b], h, c) = setup(); let token = bank.prepare_return(a, c).unwrap(); assert!(bank.quiescent().is_err()); assert_eq!(bank.handoff.bank.record(a).unwrap().last_capture, 0); let token = bank.depart_return(token).ok().unwrap(); assert_eq!(bank.handoff.bank.record(a).unwrap().state, State::Active); assert_eq!(bank.handoff.bank.record(a).unwrap().last_capture, 0); assert_eq!(h.borrow().pins, 2); physical_step(&mut bank); assert!(bank.return_landed(token).is_ok()); assert_eq!(bank.handoff.bank.snapshot().active, 0); assert_eq!(bank.handoff.bank.record(a).unwrap().last_capture, c.episode); let q = bank.quiescent().unwrap().quiescent().unwrap(); let aid = q.record(a).unwrap().thread; q.retire(a).unwrap(); q.backend.table.borrow_mut().reap(aid).unwrap(); // The real frozen Table owns stack bytes independently of Context pins. let bid = q.record(b).unwrap().thread; q.backend.table.borrow_mut().cancel(bid).unwrap(); q.retire(b).unwrap(); q.backend.table.borrow_mut().reap(bid).unwrap(); assert_eq!(h.borrow().pins, 0); assert_eq!(h.borrow().stack_pages, 0); let handoff = bank.into_handoff().ok().unwrap(); assert!(handoff.os_k1040_into_backend().is_ok()); } #[test] fn before_exit_returns_same_owned_return_and_prepared_without_state_change() { let (mut bank, [a, _], h, c) = setup(); let token = bank.prepare_return(a, c).unwrap(); let nonce = token.nonce(); let p = bank.with_return_observation(|p| p.unwrap().nonce); h.borrow_mut().commit_fail = Some(FailurePhase::BeforeSwitch); let token = match bank.depart_return(token) { Err(ReturnFailure::Before(Error::Backend, t)) => t, _ => panic!(), }; assert_eq!(token.nonce(), nonce); assert_eq!(bank.with_return_observation(|p| p.unwrap().nonce), p); assert_eq!(bank.handoff.bank.record(a).unwrap().state, State::Active); assert_eq!( h.borrow().current, Some(bank.handoff.bank.record(a).unwrap().thread) ); h.borrow_mut().commit_fail = None; land(&mut bank, token); } #[test] fn after_exit_retains_prepared_full_source_capture_and_typed_bindings() { let (mut bank, [a, _], h, c) = setup(); let token = bank.prepare_return(a, c).unwrap(); let p = bank.with_return_observation(|p| p.unwrap().nonce); h.borrow_mut().commit_fail = Some(FailurePhase::AfterSwitch); assert!(matches!( bank.depart_return(token), Err(ReturnFailure::Quarantined(Error::Backend)) )); assert!(bank.quarantined()); assert!(bank.quiescent().is_err()); assert_eq!(bank.with_return_observation(|x| x.unwrap().nonce), p); assert_eq!(bank.handoff.bank.quarantined_capture(), Some(c)); assert_eq!(bank.handoff.bank.record(a).unwrap().last_capture, 0); assert_eq!(h.borrow().pins, 2); assert_eq!(h.borrow().stack_pages, 8); assert!(h.borrow().current.is_none()); } #[test] fn wrong_actual_bootstrap_pc_q31_fpcr_sp_or_current_identity_quarantines() { for bad in 0..5 { let (mut bank, [a, _], h, c) = setup(); let token = bank.prepare_return(a, c).unwrap(); let token = bank.depart_return(token).ok().unwrap(); physical_step(&mut bank); match bad { 0 => bank.handoff.bank.backend.registers.pc += 4, 1 => bank.handoff.bank.backend.registers.q[31][1] ^= 1, 2 => bank.handoff.bank.backend.registers.fpcr ^= 1, 3 => h.borrow_mut().sp = c.address, _ => h.borrow_mut().current = Some(bank.handoff.bank.record(a).unwrap().thread), } assert!(matches!( bank.return_landed(token), Err(ReturnFailure::Quarantined(_)) )); assert!(bank.quarantined()); assert_eq!(h.borrow().pins, 2); assert_eq!(bank.handoff.bank.record(a).unwrap().last_capture, 0); assert_eq!(bank.observation().unwrap().capture, c); } } #[test] fn distinct_bootstrap_root_is_required_while_source_private_root_stays_owned() { for wrong_root in [SOURCE_ROOT, BOOTSTRAP_ROOT + 4096] { let (mut bank, [a, _], h, c) = setup(); assert_eq!(bank.handoff.bank.record(a).unwrap().root, SOURCE_ROOT); let token = bank.prepare_return(a, c).unwrap(); assert_eq!( bank.with_return_observation(|p| p.unwrap().bootstrap_root), BOOTSTRAP_ROOT ); let token = bank.depart_return(token).ok().unwrap(); physical_step(&mut bank); bank.handoff.bank.backend.root.set(wrong_root); assert!(matches!( bank.return_landed(token), Err(ReturnFailure::Quarantined(Error::Active)) )); assert!(bank.quarantined()); assert_eq!(bank.handoff.bank.record(a).unwrap().root, SOURCE_ROOT); assert_eq!(bank.handoff.bank.record(a).unwrap().last_capture, 0); assert_eq!(h.borrow().pins, 2); assert_eq!(bank.observation().unwrap().capture, c); } } #[test] fn bootstrap_landing_without_actual_frozen_table_exit_refuses_publication() { let (mut bank, [a, _], h, c) = setup(); let token = bank.prepare_return(a, c).unwrap(); h.borrow_mut().bad_registers = 9; let token = bank.depart_return(token).ok().unwrap(); physical_step(&mut bank); assert!(matches!( bank.return_landed(token), Err(ReturnFailure::Quarantined(Error::Active)) )); assert!(bank.quarantined()); assert_eq!(bank.handoff.bank.record(a).unwrap().last_capture, 0); assert_eq!( bank.handoff .bank .backend .table .borrow() .record(bank.handoff.bank.record(a).unwrap().thread) .unwrap() .state, thread::State::Running ); assert_eq!(h.borrow().pins, 2); assert_eq!(bank.observation().unwrap().capture, c); } #[test] fn finish_and_abort_failure_keep_the_exact_return_prepared() { for finish in [false, true] { let (mut bank, [a, _], h, c) = setup(); let token = bank.prepare_return(a, c).unwrap(); let p = bank.with_return_observation(|p| p.unwrap().nonce); let result = if finish { let token = bank.depart_return(token).ok().unwrap(); physical_step(&mut bank); h.borrow_mut().finish_fail = true; bank.return_landed(token) } else { h.borrow_mut().abort_fail = true; bank.abort_return(token) }; assert!(matches!( result, Err(ReturnFailure::Quarantined(Error::Backend)) )); assert_eq!(bank.with_return_observation(|x| x.unwrap().nonce), p); assert_eq!(bank.handoff.bank.record(a).unwrap().last_capture, 0); assert_eq!(bank.observation().unwrap().capture, c); assert_eq!(h.borrow().pins, 2); } } #[test] fn stale_capture_before_exit_keeps_prepared_and_same_token() { let (mut bank, [a, _], h, c) = setup(); let token = bank.prepare_return(a, c).unwrap(); let nonce = token.nonce(); h.borrow_mut().capture = None; let token = match bank.depart_return(token) { Err(ReturnFailure::Before(Error::Capture, t)) => t, _ => panic!(), }; assert_eq!(token.nonce(), nonce); assert!(h.borrow().current.is_some()); h.borrow_mut().capture = Some(c); land(&mut bank, token); } #[test] fn return_sequence_and_epoch_exhaustion_preserve_source_and_inventory() { for epoch in [false, true] { let (mut bank, [a, _], h, c) = setup(); if epoch { bank.handoff.bank.epoch = u64::MAX; } else { bank.sequence = u64::MAX; } let before = bank.handoff.bank.snapshot(); let count = h.borrow().prepares; assert!(matches!(bank.prepare_return(a, c), Err(Error::Exhausted))); assert_eq!(bank.handoff.bank.snapshot(), before); assert_eq!(h.borrow().prepares, count); assert_eq!(h.borrow().pins, 2); assert!(bank.pending.is_none()); assert!(bank.quiescent().is_ok()); } } #[test] #[allow(clippy::drop_non_drop)] fn wrong_stage_or_forgotten_return_retains_all_ownership() { let (mut bank, [a, _], h, c) = setup(); let token = bank.prepare_return(a, c).unwrap(); let token = match bank.return_landed(token) { Err(ReturnFailure::Before(Error::State, t)) => t, _ => panic!(), }; let token = bank.depart_return(token).ok().unwrap(); let token = match bank.abort_return(token) { Err(ReturnFailure::Before(Error::State, t)) => t, _ => panic!(), }; drop(token); assert_eq!(bank.observation().unwrap().stage, ReturnStage::Departed); assert!(bank.quiescent().is_err()); assert_eq!(h.borrow().pins, 2); assert_eq!(bank.observation().unwrap().capture, c); } } // NEW source-bound extension after the frozen Context/Async/Completion bytes. // The private binding remains in ContextBank while its actual inactive frozen // Table record is cancelled. No mutable backend or owner reference escapes. /// # Safety /// cancel_inactive may cancel only this exact private binding's live inactive /// Ready/Sleeping Table record, with IRQs masked. BeforeSwitch guarantees that /// every actual resource, Table record and CPU observation is unchanged. Any /// possible change or failed observation is AfterSwitch; all owners remain /// retained in the backend. It never releases the binding or its pages. pub unsafe trait InactiveBackend: Backend { fn cancel_inactive(&mut self, binding: &Self::Binding) -> Result<(), Failure>; } impl ContextBank { pub fn os_k1040_cancel_inactive(&mut self, handle: Handle) -> Result<(), Error> { let execution = self.execution(false)?; let index = self.index(handle)?; if execution.critical_depth != 0 || !execution.safe_point || self.active.is_some() || self.deferred.is_some() || self.retained_switch.is_some() || self.retained_idle.is_some() || self.slots[index].state != State::Ready { return Err(Error::Active); } let record = self.validate(index)?; if !matches!(record.state, thread::State::Ready | thread::State::Sleeping) || execution.thread == Some(record.id) || execution.sp >= record.layout.guard() && execution.sp <= record.layout.base + record.layout.bytes { return Err(Error::Active); } let result = self .backend .cancel_inactive(self.slots[index].binding.as_ref().ok_or(Error::State)?); match result { Err(Failure { error, phase: FailurePhase::BeforeSwitch, }) => return Err(error), Err(Failure { error, phase: FailurePhase::AfterSwitch, }) => { self.slots[index].state = State::Quarantined; self.faulted = true; return Err(error); } Ok(()) => {} } let after_execution = self.execution(false); if !after_execution.is_ok_and(|actual| { actual.root == execution.root && actual.thread == execution.thread && actual.daif == execution.daif && actual.critical_depth == 0 && actual.safe_point && !(actual.sp >= record.layout.guard() && actual.sp <= record.layout.base + record.layout.bytes) }) || !self.validate(index).is_ok_and(|actual| { actual.id == record.id && actual.layout == record.layout && actual.state == thread::State::Dead }) { self.slots[index].state = State::Quarantined; self.faulted = true; return Err(Error::Backend); } Ok(()) } } #[cfg(test)] mod retirement_tests { use super::*; use crate::test_support as saved; unsafe impl InactiveBackend for saved::Backend { fn cancel_inactive(&mut self, binding: &Self::Binding) -> Result<(), Failure> { let before = |error| Failure { error, phase: FailurePhase::BeforeSwitch, }; let fault = self.hardware.borrow().commit_fail; if fault == Some(FailurePhase::BeforeSwitch) { return Err(before(Error::Backend)); } if self.hardware.borrow().bad_registers == 99 { return Ok(()); // Deliberately broken backend observation. } self.table .borrow_mut() .cancel(binding.id) .map_err(|_| before(Error::State))?; if self .sleep_wait .as_ref() .is_some_and(|wait| wait.thread() == binding.id) { self.sleep_wait = None; } let bad = self.hardware.borrow().bad_registers; match bad { 100 => self.hardware.borrow_mut().cpu = 1, 101 => self.hardware.borrow_mut().daif &= !128, 102 => self.hardware.borrow_mut().current = Some(binding.id), 103 => { let layout = self.table.borrow().record(binding.id).unwrap().layout; self.hardware.borrow_mut().sp = layout.usable_bottom() + 16; } _ => {} } if fault == Some(FailurePhase::AfterSwitch) { return Err(Failure { error: Error::Backend, phase: FailurePhase::AfterSwitch, }); } Ok(()) } } #[test] fn inactive_cancel_retains_actual_stack_until_real_reap() { let (mut bank, handles, hardware) = saved::setup(); let id = bank.record(handles[1]).unwrap().thread; assert_eq!(bank.os_k1040_cancel_inactive(handles[1]), Ok(())); bank.with_backend_observation(|b| { assert_eq!( b.table.borrow().record(id).unwrap().state, thread::State::Dead ) }); assert_eq!(hardware.borrow().pins, 2); assert_eq!(hardware.borrow().stack_pages, 8); assert_eq!(bank.retire(handles[1]), Ok(())); assert_eq!(hardware.borrow().pins, 1); assert_eq!(hardware.borrow().stack_pages, 8); bank.with_backend_observation(|backend| backend.table.borrow_mut().reap(id).unwrap()); assert_eq!(hardware.borrow().stack_pages, 4); } #[test] fn before_failure_keeps_exact_metadata_and_binding_for_retry() { let (mut bank, handles, hardware) = saved::setup(); let original = bank.snapshot(); let record = bank.record(handles[1]).unwrap(); hardware.borrow_mut().commit_fail = Some(FailurePhase::BeforeSwitch); assert_eq!( bank.os_k1040_cancel_inactive(handles[1]), Err(Error::Backend) ); assert_eq!(bank.snapshot(), original); assert_eq!(bank.record(handles[1]), Ok(record)); assert_eq!(hardware.borrow().pins, 2); hardware.borrow_mut().commit_fail = None; assert_eq!(bank.os_k1040_cancel_inactive(handles[1]), Ok(())); } #[test] fn changed_table_failure_retains_and_quarantines_actual_owner() { let (mut bank, handles, hardware) = saved::setup(); let id = bank.record(handles[1]).unwrap().thread; hardware.borrow_mut().commit_fail = Some(FailurePhase::AfterSwitch); assert_eq!( bank.os_k1040_cancel_inactive(handles[1]), Err(Error::Backend) ); assert!(bank.snapshot().faulted); assert_eq!(bank.snapshot().quarantined, 1); assert_eq!(bank.snapshot().retained_bindings, 2); bank.with_backend_observation(|b| { assert_eq!( b.table.borrow().record(id).unwrap().state, thread::State::Dead ) }); assert_eq!(bank.retire(handles[1]), Err(Error::Quarantined)); assert_eq!(hardware.borrow().pins, 2); assert_eq!(hardware.borrow().stack_pages, 8); } #[test] fn false_success_without_actual_dead_table_quarantines() { let (mut bank, handles, hardware) = saved::setup(); hardware.borrow_mut().bad_registers = 99; assert_eq!( bank.os_k1040_cancel_inactive(handles[1]), Err(Error::Backend) ); assert!(bank.snapshot().faulted); assert_eq!(bank.snapshot().retained_bindings, 2); assert_eq!(hardware.borrow().pins, 2); } #[test] fn live_active_continuation_cannot_be_cancelled() { let (mut bank, handles, hardware) = saved::setup(); saved::finish(bank.start(handles[0]).unwrap()); let before = bank.snapshot(); assert_eq!( bank.os_k1040_cancel_inactive(handles[0]), Err(Error::Active) ); assert_eq!( bank.os_k1040_cancel_inactive(handles[1]), Err(Error::Active) ); assert_eq!(bank.snapshot(), before); assert_eq!(hardware.borrow().pins, 2); } #[test] fn corrupt_physical_guard_refuses_before_table_change() { let (mut bank, handles, hardware) = saved::setup(); let id = bank.record(handles[1]).unwrap().thread; let before = bank.snapshot(); hardware.borrow_mut().corrupt_guard = true; assert_eq!(bank.os_k1040_cancel_inactive(handles[1]), Err(Error::Guard)); assert_eq!(bank.snapshot(), before); bank.with_backend_observation(|b| { assert_eq!( b.table.borrow().record(id).unwrap().state, thread::State::Ready ) }); assert_eq!(hardware.borrow().pins, 2); } #[test] fn sleeping_cancel_revokes_real_wait_without_manufactured_wake() { let (mut bank, [a, b], hardware) = saved::setup_idle(); let a_id = bank.record(a).unwrap().thread; let b_id = bank.record(b).unwrap().thread; bank.backend.table.borrow_mut().exit_current(a_id).unwrap(); bank.active = None; bank.slots[a.slot].state = State::Ready; { let mut h = hardware.borrow_mut(); h.current = None; h.sp = 0x90001000; } assert!( bank.backend .sleep_wait .as_ref() .is_some_and(|wait| wait.thread() == b_id) ); assert_eq!(bank.os_k1040_cancel_inactive(b), Ok(())); assert!(bank.backend.sleep_wait.is_none()); assert_eq!( bank.backend.table.borrow().record(b_id).unwrap().state, thread::State::Dead ); assert_eq!(hardware.borrow().wake_epoch, 1); assert_eq!(hardware.borrow().pins, 2); assert_eq!(hardware.borrow().stack_pages, 8); } #[test] fn stale_and_foreign_handles_preserve_actual_table_inventory() { let (mut bank, handles, hardware) = saved::setup(); let (other, foreign, _) = saved::setup(); let before = bank.snapshot(); assert_eq!(bank.os_k1040_cancel_inactive(foreign[1]), Err(Error::Stale)); assert_eq!(bank.snapshot(), before); bank.slots[handles[1].slot].generation += 1; assert_eq!(bank.os_k1040_cancel_inactive(handles[1]), Err(Error::Stale)); assert_eq!(hardware.borrow().pins, 2); assert_eq!(bank.backend.table.borrow().snapshot().ready, 2); assert_eq!(other.snapshot().retained_bindings, 2); } #[test] fn cpu_irq_and_mask_refusals_do_not_cancel_or_release() { for field in 0..3 { let (mut bank, handles, hardware) = saved::setup(); let before = bank.snapshot(); match field { 0 => hardware.borrow_mut().cpu = 1, 1 => hardware.borrow_mut().in_irq = true, _ => hardware.borrow_mut().daif &= !128, } assert_eq!( bank.os_k1040_cancel_inactive(handles[1]), Err(Error::Context) ); assert_eq!(bank.snapshot(), before); assert_eq!(hardware.borrow().pins, 2); assert_eq!(bank.backend.table.borrow().snapshot().ready, 2); } } #[test] fn canary_and_live_source_sp_refusals_preserve_table_and_pin() { for field in 0..2 { let (mut bank, handles, hardware) = saved::setup(); let before = bank.snapshot(); let expected = if field == 0 { hardware.borrow_mut().corrupt_canary = true; Error::Canary } else { let id = bank.record(handles[1]).unwrap().thread; let record = bank.backend.table.borrow().record(id).unwrap(); hardware.borrow_mut().sp = record.layout.usable_bottom() + 16; Error::Active }; assert_eq!(bank.os_k1040_cancel_inactive(handles[1]), Err(expected)); assert_eq!(bank.snapshot(), before); assert_eq!(hardware.borrow().pins, 2); assert_eq!(bank.backend.table.borrow().snapshot().ready, 2); } } #[test] fn actual_cancel_then_changed_cpu_mask_current_or_sp_quarantines() { for bad in 100..=103 { let (mut bank, handles, hardware) = saved::setup(); let id = bank.record(handles[1]).unwrap().thread; hardware.borrow_mut().bad_registers = bad; assert_eq!( bank.os_k1040_cancel_inactive(handles[1]), Err(Error::Backend) ); assert!(bank.snapshot().faulted); assert_eq!(bank.snapshot().quarantined, 1); assert_eq!(bank.snapshot().retained_bindings, 2); assert_eq!( bank.backend.table.borrow().record(id).unwrap().state, thread::State::Dead ); assert_eq!(hardware.borrow().pins, 2); assert_eq!(hardware.borrow().stack_pages, 8); } } struct RootFlip { inner: saved::Backend, root: u64, } unsafe impl Backend for RootFlip { type Binding = saved::Binding; type Prepared = saved::Prepared; type IdlePrepared = saved::IdlePrepared; fn execution(&self) -> Execution { Execution { root: self.root, ..self.inner.execution() } } fn record(&self, b: &Self::Binding) -> Result { self.inner.record(b) } fn stack(&self, b: &Self::Binding) -> Result { self.inner.stack(b) } fn root(&self, b: &Self::Binding) -> Result { self.inner.root(b) } fn executable(&self, p: u64) -> bool { self.inner.executable(p) } fn initial_return_pc(&self, b: &Self::Binding) -> Result { self.inner.initial_return_pc(b) } fn capture_matches(&self, b: &Self::Binding, c: &Capture) -> bool { self.inner.capture_matches(b, c) } fn registers(&self) -> Registers { self.inner.registers() } fn prepare( &mut self, s: Option<&Self::Binding>, t: &Self::Binding, i: &Image, d: Disposition, ) -> Result { self.inner.prepare(s, t, i, d) } fn commit( &mut self, p: &mut Self::Prepared, s: Option<&Self::Binding>, t: &Self::Binding, i: &Image, d: Disposition, ) -> Result<(), Failure> { self.inner.commit(p, s, t, i, d) } fn finish(&mut self, p: Self::Prepared) -> Result<(), (Error, Self::Prepared)> { self.inner.finish(p) } fn abort(&mut self, p: Self::Prepared) -> Result<(), (Error, Self::Prepared)> { self.inner.abort(p) } fn release(&mut self, b: Self::Binding) -> Result<(), (Error, Self::Binding)> { self.inner.release(b) } fn ready_empty(&self) -> bool { self.inner.ready_empty() } fn wake_epoch(&self) -> u64 { self.inner.wake_epoch() } fn idle_prepare(&mut self, e: u64) -> Result { self.inner.idle_prepare(e) } fn idle_commit( &mut self, p: &mut Self::IdlePrepared, e: u64, ) -> Result { self.inner.idle_commit(p, e) } fn idle_finish( &mut self, p: Self::IdlePrepared, ) -> Result<(), (Error, Self::IdlePrepared)> { self.inner.idle_finish(p) } fn idle_abort(&mut self, p: Self::IdlePrepared) -> Result<(), (Error, Self::IdlePrepared)> { self.inner.idle_abort(p) } } unsafe impl InactiveBackend for RootFlip { fn cancel_inactive(&mut self, b: &Self::Binding) -> Result<(), Failure> { self.inner.cancel_inactive(b)?; self.root ^= 4096; Ok(()) } } #[test] fn actual_cancel_then_changed_hardware_root_quarantines() { let (inner, mut pins, hardware) = saved::backend(2); let mut bank = ContextBank::<_, 4>::new(RootFlip { inner, root: saved::ROOT, }) .ok() .unwrap(); bank.attach(pins.remove(0)).ok().unwrap(); let b = bank.attach(pins.remove(0)).ok().unwrap(); let id = bank.record(b).unwrap().thread; assert_eq!(bank.os_k1040_cancel_inactive(b), Err(Error::Backend)); assert!(bank.snapshot().faulted); assert_eq!(bank.snapshot().quarantined, 1); assert_eq!(bank.snapshot().retained_bindings, 2); assert_eq!( bank.backend.inner.table.borrow().record(id).unwrap().state, thread::State::Dead ); assert_eq!(bank.backend.execution().root, saved::ROOT ^ 4096); assert_eq!(hardware.borrow().pins, 2); assert_eq!(hardware.borrow().stack_pages, 8); } } // NEW immutable companion after frozen Context/Async/Completion/retirement. // The real Prepared remains retained; only frozen return_landed consumes it. impl CompletionBank { pub fn os_k1060_check_return_landing( &self, token: &OwnedReturn, ) -> Result { self.validate(token)?; let p = self.pending.as_ref().ok_or(Error::Pending)?; if p.stage != ReturnStage::Departed { return Err(Error::State); } let bank = &self.handoff.bank; // Repeat the actual owning-CPU / masked-task gate BEFORE returned, // source-binding, physical stack or root-owner observations. let e = bank.backend.execution(); if e.cpu != bank.cpu || e.daif & IRQ == 0 || e.daif & !0x3c0 != 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point || e.sp & 15 != 0 { return Err(Error::Context); } bank.backend.returned(&p.prepared)?; let r = bank.validate(p.source.slot)?; if r.state != thread::State::Dead || e.thread.is_some() || e.root != bank.backend.return_root(&p.prepared)? || e.sp >= r.layout.guard() && e.sp <= r.layout.base + r.layout.bytes { return Err(Error::Active); } self.observation().ok_or(Error::Pending) } } // NEW checked service bridge after generated frozen Context sources. It never // returns a mutable backend or private binding, and never branches to assembly. /// # Safety /// Captures and ticks are actual independently retained ingress/assembly /// observations. Tick read/consume touch only bounded dedicated metadata, not /// Table/allocator/Runtime/heap. CPU/root/current-thread/slice/sequence bind the /// same actual main safe point. No mutable bank/backend loan is held in IRQ. pub unsafe trait QueueServices: CompletionBackend { type QueueTick; fn queue_capture(&self) -> Option; fn queue_tick( &self, tick: &Self::QueueTick, ) -> Result; fn queue_consume_tick(&mut self, tick: Self::QueueTick); } impl CompletionBank { fn os_k1060_queue_gate(&self) -> Result<(), Error> { let bank = &self.handoff.bank; let e = bank.backend.execution(); if e.cpu != bank.cpu || e.daif & IRQ == 0 || e.daif & !0x3c0 != 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point { return Err(Error::Context); } Ok(()) } pub fn os_k1060_observe_member( &self, h: Handle, ) -> Result { self.os_k1060_queue_gate()?; let bank = &self.handoff.bank; let i = bank.index(h)?; let thread = bank.validate(i)?; let stack = bank.backend.stack(bank.binding(i)?)?; ContextBank::::stack_observation(stack, thread)?; Ok(crate::runqueue::Observation { thread, context: bank.record(h)?, stack, }) } pub fn os_k1060_capture(&self) -> Result { self.os_k1060_queue_gate()?; self.handoff .bank .backend .queue_capture() .ok_or(Error::Capture) } pub fn os_k1060_tick( &self, tick: &B::QueueTick, ) -> Result { self.os_k1060_queue_gate() .map_err(|_| crate::runqueue::Error::Context)?; self.handoff.bank.backend.queue_tick(tick) } pub fn os_k1060_consume_tick( &mut self, tick: B::QueueTick, ) -> Result<(), (crate::runqueue::Error, B::QueueTick)> { if self.os_k1060_queue_gate().is_err() { return Err((crate::runqueue::Error::Context, tick)); } self.handoff.bank.backend.queue_consume_tick(tick); Ok(()) } } // NEW custody bridge appended after the frozen CompletionBank/services bytes. /// # Safety /// Transfers only the exact nonclone Table Wait from the retained binding. /// Errors leave any acquired Wait in `owned`; no backing is released. Resolve /// consumes that exact Wait using the real Table, preserving it on Before. /// No callback switches, allocates, releases backing, or unmasks interrupts. pub unsafe trait WaitContextServices: QueueServices { /// Reads fresh physical landing evidence for this actual retained binding /// and Prepared image/nonce. It must check actual root, aligned SP on the /// binding's unique owned stack, executed landing PC, full retained wire /// frame/register image and nonce before /// returning target current. It never activates Table or changes owners. fn physical_wait_landing( &self, binding: &Self::Binding, image: &Image, nonce: u64, ) -> Result; fn take_sleep_wait( &mut self, binding: &Self::Binding, owned: &mut Option, ) -> Result<(), Failure>; fn resolve_sleep_wait( &mut self, binding: &Self::Binding, owned: &mut Option, ) -> Result<(), Failure>; } impl CompletionBank { pub fn wait_guest_physical_landing( &self, h: Handle, ) -> Result { self.os_k1060_queue_gate()?; if self.quarantined || self.pending.is_some() || self.handoff.quarantined { return Err(Error::Quarantined); } let p = self.handoff.pending.ok_or(Error::Pending)?; let bank = &self.handoff.bank; let before = bank.backend.execution(); let i = bank.index(h)?; if p.stage != HandoffStage::Departed || p.target != i || p.nonce == 0 || bank.retained_switch.is_none() { return Err(Error::State); } let actual = bank.validate(i)?; if actual.state != thread::State::Ready { return Err(Error::State); } let observed = bank .backend .physical_wait_landing(bank.binding(i)?, &p.image, p.nonce)?; if bank.backend.execution() != before || (observed.cpu, observed.irq_masked, observed.in_irq) != (before.cpu, before.daif & IRQ != 0, before.in_irq) || observed.current != Some(actual.id) { return Err(Error::Capture); } Ok(observed) } pub fn wait_guest_custody( &mut self, h: Handle, owned: &mut Option, resolve: bool, ) -> Result<(), Failure> { let before = |error| Failure { error, phase: FailurePhase::BeforeSwitch, }; self.os_k1060_queue_gate().map_err(before)?; if self.quarantined || self.pending.is_some() || self.handoff.quarantined || self.handoff.pending.is_some() { return Err(before(Error::Pending)); } let bank = &mut self.handoff.bank; let execution = bank.execution(false).map_err(before)?; let i = bank.index(h).map_err(before)?; let old = bank.validate(i).map_err(before)?; if old.state != thread::State::Sleeping || bank.slots[i].state != State::Ready || bank.active == Some(h) || (resolve && !owned.as_ref().is_some_and(|w| w.thread() == old.id)) || (!resolve && owned.is_some()) { return Err(before(Error::State)); } let binding = bank.slots[i] .binding .as_ref() .ok_or_else(|| before(Error::State))?; let result = if resolve { bank.backend.resolve_sleep_wait(binding, owned) } else { bank.backend.take_sleep_wait(binding, owned) }; // Fresh trusted execution must short circuit before any Table/stack // observation after a callback, including a successful real wake. if bank.backend.execution() != execution { bank.faulted = true; return Err(Failure { error: Error::Context, phase: FailurePhase::AfterSwitch, }); } if let Err(f) = result { if f.phase == FailurePhase::AfterSwitch { bank.faulted = true; } return Err(f); } let valid = bank.validate(i).is_ok_and(|r| { r.id == old.id && r.wait_epoch == old.wait_epoch && r.state == if resolve { thread::State::Ready } else { thread::State::Sleeping } }); let custody = if resolve { owned.is_none() } else { owned.as_ref().is_some_and(|w| w.thread() == old.id) }; if !valid || !custody { bank.faulted = true; return Err(Failure { error: Error::Context, phase: FailurePhase::AfterSwitch, }); } Ok(()) } }