//! Bounded single CPU ready membership, selection and quantum custody. //! The backend exclusively retains the frozen context and thread services. use core::marker::PhantomData; use turkos_os_context_switch::context::{self as context, Record as ContextRecord}; use turkos_os_threads::threads::{self as thread, StackObservation, ThreadId, ThreadRecord}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { Configuration, Context, Capacity, Exhausted, Duplicate, Stale, State, Guard, Empty, Priority, Tick, Expired, Backend, Quarantined, Busy, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Class { Normal, Low, } impl Class { pub fn request(value: u8) -> Result { match value { 0 => Ok(Self::Normal), 1 => Ok(Self::Low), _ => Err(Error::Priority), } } pub fn quantum(self) -> u32 { match self { Self::Normal => 4, Self::Low => 2, } } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Execution { pub cpu: u32, pub masked: bool, pub in_irq: bool, pub thread: Option, pub root: u64, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Observation { pub thread: ThreadRecord, pub context: ContextRecord, pub stack: StackObservation, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct TickObservation { pub cpu: u32, pub source: u64, pub sequence: u64, pub counter: u64, pub thread: ThreadId, pub slice: u64, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Disposition { Ready, Sleeping, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Failure { pub error: Error, pub phase: context::FailurePhase, } /// # Safety /// Backend exclusively owns live frozen Table/ContextBank services. Nonclone /// Member values pin that exact ThreadId, ContextHandle and guarded stack; Drop /// never frees backing. observe reads actual retained stack/guard/canary state /// and validates the same live context binding, with no fabricated identity. /// Each Tick owns one real timer/deferred delivery and is bound to its hardware /// source, CPU, ThreadId generation and current slice epoch. Tick observations /// and consumption never switch threads, allocate, sleep or run in IRQ here. /// prepare failure leaves complete thread/register/resource inventory unchanged. /// Prepared retains all staged ownership. commit uses the frozen context service /// and its current actual capture; BeforeSwitch means nothing visible changed, /// AfterSwitch covers any possible change. A completed switch closes IRQ custody /// and observes the new actual Running context before returning. finish/abort /// failures return the same Prepared; dropping it does not release resources. /// All operations are bounded, serialized on the initial CPU with IRQ masked. pub unsafe trait Backend { type Member; type Prepared; type Tick; fn execution(&self) -> Execution; fn observe(&self, member: &Self::Member) -> Result; fn tick(&self, tick: &Self::Tick) -> Result; fn consume_tick(&mut self, tick: Self::Tick); fn prepare( &mut self, source: Option<&Self::Member>, target: &Self::Member, epoch: u64, disposition: Disposition, ) -> Result; fn commit( &mut self, prepared: &mut Self::Prepared, source: Option<&Self::Member>, target: &Self::Member, epoch: u64, disposition: Disposition, ) -> Result<(), Failure>; fn finish(&mut self, prepared: Self::Prepared) -> Result<(), (Error, Self::Prepared)>; fn abort(&mut self, prepared: Self::Prepared) -> Result<(), (Error, Self::Prepared)>; } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Handle { queue: u64, slot: usize, generation: u64, } impl Handle { pub fn slot(self) -> usize { self.slot } pub fn generation(self) -> u64 { self.generation } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Record { pub handle: Handle, pub thread: ThreadId, pub class: Class, pub queued: bool, pub active: bool, pub remaining: u32, pub slice: u64, pub consumed_ticks: u64, pub slices: u64, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Snapshot { pub members: usize, pub queued: usize, pub active: usize, pub retired: usize, pub epoch: u64, pub consumed_ticks: u64, pub faulted: bool, pub retained_prepared: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Charge { pub remaining: u32, pub reschedule: bool, pub sequence: u64, } struct Entry { generation: u64, member: Option, thread: Option, class: Class, remaining: u32, slice: u64, consumed_ticks: u64, slices: u64, } impl Entry { fn empty() -> Self { Self { generation: 0, member: None, thread: None, class: Class::Normal, remaining: 0, slice: 0, consumed_ticks: 0, slices: 0, } } } pub struct Queue { backend: B, identity: u64, cpu: u32, entries: [Entry; N], ready: [usize; N], len: usize, active: Option, epoch: u64, tick_source: Option, last_tick: u64, last_counter: u64, total_ticks: u64, faulted: bool, retained: Option, _local: PhantomData<*mut ()>, } #[cfg(not(target_os = "none"))] fn identity() -> Result { use core::sync::atomic::{AtomicU64, Ordering}; static NEXT: AtomicU64 = AtomicU64::new(1); NEXT.try_update(Ordering::Relaxed, Ordering::Relaxed, |n| n.checked_add(1)) .map_err(|_| Error::Exhausted) } #[cfg(target_os = "none")] fn identity() -> Result { unsafe extern "C" { fn __os_k1060_runqueue_identity() -> u64; } let value = unsafe { __os_k1060_runqueue_identity() }; if value == 0 { Err(Error::Exhausted) } else { Ok(value) } } impl Queue { pub fn new(backend: B) -> Result { let e = backend.execution(); if N == 0 || N > 32 { return Err((Error::Configuration, backend)); } if !e.masked || e.in_irq || e.thread.is_some() { return Err((Error::Context, backend)); } let identity = match identity() { Ok(v) => v, Err(e) => return Err((e, backend)), }; Ok(Self { backend, identity, cpu: e.cpu, entries: core::array::from_fn(|_| Entry::empty()), ready: [0; N], len: 0, active: None, epoch: 0, tick_source: None, last_tick: 0, last_counter: 0, total_ticks: 0, faulted: false, retained: None, _local: PhantomData, }) } fn execution(&self) -> Result { if self.faulted { return Err(Error::Quarantined); } let e = self.backend.execution(); if !e.masked || e.in_irq || e.cpu != self.cpu { return Err(Error::Context); } Ok(e) } fn handle(&self, index: usize) -> Handle { Handle { queue: self.identity, slot: index, generation: self.entries[index].generation, } } fn index(&self, h: Handle) -> Result { if h.queue != self.identity || h.slot >= N { return Err(Error::Stale); } let s = &self.entries[h.slot]; if s.member.is_none() || s.generation != h.generation { return Err(Error::Stale); } Ok(h.slot) } fn member(&self, i: usize) -> Result<&B::Member, Error> { self.entries .get(i) .and_then(|e| e.member.as_ref()) .ok_or(Error::Stale) } fn observed(&self, i: usize) -> Result { let o = self.backend.observe(self.member(i)?)?; if Some(o.thread.id) != self.entries[i].thread || o.context.thread != o.thread.id { return Err(Error::Stale); } if o.context.state == context::State::Vacant || o.context.state == context::State::Quarantined || o.thread.state == thread::State::Quarantined { return Err(Error::State); } if o.stack.layout != o.thread.layout || o.stack.guard_descriptor != 0 || o.stack.low_canary != thread::LOW_CANARY || o.stack.high_canary != thread::HIGH_CANARY { return Err(Error::Guard); } if o.context.root == 0 || !o.context.root.is_multiple_of(4096) || o.context.root >> 48 != 0 { return Err(Error::State); } Ok(o) } fn eligible(&self, i: usize) -> Result { let o = self.observed(i)?; Ok(o.thread.state == thread::State::Ready && o.context.state == context::State::Ready) } fn audit_at(&self, e: Execution) -> Result<(), Error> { if self.len > N { return Err(Error::State); } for i in 0..self.len { let index = self.ready[i]; if index >= N || self.active == Some(index) || self.ready[..i].contains(&index) { return Err(Error::Duplicate); } self.observed(index)?; } for (i, entry) in self.entries.iter().enumerate() { if entry.member.is_some() { let o = self.observed(i)?; if self.entries[..i] .iter() .any(|prior| prior.member.is_some() && prior.thread == entry.thread) { return Err(Error::Duplicate); } if self.active == Some(i) { if o.thread.state != thread::State::Running || o.context.state != context::State::Active || e.thread != Some(o.thread.id) || e.root != o.context.root { return Err(Error::State); } } else if o.thread.state == thread::State::Running || o.context.state == context::State::Active { return Err(Error::State); } } } if self.active.is_none() && e.thread.is_some() { return Err(Error::State); } Ok(()) } pub fn audit(&self) -> Result<(), Error> { let e = self.execution()?; self.audit_at(e) } pub fn enqueue( &mut self, member: B::Member, class: Class, ) -> Result { let result = (|| { self.audit()?; let o = self.backend.observe(&member)?; if self .entries .iter() .any(|s| s.member.is_some() && s.thread == Some(o.thread.id)) { return Err(Error::Duplicate); } if o.thread.state != thread::State::Ready || o.context.state != context::State::Ready || o.context.thread != o.thread.id { return Err(Error::State); } if o.stack.layout != o.thread.layout || o.stack.guard_descriptor != 0 || o.stack.low_canary != thread::LOW_CANARY || o.stack.high_canary != thread::HIGH_CANARY { return Err(Error::Guard); } if o.context.root == 0 || !o.context.root.is_multiple_of(4096) || o.context.root >> 48 != 0 { return Err(Error::State); } let i = self .entries .iter() .position(|s| s.member.is_none() && s.generation != u64::MAX) .ok_or(Error::Capacity)?; if self.len == N { return Err(Error::Capacity); } let generation = self.entries[i] .generation .checked_add(1) .ok_or(Error::Exhausted)?; Ok((i, generation, o.thread.id)) })(); match result { Ok((i, generation, id)) => { self.entries[i] = Entry { generation, member: Some(member), thread: Some(id), class, remaining: 0, slice: 0, consumed_ticks: 0, slices: 0, }; self.ready[self.len] = i; self.len += 1; Ok(self.handle(i)) } Err(e) => Err((e, member)), } } pub fn requeue(&mut self, h: Handle) -> Result<(), Error> { self.audit()?; let i = self.index(h)?; if self.active == Some(i) || self.ready[..self.len].contains(&i) { return Err(Error::Duplicate); } if !self.eligible(i)? { return Err(Error::State); } if self.len == N { return Err(Error::Capacity); } self.ready[self.len] = i; self.len += 1; Ok(()) } pub fn request_priority(&mut self, h: Handle, request: u8) -> Result<(), Error> { let class = Class::request(request)?; self.audit()?; let i = self.index(h)?; if self.active == Some(i) { return Err(Error::Busy); } if !self.eligible(i)? { return Err(Error::State); } self.entries[i].class = class; Ok(()) } pub fn select(&mut self, disposition: Disposition) -> Result, Error> { self.audit()?; if self.active.is_none() && disposition == Disposition::Sleeping { return Err(Error::State); } let mut selected = None; for position in 0..self.len { if self.eligible(self.ready[position])? { selected = Some(position); break; } } let position = selected.ok_or(Error::Empty)?; let target = self.ready[position]; let epoch = self.epoch.checked_add(1).ok_or(Error::Exhausted)?; let slices = self.entries[target] .slices .checked_add(1) .ok_or(Error::Exhausted)?; let prepared = self.backend.prepare( self.active.and_then(|i| self.entries[i].member.as_ref()), self.entries[target].member.as_ref().ok_or(Error::Stale)?, epoch, disposition, )?; Ok(Selection { source: self.active, queue: self, target, position, epoch, slices, disposition, prepared: Some(prepared), }) } /// Renew policy accounting for the sole runnable member. This changes no /// hardware context; the next owned timer delivery must bind the new slice. /// A live Ready neighbour requires selection and cannot be bypassed here. pub fn renew(&mut self) -> Result { self.audit()?; let i = self.active.ok_or(Error::State)?; if self.entries[i].remaining != 0 { return Err(Error::Busy); } for member in 0..N { if member != i && self.entries[member].member.is_some() && self.eligible(member)? { return Err(Error::Busy); } } let epoch = self.epoch.checked_add(1).ok_or(Error::Exhausted)?; let slices = self.entries[i] .slices .checked_add(1) .ok_or(Error::Exhausted)?; self.epoch = epoch; self.entries[i].slice = epoch; self.entries[i].remaining = self.entries[i].class.quantum(); self.entries[i].slices = slices; Ok(self.handle(i)) } #[allow(clippy::result_large_err)] pub fn charge(&mut self, tick: B::Tick) -> Result { let result = (|| { self.audit()?; let i = self.active.ok_or(Error::State)?; let entry = &self.entries[i]; if entry.remaining == 0 { return Err(Error::Expired); } let o = self.backend.tick(&tick)?; if o.cpu != self.cpu || Some(o.thread) != entry.thread || o.slice != entry.slice || o.source == 0 || self.tick_source.is_some_and(|source| source != o.source) || o.sequence == 0 || o.sequence <= self.last_tick || o.counter < self.last_counter { return Err(Error::Tick); } let count = entry .consumed_ticks .checked_add(1) .ok_or(Error::Exhausted)?; let total = self.total_ticks.checked_add(1).ok_or(Error::Exhausted)?; Ok((i, o, count, total)) })(); match result { Ok((i, o, count, total)) => { let entry = &mut self.entries[i]; entry.remaining -= 1; entry.consumed_ticks = count; self.total_ticks = total; self.tick_source = Some(o.source); self.last_tick = o.sequence; self.last_counter = o.counter; let remaining = entry.remaining; self.backend.consume_tick(tick); Ok(Charge { remaining, reschedule: remaining == 0, sequence: o.sequence, }) } Err(e) => Err((e, tick)), } } pub fn withdraw(&mut self, h: Handle) -> Result { self.audit()?; let i = self.index(h)?; if self.active == Some(i) { return Err(Error::Busy); } let o = self.observed(i)?; if o.thread.state != thread::State::Dead && o.thread.state != thread::State::Sleeping { return Err(Error::State); } if let Some(p) = self.ready[..self.len].iter().position(|v| *v == i) { for j in p + 1..self.len { self.ready[j - 1] = self.ready[j] } self.len -= 1; self.ready[self.len] = 0; } let member = self.entries[i].member.take().ok_or(Error::Stale)?; let generation = self.entries[i].generation; self.entries[i] = Entry::empty(); self.entries[i].generation = generation; Ok(member) } pub fn record(&self, h: Handle) -> Result { let i = self.index(h)?; let e = &self.entries[i]; Ok(Record { handle: h, thread: e.thread.ok_or(Error::Stale)?, class: e.class, queued: self.ready[..self.len].contains(&i), active: self.active == Some(i), remaining: e.remaining, slice: e.slice, consumed_ticks: e.consumed_ticks, slices: e.slices, }) } pub fn snapshot(&self) -> Snapshot { Snapshot { members: self.entries.iter().filter(|s| s.member.is_some()).count(), queued: self.len, active: usize::from(self.active.is_some()), retired: self .entries .iter() .filter(|s| s.member.is_none() && s.generation == u64::MAX) .count(), epoch: self.epoch, consumed_ticks: self.total_ticks, faulted: self.faulted, retained_prepared: self.retained.is_some(), } } pub fn ready_order(&self) -> [Option; N] { core::array::from_fn(|i| (i < self.len).then(|| self.handle(self.ready[i]))) } pub fn with_backend_observation(&self, f: impl FnOnce(&B) -> R) -> R { f(&self.backend) } pub fn with_retained_observation(&self, f: impl FnOnce(Option<&B::Prepared>) -> R) -> R { f(self.retained.as_ref()) } fn quarantine(&mut self, prepared: B::Prepared) { drop(prepared); self.faulted = true; } #[allow(clippy::result_large_err)] pub fn into_backend(self) -> Result { if self.faulted || self.entries.iter().any(|e| e.member.is_some()) { return Err((Error::Busy, self)); } if let Err(e) = self.audit() { return Err((e, self)); } Ok(self.backend) } } pub enum SelectionError { Before(Error, T), Quarantined(Error), } pub struct Selection<'a, B: Backend, const N: usize> { queue: &'a mut Queue, source: Option, target: usize, position: usize, epoch: u64, slices: u64, disposition: Disposition, prepared: Option, } impl Selection<'_, B, N> { pub fn target(&self) -> Handle { self.queue.handle(self.target) } pub fn with_prepared_observation(&self, f: impl FnOnce(&B::Prepared) -> R) -> R { f(self.prepared.as_ref().unwrap()) } fn before(&self) -> Result<(), Error> { self.queue.audit()?; if self.queue.active != self.source || self.queue.ready.get(self.position) != Some(&self.target) || !self.queue.eligible(self.target)? { return Err(Error::State); } Ok(()) } fn completed(&self) -> Result<(), Error> { let e = self.queue.execution()?; let target = self.queue.observed(self.target)?; if target.thread.state != thread::State::Running || target.context.state != context::State::Active || e.thread != Some(target.thread.id) || e.root != target.context.root { return Err(Error::State); } if let Some(i) = self.source { let source = self.queue.observed(i)?; let state = match self.disposition { Disposition::Ready => thread::State::Ready, Disposition::Sleeping => thread::State::Sleeping, }; if source.thread.state != state || source.context.state != context::State::Ready { return Err(Error::State); } } Ok(()) } #[allow(clippy::result_large_err)] pub fn complete(mut self) -> Result> { if let Err(e) = self.before() { return Err(SelectionError::Before(e, self)); } let result = self.queue.backend.commit( self.prepared.as_mut().unwrap(), self.source .and_then(|i| self.queue.entries[i].member.as_ref()), self.queue.entries[self.target].member.as_ref().unwrap(), self.epoch, self.disposition, ); if let Err(f) = result { return match f.phase { context::FailurePhase::BeforeSwitch => Err(SelectionError::Before(f.error, self)), context::FailurePhase::AfterSwitch => { let p = self.prepared.take().unwrap(); self.queue.quarantine(p); Err(SelectionError::Quarantined(f.error)) } }; } if let Err(e) = self.completed() { let p = self.prepared.take().unwrap(); self.queue.quarantine(p); return Err(SelectionError::Quarantined(e)); } let p = self.prepared.take().unwrap(); if let Err((e, p)) = self.queue.backend.finish(p) { self.queue.quarantine(p); return Err(SelectionError::Quarantined(e)); } for j in self.position + 1..self.queue.len { self.queue.ready[j - 1] = self.queue.ready[j] } self.queue.len -= 1; self.queue.ready[self.queue.len] = 0; if let Some(i) = self.source && self.disposition == Disposition::Ready { self.queue.ready[self.queue.len] = i; self.queue.len += 1; } self.queue.active = Some(self.target); self.queue.epoch = self.epoch; let entry = &mut self.queue.entries[self.target]; entry.remaining = entry.class.quantum(); entry.slice = self.epoch; entry.slices = self.slices; Ok(self.queue.handle(self.target)) } pub fn abort(mut self) -> Result<(), Error> { let p = self.prepared.take().unwrap(); if let Err((e, p)) = self.queue.backend.abort(p) { self.queue.quarantine(p); return Err(e); } Ok(()) } } impl Drop for Selection<'_, B, N> { fn drop(&mut self) { if let Some(p) = self.prepared.take() && let Err((_, p)) = self.queue.backend.abort(p) { self.queue.quarantine(p); } } } #[cfg(test)] mod exhaustion_tests { use super::*; use crate::test_support::{backend, inventory, select, setup, tick}; #[test] fn retired_slot_is_skipped_and_all_retired_preserve_same_member() { let (b, mut members, c) = backend(2); let mut q = Queue::<_, 4>::new(b).ok().unwrap(); q.entries[0].generation = u64::MAX; let h = q.enqueue(members.remove(0), Class::Normal).ok().unwrap(); assert_eq!(h.slot(), 1); q.entries[2].generation = u64::MAX; q.entries[3].generation = u64::MAX; let member = members.remove(0); let id = member.thread; let member = match q.enqueue(member, Class::Normal) { Err((Error::Capacity, m)) => m, _ => panic!(), }; assert_eq!(member.thread, id); assert_eq!(q.snapshot().retired, 3); assert_eq!(inventory(&q), (2, 2, 8)); assert_eq!(c.borrow().prepares, 0); } #[test] fn epoch_and_slice_counter_exhaustion_publish_no_prefix() { for counter in 0..2 { let (mut q, _, c) = setup(2); if counter == 0 { q.epoch = u64::MAX } else { q.entries[0].slices = u64::MAX } let before = q.snapshot(); let order = q.ready_order(); assert!(matches!( q.select(Disposition::Ready), Err(Error::Exhausted) )); assert_eq!(q.snapshot(), before); assert_eq!(q.ready_order(), order); assert_eq!(c.borrow().prepares, 0); assert_eq!(inventory(&q), (2, 2, 8)); } } #[test] fn tick_counter_exhaustion_retains_exact_owned_tick() { for counter in 0..2 { let (mut q, h, c) = setup(2); select(&mut q, Disposition::Ready); if counter == 0 { q.total_ticks = u64::MAX } else { q.entries[0].consumed_ticks = u64::MAX } let t = tick(&q, h[0]); let sequence = t.nonce; let before = q.record(h[0]).unwrap(); let t = match q.charge(t) { Err((Error::Exhausted, t)) => t, _ => panic!(), }; assert_eq!(t.nonce, sequence); assert_eq!(q.record(h[0]).unwrap(), before); assert_eq!(c.borrow().consumed, 0); assert_eq!(inventory(&q), (2, 2, 8)); } } #[test] fn private_foreign_generation_and_stale_slot_refuse_complete_inventory() { let (mut q, h, c) = setup(3); let before = q.snapshot(); let forged = Handle { generation: h[0].generation + 1, ..h[0] }; assert_eq!(q.request_priority(forged, 1), Err(Error::Stale)); q.entries[2].thread = Some(q.record(h[1]).unwrap().thread); assert!(matches!(q.select(Disposition::Ready), Err(Error::Stale))); assert_eq!(q.snapshot(), before); assert_eq!(c.borrow().prepares, 0); assert_eq!(inventory(&q), (3, 3, 12)); } #[test] fn damaged_ready_length_or_duplicate_never_prepares_selection() { for counter in 0..2 { let (mut q, _, c) = setup(3); if counter == 0 { q.len = 5 } else { q.ready[2] = q.ready[0] } assert!(q.select(Disposition::Ready).is_err()); assert_eq!(c.borrow().prepares, 0); assert_eq!(inventory(&q), (3, 3, 12)); } } #[test] fn maximum_tick_sequence_is_consumed_once_without_successor_wrap() { let (mut q, h, c) = setup(2); select(&mut q, Disposition::Ready); q.last_tick = u64::MAX - 1; c.borrow_mut().sequence = u64::MAX - 1; let t = tick(&q, h[0]); let duplicate = crate::test_support::Tick { observation: t.observation, nonce: t.nonce, }; assert_eq!(q.charge(t).ok().unwrap().sequence, u64::MAX); c.borrow_mut().tick_live = Some(u64::MAX); let before = q.record(h[0]).unwrap(); assert!(matches!(q.charge(duplicate), Err((Error::Tick, _)))); assert_eq!(q.record(h[0]).unwrap(), before); assert_eq!(c.borrow().consumed, 1); assert_eq!(inventory(&q), (2, 2, 8)); } }