//! Ownership and bounded dispatch for standard GIC SGI/PPI/SPI interrupts. //! Controller operations require an exclusive loan. The platform adapter must //! mask local IRQs before taking that loan; callbacks cannot sleep or allocate. use core::sync::atomic::{AtomicU64, Ordering}; pub const MAX_INTIDS: u32 = 1020; pub const MAX_COMPLETIONS: u16 = 32; static NEXT_INSTANCE: AtomicU64 = AtomicU64::new(1); #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { Invalid, Cpu, Conflict, Capacity, Generation, NoMemory, Backing, MaskReadback, EnableReadback, InvalidAcknowledge, Foreign, Stale, Busy, Closing, AlreadyServiced, Unserviced, Budget, Retirement, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Configuration { intids: u32, cpus: u16, budget: u16, split_eoi: bool, } impl Configuration { pub fn new(intids: u32, cpus: u16, budget: u16, split_eoi: bool) -> Result { if !(32..=MAX_INTIDS).contains(&intids) || cpus == 0 || cpus > 256 || budget == 0 || budget > MAX_COMPLETIONS { return Err(Error::Invalid); } Ok(Self { intids, cpus, budget, split_eoi, }) } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Completion { Idle, Done { more: bool }, } /// Trusted physical device/interrupt boundary. Resources remain actual owned /// tokens, and no mutable backend escape is exposed by the controller. /// /// # Safety /// Acquire must either retain nothing or return the whole private resource. /// Configure runs only while masked and validates hardware CPU routing. /// Readbacks must come from volatile hardware reads after write barriers. /// One completion is bounded and cannot retire or replace its resource. A /// successful last completion clears a level source before EOI. EOI/DIR and /// the return barrier are infallible physical operations in the admitted scope. /// Release requires device quiescence and returns its entire token on failure. /// Dropping a resource or backend must retain/quarantine its backing rather /// than reclaiming it while an IRQ or DMA operation could still access it. pub unsafe trait Backend { type Resource; fn acquire(&mut self, intid: u32, owner: u64, cpu: u16) -> Result; fn validate( &self, resource: &Self::Resource, intid: u32, owner: u64, cpu: u16, ) -> Result<(), Error>; fn configure(&mut self, resource: &mut Self::Resource, intid: u32, cpu: u16); fn mask_readback(&mut self, intid: u32, cpu: u16) -> bool; fn enable_readback(&mut self, intid: u32, cpu: u16) -> bool; fn acknowledge(&mut self) -> u64; fn next_completion( &mut self, resource: &mut Self::Resource, owner: u64, ) -> Result; fn eoi(&mut self, intid: u32); fn deactivate(&mut self, intid: u32); fn return_barrier(&mut self); fn release(&mut self, resource: Self::Resource) -> Result<(), (Error, Self::Resource)>; } #[derive(Debug)] pub struct Registration { instance: u64, slot: usize, generation: u64, owner: u64, intid: u32, cpu: u16, } impl Registration { pub fn owner(&self) -> u64 { self.owner } pub fn intid(&self) -> u32 { self.intid } pub fn cpu(&self) -> u16 { self.cpu } pub fn generation(&self) -> u64 { self.generation } } #[derive(Debug)] pub struct RegistrationFailure { pub error: Error, pub retained: Option, } #[derive(Debug)] pub struct Dispatch { instance: u64, slot: usize, generation: u64, serial: u64, owner: u64, intid: u32, cpu: u16, live: bool, } impl Dispatch { pub fn owner(&self) -> u64 { self.owner } pub fn intid(&self) -> u32 { self.intid } pub fn cpu(&self) -> u16 { self.cpu } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MaskReason { Unknown, WrongCpu, Suspended, Closing, Quarantined, } #[derive(Debug)] pub enum Begin { Spurious(u32), Masked { intid: u32, reason: MaskReason }, Dispatch(Dispatch), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum State { Open, Suspended, Closing, Quarantined, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Work { pub completed: u16, pub deferred: bool, pub closing: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Close { RetainedActive, Released, } #[derive(Debug)] pub struct ReturnPermit { instance: u64, serial: u64, owner: u64, intid: u32, cpu: u16, released: bool, } impl ReturnPermit { pub fn observation(&self) -> (u64, u64, u64, u32, u16, bool) { ( self.instance, self.serial, self.owner, self.intid, self.cpu, self.released, ) } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct OwnedRecord { pub slot: usize, pub generation: u64, pub owner: u64, pub intid: u32, pub cpu: u16, pub state: State, pub active: bool, pub serviced: bool, pub masked: bool, pub callbacks: u64, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Counters { pub issued: u64, pub returned: u64, pub completions: u64, } struct Active { serial: u64, serviced: bool, } struct Route { generation: u64, owner: u64, intid: u32, cpu: u16, state: State, masked: bool, resource: R, active: Option, callbacks: u64, } pub struct Gic { backend: B, instance: u64, caller_scope: u64, configuration: Configuration, routes: [Option>; N], generations: [u64; N], counters: Counters, } impl Gic { pub fn new(caller_scope: u64, configuration: Configuration, backend: B) -> Result { if caller_scope == 0 || N == 0 { return Err(Error::Invalid); } let instance = NEXT_INSTANCE .try_update(Ordering::Relaxed, Ordering::Relaxed, |next| { next.checked_add(1) }) .map_err(|_| Error::Generation)?; Ok(Self { backend, instance, caller_scope, configuration, routes: core::array::from_fn(|_| None), generations: [0; N], counters: Counters { issued: 0, returned: 0, completions: 0, }, }) } pub fn caller_scope(&self) -> u64 { self.caller_scope } pub fn counters(&self) -> Counters { self.counters } pub fn with_backend_observation(&self, callback: impl FnOnce(&B) -> R) -> R { callback(&self.backend) } pub fn owned_records(&self) -> impl Iterator + '_ { self.routes.iter().enumerate().filter_map(|(slot, row)| { row.as_ref().map(|row| OwnedRecord { slot, generation: row.generation, owner: row.owner, intid: row.intid, cpu: row.cpu, state: row.state, active: row.active.is_some(), serviced: row.active.as_ref().is_some_and(|active| active.serviced), masked: row.masked, callbacks: row.callbacks, }) }) } fn registration(&self, slot: usize) -> Registration { let route = self.routes[slot].as_ref().expect("owned route"); Registration { instance: self.instance, slot, generation: route.generation, owner: route.owner, intid: route.intid, cpu: route.cpu, } } fn registration_slot(&self, token: &Registration, owner: u64) -> Result { if false { return Err(Error::Foreign); } if token.owner != owner { return Err(Error::Foreign); } let route = self .routes .get(token.slot) .and_then(Option::as_ref) .ok_or(Error::Stale)?; if (route.generation, route.owner, route.intid, route.cpu) != (token.generation, token.owner, token.intid, token.cpu) { return Err(Error::Stale); } self.backend .validate(&route.resource, route.intid, route.owner, route.cpu)?; Ok(token.slot) } fn dispatch_slot(&self, token: &Dispatch) -> Result { if token.instance != self.instance { return Err(Error::Foreign); } if !token.live { return Err(Error::Stale); } let route = self .routes .get(token.slot) .and_then(Option::as_ref) .ok_or(Error::Stale)?; if (route.generation, route.owner, route.intid, route.cpu) != (token.generation, token.owner, token.intid, token.cpu) || route .active .as_ref() .is_none_or(|active| active.serial != token.serial) { return Err(Error::Stale); } self.backend .validate(&route.resource, route.intid, route.owner, route.cpu)?; Ok(token.slot) } pub fn register( &mut self, intid: u32, owner: u64, cpu: u16, ) -> Result { let preflight = (|| { if owner == 0 || intid >= self.configuration.intids { return Err(Error::Invalid); } if cpu >= self.configuration.cpus { return Err(Error::Cpu); } if self .routes .iter() .flatten() .any(|route| route.intid == intid) { return Err(Error::Conflict); } self.routes .iter() .enumerate() .position(|(slot, row)| row.is_none() && self.generations[slot] != u64::MAX) .ok_or(if self.routes.iter().any(Option::is_none) { Error::Generation } else { Error::Capacity }) })(); let slot = preflight.map_err(|error| RegistrationFailure { error, retained: None, })?; if !self.backend.mask_readback(intid, cpu) { return Err(RegistrationFailure { error: Error::MaskReadback, retained: None, }); } let resource = self.backend .acquire(intid, owner, cpu) .map_err(|error| RegistrationFailure { error, retained: None, })?; let generation = self.generations[slot] + 1; self.generations[slot] = generation; self.routes[slot] = Some(Route { generation, owner, intid, cpu, state: State::Quarantined, masked: true, resource, active: None, callbacks: 0, }); let route = self.routes[slot].as_mut().expect("retained resource"); if let Err(error) = self.backend.validate(&route.resource, intid, owner, cpu) { return Err(RegistrationFailure { error, retained: Some(self.registration(slot)), }); } self.backend.configure(&mut route.resource, intid, cpu); if !self.backend.enable_readback(intid, cpu) { route.masked = self.backend.mask_readback(intid, cpu); return Err(RegistrationFailure { error: Error::EnableReadback, retained: Some(self.registration(slot)), }); } route.masked = false; route.state = State::Open; Ok(self.registration(slot)) } /// An error cannot authorize ERET: the platform must retain its exception /// frame and stop. Only Spurious/Masked or a completed Dispatch may return. pub fn begin(&mut self, cpu: u16) -> Result { if cpu >= self.configuration.cpus { return Err(Error::Cpu); } let next = self .counters .issued .checked_add(1) .ok_or(Error::Generation)?; let raw = self.backend.acknowledge(); if raw > 0x00ff_ffff { return Err(Error::InvalidAcknowledge); } let intid = raw as u32; if (1020..=1023).contains(&intid) { return Ok(Begin::Spurious(intid)); } if intid >= self.configuration.intids { return Err(Error::InvalidAcknowledge); } let slot = self .routes .iter() .position(|route| route.as_ref().is_some_and(|route| route.intid == intid)); let reason = slot .and_then(|slot| { let route = self.routes[slot].as_ref().expect("found route"); if route.cpu != cpu { Some(MaskReason::WrongCpu) } else { match route.state { State::Open => None, State::Suspended => Some(MaskReason::Suspended), State::Closing => Some(MaskReason::Closing), State::Quarantined => Some(MaskReason::Quarantined), } } }) .or_else(|| slot.is_none().then_some(MaskReason::Unknown)); if let Some(reason) = reason { if !self.backend.mask_readback(intid, cpu) { return Err(Error::MaskReadback); } self.backend.eoi(intid); if self.configuration.split_eoi { self.backend.deactivate(intid); } self.backend.return_barrier(); return Ok(Begin::Masked { intid, reason }); } let slot = slot.expect("known open route"); let route = self.routes[slot].as_mut().expect("owned route"); self.backend .validate(&route.resource, intid, route.owner, cpu)?; if route.active.is_some() { return Err(Error::Busy); } route.active = Some(Active { serial: next, serviced: false, }); self.counters.issued = next; Ok(Begin::Dispatch(Dispatch { instance: self.instance, slot, generation: route.generation, serial: next, owner: route.owner, intid, cpu, live: true, })) } pub fn service(&mut self, token: &Dispatch, limit: u16) -> Result { let slot = self.dispatch_slot(token)?; if limit == 0 || limit > self.configuration.budget { return Err(Error::Budget); } let route = self.routes[slot].as_mut().expect("owned route"); if route.active.as_ref().is_some_and(|active| active.serviced) { return Err(Error::AlreadyServiced); } if route.state == State::Closing { route.active.as_mut().expect("active route").serviced = true; return Ok(Work { completed: 0, deferred: false, closing: true, }); } if route.state != State::Open { return Err(Error::Closing); } self.counters .completions .checked_add(u64::from(limit)) .ok_or(Error::Generation)?; route .callbacks .checked_add(u64::from(limit)) .ok_or(Error::Generation)?; // Mark before entering the callback so an unexpected backend failure // cannot silently retry a potentially consumed completion. route.active.as_mut().expect("active route").serviced = true; let mut completed = 0; let mut more = false; while completed < limit { match self .backend .next_completion(&mut route.resource, route.owner) { Ok(Completion::Idle) => { more = false; break; } Ok(Completion::Done { more: pending }) => { completed += 1; self.counters.completions += 1; route.callbacks += 1; more = pending; if !pending { break; } } Err(error) => { route.masked = self.backend.mask_readback(route.intid, route.cpu); route.state = State::Quarantined; return Err(error); } } } if more { route.masked = self.backend.mask_readback(route.intid, route.cpu); route.state = if route.masked { State::Suspended } else { State::Quarantined }; if !route.masked { return Err(Error::MaskReadback); } } Ok(Work { completed, deferred: more, closing: false, }) } fn retire(&mut self, slot: usize) -> Result<(), Error> { let mut route = self.routes[slot].take().expect("owned retirement"); if route.active.is_some() || !route.masked { self.routes[slot] = Some(route); return Err(Error::Busy); } match self.backend.release(route.resource) { Ok(()) => Ok(()), Err((error, resource)) => { route.resource = resource; route.state = State::Closing; self.routes[slot] = Some(route); Err(error) } } } pub fn request_close(&mut self, token: &Registration, owner: u64) -> Result { let slot = self.registration_slot(token, owner)?; let route = self.routes[slot].as_mut().expect("owned route"); route.masked = self.backend.mask_readback(route.intid, route.cpu); if !route.masked { route.state = State::Quarantined; return Err(Error::MaskReadback); } route.state = State::Closing; if route.active.is_some() { return Ok(Close::RetainedActive); } self.retire(slot)?; Ok(Close::Released) } pub fn resume(&mut self, token: &Registration, owner: u64) -> Result<(), Error> { let slot = self.registration_slot(token, owner)?; let route = self.routes[slot].as_mut().expect("owned route"); if route.active.is_some() { return Err(Error::Busy); } if route.state != State::Suspended { return Err(Error::Closing); } if !self.backend.enable_readback(route.intid, route.cpu) { route.masked = self.backend.mask_readback(route.intid, route.cpu); route.state = State::Quarantined; return Err(Error::EnableReadback); } route.masked = false; route.state = State::Open; Ok(()) } pub fn finish(&mut self, token: &mut Dispatch) -> Result { let slot = self.dispatch_slot(token)?; let next = self .counters .returned .checked_add(1) .ok_or(Error::Generation)?; let route = self.routes[slot].as_ref().expect("owned route"); if route.state == State::Open && !route.active.as_ref().expect("active route").serviced { return Err(Error::Unserviced); } if route.state != State::Open && !route.masked { return Err(Error::MaskReadback); } let closing = route.state == State::Closing; self.backend.eoi(route.intid); if self.configuration.split_eoi { self.backend.deactivate(route.intid); } self.backend.return_barrier(); self.routes[slot].as_mut().expect("owned route").active = None; self.counters.returned = next; token.live = false; if closing { self.retire(slot)?; } Ok(ReturnPermit { instance: self.instance, serial: token.serial, owner: token.owner, intid: token.intid, cpu: token.cpu, released: closing, }) } pub fn audit(&self) -> Result<(), Error> { let active = self .routes .iter() .flatten() .filter(|route| route.active.is_some()) .count() as u64; if self.counters.issued.checked_sub(self.counters.returned) != Some(active) { return Err(Error::Backing); } for (slot, route) in self .routes .iter() .enumerate() .filter_map(|(slot, row)| row.as_ref().map(|row| (slot, row))) { if route.owner == 0 || route.intid >= self.configuration.intids || route.cpu >= self.configuration.cpus || route.generation == 0 || route.generation != self.generations[slot] { return Err(Error::Backing); } if self.routes[..slot] .iter() .flatten() .any(|other| other.intid == route.intid) { return Err(Error::Conflict); } if matches!(route.state, State::Closing | State::Suspended) && !route.masked { return Err(Error::MaskReadback); } if route .active .as_ref() .is_some_and(|active| active.serial == 0 || active.serial > self.counters.issued) { return Err(Error::Backing); } self.backend .validate(&route.resource, route.intid, route.owner, route.cpu)?; } Ok(()) } } #[cfg(test)] mod tests { use super::*; #[derive(Default)] struct Model { acknowledgements: u64, callbacks: u64, eois: u64, masks: u64, } // SAFETY: an inert host test backend; no physical IRQ evidence is claimed. unsafe impl Backend for Model { type Resource = (u32, u64, u16); fn acquire(&mut self, intid: u32, owner: u64, cpu: u16) -> Result { Ok((intid, owner, cpu)) } fn validate( &self, r: &Self::Resource, intid: u32, owner: u64, cpu: u16, ) -> Result<(), Error> { if *r == (intid, owner, cpu) { Ok(()) } else { Err(Error::Backing) } } fn configure(&mut self, _: &mut Self::Resource, _: u32, _: u16) {} fn mask_readback(&mut self, _: u32, _: u16) -> bool { self.masks += 1; true } fn enable_readback(&mut self, _: u32, _: u16) -> bool { true } fn acknowledge(&mut self) -> u64 { self.acknowledgements += 1; 30 } fn next_completion(&mut self, _: &mut Self::Resource, _: u64) -> Result { self.callbacks += 1; Ok(Completion::Done { more: false }) } fn eoi(&mut self, _: u32) { self.eois += 1 } fn deactivate(&mut self, _: u32) {} fn return_barrier(&mut self) {} fn release(&mut self, _: Self::Resource) -> Result<(), (Error, Self::Resource)> { Ok(()) } } fn new() -> Gic { Gic::new( 7, Configuration::new(128, 1, 8, true).unwrap(), Model::default(), ) .unwrap() } fn begin(g: &mut Gic) -> Dispatch { match g.begin(0).unwrap() { Begin::Dispatch(t) => t, _ => panic!("expected dispatch"), } } #[test] fn exhausted_route_slots_are_skipped_and_never_wrap() { let mut g = new::<2>(); g.generations[0] = u64::MAX; let token = g.register(30, 7, 0).unwrap(); assert_eq!(token.slot, 1); assert_eq!(g.generations[0], u64::MAX); let before = g.backend.masks; assert_eq!(g.register(33, 8, 0).unwrap_err().error, Error::Generation); assert_eq!(g.backend.masks, before); g.request_close(&token, 7).unwrap(); g.generations[1] = u64::MAX; let before = g.backend.masks; assert_eq!(g.register(30, 7, 0).unwrap_err().error, Error::Generation); assert_eq!(g.backend.masks, before); } #[test] fn issue_overflow_precedes_actual_iar_read() { let mut g = new::<1>(); g.register(30, 7, 0).unwrap(); g.counters.issued = u64::MAX; assert_eq!(g.begin(0).unwrap_err(), Error::Generation); assert_eq!(g.backend.acknowledgements, 0); assert!(g.routes[0].as_ref().unwrap().active.is_none()); } #[test] fn return_overflow_retains_live_ticket_before_eoi() { let mut g = new::<1>(); g.register(30, 7, 0).unwrap(); let mut token = begin(&mut g); g.service(&token, 1).unwrap(); g.counters.returned = u64::MAX; assert_eq!(g.finish(&mut token).unwrap_err(), Error::Generation); assert!(token.live); assert_eq!(g.backend.eois, 0); assert!(g.routes[0].as_ref().unwrap().active.is_some()); g.counters.returned = 0; g.finish(&mut token).unwrap(); assert_eq!(g.backend.eois, 1); } #[test] fn completion_overflow_refuses_before_callbacks_or_ticket_consumption() { let mut g = new::<1>(); g.register(30, 7, 0).unwrap(); let token = begin(&mut g); g.counters.completions = u64::MAX; assert_eq!(g.service(&token, 1), Err(Error::Generation)); assert_eq!(g.backend.callbacks, 0); assert!( !g.routes[0] .as_ref() .unwrap() .active .as_ref() .unwrap() .serviced ); g.counters.completions = 0; g.routes[0].as_mut().unwrap().callbacks = u64::MAX; assert_eq!(g.service(&token, 1), Err(Error::Generation)); assert_eq!(g.backend.callbacks, 0); } #[test] fn stale_dispatch_generation_never_consumes_actual_live_ticket() { let mut g = new::<1>(); g.register(30, 7, 0).unwrap(); let mut token = begin(&mut g); let stale = Dispatch { instance: token.instance, slot: token.slot, generation: token.generation + 1, serial: token.serial, owner: token.owner, intid: token.intid, cpu: token.cpu, live: true, }; assert_eq!(g.service(&stale, 1), Err(Error::Stale)); assert_eq!(g.backend.callbacks, 0); g.service(&token, 1).unwrap(); g.finish(&mut token).unwrap(); g.audit().unwrap(); } }