//! Bounded preparation of a split virtqueue descriptor table. K331 does not //! expose this frame to the device until the other rings and DMA rules exist. use crate::memory::{FrameAllocator, PAGE_SIZE}; use sha2::{Digest, Sha256}; pub const DESCRIPTOR_BYTES: u64 = 16; pub const MAX_SINGLE_PAGE_DESCRIPTORS: u32 = (PAGE_SIZE / DESCRIPTOR_BYTES) as u32; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { InvalidCount, Misaligned, OutsideRam, Overflow, Overlap, OutOfFrames, Active, Released, ReleaseFailed, InvalidBuffer, InvalidDirection, ShortResponse, IoError, UnsupportedOperation, InvalidStatus, InvalidCompletion, Timeout, Full, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ReadRetryDecision { Complete, Retry, Exhausted, } /// K397 permits two retries after the first completed EIO, never an /// unbounded requeue. Unknown block status is a protocol error, not retryable. pub fn bounded_read_retry(attempt: u8, status: u8) -> Result { if !(1..=3).contains(&attempt) { return Err(Error::InvalidCount); } match read_block_status(status) { Ok(()) => Ok(ReadRetryDecision::Complete), Err(Error::IoError) if attempt < 3 => Ok(ReadRetryDecision::Retry), Err(Error::IoError) => Ok(ReadRetryDecision::Exhausted), Err(error) => Err(error), } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct DescriptorPage { pub base: u64, pub bytes: u64, pub count: u32, } #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Descriptor { pub addr_le: u64, pub len_le: u32, pub flags_le: u16, pub next_le: u16, } pub const DESC_F_NEXT: u16 = 1; pub const DESC_F_WRITE: u16 = 2; #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct BlockReadHeader { pub type_le: u32, pub reserved_le: u32, pub sector_le: u64, } pub fn block_read_header(sector: u64) -> BlockReadHeader { BlockReadHeader { type_le: 0u32.to_le(), reserved_le: 0, sector_le: sector.to_le(), } } /// K383 is compiled only into the disposable write experiment. This creates /// the 16-byte virtio-blk OUT header but neither allocates nor submits a queue. #[cfg(any( feature = "virtio-write-header-test", feature = "virtio-metadata-order-test", feature = "p12-journal-test" ))] pub fn block_write_header(sector: u64) -> BlockReadHeader { BlockReadHeader { type_le: 1u32.to_le(), reserved_le: 0, sector_le: sector.to_le(), } } #[cfg(any( feature = "virtio-flush-request-test", feature = "virtio-metadata-order-test", feature = "p12-journal-test" ))] pub fn block_flush_header() -> BlockReadHeader { BlockReadHeader { type_le: 4u32.to_le(), reserved_le: 0, sector_le: 0, } } #[cfg(feature = "virtio-flush-request-test")] pub fn flush_descriptor_chain(header: u64, status: u64) -> [Descriptor; 2] { [ Descriptor { addr_le: header.to_le(), len_le: 16u32.to_le(), flags_le: DESC_F_NEXT.to_le(), next_le: 4u16.to_le(), }, Descriptor { addr_le: status.to_le(), len_le: 1u32.to_le(), flags_le: DESC_F_WRITE.to_le(), next_le: 0, }, ] } #[cfg(feature = "virtio-flush-request-test")] pub fn allow_flush_after_write(write_status: u8, flush_negotiated: bool) -> Result<(), Error> { if !flush_negotiated { return Err(Error::InvalidCompletion); } read_block_status(write_status) } pub fn read_descriptor_chain(header: u64, data: u64, status: u64) -> [Descriptor; 3] { [ Descriptor { addr_le: header.to_le(), len_le: 16u32.to_le(), flags_le: DESC_F_NEXT.to_le(), next_le: 1u16.to_le(), }, Descriptor { addr_le: data.to_le(), len_le: 512u32.to_le(), flags_le: (DESC_F_NEXT | DESC_F_WRITE).to_le(), next_le: 2u16.to_le(), }, Descriptor { addr_le: status.to_le(), len_le: 1u32.to_le(), flags_le: DESC_F_WRITE.to_le(), next_le: 0, }, ] } /// K384: for a block OUT request the device reads header and data. Only the /// terminal status byte is device-writable. This remains feature-gated and is /// submitted only against the disposable qcow2 experiment. #[cfg(any( feature = "virtio-write-descriptor-test", feature = "virtio-metadata-order-test", feature = "p12-journal-test" ))] pub fn write_descriptor_chain(header: u64, data: u64, status: u64) -> [Descriptor; 3] { [ Descriptor { addr_le: header.to_le(), len_le: 16u32.to_le(), flags_le: DESC_F_NEXT.to_le(), next_le: 1u16.to_le(), }, Descriptor { addr_le: data.to_le(), len_le: 512u32.to_le(), flags_le: DESC_F_NEXT.to_le(), next_le: 2u16.to_le(), }, Descriptor { addr_le: status.to_le(), len_le: 1u32.to_le(), flags_le: DESC_F_WRITE.to_le(), next_le: 0, }, ] } #[cfg(any(feature = "virtio-write-descriptor-test", feature = "p12-journal-test"))] pub fn validate_write_descriptor_chain(chain: &[Descriptor; 3]) -> Result<(), Error> { let flags = chain.map(|entry| u16::from_le(entry.flags_le)); if flags != [DESC_F_NEXT, DESC_F_NEXT, DESC_F_WRITE] { return Err(Error::InvalidDirection); } let lengths = chain.map(|entry| u32::from_le(entry.len_le)); let next = chain.map(|entry| u16::from_le(entry.next_le)); let addresses = chain.map(|entry| u64::from_le(entry.addr_le)); if lengths != [16, 512, 1] || next != [1, 2, 0] || addresses.contains(&0) || addresses[0] == addresses[1] || addresses[0] == addresses[2] || addresses[1] == addresses[2] { return Err(Error::InvalidBuffer); } Ok(()) } /// Materialize an independent three-descriptor read at a reserved table /// offset. The chain's next fields must refer to absolute table indices. pub fn read_descriptor_chain_at( first: u16, header: u64, data: u64, status: u64, ) -> Result<[Descriptor; 3], Error> { if u32::from(first) + 2 >= MAX_SINGLE_PAGE_DESCRIPTORS { return Err(Error::InvalidCount); } let mut chain = read_descriptor_chain(header, data, status); chain[0].next_le = (first + 1).to_le(); chain[1].next_le = (first + 2).to_le(); Ok(chain) } /// A block read's data buffer must contain whole 512-byte logical sectors. /// The caller separately proves that the DMA subregion stays inside its lease. pub fn read_descriptor_chain_with_len( header: u64, data: u64, status: u64, data_bytes: u32, ) -> Result<[Descriptor; 3], Error> { if data_bytes == 0 || data_bytes % 512 != 0 { return Err(Error::InvalidBuffer); } let mut chain = read_descriptor_chain(header, data, status); chain[1].len_le = data_bytes.to_le(); Ok(chain) } /// Two device-written sectors in distinct descriptors. A four-entry queue /// can hold exactly this one chain and must reject any following read. pub fn read_descriptor_chain_split_data( header: u64, data_first: u64, data_second: u64, status: u64, ) -> [Descriptor; 4] { [ Descriptor { addr_le: header.to_le(), len_le: 16u32.to_le(), flags_le: DESC_F_NEXT.to_le(), next_le: 1u16.to_le(), }, Descriptor { addr_le: data_first.to_le(), len_le: 512u32.to_le(), flags_le: (DESC_F_NEXT | DESC_F_WRITE).to_le(), next_le: 2u16.to_le(), }, Descriptor { addr_le: data_second.to_le(), len_le: 512u32.to_le(), flags_le: (DESC_F_NEXT | DESC_F_WRITE).to_le(), next_le: 3u16.to_le(), }, Descriptor { addr_le: status.to_le(), len_le: 1u32.to_le(), flags_le: DESC_F_WRITE.to_le(), next_le: 0, }, ] } /// The used-ring length includes every device-written data byte plus the /// terminal status byte. Never expose unwritten data as a successful read. pub fn validate_read_completion_length(used_len: u32, data_bytes: u32) -> Result<(), Error> { if data_bytes == 0 || data_bytes % 512 != 0 { return Err(Error::InvalidBuffer); } let expected = data_bytes.checked_add(1).ok_or(Error::Overflow)?; if used_len < expected { Err(Error::ShortResponse) } else if used_len > expected { Err(Error::InvalidBuffer) } else { Ok(()) } } /// Completion order is not contractual. Each used ID must nevertheless name /// exactly one of the two heads submitted in this queue epoch. pub fn validate_two_completion_heads(used: [u32; 2], heads: [u16; 2]) -> Result<(), Error> { if heads[0] == heads[1] || !((used[0] == u32::from(heads[0]) && used[1] == u32::from(heads[1])) || (used[0] == u32::from(heads[1]) && used[1] == u32::from(heads[0]))) { return Err(Error::InvalidCompletion); } Ok(()) } /// Wipe an entire owned request frame before it is reused for another queue /// epoch. The caller must first reset the device and retire its DMA lease. pub fn scrub_dma_request_bytes(bytes: &mut [u8]) -> Result<(), Error> { if bytes.len() != PAGE_SIZE as usize { return Err(Error::InvalidBuffer); } for byte in bytes { // DMA-visible memory must really be written; a dead-store-eliminated // zeroing pass would not establish this experiment's precondition. unsafe { core::ptr::write_volatile(byte as *mut u8, 0) }; } Ok(()) } pub fn read_block_status(status: u8) -> Result<(), Error> { match status { 0 => Ok(()), 1 => Err(Error::IoError), 2 => Err(Error::UnsupportedOperation), _ => Err(Error::InvalidStatus), } } /// A completed OUT request writes only its terminal status byte. A used-ring /// entry alone cannot turn a failed device write into accepted data. #[cfg(feature = "virtio-write-descriptor-test")] pub fn validate_write_completion( used_index: u16, used_id: u32, used_len: u32, status: u8, ) -> Result<(), Error> { if used_index != 1 || used_id != 0 { return Err(Error::InvalidCompletion); } if used_len == 0 { return Err(Error::ShortResponse); } if used_len != 1 { return Err(Error::InvalidBuffer); } read_block_status(status) } #[cfg(any( feature = "virtio-write-readback-test", feature = "virtio-flush-reopen-test" ))] pub fn write_test_pattern_byte(index: usize) -> Result { if index >= 512 { return Err(Error::InvalidBuffer); } Ok(index as u8 ^ 0xa7) } #[cfg(any( feature = "virtio-two-write-order-test", feature = "virtio-two-write-reopen-test" ))] pub fn two_write_final_pattern_byte(index: usize) -> Result { if index >= 512 { return Err(Error::InvalidBuffer); } Ok((index as u8) ^ 0x3c) } pub fn sector_sha256(sector: &[u8; 512]) -> [u8; 32] { Sha256::digest(sector).into() } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SplitRingPages { pub descriptor: u64, pub available: u64, pub used: u64, pub count: u32, pub available_bytes: u64, pub used_bytes: u64, } /// Bounded descriptor admission for a single queue epoch. A rejected /// reservation never advances the cursor, so existing chains keep ownership. /// Reuse after device completion is intentionally a separate operation. pub struct QueueAdmission { capacity: u16, occupied: u16, } impl QueueAdmission { pub fn new(capacity: u16) -> Result { if capacity == 0 || !capacity.is_power_of_two() || u32::from(capacity) > MAX_SINGLE_PAGE_DESCRIPTORS { return Err(Error::InvalidCount); } Ok(Self { capacity, occupied: 0, }) } pub fn reserve(&mut self, count: u16) -> Result { if count == 0 || count > self.capacity { return Err(Error::InvalidCount); } let end = self.occupied.checked_add(count).ok_or(Error::Full)?; if end > self.capacity { return Err(Error::Full); } let first = self.occupied; self.occupied = end; Ok(first) } pub fn occupied(&self) -> u16 { self.occupied } pub fn remaining(&self) -> u16 { self.capacity - self.occupied } /// Caller must first observe device reset and QueueReady=0. This only /// resets software admission; descriptor RAM must be scrubbed separately. pub fn reset_epoch_after_device_reset(&mut self) { self.occupied = 0; } } /// Four distinct frames owned through the entire live queue lifetime: the /// descriptor, available, used and request/data page, in that order. pub struct DmaLease { frames: [u64; 4], active: bool, released: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DmaFrameRole { Descriptor, Available, Used, RequestData, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct TaggedDmaFrame { pub address: u64, pub role: DmaFrameRole, /// QueueReady means the device may still access this physical frame. pub live_queue: bool, } impl DmaLease { pub fn allocate( allocator: &mut FrameAllocator, ram_base: u64, ram_size: u64, ) -> Result { let mut frames = [0u64; 4]; for index in 0..frames.len() { let Some(frame) = allocator.allocate() else { for allocated in frames[..index].iter().copied() { allocator .release(allocated) .map_err(|_| Error::ReleaseFailed)?; } return Err(Error::OutOfFrames); }; frames[index] = frame; } let valid = split_ring_pages(frames[0], frames[1], frames[2], 256, ram_base, ram_size) .and_then(|_| descriptor_page(frames[3], 1, ram_base, ram_size).map(|_| ())) .and_then(|_| { if frames[3] == frames[0] || frames[3] == frames[1] || frames[3] == frames[2] { Err(Error::Overlap) } else { Ok(()) } }); if let Err(error) = valid { for frame in frames { allocator.release(frame).map_err(|_| Error::ReleaseFailed)?; } return Err(error); } Ok(Self { frames, active: false, released: false, }) } pub fn frames(&self) -> Result<[u64; 4], Error> { if self.released { Err(Error::Released) } else { Ok(self.frames) } } pub fn tagged_frames(&self) -> Result<[TaggedDmaFrame; 4], Error> { if self.released { return Err(Error::Released); } let roles = [ DmaFrameRole::Descriptor, DmaFrameRole::Available, DmaFrameRole::Used, DmaFrameRole::RequestData, ]; Ok(core::array::from_fn(|index| TaggedDmaFrame { address: self.frames[index], role: roles[index], live_queue: self.active, })) } /// Only the fourth leased page may hold request buffers. A descriptor /// targeting a kernel, DTB, ring or unrelated RAM page must never become /// visible in the device-owned table. pub fn request_subregion(&self, offset: u64, length: u64) -> Result { if self.released { return Err(Error::Released); } dma_subregion(self.frames[3], offset, length) } /// Pre-submit gate for the fixed read-only TurkFS three-descriptor chain. /// Geometry, direction and every physical byte range are tied to this /// lease, not merely to the machine's broad RAM interval. pub fn validate_read_chain(&self, chain: &[Descriptor; 3]) -> Result<(), Error> { if self.released { return Err(Error::Released); } if chain.map(|entry| u16::from_le(entry.flags_le)) != [DESC_F_NEXT, DESC_F_NEXT | DESC_F_WRITE, DESC_F_WRITE] { return Err(Error::InvalidDirection); } if chain.map(|entry| u32::from_le(entry.len_le)) != [16, 512, 1] { return Err(Error::InvalidBuffer); } if chain.map(|entry| u16::from_le(entry.next_le)) != [1, 2, 0] { return Err(Error::InvalidCount); } let expected = [ self.request_subregion(0, 16)?, self.request_subregion(512, 512)?, self.request_subregion(16, 1)?, ]; if chain.map(|entry| u64::from_le(entry.addr_le)) != expected { return Err(Error::InvalidBuffer); } Ok(()) } pub fn mark_ready(&mut self) -> Result<(), Error> { if self.released { return Err(Error::Released); } self.active = true; Ok(()) } pub fn mark_reset(&mut self) -> Result<(), Error> { if self.released { return Err(Error::Released); } self.active = false; Ok(()) } pub fn release(&mut self, allocator: &mut FrameAllocator) -> Result<(), Error> { if self.released { return Err(Error::Released); } if self.active { return Err(Error::Active); } for frame in self.frames { allocator.release(frame).map_err(|_| Error::ReleaseFailed)?; } self.released = true; Ok(()) } } pub fn split_ring_pages( descriptor: u64, available: u64, used: u64, count: u32, ram_base: u64, ram_size: u64, ) -> Result { if !count.is_power_of_two() { return Err(Error::InvalidCount); } descriptor_page(descriptor, count, ram_base, ram_size)?; descriptor_page(available, 1, ram_base, ram_size)?; descriptor_page(used, 1, ram_base, ram_size)?; if descriptor == available || descriptor == used || available == used { return Err(Error::Overlap); } let available_bytes = 4 + u64::from(count) * 2; let used_bytes = 4 + u64::from(count) * 8; if available_bytes > PAGE_SIZE || used_bytes > PAGE_SIZE { return Err(Error::InvalidCount); } Ok(SplitRingPages { descriptor, available, used, count, available_bytes, used_bytes, }) } pub fn descriptor_page( frame: u64, count: u32, ram_base: u64, ram_size: u64, ) -> Result { if count == 0 || count > MAX_SINGLE_PAGE_DESCRIPTORS { return Err(Error::InvalidCount); } if frame & (PAGE_SIZE - 1) != 0 { return Err(Error::Misaligned); } let bytes = u64::from(count) .checked_mul(DESCRIPTOR_BYTES) .ok_or(Error::Overflow)?; let frame_end = frame.checked_add(PAGE_SIZE).ok_or(Error::Overflow)?; let ram_end = ram_base.checked_add(ram_size).ok_or(Error::Overflow)?; if frame < ram_base || frame_end > ram_end { return Err(Error::OutsideRam); } Ok(DescriptorPage { base: frame, bytes, count, }) } /// Return a DMA buffer address only when the whole non-empty buffer remains /// inside one exclusively owned 4 KiB page. The caller retains that page. pub fn dma_subregion(frame: u64, offset: u64, length: u64) -> Result { if frame & (PAGE_SIZE - 1) != 0 { return Err(Error::Misaligned); } if length == 0 { return Err(Error::InvalidBuffer); } let end_offset = offset.checked_add(length).ok_or(Error::Overflow)?; if end_offset > PAGE_SIZE { return Err(Error::InvalidBuffer); } let address = frame.checked_add(offset).ok_or(Error::Overflow)?; frame.checked_add(end_offset).ok_or(Error::Overflow)?; Ok(address) } /// A non-zero counter interval, checked before entering a bounded poll loop. pub fn deadline_after(start: u64, interval: u64) -> Result { if interval == 0 { return Err(Error::InvalidBuffer); } start.checked_add(interval).ok_or(Error::Overflow) } pub fn deadline_expired(now: u64, deadline: u64) -> bool { now >= deadline } /// Completion wins when observed; otherwise the caller must retain its DMA /// lease on timeout until the device has been safely quiesced. pub fn active_completion_state(used_index: u16, now: u64, deadline: u64) -> Result { if used_index != 0 { Ok(true) } else if deadline_expired(now, deadline) { Err(Error::Timeout) } else { Ok(false) } } #[cfg(test)] mod tests { extern crate std; use super::*; #[test] fn k077_dma_roles_follow_queue_ready_reset_and_release() { use crate::memory::FreeMap; let base = 0x4000_0000; let mut allocator = FrameAllocator::new(FreeMap::from_ram(base, 16 * PAGE_SIZE).unwrap()); let before = allocator.remaining_pages(); let mut lease = DmaLease::allocate(&mut allocator, base, 16 * PAGE_SIZE).unwrap(); let roles = [ DmaFrameRole::Descriptor, DmaFrameRole::Available, DmaFrameRole::Used, DmaFrameRole::RequestData, ]; let prepared = lease.tagged_frames().unwrap(); for (index, tag) in prepared.iter().enumerate() { assert_eq!(tag.role, roles[index]); assert_eq!(tag.address, lease.frames().unwrap()[index]); assert!(!tag.live_queue); } lease.mark_ready().unwrap(); assert!( lease .tagged_frames() .unwrap() .iter() .all(|tag| tag.live_queue) ); assert_eq!(lease.release(&mut allocator), Err(Error::Active)); assert_eq!(allocator.remaining_pages(), before - 4); lease.mark_reset().unwrap(); assert!( lease .tagged_frames() .unwrap() .iter() .all(|tag| !tag.live_queue) ); lease.release(&mut allocator).unwrap(); assert_eq!(allocator.remaining_pages(), before); assert_eq!(lease.tagged_frames(), Err(Error::Released)); } #[test] fn k446_dma_descriptor_fuzz_rejects_unowned_and_protected_frames() { let lease = DmaLease { frames: [0x4000_4000, 0x4000_5000, 0x4000_6000, 0x4000_9000], active: true, released: false, }; let valid = read_descriptor_chain(0x4000_9000, 0x4000_9200, 0x4000_9010); assert_eq!(lease.validate_read_chain(&valid), Ok(())); assert_eq!(lease.request_subregion(4095, 1), Ok(0x4000_9fff)); assert_eq!(lease.request_subregion(4095, 2), Err(Error::InvalidBuffer)); let protected: [u64; 5] = [ 0x4000_0000, 0x4000_2000, 0x4000_4000, 0x4000_5000, 0x4000_a000, ]; for address in protected { let mut forged = valid; forged[1].addr_le = address.to_le(); assert_eq!( lease.validate_read_chain(&forged), Err(Error::InvalidBuffer) ); assert_eq!(forged[1].addr_le, address.to_le()); } let released = DmaLease { frames: [0x4000_4000, 0x4000_5000, 0x4000_6000, 0x4000_9000], active: false, released: true, }; assert_eq!(released.request_subregion(0, 16), Err(Error::Released)); assert_eq!(released.validate_read_chain(&valid), Err(Error::Released)); let mut digest = Sha256::new(); let mut state = 0x4460_2026_d15c_0a11u64; let mut accepted = 0usize; let mut rejected = 0usize; for index in 0..32_768usize { state ^= state << 13; state ^= state >> 7; state ^= state << 17; let mut candidate = valid; if index % 128 != 0 { let slot = (state % 3) as usize; let bit = (state >> 8) as u32; match (state >> 20) & 3 { 0 => { let address = if index % 8 == 0 { protected[(state >> 32) as usize % protected.len()] } else { u64::from_le(candidate[slot].addr_le) ^ (1u64 << (bit % 64)) }; candidate[slot].addr_le = address.to_le(); } 1 => { candidate[slot].len_le = (u32::from_le(candidate[slot].len_le) ^ (1u32 << (bit % 32))).to_le() } 2 => { candidate[slot].flags_le = (u16::from_le(candidate[slot].flags_le) ^ (1u16 << (bit % 16))).to_le() } _ => { candidate[slot].next_le = (u16::from_le(candidate[slot].next_le) ^ (1u16 << (bit % 16))).to_le() } } } let before = candidate; let actual = lease.validate_read_chain(&candidate); let expected_accept = candidate == valid; assert_eq!( actual.is_ok(), expected_accept, "descriptor mutation {index}" ); assert_eq!(candidate, before, "validation changed descriptor {index}"); if actual.is_ok() { accepted += 1; } else { rejected += 1; } for entry in candidate { digest.update(u64::from_le(entry.addr_le).to_le_bytes()); digest.update(u32::from_le(entry.len_le).to_le_bytes()); digest.update(u16::from_le(entry.flags_le).to_le_bytes()); digest.update(u16::from_le(entry.next_le).to_le_bytes()); } digest.update([u8::from(actual.is_ok())]); } assert_eq!((accepted, rejected), (256, 32_512)); let hash = digest .finalize() .iter() .map(|byte| std::format!("{byte:02x}")) .collect::(); std::eprintln!( "K446 CASES=32768 OK={accepted} REJECTED={rejected} PROTECTED=5 RELEASED=2 SHA256={hash}" ); } #[test] fn k397_completed_eio_retries_twice_then_stops_without_retrying_bad_status() { assert_eq!(bounded_read_retry(1, 1), Ok(ReadRetryDecision::Retry)); assert_eq!(bounded_read_retry(2, 1), Ok(ReadRetryDecision::Retry)); assert_eq!(bounded_read_retry(3, 1), Ok(ReadRetryDecision::Exhausted)); for attempt in 1..=3 { assert_eq!( bounded_read_retry(attempt, 0), Ok(ReadRetryDecision::Complete) ); assert_eq!( bounded_read_retry(attempt, 2), Err(Error::UnsupportedOperation) ); assert_eq!(bounded_read_retry(attempt, 0xff), Err(Error::InvalidStatus)); } assert_eq!(bounded_read_retry(0, 1), Err(Error::InvalidCount)); assert_eq!(bounded_read_retry(4, 1), Err(Error::InvalidCount)); } #[test] fn k331_exactly_256_descriptors_fill_one_owned_page() { assert_eq!(DESCRIPTOR_BYTES, 16); assert_eq!(MAX_SINGLE_PAGE_DESCRIPTORS, 256); assert_eq!( descriptor_page(0x4000_4000, 256, 0x4000_0000, 0x1000_0000), Ok(DescriptorPage { base: 0x4000_4000, bytes: PAGE_SIZE, count: 256, }) ); } #[test] fn k336_queue_admission_rejects_full_without_reassigning_live_chain() { let mut queue = QueueAdmission::new(256).unwrap(); assert_eq!(queue.reserve(3), Ok(0)); assert_eq!(queue.reserve(253), Ok(3)); assert_eq!(queue.occupied(), 256); assert_eq!(queue.remaining(), 0); assert_eq!(queue.reserve(1), Err(Error::Full)); assert_eq!(queue.reserve(3), Err(Error::Full)); assert_eq!(queue.occupied(), 256); assert_eq!(queue.remaining(), 0); } #[test] fn k336_queue_admission_rejects_invalid_counts_and_crossing_capacity() { assert!(matches!(QueueAdmission::new(0), Err(Error::InvalidCount))); assert!(matches!(QueueAdmission::new(3), Err(Error::InvalidCount))); assert!(matches!(QueueAdmission::new(512), Err(Error::InvalidCount))); let mut queue = QueueAdmission::new(256).unwrap(); assert_eq!(queue.reserve(0), Err(Error::InvalidCount)); assert_eq!(queue.reserve(257), Err(Error::InvalidCount)); assert_eq!(queue.reserve(254), Ok(0)); assert_eq!(queue.reserve(3), Err(Error::Full)); assert_eq!(queue.occupied(), 254); assert_eq!(queue.remaining(), 2); assert_eq!(queue.reserve(2), Ok(254)); } #[test] fn k337_bounded_counter_deadline_rejects_zero_and_overflow() { assert_eq!(deadline_after(100, 25), Ok(125)); assert!(!deadline_expired(124, 125)); assert!(deadline_expired(125, 125)); assert!(deadline_expired(126, 125)); assert_eq!(deadline_after(100, 0), Err(Error::InvalidBuffer)); assert_eq!(deadline_after(u64::MAX - 2, 3), Err(Error::Overflow)); } #[test] fn k339_reset_epoch_allows_new_chain_only_after_explicit_reset() { let mut queue = QueueAdmission::new(256).unwrap(); assert_eq!(queue.reserve(256), Ok(0)); assert_eq!(queue.reserve(1), Err(Error::Full)); queue.reset_epoch_after_device_reset(); assert_eq!(queue.occupied(), 0); assert_eq!(queue.remaining(), 256); assert_eq!(queue.reserve(3), Ok(0)); assert_eq!(queue.occupied(), 3); } #[test] fn k343_read_header_has_exact_type_reserved_sector_and_size() { assert_eq!(core::mem::size_of::(), 16); assert_eq!(core::mem::align_of::(), 8); assert_eq!( block_read_header(1), BlockReadHeader { type_le: 0, reserved_le: 0, sector_le: 1u64.to_le() } ); } #[test] fn k344_data_descriptor_is_device_writable_and_chained_once() { let chain = read_descriptor_chain(0x4000_9000, 0x4000_9200, 0x4000_9010); assert_eq!(u32::from_le(chain[0].len_le), 16); assert_eq!(u16::from_le(chain[0].flags_le), DESC_F_NEXT); assert_eq!(u16::from_le(chain[0].next_le), 1); assert_eq!(u32::from_le(chain[1].len_le), 512); assert_eq!(u16::from_le(chain[1].flags_le), DESC_F_NEXT | DESC_F_WRITE); assert_eq!(u16::from_le(chain[1].next_le), 2); } #[cfg(feature = "virtio-write-header-test")] #[test] fn k383_write_header_is_exactly_sixteen_little_endian_bytes() { let header = block_write_header(0x0102_0304_0506_0708); assert_eq!(core::mem::size_of_val(&header), 16); assert_eq!(u32::from_le(header.type_le), 1); assert_eq!(u32::from_le(header.reserved_le), 0); assert_eq!(u64::from_le(header.sector_le), 0x0102_0304_0506_0708); let bytes = unsafe { core::slice::from_raw_parts((&header as *const BlockReadHeader).cast::(), 16) }; assert_eq!(bytes, &[1, 0, 0, 0, 0, 0, 0, 0, 8, 7, 6, 5, 4, 3, 2, 1]); assert_ne!(block_write_header(2), block_read_header(2)); } #[cfg(feature = "virtio-write-descriptor-test")] #[test] fn k384_write_data_is_device_readable_and_only_status_is_device_writable() { let chain = write_descriptor_chain(0x4000_9000, 0x4000_9200, 0x4000_9010); assert_eq!(validate_write_descriptor_chain(&chain), Ok(())); assert_eq!(chain.map(|entry| u16::from_le(entry.flags_le)), [1, 1, 2]); assert_eq!(chain.map(|entry| u32::from_le(entry.len_le)), [16, 512, 1]); assert_eq!(u64::from_le(chain[1].addr_le), 0x4000_9200); let mut wrong_data = chain; wrong_data[1].flags_le = (DESC_F_NEXT | DESC_F_WRITE).to_le(); assert_eq!( validate_write_descriptor_chain(&wrong_data), Err(Error::InvalidDirection) ); let mut wrong_status = chain; wrong_status[2].flags_le = 0; assert_eq!( validate_write_descriptor_chain(&wrong_status), Err(Error::InvalidDirection) ); let mut short_data = chain; short_data[1].len_le = 511u32.to_le(); assert_eq!( validate_write_descriptor_chain(&short_data), Err(Error::InvalidBuffer) ); let mut alias = chain; alias[1].addr_le = alias[0].addr_le; assert_eq!( validate_write_descriptor_chain(&alias), Err(Error::InvalidBuffer) ); } #[cfg(feature = "virtio-write-status-test")] #[test] fn k385_write_completion_accepts_only_exact_success_status() { assert_eq!(validate_write_completion(1, 0, 1, 0), Ok(())); assert_eq!(validate_write_completion(1, 0, 1, 1), Err(Error::IoError)); assert_eq!( validate_write_completion(1, 0, 1, 2), Err(Error::UnsupportedOperation) ); assert_eq!( validate_write_completion(1, 0, 1, 0xff), Err(Error::InvalidStatus) ); assert_eq!( validate_write_completion(0, 0, 1, 0), Err(Error::InvalidCompletion) ); assert_eq!( validate_write_completion(1, 1, 1, 0), Err(Error::InvalidCompletion) ); assert_eq!( validate_write_completion(1, 0, 0, 0), Err(Error::ShortResponse) ); assert_eq!( validate_write_completion(1, 0, 2, 0), Err(Error::InvalidBuffer) ); } #[cfg(feature = "virtio-write-readback-test")] #[test] fn k386_readback_pattern_covers_exactly_one_sector_and_checks_completion() { assert_eq!(write_test_pattern_byte(0), Ok(0xa7)); assert_eq!(write_test_pattern_byte(255), Ok(0x58)); assert_eq!(write_test_pattern_byte(256), Ok(0xa7)); assert_eq!(write_test_pattern_byte(511), Ok(0x58)); assert_eq!(write_test_pattern_byte(512), Err(Error::InvalidBuffer)); let sum: usize = (0..512) .map(|index| write_test_pattern_byte(index).unwrap() as usize) .sum(); assert_eq!(sum, 0xff00); assert_eq!(validate_read_completion_length(513, 512), Ok(())); assert_eq!( validate_read_completion_length(512, 512), Err(Error::ShortResponse) ); assert_eq!(read_block_status(1), Err(Error::IoError)); } #[cfg(feature = "virtio-two-write-order-test")] #[test] fn k390_second_write_is_a_distinct_full_sector_and_first_does_not_match() { assert_eq!(two_write_final_pattern_byte(0), Ok(0x3c)); assert_eq!(two_write_final_pattern_byte(255), Ok(0xc3)); assert_eq!(two_write_final_pattern_byte(256), Ok(0x3c)); assert_eq!(two_write_final_pattern_byte(511), Ok(0xc3)); assert_eq!(two_write_final_pattern_byte(512), Err(Error::InvalidBuffer)); let mut second = [0u8; 512]; for (index, byte) in second.iter_mut().enumerate() { *byte = two_write_final_pattern_byte(index).unwrap(); } assert_eq!( second.iter().map(|byte| *byte as usize).sum::(), 0xff00 ); assert!( second .iter() .enumerate() .all(|(index, byte)| *byte != ((index as u8) ^ 0xa7)) ); } #[cfg(feature = "virtio-flush-request-test")] #[test] fn k388_flush_has_type_four_zero_sector_and_waits_for_successful_write() { let header = block_flush_header(); assert_eq!(u32::from_le(header.type_le), 4); assert_eq!(u32::from_le(header.reserved_le), 0); assert_eq!(u64::from_le(header.sector_le), 0); let chain = flush_descriptor_chain(0x4000_9400, 0x4000_9410); assert_eq!(chain.map(|entry| u32::from_le(entry.len_le)), [16, 1]); assert_eq!(chain.map(|entry| u16::from_le(entry.flags_le)), [1, 2]); assert_eq!(chain.map(|entry| u16::from_le(entry.next_le)), [4, 0]); assert_eq!(allow_flush_after_write(0, true), Ok(())); assert_eq!(allow_flush_after_write(1, true), Err(Error::IoError)); assert_eq!( allow_flush_after_write(0, false), Err(Error::InvalidCompletion) ); } #[test] fn k345_status_descriptor_is_one_device_writable_byte_and_terminal() { let chain = read_descriptor_chain(0x4000_9000, 0x4000_9200, 0x4000_9010); assert_eq!(u64::from_le(chain[2].addr_le), 0x4000_9010); assert_eq!(u32::from_le(chain[2].len_le), 1); assert_eq!(u16::from_le(chain[2].flags_le), DESC_F_WRITE); assert_eq!(u16::from_le(chain[2].next_le), 0); } #[test] fn k350_two_sector_descriptor_has_exact_1024_byte_device_write_span() { let chain = read_descriptor_chain_with_len(0x4000_9000, 0x4000_9200, 0x4000_9010, 1024).unwrap(); assert_eq!(u32::from_le(chain[0].len_le), 16); assert_eq!(u32::from_le(chain[1].len_le), 1024); assert_eq!(u16::from_le(chain[1].flags_le), DESC_F_NEXT | DESC_F_WRITE); assert_eq!(u32::from_le(chain[2].len_le), 1); for bad in [0, 1, 511, 513, 1023] { assert_eq!( read_descriptor_chain_with_len(0, 0, 0, bad), Err(Error::InvalidBuffer) ); } } #[test] fn k351_short_used_length_cannot_turn_unwritten_bytes_into_success() { assert_eq!(validate_read_completion_length(1025, 1024), Ok(())); for short in [0, 1, 513, 1024] { assert_eq!( validate_read_completion_length(short, 1024), Err(Error::ShortResponse) ); } assert_eq!( validate_read_completion_length(1026, 1024), Err(Error::InvalidBuffer) ); assert_eq!( validate_read_completion_length(513, 0), Err(Error::InvalidBuffer) ); assert_eq!( validate_read_completion_length(0, u32::MAX), Err(Error::InvalidBuffer) ); } #[test] fn k456_short_completion_retains_four_dma_pages_until_reset() { let base = 0x4000_0000; let map = crate::memory::FreeMap::from_ram(base, 16 * PAGE_SIZE).unwrap(); let mut allocator = FrameAllocator::new(map); let mut lease = DmaLease::allocate(&mut allocator, base, 16 * PAGE_SIZE).unwrap(); let owned = lease.frames().unwrap(); assert_eq!(allocator.remaining_pages(), 12); lease.mark_ready().unwrap(); assert_eq!( validate_read_completion_length(257, 512), Err(Error::ShortResponse) ); assert_eq!(lease.release(&mut allocator), Err(Error::Active)); assert_eq!(lease.frames(), Ok(owned)); assert_eq!(allocator.remaining_pages(), 12); lease.mark_reset().unwrap(); assert_eq!(lease.release(&mut allocator), Ok(())); assert_eq!(allocator.remaining_pages(), 16); assert_eq!(lease.release(&mut allocator), Err(Error::Released)); } #[test] fn k354_four_descriptor_read_fills_actual_four_entry_queue() { let mut queue = QueueAdmission::new(4).unwrap(); let chain = read_descriptor_chain_split_data(0x4000_9000, 0x4000_9200, 0x4000_9400, 0x4000_9010); assert_eq!(queue.reserve(chain.len() as u16), Ok(0)); assert_eq!(queue.reserve(3), Err(Error::Full)); assert_eq!((queue.occupied(), queue.remaining()), (4, 0)); assert_eq!(chain.map(|d| u32::from_le(d.len_le)), [16, 512, 512, 1]); assert_eq!( chain.map(|d| u16::from_le(d.flags_le)), [ DESC_F_NEXT, DESC_F_NEXT | DESC_F_WRITE, DESC_F_NEXT | DESC_F_WRITE, DESC_F_WRITE ] ); assert_eq!(chain.map(|d| u16::from_le(d.next_le)), [1, 2, 3, 0]); assert_eq!( split_ring_pages( 0x4000_6000, 0x4000_7000, 0x4000_8000, 4, 0x4000_0000, 0x1000_0000 ) .map(|r| (r.available_bytes, r.used_bytes)), Ok((12, 36)) ); } #[test] fn k355_two_reads_use_disjoint_descriptor_chains_and_dma_regions() { let mut admission = QueueAdmission::new(8).unwrap(); let first = admission.reserve(3).unwrap(); let second = admission.reserve(3).unwrap(); assert_eq!( (first, second, admission.occupied(), admission.remaining()), (0, 3, 6, 2) ); let a = read_descriptor_chain_at(first, 0x4000_9000, 0x4000_9200, 0x4000_9010).unwrap(); let b = read_descriptor_chain_at(second, 0x4000_9020, 0x4000_9400, 0x4000_9030).unwrap(); assert_eq!(a.map(|d| u16::from_le(d.next_le)), [1, 2, 0]); assert_eq!(b.map(|d| u16::from_le(d.next_le)), [4, 5, 0]); assert_eq!( b.map(|d| u64::from_le(d.addr_le)), [0x4000_9020, 0x4000_9400, 0x4000_9030] ); assert_eq!( read_descriptor_chain_at(254, 0, 0, 0), Err(Error::InvalidCount) ); assert_eq!( split_ring_pages( 0x4000_6000, 0x4000_7000, 0x4000_8000, 8, 0x4000_0000, 0x1000_0000 ) .map(|r| (r.available_bytes, r.used_bytes)), Ok((20, 68)) ); } #[test] fn k356_completion_ids_reject_foreign_and_duplicate_heads_in_either_order() { assert_eq!(validate_two_completion_heads([0, 3], [0, 3]), Ok(())); assert_eq!(validate_two_completion_heads([3, 0], [0, 3]), Ok(())); for invalid in [[6, 3], [0, 6], [0, 0], [3, 3], [u32::MAX, 0]] { assert_eq!( validate_two_completion_heads(invalid, [0, 3]), Err(Error::InvalidCompletion) ); } assert_eq!( validate_two_completion_heads([0, 0], [0, 0]), Err(Error::InvalidCompletion) ); } #[test] fn k357_scrub_entire_dma_request_frame_without_touching_neighbors() { let mut owned = [0xccu8; PAGE_SIZE as usize + 2]; owned[1 + 512..1 + 1024].fill(0x34); owned[1 + 1024..1 + 1536].fill(0xcb); assert_eq!( scrub_dma_request_bytes(&mut owned[1..PAGE_SIZE as usize]), Err(Error::InvalidBuffer) ); assert_eq!((owned[1 + 512], owned[1 + 1024]), (0x34, 0xcb)); assert_eq!( scrub_dma_request_bytes(&mut owned[1..1 + PAGE_SIZE as usize]), Ok(()) ); assert!( owned[1..1 + PAGE_SIZE as usize] .iter() .all(|byte| *byte == 0) ); assert_eq!((owned[0], owned[PAGE_SIZE as usize + 1]), (0xcc, 0xcc)); } #[test] fn k352_block_status_distinguishes_success_io_error_and_unsupported() { assert_eq!(read_block_status(0), Ok(())); assert_eq!(read_block_status(1), Err(Error::IoError)); assert_eq!(read_block_status(2), Err(Error::UnsupportedOperation)); for invalid in [3, 0xff] { assert_eq!(read_block_status(invalid), Err(Error::InvalidStatus)); } } #[test] fn k353_active_timeout_keeps_pending_dma_ownership() { assert_eq!(active_completion_state(0, 19, 20), Ok(false)); assert_eq!(active_completion_state(0, 20, 20), Err(Error::Timeout)); assert_eq!(active_completion_state(0, 21, 20), Err(Error::Timeout)); assert_eq!(active_completion_state(1, 21, 20), Ok(true)); let mut allocator = FrameAllocator::new( crate::memory::FreeMap::from_ram(0x4000_0000, PAGE_SIZE * 16).unwrap(), ); let mut lease = DmaLease::allocate(&mut allocator, 0x4000_0000, PAGE_SIZE * 16).unwrap(); lease.mark_ready().unwrap(); assert_eq!(lease.release(&mut allocator), Err(Error::Active)); } #[test] fn k346_known_sector_hash_changes_when_one_byte_changes() { let mut sector = [0u8; 512]; for (index, byte) in sector.iter_mut().enumerate() { *byte = (index as u8) ^ 0xa5; } let expected = b"755929613d018a9cb308fd873b7b5b53078961f9be477d3650fa0710b350ad80"; assert_eq!( crate::program_hash::hex_lower(§or_sha256(§or)), *expected ); sector[511] ^= 1; assert_ne!( crate::program_hash::hex_lower(§or_sha256(§or)), *expected ); } #[test] fn k331_rejects_bad_count_alignment_ram_and_overflow() { let ram = (0x4000_0000, 0x1000_0000); assert_eq!( descriptor_page(0x4000_4000, 0, ram.0, ram.1), Err(Error::InvalidCount) ); assert_eq!( descriptor_page(0x4000_4000, 257, ram.0, ram.1), Err(Error::InvalidCount) ); assert_eq!( descriptor_page(0x4000_4008, 256, ram.0, ram.1), Err(Error::Misaligned) ); assert_eq!( descriptor_page(0x5000_0000, 256, ram.0, ram.1), Err(Error::OutsideRam) ); assert_eq!( descriptor_page(0x3fff_f000, 256, ram.0, ram.1), Err(Error::OutsideRam) ); assert_eq!( descriptor_page(u64::MAX & !(PAGE_SIZE - 1), 256, ram.0, ram.1), Err(Error::Overflow) ); assert_eq!( descriptor_page(0x4000_4000, 256, u64::MAX - 1, 0x100), Err(Error::Overflow) ); } #[test] fn split_ring_structs_and_page_geometry_match_virtio_layout() { assert_eq!(core::mem::size_of::(), 16); assert_eq!(core::mem::align_of::(), 8); assert_eq!( split_ring_pages( 0x4000_4000, 0x4000_5000, 0x4000_6000, 256, 0x4000_0000, 0x1000_0000 ), Ok(SplitRingPages { descriptor: 0x4000_4000, available: 0x4000_5000, used: 0x4000_6000, count: 256, available_bytes: 516, used_bytes: 2052, }) ); } #[test] fn split_ring_rejects_overlapping_invalid_or_outside_pages() { let ram = (0x4000_0000, 0x1000_0000); for (d, a, u, n, expected) in [ (0x4000_4000, 0x4000_4000, 0x4000_6000, 256, Error::Overlap), (0x4000_4000, 0x4000_5000, 0x4000_5000, 256, Error::Overlap), ( 0x4000_4000, 0x4000_5001, 0x4000_6000, 256, Error::Misaligned, ), ( 0x4000_4000, 0x5000_0000, 0x4000_6000, 256, Error::OutsideRam, ), ( 0x4000_4000, 0x4000_5000, 0x4000_6000, 3, Error::InvalidCount, ), ( 0x4000_4000, 0x4000_5000, 0x4000_6000, 512, Error::InvalidCount, ), ] { assert_eq!(split_ring_pages(d, a, u, n, ram.0, ram.1), Err(expected)); } } #[test] fn k332_available_ring_fits_owned_page_and_cannot_alias_descriptors() { let ring = split_ring_pages( 0x4000_4000, 0x4000_5000, 0x4000_6000, 256, 0x4000_0000, 0x1000_0000, ) .unwrap(); assert_eq!(ring.available_bytes, 4 + 256 * 2); assert_eq!(ring.available % PAGE_SIZE, 0); assert_eq!( split_ring_pages( ring.descriptor, ring.descriptor, ring.used, 256, 0x4000_0000, 0x1000_0000 ), Err(Error::Overlap) ); } #[test] fn k333_used_ring_fits_owned_page_and_cannot_alias_available() { let ring = split_ring_pages( 0x4000_4000, 0x4000_5000, 0x4000_6000, 256, 0x4000_0000, 0x1000_0000, ) .unwrap(); assert_eq!(ring.used_bytes, 4 + 256 * 8); assert_eq!(ring.used % PAGE_SIZE, 0); assert_eq!( split_ring_pages( ring.descriptor, ring.available, ring.available, 256, 0x4000_0000, 0x1000_0000 ), Err(Error::Overlap) ); } #[test] fn k334_dma_lease_blocks_release_until_queue_reset() { let mut allocator = FrameAllocator::new( crate::memory::FreeMap::from_ram(0x4000_0000, 0x1000_0000).unwrap(), ); let before = allocator.remaining_pages(); let mut lease = DmaLease::allocate(&mut allocator, 0x4000_0000, 0x1000_0000).unwrap(); let frames = lease.frames().unwrap(); assert_eq!(allocator.remaining_pages(), before - 4); assert_eq!(frames, [0x4000_0000, 0x4000_1000, 0x4000_2000, 0x4000_3000]); lease.mark_ready().unwrap(); assert_eq!(lease.release(&mut allocator), Err(Error::Active)); assert_eq!(allocator.remaining_pages(), before - 4); lease.mark_reset().unwrap(); assert_eq!(lease.release(&mut allocator), Ok(())); assert_eq!(allocator.remaining_pages(), before); assert_eq!(lease.release(&mut allocator), Err(Error::Released)); assert_eq!(lease.frames(), Err(Error::Released)); } #[test] fn k398_cancelled_write_keeps_exposed_dma_until_reset_ack_path() { let mut allocator = FrameAllocator::new( crate::memory::FreeMap::from_ram(0x4000_0000, 0x1000_0000).unwrap(), ); let before = allocator.remaining_pages(); let mut lease = DmaLease::allocate(&mut allocator, 0x4000_0000, 0x1000_0000).unwrap(); let frames = lease.frames().unwrap(); lease.mark_ready().unwrap(); assert_eq!(lease.release(&mut allocator), Err(Error::Active)); assert_eq!(lease.frames(), Ok(frames)); assert_eq!(allocator.remaining_pages(), before - 4); lease.mark_reset().unwrap(); assert_eq!(lease.release(&mut allocator), Ok(())); assert_eq!(allocator.remaining_pages(), before); assert_eq!(lease.frames(), Err(Error::Released)); assert_eq!(lease.release(&mut allocator), Err(Error::Released)); } #[test] fn k334_partial_frame_exhaustion_rolls_back_owned_pages() { let mut allocator = FrameAllocator::new( crate::memory::FreeMap::from_ram(0x4000_0000, PAGE_SIZE * 3).unwrap(), ); assert!(matches!( DmaLease::allocate(&mut allocator, 0x4000_0000, PAGE_SIZE * 3), Err(Error::OutOfFrames) )); assert_eq!(allocator.remaining_pages(), 3); } #[test] fn k335_dma_subregions_reject_zero_cross_page_and_address_overflow() { assert_eq!(dma_subregion(0x4000_9000, 0, 16), Ok(0x4000_9000)); assert_eq!(dma_subregion(0x4000_9000, 16, 1), Ok(0x4000_9010)); assert_eq!(dma_subregion(0x4000_9000, 512, 512), Ok(0x4000_9200)); assert_eq!(dma_subregion(0x4000_9000, 0, 0), Err(Error::InvalidBuffer)); assert_eq!( dma_subregion(0x4000_9000, 4095, 2), Err(Error::InvalidBuffer) ); assert_eq!( dma_subregion(0x4000_9000, u64::MAX, 2), Err(Error::Overflow) ); assert_eq!(dma_subregion(0x4000_9001, 0, 16), Err(Error::Misaligned)); assert_eq!( dma_subregion(u64::MAX & !(PAGE_SIZE - 1), 4095, 2), Err(Error::InvalidBuffer) ); assert_eq!( dma_subregion(u64::MAX & !(PAGE_SIZE - 1), 4095, 1), Err(Error::Overflow) ); } }