#![no_std] #![no_main] use core::arch::{asm, global_asm}; use core::mem::{offset_of, size_of}; use core::panic::PanicInfo; use core::ptr::{read_volatile, write_volatile}; use core::sync::atomic::{AtomicUsize, Ordering}; use turkos_bootinfo::{BOOT_INFO_MAP_CAPACITY, BOOT_INFO_PAGE_BYTES}; use turkos_kernel::bootinfo::{self, BootInfoV1}; #[cfg(feature = "irq-service-test")] mod irq_service; #[cfg(feature = "p12-journal-test")] mod journal_probe; #[cfg(feature = "user-three-test")] mod process_probe; #[cfg(feature = "irq-spurious-vector-test")] mod spurious_vector; #[cfg(feature = "user-three-test")] global_asm!(include_str!("process_probe.S")); #[cfg(feature = "heap-test")] use turkos_kernel::heap; use turkos_kernel::serial_tx::SerialLineLock; #[cfg(feature = "smp-topology-test")] use turkos_kernel::smp::{self, StackSpan}; #[cfg(any(feature = "elf-hello-test", feature = "turkfs-init-test"))] mod elf_hello_probe; #[cfg(feature = "elf-ident-test")] mod elf_ident_probe; #[cfg(feature = "elf-layout-test")] mod elf_layout_probe; #[cfg(feature = "elf-load-test")] mod elf_load_probe; #[cfg(feature = "elf-sequence-test")] mod elf_sequence_probe; #[cfg(feature = "elf-stack-test")] mod elf_stack_probe; #[cfg(feature = "ipc-basic-test")] mod ipc_basic_probe; #[cfg(feature = "ipc-multi-test")] mod ipc_multi_probe; #[cfg(feature = "ipc-timeline-test")] mod ipc_timeline_probe; #[cfg(feature = "ipc-wait-test")] mod ipc_wait_probe; #[cfg(feature = "smp-allocator-test")] mod smp_allocator_probe; #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] use turkos_kernel::irq; #[cfg(any( feature = "task-state-test", feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "ipc-wait-test", feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] use turkos_kernel::task; #[cfg(any( feature = "task-state-test", feature = "task-switch-test", feature = "task-lifecycle-test" ))] use turkos_kernel::task::Scheduler; #[cfg(any(feature = "task-preempt-test", feature = "task-sleep-test"))] mod preempt; #[cfg(feature = "serial-arguments-test")] mod serial_arguments_probe; #[cfg(feature = "serial-cancel-test")] mod serial_cancel_probe; #[cfg(feature = "serial-console-test")] mod serial_console_probe; #[cfg(feature = "serial-echo-test")] mod serial_echo_probe; #[cfg(feature = "serial-edit-test")] mod serial_edit_probe; #[cfg(feature = "serial-limit-test")] mod serial_limit_probe; #[cfg(any( feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] mod serial_line_lock_probe; #[cfg(feature = "serial-line-test")] mod serial_line_probe; #[cfg(feature = "serial-range-test")] mod serial_range_probe; #[cfg(feature = "serial-rx-test")] mod serial_rx_probe; #[cfg(feature = "serial-unknown-test")] mod serial_unknown_probe; #[cfg(feature = "serial-utf8-test")] mod serial_utf8_probe; #[cfg(feature = "task-cancel-test")] mod task_cancel_probe; #[cfg(feature = "task-guard-test")] mod task_guard; #[cfg(feature = "turkfs-cat-test")] mod turkfs_cat_probe; #[cfg(feature = "turkfs-ls-test")] mod turkfs_ls_probe; #[cfg(any( feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test" ))] mod user_dual; #[cfg(any( feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test" ))] mod user_first; #[cfg(any(feature = "user-handle-test", feature = "user-handle-transfer-test"))] mod user_handle; #[cfg(feature = "user-handle-death-test")] mod user_handle_death; #[cfg(any( feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test" ))] mod user_syscall; #[cfg(any( feature = "user-task-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test" ))] mod user_task; #[cfg(feature = "user-wait-test")] mod user_wait; #[cfg(feature = "turkfs-init-test")] mod virtio_init_probe; #[cfg(feature = "virtio-magic-test")] mod virtio_magic_probe; #[cfg(feature = "virtio-modern-features-test")] mod virtio_modern_probe; #[cfg(any( feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test" ))] mod virtio_net_arp_reply_probe; #[cfg(feature = "virtio-net-id-test")] mod virtio_net_id_probe; #[cfg(feature = "virtio-net-mac-test")] mod virtio_net_mac_probe; #[cfg(feature = "virtio-net-pool-test")] mod virtio_net_pool_probe; #[cfg(all(feature = "virtio-net-rx-test", not(feature = "virtio-net-pool-test")))] mod virtio_net_rx_probe; #[cfg(feature = "virtio-net-tx-test")] mod virtio_net_tx_probe; #[cfg(all( feature = "virtio-modern-queue-test", not(feature = "turkfs-init-test"), not(feature = "turkfs-ls-test"), not(feature = "turkfs-cat-test"), not(feature = "p12-journal-test") ))] mod virtio_queue_probe; #[cfg(any(feature = "turkfs-ls-test", feature = "turkfs-cat-test", feature = "os-runtime-product"))] mod virtio_ro_turkfs; #[cfg(feature = "os-runtime-product")] #[path = "k1290-cat-disk.rs"] mod k1290_cat_disk; #[cfg(feature = "os-runtime-product")] #[path = "k1295-ls-disk.rs"] mod k1295_ls_disk; #[cfg(feature = "os-runtime-product")] #[path = "k1295-ls-dispatch.rs"] mod k1295_ls_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1300-process-dispatch.rs"] mod k1300_process_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1305-handles-dispatch.rs"] mod k1305_handles_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1310-generation-dispatch.rs"] mod k1310_generation_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1315-rights-dispatch.rs"] mod k1315_rights_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1320-lifetime-dispatch.rs"] mod k1320_lifetime_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1325-queue-dispatch.rs"] mod k1325_queue_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1330-owned-dispatch.rs"] mod k1330_owned_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1335-transfer-dispatch.rs"] mod k1335_transfer_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1340-peer-dispatch.rs"] mod k1340_peer_dispatch; #[cfg(feature = "os-runtime-product")] #[path = "k1345-waitset-model.rs"] mod k1345_waitset_model; #[cfg(feature = "os-runtime-product")] #[path = "k1345-waitset-dispatch.rs"] mod k1345_waitset_dispatch; #[path = "k1350-readiness-model.rs"] mod k1350_readiness_model; #[path = "k1350-readiness-dispatch.rs"] mod k1350_readiness_dispatch; #[path = "k1355-deadlines-model.rs"] mod k1355_deadlines_model; #[path = "k1355-deadlines-dispatch.rs"] mod k1355_deadlines_dispatch; #[path = "k1360-teardown-model.rs"] mod k1360_teardown_model; #[path = "k1360-teardown-dispatch.rs"] mod k1360_teardown_dispatch; #[path = "k1365-shared-model.rs"] mod k1365_shared_model; #[path = "k1365-shared-dispatch.rs"] mod k1365_shared_dispatch; #[path = "k1370-seal-dispatch.rs"] mod k1370_seal_dispatch; #[path = "k1370-seal-model.rs"] mod k1370_seal_model; #[path = "k1375-lifetime-dispatch.rs"] mod k1375_lifetime_dispatch; #[path = "k1375-lifetime-model.rs"] mod k1375_lifetime_model; #[path = "k1380-ring-dispatch.rs"] mod k1380_ring_dispatch; #[path = "k1385-registry-dispatch.rs"] mod k1385_registry_dispatch; #[path = "k1390-protocol-dispatch.rs"] mod k1390_protocol_dispatch; #[path = "k1395-restart-dispatch.rs"] mod k1395_restart_dispatch; #[path = "k1400-discovery-dispatch.rs"] mod k1400_discovery_dispatch; #[path = "k1575-stat-dispatch.rs"] mod k1575_stat_dispatch; #[path = "k1580-dup-dispatch.rs"] mod k1580_dup_dispatch; #[path = "k1585-vfs-traits.rs"] mod k1585_vfs_traits; #[path = "k1590-vfs-tmpfs.rs"] mod k1590_vfs_tmpfs; #[path = "k1595-vfs-ro-cache.rs"] mod k1595_vfs_ro_cache; #[path = "k1600-vfs-unmount.rs"] mod k1600_vfs_unmount; #[path = "k1605-vfs-superblock.rs"] mod k1605_vfs_superblock; #[path = "k1610-vfs-inode-table.rs"] mod k1610_vfs_inode_table; #[path = "k1615-turkfs-reader.rs"] mod k1615_turkfs_reader; #[path = "k1620-turkfs-directory.rs"] mod k1620_turkfs_directory; #[path = "k1625-turkfs-allocation.rs"] mod k1625_turkfs_allocation; #[path = "k1630-turkfs-growth.rs"] mod k1630_turkfs_growth; #[path = "k1635-turkfs-inode.rs"] mod k1635_turkfs_inode; #[path = "k1640-turkfs-truncate.rs"] mod k1640_turkfs_truncate; #[path = "runtime/turkfs_namespace.rs"] mod os_turkfs_namespace; #[path = "k1645-turkfs-create.rs"] mod k1645_turkfs_create; #[path = "k1650-turkfs-rename.rs"] mod k1650_turkfs_rename; #[path = "runtime/turkfs_open_unlink.rs"] mod os_turkfs_open_unlink; #[path = "k1655-turkfs-unlink.rs"] mod k1655_turkfs_unlink; #[path = "k1660-turkfs-links.rs"] mod k1660_turkfs_links; #[path = "runtime/turkfs_journal.rs"] mod os_turkfs_journal; #[path = "k1665-turkfs-journal.rs"] mod k1665_turkfs_journal; #[path = "runtime/turkfs_journal_ordering.rs"] mod os_turkfs_journal_ordering; #[path = "k1670-virtio-journal.rs"] mod k1670_virtio_journal; #[path = "k1670-turkfs-ordering.rs"] mod k1670_turkfs_ordering; #[path = "runtime/turkfs_journal_reservation.rs"] mod os_turkfs_journal_reservation; #[path = "k1675-turkfs-reservation.rs"] mod k1675_turkfs_reservation; #[path = "runtime/checkpoint/checkpoint.rs"] mod os_turkfs_journal_checkpoint; #[path = "k1680-virtio-checkpoint.rs"] mod k1680_virtio_checkpoint; #[path = "k1680-turkfs-checkpoint.rs"] mod k1680_turkfs_checkpoint; #[path = "runtime/replay/replay.rs"] mod os_turkfs_recovery_replay; #[path = "k1685-virtio-replay.rs"] mod k1685_virtio_replay; #[path = "k1685-turkfs-replay.rs"] mod k1685_turkfs_replay; #[path = "k1405-device-registry.rs"] mod k1405_device_registry; #[path = "k1410-device-resources.rs"] mod k1410_device_resources; #[path = "k1415-device-driver-match.rs"] mod k1415_device_driver_match; #[path = "k1420-device-safe-detach.rs"] mod k1420_device_safe_detach; #[path = "runtime/pci_discovery.rs"] mod os_pci_discovery; #[path = "k1425-pci-ecam.rs"] mod k1425_pci_ecam; #[path = "runtime/pci_headers.rs"] mod os_pci_headers; #[path = "k1430-pci-headers.rs"] mod k1430_pci_headers; #[path = "runtime/pci_capabilities.rs"] mod os_pci_capabilities; #[path = "k1435-pci-capabilities.rs"] mod k1435_pci_capabilities; #[path = "runtime/pci_bridges.rs"] mod os_pci_bridges; #[path = "k1440-pci-bridges.rs"] mod k1440_pci_bridges; #[path = "runtime/pci_resources.rs"] mod os_pci_resources; #[path = "k1445-pci-bar-size.rs"] mod k1445_pci_bar_size; #[path = "runtime/pci_device_map.rs"] mod os_pci_device_map; #[path = "k1450-pci-device-map.rs"] mod k1450_pci_device_map; #[path = "runtime/pci_interrupts.rs"] mod os_pci_interrupts; #[path = "k1455-pci-intx.rs"] mod k1455_pci_intx; #[path = "runtime/pci_msix.rs"] mod os_pci_msix; #[path = "k1460-pci-msix.rs"] mod k1460_pci_msix; #[path = "k1470-virtio-pci-common.rs"] mod k1470_virtio_pci_common; #[path = "runtime/virtio_pci_common.rs"] mod os_virtio_pci_common; #[path = "runtime/virtio_features.rs"] mod os_virtio_features; #[path = "k1475-virtio-features.rs"] mod k1475_virtio_features; #[path = "runtime/virtio_status.rs"] mod os_virtio_status; #[path = "k1480-virtio-status.rs"] mod k1480_virtio_status; #[path = "runtime/virtio_descriptors.rs"] mod os_virtio_descriptors; #[path = "k1485-virtio-descriptors.rs"] mod k1485_virtio_descriptors; #[path = "runtime/virtio_indices.rs"] mod os_virtio_indices; #[path = "k1490-virtio-indices.rs"] mod k1490_virtio_indices; #[path = "runtime/virtio_completions.rs"] mod os_virtio_completions; #[path = "k1495-virtio-completions.rs"] mod k1495_virtio_completions; #[path = "runtime/virtio_backpressure.rs"] mod os_virtio_backpressure; #[path = "k1500-virtio-backpressure.rs"] mod k1500_virtio_backpressure; #[path = "runtime/block_geometry.rs"] mod os_block_geometry; #[path = "k1505-block-geometry.rs"] mod k1505_block_geometry; #[path = "runtime/block_io.rs"] mod os_block_io; #[path = "k1510-block-requests.rs"] mod k1510_block_requests; #[path = "k1515-block-timeouts.rs"] mod k1515_block_timeouts; #[path = "k1520-block-errors.rs"] mod k1520_block_errors; #[path = "runtime/partitions.rs"] mod os_partitions; #[path = "k1525-partitions-mbr.rs"] mod k1525_partitions_mbr; #[path = "runtime/partitions_gpt.rs"] mod os_partitions_gpt; #[path = "k1530-partitions-gpt.rs"] mod k1530_partitions_gpt; #[path = "runtime/partitions_view.rs"] mod os_partitions_view; #[path = "k1535-partitions-view.rs"] mod k1535_partitions_view; #[path = "k1540-partitions-recovery.rs"] mod k1540_partitions_recovery; #[path = "runtime/vfs_paths.rs"] mod os_vfs_paths; #[path = "k1545-vfs-paths.rs"] mod k1545_vfs_paths; #[path = "k1550-vfs-mount-cross.rs"] mod k1550_vfs_mount_cross; #[path = "runtime/vfs_lookup.rs"] mod os_vfs_lookup; #[path = "k1555-vfs-lookup.rs"] mod k1555_vfs_lookup; #[path = "runtime/vfs_symlink.rs"] mod os_vfs_symlink; #[path = "k1560-vfs-symlink.rs"] mod k1560_vfs_symlink; #[path = "runtime/vfs_open.rs"] mod os_vfs_open; #[path = "k1565-vfs-ro-open.rs"] mod k1565_vfs_ro_open; #[path = "runtime/vfs_offset.rs"] mod os_vfs_offset; #[path = "k1570-vfs-offset.rs"] mod k1570_vfs_offset; #[path = "runtime/vfs_stat.rs"] mod os_vfs_stat; #[path = "runtime/vfs_dup.rs"] mod os_vfs_dup; #[path = "runtime/vfs_traits.rs"] mod os_vfs_traits; #[path = "runtime/vfs_tmpfs.rs"] mod os_vfs_tmpfs; #[path = "runtime/vfs_ro_cache.rs"] mod os_vfs_ro_cache; #[path = "runtime/vfs_unmount.rs"] mod os_vfs_unmount; #[path = "runtime/vfs_superblock.rs"] mod os_vfs_superblock; #[path = "runtime/vfs_inode_table.rs"] mod os_vfs_inode_table; #[path = "runtime/turkfs_reader.rs"] mod os_turkfs_reader; #[path = "runtime/turkfs_directory.rs"] mod os_turkfs_directory; #[path = "runtime/turkfs_allocation.rs"] mod os_turkfs_allocation; #[path = "runtime/virtio_mmio_v2.rs"] mod os_virtio_mmio_v2; #[path = "k1465-virtio-mmio-v2.rs"] mod k1465_virtio_mmio_v2; static mut OS_K1420_NET_PROOFS: [u64; 2] = [0; 2]; #[path = "k1380-guest-ring.rs"] mod k1380_guest_ring; #[cfg(feature = "task-state-test")] fn task_state_smoke_test() -> bool { let mut scheduler = Scheduler::new(); if scheduler.run_next() != Ok(None) { return false; } write_line("TURKOS/TASK-IDLE: PASS"); let (Ok(a), Ok(b)) = (scheduler.spawn(), scheduler.spawn()) else { return false; }; if a == b || a.generation == b.generation || scheduler.state(a) != Ok(task::State::Ready) || scheduler.state(b) != Ok(task::State::Ready) { return false; } write_str("TURKOS/TASK-IDS: A-SLOT=0x"); write_hex(a.slot as usize); write_str(" A-GEN=0x"); write_hex(a.generation as usize); write_str(" B-SLOT=0x"); write_hex(b.slot as usize); write_str(" B-GEN=0x"); write_hex(b.generation as usize); write_str("\r\n"); write_line("TURKOS/TASK-STATE: PASS"); if scheduler.run_next() != Ok(Some(a)) || scheduler.yield_current(false) != Ok(Some(b)) || scheduler.yield_current(false) != Ok(Some(a)) || scheduler.ready_len() != 1 { return false; } write_line("TURKOS/TASK-QUEUE: PASS"); if scheduler.yield_current(true) != Err(task::Error::InInterrupt) || scheduler.running() != Some(a) { return false; } write_line("TURKOS/TASK-IRQ-YIELD: REJECTED"); if scheduler.sleep_current(10, 5) != Ok(Some(b)) || scheduler.state(a) != Ok(task::State::Sleeping { until: 10 }) || scheduler.wake_due(9) != Ok(0) { return false; } write_line("TURKOS/TASK-SLEEP-BEFORE: PASS"); if scheduler.wake_due(10) != Ok(1) || scheduler.wake_due(10) != Ok(0) || scheduler.yield_current(false) != Ok(Some(a)) { return false; } write_line("TURKOS/TASK-WAKE-AT: PASS"); if scheduler.exit_current(42) != Ok(Some(b)) || scheduler.state(a) != Ok(task::State::Exited { code: 42 }) || scheduler.reap(a) != Ok(42) { return false; } write_line("TURKOS/TASK-EXIT-CODE: 0x000000000000002a"); let Ok(reused) = scheduler.spawn() else { return false; }; if reused.slot != a.slot || reused.generation <= a.generation || scheduler.state(a) != Err(task::Error::InvalidTask) { return false; } write_str("TURKOS/TASK-REUSE: OLD-GEN=0x"); write_hex(a.generation as usize); write_str(" NEW-GEN=0x"); write_hex(reused.generation as usize); write_str("\r\n"); if scheduler.spawn().is_err() || scheduler.spawn().is_err() || scheduler.spawn() != Err(task::Error::Capacity) { return false; } write_line("TURKOS/TASK-CAPACITY: REJECTED"); if Scheduler::deadline(10, 4) != Ok(14) || Scheduler::deadline(u64::MAX, 1) != Err(task::Error::InvalidDeadline) || Scheduler::deadline(10, 0) != Err(task::Error::InvalidDeadline) { return false; } write_line("TURKOS/TASK-DEADLINE-OVERFLOW: REJECTED"); write_line("TURKOS/TASK-MODEL-ONLY: NO-CONTEXT-SWITCH"); true } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_switch_error(reason: &str) -> ! { write_str("TURKOS/TASK-SWITCH-ERROR: "); write_line(reason); park() } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_context_ptr(slot: usize) -> *mut task::CpuContext { // SAFETY: the only supported slots are A=0 and B=1, both owned by the // single masked-IRQ task-switch experiment. unsafe { core::ptr::addr_of_mut!(TASK_CONTEXTS) .cast::() .add(slot) } } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_step(name: &str, counter: *mut usize) { let mut sp: usize; unsafe { asm!("mov {sp}, sp", sp = out(reg) sp, options(nomem, nostack, preserves_flags)) }; // SAFETY: this test is single-CPU, IRQ-masked, and only the named task // writes its counter; no exclusive/atomic RMW is needed on QEMU/HVF. let index = unsafe { read_volatile(counter) + 1 }; unsafe { write_volatile(counter, index) }; write_str("TURKOS/TASK-STEP: ID="); write_str(name); write_str(" INDEX=0x"); write_hex(index); write_str(" SP=0x"); write_hex(sp); write_str("\r\n"); } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] #[inline(never)] fn task_yield() { // SAFETY: only one EL1 CPU runs this isolated image with IRQ masked. The // scheduler borrow ends before the assembly switches to another stack. let (old, new) = unsafe { let scheduler = &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER); let old = scheduler .running() .unwrap_or_else(|| task_switch_error("no running task")); let new = scheduler .yield_current(false) .unwrap_or_else(|_| task_switch_error("yield rejected")) .unwrap_or_else(|| task_switch_error("no ready task")); (old.slot as usize, new.slot as usize) }; if old == new { return; } // SAFETY: each context belongs to a distinct task stack; assembly saves // AAPCS64 callee registers, SIMD q8..q15 and SP before loading the next. unsafe { __turkos_swap_task_context(task_context_ptr(old), task_context_ptr(new)) }; } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_exit(code: i32) -> ! { let (old, next) = unsafe { let scheduler = &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER); let old = scheduler .running() .unwrap_or_else(|| task_switch_error("exit without task")); let next = scheduler .exit_current(code) .unwrap_or_else(|_| task_switch_error("exit rejected")); (old.slot as usize, next.map(|id| id.slot as usize)) }; let target = match next { Some(slot) => task_context_ptr(slot).cast_const(), None => core::ptr::addr_of!(TASK_BOOT_CONTEXT), }; unsafe { __turkos_swap_task_context(task_context_ptr(old), target) }; task_switch_error("exited task resumed") } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] extern "C" fn task_entry_a() -> ! { for _ in 0..4 { task_step("A", core::ptr::addr_of_mut!(TASK_A_COUNT)); task_yield(); } task_exit(0x0a) } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] extern "C" fn task_entry_b() -> ! { for _ in 0..4 { task_step("B", core::ptr::addr_of_mut!(TASK_B_COUNT)); if unsafe { read_volatile(core::ptr::addr_of!(TASK_B_COUNT)) } == 4 { write_line("TURKOS/TASK-WAIT: WAITER=B TARGET=A"); } task_yield(); } // The fourth yield leaves B ready while A exits. B then resumes on its // own stack and reads the retained exit state through A's original ID. let a = unsafe { *core::ptr::addr_of!(TASK_A_ID) } .unwrap_or_else(|| task_switch_error("missing A ID")); if unsafe { (&*core::ptr::addr_of!(TASK_SCHEDULER)).state(a) } != Ok(task::State::Exited { code: 0x0a }) { task_switch_error("waiter could not read A exit"); } write_line("TURKOS/TASK-WAIT-EXIT: WAITER=B TARGET=A CODE=0x000000000000000a"); task_exit(0x0b) } #[cfg(feature = "task-lifecycle-test")] extern "C" fn task_entry_c() -> ! { let mut sp: usize; unsafe { asm!("mov {value}, sp", value = out(reg) sp, options(nomem, nostack, preserves_flags)) }; let a_start = unsafe { core::ptr::addr_of!(TASK_A_STACK.0).cast::() as usize }; let current = unsafe { (&*core::ptr::addr_of!(TASK_SCHEDULER)).running() } .unwrap_or_else(|| task_switch_error("C has no running ID")); if current.slot != 0 || unsafe { (&*core::ptr::addr_of!(TASK_A_LEASE)).owner() } != Some(current) || sp < a_start + 16 || sp >= a_start + 16 * 1024 || sp & 15 != 0 || unsafe { read_volatile(a_start as *const u64) } != TASK_C_CANARY { task_switch_error("C stack ownership or bounds"); } unsafe { write_volatile(core::ptr::addr_of_mut!(TASK_C_COUNT), 1) }; write_str("TURKOS/TASK-LIFECYCLE-C: GEN=0x"); write_hex(current.generation as usize); write_str(" SP=0x"); write_hex(sp); write_str(" COUNT=0x"); write_hex(1); write_str("\r\n"); task_exit(0x0c) } #[cfg(feature = "task-lifecycle-test")] fn task_lifecycle_after_ab( a: task::TaskId, b: task::TaskId, a_start: usize, a_top: usize, b_start: usize, ) { let c = { let scheduler = unsafe { &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER) }; if scheduler.running().is_some() || scheduler.ready_len() != 0 || scheduler.run_next() != Ok(None) || scheduler.state(a) != Ok(task::State::Exited { code: 0x0a }) || scheduler.state(b) != Ok(task::State::Exited { code: 0x0b }) || unsafe { (&*core::ptr::addr_of!(TASK_A_LEASE)).owner() } != Some(a) || unsafe { (&*core::ptr::addr_of!(TASK_B_LEASE)).owner() } != Some(b) { task_switch_error("exited task was ready or stack owner changed"); } write_str("TURKOS/TASK-LIFECYCLE-RETIRED: A-GEN=0x"); write_hex(a.generation as usize); write_str(" B-GEN=0x"); write_hex(b.generation as usize); write_str(" READY=0x"); write_hex(scheduler.ready_len()); write_str("\r\n"); if scheduler.reap(a) != Ok(0x0a) || scheduler.reap(b) != Ok(0x0b) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).release(a) } != Ok(()) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_B_LEASE)).release(b) } != Ok(()) || scheduler.state(a) != Err(task::Error::InvalidTask) || scheduler.state(b) != Err(task::Error::InvalidTask) { task_switch_error("reap or lease release"); } unsafe { task_context_ptr(0).write(task::CpuContext::EMPTY); task_context_ptr(1).write(task::CpuContext::EMPTY); core::ptr::write_bytes(a_start as *mut u8, 0xdd, 16 * 1024); core::ptr::write_bytes(b_start as *mut u8, 0xdd, 16 * 1024); write_volatile(a_start as *mut u64, TASK_C_CANARY); write_volatile(b_start as *mut u64, TASK_B_POISON); } if unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).claim(a) } != Err(task::Error::InvalidTask) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_B_LEASE)).claim(b) } != Err(task::Error::InvalidTask) { task_switch_error("old generation reclaimed freed stack"); } write_str("TURKOS/TASK-LIFECYCLE-RELEASED: A=0x"); write_hex(a_start); write_str(" B=0x"); write_hex(b_start); write_str(" B-POISON=0x"); write_hex(unsafe { read_volatile(b_start as *const u64) } as usize); write_str("\r\n"); let c = scheduler .spawn() .unwrap_or_else(|_| task_switch_error("spawn C after reap")); if c.slot != a.slot || c.generation <= b.generation || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).claim(c) } != Ok(()) || unsafe { (&*core::ptr::addr_of!(TASK_B_LEASE)).owner() }.is_some() { task_switch_error("new generation stack claim"); } let c_entry = task_entry_c as *const () as usize; let context = task::CpuContext::initial(a_top as u64, c_entry as u64) .unwrap_or_else(|_| task_switch_error("C first context")); unsafe { task_context_ptr(0).write(context) }; write_str("TURKOS/TASK-LIFECYCLE-REUSE: OLD-GEN=0x"); write_hex(a.generation as usize); write_str(" NEW-GEN=0x"); write_hex(c.generation as usize); write_str(" STACK=0x"); write_hex(a_start); write_str("\r\n"); if scheduler.run_next() != Ok(Some(c)) { task_switch_error("C first run"); } c }; unsafe { __turkos_swap_task_context( core::ptr::addr_of_mut!(TASK_BOOT_CONTEXT), task_context_ptr(0), ); } let scheduler = unsafe { &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER) }; if unsafe { read_volatile(core::ptr::addr_of!(TASK_C_COUNT)) } != 1 || scheduler.running().is_some() || scheduler.run_next() != Ok(None) || scheduler.state(c) != Ok(task::State::Exited { code: 0x0c }) || scheduler.state(a) != Err(task::Error::InvalidTask) || scheduler.state(b) != Err(task::Error::InvalidTask) || unsafe { (&*core::ptr::addr_of!(TASK_A_LEASE)).owner() } != Some(c) || unsafe { (&*core::ptr::addr_of!(TASK_B_LEASE)).owner() }.is_some() || unsafe { read_volatile(a_start as *const u64) } != TASK_C_CANARY || unsafe { read_volatile(b_start as *const u64) } != TASK_B_POISON { task_switch_error("C exit or retired stack changed"); } write_str( "TURKOS/TASK-LIFECYCLE-EXIT: C=0x000000000000000c OLD-A=REJECTED OLD-B=REJECTED C-GUARD=0x", ); write_hex(unsafe { read_volatile(a_start as *const u64) } as usize); write_str(" B-POISON=0x"); write_hex(unsafe { read_volatile(b_start as *const u64) } as usize); write_str("\r\n"); if scheduler.reap(c) != Ok(0x0c) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).release(c) } != Ok(()) || scheduler.state(c) != Err(task::Error::InvalidTask) { task_switch_error("C final reap"); } unsafe { task_context_ptr(0).write(task::CpuContext::EMPTY) }; write_line("TURKOS/TASK-LIFECYCLE: PASS"); } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] fn task_switch_smoke_test() -> ! { // SAFETY: the two dedicated, page-aligned BSS stacks are exclusive to // this image. The low canaries remain outside every downward stack frame. let a_start = unsafe { core::ptr::addr_of_mut!(TASK_A_STACK.0).cast::() as usize }; let b_start = unsafe { core::ptr::addr_of_mut!(TASK_B_STACK.0).cast::() as usize }; let a_top = a_start + 16 * 1024; let b_top = b_start + 16 * 1024; if a_start & 4095 != 0 || b_start & 4095 != 0 || (a_start < b_top && b_start < a_top) { task_switch_error("stack bounds"); } unsafe { write_volatile(a_start as *mut u64, TASK_A_CANARY); write_volatile(b_start as *mut u64, TASK_B_CANARY); } for (name, start, top) in [("A", a_start, a_top), ("B", b_start, b_top)] { write_str("TURKOS/TASK-STACK: ID="); write_str(name); write_str(" START=0x"); write_hex(start); write_str(" TOP=0x"); write_hex(top); write_str("\r\n"); } let (a, b) = unsafe { let scheduler = &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER); let a = scheduler .spawn() .unwrap_or_else(|_| task_switch_error("spawn A")); let b = scheduler .spawn() .unwrap_or_else(|_| task_switch_error("spawn B")); (a, b) }; if a.slot != 0 || b.slot != 1 { task_switch_error("task ID order"); } #[cfg(feature = "task-lifecycle-test")] if unsafe { (&mut *core::ptr::addr_of_mut!(TASK_A_LEASE)).claim(a) } != Ok(()) || unsafe { (&mut *core::ptr::addr_of_mut!(TASK_B_LEASE)).claim(b) } != Ok(()) { task_switch_error("initial stack leases"); } unsafe { core::ptr::addr_of_mut!(TASK_A_ID).write(Some(a)) }; let a_entry = task_entry_a as *const () as usize; let b_entry = task_entry_b as *const () as usize; let a_context = task::CpuContext::initial(a_top as u64, a_entry as u64) .unwrap_or_else(|_| task_switch_error("first frame A")); let b_context = task::CpuContext::initial(b_top as u64, b_entry as u64) .unwrap_or_else(|_| task_switch_error("first frame B")); unsafe { task_context_ptr(0).write(a_context); task_context_ptr(1).write(b_context); } write_str("TURKOS/TASK-ENTRY: A=0x"); write_hex(a_entry); write_str(" B=0x"); write_hex(b_entry); write_str("\r\n"); let first = unsafe { (&mut *core::ptr::addr_of_mut!(TASK_SCHEDULER)).run_next() }; if first != Ok(Some(a)) { task_switch_error("first run"); } unsafe { __turkos_swap_task_context( core::ptr::addr_of_mut!(TASK_BOOT_CONTEXT), task_context_ptr(0), ) }; // Both task exits must have returned here without ever running an exited // context or changing either low-end stack canary. let scheduler = unsafe { &mut *core::ptr::addr_of_mut!(TASK_SCHEDULER) }; if unsafe { read_volatile(core::ptr::addr_of!(TASK_A_COUNT)) } != 4 || unsafe { read_volatile(core::ptr::addr_of!(TASK_B_COUNT)) } != 4 || scheduler.running().is_some() || scheduler.run_next() != Ok(None) || scheduler.state(a) != Ok(task::State::Exited { code: 0x0a }) || scheduler.state(b) != Ok(task::State::Exited { code: 0x0b }) || unsafe { read_volatile(a_start as *const u64) } != TASK_A_CANARY || unsafe { read_volatile(b_start as *const u64) } != TASK_B_CANARY { task_switch_error("final counters/state/guards"); } write_str("TURKOS/TASK-STACK-GUARDS: A=0x"); write_hex(TASK_A_CANARY as usize); write_str(" B=0x"); write_hex(TASK_B_CANARY as usize); write_str("\r\n"); write_line("TURKOS/TASK-EXIT: A=0x000000000000000a B=0x000000000000000b"); write_line("TURKOS/TASK-COUNTS: A=0x0000000000000004 B=0x0000000000000004"); #[cfg(feature = "task-lifecycle-test")] task_lifecycle_after_ab(a, b, a_start, a_top, b_start); write_line("TURKOS/TASK-SWITCH: PASS"); write_line("TURKOS/TASK-IDLE: EMPTY-QUEUE-WFE"); park() } #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] use turkos_kernel::mmu; use turkos_kernel::{ dtb, exception::{self, ExceptionFrame}, memory, }; global_asm!(include_str!("boot.S")); #[cfg(feature = "smp-boot-protocol-test")] global_asm!(include_str!("smp_boot.S"), vector_probe = const cfg!(feature = "smp-vector-test") as usize, timer_probe = const cfg!(feature = "smp-timer-test") as usize); #[cfg(feature = "smp-allocator-test")] global_asm!(include_str!("smp_allocator_boot.S")); #[cfg(feature = "irq-gp-test")] global_asm!(include_str!("gp_probe.S")); #[cfg(feature = "mmu-tlb-test")] global_asm!(include_str!("tlb_probe.S")); #[cfg(any( feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "ipc-wait-test", feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] global_asm!( include_str!("task_switch.S"), sp_offset = const offset_of!(task::CpuContext, sp), callee_offset = const offset_of!(task::CpuContext, callee), vector_offset = const offset_of!(task::CpuContext, vector), ); global_asm!( include_str!("vectors.S"), frame_size = const size_of::(), x3_offset = const offset_of!(ExceptionFrame, registers) + 3 * size_of::(), sp_offset = const offset_of!(ExceptionFrame, interrupted_sp), canary_offset = const offset_of!(ExceptionFrame, canary), canary_value = const exception::FRAME_CANARY, return_test = const cfg!(feature = "irq-return-test") as usize, guard_test = const cfg!(feature = "task-guard-test") as usize, user_test = const (cfg!(feature = "user-three-test") || cfg!(feature = "user-first-test") || cfg!(feature = "user-code-write-test") || cfg!(feature = "user-data-exec-test") || cfg!(feature = "user-stack-overflow-test") || cfg!(feature = "user-align-test") || cfg!(feature = "user-dual-test") || cfg!(feature = "user-dual-fault-test") || cfg!(feature = "user-dual-stack-fault-test") || cfg!(feature = "user-peer-read-test") || cfg!(feature = "user-mmio-read-test") || cfg!(feature = "user-asid-test") || cfg!(feature = "elf-load-test") || cfg!(feature = "elf-stack-test")) as usize, user_syscall = const (cfg!(feature = "user-syscall-test") || cfg!(feature = "user-copy-test") || cfg!(feature = "ipc-user-test") || cfg!(feature = "user-task-test") || cfg!(feature = "user-clock-test") || cfg!(feature = "user-handle-test") || cfg!(feature = "user-handle-transfer-test") || cfg!(feature = "user-handle-death-test") || cfg!(feature = "user-wait-test") || cfg!(feature = "user-mmio-request-test") || cfg!(feature = "elf-hello-test") || cfg!(feature = "elf-sequence-test") || cfg!(feature = "turkfs-init-test")) as usize, el0_irq = const cfg!(feature = "user-el0-irq-test") as usize, sgi_test = const (true || cfg!(feature = "irq-sgi-test") || cfg!(feature = "irq-timer-test") || cfg!(feature = "irq-unknown-test") || cfg!(feature = "task-preempt-test") || cfg!(feature = "task-sleep-test") || cfg!(feature = "user-copy-irq-test")) as usize, ); #[cfg(feature = "task-guard-test")] global_asm!(include_str!("task_guard.S")); #[cfg(any( feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test", feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test", feature = "elf-load-test", feature = "elf-stack-test" ))] global_asm!(include_str!("user_first.S"), write_fault = const cfg!(feature = "user-code-write-test") as usize, data_exec = const cfg!(feature = "user-data-exec-test") as usize, stack_overflow = const cfg!(feature = "user-stack-overflow-test") as usize, unaligned = const cfg!(feature = "user-align-test") as usize, dual_space = const (cfg!(feature = "user-dual-test") || cfg!(feature = "user-dual-fault-test") || cfg!(feature = "user-dual-stack-fault-test") || cfg!(feature = "user-peer-read-test") || cfg!(feature = "user-mmio-read-test") || cfg!(feature = "user-asid-test")) as usize, dual_fault = const (cfg!(feature = "user-dual-fault-test") || cfg!(feature = "user-peer-read-test") || cfg!(feature = "user-mmio-read-test")) as usize, dual_stack_fault = const cfg!(feature = "user-dual-stack-fault-test") as usize, elf_load = const cfg!(feature = "elf-load-test") as usize, elf_stack = const cfg!(feature = "elf-stack-test") as usize); #[cfg(any( feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] global_asm!(include_str!("elf_hello.S")); #[cfg(any( feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test" ))] global_asm!(include_str!("user_syscall.S"), copy_test = const cfg!(feature = "user-copy-test") as usize, clock_test = const cfg!(feature = "user-clock-test") as usize, mmio_test = const cfg!(feature = "user-mmio-request-test") as usize, ipc_test = const cfg!(feature = "ipc-user-test") as usize); #[cfg(feature = "user-el0-irq-test")] global_asm!(include_str!("user_el0_irq.S")); #[cfg(feature = "user-task-test")] global_asm!(include_str!("user_task.S")); #[cfg(feature = "user-handle-test")] global_asm!(include_str!("user_handle.S")); #[cfg(feature = "user-handle-transfer-test")] global_asm!(include_str!("user_handle_transfer.S")); #[cfg(feature = "user-handle-death-test")] global_asm!(include_str!("user_handle_death.S")); #[cfg(feature = "user-wait-test")] global_asm!(include_str!("user_wait.S")); #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] global_asm!( include_str!("return.S"), ctx_size = const size_of::(), sp_offset = const offset_of!(irq::IrqContext, interrupted_sp), sp_el0_offset = const offset_of!(irq::IrqContext, sp_el0), elr_offset = const offset_of!(irq::IrqContext, elr), spsr_offset = const offset_of!(irq::IrqContext, spsr), q_offset = const offset_of!(irq::IrqContext, q), fpcr_offset = const offset_of!(irq::IrqContext, fpcr), fpsr_offset = const offset_of!(irq::IrqContext, fpsr), slot_offset = const offset_of!(irq::IrqContext, vector_slot), nested_irq = const cfg!(feature = "user-copy-irq-test") as usize, ); const _: () = { let tests = cfg!(feature = "p12-journal-test") as u8 + cfg!(feature = "panic-test") as u8 + cfg!(feature = "exception-test") as u8 + cfg!(feature = "fault-test") as u8 + cfg!(feature = "frame-stress-test") as u8 + cfg!(feature = "mmu-test") as u8 + cfg!(feature = "mmu-no-map-test") as u8 + cfg!(feature = "mmu-write-test") as u8 + cfg!(feature = "mmu-nx-test") as u8 + cfg!(feature = "mmu-rodata-test") as u8 + cfg!(feature = "mmu-rodata-exec-test") as u8 + cfg!(feature = "mmu-stack-exec-test") as u8 + cfg!(feature = "mmu-mmio-exec-test") as u8 + cfg!(feature = "mmu-panic-test") as u8 + cfg!(feature = "mmu-brk-test") as u8 + cfg!(feature = "mmu-udf-test") as u8 + cfg!(feature = "mmu-guard-test") as u8 + cfg!(feature = "mmu-normal-test") as u8 + cfg!(feature = "mmu-tlb-test") as u8 + cfg!(feature = "el0-fault-test") as u8 + cfg!(feature = "nested-fault-test") as u8 + cfg!(feature = "irq-return-test") as u8 + (cfg!(feature = "irq-sgi-test") && !cfg!(feature = "smp-irq-lock-test")) as u8 + cfg!(feature = "irq-timer-test") as u8 + cfg!(feature = "irq-unknown-test") as u8 + cfg!(feature = "heap-test") as u8 + cfg!(feature = "task-state-test") as u8 + cfg!(feature = "task-switch-test") as u8 + cfg!(feature = "task-lifecycle-test") as u8 + cfg!(feature = "task-guard-test") as u8 + cfg!(feature = "user-first-test") as u8 + (cfg!(feature = "user-three-test") && !cfg!(feature = "process-exit-irq-test")) as u8 + cfg!(feature = "user-code-write-test") as u8 + cfg!(feature = "user-data-exec-test") as u8 + cfg!(feature = "user-stack-overflow-test") as u8 + cfg!(feature = "user-align-test") as u8 + cfg!(feature = "user-dual-test") as u8 + cfg!(feature = "user-dual-fault-test") as u8 + cfg!(feature = "user-dual-stack-fault-test") as u8 + cfg!(feature = "user-peer-read-test") as u8 + cfg!(feature = "user-mmio-read-test") as u8 + cfg!(feature = "user-asid-test") as u8 + cfg!(feature = "user-syscall-test") as u8 + cfg!(feature = "user-copy-test") as u8 + cfg!(feature = "ipc-user-test") as u8 + cfg!(feature = "user-copy-irq-test") as u8 + cfg!(feature = "elf-ident-test") as u8 + cfg!(feature = "elf-layout-test") as u8 + cfg!(feature = "elf-load-test") as u8 + cfg!(feature = "elf-stack-test") as u8 + cfg!(feature = "elf-hello-test") as u8 + cfg!(feature = "turkfs-init-test") as u8 + cfg!(feature = "serial-rx-test") as u8 + cfg!(feature = "serial-line-test") as u8 + cfg!(feature = "serial-edit-test") as u8 + cfg!(feature = "serial-limit-test") as u8 + cfg!(feature = "serial-console-test") as u8 + cfg!(feature = "serial-unknown-test") as u8 + cfg!(feature = "serial-utf8-test") as u8 + cfg!(feature = "serial-arguments-test") as u8 + cfg!(feature = "serial-range-test") as u8 + cfg!(feature = "serial-cancel-test") as u8 + cfg!(feature = "serial-line-lock-test") as u8 + cfg!(feature = "serial-line-unlocked-test") as u8 + cfg!(feature = "virtio-net-id-test") as u8 + cfg!(feature = "virtio-net-mac-test") as u8 + (cfg!(feature = "virtio-net-rx-test") && !cfg!(feature = "virtio-net-rx-completion-test") && !cfg!(feature = "virtio-net-pool-test")) as u8 + (cfg!(feature = "virtio-net-rx-completion-test") && !cfg!(feature = "virtio-net-broadcast-test") && !cfg!(feature = "virtio-net-unicast-test")) as u8 + (cfg!(feature = "virtio-net-broadcast-test") && !cfg!(feature = "virtio-net-arp-request-test")) as u8 + cfg!(feature = "virtio-net-arp-request-test") as u8 + cfg!(feature = "virtio-net-arp-reply-test") as u8 + cfg!(feature = "virtio-net-icmp-echo-test") as u8 + cfg!(feature = "virtio-net-reset-traffic-test") as u8 + (cfg!(feature = "virtio-net-unicast-test") && !cfg!(feature = "virtio-net-ipv4-header-test")) as u8 + (cfg!(feature = "virtio-net-ipv4-header-test") && !cfg!(feature = "virtio-net-ipv4-checksum-test")) as u8 + cfg!(feature = "virtio-net-ipv4-checksum-test") as u8 + cfg!(feature = "virtio-net-pool-test") as u8 + (cfg!(feature = "virtio-net-tx-test") && !cfg!(feature = "virtio-net-size-test") && !cfg!(feature = "virtio-net-tx-completion-test")) as u8 + cfg!(feature = "virtio-net-tx-completion-test") as u8 + (cfg!(feature = "virtio-net-size-test") && !cfg!(feature = "virtio-net-header-test")) as u8 + (cfg!(feature = "virtio-net-header-test") && !cfg!(feature = "virtio-net-type-test")) as u8 + cfg!(feature = "virtio-net-type-test") as u8 + cfg!(feature = "serial-echo-test") as u8 + cfg!(feature = "turkfs-ls-test") as u8 + cfg!(feature = "turkfs-cat-test") as u8 + cfg!(feature = "elf-sequence-test") as u8 + cfg!(feature = "ipc-basic-test") as u8 + cfg!(feature = "ipc-wait-test") as u8 + cfg!(feature = "task-cancel-test") as u8 + cfg!(feature = "ipc-multi-test") as u8 + cfg!(feature = "ipc-timeline-test") as u8 + cfg!(feature = "user-clock-test") as u8 + cfg!(feature = "user-mmio-request-test") as u8 + cfg!(feature = "user-handle-test") as u8 + cfg!(feature = "user-handle-transfer-test") as u8 + cfg!(feature = "user-handle-death-test") as u8 + cfg!(feature = "user-wait-test") as u8 + cfg!(feature = "user-task-test") as u8 + cfg!(feature = "task-preempt-test") as u8 + cfg!(feature = "task-sleep-test") as u8 + (cfg!(feature = "smp-boot-protocol-test") && !cfg!(feature = "smp-topology-test")) as u8 + (cfg!(feature = "smp-topology-test") && !cfg!(feature = "smp-vector-test")) as u8 + (cfg!(feature = "smp-vector-test") && !cfg!(feature = "smp-timer-test")) as u8 + cfg!(feature = "smp-timer-test") as u8 + (cfg!(feature = "smp-allocator-test") && !cfg!(feature = "smp-ready-test")) as u8 + (cfg!(feature = "smp-ready-test") && !cfg!(feature = "smp-serial-test")) as u8 + (cfg!(feature = "smp-serial-test") && !cfg!(feature = "smp-owner-test")) as u8 + (cfg!(feature = "smp-owner-test") && !cfg!(feature = "smp-irq-lock-test")) as u8 + (cfg!(feature = "smp-irq-lock-test") && !cfg!(feature = "smp-order-test")) as u8 + (cfg!(feature = "smp-order-test") && !cfg!(feature = "smp-fair-test")) as u8 + (cfg!(feature = "smp-fair-test") && !cfg!(feature = "smp-wait-test")) as u8 + (cfg!(feature = "smp-wait-test") && !cfg!(feature = "smp-time-test")) as u8 + (cfg!(feature = "smp-time-test") && !cfg!(feature = "smp-stats-test")) as u8 + (cfg!(feature = "smp-stats-test") && !cfg!(feature = "smp-wait-class-test")) as u8 + (cfg!(feature = "smp-wait-class-test") && !cfg!(feature = "smp-hold-test")) as u8 + (cfg!(feature = "smp-hold-test") && !cfg!(feature = "smp-counter-baseline-test")) as u8 + (cfg!(feature = "smp-counter-baseline-test") && !cfg!(feature = "smp-classified-admission-test")) as u8 + (cfg!(feature = "smp-classified-admission-test") && !cfg!(feature = "smp-race-trace-test")) as u8 + cfg!(feature = "smp-race-trace-test") as u8 - cfg!(feature = "user-copy-irq-test") as u8; assert!( tests <= 1, "test images must be built with at most one test feature" ); }; unsafe extern "C" { fn __vectors_start(); static _image_start: u8; #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static _text_start: u8; static _image_end: u8; #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static __irq_guard_page_start: u8; static __irq_stack_start: u8; static __irq_stack_top: u8; static mut __irq_stack_guard: u64; #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static _text_end: u8; #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static _rodata_end: u8; #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] static mut __irq_stack_guard: u64; #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] static __irq_stack_start: u8; #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test" ))] static __irq_stack_top: u8; #[cfg(any( feature = "irq-sgi-test", feature = "task-guard-test", feature = "smp-topology-test" ))] static mut __stack_start: u64; #[cfg(any( feature = "irq-sgi-test", feature = "task-guard-test", feature = "smp-topology-test" ))] static __stack_top: u8; #[cfg(feature = "irq-gp-test")] fn __gp_probe_run(command: usize, output: *mut u64); #[cfg(feature = "smp-boot-protocol-test")] fn __secondary_entry(); #[cfg(feature = "smp-boot-protocol-test")] static __secondary_stack_start: u8; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_stack_top: u8; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_ready: u64; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_mpidr: u64; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_context: u64; #[cfg(feature = "smp-boot-protocol-test")] static __secondary_sp: u64; #[cfg(feature = "smp-vector-test")] fn __smp_vectors_start(); #[cfg(feature = "smp-vector-test")] fn __secondary_brk_pc(); #[cfg(feature = "smp-vector-test")] static mut __secondary_vector_command: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_seen: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_esr: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_elr: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_mpidr: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_sp: u64; #[cfg(feature = "smp-vector-test")] static __secondary_fault_vbar: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_freq: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_start: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_deadline0: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_tick0: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_deadline1: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_tick1: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_mpidr: u64; #[cfg(feature = "smp-timer-test")] static __secondary_timer_seen: u64; } #[cfg(feature = "smp-boot-protocol-test")] fn psci_hvc(fid: usize, target: usize, entry: usize, context: usize) -> usize { let result; // SAFETY: this probe runs only on QEMU virt with a DTB-confirmed `hvc` // PSCI conduit. The ABI returns x0 and does not use the guest stack. unsafe { asm!( "hvc #0", inlateout("x0") fid => result, in("x1") target, in("x2") entry, in("x3") context, options(nostack) ); } result } #[cfg(feature = "smp-boot-protocol-test")] fn smp_boot_protocol_test() -> ! { const TARGET: usize = 1; const CONTEXT: usize = 0x461; const PSCI_CPU_ON: usize = 0xc400_0003; let image_start = core::ptr::addr_of!(_image_start) as usize; let image_end = core::ptr::addr_of!(_image_end) as usize; let entry = __secondary_entry as *const () as usize; let stack_start = core::ptr::addr_of!(__secondary_stack_start) as usize; let stack_top = core::ptr::addr_of!(__secondary_stack_top) as usize; if entry < image_start || entry >= image_end || entry & 3 != 0 || stack_top - stack_start != 4096 { write_line("TURKOS/SMP-ERROR: entry or stack bounds"); park(); } let version = psci_hvc(0x8400_0000, 0, 0, 0); let result = psci_hvc(PSCI_CPU_ON, TARGET, entry, CONTEXT); let mut ready = 0; for _ in 0..100_000_000 { // SAFETY: only the secondary writes the one-word publication marker. ready = unsafe { read_volatile(core::ptr::addr_of!(__secondary_ready)) }; if ready == 1 { // SAFETY: the secondary publishes MPIDR/context/SP before READY. unsafe { asm!("dsb sy", options(nostack, preserves_flags)) }; break; } core::hint::spin_loop(); } let duplicate = psci_hvc(PSCI_CPU_ON, TARGET, entry, CONTEXT); let invalid = psci_hvc(PSCI_CPU_ON, 2, entry, CONTEXT); let mpidr = unsafe { read_volatile(core::ptr::addr_of!(__secondary_mpidr)) }; let context = unsafe { read_volatile(core::ptr::addr_of!(__secondary_context)) }; let secondary_sp = unsafe { read_volatile(core::ptr::addr_of!(__secondary_sp)) }; write_str("TURKOS/SMP-PSCI: VERSION=0x"); write_hex(version); write_str(" ON=0x"); write_hex(result); write_str(" DUP=0x"); write_hex(duplicate); write_str(" BAD=0x"); write_hex(invalid); write_str("\r\nTURKOS/SMP-SECONDARY: ENTRY=0x"); write_hex(entry); write_str(" MPIDR=0x"); write_hex(mpidr as usize); write_str(" CONTEXT=0x"); write_hex(context as usize); write_str(" SP=0x"); write_hex(secondary_sp as usize); write_str(" STACK=0x"); write_hex(stack_start); write_str("..0x"); write_hex(stack_top); write_str(" READY=0x"); write_hex(ready as usize); write_str("\r\n"); if version >= 0x0001_0000 && result == 0 && duplicate == usize::MAX - 3 && invalid == usize::MAX - 1 && ready == 1 && mpidr & 0xff == TARGET as u64 && context == CONTEXT as u64 && secondary_sp == stack_top as u64 { write_line("TURKOS/SMP-PROTOCOL: PASS"); } else { write_line("TURKOS/SMP-ERROR: PSCI or secondary contract"); park(); } #[cfg(feature = "smp-topology-test")] smp_topology_test(mpidr, secondary_sp, stack_start, stack_top); #[cfg(feature = "smp-vector-test")] smp_vector_test(mpidr, stack_start, stack_top); #[cfg(feature = "smp-timer-test")] smp_timer_test(mpidr); park(); } #[cfg(feature = "smp-vector-test")] fn smp_vector_test(secondary_mpidr: u64, stack_start: usize, stack_top: usize) { let primary_vbar: usize; // SAFETY: VBAR_EL1 is a per-CPU register; this read is on the boot CPU. unsafe { asm!("mrs {value}, vbar_el1", value = out(reg) primary_vbar, options(nomem, nostack, preserves_flags)) }; let expected_primary = __vectors_start as *const () as usize; let expected_secondary = __smp_vectors_start as *const () as usize; let brk_pc = __secondary_brk_pc as *const () as usize; if primary_vbar != expected_primary || expected_primary & 2047 != 0 || expected_secondary & 2047 != 0 || expected_primary == expected_secondary { write_line("TURKOS/SMP-ERROR: vector table identity or alignment"); park(); } // SAFETY: only the primary writes the command; the secondary reads it // after the full-system barrier and event. The secondary never returns. unsafe { write_volatile(core::ptr::addr_of_mut!(__secondary_vector_command), 1); asm!("dsb sy", "sev", options(nostack, preserves_flags)); } let mut seen = 0; for _ in 0..100_000_000 { // SAFETY: the secondary publishes all fault fields before SEEN. seen = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_seen)) }; if seen != 0 { unsafe { asm!("dsb sy", options(nostack, preserves_flags)) }; break; } core::hint::spin_loop(); } let esr = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_esr)) }; let elr = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_elr)) }; let mpidr = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_mpidr)) }; let sp = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_sp)) }; let vbar = unsafe { read_volatile(core::ptr::addr_of!(__secondary_fault_vbar)) }; let primary_vbar_after: usize; unsafe { asm!("mrs {value}, vbar_el1", value = out(reg) primary_vbar_after, options(nomem, nostack, preserves_flags)) }; write_str("TURKOS/SMP-VECTOR: PRIMARY=0x"); write_hex(primary_vbar); write_str(" SECONDARY=0x"); write_hex(expected_secondary); write_str(" BRK-PC=0x"); write_hex(brk_pc); write_str(" PRIMARY-AFTER=0x"); write_hex(primary_vbar_after); write_str("\r\nTURKOS/SMP-FAULT: CPU=0x"); write_hex(mpidr as usize); write_str(" ESR=0x"); write_hex(esr as usize); write_str(" ELR=0x"); write_hex(elr as usize); write_str(" SP=0x"); write_hex(sp as usize); write_str(" VBAR=0x"); write_hex(vbar as usize); write_str(" SEEN=0x"); write_hex(seen as usize); write_str("\r\n"); if smp::validate_secondary_brk( smp::VectorFault { esr, elr, mpidr, sp, vbar, seen, }, secondary_mpidr, brk_pc as u64, expected_secondary as u64, StackSpan { start: stack_start as u64, end: stack_top as u64, sp, }, ) .is_ok() && primary_vbar_after == primary_vbar { write_line("TURKOS/SMP-VECTOR: PASS"); } else { write_line("TURKOS/SMP-ERROR: secondary exception source or vector state"); park(); } } #[cfg(feature = "smp-timer-test")] fn smp_timer_test(secondary_mpidr: u64) { use turkos_kernel::smp::TimerSample; let primary_mpidr: u64; let frequency: u64; let start: u64; // SAFETY: all physical timer registers below are accessed only on the // primary CPU; secondary uses the same banked registers in assembly. unsafe { asm!("mrs {value}, mpidr_el1", value = out(reg) primary_mpidr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, cntfrq_el0", value = out(reg) frequency, options(nomem, nostack, preserves_flags)); } let period = frequency / 100; if period == 0 { write_line("TURKOS/SMP-ERROR: timer frequency"); park(); } // SAFETY: the secondary reads command 2 after dsb/sev. It is already // parked after its controlled BRK and has a private stack/vector table. unsafe { write_volatile(core::ptr::addr_of_mut!(__secondary_vector_command), 2); asm!("dsb sy", "sev", options(nostack, preserves_flags)); asm!("mrs {value}, cntpct_el0", value = out(reg) start, options(nomem, nostack, preserves_flags)); } let Some(deadline0) = start.checked_add(period) else { write_line("TURKOS/SMP-ERROR: timer deadline overflow"); park(); }; unsafe { asm!("msr cntp_cval_el0, {deadline}", "msr cntp_ctl_el0, {enable}", "isb", deadline = in(reg) deadline0, enable = in(reg) 1u64, options(nomem, nostack)) }; let mut status = 0u64; for _ in 0..100_000_000 { unsafe { asm!("mrs {value}, cntp_ctl_el0", value = out(reg) status, options(nomem, nostack, preserves_flags)) }; if status & 4 != 0 { break; } core::hint::spin_loop(); } if status & 4 == 0 { write_line("TURKOS/SMP-ERROR: primary timer first expiry"); park(); } let tick0: u64; unsafe { asm!("mrs {value}, cntpct_el0", value = out(reg) tick0, options(nomem, nostack, preserves_flags)) }; let Some(deadline1) = tick0.checked_add(period) else { write_line("TURKOS/SMP-ERROR: timer second deadline overflow"); park(); }; unsafe { asm!("msr cntp_cval_el0, {deadline}", "isb", deadline = in(reg) deadline1, options(nomem, nostack)) }; status = 0; for _ in 0..100_000_000 { unsafe { asm!("mrs {value}, cntp_ctl_el0", value = out(reg) status, options(nomem, nostack, preserves_flags)) }; if status & 4 != 0 { break; } core::hint::spin_loop(); } let tick1: u64; unsafe { asm!("mrs {value}, cntpct_el0", value = out(reg) tick1, options(nomem, nostack, preserves_flags)); asm!("msr cntp_ctl_el0, xzr", "isb", options(nomem, nostack)); } if status & 4 == 0 { write_line("TURKOS/SMP-ERROR: primary timer second expiry"); park(); } let primary = TimerSample { mpidr: primary_mpidr, frequency, start, deadline0, tick0, deadline1, tick1, seen: 1, }; let mut seen = 0; for _ in 0..100_000_000 { seen = unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_seen)) }; if seen != 0 { unsafe { asm!("dsb sy", options(nostack, preserves_flags)) }; break; } core::hint::spin_loop(); } let secondary = TimerSample { mpidr: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_mpidr)) }, frequency: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_freq)) }, start: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_start)) }, deadline0: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_deadline0)) }, tick0: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_tick0)) }, deadline1: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_deadline1)) }, tick1: unsafe { read_volatile(core::ptr::addr_of!(__secondary_timer_tick1)) }, seen, }; for (label, sample) in [ ("TURKOS/SMP-TIMER0: CPU=0x", primary), ("TURKOS/SMP-TIMER1: CPU=0x", secondary), ] { write_str(label); write_hex(sample.mpidr as usize); write_str(" FREQ=0x"); write_hex(sample.frequency as usize); write_str(" START=0x"); write_hex(sample.start as usize); write_str(" D0=0x"); write_hex(sample.deadline0 as usize); write_str(" T0=0x"); write_hex(sample.tick0 as usize); write_str(" D1=0x"); write_hex(sample.deadline1 as usize); write_str(" T1=0x"); write_hex(sample.tick1 as usize); write_str(" SEEN=0x"); write_hex(sample.seen as usize); write_str("\r\n"); } if secondary_mpidr == secondary.mpidr && smp::validate_timer_pair(primary, secondary).is_ok() { write_line("TURKOS/SMP-TIMER: PASS"); } else { write_line("TURKOS/SMP-ERROR: per-CPU timer sample"); park(); } } #[cfg(feature = "smp-topology-test")] fn smp_topology_test( secondary_mpidr: u64, secondary_sp: u64, secondary_start: usize, secondary_top: usize, ) { let primary_mpidr: u64; let primary_sp: usize; // SAFETY: read-only per-PE identity and the current AArch64 stack pointer. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) primary_mpidr, options(nomem, nostack, preserves_flags)); asm!("mov {value}, sp", value = out(reg) primary_sp, options(nomem, nostack, preserves_flags)); } let primary_start = core::ptr::addr_of!(__stack_start) as usize; let primary_top = core::ptr::addr_of!(__stack_top) as usize; let ids = smp::distinct_pair(primary_mpidr, secondary_mpidr); write_str("TURKOS/SMP-CPU-IDS: PRIMARY=0x"); write_hex(primary_mpidr as usize); write_str(" SECONDARY=0x"); write_hex(secondary_mpidr as usize); write_str(" AFF0=0x"); write_hex(smp::affinity(primary_mpidr) as usize); write_str(" AFF1=0x"); write_hex(smp::affinity(secondary_mpidr) as usize); write_str("\r\n"); if ids.is_ok() && smp::affinity(primary_mpidr) == 0 && smp::affinity(secondary_mpidr) == 1 { write_line("TURKOS/SMP-CPU-IDS: PASS"); } else { write_line("TURKOS/SMP-ERROR: duplicate or unexpected CPU identity"); park(); } let primary = StackSpan { start: primary_start as u64, end: primary_top as u64, sp: primary_sp as u64, }; let secondary = StackSpan { start: secondary_start as u64, end: secondary_top as u64, sp: secondary_sp, }; let stacks = smp::disjoint_stacks(primary, secondary); write_str("TURKOS/SMP-STACKS: PRIMARY=0x"); write_hex(primary_start); write_str("..0x"); write_hex(primary_top); write_str(" SP0=0x"); write_hex(primary_sp); write_str(" SECONDARY=0x"); write_hex(secondary_start); write_str("..0x"); write_hex(secondary_top); write_str(" SP1=0x"); write_hex(secondary_sp as usize); write_str("\r\n"); if stacks.is_ok() && primary_top - primary_start == 16 * 1024 && secondary_top - secondary_start == 4096 { write_line("TURKOS/SMP-STACKS: PASS"); } else { write_line("TURKOS/SMP-ERROR: CPU stacks overlap or have invalid bounds"); park(); } } #[cfg(feature = "mmu-tlb-test")] unsafe extern "C" { fn __turkos_replace_l3_probe(slot: *mut u64, va_page: u64, descriptor: u64); } #[cfg(any( feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] unsafe extern "C" { fn __turkos_swap_task_context(old: *mut task::CpuContext, new: *const task::CpuContext); } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] #[repr(align(4096))] struct TaskStack([u8; 16 * 1024]); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_A_STACK: TaskStack = TaskStack([0; 16 * 1024]); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_B_STACK: TaskStack = TaskStack([0; 16 * 1024]); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_CONTEXTS: [task::CpuContext; 2] = [task::CpuContext::EMPTY; 2]; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_BOOT_CONTEXT: task::CpuContext = task::CpuContext::EMPTY; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_SCHEDULER: Scheduler = Scheduler::new(); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_A_ID: Option = None; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_A_COUNT: usize = 0; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] static mut TASK_B_COUNT: usize = 0; #[cfg(feature = "task-lifecycle-test")] static mut TASK_C_COUNT: usize = 0; #[cfg(feature = "task-lifecycle-test")] static mut TASK_A_LEASE: task::StackLease = task::StackLease::new(); #[cfg(feature = "task-lifecycle-test")] static mut TASK_B_LEASE: task::StackLease = task::StackLease::new(); #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] const TASK_A_CANARY: u64 = 0x5455_524b_5441_534b; #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] const TASK_B_CANARY: u64 = 0x5455_524b_5442_534b; #[cfg(feature = "task-lifecycle-test")] const TASK_C_CANARY: u64 = 0x5455_524b_5443_534b; #[cfg(feature = "task-lifecycle-test")] const TASK_B_POISON: u64 = 0xdddd_dddd_dddd_dddd; struct BootData { hardware: dtb::Hardware, interrupts: dtb::InterruptHardware, reservations: dtb::Reservations, static_reserved: dtb::Reservations, #[cfg(any( feature = "virtio-discover-test", feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-tx-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "user-mmio-read-test" ))] virtio_mmio: dtb::VirtioMmioNodes, #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] dtb_address: usize, dtb_size: usize, } // Bootstrap address is specific to QEMU virt. The DTB replaces it after discovery. static UART_BASE: AtomicUsize = AtomicUsize::new(0x0900_0000); static SERIAL_LINE_LOCK: SerialLineLock = SerialLineLock::new(); const UART_TX_FULL: u32 = 1 << 5; // An AArch64 BRK instruction in writable data: if PXN were ineffective, the // branch would execute BRK rather than produce the expected instruction abort. #[cfg(feature = "mmu-nx-test")] #[used] #[unsafe(link_section = ".data.mmu_nx_probe")] static mut NX_PROBE: u32 = 0xd420_0000; #[cfg(feature = "mmu-rodata-test")] #[used] #[unsafe(link_section = ".rodata.mmu_rodata_probe")] static RODATA_PROBE: u64 = 0x5455_524b_4f53_524f; #[cfg(feature = "mmu-rodata-exec-test")] #[used] #[unsafe(link_section = ".rodata.mmu_rodata_exec_probe")] static RODATA_EXEC_PROBE: u32 = 0xd420_0000; // A nonzero initialized value forces this canary into writable .data. #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] static mut MMU_DATA_CANARY: u64 = 0x5455_524b_4f53_4441; #[unsafe(no_mangle)] #[cfg_attr(feature = "smp-allocator-test", allow(unreachable_code))] #[cfg_attr( any(feature = "fiq-label-test", feature = "serror-label-test"), allow(unreachable_code) )] pub extern "C" fn kernel_main(dtb_address: usize, boot_info_address: usize) -> ! { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1300_BOOT_COUNTER), os_k960_counter()); } let os_arguments = match os_k640_arguments(dtb_address, boot_info_address) { Ok(value) => value, Err(reason) => { write_str("TURKOS/OS/K640: REJECTED reason="); write_str(reason); write_str(" BEFORE-ALLOC\r\n"); park(); } }; if os_arguments.registers().x0 != dtb_address as u64 || os_arguments.registers().x1 != boot_info_address as u64 { write_line("TURKOS/OS/K640: REJECTED reason=arguments-abi-drift BEFORE-ALLOC"); park(); } let level = install_vectors(); let os_entry = match os_k625_capture(level) { Ok(value) => value, Err(reason) => { write_str("TURKOS/OS/K625: REJECTED reason="); write_str(reason); write_str(" BEFORE-ALLOC\r\n"); park(); } }; write_line("TURKOS/BOOT0: ARM64 kernel entered"); write_str("TURKOS/OS/K625: ENTRY EL=0x"); write_hex(os_entry.0.current_el as usize); write_str(" DAIF=0x"); write_hex(os_entry.0.daif as usize); write_str(" SP=0x"); write_hex(os_entry.0.sp as usize); write_str(" START=0x"); write_hex(os_entry.0.stack_start as usize); write_str(" TOP=0x"); write_hex(os_entry.0.stack_top as usize); write_str(" PASS\r\n"); let mut synthetic_el2 = os_entry.0; synthetic_el2.current_el = 8; if turkos_kernel::runtime::entry_level::normalize(synthetic_el2) != Err(turkos_kernel::runtime::entry_level::Error::UnsupportedLevel) { write_line("TURKOS/OS/K625: REJECTED reason=synthetic-el2-accepted"); park(); } write_line("TURKOS/OS/K625: SYNTHETIC-EL2 REJECTED"); if let Err(reason) = os_k635_capture() { write_str("TURKOS/OS/K635: REJECTED reason="); write_str(reason); write_str(" BEFORE-ALLOC\r\n"); park(); } let os_sections = match os_k630_pre_mmu() { Ok(value) => value, Err(reason) => { write_str("TURKOS/OS/K630: REJECTED reason="); write_str(reason); write_str(" BEFORE-ALLOC\r\n"); park(); } }; write_str("TURKOS/OS/K630: SECTIONS TEXT=0x"); write_hex(os_sections.text_start as usize); write_str(" TEXT-END=0x"); write_hex(os_sections.text_end as usize); write_str(" RODATA-END=0x"); write_hex(os_sections.rodata_end as usize); write_str(" BSS-START=0x"); write_hex(os_sections.bss_start as usize); write_str(" BSS-END=0x"); write_hex(os_sections.bss_end as usize); write_str(" PASS\r\n"); write_str("TURKOS/EL: "); write_byte(b'0' + level as u8); write_str("\r\n"); write_str("TURKOS/DTB: 0x"); write_hex(dtb_address); write_str("\r\n"); let boot = match read_boot_data(dtb_address) { Ok(boot) => boot, Err(_) => { write_line("TURKOS/DTB-ERROR"); park(); } }; let hardware = &boot.hardware; UART_BASE.store(hardware.uart_base as usize, Ordering::Relaxed); write_str("TURKOS/UART: 0x"); write_hex(hardware.uart_base as usize); write_str("\r\nTURKOS/RAM: 0x"); write_hex(hardware.ram_base as usize); write_str("+0x"); write_hex(hardware.ram_size as usize); write_str("\r\n"); // Mandatory on the development kernel's ordinary/default entry, not a // probe feature. Rejection parks before old firmware-map reads/allocations. write_line("TURKOS/OS/K610: ENTER BEFORE-ALLOC"); write_str("TURKOS/OS/K610: INPUT DTB=0x"); write_hex(dtb_address); write_str(" BYTES=0x"); write_hex(boot.dtb_size); write_str(" X1=0x"); write_hex(boot_info_address); write_str("\r\n"); let os_header = match os_k610_header(boot_info_address, &boot) { Ok(header) => header, Err(error) => { write_str("TURKOS/OS/K610: REJECTED reason="); write_str(error.tag()); write_str(" BEFORE-ALLOC\r\n"); park(); } }; write_str("TURKOS/OS/K610: VALIDATED source="); if os_header.is_some() { write_str("uefi-bootinfo-v1 HEADER=72"); } else { write_str("direct-image HEADER=absent"); } write_str(" BEFORE-ALLOC\r\n"); write_line("TURKOS/OS/K605: ENTER BEFORE-ALLOC"); write_str("TURKOS/OS/K605: INPUT DTB=0x"); write_hex(dtb_address); write_str(" BYTES=0x"); write_hex(boot.dtb_size); write_str(" X1=0x"); write_hex(boot_info_address); write_str("\r\n"); let os_boot = match os_k605_entry(dtb_address, boot_info_address, &boot) { Ok(value) => value, Err(error) => { write_str("TURKOS/OS/K605: REJECTED reason="); write_str(os_k605_error_tag(&error)); write_str(" BEFORE-ALLOC\r\n"); park(); } }; if os_header.map(|value| value.value()) != os_boot.boot_info { write_line("TURKOS/OS/K610: REJECTED reason=header-source-mismatch BEFORE-ALLOC"); park(); } write_str("TURKOS/OS/K605: VALIDATED source="); write_str(os_boot.origin.tag()); write_str(" RAM=0x"); write_hex(os_boot.ram.start() as usize); write_str("+0x"); write_hex(os_boot.ram.bytes() as usize); write_str(" BEFORE-ALLOC\r\n"); if let Err(reason) = os_k680_fixture_start(&boot) { write_str("TURKOS/OS/K680: REJECTED reason="); write_str(reason); write_str(" BEFORE-ALLOC\r\n"); park(); } let uefi_info = match read_uefi_boot_info(boot_info_address, &boot, dtb_address) { Ok(info) => info, Err(()) => { write_line("TURKOS/BOOTINFO-ERROR"); park(); } }; write_str("TURKOS/GICD: 0x"); write_hex(boot.interrupts.gic_distributor as usize); write_str("\r\nTURKOS/GICR: 0x"); write_hex(boot.interrupts.gic_redistributor as usize); write_str("\r\nTURKOS/TIMER-PPI: 0x"); write_hex(boot.interrupts.nonsecure_phys_timer_ppi as usize); write_str("\r\nTURKOS/IRQ-MASK: ON\r\n"); #[cfg(any(feature = "fiq-label-test", feature = "serror-label-test"))] { // These fault-label images stop before the normal allocator path. let _ = &uefi_info; let slot = if cfg!(feature = "fiq-label-test") { 6 } else { 7 }; let interrupted_sp: usize; unsafe { asm!("mov {value}, sp", value = out(reg) interrupted_sp, options(nomem, nostack, preserves_flags)) }; let synthetic = ExceptionFrame { registers: [0; 31], interrupted_sp, canary: exception::FRAME_CANARY, }; write_str("TURKOS/ASYNC-LABEL-PROBE: SYNTHETIC SLOT=0x"); write_hex_byte(slot as u8); write_str("\r\n"); // A stale synchronous data-abort ESR is deliberate: an asynchronous // slot must never claim its old FAR. This calls the fatal reporter, // not the hardware FIQ/SError vector entry. kernel_exception(1, (0x25 << 26) | 0x07, 0, slot, &synthetic, 0x3c0); } write_str("TURKOS/STATIC-RESERVATIONS: 0x"); write_hex(boot.static_reserved.as_slice().len()); write_str("\r\n"); if cfg!(feature = "smp-boot-protocol-test") { #[cfg(feature = "smp-boot-protocol-test")] smp_boot_protocol_test(); } // The firmware exited before the UEFI loader jumped here. Copy the // validated map to kernel-owned BSS before the runtime planner can read it. let os_owned_map = match os_k620_snapshot(uefi_info.as_ref()) { Ok(value) => value, Err(reason) => { write_str("TURKOS/OS/K620: REJECTED reason="); write_str(reason); write_str(" BEFORE-ALLOC\r\n"); park(); } }; if let Some(ref owned) = os_owned_map { write_str("TURKOS/OS/K620: OWNED-MAP source=0x"); write_hex(owned.source_address() as usize); write_str(" copy=0x"); write_hex(owned.bytes().as_ptr() as usize); write_str(" bytes=0x"); write_hex(owned.len()); write_str(" BEFORE-ALLOC\r\n"); } else { write_line("TURKOS/OS/K620: DIRECT-NO-FIRMWARE BEFORE-ALLOC"); } if os_k720_classify(&boot, dtb_address, uefi_info.as_ref(), os_owned_map.as_ref().map(|map| map.bytes())).is_err() { write_line("TURKOS/OS/FRAME: REJECTED BEFORE-ALLOC"); park(); } let free = match plan_free_memory(&boot, dtb_address, uefi_info.as_ref(), os_owned_map.as_ref().map(|map| map.bytes())) { Ok(free) => free, Err(_) => { write_line("TURKOS/MEM-ERROR"); park(); } }; // K615: audit the finalized free map and the real frame allocator before // the ordinary 16-frame smoke test. This is the disposable branch only. if let Err(reason) = os_k620_source_mutation_probe(uefi_info.as_ref(), os_owned_map.as_ref()) { write_str("TURKOS/OS/K620: REJECTED reason="); write_str(reason); write_str("\r\n"); park(); } let (reserved_owners, reserved_pages, denied_pages) = match os_k615_regions(&boot, dtb_address, uefi_info.as_ref(), &free) { Ok(summary) => summary, Err(reason) => { write_str("TURKOS/OS/K615: REJECTED reason="); write_str(reason); write_str("\r\n"); park(); } }; write_str("TURKOS/OS/K615: RESERVED owners=0x"); write_hex(reserved_owners); write_str(" pages=0x"); write_hex(reserved_pages as usize); write_str(" denied=0x"); write_hex(denied_pages as usize); write_str(" FREE=0x"); write_hex(free.total_pages() as usize); write_str(" PASS\r\n"); if os_k720_verify_planned(&free).is_err() { write_line("TURKOS/OS/FRAME: REJECTED reason=frame-planner-inventory BEFORE-ALLOC"); park(); } if uefi_info.is_some() { write_line("TURKOS/EFI-FREEMAP: CONVENTIONAL-ONLY PASS"); } #[cfg(feature = "uefi-ownership-test")] if let Some((address, _)) = uefi_info.as_ref() { uefi_owned_endpoint_probe(free.clone(), *address); } #[cfg(feature = "smp-allocator-test")] smp_allocator_probe::run(free, &boot, level); write_str("TURKOS/FREE-RANGES: 0x"); write_hex(free.regions().len()); write_str("\r\n"); for (index, region) in free.regions().iter().enumerate() { write_str("TURKOS/FREE-RANGE: INDEX=0x"); write_hex(index); write_str(" START=0x"); write_hex(region.start as usize); write_str(" END=0x"); write_hex(region.end as usize); write_str("\r\n"); } write_str("TURKOS/FREE-PAGES: 0x"); write_hex(free.total_pages() as usize); write_str("\r\n"); #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] let mmu_free = free.clone(); if !frame_smoke_test(free) { write_line("TURKOS/FRAME-ERROR"); park(); } write_line("TURKOS/FRAME-TEST: PASS"); #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] mmu_smoke_test(mmu_free, &boot, level); #[cfg(feature = "panic-test")] { panic!("intentional smoke-test panic"); } #[cfg(feature = "exception-test")] { #[cfg(feature = "log-secret-test")] { const CANARY: &[u8; 32] = b"K453-PRIVATE-CANARY-9a7F3d2E4b6C"; let secret = core::ptr::addr_of_mut!(LOG_SECRET_PAGE).cast::(); for (index, byte) in CANARY.iter().copied().enumerate() { // SAFETY: this isolated test image owns its 4 KiB BSS page. unsafe { write_volatile(secret.add(index), byte) }; } for (index, byte) in CANARY.iter().copied().enumerate() { if unsafe { read_volatile(secret.add(index)) } != byte { write_line("TURKOS/LOG-SECRET-ERROR: seed mismatch"); park(); } } write_line("TURKOS/LOG-SECRET: SEEDED BYTES=32"); // Deliberately place canary fragments in saved GP registers. unsafe { asm!( "brk #0x42", in("x0") u64::from_be_bytes(*b"K453-PRI") as usize, in("x3") u64::from_be_bytes(*b"VATE-CAN") as usize, in("x30") u64::from_be_bytes(*b"ARY-9a7F") as usize, options(nomem, nostack, noreturn) ) } } #[cfg(not(feature = "log-secret-test"))] { // SAFETY: BRK enters the fatal vector and never returns. The three // markers identify the interrupted values, including the link register. unsafe { asm!( "brk #0x42", in("x0") 0x544b_4252_4b5f_5830usize, in("x3") 0x544b_4252_4b5f_5833usize, in("x30") 0x544b_4252_4b5f_4c52usize, options(nomem, nostack, noreturn) ) } } } #[cfg(feature = "fault-test")] { // SAFETY: UDF enters the same fatal vector and never returns. unsafe { asm!( "udf #0", in("x0") 0x544b_5544_465f_5830usize, in("x3") 0x544b_5544_465f_5833usize, in("x30") 0x544b_5544_465f_4c52usize, options(nomem, nostack, noreturn) ) } } #[cfg(feature = "frame-stress-test")] park() } fn install_vectors() -> usize { let base = __vectors_start as *const () as usize; let current_el: usize; // SAFETY: CurrentEL is readable in privileged AArch64 modes. unsafe { asm!("mrs {level}, CurrentEL", level = out(reg) current_el, options(nomem, nostack, preserves_flags)) } let level = (current_el >> 2) & 3; // SAFETY: QEMU's direct Image boot enters at EL1 or EL2; the table is // linker-aligned to 2 KiB and exceptions remain masked during setup. unsafe { match level { 1 => asm!("msr VBAR_EL1, {base}", "isb", base = in(reg) base, options(nomem, nostack)), 2 => asm!("msr VBAR_EL2, {base}", "isb", base = in(reg) base, options(nomem, nostack)), _ => { write_line("TURKOS/EL-ERROR"); park() } } } level } // The fatal vector uses 8-byte AArch64 register slots in a 272-byte frame. // Keep these assertions beside the Rust view; assembly receives the three // variable offsets from this layout through global_asm! above. const _: () = { assert!(size_of::() == 8); assert!(size_of::() == 272); assert!(offset_of!(ExceptionFrame, registers) == 0); assert!(offset_of!(ExceptionFrame, interrupted_sp) == 248); assert!(offset_of!(ExceptionFrame, canary) == 256); }; // One CPU, IRQ masked, and a non-returning fatal path: volatile state avoids // depending on an exclusive-monitor retry inside a nested exception. After // the second entry, further entries park without touching the UART again. static mut FATAL_ENTRIES: usize = 0; #[cfg(feature = "log-secret-test")] static mut LOG_SECRET_PAGE: [u8; 4096] = [0; 4096]; #[unsafe(no_mangle)] pub extern "C" fn kernel_exception( level: usize, esr: usize, elr: usize, slot: usize, frame: *const ExceptionFrame, spsr: usize, ) -> ! { if os_k880_expected_user_trap(slot, esr, elr, frame) { park(); } if os_k860_expected_user_trap(slot, esr, elr, frame) { park(); } if os_k840_expected_user_trap(slot, esr, elr, frame) { park(); } if os_k820_expected_user_fault(slot, esr, frame) { park(); } if os_k740_expected_user_svc(slot, esr, frame) { park(); } if os_k700_expected_hole_fault(level, esr, slot, frame) { park(); } if os_k635_expected_guard_fault(level, esr, slot, frame) { park(); } if os_k680_expected_user_fault(slot, esr as u64, elr as u64) { park(); } os_k680_kernel_fault(slot, esr as u64, elr as u64); let depth = core::ptr::addr_of_mut!(FATAL_ENTRIES); // SAFETY: only the boot CPU runs this fatal path, IRQ remains masked, and // the volatile store completes before any deliberate nested BRK. let prior = unsafe { read_volatile(depth) }; unsafe { write_volatile(depth, prior.saturating_add(1)) }; match exception::fatal_reentry(prior) { exception::FatalReentry::First => {} exception::FatalReentry::StopOnce => { write_line("TURKOS/EXCEPTION-NESTED: STOP"); park(); } exception::FatalReentry::ParkSilently => park(), } #[cfg(feature = "nested-fault-test")] { write_line("TURKOS/EXCEPTION-FIRST: ENTER"); // The first deliberate BRK in mmu_smoke_test reached this handler. // Trigger a second BRK *inside* it to exercise the reentry guard. unsafe { asm!("brk #0x159") }; write_line("TURKOS/EXCEPTION-NESTED: RETURNED"); } // SAFETY: the vector entry placed a complete, aligned frame on the // current exception stack before passing its address in x4. let frame = unsafe { &*frame }; if !frame.canary_valid() { write_line("TURKOS/EXCEPTION-FRAME: CORRUPT"); park(); } write_str("TURKOS/EXCEPTION: EL"); write_byte(b'0' + level as u8); write_str(" SLOT=0x"); write_hex_byte(slot as u8); write_str(" EC=0x"); let ec = ((esr >> 26) & 0x3f) as u8; write_hex_byte(ec); write_str(" ESR=0x"); write_hex(esr); write_str(" ELR=0x"); write_hex(elr); write_str(" KIND="); write_str(exception::kind_name(slot)); if let Some(kind) = exception::unexpected_el1_async(slot) { write_str(" UNEXPECTED-EL1-ASYNC="); write_str(kind); write_str(" FATAL=YES"); } write_str(" SP=0x"); write_hex(frame.interrupted_sp); write_str(" SPSR=0x"); write_hex(spsr); // Only four explicitly diagnostic fixtures may reveal saved GP values. // Ordinary fatal paths omit them; they may contain user or kernel secrets. #[cfg(all( not(feature = "log-secret-test"), any( feature = "exception-test", feature = "fault-test", feature = "mmu-brk-test", feature = "mmu-udf-test" ) ))] { write_str(" X0=0x"); write_hex(frame.registers[0]); write_str(" X3=0x"); write_hex(frame.registers[3]); write_str(" X30=0x"); write_hex(frame.registers[30]); } #[cfg(not(all( not(feature = "log-secret-test"), any( feature = "exception-test", feature = "fault-test", feature = "mmu-brk-test", feature = "mmu-udf-test" ) )))] write_str(" REGISTERS=REDACTED"); write_str(" EC-NAME="); write_str(exception::class_name(ec)); if exception::far_is_valid_for_slot(slot, esr as u64) { write_str(" FAR-VALID=YES FAR=0x"); let far: usize; // SAFETY: this ESR class defines FAR for the current privileged level. unsafe { match level { 1 => { asm!("mrs {far}, FAR_EL1", far = out(reg) far, options(nomem, nostack, preserves_flags)) } 2 => { asm!("mrs {far}, FAR_EL2", far = out(reg) far, options(nomem, nostack, preserves_flags)) } _ => park(), } } write_hex(far); } else { write_str(" FAR-VALID=NO FAR=NA"); } write_str(" FRAME-CANARY=PASS\r\n"); write_str("TURKOS/EXCEPTION-ORIGIN: "); write_str(exception::origin_name(slot)); write_str("\r\n"); park() } #[cfg(any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test" ))] const IRQ_GUARD: u64 = 0x5455_524b_4952_5147; #[cfg(feature = "irq-sgi-test")] const MAIN_STACK_GUARD: u64 = 0x5455_524b_4d41_494e; #[cfg(feature = "irq-sgi-test")] static mut IRQ_SGI_SEEN: usize = 0; #[cfg(feature = "irq-gp-test")] static mut GP_PROBE_ACTIVE: usize = 0; #[cfg(feature = "irq-gp-test")] #[unsafe(no_mangle)] static mut GP_IRQ_WAKE: usize = 0; #[cfg(feature = "irq-gp-test")] static mut GP_RETURNED: [u64; 31] = [0; 31]; #[cfg(feature = "irq-gp-test")] #[unsafe(no_mangle)] static mut GP_RETURNED_SP_EL0: u64 = 0; #[cfg(feature = "irq-gp-test")] fn gp_register_mask(registers: &[u64; 31]) -> u64 { let mut mask = 0; for (index, value) in registers.iter().enumerate() { if *value == 0x6000 + index as u64 { mask |= 1u64 << index; } } mask } #[cfg(feature = "irq-gp-test")] fn gp_return_probe( frame: *const irq::IrqContext, stack_start: usize, stack_top: usize, current_sp: usize, ) -> usize { // SAFETY: the vector placed an 816-byte frame at the top of this IRQ // stack, and this feature runs only on one CPU with a single pending SGI. let context = unsafe { &*frame }; let mask = gp_register_mask(&context.x); let valid = frame as usize == stack_top - size_of::() && irq::handler_sp_in_stack(stack_start, frame as usize, current_sp) && irq::main_stack_intact( core::ptr::addr_of!(__stack_start) as usize, core::ptr::addr_of!(__stack_top) as usize, stack_start, stack_top, context.interrupted_sp as usize, unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) }, MAIN_STACK_GUARD, ) && context.vector_slot == 5 && context.spsr & 15 == 5 && context.sp_el0 == 0x7a5a && mask == 0x7fff_ffff && unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) } == IRQ_GUARD && unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) } == MAIN_STACK_GUARD; let iar: usize; // SAFETY: the isolated guest enables SGI 1 and owns the GIC CPU interface. unsafe { asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags)) }; if !valid || irq::classify_iar(iar as u64, 1) != irq::IarClass::Expected { write_line("TURKOS/IRQ-GP-ERROR: frame or source"); return 0; } // SAFETY: EOIR and, when required, DIR receive precisely the IAR value. unsafe { asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } let previous = unsafe { read_volatile(core::ptr::addr_of!(IRQ_SGI_SEEN)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_SGI_SEEN), previous + 1) }; if previous != 0 { write_line("TURKOS/IRQ-GP-ERROR: repeated interrupt"); return 0; } write_str("TURKOS/IRQ-GP-IAR: 0x"); write_hex(iar); write_str("\r\nTURKOS/IRQ-GP-EOI: 0x"); write_hex(iar); write_str("\r\nTURKOS/IRQ-GP-SAVED-MASK: 0x"); write_hex(mask as usize); write_str("\r\nTURKOS/IRQ-GP-SAVED-SP_EL0: 0x"); write_hex(context.sp_el0 as usize); write_str("\r\n"); unsafe { write_volatile(core::ptr::addr_of_mut!(GP_IRQ_WAKE), 1) }; // WFE in gp_probe.S must wake even if the SGI arrived before WFE. unsafe { asm!("sev", options(nomem, nostack, preserves_flags)) }; 1 } #[cfg(any(feature = "irq-timer-test", feature = "irq-unknown-test"))] static mut IRQ_TIMER_SEEN: usize = 0; #[cfg(any(feature = "irq-timer-test", feature = "irq-unknown-test"))] static mut IRQ_TIMER_PERIOD: usize = 0; #[cfg(any(feature = "irq-timer-test", feature = "irq-unknown-test"))] static mut IRQ_TIMER_DEADLINE: u64 = 0; #[cfg(feature = "irq-timer-test")] #[cfg_attr(feature = "irq-service-test", allow(dead_code))] static mut IRQ_TIMER_MAIN_PROGRESS: usize = 0; #[cfg(feature = "irq-timer-storm-test")] static mut IRQ_STORM_LAST_PROGRESS: usize = 0; #[cfg(feature = "irq-timer-storm-test")] static mut IRQ_STORM_INTERLEAVED: usize = 0; #[cfg(feature = "irq-timer-storm-test")] static mut IRQ_STORM_SKIPPED: usize = 0; #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] static mut IRQ_UNKNOWN_GICR: usize = 0; #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] static mut IRQ_UNKNOWN_HANDLED: usize = 0; #[cfg(any(feature = "task-preempt-test", feature = "task-sleep-test"))] #[unsafe(no_mangle)] pub extern "C" fn kernel_return_probe(frame: *const irq::IrqContext) -> usize { preempt::return_probe(frame) } #[cfg(all( not(any(feature = "task-preempt-test", feature = "task-sleep-test")), any( feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test" ) ))] #[cfg_attr(feature = "smp-irq-lock-test", allow(unreachable_code))] #[cfg_attr(feature = "irq-spurious-vector-test", allow(unreachable_code))] #[unsafe(no_mangle)] pub extern "C" fn kernel_return_probe(frame: *const irq::IrqContext) -> usize { let stack_start = core::ptr::addr_of!(__irq_stack_start) as usize; let stack_top = core::ptr::addr_of!(__irq_stack_top) as usize; let mut current_sp: usize; #[cfg(feature = "irq-return-test")] let esr: usize; // SAFETY: this handler is called only by an isolated EL1 test vector. unsafe { asm!("mov {value}, sp", value = out(reg) current_sp, options(nomem, nostack, preserves_flags)); #[cfg(feature = "irq-return-test")] asm!("mrs {value}, ESR_EL1", value = out(reg) esr, options(nomem, nostack, preserves_flags)); } #[cfg(feature = "smp-irq-lock-test")] return smp_allocator_probe::irq_lock_probe(frame, stack_start, stack_top, current_sp); #[cfg(feature = "irq-service-test")] if irq_service::active() { return irq_service::handle(frame, stack_start, stack_top, current_sp); } #[cfg(feature = "irq-spurious-vector-test")] return spurious_vector::handle(frame, stack_start, stack_top, current_sp); #[cfg(feature = "process-exit-irq-test")] if process_probe::exit_irq_active() { return process_probe::handle_exit_irq(frame, stack_start, stack_top, current_sp); } let frame_address = frame as usize; #[cfg(feature = "irq-gp-test")] if unsafe { read_volatile(core::ptr::addr_of!(GP_PROBE_ACTIVE)) } != 0 { return gp_return_probe(frame, stack_start, stack_top, current_sp); } // SAFETY: the vector wrote the complete frame at the top of the dedicated // stack before calling Rust; the low guard was initialized by the probe. // A leaf Rust handler may keep SP exactly at the frame base; the first // local stack allocation, if any, moves it below. Both states are valid. #[allow(unused_mut)] let mut valid = unsafe { frame_address == stack_top - size_of::() && irq::handler_sp_in_stack(stack_start, frame_address, current_sp) && read_volatile(core::ptr::addr_of!(__irq_stack_guard)) == IRQ_GUARD && (*frame).interrupted_sp & 15 == 0 && ((*frame).interrupted_sp < stack_start as u64 || (*frame).interrupted_sp >= stack_top as u64) && (*frame).elr != 0 && ((*frame).spsr & 0xf) == 5 && irq::return_slot_matches((*frame).vector_slot, !cfg!(feature = "irq-return-test")) && (*frame).x[0] == 0x5455_524b_4952_5130 && (*frame).x[3] == 0x5455_524b_4952_5133 && (*frame).x[16] == 0x5455_524b_4952_5116 && (*frame).x[17] == 0x5455_524b_4952_5117 && (*frame).x[30] == 0x5455_524b_4952_514c && (*frame).q[0] == [0x5a5a_5a5a_5a5a_5a5a; 2] && (*frame).q[31] == [0xa5a5_a5a5_a5a5_a5a5; 2] }; #[cfg(feature = "irq-sgi-test")] { let main_start = core::ptr::addr_of!(__stack_start) as usize; let main_top = core::ptr::addr_of!(__stack_top) as usize; let main_guard = unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) }; valid &= irq::main_stack_intact( main_start, main_top, stack_start, stack_top, unsafe { (*frame).interrupted_sp as usize }, main_guard, MAIN_STACK_GUARD, ); } #[cfg(feature = "irq-return-test")] { valid &= ((esr >> 26) & 0x3f) == 0x15 && (esr & 0xffff) == 0x42; if valid { write_str("TURKOS/SVC-VECTOR-SLOT: 0x"); write_hex_byte(unsafe { (*frame).vector_slot as u8 }); write_str("\r\n"); write_str("TURKOS/SVC-RETURN-ESR: 0x"); write_hex(esr); write_str("\r\n"); write_line("TURKOS/SVC-RETURN-HANDLER: PASS"); return 1; } write_str("TURKOS/SVC-RETURN-ERROR: context esr=0x"); write_hex(esr); write_str(" frame=0x"); write_hex(frame_address); write_str(" stack=0x"); write_hex(current_sp); write_str(" guard=0x"); write_hex(unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) as usize }); write_str(" old=0x"); write_hex(unsafe { (*frame).interrupted_sp as usize }); write_str(" spsr=0x"); write_hex(unsafe { (*frame).spsr as usize }); write_str(" x0=0x"); write_hex(unsafe { (*frame).x[0] as usize }); write_str(" x3=0x"); write_hex(unsafe { (*frame).x[3] as usize }); write_str(" x16=0x"); write_hex(unsafe { (*frame).x[16] as usize }); write_str(" x17=0x"); write_hex(unsafe { (*frame).x[17] as usize }); write_str(" x30=0x"); write_hex(unsafe { (*frame).x[30] as usize }); write_str(" q0=0x"); write_hex(unsafe { (*frame).q[0][0] as usize }); write_str(" q31=0x"); write_hex(unsafe { (*frame).q[31][0] as usize }); write_str("\r\n"); return 0; } #[cfg(feature = "irq-sgi-test")] { let iar: usize; // SAFETY: setup enabled the GICv3 system-register interface before // unmasking IRQ. Reading IAR acknowledges one Group 1 interrupt. unsafe { asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags)); } valid &= (iar & 0x00ff_ffff) == 1; valid &= irq::nzcv_matches(unsafe { (*frame).spsr }, irq::IRQ_PROBE_NZCV); if !valid { write_str("TURKOS/IRQ-SGI-ERROR: iar=0x"); write_hex(iar); write_str(" frame=0x"); write_hex(frame_address); write_str(" stack=0x"); write_hex(current_sp); write_str(" old=0x"); write_hex(unsafe { (*frame).interrupted_sp as usize }); write_str(" spsr=0x"); write_hex(unsafe { (*frame).spsr as usize }); write_str(" x0=0x"); write_hex(unsafe { (*frame).x[0] as usize }); write_str(" x3=0x"); write_hex(unsafe { (*frame).x[3] as usize }); write_str(" x16=0x"); write_hex(unsafe { (*frame).x[16] as usize }); write_str(" x17=0x"); write_hex(unsafe { (*frame).x[17] as usize }); write_str(" x30=0x"); write_hex(unsafe { (*frame).x[30] as usize }); write_str(" q0=0x"); write_hex(unsafe { (*frame).q[0][0] as usize }); write_str(" q31=0x"); write_hex(unsafe { (*frame).q[31][0] as usize }); write_str("\r\n"); return 0; } // SAFETY: the acknowledged SGI is edge-triggered. EOIR drops its // priority and deactivates it in EOImode 0; mode 1 needs DIR as well. unsafe { asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } // SAFETY: this dedicated guest has one CPU. IRQ is masked on entry, // so a plain volatile count avoids an exclusive/atomic access while // still detecting an unexpected second interrupt. let previous = unsafe { read_volatile(core::ptr::addr_of!(IRQ_SGI_SEEN)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_SGI_SEEN), previous + 1) }; if previous != 0 { write_line("TURKOS/IRQ-SGI-ERROR: repeated interrupt"); return 0; } write_str("TURKOS/IRQ-SGI-IAR: 0x"); write_hex(iar); write_str("\r\n"); write_str("TURKOS/IRQ-SGI-EOI: 0x"); write_hex(iar); write_str("\r\n"); if cfg!(feature = "irq-guard-test") { let guard_page = core::ptr::addr_of!(__irq_guard_page_start) as usize; let target = guard_page + memory::PAGE_SIZE as usize - 1; write_str("TURKOS/IRQ-GUARD-PAGE: 0x"); write_hex(guard_page); write_str("\r\nTURKOS/IRQ-GUARD-STACK: 0x"); write_hex(stack_start); write_str("-0x"); write_hex(stack_top); write_str(" HANDLER-SP=0x"); write_hex(current_sp); write_str("\r\nTURKOS/IRQ-GUARD-TARGET: 0x"); write_hex(target); write_str("\r\n"); write_line("TURKOS/IRQ-GUARD-PROBE: WRITE-LOWER-PAGE"); // SAFETY: this isolated guest intentionally writes the final byte // of the unmapped page below its IRQ stack. The fatal same-EL // vector must report FAR and park; returning is a test failure. unsafe { write_volatile(target as *mut u8, 0xa5) }; write_line("TURKOS/IRQ-GUARD-ERROR: write unexpectedly returned"); return 0; } write_str("TURKOS/IRQ-VECTOR-SLOT: 0x"); write_hex_byte(unsafe { (*frame).vector_slot as u8 }); write_str("\r\n"); write_str("TURKOS/IRQ-SGI-SPSR: 0x"); write_hex(unsafe { (*frame).spsr as usize }); write_str("\r\n"); write_str("TURKOS/IRQ-STACK-BOUNDS: MAIN=0x"); write_hex(core::ptr::addr_of!(__stack_start) as usize); write_str("-0x"); write_hex(core::ptr::addr_of!(__stack_top) as usize); write_str(" IRQ=0x"); write_hex(stack_start); write_str("-0x"); write_hex(stack_top); write_str(" OLD-SP=0x"); write_hex(unsafe { (*frame).interrupted_sp as usize }); write_str(" HANDLER-SP=0x"); write_hex(current_sp); write_str(" GUARD=0x"); write_hex(unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) as usize }); write_str("\r\n"); write_line("TURKOS/IRQ-SGI-HANDLER: PASS"); return 1; } #[cfg(any(feature = "irq-timer-test", feature = "irq-unknown-test"))] { let iar: usize; // SAFETY: this image enabled the Group 1 CPU interface for PPI 30; // the negative image additionally enables SGI 2 to test rejection. unsafe { asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags)) }; match irq::classify_iar(iar as u64, 30) { irq::IarClass::Unexpected(intid) => { // The isolated K457 image owns SGI 2. Disable the exact // redistributor source, wait for RWP, and verify enable // readback before allowing ERET. Other images stay fatal. #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] let masked = if intid == 2 { let base = unsafe { read_volatile(core::ptr::addr_of!(IRQ_UNKNOWN_GICR)) }; if base == 0 { false } else { let bit = irq::sgi_disable_bit(intid).unwrap_or(0); let control = base as *const u32; let enabled = (base + 0x1_0100) as *const u32; let disable = (base + 0x1_0180) as *mut u32; // SAFETY: the validated 128 KiB GICR Device window // was mapped before IRQ unmasking in this image. unsafe { write_volatile(disable, bit); asm!("dsb sy", options(nostack, preserves_flags)); let mut ready = false; for _ in 0..1_000_000 { if read_volatile(control) & 8 == 0 { ready = true; break; } } ready && read_volatile(enabled) & bit == 0 } } } else { false }; #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] if masked { write_line("TURKOS/IRQ-UNKNOWN-ORDER: MASK-READBACK=OFF BEFORE-EOI"); } // SAFETY: an unexpected acknowledged source must be EOI'd. // Stop the timer so an unrelated PPI cannot storm meanwhile. unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)); asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } write_str("TURKOS/IRQ-UNKNOWN-EOI: 0x"); write_hex(intid as usize); write_str("\r\n"); write_line("TURKOS/IRQ-UNKNOWN: REJECTED"); #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] if masked { // A repeat delivery must become count=2, never be hidden // by overwriting the first observation with a constant. unsafe { let counter = core::ptr::addr_of_mut!(IRQ_UNKNOWN_HANDLED); write_volatile(counter, read_volatile(counter).saturating_add(1)); } return 1; } #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] write_line("TURKOS/IRQ-WRONG-MASK: FAIL"); return 0; } irq::IarClass::Special(intid) => { // Special IAR values did not acknowledge an interrupt: no EOI. unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; write_str("TURKOS/IRQ-SPECIAL-NO-EOI: 0x"); write_hex(intid as usize); write_str("\r\n"); return 0; } irq::IarClass::Expected => {} } if !valid { // SAFETY: disabling the timer prevents a repeated level signal. unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; write_str("TURKOS/IRQ-TIMER-ERROR: context/intid=0x"); write_hex(iar & 0x00ff_ffff); write_str(" frame=0x"); write_hex(frame_address); write_str(" stack=0x"); write_hex(current_sp); write_str(" guard=0x"); write_hex(unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) as usize }); write_str(" old=0x"); write_hex(unsafe { (*frame).interrupted_sp as usize }); write_str(" spsr=0x"); write_hex(unsafe { (*frame).spsr as usize }); write_str(" x0=0x"); write_hex(unsafe { (*frame).x[0] as usize }); write_str(" x3=0x"); write_hex(unsafe { (*frame).x[3] as usize }); write_str(" x16=0x"); write_hex(unsafe { (*frame).x[16] as usize }); write_str(" x17=0x"); write_hex(unsafe { (*frame).x[17] as usize }); write_str(" x30=0x"); write_hex(unsafe { (*frame).x[30] as usize }); write_str(" q0=0x"); write_hex(unsafe { (*frame).q[0][0] as usize }); write_str(" q31=0x"); write_hex(unsafe { (*frame).q[31][0] as usize }); write_str("\r\n"); return 0; } // SAFETY: one PE owns this isolated counter and IRQ is masked on entry. let previous = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_SEEN)) }; if previous >= 100 { unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; write_line("TURKOS/IRQ-TIMER-ERROR: extra tick"); return 0; } let count = previous + 1; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_SEEN), count) }; #[cfg(feature = "irq-timer-storm-test")] { let progress = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_MAIN_PROGRESS)) }; let last = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_LAST_PROGRESS)) }; if count > 1 && progress > last { let samples = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_INTERLEAVED)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_STORM_INTERLEAVED), samples + 1) }; } unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_STORM_LAST_PROGRESS), progress) }; } #[cfg(feature = "irq-timer-late-test")] if count == 50 { // Deliberately hold IRQ 50 beyond three 10 ms periods while IRQ // remains masked on entry. The normal timer image never does this. let period = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_PERIOD)) } as u64; let deadline = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_DEADLINE)) }; if let Some(target) = period .checked_mul(3) .and_then(|delay| deadline.checked_add(delay)) { loop { let now: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) now, options(nomem, nostack, preserves_flags)) }; if now >= target { break; } core::hint::spin_loop(); } } } // Read the counter before UART output. Advance the absolute compare // value before EOI so a late handler cannot immediately storm. let counter: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) counter, options(nomem, nostack, preserves_flags)) }; let mut deadline_ok = true; let mut skipped = 0; // Clear the level source before EOI. The hundredth tick stops it. if count < 100 { let period = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_PERIOD)) } as u64; let previous_deadline = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_DEADLINE)) }; if let Some((next, missed)) = irq::next_timer_deadline(previous_deadline, period, counter) { skipped = missed; #[cfg(feature = "irq-timer-storm-test")] { let before = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_SKIPPED)) }; unsafe { write_volatile( core::ptr::addr_of_mut!(IRQ_STORM_SKIPPED), before.saturating_add(missed as usize), ) }; } unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_DEADLINE), next); asm!("msr CNTP_CVAL_EL0, {value}", "isb", value = in(reg) next, options(nomem, nostack)); } } else { deadline_ok = false; unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; } } else { unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; } // SAFETY: the acknowledged PPI is valid; EOImode 1 also needs DIR. unsafe { asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: usize; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } if !deadline_ok { write_line("TURKOS/IRQ-TIMER-ERROR: deadline overflow"); return 0; } // K458: serial diagnostics can take longer than a 100 us period. // Keep the timer level deasserted while printing, then coalesce any // elapsed deadlines just before ERET so main gets a real time slice. #[cfg(feature = "irq-timer-storm-test")] if count < 100 { unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)) }; } write_str("TURKOS/IRQ-TIMER-IAR: 0x"); write_hex(iar); write_str("\r\n"); write_str("TURKOS/IRQ-TIMER-EOI: 0x"); write_hex(iar); write_str("\r\n"); if skipped != 0 { write_str("TURKOS/IRQ-TIMER-LATE: 0x"); write_hex(skipped as usize); write_str("\r\n"); } write_str("TURKOS/IRQ-TIMER-COUNTER: 0x"); write_hex(counter as usize); write_str("\r\nTURKOS/IRQ-TIMER-TICK: 0x"); write_hex(count); write_str("\r\n"); #[cfg(feature = "irq-timer-storm-test")] if count < 100 { let now_after_log: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) now_after_log, options(nomem, nostack, preserves_flags)) }; let previous_deadline = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_DEADLINE)) }; let period = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_PERIOD)) } as u64; let Some((next, missed)) = irq::next_timer_deadline(previous_deadline, period, now_after_log) else { write_line("TURKOS/IRQ-TIMER-ERROR: storm deadline overflow"); return 0; }; let before = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_SKIPPED)) }; unsafe { write_volatile( core::ptr::addr_of_mut!(IRQ_STORM_SKIPPED), before.saturating_add(missed as usize), ); write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_DEADLINE), next); asm!("msr CNTP_CVAL_EL0, {value}", "msr CNTP_CTL_EL0, {enable}", "isb", value = in(reg) next, enable = in(reg) 1usize, options(nomem, nostack)); } } return 1; } } #[cfg(feature = "irq-return-test")] fn irq_return_smoke_test() { // SAFETY: only the dedicated image writes this private guard before SVC. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; let x0: usize; let x3: usize; let x16: usize; let x17: usize; let x30: usize; let q0_low: usize; let q31_low: usize; // SAFETY: this image alone routes EL1h synchronous exceptions to the // returning vector. The handler restores every GP/SIMD register and ERET // resumes at the instruction after this deliberate SVC. unsafe { asm!( "movi v0.16b, #0x5a", "movi v31.16b, #0xa5", "svc #0x42", "umov {q0_low}, v0.d[0]", "umov {q31_low}, v31.d[0]", q0_low = lateout(reg) q0_low, q31_low = lateout(reg) q31_low, inout("x0") 0x5455_524b_4952_5130usize => x0, inout("x3") 0x5455_524b_4952_5133usize => x3, inout("x16") 0x5455_524b_4952_5116usize => x16, inout("x17") 0x5455_524b_4952_5117usize => x17, inout("x30") 0x5455_524b_4952_514cusize => x30, out("v0") _, out("v31") _, ); } let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) }; if x0 == 0x5455_524b_4952_5130 && x3 == 0x5455_524b_4952_5133 && x16 == 0x5455_524b_4952_5116 && x17 == 0x5455_524b_4952_5117 && x30 == 0x5455_524b_4952_514c && q0_low == 0x5a5a_5a5a_5a5a_5a5a && q31_low == 0xa5a5_a5a5_a5a5_a5a5 && guard == IRQ_GUARD { write_line("TURKOS/SVC-RETURN: PASS"); } else { write_line("TURKOS/SVC-RETURN-ERROR: restored registers"); } } #[cfg(any( feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-copy-irq-test", feature = "user-el0-irq-test" ))] fn prepare_test_interrupt(boot: &BootData, intid: usize) -> bool { if intid != 1 && intid != 2 && intid != 30 { return false; } let gicd = boot.interrupts.gic_distributor; let gicr = boot.interrupts.gic_redistributor; if gicd == 0 || gicr == 0 || gicd & 0xffff != 0 || gicr & 0xffff != 0 { return false; } // SAFETY: DTB discovery established the distributor/redistributor sizes. // This isolated single-CPU experiment configures one SGI or PPI before // translation is enabled, so these physical MMIO addresses are reachable. unsafe { let register = |base: u64, offset: usize| (base as usize + offset) as *mut u32; let distributor_id = read_volatile(register(gicd, 0xffe8)); let redistributor_id = read_volatile(register(gicr, 0xffe8)); let redistributor_type = read_volatile((gicr as usize + 0x8) as *const u64); let mpidr: u64; asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)); write_str("TURKOS/GICD-PIDR2: 0x"); write_hex(distributor_id as usize); write_str("\r\nTURKOS/GICR-PIDR2: 0x"); write_hex(redistributor_id as usize); write_str("\r\nTURKOS/GICR-TYPER: 0x"); write_hex(redistributor_type as usize); write_str("\r\nTURKOS/GIC-MPIDR: 0x"); write_hex(mpidr as usize); write_str("\r\n"); if irq::gic_arch_revision(distributor_id).is_none() || irq::gic_arch_revision(redistributor_id).is_none() || !irq::redistributor_matches_mpidr(redistributor_type, mpidr) { write_line("TURKOS/GIC-IDENTITY: FAIL"); return false; } write_line("TURKOS/GIC-IDENTITY: PASS"); let waker = register(gicr, 0x14); write_volatile(waker, read_volatile(waker) & !2); asm!("dsb sy", options(nostack, preserves_flags)); let mut awake = false; for _ in 0..1_000_000 { if read_volatile(waker) & 4 == 0 { awake = true; break; } } if !awake { return false; } let distributor = register(gicd, 0); write_volatile(distributor, read_volatile(distributor) | 0x22); // ARE_NS, Group 1 NS let mut ready = false; for _ in 0..1_000_000 { if read_volatile(distributor) & (1 << 31) == 0 { ready = true; break; } } if !ready { return false; } let group = register(gicr, 0x1_0080); let bit = 1u32 << intid; write_volatile(group, read_volatile(group) | bit); write_volatile((gicr as usize + 0x1_0400 + intid) as *mut u8, 0x80); write_volatile(register(gicr, 0x1_0100), bit); // GICR_ISENABLER0 asm!("dsb sy", options(nostack, preserves_flags)); let mut redistributor_ready = false; for _ in 0..1_000_000 { if read_volatile(register(gicr, 0)) & 8 == 0 { redistributor_ready = true; break; } } if !redistributor_ready { return false; } let sre: usize; asm!("mrs {value}, ICC_SRE_EL1", value = out(reg) sre, options(nomem, nostack, preserves_flags)); asm!("msr ICC_SRE_EL1, {value}", "isb", value = in(reg) (sre | 1), options(nomem, nostack)); let sre_after: usize; asm!("mrs {value}, ICC_SRE_EL1", value = out(reg) sre_after, options(nomem, nostack, preserves_flags)); if sre_after & 1 == 0 { return false; } asm!( "msr ICC_PMR_EL1, {priority}", "msr ICC_BPR1_EL1, xzr", "msr ICC_IGRPEN1_EL1, {enable}", "isb", priority = in(reg) 0xffusize, enable = in(reg) 1usize, options(nomem, nostack) ); } true } #[cfg(feature = "irq-sgi-test")] fn irq_sgi_smoke_test() { // SAFETY: only this test image owns the guard and unmasks IRQ inside the // bounded inline-assembly interval below. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; unsafe { write_volatile(core::ptr::addr_of_mut!(__stack_start), MAIN_STACK_GUARD) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_SGI_SEEN), 0) }; let mpidr: u64; // SAFETY: MPIDR_EL1 identifies the only running PE in this test image. unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)) }; let Some(sgi_command) = irq::sgi1r_for_mpidr(mpidr, 1) else { write_line("TURKOS/IRQ-SGI-ERROR: target affinity"); return; }; write_line("TURKOS/IRQ-SGI: ARMED"); if cfg!(feature = "irq-gp-test") { irq_gp_smoke_test(sgi_command); return; } let x0: usize; let x3: usize; let x16: usize; let x17: usize; let x30: usize; let q0_low: usize; let q31_low: usize; let observed: usize; let returned_nzcv: usize; // SAFETY: GICv3 SGI 1 has been configured for this CPU. The IRQ vector // acknowledges/EOIs it, sets the flag, restores context and ERETs. The // timeout always remasks IRQ before returning to Rust. unsafe { asm!( "movi v0.16b, #0x5a", "movi v31.16b, #0xa5", "msr nzcv, {expected_nzcv}", "msr daifclr, #2", "msr ICC_SGI1R_EL1, {sgi}", "isb", "2:", "ldr {observed}, [{flag}]", "cbnz {observed}, 3f", "sub {budget}, {budget}, #1", "cbnz {budget}, 2b", "3:", "msr daifset, #2", "mrs {returned_nzcv}, nzcv", "umov {q0_low}, v0.d[0]", "umov {q31_low}, v31.d[0]", expected_nzcv = in(reg) irq::IRQ_PROBE_NZCV as usize, sgi = in(reg) sgi_command, flag = in(reg) core::ptr::addr_of!(IRQ_SGI_SEEN) as usize, budget = inout(reg) 50_000_000usize => _, observed = out(reg) observed, returned_nzcv = lateout(reg) returned_nzcv, q0_low = lateout(reg) q0_low, q31_low = lateout(reg) q31_low, inout("x0") 0x5455_524b_4952_5130usize => x0, inout("x3") 0x5455_524b_4952_5133usize => x3, inout("x16") 0x5455_524b_4952_5116usize => x16, inout("x17") 0x5455_524b_4952_5117usize => x17, inout("x30") 0x5455_524b_4952_514cusize => x30, out("v0") _, out("v31") _, ); } let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) }; let main_guard = unsafe { read_volatile(core::ptr::addr_of!(__stack_start)) }; let count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_SGI_SEEN)) }; if observed == 1 && count == 1 && x0 == 0x5455_524b_4952_5130 && x3 == 0x5455_524b_4952_5133 && x16 == 0x5455_524b_4952_5116 && x17 == 0x5455_524b_4952_5117 && x30 == 0x5455_524b_4952_514c && q0_low == 0x5a5a_5a5a_5a5a_5a5a && q31_low == 0xa5a5_a5a5_a5a5_a5a5 && irq::nzcv_matches(returned_nzcv as u64, irq::IRQ_PROBE_NZCV) && guard == IRQ_GUARD && main_guard == MAIN_STACK_GUARD { write_line("TURKOS/IRQ-SGI-NZCV: PASS"); write_str("TURKOS/IRQ-MAIN-STACK-GUARD: 0x"); write_hex(main_guard as usize); write_str("\r\n"); write_line("TURKOS/IRQ-MAIN-STACK: PASS"); write_str("TURKOS/IRQ-SGI-COUNT: 0x"); write_hex(count); write_str("\r\n"); write_line("TURKOS/IRQ-SGI: PASS"); } else { write_line("TURKOS/IRQ-SGI-ERROR: timeout or restored registers"); } } #[cfg(feature = "irq-sgi-test")] fn irq_gp_smoke_test(sgi_command: u64) { #[cfg(feature = "irq-gp-test")] { let caller_sp_el0: usize; unsafe { asm!("mrs {value}, SP_EL0", value = out(reg) caller_sp_el0, options(nomem, nostack, preserves_flags)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(GP_IRQ_WAKE), 0) }; unsafe { write_volatile(core::ptr::addr_of_mut!(GP_PROBE_ACTIVE), 1) }; // SAFETY: the isolated assembly routine saves its caller's callee- // saved registers, owns IRQ unmasking, and copies all 31 return values // before restoring the Rust ABI state. unsafe { __gp_probe_run( sgi_command as usize, core::ptr::addr_of_mut!(GP_RETURNED).cast(), ) }; unsafe { write_volatile(core::ptr::addr_of_mut!(GP_PROBE_ACTIVE), 0) }; let returned = unsafe { read_volatile(core::ptr::addr_of!(GP_RETURNED)) }; let returned_sp_el0 = unsafe { read_volatile(core::ptr::addr_of!(GP_RETURNED_SP_EL0)) }; let after_sp_el0: usize; unsafe { asm!("mrs {value}, SP_EL0", value = out(reg) after_sp_el0, options(nomem, nostack, preserves_flags)) }; let mask = gp_register_mask(&returned); let count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_SGI_SEEN)) }; write_str("TURKOS/IRQ-GP-RESTORED-MASK: 0x"); write_hex(mask as usize); write_str("\r\nTURKOS/IRQ-GP-RESTORED-SP_EL0: 0x"); write_hex(returned_sp_el0 as usize); write_str("\r\nTURKOS/IRQ-GP-CALLER-SP_EL0: "); write_str(if after_sp_el0 == caller_sp_el0 { "PASS" } else { "FAIL" }); write_str("\r\nTURKOS/IRQ-GP-COUNT: 0x"); write_hex(count); write_str("\r\n"); if count == 1 && mask == 0x7fff_ffff && returned_sp_el0 == 0x7a5a && after_sp_el0 == caller_sp_el0 { write_line("TURKOS/IRQ-GP: PASS"); } else { write_line("TURKOS/IRQ-GP-ERROR: restored registers"); } } #[cfg(not(feature = "irq-gp-test"))] { let _ = sgi_command; } } #[cfg(feature = "irq-timer-test")] #[cfg_attr(feature = "irq-service-test", allow(dead_code))] fn irq_timer_smoke_test(gicr: u64) { // SAFETY: the dedicated image alone enables PPI 30 and uses this stack. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_SEEN), 0) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_MAIN_PROGRESS), 0) }; let frequency: usize; unsafe { asm!("mrs {value}, CNTFRQ_EL0", value = out(reg) frequency, options(nomem, nostack, preserves_flags)) }; let period_ns = if cfg!(feature = "irq-timer-storm-test") { 100_000 } else { 10_000_000 }; let Some(period_ticks) = irq::timer_ticks_for_ns(frequency as u64, period_ns) else { write_line("TURKOS/IRQ-TIMER-ERROR: frequency"); return; }; let period = period_ticks as usize; // The supported guest is AArch64. write_str("TURKOS/IRQ-TIMER-FREQ: 0x"); write_hex(frequency); write_str("\r\nTURKOS/IRQ-TIMER: ARMED\r\n"); #[cfg(feature = "irq-timer-storm-test")] { write_str("TURKOS/IRQ-STORM-PERIOD-NS: 0x"); write_hex(period_ns as usize); write_str("\r\n"); } let now: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) now, options(nomem, nostack, preserves_flags)) }; let Some((first_deadline, _)) = irq::next_timer_deadline(now, period_ticks, now) else { write_line("TURKOS/IRQ-TIMER-ERROR: first deadline overflow"); return; }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_PERIOD), period) }; unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_TIMER_DEADLINE), first_deadline) }; let (x0, x3, x16, x17, x30, q0_low, q31_low, observed, progress): ( usize, usize, usize, usize, usize, usize, usize, usize, usize, ); // SAFETY: the handler advances absolute deadlines, then disables the timer. The bounded // loop always remasks IRQ and disables the timer before Rust resumes. unsafe { asm!( "movi v0.16b, #0x5a", "movi v31.16b, #0xa5", "msr CNTP_CVAL_EL0, {deadline}", "msr CNTP_CTL_EL0, {enable}", "isb", "msr daifclr, #2", "2:", "add {progress}, {progress}, #1", "str {progress}, [{main_flag}]", "ldr {observed}, [{flag}]", "cmp {observed}, #100", "b.ge 3f", "subs {budget}, {budget}, #1", "b.ne 2b", "3:", "msr daifset, #2", "msr CNTP_CTL_EL0, xzr", "isb", "umov {q0_low}, v0.d[0]", "umov {q31_low}, v31.d[0]", deadline = in(reg) first_deadline, enable = in(reg) 1usize, flag = in(reg) core::ptr::addr_of!(IRQ_TIMER_SEEN) as usize, main_flag = in(reg) core::ptr::addr_of_mut!(IRQ_TIMER_MAIN_PROGRESS) as usize, budget = inout(reg) 10_000_000_000usize => _, progress = inout(reg) 0usize => progress, observed = out(reg) observed, q0_low = lateout(reg) q0_low, q31_low = lateout(reg) q31_low, inout("x0") 0x5455_524b_4952_5130usize => x0, inout("x3") 0x5455_524b_4952_5133usize => x3, inout("x16") 0x5455_524b_4952_5116usize => x16, inout("x17") 0x5455_524b_4952_5117usize => x17, inout("x30") 0x5455_524b_4952_514cusize => x30, out("v0") _, out("v31") _, ); } let count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_TIMER_SEEN)) }; let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) }; let ppi_enabled_before: bool; let mut ppi_disabled_after = false; // SAFETY: only the timer image maps the validated 128 KiB GICR Device // window. IRQ is masked and CNTP_CTL_EL0 is already clear here. unsafe { let isenabler = (gicr as usize + 0x1_0100) as *const u32; let icenabler = (gicr as usize + 0x1_0180) as *mut u32; let control = gicr as *const u32; ppi_enabled_before = read_volatile(isenabler) & (1 << 30) != 0; write_volatile(icenabler, 1 << 30); asm!("dsb sy", options(nostack, preserves_flags)); for _ in 0..1_000_000 { if read_volatile(control) & 8 == 0 { ppi_disabled_after = read_volatile(isenabler) & (1 << 30) == 0; break; } } } write_str("TURKOS/IRQ-TIMER-PPI-ENABLED-BEFORE: 0x"); write_hex(ppi_enabled_before as usize); write_str("\r\n"); if ppi_enabled_before && ppi_disabled_after { write_line("TURKOS/IRQ-TIMER-PPI-MASK: PASS"); } else { write_line("TURKOS/IRQ-TIMER-PPI-MASK: FAIL"); } if observed == 100 && count == 100 && ppi_enabled_before && ppi_disabled_after && progress > 100 && x0 == 0x5455_524b_4952_5130 && x3 == 0x5455_524b_4952_5133 && x16 == 0x5455_524b_4952_5116 && x17 == 0x5455_524b_4952_5117 && x30 == 0x5455_524b_4952_514c && q0_low == 0x5a5a_5a5a_5a5a_5a5a && q31_low == 0xa5a5_a5a5_a5a5_a5a5 && guard == IRQ_GUARD { write_str("TURKOS/IRQ-TIMER-FIRST-DEADLINE: 0x"); write_hex(first_deadline as usize); write_str("\r\n"); write_str("TURKOS/IRQ-TIMER-COUNT: 0x"); write_hex(count); write_str("\r\nTURKOS/IRQ-TIMER-PROGRESS: 0x"); write_hex(progress); write_str("\r\n"); write_line("TURKOS/IRQ-TIMER: PASS"); #[cfg(feature = "irq-timer-storm-test")] { let interleaved = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_INTERLEAVED)) }; let skipped = unsafe { read_volatile(core::ptr::addr_of!(IRQ_STORM_SKIPPED)) }; write_str("TURKOS/IRQ-STORM-INTERLEAVED: 0x"); write_hex(interleaved); write_str("\r\nTURKOS/IRQ-STORM-SKIPPED: 0x"); write_hex(skipped); write_str("\r\n"); write_line( if irq::storm_main_progress_ok(interleaved, count, progress) { "TURKOS/IRQ-STORM: PASS" } else { "TURKOS/IRQ-STORM: FAIL" }, ); } } else { write_line("TURKOS/IRQ-TIMER-ERROR: timeout or restored registers"); } } #[cfg(all(feature = "irq-unknown-test", not(feature = "irq-spurious-test")))] fn irq_unknown_smoke_test(gicr: u64) -> ! { // SAFETY: only this negative test image enables SGI 2 alongside PPI 30. unsafe { write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD) }; let mpidr: u64; unsafe { asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags)) }; let Some(command) = irq::sgi1r_for_mpidr(mpidr, 2) else { write_line("TURKOS/IRQ-UNKNOWN-ERROR: target affinity"); park(); }; let enabled_before = unsafe { read_volatile((gicr as usize + 0x1_0100) as *const u32) & 4 != 0 }; write_str("TURKOS/IRQ-WRONG-MASK-ENABLED-BEFORE: 0x"); write_hex(enabled_before as usize); write_str("\r\n"); if !enabled_before { write_line("TURKOS/IRQ-WRONG-MASK: FAIL"); park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(IRQ_UNKNOWN_GICR), gicr as usize) }; write_line("TURKOS/IRQ-UNKNOWN: ARMED"); // The first SGI reaches the real vector; the second is deliberately sent // while interrupts are still unmasked to prove the source stays disabled. unsafe { asm!("msr daifclr, #2", "msr ICC_SGI1R_EL1, {command}", "isb", command = in(reg) command, options(nostack)); } let mut progress = 0usize; for _ in 0..10_000_000 { progress += 1; if unsafe { read_volatile(core::ptr::addr_of!(IRQ_UNKNOWN_HANDLED)) } != 0 { break; } core::hint::spin_loop(); } let first_count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_UNKNOWN_HANDLED)) }; let masked_after = unsafe { read_volatile((gicr as usize + 0x1_0100) as *const u32) & 4 == 0 }; if first_count == 1 && masked_after { unsafe { asm!("msr ICC_SGI1R_EL1, {command}", "isb", command = in(reg) command, options(nostack)) }; write_line("TURKOS/IRQ-WRONG-MASK-SECOND-SGI: SENT"); for _ in 0..500_000 { progress += 1; core::hint::spin_loop(); } } unsafe { asm!("msr daifset, #2", "isb", options(nostack)) }; let final_count = unsafe { read_volatile(core::ptr::addr_of!(IRQ_UNKNOWN_HANDLED)) }; let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) }; write_str("TURKOS/IRQ-WRONG-MASK-MASKED-AFTER: 0x"); write_hex(masked_after as usize); write_str("\r\nTURKOS/IRQ-WRONG-MASK-COUNT: 0x"); write_hex(final_count); write_str("\r\nTURKOS/IRQ-WRONG-MASK-PROGRESS: 0x"); write_hex(progress); write_str("\r\n"); if first_count == 1 && final_count == 1 && masked_after && progress > 500_000 && guard == IRQ_GUARD { write_line("TURKOS/IRQ-WRONG-MASK: PASS"); } else { write_line("TURKOS/IRQ-WRONG-MASK: FAIL"); } park() } #[cfg(feature = "irq-spurious-test")] fn irq_spurious_smoke_test() -> ! { // The GIC CPU interface is enabled but no SGI was sent and the physical // timer remains disabled. Polling the IAR is permitted at EL1; 1023 did // not acknowledge an interrupt, so neither read is followed by an EOI. let first: u64; let second: u64; unsafe { asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) first, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) second, options(nomem, nostack, preserves_flags)); } write_str("TURKOS/IRQ-SPURIOUS-FIRST: 0x"); write_hex(first as usize); write_str("\r\nTURKOS/IRQ-SPURIOUS-SECOND: 0x"); write_hex(second as usize); write_str("\r\n"); if irq::classify_iar(first, 30) == irq::IarClass::Special(1023) && irq::classify_iar(second, 30) == irq::IarClass::Special(1023) { write_line("TURKOS/IRQ-SPURIOUS-NO-EOI: PASS"); } else { write_line("TURKOS/IRQ-SPURIOUS-ERROR: unexpected IAR"); } park() } #[cfg(feature = "mmu-tlb-test")] fn mmu_tlb_probe(allocator: &mut memory::FrameAllocator, tables: &mmu::Tables) -> ! { const A: u64 = 0x5455_524b_544c_4241; const B: u64 = 0x5455_524b_544c_4242; let before_pages = allocator.remaining_pages(); let Some(first) = allocator.allocate() else { write_line("TURKOS/TLB-ERROR: first frame unavailable"); park(); }; let Some(second) = allocator.allocate() else { write_line("TURKOS/TLB-ERROR: second frame unavailable"); park(); }; if first == second { write_line("TURKOS/TLB-ERROR: duplicate frames"); park(); } // SAFETY: both pages are exclusively allocated, writable and initially // identity-mapped. The first words are restored before ownership returns. let (saved_first, saved_second) = unsafe { let old_first = read_volatile(first as *const u64); let old_second = read_volatile(second as *const u64); write_volatile(first as *mut u64, A); write_volatile(second as *mut u64, B); (old_first, old_second) }; // SAFETY: the allocated pages are data frames. The table helper rejects // any frame outside the first 2 MiB or a descriptor not matching A. let Ok((slot, second_descriptor)) = (unsafe { tables.probe_replacement(first, first, second) }) else { write_line("TURKOS/TLB-ERROR: unsafe first replacement"); park(); }; write_str("TURKOS/TLB-VA: 0x"); write_hex(first as usize); write_str("\r\nTURKOS/TLB-PA-B: 0x"); write_hex(second as usize); write_str("\r\nTURKOS/TLB-BEFORE: 0x"); let initial = unsafe { read_volatile(first as *const u64) }; write_hex(initial as usize); write_str("\r\n"); if initial != A || unsafe { read_volatile(second as *const u64) } != B { write_line("TURKOS/TLB-ERROR: initial physical canaries"); park(); } // SAFETY: the assembly performs break-before-make with a local address // operand and inner-shareable TLBI; all pages and the L3 slot remain owned. unsafe { __turkos_replace_l3_probe(slot, first >> 12, second_descriptor) }; let after = unsafe { read_volatile(first as *const u64) }; write_str("TURKOS/TLB-AFTER: 0x"); write_hex(after as usize); write_str("\r\n"); if after != B || unsafe { read_volatile(slot) } != second_descriptor { write_line("TURKOS/TLB-ERROR: old translation remained"); park(); } let Ok((slot, first_descriptor)) = (unsafe { tables.probe_replacement(first, second, first) }) else { write_line("TURKOS/TLB-ERROR: unsafe restore replacement"); park(); }; unsafe { __turkos_replace_l3_probe(slot, first >> 12, first_descriptor) }; let restored = unsafe { read_volatile(first as *const u64) }; write_str("TURKOS/TLB-RESTORED: 0x"); write_hex(restored as usize); write_str("\r\n"); if restored != A || unsafe { read_volatile(slot) } != first_descriptor { write_line("TURKOS/TLB-ERROR: identity restore failed"); park(); } unsafe { write_volatile(first as *mut u64, saved_first); write_volatile(second as *mut u64, saved_second); } if allocator.release(second).is_err() || allocator.release(first).is_err() { write_line("TURKOS/TLB-ERROR: frame release"); park(); } let after_pages = allocator.remaining_pages(); write_str("TURKOS/TLB-FRAME-ACCOUNT: BEFORE=0x"); write_hex(before_pages as usize); write_str(" AFTER=0x"); write_hex(after_pages as usize); write_str("\r\n"); if before_pages != after_pages { write_line("TURKOS/TLB-ERROR: frame ownership drift"); park(); } write_line("TURKOS/TLB-BBM: PASS"); park() } #[cfg(feature = "heap-test")] fn write_heap_allocation(kind: &str, handle: heap::Handle, size: usize, align: usize) { write_str("TURKOS/HEAP-ALLOC: KIND="); write_str(kind); write_str(" ADDR=0x"); write_hex(handle.address); write_str(" SIZE=0x"); write_hex(size); write_str(" ALIGN=0x"); write_hex(align); write_str("\r\n"); } #[cfg(feature = "heap-test")] fn heap_object_smoke_test() -> bool { // SAFETY: HeapImage alone owns the four mapped writable pages; this // experiment is single-CPU and never hands pointers to another context. // Metadata lives in BSS to keep the 16 KiB early boot stack bounded. static mut HEAP_STORAGE: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); let storage = core::ptr::addr_of_mut!(HEAP_STORAGE).cast::(); let Ok(()) = (unsafe { heap::Heap::init_in_place(storage, mmu::HEAP_WINDOW_START as usize) }) else { return false; }; let heap = unsafe { &mut *storage }; let Ok(small) = heap.allocate(16, 16) else { return false; }; let Ok(aligned) = heap.allocate(32, 64) else { return false; }; let Ok(page) = heap.allocate(4096, 4096) else { return false; }; let Ok(crossing) = heap.allocate(5000, 8) else { return false; }; if small.address % 16 != 0 || aligned.address % 64 != 0 || page.address % 4096 != 0 || crossing.address / 4096 == (crossing.address + 4999) / 4096 { return false; } for (kind, handle, size, align) in [ ("SMALL16", small, 16, 16), ("ALIGN64", aligned, 32, 64), ("PAGE", page, 4096, 4096), ("CROSS", crossing, 5000, 8), ] { // SAFETY: first and last requested bytes belong to this allocation. unsafe { write_volatile(handle.address as *mut u8, 0x5a); write_volatile((handle.address + size - 1) as *mut u8, 0xa5); if read_volatile(handle.address as *const u8) != 0x5a || read_volatile((handle.address + size - 1) as *const u8) != 0xa5 { return false; } } write_heap_allocation(kind, handle, size, align); } if heap.allocate(0, 8) != Err(heap::Error::Invalid) { return false; } write_line("TURKOS/HEAP-ZERO: PASS"); if heap.allocate(usize::MAX, 8) != Err(heap::Error::Overflow) || heap.allocate(1, 3) != Err(heap::Error::Invalid) { return false; } write_line("TURKOS/HEAP-OVERFLOW: PASS"); if heap.allocate(heap::CAPACITY - 8, 1) != Err(heap::Error::OutOfMemory) { return false; } write_line("TURKOS/HEAP-OOM: PASS"); let trailer = small.address + 16; // SAFETY: the eight-byte trailer was written by Heap::allocate; altering // one bit must make release fail without changing ownership. let previous = unsafe { core::ptr::read_unaligned(trailer as *const u64) }; unsafe { core::ptr::write_unaligned(trailer as *mut u64, previous ^ 1) }; if heap.release(small) != Err(heap::Error::CorruptedCanary) || heap.active_count() != 4 { return false; } unsafe { core::ptr::write_unaligned(trailer as *mut u64, previous) }; write_line("TURKOS/HEAP-CANARY: PASS"); if heap.release(heap::Handle { address: small.address - 8, ..small }) != Err(heap::Error::Invalid) { return false; } write_line("TURKOS/HEAP-FOREIGN: PASS"); for handle in [crossing, page, aligned, small] { if heap.release(handle).is_err() { return false; } } write_line("TURKOS/HEAP-REVERSE-FREE: PASS"); if heap.release(small) != Err(heap::Error::DoubleFree) { return false; } write_line("TURKOS/HEAP-DOUBLE-FREE: PASS"); if heap.active_count() != 0 || heap.free_bytes() != heap::CAPACITY || heap.largest_free() != heap::CAPACITY { return false; } write_line("TURKOS/HEAP-COALESCE: PASS"); let Ok(full) = heap.allocate(heap::CAPACITY - 8, 1) else { return false; }; if heap.allocate(1, 1) != Err(heap::Error::OutOfMemory) || heap.release(full).is_err() { return false; } write_line("TURKOS/HEAP-OOM-RECOVERY: PASS"); let mut small_handles: [Option; 32] = [None; 32]; for index in 0..small_handles.len() { let Ok(handle) = heap.allocate(32, 16) else { return false; }; if small_handles[..index] .iter() .flatten() .any(|prior| prior.address == handle.address) { return false; } small_handles[index] = Some(handle); } for handle in small_handles.into_iter().rev().flatten() { if heap.release(handle).is_err() { return false; } } write_line("TURKOS/HEAP-SMALL-MANY: PASS"); write_str("TURKOS/HEAP-ACCOUNT: USED=0x"); write_hex(heap.used_bytes()); write_str(" FREE=0x"); write_hex(heap.free_bytes()); write_str(" LARGEST=0x"); write_hex(heap.largest_free()); write_str(" ACTIVE=0x"); write_hex(heap.active_count()); write_str("\r\n"); heap.used_bytes() == 0 && heap.free_bytes() == heap::CAPACITY && heap.largest_free() == heap::CAPACITY && heap.active_count() == 0 } #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] fn mmu_smoke_test(free: memory::FreeMap, boot: &BootData, level: usize) -> ! { if level != 1 { write_line("TURKOS/MMU-ERROR: EL1 required"); park(); } let mut os_runtime = match os_k660_new(free, boot.hardware.uart_base) { Ok(value) => value, Err(reason) => { write_str("TURKOS/OS/K660: REJECTED reason="); write_line(reason); park(); } }; let allocator = &mut os_runtime.memory_mut().expect("runtime allocator").allocator; let hardware = &boot.hardware; let layout = mmu::Layout { ram_base: hardware.ram_base, ram_size: hardware.ram_size, uart_base: hardware.uart_base, image_start: core::ptr::addr_of!(_image_start) as u64, text_start: core::ptr::addr_of!(_text_start) as u64, text_end: core::ptr::addr_of!(_text_end) as u64, rodata_end: core::ptr::addr_of!(_rodata_end) as u64, image_end: core::ptr::addr_of!(_image_end) as u64, irq_guard_page: Some(core::ptr::addr_of!(__irq_guard_page_start) as u64), heap_start: if cfg!(feature = "heap-test") { Some(mmu::HEAP_WINDOW_START) } else { None }, }; #[cfg(any( feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-copy-irq-test", feature = "user-el0-irq-test" ))] { #[cfg(feature = "irq-sgi-test")] let intid = 1; #[cfg(feature = "user-copy-irq-test")] let intid = 1; #[cfg(feature = "user-el0-irq-test")] let intid = boot.interrupts.nonsecure_phys_timer_ppi as usize; #[cfg(feature = "irq-timer-test")] let intid = boot.interrupts.nonsecure_phys_timer_ppi as usize; #[cfg(feature = "irq-unknown-test")] let intid = 2; #[cfg(any(feature = "task-preempt-test", feature = "task-sleep-test"))] let intid = boot.interrupts.nonsecure_phys_timer_ppi as usize; if !prepare_test_interrupt(boot, intid) { write_line("TURKOS/IRQ-ERROR: GIC setup"); park(); } #[cfg(feature = "user-el0-irq-test")] user_syscall::set_el0_irq_intid(intid); #[cfg(feature = "irq-unknown-test")] if !prepare_test_interrupt(boot, 30) { write_line("TURKOS/IRQ-ERROR: PPI setup"); park(); } } // SAFETY: FreeMap excludes the kernel, DTB, and reserved memory. This // dedicated guest keeps every table frame allocated until it is stopped. let free_before_tables = allocator.remaining_pages(); write_str("TURKOS/MMU-LINKER: TEXT-START=0x"); write_hex(layout.text_start as usize); write_str(" TEXT-END=0x"); write_hex(layout.text_end as usize); write_str(" RODATA-END=0x"); write_hex(layout.rodata_end as usize); write_str("\r\n"); #[allow(unused_mut)] let mut tables = match unsafe { mmu::build_identity_tables(allocator, layout, boot.static_reserved.as_slice()) } { Ok(tables) => tables, Err(_) => { let sctlr_after_failure: u64; // SAFETY: SCTLR_EL1 is readable at EL1 even when table planning fails. unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr_after_failure, options(nomem, nostack, preserves_flags)) }; if sctlr_after_failure & 1 == 0 && allocator.remaining_pages() == free_before_tables { write_line("TURKOS/MMU-BUILD-FAIL-PRE-TTBR: PASS"); } write_line("TURKOS/MMU-ERROR: table build"); park() } }; #[cfg(any( feature = "irq-timer-test", feature = "irq-unknown-test", feature = "task-preempt-test", feature = "task-sleep-test", feature = "user-mmio-read-test", feature = "user-el0-irq-test" ))] { let gicr = boot.interrupts.gic_redistributor; #[cfg(feature = "user-mmio-read-test")] if gicr != 0x080a_0000 { write_line("TURKOS/USER-MMIO-AUDIT-ERROR: unexpected DTB GICR window"); park(); } // SAFETY: DTB validation established a 128 KiB redistributor window; // this isolated timer image maps it before installing TTBR0. if unsafe { tables.map_device_window(allocator, gicr, 0x2_0000, layout.uart_base) } .is_err() { write_line("TURKOS/MMU-ERROR: GICR window"); park(); } write_str("TURKOS/MMU-GICR-WINDOW: 0x"); write_hex(gicr as usize); write_str("+0x"); write_hex(0x2_0000); write_str("\r\n"); } #[cfg(any( feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-tx-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "user-mmio-read-test" ))] { let window = dtb::virtio_page_window( &boot.virtio_mmio, boot.hardware.ram_base, boot.hardware.ram_size, ) .unwrap_or_else(|_| { write_line("TURKOS/VIRTIO-MAGIC-ERROR: invalid DTB page window"); park() }); #[cfg(feature = "user-mmio-read-test")] if window.start != 0x0a00_0000 || window.end != 0x0a00_4000 { write_line("TURKOS/USER-MMIO-AUDIT-ERROR: unexpected DTB virtio window"); park(); } // SAFETY: the DTB slots passed the RAM-disjoint page-window check; // this isolated image owns the added table until it stops. if unsafe { tables.map_device_window( allocator, window.start, window.end - window.start, layout.uart_base, ) } .is_err() { write_line("TURKOS/VIRTIO-MAGIC-ERROR: MMU Device mapping"); park() } write_str("TURKOS/MMU-VIRTIO-WINDOW: 0x"); write_hex(window.start as usize); write_str("+0x"); write_hex((window.end - window.start) as usize); write_str("\r\n"); } let gic_start = boot.interrupts.gic_distributor.min(boot.interrupts.gic_redistributor); let gic_end = (boot.interrupts.gic_distributor + 0x1_0000).max(boot.interrupts.gic_redistributor + 0x2_0000); if unsafe { tables.map_device_window(allocator, gic_start, gic_end - gic_start, layout.uart_base) }.is_err() { write_line("TURKOS/OS/K660: REJECTED reason=gic-device-map"); park(); } let dma_blob = unsafe { core::slice::from_raw_parts(boot.dtb_address as *const u8, boot.dtb_size) }; let dma_nodes = dtb::discover_virtio_mmio(dma_blob).expect("bounded DTB VirtIO nodes"); let dma_window = dtb::virtio_page_window(&dma_nodes, boot.hardware.ram_base, boot.hardware.ram_size).expect("bounded DTB VirtIO window"); if unsafe { tables.map_device_window(allocator, dma_window.start, dma_window.end-dma_window.start, layout.uart_base) }.is_err() { write_line("TURKOS/OS/DMA: REJECTED reason=dma-mmio-map"); park(); } let pci_mode = if cfg!(feature = "os-runtime-product") { os_k1100_product_probe_mode(boot).ok() } else { None }; if matches!(pci_mode, Some(76) | Some(77) | Some(78) | Some(79) | Some(80) | Some(81) | Some(82) | Some(83) | Some(85) | Some(86) | Some(87)) { let ecam_blob = unsafe { core::slice::from_raw_parts(boot.dtb_address as *const u8, boot.dtb_size) }; let host = dtb::discover_pci_ecam(ecam_blob).unwrap_or_else(|_| { write_line("TURKOS/OS/K1425: REJECTED reason=ecam-dtb"); park(); }); let window = os_pci_discovery::EcamWindow::new( host.base, host.size, host.first_bus, host.last_bus).unwrap_or_else(|_| { write_line("TURKOS/OS/K1425: REJECTED reason=ecam-window"); park(); }); let slot = if pci_mode == Some(83) { 4 } else if matches!(pci_mode, Some(79) | Some(80) | Some(81) | Some(82)) { 3 } else { 1 }; let config = window.address(0, slot, 0, 0).unwrap_or_else(|_| { write_line("TURKOS/OS/K1425: REJECTED reason=ecam-bdf"); park(); }); if unsafe { tables.map_ecam_config_page(allocator, config.physical) }.is_err() { write_line("TURKOS/OS/K1425: REJECTED reason=ecam-map"); park(); } if matches!(pci_mode, Some(85) | Some(86) | Some(87)) { let second = window.address(0, 2, 0, 0).unwrap_or_else(|_| park()); if unsafe { tables.map_ecam_peer_page(config.physical, second.physical) }.is_err() { write_line("TURKOS/OS/K1470: REJECTED reason=second-ecam-map"); park(); } let (start, end) = host.memory_aperture.unwrap_or_else(|| park()); let bar = end.checked_sub(0x4000).unwrap_or_else(|| park()); if bar < start || bar & 0x3fff != 0 || unsafe { tables.map_device_window(allocator, bar, 0x1000, layout.uart_base) }.is_err() || !tables.device_page_matches(bar) { write_line("TURKOS/OS/K1470: REJECTED reason=common-device-map"); park(); } } if pci_mode == Some(77) { let absent = window.address(0, 31, 0, 0).unwrap_or_else(|_| { write_line("TURKOS/OS/K1430: REJECTED reason=absent-bdf"); park(); }); if unsafe { tables.map_ecam_peer_page(config.physical, absent.physical) }.is_err() { write_line("TURKOS/OS/K1430: REJECTED reason=absent-map"); park(); } } if matches!(pci_mode, Some(79) | Some(80) | Some(81) | Some(82)) { let nested = window.address(1, 1, 0, 0).unwrap_or_else(|_| { write_line("TURKOS/OS/K1440: REJECTED reason=nested-bdf"); park(); }); let leaf = window.address(2, 1, 0, 0).unwrap_or_else(|_| { write_line("TURKOS/OS/K1440: REJECTED reason=leaf-bdf"); park(); }); if unsafe { tables.map_ecam_peer_page(config.physical, nested.physical) }.is_err() || unsafe { tables.map_ecam_next_block_page(allocator, config.physical, leaf.physical) }.is_err() { write_line("TURKOS/OS/K1440: REJECTED reason=bridge-maps"); park(); } if pci_mode == Some(82) { let edu = window.address(2, 2, 0, 0).unwrap_or_else(|_| { write_line("TURKOS/OS/K1455: REJECTED reason=edu-bdf"); park(); }); if unsafe { tables.map_ecam_peer_page(leaf.physical, edu.physical) }.is_err() { write_line("TURKOS/OS/K1455: REJECTED reason=edu-config-map"); park(); } } } if pci_mode == Some(81) { let (start, end) = host.memory_aperture.unwrap_or_else(|| { write_line("TURKOS/OS/K1450: REJECTED reason=pci-memory-aperture"); park(); }); if end <= start || end - start < 0x1000 || unsafe { tables.map_device_window(allocator, start, 0x1000, layout.uart_base) }.is_err() || !tables.device_page_matches(start) { write_line("TURKOS/OS/K1450: REJECTED reason=bar-device-map"); park(); } } if pci_mode == Some(82) { let (start, end) = host.memory_aperture.unwrap_or_else(|| { write_line("TURKOS/OS/K1455: REJECTED reason=pci-memory-aperture"); park(); }); let edu_bar = start.checked_add(0x10_0000).unwrap_or_else(|| park()); if edu_bar.checked_add(0x1000).is_none_or(|last| last > end) || unsafe { tables.map_device_window(allocator, edu_bar, 0x1000, layout.uart_base) }.is_err() || !tables.device_page_matches(edu_bar) { write_line("TURKOS/OS/K1455: REJECTED reason=edu-device-map"); park(); } } if pci_mode == Some(83) { let (start, end) = host.memory_aperture.unwrap_or_else(|| { write_line("TURKOS/OS/K1460: REJECTED reason=pci-memory-aperture"); park(); }); let msi = dtb::discover_pci_msi_frame(ecam_blob).unwrap_or_else(|_| { write_line("TURKOS/OS/K1460: REJECTED reason=msi-dtb"); park(); }); if end <= start || end - start < 0x1000 || unsafe { tables.map_device_window(allocator, start, 0x1000, layout.uart_base) }.is_err() || !tables.device_page_matches(start) || !tables.device_page_matches(msi.base) { write_line("TURKOS/OS/K1460: REJECTED reason=device-map"); park(); } } } let free_after_tables = allocator.remaining_pages(); let tagged_table_frames = tables.tagged_frames().count(); let table_tags_valid = tagged_table_frames == tables.frame_count() && tables.tagged_frames().all(|tag| { tag.address != 0 && tag.address % memory::PAGE_SIZE == 0 && tables .tagged_frames() .filter(|other| other.address == tag.address) .count() == 1 }); if !table_tags_valid || free_before_tables.checked_sub(free_after_tables) != Some(tables.frame_count() as u64) { write_line("TURKOS/MMU-ERROR: table ownership count"); park(); } write_str("TURKOS/MMU-TABLE-ACCOUNT: BEFORE=0x"); write_hex(free_before_tables as usize); write_str(" AFTER=0x"); write_hex(free_after_tables as usize); write_str(" COUNT=0x"); write_hex(tables.frame_count()); write_str("\r\n"); #[cfg(feature = "heap-test")] let heap_pages = { if !tables.heap_window_unmapped() { write_line("TURKOS/HEAP-ERROR: virtual window was mapped"); park(); } write_str("TURKOS/HEAP-WINDOW: START=0x"); write_hex(mmu::HEAP_WINDOW_START as usize); write_str(" END=0x"); write_hex( (mmu::HEAP_WINDOW_START + mmu::HEAP_WINDOW_PAGES as u64 * memory::PAGE_SIZE) as usize, ); write_str("\r\nTURKOS/HEAP-UNMAPPED-BEFORE: PASS\r\n"); let before = allocator.remaining_pages(); // SAFETY: the free map reserved the heap's identity range before // building tables, and TTBR0 has not yet been installed. let pages = match unsafe { tables.map_heap_prefix(allocator) } { Ok(pages) => pages, Err(_) => { write_line("TURKOS/HEAP-ERROR: map prefix"); park(); } }; if !tables.heap_tail_unmapped() || before.checked_sub(allocator.remaining_pages()) != Some(pages.count() as u64) { write_line("TURKOS/HEAP-ERROR: map ownership"); park(); } write_str("TURKOS/HEAP-FRAME-ACCOUNT: BEFORE=0x"); write_hex(before as usize); write_str(" AFTER=0x"); write_hex(allocator.remaining_pages() as usize); write_str(" COUNT=0x"); write_hex(pages.count()); write_str("\r\nTURKOS/HEAP-TAIL-UNMAPPED: PASS\r\n"); for (index, frame) in pages.frames.iter().copied().enumerate() { write_str("TURKOS/HEAP-MAP: INDEX=0x"); write_hex(index); write_str(" VA=0x"); write_hex((mmu::HEAP_WINDOW_START + index as u64 * memory::PAGE_SIZE) as usize); write_str(" PA=0x"); write_hex(frame as usize); write_str("\r\n"); } pages }; let sctlr_before: u64; let mmfr0: u64; // SAFETY: these privileged system registers are readable at EL1. unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr_before, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ID_AA64MMFR0_EL1", value = out(reg) mmfr0, options(nomem, nostack, preserves_flags)); } if sctlr_before & 1 != 0 || (mmfr0 & 0xf) == 7 { write_line("TURKOS/MMU-ERROR: boot register state"); park(); } let Some(tcr) = mmu::tcr_el1((mmfr0 & 0xf).min(5)) else { write_line("TURKOS/MMU-ERROR: physical address width"); park() }; write_str("TURKOS/MMU-ROOT: 0x"); write_hex(tables.root as usize); write_str("\r\nTURKOS/MMU-TABLES: 0x"); write_hex(tables.frame_count()); write_str("\r\nTURKOS/MMU-BLOCKS: 0x"); write_hex(tables.block_count()); write_str("\r\n"); write_line("TURKOS/MMU-PRE-ENABLE: PASS"); // SAFETY: the root is a 4 KiB-aligned L1 table. Identity mapping covers // the current PC, stack, VBAR, UART, and all table pages. IRQ remains masked. unsafe { asm!( "dsb sy", "msr MAIR_EL1, {mair}", "msr TCR_EL1, {tcr}", "msr TTBR0_EL1, {root}", "isb", "tlbi vmalle1", "dsb sy", "isb", "msr SCTLR_EL1, {sctlr}", "isb", mair = in(reg) mmu::MAIR_EL1, tcr = in(reg) tcr, root = in(reg) tables.root, sctlr = in(reg) (sctlr_before | 1), options(nostack) ); } let sctlr_after: u64; // SAFETY: SCTLR_EL1 remains readable after enabling translation. unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr_after, options(nomem, nostack, preserves_flags)) }; if sctlr_after & 1 == 0 { write_line("TURKOS/MMU-ERROR: SCTLR.M is clear"); park(); } write_line("TURKOS/MMU: ON"); let actual_ttbr: u64; let actual_mair: u64; let actual_tcr: u64; let actual_vbar: u64; let actual_sp: u64; // SAFETY: all registers are readable in the EL1-only smoke-test context. unsafe { asm!("mrs {value}, TTBR0_EL1", value = out(reg) actual_ttbr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, MAIR_EL1", value = out(reg) actual_mair, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, TCR_EL1", value = out(reg) actual_tcr, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, VBAR_EL1", value = out(reg) actual_vbar, options(nomem, nostack, preserves_flags)); asm!("mov {value}, sp", value = out(reg) actual_sp, options(nomem, nostack, preserves_flags)); } let vector_address = __vectors_start as *const () as u64; if actual_ttbr != tables.root || actual_ttbr & (memory::PAGE_SIZE - 1) != 0 || actual_mair != mmu::MAIR_EL1 || actual_tcr != tcr || actual_vbar != vector_address || actual_vbar < layout.text_start || actual_vbar >= layout.text_end || actual_sp < hardware.ram_base || actual_sp > hardware.ram_base + hardware.ram_size || actual_sp & 15 != 0 { write_line("TURKOS/MMU-ERROR: post-enable registers"); park(); } write_str("TURKOS/MMU-TTBR0: 0x"); write_hex(actual_ttbr as usize); write_str("\r\nTURKOS/MMU-MAIR: 0x"); write_hex(actual_mair as usize); write_str("\r\nTURKOS/MMU-TCR: 0x"); write_hex(actual_tcr as usize); write_str("\r\nTURKOS/MMU-VBAR: 0x"); write_hex(actual_vbar as usize); write_str("\r\nTURKOS/MMU-SP: 0x"); write_hex(actual_sp as usize); write_str("\r\n"); let dtb_bytes = boot.dtb_address as *const u8; // SAFETY: read_boot_data already validated this FDT header and the DTB is // still identity-mapped RAM while the same guest is running. let dtb_magic = unsafe { u32::from_be_bytes([ read_volatile(dtb_bytes), read_volatile(dtb_bytes.add(1)), read_volatile(dtb_bytes.add(2)), read_volatile(dtb_bytes.add(3)), ]) }; if dtb_magic != 0xd00d_feed { write_line("TURKOS/MMU-ERROR: DTB lost after enable"); park(); } write_line("TURKOS/MMU-DTB: PASS"); let free_before_probe = allocator.remaining_pages(); let mut probe_frames = [0u64; 16]; let mut probe_saved = [0u64; 16]; for index in 0..probe_frames.len() { let Some(frame) = allocator.allocate() else { write_line("TURKOS/MMU-ERROR: post-enable frame unavailable"); park(); }; probe_frames[index] = frame; // SAFETY: each frame is exclusively owned, writable, and identity-mapped. unsafe { probe_saved[index] = read_volatile(frame as *const u64); let marker = 0x5455_524b_4f53_0000u64 | index as u64; write_volatile(frame as *mut u64, marker); if read_volatile(frame as *const u64) != marker { write_line("TURKOS/MMU-ERROR: post-enable frame mismatch"); park(); } } } for index in (0..probe_frames.len()).rev() { // SAFETY: the frame is still owned and its original first word is restored. unsafe { write_volatile(probe_frames[index] as *mut u64, probe_saved[index]) }; if allocator.release(probe_frames[index]).is_err() { write_line("TURKOS/MMU-ERROR: post-enable frame release"); park(); } } if allocator.remaining_pages() != free_before_probe { write_line("TURKOS/MMU-ERROR: post-enable frame ownership"); park(); } write_str("TURKOS/MMU-FRAME-COUNT: 0x"); write_hex(probe_frames.len()); write_str("\r\n"); write_line("TURKOS/MMU-FRAME: PASS"); let data_address = core::ptr::addr_of_mut!(MMU_DATA_CANARY) as u64; if data_address < layout.rodata_end || data_address >= layout.image_end { write_line("TURKOS/MMU-ERROR: data canary outside data"); park(); } // SAFETY: this initialized static belongs to the kernel's writable data // section, and this single-CPU smoke test restores its original value. unsafe { let previous = read_volatile(data_address as *const u64); write_volatile(data_address as *mut u64, previous ^ u64::MAX); if read_volatile(data_address as *const u64) != previous ^ u64::MAX { write_line("TURKOS/MMU-ERROR: data canary mismatch"); park(); } write_volatile(data_address as *mut u64, previous); } write_line("TURKOS/MMU-DATA-WRITE: PASS"); #[cfg(feature = "heap-test")] { for (index, frame) in heap_pages.frames.iter().copied().enumerate() { let address = mmu::HEAP_WINDOW_START + index as u64 * memory::PAGE_SIZE; let marker = 0x5455_524b_4845_4150u64 ^ index as u64; // SAFETY: the heap VA is backed by this exclusively owned frame; // both aliases are readable in the isolated early-boot test. unsafe { write_volatile(address as *mut u64, marker); if read_volatile(address as *const u64) != marker || read_volatile(frame as *const u64) != marker { write_line("TURKOS/HEAP-ERROR: VA/PA mismatch"); park(); } } } write_line("TURKOS/HEAP-VA-PA: PASS"); if !heap_object_smoke_test() { write_line("TURKOS/HEAP-ERROR: object contract"); park(); } let before_return = allocator.remaining_pages(); for (index, frame) in heap_pages.frames.iter().copied().enumerate().rev() { let address = mmu::HEAP_WINDOW_START + index as u64 * memory::PAGE_SIZE; // SAFETY: the object test proved no active handles and complete // coalescing. This CPU is the only user of these VAs. The valid // descriptor is cleared before targeted TLBI and frame reuse. if unsafe { tables.detach_heap_page(index, frame) }.is_err() { write_line("TURKOS/HEAP-ERROR: detach descriptor"); park(); } unsafe { asm!( "dsb ishst", "tlbi vaae1is, {page}", "dsb ish", "isb", page = in(reg) address >> 12, options(nostack) ); } if allocator.release(frame).is_err() { write_line("TURKOS/HEAP-ERROR: return frame"); park(); } write_str("TURKOS/HEAP-RETURN: INDEX=0x"); write_hex(index); write_str(" VA=0x"); write_hex(address as usize); write_str(" PA=0x"); write_hex(frame as usize); write_str("\r\n"); } if !tables.heap_window_unmapped() { write_line("TURKOS/HEAP-ERROR: descriptor remained"); park(); } write_line("TURKOS/HEAP-UNMAPPED-AFTER: PASS"); let after_return = allocator.remaining_pages(); write_str("TURKOS/HEAP-FRAME-RETURN: BEFORE=0x"); write_hex(before_return as usize); write_str(" AFTER=0x"); write_hex(after_return as usize); write_str(" COUNT=0x"); write_hex(heap_pages.count()); write_str("\r\n"); if after_return != before_return + heap_pages.count() as u64 { write_line("TURKOS/HEAP-ERROR: return accounting"); park(); } write_line("TURKOS/HEAP: PASS"); #[cfg(feature = "heap-panic-test")] { write_line("TURKOS/HEAP-PANIC-READY: ALL-PAGES-UNMAPPED"); panic!("heap unavailable by design"); } #[cfg(not(feature = "heap-panic-test"))] park(); } #[cfg(feature = "mmu-tlb-test")] mmu_tlb_probe(allocator, &tables); #[cfg(feature = "task-state-test")] { if task_state_smoke_test() { write_line("TURKOS/TASK-MODEL: PASS"); } else { write_line("TURKOS/TASK-ERROR: state contract"); } park(); } #[cfg(any(feature = "task-switch-test", feature = "task-lifecycle-test"))] task_switch_smoke_test(); #[cfg(feature = "task-guard-test")] task_guard::smoke_test(&tables, layout); #[cfg(feature = "user-three-test")] process_probe::run(allocator, &tables, layout); #[cfg(any( feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test" ))] user_first::smoke_test(allocator, &tables, layout); #[cfg(any( feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test" ))] user_dual::smoke_test(allocator, &tables, layout); #[cfg(any( feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test" ))] user_syscall::smoke_test(allocator, &tables, layout); #[cfg(feature = "elf-ident-test")] elf_ident_probe::smoke_test(); #[cfg(feature = "elf-layout-test")] elf_layout_probe::smoke_test(); #[cfg(feature = "elf-load-test")] elf_load_probe::smoke_test(allocator, &tables, layout); #[cfg(feature = "elf-stack-test")] elf_stack_probe::smoke_test(allocator, &tables, layout); #[cfg(feature = "elf-hello-test")] elf_hello_probe::smoke_test(allocator, &tables, layout); #[cfg(feature = "elf-sequence-test")] elf_sequence_probe::smoke_test(allocator, &tables, layout); #[cfg(feature = "ipc-basic-test")] ipc_basic_probe::smoke_test(); #[cfg(feature = "ipc-wait-test")] ipc_wait_probe::smoke_test(); #[cfg(feature = "task-cancel-test")] task_cancel_probe::smoke_test(); #[cfg(feature = "ipc-multi-test")] ipc_multi_probe::smoke_test(); #[cfg(feature = "ipc-timeline-test")] ipc_timeline_probe::smoke_test(); #[cfg(feature = "virtio-discover-test")] virtio_discovery_smoke_test(&boot); #[cfg(feature = "virtio-magic-test")] virtio_magic_probe::smoke_test(&boot); #[cfg(feature = "virtio-net-id-test")] virtio_net_id_probe::smoke_test(&boot); #[cfg(feature = "virtio-net-mac-test")] virtio_net_mac_probe::smoke_test(&boot); #[cfg(all(feature = "virtio-net-rx-test", not(feature = "virtio-net-pool-test")))] virtio_net_rx_probe::smoke_test(&boot, allocator); #[cfg(any( feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test" ))] virtio_net_arp_reply_probe::smoke_test(&boot, allocator); #[cfg(feature = "virtio-net-pool-test")] virtio_net_pool_probe::smoke_test(&boot, allocator); #[cfg(all( feature = "virtio-net-tx-test", not(feature = "virtio-net-tx-completion-test") ))] virtio_net_tx_probe::smoke_test(&boot, allocator); #[cfg(feature = "virtio-net-tx-completion-test")] virtio_net_tx_probe::completion_smoke_test(&boot, allocator); #[cfg(feature = "virtio-modern-features-test")] virtio_modern_probe::smoke_test(&boot, allocator); #[cfg(feature = "p12-journal-test")] journal_probe::run(&boot, allocator); #[cfg(feature = "turkfs-init-test")] virtio_init_probe::smoke_test(&boot, allocator, &tables, layout); #[cfg(feature = "serial-rx-test")] serial_rx_probe::smoke_test(); #[cfg(feature = "serial-line-test")] serial_line_probe::smoke_test(); #[cfg(feature = "serial-edit-test")] serial_edit_probe::smoke_test(); #[cfg(feature = "serial-limit-test")] serial_limit_probe::smoke_test(); #[cfg(feature = "serial-console-test")] serial_console_probe::smoke_test(); #[cfg(feature = "serial-unknown-test")] serial_unknown_probe::smoke_test(); #[cfg(feature = "serial-utf8-test")] serial_utf8_probe::smoke_test(); #[cfg(feature = "serial-arguments-test")] serial_arguments_probe::smoke_test(); #[cfg(feature = "serial-range-test")] serial_range_probe::smoke_test(); #[cfg(feature = "serial-cancel-test")] serial_cancel_probe::smoke_test(); #[cfg(any( feature = "serial-line-lock-test", feature = "serial-line-unlocked-test" ))] serial_line_lock_probe::smoke_test(); #[cfg(feature = "serial-echo-test")] serial_echo_probe::smoke_test(); #[cfg(feature = "turkfs-ls-test")] turkfs_ls_probe::smoke_test(&boot, allocator); #[cfg(feature = "turkfs-cat-test")] turkfs_cat_probe::smoke_test(&boot, allocator); #[cfg(all( feature = "virtio-modern-queue-test", not(feature = "turkfs-init-test"), not(feature = "turkfs-ls-test"), not(feature = "turkfs-cat-test"), not(feature = "p12-journal-test") ))] virtio_queue_probe::smoke_test(&boot, allocator); #[cfg(feature = "user-task-test")] user_task::smoke_test(allocator, &tables, layout); #[cfg(feature = "user-handle-test")] user_task::smoke_test(allocator, &tables, layout); #[cfg(feature = "user-handle-transfer-test")] user_task::smoke_test(allocator, &tables, layout); #[cfg(feature = "user-handle-death-test")] user_task::smoke_test(allocator, &tables, layout); #[cfg(feature = "user-wait-test")] user_task::smoke_test(allocator, &tables, layout); #[cfg(any(feature = "task-preempt-test", feature = "task-sleep-test"))] preempt::smoke_test(); #[cfg(feature = "mmu-no-map-test")] { let Some(region) = boot.static_reserved.as_slice().iter().find(|region| { region.no_map && ((region.start == hardware.ram_base && region.size == 4 * memory::PAGE_SIZE) || (region.start == hardware.ram_base + (1 << 21) + memory::PAGE_SIZE && region.size == memory::PAGE_SIZE) || (region.start == hardware.ram_base + (1 << 21) && region.size == (1 << 21))) }) else { write_line("TURKOS/MMU-ERROR: no-map fixture missing"); park(); }; let first_hole = region.start == hardware.ram_base && region.size == 4 * memory::PAGE_SIZE; let distant_hole = region.start == hardware.ram_base + (1 << 21) + memory::PAGE_SIZE && region.size == memory::PAGE_SIZE; let full_block_hole = region.start == hardware.ram_base + (1 << 21) && region.size == (1 << 21); if !first_hole && !distant_hole && !full_block_hole { write_line("TURKOS/MMU-ERROR: unexpected no-map fixture"); park(); } let adjacent = (region.start + region.size + memory::PAGE_SIZE - 1) & !(memory::PAGE_SIZE - 1); if distant_hole || full_block_hole { // SAFETY: the page immediately before the isolated distant hole // remains in the same split L2 block and must retain a descriptor. unsafe { read_volatile((region.start - memory::PAGE_SIZE) as *const u64) }; write_line("TURKOS/MMU-PREVIOUS: PASS"); } // SAFETY: both dedicated fixtures leave the next RAM page mapped. The // read proves the hole is local even after an L2 block is split. unsafe { read_volatile(adjacent as *const u64) }; write_line("TURKOS/MMU-ADJACENT: PASS"); write_line("TURKOS/MMU-PROBE: NO-MAP"); // SAFETY: the read intentionally traps on a reserved, unmapped page. unsafe { read_volatile(region.start as *const u64) }; write_line("TURKOS/MMU-ERROR: no-map read returned"); park() } #[cfg(feature = "mmu-write-test")] { write_line("TURKOS/MMU-PROBE: TEXT-WRITE"); // SAFETY: this intentionally triggers an EL1 write-permission fault. unsafe { write_volatile(core::ptr::addr_of!(_image_start) as *mut u64, 0) }; write_line("TURKOS/MMU-ERROR: text write returned"); park() } #[cfg(feature = "mmu-nx-test")] { let target = core::ptr::addr_of_mut!(NX_PROBE) as u64; if target < layout.rodata_end || target >= layout.image_end || target & 3 != 0 { write_line("TURKOS/MMU-ERROR: NX probe outside data"); park(); } write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: DATA-EXEC"); // SAFETY: the dedicated test intentionally branches to an XN data // page. The fatal vector reports the instruction-abort frame. unsafe { asm!("br {target}", target = in(reg) target, options(noreturn)) } } #[cfg(feature = "mmu-rodata-test")] { let target = core::ptr::addr_of!(RODATA_PROBE) as u64; if target < layout.text_end || target >= layout.rodata_end { write_line("TURKOS/MMU-ERROR: rodata probe outside rodata"); park(); } write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: RODATA-WRITE"); // SAFETY: the dedicated test intentionally writes to a read-only // page. The fatal vector reports the data-abort frame. unsafe { write_volatile(target as *mut u64, 0) }; write_line("TURKOS/MMU-ERROR: rodata write returned"); park() } #[cfg(feature = "mmu-rodata-exec-test")] { let target = core::ptr::addr_of!(RODATA_EXEC_PROBE) as u64; if target < layout.text_end || target >= layout.rodata_end || target & 3 != 0 { write_line("TURKOS/MMU-ERROR: rodata exec probe outside rodata"); park(); } write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: RODATA-EXEC"); // SAFETY: this dedicated image intentionally branches to an XN rodata page. unsafe { asm!("br {target}", target = in(reg) target, options(noreturn)) } } #[cfg(feature = "mmu-stack-exec-test")] { let target: u64; // SAFETY: reading SP into a register does not modify guest state. unsafe { asm!("mov {target}, sp", target = out(reg) target, options(nomem, nostack, preserves_flags)) }; if target < hardware.ram_base || target >= hardware.ram_base + hardware.ram_size { write_line("TURKOS/MMU-ERROR: stack probe outside RAM"); park(); } write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: STACK-EXEC"); // SAFETY: this dedicated image intentionally branches to an XN stack page. unsafe { asm!("br {target}", target = in(reg) target, options(noreturn)) } } #[cfg(feature = "mmu-mmio-exec-test")] { let target = hardware.uart_base; write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: UART-EXEC"); // SAFETY: this dedicated image intentionally attempts instruction fetch // from the mapped, privileged-XN UART Device page. unsafe { asm!("br {target}", target = in(reg) target, options(noreturn)) } } #[cfg(feature = "mmu-test")] { // 0x2000_0000 is outside the one mapped PL011 page and installed RAM. // SAFETY: the read intentionally triggers an EL1 translation fault. unsafe { read_volatile(0x2000_0000 as *const u64) }; write_line("TURKOS/MMU-ERROR: unmapped read returned"); park() } #[cfg(feature = "mmu-panic-test")] { write_line("TURKOS/MMU-PROBE: PANIC"); panic!("intentional post-MMU panic"); } #[cfg(feature = "mmu-brk-test")] { write_line("TURKOS/MMU-PROBE: BRK"); // SAFETY: this dedicated image intentionally enters the fatal EL1 // synchronous vector after identity translation is enabled. unsafe { asm!( "brk #0x42", in("x0") 0x544b_4252_4b5f_5830usize, in("x3") 0x544b_4252_4b5f_5833usize, in("x30") 0x544b_4252_4b5f_4c52usize, options(nomem, nostack, noreturn) ) } } #[cfg(feature = "mmu-udf-test")] { write_line("TURKOS/MMU-PROBE: UDF"); // SAFETY: this dedicated image intentionally enters the fatal EL1 // synchronous vector with an undefined instruction after MMU enable. unsafe { asm!( "udf #0", in("x0") 0x544b_5544_465f_5830usize, in("x3") 0x544b_5544_465f_5833usize, in("x30") 0x544b_5544_465f_4c52usize, options(nomem, nostack, noreturn) ) } } #[cfg(feature = "mmu-guard-test")] { let target = (layout.image_end + memory::PAGE_SIZE - 1) & !(memory::PAGE_SIZE - 1); write_str("TURKOS/MMU-IMAGE-END: 0x"); write_hex(layout.image_end as usize); write_str("\r\n"); write_str("TURKOS/MMU-PROBE-ADDR: 0x"); write_hex(target as usize); write_str("\r\n"); write_line("TURKOS/MMU-PROBE: IMAGE-GUARD"); // SAFETY: the first page beyond the image is intentionally unmapped. unsafe { read_volatile(target as *const u64) }; write_line("TURKOS/MMU-ERROR: guard read returned"); park() } #[cfg(feature = "irq-return-test")] { irq_return_smoke_test(); park() } #[cfg(feature = "el0-fault-test")] { let target = layout.text_start as usize; write_str("TURKOS/EL0-FAULT-TARGET: 0x"); write_hex(target); write_str("\r\n"); write_line("TURKOS/EL0-ENTRY: ATTEMPT"); // This image deliberately branches to EL1-only text from EL0. No user // instruction runs and SP_EL0 is never dereferenced; it is not a user // task or a usable EL0 stack. DAIF stays masked during the probe. unsafe { asm!( "msr SP_EL0, {stack}", "msr ELR_EL1, {target}", "msr SPSR_EL1, {spsr}", "isb", "eret", stack = in(reg) 0usize, target = in(reg) target, spsr = in(reg) 0x3c0usize, options(noreturn) ); } } #[cfg(feature = "nested-fault-test")] { write_line("TURKOS/MMU-PROBE: NESTED-FATAL"); unsafe { asm!("brk #0x158") }; write_line("TURKOS/EXCEPTION-FIRST: RETURNED"); park(); } #[cfg(feature = "irq-sgi-test")] { irq_sgi_smoke_test(); park() } #[cfg(all(feature = "irq-timer-test", not(feature = "process-exit-irq-test")))] { #[cfg(feature = "irq-service-test")] irq_service::run(&allocator, boot.interrupts.gic_redistributor); #[cfg(not(feature = "irq-service-test"))] { irq_timer_smoke_test(boot.interrupts.gic_redistributor); park() } } #[cfg(feature = "irq-spurious-test")] irq_spurious_smoke_test(); #[cfg(feature = "irq-spurious-vector-test")] spurious_vector::run(boot.interrupts.gic_redistributor); #[cfg(all( feature = "irq-unknown-test", not(feature = "irq-spurious-test"), not(feature = "irq-spurious-vector-test") ))] irq_unknown_smoke_test(boot.interrupts.gic_redistributor); #[cfg(not(any( feature = "mmu-test", feature = "user-three-test", feature = "mmu-no-map-test", feature = "mmu-write-test", feature = "mmu-nx-test", feature = "mmu-rodata-test", feature = "mmu-rodata-exec-test", feature = "mmu-stack-exec-test", feature = "mmu-mmio-exec-test", feature = "mmu-panic-test", feature = "mmu-brk-test", feature = "mmu-udf-test", feature = "mmu-guard-test", feature = "mmu-tlb-test", feature = "el0-fault-test", feature = "nested-fault-test", feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-timer-test", feature = "irq-unknown-test", feature = "heap-test", feature = "task-state-test", feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-guard-test", feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test", feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-clock-test", feature = "user-mmio-request-test", feature = "elf-ident-test", feature = "elf-layout-test", feature = "elf-load-test", feature = "elf-stack-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "turkfs-init-test", feature = "serial-rx-test", feature = "serial-line-test", feature = "serial-edit-test", feature = "serial-limit-test", feature = "serial-console-test", feature = "serial-echo-test", feature = "turkfs-ls-test", feature = "turkfs-cat-test", feature = "ipc-basic-test", feature = "ipc-wait-test", feature = "task-cancel-test", feature = "ipc-multi-test", feature = "ipc-timeline-test", feature = "virtio-discover-test", feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-tx-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-task-test", feature = "task-preempt-test", feature = "task-sleep-test" )))] { // Measure the live mask, not only the early boot's informational label. // Acceptance probes may unmask IRQ in their own feature-gated images; // the ordinary Image must arrive here with IRQ still masked. let daif: u64; unsafe { asm!("mrs {value}, DAIF", value = out(reg) daif, options(nomem, nostack)) }; write_str("TURKOS/IRQ-DEFAULT-DAIF: 0x"); write_hex(daif as usize); write_str("\r\n"); let (ro_leaf, ro_value, data_leaf, data_value) = match os_k630_live_mmu(&tables) { Ok(value) => value, Err(reason) => { write_str("TURKOS/OS/K630: REJECTED reason="); write_str(reason); write_str(" AFTER-MMU\r\n"); park(); } }; write_str("TURKOS/OS/K630: RO-LEAF=0x"); write_hex(ro_leaf as usize); write_str(" VALUE=0x"); write_hex(ro_value as usize); write_str(" DATA-LEAF=0x"); write_hex(data_leaf as usize); write_str(" VALUE=0x"); write_hex(data_value as usize); write_str(" PASS\r\n"); if let Err(reason) = os_k635_post_mmu(&tables, &allocator, layout) { write_str("TURKOS/OS/K635: REJECTED reason="); write_str(reason); write_str(" AFTER-MMU\r\n"); park(); } if let Err(reason) = os_k700_live_platform(allocator, &tables, boot) { write_str("TURKOS/OS/PLATFORM: REJECTED reason="); write_line(reason); park(); } if unsafe { read_volatile(core::ptr::addr_of!(OS_K700_MODE)) } == 1 { os_k700_hole_read(); } if let Err(reason) = os_k660_accept(&mut os_runtime, boot) { write_str("TURKOS/OS/K660: REJECTED reason="); write_line(reason); park(); } if let Err(reason) = os_k680_after_runtime(&mut os_runtime, &tables, boot) { write_str("TURKOS/OS/K680: REJECTED reason="); write_line(reason); park(); } if let Err(reason) = os_k700_finish_platform() { write_str("TURKOS/OS/PLATFORM: REJECTED reason="); write_line(reason); park(); } write_line("TURKOS/OS/K685: PASS"); write_line("TURKOS/OS/K690: PASS"); write_line("TURKOS/OS/K695: PASS"); write_line("TURKOS/OS/K700: LOCAL-CPU-GUEST PASS M1-FRESH-IMAGE=PENDING"); if let Err(reason) = os_k740_on_stack(&mut os_runtime, &tables, boot) { write_str("TURKOS/OS/METADATA: REJECTED reason="); write_line(reason); park(); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_enabled(boot).unwrap_or_else(|_|park()) { write_line("TURKOS/OS/K780: RETURN-POOL READY"); while unsafe{read_volatile(core::ptr::addr_of!(OS_K780_RETURN_POOL_GATE))}==0 { unsafe{asm!("yield",options(nomem,nostack,preserves_flags))}; } } if let Err(reason) = os_k800_on_stack(&mut os_runtime, &tables, boot) { write_str("TURKOS/OS/DMA: REJECTED reason="); write_line(reason); park(); } if let Err(reason) = os_k820_on_stack(&mut os_runtime,&tables,boot) { write_str("TURKOS/OS/PAGE-TABLES: REJECTED reason="); write_line(reason); park(); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_enabled(boot).unwrap_or_else(|_|park()) { write_line("TURKOS/OS/K840: RETURN-POOL READY"); while unsafe{read_volatile(core::ptr::addr_of!(OS_K840_RETURN_POOL_GATE))}==0 { unsafe{asm!("yield",options(nomem,nostack,preserves_flags))}; } } if let Err(reason) = os_k860_on_stack(&mut os_runtime,&tables,boot) { write_str("TURKOS/OS/MAPPING: REJECTED reason="); write_line(reason); park(); } if let Err(reason) = os_k880_on_stack(&mut os_runtime,&tables,boot) { write_str("TURKOS/OS/USER-COPY: REJECTED reason="); write_line(reason); park(); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_enabled(boot).unwrap_or_else(|_|park()) { write_line("TURKOS/OS/K880: RETURN-POOL READY"); while unsafe{read_volatile(core::ptr::addr_of!(OS_K880_RETURN_POOL_GATE))}==0 { unsafe{asm!("yield",options(nomem,nostack,preserves_flags))}; } } if let Err(reason) = os_k900_on_stack(&mut os_runtime,&tables,boot) { write_str("TURKOS/OS/PAGE-FAULTS: REJECTED reason="); write_line(reason); park(); } if let Err(reason) = os_k720_accept(&mut os_runtime.memory_mut().expect("frame authority").allocator, &tables) { write_str("TURKOS/OS/FRAME: REJECTED reason="); write_line(reason); park(); } write_line("TURKOS/MMU-NORMAL: PASS"); write_line("TURKOS/OS/K605: PASS"); write_line("TURKOS/OS/K610: PASS"); write_line("TURKOS/OS/K615: PASS"); write_line("TURKOS/OS/K620: PASS"); write_line("TURKOS/OS/K625: PASS"); write_line("TURKOS/OS/K630: PASS"); write_line("TURKOS/OS/K635: PASS"); write_line("TURKOS/OS/K640: PASS"); os_k900_kernel_fatal_if_requested(&mut os_runtime); os_k920_fatal_if_requested(&mut os_runtime,&tables,boot); os_k1020_fatal_if_requested(&mut os_runtime,&tables,boot); park() } } fn read_boot_data(address: usize) -> Result { if address == 0 || address & 7 != 0 { return Err(dtb::Error::Invalid); } // SAFETY: the trusted QEMU boot contract passes an identity-mapped, readable // DTB pointer in x0. Check its 40-byte header before bounding the full view. let header = unsafe { core::slice::from_raw_parts(address as *const u8, 40) }; let size = dtb::total_size(header)?; // SAFETY: QEMU supplies the entire blob at this address; size is capped at 2 MiB. let blob = unsafe { core::slice::from_raw_parts(address as *const u8, size) }; Ok(BootData { hardware: os_k700_capture_platform(blob)?, interrupts: dtb::discover_interrupts(blob)?, reservations: dtb::reservations(blob)?, static_reserved: dtb::static_reserved(blob)?, #[cfg(any( feature = "virtio-discover-test", feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-tx-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "user-mmio-read-test" ))] virtio_mmio: dtb::discover_virtio_mmio(blob)?, #[cfg(not(any( feature = "panic-test", feature = "exception-test", feature = "fault-test", feature = "frame-stress-test" )))] dtb_address: address, dtb_size: size, }) } fn read_uefi_boot_info( address: usize, boot: &BootData, dtb_address: usize, ) -> Result, ()> { // QEMU direct -kernel keeps x1=0. Only the UEFI loader supplies BootInfo. if address == 0 { return Ok(None); } bootinfo::validate_uefi_boot_info_pointer(address as u64).map_err(|_| ())?; let ram_start = boot.hardware.ram_base; let ram_end = ram_start.checked_add(boot.hardware.ram_size).ok_or(())?; let header_end = (address as u64) .checked_add(core::mem::size_of::() as u64) .ok_or(())?; if (address as u64) < ram_start || header_end > ram_end { return Err(()); } // SAFETY: the DTB-proven RAM extent contains the entire V1 header. This // remains a trusted UEFI/QEMU boot contract, not arbitrary physical DMA. let info = unsafe { core::ptr::read_unaligned(address as *const BootInfoV1) }; let kernel_start = core::ptr::addr_of!(_image_start) as u64; let kernel_bytes = (core::ptr::addr_of!(_image_end) as u64) .checked_sub(kernel_start) .ok_or(())?; info.validate_handoff( address as u64, ram_start, boot.hardware.ram_size, kernel_start, kernel_bytes, dtb_address as u64, boot.dtb_size as u64, ) .map_err(|_| ())?; // SAFETY: validate_handoff bounded the complete transferred map to a // fixed, page-aligned 64 KiB RAM allocation disjoint from Image and DTB. let map = unsafe { core::slice::from_raw_parts( info.memory_map_ptr as *const u8, info.memory_map_bytes as usize, ) }; let summary = bootinfo::inspect_uefi_memory_map(&info, address as u64, map).map_err(|_| ())?; write_str("TURKOS/BOOTINFO: phys=0x"); write_hex(address); write_str(" map=0x"); write_hex(info.memory_map_ptr as usize); write_str(" bytes=0x"); write_hex(info.memory_map_bytes as usize); write_str(" descriptors=0x"); write_hex(summary.descriptors); write_str(" loader-data=0x"); write_hex(summary.loader_data_descriptors); write_str(" PASS\r\n"); for raw in map.chunks_exact(info.descriptor_bytes as usize) { let descriptor = bootinfo::parse_memory_descriptor(raw).map_err(|_| ())?; write_str("TURKOS/EFI-DESCRIPTOR: TYPE=0x"); write_hex(descriptor.kind as usize); write_str(" START=0x"); write_hex(descriptor.start as usize); write_str(" END=0x"); write_hex(descriptor.end as usize); write_str("\r\n"); } Ok(Some((address as u64, info))) } #[cfg(feature = "virtio-discover-test")] fn virtio_discovery_smoke_test(boot: &BootData) -> ! { let nodes = boot.virtio_mmio.as_slice(); if nodes.len() != 32 { write_line("TURKOS/VIRTIO-DISCOVER-ERROR: slot count"); park(); } let mut min = u64::MAX; let mut max = 0u64; for node in nodes { if node.size != 0x200 { write_line("TURKOS/VIRTIO-DISCOVER-ERROR: slot size"); park(); } min = min.min(node.base); max = max.max(node.base); } if min != 0x0a00_0000 || max != 0x0a00_3e00 { write_line("TURKOS/VIRTIO-DISCOVER-ERROR: slot span"); park(); } write_str("TURKOS/VIRTIO-SLOTS: COUNT=0x"); write_hex(nodes.len()); write_str(" MIN=0x"); write_hex(min as usize); write_str(" MAX=0x"); write_hex(max as usize); write_str(" SIZE=0x"); write_hex(nodes[0].size as usize); write_str("\r\n"); write_line("TURKOS/VIRTIO-DISCOVER: PASS"); let window = dtb::virtio_page_window( &boot.virtio_mmio, boot.hardware.ram_base, boot.hardware.ram_size, ) .unwrap_or_else(|_| { write_line("TURKOS/VIRTIO-RANGE-ERROR: page window"); park() }); write_str("TURKOS/VIRTIO-PAGE-WINDOW: START=0x"); write_hex(window.start as usize); write_str(" END=0x"); write_hex(window.end as usize); write_str(" PAGES=0x"); write_hex(window.pages); write_str("\r\n"); write_line("TURKOS/VIRTIO-RANGE: PASS"); park() } fn plan_free_memory( boot: &BootData, dtb_address: usize, uefi_info: Option<&(u64, BootInfoV1)>, owned_map: Option<&[u8]>, ) -> Result { let mut free = if let Some((_, info)) = uefi_info { // read_uefi_boot_info proved the map's complete span is inside RAM, // validated every descriptor and checked both owned allocations. let raw = owned_map.ok_or(memory::Error::Invalid)?; memory::FreeMap::from_uefi_conventional( boot.hardware.ram_base, boot.hardware.ram_size, raw, info.descriptor_bytes as usize, )? } else { memory::FreeMap::from_ram(boot.hardware.ram_base, boot.hardware.ram_size)? }; os_k700_constrain_free(&mut free, boot)?; let kernel_start = core::ptr::addr_of!(_image_start) as u64; let kernel_end = core::ptr::addr_of!(_image_end) as u64; let kernel_size = kernel_end .checked_sub(kernel_start) .ok_or(memory::Error::Invalid)?; free.reserve(kernel_start, kernel_size)?; // The first whole page beyond the image is never given to the allocator. // The MMU leaves the same page unmapped to expose image-end overruns. let guard_start = kernel_end .checked_add(memory::PAGE_SIZE - 1) .ok_or(memory::Error::Invalid)? & !(memory::PAGE_SIZE - 1); free.reserve(guard_start, memory::PAGE_SIZE)?; free.reserve(dtb_address as u64, boot.dtb_size as u64)?; if let Some((address, info)) = uefi_info { free.reserve(*address, BOOT_INFO_PAGE_BYTES)?; free.reserve(info.memory_map_ptr, BOOT_INFO_MAP_CAPACITY)?; } for region in boot.reservations.as_slice() { free.reserve(region.start, region.size)?; } for region in boot.static_reserved.as_slice() { free.reserve(region.start, region.size)?; } #[cfg(feature = "heap-test")] free.reserve( mmu::HEAP_WINDOW_START, mmu::HEAP_WINDOW_PAGES as u64 * memory::PAGE_SIZE, )?; Ok(free) } #[cfg(feature = "uefi-ownership-test")] fn uefi_owned_endpoint_probe(free: memory::FreeMap, foreign: u64) { let first = free.regions()[0].start; let last = free.regions().last().unwrap().end - memory::PAGE_SIZE; let before = free.total_pages(); let mut allocator = memory::FrameAllocator::new(free); for (role, address) in [("FIRST", first), ("LAST", last)] { if allocator.allocate_at(address) != Ok(address) { write_line("TURKOS/OWNED-PROBE-ERROR: endpoint allocation"); park(); } // SAFETY: allocate_at admitted the entire frame under the validated // firmware/DTB FreeMap, disjoint from Image, DTB, payload and map. // This pre-MMU experiment owns it exclusively and restores both words. let start = address as *mut u64; let end = (address + memory::PAGE_SIZE - 8) as *mut u64; let canary = 0x4f57_4e45_4450_4147u64 ^ address; let good = unsafe { let old_start = read_volatile(start); let old_end = read_volatile(end); write_volatile(start, canary); write_volatile(end, !canary); let good = read_volatile(start) == canary && read_volatile(end) == !canary; write_volatile(start, old_start); write_volatile(end, old_end); good }; if !good { write_line("TURKOS/OWNED-PROBE-ERROR: canary"); park(); } write_str("TURKOS/OWNED-PAGE: ROLE="); write_str(role); write_str(" ADDR=0x"); write_hex(address as usize); write_str(" PASS\r\n"); } if allocator.allocate_at(foreign) != Err(memory::FrameRequestError::OutsideAuthority) || allocator.remaining_pages() + 2 != before || allocator.release_many(&[first, last]).is_err() || allocator.remaining_pages() != before || allocator.release(first).is_ok() { write_line("TURKOS/OWNED-PROBE-ERROR: foreign/accounting"); park(); } write_str("TURKOS/OWNED-FOREIGN: ADDR=0x"); write_hex(foreign as usize); write_str(" REJECTED\r\n"); write_line("TURKOS/OWNED-PROBE: ENDPOINTS+FOREIGN+RESTORE PASS"); } fn frame_smoke_test(free: memory::FreeMap) -> bool { #[cfg(feature = "frame-stress-test")] const TEST_PAGES: usize = 128; #[cfg(not(feature = "frame-stress-test"))] const TEST_PAGES: usize = 16; let initial_pages = free.total_pages(); let mut allocator = memory::FrameAllocator::new(free); let mut frames = [0; TEST_PAGES]; for index in 0..TEST_PAGES { let Some(address) = allocator.allocate() else { return false; }; if frames[..index].contains(&address) { return false; } frames[index] = address; // SAFETY: finalized free RAM excludes image, DTB and both reservation // sources. Direct QEMU Image boot uses an identity-mapped RAM view. unsafe { let first = address as *mut u64; let last = (address + memory::PAGE_SIZE - core::mem::size_of::() as u64) as *mut u64; let before_first = read_volatile(first); let before_last = read_volatile(last); let first_canary = 0x5455_524b_4f53_4652 ^ address; let last_canary = !first_canary; write_volatile(first, first_canary); write_volatile(last, last_canary); let passed = read_volatile(first) == first_canary && read_volatile(last) == last_canary; write_volatile(first, before_first); write_volatile(last, before_last); if !passed { write_str("TURKOS/FRAME-ERROR-ADDRESS: 0x"); write_hex(address as usize); write_str("\r\n"); return false; } } } let address = frames[0]; write_str("TURKOS/FRAME-ADDRESS: 0x"); write_hex(address as usize); write_str("\r\nTURKOS/FRAME-COUNT: 0x"); write_hex(TEST_PAGES); write_str("\r\nTURKOS/FRAME-EDGE: PASS\r\n"); for frame in frames.into_iter().rev() { if allocator.release(frame).is_err() { return false; } } if allocator.remaining_pages() != initial_pages { return false; } let before_duplicate = allocator.remaining_pages(); if allocator.release(address) != Err(memory::Error::Invalid) || allocator.remaining_pages() != before_duplicate { return false; } let mut reclaimed = [0; TEST_PAGES]; for index in 0..TEST_PAGES { let Some(frame) = allocator.allocate() else { return false; }; if frame != frames[index] || reclaimed[..index].contains(&frame) { return false; } reclaimed[index] = frame; } for frame in reclaimed.into_iter().rev() { if allocator.release(frame).is_err() { return false; } } if allocator.remaining_pages() != initial_pages { return false; } write_str("TURKOS/FRAME-DOUBLE-FREE: ADDRESS=0x"); write_hex(address as usize); write_str(" BEFORE=0x"); write_hex(before_duplicate as usize); write_str(" AFTER=0x"); write_hex(allocator.remaining_pages() as usize); write_str(" RECLAIMED=0x"); write_hex(TEST_PAGES); write_str(" RESULT=PASS\r\n"); write_str("TURKOS/FRAME-ACCOUNT: BEFORE=0x"); write_hex(initial_pages as usize); write_str(" AFTER=0x"); write_hex(allocator.remaining_pages() as usize); write_str("\r\n"); true } fn write_byte(byte: u8) { let base = UART_BASE.load(Ordering::Relaxed); // SAFETY: before discovery QEMU virt maps PL011 at the bootstrap address; // afterwards the DTB `reg` value identifies the same MMIO device. unsafe { while read_volatile((base + 0x18) as *const u32) & UART_TX_FULL != 0 {} write_volatile(base as *mut u32, byte as u32); } } fn write_str(value: &str) { for byte in value.bytes() { write_byte(byte); } } fn write_line(value: &str) { // Before SCTLR_EL1.M is set, this boot CPU has no competing tasks and // QEMU/HVF can reject exclusive accesses to the untranslated memory type. // The lock becomes usable only after RAM is mapped as Normal memory. let sctlr: usize; unsafe { asm!("mrs {value}, SCTLR_EL1", value = out(reg) sctlr, options(nomem, nostack, preserves_flags)) }; if sctlr & 1 == 0 { write_str(value); write_str("\r\n"); return; } // A one-CPU timer IRQ cannot enter another writer while this task owns // the line lock. Preserve the caller's previous I bit, including boot // paths that already run with IRQ masked. let daif: usize; unsafe { asm!( "mrs {saved}, daif", "msr daifset, #2", "isb", saved = out(reg) daif, options(nostack, preserves_flags) ) }; let guard = SERIAL_LINE_LOCK.lock(); write_str(value); write_str("\r\n"); drop(guard); if daif & (1 << 7) == 0 { unsafe { asm!("msr daifclr, #2", "isb", options(nostack, preserves_flags)) }; } } fn write_hex(value: usize) { for shift in (0..16).rev() { let nibble = ((value >> (shift * 4)) & 0xf) as u8; write_byte(if nibble < 10 { b'0' + nibble } else { b'a' + nibble - 10 }); } } fn write_hex_byte(value: u8) { for shift in [4, 0] { let nibble = (value >> shift) & 0xf; write_byte(if nibble < 10 { b'0' + nibble } else { b'a' + nibble - 10 }); } } fn park() -> ! { loop { // SAFETY: wfe only waits for an event and does not access memory. unsafe { asm!("wfe", options(nomem, nostack, preserves_flags)) } } } #[panic_handler] fn panic(_info: &PanicInfo) -> ! { os_k680_panic(); write_line("TURKOS/PANIC"); park() } // K400 gate in this crate root too: `cargo rustc --bin turkos-kernel -- --cfg // 'feature="production-write"'` reaches only this binary crate, never lib.rs. // Kept last so default-build line numbers (panic locations) do not move. #[cfg(feature = "production-write")] compile_error!( "K400 production writes are blocked: journal/replay, orphan recovery, failure matrix and release evidence are not complete" ); // Inserted only into the isolated development kernel's main.rs. // // Unsafe boundary: reuse the existing, trusted QEMU/UEFI bootstrap contract, // not a general physical-memory read API. read_boot_data has already validated // the readable DTB. IRQs are masked, firmware has exited on the UEFI path, and // no runtime allocation/map-descriptor read occurs before this function. // A malicious DTB that lies about physical RAM or a DMA adversary is outside // this acceptance. Numeric checks do not establish general pointer provenance. fn os_k605_entry( dtb_address: usize, boot_info_address: usize, boot: &BootData, ) -> Result< turkos_kernel::runtime::boot_contract::ResolvedBoot, turkos_kernel::runtime::boot_contract::BootError, > { use turkos_kernel::runtime::boot_contract::{ self, BootError, EntryRegisters, LocatedBytes, PhysicalRange, }; let ram_start = boot.hardware.ram_base; let ram_end = ram_start .checked_add(boot.hardware.ram_size) .ok_or(BootError::RamRange(boot_contract::RangeError::Overflow))?; let dtb_end = (dtb_address as u64) .checked_add(boot.dtb_size as u64) .ok_or(BootError::DtbOutsideRam)?; if (dtb_address as u64) < ram_start || dtb_end > ram_end { return Err(BootError::DtbOutsideRam); } // SAFETY: the trusted entry reader already bounded and parsed this exact // readable DTB slice; we additionally require it inside discovered RAM. let blob = unsafe { core::slice::from_raw_parts(dtb_address as *const u8, boot.dtb_size) }; let dtb = LocatedBytes::new(dtb_address as u64, blob).map_err(|_| BootError::DtbOutsideRam)?; let payload = if boot_info_address == 0 { None } else { turkos_bootinfo::validate_boot_info_pointer(boot_info_address as u64) .map_err(BootError::BootInfoPointer)?; let bytes = core::mem::size_of::(); let end = (boot_info_address as u64) .checked_add(bytes as u64) .ok_or(BootError::Handoff( turkos_bootinfo::BootInfoHandoffError::Range, ))?; if (boot_info_address as u64) < ram_start || end > ram_end { return Err(BootError::Handoff( turkos_bootinfo::BootInfoHandoffError::Range, )); } // SAFETY: this boot contract transfers a persistent, identity-mapped // RAM header after ExitBootServices, not an arbitrary external pointer. // Numeric bounds precede the read. resolve must validate the bytes and // all reservations before the existing map reader/allocator is called. let bytes = unsafe { core::slice::from_raw_parts(boot_info_address as *const u8, bytes) }; Some( LocatedBytes::new(boot_info_address as u64, bytes) .map_err(|_| BootError::HeaderTruncated)?, ) }; let image_start = core::ptr::addr_of!(_image_start) as u64; let image_bytes = (core::ptr::addr_of!(_image_end) as u64) .checked_sub(image_start) .ok_or(BootError::ImageOutsideRam)?; let image = PhysicalRange::new(image_start, image_bytes).map_err(|_| BootError::ImageOutsideRam)?; boot_contract::resolve_platform( EntryRegisters { dtb_address: dtb_address as u64, boot_info_address: boot_info_address as u64, }, dtb, payload, image, dtb::Hardware { uart_base: boot.hardware.uart_base, ram_base: boot.hardware.ram_base, ram_size: boot.hardware.ram_size, }, ) } fn os_k605_error_tag(error: &turkos_kernel::runtime::boot_contract::BootError) -> &'static str { use turkos_bootinfo::{BootInfoHandoffError as H, BootInfoHeaderError as B}; use turkos_kernel::runtime::boot_contract::BootError as E; match error { E::Header(B::Magic) => "header-magic", E::Header(B::Version) => "header-version", E::Header(B::Size) => "header-size", E::Handoff(H::Overlap) => "handoff-overlap", E::Handoff(H::Range) => "handoff-range", E::Handoff(H::Map) => "handoff-map", E::BootInfoPointer(_) => "bootinfo-pointer", E::Dtb(_) | E::DtbPointer | E::DtbOutsideRam | E::DtbAddressMismatch => "dtb", _ => "other", } } // Present only in the generated development kernel. The existing DTB reader // has already parsed a trusted QEMU/UEFI DTB and established the RAM extent. // IRQs remain masked and neither runtime allocation nor EFI map inspection // has occurred. These raw reads do not establish arbitrary-pointer safety. #[derive(Clone, Copy, Debug)] enum K610EntryError { Pointer, Range, Header(turkos_kernel::runtime::boot_header::HeaderError), } impl K610EntryError { fn tag(self) -> &'static str { match self { Self::Pointer => "bootinfo-pointer", Self::Range => "header-outside-ram", Self::Header(error) => error.tag(), } } } fn os_k610_header( address: usize, boot: &BootData, ) -> Result, K610EntryError> { use turkos_kernel::runtime::boot_header::{self, PREFIX_BYTES, V1_BYTES}; if address == 0 { return Ok(None); } turkos_bootinfo::validate_boot_info_pointer(address as u64) .map_err(|_| K610EntryError::Pointer)?; let ram_start = boot.hardware.ram_base; let ram_end = ram_start .checked_add(boot.hardware.ram_size) .ok_or(K610EntryError::Range)?; let prefix_end = (address as u64) .checked_add(PREFIX_BYTES as u64) .ok_or(K610EntryError::Range)?; if (address as u64) < ram_start || prefix_end > ram_end { return Err(K610EntryError::Range); } // SAFETY: the selected trusted QEMU/UEFI boot profile identity maps this // bounded RAM prefix. The parser below checks magic/version/declared size // before any read of the remainder of the 72-byte header. let prefix = unsafe { core::slice::from_raw_parts(address as *const u8, PREFIX_BYTES) }; boot_header::parse_prefix(prefix).map_err(K610EntryError::Header)?; let body_end = (address as u64) .checked_add(V1_BYTES as u64) .ok_or(K610EntryError::Range)?; if body_end > ram_end { return Err(K610EntryError::Range); } // SAFETY: the complete identity-mapped V1 extent is inside the DTB RAM // profile; no payload address is used as an allocator authority here. let bytes = unsafe { core::slice::from_raw_parts(address as *const u8, V1_BYTES) }; boot_header::parse_complete(bytes) .map(Some) .map_err(K610EntryError::Header) } // Injected only into a disposable development source copy. This checks the // existing finalized FreeMap, never replaces its authority or claims UEFI // LoaderData pages are available. All range reads are metadata-only. fn os_k615_regions( boot: &BootData, dtb_address: usize, uefi_info: Option<&(u64, BootInfoV1)>, free: &memory::FreeMap, ) -> Result<(usize, u64, u64), &'static str> { use turkos_kernel::runtime::regions::{Error, Inputs, Plan, Range, PAGE_BYTES}; let make = |start, bytes| Range::from_start_bytes(start, bytes).map_err(Error::tag); let image_start = core::ptr::addr_of!(_image_start) as u64; let image_end = core::ptr::addr_of!(_image_end) as u64; let image = Range::from_endpoints(image_start, image_end).map_err(Error::tag)?; let inputs = Inputs { ram: make(boot.hardware.ram_base, boot.hardware.ram_size)?, image, dtb: make(dtb_address as u64, boot.dtb_size as u64)?, payload: uefi_info .map(|(address, _)| make(*address, BOOT_INFO_PAGE_BYTES)) .transpose()?, map: uefi_info .map(|(_, info)| make(info.memory_map_ptr, BOOT_INFO_MAP_CAPACITY)) .transpose()?, }; let plan = Plan::create(inputs).map_err(Error::tag)?; let initial_pages = free.total_pages(); for region in free.regions() { plan.check_free_region(Range::from_endpoints(region.start, region.end).map_err(Error::tag)?) .map_err(Error::tag)?; } // Prove the counterexample is rejected by this same guest Image. It does // not mutate the real handoff or claim a physical malicious-loader run. let mut bad = inputs; bad.payload = Some(make(image_start + PAGE_BYTES, PAGE_BYTES)?); if bad.map.is_none() { bad.map = Some(inputs.dtb); } if Plan::create(bad) != Err(Error::Overlap) || free.total_pages() != initial_pages { return Err("synthetic-overlap-not-rejected"); } write_line("TURKOS/OS/K615: SYNTHETIC-PAYLOAD-IMAGE-OVERLAP REJECTED"); let mut allocator = memory::FrameAllocator::new(free.clone()); let before = allocator.remaining_pages(); let mut denied = 0u64; for reservation in plan.as_slice().iter().flatten() { let mut address = reservation.pages.start; while address < reservation.pages.end { if allocator.allocate_at(address) != Err(memory::FrameRequestError::OutsideAuthority) { return Err("reserved-frame-issued"); } denied += 1; address += PAGE_BYTES; } } if denied != plan.page_count() || allocator.remaining_pages() != before || allocator.free_regions().map_err(|_| "free-map-corrupt")? != free.regions() { return Err("failed-request-changed-ownership"); } // Healthy service still progresses after the failed requests, and the // exact free-region inventory is restored before normal boot continues. let first = free.regions()[0].start; let last = free.regions().last().ok_or("empty-free-map")?.end - PAGE_BYTES; if first == last || allocator.allocate_at(first) != Ok(first) || allocator.allocate_at(last) != Ok(last) || allocator.release_many(&[first, last]).is_err() || allocator.remaining_pages() != before || allocator.free_regions().map_err(|_| "free-map-corrupt")? != free.regions() { return Err("healthy-frame-service-failed"); } Ok((plan.as_slice().len(), plan.page_count(), denied)) } // Generated development Image only. Single-CPU bootstrap, IRQs masked. The // original map is validated by read_uefi_boot_info before this function. // After the snapshot, plan_free_memory is forced to use this BSS copy. #[unsafe(no_mangle)] static mut OS_K620_MAP_COPY: [u8; turkos_kernel::runtime::finalize::CAPACITY] = [0; turkos_kernel::runtime::finalize::CAPACITY]; fn os_k620_snapshot( uefi_info: Option<&(u64, BootInfoV1)>, ) -> Result>, &'static str> { use turkos_kernel::runtime::finalize::{self, ServiceGate}; let Some((_, info)) = uefi_info else { return Ok(None); }; let mut gate = ServiceGate::new(); gate.exit().map_err(|error| error.tag())?; if gate.request_service() != Err(finalize::Error::ServiceAfterExit) { return Err("post-exit-service-accepted"); } write_line("TURKOS/OS/K620: POST-EXIT-SERVICE REJECTED"); // SAFETY: read_uefi_boot_info already checked that the full map lies in // the trusted identity-mapped QEMU RAM and validated descriptor length. let source = unsafe { core::slice::from_raw_parts(info.memory_map_ptr as *const u8, info.memory_map_bytes as usize) }; // SAFETY: exactly one boot CPU calls this once before IRQs or other users // of this generated static exist. The copy remains in the kernel Image. let destination = unsafe { core::slice::from_raw_parts_mut( core::ptr::addr_of_mut!(OS_K620_MAP_COPY).cast::(), finalize::CAPACITY, ) }; finalize::snapshot(info.memory_map_ptr, source, destination, gate) .map(Some) .map_err(|error| error.tag()) } fn os_k620_source_mutation_probe( uefi_info: Option<&(u64, BootInfoV1)>, owned: Option<&turkos_kernel::runtime::finalize::OwnedMap<'_>>, ) -> Result<(), &'static str> { let Some((_, info)) = uefi_info else { if owned.is_some() { return Err("direct-has-firmware-map"); } return Ok(()); }; let map = owned.ok_or("missing-owned-map")?; if map.source_address() != info.memory_map_ptr || map.len() != info.memory_map_bytes as usize { return Err("wrong-owned-map"); } let before = map.bytes()[0]; // SAFETY: the source is a reserved loader-owned map page in the trusted // guest. This *test-only* probe saves/restores one byte after the runtime // planner has consumed only the BSS copy. It never publishes the mutation. let source = info.memory_map_ptr as *mut u8; let original = unsafe { read_volatile(source) }; unsafe { write_volatile(source, original ^ 0xff) }; let stable = map.bytes()[0] == before && unsafe { read_volatile(source) } != before; unsafe { write_volatile(source, original) }; if !stable || unsafe { read_volatile(source) } != original { return Err("owned-map-follows-source"); } write_line("TURKOS/OS/K620: SOURCE-MUTATION COPY-STABLE PASS"); Ok(()) } // Platform register boundary for the isolated default development Image. // boot.S already masked DAIF, selected the boot CPU and installed the stack. fn os_k625_capture( installed_level: usize, ) -> Result< ( turkos_kernel::runtime::entry_level::Snapshot, turkos_kernel::runtime::entry_level::Normalized, ), &'static str, > { use turkos_kernel::runtime::entry_level::{self, Snapshot}; unsafe extern "C" { static __stack_start: u8; static __stack_top: u8; } let current_el: u64; let daif: u64; let sp: u64; // SAFETY: CurrentEL, DAIF and SP are readable in privileged AArch64. // IRQs remain masked and this capture precedes any runtime allocation. unsafe { asm!( "mrs {el}, CurrentEL", "mrs {daif}, DAIF", "mov {sp}, sp", el = out(reg) current_el, daif = out(reg) daif, sp = out(reg) sp, options(nomem, nostack, preserves_flags), ); } let snapshot = Snapshot { current_el, daif, sp, stack_start: core::ptr::addr_of!(__stack_start) as u64, stack_top: core::ptr::addr_of!(__stack_top) as u64, }; let normalized = entry_level::normalize(snapshot).map_err(|error| error.tag())?; if installed_level != usize::from(normalized.level) { return Err("vector-level-mismatch"); } Ok((snapshot, normalized)) } #[used] #[unsafe(no_mangle)] #[unsafe(link_section = ".rodata.k630_canary")] static OS_K630_RODATA_CANARY: [u8; 32] = *b"K630-RODATA-CANARY-31-BYTES-0000"; #[used] #[unsafe(no_mangle)] #[unsafe(link_section = ".data.k630_canary")] static mut OS_K630_DATA_CANARY: u64 = 0x4b36_3330_4441_5441; #[used] #[unsafe(no_mangle)] #[unsafe(link_section = ".bss.k630_canary")] static mut OS_K630_BSS_CANARY: [u8; 4096] = [0; 4096]; fn os_k630_pre_mmu() -> Result { use turkos_kernel::runtime::sections::{self, Access, Error, Sections}; unsafe extern "C" { static __bss_start: u8; static __bss_end: u8; } let layout = Sections { image_start: core::ptr::addr_of!(_image_start) as u64, text_start: core::ptr::addr_of!(_text_start) as u64, text_end: core::ptr::addr_of!(_text_end) as u64, rodata_end: core::ptr::addr_of!(_rodata_end) as u64, bss_start: core::ptr::addr_of!(__bss_start) as u64, bss_end: core::ptr::addr_of!(__bss_end) as u64, } .validate() .map_err(Error::tag)?; let rodata = core::ptr::addr_of!(OS_K630_RODATA_CANARY) as u64; let data = core::ptr::addr_of!(OS_K630_DATA_CANARY) as u64; let bss = core::ptr::addr_of!(OS_K630_BSS_CANARY) as u64; if layout.access(rodata) != Some(Access::ReadOnlyNoExec) || layout.access(data) != Some(Access::ReadWriteNoExec) || layout.access(bss) != Some(Access::ReadWriteNoExec) { return Err("canary-section-mismatch"); } let mut byte = 0; while byte < 4096 { // SAFETY: the dedicated static BSS canary is inside the validated // linked Image; boot.S zeroed it before Rust entry. if unsafe { read_volatile(core::ptr::addr_of!(OS_K630_BSS_CANARY).cast::().add(byte)) } != 0 { return Err("bss-not-zero"); } byte += 1; } let mut bad = layout; bad.bss_start = layout.text_start + 4096; if bad.validate() != Err(sections::Error::Overlap) { return Err("data-text-overlap-accepted"); } write_line("TURKOS/OS/K630: BSS-ZERO 4096 PASS"); write_line("TURKOS/OS/K630: SYNTHETIC-DATA-TEXT-OVERLAP REJECTED"); Ok(layout) } fn os_k630_leaf( tables: &mmu::Tables, root: u64, address: u64, ) -> Result<(u64, u64), &'static str> { const FRAME_MASK: u64 = 0x0000_ffff_ffff_f000; let mut table = root; for shift in [30, 21, 12] { if table == 0 || !tables.owns_table_frame(table) { return Err("unowned-table-frame"); } let slot = (address >> shift) & 511; let entry = table + slot * 8; // SAFETY: Tables owns this aligned frame and the index stays in its // 4-KiB extent; mappings are stable while IRQs remain masked. let descriptor = unsafe { read_volatile(entry as *const u64) }; if descriptor & 3 != 3 { return Err("invalid-page-descriptor"); } if shift == 12 { return Ok((entry, descriptor)); } table = descriptor & FRAME_MASK; } Err("missing-page-leaf") } fn os_k630_live_mmu(tables: &mmu::Tables) -> Result<(u64, u64, u64, u64), &'static str> { const FRAME_MASK: u64 = 0x0000_ffff_ffff_f000; const AP_READ_ONLY: u64 = 0b10 << 6; const PXN: u64 = 1 << 53; let ttbr: u64; // SAFETY: TTBR0_EL1 is readable after MMU activation at EL1. unsafe { asm!("mrs {value}, TTBR0_EL1", value = out(reg) ttbr, options(nomem, nostack, preserves_flags)) }; let root = ttbr & FRAME_MASK; let rodata = core::ptr::addr_of!(OS_K630_RODATA_CANARY) as u64; let data = core::ptr::addr_of!(OS_K630_DATA_CANARY) as u64; let (ro_leaf, ro_value) = os_k630_leaf(tables, root, rodata)?; let (data_leaf, data_value) = os_k630_leaf(tables, root, data)?; if ro_value & AP_READ_ONLY == 0 || ro_value & PXN == 0 || data_value & AP_READ_ONLY != 0 || data_value & PXN == 0 { return Err("rodata-data-permissions"); } if unsafe { read_volatile(core::ptr::addr_of!(OS_K630_DATA_CANARY)) } != 0x4b36_3330_4441_5441 { return Err("data-canary-drift"); } write_line("TURKOS/OS/K630: RODATA-RO DATA-RW PASS"); Ok((ro_leaf, ro_value, data_leaf, data_value)) } // K635 platform boundary. The isolated boot.S reserves a real guard page and // seeds both stack canaries before its first call into Rust. The boot CPU owns // this singleton; DAIF remains masked throughout the acceptance path. static mut OS_K635_STACK_OWNER: Option = None; static mut OS_K635_GUARD_FAULT_ARMED: bool = false; fn os_k635_sp() -> u64 { let sp: u64; // SAFETY: privileged AArch64 SP capture does not change the stack. unsafe { asm!("mov {value}, sp", value = out(reg) sp, options(nomem, nostack, preserves_flags)) }; sp } fn os_k635_canaries(stack_start: u64) -> [u64; 2] { // SAFETY: callers obtained the page-aligned, linked boot-stack extent from // the image. These two words are reserved and seeded before Rust entry. unsafe { [ read_volatile(stack_start as *const u64), read_volatile((stack_start + 8) as *const u64), ] } } fn os_k635_capture() -> Result<(), &'static str> { use turkos_kernel::runtime::entry_stack::{Error, Layout, Owner, StackOwner}; unsafe extern "C" { static __boot_guard_page_start: u8; static __stack_start: u8; static __stack_top: u8; } let layout = Layout { image_start: core::ptr::addr_of!(_image_start) as u64, image_end: core::ptr::addr_of!(_image_end) as u64, guard_start: core::ptr::addr_of!(__boot_guard_page_start) as u64, stack_start: core::ptr::addr_of!(__stack_start) as u64, stack_top: core::ptr::addr_of!(__stack_top) as u64, }; let mut owner = StackOwner::claim(layout, Owner::BootCpu(0)).map_err(Error::tag)?; let sp = os_k635_sp(); owner .activate(sp, os_k635_canaries(layout.stack_start)) .map_err(Error::tag)?; let before = owner.snapshot(); if owner.check_access(layout.stack_start - 8, 8) != Err(Error::GuardAccess) || owner.snapshot() != before || owner .verify(sp, os_k635_canaries(layout.stack_start)) .is_err() { return Err("guard-reject-ownership-drift"); } // SAFETY: one boot CPU, IRQs masked, exactly one invocation before any // runtime allocation. The checked owner is published only after success. unsafe { let destination = core::ptr::addr_of_mut!(OS_K635_STACK_OWNER); if (*destination).is_some() { return Err("duplicate-stack-owner"); } write_volatile(destination, Some(owner)); } write_str("TURKOS/OS/K635: STACK GUARD=0x"); write_hex(layout.guard_start as usize); write_str(" START=0x"); write_hex(layout.stack_start as usize); write_str(" TOP=0x"); write_hex(layout.stack_top as usize); write_str(" SP=0x"); write_hex(sp as usize); write_str(" PASS\r\n"); write_line("TURKOS/OS/K635: BOOT-CANARY PAIR PASS"); write_line( "TURKOS/OS/K635: SYNTHETIC-GUARD-ACCESS REJECTED OWNER=BOOT-CPU0 IMAGE-PAGES=5 ALLOCATOR-PAGES=0 UNCHANGED", ); Ok(()) } fn os_k635_post_mmu( tables: &mmu::Tables, allocator: &memory::FrameAllocator, layout: mmu::Layout, ) -> Result<(), &'static str> { use turkos_kernel::runtime::entry_stack::{Error, GuardEvidence, State}; // SAFETY: capture initialized the singleton on this CPU before reaching // MMU setup. IRQs are still masked; no other reference exists. let owner = unsafe { (*core::ptr::addr_of_mut!(OS_K635_STACK_OWNER)).as_mut() } .ok_or("stack-owner-missing")?; let before = owner.snapshot(); owner .verify(os_k635_sp(), os_k635_canaries(before.layout.stack_start)) .map_err(Error::tag)?; let free_before = allocator.remaining_pages(); let frames_before = tables.frame_count(); // SAFETY: the image exclusively owns this page immediately below the live // boot stack. The method checks image boundaries and the writable leaf. // Only this CPU accesses the owned table; none accesses the guard VA. unsafe { tables.unmap_task_stack_guard( before.layout.guard_start, before.layout.stack_start, layout.image_start, layout.image_end, ) } .map_err(|_| "boot-guard-unmap")?; // SAFETY: leaf invalidation precedes this complete local EL1 invalidation; // DAIF is masked and the guard contains no live stack data or instructions. unsafe { asm!( "dsb ishst", "tlbi vmalle1", "dsb ish", "isb", options(nostack, preserves_flags) ) }; if !tables.task_stack_guard_unmapped(before.layout.guard_start) || free_before != allocator.remaining_pages() || frames_before != tables.frame_count() { return Err("boot-guard-table-or-owner-drift"); } owner .protect_guard(GuardEvidence { owner: before.owner, address: before.layout.guard_start, descriptor: 0, tlbi_complete: true, }) .map_err(Error::tag)?; let after = owner.snapshot(); if after.state != State::Protected || after.layout != before.layout || after.owner != before.owner || after.image_reserved_pages != before.image_reserved_pages || after.allocator_pages != before.allocator_pages { return Err("boot-guard-publication-drift"); } let sp = os_k635_sp(); owner .verify(sp, os_k635_canaries(before.layout.stack_start)) .map_err(Error::tag)?; if owner.check_access(before.layout.stack_start - 8, 8) != Err(Error::GuardAccess) { return Err("protected-guard-accepted"); } write_str("TURKOS/OS/K635: LIVE-SP=0x"); write_hex(sp as usize); write_str(" CANARY=0x"); write_hex(os_k635_canaries(before.layout.stack_start)[0] as usize); write_str(" PASS\r\n"); write_str("TURKOS/OS/K635: GUARD-UNMAPPED=0x"); write_hex(before.layout.guard_start as usize); write_str(" DESCRIPTOR=0 TLBI=COMPLETE PASS\r\n"); write_str("TURKOS/OS/K635: OWNERSHIP IMAGE-PAGES=5 ALLOCATOR-PAGES=0 FREE-BEFORE=0x"); write_hex(free_before as usize); write_str(" FREE-AFTER=0x"); write_hex(allocator.remaining_pages() as usize); write_str(" TABLE-FRAMES=0x"); write_hex(frames_before); write_str(" UNCHANGED PASS\r\n"); Ok(()) } fn os_k635_guard_read() -> ! { use turkos_kernel::runtime::entry_stack::State; // SAFETY: one CPU owns this singleton; capture and protection completed. let owner = unsafe { (*core::ptr::addr_of!(OS_K635_STACK_OWNER)).as_ref() }; let Some(owner) = owner else { park() }; let state = owner.snapshot(); if state.state != State::Protected { write_line("TURKOS/OS/K635: GUARD-READ NOT-PROTECTED"); park(); } let target = state.layout.stack_start - 1; // SAFETY: this single-CPU negative image owns the fault expectation flag. unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K635_GUARD_FAULT_ARMED), true) }; write_str("TURKOS/OS/K635: GUARD-READ ARMED TARGET=0x"); write_hex(target as usize); write_str("\r\n"); // SAFETY: deliberate negative acceptance access to the confirmed unmapped // guard. It must enter the synchronous exception vector and never return. unsafe { read_volatile(target as *const u8) }; write_line("TURKOS/OS/K635: GUARD-READ RETURNED ERROR"); park() } fn os_k635_expected_guard_fault( level: usize, esr: usize, slot: usize, frame: *const ExceptionFrame, ) -> bool { // SAFETY: only the negative probe arms this flag on the boot CPU. if !unsafe { read_volatile(core::ptr::addr_of!(OS_K635_GUARD_FAULT_ARMED)) } { return false; } let far: u64; // SAFETY: synchronous fault reporting occurs at EL1 and FAR is readable. unsafe { asm!("mrs {value}, FAR_EL1", value = out(reg) far, options(nomem, nostack, preserves_flags)) }; // SAFETY: initialized before MMU setup, one CPU and IRQs masked. Vectors // passed their complete aligned fatal exception frame in x4. let owner = unsafe { (*core::ptr::addr_of!(OS_K635_STACK_OWNER)).as_ref() }; let Some(owner) = owner else { return false }; let state = owner.snapshot(); let context = unsafe { &*frame }; let expected = level == 1 && slot == 4 && (esr >> 26) & 0x3f == 0x25 && esr & 0x3f == 0x07 && esr & (1 << 6) == 0 && far == state.layout.stack_start - 1 && context.canary_valid() && owner .verify( context.interrupted_sp as u64, os_k635_canaries(state.layout.stack_start), ) .is_ok(); write_str("TURKOS/OS/K635: GUARD-FAULT ESR=0x"); write_hex(esr); write_str(" FAR=0x"); write_hex(far as usize); write_str(" SP=0x"); write_hex(context.interrupted_sp); write_str(if expected { " CANARY=PASS OWNER=UNCHANGED PASS\r\n" } else { " REJECTED\r\n" }); expected } // Generated development Image only. The capture lives in initialized .data: // boot.S writes it before x0/x1 are reused; BSS clearing cannot erase it. // The acceptance profile has exactly one vCPU with all DAIF bits masked. #[used] #[unsafe(no_mangle)] #[unsafe(link_section = ".data.k640_arguments")] static mut OS_K640_LOADER_ARGUMENTS: turkos_kernel::runtime::arguments::LoaderWitness = turkos_kernel::runtime::arguments::LoaderWitness { registers: turkos_kernel::runtime::arguments::Registers { x0: u64::MAX, x1: u64::MAX, }, cookie: u64::MAX, }; fn os_k640_arguments( dtb_address: usize, boot_info_address: usize, ) -> Result { use turkos_kernel::runtime::arguments::{self, BootPath, Error, Gate, Registers, State}; // SAFETY: this is an aligned linked .data static, written once by primary // entry assembly before Rust. One vCPU, masked IRQs and no aliases exist. // Reading the numeric witness does not read any loader-owned buffer. let loader = unsafe { read_volatile(core::ptr::addr_of!(OS_K640_LOADER_ARGUMENTS)) }; let observed = Registers { x0: dtb_address as u64, x1: boot_info_address as u64, }; write_str("TURKOS/OS/K640: ARGUMENTS witness-x0=0x"); write_hex(loader.registers.x0 as usize); write_str(" witness-x1=0x"); write_hex(loader.registers.x1 as usize); write_str(" observed-x0=0x"); write_hex(observed.x0 as usize); write_str(" observed-x1=0x"); write_hex(observed.x1 as usize); write_str("\r\n"); // Reject a live assembly clobber before DTB/BootInfo parsing or allocation. arguments::verify(loader, observed).map_err(Error::tag)?; let mut gate = Gate::new(); let before = gate; let mut changed = observed; changed.x1 ^= 1; if gate.publish(loader, changed) != Err(Error::X1Changed) || gate != before || gate.state() != State::Unpublished || gate.record_count() != 0 { return Err("arguments-partial-publication"); } write_line("TURKOS/OS/K640: SYNTHETIC-X1-CLOBBER REJECTED RECORDS=0"); let accepted = gate.publish(loader, observed).map_err(Error::tag)?; if accepted.registers() != observed || gate.published() != Some(accepted) || gate.state() != State::Published || gate.record_count() != 1 { return Err("arguments-publication-mismatch"); } write_str("TURKOS/OS/K640: LOADER-EQUAL source="); write_str(match accepted.path() { BootPath::Direct => "DIRECT", BootPath::Uefi => "UEFI", }); write_str(" RECORDS=1 REGIONS-ADOPTED=0 PASS\r\n"); Ok(accepted) } // Append only to the isolated kernel main.rs. Copy runtime.rs to the isolated // library's runtime/services.rs and add `pub mod services` there. // Integration order: // 1. replace the mmu_smoke_test allocator construction with os_k660_new; // all existing MMU work borrows runtime.memory_mut()?.allocator; // 2. map boot GICD (64 KiB) and GICR (128 KiB) as Device before MMU activation; // 3. route vector slot 5 to return.S (sgi_test=1), include return.S with the // existing IrqContext offsets, and use this kernel_return_probe; // 4. after MMU checks call os_k660_accept, then retain the runtime in the // never-returning guest loop. Product and acceptance have distinct manifests. // Hardware scope: one CPU, GICv3/v4 PPI 30, bounded cooperative scheduler work. // No virtio DMA is enabled. The live-DMA rejection below is a synthetic // counterexample, explicitly labeled, alongside real frame ownership rollback. use turkos_kernel::runtime::services as os_k660_policy; struct OsK660Uart { base: u64, } struct OsK660Memory { allocator: memory::FrameAllocator, } struct OsK660Irq { gicr: u64, installed: bool, } struct OsK660Tasks { scheduler: turkos_kernel::task::Scheduler, id: Option, frame: Option, synthetic_dma_active: bool, progress: u64, } impl os_k660_policy::Service for OsK660Uart { fn ownership(&self) -> os_k660_policy::Ownership { os_k660_policy::Ownership::default() } fn activity(&self) -> os_k660_policy::Activity { os_k660_policy::Activity::default() } } impl os_k660_policy::Service for OsK660Memory { fn ownership(&self) -> os_k660_policy::Ownership { // MMU frames belong to the already validated Tables owner. The // allocator owns their authority/metadata, not a second frame lease. os_k660_policy::Ownership::default() } fn activity(&self) -> os_k660_policy::Activity { os_k660_policy::Activity::default() } } impl os_k660_policy::Service for OsK660Irq { fn ownership(&self) -> os_k660_policy::Ownership { os_k660_policy::Ownership { irq_bindings: self.installed.into(), ..os_k660_policy::Ownership::default() } } fn activity(&self) -> os_k660_policy::Activity { // Single CPU; callers keep IRQ masked while observing this value. os_k660_policy::Activity { irq_callbacks: unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) }, dma_requests: 0, } } } impl os_k660_policy::Service for OsK660Tasks { fn ownership(&self) -> os_k660_policy::Ownership { os_k660_policy::Ownership { frames: self.frame.is_some().into(), tasks: self.id.is_some().into(), dma_buffers: self.synthetic_dma_active.into(), ..os_k660_policy::Ownership::default() } } fn activity(&self) -> os_k660_policy::Activity { os_k660_policy::Activity { irq_callbacks: unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) }, dma_requests: self.synthetic_dma_active.into(), } } } type OsK660Runtime = os_k660_policy::Runtime; // Stable storage avoids copying the allocator's bounded metadata onto the // 16-KiB boot stack. Only the boot CPU initializes and borrows this object; // interrupt handlers never borrow the runtime or allocator. static mut OS_K660_RUNTIME_STORAGE: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); static mut OS_K660_RUNTIME_INITIALIZED: bool = false; #[used] #[unsafe(no_mangle)] #[unsafe(link_section = ".rodata.os_runtime_manifest")] static OS_K660_MANIFEST: [u8; 25] = if cfg!(feature = "os-runtime-acceptance") { *b"K660:ACCEPTANCE:PROBES=07" } else { *b"K660:PRODUCT:PROBES=00000" }; #[unsafe(no_mangle)] static mut OS_K660_IRQ_COUNT: u64 = 0; static mut OS_K660_IRQ_ACTIVE: u32 = 0; static mut OS_K660_IRQ_LAST_COUNTER: u64 = 0; const OS_K660_IRQ_GUARD: u64 = 0x4b36_3630_4952_5147; #[unsafe(no_mangle)] static mut OS_K660_RUNTIME_EVIDENCE: [u64; 9] = [0; 9]; #[unsafe(no_mangle)] static mut OS_K660_GIC_ENABLE_READBACK: u32 = 0; #[allow(clippy::result_large_err)] fn os_k660_new( free: memory::FreeMap, uart_base: u64, ) -> Result<&'static mut OsK660Runtime, &'static str> { use os_k660_policy::{Manifest, Profile}; let manifest = Manifest { profile: if cfg!(feature = "os-runtime-acceptance") { Profile::Acceptance } else { Profile::Product }, probes: if cfg!(feature = "os-runtime-acceptance") { os_k660_policy::KNOWN_PROBES } else { 0 }, } .validate() .map_err(os_k660_policy::Error::tag)?; // SAFETY: this one-CPU entry executes once with IRQ masked, before any // service publishes a reference. The explicit guard forbids overwriting a // live owner; the never-returning boot function retains the unique borrow. let runtime = unsafe { if read_volatile(core::ptr::addr_of!(OS_K660_RUNTIME_INITIALIZED)) { return Err("duplicate-runtime"); } let storage = core::ptr::addr_of_mut!(OS_K660_RUNTIME_STORAGE).cast::(); storage.write(OsK660Runtime::new(manifest)); write_volatile(core::ptr::addr_of_mut!(OS_K660_RUNTIME_INITIALIZED), true); &mut *storage }; runtime .start_boot(|| { if uart_base == 0 || UART_BASE.load(Ordering::Relaxed) as u64 != uart_base { return Err(os_k660_policy::InitFailure { cause: os_k660_policy::Cause::Hardware, acquired: None, }); } // Boot discovery has configured this bootstrap UART. The service adopts // that one mapping; it does not register a second UART/allocator. Ok(OsK660Uart { base: uart_base }) }) .map_err(os_k660_policy::Error::tag)?; write_line("TURKOS/OS/K645: STAGE BOOT UART=1"); if cfg!(feature = "os-runtime-acceptance") { let before = runtime.snapshot().map_err(os_k660_policy::Error::tag)?; if runtime.start_tasks(|_, _| panic!("scheduler ran before memory")) != Err(os_k660_policy::Error::Dependency { service: os_k660_policy::ServiceId::Scheduler, have: os_k660_policy::Stage::Boot, }) || runtime.snapshot().map_err(os_k660_policy::Error::tag)? != before { return Err("scheduler-before-memory-accepted"); } write_line("TURKOS/OS/K645: SYNTHETIC-SCHEDULER-BEFORE-MEMORY REJECTED"); } runtime .start_memory(|uart| { if uart.base != uart_base { return Err(os_k660_policy::InitFailure { cause: os_k660_policy::Cause::Corrupt, acquired: None, }); } Ok(OsK660Memory { allocator: os_k720_allocator(free), }) }) .map_err(os_k660_policy::Error::tag)?; write_line("TURKOS/OS/K645: STAGE MEMORY ALLOCATOR=1"); Ok(runtime) } /// IRQ must stay masked; GICD/GICR must already have validated Device mappings. fn os_k660_install_irq( boot: &BootData, ) -> Result> { use os_k660_policy::{Cause, InitFailure}; let gicd = boot.interrupts.gic_distributor; let gicr = boot.interrupts.gic_redistributor; let mut owner = OsK660Irq { gicr, installed: false, }; let mut setup = || -> Result<(), Cause> { if gicd == 0 || gicr == 0 || gicd & 0xffff != 0 || gicr & 0xffff != 0 || boot.interrupts.nonsecure_phys_timer_ppi != 30 { return Err(Cause::Hardware); } // SAFETY: this isolated single-CPU Image owns validated Device GIC // windows; all setup and partial rollback occur with IRQ masked. unsafe { let reg = |base: u64, offset: u64| (base + offset) as *mut u32; let mpidr: u64; let daif: u64; asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack)); asm!("mrs {value}, DAIF", value = out(reg) daif, options(nomem, nostack)); if daif & 0x80 == 0 || turkos_kernel::irq::gic_arch_revision(read_volatile(reg(gicd, 0xffe8))).is_none() || turkos_kernel::irq::gic_arch_revision(read_volatile(reg(gicr, 0xffe8))).is_none() || !turkos_kernel::irq::redistributor_matches_mpidr( read_volatile((gicr + 8) as *const u64), mpidr, ) { return Err(Cause::Hardware); } let waker = reg(gicr, 0x14); write_volatile(waker, read_volatile(waker) & !2); asm!("dsb sy", options(nostack)); let mut awake = false; for _ in 0..1_000_000 { if read_volatile(waker) & 4 == 0 { awake = true; break; } } if !awake { return Err(Cause::Busy); } let control = reg(gicd, 0); write_volatile(control, read_volatile(control) | 0x22); let mut ready = false; for _ in 0..1_000_000 { if read_volatile(control) & (1 << 31) == 0 { ready = true; break; } } if !ready { return Err(Cause::Busy); } let group = reg(gicr, 0x1_0080); write_volatile(group, read_volatile(group) | (1 << 30)); write_volatile((gicr + 0x1_0400 + 30) as *mut u8, 0x80); write_volatile(reg(gicr, 0x1_0100), 1 << 30); owner.installed = true; asm!("dsb sy", options(nostack)); ready = false; for _ in 0..1_000_000 { if read_volatile(reg(gicr, 0)) & 8 == 0 { ready = true; break; } } if !ready || read_volatile(reg(gicr, 0x1_0100)) & (1 << 30) == 0 { return Err(Cause::Busy); } let sre: u64; asm!("mrs {value}, ICC_SRE_EL1", value = out(reg) sre, options(nomem, nostack)); asm!("msr ICC_SRE_EL1, {value}", "isb", value = in(reg) (sre | 1), options(nomem, nostack)); let sre_after: u64; asm!("mrs {value}, ICC_SRE_EL1", value = out(reg) sre_after, options(nomem, nostack)); if sre_after & 1 == 0 { return Err(Cause::Hardware); } asm!("msr CNTP_CTL_EL0, xzr", "msr ICC_PMR_EL1, {priority}", "msr ICC_BPR1_EL1, xzr", "msr ICC_IGRPEN1_EL1, {enable}", "isb", priority = in(reg) 0xffu64, enable = in(reg) 1u64, options(nomem, nostack)); write_volatile( core::ptr::addr_of_mut!(__irq_stack_guard), OS_K660_IRQ_GUARD, ); } Ok(()) }; let result = setup(); match result { Ok(()) => Ok(owner), Err(cause) => Err(InitFailure { cause, acquired: Some(owner), }), } } #[unsafe(no_mangle)] pub extern "C" fn kernel_return_probe(frame: *const turkos_kernel::irq::IrqContext) -> usize { if frame.is_null() { return 0; } let low = core::ptr::addr_of!(__irq_stack_start) as usize; let high = core::ptr::addr_of!(__irq_stack_top) as usize; let sp: usize; let iar: u64; let now: u64; // SAFETY: return.S passes its one guarded frame at EL1 with IRQ masked. unsafe { asm!("mov {value}, sp", value = out(reg) sp, options(nomem, nostack)); asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack)); asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack)); asm!("mrs {value}, CNTPCT_EL0", value = out(reg) now, options(nomem, nostack)); } if !matches!( turkos_kernel::irq::classify_iar(iar, 30), turkos_kernel::irq::IarClass::Expected ) { if !matches!( turkos_kernel::irq::classify_iar(iar, 30), turkos_kernel::irq::IarClass::Special(_) ) { unsafe { asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } } return 0; } let valid = frame as usize == high - size_of::() && turkos_kernel::irq::handler_sp_in_stack(low, frame as usize, sp) && unsafe { (*frame).vector_slot == 5 && (*frame).spsr & 0x8f == 5 } && unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) } == OS_K660_IRQ_GUARD; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE), 1); asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); let control: u64; asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack)); if control & 2 != 0 { asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack)); } if valid { if !os_k780_irq_gate() { return 0; } let count = read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)); if let Some(next) = count.checked_add(1) { write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT), next); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER), now); } else { return 0; } } write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE), 0); } usize::from(valid) } #[allow(clippy::result_large_err)] fn os_k660_tasks( memory: &mut OsK660Memory, irq: &mut OsK660Irq, ) -> Result> { use os_k660_policy::{Cause, InitFailure}; if !irq.installed { return Err(InitFailure { cause: Cause::Unavailable, acquired: None, }); } let frame = memory.allocator.allocate().ok_or(InitFailure { cause: Cause::Exhausted, acquired: None, })?; let mut owner = OsK660Tasks { scheduler: turkos_kernel::task::Scheduler::new(), id: None, frame: Some(frame), synthetic_dma_active: false, progress: 0, }; match owner.scheduler.spawn() { Ok(id) => { owner.id = Some(id); Ok(owner) } Err(_) => Err(InitFailure { cause: Cause::Exhausted, acquired: Some(owner), }), } } struct OsK660Rollback { clear_synthetic_dma: bool, } impl os_k660_policy::Rollback for OsK660Rollback { fn quiesce_uart(&mut self, _: &mut OsK660Uart) -> Result<(), os_k660_policy::Cause> { Ok(()) } fn release_uart(&mut self, _: &mut OsK660Uart) -> Result<(), os_k660_policy::Cause> { Ok(()) } fn quiesce_memory(&mut self, _: &mut OsK660Memory) -> Result<(), os_k660_policy::Cause> { Ok(()) } fn release_memory(&mut self, _: &mut OsK660Memory) -> Result<(), os_k660_policy::Cause> { Ok(()) } fn quiesce_irq(&mut self, _: &mut OsK660Irq) -> Result<(), os_k660_policy::Cause> { unsafe { asm!( "msr DAIFSet, #2", "msr CNTP_CTL_EL0, xzr", "dsb sy", "isb", options(nostack) ); } Ok(()) } fn release_irq( &mut self, irq: &mut OsK660Irq, _: &mut OsK660Memory, ) -> Result<(), os_k660_policy::Cause> { unsafe { write_volatile((irq.gicr + 0x1_0180) as *mut u32, 1 << 30); asm!("dsb sy", options(nostack)); let mut ready = false; for _ in 0..1_000_000 { if read_volatile(irq.gicr as *const u32) & 8 == 0 { ready = true; break; } } if !ready || read_volatile((irq.gicr + 0x1_0100) as *const u32) & (1 << 30) != 0 { return Err(os_k660_policy::Cause::Busy); } } irq.installed = false; Ok(()) } fn quiesce_tasks( &mut self, tasks: &mut OsK660Tasks, irq: &mut OsK660Irq, ) -> Result<(), os_k660_policy::Cause> { self.quiesce_irq(irq)?; if self.clear_synthetic_dma { tasks.synthetic_dma_active = false; } Ok(()) } fn release_tasks( &mut self, tasks: &mut OsK660Tasks, memory: &mut OsK660Memory, _: &mut OsK660Irq, ) -> Result<(), os_k660_policy::Cause> { if let Some(id) = tasks.id { if tasks.scheduler.run_next() != Ok(Some(id)) || tasks.scheduler.exit_current(0) != Ok(None) || tasks.scheduler.reap(id) != Ok(0) { return Err(os_k660_policy::Cause::Corrupt); } tasks.id = None; } if let Some(frame) = tasks.frame { memory .allocator .release(frame) .map_err(|_| os_k660_policy::Cause::Corrupt)?; tasks.frame = None; } Ok(()) } } #[allow(clippy::result_large_err)] fn os_k660_accept(runtime: &mut OsK660Runtime, boot: &BootData) -> Result<(), &'static str> { use os_k660_policy::{Cause, Error, InitFailure, Manifest, Profile, ServiceId, Stage}; let is_acceptance = runtime.manifest().profile == Profile::Acceptance; if is_acceptance { let before = runtime.snapshot().map_err(Error::tag)?; let free = runtime .memory() .ok_or("allocator-missing")? .allocator .remaining_pages(); if runtime.start_memory(|_| panic!("duplicate allocator initializer ran")) != Err(Error::Duplicate(ServiceId::Allocator)) || runtime.snapshot().map_err(Error::tag)? != before || runtime .memory() .ok_or("allocator-lost")? .allocator .remaining_pages() != free { return Err("duplicate-allocator-accepted"); } write_line("TURKOS/OS/K650: DUPLICATE-ALLOCATOR REJECTED OWNER-PRESERVED"); if (Manifest { profile: Profile::Product, probes: os_k660_policy::PROBE_INIT_FAILURE, }) .validate() != Err(Error::ProductProbe) { return Err("negative-probe-product-contamination"); } write_line("TURKOS/OS/K655: NEGATIVE-PROBE-IN-PRODUCT REJECTED"); } runtime .start_irq(|_| os_k660_install_irq(boot)) .map_err(Error::tag)?; write_line("TURKOS/OS/K645: STAGE IRQ GIC-PPI=30 INSTALLED"); if is_acceptance { let before = runtime.snapshot().map_err(Error::tag)?; let free = runtime .memory() .ok_or("allocator-missing")? .allocator .remaining_pages(); let result = runtime.start_tasks(|memory, irq| { let mut owner = os_k660_tasks(memory, irq)?; owner.synthetic_dma_active = true; Err(InitFailure { cause: Cause::Injected, acquired: Some(owner), }) }); if result != Err(Error::Init { service: ServiceId::Scheduler, cause: Cause::Injected, }) || runtime.snapshot().map_err(Error::tag)?.stage != Stage::Irq || runtime .memory() .ok_or("allocator-missing")? .allocator .remaining_pages() .checked_add(1) != Some(free) { return Err("failure-acquisition-account"); } let retained = runtime.snapshot().map_err(Error::tag)?; let acquired_free = runtime .memory() .ok_or("allocator-missing")? .allocator .remaining_pages(); let mut rollback = OsK660Rollback { clear_synthetic_dma: false, }; if runtime.rollback_failed(&mut rollback) != Err(Error::LiveDma(ServiceId::Scheduler)) || runtime.snapshot().map_err(Error::tag)? != retained || runtime .memory() .ok_or("allocator-missing")? .allocator .remaining_pages() .checked_add(1) != Some(free) { return Err("live-dma-early-release"); } write_line("TURKOS/OS/K660: SYNTHETIC-LIVE-DMA REJECTED FRAME-RETAINED"); let rejected_free = runtime .memory() .ok_or("allocator-missing")? .allocator .remaining_pages(); rollback.clear_synthetic_dma = true; runtime.rollback_failed(&mut rollback).map_err(Error::tag)?; if runtime.snapshot().map_err(Error::tag)? != before || runtime .memory() .ok_or("allocator-missing")? .allocator .remaining_pages() != free { return Err("failed-init-rollback-leak"); } write_line("TURKOS/OS/K660: FAILED-TASK-INIT ROLLBACK FRAME-ACCOUNT-PRESERVED"); unsafe { let destination = core::ptr::addr_of_mut!(OS_K660_RUNTIME_EVIDENCE).cast::(); for (index, value) in [ free, acquired_free, rejected_free, runtime .memory() .ok_or("allocator-missing")? .allocator .remaining_pages(), ] .into_iter() .enumerate() { write_volatile(destination.add(5 + index), value); } } } runtime.start_tasks(os_k660_tasks).map_err(Error::tag)?; if runtime.snapshot().map_err(Error::tag)?.registered != [true; 4] { return Err("runtime-registrations"); } write_line("TURKOS/OS/K645: STAGE TASKS SCHEDULER=1"); write_str("TURKOS/OS/K655: MANIFEST "); for byte in OS_K660_MANIFEST { write_byte(byte); } write_str("\r\n"); os_k660_work(runtime)?; if is_acceptance { write_line("TURKOS/OS/K645: PASS"); write_line("TURKOS/OS/K650: PASS"); write_line("TURKOS/OS/K655: PASS"); write_line("TURKOS/OS/K660: PASS"); } else { write_line("TURKOS/OS/K660: PRODUCT-READY"); } Ok(()) } fn os_k660_work(runtime: &mut OsK660Runtime) -> Result<(), &'static str> { use os_k660_policy::Error; let (_, memory, irq, tasks) = runtime.running().map_err(Error::tag)?; if !irq.installed { return Err("irq-service-missing"); } let free_before = memory.allocator.remaining_pages(); let id = tasks.id.ok_or("task-missing")?; let frame = tasks.frame.ok_or("task-frame-missing")?; let frequency: u64; unsafe { asm!("mrs {value}, CNTFRQ_EL0", value = out(reg) frequency, options(nomem, nostack)); } let interval = turkos_kernel::irq::timer_ticks_for_ns(frequency, 1_000_000).ok_or("timer-frequency")?; if tasks.scheduler.run_next() != Ok(Some(id)) { return Err("scheduler-selection"); } for iteration in 1..=4u64 { if tasks.scheduler.running() != Some(id) { return Err("scheduler-selection"); } let count_before = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; let now: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) now, options(nomem, nostack)); } let deadline = now.checked_add(interval).ok_or("timer-deadline")?; let timeout = now.checked_add(frequency).ok_or("timer-timeout")?; // Real scheduler-selected work writes only its acquired frame. This // bounded cooperative worker does not yet switch independent stacks. unsafe { write_volatile(frame as *mut u64, iteration ^ 0x4b36_3630_574f_524b); } unsafe { asm!("msr CNTP_CVAL_EL0, {value}", "msr CNTP_CTL_EL0, {enable}", "isb", "msr DAIFClr, #2", value = in(reg) deadline, enable = in(reg) 1u64, options(nostack)); } loop { let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; let current: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value = out(reg) current, options(nomem, nostack)); } if count != count_before { break; } if current > timeout { unsafe { asm!( "msr DAIFSet, #2", "msr CNTP_CTL_EL0, xzr", "isb", options(nostack) ); } return Err("runtime-timer-timeout"); } tasks.progress = tasks .progress .checked_add(1) .ok_or("task-progress-overflow")?; } unsafe { asm!("msr DAIFSet, #2", "isb", options(nostack)); } if unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) } != count_before + 1 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)) } < deadline || unsafe { read_volatile(frame as *const u64) } != iteration ^ 0x4b36_3630_574f_524b || memory.allocator.remaining_pages() != free_before || tasks.scheduler.yield_current(false) != Ok(Some(id)) { return Err("runtime-task-irq-account"); } } if tasks.progress == 0 { return Err("runtime-task-no-progress"); } let gic_enable = unsafe { read_volatile((irq.gicr + 0x1_0100) as *const u32) }; if gic_enable & (1 << 30) == 0 { return Err("runtime-gic-source-not-enabled"); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K660_GIC_ENABLE_READBACK), gic_enable, ); } unsafe { let destination = core::ptr::addr_of_mut!(OS_K660_RUNTIME_EVIDENCE).cast::(); for (index, value) in [ read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)), tasks.progress, free_before, memory.allocator.remaining_pages(), frame, ] .into_iter() .enumerate() { write_volatile(destination.add(index), value); } } write_str("TURKOS/OS/K645: LIVE TIMER-DELIVERIES=4 TASK-PROGRESS=0x"); write_hex(tasks.progress as usize); write_str(" FREE-PAGES=0x"); write_hex(free_before as usize); write_str(" TASK-FRAME=0x"); write_hex(frame as usize); write_str(" GIC-ENABLE=0x"); write_hex(gic_enable as usize); write_str("\r\n"); Ok(()) } global_asm!(include_str!("return.S"), ctx_size = const size_of::(), sp_offset = const offset_of!(turkos_kernel::irq::IrqContext, interrupted_sp), sp_el0_offset = const offset_of!(turkos_kernel::irq::IrqContext, sp_el0), elr_offset = const offset_of!(turkos_kernel::irq::IrqContext, elr), spsr_offset = const offset_of!(turkos_kernel::irq::IrqContext, spsr), q_offset = const offset_of!(turkos_kernel::irq::IrqContext, q), fpcr_offset = const offset_of!(turkos_kernel::irq::IrqContext, fpcr), fpsr_offset = const offset_of!(turkos_kernel::irq::IrqContext, fpsr), slot_offset = const offset_of!(turkos_kernel::irq::IrqContext, vector_slot), nested_irq = const 0); #[cfg(any(feature = "p12-journal-test", feature = "uefi-ownership-test", feature = "user-three-test", feature = "irq-service-test", feature = "irq-spurious-vector-test", feature = "irq-spurious-vector-guard-test", feature = "process-exit-irq-test", feature = "process-exit-stale-irq-test", feature = "panic-test", feature = "exception-test", feature = "fiq-label-test", feature = "serror-label-test", feature = "log-secret-test", feature = "fault-test", feature = "frame-stress-test", feature = "mmu-test", feature = "mmu-no-map-test", feature = "mmu-write-test", feature = "mmu-nx-test", feature = "mmu-rodata-test", feature = "mmu-rodata-exec-test", feature = "mmu-stack-exec-test", feature = "mmu-mmio-exec-test", feature = "mmu-panic-test", feature = "mmu-brk-test", feature = "mmu-udf-test", feature = "mmu-guard-test", feature = "mmu-normal-test", feature = "mmu-tlb-test", feature = "el0-fault-test", feature = "nested-fault-test", feature = "irq-return-test", feature = "irq-sgi-test", feature = "irq-guard-test", feature = "irq-gp-test", feature = "heap-test", feature = "heap-panic-test", feature = "task-state-test", feature = "task-switch-test", feature = "task-lifecycle-test", feature = "task-preempt-test", feature = "task-ipc-timer-test", feature = "task-fair-test", feature = "task-guard-test", feature = "user-first-test", feature = "user-code-write-test", feature = "user-data-exec-test", feature = "user-stack-overflow-test", feature = "user-align-test", feature = "user-dual-test", feature = "user-dual-fault-test", feature = "user-dual-stack-fault-test", feature = "user-peer-read-test", feature = "user-mmio-read-test", feature = "user-asid-test", feature = "user-syscall-test", feature = "user-copy-test", feature = "ipc-user-test", feature = "user-copy-irq-test", feature = "user-el0-irq-test", feature = "user-task-test", feature = "user-clock-test", feature = "user-handle-test", feature = "user-handle-transfer-test", feature = "user-handle-death-test", feature = "user-wait-test", feature = "user-mmio-request-test", feature = "elf-ident-test", feature = "elf-layout-test", feature = "elf-load-test", feature = "elf-stack-test", feature = "elf-hello-test", feature = "elf-sequence-test", feature = "serial-run-test", feature = "serial-wait-test", feature = "ipc-basic-test", feature = "ipc-wait-test", feature = "task-cancel-test", feature = "ipc-multi-test", feature = "ipc-timeline-test", feature = "virtio-discover-test", feature = "virtio-magic-test", feature = "virtio-net-id-test", feature = "virtio-net-mac-test", feature = "virtio-net-rx-test", feature = "virtio-net-rx-completion-test", feature = "virtio-net-broadcast-test", feature = "virtio-net-arp-request-test", feature = "virtio-net-arp-reply-test", feature = "virtio-net-icmp-echo-test", feature = "virtio-net-reset-traffic-test", feature = "virtio-net-unicast-test", feature = "virtio-net-ipv4-header-test", feature = "virtio-net-ipv4-checksum-test", feature = "virtio-net-pool-test", feature = "virtio-net-tx-test", feature = "virtio-net-tx-completion-test", feature = "virtio-net-size-test", feature = "virtio-net-header-test", feature = "virtio-net-type-test", feature = "virtio-modern-features-test", feature = "virtio-modern-queue-test", feature = "virtio-partial-completion-test", feature = "virtio-write-gate-test", feature = "production-write", feature = "virtio-write-header-test", feature = "virtio-write-descriptor-test", feature = "virtio-write-cancel-test", feature = "virtio-write-status-test", feature = "virtio-write-readback-test", feature = "virtio-flush-gate-test", feature = "virtio-flush-request-test", feature = "virtio-flush-reopen-test", feature = "virtio-two-write-order-test", feature = "virtio-two-write-reopen-test", feature = "virtio-partial-record-test", feature = "virtio-crash-after-first-test", feature = "virtio-crash-after-second-test", feature = "virtio-metadata-order-test", feature = "virtio-metadata-precommit-test", feature = "virtio-io-retry-test", feature = "virtio-io-retry-exhaust-test", feature = "turkfs-format-test", feature = "turkfs-bitmap-test", feature = "turkfs-enospc-test", feature = "turkfs-init-test", feature = "turkfs-init-exit-test", feature = "serial-rx-test", feature = "serial-line-test", feature = "serial-edit-test", feature = "serial-limit-test", feature = "serial-console-test", feature = "serial-unknown-test", feature = "serial-utf8-test", feature = "serial-arguments-test", feature = "serial-range-test", feature = "serial-cancel-test", feature = "serial-line-lock-test", feature = "serial-line-unlocked-test", feature = "serial-echo-test", feature = "turkfs-ls-test", feature = "turkfs-cat-test", feature = "virtio-queue-timeout-test", feature = "virtio-io-error-test", feature = "virtio-active-timeout-test", feature = "virtio-disk-absence-test", feature = "task-sleep-test", feature = "irq-timer-test", feature = "irq-timer-late-test", feature = "irq-timer-storm-test", feature = "irq-unknown-test", feature = "irq-spurious-test", feature = "smp-boot-protocol-test", feature = "smp-topology-test", feature = "smp-vector-test", feature = "smp-timer-test", feature = "smp-allocator-test", feature = "smp-ready-test", feature = "smp-serial-test", feature = "smp-owner-test", feature = "smp-irq-lock-test", feature = "smp-order-test", feature = "smp-fair-test", feature = "smp-wait-test", feature = "smp-time-test", feature = "smp-stats-test", feature = "smp-wait-class-test", feature = "smp-hold-test", feature = "smp-counter-baseline-test", feature = "smp-classified-admission-test", feature = "smp-race-trace-test"))] compile_error!("legacy probes cannot enter the runtime product or acceptance image"); #[cfg(all(feature="os-runtime-product", feature="os-runtime-acceptance"))] compile_error!("runtime image profiles must be exclusive"); #[cfg(all(feature="os-runtime-product", feature="os-runtime-negative-probe"))] compile_error!("negative probes cannot enter the product image"); #[cfg(not(any(feature="os-runtime-product", feature="os-runtime-acceptance")))] compile_error!("runtime image profile is required"); // Generated development Image only; one CPU, IRQs masked. Fatal paths never // use write_line, SERIAL_LINE_LOCK, panic text, file paths, or heap allocation. use turkos_kernel::runtime::diagnostics as os_diagnostics; static mut OS_K680_LOG: Option> = None; static mut OS_K680_FATAL: os_diagnostics::FatalGuard = os_diagnostics::FatalGuard::new(); const OS_K680_SECRET_CANARY: &[u8] = b"K680-PRIVATE-CANARY-9a7F-DO-NOT-LOG"; #[used] #[unsafe(no_mangle)] static OS_K680_SECRET_BYTES: [u8; b"K680-PRIVATE-CANARY-9a7F-DO-NOT-LOG".len()] = *b"K680-PRIVATE-CANARY-9a7F-DO-NOT-LOG"; static mut OS_K680_MODE: os_diagnostics::ProbeMode = os_diagnostics::ProbeMode::Normal; #[unsafe(no_mangle)] static mut OS_K680_EVIDENCE: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K680_USER_STATE: [u64; 5] = [0; 5]; #[unsafe(no_mangle)] static mut OS_K680_RUNTIME_BASELINE: [u64; 9] = [0; 9]; #[unsafe(no_mangle)] static mut OS_K680_BASELINE_IRQ_COUNT: u64 = 0; #[repr(C, align(4096))] struct OsK680HeapBacking([u8; turkos_kernel::heap::CAPACITY]); static mut OS_K680_HEAP_BACKING: OsK680HeapBacking = OsK680HeapBacking([0; turkos_kernel::heap::CAPACITY]); static mut OS_K680_HEAP: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); fn os_k680_fixture_start(boot: &BootData) -> Result<(), &'static str> { let start = boot.dtb_address as u64; let end = start .checked_add(boot.dtb_size as u64) .ok_or("diagnostics-dtb-overflow")?; let ram_end = boot .hardware .ram_base .checked_add(boot.hardware.ram_size) .ok_or("diagnostics-ram-overflow")?; if !(40..=2 * 1024 * 1024).contains(&boot.dtb_size) || start < boot.hardware.ram_base || end > ram_end { return Err("diagnostics-dtb-range"); } // SAFETY: read_boot_data validated the FDT, and the checked full span is // inside trusted identity-mapped RAM. This bounded parser retains no slice. let blob = unsafe { core::slice::from_raw_parts(start as *const u8, boot.dtb_size) }; let fixture = os_diagnostics::boot_fixture(blob).map_err(os_diagnostics::Error::tag)?; if cfg!(feature = "os-runtime-product") && fixture.mode != os_diagnostics::ProbeMode::Normal { return Err("diagnostics-probe-in-product"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K680_MODE), fixture.mode) }; os_k680_start(fixture.boot.value())?; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K680_EVIDENCE).cast::(), fixture.boot.value(), ) }; write_str("TURKOS/OS/K665: FIXTURE BOOT-ID=0x"); write_hex(fixture.boot.value() as usize); write_str(" DTB=0x"); write_hex(start as usize); write_str("\r\n"); Ok(()) } struct OsK680Uart; impl os_diagnostics::TrySink for OsK680Uart { fn try_byte(&mut self, byte: u8) -> bool { let base = UART_BASE.load(Ordering::Relaxed); // SAFETY: the existing boot/MMU path validates and maps this PL011. // One status read and at most one data write; no readiness spin/lock. unsafe { if read_volatile((base + 0x18) as *const u32) & (1 << 5) != 0 { return false; } write_volatile(base as *mut u32, u32::from(byte)); } true } } fn os_k680_log() -> Result, &'static str> { // SAFETY: a copied value is read without keeping a static reference. // The single boot CPU owns the log, IRQs remain masked for these hooks. unsafe { read_volatile(core::ptr::addr_of!(OS_K680_LOG)) }.ok_or("diagnostics-not-started") } fn os_k680_emit(record: os_diagnostics::Record) -> Result<(), &'static str> { let outcome = os_diagnostics::emit_once( &mut os_diagnostics::FatalGuard::new(), record, &mut OsK680Uart, os_diagnostics::MAX_FATAL_POLLS, ); match outcome { os_diagnostics::FatalOutcome::Complete { .. } => Ok(()), _ => Err("diagnostics-transport-stalled"), } } // boot_id is a nonzero generation supplied by the trusted per-boot fixture. // Reusing the fixed Image hash as this generation would defeat K665. fn os_k680_start(boot_id: u64) -> Result<(), &'static str> { let boot = os_diagnostics::BootId::new(boot_id).map_err(os_diagnostics::Error::tag)?; if unsafe { read_volatile(core::ptr::addr_of!(OS_K680_LOG)) }.is_some() { return Err("diagnostics-already-started"); } let mut log = os_diagnostics::EventLog::<8>::new(boot); let entered = log .append( boot, os_diagnostics::Service::Boot, os_diagnostics::Input::Public(os_diagnostics::PublicData::Event( os_diagnostics::EventCode::Entered, )), ) .map_err(os_diagnostics::Error::tag)?; // SAFETY: exactly one primary entry publishes this owned fixed array. unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K680_LOG), Some(log)) }; os_k680_emit(entered) } fn os_k680_append( service: os_diagnostics::Service, input: os_diagnostics::Input<'_>, ) -> Result { let mut log = os_k680_log()?; let record = log .append(log.boot(), service, input) .map_err(os_diagnostics::Error::tag)?; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K680_LOG), Some(log)) }; os_k680_emit(record)?; Ok(record) } fn os_k680_self_check() -> Result<(), &'static str> { use os_diagnostics::{ AddressKind, BootId, Error, EventCode, EventLog, FatalGuard, FatalOutcome, Input, PublicAddress, PublicData, RawFault, Record, Service, TrySink, }; let mut log = os_k680_log()?; let before = log; let boot = log.boot(); let other = BootId::new(if boot.value() == 1 { 2 } else { 1 }).map_err(Error::tag)?; let mut old = EventLog::<1>::new(other); let stale = old .append( other, Service::Boot, Input::Public(PublicData::Event(EventCode::Entered)), ) .map_err(Error::tag)?; if log.replay(stale) != Err(Error::StaleBoot) || log != before { return Err("diagnostics-stale-log-accepted"); } write_line("TURKOS/OS/K665: STALE-BOOT REJECTED STATE-STABLE"); let register_canary = [0x4b36_3830_5345_4352u64; 31]; for private in [ Input::Secret(OS_K680_SECRET_CANARY), Input::HostPath("/keys/master.key"), Input::Registers(®ister_canary), ] { if log.append(boot, Service::Diagnostics, private) != Err(Error::PrivateData) || log != before || Record::emergency(boot, log.next_sequence(), Service::Diagnostics, private) != Err(Error::PrivateData) { return Err("diagnostics-private-input-accepted"); } } write_line("TURKOS/OS/K680: SECRET HOST-PATH REGISTERS REJECTED STATE-STABLE"); let bad_far = RawFault { slot: 4, esr: 0x3c << 26, task: None, elr: PublicAddress { kind: AddressKind::KernelVirtual, value: os_k680_self_check as *const () as u64, }, far: Some(PublicAddress { kind: AddressKind::KernelVirtual, value: 0x4b36_3830_5345_4352, }), }; if os_diagnostics::classify_fault(bad_far) != Err(Error::InvalidFar) || log != before { return Err("diagnostics-invalid-far-accepted"); } write_line("TURKOS/OS/K675: INVALID-FAR REJECTED STATE-STABLE"); struct NotReady { polls: usize, } impl TrySink for NotReady { fn try_byte(&mut self, _: u8) -> bool { self.polls += 1; false } } let record = Record::emergency( boot, log.next_sequence(), Service::Diagnostics, Input::Public(PublicData::Event(EventCode::Panic)), ) .map_err(Error::tag)?; let mut guard = FatalGuard::new(); let mut sink = NotReady { polls: 0 }; if os_diagnostics::emit_once(&mut guard, record, &mut sink, 8) != (FatalOutcome::TransportStalled { bytes: 0, polls: 8 }) || os_diagnostics::emit_once(&mut guard, record, &mut sink, 4096) != FatalOutcome::SilentReentry || sink.polls != 8 { return Err("diagnostics-recursive-uart-access"); } write_line("TURKOS/OS/K670: UART-NOT-READY BOUNDED REENTRY-SILENT PASS"); os_k680_append( Service::Diagnostics, Input::Public(PublicData::Event(EventCode::Ready)), )?; Ok(()) } fn os_k680_fatal_enter() -> Option { // SAFETY: no mutable reference to the static survives this function. // Commit before record construction or I/O, so a synchronous nested fault // observes the entered flag even if the UART was never ready. let mut guard = unsafe { read_volatile(core::ptr::addr_of!(OS_K680_FATAL)) }; let evidence = core::ptr::addr_of_mut!(OS_K680_EVIDENCE).cast::(); unsafe { write_volatile( evidence.add(3), read_volatile(evidence.add(3)).saturating_add(1), ) }; let permit = guard.enter(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K680_FATAL), guard) }; permit } fn os_k680_panic() -> ! { let Some(permit) = os_k680_fatal_enter() else { park() }; let log = match os_k680_log() { Ok(log) => log, Err(_) => park(), }; let record = match os_diagnostics::Record::emergency( log.boot(), log.next_sequence(), os_diagnostics::Service::Diagnostics, os_diagnostics::Input::Public(os_diagnostics::PublicData::Event( os_diagnostics::EventCode::Panic, )), ) { Ok(record) => record, Err(_) => park(), }; let _ = os_diagnostics::emit_entered( permit, record, &mut OsK680Uart, os_diagnostics::MAX_FATAL_POLLS, ); park() } fn os_k680_fault_capture( slot: usize, esr: u64, elr: u64, task: Option, ) -> Result { let slot = u8::try_from(slot).map_err(|_| "diagnostics-invalid-vector")?; let kind = if slot < 8 { os_diagnostics::AddressKind::KernelVirtual } else { os_diagnostics::AddressKind::UserVirtual }; let far = if os_diagnostics::far_is_valid(slot, esr) { let value: u64; // SAFETY: this hook is called by an actual EL1 exception handler and // the synchronous ESR class defines FAR. Invalid FAR is never read. unsafe { asm!("mrs {value}, FAR_EL1", value = out(reg) value, options(nomem, nostack, preserves_flags)) }; Some(os_diagnostics::PublicAddress { kind, value }) } else { None }; os_diagnostics::classify_fault(os_diagnostics::RawFault { slot, esr, task, elr: os_diagnostics::PublicAddress { kind, value: elr }, far, }) .map_err(os_diagnostics::Error::tag) } // The existing process handler decides which task to terminate. This hook // reports the actual lower-EL capture and returns to that handler's policy. fn os_k680_process_fault(slot: usize, esr: u64, elr: u64, task: u64) -> Result<(), &'static str> { let report = os_k680_fault_capture(slot, esr, elr, Some(task))?; if report.context() != os_diagnostics::Context::Process(task) { return Err("diagnostics-process-context"); } os_k680_append( os_diagnostics::Service::Tasks, os_diagnostics::Input::Public(os_diagnostics::PublicData::Fault(report)), )?; Ok(()) } fn os_k680_kernel_fault(slot: usize, esr: u64, elr: u64) -> ! { let Some(permit) = os_k680_fatal_enter() else { park() }; if unsafe { read_volatile(core::ptr::addr_of!(OS_K680_MODE)) } == os_diagnostics::ProbeMode::RecursiveBrk { // Actual synchronous reentry before the first record reaches UART. unsafe { asm!("brk #0x681", options(nomem, nostack)) }; park(); } let report = match os_k680_fault_capture(slot, esr, elr, None) { Ok(report) => report, Err(_) => park(), }; let log = match os_k680_log() { Ok(log) => log, Err(_) => park(), }; let record = match os_diagnostics::Record::emergency( log.boot(), log.next_sequence(), os_diagnostics::Service::Diagnostics, os_diagnostics::Input::Public(os_diagnostics::PublicData::Fault(report)), ) { Ok(record) => record, Err(_) => park(), }; let _ = os_diagnostics::emit_entered( permit, record, &mut OsK680Uart, os_diagnostics::MAX_FATAL_POLLS, ); park() } fn os_k680_after_runtime( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { use os_diagnostics::ProbeMode; // A separate real RAM checkpoint precedes all diagnostics work. Preserve // the runtime service's original nine words and independently read its // actual cumulative IRQ counter. Never reset or relabel the live counter. let baseline = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_RUNTIME_EVIDENCE)) }; let irq_count = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; if baseline[0] != irq_count || irq_count != 4 { return Err("diagnostics-runtime-baseline"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K680_RUNTIME_BASELINE), baseline); write_volatile( core::ptr::addr_of_mut!(OS_K680_BASELINE_IRQ_COUNT), irq_count, ); } if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(boot)? { os_k680_self_check()?; } let mode = unsafe { read_volatile(core::ptr::addr_of!(OS_K680_MODE)) }; match mode { ProbeMode::Normal => { if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(boot)? { os_k680_user_probe(runtime, tables, boot)?; } else { os_k680_append( os_diagnostics::Service::Diagnostics, os_diagnostics::Input::Public(os_diagnostics::PublicData::Event( os_diagnostics::EventCode::Ready, )), )?; } } ProbeMode::PanicHeldLock => { // Real mapped fixed Heap: fill its only free region, observe OOM, // then panic while the real serial line lock is still held. let storage = core::ptr::addr_of_mut!(OS_K680_HEAP).cast::(); let backing = core::ptr::addr_of_mut!(OS_K680_HEAP_BACKING).cast::() as usize; unsafe { turkos_kernel::heap::Heap::init_in_place(storage, backing) } .map_err(|_| "diagnostics-heap-init")?; let heap = unsafe { &mut *storage }; heap.allocate(turkos_kernel::heap::CAPACITY - 8, 8) .map_err(|_| "diagnostics-heap-fill")?; if heap.allocate(1, 8) != Err(turkos_kernel::heap::Error::OutOfMemory) { return Err("diagnostics-heap-not-full"); } let evidence = core::ptr::addr_of_mut!(OS_K680_EVIDENCE).cast::(); unsafe { write_volatile(evidence.add(4), heap.used_bytes() as u64); write_volatile(evidence.add(5), heap.active_count() as u64) }; write_line("TURKOS/OS/K670: REAL-HEAP-FULL SERIAL-LOCK-ARMED"); let _held = SERIAL_LINE_LOCK.lock(); unsafe { write_volatile(evidence.add(6), u64::from(SERIAL_LINE_LOCK.is_locked())) }; panic!("private panic text must never be formatted"); } ProbeMode::InvalidFarBrk | ProbeMode::RecursiveBrk => { write_line(if mode == ProbeMode::InvalidFarBrk { "TURKOS/OS/K675: BRK-POISONED-FAR-REGISTERS ARMED" } else { "TURKOS/OS/K670: RECURSIVE-BRK ARMED" }); unsafe { asm!("msr FAR_EL1, {secret}", "brk #0x680", secret=in(reg) 0x4b36_3830_5345_4352u64, in("x0") 0x4b36_3830_5052_4956u64, in("x3") 0x4154_452d_4341_4e41u64, options(nomem,nostack,noreturn)) }; } } let log = os_k680_log()?; let evidence = core::ptr::addr_of_mut!(OS_K680_EVIDENCE).cast::(); unsafe { write_volatile(evidence.add(1), log.next_sequence()); write_volatile(evidence.add(2), log.len() as u64) }; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K665: PASS"); write_line("TURKOS/OS/K670: PASS"); write_line("TURKOS/OS/K675: PASS"); write_line("TURKOS/OS/K680: PASS"); } else { write_line("TURKOS/OS/K680: PRODUCT-READY"); } Ok(()) } fn os_k680_user_probe( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { let (_, memory, _, tasks) = runtime.running().map_err(os_k660_policy::Error::tag)?; let id = tasks.id.ok_or("diagnostics-task-missing")?; let free = memory.allocator.remaining_pages(); let user = unsafe { tables.build_user_space( &mut memory.allocator, boot.hardware.ram_base, boot.hardware.ram_size, ) } .map_err(|_| "diagnostics-user-space")?; // Three real EL0 instructions: point at the unmapped user guard; read it. let code = [0xd284_0001u32, 0xf2b0_0001u32, 0xf940_0020u32]; for (index, word) in code.into_iter().enumerate() { unsafe { write_volatile((user.code_frame() as *mut u32).add(index), word) }; } unsafe { asm!("dc cvau, {code}", "dsb ish", "ic ivau, {code}", "dsb ish", "isb", code=in(reg) user.code_frame(), options(nostack)) }; let state = core::ptr::addr_of_mut!(OS_K680_USER_STATE).cast::(); unsafe { write_volatile(state.add(0), tables.root); write_volatile(state.add(2), id.generation); write_volatile(state.add(3), 1); write_volatile(state.add(4), 0) }; unsafe extern "C" { fn __os_k680_enter_user(root: u64); } unsafe { __os_k680_enter_user(user.root) }; if unsafe { read_volatile(state.add(4)) } != 1 { return Err("diagnostics-user-fault-missing"); } // Resume assembly restored the kernel root and completed TLBI before return. unsafe { user.release_after_tlbi(&mut memory.allocator, tables.root) } .map_err(|_| "diagnostics-user-release")?; if memory.allocator.remaining_pages() != free || tasks.scheduler.exit_current(-14) != Ok(None) || tasks.scheduler.reap(id) != Ok(-14) { return Err("diagnostics-task-fault-policy"); } let evidence = core::ptr::addr_of_mut!(OS_K680_EVIDENCE).cast::(); unsafe { write_volatile(evidence.add(4), free); write_volatile(evidence.add(5), memory.allocator.remaining_pages()) }; let next = tasks .scheduler .spawn() .map_err(|_| "diagnostics-replacement-task")?; tasks.id = Some(next); write_line("TURKOS/OS/K675: REAL-EL0-TRANSLATION-FAULT TASK-REAPED SIX-FRAMES-RESTORED"); os_k660_work(runtime)?; write_line("TURKOS/OS/K675: KERNEL-SURVIVED REPLACEMENT-TASK-TIMER-PROGRESS PASS"); Ok(()) } fn os_k680_expected_user_fault(slot: usize, esr: u64, elr: u64) -> bool { let state = core::ptr::addr_of_mut!(OS_K680_USER_STATE).cast::(); if unsafe { read_volatile(state.add(3)) } != 1 { return false; } if slot != 8 || esr >> 26 & 63 != 0x24 || esr & 0x3f != 7 { park(); } let task = unsafe { read_volatile(state.add(2)) }; if os_k680_process_fault(slot, esr, elr, task).is_err() { park(); } let far: u64; unsafe { asm!("mrs {value}, FAR_EL1",value=out(reg)far,options(nomem,nostack)) }; if far != mmu::USER_GUARD_VA { park(); } let evidence = core::ptr::addr_of_mut!(OS_K680_EVIDENCE).cast::(); unsafe { write_volatile(evidence.add(6), esr); write_volatile(evidence.add(7), far); write_volatile(state.add(3), 0); write_volatile(state.add(4), 1) }; unsafe extern "C" { fn __os_k680_resume_kernel() -> !; } unsafe { __os_k680_resume_kernel() } } global_asm!( r#" .section .text.os_k680_user,"ax" .global __os_k680_enter_user __os_k680_enter_user: stp x19, x20, [sp, #-16]! stp x21, x22, [sp, #-16]! stp x23, x24, [sp, #-16]! stp x25, x26, [sp, #-16]! stp x27, x28, [sp, #-16]! stp x29, x30, [sp, #-16]! stp d8, d9, [sp, #-16]! stp d10, d11, [sp, #-16]! stp d12, d13, [sp, #-16]! stp d14, d15, [sp, #-16]! adrp x1, OS_K680_USER_STATE add x1, x1, :lo12:OS_K680_USER_STATE mov x2, sp str x2, [x1, #8] msr TTBR0_EL1, x0 dsb ish tlbi vmalle1 dsb ish isb movz x0, #0x4000 movk x0, #0x8000, lsl #16 msr SP_EL0, x0 movz x0, #0x8000, lsl #16 msr ELR_EL1, x0 mov x0, #0x3c0 msr SPSR_EL1, x0 isb eret .global __os_k680_resume_kernel __os_k680_resume_kernel: adrp x0, OS_K680_USER_STATE add x0, x0, :lo12:OS_K680_USER_STATE ldp x1, x2, [x0] msr TTBR0_EL1, x1 dsb ish tlbi vmalle1 dsb ish isb mov sp, x2 ldp d14, d15, [sp], #16 ldp d12, d13, [sp], #16 ldp d10, d11, [sp], #16 ldp d8, d9, [sp], #16 ldp x29, x30, [sp], #16 ldp x27, x28, [sp], #16 ldp x25, x26, [sp], #16 ldp x23, x24, [sp], #16 ldp x21, x22, [sp], #16 ldp x19, x20, [sp], #16 ret "# ); // Platform scratch lives in image-owned BSS. Parsing never allocates frames; // this boot CPU alone publishes the discovery before allocator setup. static mut OS_K700_WORKSPACE: turkos_kernel::runtime::platform::Workspace = turkos_kernel::runtime::platform::Workspace::new(); static mut OS_K700_PLATFORM: Option = None; static mut OS_K700_EXPECTED_HOLE: u64 = 0; static mut OS_K700_HOLE_TARGET: u64 = 0; static mut OS_K700_MODE: u32 = 0; #[unsafe(no_mangle)] static mut OS_K700_EVIDENCE: [u64; 64] = [0; 64]; fn os_k700_capture_platform(blob: &[u8]) -> Result { use turkos_kernel::runtime::{platform, platform_features}; // SAFETY: one boot CPU, before allocator and before any IRQ unmask. Scratch // is private BSS rather than a 22-KiB allocation on the 16-KiB stack. let scratch = unsafe { &mut *core::ptr::addr_of_mut!(OS_K700_WORKSPACE) }; let found = match platform::discover(blob, platform::Limits::DEFAULT, scratch) { Ok(value) => value, Err(error) => { write_str("TURKOS/OS/PLATFORM: REJECTED reason="); write_str(error.tag()); write_str(" BEFORE-ALLOC\r\n"); return Err(dtb::Error::Unsupported); } }; let mode = platform::chosen_u32(blob, scratch, b"turkos,platform-mode") .map_err(|_| dtb::Error::Invalid)? .unwrap_or(0); if mode > 1 || cfg!(feature = "os-runtime-product") && mode != 0 { return Err(dtb::Error::Unsupported); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K700_MODE), mode) }; let pfr0: u64; let mmfr0: u64; let counter_frequency: u64; let current_el: u64; // SAFETY: readable architectural capabilities in the EL1 guest. This // gate runs before MMU allocation and does not enable optional ISA paths. unsafe { asm!("mrs {value}, ID_AA64PFR0_EL1", value = out(reg) pfr0, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ID_AA64MMFR0_EL1", value = out(reg) mmfr0, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, CNTFRQ_EL0", value = out(reg) counter_frequency, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, CurrentEL", value = out(reg) current_el, options(nomem, nostack, preserves_flags)); } let registers = platform_features::Registers { pfr0, mmfr0, counter_frequency, current_el, }; write_str("TURKOS/OS/PLATFORM: CPU-SNAPSHOT PFR0=0x"); write_hex(pfr0 as usize); write_str(" MMFR0=0x"); write_hex(mmfr0 as usize); write_str(" CNTFRQ=0x"); write_hex(counter_frequency as usize); write_str(" CURRENT-EL=0x"); write_hex(current_el as usize); write_str("\r\n"); let features = match platform_features::admit(registers, platform_features::REQUIRED) { Ok(value) => value, Err(error) => { write_str("TURKOS/OS/PLATFORM: REJECTED reason="); write_str(error.tag()); write_str(" BEFORE-ALLOC\r\n"); return Err(dtb::Error::Unsupported); } }; unsafe { let evidence = core::ptr::addr_of_mut!(OS_K700_EVIDENCE).cast::(); for (index, value) in [pfr0, mmfr0, counter_frequency, features.available as u64] .iter() .enumerate() { write_volatile(evidence.add(24 + index), *value); } } let mut absent = registers; absent.pfr0 |= 15 << 20; if platform_features::admit(absent, platform_features::REQUIRED) != Err(platform_features::Error::RequiredMissing) || platform_features::admit(registers, platform_features::REQUIRED | (1 << 31)) != Err(platform_features::Error::UnknownRequirement) || platform_features::admit(registers, platform_features::REQUIRED) != Ok(features) { return Err(dtb::Error::Invalid); } let banks = found.banks.as_slice(); let reserved = dtb::static_reserved(blob)?; for pair in banks.windows(2) { if pair[0].end < pair[1].start && !reserved.as_slice().iter().any(|region| { region.no_map && region.start <= pair[0].end && region .start .checked_add(region.size) .is_some_and(|end| end >= pair[1].start) }) { return Err(dtb::Error::Unsupported); } } let first = banks.first().ok_or(dtb::Error::MissingDevice)?; let last = banks.last().ok_or(dtb::Error::MissingDevice)?; let hardware = dtb::Hardware { uart_base: found.devices.uart.region.start, ram_base: first.start, ram_size: last .end .checked_sub(first.start) .ok_or(dtb::Error::Invalid)?, }; write_str("TURKOS/OS/PLATFORM: TREE BYTES=0x"); write_hex(found.tree.blob_bytes); write_str(" NODES=0x"); write_hex(found.tree.nodes); write_str(" DEPTH=0x"); write_hex(found.tree.depth); write_str(" PROPERTIES=0x"); write_hex(found.tree.properties); write_str(" REFERENCES=0x"); write_hex(found.tree.references); write_str(" BOUNDED PASS\r\n"); for (index, bank) in banks.iter().enumerate() { write_str("TURKOS/OS/PLATFORM: BANK INDEX=0x"); write_hex(index); write_str(" START=0x"); write_hex(bank.start as usize); write_str(" END=0x"); write_hex(bank.end as usize); write_str("\r\n"); } write_str("TURKOS/OS/PLATFORM: CPU PFR0=0x"); write_hex(pfr0 as usize); write_str(" MMFR0=0x"); write_hex(mmfr0 as usize); write_str(" CNTFRQ=0x"); write_hex(counter_frequency as usize); write_str(" REQUIRED=0x"); write_hex(features.required as usize); write_str(" AVAILABLE=0x"); write_hex(features.available as usize); write_str(" PA-BITS=0x"); write_hex(features.physical_bits as usize); write_str(" PASS\r\n"); write_line( "TURKOS/OS/PLATFORM: SYNTHETIC-MISSING-SIMD UNKNOWN-REQUIREMENT REJECTED OWNER=UNCHANGED", ); // SAFETY: the singleton is published once after the complete parser and // CPU gate succeeded; no resources were acquired on either failure path. unsafe { let destination = core::ptr::addr_of_mut!(OS_K700_PLATFORM); if (*destination).is_some() { return Err(dtb::Error::Invalid); } write_volatile(destination, Some(found)); } Ok(hardware) } fn os_k700_constrain_free( free: &mut memory::FreeMap, boot: &BootData, ) -> Result<(), memory::Error> { // SAFETY: platform discovery precedes this single-CPU allocator planning. let found = unsafe { (*core::ptr::addr_of!(OS_K700_PLATFORM)).as_ref() } .ok_or(memory::Error::Invalid)?; let banks = found.banks.as_slice(); let mut previous_end = boot.hardware.ram_base; for bank in banks { if bank.start > previous_end { free.reserve(previous_end, bank.start - previous_end)?; } previous_end = bank.end; } for region in free.regions() { let mut covered = region.start; for bank in banks { if covered >= bank.start && covered < bank.end { covered = bank.end.min(region.end); } } if covered != region.end { return Err(memory::Error::Invalid); } } write_line("TURKOS/OS/PLATFORM: FREE-MAP DTB-BANKS FIRMWARE-INTERSECTION HOLES-WITHHELD PASS"); Ok(()) } fn os_k700_translation(tables: &mmu::Tables, address: u64) -> Result { const FRAME_MASK: u64 = 0x0000_ffff_ffff_f000; let root: u64; // SAFETY: TTBR0_EL1 readable at EL1 after MMU activation. unsafe { asm!("mrs {value}, TTBR0_EL1", value = out(reg) root, options(nomem, nostack, preserves_flags)) }; let mut table = root & FRAME_MASK; for shift in [30, 21, 12] { if !tables.owns_table_frame(table) { return Err("unowned-platform-table"); } let slot = table + ((address >> shift) & 511) * 8; // SAFETY: Tables owns the complete aligned frame; index is 0..511. let descriptor = unsafe { read_volatile(slot as *const u64) }; if descriptor & 1 == 0 { return Ok(0); } if shift == 12 || descriptor & 3 == 1 { return Ok(descriptor); } if descriptor & 3 != 3 { return Err("bad-platform-descriptor"); } table = descriptor & FRAME_MASK; } Err("missing-platform-translation") } fn os_k700_live_platform( allocator: &mut memory::FrameAllocator, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { // SAFETY: initialized once before allocation; stable on this boot CPU. let found = unsafe { (*core::ptr::addr_of!(OS_K700_PLATFORM)).as_ref() } .ok_or("platform-not-published")?; let devices = found.devices; let sre_before: u64; let sre: u64; let control: u64; let daif: u64; // SAFETY: the bounded DTB identified this virtual GICv3 interface. IRQs // remain masked. Enable its architectural system-register interface and // capture real readback; no host-projected feature bit is manufactured. unsafe { asm!("mrs {value}, DAIF", value = out(reg) daif, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ICC_SRE_EL1", value = out(reg) sre_before, options(nomem, nostack, preserves_flags)); asm!("msr ICC_SRE_EL1, {value}", "isb", value = in(reg) (sre_before | 1), options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ICC_SRE_EL1", value = out(reg) sre, options(nomem, nostack, preserves_flags)); asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags)); } if daif & 0x80 == 0 || sre & 1 == 0 { return Err("virtual-gic-interface-readback"); } unsafe { let evidence = core::ptr::addr_of_mut!(OS_K700_EVIDENCE).cast::(); write_volatile(evidence.add(48), sre); write_volatile(evidence.add(49), control); } write_str("TURKOS/OS/PLATFORM: GIC-READBACK MASKED SRE=0x"); write_hex(sre as usize); write_str(" CTLR=0x"); write_hex(control as usize); write_str(" PLATFORM-INTERFACE PASS\r\n"); if boot.hardware.uart_base != devices.uart.region.start || boot.interrupts.gic_distributor != devices.gic_distributor.region.start || boot.interrupts.gic_redistributor != devices.gic_redistributor.region.start || boot.interrupts.nonsecure_phys_timer_ppi != devices.timer_ppi { return Err("platform-device-provenance"); } for (name, record) in [ ("UART", devices.uart), ("GICD", devices.gic_distributor), ("GICR", devices.gic_redistributor), ] { write_str("TURKOS/OS/PLATFORM: DEVICE "); write_str(name); write_str(" ADDRESS=0x"); write_hex(record.region.start as usize); write_str(" NODE=0x"); write_hex(record.node as usize); write_str(" REG-OFFSET=0x"); write_hex(record.reg_property_offset as usize); write_str(" DTB-BOUND PASS\r\n"); } write_str("TURKOS/OS/PLATFORM: DEVICE TIMER PPI=0x"); write_hex(devices.timer_ppi as usize); write_str(" NODE=0x"); write_hex(devices.timer_node as usize); write_str(" INTERRUPTS-OFFSET=0x"); write_hex(devices.timer_property_offset as usize); write_str(" DTB-BOUND PASS\r\n"); let before = allocator.remaining_pages(); let banks = found.banks.as_slice(); let mut canary_mask = 0u64; for (index, bank) in banks.iter().enumerate() { let regions = allocator .free_regions() .map_err(|_| "platform-free-metadata")?; let first = regions .iter() .find(|region| region.start >= bank.start && region.end <= bank.end) .ok_or("platform-bank-empty")? .start; let last = regions .iter() .rev() .find(|region| region.start >= bank.start && region.end <= bank.end) .ok_or("platform-bank-empty")? .end - memory::PAGE_SIZE; for (endpoint, address) in [(0usize, first), (1, last)] { let snapshot = allocator.remaining_pages(); let frame = allocator .allocate_at(address) .map_err(|_| "platform-endpoint-acquire")?; if frame != address || allocator.remaining_pages() + 1 != snapshot || !matches!(os_k700_translation(tables, frame), Ok(value) if value != 0) { allocator .release(frame) .map_err(|_| "platform-endpoint-rollback")?; return Err("platform-endpoint-owner"); } // SAFETY: FrameAllocator just admitted this exclusive, mapped page // from the constrained authority. Restore bytes before release. unsafe { let first_saved = read_volatile(frame as *const u64); let last_saved = read_volatile((frame + memory::PAGE_SIZE - 8) as *const u64); write_volatile(frame as *mut u64, 0x4b36_3935_4241_4e4b); write_volatile( (frame + memory::PAGE_SIZE - 8) as *mut u64, !0x4b36_3935_4241_4e4bu64, ); let valid = read_volatile(frame as *const u64) == 0x4b36_3935_4241_4e4b && read_volatile((frame + memory::PAGE_SIZE - 8) as *const u64) == !0x4b36_3935_4241_4e4bu64; write_volatile(frame as *mut u64, first_saved); write_volatile((frame + memory::PAGE_SIZE - 8) as *mut u64, last_saved); if !valid { allocator .release(frame) .map_err(|_| "platform-endpoint-rollback")?; return Err("platform-endpoint-canary"); } canary_mask |= 1 << (index * 2 + endpoint); } allocator .release(frame) .map_err(|_| "platform-endpoint-release")?; if allocator.remaining_pages() != snapshot { return Err("platform-endpoint-leak"); } unsafe { let evidence = core::ptr::addr_of_mut!(OS_K700_EVIDENCE).cast::(); write_volatile(evidence.add(4 + index * 2 + endpoint), frame); write_volatile(evidence.add(28 + index * 2), bank.start); write_volatile(evidence.add(29 + index * 2), bank.end); } write_str("TURKOS/OS/PLATFORM: ALLOC BANK=0x"); write_hex(index); write_str(" ENDPOINT=0x"); write_hex(endpoint); write_str(" FRAME=0x"); write_hex(frame as usize); write_str(" OWNED-CANARY-RELEASE PASS\r\n"); } } for reservation in boot.static_reserved.as_slice() { if !reservation.no_map { continue; } let end = reservation .start .checked_add(reservation.size) .ok_or("platform-hole-overflow")?; let target = reservation.start; let snapshot = allocator.remaining_pages(); if allocator.allocate_at(target) != Err(memory::FrameRequestError::OutsideAuthority) || allocator.remaining_pages() != snapshot || os_k700_translation(tables, target)? != 0 || os_k700_translation(tables, end - memory::PAGE_SIZE)? != 0 { return Err("platform-hole-authority"); } write_str("TURKOS/OS/PLATFORM: HOLE START=0x"); write_hex(target as usize); write_str(" END=0x"); write_hex(end as usize); write_str(" ALLOC=OUTSIDE-AUTHORITY TRANSLATION=UNMAPPED OWNER=UNCHANGED PASS\r\n"); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K700_HOLE_TARGET), target) }; } if allocator.remaining_pages() != before { return Err("platform-ownership-drift"); } unsafe { let evidence = core::ptr::addr_of_mut!(OS_K700_EVIDENCE).cast::(); for (index, value) in [ before, allocator.remaining_pages(), banks.len() as u64, (banks.len() * 2) as u64, ] .iter() .enumerate() { write_volatile(evidence.add(index), *value); } write_volatile(evidence.add(20), canary_mask); } write_str("TURKOS/OS/PLATFORM: OWNERSHIP FREE-BEFORE=0x"); write_hex(before as usize); write_str(" FREE-AFTER=0x"); write_hex(allocator.remaining_pages() as usize); write_str(" UNCHANGED PASS\r\n"); Ok(()) } fn os_k700_finish_platform() -> Result<(), &'static str> { // SAFETY: K660/K680 completed actual timer deliveries on this CPU and // masked IRQs again. These are the same live counters captured by QMP. let deliveries = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; let evidence = core::ptr::addr_of_mut!(OS_K700_EVIDENCE).cast::(); let sre = unsafe { read_volatile(evidence.add(48)) }; let control = unsafe { read_volatile(evidence.add(49)) }; if deliveries < 4 || sre & 1 == 0 { return Err("virtual-gic-delivery-proof"); } unsafe { write_volatile(evidence.add(50), deliveries as u64) }; write_str("TURKOS/OS/PLATFORM: GIC-INTERFACE SRE=0x"); write_hex(sre as usize); write_str(" CTLR=0x"); write_hex(control as usize); write_str(" IRQ-DELIVERIES=0x"); write_hex(deliveries as usize); write_str(" REAL PASS\r\n"); Ok(()) } fn os_k700_hole_read() -> ! { // SAFETY: live acceptance stored a verified no-map reservation endpoint. let target = unsafe { read_volatile(core::ptr::addr_of!(OS_K700_HOLE_TARGET)) }; if target == 0 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K700_EXPECTED_HOLE), target) }; write_str("TURKOS/OS/PLATFORM: HOLE-READ ARMED TARGET=0x"); write_hex(target as usize); write_str("\r\n"); // SAFETY: deliberate negative access to a verified unmapped hole. Fatal // vectors must capture its real FAR/ESR; no success path returns here. unsafe { read_volatile(target as *const u64) }; write_line("TURKOS/OS/PLATFORM: HOLE-READ RETURNED ERROR"); park() } fn os_k700_expected_hole_fault( level: usize, esr: usize, slot: usize, frame: *const ExceptionFrame, ) -> bool { let target = unsafe { read_volatile(core::ptr::addr_of!(OS_K700_EXPECTED_HOLE)) }; if target == 0 { return false; } let far: u64; unsafe { asm!("mrs {value}, FAR_EL1", value = out(reg) far, options(nomem, nostack, preserves_flags)) }; let context = unsafe { &*frame }; let valid = level == 1 && slot == 4 && (esr >> 26) & 0x3f == 0x25 && matches!(esr & 0x3f, 6 | 7) && esr & (1 << 6) == 0 && far == target && context.canary_valid(); write_str("TURKOS/OS/PLATFORM: HOLE-FAULT ESR=0x"); write_hex(esr); write_str(" FAR=0x"); write_hex(far as usize); write_str(if valid { " FRAME-CANARY=PASS OWNER=UNCHANGED PASS\r\n" } else { " REJECTED\r\n" }); valid } // Isolated development kernel only. Firmware classification runs before any // frame allocator exists, validates the finalized free map, then the actual // memory::FrameAllocator façade owns that exact admitted pool throughout MMU, // runtime tasks, endpoint tests and later frame-metadata consumers. use turkos_kernel::runtime::frame_authority as os_k720_frame; static mut OS_K720_DESCRIPTORS: [os_k720_frame::Descriptor; os_k720_frame::MAX_DESCRIPTORS] = [os_k720_frame::Descriptor { kind: 0, region: turkos_kernel::runtime::platform_ram::Region::EMPTY, }; os_k720_frame::MAX_DESCRIPTORS]; static mut OS_K720_RESERVATIONS: [turkos_kernel::runtime::platform_ram::Region; os_k720_frame::MAX_REGIONS] = [turkos_kernel::runtime::platform_ram::Region::EMPTY; os_k720_frame::MAX_REGIONS]; static mut OS_K720_CLASSIFIED: Option = None; #[unsafe(no_mangle)] static mut OS_K720_EVIDENCE: [u64; 64] = [0; 64]; #[unsafe(no_mangle)] static mut OS_K720_FREE_INVENTORY: [u64; os_k720_frame::MAX_REGIONS * 2] = [0; os_k720_frame::MAX_REGIONS * 2]; #[unsafe(no_mangle)] static mut OS_K720_LIVE_INVENTORY: [u64; os_k720_frame::MAX_LIVE] = [0; os_k720_frame::MAX_LIVE]; fn os_k720_classify( boot: &BootData, dtb_address: usize, info: Option<&(u64, BootInfoV1)>, owned_map: Option<&[u8]>, ) -> Result<(), memory::Error> { use turkos_kernel::runtime::platform_ram::Region; let found = unsafe { (*core::ptr::addr_of!(OS_K700_PLATFORM)).as_ref() } .ok_or(memory::Error::Invalid)?; // SAFETY: one boot CPU owns both scratch arrays before allocation/IRQ. let reservations = unsafe { &mut *core::ptr::addr_of_mut!(OS_K720_RESERVATIONS) }; let mut len = 0; let mut add = |start: u64, bytes: u64| -> Result<(), memory::Error> { if len == reservations.len() || bytes == 0 { return Err(memory::Error::Invalid); } reservations[len] = Region { start, end: start.checked_add(bytes).ok_or(memory::Error::Invalid)?, }; len += 1; Ok(()) }; let image_start = core::ptr::addr_of!(_image_start) as u64; let image_end = core::ptr::addr_of!(_image_end) as u64; add( image_start, image_end .checked_sub(image_start) .ok_or(memory::Error::Invalid)?, )?; let image_guard = image_end.checked_add(4095).ok_or(memory::Error::Invalid)? & !4095; add(image_guard, 4096)?; add(dtb_address as u64, boot.dtb_size as u64)?; for reserved in boot .reservations .as_slice() .iter() .chain(boot.static_reserved.as_slice()) { add(reserved.start, reserved.size)?; } let plan = if let Some((address, header)) = info { add(*address, BOOT_INFO_PAGE_BYTES)?; add(header.memory_map_ptr, BOOT_INFO_MAP_CAPACITY)?; let map = owned_map.ok_or(memory::Error::Invalid)?; let stride = header.descriptor_bytes as usize; if stride < 40 || stride % 8 != 0 || map.is_empty() || map.len() % stride != 0 || map.len() / stride > os_k720_frame::MAX_DESCRIPTORS { return Err(memory::Error::Invalid); } let descriptors = unsafe { &mut *core::ptr::addr_of_mut!(OS_K720_DESCRIPTORS) }; for (index, raw) in map.chunks_exact(stride).enumerate() { let entry = bootinfo::parse_memory_descriptor(raw).map_err(|_| memory::Error::Invalid)?; descriptors[index] = os_k720_frame::Descriptor { kind: entry.kind, region: Region { start: entry.start, end: entry.end, }, }; } let count = map.len() / stride; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K720_EVIDENCE) .cast::() .add(44), count as u64, ); let output = core::ptr::addr_of_mut!(OS_K720_EVIDENCE).cast::(); write_volatile(output.add(48), stride as u64); write_volatile(output.add(49), map.len() as u64); write_volatile(output.add(50), header.memory_map_ptr); write_volatile(output.add(51), *address); } os_k720_frame::Plan::firmware(&found.banks, &reservations[..len], &descriptors[..count]) } else { os_k720_frame::Plan::direct(&found.banks, &reservations[..len]) }; let plan = match plan { Ok(value) => value, Err(error) => { write_str("TURKOS/OS/FRAME: REJECTED reason="); write_str(error.tag()); write_str(" BEFORE-ALLOC\r\n"); return Err(memory::Error::Invalid); } }; // Counterexample uses a typed scratch descriptor, never corrupts the // source firmware map or published authority. Reclassification must reject // an unknown type even if its address lies outside every admitted bank. let unknown = os_k720_frame::Descriptor { kind: 99, region: Region { start: 0, end: 4096, }, }; if !matches!( os_k720_frame::Plan::firmware(&found.banks, &[], &[unknown]), Err(os_k720_frame::Error::UnknownFirmwareType(99)) ) { return Err(memory::Error::Invalid); } let audit = plan.boot_audit(); unsafe { let output = core::ptr::addr_of_mut!(OS_K720_EVIDENCE).cast::(); for (index, value) in [ audit.ram_full_pages, audit.firmware_withheld_pages, audit.boot_reserved_pages, audit.authorized_pages, ] .into_iter() .enumerate() { write_volatile(output.add(11 + index), value); } if (*core::ptr::addr_of!(OS_K720_CLASSIFIED)).is_some() { return Err(memory::Error::Invalid); } core::ptr::addr_of_mut!(OS_K720_CLASSIFIED).write(Some(plan)); } write_line("TURKOS/OS/K705: SYNTHETIC-UNKNOWN-FIRMWARE-TYPE REJECTED BEFORE-ALLOC"); Ok(()) } fn os_k720_verify_planned(free: &memory::FreeMap) -> Result<(), memory::Error> { let plan = unsafe { (*core::ptr::addr_of!(OS_K720_CLASSIFIED)).as_ref() } .ok_or(memory::Error::Invalid)?; plan.matches_admitted(free.regions()) .map_err(|_| memory::Error::Invalid)?; write_line("TURKOS/OS/K705: DTB-FIRMWARE-INTERSECTION FINAL-FREE-MAP-EQUAL BEFORE-ALLOC PASS"); Ok(()) } fn os_k720_allocator(free: memory::FreeMap) -> memory::FrameAllocator { // SAFETY: runtime start_memory is single CPU and executes once; taking // this nonclone plan transfers authority rather than reconstructing it. let plan = unsafe { (*core::ptr::addr_of_mut!(OS_K720_CLASSIFIED)).take() } .expect("classified physical authority"); memory::FrameAllocator::from_plan(free, plan) } fn os_k720_accept( allocator: &mut memory::FrameAllocator, tables: &mmu::Tables, ) -> Result<(), &'static str> { use os_k720_frame::Error; use turkos_kernel::runtime::platform_ram::Region; let found = unsafe { (*core::ptr::addr_of!(OS_K700_PLATFORM)).as_ref() } .ok_or("frame-platform-missing")?; let before = allocator.os_authority().audit().map_err(Error::tag)?; if !matches!( allocator.reserve_dynamic(Region::EMPTY), Err(Error::Invalid) ) || allocator.os_authority().audit().map_err(Error::tag)? != before { return Err("frame-empty-reservation-accepted"); } let task = unsafe { read_volatile( core::ptr::addr_of!(OS_K660_RUNTIME_EVIDENCE) .cast::() .add(4), ) }; let task_before = unsafe { read_volatile(task as *const u64) }; let mut live = [0u64; os_k720_frame::MAX_LIVE]; let mut live_len = 0; for tag in tables.tagged_frames() { if live_len == live.len() { return Err("frame-live-capacity"); } live[live_len] = tag.address; live_len += 1; } if live_len + 1 >= live.len() { return Err("frame-live-capacity"); } live[live_len] = task; live_len += 1; allocator .os_authority() .audit_observed( allocator.os_authority().free_regions(), &live[..live_len], &[], ) .map_err(Error::tag)?; live[live_len] = task; if allocator.os_authority().audit_observed( allocator.os_authority().free_regions(), &live[..live_len + 1], &[], ) != Err(Error::Audit) || allocator.os_authority().audit_observed( allocator.os_authority().free_regions(), &live[..live_len - 1], &[], ) != Err(Error::Audit) { return Err("frame-owner-inventory-accepted"); } write_line("TURKOS/OS/K720: MISSING-OWNER DUPLICATE-OWNER REJECTED LEDGER=UNCHANGED"); let first = allocator .os_authority() .free_regions() .first() .ok_or("frame-no-free")? .start; let reservation = allocator .reserve_dynamic(Region { start: first, end: first + 8192, }) .map_err(Error::tag)?; let reserved = allocator.os_authority().audit().map_err(Error::tag)?; if reserved.free_pages + 2 != before.free_pages || reserved.reserved_pages != 2 || allocator.allocate_at(first) != Err(memory::FrameRequestError::NotFree) { return Err("frame-reservation-not-withheld"); } if !matches!( allocator.reserve_dynamic(reservation.region()), Err(Error::Reserved) ) || allocator.os_authority().audit().map_err(Error::tag)? != reserved || allocator.os_authority().audit_observed( allocator.os_authority().free_regions(), &live[..live_len], &[reservation.region()], ) != Ok(reserved) || allocator.os_authority().audit_observed( allocator.os_authority().free_regions(), &live[..live_len], &[], ) != Err(Error::Audit) { return Err("frame-reservation-inventory"); } if !matches!( allocator.reserve_dynamic(Region { start: task, end: task + 4096 }), Err(Error::LiveReservation) ) || allocator.os_authority().audit().map_err(Error::tag)? != reserved || unsafe { read_volatile(task as *const u64) } != task_before { return Err("frame-live-reservation-accepted"); } allocator .release_dynamic(&reservation) .map_err(Error::tag)?; if allocator.os_authority().audit().map_err(Error::tag)? != before { return Err("frame-reservation-leak"); } write_line("TURKOS/OS/K710: EMPTY-RANGE RESERVED LIVE-TASK-RANGE REJECTED ACCOUNT=CONSERVED"); let mut canary_mask = 0u64; let mut endpoint_count = 0; for (bank_index, bank) in found.banks.as_slice().iter().enumerate() { let (first, last) = allocator .os_authority() .endpoints(*bank) .map_err(Error::tag)?; for (endpoint, address) in [(0, first), (1, last)] { allocator .allocate_at(address) .map_err(|_| "frame-endpoint-allocation")?; if os_k700_translation(tables, address)? == 0 { return Err("frame-endpoint-not-mapped"); } let canary = 0x4b37_3135_4241_4e4bu64 ^ (bank_index as u64 * 2 + endpoint as u64); // SAFETY: the central authority just leased this complete identity // mapped free RAM page; live/boot-reserved/hole pages are excluded. let page = unsafe { core::slice::from_raw_parts_mut(address as *mut u8, 4096) }; for offset in 0..4096 { unsafe { write_volatile(page.as_mut_ptr().add(offset), 0xa7); } } allocator .prepare_owned_page(address, page, canary) .map_err(Error::tag)?; for offset in 8..4088 { if unsafe { read_volatile(page.as_ptr().add(offset)) } != 0 { return Err("frame-interior-erasure"); } } if unsafe { read_volatile(address as *const u64) } != canary || unsafe { read_volatile((address + 4088) as *const u64) } != !canary { return Err("frame-endpoint-canary"); } let acquired = allocator.os_authority().audit().map_err(Error::tag)?; unsafe { write_volatile(address as *mut u8, page[0] ^ 1); } if allocator.verify_owned_page(address, page, canary) != Err(Error::PageContents) || allocator.os_authority().audit().map_err(Error::tag)? != acquired { return Err("frame-corrupt-canary-accepted"); } unsafe { write_volatile(address as *mut u8, page[0] ^ 1); } allocator .verify_owned_page(address, page, canary) .map_err(Error::tag)?; allocator .release(address) .map_err(|_| "frame-endpoint-release")?; if allocator.os_authority().audit().map_err(Error::tag)? != before { return Err("frame-endpoint-account"); } unsafe { let output = core::ptr::addr_of_mut!(OS_K720_EVIDENCE).cast::(); write_volatile(output.add(16 + bank_index * 4 + endpoint * 2), address); write_volatile(output.add(17 + bank_index * 4 + endpoint * 2), canary); } canary_mask |= 1 << (bank_index * 2 + endpoint); endpoint_count += 1; write_str("TURKOS/OS/K715: BANK=0x"); write_hex(bank_index); write_str(" ENDPOINT=0x"); write_hex(endpoint); write_str(" FRAME=0x"); write_hex(address as usize); write_str(" ERASED=4096 CANARY=0x"); write_hex(canary as usize); write_str(" RELEASED PASS\r\n"); } } if unsafe { read_volatile(task as *const u64) } != task_before { return Err("frame-task-canary-drift"); } allocator .os_authority() .audit_observed( allocator.os_authority().free_regions(), &live[..live_len], &[], ) .map_err(Error::tag)?; let after = allocator.os_authority().audit().map_err(Error::tag)?; if after != before { return Err("frame-final-account"); } unsafe { let output = core::ptr::addr_of_mut!(OS_K720_EVIDENCE).cast::(); for (index, value) in [ before.authorized_pages, before.free_pages, after.free_pages, before.live_pages, after.live_pages, 2, reserved.free_pages, reserved.live_pages, endpoint_count, canary_mask, found.banks.as_slice().len() as u64, ] .into_iter() .enumerate() { write_volatile(output.add(index), value); } write_volatile(output.add(40), task_before); write_volatile(output.add(41), read_volatile(task as *const u64)); write_volatile(output.add(42), tables.frame_count() as u64); write_volatile(output.add(43), task); write_volatile( output.add(45), allocator.os_authority().free_regions().len() as u64, ); write_volatile(output.add(46), live_len as u64); let free_output = core::ptr::addr_of_mut!(OS_K720_FREE_INVENTORY).cast::(); for (index, region) in allocator.os_authority().free_regions().iter().enumerate() { write_volatile(free_output.add(index * 2), region.start); write_volatile(free_output.add(index * 2 + 1), region.end); } let live_output = core::ptr::addr_of_mut!(OS_K720_LIVE_INVENTORY).cast::(); for (index, address) in live[..live_len].iter().enumerate() { write_volatile(live_output.add(index), *address); } } write_line("TURKOS/OS/K705: PASS"); write_line("TURKOS/OS/K710: PASS"); write_line("TURKOS/OS/K715: PASS"); write_line("TURKOS/OS/K720: LOCAL-GUEST PASS K700-FRESH-M1=PENDING"); Ok(()) } // The registry exclusively borrows the one boot runtime FrameAllocator. // Its backend uses that allocator's central authority, never a detached pool. use turkos_kernel::runtime::frame_metadata as os_k740_metadata; #[unsafe(no_mangle)] static mut OS_K740_EVIDENCE: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K740_SEED_SNAPSHOT: [u8; 4096] = [0; 4096]; #[unsafe(no_mangle)] static mut OS_K740_ZERO_SNAPSHOT: [u8; 4096] = [0; 4096]; #[unsafe(no_mangle)] static mut OS_K740_USER_STATE: [u64; 12] = [0; 12]; #[unsafe(no_mangle)] static mut OS_K740_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K740_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[repr(C, align(4096))] struct OsK740Stack { guard: [u8;4096], usable: [u8;65536] } #[unsafe(no_mangle)] static mut OS_K740_STACK: OsK740Stack = OsK740Stack {guard:[0;4096],usable:[0;65536]}; #[unsafe(no_mangle)] static mut OS_K740_STACK_EVIDENCE: [u64;24] = [0;24]; static mut OS_K740_ERROR: Option<&'static str> = None; const OS_K740_LOW_CANARY:u64=0x4b37_3430_4c4f_5721; const OS_K740_HIGH_CANARY:u64=0x4b37_3430_4849_4748; fn os_k740_stack_word(slot:usize,value:u64) { unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K740_STACK_EVIDENCE).cast::().add(slot),value)}; } fn os_k740_on_stack(runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData) -> Result<(), &'static str> { unsafe extern "C" {static __stack_start:u8;static __stack_top:u8;} let guard=core::ptr::addr_of!(OS_K740_STACK) as u64;let low=guard+4096;let end=low+65536;let high=end-8; let before_sp:u64;unsafe {asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack));} let original_low=core::ptr::addr_of!(__stack_start) as u64;let original_top=core::ptr::addr_of!(__stack_top) as u64; if before_sp<=original_low+8||before_sp>=original_top {return Err("metadata-original-stack-range");} let original_canary=unsafe {read_volatile(original_low as *const u64)}; let original_second=unsafe {read_volatile((original_low+8) as *const u64)}; if original_canary!=0x4b36_3335_5354_414b||original_second!=!0x4b36_3335_5354_414bu64 {return Err("metadata-original-stack-canaries");} unsafe {tables.unmap_metadata_stack_guard(guard,low,end,core::ptr::addr_of!(_image_start) as u64,core::ptr::addr_of!(_image_end) as u64)}.map_err(|_|"metadata-stack-guard")?; if !tables.task_stack_guard_unmapped(guard)||!tables.task_stack_guard_unmapped(original_low-4096) {return Err("metadata-stack-guard-descriptor");} unsafe {write_volatile(low as *mut u64,OS_K740_LOW_CANARY);write_volatile(high as *mut u64,OS_K740_HIGH_CANARY)}; for(slot,value) in [(1,low),(2,end-16),(7,0),(8,OS_K740_LOW_CANARY),(9,OS_K740_HIGH_CANARY),(10,original_low),(11,original_canary),(13,guard),(14,low),(15,high), (16,original_second),(18,tables.metadata_stack_guard_descriptor_address(guard).ok_or("metadata-guard-slot")?),(19,tables.metadata_stack_guard_descriptor_address(original_low-4096).ok_or("metadata-boot-guard-slot")?),(20,original_low),(21,original_top)] {os_k740_stack_word(slot,value);} unsafe extern "C" {fn __os_k740_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8)->u64;} let result=unsafe {__os_k740_on_stack((runtime as *mut OsK660Runtime).cast(),(tables as *const mmu::Tables).cast(),(boot as *const BootData).cast())}; let after_sp:u64;unsafe {asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack));} let state=unsafe {&*core::ptr::addr_of!(OS_K740_STACK_EVIDENCE)}; if before_sp!=after_sp||state[0]!=state[6]||state[3]!=end-16 || state[4]<=low+8||state[4]>end-16||state[5]<=low+8||state[5]>end-16 || unsafe {read_volatile(low as *const u64)}!=OS_K740_LOW_CANARY || unsafe {read_volatile(high as *const u64)}!=OS_K740_HIGH_CANARY || unsafe {read_volatile(original_low as *const u64)}!=original_canary || unsafe {read_volatile((original_low+8) as *const u64)}!=original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low-4096) {return Err("metadata-stack-restoration");} os_k740_stack_word(12,unsafe {read_volatile(original_low as *const u64)}); os_k740_stack_word(17,unsafe {read_volatile((original_low+8) as *const u64)}); if result==0 {write_line("TURKOS/OS/K740: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS");Ok(())} else {Err(unsafe {read_volatile(core::ptr::addr_of!(OS_K740_ERROR))}.unwrap_or("metadata-stack-return"))} } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k740_accept_body(runtime: *mut u8, tables: *const u8, boot: *const u8)->u64 { let sp:u64;unsafe {asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack));}os_k740_stack_word(4,sp); let result=os_k740_accept(unsafe {&mut *runtime.cast::()},unsafe {&*tables.cast::()},unsafe {&*boot.cast::()}); let result = result.and_then(|()| { let runtime = unsafe { &mut *runtime.cast::() }; let tables = unsafe { &*tables.cast::() }; os_k760_accept(&mut runtime.memory_mut().expect("buddy authority").allocator, tables)?; os_k780_accept(runtime,tables,unsafe {&*boot.cast::()}) }); let sp:u64;unsafe {asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack));}os_k740_stack_word(5,sp); match result {Ok(())=>0,Err(reason)=>{unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K740_ERROR),Some(reason))};1}} } #[derive(Clone, Copy, PartialEq, Eq)] struct OsK740Inventory { words: [u64; 328], leases: [(u64, u64); 4], } static mut OS_K740_COMPARE: [OsK740Inventory; 10] = [OsK740Inventory {words:[0;328],leases:[(0,0);4]};10]; struct OsK740Backend<'a> { authority: &'a mut os_k720_frame::Allocator, leases: [Option; 4], partial_zero: bool, quarantine: bool, } impl os_k740_metadata::Backend for OsK740Backend<'_> { fn allocate(&mut self) -> Result { let slot = self.leases.iter().position(Option::is_none).ok_or(os_k740_metadata::Error::Backend)?; let lease = self.authority.allocate().map_err(|_| os_k740_metadata::Error::NoMemory)?; let address = lease.address(); self.leases[slot] = Some(lease); let evidence = core::ptr::addr_of_mut!(OS_K740_EVIDENCE).cast::(); unsafe { write_volatile(evidence.add(24), read_volatile(evidence.add(24)) + 1) }; // Deliberate fault-injection backing, after real authority acquisition. if self.partial_zero { unsafe { core::ptr::write_bytes(address as *mut u8, 0xa5, 4096) } }; Ok(address) } fn release(&mut self, address: u64) -> Result<(), os_k740_metadata::Error> { if self.quarantine { return Err(os_k740_metadata::Error::Backend); } let slot = self.leases.iter().position(|token| token.as_ref().is_some_and(|lease| lease.address() == address)).ok_or(os_k740_metadata::Error::Backend)?; self.authority.release(self.leases[slot].as_ref().ok_or(os_k740_metadata::Error::Backend)?).map_err(|_| os_k740_metadata::Error::Backend)?; self.leases[slot] = None; let evidence = core::ptr::addr_of_mut!(OS_K740_EVIDENCE).cast::(); unsafe { write_volatile(evidence.add(25), read_volatile(evidence.add(25)) + 1) }; Ok(()) } fn zero_page(&mut self, address: u64) -> Result<(), os_k740_metadata::Error> { self.validate_address(address)?; // SAFETY: the registry's backend just acquired this real identity-mapped // page from central authority and retains that lease until release. unsafe { core::ptr::write_bytes(address as *mut u8, 0, if self.partial_zero {64} else {4096}) }; Ok(()) } fn read_byte(&self, address: u64, offset: usize) -> Result { if offset >= 4096 { return Err(os_k740_metadata::Error::BadAddress); } self.validate_address(address)?; Ok(unsafe { read_volatile((address as *const u8).add(offset)) }) } fn available_pages(&self) -> u64 { self.authority.remaining_pages() } } impl OsK740Backend<'_> { fn validate_address(&self,address:u64)->Result<(),os_k740_metadata::Error> { let lease=self.leases.iter().flatten().find(|lease|lease.address()==address).ok_or(os_k740_metadata::Error::BadAddress)?; self.authority.validate_lease(lease).map_err(|_|os_k740_metadata::Error::Backend) } #[inline(never)] fn inventory(&self) -> OsK740Inventory { let mut snapshot = os_k740_authority_inventory(self.authority); for (index, lease) in self.leases.iter().enumerate() { if let Some(lease) = lease { snapshot.leases[index] = (lease.address(), lease.generation()); } } snapshot } #[inline(never)] fn save(&self, slot: usize) { unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K740_COMPARE).cast::().add(slot),self.inventory())}; } #[inline(never)] fn matches(&self, slot: usize) -> bool { self.inventory()==unsafe {read_volatile(core::ptr::addr_of!(OS_K740_COMPARE).cast::().add(slot))} } } #[inline(never)] fn os_k740_save_authority(authority: &os_k720_frame::Allocator, slot: usize) { unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K740_COMPARE).cast::().add(slot),os_k740_authority_inventory(authority))}; } #[inline(never)] fn os_k740_authority_matches(authority: &os_k720_frame::Allocator, slot: usize) -> bool { os_k740_authority_inventory(authority)==unsafe {read_volatile(core::ptr::addr_of!(OS_K740_COMPARE).cast::().add(slot))} } #[inline(never)] fn os_k740_capture_authority(authority: &os_k720_frame::Allocator, baseline: usize) { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K740_AUTHORITY_BEFORE),(*core::ptr::addr_of!(OS_K740_COMPARE).cast::().add(baseline)).words); write_volatile(core::ptr::addr_of_mut!(OS_K740_AUTHORITY_AFTER),os_k740_authority_inventory(authority).words); } } #[inline(never)] fn os_k740_authority_inventory(authority: &os_k720_frame::Allocator) -> OsK740Inventory { let mut words = [0; 328]; words[0] = authority.free_regions().len() as u64; words[3] = authority.remaining_pages(); for (index, region) in authority.free_regions().iter().enumerate() { words[4 + 2 * index] = region.start; words[5 + 2 * index] = region.end; } for (index, address) in authority.live_addresses().enumerate() { words[132 + index] = address; words[1] += 1; } for (index, region) in authority.reserved_regions().enumerate() { words[196 + 2 * index] = region.start; words[197 + 2 * index] = region.end; words[2] += 1; } OsK740Inventory { words, leases: [(0, 0); 4] } } fn os_k740_error(error: os_k740_metadata::Error) -> &'static str { use os_k740_metadata::Error::*; match error { Owner => "metadata-owner", Capacity => "metadata-capacity", NoMemory => "metadata-no-memory", Backend => "metadata-backend", BadAddress => "metadata-address", DuplicateAddress => "metadata-duplicate-address", Erasure => "metadata-erasure", Foreign => "metadata-foreign-owner", Stale => "metadata-stale-handle", Pinned => "metadata-pinned", PinCapacity => "metadata-pin-capacity", PinConsumed => "metadata-pin-consumed", Generation => "metadata-generation", } } fn os_k740_snapshot(slot: usize, value: u64) { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K740_EVIDENCE).cast::().add(slot), value) }; } fn os_k740_user_code(user: &mmu::UserSpace, seed: bool) -> Result<(), &'static str> { unsafe extern "C" { static __os_k740_seed_start: u8; static __os_k740_seed_end: u8; static __os_k740_zero_start: u8; static __os_k740_zero_end: u8; } let (start,end) = if seed { (core::ptr::addr_of!(__os_k740_seed_start),core::ptr::addr_of!(__os_k740_seed_end)) } else { (core::ptr::addr_of!(__os_k740_zero_start),core::ptr::addr_of!(__os_k740_zero_end)) }; let length=(end as usize).checked_sub(start as usize).ok_or("metadata-user-code")?; if length==0 || length>4096 { return Err("metadata-user-code"); } unsafe { core::ptr::copy_nonoverlapping(start,user.code_frame() as *mut u8,length); asm!("dc cvau, {code}","dsb ish","ic ivau, {code}","dsb ish","isb",code=in(reg)user.code_frame(),options(nostack)); } Ok(()) } fn os_k740_switch(user: &mmu::UserSpace, kernel_root: u64, owner: u64, seed: bool) -> Result<(), &'static str> { let state=core::ptr::addr_of_mut!(OS_K740_USER_STATE).cast::(); let base=core::ptr::addr_of_mut!(OS_K680_USER_STATE).cast::(); unsafe { write_volatile(base,kernel_root); write_volatile(base.add(3),0); write_volatile(state.add(8),if seed {1} else {2}); write_volatile(state.add(9),0); write_volatile(state.add(if seed {0} else {1}),user.root); write_volatile(state.add(if seed {2} else {3}),owner); } unsafe extern "C" { fn __os_k680_enter_user(root: u64); } unsafe { __os_k680_enter_user(user.root) }; if unsafe {read_volatile(state.add(9))}!=1 {return Err("metadata-el0-return");} Ok(()) } fn os_k740_expected_user_svc(slot: usize, esr: usize, frame: *const ExceptionFrame) -> bool { let state=core::ptr::addr_of_mut!(OS_K740_USER_STATE).cast::(); let mode=unsafe {read_volatile(state.add(8))}; if mode==0 {return false;} if slot!=8 || (esr>>26)&63!=0x15 || esr&0xffff!=if mode==1 {0x740} else {0x741} || frame.is_null() {park();} let saved=unsafe {&*frame}; if !saved.canary_valid() {park();} let (x0,x1)=(saved.registers[0] as u64,saved.registers[1] as u64); if mode==1 && (x0!=4096 || x1!=0xa5) || mode==2 && (x0!=0 || x1!=4096) {park();} unsafe { write_volatile(state.add(if mode==1 {4} else {6}),x0); write_volatile(state.add(if mode==1 {5} else {7}),x1); write_volatile(state.add(8),0);write_volatile(state.add(9),1); write_volatile(state.add(10),(esr>>26) as u64&63);write_volatile(state.add(11),slot as u64); } unsafe extern "C" {fn __os_k680_resume_kernel()->!;} unsafe {__os_k680_resume_kernel()} } // Table ownership is observed separately from the borrowed metadata frame. fn os_k740_inventory( authority: &os_k720_frame::Allocator, tables: &mmu::Tables, task: u64, users: &[&mmu::UserSpace], metadata: &[u64], ) -> Result<(), &'static str> { let mut live=[0u64;64];let mut count=0; for frame in tables.tagged_frames() { live[count]=frame.address; count+=1; } live[count]=task;count+=1; for user in users { for frame in user.owned_frames() {live[count]=frame;count+=1;} } for frame in metadata {if count==64 {return Err("metadata-inventory-capacity");} live[count]=*frame;count+=1;} authority.audit_observed(authority.free_regions(),&live[..count],&[]).map_err(|_|"metadata-inventory-audit")?; Ok(()) } fn os_k740_quarantine(allocator: &mut memory::FrameAllocator, tables: &mmu::Tables, task: u64, owner: u64) -> Result<(), &'static str> { use os_k740_metadata::{Error, Registry}; os_k740_save_authority(allocator.os_authority(),8); let free_before=allocator.remaining_pages(); let backend=OsK740Backend {authority:allocator.os_authority_mut(),leases:core::array::from_fn(|_|None),partial_zero:true,quarantine:true}; let mut registry=Registry::<_,1>::new(0x742,backend).map_err(os_k740_error)?; if registry.allocate_zeroed(owner)!=Err(Error::Erasure) {return Err("metadata-quarantine-erasure");} let snapshot=registry.snapshot(); if snapshot.live_pages!=0||snapshot.quarantined_pages!=1||snapshot.free_metadata!=0||snapshot.backend_free_pages+1!=free_before {return Err("metadata-quarantine-publication");} let record=registry.owned_records().next().ok_or("metadata-quarantine-record")?; if !record.quarantined||record.owner!=owner||record.pins!=0 {return Err("metadata-quarantine-record");} registry.with_backend_observation(|backend|backend.save(9)); if registry.allocate_zeroed(owner)!=Err(Error::Capacity)||registry.snapshot()!=snapshot||!registry.with_backend_observation(|backend|backend.matches(9)) {return Err("metadata-quarantine-reuse");} registry.with_backend_observation(|backend| os_k740_inventory(backend.authority,tables,task,&[],&[record.address]))?; registry.with_backend_observation(|backend|os_k740_capture_authority(backend.authority,8)); for (index,value) in [(0,free_before),(1,snapshot.backend_free_pages),(2,record.address),(4,record.generation),(6,owner),(13,1),(17,1)] {os_k740_snapshot(index,value);} write_line("TURKOS/OS/K740: FAILED-ROLLBACK QUARANTINED NO-HANDLE NO-REUSE AUTHORITY-LEASE-RETAINED"); // This disposable fixture intentionally parks with registry and its real // backing lease alive. It cannot promote to successful reclaim evidence. park(); } fn os_k740_accept(runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData) -> Result<(), &'static str> { use os_k740_metadata::{Error,Registry}; if !cfg!(feature="os-runtime-acceptance") && !os_k1100_product_probe_enabled(boot)? { // Keep independent QMP control symbols present in the product build. unsafe { let _=read_volatile(core::ptr::addr_of!(OS_K740_EVIDENCE)); let _=read_volatile(core::ptr::addr_of!(OS_K740_SEED_SNAPSHOT)); let _=read_volatile(core::ptr::addr_of!(OS_K740_ZERO_SNAPSHOT)); let _=read_volatile(core::ptr::addr_of!(OS_K740_AUTHORITY_BEFORE)); let _=read_volatile(core::ptr::addr_of!(OS_K740_AUTHORITY_AFTER)); } write_line("TURKOS/OS/K740: PRODUCT-CONTROL NO-METADATA-PROBE"); return Ok(()); } let (_,memory,_,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?; let task_frame=tasks.frame.ok_or("metadata-task-frame")?; let blob=unsafe {core::slice::from_raw_parts(boot.dtb_address as *const u8,boot.dtb_size)}; let mode=turkos_kernel::runtime::platform::chosen_u32(blob,unsafe {&*core::ptr::addr_of!(OS_K700_WORKSPACE)},b"turkos,metadata-mode").map_err(|_|"metadata-fixture")?.unwrap_or(0); if mode>1 {return Err("metadata-mode");} if mode==1 {os_k740_quarantine(&mut memory.allocator,tables,task_frame,tasks.id.ok_or("metadata-task")?.generation)?;park();} os_k740_inventory(memory.allocator.os_authority(),tables,task_frame,&[],&[])?; let free_before=memory.allocator.remaining_pages();os_k740_snapshot(0,free_before); os_k740_save_authority(memory.allocator.os_authority(),0); let mut user_a=unsafe {tables.build_user_space(&mut memory.allocator,boot.hardware.ram_base,boot.hardware.ram_size)}.map_err(|_|"metadata-user-a")?; let mut user_b=unsafe {tables.build_user_space(&mut memory.allocator,boot.hardware.ram_base,boot.hardware.ram_size)}.map_err(|_|"metadata-user-b")?; os_k740_user_code(&user_a,true)?;os_k740_user_code(&user_b,false)?; os_k740_snapshot(18,user_a.data_frame());os_k740_snapshot(19,user_b.data_frame()); os_k740_inventory(memory.allocator.os_authority(),tables,task_frame,&[&user_a,&user_b],&[])?; let id_a=tasks.id.ok_or("metadata-task-a")?; if tasks.scheduler.running()!=Some(id_a) {return Err("metadata-process-a-not-selected");} let owner_a=id_a.generation; os_k740_snapshot(28,id_a.slot as u64); let backend=OsK740Backend {authority:memory.allocator.os_authority_mut(),leases:core::array::from_fn(|_|None),partial_zero:false,quarantine:false}; let mut registry=Registry::<_,4>::new(0x740,backend).map_err(os_k740_error)?; let initial=registry.snapshot(); let a=registry.allocate_zeroed(owner_a).map_err(os_k740_error)?; os_k740_snapshot(2,a.address());os_k740_snapshot(4,a.generation()); let mut client_pin=registry.pin(a,owner_a).map_err(os_k740_error)?; let stable=registry.snapshot(); registry.with_backend_observation(|backend|backend.save(1)); if registry.release(a,owner_a)!=Err(Error::Pinned)||registry.snapshot()!=stable||!registry.with_backend_observation(|backend|backend.matches(1)) {return Err("metadata-pinned-release");} let old_descriptor=user_a.descriptor(1); if unsafe {user_a.borrow_metadata_data(&mut registry,a,owner_a,0)}.is_ok()||user_a.descriptor(1)!=old_descriptor||registry.snapshot()!=stable||!registry.with_backend_observation(|backend|backend.matches(1)) {return Err("metadata-failed-setup-publication");} registry.with_backend_observation(|backend| os_k740_inventory(backend.authority,tables,task_frame,&[&user_a,&user_b],&[a.address()]))?; write_line("TURKOS/OS/K730: FAILED-BORROW-SETUP REJECTED OWNER-INVENTORY-STABLE"); let borrowed=unsafe {user_a.borrow_metadata_data(&mut registry,a,owner_a,tables.root)}.map_err(|_|"metadata-borrow-a")?; os_k740_snapshot(20,user_a.descriptor(1).ok_or("metadata-descriptor-a")?); registry.with_backend_observation(|backend|backend.save(2)); if registry.snapshot().pins!=2||registry.release(a,owner_a)!=Err(Error::Pinned)||!registry.with_backend_observation(|backend|backend.matches(2)) {return Err("metadata-live-mapping-pin");} os_k740_snapshot(14,registry.snapshot().pins as u64); os_k740_switch(&user_a,tables.root,owner_a,true)?; unsafe {user_a.restore_metadata_data(&mut registry,borrowed,tables.root)}.map_err(|_|"metadata-detach-a")?; os_k740_snapshot(21,user_a.descriptor(1).ok_or("metadata-descriptor-a")?); for index in 0..4096 { let byte=registry.read_byte(a,owner_a,index).map_err(os_k740_error)?; if byte!=0xa5 {return Err("metadata-el0-seed-incomplete");} unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K740_SEED_SNAPSHOT).cast::().add(index),byte)}; } let one_pin=registry.snapshot(); registry.with_backend_observation(|backend|backend.save(3)); if registry.release(a,owner_a)!=Err(Error::Pinned)||registry.snapshot()!=one_pin||!registry.with_backend_observation(|backend|backend.matches(3)) {return Err("metadata-pin-lifetime");} os_k740_snapshot(15,registry.snapshot().pins as u64); registry.unpin(&mut client_pin,owner_a).map_err(os_k740_error)?; let unpinned=registry.snapshot();registry.with_backend_observation(|backend|backend.save(4)); if registry.unpin(&mut client_pin,owner_a)!=Err(Error::PinConsumed)||registry.snapshot()!=unpinned||!registry.with_backend_observation(|backend|backend.matches(4)) {return Err("metadata-double-unpin");} os_k740_snapshot(16,registry.snapshot().pins as u64); let id_a=tasks.id.ok_or("metadata-task-a")?; if tasks.scheduler.exit_current(0)!=Ok(None)||tasks.scheduler.reap(id_a)!=Ok(0) {return Err("metadata-retire-process-a");} os_k740_snapshot(30,1); registry.release(a,owner_a).map_err(os_k740_error)?; let id_b=tasks.scheduler.spawn().map_err(|_|"metadata-task-b")?; if tasks.scheduler.run_next()!=Ok(Some(id_b)) {return Err("metadata-select-process-b");} tasks.id=Some(id_b);let owner_b=id_b.generation; os_k740_snapshot(29,id_b.slot as u64); let b=registry.allocate_zeroed(owner_b).map_err(os_k740_error)?; if b.address()!=a.address()||b.generation()<=a.generation()||owner_b==owner_a {return Err("metadata-page-not-reused");} os_k740_snapshot(3,b.address());os_k740_snapshot(5,b.generation());os_k740_snapshot(6,owner_a);os_k740_snapshot(7,owner_b); let before_stale=registry.snapshot(); registry.with_backend_observation(|backend|backend.save(5)); if registry.release(a,owner_a)!=Err(Error::Stale)||registry.release(b,owner_a)!=Err(Error::Foreign)||registry.snapshot()!=before_stale||!registry.with_backend_observation(|backend|backend.matches(5)) {return Err("metadata-stale-owner-free");} write_line("TURKOS/OS/K725: STALE-HANDLE FOREIGN-OWNER REJECTED LIVE-PAGE-PRESERVED"); let borrowed=unsafe {user_b.borrow_metadata_data(&mut registry,b,owner_b,tables.root)}.map_err(|_|"metadata-borrow-b")?; os_k740_snapshot(22,user_b.descriptor(1).ok_or("metadata-descriptor-b")?); os_k740_switch(&user_b,tables.root,owner_b,false)?; unsafe {user_b.restore_metadata_data(&mut registry,borrowed,tables.root)}.map_err(|_|"metadata-detach-b")?; os_k740_snapshot(23,user_b.descriptor(1).ok_or("metadata-descriptor-b")?); for index in 0..4096 { let byte=registry.read_byte(b,owner_b,index).map_err(os_k740_error)?; if byte!=0 {return Err("metadata-el0-zero-incomplete");} unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K740_ZERO_SNAPSHOT).cast::().add(index),byte)}; } // Eight genuine capability tokens exercise the bounded pin table. let mut pins:[Option;8]=core::array::from_fn(|_|None); for token in &mut pins {*token=Some(registry.pin(b,owner_b).map_err(os_k740_error)?);} let full_pins=registry.snapshot(); registry.with_backend_observation(|backend|backend.save(6)); if !matches!(registry.pin(b,owner_b),Err(Error::PinCapacity))||registry.snapshot()!=full_pins||!registry.with_backend_observation(|backend|backend.matches(6)) {return Err("metadata-pin-overflow");} for token in pins.iter_mut() {registry.unpin(token.as_mut().ok_or("metadata-pin-token")?,owner_b).map_err(os_k740_error)?;} registry.release(b,owner_b).map_err(os_k740_error)?; write_line("TURKOS/OS/K730: TWO-EL0-CONSUMERS SAME-PHYSICAL-PAGE 4096-ZERO PASS"); write_line("TURKOS/OS/K735: TWO-PINS LAST-UNPIN SINGLE-RELEASE OVERFLOW DOUBLE-UNPIN REJECTED"); let mut capacity:[Option;4]=[None;4]; for handle in &mut capacity {*handle=Some(registry.allocate_zeroed(owner_b).map_err(os_k740_error)?);} let full=registry.snapshot();registry.with_backend_observation(|backend|backend.save(7));let allocation_calls=unsafe {read_volatile(core::ptr::addr_of!(OS_K740_EVIDENCE).cast::().add(24))}; if registry.allocate_zeroed(owner_b)!=Err(Error::Capacity)||registry.snapshot()!=full||!registry.with_backend_observation(|backend|backend.matches(7))||unsafe {read_volatile(core::ptr::addr_of!(OS_K740_EVIDENCE).cast::().add(24))}!=allocation_calls {return Err("metadata-enospc-acquired-page");} let mut exact=[0u64;4];let mut owned=0; for record in registry.owned_records() {if record.quarantined||record.owner!=owner_b||record.pins!=0 {return Err("metadata-owned-record");}exact[owned]=record.address;owned+=1;} if owned!=4 {return Err("metadata-owned-count");} registry.with_backend_observation(|backend| os_k740_inventory(backend.authority,tables,task_frame,&[&user_a,&user_b],&exact))?; // Backend audit observes exactly the same physical inventory, not totals. // The registry still uniquely owns this backend, so no raw allocator call // is made while any of its records are live. for handle in capacity.into_iter().flatten() {registry.release(handle,owner_b).map_err(os_k740_error)?;} if registry.snapshot()!=initial {return Err("metadata-registry-resource-leak");} drop(registry); os_k740_inventory(memory.allocator.os_authority(),tables,task_frame,&[&user_a,&user_b],&[])?; unsafe {user_a.release_after_tlbi(&mut memory.allocator,tables.root)}.map_err(|_|"metadata-release-user-a")?; unsafe {user_b.release_after_tlbi(&mut memory.allocator,tables.root)}.map_err(|_|"metadata-release-user-b")?; os_k740_inventory(memory.allocator.os_authority(),tables,task_frame,&[],&[])?; // Same normal Image also runs the planned partial-erasure counterexample // on an actual acquired page and verifies rollback against central ledger. let negative_before=memory.allocator.remaining_pages(); let partial=OsK740Backend {authority:memory.allocator.os_authority_mut(),leases:core::array::from_fn(|_|None),partial_zero:true,quarantine:false}; let mut negative=Registry::<_,1>::new(0x741,partial).map_err(os_k740_error)?; if negative.allocate_zeroed(owner_b)!=Err(Error::Erasure)||negative.snapshot().live_pages!=0||negative.snapshot().backend_free_pages!=negative_before {return Err("metadata-partial-erasure-accepted");} negative.with_backend_observation(|backend| os_k740_inventory(backend.authority,tables,task_frame,&[],&[]))?; drop(negative); write_line("TURKOS/OS/K730: REAL-PARTIAL-ERASURE REJECTED CENTRAL-INVENTORY-RESTORED"); if memory.allocator.remaining_pages()!=free_before {return Err("metadata-authority-resource-leak");} if !os_k740_authority_matches(memory.allocator.os_authority(),0) {return Err("metadata-exact-authority-resource-leak");} os_k740_capture_authority(memory.allocator.os_authority(),0); os_k740_snapshot(1,memory.allocator.remaining_pages());os_k740_snapshot(8,4096);os_k740_snapshot(9,0);os_k740_snapshot(10,4);os_k740_snapshot(11,8); os_k740_snapshot(12,memory.allocator.os_authority().audit().map_err(|_|"metadata-final-audit")?.live_pages); os_k740_snapshot(17,0x7f); write_line("TURKOS/OS/K740: ENOSPC BEFORE-ALLOC AUTHORITY-METADATA-TABLE-INVENTORY RESTORED"); // B is retired before the reusable runtime worker runs again. if tasks.scheduler.exit_current(0)!=Ok(None)||tasks.scheduler.reap(id_b)!=Ok(0) {return Err("metadata-retire-process-b");} os_k740_snapshot(31,1); let next=tasks.scheduler.spawn().map_err(|_|"metadata-replacement-task")?;tasks.id=Some(next); write_line("TURKOS/OS/K725: PASS");write_line("TURKOS/OS/K730: PASS");write_line("TURKOS/OS/K735: PASS");write_line("TURKOS/OS/K740: PASS"); Ok(()) } global_asm!(r#" .section .text.os_k740_stack,"ax" .balign 4 .global __os_k740_on_stack __os_k740_on_stack: adrp x9, OS_K740_STACK_EVIDENCE add x9, x9, :lo12:OS_K740_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K740_STACK add x9, x9, :lo12:OS_K740_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K740_STACK_EVIDENCE add x9, x9, :lo12:OS_K740_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k740_accept_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K740_STACK_EVIDENCE add x9, x9, :lo12:OS_K740_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret .section .text.os_k740_user_blobs,"ax" .balign 4 .global __os_k740_seed_start __os_k740_seed_start: movz x1, #0x1000 movk x1, #0x8000, lsl #16 mov x2, #4096 mov w3, #0xa5 1: strb w3, [x1], #1 subs x2, x2, #1 b.ne 1b mov x0, #4096 mov x1, #0xa5 svc #0x740 b . .global __os_k740_seed_end __os_k740_seed_end: .global __os_k740_zero_start __os_k740_zero_start: movz x1, #0x1000 movk x1, #0x8000, lsl #16 mov x2, #4096 mov x0, #0 1: ldrb w3, [x1], #1 orr x0, x0, x3 subs x2, x2, #1 b.ne 1b mov x1, #4096 svc #0x741 b . .global __os_k740_zero_end __os_k740_zero_end: "#); // Image-owned BSS holds bounded metadata; no large workspace uses boot stack. static mut OS_K760_WORKSPACE: turkos_kernel::runtime::buddy::Workspace = turkos_kernel::runtime::buddy::Workspace::new(); static mut OS_K760_NEXT_IDENTITY: u64 = 1; #[unsafe(no_mangle)] unsafe extern "C" fn __os_k760_workspace_identity() -> u64 { let daif: u64; unsafe { asm!("mrs {value}, DAIF", value = out(reg) daif, options(nomem, nostack, preserves_flags)) }; if daif & 0x80 == 0 { return 0; } // SAFETY: the accepted machine is one CPU, IRQs are actually masked and // every source-copy Buddy uses this same Image-owned counter. No caller // can repeat an identity, and wrapping fails before any ledger is published. let identity = unsafe { read_volatile(core::ptr::addr_of!(OS_K760_NEXT_IDENTITY)) }; let Some(next) = identity.checked_add(1) else { return 0; }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K760_NEXT_IDENTITY), next) }; identity } #[unsafe(no_mangle)] static mut OS_K760_EVIDENCE: [u64; 128] = [0; 128]; #[unsafe(no_mangle)] static mut OS_K760_INVENTORY: [u64; 768] = [0; 768]; fn os_k760_word(index: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K760_EVIDENCE) .cast::() .add(index), value, ) }; } fn os_k760_inventory( owner: &turkos_kernel::runtime::frame_authority::Allocator, after: bool, ) -> Result<(), &'static str> { let free = owner.free_regions(); let inventory = core::ptr::addr_of_mut!(OS_K760_INVENTORY).cast::(); for (index, region) in free.iter().enumerate() { unsafe { write_volatile( inventory.add(if after { 128 } else { 0 } + index * 2), region.start, ); write_volatile( inventory.add(if after { 128 } else { 0 } + index * 2 + 1), region.end, ); } } let mut live = 0; for address in owner.live_addresses() { unsafe { write_volatile(inventory.add(if after { 320 } else { 256 } + live), address) }; live += 1; } let mut reserved = 0; for region in owner.reserved_regions() { unsafe { write_volatile( inventory.add(if after { 512 } else { 384 } + reserved * 2), region.start, ); write_volatile( inventory.add(if after { 512 } else { 384 } + reserved * 2 + 1), region.end, ) }; reserved += 1; } os_k760_word(if after { 11 } else { 10 }, free.len() as u64); os_k760_word(if after { 13 } else { 12 }, live as u64); os_k760_word(if after { 15 } else { 14 }, reserved as u64); Ok(()) } fn os_k760_page_canary( address: u64, index: usize, tables: &mmu::Tables, ) -> Result<(), &'static str> { if !matches!(os_k700_translation(tables, address), Ok(value) if value != 0) || !matches!(os_k700_translation(tables, address + memory::PAGE_SIZE - 8), Ok(value) if value != 0) { return Err("buddy-owned-page-unmapped"); } // SAFETY: caller holds this page's actual central typed Lease inside the // exclusively borrowed Buddy. Restore both saved words before any release. let (first, last) = unsafe { ( read_volatile(address as *const u64), read_volatile((address + memory::PAGE_SIZE - 8) as *const u64), ) }; let canary = 0x4b37_3630_a17e_0000u64 ^ address ^ index as u64; let valid = unsafe { write_volatile(address as *mut u64, canary); write_volatile((address + memory::PAGE_SIZE - 8) as *mut u64, !canary); let valid = read_volatile(address as *const u64) == canary && read_volatile((address + memory::PAGE_SIZE - 8) as *const u64) == !canary; write_volatile(address as *mut u64, first); write_volatile((address + memory::PAGE_SIZE - 8) as *mut u64, last); valid }; if !valid { return Err("buddy-owned-canary"); } os_k760_word(64 + index, address); os_k760_word(80 + index, first); os_k760_word(96 + index, last); Ok(()) } fn os_k760_accept( allocator: &mut memory::FrameAllocator, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::{ buddy::{BlockLease, Buddy, Error, FreeBlock}, platform_ram::Region, }; let owner = allocator.os_authority_mut(); let daif: u64; unsafe { asm!("mrs {value}, DAIF", value = out(reg) daif, options(nomem, nostack, preserves_flags)) }; if daif & 0x80 == 0 { return Err("buddy-irq-unmasked"); } let first_identity = unsafe { read_volatile(core::ptr::addr_of!(OS_K760_NEXT_IDENTITY)) }; os_k760_word(54, daif); os_k760_word(55, first_identity); let before = owner.audit().map_err(|_| "buddy-authority-before")?; os_k760_inventory(owner, false)?; let align = 16 * memory::PAGE_SIZE; let pool = owner .free_regions() .iter() .find_map(|region| { // Leave endpoint and the later K720 held-page samples outside this // interior pool, preserving the earlier checkpoint's restored bytes. let start = region.start.checked_add(2 * align - 1)? & !(align - 1); let end = start.checked_add(16 * memory::PAGE_SIZE)?; (end <= region.end).then_some(Region { start, end }) }) .ok_or("buddy-pool-unavailable")?; os_k760_word(0, 0x4b37_3630_4255_4444); os_k760_word(1, before.free_pages); os_k760_word(3, before.live_pages); os_k760_word(5, before.reserved_pages); os_k760_word(7, pool.start); os_k760_word(8, pool.end); // SAFETY: boot CPU only, actual central authority borrowed exclusively; // this image-owned BSS workspace is never concurrently accessed by IRQs. let workspace = unsafe { &mut *core::ptr::addr_of_mut!(OS_K760_WORKSPACE) }; let mut buddy = Buddy::initialize_region(workspace, owner, 760, pool).map_err(|_| "buddy-init")?; let stable = buddy.audit().map_err(|_| "buddy-audit")?; if buddy.acquire_at(pool.start + memory::PAGE_SIZE, 2).err() != Some(Error::Misaligned) || buddy.acquire(4).err() != Some(Error::InvalidOrder) || buddy.audit().map_err(|_| "buddy-order-audit")? != stable { return Err("buddy-negative-order"); } os_k760_word(48, 1); os_k760_word(49, 1); let mut leases: [Option; 4] = [const { None }; 4]; let mut page = 0; for order in 0..4 { let lease = buddy .acquire(order as u8) .map_err(|_| "buddy-order-acquire")?; os_k760_word(16 + order * 2, lease.address()); os_k760_word(17 + order * 2, lease.pages()); for index in 0..lease.pages() { os_k760_page_canary(lease.address() + index * memory::PAGE_SIZE, page, tables)?; page += 1; } write_str("TURKOS/OS/BUDDY: ORDER=0x"); write_hex(order); write_str(" ADDRESS=0x"); write_hex(lease.address() as usize); write_str(" PAGES=0x"); write_hex(lease.pages() as usize); write_str(" REAL-OWNED-CANARY PASS\r\n"); leases[order] = Some(lease); } let peak = buddy.audit().map_err(|_| "buddy-peak-audit")?; if page != 15 || peak.live_pages != 15 || peak.central_live_pages != before.live_pages + 15 || peak.central_free_pages + 15 != before.free_pages { return Err("buddy-order-inventory"); } os_k760_word(9, (1 << 15) - 1); os_k760_word(52, peak.live_pages); os_k760_word(53, before.free_pages - peak.central_free_pages); for lease in &mut leases { buddy .relinquish(lease.take().ok_or("buddy-order-token")?) .map_err(|_| "buddy-order-release")?; } if buddy.audit().map_err(|_| "buddy-order-return")? != stable { return Err("buddy-order-drift"); } let root = FreeBlock::new(pool.start, 4).map_err(|_| "buddy-root")?; let (low, high) = buddy.split_free(root).map_err(|_| "buddy-split-root")?; let (first, second) = buddy.split_free(low).map_err(|_| "buddy-split")?; let split = buddy.audit().map_err(|_| "buddy-split-audit")?; if first.pages() != 4 || second.pages() != 4 || first.region().end != second.address() || second.region().end != low.region().end || buddy.merge_free(first, first) != Err(Error::NotBuddy) || buddy.split_free(low) != Err(Error::NotFree) || buddy.audit().map_err(|_| "buddy-split-reject-audit")? != split { return Err("buddy-split-partition"); } for (index, value) in [ low.address(), low.pages(), first.address(), first.pages(), second.address(), second.pages(), ] .iter() .enumerate() { os_k760_word(24 + index, *value); } os_k760_word(30, stable.free_pages); os_k760_word(31, split.free_pages); os_k760_word(50, 1); let a = buddy .acquire_at(first.address(), 2) .map_err(|_| "buddy-merge-first")?; let live = buddy.audit().map_err(|_| "buddy-live-audit")?; if buddy.merge_free(first, second) != Err(Error::NotFree) || buddy.audit().map_err(|_| "buddy-live-reject-audit")? != live { return Err("buddy-merge-live"); } os_k760_word(51, 1); let b = buddy .acquire_at(second.address(), 2) .map_err(|_| "buddy-merge-second")?; buddy .relinquish(a) .map_err(|_| "buddy-merge-release-first")?; if buddy.acquire_at(low.address(), 3).err() != Some(Error::NoMemory) { return Err("buddy-live-coalesced"); } buddy .relinquish(b) .map_err(|_| "buddy-merge-release-second")?; let restored = buddy .acquire_at(low.address(), 3) .map_err(|_| "buddy-coalesced-acquire")?; os_k760_word(40, restored.address()); os_k760_word(41, restored.pages()); buddy .relinquish(restored) .map_err(|_| "buddy-coalesced-release")?; let _ = high; if buddy.audit().map_err(|_| "buddy-merge-return")? != stable { return Err("buddy-merge-drift"); } buddy.close().map_err(|_| "buddy-close-orders")?; let mut buddy = Buddy::initialize_region( workspace, owner, 760, Region { start: pool.start, end: pool.start + 8 * memory::PAGE_SIZE, }, ) .map_err(|_| "buddy-fragment-init")?; let mut fragments: [Option; 8] = [const { None }; 8]; for (index, lease) in fragments.iter_mut().enumerate() { *lease = Some( buddy .acquire_at(pool.start + index as u64 * memory::PAGE_SIZE, 0) .map_err(|_| "buddy-fragment-acquire")?, ); } for index in [0, 2, 4, 6] { buddy .relinquish(fragments[index].take().ok_or("buddy-fragment-token")?) .map_err(|_| "buddy-fragment-release")?; } let fragmented = buddy.audit().map_err(|_| "buddy-fragment-audit")?; if fragmented.free_pages != 4 || fragmented.live_pages != 4 || buddy.acquire(3).err() != Some(Error::NoMemory) || buddy.audit().map_err(|_| "buddy-enospc-audit")? != fragmented { return Err("buddy-contiguous-scatter"); } os_k760_word(32, fragmented.free_pages); os_k760_word(33, fragmented.live_pages); os_k760_word(34, 1); os_k760_word(36, fragmented.free_pages); os_k760_word(37, fragmented.free_pages); os_k760_word(38, fragmented.live_pages); os_k760_word(39, fragmented.live_pages); let small = buddy.acquire(0).map_err(|_| "buddy-small-progress")?; os_k760_word(35, 1); buddy.relinquish(small).map_err(|_| "buddy-small-release")?; for token in &mut fragments { if let Some(token) = token.take() { buddy .relinquish(token) .map_err(|_| "buddy-fragment-final-release")?; } } let final_pool = buddy.audit().map_err(|_| "buddy-pool-final")?; os_k760_word(42, final_pool.free_pages); os_k760_word(43, final_pool.live_pages); buddy.close().map_err(|_| "buddy-close-fragment")?; let after = owner.audit().map_err(|_| "buddy-authority-after")?; let next_identity = unsafe { read_volatile(core::ptr::addr_of!(OS_K760_NEXT_IDENTITY)) }; if first_identity.checked_add(2) != Some(next_identity) { return Err("buddy-workspace-identities"); } os_k760_word(56, next_identity); os_k760_inventory(owner, true)?; os_k760_word(2, after.free_pages); os_k760_word(4, after.live_pages); os_k760_word(6, after.reserved_pages); if before != after { return Err("buddy-central-drift"); } let inventory = core::ptr::addr_of!(OS_K760_INVENTORY).cast::(); for (before, after, words) in [(0, 128, 128), (256, 320, 64), (384, 512, 128)] { for index in 0..words { if unsafe { read_volatile(inventory.add(before + index)) != read_volatile(inventory.add(after + index)) } { return Err("buddy-exact-inventory-drift"); } } } write_line("TURKOS/OS/BUDDY: SPLIT 8=4+4 EXACT DUPLICATE-CHILD REJECTED OWNER=UNCHANGED PASS"); write_line("TURKOS/OS/BUDDY: MERGE FREE-SAME-ORDER LIVE-BUDDY REJECTED LARGE8-RESTORED PASS"); write_line( "TURKOS/OS/BUDDY: FRAGMENT FREE4 LIVE4 CONTIGUOUS8=ENOSPC SCATTER=NEVER EXACT-INVENTORY UNCHANGED PASS", ); write_line( "TURKOS/OS/BUDDY: CENTRAL TYPED-PAGE-LEASES ALL-RETURNED EXACT-INVENTORY UNCHANGED PASS", ); for marker in [ "TURKOS/OS/K745: PASS", "TURKOS/OS/K750: PASS", "TURKOS/OS/K755: PASS", "TURKOS/OS/K760: PASS", ] { write_line(marker); } Ok(()) } // This adapter lives only in the later heap overlay. All backing tokens remain // private to Heap; Buddy owns the exclusive central-authority borrow. struct OsHeapBuddyBackend<'a> { buddy: turkos_kernel::runtime::buddy::Buddy<'a>, } impl turkos_kernel::runtime::kernel_heap::Backend for OsHeapBuddyBackend<'_> { type Lease = turkos_kernel::runtime::buddy::BlockLease; fn acquire(&mut self, order: u8) -> Result, turkos_kernel::runtime::kernel_heap::Error> { use turkos_kernel::runtime::kernel_heap::{Block, Error}; let lease = self.buddy.acquire(order).map_err(|error| match error { turkos_kernel::runtime::buddy::Error::NoMemory => Error::NoMemory, _ => Error::Backing, })?; // This ARM64 kernel has usize=u64. Conversion is exact for the checked // physical RAM banks; order0..3 gives at most eight owned pages. let address = lease.address() as usize; let pages = lease.pages() as usize; Ok(Block { address, pages, lease }) } fn relinquish(&mut self, lease: Self::Lease) -> Result<(), (turkos_kernel::runtime::kernel_heap::Error, Self::Lease)> { self.buddy.relinquish(lease).map_err(|(_, lease)| (turkos_kernel::runtime::kernel_heap::Error::Backing, lease)) } } use turkos_kernel::runtime::kernel_heap as os_k780_heap; type OsK780Heap<'a> = os_k780_heap::Heap,16,16>; static mut OS_K780_WORKSPACE: turkos_kernel::runtime::buddy::Workspace = turkos_kernel::runtime::buddy::Workspace::new(); static mut OS_K780_STORAGE: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K780_COMPARE: [[u64;512];3] = [[0;512];3]; static mut OS_K780_IRQ_ARMED:u64=0; #[unsafe(no_mangle)] static mut OS_K780_PRE_HEAP_IRQ_COUNT:u64=0; #[unsafe(no_mangle)] static mut OS_K780_PRE_DIAGNOSTIC:[u64;8]=[0;8]; #[unsafe(no_mangle)] static mut OS_K780_FINAL_DIAGNOSTIC:[u64;3]=[0;3]; #[unsafe(no_mangle)] static mut OS_K780_EVIDENCE:[u64;64]=[0;64]; #[unsafe(no_mangle)] static mut OS_K780_OBJECTS:[u64;96]=[0;96]; #[unsafe(no_mangle)] static mut OS_K780_METADATA_FULL:[u64;80]=[0;80]; #[unsafe(no_mangle)] static mut OS_K780_CANARY:[u8;8416]=[0;8416]; #[unsafe(no_mangle)] static mut OS_K780_AUTHORITY_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K780_AUTHORITY_LIVE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K780_AUTHORITY_AFTER:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K780_AUTHORITY_FULL:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K780_AUTHORITY_IRQ_AFTER:[u64;328]=[0;328]; fn os_k780_word(slot:usize,value:u64) {unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K780_EVIDENCE).cast::().add(slot),value)};} fn os_k780_get(slot:usize)->u64 {unsafe {read_volatile(core::ptr::addr_of!(OS_K780_EVIDENCE).cast::().add(slot))}} struct OsK780Backend<'a> {inner:OsHeapBuddyBackend<'a>} impl os_k780_heap::Backend for OsK780Backend<'_> { type Lease=turkos_kernel::runtime::buddy::BlockLease; fn acquire(&mut self,order:u8)->Result,os_k780_heap::Error> { os_k780_word(24,os_k780_get(24)+1); self.inner.buddy.audit().map_err(|_|os_k780_heap::Error::Audit)?; let block=os_k780_heap::Backend::acquire(&mut self.inner,order)?; unsafe {core::ptr::write_bytes(block.address as *mut u8,0,block.pages*4096)}; os_k780_word(25,os_k780_get(25)+1);Ok(block) } fn relinquish(&mut self,lease:Self::Lease)->Result<(),(os_k780_heap::Error,Self::Lease)> { if self.inner.buddy.audit().is_err() {return Err((os_k780_heap::Error::Audit,lease));} unsafe {core::ptr::write_bytes(lease.address() as *mut u8,0,lease.pages() as usize*4096)}; match os_k780_heap::Backend::relinquish(&mut self.inner,lease) {Ok(())=>{os_k780_word(26,os_k780_get(26)+1);Ok(())},Err(error)=>Err(error)} } } // Called from the IRQ handler before any heap reference could be formed. // Heap::context is the private frozen policy; this helper delegates to it. fn os_k780_irq_gate()->bool { if unsafe {read_volatile(core::ptr::addr_of!(OS_K780_IRQ_ARMED))}==0 {return true;} let before=os_k780_get(24); let refused=os_k780_heap::context_permission::,16,16>(os_k780_heap::Context::Interrupt)==Err(os_k780_heap::Error::WouldBlock); if refused&&before==os_k780_get(24) {os_k780_word(31,os_k780_get(31)+1);true} else {os_k780_word(32,1);false} } #[inline(never)] fn os_k780_authority_words(authority:&os_k720_frame::Allocator,target:*mut u64) { for i in 0..328 {unsafe {write_volatile(target.add(i),0)};} unsafe {write_volatile(target,authority.free_regions().len() as u64);write_volatile(target.add(3),authority.remaining_pages())}; for(i,region) in authority.free_regions().iter().enumerate() {unsafe {write_volatile(target.add(4+2*i),region.start);write_volatile(target.add(5+2*i),region.end)};} for(i,address) in authority.live_addresses().enumerate() {unsafe {write_volatile(target.add(132+i),address);write_volatile(target.add(1),i as u64+1)};} for(i,region) in authority.reserved_regions().enumerate() {unsafe {write_volatile(target.add(196+2*i),region.start);write_volatile(target.add(197+2*i),region.end);write_volatile(target.add(2),i as u64+1)};} } #[inline(never)] fn os_k780_snapshot(heap:&OsK780Heap<'_>,slot:usize)->Result<(),&'static str> { heap.audit().map_err(|_|"heap-object-audit")?; let output=unsafe {&mut *core::ptr::addr_of_mut!(OS_K780_COMPARE).cast::<[u64;512]>().add(slot)}; output.fill(0); heap.with_backend_observation(|backend|backend.inner.buddy.with_authority_observation(|authority|os_k780_authority_words(authority,output.as_mut_ptr()))); for(i,row) in heap.object_records().enumerate() {let at=328+5*i;output[at..at+5].copy_from_slice(&[row.address as u64,row.owner as u64,row.generation,row.requested as u64,row.capacity as u64]);} for(i,row) in heap.backing_records().enumerate() {let at=408+3*i;output[at..at+3].copy_from_slice(&[row.address as u64,row.pages as u64,match row.kind {os_k780_heap::Kind::Slab(class)=>class as u64,os_k780_heap::Kind::Large=>0,os_k780_heap::Kind::Quarantined=>u64::MAX}]);} heap.with_backend_observation(|backend| { let buddy=&backend.inner.buddy;buddy.audit().map_err(|_|"heap-buddy-audit")?; if buddy.free_blocks().len()>8 {return Err("heap-free-block-capacity");} for(i,block) in buddy.free_blocks().iter().enumerate() {output[456+2*i]=block.address();output[457+2*i]=block.pages();} Ok(()) }) } fn os_k780_matches(heap:&OsK780Heap<'_>,slot:usize)->Result { os_k780_snapshot(heap,2)?; let saved=unsafe {&*core::ptr::addr_of!(OS_K780_COMPARE).cast::<[u64;512]>().add(slot)}; let actual=unsafe {&*core::ptr::addr_of!(OS_K780_COMPARE).cast::<[u64;512]>().add(2)}; Ok(saved==actual) } fn os_k780_exact_inventory(heap:&OsK780Heap<'_>,tables:&mmu::Tables,task:u64)->Result<(),&'static str> { let mut live=[0u64;64];let mut count=0; for frame in tables.tagged_frames() {live[count]=frame.address;count+=1;}live[count]=task;count+=1; let mut blocks=[(0u64,0u8);16];let mut block_count=0; for record in heap.backing_records() { if record.kind==os_k780_heap::Kind::Quarantined||block_count==16 {return Err("heap-quarantined-backing");} blocks[block_count]=(record.address as u64,(record.pages as u64).trailing_zeros() as u8);block_count+=1; for page in 0..record.pages {if count==64 {return Err("heap-inventory-capacity");}live[count]=record.address as u64+page as u64*4096;count+=1;} } heap.with_backend_observation(|backend| { let buddy=&backend.inner.buddy;let mut seen=0; for block in buddy.live_blocks() {if !blocks[..block_count].contains(&block) {return Err("heap-backing-buddy-mismatch");}seen+=1;} if seen!=block_count {return Err("heap-backing-buddy-count");} buddy.with_authority_observation(|authority|authority.audit_observed(authority.free_regions(),&live[..count],&[]).map_err(|_|"heap-central-inventory"))?; buddy.audit().map_err(|_|"heap-central-generations")?;Ok(()) }) } fn os_k780_timers()->Result<(),&'static str> { let frequency:u64;unsafe {asm!("mrs {v}, CNTFRQ_EL0",v=out(reg)frequency,options(nomem,nostack));} let interval=turkos_kernel::irq::timer_ticks_for_ns(frequency,1_000_000).ok_or("heap-timer-frequency")?; unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K780_IRQ_ARMED),1)}; for index in 0..4 { let count=unsafe {read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))};let now:u64; unsafe {asm!("mrs {v}, CNTPCT_EL0",v=out(reg)now,options(nomem,nostack));} let deadline=now.checked_add(interval).ok_or("heap-timer-overflow")?;let limit=now.checked_add(frequency).ok_or("heap-timer-overflow")?; unsafe {asm!("msr CNTP_CVAL_EL0, {deadline}","msr CNTP_CTL_EL0, {enable}","isb","msr DAIFClr, #2",deadline=in(reg)deadline,enable=in(reg)1u64,options(nostack));} loop { if unsafe {read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}!=count {break;} let now:u64;unsafe {asm!("mrs {v}, CNTPCT_EL0",v=out(reg)now,options(nomem,nostack));} if now>limit {unsafe {asm!("msr DAIFSet, #2","msr CNTP_CTL_EL0, xzr","isb",options(nostack));}return Err("heap-timer-timeout");} } unsafe {asm!("msr DAIFSet, #2","isb",options(nostack));} let after=unsafe {read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}; if after!=count+1||unsafe {read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER))}Result<(),&'static str> { use os_k780_heap::{Context,Error,Handle,Heap}; let before_irq=unsafe {read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}; unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K780_PRE_HEAP_IRQ_COUNT),before_irq);write_volatile(core::ptr::addr_of_mut!(OS_K780_PRE_DIAGNOSTIC),read_volatile(core::ptr::addr_of!(OS_K680_EVIDENCE)))}; let log=os_k680_log()?;unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K780_FINAL_DIAGNOSTIC),[log.boot().value(),log.next_sequence(),log.len() as u64])}; if !cfg!(feature="os-runtime-acceptance") && !os_k1100_product_probe_enabled(boot)? { unsafe {let _=read_volatile(core::ptr::addr_of!(OS_K780_EVIDENCE));let _=read_volatile(core::ptr::addr_of!(OS_K780_OBJECTS));let _=read_volatile(core::ptr::addr_of!(OS_K780_METADATA_FULL));let _=read_volatile(core::ptr::addr_of!(OS_K780_CANARY));let _=read_volatile(core::ptr::addr_of!(OS_K780_AUTHORITY_BEFORE));let _=read_volatile(core::ptr::addr_of!(OS_K780_AUTHORITY_LIVE));let _=read_volatile(core::ptr::addr_of!(OS_K780_AUTHORITY_AFTER));let _=read_volatile(core::ptr::addr_of!(OS_K780_AUTHORITY_FULL));let _=read_volatile(core::ptr::addr_of!(OS_K780_AUTHORITY_IRQ_AFTER));} write_line("TURKOS/OS/K780: PRODUCT-CONTROL NO-HEAP-PROBE");return Ok(()); } if before_irq!=8 {return Err("heap-real-prerequisite-irq");} let(_,memory,_,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?; let task=tasks.frame.ok_or("heap-task-frame")?;let owner=u32::try_from(tasks.id.ok_or("heap-owner-task")?.generation).map_err(|_|"heap-owner-generation")?; let authority=memory.allocator.os_authority_mut(); os_k780_authority_words(authority,core::ptr::addr_of_mut!(OS_K780_AUTHORITY_BEFORE).cast()); let buddy_end=unsafe {read_volatile(core::ptr::addr_of!(OS_K760_EVIDENCE).cast::().add(8))}; let pool=authority.free_regions().iter().find_map(|region| {let start=(region.start.max(buddy_end+32768)+32767)&!32767;let end=start.checked_add(32768)?;(end<=region.end).then_some(turkos_kernel::runtime::platform_ram::Region{start,end})}).ok_or("heap-eight-page-pool")?; os_k780_word(0,0x4b37_3830_4845_4150);os_k780_word(1,pool.start);os_k780_word(2,pool.end);os_k780_word(3,owner as u64);os_k780_word(28,before_irq); let workspace=unsafe {&mut *core::ptr::addr_of_mut!(OS_K780_WORKSPACE)}; let buddy=turkos_kernel::runtime::buddy::Buddy::initialize_region(workspace,authority,780,pool).map_err(|_|"heap-buddy-init")?; let backend=OsK780Backend{inner:OsHeapBuddyBackend{buddy}}; let storage=core::ptr::addr_of_mut!(OS_K780_STORAGE).cast::>(); unsafe {Heap::init_in_place(storage,backend,780)}.map_err(|_|"heap-in-place-init")?; let heap=unsafe {&mut *storage}; let mut handles:[Option;16]=[None;16];let layouts=[(24,8),(64,16),(128,32),(8192,4096)]; for(i,(size,align)) in layouts.into_iter().enumerate() {handles[i]=Some(heap.allocate(owner,size,align,Context::Task).map_err(|_|"heap-real-allocation")?);} os_k780_exact_inventory(heap,tables,task)?; heap.with_backend_observation(|backend|backend.inner.buddy.with_authority_observation(|authority|os_k780_authority_words(authority,core::ptr::addr_of_mut!(OS_K780_AUTHORITY_LIVE).cast()))); let peak=heap.audit().map_err(|_|"heap-peak-audit")?;os_k780_word(4,peak.objects as u64);os_k780_word(5,peak.requested_bytes as u64);os_k780_word(6,peak.capacity_bytes as u64);os_k780_word(7,peak.backing_pages as u64); let mut at=0; for(i,record) in heap.object_records().enumerate() { let output=core::ptr::addr_of_mut!(OS_K780_OBJECTS).cast::(); for(j,value) in [record.address as u64,record.owner as u64,record.generation,record.requested as u64,record.capacity as u64,layouts[i].1 as u64].into_iter().enumerate() {unsafe {write_volatile(output.add(i*6+j),value)};} for offset in 0..record.capacity {unsafe {write_volatile((record.address as *mut u8).add(offset),0x61+i as u8)};} for offset in 0..record.capacity {let byte=unsafe {read_volatile((record.address as *const u8).add(offset))};if byte!=0x61+i as u8 {return Err("heap-full-byte-canary");}unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K780_CANARY).cast::().add(at+offset),byte)};} at+=record.capacity; } if at!=8416 {return Err("heap-canary-checkpoint-length");} os_k780_snapshot(heap,0)?;let first=handles[0].ok_or("heap-first-handle")?; if heap.release(first,owner+1,Context::Task)!=Err(Error::ForeignHandle)||heap.release_pointer(first,owner,first.address()+1,Context::Task)!=Err(Error::ForeignPointer)||!os_k780_matches(heap,0)? {return Err("heap-foreign-interior-accepted");} heap.release(first,owner,Context::Task).map_err(|_|"heap-release-first")?;handles[0]=None; os_k780_snapshot(heap,0)?; if heap.release(first,owner,Context::Task)!=Err(Error::StaleHandle)||!os_k780_matches(heap,0)? {return Err("heap-double-free-accepted");} let replacement=heap.allocate(owner,24,8,Context::Task).map_err(|_|"heap-reuse-cell")?; if replacement.address()!=first.address()||replacement.generation()<=first.generation() {return Err("heap-slot-not-reused");} handles[0]=Some(replacement);os_k780_word(8,first.address() as u64);os_k780_word(9,replacement.address() as u64);os_k780_word(10,first.generation());os_k780_word(11,replacement.generation()); os_k780_snapshot(heap,0)?; if heap.release(first,owner,Context::Task)!=Err(Error::StaleHandle)||!os_k780_matches(heap,0)? {return Err("heap-stale-generation-accepted");} for handle in &mut handles {if let Some(handle)=handle.take() {heap.release(handle,owner,Context::Task).map_err(|_|"heap-canary-release")?;}} for handle in &mut handles {*handle=Some(heap.allocate(owner,24,8,Context::Task).map_err(|_|"heap-metadata-fill")?);} for(i,record) in heap.object_records().enumerate() {for(j,value) in [record.address as u64,record.owner as u64,record.generation,record.requested as u64,record.capacity as u64].into_iter().enumerate() {unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K780_METADATA_FULL).cast::().add(i*5+j),value)};}} os_k780_snapshot(heap,0)?;let calls=os_k780_get(24); if heap.allocate(owner,24,8,Context::Task)!=Err(Error::MetadataFull)||os_k780_get(24)!=calls||!os_k780_matches(heap,0)? {return Err("heap-metadata-enospc-acquired");} os_k780_word(12,heap.snapshot().objects as u64);os_k780_word(13,1); for handle in &mut handles {heap.release(handle.take().ok_or("heap-fill-handle")?,owner,Context::Task).map_err(|_|"heap-fill-release")?;} for handle in handles.iter_mut().take(8) {*handle=Some(heap.allocate(owner,4096,4096,Context::Task).map_err(|_|"heap-backing-fill")?);} os_k780_exact_inventory(heap,tables,task)?;os_k780_snapshot(heap,0)?;let calls=os_k780_get(24); heap.with_backend_observation(|backend|backend.inner.buddy.with_authority_observation(|authority|os_k780_authority_words(authority,core::ptr::addr_of_mut!(OS_K780_AUTHORITY_FULL).cast()))); if heap.allocate(owner,4096,4096,Context::Task)!=Err(Error::NoMemory)||os_k780_get(24)!=calls+1||!os_k780_matches(heap,0)? {return Err("heap-real-enospc");} os_k780_word(14,heap.snapshot().backing_pages as u64);os_k780_word(15,1); let calls=os_k780_get(24);os_k780_word(30,calls); { let _held=SERIAL_LINE_LOCK.lock();os_k780_word(16,u64::from(SERIAL_LINE_LOCK.is_locked())); let record=os_k680_append(os_diagnostics::Service::Memory,os_diagnostics::Input::Public(os_diagnostics::PublicData::Event(os_diagnostics::EventCode::Rejected)))?; os_k780_word(17,record.sequence());os_k780_word(18,1); } if os_k780_get(24)!=calls||!os_k780_matches(heap,0)? {return Err("heap-diagnostics-acquired-backing");} os_k780_timers()?; heap.with_backend_observation(|backend|backend.inner.buddy.with_authority_observation(|authority|os_k780_authority_words(authority,core::ptr::addr_of_mut!(OS_K780_AUTHORITY_IRQ_AFTER).cast()))); if os_k780_get(24)!=calls||!os_k780_matches(heap,0)?||os_k780_get(31)!=4 {return Err("heap-irq-acquired-backing");} let irq=unsafe {read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))};if irq!=before_irq+4 {return Err("heap-cumulative-irq");}os_k780_word(29,irq); let log=os_k680_log()?;unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K780_FINAL_DIAGNOSTIC),[log.boot().value(),log.next_sequence(),log.len() as u64])}; for handle in &mut handles {if let Some(handle)=handle.take() {heap.release(handle,owner,Context::Task).map_err(|_|"heap-pressure-release")?;}} if heap.audit().map_err(|_|"heap-final-audit")?!=os_k780_heap::Snapshot::default() {return Err("heap-owner-leak");} os_k780_exact_inventory(heap,tables,task)?; let backend=unsafe {Heap::take_backend_in_place(storage)}.map_err(|_|"heap-backend-still-live")?; backend.inner.buddy.close().map_err(|_|"heap-buddy-still-live")?; os_k780_authority_words(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K780_AUTHORITY_AFTER).cast()); let before=unsafe {&*core::ptr::addr_of!(OS_K780_AUTHORITY_BEFORE)};let after=unsafe {&*core::ptr::addr_of!(OS_K780_AUTHORITY_AFTER)}; if before!=after {return Err("heap-central-inventory-leak");} os_k780_word(19,0x3f); write_line("TURKOS/OS/K765: SLAB24 SLAB64 SLAB128 REAL-OWNED BYTE-CANARIES PASS"); write_line("TURKOS/OS/K770: LARGE8192 CONTIGUOUS OWNED-BLOCK EXACT-INVENTORY PASS"); write_line("TURKOS/OS/K775: FOREIGN INTERIOR STALE DOUBLE-FREE REJECTED EXACT-INVENTORY UNCHANGED PASS"); write_line("TURKOS/OS/K780: FULL-BUDDY8 FULL-METADATA16 BOUNDED-DIAGNOSTIC HELD-LOCK REAL-IRQ4 WOULD-BLOCK4 ZERO-ACQUIRES PASS"); for marker in ["TURKOS/OS/K765: PASS","TURKOS/OS/K770: PASS","TURKOS/OS/K775: PASS","TURKOS/OS/K780: PASS"] {write_line(marker);}Ok(()) } // Isolated QEMU VirtIO guest integration. Hardware counters below are written // only by volatile CPU accesses and actual ARM64 barrier callbacks. The host // debugger is authorized to write one checkpoint byte and no result evidence. use turkos_kernel::runtime::dma as os_k800_dma; #[unsafe(no_mangle)] static mut OS_K800_EVIDENCE: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K800_QUEUES: [u64; 128] = [0; 128]; #[unsafe(no_mangle)] static mut OS_K800_AUTHORITY: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K800_REGISTRY: [u64; 24] = [0; 24]; #[unsafe(no_mangle)] static mut OS_K800_CAPTURE_GATE: u8 = 0; static mut OS_K800_WORKSPACE: os_k800_dma::Workspace = os_k800_dma::Workspace::new(); fn os_k800_word(queue: usize, offset: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K800_QUEUES) .cast::() .add(queue * 64 + offset), value, ) }; } fn os_k800_increment(queue: usize, offset: usize) { unsafe { let pointer = core::ptr::addr_of_mut!(OS_K800_QUEUES) .cast::() .add(queue * 64 + offset); write_volatile(pointer, read_volatile(pointer) + 1) }; } unsafe fn os_k800_read(base: u64, offset: u64) -> u32 { unsafe { read_volatile((base + offset) as *const u32) } } unsafe fn os_k800_write(base: u64, offset: u64, value: u32) { unsafe { write_volatile((base + offset) as *mut u32, value) } } unsafe fn os_k800_write64(base: u64, offset: u64, value: u64) { unsafe { os_k800_write(base, offset, value as u32); os_k800_write(base, offset + 4, (value >> 32) as u32) } } fn os_k800_clock() -> u64 { let value: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0",value=out(reg)value,options(nomem,nostack,preserves_flags)) }; value } struct OsK800Backend<'a> { authority: &'a mut os_k720_frame::Allocator, leases: [Option; 4], addresses: [u64; 4], } impl OsK800Backend<'_> { fn validates(&self, address: u64) -> Result<(), os_k740_metadata::Error> { let lease = self .leases .iter() .flatten() .find(|lease| lease.address() == address) .ok_or(os_k740_metadata::Error::BadAddress)?; self.authority .validate_lease(lease) .map_err(|_| os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK800Backend<'_> { fn allocate(&mut self) -> Result { let slot = self .leases .iter() .position(Option::is_none) .ok_or(os_k740_metadata::Error::Capacity)?; let lease = self .authority .allocate_at(self.addresses[slot]) .map_err(|_| os_k740_metadata::Error::NoMemory)?; let address = lease.address(); self.leases[slot] = Some(lease); Ok(address) } fn release(&mut self, address: u64) -> Result<(), os_k740_metadata::Error> { let slot = self .leases .iter() .position(|lease| { lease .as_ref() .is_some_and(|lease| lease.address() == address) }) .ok_or(os_k740_metadata::Error::BadAddress)?; self.authority .release( self.leases[slot] .as_ref() .ok_or(os_k740_metadata::Error::Backend)?, ) .map_err(|_| os_k740_metadata::Error::Backend)?; self.leases[slot] = None; Ok(()) } fn zero_page(&mut self, address: u64) -> Result<(), os_k740_metadata::Error> { self.validates(address)?; for offset in 0..4096 { unsafe { write_volatile((address as *mut u8).add(offset), 0) } } Ok(()) } fn read_byte(&self, address: u64, offset: usize) -> Result { if offset >= 4096 { return Err(os_k740_metadata::Error::BadAddress); } self.validates(address)?; Ok(unsafe { read_volatile((address as *const u8).add(offset)) }) } fn available_pages(&self) -> u64 { self.authority.remaining_pages() } } struct OsK800Device { binding: os_k800_dma::Binding, available: u64, used: u64, withhold_reset: bool, } impl os_k800_dma::Device for OsK800Device { fn binding(&self) -> os_k800_dma::Binding { self.binding } fn publish_barrier(&mut self) { unsafe { asm!("dmb oshst", options(nostack, preserves_flags)) }; os_k800_increment(self.binding.queue as usize, 20) } fn store_available_index(&mut self, index: u16) { unsafe { write_volatile((self.available + 2) as *mut u16, index.to_le()) }; os_k800_word(self.binding.queue as usize, 16, index as u64) } fn notify(&mut self, queue: u16) { unsafe { os_k800_write(self.binding.device, 0x50, queue as u32) } } fn read_used_index(&mut self) -> u16 { let value = unsafe { u16::from_le(read_volatile((self.used + 2) as *const u16)) }; os_k800_word(self.binding.queue as usize, 17, value as u64); value } fn consume_barrier(&mut self) { unsafe { asm!("dmb oshld", options(nostack, preserves_flags)) }; os_k800_increment(self.binding.queue as usize, 21) } fn read_used_element(&mut self) -> os_k800_dma::Completion { let id = unsafe { u32::from_le(read_volatile((self.used + 4) as *const u32)) }; let length = unsafe { u32::from_le(read_volatile((self.used + 8) as *const u32)) }; os_k800_word(self.binding.queue as usize, 18, id as u64); os_k800_word(self.binding.queue as usize, 19, length as u64); os_k800_dma::Completion { id, length } } fn write_reset(&mut self) { if self.withhold_reset { write_line( "TURKOS/OS/K795: CONTROL WITHHELD-RESET-WRITE REAL-DEVICE-READBACK-REQUIRED", ); } else { unsafe { os_k800_write(self.binding.device, 0x70, 0) }; os_k800_increment(self.binding.queue as usize, 22) } } fn reset_barrier(&mut self) { unsafe { asm!("dsb sy", options(nostack, preserves_flags)) }; os_k800_increment(self.binding.queue as usize, 23) } fn read_status(&mut self) -> u32 { let value = unsafe { os_k800_read(self.binding.device, 0x70) }; os_k800_word(self.binding.queue as usize, 24, value as u64); value } fn read_queue_ready(&mut self, queue: u16) -> u32 { unsafe { os_k800_write(self.binding.device, 0x30, queue as u32) }; let value = unsafe { os_k800_read(self.binding.device, 0x44) }; os_k800_word(self.binding.queue as usize, 25, value as u64); value } } fn os_k800_packet(boot_id: u64, queue: usize) -> [u8; 60] { let mut frame = [0u8; 60]; let nic = [0x52, 0x54, 0, 0x12, 0x34, 0x56]; let peer = [2, 0, 0, 0, 0, 1]; frame[..6].copy_from_slice(if queue == 0 { &nic } else { &peer }); frame[6..12].copy_from_slice(if queue == 0 { &peer } else { &nic }); frame[12..14].copy_from_slice(&[0x88, 0xb5]); frame[14..22].copy_from_slice(&boot_id.to_be_bytes()); frame[22] = queue as u8; for offset in 23..60 { frame[offset] = (0x40 + queue * 64 + offset) as u8; } frame } fn os_k800_capture( scope: &os_k800_dma::Dma<'_, OsK800Backend<'_>, 4>, stage: u64, ) -> Result<(), &'static str> { let snapshot = scope.audit().map_err(os_k800_dma::Error::tag)?; scope.observe_registry(|registry| { registry.with_backend_observation(|backend| unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K800_AUTHORITY), os_k740_authority_inventory(backend.authority).words, ) }); let output = core::ptr::addr_of_mut!(OS_K800_REGISTRY).cast::(); unsafe { for index in 0..24 { write_volatile(output.add(index), 0) } }; for (index, record) in registry.owned_records().enumerate() { for (field, value) in [ record.address, record.owner, record.generation, record.pins as u64, record.quarantined as u64, ] .into_iter() .enumerate() { unsafe { write_volatile(output.add(index * 5 + field), value) } } } }); unsafe { let output = core::ptr::addr_of_mut!(OS_K800_EVIDENCE).cast::(); for (index, value) in [ (7, stage), (8, snapshot.pins as u64), (9, snapshot.registry_pins as u64), (10, snapshot.active as u64), (11, snapshot.quarantined as u64), ] { write_volatile(output.add(index), value) } write_volatile(core::ptr::addr_of_mut!(OS_K800_CAPTURE_GATE), 0) }; write_str("TURKOS/OS/K800: CAPTURE STAGE=0x"); write_hex(stage as usize); write_str(" GATE=0x"); write_hex(core::ptr::addr_of!(OS_K800_CAPTURE_GATE) as usize); write_line(" READY"); let frequency: u64; unsafe { asm!("mrs {value}, CNTFRQ_EL0",value=out(reg)frequency,options(nomem,nostack,preserves_flags)) }; let deadline = os_k800_clock() .checked_add(frequency.checked_mul(45).ok_or("dma-capture-budget")?) .ok_or("dma-capture-deadline")?; while unsafe { read_volatile(core::ptr::addr_of!(OS_K800_CAPTURE_GATE)) } != 1 { if os_k800_clock() >= deadline { return Err("dma-capture-timeout"); } core::hint::spin_loop() } Ok(()) } fn os_k800_configure(base: u64, queue: usize, frames: [u64; 4]) -> Result<(), &'static str> { unsafe { os_k800_write(base, 0x70, 0); asm!("dsb sy", options(nostack, preserves_flags)); if os_k800_read(base, 0x70) != 0 { return Err("dma-initial-reset"); }; os_k800_write(base, 0x70, 3); os_k800_write(base, 0x14, 0); let low = os_k800_read(base, 0x10); os_k800_write(base, 0x14, 1); let high = os_k800_read(base, 0x10); if low & 32 == 0 || high & 1 == 0 { return Err("dma-net-features"); }; os_k800_word(queue, 7, low as u64); os_k800_word(queue, 8, high as u64); os_k800_write(base, 0x24, 0); os_k800_write(base, 0x20, 32); os_k800_write(base, 0x24, 1); os_k800_write(base, 0x20, 1); os_k800_write(base, 0x70, 0x0b); if os_k800_read(base, 0x70) != 0x0b { return Err("dma-features-rejected"); }; os_k800_write(base, 0x30, queue as u32); let max = os_k800_read(base, 0x34); if max < 8 || os_k800_read(base, 0x44) != 0 { return Err("dma-queue-admission"); }; os_k800_word(queue, 9, max as u64); os_k800_write(base, 0x38, 8); os_k800_write64(base, 0x80, frames[0]); os_k800_write64(base, 0x90, frames[1]); os_k800_write64(base, 0xa0, frames[2]); asm!("dmb oshst", options(nostack, preserves_flags)); os_k800_write(base, 0x44, 1); let ready = os_k800_read(base, 0x44); if ready != 1 { return Err("dma-ready-readback"); }; os_k800_word(queue, 11, ready as u64); os_k800_write(base, 0x70, 0x0f); let status = os_k800_read(base, 0x70); if status != 15 { return Err("dma-driver-ready"); }; os_k800_word(queue, 10, status as u64); } Ok(()) } fn os_k800_accept( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { use os_k800_dma::{Binding, Direction, Error}; use turkos_kernel::runtime::device_registry::{DetachError, DeviceClass, DeviceIdentity, DeviceKey, ResourceKind, ResourceRange, SafeRegistry, Transport}; let daif: u64; unsafe { asm!("mrs {value}, DAIF",value=out(reg)daif,options(nomem,nostack,preserves_flags)) }; if daif & 0x80 == 0 { return Err("dma-irq-unmasked"); }; let blob = unsafe { core::slice::from_raw_parts(boot.dtb_address as *const u8, boot.dtb_size) }; let mode = turkos_kernel::runtime::platform::chosen_u32( blob, unsafe { &*core::ptr::addr_of!(OS_K700_WORKSPACE) }, b"turkos,dma-mode", ) .map_err(|_| "dma-fixture")? .unwrap_or(0); if mode > 1 || mode != 0 && cfg!(feature = "os-runtime-product") { return Err("dma-mode"); } let k1420_mode = cfg!(feature = "os-runtime-product") && os_k1100_product_probe_mode(boot)? == 75; let dma_nodes = dtb::discover_virtio_mmio(blob).map_err(|_| "dma-dtb-devices")?; let mut device = None; for node in dma_nodes.as_slice() { let magic = unsafe { os_k800_read(node.base, 0) }; let version = unsafe { os_k800_read(node.base, 4) }; let id = unsafe { os_k800_read(node.base, 8) }; if magic != 0x7472_6976 { return Err("dma-mmio-magic"); }; if id == 1 { if version != 2 || device.replace(node.base).is_some() { return Err("dma-net-device"); } } } let base = device.ok_or("dma-no-net")?; let net_irq = if k1420_mode { turkos_kernel::runtime::platform::os_k940_virtio_spi( blob, unsafe { &*core::ptr::addr_of!(OS_K700_WORKSPACE) }, base, boot.interrupts.gic_distributor) .map_err(|_| "k1420-net-spi")?.0 } else { 0 }; let descriptor = os_k700_translation(tables, base)?; if descriptor & 1 == 0 || (descriptor >> 2) & 7 != 1 { return Err("dma-mmio-not-device-mapped"); } let boot_id = unsafe { read_volatile(core::ptr::addr_of!(OS_K680_EVIDENCE).cast::()) }; let (_, memory, _, tasks) = runtime.running().map_err(os_k660_policy::Error::tag)?; let task = tasks.frame.ok_or("dma-task")?; let task_canary = unsafe { read_volatile(task as *const u64) }; let free_before = memory.allocator.remaining_pages(); if mode == 1 { // This separate fault-control boot captures a completed predecessor // ledger before retaining DMA backing. The later current DMA inventory // independently accounts the four pinned pages; no count is reset. os_k720_accept(&mut memory.allocator, tables)?; if unsafe { read_volatile( core::ptr::addr_of!(OS_K640_LOADER_ARGUMENTS) .cast::() .add(2), ) } != 0x640 { return Err("dma-predecessor-entry-witness"); } write_line( "TURKOS/OS/K800: CONTROL PREDECESSOR ENTRY-MMU-FRAME-CHECKS-COMPLETE BEFORE-DMA-PINS", ); write_line("TURKOS/OS/K640: PASS"); } unsafe { let output = core::ptr::addr_of_mut!(OS_K800_EVIDENCE).cast::(); for (index, value) in [ (0, base), (1, 0x7472_6976), (2, 2), (3, 1), (6, mode as u64), (21, free_before), (25, task_canary), (27, daif), (28, if mode == 1 { 0x640 } else { 0 }), (29, descriptor), ] { write_volatile(output.add(index), value) } }; // Preserve physical predecessor checkpoints in the low free pool. These // four interior high pages are still acquired by the same typed authority; // the top endpoint remains available for the later full-page K720 proof. let dma_region = memory .allocator .os_authority() .free_regions() .iter() .rev() .find(|region| region.end - region.start >= 12 * 4096) .ok_or("dma-interior-pool")?; let dma_start = dma_region.end - 8 * 4096; let backend = OsK800Backend { authority: memory.allocator.os_authority_mut(), leases: core::array::from_fn(|_| None), addresses: core::array::from_fn(|index| dma_start + index as u64 * 4096), }; let mut registry = os_k740_metadata::Registry::<_, 4>::new(800, backend).map_err(|_| "dma-registry")?; for queue in 0..2 { let available_before = registry.snapshot().backend_free_pages; let mut handles = [None; 4]; for slot in &mut handles { *slot = Some( registry .allocate_zeroed(800) .map_err(|_| "dma-buffer-acquire")?, ) } let frames = handles.map(|handle| handle.expect("DMA handle").address()); let mut safe = SafeRegistry::<1, 6>::new(); let safe_handle = if k1420_mode { let identity = DeviceIdentity { transport: Transport::Mmio, vendor_id: unsafe { os_k800_read(base, 12) }, device_id: 1, class: DeviceClass::Network }; let key = DeviceKey { transport: Transport::Mmio, location: base }; let ranges = [ ResourceRange {kind: ResourceKind::Mmio, start: base, length: 0x200}, ResourceRange {kind: ResourceKind::Irq, start: u64::from(net_irq), length: 1}, ResourceRange {kind: ResourceKind::Dma, start: frames[0], length: 4096}, ResourceRange {kind: ResourceKind::Dma, start: frames[1], length: 4096}, ResourceRange {kind: ResourceKind::Dma, start: frames[2], length: 4096}, ResourceRange {kind: ResourceKind::Dma, start: frames[3], length: 4096}, ]; Some(safe.attach(key, identity, &ranges, 1) .map_err(|_| "k1420-net-attach")?) } else { None }; let binding = Binding::new(base, queue as u16, queue as u64 + 1).map_err(Error::tag)?; let workspace = unsafe { &mut *core::ptr::addr_of_mut!(OS_K800_WORKSPACE) }; let mut scope = os_k800_dma::Dma::initialize(workspace, &mut registry, 800 + queue as u64, binding) .map_err(Error::tag)?; let mut tickets: [Option; 4] = core::array::from_fn(|_| None); for index in 0..4 { tickets[index] = Some( scope .pin(handles[index].expect("DMA handle"), 800) .map_err(Error::tag)?, ) } let buffer = tickets[3].as_ref().ok_or("dma-buffer-ticket")?; let direction = if queue == 0 { Direction::DeviceWrites } else { Direction::DeviceReads }; let length = if queue == 0 { 2048 } else { 72 }; let span = scope .descriptor(buffer, binding, 0, length, direction) .map_err(Error::tag)?; let before = scope.audit().map_err(Error::tag)?; if scope.descriptor( buffer, Binding { queue: (queue ^ 1) as u16, ..binding }, 0, 16, direction, ) != Err(Error::Foreign) || scope.descriptor(buffer, binding, 4095, 2, direction) != Err(Error::Bounds) || scope.descriptor(buffer, binding, u64::MAX, 2, direction) != Err(Error::Overflow) || scope.audit().map_err(Error::tag)? != before { return Err("dma-rejected-descriptor-state"); } for handle in handles.iter().flatten() { if scope.ordinary_release(*handle, 800) != Err(os_k740_metadata::Error::Pinned) { return Err("dma-ordinary-release-accepted"); }; os_k800_increment(queue, 6) } let expected = os_k800_packet(boot_id, queue); unsafe { write_volatile( frames[0] as *mut turkos_kernel::virtio_ring::Descriptor, turkos_kernel::virtio_ring::Descriptor { addr_le: span.address.to_le(), len_le: span.length.to_le(), flags_le: (if queue == 0 { 2u16 } else { 0u16 }).to_le(), next_le: 0, }, ); write_volatile((frames[1] + 4) as *mut u16, 0); write_volatile((frames[3] + 2048) as *mut u8, 0xa5); if queue == 1 { for (index, byte) in expected.iter().enumerate() { write_volatile((frames[3] + 12 + index as u64) as *mut u8, *byte) } } } let references = [ tickets[0].as_ref().unwrap(), tickets[1].as_ref().unwrap(), tickets[2].as_ref().unwrap(), tickets[3].as_ref().unwrap(), ]; scope.activate(&references).map_err(Error::tag)?; for (offset, value) in [ (0, base), (1, queue as u64), (2, binding.epoch), (3, 4), (4, available_before), (5, available_before - 4), (12, frames[0]), (13, frames[1]), (14, frames[2]), (15, frames[3]), (30, mode as u64), ] { os_k800_word(queue, offset, value) } os_k800_configure(base, queue, frames)?; let mut io = OsK800Device { binding, available: frames[1], used: frames[2], withhold_reset: mode == 1 && queue == 1, }; scope .publish(&references, 0, span, &mut io) .map_err(Error::tag)?; if let Some(handle) = safe_handle { safe.post(handle, 1).map_err(|_| "k1420-net-post")?; let before = safe.snapshot(); if safe.finish_detach(handle) != Err(DetachError::ResetRequired) || safe.snapshot() != before { return Err("k1420-net-early-free"); } } write_str("TURKOS/OS/K790: QUEUE=0x"); write_hex(queue); write_line(if queue == 0 { " RX WAITING-HOST-FRAME" } else { " TX WAITING-USED" }); let frequency: u64; unsafe { asm!("mrs {value}, CNTFRQ_EL0",value=out(reg)frequency,options(nomem,nostack,preserves_flags)) }; let deadline = os_k800_clock() .checked_add(frequency.checked_mul(5).ok_or("dma-time-budget")?) .ok_or("dma-time-overflow")?; let completion = loop { match scope.consume(&mut io) { Ok(value) => break value, Err(Error::WouldBlock) => { if os_k800_clock() >= deadline { return Err("dma-completion-timeout"); }; core::hint::spin_loop() } Err(error) => return Err(error.tag()), } }; if completion.id != 0 || completion.length != if queue == 0 { 72 } else { 0 } { return Err("dma-completion-shape"); } for (index, byte) in expected.iter().enumerate() { if unsafe { read_volatile((frames[3] + 12 + index as u64) as *const u8) } != *byte { return Err("dma-packet-mismatch"); } } if unsafe { read_volatile((frames[3] + 2048) as *const u8) } != 0xa5 { return Err("dma-buffer-guard"); } for handle in handles.iter().flatten() { if scope.ordinary_release(*handle, 800) != Err(os_k740_metadata::Error::Pinned) { return Err("dma-completed-release-accepted"); }; os_k800_increment(queue, 6) } os_k800_word(queue, 31, 1); os_k800_capture(&scope, queue as u64 + 1)?; if let Some(handle) = safe_handle { safe.stop_admissions(handle).map_err(|_| "k1420-net-stop")?; } match scope.reset(&mut io) { Ok(witness) => { if witness.status != 0 || witness.queue_ready != 0 { return Err("dma-reset-witness"); } } Err(Error::ResetUnanswered) if mode == 1 && queue == 1 => { let snapshot = scope.audit().map_err(Error::tag)?; if snapshot.quarantined != 4 || snapshot.pins != 4 || snapshot.registry_pins != 4 { return Err("dma-quarantine-pin"); }; for handle in handles.iter().flatten() { if scope.ordinary_release(*handle, 800) != Err(os_k740_metadata::Error::Pinned) { return Err("dma-quarantine-release"); }; os_k800_increment(queue, 6) } os_k800_word(queue, 28, snapshot.pins as u64); os_k800_word(queue, 29, snapshot.quarantined as u64); scope.observe_registry(|registry| { registry.with_backend_observation(|backend| unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K800_AUTHORITY), os_k740_authority_inventory(backend.authority).words, ) }) }); unsafe { let output = core::ptr::addr_of_mut!(OS_K800_EVIDENCE).cast::(); write_volatile(output.add(11), 4); write_volatile(output.add(26), read_volatile(task as *const u64)); }; write_line( "TURKOS/OS/K795: REAL-STATUS=15 QUEUE-READY=1 UNANSWERED-RESET QUARANTINED PIN=4 BACKING=RETAINED ORDINARY-FREE=REFUSED PASS", ); park() } Err(error) => return Err(error.tag()), } if let Some(handle) = safe_handle { safe.mark_reset(handle).map_err(|_| "k1420-net-reset-model")?; if safe.drain_after_reset(handle) != Ok(1) { return Err("k1420-net-drain"); } k1420_device_safe_detach::mask_spi(boot.interrupts.gic_distributor, net_irq)?; safe.mark_irq_detached(handle).map_err(|_| "k1420-net-irq-model")?; safe.finish_detach(handle).map_err(|_| "k1420-net-finish")?; if safe.binding_count() != 0 || safe.resource_count() != 0 || scope.audit().map_err(Error::tag)?.pins != 4 { return Err("k1420-net-owner-before-physical-free"); } } for ticket in &mut tickets { scope .retire(ticket.take().ok_or("dma-pin-token")?) .map_err(|(error, _)| error.tag())? } for handle in handles.iter().flatten() { scope .ordinary_release(*handle, 800) .map_err(|_| "dma-release-after-reset")?; os_k800_increment(queue, 26) } let snapshot = scope.audit().map_err(Error::tag)?; os_k800_word(queue, 28, snapshot.registry_pins as u64); let after = scope.observe_registry(|registry| registry.snapshot().backend_free_pages); os_k800_word(queue, 27, after); if after != available_before || snapshot.pins != 0 || snapshot.registry_pins != 0 { return Err("dma-reset-reclaim-account"); }; scope.close().map_err(|(error, _)| error.tag())?; if k1420_mode { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1420_NET_PROOFS).cast::().add(queue), 1); } write_str("TURKOS/OS/K1420-FULL: QUEUE=0x"); write_hex(queue); write_str(" NET=0x"); write_hex(base as usize); write_str(" IRQ=0x"); write_hex(net_irq as usize); write_line(" RESET=0 QUEUE-READY=0 DRAIN=1 IRQ-MASKED=1 UNPINNED=4 RELEASED=4 CLAIMS=0 FREE-EQUAL=1"); } write_str("TURKOS/OS/K790: REAL-COMPLETION QUEUE=0x"); write_hex(queue); write_line( " USED=1 ID=0 PACKET=60 RESET-CPU-READBACK=0 QUEUE-READY=0 UNPINNED=4 RELEASED=4 PASS", ); } if registry.snapshot().pins != 0 || registry.snapshot().live_pages != 0 || registry.snapshot().backend_free_pages != free_before { return Err("dma-final-conservation"); }; registry.with_backend_observation(|backend| unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K800_AUTHORITY), os_k740_authority_inventory(backend.authority).words, ) }); if registry.owned_records().next().is_some() { return Err("dma-final-records"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K800_REGISTRY), [0; 24]) }; drop(registry); if memory.allocator.remaining_pages() != free_before || unsafe { read_volatile(task as *const u64) } != task_canary { return Err("dma-task-canary"); }; unsafe { let output = core::ptr::addr_of_mut!(OS_K800_EVIDENCE).cast::(); write_volatile(output.add(24), free_before); write_volatile(output.add(26), read_volatile(task as *const u64)); write_volatile(output.add(8), 0); write_volatile(output.add(9), 0); write_volatile(output.add(10), 0); }; write_line( "TURKOS/OS/K785: DEVICE-QUEUE-BOUND REAL-REGISTRY-PINS WRONG-QUEUE BOUNDS OVERFLOW ORDINARY-FREE-REFUSED PASS", ); write_line("TURKOS/OS/K790: REAL-VIRTIO RX TX PUBLISH-DMB-OSHST CONSUME-DMB-OSHLD PASS"); write_line( "TURKOS/OS/K795: VOLATILE-CPU-STATUS QUEUE-READY RESET-READBACK BEFORE-UNPIN DSB-SY PASS", ); write_line( "TURKOS/OS/K800: LOCAL-GUEST PASS K700-FRESH-M1=PENDING PHYSICAL-NONCOHERENT-CACHE=PENDING", ); Ok(()) } #[repr(C, align(4096))] struct OsK800Stack { guard: [u8; 4096], usable: [u8; 65536], } #[unsafe(no_mangle)] static mut OS_K800_STACK: OsK800Stack = OsK800Stack { guard: [0; 4096], usable: [0; 65536], }; #[unsafe(no_mangle)] static mut OS_K800_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K800_ERROR: Option<&'static str> = None; const OS_K800_LOW_CANARY: u64 = 0x4b38_3030_4c4f_5721; const OS_K800_HIGH_CANARY: u64 = 0x4b38_3030_4849_4748; fn os_k800_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K800_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } fn os_k800_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { unsafe extern "C" { static __stack_start: u8; static __stack_top: u8; } let guard = core::ptr::addr_of!(OS_K800_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_low = core::ptr::addr_of!(__stack_start) as u64; let original_top = core::ptr::addr_of!(__stack_top) as u64; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("dma-original-stack-range"); } let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_low + 8) as *const u64) }; if original_canary != 0x4b36_3335_5354_414b || original_second != !0x4b36_3335_5354_414bu64 { return Err("dma-original-stack-canaries"); } unsafe { tables.unmap_metadata_stack_guard( guard, low, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64, ) } .map_err(|_| "dma-stack-guard")?; if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("dma-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K800_LOW_CANARY); write_volatile(high as *mut u64, OS_K800_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K800_LOW_CANARY), (9, OS_K800_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("dma-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("dma-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k800_stack_word(slot, value); } unsafe extern "C" { fn __os_k800_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k800_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K800_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K800_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K800_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_low + 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("dma-stack-restoration"); } os_k800_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k800_stack_word(17, unsafe { read_volatile((original_low + 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K800: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); Ok(()) } else { Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K800_ERROR)) } .unwrap_or("dma-stack-return")) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k800_accept_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k800_stack_word(4, sp); let result = os_k800_accept( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k800_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K800_ERROR), Some(reason)) }; 1 } } } global_asm!( r#" .section .text.os_k800_stack,"ax" .balign 4 .global __os_k800_on_stack __os_k800_on_stack: adrp x9, OS_K800_STACK_EVIDENCE add x9, x9, :lo12:OS_K800_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K800_STACK add x9, x9, :lo12:OS_K800_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K800_STACK_EVIDENCE add x9, x9, :lo12:OS_K800_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k800_accept_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K800_STACK_EVIDENCE add x9, x9, :lo12:OS_K800_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret "# ); #[unsafe(no_mangle)] static mut OS_K820_EVIDENCE:[u64;128]=[0;128]; #[unsafe(no_mangle)] static mut OS_K820_USER_STATE:[u64;16]=[0;16]; #[unsafe(no_mangle)] static mut OS_K820_RECORDS:[u64;192]=[0;192]; #[unsafe(no_mangle)] static mut OS_K820_REGISTRY_RECORDS:[u64;192]=[0;192]; #[unsafe(no_mangle)] static mut OS_K820_AUTHORITY_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K820_AUTHORITY_LIVE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K820_AUTHORITY_AFTER:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K820_CAPTURE_GATE:u8=0; fn os_k820_word(index:usize,value:u64){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K820_EVIDENCE).cast::().add(index),value)};} fn os_k820_authority(authority:&os_k720_frame::Allocator,target:*mut u64){ let free=authority.free_regions();let mut live=0;let mut reserved_count=0; unsafe{for i in 0..328{write_volatile(target.add(i),0);}write_volatile(target,free.len() as u64);write_volatile(target.add(3),authority.remaining_pages());} for(index,region)in free.iter().enumerate(){unsafe{write_volatile(target.add(4+index*2),region.start);write_volatile(target.add(5+index*2),region.end)};} for address in authority.live_addresses(){unsafe{write_volatile(target.add(132+live),address)};live+=1;} unsafe{write_volatile(target.add(1),live as u64)}; for(index,region)in authority.reserved_regions().enumerate(){unsafe{write_volatile(target.add(196+index*2),region.start);write_volatile(target.add(197+index*2),region.end)};reserved_count+=1;}unsafe{write_volatile(target.add(2),reserved_count)}; } fn os_k820_controls(allocator:&mut memory::FrameAllocator,tables:&mmu::Tables)->Result<(),&'static str>{ use os_k820_tree::{Error,Tree}; let geometry=os_k820_descriptor::Geometry::new(48,40).map_err(|_|"page-tables-geometry")?; let backend=OsK820Backend::new(allocator.os_authority_mut(),tables)?;let mut tree=Tree::<_,16>::new(backend.identity(),geometry,backend).map_err(|_|"page-tables-control-tree")?; let root=tree.new_root().map_err(|_|"page-tables-control-root")?;let page=tree.allocate_page().map_err(|_|"page-tables-control-page")?; let before=tree.with_backend_observation(|b|b.with_authority_observation(|a|a.remaining_pages())); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K820_ALLOC_BUDGET),1)}; let rw=os_k820_descriptor::Permissions{user:true,write:true,execute:false}; if tree.map(&root,0x8000_0000,&page,os_k820_descriptor::Memory::Normal,rw)!=Err(Error::NoMemory) ||unsafe{read_volatile(root.address() as *const u64)}!=0 ||tree.with_backend_observation(|b|b.with_authority_observation(|a|a.remaining_pages()))!=before ||tree.owned_records().count()!=2 {return Err("page-tables-second-table-enospc");} os_k820_word(16,1); let next=page.address()+4096; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K820_ALLOC_BUDGET),1);write_volatile(core::ptr::addr_of_mut!(OS_K820_FAIL_RELEASE),next)}; if tree.map(&root,0x8000_0000,&page,os_k820_descriptor::Memory::Normal,rw)!=Err(Error::Retained) ||unsafe{read_volatile(root.address() as *const u64)}!=0 ||tree.owned_records().filter(|r|r.quarantined&&r.address==next).count()!=1 ||tree.with_backend_observation(|b|b.with_authority_observation(|a|a.remaining_pages()))+1!=before ||tree.reap_quarantine()!=Err(Error::Retained){return Err("page-tables-retained-rollback");} tree.audit().map_err(|_|"page-tables-retained-audit")?;os_k820_word(17,1); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K820_ALLOC_BUDGET),u64::MAX);write_volatile(core::ptr::addr_of_mut!(OS_K820_FAIL_RELEASE),0)}; if tree.reap_quarantine()!=Ok(1){return Err("page-tables-quarantine-retry");} tree.release_page(page).map_err(|_|"page-tables-control-page-release")?;tree.release_root(root).map_err(|_|"page-tables-control-root-release")?; if tree.owned_records().count()!=0{return Err("page-tables-control-final-owner");}os_k820_word(18,1); write_line("TURKOS/OS/K805: SECOND-TABLE-ENOSPC NO-PUBLISHED-PREFIX FAILED-ROLLBACK-HANDLE-RETAINED RETRY-RESTORED PASS");Ok(()) } fn os_k820_code(page:&os_k820_tree::Page,begin:*const u8,end:*const u8)->Result<(),&'static str>{ let size=end as usize-begin as usize;if size==0||size>4096{return Err("page-tables-code-template");} for offset in 0..size {unsafe{write_volatile((page.address() as *mut u8).add(offset),read_volatile(begin.add(offset)))};} let ctr:u64;unsafe{asm!("mrs {ctr}, CTR_EL0",ctr=out(reg)ctr,options(nomem,nostack,preserves_flags))}; let dline=4u64<<((ctr>>16)&15);let iline=4u64<<(ctr&15); let mut address=page.address();while addressResult<(),&'static str>{ let state=core::ptr::addr_of_mut!(OS_K820_USER_STATE).cast::(); unsafe{write_volatile(state,mode);write_volatile(state.add(2),kernel);write_volatile(state.add(3),tcr);write_volatile(state.add(4),0);} unsafe extern "C"{fn __os_k820_enter_user(root:u64,entry:u64);} unsafe{__os_k820_enter_user(root.address()|(0x20u64<<48),entry)}; if unsafe{read_volatile(state.add(4))}!=mode{return Err("page-tables-missing-permission-fault");}Ok(()) } fn os_k820_expected_user_fault(slot:usize,_esr:usize,frame:*const ExceptionFrame)->bool{ let state=core::ptr::addr_of_mut!(OS_K820_USER_STATE).cast::();let mode=unsafe{read_volatile(state)};if mode==0{return false;} if frame.is_null()||slot!=8||!unsafe{&*frame}.canary_valid(){park();} let(esr,far,elr,ttbr,tcr,mair,sp_el0):(u64,u64,u64,u64,u64,u64,u64); unsafe{asm!("mrs {esr}, ESR_EL1","mrs {far}, FAR_EL1","mrs {elr}, ELR_EL1","mrs {ttbr}, TTBR0_EL1","mrs {tcr}, TCR_EL1","mrs {mair}, MAIR_EL1","mrs {sp_el0}, SP_EL0",esr=out(reg)esr,far=out(reg)far,elr=out(reg)elr,ttbr=out(reg)ttbr,tcr=out(reg)tcr,mair=out(reg)mair,sp_el0=out(reg)sp_el0,options(nomem,nostack,preserves_flags))}; let class=esr>>26&63;if esr&63!=15||(mode==1&&(class!=0x24||esr&(1<<6)==0||far!=0x8000_0000||elr!=0x8000_0004))||(mode==2&&(class!=0x20||far!=0x8000_1000||elr!=0x8000_1000)){park();} let at=if mode==1{32}else{48};let saved=unsafe{&*frame}; for(i,value)in [esr,far,elr,ttbr,tcr,mair,slot as u64,1,frame as u64,sp_el0,saved.registers[0] as u64,mode].into_iter().enumerate(){os_k820_word(at+i,value);} unsafe{write_volatile(state,0);write_volatile(state.add(4),mode)}; unsafe extern "C"{fn __os_k820_resume_kernel()->!;} unsafe{__os_k820_resume_kernel()} } fn os_k820_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,_boot:&BootData)->Result<(),&'static str>{ use os_k820_tree::{Tree,Error};use os_k820_descriptor::{Geometry,Memory,Permissions}; let allocator=&mut runtime.memory_mut().map_err(|_|"page-tables-authority")?.allocator; let(original_tcr,original_ttbr,mair):(u64,u64,u64);unsafe{asm!("mrs {tcr}, TCR_EL1","mrs {root}, TTBR0_EL1","mrs {mair}, MAIR_EL1",tcr=out(reg)original_tcr,root=out(reg)original_ttbr,mair=out(reg)mair,options(nomem,nostack,preserves_flags))}; if original_ttbr&OS_K820_PA_MASK!=tables.root||original_tcr&63!=25||mair&0xff!=0xff{return Err("page-tables-original-control");} let free_before=allocator.remaining_pages();os_k820_authority(allocator.os_authority(),core::ptr::addr_of_mut!(OS_K820_AUTHORITY_BEFORE).cast()); for(i,v)in [(0,0x4b3832305441424c),(1,tables.root),(2,original_tcr),(3,mair),(4,free_before),(22,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))})]{os_k820_word(i,v);} os_k820_controls(allocator,tables)?; let geometry=Geometry::new(48,40).map_err(|_|"page-tables-geometry")?;let backend=OsK820Backend::new(allocator.os_authority_mut(),tables)?;let identity=backend.identity();let mut tree=Tree::<_,32>::new(identity,geometry,backend).map_err(|_|"page-tables-tree")?; let root=tree.new_root().map_err(|_|"page-tables-root")?; for index in 0..512 {let raw=unsafe{read_volatile((tables.root as *const u64).add(index))};if raw!=0 {if raw&3!=3{return Err("page-tables-shared-kernel-entry");}tree.share_table(&root,(index as u64)<<30,1,raw&OS_K820_PA_MASK).map_err(|_|"page-tables-share-kernel")?;}} let code=tree.allocate_page().map_err(|_|"page-tables-code")?;let execute=tree.allocate_page().map_err(|_|"page-tables-execute-code")?;let data=tree.allocate_page().map_err(|_|"page-tables-data")?;let stack=tree.allocate_page().map_err(|_|"page-tables-stack")?; unsafe extern "C"{static __os_k820_write_start:u8;static __os_k820_write_end:u8;static __os_k820_execute_start:u8;static __os_k820_execute_end:u8;} os_k820_code(&code,core::ptr::addr_of!(__os_k820_write_start),core::ptr::addr_of!(__os_k820_write_end))?;os_k820_code(&execute,core::ptr::addr_of!(__os_k820_execute_start),core::ptr::addr_of!(__os_k820_execute_end))?; let rx=Permissions{user:true,write:false,execute:true};let rw=Permissions{user:true,write:true,execute:false};let ro=Permissions{user:true,write:false,execute:false}; for(va,page,permissions)in [(0x8000_0000,&code,rx),(0x8000_4000,&execute,rx),(0x8000_1000,&data,rw),(0x8000_3000,&stack,rw)]{tree.map(&root,va,page,Memory::Normal,permissions).map_err(|_|"page-tables-user-map")?;} let known=(7u64<<39)|(13<<30)|(29<<21)|(31<<12)|0xabc;tree.map(&root,known&!4095,&data,Memory::Normal,ro).map_err(|_|"page-tables-known-map")?; let walked=tree.walk(&root,known).map_err(|_|"page-tables-known-walk")?;if walked.physical!=data.address()+0xabc{return Err("page-tables-known-translation");} tree.audit().map_err(|_|"page-tables-owned-table-audit")?; for(i,v)in [(6,identity),(7,root.address()),(8,known),(9,data.address()+0xabc),(10,walked.physical),(11,code.address()),(12,execute.address()),(13,data.address()),(14,stack.address())]{os_k820_word(i,v);} tree.activate(&root,0x20).map_err(|_|"page-tables-root-activate")?; let(par,active_tcr,active_ttbr,daif):(u64,u64,u64,u64);unsafe{asm!("at s1e1r, {known}","isb","mrs {par}, PAR_EL1","mrs {tcr}, TCR_EL1","mrs {ttbr}, TTBR0_EL1","mrs {daif}, DAIF",known=in(reg)known,par=out(reg)par,tcr=out(reg)active_tcr,ttbr=out(reg)active_ttbr,daif=out(reg)daif,options(nostack,preserves_flags))}; if par&1!=0||(par&OS_K820_PA_MASK)+(known&4095)!=walked.physical||active_tcr&63!=16||active_ttbr&OS_K820_PA_MASK!=root.address()||daif&0x80==0{return Err("page-tables-hardware-translation");} for(i,v)in [(24,par),(25,active_tcr),(26,active_ttbr),(27,daif)]{os_k820_word(i,v);} let Err((root_error,root))=tree.release_root(root) else {return Err("page-tables-active-root-was-freed");}; if root_error!=Error::ActiveRoot{return Err("page-tables-active-root-error");}os_k820_word(15,1); tree.restore_shared_root(tables.root,Geometry::new(39,40).map_err(|_|"page-tables-original-geometry")?,0).map_err(|_|"page-tables-restore-shared-root")?; if cfg!(feature="os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? {os_k820_user(&root,1,0x8000_0000,tables.root,original_tcr)?;os_k820_user(&root,2,0x8000_4000,tables.root,original_tcr)?;write_line("TURKOS/OS/K810: REAL-EL0 RX-CODE-WRITE RW-DATA-EXECUTE PERMISSION-FAULTS KERNEL-RECOVERED PASS");} else {write_line("TURKOS/OS/K810: PRODUCT-CONTROL OWNED-TABLES NO-EL0-PROBE");} let mut count=0;for r in tree.owned_records(){let p=unsafe{core::ptr::addr_of_mut!(OS_K820_RECORDS).cast::().add(count*6)};unsafe{for(i,v)in [r.address,r.generation,r.table as u64,r.root as u64,r.quarantined as u64,r.mappings as u64].into_iter().enumerate(){write_volatile(p.add(i),v)}};count+=1;} os_k820_word(19,count as u64);tree.with_backend_observation(|b|b.with_authority_observation(|a|os_k820_authority(a,core::ptr::addr_of_mut!(OS_K820_AUTHORITY_LIVE).cast()))); tree.with_backend_observation(|b|{for(index,(r,central_generation))in b.observed_handles().enumerate(){let p=unsafe{core::ptr::addr_of_mut!(OS_K820_REGISTRY_RECORDS).cast::().add(index*6)};unsafe{for(i,v)in [r.address,r.owner,r.generation,r.pins as u64,r.quarantined as u64,central_generation].into_iter().enumerate(){write_volatile(p.add(i),v)}};}}); write_line("TURKOS/OS/K815: ACTUAL-FOUR-LEVEL OWNED-TABLE-WALK KNOWN-VA-TO-PA PASS");write_line("TURKOS/OS/K820: CAPTURE OWNED-TABLES"); while unsafe{read_volatile(core::ptr::addr_of!(OS_K820_CAPTURE_GATE))}==0 {unsafe{asm!("yield",options(nomem,nostack,preserves_flags))};} tree.detach_all(&root).map_err(|_|"page-tables-detach-tlbi")?;tree.release_root(root).map_err(|_|"page-tables-root-release")?; for page in [code,execute,data,stack]{tree.release_page(page).map_err(|_|"page-tables-data-release")?;} tree.audit().map_err(|_|"page-tables-final-audit")?;if tree.owned_records().count()!=0{return Err("page-tables-final-live-records");} tree.with_backend_observation(|b|b.with_authority_observation(|a|os_k820_authority(a,core::ptr::addr_of_mut!(OS_K820_AUTHORITY_AFTER).cast()))); let(after_tcr,after_ttbr):(u64,u64);unsafe{asm!("mrs {tcr}, TCR_EL1","mrs {root}, TTBR0_EL1",tcr=out(reg)after_tcr,root=out(reg)after_ttbr,options(nomem,nostack,preserves_flags))}; os_k820_word(20,after_ttbr);os_k820_word(21,after_tcr);os_k820_word(23,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}); let after=tree.with_backend_observation(|b|b.with_authority_observation(|a|a.remaining_pages()));os_k820_word(5,after); if after!=free_before||after_ttbr!=original_ttbr||after_tcr!=original_tcr||unsafe{&*core::ptr::addr_of!(OS_K820_AUTHORITY_BEFORE)}!=unsafe{&*core::ptr::addr_of!(OS_K820_AUTHORITY_AFTER)}{return Err("page-tables-authority-restoration");} write_line("TURKOS/OS/K820: DETACH DSB-TLBI-DSB-ISB BEFORE-RECLAIM ACTIVE-TTBR-FREE-REFUSED EXACT-AUTHORITY-RESTORED PASS");write_line("TURKOS/OS/K805: PASS");write_line("TURKOS/OS/K815: PASS");write_line("TURKOS/OS/K820: PASS");Ok(()) } global_asm!(r#" .section .text.os_k820_user,"ax" .global __os_k820_write_start __os_k820_write_start: movz x0, #0x8000, lsl #16 str xzr, [x0] svc #0x820 .global __os_k820_write_end __os_k820_write_end: .global __os_k820_execute_start __os_k820_execute_start: movz x0, #0x1000 movk x0, #0x8000, lsl #16 br x0 svc #0x821 .global __os_k820_execute_end __os_k820_execute_end: .global __os_k820_enter_user __os_k820_enter_user: stp x19,x20,[sp,#-16]! stp x21,x22,[sp,#-16]! stp x23,x24,[sp,#-16]! stp x25,x26,[sp,#-16]! stp x27,x28,[sp,#-16]! stp x29,x30,[sp,#-16]! stp d8,d9,[sp,#-16]! stp d10,d11,[sp,#-16]! stp d12,d13,[sp,#-16]! stp d14,d15,[sp,#-16]! adrp x9,OS_K820_USER_STATE add x9,x9,:lo12:OS_K820_USER_STATE mov x10,sp str x10,[x9,#8] mrs x10,TCR_EL1 bic x10,x10,#0x3f orr x10,x10,#0x10 dsb ishst msr TCR_EL1,x10 msr TTBR0_EL1,x0 tlbi vmalle1is dsb ish isb movz x0,#0x4000 movk x0,#0x8000,lsl #16 msr SP_EL0,x0 msr ELR_EL1,x1 mov x0,#0x3c0 msr SPSR_EL1,x0 isb eret .global __os_k820_resume_kernel __os_k820_resume_kernel: adrp x9,OS_K820_USER_STATE add x9,x9,:lo12:OS_K820_USER_STATE ldr x10,[x9,#24] msr TCR_EL1,x10 ldr x10,[x9,#16] msr TTBR0_EL1,x10 dsb ish tlbi vmalle1is dsb ish isb ldr x10,[x9,#8] mov sp,x10 ldp d14,d15,[sp],#16 ldp d12,d13,[sp],#16 ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 ldp x27,x28,[sp],#16 ldp x25,x26,[sp],#16 ldp x23,x24,[sp],#16 ldp x21,x22,[sp],#16 ldp x19,x20,[sp],#16 ret "#); use turkos_kernel::runtime::page_tables::{descriptor as os_k820_descriptor,tree as os_k820_tree}; const OS_K820_OWNER:u64=0x820; const OS_K820_PA_MASK:u64=0x0000_ffff_ffff_f000; #[unsafe(no_mangle)] static mut OS_K820_ALLOC_BUDGET:u64=u64::MAX; #[unsafe(no_mangle)] static mut OS_K820_FAIL_RELEASE:u64=0; #[unsafe(no_mangle)] static mut OS_K820_EVENTS:[u64;4096]=[0;4096]; #[unsafe(no_mangle)] static mut OS_K820_EVENT_COUNT:u64=0; fn os_k820_event(kind:u64,address:u64,value:u64){ let n=unsafe{read_volatile(core::ptr::addr_of!(OS_K820_EVENT_COUNT))}; if n>=1365 {park();} unsafe{let p=core::ptr::addr_of_mut!(OS_K820_EVENTS).cast::().add(n as usize*3);write_volatile(p,kind);write_volatile(p.add(1),address);write_volatile(p.add(2),value);write_volatile(core::ptr::addr_of_mut!(OS_K820_EVENT_COUNT),n+1);} } struct OsK820Frames<'a>{authority:&'a mut os_k720_frame::Allocator,leases:[Option;32],addresses:[u64;32]} impl OsK820Frames<'_>{ fn validate(&self,address:u64)->Result<(),os_k740_metadata::Error>{ let lease=self.leases.iter().flatten().find(|lease|lease.address()==address).ok_or(os_k740_metadata::Error::BadAddress)?; self.authority.validate_lease(lease).map_err(|_|os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK820Frames<'_>{ fn allocate(&mut self)->Result{ let budget=unsafe{read_volatile(core::ptr::addr_of!(OS_K820_ALLOC_BUDGET))}; if budget==0{return Err(os_k740_metadata::Error::NoMemory);} if budget!=u64::MAX {unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K820_ALLOC_BUDGET),budget-1)};} let slot=self.leases.iter().position(Option::is_none).ok_or(os_k740_metadata::Error::Capacity)?; let lease=self.authority.allocate_at(self.addresses[slot]).map_err(|_|os_k740_metadata::Error::NoMemory)?; let address=lease.address();self.leases[slot]=Some(lease);os_k820_event(1,address,0);Ok(address) } fn release(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{ self.validate(address)?; if unsafe{read_volatile(core::ptr::addr_of!(OS_K820_FAIL_RELEASE))}==address {os_k820_event(9,address,0);return Err(os_k740_metadata::Error::Backend);} let slot=self.leases.iter().position(|lease|lease.as_ref().is_some_and(|lease|lease.address()==address)).ok_or(os_k740_metadata::Error::BadAddress)?; for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};} unsafe{asm!("dsb ish",options(nostack,preserves_flags))}; self.authority.release(self.leases[slot].as_ref().ok_or(os_k740_metadata::Error::Backend)?).map_err(|_|os_k740_metadata::Error::Backend)?; self.leases[slot]=None;os_k820_event(8,address,0);Ok(()) } fn zero_page(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{self.validate(address)?;for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};}Ok(())} fn read_byte(&self,address:u64,offset:usize)->Result{if offset>=4096{return Err(os_k740_metadata::Error::BadAddress);}self.validate(address)?;Ok(unsafe{read_volatile((address as *const u8).add(offset))})} fn available_pages(&self)->u64{self.authority.remaining_pages()} } pub struct OsK820Backend<'a>{identity:u64,registry:os_k740_metadata::Registry,32>,tables:&'a mmu::Tables} impl<'a> OsK820Backend<'a>{ pub fn new(authority:&'a mut os_k720_frame::Allocator,tables:&'a mmu::Tables)->Result{ let region=authority.free_regions().iter().rev().find(|r|r.end-r.start>=40*4096).ok_or("page-tables-interior-pool")?; let base=region.end-36*4096; let addresses=core::array::from_fn(|i|base+i as u64*4096); let frames=OsK820Frames{authority,leases:core::array::from_fn(|_|None),addresses}; unsafe extern "C"{fn __os_k760_workspace_identity()->u64;} let identity=unsafe{__os_k760_workspace_identity()};if identity==0{return Err("page-tables-registry-identity");} let registry=os_k740_metadata::Registry::new(identity,frames).map_err(|_|"page-tables-registry")?; Ok(Self{identity,registry,tables}) } pub fn identity(&self)->u64{self.identity} pub fn with_authority_observation(&self,callback:impl FnOnce(&os_k720_frame::Allocator)->R)->R{self.registry.with_backend_observation(|frames|callback(frames.authority))} pub fn registry_records(&self)->impl Iterator+'_ {self.registry.owned_records()} pub fn observed_handles(&self)->impl Iterator+'_ { self.registry.owned_records().map(|r|{let generation=self.registry.with_backend_observation(|frames|frames.leases.iter().flatten().find(|lease|lease.address()==r.address).map(|lease|lease.generation()).unwrap_or(0));(r,generation)}) } fn owned_kernel(&self,address:u64)->bool{self.tables.tagged_frames().any(|frame|frame.address==address)} fn shared_walk(&self,address:u64,level:u8,visited:&mut [u64;4],count:usize)->Result<(),os_k820_tree::Error>{ if level>3||count>=4||visited[..count].contains(&address)||!self.owned_kernel(address){return Err(os_k820_tree::Error::Shared);} visited[count]=address; for index in 0..512 { let raw=unsafe{read_volatile((address as *const u64).add(index))};if raw==0{continue;} match raw&3 { 3 if level<3=>self.shared_walk(raw&OS_K820_PA_MASK,level+1,visited,count+1)?, 1 if level<3=>{if raw&(1<<6)!=0{return Err(os_k820_tree::Error::Shared);}}, 3 if level==3=>{if raw&(1<<6)!=0{return Err(os_k820_tree::Error::Shared);}}, _=>return Err(os_k820_tree::Error::Shared), } } Ok(()) } fn kernel_level(&self,address:u64)->Result{ let mut pending=[(0u64,0u8);24];pending[0]=(self.tables.root,1);let mut len=1;let mut at=0; while at{ type Handle=os_k740_metadata::Handle; fn allocate_zeroed(&mut self)->Result{self.registry.allocate_zeroed(OS_K820_OWNER).map_err(|e|if e==os_k740_metadata::Error::NoMemory{os_k820_tree::Error::NoMemory}else{os_k820_tree::Error::Backend})} fn address(&self,handle:Self::Handle)->u64{handle.address()} fn validate(&self,handle:Self::Handle)->Result<(),os_k820_tree::Error>{self.registry.read_byte(handle,OS_K820_OWNER,4095).map(|_|()).map_err(|_|os_k820_tree::Error::Stale)} fn release(&mut self,handle:Self::Handle)->Result<(),os_k820_tree::Error>{self.registry.release(handle,OS_K820_OWNER).map_err(|_|os_k820_tree::Error::Backend)} fn entry(&self,handle:Self::Handle,index:usize)->Result{if index>=512{return Err(os_k820_tree::Error::Backend);}self.validate(handle)?;Ok(unsafe{read_volatile((handle.address() as *const u64).add(index))})} fn write_entry(&mut self,handle:Self::Handle,index:usize,value:u64)->Result<(),os_k820_tree::Error>{if index>=512{return Err(os_k820_tree::Error::Backend);}self.validate(handle)?;unsafe{write_volatile((handle.address() as *mut u64).add(index),value)};os_k820_event(2,handle.address()+index as u64*8,value);Ok(())} fn validate_shared_table(&self,address:u64)->Result<(),os_k820_tree::Error>{let level=self.kernel_level(address)?;self.shared_walk(address,level,&mut[0;4],0)} fn validate_shared_page(&self,_address:u64)->Result<(),os_k820_tree::Error>{Err(os_k820_tree::Error::Shared)} fn shared_entry(&self,address:u64,index:usize)->Result{if index>=512{return Err(os_k820_tree::Error::Shared);}self.validate_shared_table(address)?;Ok(unsafe{read_volatile((address as *const u64).add(index))})} fn active_root(&self)->u64{let value:u64;unsafe{asm!("mrs {value}, TTBR0_EL1",value=out(reg)value,options(nomem,nostack,preserves_flags))};value&OS_K820_PA_MASK} fn invalidate(&mut self,root:u64,va:Option){ unsafe{asm!("dsb ishst",options(nostack,preserves_flags))};os_k820_event(3,root,va.unwrap_or(u64::MAX)); unsafe{asm!("tlbi vmalle1is",options(nostack,preserves_flags))};os_k820_event(4,root,va.unwrap_or(u64::MAX)); unsafe{asm!("dsb ish",options(nostack,preserves_flags))};os_k820_event(5,root,0);unsafe{asm!("isb",options(nostack,preserves_flags))};os_k820_event(6,root,0); } fn select_root(&mut self,root:u64,geometry:os_k820_descriptor::Geometry,asid:u16){ let old:u64;unsafe{asm!("mrs {old}, TCR_EL1",old=out(reg)old,options(nomem,nostack,preserves_flags))};let tcr=(old&!63)|(64-geometry.virtual_bits() as u64); let ttbr=root|(asid as u64)<<48; unsafe{asm!("dsb ishst","msr TCR_EL1, {tcr}","msr TTBR0_EL1, {ttbr}",tcr=in(reg)tcr,ttbr=in(reg)ttbr,options(nostack,preserves_flags))}; if tcr!=old {unsafe{asm!("tlbi vmalle1is","dsb ish",options(nostack,preserves_flags))};} unsafe{asm!("isb",options(nostack,preserves_flags))};os_k820_event(7,root,asid as u64); } } #[repr(C, align(4096))] struct OsK820Stack { guard: [u8; 4096], usable: [u8; 65536], } #[unsafe(no_mangle)] static mut OS_K820_STACK: OsK820Stack = OsK820Stack { guard: [0; 4096], usable: [0; 65536], }; #[unsafe(no_mangle)] static mut OS_K820_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K820_ERROR: Option<&'static str> = None; const OS_K820_LOW_CANARY: u64 = 0x4b38_3230_4c4f_5721; const OS_K820_HIGH_CANARY: u64 = 0x4b38_3230_4849_4748; fn os_k820_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K820_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } fn os_k820_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { unsafe extern "C" { static __stack_start: u8; static __stack_top: u8; } let guard = core::ptr::addr_of!(OS_K820_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_low = core::ptr::addr_of!(__stack_start) as u64; let original_top = core::ptr::addr_of!(__stack_top) as u64; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("page-tables-original-stack-range"); } let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_low + 8) as *const u64) }; if original_canary != 0x4b36_3335_5354_414b || original_second != !0x4b36_3335_5354_414bu64 { return Err("page-tables-original-stack-canaries"); } unsafe { tables.unmap_metadata_stack_guard( guard, low, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64, ) } .map_err(|_| "page-tables-stack-guard")?; if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("page-tables-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K820_LOW_CANARY); write_volatile(high as *mut u64, OS_K820_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K820_LOW_CANARY), (9, OS_K820_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("page-tables-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("page-tables-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k820_stack_word(slot, value); } unsafe extern "C" { fn __os_k820_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k820_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K820_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K820_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K820_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_low + 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("page-tables-stack-restoration"); } os_k820_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k820_stack_word(17, unsafe { read_volatile((original_low + 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K820: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); Ok(()) } else { Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K820_ERROR)) } .unwrap_or("page-tables-stack-return")) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k820_accept_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k820_stack_word(4, sp); let result = os_k820_guarded_work( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k820_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K820_ERROR), Some(reason)) }; 1 } } } fn os_k820_guarded_work(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str>{ os_k820_accept(runtime,tables,boot)?; os_k840_accept(runtime,tables,boot) } global_asm!( r#" .section .text.os_k820_stack,"ax" .balign 4 .global __os_k820_on_stack __os_k820_on_stack: adrp x9, OS_K820_STACK_EVIDENCE add x9, x9, :lo12:OS_K820_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K820_STACK add x9, x9, :lo12:OS_K820_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K820_STACK_EVIDENCE add x9, x9, :lo12:OS_K820_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k820_accept_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K820_STACK_EVIDENCE add x9, x9, :lo12:OS_K820_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret "# ); use turkos_kernel::runtime::page_tables::{descriptor as os_k840_page_descriptor,tree as os_k840_page_tree}; const OS_K840_PAGE_OWNER:u64=0x840; const OS_K840_PAGE_PA_MASK:u64=0x0000_ffff_ffff_f000; #[unsafe(no_mangle)] static mut OS_K840_PAGE_ALLOC_BUDGET:u64=u64::MAX; #[unsafe(no_mangle)] static mut OS_K840_PAGE_FAIL_RELEASE:u64=0; #[unsafe(no_mangle)] static mut OS_K840_PAGE_EVENTS:[u64;4096]=[0;4096]; #[unsafe(no_mangle)] static mut OS_K840_PAGE_EVENT_COUNT:u64=0; fn os_k840_page_event(kind:u64,address:u64,value:u64){ let n=unsafe{read_volatile(core::ptr::addr_of!(OS_K840_PAGE_EVENT_COUNT))}; if n>=1365 {park();} unsafe{let p=core::ptr::addr_of_mut!(OS_K840_PAGE_EVENTS).cast::().add(n as usize*3);write_volatile(p,kind);write_volatile(p.add(1),address);write_volatile(p.add(2),value);write_volatile(core::ptr::addr_of_mut!(OS_K840_PAGE_EVENT_COUNT),n+1);} } struct OsK840PageFrames<'a>{authority:&'a mut os_k720_frame::Allocator,leases:[Option;32],addresses:[u64;32]} impl OsK840PageFrames<'_>{ fn validate(&self,address:u64)->Result<(),os_k740_metadata::Error>{ let lease=self.leases.iter().flatten().find(|lease|lease.address()==address).ok_or(os_k740_metadata::Error::BadAddress)?; self.authority.validate_lease(lease).map_err(|_|os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK840PageFrames<'_>{ fn allocate(&mut self)->Result{ let budget=unsafe{read_volatile(core::ptr::addr_of!(OS_K840_PAGE_ALLOC_BUDGET))}; if budget==0{return Err(os_k740_metadata::Error::NoMemory);} if budget!=u64::MAX {unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K840_PAGE_ALLOC_BUDGET),budget-1)};} let slot=self.leases.iter().position(Option::is_none).ok_or(os_k740_metadata::Error::Capacity)?; let lease=self.authority.allocate_at(self.addresses[slot]).map_err(|_|os_k740_metadata::Error::NoMemory)?; let address=lease.address();self.leases[slot]=Some(lease);os_k840_page_event(1,address,0);Ok(address) } fn release(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{ self.validate(address)?; if unsafe{read_volatile(core::ptr::addr_of!(OS_K840_PAGE_FAIL_RELEASE))}==address {os_k840_page_event(9,address,0);return Err(os_k740_metadata::Error::Backend);} let slot=self.leases.iter().position(|lease|lease.as_ref().is_some_and(|lease|lease.address()==address)).ok_or(os_k740_metadata::Error::BadAddress)?; for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};} unsafe{asm!("dsb ish",options(nostack,preserves_flags))}; self.authority.release(self.leases[slot].as_ref().ok_or(os_k740_metadata::Error::Backend)?).map_err(|_|os_k740_metadata::Error::Backend)?; self.leases[slot]=None;os_k840_page_event(8,address,0);Ok(()) } fn zero_page(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{self.validate(address)?;for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};}Ok(())} fn read_byte(&self,address:u64,offset:usize)->Result{if offset>=4096{return Err(os_k740_metadata::Error::BadAddress);}self.validate(address)?;Ok(unsafe{read_volatile((address as *const u8).add(offset))})} fn available_pages(&self)->u64{self.authority.remaining_pages()} } pub struct OsK840PageBackend<'a>{identity:u64,registry:os_k740_metadata::Registry,32>,tables:&'a mmu::Tables} impl<'a> OsK840PageBackend<'a>{ pub fn new(authority:&'a mut os_k720_frame::Allocator,tables:&'a mmu::Tables)->Result{ let region=authority.free_regions().iter().rev().find(|r|r.end-r.start>=40*4096).ok_or("page-tables-interior-pool")?; let base=region.end-36*4096; let addresses=core::array::from_fn(|i|base+i as u64*4096); let frames=OsK840PageFrames{authority,leases:core::array::from_fn(|_|None),addresses}; unsafe extern "C"{fn __os_k760_workspace_identity()->u64;} let identity=unsafe{__os_k760_workspace_identity()};if identity==0{return Err("page-tables-registry-identity");} let registry=os_k740_metadata::Registry::new(identity,frames).map_err(|_|"page-tables-registry")?; Ok(Self{identity,registry,tables}) } pub fn identity(&self)->u64{self.identity} pub fn with_authority_observation(&self,callback:impl FnOnce(&os_k720_frame::Allocator)->R)->R{self.registry.with_backend_observation(|frames|callback(frames.authority))} pub fn registry_records(&self)->impl Iterator+'_ {self.registry.owned_records()} pub fn observed_handles(&self)->impl Iterator+'_ { self.registry.owned_records().map(|r|{let generation=self.registry.with_backend_observation(|frames|frames.leases.iter().flatten().find(|lease|lease.address()==r.address).map(|lease|lease.generation()).unwrap_or(0));(r,generation)}) } fn owned_kernel(&self,address:u64)->bool{self.tables.tagged_frames().any(|frame|frame.address==address)} fn shared_walk(&self,address:u64,level:u8,visited:&mut [u64;4],count:usize)->Result<(),os_k840_page_tree::Error>{ if level>3||count>=4||visited[..count].contains(&address)||!self.owned_kernel(address){return Err(os_k840_page_tree::Error::Shared);} visited[count]=address; for index in 0..512 { let raw=unsafe{read_volatile((address as *const u64).add(index))};if raw==0{continue;} match raw&3 { 3 if level<3=>self.shared_walk(raw&OS_K840_PAGE_PA_MASK,level+1,visited,count+1)?, 1 if level<3=>{if raw&(1<<6)!=0{return Err(os_k840_page_tree::Error::Shared);}}, 3 if level==3=>{if raw&(1<<6)!=0{return Err(os_k840_page_tree::Error::Shared);}}, _=>return Err(os_k840_page_tree::Error::Shared), } } Ok(()) } fn kernel_level(&self,address:u64)->Result{ let mut pending=[(0u64,0u8);24];pending[0]=(self.tables.root,1);let mut len=1;let mut at=0; while at{ type Handle=os_k740_metadata::Handle; fn allocate_zeroed(&mut self)->Result{self.registry.allocate_zeroed(OS_K840_PAGE_OWNER).map_err(|e|if e==os_k740_metadata::Error::NoMemory{os_k840_page_tree::Error::NoMemory}else{os_k840_page_tree::Error::Backend})} fn address(&self,handle:Self::Handle)->u64{handle.address()} fn validate(&self,handle:Self::Handle)->Result<(),os_k840_page_tree::Error>{self.registry.read_byte(handle,OS_K840_PAGE_OWNER,4095).map(|_|()).map_err(|_|os_k840_page_tree::Error::Stale)} fn release(&mut self,handle:Self::Handle)->Result<(),os_k840_page_tree::Error>{self.registry.release(handle,OS_K840_PAGE_OWNER).map_err(|_|os_k840_page_tree::Error::Backend)} fn entry(&self,handle:Self::Handle,index:usize)->Result{if index>=512{return Err(os_k840_page_tree::Error::Backend);}self.validate(handle)?;Ok(unsafe{read_volatile((handle.address() as *const u64).add(index))})} fn write_entry(&mut self,handle:Self::Handle,index:usize,value:u64)->Result<(),os_k840_page_tree::Error>{if index>=512{return Err(os_k840_page_tree::Error::Backend);}self.validate(handle)?;unsafe{write_volatile((handle.address() as *mut u64).add(index),value)};os_k840_page_event(2,handle.address()+index as u64*8,value);Ok(())} fn validate_shared_table(&self,address:u64)->Result<(),os_k840_page_tree::Error>{let level=self.kernel_level(address)?;self.shared_walk(address,level,&mut[0;4],0)} fn validate_shared_page(&self,_address:u64)->Result<(),os_k840_page_tree::Error>{Err(os_k840_page_tree::Error::Shared)} fn shared_entry(&self,address:u64,index:usize)->Result{if index>=512{return Err(os_k840_page_tree::Error::Shared);}self.validate_shared_table(address)?;Ok(unsafe{read_volatile((address as *const u64).add(index))})} fn active_root(&self)->u64{let value:u64;unsafe{asm!("mrs {value}, TTBR0_EL1",value=out(reg)value,options(nomem,nostack,preserves_flags))};value&OS_K840_PAGE_PA_MASK} fn invalidate(&mut self,root:u64,va:Option){ unsafe{asm!("dsb ishst",options(nostack,preserves_flags))};os_k840_page_event(3,root,va.unwrap_or(u64::MAX)); unsafe{asm!("tlbi vmalle1is",options(nostack,preserves_flags))};os_k840_page_event(4,root,va.unwrap_or(u64::MAX)); unsafe{asm!("dsb ish",options(nostack,preserves_flags))};os_k840_page_event(5,root,0);unsafe{asm!("isb",options(nostack,preserves_flags))};os_k840_page_event(6,root,0); } fn select_root(&mut self,root:u64,geometry:os_k840_page_descriptor::Geometry,asid:u16){ let old:u64;unsafe{asm!("mrs {old}, TCR_EL1",old=out(reg)old,options(nomem,nostack,preserves_flags))};let tcr=(old&!63)|(64-geometry.virtual_bits() as u64); let ttbr=root|(asid as u64)<<48; unsafe{asm!("dsb ishst","msr TCR_EL1, {tcr}","msr TTBR0_EL1, {ttbr}",tcr=in(reg)tcr,ttbr=in(reg)ttbr,options(nostack,preserves_flags))}; if tcr!=old {unsafe{asm!("tlbi vmalle1is","dsb ish",options(nostack,preserves_flags))};} unsafe{asm!("isb",options(nostack,preserves_flags))};os_k840_page_event(7,root,asid as u64); } } use turkos_kernel::runtime::address_spaces as os_k840_spaces; type OsK840Tree<'a> = os_k820_tree::Tree, 32>; type OsK840Spaces<'a> = os_k840_spaces::Spaces, 3, 2>; #[unsafe(no_mangle)] static mut OS_K840_EVIDENCE: [u64; 64] = [0; 64]; #[unsafe(no_mangle)] static mut OS_K840_DIAGNOSTIC_CHECKPOINT: [u64; 6] = [0; 6]; #[unsafe(no_mangle)] static mut OS_K840_IDS: [u64; 24] = [0; 24]; #[unsafe(no_mangle)] static mut OS_K840_LIVE_IDS: [u64; 18] = [0; 18]; #[unsafe(no_mangle)] static mut OS_K840_OWNED: [u64; 192] = [0; 192]; #[unsafe(no_mangle)] static mut OS_K840_REGISTRY: [u64; 192] = [0; 192]; #[unsafe(no_mangle)] static mut OS_K840_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K840_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K840_AUTHORITY_LIVE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K840_TLBI: [u64; 16] = [0; 16]; #[unsafe(no_mangle)] static mut OS_K840_GENERATION_CONTROL: [u64; 48] = [0; 48]; #[unsafe(no_mangle)] static mut OS_K840_GENERATION_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K840_GENERATION_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K840_CAPTURE_GATE: u8 = 0; static mut OS_K840_COMPARE: [u64; 328] = [0; 328]; static mut OS_K840_STORAGE: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); fn os_k840_word(index: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K840_EVIDENCE) .cast::() .add(index), value, ) }; } fn os_k840_tree_error(error: os_k820_tree::Error) -> os_k840_spaces::Error { match error { os_k820_tree::Error::Capacity | os_k820_tree::Error::NoMemory => { os_k840_spaces::Error::NoMemory } os_k820_tree::Error::ActiveRoot => os_k840_spaces::Error::Active, _ => os_k840_spaces::Error::Backing, } } fn os_k840_capture_inventory(authority: &os_k720_frame::Allocator, output: *mut u64) { let inventory = os_k740_authority_inventory(authority); for (index, value) in inventory.words.into_iter().enumerate() { unsafe { write_volatile(output.add(index), value) }; } } fn os_k840_inventory_equal(authority: &os_k720_frame::Allocator, expected: *const u64) -> bool { let actual = os_k740_authority_inventory(authority); actual .words .iter() .enumerate() .all(|(index, value)| *value == unsafe { read_volatile(expected.add(index)) }) } #[unsafe(no_mangle)] #[inline(never)] unsafe extern "C" fn __os_k840_targeted_asid(operand: u64, evidence: *mut u64) { unsafe { asm!("dsb ish", options(nostack, preserves_flags)); write_volatile(evidence.add(6), 1); asm!("tlbi aside1is, x0",in("x0")operand,options(nostack,preserves_flags)); write_volatile(evidence.add(7), 1); asm!("dsb ish", options(nostack, preserves_flags)); write_volatile(evidence.add(8), 1); asm!("isb", options(nostack, preserves_flags)); write_volatile(evidence.add(9), 1); } } fn os_k840_capture(stage: u64) -> Result<(), &'static str> { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K840_CAPTURE_GATE), 0) }; write_str("TURKOS/OS/K840: CAPTURE STAGE=0x"); write_hex(stage as usize); write_str(" GATE=0x"); write_hex(core::ptr::addr_of!(OS_K840_CAPTURE_GATE) as usize); write_line(" READY"); let (start, frequency): (u64, u64); unsafe { asm!("mrs {start}, CNTPCT_EL0","mrs {frequency}, CNTFRQ_EL0",start=out(reg)start,frequency=out(reg)frequency,options(nomem,nostack,preserves_flags)) }; let end = start .checked_add( frequency .checked_mul(90) .ok_or("address-space-capture-frequency")?, ) .ok_or("address-space-capture-deadline")?; while unsafe { read_volatile(core::ptr::addr_of!(OS_K840_CAPTURE_GATE)) } != 1 { let now: u64; unsafe { asm!("mrs {now}, CNTPCT_EL0",now=out(reg)now,options(nomem,nostack,preserves_flags)) }; if now >= end { return Err("address-space-capture-timeout"); } core::hint::spin_loop(); } Ok(()) } struct OsK840Root { root: Option, pages: [Option; 3], owner: u64, } impl OsK840Root { fn address(&self) -> u64 { self.root.as_ref().map_or(0, |root| root.address()) } fn data(&self) -> u64 { self.pages[1].as_ref().map_or(0, |page| page.address()) } } struct OsK840Backend<'a> { tree: OsK840Tree<'a>, kernel_root: u64, configuration: os_k840_spaces::Configuration, retired: Option, fail_first: bool, retain_first: bool, requests: u64, selections: u64, last_new_root: u64, last_new_data: u64, } impl OsK840Backend<'_> { fn authority(&self, callback: impl FnOnce(&os_k720_frame::Allocator) -> R) -> R { self.tree .with_backend_observation(|backend| backend.with_authority_observation(callback)) } fn capture(&self) -> Result<(), os_k840_spaces::Error> { self.tree.audit().map_err(os_k840_tree_error)?; let owned = core::ptr::addr_of_mut!(OS_K840_OWNED).cast::(); let registry = core::ptr::addr_of_mut!(OS_K840_REGISTRY).cast::(); unsafe { core::ptr::write_bytes(owned, 0, 192); core::ptr::write_bytes(registry, 0, 192) }; for (index, row) in self.tree.owned_records().enumerate() { if index >= 32 { return Err(os_k840_spaces::Error::Backing); } for (field, value) in [ row.address, row.generation, u64::from(row.table), u64::from(row.root), u64::from(row.quarantined), row.mappings as u64, ] .into_iter() .enumerate() { unsafe { write_volatile(owned.add(index * 6 + field), value) }; } } self.tree.with_backend_observation(|backend| { for (index, (row, central_generation)) in backend.observed_handles().enumerate() { for (field, value) in [ row.address, row.owner, row.generation, row.pins as u64, u64::from(row.quarantined), central_generation, ] .into_iter() .enumerate() { unsafe { write_volatile(registry.add(index * 6 + field), value) }; } } }); self.authority(|authority| { os_k840_capture_inventory( authority, core::ptr::addr_of_mut!(OS_K840_AUTHORITY_LIVE).cast(), ) }); Ok(()) } fn discard(&mut self, bundle: &mut OsK840Root) -> Result<(), os_k840_spaces::Error> { if let Some(root) = bundle.root.as_ref() { self.tree.detach_all(root).map_err(os_k840_tree_error)?; } for page in &mut bundle.pages { if let Some(token) = page.take() { if let Err((error, token)) = self.tree.release_page(token) { *page = Some(token); return Err(os_k840_tree_error(error)); } } } if let Some(root) = bundle.root.take() { if let Err((error, root)) = self.tree.release_root(root) { bundle.root = Some(root); return Err(os_k840_tree_error(error)); } } self.tree.audit().map_err(os_k840_tree_error) } fn drain_retired(&mut self) -> Result<(), os_k840_spaces::Error> { if let Some(mut bundle) = self.retired.take() { if let Err(error) = self.discard(&mut bundle) { self.retired = Some(bundle); return Err(error); } } Ok(()) } fn failed_root_control(&mut self) -> Result<(), os_k840_spaces::Error> { let mut pages: [Option; 32] = core::array::from_fn(|_| None); for page in &mut pages { *page = Some(self.tree.allocate_page().map_err(os_k840_tree_error)?); } os_k840_word(38, pages[0].as_ref().expect("full pool").address()); os_k840_word(39, pages[31].as_ref().expect("full pool").address() + 4096); self.capture()?; os_k840_word(33, 32); os_k840_capture(1).map_err(|_| os_k840_spaces::Error::Backing)?; if self.tree.new_root() != Err(os_k820_tree::Error::Capacity) { return Err(os_k840_spaces::Error::Backing); } for page in pages.into_iter().flatten() { self.tree .release_page(page) .map_err(|(error, _)| os_k840_tree_error(error))?; } self.tree.audit().map_err(os_k840_tree_error)?; if self.tree.owned_records().count() != 0 || !self.authority(|authority| { os_k840_inventory_equal( authority, core::ptr::addr_of!(OS_K840_AUTHORITY_BEFORE).cast(), ) }) { return Err(os_k840_spaces::Error::Backing); } Err(os_k840_spaces::Error::NoMemory) } fn user_code(&self, bundle: &OsK840Root) -> Result<(), os_k840_spaces::Error> { unsafe extern "C" { static __os_k840_user_start: u8; static __os_k840_user_end: u8; } let start = core::ptr::addr_of!(__os_k840_user_start) as usize; let end = core::ptr::addr_of!(__os_k840_user_end) as usize; let length = end .checked_sub(start) .filter(|length| *length > 0 && *length <= 4096) .ok_or(os_k840_spaces::Error::Backing)?; let code = bundle.pages[0] .as_ref() .ok_or(os_k840_spaces::Error::Backing)? .address(); for offset in 0..length { unsafe { write_volatile( (code as *mut u8).add(offset), read_volatile((start as *const u8).add(offset)), ) }; } let ctr: u64; unsafe { asm!("mrs {ctr}, CTR_EL0",ctr=out(reg)ctr,options(nomem,nostack,preserves_flags)) }; let dline = 4u64 << (ctr >> 16 & 15); let iline = 4u64 << (ctr & 15); if dline > 4096 || iline > 4096 { return Err(os_k840_spaces::Error::Backing); } let mut address = code; while address < code + 4096 { unsafe { asm!("dc cvau, {address}",address=in(reg)address,options(nostack,preserves_flags)) }; address += dline; } unsafe { asm!("dsb ish", options(nostack, preserves_flags)) }; address = code; while address < code + 4096 { unsafe { asm!("ic ivau, {address}",address=in(reg)address,options(nostack,preserves_flags)) }; address += iline; } unsafe { asm!("dsb ish", "isb", options(nostack, preserves_flags)) }; os_k840_word(60, length as u64); Ok(()) } } impl os_k840_spaces::Backend for OsK840Backend<'_> { type Root = OsK840Root; fn acquire_root(&mut self, owner: u64) -> Result { self.requests += 1; if self.fail_first { self.fail_first = false; return self .failed_root_control() .and_then(|()| Err(os_k840_spaces::Error::NoMemory)); } let root = self.tree.new_draft_root().map_err(os_k840_tree_error)?; let mut bundle = OsK840Root { root: Some(root), pages: core::array::from_fn(|_| None), owner, }; let built = (|| { for page in &mut bundle.pages { *page = Some(self.tree.allocate_page().map_err(os_k840_tree_error)?); } self.user_code(&bundle)?; let root = bundle.root.as_ref().ok_or(os_k840_spaces::Error::Backing)?; for index in 0..2 { let raw = unsafe { read_volatile((self.kernel_root as *const u64).add(index)) }; if raw & 3 != 3 { return Err(os_k840_spaces::Error::Backing); } self.tree .share_table(root, (index as u64) << 30, 1, raw & OS_K840_PAGE_PA_MASK) .map_err(os_k840_tree_error)?; } for (index, va, permissions) in [ ( 0, mmu::USER_CODE_VA, os_k820_descriptor::Permissions { user: true, write: false, execute: true, }, ), ( 1, mmu::USER_DATA_VA, os_k820_descriptor::Permissions { user: true, write: true, execute: false, }, ), ( 2, mmu::USER_STACK_VA, os_k820_descriptor::Permissions { user: true, write: true, execute: false, }, ), ] { self.tree .map( root, va, bundle.pages[index] .as_ref() .ok_or(os_k840_spaces::Error::Backing)?, os_k820_descriptor::Memory::Normal, permissions, ) .map_err(os_k840_tree_error)?; } self.tree.publish_root(root).map_err(os_k840_tree_error)?; self.tree.audit().map_err(os_k840_tree_error)?; Ok::<(), os_k840_spaces::Error>(()) })(); if let Err(error) = built { if self.discard(&mut bundle).is_err() { self.retired = Some(bundle); return Err(os_k840_spaces::Error::Retirement); } return Err(error); } self.last_new_root = bundle.address(); self.last_new_data = bundle.data(); Ok(bundle) } fn root_address(&self, root: &OsK840Root) -> u64 { root.address() } fn validate_root(&self, root: &OsK840Root) -> Result<(), os_k840_spaces::Error> { if root.owner == 0 || root.address() == 0 || !self .tree .owned_records() .any(|record| record.root && record.address == root.address()) { return Err(os_k840_spaces::Error::Backing); } self.tree.audit().map_err(os_k840_tree_error) } fn invalidate_recycled(&mut self, asid: u16, old_generation: u64, new_generation: u64) { let (ttbr, tcr) = os_k840_registers(); let Some(retired) = self.retired.as_ref() else { park(); }; if ttbr != self.kernel_root || old_generation != 1 || new_generation != 2 || asid != 1 { park(); } let operand = self .configuration .tlbi_operand(asid) .unwrap_or_else(|_| park()); let evidence = core::ptr::addr_of_mut!(OS_K840_TLBI).cast::(); unsafe { for (index, value) in [ asid as u64, old_generation, new_generation, operand, ttbr, tcr, 0, 0, 0, 0, read_volatile( core::ptr::addr_of!(OS_K840_USER_STATE) .cast::() .add(17), ), retired.address(), retired.data(), self.last_new_root, self.last_new_data, read_volatile(core::ptr::addr_of!(OS_K840_PAGE_EVENT_COUNT)), ] .into_iter() .enumerate() { write_volatile(evidence.add(index), value) } __os_k840_targeted_asid(operand, evidence); } os_k840_word(21, 1); os_k840_word(22, operand); } fn select(&mut self, root: &OsK840Root, asid: u16) -> Result<(), os_k840_spaces::Error> { self.validate_root(root)?; let ttbr = self.configuration.ttbr(root.address(), asid)?; let before_tcr = os_k840_registers().1; self.tree .activate( root.root.as_ref().ok_or(os_k840_spaces::Error::Backing)?, asid, ) .map_err(os_k840_tree_error)?; if os_k840_registers() != (ttbr, before_tcr) { return Err(os_k840_spaces::Error::Configuration); } self.selections += 1; Ok(()) } fn restore_kernel(&mut self) -> Result<(), os_k840_spaces::Error> { let geometry = self.tree.geometry(); self.tree .restore_shared_root(self.kernel_root, geometry, 0) .map_err(os_k840_tree_error)?; if os_k840_registers().0 != self.kernel_root { return Err(os_k840_spaces::Error::Configuration); } Ok(()) } fn retire_root( &mut self, mut root: OsK840Root, _asid: u16, _generation: u64, ) -> Result<(), (os_k840_spaces::Error, OsK840Root)> { if os_k840_registers().0 & OS_K840_PAGE_PA_MASK == root.address() { return Err((os_k840_spaces::Error::Active, root)); } if let Err(error) = self.validate_root(&root) { return Err((error, root)); } if self.retain_first { if self.retired.is_some() { return Err((os_k840_spaces::Error::Retirement, root)); } if let Err(error) = self .tree .quiesce(root.root.as_ref().expect("validated root")) { return Err((os_k840_tree_error(error), root)); } self.retain_first = false; self.retired = Some(root); return Ok(()); } self.discard(&mut root).map_err(|error| (error, root)) } } fn os_k840_id( spaces: &OsK840Spaces<'_>, handle: os_k840_spaces::Handle, owner: u64, task_slot: u64, index: usize, ) -> Result<(), &'static str> { let observed = spaces .observe(handle, owner) .map_err(|_| "address-space-id-observation")?; let data = spaces .with_root_observation(handle, owner, |_, root| root.data()) .map_err(|_| "address-space-data-observation")?; let output = core::ptr::addr_of_mut!(OS_K840_IDS).cast::(); for (field, value) in [ owner, observed.slot as u64, observed.generation, observed.root, observed.asid as u64, observed.asid_generation, task_slot, data, ] .into_iter() .enumerate() { unsafe { write_volatile(output.add(index * 8 + field), value) }; } Ok(()) } fn os_k840_live_ids(spaces: &OsK840Spaces<'_>) -> Result<(), &'static str> { spaces.audit().map_err(|_| "address-space-live-id-audit")?; let output = core::ptr::addr_of_mut!(OS_K840_LIVE_IDS).cast::(); unsafe { core::ptr::write_bytes(output, 0, 18) }; for (index, row) in spaces.owned_records().enumerate() { for (field, value) in [ row.owner, row.slot as u64, row.generation, row.root, row.asid as u64, row.asid_generation, ] .into_iter() .enumerate() { unsafe { write_volatile(output.add(index * 6 + field), value) }; } } Ok(()) } fn os_k840_run( spaces: &mut OsK840Spaces<'_>, handle: os_k840_spaces::Handle, id: turkos_kernel::task::TaskId, mode: u64, operation: u64, target: u64, seed: u64, expected: u64, ) -> Result<(), &'static str> { let observed = spaces .activate(handle, id.generation) .map_err(|_| "address-space-root-selection")?; let kernel = spaces.with_backend_observation(|backend| backend.kernel_root); os_k840_enter( mode, id.generation, id.slot as u64, spaces .configuration() .ttbr(observed.root, observed.asid) .map_err(|_| "address-space-ttbr-encoding")?, kernel, operation, target, seed, expected, ) } fn os_k840_accept( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { let diagnostic = os_k680_log()?; let diagnostic_before = [ diagnostic.boot().value(), diagnostic.next_sequence(), diagnostic.len() as u64, ]; unsafe { for (index, value) in diagnostic_before.into_iter().enumerate() { write_volatile( core::ptr::addr_of_mut!(OS_K840_DIAGNOSTIC_CHECKPOINT) .cast::() .add(index), value, ); } } if !cfg!(feature = "os-runtime-acceptance") && !os_k1100_product_probe_enabled(boot)? { unsafe { for (index, value) in diagnostic_before.into_iter().enumerate() { write_volatile( core::ptr::addr_of_mut!(OS_K840_DIAGNOSTIC_CHECKPOINT) .cast::() .add(index + 3), value, ); } } unsafe extern "C" { static __os_k840_user_start: u8; static __os_k840_user_end: u8; } let _ = unsafe { read_volatile(core::ptr::addr_of!(__os_k840_user_start)) }; let _ = unsafe { read_volatile(core::ptr::addr_of!(__os_k840_user_end).sub(1)) }; let _ = unsafe { read_volatile((__os_k840_targeted_asid as *const ()).cast::()) }; let _ = unsafe { read_volatile(core::ptr::addr_of!(OS_K840_LIVE_IDS).cast::()) }; // The product control exposes the identical capture ABI. Volatile // observations retain these zero BSS symbols without executing probes. for pointer in [ core::ptr::addr_of!(OS_K840_EVIDENCE).cast::(), core::ptr::addr_of!(OS_K840_IDS).cast(), core::ptr::addr_of!(OS_K840_OWNED).cast(), core::ptr::addr_of!(OS_K840_REGISTRY).cast(), core::ptr::addr_of!(OS_K840_AUTHORITY_BEFORE).cast(), core::ptr::addr_of!(OS_K840_AUTHORITY_AFTER).cast(), core::ptr::addr_of!(OS_K840_AUTHORITY_LIVE).cast(), core::ptr::addr_of!(OS_K840_TLBI).cast(), core::ptr::addr_of!(OS_K840_GENERATION_CONTROL).cast(), core::ptr::addr_of!(OS_K840_GENERATION_BEFORE).cast(), core::ptr::addr_of!(OS_K840_GENERATION_AFTER).cast(), core::ptr::addr_of!(OS_K840_CAPTURE_GATE).cast(), core::ptr::addr_of!(OS_K840_PAGE_EVENTS).cast(), core::ptr::addr_of!(OS_K840_PAGE_EVENT_COUNT).cast(), ] { let _ = unsafe { read_volatile(pointer) }; } write_line("TURKOS/OS/K840: PRODUCT-CONTROL ADDRESS-SPACES-PROBE-ABSENT"); return Ok(()); } use os_k840_spaces::Error; let (mmfr0, tcr, sp): (u64, u64, u64); unsafe { asm!("mrs {mmfr0}, ID_AA64MMFR0_EL1","mrs {tcr}, TCR_EL1","mov {sp}, sp",mmfr0=out(reg)mmfr0,tcr=out(reg)tcr,sp=out(reg)sp,options(nomem,nostack,preserves_flags)) }; let configuration = os_k840_spaces::Configuration::from_registers(mmfr0, tcr) .map_err(|_| "address-space-hardware-configuration")?; let irq = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; let (_, memory, _, tasks) = runtime.running().map_err(os_k660_policy::Error::tag)?; let a_id = tasks.id.ok_or("address-space-task-a")?; if tasks.scheduler.running().is_none() && tasks.scheduler.run_next() != Ok(Some(a_id)) { return Err("address-space-a-initial-selection"); } if tasks.scheduler.running() != Some(a_id) { return Err("address-space-a-not-selected"); } let old_events = unsafe { read_volatile(core::ptr::addr_of!(OS_K820_EVENT_COUNT)) }; for (index, value) in [ (0, 0x4b38343053504143), (1, mmfr0), (2, tcr), (3, configuration.hardware_bits() as u64), (4, configuration.active_bits() as u64), (5, 2), (6, 3), (7, irq), (11, memory.allocator.remaining_pages()), (23, sp), (54, tcr), (57, old_events), (61, boot.hardware.uart_base), (62, core::ptr::addr_of!(OS_K840_EVIDENCE) as u64), ] { os_k840_word(index, value) } os_k840_capture_inventory( memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K840_AUTHORITY_BEFORE).cast(), ); let page_backend = OsK840PageBackend::new(memory.allocator.os_authority_mut(), tables)?; let identity = page_backend.identity(); let geometry = os_k820_descriptor::Geometry::new( 64 - (tcr & 63) as u8, match tcr >> 32 & 7 { 0 => 32, 1 => 36, 2 => 40, 3 => 42, 4 => 44, 5 => 48, _ => return Err("address-space-physical-width"), }, ) .map_err(|_| "address-space-geometry")?; let tree = os_k820_tree::Tree::new(identity, geometry, page_backend) .map_err(|_| "address-space-tree")?; let backend = OsK840Backend { tree, kernel_root: tables.root, configuration, retired: None, fail_first: true, retain_first: true, requests: 0, selections: 0, last_new_root: 0, last_new_data: 0, }; let storage = core::ptr::addr_of_mut!(OS_K840_STORAGE).cast::>(); unsafe { storage.write( os_k840_spaces::Spaces::new(identity, configuration, backend) .map_err(|_| "address-space-registry")?, ) }; let spaces = unsafe { &mut *storage }; if spaces.create(a_id.generation) != Err(Error::NoMemory) || spaces.owned_records().count() != 0 || !spaces.with_backend_observation(|backend| { backend.authority(|authority| { os_k840_inventory_equal( authority, core::ptr::addr_of!(OS_K840_AUTHORITY_BEFORE).cast(), ) }) }) { return Err("address-space-atomic-failed-root"); } os_k840_word(13, 1); os_k840_word(14, 0); os_k840_word( 15, spaces.with_backend_observation(|backend| backend.requests), ); let a = spaces .create(a_id.generation) .map_err(|_| "address-space-create-a")?; let b_id = tasks .scheduler .spawn() .map_err(|_| "address-space-task-b")?; let b = spaces .create(b_id.generation) .map_err(|_| "address-space-create-b")?; if a.generation() != 1 || b.generation() != 1 || a_id == b_id { return Err("address-space-atomic-generation"); } os_k840_id(spaces, a, a_id.generation, a_id.slot as u64, 0)?; os_k840_id(spaces, b, b_id.generation, b_id.slot as u64, 1)?; spaces.with_backend_observation(|backend| { backend.authority(|authority| { os_k840_capture_inventory(authority, core::ptr::addr_of_mut!(OS_K840_COMPARE).cast()) }) }); spaces .activate(a, a_id.generation) .map_err(|_| "address-space-active-control")?; let stable = spaces .observe(a, a_id.generation) .map_err(|_| "address-space-active-observation")?; if spaces.release(a, a_id.generation) != Err(Error::Active) || spaces.observe(a, a_id.generation) != Ok(stable) || !spaces.with_backend_observation(|backend| { backend.authority(|authority| { os_k840_inventory_equal(authority, core::ptr::addr_of!(OS_K840_COMPARE).cast()) }) }) { return Err("address-space-active-release"); } spaces .restore_kernel() .map_err(|_| "address-space-kernel-restore")?; if spaces.activate(a, b_id.generation) != Err(Error::Foreign) || !spaces.with_backend_observation(|backend| { backend.authority(|authority| { os_k840_inventory_equal(authority, core::ptr::addr_of!(OS_K840_COMPARE).cast()) }) }) { return Err("address-space-foreign-owner"); } os_k840_word(29, 1); os_k840_run(spaces, a, a_id, 1, 1, mmu::USER_DATA_VA, 0xa1, 0xa1)?; if tasks.scheduler.yield_current(false) != Ok(Some(b_id)) { return Err("address-space-a-b-selection"); } tasks.id = Some(b_id); os_k840_run(spaces, b, b_id, 2, 1, mmu::USER_DATA_VA, 0xb2, 0xb2)?; if tasks.scheduler.yield_current(false) != Ok(Some(a_id)) { return Err("address-space-b-a-selection"); } tasks.id = Some(a_id); os_k840_run(spaces, a, a_id, 3, 0, mmu::USER_DATA_VA, 0, 0xa1)?; if tasks.scheduler.yield_current(false) != Ok(Some(b_id)) { return Err("address-space-a-b-return-selection"); } tasks.id = Some(b_id); os_k840_run(spaces, b, b_id, 4, 0, mmu::USER_DATA_VA, 0, 0xb2)?; os_k840_live_ids(spaces)?; spaces .with_backend_observation(|backend| backend.capture()) .map_err(|_| "address-space-a-b-inventory")?; os_k840_capture(2)?; write_line( "TURKOS/OS/K830: REAL-EL0 A-B-A DISTINCT-ROOTS DISTINCT-ASIDS FULL4096-ISOLATION PASS", ); let c_id = tasks .scheduler .spawn() .map_err(|_| "address-space-task-c")?; let requests = spaces.with_backend_observation(|backend| backend.requests); if spaces.create(c_id.generation) != Err(Error::AsidsFull) || spaces.with_backend_observation(|backend| backend.requests) != requests || !spaces.with_backend_observation(|backend| { backend.authority(|authority| { os_k840_inventory_equal(authority, core::ptr::addr_of!(OS_K840_COMPARE).cast()) }) }) { return Err("address-space-scarce-asid-before-root"); } os_k840_word(32, 1); if tasks.scheduler.yield_current(false) != Ok(Some(a_id)) || tasks.scheduler.exit_current(0) != Ok(Some(c_id)) || tasks.scheduler.reap(a_id) != Ok(0) { return Err("address-space-retire-task-a"); } tasks.id = Some(c_id); os_k840_word(51, 1); spaces .release(a, a_id.generation) .map_err(|_| "address-space-retain-inactive-root-a")?; let c = spaces .create(c_id.generation) .map_err(|_| "address-space-create-c")?; os_k840_id(spaces, c, c_id.generation, c_id.slot as u64, 2)?; if spaces.activate(a, a_id.generation) != Err(Error::Stale) { return Err("address-space-stale-id"); } os_k840_word(30, 1); os_k840_run(spaces, c, c_id, 5, 0, mmu::USER_DATA_VA, 0, 0)?; os_k840_live_ids(spaces)?; spaces .with_backend_observation(|backend| backend.capture()) .map_err(|_| "address-space-recycled-inventory")?; os_k840_capture(3)?; os_k840_run(spaces, c, c_id, 6, 1, mmu::USER_DATA_VA, 0xc3, 0xc3)?; os_k840_run(spaces, c, c_id, 7, 0, mmu::USER_DATA_VA, 0, 0xc3)?; let kernel_target = core::ptr::addr_of!(OS_K840_EVIDENCE) as u64; os_k840_run(spaces, c, c_id, 8, 2, kernel_target, 0, 0)?; os_k840_run(spaces, c, c_id, 9, 2, boot.hardware.uart_base, 0, 0)?; if tasks.scheduler.running() != Some(c_id) || tasks.scheduler.exit_current(-14) != Ok(Some(b_id)) || tasks.scheduler.reap(c_id) != Ok(-14) { return Err("address-space-fault-task-c-reap"); } tasks.id = Some(b_id); os_k840_word(53, 1); spaces .release(c, c_id.generation) .map_err(|_| "address-space-release-c")?; if spaces.release(c, c_id.generation) != Err(Error::Stale) { return Err("address-space-double-release"); } os_k840_word(31, 1); spaces .release(b, b_id.generation) .map_err(|_| "address-space-release-b")?; spaces .audit() .map_err(|_| "address-space-empty-registry-audit")?; let control = core::ptr::addr_of_mut!(OS_K840_GENERATION_CONTROL).cast::(); let (control_root, control_tcr) = os_k840_registers(); let control_requests = spaces.with_backend_observation(|backend| backend.requests); let control_events = unsafe { read_volatile(core::ptr::addr_of!(OS_K840_PAGE_EVENT_COUNT)) }; let tlbi_before = unsafe { read_volatile(core::ptr::addr_of!(OS_K840_TLBI)) }; spaces.with_backend_observation(|backend| { backend.authority(|authority| { os_k840_capture_inventory( authority, core::ptr::addr_of_mut!(OS_K840_GENERATION_BEFORE).cast(), ) }) }); let (retired_ids, retired_asids) = spaces .generation_exhaustion_fixture(b_id.generation) .map_err(|_| "address-space-generation-retirement-control")?; let (control_after_root, control_after_tcr) = os_k840_registers(); let control_after_requests = spaces.with_backend_observation(|backend| backend.requests); let control_after_events = unsafe { read_volatile(core::ptr::addr_of!(OS_K840_PAGE_EVENT_COUNT)) }; let tlbi_after = unsafe { read_volatile(core::ptr::addr_of!(OS_K840_TLBI)) }; spaces.with_backend_observation(|backend| { backend.authority(|authority| { os_k840_capture_inventory( authority, core::ptr::addr_of_mut!(OS_K840_GENERATION_AFTER).cast(), ) }) }); if (control_after_root, control_after_tcr) != (control_root, control_tcr) || control_after_requests != control_requests || control_after_events != control_events || tlbi_after != tlbi_before || !spaces.with_backend_observation(|backend| { backend.authority(|authority| { os_k840_inventory_equal( authority, core::ptr::addr_of!(OS_K840_GENERATION_BEFORE).cast(), ) }) }) { return Err("address-space-generation-preflight-state-change"); } for (index, value) in [ control_root, control_after_root, control_tcr, control_after_tcr, control_requests, control_after_requests, control_events, control_after_events, 1, 1, retired_ids as u64, retired_asids as u64, ] .into_iter() .enumerate() { unsafe { write_volatile(control.add(index), value) }; } for index in 0..16 { unsafe { write_volatile(control.add(12 + index), tlbi_before[index]); write_volatile(control.add(28 + index), tlbi_after[index]); } } os_k840_word(34, retired_ids as u64); os_k840_word(35, retired_asids as u64); os_k840_live_ids(spaces)?; let mut backend = unsafe { storage.read() } .into_backend() .map_err(|_| "address-space-backend-retained-id")?; backend .drain_retired() .map_err(|_| "address-space-drain-retained-root")?; backend .capture() .map_err(|_| "address-space-final-inventory")?; if backend.tree.owned_records().count() != 0 || !backend.authority(|authority| { os_k840_inventory_equal( authority, core::ptr::addr_of!(OS_K840_AUTHORITY_BEFORE).cast(), ) }) { return Err("address-space-exact-authority-leak"); } backend.authority(|authority| { os_k840_capture_inventory( authority, core::ptr::addr_of_mut!(OS_K840_AUTHORITY_AFTER).cast(), ) }); let final_free = backend.authority(|authority| authority.remaining_pages()); let requests = backend.requests; let selections = backend.selections; drop(backend); if tasks.scheduler.exit_current(0) != Ok(None) || tasks.scheduler.reap(b_id) != Ok(0) { return Err("address-space-retire-task-b"); } os_k840_word(52, 1); tasks.id = Some( tasks .scheduler .spawn() .map_err(|_| "address-space-replacement-task")?, ); let (final_root, final_tcr) = os_k840_registers(); let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack,preserves_flags)) }; let final_irq = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; let final_events = unsafe { read_volatile(core::ptr::addr_of!(OS_K820_EVENT_COUNT)) }; let diagnostic = os_k680_log()?; let diagnostic_after = [ diagnostic.boot().value(), diagnostic.next_sequence(), diagnostic.len() as u64, ]; if diagnostic_after != diagnostic_before { return Err("address-space-event-log-checkpoint-change"); } unsafe { for (index, value) in diagnostic_after.into_iter().enumerate() { write_volatile( core::ptr::addr_of_mut!(OS_K840_DIAGNOSTIC_CHECKPOINT) .cast::() .add(index + 3), value, ); } } if final_tcr != tcr || final_root != tables.root || final_irq != irq || final_events != old_events { return Err("address-space-predecessor-checkpoint-change"); } for (index, value) in [ (8, final_irq), (9, diagnostic_before[1]), (10, diagnostic_after[1]), (12, final_free), (16, a.generation()), (17, b.generation()), (18, c.generation()), (19, 1), (20, 2), (24, after_sp), (25, 0xa1), (26, 0xb2), (27, 0), (28, 0xc3), (36, requests), (37, selections), (48, a_id.slot as u64), (49, b_id.slot as u64), (50, c_id.slot as u64), (55, final_tcr), (56, final_root), (58, final_events), (59, unsafe { read_volatile(core::ptr::addr_of!(OS_K840_PAGE_EVENT_COUNT)) }), (63, 255), ] { os_k840_word(index, value) } write_line( "TURKOS/OS/K825: REAL-ROOT-ENOSPC ATOMIC-ID-ASID-UNREGISTERED EXACT-AUTHORITY-RESTORED PASS", ); write_line( "TURKOS/OS/K835: REAL-TARGETED-ASIDE1IS CHECKED-GENERATION OLD-ROOT-RETAINED NEW-PHYSICAL-DATA ZERO4096 PASS", ); write_line( "TURKOS/OS/K840: REAL-EL0 SHARED-KERNEL-AND-MMIO PERMISSION-FAULTS EXACT-AUTHORITY-RESTORED PASS", ); write_line( "TURKOS/OS/K840: EMPTY-HISTORY-GENERATION-EXHAUSTION CONTROL RETIRED-BEFORE-ACQUIRE PASS", ); for line in [ "TURKOS/OS/K825: PASS", "TURKOS/OS/K830: PASS", "TURKOS/OS/K835: PASS", "TURKOS/OS/K840: PASS", ] { write_line(line) } Ok(()) } // New EL0 entry/return path intentionally preserves the ASID-tagged TLB across // ordinary A-B-A switches. Root selection and targeted retirement belong to // the new address-space backend; neither entry nor return executes global TLBI. #[unsafe(no_mangle)] static mut OS_K840_USER_STATE: [u64; 24] = [0; 24]; #[unsafe(no_mangle)] static mut OS_K840_TRAPS: [u64; 144] = [0; 144]; fn os_k840_registers() -> (u64, u64) { let (ttbr, tcr): (u64, u64); unsafe { asm!("mrs {ttbr}, TTBR0_EL1", "mrs {tcr}, TCR_EL1",ttbr=out(reg)ttbr,tcr=out(reg)tcr,options(nomem,nostack,preserves_flags)) }; (ttbr, tcr) } fn os_k840_enter( mode: u64, owner: u64, task_slot: u64, ttbr: u64, kernel_root: u64, operation: u64, target: u64, seed: u64, expected: u64, ) -> Result<(), &'static str> { let (actual, tcr) = os_k840_registers(); if actual != ttbr || actual >> 48 == 0 || mode == 0 || mode > 9 { return Err("address-space-entry-ttbr"); } let state = core::ptr::addr_of_mut!(OS_K840_USER_STATE).cast::(); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags)) }; unsafe { for (index, value) in [ (0, kernel_root), (2, ttbr), (3, owner), (4, mode), (5, 0), (6, actual), (7, tcr), (14, expected), (15, operation), (16, target), (18, task_slot), (19, sp), (21, read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))), ] { write_volatile(state.add(index), value) } } unsafe extern "C" { fn __os_k840_enter_user(operation: u64, target: u64, seed: u64); } unsafe { __os_k840_enter_user(operation, target, seed) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack,preserves_flags)) }; unsafe { write_volatile(state.add(20), after_sp) }; if unsafe { read_volatile(state.add(5)) } != 1 || sp != after_sp || os_k840_registers() != (kernel_root, tcr) || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) } != unsafe { read_volatile(state.add(21)) } { return Err("address-space-entry-return"); } Ok(()) } fn os_k840_expected_user_trap( slot: usize, esr: usize, elr: usize, frame: *const ExceptionFrame, ) -> bool { let state = core::ptr::addr_of_mut!(OS_K840_USER_STATE).cast::(); let mode = unsafe { read_volatile(state.add(4)) }; if mode == 0 { return false; } if frame.is_null() || slot != 8 { park(); } let saved = unsafe { &*frame }; if !saved.canary_valid() { park(); } let (ttbr, tcr) = os_k840_registers(); if ttbr != unsafe { read_volatile(state.add(2)) } || tcr != unsafe { read_volatile(state.add(7)) } || ttbr >> 48 == 0 || (elr as u64) < mmu::USER_CODE_VA || (elr as u64) >= mmu::USER_DATA_VA { park(); } let ec = (esr >> 26) & 63; let far: u64; unsafe { asm!("mrs {far}, FAR_EL1",far=out(reg)far,options(nomem,nostack,preserves_flags)) }; let (x0, x1) = (saved.registers[0] as u64, saved.registers[1] as u64); if mode <= 7 { if ec != 0x15 || esr & 0xffff != 0x840 || x0 != unsafe { read_volatile(state.add(14)) } || x1 != 4096 { park(); } } else if ec != 0x24 || !(13..=15).contains(&(esr & 63)) || esr & ((1 << 6) | (1 << 7) | (1 << 10)) != 0 || far != unsafe { read_volatile(state.add(16)) } { park(); } let index = unsafe { read_volatile(state.add(17)) } as usize; if index >= 12 { park(); } let trace = core::ptr::addr_of_mut!(OS_K840_TRAPS).cast::(); let irq = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; let owner = unsafe { read_volatile(state.add(3)) }; let task_slot = unsafe { read_volatile(state.add(18)) }; unsafe { for (field, value) in [ mode, ttbr, tcr, owner, task_slot, slot as u64, esr as u64, if mode <= 7 { 0 } else { far }, elr as u64, x0, x1, irq, ] .into_iter() .enumerate() { write_volatile(trace.add(index * 12 + field), value) } for (field, value) in [ (4, 0), (5, 1), (6, ttbr), (8, slot as u64), (9, esr as u64), (10, if mode <= 7 { 0 } else { far }), (11, elr as u64), (12, x0), (13, x1), (17, (index + 1) as u64), ] { write_volatile(state.add(field), value) } } unsafe extern "C" { fn __os_k840_resume_kernel() -> !; } unsafe { __os_k840_resume_kernel() } } global_asm!( r#" .section .text.os_k840_entry,"ax" .global __os_k840_enter_user __os_k840_enter_user: stp x19, x20, [sp, #-16]! stp x21, x22, [sp, #-16]! stp x23, x24, [sp, #-16]! stp x25, x26, [sp, #-16]! stp x27, x28, [sp, #-16]! stp x29, x30, [sp, #-16]! stp d8, d9, [sp, #-16]! stp d10, d11, [sp, #-16]! stp d12, d13, [sp, #-16]! stp d14, d15, [sp, #-16]! adrp x3, OS_K840_USER_STATE add x3, x3, :lo12:OS_K840_USER_STATE mov x4, sp str x4, [x3, #8] dsb ishst isb movz x3, #0x4000 movk x3, #0x8000, lsl #16 msr SP_EL0, x3 movz x3, #0x8000, lsl #16 msr ELR_EL1, x3 mov x3, #0x3c0 msr SPSR_EL1, x3 isb eret .global __os_k840_resume_kernel __os_k840_resume_kernel: adrp x0, OS_K840_USER_STATE add x0, x0, :lo12:OS_K840_USER_STATE ldp x1, x2, [x0] dsb ishst msr TTBR0_EL1, x1 isb mov sp, x2 ldp d14, d15, [sp], #16 ldp d12, d13, [sp], #16 ldp d10, d11, [sp], #16 ldp d8, d9, [sp], #16 ldp x29, x30, [sp], #16 ldp x27, x28, [sp], #16 ldp x25, x26, [sp], #16 ldp x23, x24, [sp], #16 ldp x21, x22, [sp], #16 ldp x19, x20, [sp], #16 ret .section .text.os_k840_user,"ax" .balign 16 .global __os_k840_user_start __os_k840_user_start: cmp x0, #2 b.eq 4f cbz x0, 2f mov x3, #4096 mov x4, x1 1: strb w2, [x4], #1 subs x3, x3, #1 b.ne 1b 2: mov x3, #4096 mov x4, x1 mov x0, #0 3: ldrb w5, [x4], #1 orr x0, x0, x5 subs x3, x3, #1 b.ne 3b mov x1, #4096 svc #0x840 b . 4: ldr x0, [x1] mov x1, #0 svc #0x840 b . .global __os_k840_user_end __os_k840_user_end: "# ); use turkos_kernel::runtime::page_tables::{descriptor as os_k860_page_descriptor,tree as os_k860_page_tree}; const OS_K860_PAGE_OWNER:u64=0x860; const OS_K860_PAGE_PA_MASK:u64=0x0000_ffff_ffff_f000; #[unsafe(no_mangle)] static mut OS_K860_PAGE_ALLOC_BUDGET:u64=u64::MAX; #[unsafe(no_mangle)] static mut OS_K860_PAGE_FAIL_RELEASE:u64=0; #[unsafe(no_mangle)] static mut OS_K860_PAGE_EVENTS:[u64;4096]=[0;4096]; #[unsafe(no_mangle)] static mut OS_K860_PAGE_EVENT_COUNT:u64=0; fn os_k860_page_event(kind:u64,address:u64,value:u64){ let n=unsafe{read_volatile(core::ptr::addr_of!(OS_K860_PAGE_EVENT_COUNT))}; if n>=1365 {park();} unsafe{let p=core::ptr::addr_of_mut!(OS_K860_PAGE_EVENTS).cast::().add(n as usize*3);write_volatile(p,kind);write_volatile(p.add(1),address);write_volatile(p.add(2),value);write_volatile(core::ptr::addr_of_mut!(OS_K860_PAGE_EVENT_COUNT),n+1);} } struct OsK860PageFrames<'a>{authority:&'a mut os_k720_frame::Allocator,leases:[Option;32],addresses:[u64;32]} impl OsK860PageFrames<'_>{ fn validate(&self,address:u64)->Result<(),os_k740_metadata::Error>{ let lease=self.leases.iter().flatten().find(|lease|lease.address()==address).ok_or(os_k740_metadata::Error::BadAddress)?; self.authority.validate_lease(lease).map_err(|_|os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK860PageFrames<'_>{ fn allocate(&mut self)->Result{ let budget=unsafe{read_volatile(core::ptr::addr_of!(OS_K860_PAGE_ALLOC_BUDGET))}; if budget==0{return Err(os_k740_metadata::Error::NoMemory);} if budget!=u64::MAX {unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K860_PAGE_ALLOC_BUDGET),budget-1)};} let slot=self.leases.iter().position(Option::is_none).ok_or(os_k740_metadata::Error::Capacity)?; let lease=self.authority.allocate_at(self.addresses[slot]).map_err(|_|os_k740_metadata::Error::NoMemory)?; let address=lease.address();self.leases[slot]=Some(lease);os_k860_page_event(1,address,0);Ok(address) } fn release(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{ self.validate(address)?; if unsafe{read_volatile(core::ptr::addr_of!(OS_K860_PAGE_FAIL_RELEASE))}==address {os_k860_page_event(9,address,0);return Err(os_k740_metadata::Error::Backend);} let slot=self.leases.iter().position(|lease|lease.as_ref().is_some_and(|lease|lease.address()==address)).ok_or(os_k740_metadata::Error::BadAddress)?; for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};} unsafe{asm!("dsb ish",options(nostack,preserves_flags))}; self.authority.release(self.leases[slot].as_ref().ok_or(os_k740_metadata::Error::Backend)?).map_err(|_|os_k740_metadata::Error::Backend)?; self.leases[slot]=None;os_k860_page_event(8,address,0);Ok(()) } fn zero_page(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{self.validate(address)?;for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};}Ok(())} fn read_byte(&self,address:u64,offset:usize)->Result{if offset>=4096{return Err(os_k740_metadata::Error::BadAddress);}self.validate(address)?;Ok(unsafe{read_volatile((address as *const u8).add(offset))})} fn available_pages(&self)->u64{self.authority.remaining_pages()} } pub struct OsK860PageBackend<'a>{identity:u64,registry:os_k740_metadata::Registry,32>,tables:&'a mmu::Tables} impl<'a> OsK860PageBackend<'a>{ pub fn new(authority:&'a mut os_k720_frame::Allocator,tables:&'a mmu::Tables)->Result{ let region=authority.free_regions().iter().rev().find(|r|r.end-r.start>=40*4096).ok_or("page-tables-interior-pool")?; let base=region.end-36*4096; let addresses=core::array::from_fn(|i|base+i as u64*4096); let frames=OsK860PageFrames{authority,leases:core::array::from_fn(|_|None),addresses}; unsafe extern "C"{fn __os_k760_workspace_identity()->u64;} let identity=unsafe{__os_k760_workspace_identity()};if identity==0{return Err("page-tables-registry-identity");} let registry=os_k740_metadata::Registry::new(identity,frames).map_err(|_|"page-tables-registry")?; Ok(Self{identity,registry,tables}) } pub fn identity(&self)->u64{self.identity} pub fn with_authority_observation(&self,callback:impl FnOnce(&os_k720_frame::Allocator)->R)->R{self.registry.with_backend_observation(|frames|callback(frames.authority))} pub fn registry_records(&self)->impl Iterator+'_ {self.registry.owned_records()} pub fn observed_handles(&self)->impl Iterator+'_ { self.registry.owned_records().map(|r|{let generation=self.registry.with_backend_observation(|frames|frames.leases.iter().flatten().find(|lease|lease.address()==r.address).map(|lease|lease.generation()).unwrap_or(0));(r,generation)}) } fn owned_kernel(&self,address:u64)->bool{self.tables.tagged_frames().any(|frame|frame.address==address)} fn shared_walk(&self,address:u64,level:u8,visited:&mut [u64;4],count:usize)->Result<(),os_k860_page_tree::Error>{ if level>3||count>=4||visited[..count].contains(&address)||!self.owned_kernel(address){return Err(os_k860_page_tree::Error::Shared);} visited[count]=address; for index in 0..512 { let raw=unsafe{read_volatile((address as *const u64).add(index))};if raw==0{continue;} match raw&3 { 3 if level<3=>self.shared_walk(raw&OS_K860_PAGE_PA_MASK,level+1,visited,count+1)?, 1 if level<3=>{if raw&(1<<6)!=0{return Err(os_k860_page_tree::Error::Shared);}}, 3 if level==3=>{if raw&(1<<6)!=0{return Err(os_k860_page_tree::Error::Shared);}}, _=>return Err(os_k860_page_tree::Error::Shared), } } Ok(()) } fn kernel_level(&self,address:u64)->Result{ let mut pending=[(0u64,0u8);24];pending[0]=(self.tables.root,1);let mut len=1;let mut at=0; while at{ type Handle=os_k740_metadata::Handle; fn allocate_zeroed(&mut self)->Result{self.registry.allocate_zeroed(OS_K860_PAGE_OWNER).map_err(|e|if e==os_k740_metadata::Error::NoMemory{os_k860_page_tree::Error::NoMemory}else{os_k860_page_tree::Error::Backend})} fn address(&self,handle:Self::Handle)->u64{handle.address()} fn validate(&self,handle:Self::Handle)->Result<(),os_k860_page_tree::Error>{self.registry.read_byte(handle,OS_K860_PAGE_OWNER,4095).map(|_|()).map_err(|_|os_k860_page_tree::Error::Stale)} fn release(&mut self,handle:Self::Handle)->Result<(),os_k860_page_tree::Error>{self.registry.release(handle,OS_K860_PAGE_OWNER).map_err(|_|os_k860_page_tree::Error::Backend)} fn entry(&self,handle:Self::Handle,index:usize)->Result{if index>=512{return Err(os_k860_page_tree::Error::Backend);}self.validate(handle)?;Ok(unsafe{read_volatile((handle.address() as *const u64).add(index))})} fn write_entry(&mut self,handle:Self::Handle,index:usize,value:u64)->Result<(),os_k860_page_tree::Error>{if index>=512{return Err(os_k860_page_tree::Error::Backend);}self.validate(handle)?;unsafe{write_volatile((handle.address() as *mut u64).add(index),value)};os_k860_page_event(2,handle.address()+index as u64*8,value);Ok(())} fn validate_shared_table(&self,address:u64)->Result<(),os_k860_page_tree::Error>{let level=self.kernel_level(address)?;self.shared_walk(address,level,&mut[0;4],0)} fn validate_shared_page(&self,_address:u64)->Result<(),os_k860_page_tree::Error>{Err(os_k860_page_tree::Error::Shared)} fn shared_entry(&self,address:u64,index:usize)->Result{if index>=512{return Err(os_k860_page_tree::Error::Shared);}self.validate_shared_table(address)?;Ok(unsafe{read_volatile((address as *const u64).add(index))})} fn active_root(&self)->u64{let value:u64;unsafe{asm!("mrs {value}, TTBR0_EL1",value=out(reg)value,options(nomem,nostack,preserves_flags))};value&OS_K860_PAGE_PA_MASK} fn invalidate(&mut self,root:u64,va:Option){ unsafe{asm!("dsb ishst",options(nostack,preserves_flags))};os_k860_page_event(3,root,va.unwrap_or(u64::MAX)); unsafe{asm!("tlbi vmalle1is",options(nostack,preserves_flags))};os_k860_page_event(4,root,va.unwrap_or(u64::MAX)); unsafe{asm!("dsb ish",options(nostack,preserves_flags))};os_k860_page_event(5,root,0);unsafe{asm!("isb",options(nostack,preserves_flags))};os_k860_page_event(6,root,0); } fn select_root(&mut self,root:u64,geometry:os_k860_page_descriptor::Geometry,asid:u16){ let old:u64;unsafe{asm!("mrs {old}, TCR_EL1",old=out(reg)old,options(nomem,nostack,preserves_flags))};let tcr=(old&!63)|(64-geometry.virtual_bits() as u64); let ttbr=root|(asid as u64)<<48; unsafe{asm!("dsb ishst","msr TCR_EL1, {tcr}","msr TTBR0_EL1, {ttbr}",tcr=in(reg)tcr,ttbr=in(reg)ttbr,options(nostack,preserves_flags))}; if tcr!=old {unsafe{asm!("tlbi vmalle1is","dsb ish",options(nostack,preserves_flags))};} unsafe{asm!("isb",options(nostack,preserves_flags))};os_k860_page_event(7,root,asid as u64); } } use turkos_kernel::runtime::{address_spaces as os_k860_spaces, mapping as os_k860_mapping}; type OsK860Tree<'a> = os_k820_tree::Tree, 32>; const OS_K860_OWNER: u64 = 0x860; const OS_K860_DATA: u64 = 0x8020_0000; const OS_K860_WINDOW_END: u64 = OS_K860_DATA + 16 * 4096; #[derive(Clone, Copy, PartialEq, Eq)] struct OsK860Value { address: u64, value: os_k860_mapping::Value, } struct OsK860Root { root: Option, // These owners survive every replacement/retirement of table roots. pages: [Option; 5], owner: u64, } impl OsK860Root { fn address(&self) -> u64 { self.root.as_ref().map_or(0, |root| root.address()) } fn reference(&self, index: usize) -> Result { self.pages .get(index) .and_then(Option::as_ref) .map(os_k820_tree::Page::reference) .ok_or(os_k860_mapping::Error::Backing) } } struct OsK860SpaceBackend<'a> { tree: OsK860Tree<'a>, kernel_root: u64, geometry: os_k820_descriptor::Geometry, retained: Option, } fn os_k860_tree_error(error: os_k820_tree::Error) -> os_k860_mapping::Error { match error { os_k820_tree::Error::NoMemory | os_k820_tree::Error::Capacity => { os_k860_mapping::Error::NoMemory } _ => os_k860_mapping::Error::Backing, } } fn os_k860_space_error(_: os_k860_spaces::Error) -> os_k860_mapping::Error { os_k860_mapping::Error::Backing } impl OsK860SpaceBackend<'_> { fn retain_failed_root(&mut self, root: os_k820_tree::Root, owner: u64) -> ! { if self.retained.is_some() { // Containment keeps both the earlier BSS token and this live token. let retained_root = root; loop { core::hint::black_box(&retained_root); unsafe { asm!("wfe", options(nomem, nostack, preserves_flags)) }; } } self.retained = Some(OsK860Root { root: Some(root), pages: core::array::from_fn(|_| None), owner, }); park() } fn authority(&self, callback: impl FnOnce(&os_k720_frame::Allocator) -> R) -> R { self.tree .with_backend_observation(|backend| backend.with_authority_observation(callback)) } fn fixed( &mut self, root: &os_k820_tree::Root, bundle: &OsK860Root, ) -> Result<(), os_k860_mapping::Error> { for index in 0..2 { let raw = unsafe { read_volatile((self.kernel_root as *const u64).add(index)) }; if raw & 3 != 3 { return Err(os_k860_mapping::Error::Backing); } self.tree .share_table(root, (index as u64) << 30, 1, raw & OS_K860_PAGE_PA_MASK) .map_err(os_k860_tree_error)?; } for (index, va, permissions) in [ (0, mmu::USER_CODE_VA, os_k860_rx()), (1, mmu::USER_STACK_VA, os_k860_rw()), ] { self.tree .map_reference( root, va, bundle.reference(index)?, os_k820_descriptor::Memory::Normal, permissions, ) .map_err(os_k860_tree_error)?; } Ok(()) } fn discard(&mut self, bundle: &mut OsK860Root) -> Result<(), os_k860_spaces::Error> { if let Some(root) = &bundle.root { self.tree .detach_all(root) .map_err(|_| os_k860_spaces::Error::Retirement)?; } if let Some(root) = bundle.root.take() { if let Err((_, root)) = self.tree.release_root(root) { bundle.root = Some(root); return Err(os_k860_spaces::Error::Retirement); } } for page in &mut bundle.pages { if let Some(token) = page.take() { if let Err((_, token)) = self.tree.release_page(token) { *page = Some(token); return Err(os_k860_spaces::Error::Retirement); } } } self.tree .audit() .map_err(|_| os_k860_spaces::Error::Backing) } } impl os_k860_spaces::Backend for OsK860SpaceBackend<'_> { type Root = OsK860Root; fn acquire_root(&mut self, owner: u64) -> Result { if self.retained.is_some() { return Err(os_k860_spaces::Error::Retirement); } let root = self .tree .new_draft_root() .map_err(|_| os_k860_spaces::Error::NoMemory)?; let mut bundle = OsK860Root { root: Some(root), pages: core::array::from_fn(|_| None), owner, }; let built = (|| { for page in &mut bundle.pages { *page = Some( self.tree .allocate_page() .map_err(|_| os_k860_spaces::Error::NoMemory)?, ); } os_k860_initialize_pages(&bundle).map_err(|_| os_k860_spaces::Error::Backing)?; self.fixed( bundle.root.as_ref().ok_or(os_k860_spaces::Error::Backing)?, &bundle, ) .map_err(|_| os_k860_spaces::Error::Backing)?; self.tree .publish_root(bundle.root.as_ref().ok_or(os_k860_spaces::Error::Backing)?) .map_err(|_| os_k860_spaces::Error::Backing)?; self.tree .audit() .map_err(|_| os_k860_spaces::Error::Backing) })(); if let Err(error) = built { if self.discard(&mut bundle).is_err() { self.retained = Some(bundle); return Err(os_k860_spaces::Error::Retirement); } return Err(error); } Ok(bundle) } fn root_address(&self, root: &OsK860Root) -> u64 { root.address() } fn validate_root(&self, root: &OsK860Root) -> Result<(), os_k860_spaces::Error> { if root.owner == 0 || root.address() == 0 || root.pages.iter().any(Option::is_none) { return Err(os_k860_spaces::Error::Backing); } if !self .tree .owned_records() .any(|row| row.root && row.address == root.address()) { return Err(os_k860_spaces::Error::Backing); } for page in root.pages.iter().flatten() { if !self .tree .owned_records() .any(|row| !row.table && row.address == page.address() && !row.quarantined) { return Err(os_k860_spaces::Error::Backing); } } self.tree .audit() .map_err(|_| os_k860_spaces::Error::Backing) } fn invalidate_recycled(&mut self, asid: u16, old: u64, new: u64) { if asid == 0 || old == 0 || new != old + 1 { park(); } os_k860_sync(0x11, asid as u64); } fn select(&mut self, root: &OsK860Root, asid: u16) -> Result<(), os_k860_spaces::Error> { self.validate_root(root)?; self.tree .activate( root.root.as_ref().ok_or(os_k860_spaces::Error::Backing)?, asid, ) .map_err(|_| os_k860_spaces::Error::Backing) } fn restore_kernel(&mut self) -> Result<(), os_k860_spaces::Error> { self.tree .restore_shared_root(self.kernel_root, self.geometry, 0) .map_err(|_| os_k860_spaces::Error::Backing) } fn retire_root( &mut self, mut root: OsK860Root, _asid: u16, _generation: u64, ) -> Result<(), (os_k860_spaces::Error, OsK860Root)> { match self.discard(&mut root) { Ok(()) => Ok(()), Err(error) => Err((error, root)), } } } struct OsK860Prepared { root: os_k820_tree::Root, map: [Option; 16], } struct OsK860Backend<'a> { spaces: os_k860_spaces::Spaces, 1, 1>, handle: os_k860_spaces::Handle, owner: os_k860_mapping::Owner, map: [Option; 16], prepares: u64, commits: u64, rollbacks: u64, fillers: [Option; 24], } impl OsK860Backend<'_> { fn observe( &self, callback: impl FnOnce(&OsK860SpaceBackend<'_>, &OsK860Root) -> R, ) -> Result { self.spaces .with_root_observation(self.handle, self.owner.id, callback) .map_err(os_k860_space_error) } fn transaction( &mut self, callback: impl FnOnce(&mut OsK860SpaceBackend<'_>, &mut OsK860Root) -> R, ) -> Result { unsafe { self.spaces .with_owned_root_transaction(self.handle, self.owner.id, callback) } .map_err(os_k860_space_error) } fn activate(&mut self) -> Result { self.spaces .activate(self.handle, self.owner.id) .map(|row| row.root | (row.asid as u64) << 48) .map_err(os_k860_space_error) } fn full_draft_control(&mut self) -> Result { let held = self.observe(|backend, _| 32usize - backend.tree.owned_records().count() - 3)?; if held > 24 { return Err(os_k860_mapping::Error::Backing); } for index in 0..held { let page = self.transaction(|backend, _| { backend.tree.allocate_page().map_err(os_k860_tree_error) })??; self.fillers[index] = Some(page); } Ok(held) } fn return_fillers(&mut self) -> Result<(), os_k860_mapping::Error> { for index in 0..24 { if let Some(page) = self.fillers[index].take() { let result = self.transaction(|backend, _| backend.tree.release_page(page))?; if let Err((error, page)) = result { self.fillers[index] = Some(page); return Err(os_k860_tree_error(error)); } } } Ok(()) } } impl os_k860_mapping::Backend for OsK860Backend<'_> { type FrameRef = os_k820_tree::PageRef; type Prepared = OsK860Prepared; fn owner(&self) -> os_k860_mapping::Owner { self.owner } fn validate_frame( &self, owner: os_k860_mapping::Owner, frame: Self::FrameRef, ) -> Result<(), os_k860_mapping::Error> { if owner != self.owner { return Err(os_k860_mapping::Error::Foreign); } self.observe(|backend, bundle| { if !bundle.pages[2..] .iter() .flatten() .any(|page| page.reference() == frame) { return Err(os_k860_mapping::Error::Backing); } backend.tree.audit().map_err(os_k860_tree_error) })? } fn current( &self, address: u64, ) -> Result>, os_k860_mapping::Error> { let expected = self .map .iter() .flatten() .find(|row| row.address == address) .map(|row| row.value); self.observe(|backend, bundle| { match backend.tree.walk( bundle .root .as_ref() .ok_or(os_k860_mapping::Error::Backing)?, address, ) { Ok(walked) => { let value = expected.ok_or(os_k860_mapping::Error::Backing)?; if walked.physical != value.frame.address() || walked.permissions != value.permissions || walked.memory != os_k820_descriptor::Memory::Normal { return Err(os_k860_mapping::Error::Backing); } Ok(expected) } Err(os_k820_tree::Error::Descriptor(os_k820_descriptor::Error::Unmapped)) if expected.is_none() => { Ok(None) } _ => Err(os_k860_mapping::Error::Backing), } })? } fn prepare( &mut self, changes: &[os_k860_mapping::Change], ) -> Result { self.prepares = self .prepares .checked_add(1) .ok_or(os_k860_mapping::Error::Generation)?; let mut map = self.map; for change in changes { if os_k860_mapping::Backend::current(self, change.virtual_address)? != change.before { return Err(os_k860_mapping::Error::Backing); } let slot = map .iter() .position(|row| row.is_some_and(|row| row.address == change.virtual_address)) .or_else(|| map.iter().position(Option::is_none)) .ok_or(os_k860_mapping::Error::Capacity)?; map[slot] = change.after.map(|value| OsK860Value { address: change.virtual_address, value, }); } let result = self.transaction(|backend, bundle| { let draft = backend.tree.new_draft_root().map_err(os_k860_tree_error)?; let built = (|| { backend.fixed(&draft, bundle)?; for row in map.iter().flatten() { backend .tree .map_reference( &draft, row.address, row.value.frame, os_k820_descriptor::Memory::Normal, row.value.permissions, ) .map_err(os_k860_tree_error)?; } backend.tree.audit().map_err(os_k860_tree_error) })(); if let Err(error) = built { if backend.tree.detach_all(&draft).is_err() { backend.retain_failed_root(draft, bundle.owner); } if let Err((_, draft)) = backend.tree.release_root(draft) { backend.retain_failed_root(draft, bundle.owner); } return Err(error); } Ok(OsK860Prepared { root: draft, map }) })?; if result.is_err() { self.rollbacks = self .rollbacks .checked_add(1) .ok_or(os_k860_mapping::Error::Generation)?; } result } fn commit(&mut self, prepared: Self::Prepared) { let OsK860Prepared { root: draft, map } = prepared; let asid = self .spaces .observe(self.handle, self.owner.id) .unwrap_or_else(|_| park()) .asid; self.transaction(|backend,bundle|{ use os_k860_spaces::Backend; backend.restore_kernel().unwrap_or_else(|_|park()); let old=bundle.root.as_ref().unwrap_or_else(||park()); backend.tree.quiesce(old).unwrap_or_else(|_|park()); os_k860_sync(0x12,old.address()); if backend.tree.publish_root(&draft).is_err() { backend.retain_failed_root(draft,bundle.owner); } if backend.tree.activate(&draft,asid).is_err() { backend.retain_failed_root(draft,bundle.owner); } let actual:u64;unsafe{asm!("mrs {actual}, TTBR0_EL1",actual=out(reg)actual,options(nomem,nostack,preserves_flags))}; if actual!=draft.address()|(asid as u64)<<48{backend.retain_failed_root(draft,bundle.owner);} os_k860_page_event(0x13,old.address(),actual); let old=bundle.root.replace(draft).unwrap_or_else(||park()); if backend.tree.detach_all(&old).is_err(){backend.retain_failed_root(old,bundle.owner);} if let Err((_,old))=backend.tree.release_root(old){backend.retain_failed_root(old,bundle.owner);} backend.tree.audit().unwrap_or_else(|_|park()); }).unwrap_or_else(|_|park()); self.map = map; self.commits = self.commits.checked_add(1).unwrap_or_else(|| park()); } } fn os_k860_rw() -> os_k820_descriptor::Permissions { os_k820_descriptor::Permissions { user: true, write: true, execute: false, } } fn os_k860_ro() -> os_k820_descriptor::Permissions { os_k820_descriptor::Permissions { user: true, write: false, execute: false, } } fn os_k860_rx() -> os_k820_descriptor::Permissions { os_k820_descriptor::Permissions { user: true, write: false, execute: true, } } fn os_k860_sync(kind: u64, address: u64) { unsafe { asm!("dsb ishst", options(nostack, preserves_flags)) }; os_k860_page_event(kind, address, 1); unsafe { asm!("tlbi vmalle1is", options(nostack, preserves_flags)) }; os_k860_page_event(kind, address, 2); unsafe { asm!("dsb ish", options(nostack, preserves_flags)) }; os_k860_page_event(kind, address, 3); unsafe { asm!("isb", options(nostack, preserves_flags)) }; os_k860_page_event(kind, address, 4); } #[unsafe(no_mangle)] static mut OS_K860_EVIDENCE: [u64; 128] = [0; 128]; #[unsafe(no_mangle)] static mut OS_K860_STATE: [u64; 64] = [0; 64]; #[unsafe(no_mangle)] static mut OS_K860_RECORDS: [u64; 96] = [0; 96]; #[unsafe(no_mangle)] static mut OS_K860_OWNED: [u64; 192] = [0; 192]; #[unsafe(no_mangle)] static mut OS_K860_REGISTRY: [u64; 192] = [0; 192]; #[unsafe(no_mangle)] static mut OS_K860_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K860_AUTHORITY_CURRENT: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K860_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K860_POOL_BEFORE: [u8; 32 * 4096] = [0; 32 * 4096]; #[unsafe(no_mangle)] static mut OS_K860_CAPTURE_GATE: u8 = 0; #[unsafe(no_mangle)] static mut OS_K860_NONCE_MASK: u64 = 0; static mut OS_K860_NEXT_NONCE: u64 = 1; type OsK860Mapping<'a> = os_k860_mapping::Mapping, 16>; static mut OS_K860_MAPPING: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K860_EMPTY_BACKEND: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K860_MAPPING_PRESENT: u64 = 0; #[unsafe(no_mangle)] unsafe extern "C" fn __os_k860_mapping_identity() -> u64 { let daif: u64; unsafe { asm!("mrs {daif}, DAIF",daif=out(reg)daif,options(nomem,nostack,preserves_flags)) }; if daif & 0x80 == 0 { return 0; } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K860_NONCE_MASK), daif) }; let current = unsafe { read_volatile(core::ptr::addr_of!(OS_K860_NEXT_NONCE)) }; let Some(next) = current.checked_add(1) else { return 0; }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K860_NEXT_NONCE), next) }; current } fn os_k860_word(index: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K860_EVIDENCE) .cast::() .add(index), value, ) } } #[inline(never)] unsafe fn os_k860_build_at<'a>( destination: *mut OsK860Mapping<'a>, authority: &'a mut os_k720_frame::Allocator, tables: &'a mmu::Tables, ) -> Result<(), &'static str> { let page_backend = OsK860PageBackend::new(authority, tables)?; let pool = page_backend .registry .with_backend_observation(|frames| frames.addresses[0]); for offset in 0..32 * 4096 { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K860_POOL_BEFORE) .cast::() .add(offset), read_volatile((pool as *const u8).add(offset)), ) }; } let identity = page_backend.identity(); let geometry = os_k820_descriptor::Geometry::new(39, 40).map_err(|_| "mapping-geometry")?; let tree = OsK860Tree::new(identity, geometry, page_backend).map_err(|_| "mapping-tree")?; let (mmfr0, tcr): (u64, u64); unsafe { asm!("mrs {mmfr0}, ID_AA64MMFR0_EL1","mrs {tcr}, TCR_EL1",mmfr0=out(reg)mmfr0,tcr=out(reg)tcr,options(nomem,nostack,preserves_flags)) }; let configuration = os_k860_spaces::Configuration::from_registers(mmfr0, tcr) .map_err(|_| "mapping-asid-configuration")?; let space_backend = OsK860SpaceBackend { tree, kernel_root: tables.root, geometry, retained: None, }; let mut spaces = os_k860_spaces::Spaces::<_, 1, 1>::new(identity, configuration, space_backend) .map_err(|_| "mapping-space-registry")?; let handle = spaces .create(OS_K860_OWNER) .map_err(|_| "mapping-space-root")?; let owner = os_k860_mapping::Owner { id: handle.owner(), generation: handle.generation(), }; let backend = OsK860Backend { spaces, handle, owner, map: [None; 16], prepares: 0, commits: 0, rollbacks: 0, fillers: core::array::from_fn(|_| None), }; unsafe { os_k860_mapping::Mapping::initialize_at( destination, backend, 0x860, os_k860_mapping::Window::new(OS_K860_DATA, OS_K860_WINDOW_END, 39) .map_err(|_| "mapping-window")?, ) } .map_err(|_| "mapping-initialize") } fn os_k860_capture(mapper: &mut OsK860Mapping<'_>, stage: u64) -> Result<(), &'static str> { let actual = unsafe { mapper.with_backing_allocation(|backend| backend.activate()) } .map_err(|_| "mapping-capture-pin")? .map_err(|_| "mapping-capture-activate")?; let (ttbr, tcr) = os_k860_registers(); if actual != ttbr { return Err("mapping-capture-root"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K860_STATE), [0; 64]); write_volatile(core::ptr::addr_of_mut!(OS_K860_RECORDS), [0; 96]); write_volatile(core::ptr::addr_of_mut!(OS_K860_OWNED), [0; 192]); write_volatile(core::ptr::addr_of_mut!(OS_K860_REGISTRY), [0; 192]) }; let mut records = 0; let mut guards = 0; let mut pins = 0; for (address, value, generation, held) in mapper.records() { let (raw, physical) = if let Some(value) = value { ( os_k820_descriptor::page( os_k820_descriptor::Geometry::new(39, 40) .map_err(|_| "mapping-capture-geometry")?, value.frame.address(), os_k820_descriptor::Memory::Normal, value.permissions, ) .map_err(|_| "mapping-capture-pte")?, value.frame.address(), ) } else { guards += 1; (0, 0) }; let p = unsafe { core::ptr::addr_of_mut!(OS_K860_RECORDS) .cast::() .add(records * 6) }; unsafe { for (i, v) in [ address, physical, raw, generation, held as u64, (value.is_none()) as u64, ] .into_iter() .enumerate() { write_volatile(p.add(i), v) } }; records += 1; pins += held as u64; } mapper.with_backend_observation(|backend| { backend .observe(|space, bundle| { space.tree.audit().map_err(|_| "mapping-capture-tree")?; let observation = backend .spaces .observe(backend.handle, backend.owner.id) .map_err(|_| "mapping-capture-namespace")?; let mut owned = 0; let mut data = 0; let mut tables = 0; for row in space.tree.owned_records() { let p = unsafe { core::ptr::addr_of_mut!(OS_K860_OWNED) .cast::() .add(owned * 6) }; unsafe { for (i, v) in [ row.address, row.generation, row.table as u64, row.root as u64, row.quarantined as u64, row.mappings as u64, ] .into_iter() .enumerate() { write_volatile(p.add(i), v) } }; owned += 1; if row.table { tables += 1 } else { data += 1 }; } let mut registry = 0; space.tree.with_backend_observation(|page_backend| { for (row, central_generation) in page_backend.observed_handles() { let p = unsafe { core::ptr::addr_of_mut!(OS_K860_REGISTRY) .cast::() .add(registry * 6) }; unsafe { for (i, v) in [ row.address, row.owner, row.generation, row.pins as u64, row.quarantined as u64, central_generation, ] .into_iter() .enumerate() { write_volatile(p.add(i), v) } }; registry += 1; } }); space.authority(|authority| unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K860_AUTHORITY_CURRENT), os_k740_authority_inventory(authority).words, ) }); let p = core::ptr::addr_of_mut!(OS_K860_STATE).cast::(); unsafe { for (i, v) in [ (0, stage), (1, backend.owner.id), (2, backend.owner.generation), (3, backend.handle.slot() as u64), (4, observation.asid as u64), (5, observation.asid_generation), (6, bundle.address()), (7, ttbr), (8, tcr), (9, backend.prepares), (10, backend.commits), (11, backend.rollbacks), (12, records as u64), (13, owned as u64), (14, backend.spaces.owned_records().count() as u64), (15, registry as u64), (16, data as u64), (17, tables as u64), (18, guards), (19, pins), (20, read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))), ( 21, read_volatile(core::ptr::addr_of!(OS_K860_PAGE_EVENT_COUNT)), ), ( 22, backend.fillers.iter().filter(|page| page.is_some()).count() as u64, ), (24, read_volatile(core::ptr::addr_of!(OS_K860_NONCE_MASK))), ] { write_volatile(p.add(i), v) } }; Ok::<(), &'static str>(()) }) .map_err(|_| "mapping-capture-owner")? })?; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K860_CAPTURE_GATE), 0) }; write_str("TURKOS/OS/K860: CAPTURE STAGE=0x"); write_hex(stage as usize); write_str(" GATE=0x"); write_hex(core::ptr::addr_of!(OS_K860_CAPTURE_GATE) as usize); write_line(" READY"); let (start, frequency): (u64, u64); unsafe { asm!("mrs {start}, CNTPCT_EL0","mrs {frequency}, CNTFRQ_EL0",start=out(reg)start,frequency=out(reg)frequency,options(nomem,nostack,preserves_flags)) }; let end = start .checked_add( frequency .checked_mul(90) .ok_or("mapping-capture-frequency")?, ) .ok_or("mapping-capture-deadline")?; while unsafe { read_volatile(core::ptr::addr_of!(OS_K860_CAPTURE_GATE)) } != 1 { let now: u64; unsafe { asm!("mrs {now}, CNTPCT_EL0",now=out(reg)now,options(nomem,nostack,preserves_flags)) }; if now >= end { return Err("mapping-capture-timeout"); } core::hint::spin_loop(); } Ok(()) } fn os_k860_run( mapper: &mut OsK860Mapping<'_>, mode: u64, operation: u64, target: u64, seed: u64, expected: u64, ) -> Result<(), &'static str> { let (ttbr, owner, kernel) = unsafe { mapper.with_backing_allocation(|backend| { let ttbr = backend.activate()?; let kernel = backend.observe(|space, _| space.kernel_root)?; Ok::<_, os_k860_mapping::Error>((ttbr, backend.owner, kernel)) }) } .map_err(|_| "mapping-run-pin")? .map_err(|_| "mapping-run-root")?; os_k860_enter(mode, owner, ttbr, kernel, operation, target, seed, expected) } #[inline(never)] fn os_k860_accept( runtime: &mut OsK660Runtime, tables: &mmu::Tables, _boot: &BootData, ) -> Result<(), &'static str> { use os_k860_mapping::Error; let (before_ttbr, before_tcr) = os_k860_registers(); let irq = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; let memory = runtime.memory_mut().map_err(|_| "mapping-memory")?; let authority = memory.allocator.os_authority_mut(); let before = os_k740_authority_inventory(authority).words; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K860_AUTHORITY_BEFORE), before) }; if unsafe { read_volatile(core::ptr::addr_of!(OS_K860_MAPPING_PRESENT)) } != 0 { return Err("mapping-bss-already-initialized"); } let storage = core::ptr::addr_of_mut!(OS_K860_MAPPING).cast::>(); unsafe { os_k860_build_at(storage, authority, tables)?; write_volatile(core::ptr::addr_of_mut!(OS_K860_MAPPING_PRESENT), 1) }; let mapper = unsafe { &mut *storage }; let (owner, frames) = mapper .with_backend_observation(|backend| { backend.observe(|_, bundle| { Ok::<_, Error>(( backend.owner, [ bundle.reference(2)?, bundle.reference(3)?, bundle.reference(4)?, ], )) }) }) .map_err(|_| "mapping-data-owner")? .map_err(|_| "mapping-data-refs")?; for (i, v) in [ (0, 0x4b3836304d415050), (1, tables.root), (2, before_ttbr), (3, before_tcr), (4, before[3]), (6, owner.id), (7, owner.generation), (23, irq), (26, OS_K860_DATA), (27, OS_K860_WINDOW_END), (28, frames[0].address()), (29, frames[1].address()), (30, frames[2].address()), ] { os_k860_word(i, v) } mapper .with_backend_observation(|backend| { backend.observe(|_, bundle| { os_k860_word( 31, bundle.pages[0].as_ref().ok_or(Error::Backing)?.address(), ); os_k860_word( 32, bundle.pages[1].as_ref().ok_or(Error::Backing)?.address(), ); let pool = backend.spaces.with_backend_observation(|space| { space.tree.with_backend_observation(|pages| { pages.registry.with_backend_observation(|frames| { (frames.addresses[0], frames.addresses[31] + 4096) }) }) }); os_k860_word(38, pool.0); os_k860_word(39, pool.1); Ok::<(), Error>(()) }) }) .map_err(|_| "mapping-fixed-owner")? .map_err(|_| "mapping-fixed-pages")?; mapper .map(owner, OS_K860_DATA + 2 * 4096, &frames[2..], os_k860_rw()) .map_err(|_| "mapping-third-page")?; os_k860_capture(mapper, 1)?; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? { if mapper.map(owner, OS_K860_DATA, &frames, os_k860_rw()) != Err(Error::Conflict) { return Err("mapping-third-conflict-prefix"); } os_k860_run(mapper, 4, 3, OS_K860_DATA, 0, 0)?; os_k860_run(mapper, 4, 3, OS_K860_DATA + 4096, 0, 0)?; os_k860_run(mapper, 1, 0, OS_K860_DATA + 8192, 0, 0xa3)?; os_k860_word(34, 1); } os_k860_capture(mapper, 2)?; mapper .unmap(owner, OS_K860_DATA + 8192, 1) .map_err(|_| "mapping-third-unmap")?; mapper .map(owner, OS_K860_DATA, &frames, os_k860_rw()) .map_err(|_| "mapping-three-pages")?; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? { let held = unsafe { mapper.with_backing_allocation(|backend| backend.full_draft_control()) } .map_err(|_| "mapping-enospc-control")? .map_err(|_| "mapping-enospc-fillers")?; os_k860_word(15, held as u64); } os_k860_capture(mapper, 3)?; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? { if mapper.protect(owner, OS_K860_DATA, 3, os_k860_ro()) != Err(Error::NoMemory) { return Err("mapping-enospc-prefix"); } for i in 0..3 { os_k860_run( mapper, 2, 1, OS_K860_DATA + i * 4096 + 4095, 0xa1 + i, 0xa1 + i, )?; } os_k860_word(35, 1); } os_k860_capture(mapper, 4)?; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? { unsafe { mapper.with_backing_allocation(|backend| backend.return_fillers()) } .map_err(|_| "mapping-filler-pin")? .map_err(|_| "mapping-filler-return")?; let foreign = os_k860_mapping::Owner { id: owner.id + 1, ..owner }; if mapper.unmap(foreign, OS_K860_DATA, 1) != Err(Error::Foreign) { return Err("mapping-foreign-owner"); } let foreign = os_k860_mapping::Owner { generation: owner.generation + 1, ..owner }; if mapper.protect(foreign, OS_K860_DATA, 1, os_k860_ro()) != Err(Error::Foreign) { return Err("mapping-foreign-generation"); } os_k860_word(21, 2); if mapper.map(owner, OS_K860_DATA + 4 * 4096, &frames[..1], os_k860_rx()) != Err(Error::Permissions) { return Err("mapping-wx-alias"); } os_k860_word(20, 1); let mut pin = mapper .pin(owner, OS_K860_DATA, 3, true) .map_err(|_| "mapping-pin")?; if mapper.unmap(owner, OS_K860_DATA + 4096, 1) != Err(Error::Busy) { return Err("mapping-pinned-unmap"); } os_k860_word(18, 1); if mapper.protect(owner, OS_K860_DATA, 3, os_k860_ro()) != Err(Error::Busy) { return Err("mapping-pinned-protect"); } os_k860_word(19, 1); mapper.unpin(&mut pin).map_err(|_| "mapping-unpin")?; } let before_guard = mapper.with_backend_observation(|backend| { ( backend.prepares, backend.commits, backend .observe(|space, _| { space.authority(|authority| os_k740_authority_inventory(authority).words) }) .map_err(|_| "mapping-guard-before"), ) }); mapper .guard(owner, OS_K860_DATA + 10 * 4096, 1) .map_err(|_| "mapping-guard")?; let after_guard = mapper.with_backend_observation(|backend| { ( backend.prepares, backend.commits, backend .observe(|space, _| { space.authority(|authority| os_k740_authority_inventory(authority).words) }) .map_err(|_| "mapping-guard-after"), ) }); if before_guard != after_guard { return Err("mapping-guard-backing"); } os_k860_word(16, before_guard.0); os_k860_word(17, after_guard.0); mapper .unmap(owner, OS_K860_DATA + 4096, 1) .map_err(|_| "mapping-middle-unmap")?; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? { os_k860_run(mapper, 1, 0, OS_K860_DATA, 0, 255)?; os_k860_run(mapper, 1, 0, OS_K860_DATA + 8192, 0, 0xa3)?; os_k860_run(mapper, 4, 3, OS_K860_DATA + 4096, 0, 0)?; if mapper.pin(owner, OS_K860_DATA, 3, false).is_ok() { return Err("mapping-missing-last-prefix-pin"); } os_k860_word(36, 1); } os_k860_capture(mapper, 5)?; mapper .protect(owner, OS_K860_DATA, 1, os_k860_ro()) .map_err(|_| "mapping-ro")?; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? { os_k860_run(mapper, 1, 0, OS_K860_DATA, 0, 255)?; os_k860_run(mapper, 5, 1, OS_K860_DATA, 0xaa, 0)?; os_k860_run(mapper, 6, 2, OS_K860_DATA, 0, 0)?; if mapper.pin(owner, OS_K860_DATA, 1, true).is_ok() { return Err("mapping-ro-write-pin"); } os_k860_word(37, 1); } os_k860_capture(mapper, 6)?; mapper .protect(owner, OS_K860_DATA, 1, os_k860_rx()) .map_err(|_| "mapping-rx")?; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? { os_k860_run(mapper, 3, 2, OS_K860_DATA, 0, 0x86)?; if mapper.map(owner, OS_K860_DATA + 4 * 4096, &frames[..1], os_k860_rw()) != Err(Error::Permissions) { return Err("mapping-xw-alias"); } os_k860_word(20, 2); } os_k860_capture(mapper, 7)?; let totals = mapper .with_backend_observation(|backend| (backend.prepares, backend.commits, backend.rollbacks)); os_k860_word(12, totals.0); os_k860_word(13, totals.1); os_k860_word(14, totals.2); mapper .unmap(owner, OS_K860_DATA, 1) .map_err(|_| "mapping-final-first")?; mapper .unmap(owner, OS_K860_DATA + 8192, 1) .map_err(|_| "mapping-final-last")?; mapper .unmap(owner, OS_K860_DATA + 10 * 4096, 1) .map_err(|_| "mapping-final-guard")?; let destination = core::ptr::addr_of_mut!(OS_K860_EMPTY_BACKEND).cast::>(); unsafe { os_k860_mapping::Mapping::extract_empty_backend_at(storage, destination) } .map_err(|_| "mapping-final-pins")?; let backend = unsafe { &mut *destination }; backend .return_fillers() .map_err(|_| "mapping-final-fillers")?; backend .spaces .restore_kernel() .map_err(|_| "mapping-final-kernel")?; backend .spaces .release(backend.handle, owner.id) .map_err(|_| "mapping-final-space-release")?; backend .spaces .audit() .map_err(|_| "mapping-final-identities")?; if backend.spaces.owned_records().next().is_some() || !backend.spaces.with_backend_observation(|spaces| { spaces.tree.owned_records().count() == 0 && spaces.retained.is_none() }) { return Err("mapping-final-owner-retained"); } let after = backend.spaces.with_backend_observation(|spaces| { spaces.authority(|authority| os_k740_authority_inventory(authority).words) }); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K860_AUTHORITY_AFTER), after) }; os_k860_word(5, after[3]); os_k860_word(24, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }); os_k860_word(22, unsafe { read_volatile( core::ptr::addr_of!(OS_K860_USER_STATE) .cast::() .add(17), ) }); if before != after || os_k860_registers() != (before_ttbr, before_tcr) || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) } != irq { return Err("mapping-final-conservation"); } unsafe { core::ptr::drop_in_place(destination); write_volatile(core::ptr::addr_of_mut!(OS_K860_MAPPING_PRESENT), 0) }; if cfg!(feature = "os-runtime-acceptance") || os_k1100_product_probe_enabled(_boot)? { write_line( "TURKOS/OS/K845: THIRD-PAGE-CONFLICT TWO-ABSENT-PREFIX-PAGES REAL-EL0 OLD-ROOT-UNCHANGED PASS", ); write_line( "TURKOS/OS/K850: PRIVATE-COMPLETE-DRAFT ACTUAL-OWNED-TABLE-POOL-ENOSPC NO-VISIBLE-PREFIX OLD-RW-SURVIVES PASS", ); write_line( "TURKOS/OS/K855: MIDDLE-UNMAP BOTH-ENDPOINTS-READABLE GUARD-ZERO-BACKING FOREIGN-OWNER PIN-DESTRUCTIVE-REFUSAL PASS", ); write_line( "TURKOS/OS/K860: RW-RO-RX REAL-EL0 WRITE-EXECUTE-FAULTS PHYSICAL-W-X-ALIAS-REFUSED DATA-OWNERS-SURVIVE-ROOT-RETIREMENT PASS", ); } if cfg!(feature = "os-runtime-product") && !os_k1100_product_probe_enabled(_boot)? { write_line("TURKOS/OS/K860: PRODUCT-CONTROL OWNED-RANGE-ROOTS NO-EL0-PROBE"); } write_line( "TURKOS/OS/K860: EXACT-AUTHORITY-RESTORED OWNER-ID-GENERATION-ASID-CONSERVED TABLES-RETIRED-AFTER-TLBI PASS", ); Ok(()) } #[unsafe(no_mangle)] static mut OS_K860_USER_STATE: [u64; 24] = [0; 24]; #[unsafe(no_mangle)] static mut OS_K860_TRAPS: [u64; 256] = [0; 256]; fn os_k860_registers() -> (u64, u64) { let (ttbr, tcr): (u64, u64); unsafe { asm!("mrs {ttbr}, TTBR0_EL1","mrs {tcr}, TCR_EL1",ttbr=out(reg)ttbr,tcr=out(reg)tcr,options(nomem,nostack,preserves_flags)) }; (ttbr, tcr) } fn os_k860_enter( mode: u64, owner: os_k860_mapping::Owner, ttbr: u64, kernel_root: u64, operation: u64, target: u64, seed: u64, expected: u64, ) -> Result<(), &'static str> { let (actual, tcr) = os_k860_registers(); if actual != ttbr || actual >> 48 == 0 || mode == 0 || mode > 6 || tcr & 63 != 25 { return Err("mapping-user-entry-root"); } let state = core::ptr::addr_of_mut!(OS_K860_USER_STATE).cast::(); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)) }; unsafe { for (index, value) in [ (0, kernel_root), (2, ttbr), (3, owner.id), (4, mode), (5, 0), (6, actual), (7, tcr), (14, expected), (15, operation), (16, target), (18, owner.generation), (19, sp), (21, read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))), ] { write_volatile(state.add(index), value) } }; unsafe extern "C" { fn __os_k860_enter_user(operation: u64, target: u64, seed: u64); } unsafe { __os_k860_enter_user(operation, target, seed) }; let after: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after,options(nomem,nostack)); write_volatile(state.add(20), after) }; if unsafe { read_volatile(state.add(5)) } != 1 || sp != after || os_k860_registers() != (kernel_root, tcr) || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) } != unsafe { read_volatile(state.add(21)) } { return Err("mapping-user-return"); } Ok(()) } fn os_k860_expected_user_trap( slot: usize, esr: usize, elr: usize, frame: *const ExceptionFrame, ) -> bool { let state = core::ptr::addr_of_mut!(OS_K860_USER_STATE).cast::(); let mode = unsafe { read_volatile(state.add(4)) }; if mode == 0 { return false; } if slot != 8 || frame.is_null() { park(); } let saved = unsafe { &*frame }; if !saved.canary_valid() { park(); } let (ttbr, tcr) = os_k860_registers(); if ttbr != unsafe { read_volatile(state.add(2)) } || tcr != unsafe { read_volatile(state.add(7)) } || ttbr >> 48 == 0 { park(); } let far: u64; unsafe { asm!("mrs {far}, FAR_EL1",far=out(reg)far,options(nomem,nostack,preserves_flags)) }; let ec = (esr >> 26) & 63; let target = unsafe { read_volatile(state.add(16)) }; let (x0, x1) = (saved.registers[0] as u64, saved.registers[1] as u64); if mode <= 3 { let length = if mode == 1 { 4096 } else if mode == 2 { 1 } else { 0 }; if ec != 0x15 || esr & 0xffff != 0x860 || x0 != unsafe { read_volatile(state.add(14)) } || x1 != length { park(); } if mode == 3 { if elr as u64 != target + 12 { park(); } } else if (elr as u64) < mmu::USER_CODE_VA || (elr as u64) >= mmu::USER_DATA_VA { park(); } } else { let (expected_ec, dfsc, write) = if mode == 6 { (0x20, 15, false) } else if mode == 5 { (0x24, 15, true) } else { (0x24, 7, false) }; if ec != expected_ec || esr & 63 != dfsc || esr & (1 << 10) != 0 || (esr & (1 << 6) != 0) != write || far != target { park(); } if mode == 6 { if elr as u64 != target { park(); } } else if (elr as u64) < mmu::USER_CODE_VA || (elr as u64) >= mmu::USER_DATA_VA { park(); } } let index = unsafe { read_volatile(state.add(17)) } as usize; if index >= 16 { park(); } let trace = core::ptr::addr_of_mut!(OS_K860_TRAPS).cast::(); let irq = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; unsafe { for (field, value) in [ mode, ttbr, tcr, read_volatile(state.add(3)), read_volatile(state.add(18)), ttbr >> 48, slot as u64, esr as u64, if mode <= 3 { 0 } else { far }, elr as u64, x0, x1, irq, read_volatile(state.add(15)), target, read_volatile(state.add(14)), ] .into_iter() .enumerate() { write_volatile(trace.add(index * 16 + field), value) } for (field, value) in [ (4, 0), (5, 1), (6, ttbr), (8, slot as u64), (9, esr as u64), (10, if mode <= 3 { 0 } else { far }), (11, elr as u64), (12, x0), (13, x1), (17, index as u64 + 1), ] { write_volatile(state.add(field), value) } } unsafe extern "C" { fn __os_k860_resume_kernel() -> !; } unsafe { __os_k860_resume_kernel() } } fn os_k860_initialize_pages(bundle: &OsK860Root) -> Result<(), os_k860_mapping::Error> { unsafe extern "C" { static __os_k860_user_start: u8; static __os_k860_user_end: u8; static __os_k860_execute_start: u8; static __os_k860_execute_end: u8; } for (index, page) in bundle.pages.iter().enumerate() { let address = page .as_ref() .ok_or(os_k860_mapping::Error::Backing)? .address(); if index >= 2 { for offset in 0..4096 { unsafe { write_volatile((address as *mut u8).add(offset), 0xa1 + (index - 2) as u8) } } } } for (index, start, end) in [ ( 0, core::ptr::addr_of!(__os_k860_user_start) as usize, core::ptr::addr_of!(__os_k860_user_end) as usize, ), ( 2, core::ptr::addr_of!(__os_k860_execute_start) as usize, core::ptr::addr_of!(__os_k860_execute_end) as usize, ), ] { let len = end .checked_sub(start) .filter(|len| *len > 0 && *len <= 4096) .ok_or(os_k860_mapping::Error::Backing)?; let address = bundle.pages[index] .as_ref() .ok_or(os_k860_mapping::Error::Backing)? .address(); for offset in 0..len { unsafe { write_volatile( (address as *mut u8).add(offset), read_volatile((start as *const u8).add(offset)), ) } } let ctr: u64; unsafe { asm!("mrs {ctr}, CTR_EL0",ctr=out(reg)ctr,options(nomem,nostack,preserves_flags)) }; let dl = 4u64 << (ctr >> 16 & 15); let il = 4u64 << (ctr & 15); if dl > 4096 || il > 4096 { return Err(os_k860_mapping::Error::Backing); } let mut at = address; while at < address + 4096 { unsafe { asm!("dc cvau, {at}",at=in(reg)at,options(nostack,preserves_flags)) }; at += dl; } unsafe { asm!("dsb ish", options(nostack, preserves_flags)) }; at = address; while at < address + 4096 { unsafe { asm!("ic ivau, {at}",at=in(reg)at,options(nostack,preserves_flags)) }; at += il; } unsafe { asm!("dsb ish", "isb", options(nostack, preserves_flags)) }; } Ok(()) } global_asm!( r#" .section .text.os_k860_entry,"ax" .global __os_k860_enter_user __os_k860_enter_user: stp x19,x20,[sp,#-16]! stp x21,x22,[sp,#-16]! stp x23,x24,[sp,#-16]! stp x25,x26,[sp,#-16]! stp x27,x28,[sp,#-16]! stp x29,x30,[sp,#-16]! stp d8,d9,[sp,#-16]! stp d10,d11,[sp,#-16]! stp d12,d13,[sp,#-16]! stp d14,d15,[sp,#-16]! adrp x3,OS_K860_USER_STATE add x3,x3,:lo12:OS_K860_USER_STATE mov x4,sp str x4,[x3,#8] dsb ishst isb movz x3,#0x4000 movk x3,#0x8000,lsl #16 msr SP_EL0,x3 movz x3,#0x8000,lsl #16 msr ELR_EL1,x3 mov x3,#0x3c0 msr SPSR_EL1,x3 isb eret .global __os_k860_resume_kernel __os_k860_resume_kernel: adrp x0,OS_K860_USER_STATE add x0,x0,:lo12:OS_K860_USER_STATE ldp x1,x2,[x0] dsb ishst msr TTBR0_EL1,x1 isb mov sp,x2 ldp d14,d15,[sp],#16 ldp d12,d13,[sp],#16 ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 ldp x27,x28,[sp],#16 ldp x25,x26,[sp],#16 ldp x23,x24,[sp],#16 ldp x21,x22,[sp],#16 ldp x19,x20,[sp],#16 ret .section .text.os_k860_user,"ax" .balign 16 .global __os_k860_user_start __os_k860_user_start: cmp x0,#2 b.eq 5f cmp x0,#3 b.eq 6f cbnz x0,4f mov x3,#4096 mov x4,x1 mov x0,#0 1: ldrb w5,[x4],#1 orr x0,x0,x5 subs x3,x3,#1 b.ne 1b mov x1,#4096 svc #0x860 b . 4: .global __os_k860_user_store __os_k860_user_store: strb w2,[x1] ldrb w0,[x1] mov x1,#1 svc #0x860 b . 5: br x1 6: .global __os_k860_user_load __os_k860_user_load: ldrb w0,[x1] mov x1,#1 svc #0x860 b . .global __os_k860_user_end __os_k860_user_end: .global __os_k860_execute_start __os_k860_execute_start: mov x0,#0x86 mov x1,#0 svc #0x860 b . .global __os_k860_execute_end __os_k860_execute_end: "# ); #[repr(C, align(4096))] struct OsK860Stack { guard: [u8; 4096], usable: [u8; 65536], } #[unsafe(no_mangle)] static mut OS_K860_STACK: OsK860Stack = OsK860Stack { guard: [0; 4096], usable: [0; 65536], }; #[unsafe(no_mangle)] static mut OS_K860_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K860_ERROR: Option<&'static str> = None; const OS_K860_LOW_CANARY: u64 = 0x4b38_3630_4c4f_5721; const OS_K860_HIGH_CANARY: u64 = 0x4b38_3630_4849_4748; fn os_k860_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K860_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } fn os_k860_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { unsafe extern "C" { static __stack_start: u8; static __stack_top: u8; } let guard = core::ptr::addr_of!(OS_K860_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_low = core::ptr::addr_of!(__stack_start) as u64; let original_top = core::ptr::addr_of!(__stack_top) as u64; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("mapping-original-stack-range"); } let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_low + 8) as *const u64) }; if original_canary != 0x4b36_3335_5354_414b || original_second != !0x4b36_3335_5354_414bu64 { return Err("mapping-original-stack-canaries"); } unsafe { tables.unmap_metadata_stack_guard( guard, low, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64, ) } .map_err(|_| "mapping-stack-guard")?; if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("mapping-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K860_LOW_CANARY); write_volatile(high as *mut u64, OS_K860_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K860_LOW_CANARY), (9, OS_K860_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("mapping-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("mapping-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k860_stack_word(slot, value); } unsafe extern "C" { fn __os_k860_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k860_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K860_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K860_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K860_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_low + 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("mapping-stack-restoration"); } os_k860_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k860_stack_word(17, unsafe { read_volatile((original_low + 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K860: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); Ok(()) } else { Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K860_ERROR)) } .unwrap_or("mapping-stack-return")) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k860_accept_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k860_stack_word(4, sp); let result = os_k860_guarded_work( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k860_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K860_ERROR), Some(reason)) }; 1 } } } fn os_k860_guarded_work( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { os_k860_accept(runtime, tables, boot) } global_asm!( r#" .section .text.os_k860_stack,"ax" .balign 4 .global __os_k860_on_stack __os_k860_on_stack: adrp x9, OS_K860_STACK_EVIDENCE add x9, x9, :lo12:OS_K860_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K860_STACK add x9, x9, :lo12:OS_K860_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K860_STACK_EVIDENCE add x9, x9, :lo12:OS_K860_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k860_accept_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K860_STACK_EVIDENCE add x9, x9, :lo12:OS_K860_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret "# ); use turkos_kernel::runtime::page_tables::{descriptor as os_k880_page_descriptor,tree as os_k880_page_tree}; const OS_K880_PAGE_OWNER:u64=0x880; const OS_K880_PAGE_PA_MASK:u64=0x0000_ffff_ffff_f000; #[unsafe(no_mangle)] static mut OS_K880_PAGE_ALLOC_BUDGET:u64=u64::MAX; #[unsafe(no_mangle)] static mut OS_K880_PAGE_FAIL_RELEASE:u64=0; #[unsafe(no_mangle)] static mut OS_K880_PAGE_EVENTS:[u64;4096]=[0;4096]; #[unsafe(no_mangle)] static mut OS_K880_PAGE_EVENT_COUNT:u64=0; fn os_k880_page_event(kind:u64,address:u64,value:u64){ let n=unsafe{read_volatile(core::ptr::addr_of!(OS_K880_PAGE_EVENT_COUNT))}; if n>=1365 {park();} unsafe{let p=core::ptr::addr_of_mut!(OS_K880_PAGE_EVENTS).cast::().add(n as usize*3);write_volatile(p,kind);write_volatile(p.add(1),address);write_volatile(p.add(2),value);write_volatile(core::ptr::addr_of_mut!(OS_K880_PAGE_EVENT_COUNT),n+1);} } struct OsK880PageFrames<'a>{authority:&'a mut os_k720_frame::Allocator,leases:[Option;32],addresses:[u64;32]} impl OsK880PageFrames<'_>{ fn validate(&self,address:u64)->Result<(),os_k740_metadata::Error>{ let lease=self.leases.iter().flatten().find(|lease|lease.address()==address).ok_or(os_k740_metadata::Error::BadAddress)?; self.authority.validate_lease(lease).map_err(|_|os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK880PageFrames<'_>{ fn allocate(&mut self)->Result{ let budget=unsafe{read_volatile(core::ptr::addr_of!(OS_K880_PAGE_ALLOC_BUDGET))}; if budget==0{return Err(os_k740_metadata::Error::NoMemory);} if budget!=u64::MAX {unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_PAGE_ALLOC_BUDGET),budget-1)};} let slot=self.leases.iter().position(Option::is_none).ok_or(os_k740_metadata::Error::Capacity)?; let lease=self.authority.allocate_at(self.addresses[slot]).map_err(|_|os_k740_metadata::Error::NoMemory)?; let address=lease.address();self.leases[slot]=Some(lease);os_k880_page_event(1,address,0);Ok(address) } fn release(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{ self.validate(address)?; if unsafe{read_volatile(core::ptr::addr_of!(OS_K880_PAGE_FAIL_RELEASE))}==address {os_k880_page_event(9,address,0);return Err(os_k740_metadata::Error::Backend);} let slot=self.leases.iter().position(|lease|lease.as_ref().is_some_and(|lease|lease.address()==address)).ok_or(os_k740_metadata::Error::BadAddress)?; for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};} unsafe{asm!("dsb ish",options(nostack,preserves_flags))}; self.authority.release(self.leases[slot].as_ref().ok_or(os_k740_metadata::Error::Backend)?).map_err(|_|os_k740_metadata::Error::Backend)?; self.leases[slot]=None;os_k880_page_event(8,address,0);Ok(()) } fn zero_page(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{self.validate(address)?;for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};}Ok(())} fn read_byte(&self,address:u64,offset:usize)->Result{if offset>=4096{return Err(os_k740_metadata::Error::BadAddress);}self.validate(address)?;Ok(unsafe{read_volatile((address as *const u8).add(offset))})} fn available_pages(&self)->u64{self.authority.remaining_pages()} } pub struct OsK880PageBackend<'a>{identity:u64,registry:os_k740_metadata::Registry,32>,tables:&'a mmu::Tables} impl<'a> OsK880PageBackend<'a>{ pub fn new(authority:&'a mut os_k720_frame::Allocator,tables:&'a mmu::Tables)->Result{ let region=authority.free_regions().iter().rev().find(|r|r.end-r.start>=40*4096).ok_or("page-tables-interior-pool")?; let base=region.end-36*4096; let addresses=core::array::from_fn(|i|base+i as u64*4096); let frames=OsK880PageFrames{authority,leases:core::array::from_fn(|_|None),addresses}; unsafe extern "C"{fn __os_k760_workspace_identity()->u64;} let identity=unsafe{__os_k760_workspace_identity()};if identity==0{return Err("page-tables-registry-identity");} let registry=os_k740_metadata::Registry::new(identity,frames).map_err(|_|"page-tables-registry")?; Ok(Self{identity,registry,tables}) } pub fn identity(&self)->u64{self.identity} pub fn with_authority_observation(&self,callback:impl FnOnce(&os_k720_frame::Allocator)->R)->R{self.registry.with_backend_observation(|frames|callback(frames.authority))} pub fn registry_records(&self)->impl Iterator+'_ {self.registry.owned_records()} pub fn observed_handles(&self)->impl Iterator+'_ { self.registry.owned_records().map(|r|{let generation=self.registry.with_backend_observation(|frames|frames.leases.iter().flatten().find(|lease|lease.address()==r.address).map(|lease|lease.generation()).unwrap_or(0));(r,generation)}) } fn owned_kernel(&self,address:u64)->bool{self.tables.tagged_frames().any(|frame|frame.address==address)} fn shared_walk(&self,address:u64,level:u8,visited:&mut [u64;4],count:usize)->Result<(),os_k880_page_tree::Error>{ if level>3||count>=4||visited[..count].contains(&address)||!self.owned_kernel(address){return Err(os_k880_page_tree::Error::Shared);} visited[count]=address; for index in 0..512 { let raw=unsafe{read_volatile((address as *const u64).add(index))};if raw==0{continue;} match raw&3 { 3 if level<3=>self.shared_walk(raw&OS_K880_PAGE_PA_MASK,level+1,visited,count+1)?, 1 if level<3=>{if raw&(1<<6)!=0{return Err(os_k880_page_tree::Error::Shared);}}, 3 if level==3=>{if raw&(1<<6)!=0{return Err(os_k880_page_tree::Error::Shared);}}, _=>return Err(os_k880_page_tree::Error::Shared), } } Ok(()) } fn kernel_level(&self,address:u64)->Result{ let mut pending=[(0u64,0u8);24];pending[0]=(self.tables.root,1);let mut len=1;let mut at=0; while at{ type Handle=os_k740_metadata::Handle; fn allocate_zeroed(&mut self)->Result{self.registry.allocate_zeroed(OS_K880_PAGE_OWNER).map_err(|e|if e==os_k740_metadata::Error::NoMemory{os_k880_page_tree::Error::NoMemory}else{os_k880_page_tree::Error::Backend})} fn address(&self,handle:Self::Handle)->u64{handle.address()} fn validate(&self,handle:Self::Handle)->Result<(),os_k880_page_tree::Error>{self.registry.read_byte(handle,OS_K880_PAGE_OWNER,4095).map(|_|()).map_err(|_|os_k880_page_tree::Error::Stale)} fn release(&mut self,handle:Self::Handle)->Result<(),os_k880_page_tree::Error>{self.registry.release(handle,OS_K880_PAGE_OWNER).map_err(|_|os_k880_page_tree::Error::Backend)} fn entry(&self,handle:Self::Handle,index:usize)->Result{if index>=512{return Err(os_k880_page_tree::Error::Backend);}self.validate(handle)?;Ok(unsafe{read_volatile((handle.address() as *const u64).add(index))})} fn write_entry(&mut self,handle:Self::Handle,index:usize,value:u64)->Result<(),os_k880_page_tree::Error>{if index>=512{return Err(os_k880_page_tree::Error::Backend);}self.validate(handle)?;unsafe{write_volatile((handle.address() as *mut u64).add(index),value)};os_k880_page_event(2,handle.address()+index as u64*8,value);Ok(())} fn validate_shared_table(&self,address:u64)->Result<(),os_k880_page_tree::Error>{let level=self.kernel_level(address)?;self.shared_walk(address,level,&mut[0;4],0)} fn validate_shared_page(&self,_address:u64)->Result<(),os_k880_page_tree::Error>{Err(os_k880_page_tree::Error::Shared)} fn shared_entry(&self,address:u64,index:usize)->Result{if index>=512{return Err(os_k880_page_tree::Error::Shared);}self.validate_shared_table(address)?;Ok(unsafe{read_volatile((address as *const u64).add(index))})} fn active_root(&self)->u64{let value:u64;unsafe{asm!("mrs {value}, TTBR0_EL1",value=out(reg)value,options(nomem,nostack,preserves_flags))};value&OS_K880_PAGE_PA_MASK} fn invalidate(&mut self,root:u64,va:Option){ unsafe{asm!("dsb ishst",options(nostack,preserves_flags))};os_k880_page_event(3,root,va.unwrap_or(u64::MAX)); unsafe{asm!("tlbi vmalle1is",options(nostack,preserves_flags))};os_k880_page_event(4,root,va.unwrap_or(u64::MAX)); unsafe{asm!("dsb ish",options(nostack,preserves_flags))};os_k880_page_event(5,root,0);unsafe{asm!("isb",options(nostack,preserves_flags))};os_k880_page_event(6,root,0); } fn select_root(&mut self,root:u64,geometry:os_k880_page_descriptor::Geometry,asid:u16){ let old:u64;unsafe{asm!("mrs {old}, TCR_EL1",old=out(reg)old,options(nomem,nostack,preserves_flags))};let tcr=(old&!63)|(64-geometry.virtual_bits() as u64); let ttbr=root|(asid as u64)<<48; unsafe{asm!("dsb ishst","msr TCR_EL1, {tcr}","msr TTBR0_EL1, {ttbr}",tcr=in(reg)tcr,ttbr=in(reg)ttbr,options(nostack,preserves_flags))}; if tcr!=old {unsafe{asm!("tlbi vmalle1is","dsb ish",options(nostack,preserves_flags))};} unsafe{asm!("isb",options(nostack,preserves_flags))};os_k880_page_event(7,root,asid as u64); } } use turkos_kernel::runtime::{address_spaces as os_k880_spaces, user_copy_mapping as os_k880_mapping}; type OsK880Tree<'a> = os_k820_tree::Tree, 32>; const OS_K880_OWNER: u64 = 0x880; const OS_K880_DATA: u64 = 0x8020_0000; const OS_K880_WINDOW_END: u64 = OS_K880_DATA + 16 * 4096; #[derive(Clone, Copy, PartialEq, Eq)] struct OsK880Value { address: u64, value: os_k880_mapping::Value, } struct OsK880Root { root: Option, // These owners survive every replacement/retirement of table roots. pages: [Option; 19], owner: u64, } impl OsK880Root { fn address(&self) -> u64 { self.root.as_ref().map_or(0, |root| root.address()) } fn reference(&self, index: usize) -> Result { self.pages .get(index) .and_then(Option::as_ref) .map(os_k820_tree::Page::reference) .ok_or(os_k880_mapping::Error::Backing) } } struct OsK880SpaceBackend<'a> { tree: OsK880Tree<'a>, kernel_root: u64, geometry: os_k820_descriptor::Geometry, retained: Option, } fn os_k880_tree_error(error: os_k820_tree::Error) -> os_k880_mapping::Error { match error { os_k820_tree::Error::NoMemory | os_k820_tree::Error::Capacity => { os_k880_mapping::Error::NoMemory } _ => os_k880_mapping::Error::Backing, } } fn os_k880_space_error(_: os_k880_spaces::Error) -> os_k880_mapping::Error { os_k880_mapping::Error::Backing } impl OsK880SpaceBackend<'_> { fn retain_failed_root(&mut self, root: os_k820_tree::Root, owner: u64) -> ! { if self.retained.is_some() { // Containment keeps both the earlier BSS token and this live token. let retained_root = root; loop { core::hint::black_box(&retained_root); unsafe { asm!("wfe", options(nomem, nostack, preserves_flags)) }; } } self.retained = Some(OsK880Root { root: Some(root), pages: core::array::from_fn(|_| None), owner, }); park() } fn authority(&self, callback: impl FnOnce(&os_k720_frame::Allocator) -> R) -> R { self.tree .with_backend_observation(|backend| backend.with_authority_observation(callback)) } fn fixed( &mut self, root: &os_k820_tree::Root, bundle: &OsK880Root, ) -> Result<(), os_k880_mapping::Error> { for index in 0..2 { let raw = unsafe { read_volatile((self.kernel_root as *const u64).add(index)) }; if raw & 3 != 3 { return Err(os_k880_mapping::Error::Backing); } self.tree .share_table(root, (index as u64) << 30, 1, raw & OS_K880_PAGE_PA_MASK) .map_err(os_k880_tree_error)?; } for (index, va, permissions) in [ (0, mmu::USER_CODE_VA, os_k880_rx()), (1, mmu::USER_STACK_VA, os_k880_rw()), (2, 0x8000_5000, os_k880_rw()), ] { self.tree .map_reference( root, va, bundle.reference(index)?, os_k820_descriptor::Memory::Normal, permissions, ) .map_err(os_k880_tree_error)?; } Ok(()) } fn discard(&mut self, bundle: &mut OsK880Root) -> Result<(), os_k880_spaces::Error> { if let Some(root) = &bundle.root { self.tree .detach_all(root) .map_err(|_| os_k880_spaces::Error::Retirement)?; } if let Some(root) = bundle.root.take() { if let Err((_, root)) = self.tree.release_root(root) { bundle.root = Some(root); return Err(os_k880_spaces::Error::Retirement); } } for page in &mut bundle.pages { if let Some(token) = page.take() { if let Err((_, token)) = self.tree.release_page(token) { *page = Some(token); return Err(os_k880_spaces::Error::Retirement); } } } self.tree .audit() .map_err(|_| os_k880_spaces::Error::Backing) } } impl os_k880_spaces::Backend for OsK880SpaceBackend<'_> { type Root = OsK880Root; fn acquire_root(&mut self, owner: u64) -> Result { if self.retained.is_some() { return Err(os_k880_spaces::Error::Retirement); } let root = self .tree .new_draft_root() .map_err(|_| os_k880_spaces::Error::NoMemory)?; let mut bundle = OsK880Root { root: Some(root), pages: core::array::from_fn(|_| None), owner, }; let built = (|| { for page in &mut bundle.pages { *page = Some( self.tree .allocate_page() .map_err(|_| os_k880_spaces::Error::NoMemory)?, ); } os_k880_initialize_pages(&bundle).map_err(|_| os_k880_spaces::Error::Backing)?; self.fixed( bundle.root.as_ref().ok_or(os_k880_spaces::Error::Backing)?, &bundle, ) .map_err(|_| os_k880_spaces::Error::Backing)?; self.tree .publish_root(bundle.root.as_ref().ok_or(os_k880_spaces::Error::Backing)?) .map_err(|_| os_k880_spaces::Error::Backing)?; self.tree .audit() .map_err(|_| os_k880_spaces::Error::Backing) })(); if let Err(error) = built { if self.discard(&mut bundle).is_err() { self.retained = Some(bundle); return Err(os_k880_spaces::Error::Retirement); } return Err(error); } Ok(bundle) } fn root_address(&self, root: &OsK880Root) -> u64 { root.address() } fn validate_root(&self, root: &OsK880Root) -> Result<(), os_k880_spaces::Error> { if root.owner == 0 || root.address() == 0 || root.pages.iter().any(Option::is_none) { return Err(os_k880_spaces::Error::Backing); } if !self .tree .owned_records() .any(|row| row.root && row.address == root.address()) { return Err(os_k880_spaces::Error::Backing); } for page in root.pages.iter().flatten() { if !self .tree .owned_records() .any(|row| !row.table && row.address == page.address() && !row.quarantined) { return Err(os_k880_spaces::Error::Backing); } } self.tree .audit() .map_err(|_| os_k880_spaces::Error::Backing) } fn invalidate_recycled(&mut self, asid: u16, old: u64, new: u64) { if asid == 0 || old == 0 || new != old + 1 { park(); } os_k880_sync(0x11, asid as u64); } fn select(&mut self, root: &OsK880Root, asid: u16) -> Result<(), os_k880_spaces::Error> { self.validate_root(root)?; self.tree .activate( root.root.as_ref().ok_or(os_k880_spaces::Error::Backing)?, asid, ) .map_err(|_| os_k880_spaces::Error::Backing) } fn restore_kernel(&mut self) -> Result<(), os_k880_spaces::Error> { self.tree .restore_shared_root(self.kernel_root, self.geometry, 0) .map_err(|_| os_k880_spaces::Error::Backing) } fn retire_root( &mut self, mut root: OsK880Root, _asid: u16, _generation: u64, ) -> Result<(), (os_k880_spaces::Error, OsK880Root)> { match self.discard(&mut root) { Ok(()) => Ok(()), Err(error) => Err((error, root)), } } } struct OsK880Prepared { root: os_k820_tree::Root, map: [Option; 16], } struct OsK880Backend<'a> { spaces: os_k880_spaces::Spaces, 1, 1>, handle: os_k880_spaces::Handle, owner: os_k880_mapping::Owner, map: [Option; 16], prepares: u64, commits: u64, rollbacks: u64, fillers: [Option; 24], } impl OsK880Backend<'_> { fn observe( &self, callback: impl FnOnce(&OsK880SpaceBackend<'_>, &OsK880Root) -> R, ) -> Result { self.spaces .with_root_observation(self.handle, self.owner.id, callback) .map_err(os_k880_space_error) } fn transaction( &mut self, callback: impl FnOnce(&mut OsK880SpaceBackend<'_>, &mut OsK880Root) -> R, ) -> Result { unsafe { self.spaces .with_owned_root_transaction(self.handle, self.owner.id, callback) } .map_err(os_k880_space_error) } fn activate(&mut self) -> Result { self.spaces .activate(self.handle, self.owner.id) .map(|row| row.root | (row.asid as u64) << 48) .map_err(os_k880_space_error) } fn full_draft_control(&mut self) -> Result { let held = self.observe(|backend, _| 32usize - backend.tree.owned_records().count() - 3)?; if held > 24 { return Err(os_k880_mapping::Error::Backing); } for index in 0..held { let page = self.transaction(|backend, _| { backend.tree.allocate_page().map_err(os_k880_tree_error) })??; self.fillers[index] = Some(page); } Ok(held) } fn return_fillers(&mut self) -> Result<(), os_k880_mapping::Error> { for index in 0..24 { if let Some(page) = self.fillers[index].take() { let result = self.transaction(|backend, _| backend.tree.release_page(page))?; if let Err((error, page)) = result { self.fillers[index] = Some(page); return Err(os_k880_tree_error(error)); } } } Ok(()) } } impl os_k880_mapping::Backend for OsK880Backend<'_> { type FrameRef = os_k820_tree::PageRef; type Prepared = OsK880Prepared; fn owner(&self) -> os_k880_mapping::Owner { self.owner } fn validate_frame( &self, owner: os_k880_mapping::Owner, frame: Self::FrameRef, ) -> Result<(), os_k880_mapping::Error> { if owner != self.owner { return Err(os_k880_mapping::Error::Foreign); } self.observe(|backend, bundle| { if !bundle.pages[3..] .iter() .flatten() .any(|page| page.reference() == frame) { return Err(os_k880_mapping::Error::Backing); } backend.tree.audit().map_err(os_k880_tree_error) })? } fn current( &self, address: u64, ) -> Result>, os_k880_mapping::Error> { let expected = self .map .iter() .flatten() .find(|row| row.address == address) .map(|row| row.value); self.observe(|backend, bundle| { match backend.tree.walk( bundle .root .as_ref() .ok_or(os_k880_mapping::Error::Backing)?, address, ) { Ok(walked) => { let value = expected.ok_or(os_k880_mapping::Error::Backing)?; if walked.physical != value.frame.address() || walked.permissions != value.permissions || walked.memory != os_k820_descriptor::Memory::Normal { return Err(os_k880_mapping::Error::Backing); } Ok(expected) } Err(os_k820_tree::Error::Descriptor(os_k820_descriptor::Error::Unmapped)) if expected.is_none() => { Ok(None) } _ => Err(os_k880_mapping::Error::Backing), } })? } fn prepare( &mut self, changes: &[os_k880_mapping::Change], ) -> Result { self.prepares = self .prepares .checked_add(1) .ok_or(os_k880_mapping::Error::Generation)?; let mut map = self.map; for change in changes { if os_k880_mapping::Backend::current(self, change.virtual_address)? != change.before { return Err(os_k880_mapping::Error::Backing); } let slot = map .iter() .position(|row| row.is_some_and(|row| row.address == change.virtual_address)) .or_else(|| map.iter().position(Option::is_none)) .ok_or(os_k880_mapping::Error::Capacity)?; map[slot] = change.after.map(|value| OsK880Value { address: change.virtual_address, value, }); } let result = self.transaction(|backend, bundle| { let draft = backend.tree.new_draft_root().map_err(os_k880_tree_error)?; let built = (|| { backend.fixed(&draft, bundle)?; for row in map.iter().flatten() { backend .tree .map_reference( &draft, row.address, row.value.frame, os_k820_descriptor::Memory::Normal, row.value.permissions, ) .map_err(os_k880_tree_error)?; } backend.tree.audit().map_err(os_k880_tree_error) })(); if let Err(error) = built { if backend.tree.detach_all(&draft).is_err() { backend.retain_failed_root(draft, bundle.owner); } if let Err((_, draft)) = backend.tree.release_root(draft) { backend.retain_failed_root(draft, bundle.owner); } return Err(error); } Ok(OsK880Prepared { root: draft, map }) })?; if result.is_err() { self.rollbacks = self .rollbacks .checked_add(1) .ok_or(os_k880_mapping::Error::Generation)?; } result } fn commit(&mut self, prepared: Self::Prepared) { let OsK880Prepared { root: draft, map } = prepared; let asid = self .spaces .observe(self.handle, self.owner.id) .unwrap_or_else(|_| park()) .asid; self.transaction(|backend,bundle|{ use os_k880_spaces::Backend; backend.restore_kernel().unwrap_or_else(|_|park()); let old=bundle.root.as_ref().unwrap_or_else(||park()); backend.tree.quiesce(old).unwrap_or_else(|_|park()); os_k880_sync(0x12,old.address()); if backend.tree.publish_root(&draft).is_err() { backend.retain_failed_root(draft,bundle.owner); } if backend.tree.activate(&draft,asid).is_err() { backend.retain_failed_root(draft,bundle.owner); } let actual:u64;unsafe{asm!("mrs {actual}, TTBR0_EL1",actual=out(reg)actual,options(nomem,nostack,preserves_flags))}; if actual!=draft.address()|(asid as u64)<<48{backend.retain_failed_root(draft,bundle.owner);} os_k880_page_event(0x13,old.address(),actual); let old=bundle.root.replace(draft).unwrap_or_else(||park()); if backend.tree.detach_all(&old).is_err(){backend.retain_failed_root(old,bundle.owner);} if let Err((_,old))=backend.tree.release_root(old){backend.retain_failed_root(old,bundle.owner);} backend.tree.audit().unwrap_or_else(|_|park()); }).unwrap_or_else(|_|park()); self.map = map; self.commits = self.commits.checked_add(1).unwrap_or_else(|| park()); } } fn os_k880_rw() -> os_k820_descriptor::Permissions { os_k820_descriptor::Permissions { user: true, write: true, execute: false, } } fn os_k880_ro() -> os_k820_descriptor::Permissions { os_k820_descriptor::Permissions { user: true, write: false, execute: false, } } fn os_k880_rx() -> os_k820_descriptor::Permissions { os_k820_descriptor::Permissions { user: true, write: false, execute: true, } } fn os_k880_sync(kind: u64, address: u64) { unsafe { asm!("dsb ishst", options(nostack, preserves_flags)) }; os_k880_page_event(kind, address, 1); unsafe { asm!("tlbi vmalle1is", options(nostack, preserves_flags)) }; os_k880_page_event(kind, address, 2); unsafe { asm!("dsb ish", options(nostack, preserves_flags)) }; os_k880_page_event(kind, address, 3); unsafe { asm!("isb", options(nostack, preserves_flags)) }; os_k880_page_event(kind, address, 4); } use turkos_kernel::runtime::user_copy as os_k880_copy; type OsK880Map<'a> = os_k880_mapping::Mapping,16>; type OsK880Range<'r,'b> = os_k880_copy::PinnedRange<'r,OsK880Backend<'b>,16>; #[unsafe(no_mangle)] static mut OS_K880_EVIDENCE:[u64;128]=[0;128]; #[unsafe(no_mangle)] static mut OS_K880_USER_STATE:[u64;32]=[0;32]; #[unsafe(no_mangle)] static mut OS_K880_RECORDS:[u64;192]=[0;192]; #[unsafe(no_mangle)] static mut OS_K880_REGISTRY:[u64;192]=[0;192]; #[unsafe(no_mangle)] static mut OS_K880_MAP_RECORDS:[u64;96]=[0;96]; #[unsafe(no_mangle)] static mut OS_K880_OWNED_PAGES:[u64;19]=[0;19]; #[unsafe(no_mangle)] static mut OS_K880_AUTHORITY_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K880_AUTHORITY_LIVE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K880_AUTHORITY_AFTER:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K880_REFUSAL_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K880_REFUSAL_AFTER:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K880_READ_POSITIVE:[u8;8225]=[0;8225]; #[unsafe(no_mangle)] static mut OS_K880_READ_NEGATIVE:[u8;8225]=[0;8225]; #[unsafe(no_mangle)] static mut OS_K880_WRITE_SOURCE:[u8;7032]=[0;7032]; #[unsafe(no_mangle)] static mut OS_K880_SHORT_OUTPUT:[u8;32]=[0;32]; #[unsafe(no_mangle)] static mut OS_K880_LONG_OUTPUT:[u8;32]=[0;32]; #[unsafe(no_mangle)] static mut OS_K880_LONG_SOURCE:[u8;65568]=[0;65568]; unsafe extern "C"{static OS_K880_RACE_OUTPUT:u8;} global_asm!(".global OS_K880_RACE_OUTPUT\n.set OS_K880_RACE_OUTPUT, OS_K880_LONG_SOURCE"); #[unsafe(no_mangle)] static mut OS_K880_STRING_WORKSPACE:os_k880_copy::StringWorkspace<65536>=os_k880_copy::StringWorkspace::new(); #[unsafe(no_mangle)] static mut OS_K880_CAPTURE_GATE:u8=0; #[unsafe(no_mangle)] static mut OS_K880_MAPPING_NONCE:u64=1; #[unsafe(no_mangle)] static mut OS_K880_LAST_TIMER:u64=0; static mut OS_K880_ACTIVE_RANGE:*mut OsK880Range<'static,'static>=core::ptr::null_mut(); static mut OS_K880_TASKS:*mut OsK660Tasks=core::ptr::null_mut(); static mut OS_K880_MAPPING_STORAGE:core::mem::MaybeUninit>=core::mem::MaybeUninit::uninit(); static mut OS_K880_BACKEND_STORAGE:core::mem::MaybeUninit>=core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] extern "C" fn __os_k880_mapping_identity()->u64 { let daif:u64;unsafe{asm!("mrs {value}, DAIF",value=out(reg)daif,options(nomem,nostack,preserves_flags))}; if daif&0x80==0{return 0;} let next=unsafe{read_volatile(core::ptr::addr_of!(OS_K880_MAPPING_NONCE))}; let Some(after)=next.checked_add(1) else{return 0;}; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_MAPPING_NONCE),after)};next } fn os_k880_word(index:usize,value:u64){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_EVIDENCE).cast::().add(index),value)};} fn os_k880_buffer(pointer:*mut u8,bytes:usize,value:u8){for i in 0..bytes{unsafe{write_volatile(pointer.add(i),value)};}} fn os_k880_copy_error(_:os_k880_copy::Error)->&'static str{"user-copy-contract"} fn os_k880_mapping_error(_:os_k880_mapping::Error)->&'static str{"user-copy-mapping"} fn os_k880_window()->Result{os_k880_copy::UserWindow::new(OS_K880_DATA,OS_K880_WINDOW_END,39).map_err(os_k880_copy_error)} fn os_k880_inventory(map:&OsK880Map<'_>,target:*mut u64)->Result<(),&'static str>{map.with_backend_observation(|backend|backend.observe(|b,_|b.authority(|a|os_k820_authority(a,target)))).map_err(os_k880_mapping_error)} // SAFETY: each inert reference is checked against the actual private retained // Page owner, Tree metadata generation, central lease and live Normal PTE by // Mapping::pin. Kernel aliases are identity-mapped owned RAM and stay retained. unsafe impl os_k880_copy::PinnedMemory for OsK880Backend<'_>{ unsafe fn read_ram(&self,frame:os_k820_tree::PageRef,offset:usize,destination:&mut[u8]){ if offset.checked_add(destination.len()).is_none_or(|end|end>4096)||os_k880_mapping::Backend::validate_frame(self,self.owner,frame).is_err(){park();} for(i,byte)in destination.iter_mut().enumerate(){*byte=unsafe{read_volatile((frame.address() as *const u8).add(offset+i))};} } unsafe fn write_ram(&self,frame:os_k820_tree::PageRef,offset:usize,source:&[u8]){ if offset.checked_add(source.len()).is_none_or(|end|end>4096)||os_k880_mapping::Backend::validate_frame(self,self.owner,frame).is_err(){park();} for(i,byte)in source.iter().enumerate(){unsafe{write_volatile((frame.address() as *mut u8).add(offset+i),*byte)};} } } fn os_k880_initialize_pages(bundle:&OsK880Root)->Result<(),os_k880_mapping::Error>{ let page=bundle.pages[0].as_ref().ok_or(os_k880_mapping::Error::Backing)?; unsafe extern "C"{static __os_k880_user_start:u8;static __os_k880_user_end:u8;} os_k820_code(page,core::ptr::addr_of!(__os_k880_user_start),core::ptr::addr_of!(__os_k880_user_end)).map_err(|_|os_k880_mapping::Error::Backing)?; for page in bundle.pages[1..].iter().flatten(){for offset in 0..4096{if unsafe{read_volatile((page.address() as *const u8).add(offset))}!=0{return Err(os_k880_mapping::Error::Backing);}}} Ok(()) } fn os_k880_snapshot(map:&OsK880Map<'_>)->Result<(),&'static str>{ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_LAST_TIMER),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)))}; let(ttbr,tcr,daif,first,third):(u64,u64,u64,u64,u64);unsafe{asm!("mrs {ttbr}, TTBR0_EL1","mrs {tcr}, TCR_EL1","mrs {daif}, DAIF","at s1e1r, {address}","isb","mrs {first}, PAR_EL1",ttbr=out(reg)ttbr,tcr=out(reg)tcr,daif=out(reg)daif,address=in(reg)OS_K880_DATA,first=out(reg)first,options(nostack,preserves_flags));asm!("at s1e1r, {address}","isb","mrs {third}, PAR_EL1",address=in(reg)(OS_K880_DATA+8192),third=out(reg)third,options(nostack,preserves_flags))}; for(i,value)in [(35,ttbr),(36,tcr),(37,daif),(38,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}),(68,first),(69,third)]{os_k880_word(i,value);} unsafe{core::ptr::write_bytes(core::ptr::addr_of_mut!(OS_K880_RECORDS).cast::(),0,192);core::ptr::write_bytes(core::ptr::addr_of_mut!(OS_K880_REGISTRY).cast::(),0,192);core::ptr::write_bytes(core::ptr::addr_of_mut!(OS_K880_MAP_RECORDS).cast::(),0,96)}; for(index,(va,value,generation,pins))in map.records().enumerate(){ let permissions=value.map_or(0,|v|1|(v.permissions.write as u64)<<1|(v.permissions.execute as u64)<<2); let address=value.map_or(0,|v|v.frame.address()); for(i,v)in [va,address,generation,pins as u64,permissions,1].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_MAP_RECORDS).cast::().add(index*6+i),v)};} } os_k880_word(31,map.records().count() as u64); map.with_backend_observation(|backend|backend.observe(|b,bundle|{ let mut count=0; for(index,r)in b.tree.owned_records().enumerate(){for(i,v)in [r.address,r.generation,r.table as u64,r.root as u64,r.quarantined as u64,r.mappings as u64].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_RECORDS).cast::().add(index*6+i),v)};}count+=1;} os_k880_word(32,count);os_k880_word(11,bundle.address()); for(index,page)in bundle.pages.iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_OWNED_PAGES).cast::().add(index),page.as_ref().map_or(0,|p|p.address()))};} b.tree.with_backend_observation(|p|{for(index,(r,central_generation))in p.observed_handles().enumerate(){for(i,v)in [r.address,r.owner,r.generation,r.pins as u64,r.quarantined as u64,central_generation].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_REGISTRY).cast::().add(index*6+i),v)};}}}); b.authority(|authority|os_k820_authority(authority,core::ptr::addr_of_mut!(OS_K880_AUTHORITY_LIVE).cast())); })).map_err(os_k880_mapping_error) } fn os_k880_capture(stage:u64)->Result<(),&'static str>{ os_k880_word(29,stage);unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_CAPTURE_GATE),0)}; write_str("TURKOS/OS/K880: CAPTURE STAGE=0x");write_hex(stage as usize);write_line(" READY"); let(start,frequency):(u64,u64);unsafe{asm!("mrs {start}, CNTPCT_EL0","mrs {frequency}, CNTFRQ_EL0",start=out(reg)start,frequency=out(reg)frequency,options(nomem,nostack,preserves_flags))}; let end=start.checked_add(frequency.checked_mul(90).ok_or("user-copy-frequency")?).ok_or("user-copy-deadline")?; while unsafe{read_volatile(core::ptr::addr_of!(OS_K880_CAPTURE_GATE))}!=1{let now:u64;unsafe{asm!("mrs {now}, CNTPCT_EL0",now=out(reg)now,options(nomem,nostack,preserves_flags))};if now>=end{return Err("user-copy-capture-timeout");}core::hint::spin_loop();}Ok(()) } fn os_k880_user(root:u64,mode:u64,entry:u64,stack:u64,kernel:u64)->Result<(),&'static str>{ let state=core::ptr::addr_of_mut!(OS_K880_USER_STATE).cast::(); unsafe{write_volatile(state,mode);write_volatile(state.add(2),kernel);write_volatile(state.add(3),0);write_volatile(state.add(4),root);write_volatile(state.add(5),stack)}; unsafe extern "C"{fn __os_k880_enter_user(root:u64,entry:u64,stack:u64);} unsafe{__os_k880_enter_user(root,entry,stack)}; if unsafe{read_volatile(state.add(3))}!=mode{return Err("user-copy-user-svc-missing");}Ok(()) } fn os_k880_expected_user_trap(slot:usize,_esr:usize,_elr:usize,frame:*const ExceptionFrame)->bool{ let state=core::ptr::addr_of_mut!(OS_K880_USER_STATE).cast::();let mode=unsafe{read_volatile(state)};if mode==0{return false;} if slot!=8||frame.is_null()||!unsafe{&*frame}.canary_valid(){park();} let(esr,far,elr,ttbr,sp,spsr):(u64,u64,u64,u64,u64,u64);unsafe{asm!("mrs {esr}, ESR_EL1","mrs {far}, FAR_EL1","mrs {elr}, ELR_EL1","mrs {ttbr}, TTBR0_EL1","mrs {sp}, SP_EL0","mrs {spsr}, SPSR_EL1",esr=out(reg)esr,far=out(reg)far,elr=out(reg)elr,ttbr=out(reg)ttbr,sp=out(reg)sp,spsr=out(reg)spsr,options(nomem,nostack,preserves_flags))}; let saved=unsafe{&*frame};if esr>>26&63!=0x15||esr&65535!=(0x8800+mode-1)||ttbr!=unsafe{read_volatile(state.add(4))}||sp!=unsafe{read_volatile(state.add(5))}||spsr&15!=0{park();} let at=if mode==1{40}else{56}; for(i,v)in [esr,far,elr,ttbr,sp,spsr,slot as u64,frame as u64,saved.registers[0] as u64,saved.registers[1] as u64,saved.registers[8] as u64,mode].into_iter().enumerate(){os_k880_word(at+i,v);} if mode==2{ let range=unsafe{read_volatile(core::ptr::addr_of!(OS_K880_ACTIVE_RANGE))};let tasks=unsafe{read_volatile(core::ptr::addr_of!(OS_K880_TASKS))}; if range.is_null()||tasks.is_null()||saved.registers[0] as u64!=OS_K880_DATA||saved.registers[1]!=16||saved.registers[8]!=2{park();} let id=unsafe{(&*tasks).scheduler.running()}.unwrap_or_else(||park()); if id.slot as u64!=unsafe{read_volatile(core::ptr::addr_of!(OS_K880_EVIDENCE).cast::().add(20))}||id.generation!=unsafe{read_volatile(core::ptr::addr_of!(OS_K880_EVIDENCE).cast::().add(21))}{park();} os_k880_word(70,id.slot as u64);os_k880_word(71,id.generation); let range=unsafe{&mut *range};let owner=os_k880_mapping::Owner{id:OS_K880_OWNER,generation:unsafe{read_volatile(core::ptr::addr_of!(OS_K880_EVIDENCE).cast::().add(9))}}; if range.attempt_unmap(owner,OS_K880_DATA,16)!=Err(os_k880_mapping::Error::Busy)||range.attempt_protect(owner,OS_K880_DATA,16,os_k880_ro())!=Err(os_k880_mapping::Error::Busy){park();} os_k880_word(22,1);os_k880_word(23,1); let backing=unsafe{range.attempt_backing_request(|backend|{os_k880_word(34,1);backend.transaction(|b,bundle|{let page=bundle.pages[3].take().unwrap_or_else(||park());if let Err((_,page))=b.tree.release_page(page){bundle.pages[3]=Some(page);}})})}; if backing!=Err(os_k880_mapping::Error::Busy){park();}os_k880_word(33,1); } unsafe{write_volatile(state,0);write_volatile(state.add(3),mode)}; unsafe extern "C"{fn __os_k880_resume_kernel()->!;} unsafe{__os_k880_resume_kernel()} } fn os_k880_timer_progress()->Result<(),&'static str>{ let frequency:u64;unsafe{asm!("mrs {frequency}, CNTFRQ_EL0",frequency=out(reg)frequency,options(nomem,nostack,preserves_flags))}; let interval=turkos_kernel::irq::timer_ticks_for_ns(frequency,1_000_000).ok_or("user-copy-timer-interval")?; for _ in 0..4{let before=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))};let now:u64;unsafe{asm!("mrs {now}, CNTPCT_EL0",now=out(reg)now,options(nomem,nostack,preserves_flags))};let deadline=now.checked_add(interval).ok_or("user-copy-timer-deadline")?;let end=now.checked_add(frequency).ok_or("user-copy-timer-timeout")?; unsafe{asm!("msr CNTP_CVAL_EL0, {deadline}","msr CNTP_CTL_EL0, {enable}","isb","msr DAIFClr, #2",deadline=in(reg)deadline,enable=in(reg)1u64,options(nostack,preserves_flags))}; while unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}==before{let current:u64;unsafe{asm!("mrs {current}, CNTPCT_EL0",current=out(reg)current,options(nomem,nostack,preserves_flags))};if current>end{unsafe{asm!("msr DAIFSet, #2","msr CNTP_CTL_EL0, xzr","isb",options(nostack,preserves_flags))};return Err("user-copy-timer-timeout");}} unsafe{asm!("msr DAIFSet, #2","isb",options(nostack,preserves_flags))};if unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}!=before+1||unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER))}(destination:*mut OsK880Map<'a>,authority:&'a mut os_k720_frame::Allocator,tables:&'a mmu::Tables,tcr:u64,pool:u64)->Result<(os_k880_mapping::Owner,[os_k820_tree::PageRef;16]),&'static str>{ let page=OsK880PageBackend::new(authority,tables)?;let identity=page.identity(); let geometry=os_k820_descriptor::Geometry::new(39,40).map_err(|_|"user-copy-geometry")?; let tree=OsK880Tree::new(identity,geometry,page).map_err(|_|"user-copy-tree")?; let mmfr0:u64;unsafe{asm!("mrs {value}, ID_AA64MMFR0_EL1",value=out(reg)mmfr0,options(nomem,nostack,preserves_flags))}; let config=os_k880_spaces::Configuration::from_registers(mmfr0,tcr).map_err(|_|"user-copy-space-config")?; let backend=OsK880SpaceBackend{tree,kernel_root:tables.root,geometry,retained:None}; let mut spaces=os_k880_spaces::Spaces::<_,1,1>::new(identity,config,backend).map_err(|_|"user-copy-spaces")?; let handle=spaces.create(OS_K880_OWNER).map_err(|_|"user-copy-space-acquire")?; let owner=os_k880_mapping::Owner{id:handle.owner(),generation:handle.generation()}; let backend=OsK880Backend{spaces,handle,owner,map:[None;16],prepares:0,commits:0,rollbacks:0,fillers:core::array::from_fn(|_|None)}; let actual_pool=backend.observe(|_,bundle|bundle.address()).map_err(os_k880_mapping_error)?;if actual_pool!=pool{return Err("user-copy-pool-binding");}os_k880_word(30,actual_pool); let frames=backend.observe(|_,bundle|core::array::from_fn(|index|bundle.reference(index+3).unwrap_or_else(|_|park()))).map_err(os_k880_mapping_error)?; let window=os_k880_mapping::Window::new(OS_K880_DATA,OS_K880_WINDOW_END,39).map_err(os_k880_mapping_error)?; unsafe{OsK880Map::initialize_at(destination,backend,0x880,window)}.map_err(os_k880_mapping_error)?;Ok((owner,frames)) } #[inline(never)] fn os_k880_product_configuration()->Result<(),&'static str>{ unsafe extern "C"{static __os_k880_user_start:u8;static __os_k880_beta_start:u8;static __os_k880_user_end:u8;} // Read-only startup state retains the zero-sized-work product evidence // objects through linker GC. No user frame, copy pin or allocation is made. let pointers=[ core::ptr::addr_of!(OS_K880_EVIDENCE).cast::(),core::ptr::addr_of!(OS_K880_USER_STATE).cast(),core::ptr::addr_of!(OS_K880_RECORDS).cast(),core::ptr::addr_of!(OS_K880_REGISTRY).cast(), core::ptr::addr_of!(OS_K880_MAP_RECORDS).cast(),core::ptr::addr_of!(OS_K880_OWNED_PAGES).cast(),core::ptr::addr_of!(OS_K880_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K880_AUTHORITY_LIVE).cast(),core::ptr::addr_of!(OS_K880_AUTHORITY_AFTER).cast(), core::ptr::addr_of!(OS_K880_REFUSAL_BEFORE).cast(),core::ptr::addr_of!(OS_K880_REFUSAL_AFTER).cast(),core::ptr::addr_of!(OS_K880_READ_POSITIVE).cast(),core::ptr::addr_of!(OS_K880_READ_NEGATIVE).cast(), core::ptr::addr_of!(OS_K880_WRITE_SOURCE).cast(),core::ptr::addr_of!(OS_K880_SHORT_OUTPUT).cast(),core::ptr::addr_of!(OS_K880_LONG_OUTPUT).cast(),core::ptr::addr_of!(OS_K880_LONG_SOURCE).cast(),core::ptr::addr_of!(OS_K880_RACE_OUTPUT).cast(), core::ptr::addr_of!(OS_K880_STRING_WORKSPACE).cast(),core::ptr::addr_of!(OS_K880_CAPTURE_GATE),core::ptr::addr_of!(OS_K880_PAGE_EVENTS).cast(),core::ptr::addr_of!(OS_K880_PAGE_EVENT_COUNT).cast(), core::ptr::addr_of!(OS_K880_LAST_TIMER).cast(), core::ptr::addr_of!(__os_k880_user_start),core::ptr::addr_of!(__os_k880_beta_start),core::ptr::addr_of!(__os_k880_user_end), ]; for pointer in &pointers[..23]{if unsafe{read_volatile(*pointer)}!=0{return Err("user-copy-product-configuration");}} for pointer in &pointers[23..]{let _instruction=unsafe{read_volatile(*pointer)};} if unsafe{read_volatile(core::ptr::addr_of!(OS_K880_MAPPING_NONCE))}!=1{return Err("user-copy-product-nonce");}Ok(()) } #[inline(never)] fn os_k880_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,_boot:&BootData)->Result<(),&'static str>{ #[cfg(feature="os-runtime-product")]{ if !os_k1100_product_probe_enabled(_boot)? { let _=(runtime,tables);os_k880_product_configuration()?; write_line("TURKOS/OS/K880: PRODUCT-CONTROL NO-USER-COPY-PROBE");return Ok(()); } } #[cfg(any(feature="os-runtime-acceptance",feature="os-runtime-product"))]{ let(_,memory,_,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?;let alpha=tasks.id.ok_or("user-copy-task-alpha")?; // K740 retained the replacement worker in Ready after retiring B. if tasks.scheduler.running().is_none()&&tasks.scheduler.run_next()!=Ok(Some(alpha)){return Err("user-copy-alpha-selection");} if tasks.scheduler.running()!=Some(alpha){return Err("user-copy-task-alpha-running");} let(ttbr,tcr,daif):(u64,u64,u64);unsafe{asm!("mrs {ttbr}, TTBR0_EL1","mrs {tcr}, TCR_EL1","mrs {daif}, DAIF",ttbr=out(reg)ttbr,tcr=out(reg)tcr,daif=out(reg)daif,options(nomem,nostack,preserves_flags))}; if ttbr&OS_K880_PAGE_PA_MASK!=tables.root||tcr&63!=25||daif&0x80==0{return Err("user-copy-original-control");} os_k820_authority(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K880_AUTHORITY_BEFORE).cast()); for(i,v)in [(0,0x4b383830434f5059),(1,tables.root),(2,ttbr),(3,tcr),(4,daif),(5,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}),(18,alpha.slot as u64),(19,alpha.generation)]{os_k880_word(i,v);} let pool=memory.allocator.os_authority().free_regions().iter().rev().find(|r|r.end-r.start>=40*4096).ok_or("user-copy-pool-before")?.end-36*4096; os_k880_word(30,pool);os_k880_word(35,ttbr);os_k880_word(36,tcr);os_k880_word(37,daif);os_k880_word(38,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_LAST_TIMER),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)))}; os_k880_capture(0)?; let storage=core::ptr::addr_of_mut!(OS_K880_MAPPING_STORAGE).cast::>(); let(owner,frames)=os_k880_initialize(storage,memory.allocator.os_authority_mut(),tables,tcr,pool)?; let mut map=unsafe{&mut *storage}; map.map(owner,OS_K880_DATA,&frames,os_k880_rw()).map_err(os_k880_mapping_error)?; let root=map.with_backend_observation(|b|b.spaces.observe(b.handle,b.owner.id)).map_err(|_|"user-copy-root-observation")?; os_k880_word(8,owner.id);os_k880_word(9,owner.generation);os_k880_word(10,root.asid as u64); let user_root=root.root|(root.asid as u64)<<48; os_k880_user(user_root,1,mmu::USER_CODE_VA,mmu::USER_STACK_TOP,tables.root)?;os_k880_word(25,mmu::USER_STACK_TOP); let positive=core::ptr::addr_of_mut!(OS_K880_READ_POSITIVE).cast::();let negative=core::ptr::addr_of_mut!(OS_K880_READ_NEGATIVE).cast::(); os_k880_buffer(positive,8225,0xcc);os_k880_buffer(negative,8225,0xcc); os_k880_copy::copy_from_user(&mut map,owner,os_k880_window()?,OS_K880_DATA+4095,unsafe{core::slice::from_raw_parts_mut(positive.add(16),8193)}).map_err(os_k880_copy_error)?; for i in 0..8193{let expected=if i==0{0xa0}else if i<=4096{0xa1}else{0xa2};if unsafe{read_volatile(positive.add(16+i))}!=expected{return Err("user-copy-three-page-read");}} os_k880_word(12,8193); map.unmap(owner,OS_K880_DATA+8192,1).map_err(os_k880_mapping_error)?; os_k880_inventory(&map,core::ptr::addr_of_mut!(OS_K880_REFUSAL_BEFORE).cast())?; if os_k880_copy::copy_from_user(&mut map,owner,os_k880_window()?,OS_K880_DATA+4095,unsafe{core::slice::from_raw_parts_mut(negative.add(16),8193)})!=Err(os_k880_copy::Error::Mapping(os_k880_mapping::Error::Unmapped)){return Err("user-copy-missing-last-error");} os_k880_inventory(&map,core::ptr::addr_of_mut!(OS_K880_REFUSAL_AFTER).cast())?; if (0..8225).any(|i|unsafe{read_volatile(negative.add(i))}!=0xcc)||unsafe{&*core::ptr::addr_of!(OS_K880_REFUSAL_BEFORE)}!=unsafe{&*core::ptr::addr_of!(OS_K880_REFUSAL_AFTER)}{return Err("user-copy-read-prefix-or-owner");} os_k880_word(16,1);os_k880_snapshot(&map)?;os_k880_capture(1)?; map.map(owner,OS_K880_DATA+8192,&frames[2..3],os_k880_rw()).map_err(os_k880_mapping_error)?; map.protect(owner,OS_K880_DATA+4096,1,os_k880_rx()).map_err(os_k880_mapping_error)?; let source=core::ptr::addr_of_mut!(OS_K880_WRITE_SOURCE).cast::();os_k880_buffer(source,7032,0xcc);os_k880_buffer(unsafe{source.add(16)},7000,0x5a); os_k880_snapshot(&map)?;os_k880_inventory(&map,core::ptr::addr_of_mut!(OS_K880_REFUSAL_BEFORE).cast())?;os_k880_capture(2)?; if os_k880_copy::copy_to_user(&mut map,owner,os_k880_window()?,OS_K880_DATA+333,unsafe{core::slice::from_raw_parts(source.add(16),7000)})!=Err(os_k880_copy::Error::Mapping(os_k880_mapping::Error::Permissions)){return Err("user-copy-rx-last-error");} os_k880_inventory(&map,core::ptr::addr_of_mut!(OS_K880_REFUSAL_AFTER).cast())?; if unsafe{&*core::ptr::addr_of!(OS_K880_REFUSAL_BEFORE)}!=unsafe{&*core::ptr::addr_of!(OS_K880_REFUSAL_AFTER)}{return Err("user-copy-write-refusal-owner");} os_k880_word(17,1);os_k880_snapshot(&map)?;os_k880_capture(3)?; map.protect(owner,OS_K880_DATA+4096,1,os_k880_rw()).map_err(os_k880_mapping_error)?; os_k880_copy::copy_to_user(&mut map,owner,os_k880_window()?,OS_K880_DATA+333,unsafe{core::slice::from_raw_parts(source.add(16),7000)}).map_err(os_k880_copy_error)?;os_k880_word(13,7000); os_k880_snapshot(&map)?;os_k880_capture(4)?; os_k880_buffer(core::ptr::addr_of_mut!(OS_K880_SHORT_OUTPUT).cast(),32,0xcc);os_k880_buffer(core::ptr::addr_of_mut!(OS_K880_LONG_OUTPUT).cast(),32,0xcc); os_k880_copy::copy_to_user(&mut map,owner,os_k880_window()?,OS_K880_DATA+4093,b"turkos\0").map_err(os_k880_copy_error)?; let workspace=unsafe{&mut *core::ptr::addr_of_mut!(OS_K880_STRING_WORKSPACE)}; let short=unsafe{core::slice::from_raw_parts_mut(core::ptr::addr_of_mut!(OS_K880_SHORT_OUTPUT).cast::(),32)}; if os_k880_copy::copy_cstr(&mut map,owner,os_k880_window()?,OS_K880_DATA+4093,32,short,workspace)!=Ok(6)||&short[..6]!=b"turkos"||short[6..].iter().any(|v|*v!=0xcc){return Err("user-copy-short-string");}os_k880_word(14,6); let long_source=core::ptr::addr_of_mut!(OS_K880_LONG_SOURCE).cast::();os_k880_buffer(long_source,65568,0xcc);os_k880_buffer(unsafe{long_source.add(16)},65536,0x7e); os_k880_copy::copy_to_user(&mut map,owner,os_k880_window()?,OS_K880_DATA,unsafe{core::slice::from_raw_parts(long_source.add(16),65536)}).map_err(os_k880_copy_error)?; let long=unsafe{core::slice::from_raw_parts_mut(core::ptr::addr_of_mut!(OS_K880_LONG_OUTPUT).cast::(),32)}; if os_k880_copy::copy_cstr(&mut map,owner,os_k880_window()?,OS_K880_DATA,65536,long,workspace)!=Err(os_k880_copy::Error::NameTooLong)||long.iter().any(|v|*v!=0xcc){return Err("user-copy-bounded-string");}os_k880_word(15,1); os_k880_snapshot(&map)?;os_k880_capture(5)?; let beta=tasks.scheduler.spawn().map_err(|_|"user-copy-task-beta")?; if beta==alpha||beta.generation<=alpha.generation||tasks.scheduler.running()!=Some(alpha){return Err("user-copy-beta-preparation");} os_k880_word(20,beta.slot as u64);os_k880_word(21,beta.generation); let root=map.with_backend_observation(|b|b.spaces.observe(b.handle,b.owner.id)).map_err(|_|"user-copy-beta-root")?;let beta_root=root.root|(root.asid as u64)<<48; os_k880_word(27,root.root);os_k880_word(26,0x8000_6000); let mut range=os_k880_copy::PinnedRange::acquire(&mut map,owner,os_k880_window()?,OS_K880_DATA,65536,false).map_err(os_k880_copy_error)?; let before_dispatch=tasks.scheduler.running().ok_or("user-copy-alpha-before-dispatch")?; if before_dispatch!=alpha{return Err("user-copy-alpha-before-dispatch");} os_k880_word(72,before_dispatch.slot as u64);os_k880_word(73,before_dispatch.generation); range.with_mapping_observation(os_k880_snapshot)?;os_k880_capture(6)?; if tasks.scheduler.yield_current(false)!=Ok(Some(beta)){return Err("user-copy-beta-selection");} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_TASKS),tasks as *mut _);write_volatile(core::ptr::addr_of_mut!(OS_K880_ACTIVE_RANGE),(&mut range as *mut OsK880Range<'_,'_>).cast())}; unsafe extern "C"{static __os_k880_beta_start:u8;static __os_k880_user_start:u8;} let beta_entry=mmu::USER_CODE_VA+(core::ptr::addr_of!(__os_k880_beta_start) as u64-core::ptr::addr_of!(__os_k880_user_start) as u64); os_k880_user(beta_root,2,beta_entry,0x8000_6000,tables.root)?; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K880_ACTIVE_RANGE),core::ptr::null_mut());write_volatile(core::ptr::addr_of_mut!(OS_K880_TASKS),core::ptr::null_mut())}; if tasks.scheduler.exit_current(0x880)!=Ok(Some(alpha))||tasks.scheduler.reap(beta)!=Ok(0x880)||tasks.scheduler.running()!=Some(alpha){return Err("user-copy-beta-retire");} let after_dispatch=tasks.scheduler.running().ok_or("user-copy-alpha-after-dispatch")?; os_k880_word(74,after_dispatch.slot as u64);os_k880_word(75,after_dispatch.generation); os_k880_timer_progress()?;os_k880_word(6,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}); // The earlier full-range source has finished its immutable borrow. // Reuse that actual storage as the guarded destination, resetting it // before the pinned read so the captured result must change bytes. let race=core::ptr::addr_of!(OS_K880_RACE_OUTPUT) as *mut u8;os_k880_buffer(race,65568,0xcc); range.with_mapping_observation(os_k880_snapshot)?;os_k880_capture(7)?; range.read_into(unsafe{core::slice::from_raw_parts_mut(race.add(16),65536)}).map_err(os_k880_copy_error)?;drop(range); if (0..65536).any(|i|unsafe{read_volatile(race.add(16+i))}!=0x7e)||map.records().any(|row|row.3!=0){return Err("user-copy-race-result");}os_k880_word(24,16); os_k880_snapshot(&map)?;os_k880_capture(8)?; map.unmap(owner,OS_K880_DATA,16).map_err(os_k880_mapping_error)?; let destination=core::ptr::addr_of_mut!(OS_K880_BACKEND_STORAGE).cast::>(); unsafe{OsK880Map::extract_empty_backend_at(storage,destination)}.map_err(os_k880_mapping_error)?; let backend=unsafe{&mut *destination}; backend.spaces.restore_kernel().map_err(|_|"user-copy-final-root")?;backend.spaces.release(backend.handle,owner.id).map_err(|_|"user-copy-retirement")?; backend.spaces.with_backend_observation(|b|b.authority(|a|os_k820_authority(a,core::ptr::addr_of_mut!(OS_K880_AUTHORITY_AFTER).cast()))); if backend.spaces.owned_records().count()!=0||backend.spaces.with_backend_observation(|b|b.tree.owned_records().count())!=0||unsafe{&*core::ptr::addr_of!(OS_K880_AUTHORITY_BEFORE)}!=unsafe{&*core::ptr::addr_of!(OS_K880_AUTHORITY_AFTER)}{return Err("user-copy-authority-restoration");} let(after_ttbr,after_tcr):(u64,u64);unsafe{asm!("mrs {ttbr}, TTBR0_EL1","mrs {tcr}, TCR_EL1",ttbr=out(reg)after_ttbr,tcr=out(reg)after_tcr,options(nomem,nostack,preserves_flags))};if after_ttbr!=ttbr||after_tcr!=tcr{return Err("user-copy-register-restoration");}os_k880_word(7,1); unsafe{core::ptr::drop_in_place(destination)}; write_line("TURKOS/OS/K865: THREE-PAGE-8193-READ MISSING-LAST ZERO-KERNEL-PREFIX PASS");write_line("TURKOS/OS/K870: TWO-PAGE-WRITE RX-LAST ZERO-USER-PREFIX PASS");write_line("TURKOS/OS/K875: CROSS-PAGE-NUL 64K-ENAMETOOLONG BOUNDED-KERNEL-OUTPUT PASS");write_line("TURKOS/OS/K880: REAL-EL0-BETA-SVC UNMAP-PROTECT-FREE-BUSY LIVE-PINS FOUR-TIMER-IRQS COPY-COMPLETE EXACT-AUTHORITY-RESTORED PASS");Ok(()) } } global_asm!(r#" .section .text.os_k880_user,"ax" .balign 4 .global __os_k880_user_start __os_k880_user_start: movz x0,#0x8020,lsl #16 mov x1,#0xa0 mov x2,#16 1: mov x3,#4096 2: strb w1,[x0],#1 subs x3,x3,#1 b.ne 2b add x1,x1,#1 subs x2,x2,#1 b.ne 1b svc #0x8800 b . .global __os_k880_beta_start __os_k880_beta_start: movz x0,#0x8020,lsl #16 mov x1,#16 mov x8,#2 svc #0x8801 b . .global __os_k880_user_end __os_k880_user_end: .global __os_k880_enter_user __os_k880_enter_user: stp x19,x20,[sp,#-16]! stp x21,x22,[sp,#-16]! stp x23,x24,[sp,#-16]! stp x25,x26,[sp,#-16]! stp x27,x28,[sp,#-16]! stp x29,x30,[sp,#-16]! stp d8,d9,[sp,#-16]! stp d10,d11,[sp,#-16]! stp d12,d13,[sp,#-16]! stp d14,d15,[sp,#-16]! adrp x9,OS_K880_USER_STATE add x9,x9,:lo12:OS_K880_USER_STATE mov x10,sp str x10,[x9,#8] dsb ishst msr TTBR0_EL1,x0 isb msr SP_EL0,x2 msr ELR_EL1,x1 mov x0,#0x3c0 msr SPSR_EL1,x0 isb eret .global __os_k880_resume_kernel __os_k880_resume_kernel: adrp x9,OS_K880_USER_STATE add x9,x9,:lo12:OS_K880_USER_STATE ldr x10,[x9,#16] msr TTBR0_EL1,x10 isb ldr x10,[x9,#8] mov sp,x10 ldp d14,d15,[sp],#16 ldp d12,d13,[sp],#16 ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 ldp x27,x28,[sp],#16 ldp x25,x26,[sp],#16 ldp x23,x24,[sp],#16 ldp x21,x22,[sp],#16 ldp x19,x20,[sp],#16 ret "#); unsafe extern "C" { static OS_K880_STACK: u8; } global_asm!(".global OS_K880_STACK\n.set OS_K880_STACK, OS_K860_STACK"); #[unsafe(no_mangle)] static mut OS_K880_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K880_ERROR: Option<&'static str> = None; const OS_K880_LOW_CANARY: u64 = 0x4b38_3630_4c4f_5721; const OS_K880_HIGH_CANARY: u64 = 0x4b38_3630_4849_4748; fn os_k880_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K880_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } fn os_k880_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { unsafe extern "C" { static __stack_start: u8; static __stack_top: u8; } let guard = core::ptr::addr_of!(OS_K880_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_low = core::ptr::addr_of!(__stack_start) as u64; let original_top = core::ptr::addr_of!(__stack_top) as u64; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("user-copy-original-stack-range"); } let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_low + 8) as *const u64) }; if original_canary != 0x4b36_3335_5354_414b || original_second != !0x4b36_3335_5354_414bu64 { return Err("user-copy-original-stack-canaries"); } let previous_guard = unsafe { read_volatile(core::ptr::addr_of!(OS_K860_STACK_EVIDENCE).cast::().add(13)) }; if guard != previous_guard || guard % 4096 != 0 || guard < core::ptr::addr_of!(_image_start) as u64 || end > core::ptr::addr_of!(_image_end) as u64 || end > 0x8000_0000 || !tables.task_stack_guard_unmapped(guard) { return Err("user-copy-reused-stack-guard"); } if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("user-copy-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K880_LOW_CANARY); write_volatile(high as *mut u64, OS_K880_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K880_LOW_CANARY), (9, OS_K880_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("user-copy-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("user-copy-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k880_stack_word(slot, value); } unsafe extern "C" { fn __os_k880_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k880_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K880_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K880_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K880_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_low + 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("user-copy-stack-restoration"); } os_k880_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k880_stack_word(17, unsafe { read_volatile((original_low + 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K880: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); Ok(()) } else { Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K880_ERROR)) } .unwrap_or("user-copy-stack-return")) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k880_accept_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k880_stack_word(4, sp); let result = os_k880_guarded_work( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k880_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K880_ERROR), Some(reason)) }; 1 } } } fn os_k880_guarded_work(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str>{ os_k880_accept(runtime,tables,boot) } global_asm!( r#" .section .text.os_k880_stack,"ax" .balign 4 .global __os_k880_on_stack __os_k880_on_stack: adrp x9, OS_K880_STACK_EVIDENCE add x9, x9, :lo12:OS_K880_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K880_STACK add x9, x9, :lo12:OS_K880_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K880_STACK_EVIDENCE add x9, x9, :lo12:OS_K880_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k880_accept_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K880_STACK_EVIDENCE add x9, x9, :lo12:OS_K880_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret "# ); const OS_K900_PAGE_OWNER:u64=0x900; const OS_K900_PAGE_PA_MASK:u64=0x0000_ffff_ffff_f000; #[unsafe(no_mangle)] static mut OS_K900_PAGE_ALLOC_BUDGET:u64=u64::MAX; #[unsafe(no_mangle)] static mut OS_K900_PAGE_FAIL_RELEASE:u64=0; #[unsafe(no_mangle)] static mut OS_K900_PAGE_EVENT_COUNT:u64=0; fn os_k900_page_event(kind:u64,address:u64,value:u64){ let n=unsafe{read_volatile(core::ptr::addr_of!(OS_K900_PAGE_EVENT_COUNT))}; if n>=1365 {park();} unsafe{let p=core::ptr::addr_of_mut!(OS_K900_PAGE_EVENTS).cast::().add(n as usize*3);write_volatile(p,kind);write_volatile(p.add(1),address);write_volatile(p.add(2),value);write_volatile(core::ptr::addr_of_mut!(OS_K900_PAGE_EVENT_COUNT),n+1);} } struct OsK900PageFrames<'a>{authority:&'a mut os_k720_frame::Allocator,leases:[Option;32],addresses:[u64;32]} impl OsK900PageFrames<'_>{ fn validate(&self,address:u64)->Result<(),os_k740_metadata::Error>{ let lease=self.leases.iter().flatten().find(|lease|lease.address()==address).ok_or(os_k740_metadata::Error::BadAddress)?; self.authority.validate_lease(lease).map_err(|_|os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK900PageFrames<'_>{ fn allocate(&mut self)->Result{ let budget=unsafe{read_volatile(core::ptr::addr_of!(OS_K900_PAGE_ALLOC_BUDGET))}; if budget==0{return Err(os_k740_metadata::Error::NoMemory);} if budget!=u64::MAX {unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K900_PAGE_ALLOC_BUDGET),budget-1)};} let slot=self.leases.iter().position(Option::is_none).ok_or(os_k740_metadata::Error::Capacity)?; let lease=self.authority.allocate_at(self.addresses[slot]).map_err(|_|os_k740_metadata::Error::NoMemory)?; let address=lease.address();let generation=lease.generation();self.leases[slot]=Some(lease);os_k900_page_event(1,address,generation);Ok(address) } fn release(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{ self.validate(address)?; if unsafe{read_volatile(core::ptr::addr_of!(OS_K900_PAGE_FAIL_RELEASE))}==address {os_k900_page_event(9,address,0);return Err(os_k740_metadata::Error::Backend);} let slot=self.leases.iter().position(|lease|lease.as_ref().is_some_and(|lease|lease.address()==address)).ok_or(os_k740_metadata::Error::BadAddress)?; for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};} unsafe{asm!("dsb ish",options(nostack,preserves_flags))}; self.authority.release(self.leases[slot].as_ref().ok_or(os_k740_metadata::Error::Backend)?).map_err(|_|os_k740_metadata::Error::Backend)?; self.leases[slot]=None;os_k900_page_event(8,address,0);Ok(()) } fn zero_page(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{self.validate(address)?;for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};}Ok(())} fn read_byte(&self,address:u64,offset:usize)->Result{if offset>=4096{return Err(os_k740_metadata::Error::BadAddress);}self.validate(address)?;Ok(unsafe{read_volatile((address as *const u8).add(offset))})} fn available_pages(&self)->u64{self.authority.remaining_pages()} } pub struct OsK900PageBackend<'a>{identity:u64,registry:os_k740_metadata::Registry,32>,tables:&'a mmu::Tables} impl<'a> OsK900PageBackend<'a>{ pub fn new(authority:&'a mut os_k720_frame::Allocator,tables:&'a mmu::Tables)->Result{ let region=authority.free_regions().iter().rev().find(|r|r.end-r.start>=40*4096).ok_or("page-tables-interior-pool")?; let base=region.end-36*4096; let addresses=core::array::from_fn(|i|base+i as u64*4096); let frames=OsK900PageFrames{authority,leases:core::array::from_fn(|_|None),addresses}; unsafe extern "C"{fn __os_k760_workspace_identity()->u64;} let identity=unsafe{__os_k760_workspace_identity()};if identity==0{return Err("page-tables-registry-identity");} let registry=os_k740_metadata::Registry::new(identity,frames).map_err(|_|"page-tables-registry")?; Ok(Self{identity,registry,tables}) } pub fn identity(&self)->u64{self.identity} pub fn with_authority_observation(&self,callback:impl FnOnce(&os_k720_frame::Allocator)->R)->R{self.registry.with_backend_observation(|frames|callback(frames.authority))} pub fn registry_records(&self)->impl Iterator+'_ {self.registry.owned_records()} pub fn observed_handles(&self)->impl Iterator+'_ { self.registry.owned_records().map(|r|{let generation=self.registry.with_backend_observation(|frames|frames.leases.iter().flatten().find(|lease|lease.address()==r.address).map(|lease|lease.generation()).unwrap_or(0));(r,generation)}) } fn owned_kernel(&self,address:u64)->bool{self.tables.tagged_frames().any(|frame|frame.address==address)} fn shared_walk(&self,address:u64,level:u8,visited:&mut [u64;4],count:usize)->Result<(),os_k820_tree::Error>{ if level>3||count>=4||visited[..count].contains(&address)||!self.owned_kernel(address){return Err(os_k820_tree::Error::Shared);} visited[count]=address; for index in 0..512 { let raw=unsafe{read_volatile((address as *const u64).add(index))};if raw==0{continue;} match raw&3 { 3 if level<3=>self.shared_walk(raw&OS_K900_PAGE_PA_MASK,level+1,visited,count+1)?, 1 if level<3=>{if raw&(1<<6)!=0{return Err(os_k820_tree::Error::Shared);}}, 3 if level==3=>{if raw&(1<<6)!=0{return Err(os_k820_tree::Error::Shared);}}, _=>return Err(os_k820_tree::Error::Shared), } } Ok(()) } fn kernel_level(&self,address:u64)->Result{ let mut pending=[(0u64,0u8);24];pending[0]=(self.tables.root,1);let mut len=1;let mut at=0; while at{ type Handle=os_k740_metadata::Handle; fn allocate_zeroed(&mut self)->Result{let handle=self.registry.allocate_zeroed(OS_K900_PAGE_OWNER).map_err(|e|if e==os_k740_metadata::Error::NoMemory{os_k820_tree::Error::NoMemory}else{os_k820_tree::Error::Backend})?;os_k900_page_event(0x40,handle.address(),handle.generation());Ok(handle)} fn address(&self,handle:Self::Handle)->u64{handle.address()} fn validate(&self,handle:Self::Handle)->Result<(),os_k820_tree::Error>{self.registry.read_byte(handle,OS_K900_PAGE_OWNER,4095).map(|_|()).map_err(|_|os_k820_tree::Error::Stale)} fn release(&mut self,handle:Self::Handle)->Result<(),os_k820_tree::Error>{self.registry.release(handle,OS_K900_PAGE_OWNER).map_err(|_|os_k820_tree::Error::Backend)} fn entry(&self,handle:Self::Handle,index:usize)->Result{if index>=512{return Err(os_k820_tree::Error::Backend);}self.validate(handle)?;Ok(unsafe{read_volatile((handle.address() as *const u64).add(index))})} fn write_entry(&mut self,handle:Self::Handle,index:usize,value:u64)->Result<(),os_k820_tree::Error>{if index>=512{return Err(os_k820_tree::Error::Backend);}self.validate(handle)?;unsafe{write_volatile((handle.address() as *mut u64).add(index),value)};os_k900_page_event(2,handle.address()+index as u64*8,value);Ok(())} fn validate_shared_table(&self,address:u64)->Result<(),os_k820_tree::Error>{let level=self.kernel_level(address)?;self.shared_walk(address,level,&mut[0;4],0)} fn validate_shared_page(&self,_address:u64)->Result<(),os_k820_tree::Error>{Err(os_k820_tree::Error::Shared)} fn shared_entry(&self,address:u64,index:usize)->Result{if index>=512{return Err(os_k820_tree::Error::Shared);}self.validate_shared_table(address)?;Ok(unsafe{read_volatile((address as *const u64).add(index))})} fn active_root(&self)->u64{let value:u64;unsafe{asm!("mrs {value}, TTBR0_EL1",value=out(reg)value,options(nomem,nostack,preserves_flags))};value&OS_K900_PAGE_PA_MASK} fn invalidate(&mut self,root:u64,va:Option){ unsafe{asm!("dsb ishst",options(nostack,preserves_flags))};os_k900_page_event(3,root,va.unwrap_or(u64::MAX)); unsafe{asm!("tlbi vmalle1is",options(nostack,preserves_flags))};os_k900_page_event(4,root,va.unwrap_or(u64::MAX)); unsafe{asm!("dsb ish",options(nostack,preserves_flags))};os_k900_page_event(5,root,0);unsafe{asm!("isb",options(nostack,preserves_flags))};os_k900_page_event(6,root,0); } fn select_root(&mut self,root:u64,geometry:os_k820_descriptor::Geometry,asid:u16){ let old:u64;unsafe{asm!("mrs {old}, TCR_EL1",old=out(reg)old,options(nomem,nostack,preserves_flags))};let tcr=(old&!63)|(64-geometry.virtual_bits() as u64); let ttbr=root|(asid as u64)<<48; unsafe{asm!("dsb ishst","msr TCR_EL1, {tcr}","msr TTBR0_EL1, {ttbr}",tcr=in(reg)tcr,ttbr=in(reg)ttbr,options(nostack,preserves_flags))}; if tcr!=old {unsafe{asm!("tlbi vmalle1is","dsb ish",options(nostack,preserves_flags))};} unsafe{asm!("isb",options(nostack,preserves_flags))};os_k900_page_event(7,root,asid as u64); } } // Historical predecessor after-inventory is immutable and independently bound // to the newly observed checkpoint0 central ledger. Fatal before-inventory is // the completed normal after-inventory. Fatal after reuses the obsolete final // live scratch only after its complete physical checkpoint has been archived. unsafe extern "C" { static mut OS_K900_PAGE_EVENTS: [u64; 4096]; static mut OS_K900_AUTHORITY_BEFORE: [u64; 328]; static mut OS_K900_KERNEL_BEFORE: [u64; 328]; static mut OS_K900_KERNEL_AFTER: [u64; 328]; } global_asm!( ".global OS_K900_PAGE_EVENTS\n.set OS_K900_PAGE_EVENTS, OS_K880_PAGE_EVENTS\n.type OS_K900_PAGE_EVENTS, %object\n.size OS_K900_PAGE_EVENTS, 32768\n.global OS_K900_AUTHORITY_BEFORE\n.set OS_K900_AUTHORITY_BEFORE, OS_K880_AUTHORITY_AFTER\n.type OS_K900_AUTHORITY_BEFORE, %object\n.size OS_K900_AUTHORITY_BEFORE, 2624\n.global OS_K900_KERNEL_BEFORE\n.set OS_K900_KERNEL_BEFORE, OS_K900_AUTHORITY_AFTER\n.type OS_K900_KERNEL_BEFORE, %object\n.size OS_K900_KERNEL_BEFORE, 2624\n.global OS_K900_KERNEL_AFTER\n.set OS_K900_KERNEL_AFTER, OS_K900_AUTHORITY_LIVE\n.type OS_K900_KERNEL_AFTER, %object\n.size OS_K900_KERNEL_AFTER, 2624" ); use turkos_kernel::runtime::{ mapping as os_k900_mapping, page_fault_frame as os_k900_frame, page_faults::classify as os_k900_classify, page_faults::pager as os_k900_pager, }; type OsK900Tree<'a> = os_k820_tree::Tree, 32>; type OsK900Pager<'a> = os_k900_pager::Pager, 16>; const OS_K900_CODE: u64 = 0x8000_0000; const OS_K900_ANON: u64 = 0x8001_0000; const OS_K900_COW: u64 = 0x8001_4000; const OS_K900_USER_STACK: u64 = 0x8001_f000; const OS_K900_A: os_k900_mapping::Owner = os_k900_mapping::Owner { id: 0x9001, generation: 1, }; const OS_K900_B: os_k900_mapping::Owner = os_k900_mapping::Owner { id: 0x9002, generation: 1, }; #[derive(Clone, Copy, PartialEq, Eq)] struct OsK900Map { owner: os_k900_mapping::Owner, address: u64, value: os_k900_pager::Value, } struct OsK900Prepared { roots: [Option; 2], map: [Option; 16], } struct OsK900FaultBackend<'a> { tree: OsK900Tree<'a>, kernel: u64, geometry: os_k820_descriptor::Geometry, roots: [Option; 2], fixed: [Option; 2], map: [Option; 16], retained: [Option; 4], } fn os_k900_tree_error(error: os_k820_tree::Error) -> os_k900_pager::Error { match error { os_k820_tree::Error::NoMemory | os_k820_tree::Error::Capacity => { os_k900_pager::Error::NoMemory } os_k820_tree::Error::Retained => os_k900_pager::Error::Retained, _ => os_k900_pager::Error::Backend, } } fn os_k900_rw() -> os_k820_descriptor::Permissions { os_k900_pager::USER_RW } fn os_k900_rx() -> os_k820_descriptor::Permissions { os_k820_descriptor::Permissions { user: true, write: false, execute: true, } } impl<'a> OsK900FaultBackend<'a> { fn index(owner: os_k900_mapping::Owner) -> Result { if owner == OS_K900_A { Ok(0) } else if owner == OS_K900_B { Ok(1) } else { Err(os_k900_pager::Error::Foreign) } } fn fixed(&mut self, root: &os_k820_tree::Root) -> Result<(), os_k900_pager::Error> { for index in 0..2 { let raw = unsafe { read_volatile((self.kernel as *const u64).add(index)) }; if raw & 3 != 3 { return Err(os_k900_pager::Error::Backend); } self.tree .share_table(root, (index as u64) << 30, 1, raw & OS_K900_PAGE_PA_MASK) .map_err(os_k900_tree_error)?; } for (index, address, permissions) in [ (0, OS_K900_CODE, os_k900_rx()), (1, OS_K900_USER_STACK, os_k900_rw()), ] { self.tree .map_reference( root, address, self.fixed[index] .as_ref() .ok_or(os_k900_pager::Error::Backend)? .reference(), os_k820_descriptor::Memory::Normal, permissions, ) .map_err(os_k900_tree_error)?; } Ok(()) } fn authority(&self, callback: impl FnOnce(&os_k720_frame::Allocator) -> R) -> R { self.tree .with_backend_observation(|b| b.with_authority_observation(callback)) } fn activate(&mut self, owner: os_k900_mapping::Owner) -> Result<(), os_k900_pager::Error> { let i = Self::index(owner)?; self.tree .activate( self.roots[i] .as_ref() .ok_or(os_k900_pager::Error::Foreign)?, i as u16 + 1, ) .map_err(os_k900_tree_error) } fn restore(&mut self) -> Result<(), os_k900_pager::Error> { self.tree .restore_shared_root(self.kernel, self.geometry, 0) .map_err(os_k900_tree_error) } fn retain(&mut self, root: os_k820_tree::Root) { if let Some(slot) = self.retained.iter_mut().find(|r| r.is_none()) { *slot = Some(root); } else { let retained = root; loop { core::hint::black_box(&retained); unsafe { asm!("wfe", options(nomem, nostack, preserves_flags)) }; } } } fn discard(&mut self, root: os_k820_tree::Root) -> Result<(), os_k900_pager::Error> { if self.tree.detach_all(&root).is_err() { self.retain(root); return Err(os_k900_pager::Error::Retained); } if let Err((_, root)) = self.tree.release_root(root) { self.retain(root); return Err(os_k900_pager::Error::Retained); } Ok(()) } fn reap(&mut self) -> Result<(), os_k900_pager::Error> { self.tree.reap_quarantine().map_err(os_k900_tree_error)?; for i in 0..4 { if let Some(root) = self.retained[i].take() { self.discard(root)?; } } self.tree.audit().map_err(os_k900_tree_error) } fn finish(&mut self) -> Result<(), os_k900_pager::Error> { self.restore()?; for i in 0..2 { if let Some(root) = self.roots[i].take() { self.discard(root)?; } } for i in 0..2 { if let Some(page) = self.fixed[i].take() { if let Err((_, page)) = self.tree.release_page(page) { self.fixed[i] = Some(page); return Err(os_k900_pager::Error::Retained); } } } self.reap()?; if self.tree.owned_records().count() != 0 { return Err(os_k900_pager::Error::Retained); } Ok(()) } } impl os_k900_pager::Backend for OsK900FaultBackend<'_> { type Lease = os_k820_tree::Page; type Reference = os_k820_tree::PageRef; type Prepared = OsK900Prepared; fn current_owner(&self) -> Option { let raw: u64; unsafe { asm!("mrs {raw},TTBR0_EL1",raw=out(reg)raw,options(nomem,nostack,preserves_flags)) }; for (i, owner) in [OS_K900_A, OS_K900_B].into_iter().enumerate() { if self.roots[i] .as_ref() .is_some_and(|r| raw == r.address() | ((i as u64 + 1) << 48)) { return Some(owner); } } None } fn validate_owner(&self, owner: os_k900_mapping::Owner) -> Result<(), os_k900_pager::Error> { let i = Self::index(owner)?; self.tree .walk( self.roots[i] .as_ref() .ok_or(os_k900_pager::Error::Foreign)?, OS_K900_CODE, ) .map_err(os_k900_tree_error)?; Ok(()) } fn allocate_zeroed(&mut self) -> Result { self.tree.allocate_page().map_err(os_k900_tree_error) } fn reference(&self, page: &Self::Lease) -> Self::Reference { page.reference() } fn physical_page(&self, page: &Self::Lease) -> u64 { page.address() } fn validate(&self, page: &Self::Lease) -> Result<(), os_k900_pager::Error> { self.tree .os_k900_validate_page(page) .map_err(os_k900_tree_error) } fn release(&mut self, page: Self::Lease) -> Result<(), (os_k900_pager::Error, Self::Lease)> { self.tree .release_page(page) .map_err(|(e, p)| (os_k900_tree_error(e), p)) } fn initialize( &mut self, page: &Self::Lease, bytes: &[u8; 4096], ) -> Result<(), os_k900_pager::Error> { self.validate(page)?; for (i, b) in bytes.iter().enumerate() { unsafe { write_volatile((page.address() as *mut u8).add(i), *b) }; } unsafe { asm!("dsb ish", options(nostack, preserves_flags)) }; Ok(()) } fn copy_page( &mut self, source: &Self::Lease, target: &Self::Lease, ) -> Result<(), os_k900_pager::Error> { self.validate(source)?; self.validate(target)?; for i in 0..4096 { unsafe { write_volatile( (target.address() as *mut u8).add(i), read_volatile((source.address() as *const u8).add(i)), ) }; } unsafe { asm!("dsb ish", options(nostack, preserves_flags)) }; os_k900_page_event(0x20, source.address(), target.address()); Ok(()) } fn current( &self, owner: os_k900_mapping::Owner, address: u64, ) -> Result>, os_k900_pager::Error> { let i = Self::index(owner)?; let value = self .map .iter() .flatten() .find(|m| m.owner == owner && m.address == address) .map(|m| m.value); let walked = self.tree.walk( self.roots[i] .as_ref() .ok_or(os_k900_pager::Error::Foreign)?, address, ); match (value, walked) { (None, Err(os_k820_tree::Error::Descriptor(os_k820_descriptor::Error::Unmapped))) => { Ok(None) } (Some(v), Ok(t)) if t.physical == v.frame.address() && t.permissions == v.permissions => { self.tree .os_k900_validate_reference(v.frame) .map_err(os_k900_tree_error)?; Ok(Some(v)) } _ => Err(os_k900_pager::Error::Backend), } } fn prepare( &mut self, changes: &[os_k900_pager::Change], ) -> Result { if self.retained.iter().any(Option::is_some) { return Err(os_k900_pager::Error::Retained); } let mut map = self.map; let mut affected = [false; 2]; for c in changes { let i = Self::index(c.owner)?; if self.current(c.owner, c.address)? != c.before { return Err(os_k900_pager::Error::Conflict); } affected[i] = true; if let Some(v) = c.after { self.tree .os_k900_validate_reference(v.frame) .map_err(os_k900_tree_error)?; } let slot = map .iter() .position(|m| m.is_some_and(|m| m.owner == c.owner && m.address == c.address)); match (c.after, slot) { (Some(value), Some(s)) => { map[s] = Some(OsK900Map { owner: c.owner, address: c.address, value, }) } (Some(value), None) => { let s = map .iter() .position(Option::is_none) .ok_or(os_k900_pager::Error::Capacity)?; map[s] = Some(OsK900Map { owner: c.owner, address: c.address, value, }); } (None, Some(s)) => map[s] = None, (None, None) => {} } } let mut prepared = OsK900Prepared { roots: core::array::from_fn(|_| None), map, }; let built = (|| { for i in 0..2 { if affected[i] { let root = self.tree.new_draft_root().map_err(os_k900_tree_error)?; prepared.roots[i] = Some(root); let root = prepared.roots[i] .as_ref() .ok_or(os_k900_pager::Error::Backend)?; self.fixed(root)?; let owner = [OS_K900_A, OS_K900_B][i]; for m in map.iter().flatten().filter(|m| m.owner == owner) { self.tree .map_reference( root, m.address, m.value.frame, os_k820_descriptor::Memory::Normal, m.value.permissions, ) .map_err(os_k900_tree_error)?; } } } Ok(()) })(); if let Err(error) = built { let mut retained = false; for root in &mut prepared.roots { if let Some(root) = root.take() { retained |= self.discard(root).is_err(); } } return Err(if retained { os_k900_pager::Error::Retained } else { error }); } self.tree.audit().map_err(os_k900_tree_error)?; Ok(prepared) } fn commit(&mut self, mut prepared: Self::Prepared) { let active = self.current_owner(); if self.restore().is_err() { park(); } for i in 0..2 { if let Some(root) = prepared.roots[i].take() { let old = self.roots[i].take().unwrap_or_else(|| park()); if self.tree.quiesce(&old).is_err() { park(); } os_k900_page_sync(0x12, old.address()); if self.tree.publish_root(&root).is_err() { self.roots[i] = Some(old); self.retain(root); park(); } self.roots[i] = Some(root); if self.discard(old).is_err() { park(); } } } self.map = prepared.map; if let Some(owner) = active { if self.activate(owner).is_err() { park(); } } os_k900_page_event(0x21, 0, 1); } } fn os_k900_page_sync(kind: u64, address: u64) { unsafe { asm!("dsb ishst", options(nostack, preserves_flags)) }; os_k900_page_event(kind, address, 1); unsafe { asm!("tlbi vmalle1is", options(nostack, preserves_flags)) }; os_k900_page_event(kind, address, 2); unsafe { asm!("dsb ish", options(nostack, preserves_flags)) }; os_k900_page_event(kind, address, 3); unsafe { asm!("isb", options(nostack, preserves_flags)) }; os_k900_page_event(kind, address, 4); } impl os_k900_frame::ReturnAuthority for OsK900FaultBackend<'_> { fn owner_root(&self, user: os_k900_frame::User) -> Result<(), os_k900_frame::Error> { let owner = os_k900_mapping::Owner { id: user.owner, generation: user.generation, }; let i = Self::index(owner).map_err(|_| os_k900_frame::Error::Owner)?; if os_k900_pager::Backend::current_owner(self) != Some(owner) || user.asid != i as u16 + 1 || self.roots[i].as_ref().map(|r| r.address()) != Some(user.root) { return Err(os_k900_frame::Error::Root); } Ok(()) } fn executable( &self, user: os_k900_frame::User, entry: u64, ) -> Result<(), os_k900_frame::Error> { let i = Self::index(os_k900_mapping::Owner { id: user.owner, generation: user.generation, }) .map_err(|_| os_k900_frame::Error::Owner)?; let t = self .tree .walk( self.roots[i].as_ref().ok_or(os_k900_frame::Error::Root)?, entry, ) .map_err(|_| os_k900_frame::Error::Entry)?; if !t.permissions.user || !t.permissions.execute || t.permissions.write { return Err(os_k900_frame::Error::Entry); } Ok(()) } fn writable_stack( &self, user: os_k900_frame::User, sp: u64, ) -> Result<(), os_k900_frame::Error> { let i = Self::index(os_k900_mapping::Owner { id: user.owner, generation: user.generation, }) .map_err(|_| os_k900_frame::Error::Owner)?; let t = self .tree .walk( self.roots[i].as_ref().ok_or(os_k900_frame::Error::Root)?, sp, ) .map_err(|_| os_k900_frame::Error::Stack)?; if !t.permissions.user || !t.permissions.write || t.permissions.execute { return Err(os_k900_frame::Error::Stack); } Ok(()) } } #[unsafe(no_mangle)] static mut OS_K900_EVIDENCE: [u64; 48] = [0; 48]; #[unsafe(no_mangle)] static mut OS_K900_RECORDS: [u64; 72] = [0; 72]; #[unsafe(no_mangle)] static mut OS_K900_REGISTRY: [u64; 72] = [0; 72]; #[unsafe(no_mangle)] static mut OS_K900_OWNED: [u64; 24] = [0; 24]; #[unsafe(no_mangle)] static mut OS_K900_MAPPINGS: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K900_AUTHORITY_LIVE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K900_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K900_CAPTURE_GATE: u8 = 0; fn os_k900_word(index: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K900_EVIDENCE) .cast::() .add(index), value, ) }; } fn os_k900_snapshot(pager: &OsK900Pager<'_>) { for target in [ core::ptr::addr_of_mut!(OS_K900_RECORDS).cast::(), core::ptr::addr_of_mut!(OS_K900_REGISTRY).cast::(), ] { for i in 0..72 { unsafe { write_volatile(target.add(i), 0) }; } } for i in 0..32 { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K900_MAPPINGS) .cast::() .add(i), 0, ) }; } pager.with_backend_observation(|b| { b.authority(|a| { os_k820_authority(a, core::ptr::addr_of_mut!(OS_K900_AUTHORITY_LIVE).cast()) }); let mut count = 0; for (r, index) in b.tree.owned_records().zip(0..32) { if index >= 12 { park(); } let target = unsafe { core::ptr::addr_of_mut!(OS_K900_RECORDS) .cast::() .add(index * 6) }; for (i, v) in [ r.address, r.generation, r.table as u64, r.root as u64, r.quarantined as u64, r.mappings as u64, ] .into_iter() .enumerate() { unsafe { write_volatile(target.add(i), v) }; } count += 1; } os_k900_word(20, count); b.tree.with_backend_observation(|backend| { for (index, (r, g)) in backend.observed_handles().enumerate() { if index >= 12 { park(); } let target = unsafe { core::ptr::addr_of_mut!(OS_K900_REGISTRY) .cast::() .add(index * 6) }; for (i, v) in [ r.address, r.owner, r.generation, r.pins as u64, r.quarantined as u64, g, ] .into_iter() .enumerate() { unsafe { write_volatile(target.add(i), v) }; } } }); for i in 0..2 { os_k900_word(10 + i, b.roots[i].as_ref().map_or(0, |r| r.address())); } os_k900_word(12, b.fixed[0].as_ref().map_or(0, |p| p.address())); os_k900_word(13, b.fixed[1].as_ref().map_or(0, |p| p.address())); }); for i in 0..24 { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K900_OWNED).cast::().add(i), 0, ) }; } for (index, (page, references, quarantined)) in pager.owned_pages().enumerate() { if index >= 4 { park(); } let target = unsafe { core::ptr::addr_of_mut!(OS_K900_OWNED) .cast::() .add(index * 6) }; let generation = pager .with_backend_observation(|b| b.tree.os_k900_reference_generation(page)) .unwrap_or_else(|_| park()); for (i, v) in [ page.address(), references as u64, quarantined as u64, generation, 0, 0, ] .into_iter() .enumerate() { unsafe { write_volatile(target.add(i), v) }; } } let mut count = 0; for (owner, address, page, cow) in pager.mappings() { if count >= 4 { park(); } let target = unsafe { core::ptr::addr_of_mut!(OS_K900_MAPPINGS) .cast::() .add(count * 8) }; let (value, generation) = pager.with_backend_observation(|b| { ( os_k900_pager::Backend::current(b, owner, address) .unwrap_or_else(|_| park()) .unwrap_or_else(|| park()), b.tree .os_k900_reference_generation(page) .unwrap_or_else(|_| park()), ) }); let permissions = (value.permissions.user as u64) | ((value.permissions.write as u64) << 1) | ((value.permissions.execute as u64) << 2); for (i, v) in [ owner.id, owner.generation, address, page.address(), cow as u64, permissions, generation, 1, ] .into_iter() .enumerate() { unsafe { write_volatile(target.add(i), v) }; } count += 1; } os_k900_word(21, count as u64); let snapshot = pager.snapshot(); for (i, v) in [ snapshot.processes, snapshot.regions, snapshot.mappings, snapshot.owned_pages, snapshot.quarantined_pages, ] .into_iter() .enumerate() { os_k900_word(22 + i, v as u64); } let (ttbr, tcr, vbar, daif): (u64, u64, u64, u64); unsafe { asm!("mrs {r},TTBR0_EL1","mrs {t},TCR_EL1","mrs {v},VBAR_EL1","mrs {d},DAIF",r=out(reg)ttbr,t=out(reg)tcr,v=out(reg)vbar,d=out(reg)daif,options(nomem,nostack,preserves_flags)) }; for (i, v) in [ (14, ttbr), (15, tcr), (16, vbar), (17, daif), (18, unsafe { read_volatile(core::ptr::addr_of!(OS_K900_FAULT_COUNT)) }), (19, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }), ] { os_k900_word(i, v); } } fn os_k900_capture(pager: &OsK900Pager<'_>, stage: u64) { os_k900_word(9, stage); os_k900_snapshot(pager); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_CAPTURE_GATE), 0) }; write_str("TURKOS/OS/K900: CAPTURE STAGE=0x"); write_hex(stage as usize); write_line(""); while unsafe { read_volatile(core::ptr::addr_of!(OS_K900_CAPTURE_GATE)) } == 0 { unsafe { asm!("yield", options(nomem, nostack, preserves_flags)) }; } } fn os_k900_capture_initial() { os_k900_word(9, 0); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_CAPTURE_GATE), 0) }; write_line("TURKOS/OS/K900: CAPTURE STAGE=0x0000000000000000"); while unsafe { read_volatile(core::ptr::addr_of!(OS_K900_CAPTURE_GATE)) } == 0 { unsafe { asm!("yield", options(nomem, nostack, preserves_flags)) }; } } fn os_k900_entry(label: *const u8) -> u64 { unsafe extern "C" { static __os_k900_user_start: u8; } OS_K900_CODE + (label as u64 - core::ptr::addr_of!(__os_k900_user_start) as u64) } fn os_k900_accept( runtime: &mut OsK660Runtime, tables: &mmu::Tables, _boot: &BootData, ) -> Result<(), &'static str> { #[cfg(feature = "os-runtime-product")] { if !os_k1100_product_probe_enabled(_boot)? { let _ = (runtime, tables); os_k900_product_configuration(); write_line("TURKOS/OS/K900: PRODUCT-CONTROL NO-PAGE-FAULT-PROBE"); return Ok(()); } } #[cfg(any(feature = "os-runtime-acceptance", feature = "os-runtime-product"))] { use os_k900_pager::{Error, Pager, Region, RegionKind}; os_k900_read_mode(_boot)?; let allocator = &mut runtime .memory_mut() .map_err(|_| "page-fault-authority")? .allocator; let authority = allocator.os_authority_mut(); let region = authority .free_regions() .iter() .rev() .find(|r| r.end - r.start >= 40 * 4096) .ok_or("page-fault-pool")?; let pool = region.end - 36 * 4096; let (ttbr, tcr, vbar, daif): (u64, u64, u64, u64); unsafe { asm!("mrs {r},TTBR0_EL1","mrs {t},TCR_EL1","mrs {v},VBAR_EL1","mrs {d},DAIF",r=out(reg)ttbr,t=out(reg)tcr,v=out(reg)vbar,d=out(reg)daif,options(nomem,nostack,preserves_flags)) }; if ttbr != tables.root || tcr & 63 != 25 || daif & 128 == 0 { return Err("page-fault-baseline"); } for (i, v) in [ (0, 0x4b39303050414745), (1, ttbr), (2, tcr), (3, vbar), (4, daif), (5, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }), (6, pool), ] { os_k900_word(i, v); } os_k820_authority( authority, core::ptr::addr_of_mut!(OS_K900_AUTHORITY_LIVE).cast(), ); os_k900_word(46, unsafe { read_volatile(core::ptr::addr_of!(OS_K880_PAGE_EVENT_COUNT)) }); os_k900_capture_initial(); os_k900_begin_event_loan()?; let backend = OsK900PageBackend::new(authority, tables)?; let geometry = os_k820_descriptor::Geometry::new(39, 40).map_err(|_| "page-fault-geometry")?; let identity = backend.identity(); let mut tree = OsK900Tree::new(identity, geometry, backend).map_err(|_| "page-fault-tree")?; let code = tree.allocate_page().map_err(|_| "page-fault-code")?; let stack = tree.allocate_page().map_err(|_| "page-fault-stack")?; unsafe extern "C" { static __os_k900_user_start: u8; static __os_k900_user_end: u8; static __os_k900_anon: u8; static __os_k900_quota: u8; static __os_k900_stack: u8; static __os_k900_heap_sp: u8; static __os_k900_cow_write: u8; static __os_k900_cow_fail: u8; static __os_k900_hash: u8; static __os_k900_hash_fail: u8; } os_k820_code( &code, core::ptr::addr_of!(__os_k900_user_start), core::ptr::addr_of!(__os_k900_user_end), )?; let mut backend = OsK900FaultBackend { tree, kernel: tables.root, geometry, roots: core::array::from_fn(|_| None), fixed: [Some(code), Some(stack)], map: [None; 16], retained: core::array::from_fn(|_| None), }; for i in 0..2 { let root = backend.tree.new_root().map_err(|_| "page-fault-root")?; backend.fixed(&root).map_err(|_| "page-fault-fixed-map")?; backend.roots[i] = Some(root); } let mut pager = Pager::new(backend); pager .register(OS_K900_A, 1) .map_err(|_| "page-fault-register-anon")?; pager .add_region(Region { owner: OS_K900_A, start: OS_K900_ANON, end: OS_K900_ANON + 8192, kind: RegionKind::Anonymous, }) .map_err(|_| "page-fault-anon-region")?; let user_stack = OS_K900_USER_STACK + 4096 - 16; let mut refusal_before = [0u64; 328]; pager.with_backend_observation(|b| { b.authority(|a| os_k820_authority(a, refusal_before.as_mut_ptr())) }); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_PAGE_ALLOC_BUDGET), 3) }; let control = os_k900_run_user( &mut pager, OS_K900_A, 1, os_k900_entry(core::ptr::addr_of!(__os_k900_anon)), user_stack, )?; if control.0 != Error::NoMemory as u64 + 1 { return Err("page-fault-second-table-enospc"); } os_k900_word(32, control.0); if !pager.with_backend_observation(|b| { b.authority(|a| os_k900_inventory_matches(a, &refusal_before)) }) { return Err("page-fault-enospc-inventory"); } os_k900_capture(&pager, 1); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_PAGE_ALLOC_BUDGET), 3); write_volatile( core::ptr::addr_of_mut!(OS_K900_PAGE_FAIL_RELEASE), pool + 8 * 4096, ) }; let control = os_k900_run_user( &mut pager, OS_K900_A, 2, os_k900_entry(core::ptr::addr_of!(__os_k900_anon)), user_stack, )?; if control.0 != Error::Retained as u64 + 1 || pager.snapshot().quarantined_pages != 1 { return Err("page-fault-retained-rollback"); } os_k900_word(33, control.0); pager.audit().map_err(|_| "page-fault-retained-audit")?; os_k900_capture(&pager, 2); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_PAGE_ALLOC_BUDGET), u64::MAX); write_volatile(core::ptr::addr_of_mut!(OS_K900_PAGE_FAIL_RELEASE), 0) }; if pager .reap_quarantine() .map_err(|_| "page-fault-retained-retry")? != 1 { return Err("page-fault-retained-count"); } unsafe { pager.os_k900_with_backend_mut(|b| b.reap()) } .map_err(|_| "page-fault-table-retry")?; let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K900_FAULT_COUNT)) }; let anon = os_k900_run_user( &mut pager, OS_K900_A, 3, os_k900_entry(core::ptr::addr_of!(__os_k900_anon)), user_stack, )?; if anon != (0, 0) || unsafe { read_volatile(core::ptr::addr_of!(OS_K900_FAULT_COUNT)) } != count + 1 { return Err("page-fault-anonymous-zero-repeat"); } os_k900_word(34, 1); let quota = os_k900_run_user( &mut pager, OS_K900_A, 4, os_k900_entry(core::ptr::addr_of!(__os_k900_quota)), user_stack, )?; if quota.0 != Error::Quota as u64 + 1 { return Err("page-fault-quota"); } os_k900_word(35, quota.0); os_k900_capture(&pager, 3); pager .detach_owner(OS_K900_A) .map_err(|_| "page-fault-anon-detach")?; pager .register(OS_K900_A, 8) .map_err(|_| "page-fault-register-stack")?; pager .add_region(Region { owner: OS_K900_A, start: OS_K900_USER_STACK - 4 * 4096, end: OS_K900_USER_STACK + 4096, kind: RegionKind::Stack { resident_low: OS_K900_USER_STACK, }, }) .map_err(|_| "page-fault-stack-region")?; pager .add_region(Region { owner: OS_K900_A, start: OS_K900_ANON, end: OS_K900_ANON + 8192, kind: RegionKind::Anonymous, }) .map_err(|_| "page-fault-stack-anon")?; for i in 1..=2 { let value = os_k900_run_user( &mut pager, OS_K900_A, 4 + i, os_k900_entry(core::ptr::addr_of!(__os_k900_stack)), OS_K900_USER_STACK - (i - 1) * 4096 - 16, )?; if value != (0x91, 0x91) { return Err("page-fault-adjacent-stack"); } } let gap = os_k900_run_user( &mut pager, OS_K900_A, 7, os_k900_entry(core::ptr::addr_of!(__os_k900_stack)), OS_K900_USER_STACK - 3 * 4096 - 16, )?; if gap.0 != Error::Stack as u64 + 1 { return Err("page-fault-stack-gap"); } let jump = os_k900_run_user( &mut pager, OS_K900_A, 8, os_k900_entry(core::ptr::addr_of!(__os_k900_heap_sp)), OS_K900_ANON + 16, )?; if jump.0 != Error::Stack as u64 + 1 { return Err("page-fault-heap-sp"); } os_k900_word(36, gap.0); os_k900_word(37, jump.0); os_k900_capture(&pager, 4); pager .detach_owner(OS_K900_A) .map_err(|_| "page-fault-stack-detach")?; pager .register(OS_K900_A, 4) .map_err(|_| "page-fault-cow-a")?; pager .register(OS_K900_B, 4) .map_err(|_| "page-fault-cow-b")?; pager .share_cow( OS_K900_A, OS_K900_B, OS_K900_COW, OS_K900_COW, &[0x5a; 4096], ) .map_err(|_| "page-fault-cow-share")?; let before = os_k900_run_user( &mut pager, OS_K900_B, 9, os_k900_entry(core::ptr::addr_of!(__os_k900_hash)), user_stack, )?; let wrote = os_k900_run_user( &mut pager, OS_K900_A, 10, os_k900_entry(core::ptr::addr_of!(__os_k900_cow_write)), user_stack, )?; let after = os_k900_run_user( &mut pager, OS_K900_B, 11, os_k900_entry(core::ptr::addr_of!(__os_k900_hash)), user_stack, )?; if before.0 != 0x8ab9_e262_b32d_a325 || before.0 != after.0 || wrote != (0xa1, 0xa1) { return Err("page-fault-cow-private-copy"); } for (i, v) in [(38, before.0), (39, after.0), (40, wrote.0)] { os_k900_word(i, v); } os_k900_capture(&pager, 5); pager .share_cow( OS_K900_A, OS_K900_B, OS_K900_COW + 4096, OS_K900_COW + 4096, &[0x6b; 4096], ) .map_err(|_| "page-fault-cow-negative-share")?; let mut refusal_before = [0u64; 328]; pager.with_backend_observation(|b| { b.authority(|a| os_k820_authority(a, refusal_before.as_mut_ptr())) }); os_k900_capture(&pager, 6); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_PAGE_ALLOC_BUDGET), 0) }; let failed = os_k900_run_user( &mut pager, OS_K900_A, 12, os_k900_entry(core::ptr::addr_of!(__os_k900_cow_fail)), user_stack, )?; if failed.0 != Error::NoMemory as u64 + 1 { return Err("page-fault-cow-enospc"); } let b = os_k900_run_user( &mut pager, OS_K900_B, 13, os_k900_entry(core::ptr::addr_of!(__os_k900_hash_fail)), user_stack, )?; if b.0 != 0x3be2_3752_3bb6_f325 { return Err("page-fault-cow-enospc-b-changed"); } if !pager.with_backend_observation(|b| { b.authority(|a| os_k900_inventory_matches(a, &refusal_before)) }) { return Err("page-fault-cow-enospc-inventory"); } os_k900_word(41, failed.0); os_k900_word(42, b.0); pager.audit().map_err(|_| "page-fault-live-audit")?; os_k900_capture(&pager, 7); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_PAGE_ALLOC_BUDGET), u64::MAX) }; pager .detach_owner(OS_K900_A) .map_err(|_| "page-fault-cow-a-detach")?; pager .detach_owner(OS_K900_B) .map_err(|_| "page-fault-cow-b-detach")?; pager.audit().map_err(|_| "page-fault-final-audit")?; unsafe { pager.os_k900_with_backend_mut(|b| b.finish()) } .map_err(|_| "page-fault-root-cleanup")?; pager.with_backend_observation(|b| { b.authority(|a| { os_k820_authority(a, core::ptr::addr_of_mut!(OS_K900_AUTHORITY_AFTER).cast()) }) }); let (after_ttbr, after_tcr, after_vbar): (u64, u64, u64); unsafe { asm!("mrs {r},TTBR0_EL1","mrs {t},TCR_EL1","mrs {v},VBAR_EL1",r=out(reg)after_ttbr,t=out(reg)after_tcr,v=out(reg)after_vbar,options(nomem,nostack,preserves_flags)) }; for (i, v) in [ (7, 1), (8, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }), (43, after_ttbr), (44, after_tcr), (45, after_vbar), ] { os_k900_word(i, v); } if after_ttbr != ttbr || after_tcr != tcr || after_vbar != vbar || unsafe { &*core::ptr::addr_of!(OS_K900_AUTHORITY_BEFORE) } != unsafe { &*core::ptr::addr_of!(OS_K900_AUTHORITY_AFTER) } { return Err("page-fault-restoration"); } write_line( "TURKOS/OS/K885: ACTUAL-LOWER-EL TRANSLATION-PERMISSION RAW-ESR-FAR-ELR FAIL-CLOSED CLASSIFICATION PASS", ); write_line( "TURKOS/OS/K890: ACTUAL-EL0 ANONYMOUS ZERO-FIRST-ACCESS SECOND-ACCESS-NO-ALLOCATION QUOTA-ENOSPC EXACT-INVENTORY PASS", ); write_line( "TURKOS/OS/K895: ACTUAL-EL0 ADJACENT-STACK-GROWTH GAP-HEAP-SP-REFUSED PRIVATE-COW B-UNCHANGED ENOSPC-BOTH-UNCHANGED PASS", ); write_line( "TURKOS/OS/K900: FULL-PHYSICAL-OWNERSHIP DRAFT-PREFLIGHT DSB-TLBI-DSB-ISB BEFORE-RECLAIM EXACT-AUTHORITY-RESTORED PASS", ); Ok(()) } } fn os_k900_product_configuration() { unsafe extern "C" { static __os_k900_user_start: u8; static __os_k900_user_end: u8; static __os_k900_anon: u8; static __os_k900_quota: u8; static __os_k900_stack: u8; static __os_k900_heap_sp: u8; static __os_k900_cow_write: u8; static __os_k900_cow_fail: u8; static __os_k900_vectors: u8; static __os_k900_kernel_fault_instruction: u8; } let symbols = [ core::ptr::addr_of!(OS_K900_EVIDENCE) as *const u8 as usize, core::ptr::addr_of!(OS_K900_USER_STATE) as *const u8 as usize, core::ptr::addr_of!(OS_K900_FAULTS) as *const u8 as usize, core::ptr::addr_of!(OS_K900_FAULT_COUNT) as *const u8 as usize, core::ptr::addr_of!(OS_K900_RECORDS) as *const u8 as usize, core::ptr::addr_of!(OS_K900_REGISTRY) as *const u8 as usize, core::ptr::addr_of!(OS_K900_MAPPINGS) as *const u8 as usize, core::ptr::addr_of!(OS_K900_OWNED) as *const u8 as usize, core::ptr::addr_of!(OS_K900_AUTHORITY_BEFORE) as *const u8 as usize, core::ptr::addr_of!(OS_K900_AUTHORITY_LIVE) as *const u8 as usize, core::ptr::addr_of!(OS_K900_AUTHORITY_AFTER) as *const u8 as usize, core::ptr::addr_of!(OS_K900_CAPTURE_GATE) as *const u8 as usize, core::ptr::addr_of!(OS_K900_PAGE_EVENTS) as *const u8 as usize, core::ptr::addr_of!(OS_K900_PAGE_EVENT_COUNT) as *const u8 as usize, core::ptr::addr_of!(OS_K900_KERNEL_FAULT) as *const u8 as usize, core::ptr::addr_of!(OS_K900_KERNEL_BEFORE) as *const u8 as usize, core::ptr::addr_of!(OS_K900_KERNEL_AFTER) as *const u8 as usize, core::ptr::addr_of!(__os_k900_user_start) as usize, core::ptr::addr_of!(__os_k900_user_end) as usize, core::ptr::addr_of!(__os_k900_anon) as usize, core::ptr::addr_of!(__os_k900_quota) as usize, core::ptr::addr_of!(__os_k900_stack) as usize, core::ptr::addr_of!(__os_k900_heap_sp) as usize, core::ptr::addr_of!(__os_k900_cow_write) as usize, core::ptr::addr_of!(__os_k900_cow_fail) as usize, core::ptr::addr_of!(__os_k900_vectors) as usize, core::ptr::addr_of!(__os_k900_kernel_fault_instruction) as usize, ]; core::hint::black_box(symbols); } fn os_k900_inventory_matches(a: &os_k720_frame::Allocator, b: &[u64; 328]) -> bool { let free = a.free_regions(); if b[0] != free.len() as u64 || b[3] != a.remaining_pages() { return false; } for (i, r) in free.iter().enumerate() { if b[4 + i * 2] != r.start || b[5 + i * 2] != r.end { return false; } } let mut live = 0; for address in a.live_addresses() { if live >= 64 || b[132 + live] != address { return false; } live += 1; } if b[1] != live as u64 { return false; } let mut reserved = 0; for r in a.reserved_regions() { if reserved >= 64 || b[196 + reserved * 2] != r.start || b[197 + reserved * 2] != r.end { return false; } reserved += 1; } b[2] == reserved as u64 } fn os_k900_begin_event_loan() -> Result<(), &'static str> { let expected = unsafe { read_volatile(core::ptr::addr_of!(OS_K900_EVIDENCE).cast::().add(46)) }; if expected == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K880_PAGE_EVENT_COUNT)) } != expected || unsafe { read_volatile(core::ptr::addr_of!(OS_K900_PAGE_EVENT_COUNT)) } != 0 { return Err("page-fault-event-loan-baseline"); } for index in 0..4096 { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K900_PAGE_EVENTS) .cast::() .add(index), 0, ) }; } Ok(()) } #[unsafe(no_mangle)] static mut OS_K900_USER_STATE: [u64; 13] = [0; 13]; #[unsafe(no_mangle)] static mut OS_K900_FAULTS: [u64; 160] = [0; 160]; #[unsafe(no_mangle)] static mut OS_K900_FAULT_COUNT: u64 = 0; static mut OS_K900_PAGER_POINTER: *mut u8 = core::ptr::null_mut(); fn os_k900_user_word(index: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K900_USER_STATE) .cast::() .add(index), value, ) }; } fn os_k900_run_user( pager: &mut OsK900Pager<'_>, owner: os_k900_mapping::Owner, mode: u64, entry: u64, stack: u64, ) -> Result<(u64, u64), &'static str> { unsafe { pager.os_k900_with_backend_mut(|b| b.activate(owner)) } .map_err(|_| "page-fault-user-root")?; let (vbar, ttbr, tcr): (u64, u64, u64); unsafe { asm!("mrs {v},VBAR_EL1","mrs {r},TTBR0_EL1","mrs {t},TCR_EL1",v=out(reg)vbar,r=out(reg)ttbr,t=out(reg)tcr,options(nomem,nostack,preserves_flags)) }; let kernel = pager.with_backend_observation(|b| b.kernel); for (i, v) in [ (0, mode), (2, kernel), (3, tcr), (4, 0), (5, vbar), (6, owner.id), (7, owner.generation), (8, ttbr), (9, 0), (10, 0), (11, 0), (12, 0), ] { os_k900_user_word(i, v); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K900_PAGER_POINTER), (pager as *mut OsK900Pager<'_>).cast(), ) }; unsafe extern "C" { fn __os_k900_enter_user(entry: u64, stack: u64); } unsafe { __os_k900_enter_user(entry, stack) }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K900_PAGER_POINTER), core::ptr::null_mut(), ) }; let state = unsafe { &*core::ptr::addr_of!(OS_K900_USER_STATE) }; if state[4] != mode { return Err("page-fault-user-completion"); } Ok((state[9], state[10])) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k900_fault(frame: *mut os_k900_frame::Frame) -> u64 { let guard = core::ptr::addr_of!(OS_K900_STACK) as u64; let stack = os_k900_frame::Stack::new(guard + 4096, guard + 4096 + 65536).unwrap_or_else(|_| park()); if stack.frame_address(frame as u64).is_err() { park(); } let saved = unsafe { &*frame }; if os_k900_frame::validate_ingress(saved, frame as u64, stack).is_err() { park(); } if saved.vector_slot == 4 { os_k900_kernel_fatal(saved, frame as u64); } let pointer = unsafe { read_volatile(core::ptr::addr_of!(OS_K900_PAGER_POINTER)) }; if pointer.is_null() { park(); } let pager = unsafe { &mut *pointer.cast::>() }; let state = unsafe { &*core::ptr::addr_of!(OS_K900_USER_STATE) }; let mode = state[0]; let owner = os_k900_mapping::Owner { id: state[6], generation: state[7], }; let (ttbr, tcr): (u64, u64); unsafe { asm!("mrs {r},TTBR0_EL1","mrs {t},TCR_EL1",r=out(reg)ttbr,t=out(reg)tcr,options(nomem,nostack,preserves_flags)) }; if pager.with_backend_observation(|b| os_k900_pager::Backend::current_owner(b)) != Some(owner) { park(); } if saved.esr >> 26 & 63 == 0x15 { if saved.esr & 65535 != 0x900 { park(); } os_k900_user_word(9, saved.x[0]); os_k900_user_word(10, saved.x[1]); os_k900_user_word(11, saved.x[2]); os_k900_user_word(12, saved.x[3]); os_k900_user_word(4, mode); return 1; } let evidence = os_k900_classify::Evidence { vector: saved.vector_slot as u8, esr: saved.esr, far: Some(saved.far), elr: saved.elr, spsr: saved.spsr, sp_el0: saved.sp_el0, }; let fault = match os_k900_classify::classify(evidence) { os_k900_classify::Classification::User(f) => f, _ => park(), }; let resolved = pager.resolve(owner, fault); let mut physical = 0; let mut kind = 0; let mut error = 0; let mut allocated = 0; match resolved { Ok(r) => { physical = r.frame.address(); kind = match r.kind { os_k900_pager::Resolved::Present => 1, os_k900_pager::Resolved::Anonymous => 2, os_k900_pager::Resolved::Stack => 3, os_k900_pager::Resolved::CopyOnWrite => 4, }; allocated = r.allocated as u64; } Err(e) => error = e as u64 + 1, } let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K900_FAULT_COUNT)) }; if count >= 10 { park(); } let row = unsafe { core::ptr::addr_of_mut!(OS_K900_FAULTS) .cast::() .add(count as usize * 16) }; for (i, v) in [ mode, owner.id, saved.esr, saved.far, saved.elr, saved.spsr, saved.sp_el0, ttbr, tcr, frame as u64, kind, error, physical, saved.canary, saved.current_el, allocated, ] .into_iter() .enumerate() { unsafe { write_volatile(row.add(i), v) }; } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_FAULT_COUNT), count + 1) }; if error != 0 { os_k900_user_word(9, error); os_k900_user_word(4, mode); return 1; } let user = pager.with_backend_observation(|b| { let i = OsK900FaultBackend::index(owner).unwrap_or_else(|_| park()); os_k900_frame::User { owner: owner.id, generation: owner.generation, root: b.roots[i].as_ref().unwrap_or_else(|| park()).address(), asid: i as u16 + 1, } }); if pager .with_backend_observation(|b| { os_k900_frame::authorize_user_return(saved, frame as u64, stack, user, b) }) .is_err() { park(); } 0 } global_asm!(include_str!("k900-page-faults-vectors.S")); static mut OS_K900_MODE: u32 = 0; static mut OS_K900_FATAL_AUTHORITY: *const os_k720_frame::Allocator = core::ptr::null(); #[unsafe(no_mangle)] static mut OS_K900_KERNEL_FAULT: [u64; 32] = [0; 32]; fn os_k900_read_mode(boot: &BootData) -> Result<(), &'static str> { let blob = unsafe { core::slice::from_raw_parts(boot.dtb_address as *const u8, boot.dtb_size) }; let scratch = unsafe { &mut *core::ptr::addr_of_mut!(OS_K700_WORKSPACE) }; let mode = turkos_kernel::runtime::platform::chosen_u32(blob, scratch, b"turkos,page-fault-mode") .map_err(|_| "page-fault-mode")? .unwrap_or(0); if mode > 1 { return Err("page-fault-mode"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_MODE), mode) }; Ok(()) } fn os_k900_kernel_fatal_if_requested(runtime: &mut OsK660Runtime) { if unsafe { read_volatile(core::ptr::addr_of!(OS_K900_MODE)) } != 1 { return; } let authority = runtime .memory_mut() .unwrap_or_else(|_| park()) .allocator .os_authority(); os_k820_authority( authority, core::ptr::addr_of_mut!(OS_K900_KERNEL_AFTER).cast(), ); if unsafe { &*core::ptr::addr_of!(OS_K900_KERNEL_BEFORE) } != unsafe { &*core::ptr::addr_of!(OS_K900_KERNEL_AFTER) } { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_FATAL_AUTHORITY), authority) }; unsafe extern "C" { fn __os_k900_fatal_on_stack() -> !; } unsafe { __os_k900_fatal_on_stack() } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k900_fatal_body() -> ! { unsafe extern "C" { fn __os_k900_vectors(); fn __os_k900_kernel_fault_instruction(target: u64); } let vector = __os_k900_vectors as *const () as u64; unsafe { asm!("msr VBAR_EL1,{vector}","isb",vector=in(reg)vector,options(nostack,preserves_flags)) }; unsafe { __os_k900_kernel_fault_instruction(0x8000_0000) }; park() } fn os_k900_kernel_fatal(frame: &os_k900_frame::Frame, address: u64) -> ! { let evidence = os_k900_classify::Evidence { vector: frame.vector_slot as u8, esr: frame.esr, far: Some(frame.far), elr: frame.elr, spsr: frame.spsr, sp_el0: frame.sp_el0, }; if os_k900_classify::classify(evidence) != os_k900_classify::Classification::Kernel { park(); } let authority = unsafe { read_volatile(core::ptr::addr_of!(OS_K900_FATAL_AUTHORITY)) }; if authority.is_null() { park(); } os_k820_authority( unsafe { &*authority }, core::ptr::addr_of_mut!(OS_K900_KERNEL_AFTER).cast(), ); if unsafe { &*core::ptr::addr_of!(OS_K900_KERNEL_BEFORE) } != unsafe { &*core::ptr::addr_of!(OS_K900_KERNEL_AFTER) } { park(); } let (ttbr, tcr, vbar, daif): (u64, u64, u64, u64); unsafe { asm!("mrs {r},TTBR0_EL1","mrs {t},TCR_EL1","mrs {v},VBAR_EL1","mrs {d},DAIF",r=out(reg)ttbr,t=out(reg)tcr,v=out(reg)vbar,d=out(reg)daif,options(nomem,nostack,preserves_flags)) }; let target = core::ptr::addr_of_mut!(OS_K900_KERNEL_FAULT).cast::(); for (i, v) in [ 1, frame.esr, frame.far, frame.elr, frame.spsr, frame.interrupted_sp, frame.sp_el0, frame.vector_slot, frame.current_el, frame.canary, address, ttbr, tcr, vbar, daif, unsafe { read_volatile(core::ptr::addr_of!(OS_K900_FAULT_COUNT)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K900_PAGE_EVENT_COUNT)) }, ] .into_iter() .enumerate() { unsafe { write_volatile(target.add(i), v) }; } write_line( "TURKOS/OS/K885: REAL-CURRENT-EL KERNEL-FAULT NO-DEMAND-ALLOCATION EXACT-LEDGER FATAL-PARKED", ); park() } #[repr(C, align(4096))] struct OsK900Stack { guard: [u8; 4096], usable: [u8; 65536], } unsafe extern "C" { static OS_K900_STACK: OsK900Stack; } global_asm!(".global OS_K900_STACK\n.set OS_K900_STACK, OS_K820_STACK\n.type OS_K900_STACK, %object\n.size OS_K900_STACK, 69632"); #[unsafe(no_mangle)] static mut OS_K900_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K900_ERROR: Option<&'static str> = None; const OS_K900_LOW_CANARY: u64 = 0x4b38_3230_4c4f_5721; const OS_K900_HIGH_CANARY: u64 = 0x4b38_3230_4849_4748; fn os_k900_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K900_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } fn os_k900_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { unsafe extern "C" { static __stack_start: u8; static __stack_top: u8; } let guard = core::ptr::addr_of!(OS_K900_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_low = core::ptr::addr_of!(__stack_start) as u64; let original_top = core::ptr::addr_of!(__stack_top) as u64; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("page-faults-original-stack-range"); } let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_low + 8) as *const u64) }; if original_canary != 0x4b36_3335_5354_414b || original_second != !0x4b36_3335_5354_414bu64 { return Err("page-faults-original-stack-canaries"); } let prior_guard=unsafe {read_volatile(core::ptr::addr_of!(OS_K820_STACK_EVIDENCE).cast::().add(13))}; let prior_low=unsafe {read_volatile(core::ptr::addr_of!(OS_K820_STACK_EVIDENCE).cast::().add(1))}; let prior_top=unsafe {read_volatile(core::ptr::addr_of!(OS_K820_STACK_EVIDENCE).cast::().add(2))}+16; if guard!=prior_guard||low!=prior_low||end!=prior_top ||guardcore::ptr::addr_of!(_image_end)as u64 ||unsafe{read_volatile(low as *const u64)}!=OS_K900_LOW_CANARY ||unsafe{read_volatile(high as *const u64)}!=OS_K900_HIGH_CANARY ||!tables.task_stack_guard_unmapped(guard){return Err("page-faults-returned-stack-guard");} if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("page-faults-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K900_LOW_CANARY); write_volatile(high as *mut u64, OS_K900_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K900_LOW_CANARY), (9, OS_K900_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("page-faults-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("page-faults-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k900_stack_word(slot, value); } unsafe extern "C" { fn __os_k900_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k900_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K900_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K900_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K900_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_low + 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("page-faults-stack-restoration"); } os_k900_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k900_stack_word(17, unsafe { read_volatile((original_low + 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K900: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); Ok(()) } else { Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K900_ERROR)) } .unwrap_or("page-faults-stack-return")) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k900_accept_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k900_stack_word(4, sp); let result = os_k900_guarded_work( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k900_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K900_ERROR), Some(reason)) }; 1 } } } #[unsafe(no_mangle)] static mut OS_K780_RETURN_POOL_GATE:u8=0; #[unsafe(no_mangle)] static mut OS_K840_RETURN_POOL_GATE:u8=0; #[unsafe(no_mangle)] static mut OS_K880_RETURN_POOL_GATE:u8=0; #[unsafe(no_mangle)] static mut OS_K900_RETURN_POOL_GATE:u8=0; #[unsafe(no_mangle)] static mut OS_K920_RETURN_POOL_GATE:u8=0; fn os_k900_guarded_work(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str>{ os_k900_accept(runtime,tables,boot)?; if cfg!(feature="os-runtime-product") && os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K900: RETURN-POOL READY"); while unsafe{read_volatile(core::ptr::addr_of!(OS_K900_RETURN_POOL_GATE))}==0 { unsafe{asm!("yield",options(nomem,nostack,preserves_flags))}; } } os_k920_accept(runtime,tables,boot)?; if cfg!(feature="os-runtime-product") && os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K920: RETURN-POOL READY"); while unsafe{read_volatile(core::ptr::addr_of!(OS_K920_RETURN_POOL_GATE))}==0 { unsafe{asm!("yield",options(nomem,nostack,preserves_flags))}; } } os_k940_accept(runtime,tables,boot)?; os_k960_accept(runtime,tables,boot)?; os_k980_accept(runtime,tables,boot)?; os_k1000_accept(runtime,tables,boot)?; os_k1020_accept(runtime,tables,boot)?; os_k1040_on_stack(runtime,tables,boot)?; os_k1060_on_stack(runtime,tables,boot)?; os_k1080_on_stack(runtime,tables,boot)?; os_k1100_on_stack(runtime,tables,boot) } global_asm!( r#" .section .text.os_k900_stack,"ax" .balign 4 .global __os_k900_on_stack __os_k900_on_stack: adrp x9, OS_K900_STACK_EVIDENCE add x9, x9, :lo12:OS_K900_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K900_STACK add x9, x9, :lo12:OS_K900_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K900_STACK_EVIDENCE add x9, x9, :lo12:OS_K900_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k900_accept_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K900_STACK_EVIDENCE add x9, x9, :lo12:OS_K900_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret "# ); use turkos_kernel::runtime::page_tables::{descriptor as os_k920_page_descriptor,tree as os_k920_page_tree}; const OS_K920_PAGE_OWNER:u64=0x920; const OS_K920_PAGE_PA_MASK:u64=0x0000_ffff_ffff_f000; #[unsafe(no_mangle)] static mut OS_K920_PAGE_ALLOC_BUDGET:u64=u64::MAX; #[unsafe(no_mangle)] static mut OS_K920_PAGE_FAIL_RELEASE:u64=0; #[unsafe(no_mangle)] static mut OS_K920_PAGE_EVENTS:[u64;384]=[0;384]; #[unsafe(no_mangle)] static mut OS_K920_PAGE_EVENT_COUNT:u64=0; fn os_k920_page_event(kind:u64,address:u64,value:u64){ let n=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_PAGE_EVENT_COUNT))}; if n>=128 {park();} unsafe{let p=core::ptr::addr_of_mut!(OS_K920_PAGE_EVENTS).cast::().add(n as usize*3);write_volatile(p,kind);write_volatile(p.add(1),address);write_volatile(p.add(2),value);write_volatile(core::ptr::addr_of_mut!(OS_K920_PAGE_EVENT_COUNT),n+1);} } struct OsK920PageFrames<'a>{authority:&'a mut os_k720_frame::Allocator,leases:[Option;8],addresses:[u64;8]} impl OsK920PageFrames<'_>{ fn validate(&self,address:u64)->Result<(),os_k740_metadata::Error>{ let lease=self.leases.iter().flatten().find(|lease|lease.address()==address).ok_or(os_k740_metadata::Error::BadAddress)?; self.authority.validate_lease(lease).map_err(|_|os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK920PageFrames<'_>{ fn allocate(&mut self)->Result{ let budget=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_PAGE_ALLOC_BUDGET))}; if budget==0{return Err(os_k740_metadata::Error::NoMemory);} if budget!=u64::MAX {unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_PAGE_ALLOC_BUDGET),budget-1)};} let slot=self.leases.iter().position(Option::is_none).ok_or(os_k740_metadata::Error::Capacity)?; let lease=self.authority.allocate_at(self.addresses[slot]).map_err(|_|os_k740_metadata::Error::NoMemory)?; let address=lease.address();let generation=lease.generation();self.leases[slot]=Some(lease);os_k920_page_event(1,address,generation);Ok(address) } fn release(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{ self.validate(address)?; if unsafe{read_volatile(core::ptr::addr_of!(OS_K920_PAGE_FAIL_RELEASE))}==address {os_k920_page_event(9,address,0);return Err(os_k740_metadata::Error::Backend);} let slot=self.leases.iter().position(|lease|lease.as_ref().is_some_and(|lease|lease.address()==address)).ok_or(os_k740_metadata::Error::BadAddress)?; for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};} unsafe{asm!("dsb ish",options(nostack,preserves_flags))}; self.authority.release(self.leases[slot].as_ref().ok_or(os_k740_metadata::Error::Backend)?).map_err(|_|os_k740_metadata::Error::Backend)?; self.leases[slot]=None;os_k920_page_event(8,address,0);Ok(()) } fn zero_page(&mut self,address:u64)->Result<(),os_k740_metadata::Error>{self.validate(address)?;for offset in 0..4096 {unsafe{write_volatile((address as *mut u8).add(offset),0)};}Ok(())} fn read_byte(&self,address:u64,offset:usize)->Result{if offset>=4096{return Err(os_k740_metadata::Error::BadAddress);}self.validate(address)?;Ok(unsafe{read_volatile((address as *const u8).add(offset))})} fn available_pages(&self)->u64{self.authority.remaining_pages()} } pub struct OsK920PageBackend<'a>{identity:u64,registry:os_k740_metadata::Registry,8>,tables:&'a mmu::Tables} impl<'a> OsK920PageBackend<'a>{ pub fn new(authority:&'a mut os_k720_frame::Allocator,tables:&'a mmu::Tables)->Result{ let region=authority.free_regions().iter().rev().find(|r|r.end-r.start>=80*4096).ok_or("page-tables-interior-pool")?; let base=region.end-76*4096; let addresses=core::array::from_fn(|i|base+i as u64*4096); let frames=OsK920PageFrames{authority,leases:core::array::from_fn(|_|None),addresses}; unsafe extern "C"{fn __os_k760_workspace_identity()->u64;} let identity=unsafe{__os_k760_workspace_identity()};if identity==0{return Err("page-tables-registry-identity");} let registry=os_k740_metadata::Registry::new(identity,frames).map_err(|_|"page-tables-registry")?; Ok(Self{identity,registry,tables}) } pub fn identity(&self)->u64{self.identity} pub fn with_authority_observation(&self,callback:impl FnOnce(&os_k720_frame::Allocator)->R)->R{self.registry.with_backend_observation(|frames|callback(frames.authority))} pub fn registry_records(&self)->impl Iterator+'_ {self.registry.owned_records()} pub fn observed_handles(&self)->impl Iterator+'_ { self.registry.owned_records().map(|r|{let generation=self.registry.with_backend_observation(|frames|frames.leases.iter().flatten().find(|lease|lease.address()==r.address).map(|lease|lease.generation()).unwrap_or(0));(r,generation)}) } fn owned_kernel(&self,address:u64)->bool{self.tables.tagged_frames().any(|frame|frame.address==address)} fn shared_walk(&self,address:u64,level:u8,visited:&mut [u64;4],count:usize)->Result<(),os_k920_page_tree::Error>{ if level>3||count>=4||visited[..count].contains(&address)||!self.owned_kernel(address){return Err(os_k920_page_tree::Error::Shared);} visited[count]=address; for index in 0..512 { let raw=unsafe{read_volatile((address as *const u64).add(index))};if raw==0{continue;} match raw&3 { 3 if level<3=>self.shared_walk(raw&OS_K920_PAGE_PA_MASK,level+1,visited,count+1)?, 1 if level<3=>{if raw&(1<<6)!=0{return Err(os_k920_page_tree::Error::Shared);}}, 3 if level==3=>{if raw&(1<<6)!=0{return Err(os_k920_page_tree::Error::Shared);}}, _=>return Err(os_k920_page_tree::Error::Shared), } } Ok(()) } fn kernel_level(&self,address:u64)->Result{ let mut pending=[(0u64,0u8);24];pending[0]=(self.tables.root,1);let mut len=1;let mut at=0; while at{ type Handle=os_k740_metadata::Handle; fn allocate_zeroed(&mut self)->Result{let handle=self.registry.allocate_zeroed(OS_K920_PAGE_OWNER).map_err(|e|if e==os_k740_metadata::Error::NoMemory{os_k920_page_tree::Error::NoMemory}else{os_k920_page_tree::Error::Backend})?;os_k920_page_event(0x40,handle.address(),handle.generation());Ok(handle)} fn address(&self,handle:Self::Handle)->u64{handle.address()} fn validate(&self,handle:Self::Handle)->Result<(),os_k920_page_tree::Error>{self.registry.read_byte(handle,OS_K920_PAGE_OWNER,4095).map(|_|()).map_err(|_|os_k920_page_tree::Error::Stale)} fn release(&mut self,handle:Self::Handle)->Result<(),os_k920_page_tree::Error>{self.registry.release(handle,OS_K920_PAGE_OWNER).map_err(|_|os_k920_page_tree::Error::Backend)} fn entry(&self,handle:Self::Handle,index:usize)->Result{if index>=512{return Err(os_k920_page_tree::Error::Backend);}self.validate(handle)?;Ok(unsafe{read_volatile((handle.address() as *const u64).add(index))})} fn write_entry(&mut self,handle:Self::Handle,index:usize,value:u64)->Result<(),os_k920_page_tree::Error>{if index>=512{return Err(os_k920_page_tree::Error::Backend);}self.validate(handle)?;unsafe{write_volatile((handle.address() as *mut u64).add(index),value)};os_k920_page_event(2,handle.address()+index as u64*8,value);Ok(())} fn validate_shared_table(&self,address:u64)->Result<(),os_k920_page_tree::Error>{let level=self.kernel_level(address)?;self.shared_walk(address,level,&mut[0;4],0)} fn validate_shared_page(&self,_address:u64)->Result<(),os_k920_page_tree::Error>{Err(os_k920_page_tree::Error::Shared)} fn shared_entry(&self,address:u64,index:usize)->Result{if index>=512{return Err(os_k920_page_tree::Error::Shared);}self.validate_shared_table(address)?;Ok(unsafe{read_volatile((address as *const u64).add(index))})} fn active_root(&self)->u64{let value:u64;unsafe{asm!("mrs {value}, TTBR0_EL1",value=out(reg)value,options(nomem,nostack,preserves_flags))};value&OS_K920_PAGE_PA_MASK} fn invalidate(&mut self,root:u64,va:Option){ unsafe{asm!("dsb ishst",options(nostack,preserves_flags))};os_k920_page_event(3,root,va.unwrap_or(u64::MAX)); unsafe{asm!("tlbi vmalle1is",options(nostack,preserves_flags))};os_k920_page_event(4,root,va.unwrap_or(u64::MAX)); unsafe{asm!("dsb ish",options(nostack,preserves_flags))};os_k920_page_event(5,root,0);unsafe{asm!("isb",options(nostack,preserves_flags))};os_k920_page_event(6,root,0); } fn select_root(&mut self,root:u64,geometry:os_k920_page_descriptor::Geometry,asid:u16){ let old:u64;unsafe{asm!("mrs {old}, TCR_EL1",old=out(reg)old,options(nomem,nostack,preserves_flags))};let tcr=(old&!63)|(64-geometry.virtual_bits() as u64); let ttbr=root|(asid as u64)<<48; unsafe{asm!("dsb ishst","msr TCR_EL1, {tcr}","msr TTBR0_EL1, {ttbr}",tcr=in(reg)tcr,ttbr=in(reg)ttbr,options(nostack,preserves_flags))}; if tcr!=old {unsafe{asm!("tlbi vmalle1is","dsb ish",options(nostack,preserves_flags))};} unsafe{asm!("isb",options(nostack,preserves_flags))};os_k920_page_event(7,root,asid as u64); } } // Isolated typed ownership adapter. Each private Root bundle keeps its code, // data and user-stack Page tokens until the checked space has returned to the // kernel and complete Tree invalidation precedes retirement. use turkos_kernel::runtime::address_spaces as os_k920_spaces; type OsK920Tree<'a> = os_k820_tree::Tree, 8>; type OsK920Spaces<'a> = os_k920_spaces::Spaces, 1, 1>; struct OsK920Root { root: Option, pages: [Option; 3], owner: u64, } impl OsK920Root { fn address(&self) -> u64 { self.root.as_ref().map_or(0, |root| root.address()) } fn data(&self) -> u64 { self.pages[1].as_ref().map_or(0, |page| page.address()) } } struct OsK920Backend<'a> { tree: OsK920Tree<'a>, kernel_root: u64, configuration: os_k920_spaces::Configuration, quarantined: Option, requests: u64, selections: u64, last_new_root: u64, last_new_data: u64, } impl OsK920Backend<'_> { fn authority(&self, callback: impl FnOnce(&os_k720_frame::Allocator) -> R) -> R { self.tree .with_backend_observation(|backend| backend.with_authority_observation(callback)) } fn capture(&self) -> Result<(), os_k920_spaces::Error> { self.tree.audit().map_err(os_k920_tree_error)?; let owned = core::ptr::addr_of_mut!(OS_K920_OWNED).cast::(); let registry = core::ptr::addr_of_mut!(OS_K920_REGISTRY).cast::(); unsafe { core::ptr::write_bytes(owned, 0, 48); core::ptr::write_bytes(registry, 0, 48) }; for (index, row) in self.tree.owned_records().enumerate() { if index >= 8 { return Err(os_k920_spaces::Error::Backing); } for (field, value) in [ row.address, row.generation, u64::from(row.table), u64::from(row.root), u64::from(row.quarantined), row.mappings as u64, ] .into_iter() .enumerate() { unsafe { write_volatile(owned.add(index * 6 + field), value) }; } } self.tree.with_backend_observation(|backend| { for (index, (row, central_generation)) in backend.observed_handles().enumerate() { for (field, value) in [ row.address, row.owner, row.generation, row.pins as u64, u64::from(row.quarantined), central_generation, ] .into_iter() .enumerate() { unsafe { write_volatile(registry.add(index * 6 + field), value) }; } } }); self.authority(|authority| { os_k920_capture_inventory( authority, core::ptr::addr_of_mut!(OS_K920_AUTHORITY_LIVE).cast(), ) }); Ok(()) } fn discard(&mut self, bundle: &mut OsK920Root) -> Result<(), os_k920_spaces::Error> { if let Some(root) = bundle.root.as_ref() { self.tree.detach_all(root).map_err(os_k920_tree_error)?; } for page in &mut bundle.pages { if let Some(token) = page.take() { if let Err((error, token)) = self.tree.release_page(token) { *page = Some(token); return Err(os_k920_tree_error(error)); } } } if let Some(root) = bundle.root.take() { if let Err((error, root)) = self.tree.release_root(root) { bundle.root = Some(root); return Err(os_k920_tree_error(error)); } } self.tree.audit().map_err(os_k920_tree_error) } fn user_code(&self, bundle: &OsK920Root) -> Result<(), os_k920_spaces::Error> { unsafe extern "C" { static __os_k920_user_code: u8; static __os_k920_user_code_end: u8; } let start = core::ptr::addr_of!(__os_k920_user_code) as usize; let end = core::ptr::addr_of!(__os_k920_user_code_end) as usize; let length = end .checked_sub(start) .filter(|length| *length > 0 && *length <= 4096) .ok_or(os_k920_spaces::Error::Backing)?; let code = bundle.pages[0] .as_ref() .ok_or(os_k920_spaces::Error::Backing)? .address(); for offset in 0..length { unsafe { write_volatile( (code as *mut u8).add(offset), read_volatile((start as *const u8).add(offset)), ) }; } let ctr: u64; unsafe { asm!("mrs {ctr}, CTR_EL0",ctr=out(reg)ctr,options(nomem,nostack,preserves_flags)) }; let dline = 4u64 << (ctr >> 16 & 15); let iline = 4u64 << (ctr & 15); if dline > 4096 || iline > 4096 { return Err(os_k920_spaces::Error::Backing); } let mut address = code; while address < code + 4096 { unsafe { asm!("dc cvau, {address}",address=in(reg)address,options(nostack,preserves_flags)) }; address += dline; } unsafe { asm!("dsb ish", options(nostack, preserves_flags)) }; address = code; while address < code + 4096 { unsafe { asm!("ic ivau, {address}",address=in(reg)address,options(nostack,preserves_flags)) }; address += iline; } unsafe { asm!("dsb ish", "isb", options(nostack, preserves_flags)) }; os_k920_word(60, length as u64); Ok(()) } } impl os_k920_spaces::Backend for OsK920Backend<'_> { type Root = OsK920Root; fn acquire_root(&mut self, owner: u64) -> Result { self.requests += 1; let root = self.tree.new_draft_root().map_err(os_k920_tree_error)?; let mut bundle = OsK920Root { root: Some(root), pages: core::array::from_fn(|_| None), owner, }; let built = (|| { for page in &mut bundle.pages { *page = Some(self.tree.allocate_page().map_err(os_k920_tree_error)?); } self.user_code(&bundle)?; let root = bundle.root.as_ref().ok_or(os_k920_spaces::Error::Backing)?; for index in 0..2 { let raw = unsafe { read_volatile((self.kernel_root as *const u64).add(index)) }; if raw & 3 != 3 { return Err(os_k920_spaces::Error::Backing); } self.tree .share_table(root, (index as u64) << 30, 1, raw & OS_K920_PAGE_PA_MASK) .map_err(os_k920_tree_error)?; } for (index, va, permissions) in [ ( 0, mmu::USER_CODE_VA, os_k820_descriptor::Permissions { user: true, write: false, execute: true, }, ), ( 1, mmu::USER_DATA_VA, os_k820_descriptor::Permissions { user: true, write: true, execute: false, }, ), ( 2, mmu::USER_STACK_VA, os_k820_descriptor::Permissions { user: true, write: true, execute: false, }, ), ] { self.tree .map( root, va, bundle.pages[index] .as_ref() .ok_or(os_k920_spaces::Error::Backing)?, os_k820_descriptor::Memory::Normal, permissions, ) .map_err(os_k920_tree_error)?; } self.tree.publish_root(root).map_err(os_k920_tree_error)?; self.tree.audit().map_err(os_k920_tree_error)?; Ok::<(), os_k920_spaces::Error>(()) })(); if let Err(error) = built { if self.discard(&mut bundle).is_err() { self.quarantined = Some(bundle); return Err(os_k920_spaces::Error::Retirement); } return Err(error); } self.last_new_root = bundle.address(); self.last_new_data = bundle.data(); Ok(bundle) } fn root_address(&self, root: &OsK920Root) -> u64 { root.address() } fn validate_root(&self, root: &OsK920Root) -> Result<(), os_k920_spaces::Error> { if root.owner == 0 || root.address() == 0 || !self .tree .owned_records() .any(|record| record.root && record.address == root.address()) { return Err(os_k920_spaces::Error::Backing); } self.tree.audit().map_err(os_k920_tree_error) } fn invalidate_recycled(&mut self, asid: u16, old_generation: u64, new_generation: u64) { let (ttbr, _, _, _) = os_k920_registers(); if ttbr != self.kernel_root || asid != 1 || old_generation == 0 || old_generation.checked_add(1) != Some(new_generation) { park(); } let operand = self.configuration.tlbi_operand(asid).unwrap_or_else(|_|park()); unsafe { asm!("dsb ish", "tlbi aside1is, {operand}", "dsb ish", "isb", operand=in(reg)operand, options(nostack,preserves_flags)) }; } fn select(&mut self, root: &OsK920Root, asid: u16) -> Result<(), os_k920_spaces::Error> { self.validate_root(root)?; let ttbr = self.configuration.ttbr(root.address(), asid)?; let before_tcr = os_k920_registers().1; self.tree .activate( root.root.as_ref().ok_or(os_k920_spaces::Error::Backing)?, asid, ) .map_err(os_k920_tree_error)?; if os_k920_registers().0 != ttbr || os_k920_registers().1 != before_tcr { return Err(os_k920_spaces::Error::Configuration); } self.selections += 1; Ok(()) } fn restore_kernel(&mut self) -> Result<(), os_k920_spaces::Error> { let geometry = self.tree.geometry(); self.tree .restore_shared_root(self.kernel_root, geometry, 0) .map_err(os_k920_tree_error)?; if os_k920_registers().0 != self.kernel_root { return Err(os_k920_spaces::Error::Configuration); } Ok(()) } fn retire_root( &mut self, mut root: OsK920Root, _asid: u16, _generation: u64, ) -> Result<(), (os_k920_spaces::Error, OsK920Root)> { if os_k920_registers().0 & OS_K920_PAGE_PA_MASK == root.address() { return Err((os_k920_spaces::Error::Active, root)); } if let Err(error) = self.validate_root(&root) { return Err((error, root)); } self.discard(&mut root).map_err(|error| (error, root)) } } use turkos_kernel::runtime::exceptions as os_k920_exception; // The private controller is initialized only while DAIF.I is masked. Its // mutable borrow lasts one checked operation; a live token holds no borrow. static mut OS_K920_CONTROLLER: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K920_KERNEL_FRAME: u64 = 0; #[unsafe(no_mangle)] static mut OS_K920_IRQ_SEEN: u64 = 0; #[unsafe(no_mangle)] static mut OS_K920_IRQ_WITNESS: [u64; 10] = [0; 10]; #[unsafe(no_mangle)] static mut OS_K920_STATE: [u64; 32] = [0; 32]; const OS_K920_ZERO_FRAME: os_k920_exception::Frame = os_k920_exception::Frame { current_el: 0, canary: 0, ..os_k920_exception::Frame::EMPTY }; #[unsafe(no_mangle)] static mut OS_K920_FRAMES: [os_k920_exception::Frame; 8] = [OS_K920_ZERO_FRAME; 8]; #[unsafe(no_mangle)] static mut OS_K920_FRAME_ADDRESSES: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K920_NESTING: [u64; 128] = [0; 128]; #[unsafe(no_mangle)] static mut OS_K920_CANDIDATES: [u64; 56] = [0; 56]; #[unsafe(no_mangle)] static mut OS_K920_CANDIDATE_RESULTS: [u64; 112] = [0; 112]; unsafe extern "C" { static mut OS_K920_FATAL_FRAMES: u8; } // Negative frames occupy normal trace slots4/5. This inert ELF alias retains // both exact848-byte frames without a second copy of the same measured bytes. global_asm!(".global OS_K920_FATAL_FRAMES\n.set OS_K920_FATAL_FRAMES, OS_K920_FRAMES+3392\n.size OS_K920_FATAL_FRAMES, 1696"); #[unsafe(no_mangle)] static mut OS_K920_FATAL: [u64; 32] = [0; 32]; fn os_k920_state(index: usize) -> u64 { unsafe { read_volatile(core::ptr::addr_of!(OS_K920_STATE).cast::().add(index)) } } fn os_k920_set_state(index: usize, value: u64) { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K920_STATE).cast::().add(index), value) }; } fn os_k920_stack() -> Result { os_k920_exception::Stack::new(os_k920_state(11), os_k920_state(12)) .map_err(|_| "exception-stack-bounds") } fn os_k920_registers() -> (u64, u64, u64, u64) { let (ttbr, tcr, vbar, daif): (u64, u64, u64, u64); unsafe { asm!("mrs {ttbr}, TTBR0_EL1", "mrs {tcr}, TCR_EL1", "mrs {vbar}, VBAR_EL1", "mrs {daif}, DAIF", ttbr=out(reg)ttbr,tcr=out(reg)tcr,vbar=out(reg)vbar, daif=out(reg)daif, options(nomem,nostack,preserves_flags)) }; (ttbr, tcr, vbar, daif) } fn os_k920_snapshot(index: usize, phase: u64, frame: u64, slot: u64) { if index >= 16 { park(); } let controller = core::ptr::addr_of!(OS_K920_CONTROLLER) .cast::>(); let snapshot = unsafe { (&*controller).snapshot() }; let (ttbr, _, _, _) = os_k920_registers(); for (field, value) in [phase, frame, slot, snapshot.depth as u64, snapshot.issued, snapshot.completed, snapshot.high_water as u64, ttbr].into_iter().enumerate() { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K920_NESTING).cast::() .add(index*8+field),value) }; } } fn os_k920_save_frame(frame: &os_k920_exception::Frame, address: u64) -> usize { let index = os_k920_state(9) as usize; if index >= 8 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K920_FRAMES) .cast::().add(index), *frame); write_volatile(core::ptr::addr_of_mut!(OS_K920_FRAME_ADDRESSES) .cast::().add(index), address); } os_k920_set_state(9,index as u64+1); index } const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k920_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); // Read-only continuation API. This exact raw bit25 state was acquired only // from the trusted real EL1t frame before the normal timer/IRQ probes. #[unsafe(no_mangle)] #[inline(never)] pub extern "C" fn __os_k920_kernel_tco_observed()->u64{os_k920_state(30)} // Narrow extension for the raw bit25 status observed at trusted EL1t ingress. // SPSR_EL1.TCO is preserved across exception return (Arm DDI0601 SPSR_EL1). // This helper does not write the CPU frame or install a user-return policy. // The original full frame, including measured TCO, remains the ERET source. const OS_K920_KERNEL_TCO:u64=1<<25; fn os_k920_kernel_irq_return(frame:&os_k920_exception::Frame,address:u64, stack:os_k920_exception::Stack,text_start:u64,text_end:u64,observed_tco:u64) ->Result { if observed_tco&!OS_K920_KERNEL_TCO!=0||frame.spsr&OS_K920_KERNEL_TCO!=observed_tco { return Err(os_k920_exception::Error::Status); } // The core's trusted kernel gate checks every remaining status bit, // source EL, vector, owner stack interval and executable kernel entry. let mut baseline=*frame;baseline.spsr&=!OS_K920_KERNEL_TCO; os_k920_exception::authorize_kernel_irq_return(&baseline,address,stack,text_start,text_end) } #[unsafe(no_mangle)] static mut OS_K920_IRQ_ADDRESSES:[u64;2]=[0;2]; fn os_k920_irq_addresses(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_IRQ_ADDRESSES).cast::(),core::ptr::addr_of!(OS_K660_IRQ_ACTIVE) as u64);write_volatile(core::ptr::addr_of_mut!(OS_K920_IRQ_ADDRESSES).cast::().add(1),core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER) as u64)}} #[unsafe(no_mangle)] static mut OS_K920_EVIDENCE: [u64; 80] = [0; 80]; #[unsafe(no_mangle)] static mut OS_K920_DIAGNOSTIC_CHECKPOINT: [u64; 6] = [0; 6]; #[unsafe(no_mangle)] static mut OS_K920_PRE_IRQ_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K920_PRE_IRQ_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K920_PRE_IRQ_LAST: u64 = 0; #[unsafe(no_mangle)] static mut OS_K920_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K920_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K920_AUTHORITY_LIVE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K920_CONTROL_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; unsafe extern "C" { static mut OS_K920_CONTROL_AUTHORITY_AFTER: u8; } // This view is measured at checkpoint1 immediately after the controls. Later // live/final snapshots use the same buffer under their own explicit stages. global_asm!(".global OS_K920_CONTROL_AUTHORITY_AFTER\n.set OS_K920_CONTROL_AUTHORITY_AFTER, OS_K920_AUTHORITY_LIVE\n.size OS_K920_CONTROL_AUTHORITY_AFTER, 2624"); #[unsafe(no_mangle)] static mut OS_K920_OWNED: [u64; 48] = [0; 48]; #[unsafe(no_mangle)] static mut OS_K920_REGISTRY: [u64; 48] = [0; 48]; #[unsafe(no_mangle)] static mut OS_K920_CAPTURE_GATE: u8 = 0; #[unsafe(no_mangle)] static mut OS_K920_IMAGE_SPAN: [u64; 24] = [0; 24]; static mut OS_K920_SPACES: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K920_HANDLE: Option = None; fn os_k920_word(index: usize, value: u64) { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K920_EVIDENCE).cast::().add(index),value) }; } fn os_k920_diagnostics(offset:usize) { // Read the actual EventLog by immutable observation, without cloning its // record array or advancing/resetting its real sequence. let log=unsafe{&*core::ptr::addr_of!(OS_K680_LOG)}.as_ref().unwrap_or_else(||park()); for(index,value)in [log.boot().value(),log.next_sequence(),log.len() as u64].into_iter().enumerate(){ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_DIAGNOSTIC_CHECKPOINT).cast::().add(offset+index),value)}; } } fn os_k920_tree_error(error: os_k820_tree::Error) -> os_k920_spaces::Error { match error { os_k820_tree::Error::Capacity | os_k820_tree::Error::NoMemory => os_k920_spaces::Error::NoMemory, os_k820_tree::Error::ActiveRoot => os_k920_spaces::Error::Active, _ => os_k920_spaces::Error::Backing, } } fn os_k920_capture_inventory(authority: &os_k720_frame::Allocator, output: *mut u64) { for (index,value) in os_k740_authority_inventory(authority).words.into_iter().enumerate() { unsafe { write_volatile(output.add(index),value) }; } } fn os_k920_capture(stage:u64)->Result<(),&'static str> { unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K920_CAPTURE_GATE),0)}; write_str("TURKOS/OS/K920: CAPTURE STAGE=0x");write_hex(stage as usize); write_str(" GATE=0x");write_hex(core::ptr::addr_of!(OS_K920_CAPTURE_GATE) as usize);write_line(" READY"); let(start,frequency):(u64,u64);unsafe{asm!("mrs {start}, CNTPCT_EL0","mrs {frequency}, CNTFRQ_EL0",start=out(reg)start,frequency=out(reg)frequency,options(nomem,nostack,preserves_flags))}; let end=start.checked_add(frequency.checked_mul(90).ok_or("exception-capture-frequency")?).ok_or("exception-capture-deadline")?; while unsafe{read_volatile(core::ptr::addr_of!(OS_K920_CAPTURE_GATE))}!=1 { let now:u64;unsafe{asm!("mrs {now}, CNTPCT_EL0",now=out(reg)now,options(nomem,nostack,preserves_flags))}; if now>=end{return Err("exception-capture-timeout");}core::hint::spin_loop(); }Ok(()) } fn os_k920_handle()->os_k920_spaces::Handle { unsafe{read_volatile(core::ptr::addr_of!(OS_K920_HANDLE))}.unwrap_or_else(||park()) } fn os_k920_user()->os_k920_exception::User { os_k920_exception::User {owner:os_k920_state(5),generation:os_k920_state(6),root:os_k920_state(3),asid:os_k920_state(4) as u16} } fn os_k920_error(error:os_k920_exception::Error)->u64 { use os_k920_exception::Error; match error {Error::WrongOrigin=>1,Error::Stack=>2,Error::Canary=>3,Error::WrongLevel=>4, Error::Root=>5,Error::Owner=>6,Error::Entry=>7,Error::Status=>8,Error::Nesting=>9, Error::Fatal=>10,_=>255} } struct OsK920ReturnAuthority; impl os_k920_exception::ReturnAuthority for OsK920ReturnAuthority { fn owner_root(&self,user:os_k920_exception::User)->Result<(),os_k920_exception::Error> { os_k920_set_state(24,os_k920_state(24)+1); let spaces=unsafe{&*core::ptr::addr_of!(OS_K920_SPACES).cast::>()}; let observed=spaces.observe(os_k920_handle(),user.owner).map_err(|_|os_k920_exception::Error::Owner)?; if user.owner!=os_k920_state(5)||user.generation!=observed.generation{return Err(os_k920_exception::Error::Owner);} let ttbr=spaces.configuration().ttbr(observed.root,observed.asid).map_err(|_|os_k920_exception::Error::Root)?; if user.root!=observed.root||user.asid!=observed.asid||os_k920_registers().0!=ttbr{return Err(os_k920_exception::Error::Root);} Ok(()) } fn executable(&self,user:os_k920_exception::User,entry:u64)->Result<(),os_k920_exception::Error> { os_k920_set_state(25,os_k920_state(25)+1); let spaces=unsafe{&*core::ptr::addr_of!(OS_K920_SPACES).cast::>()}; spaces.with_root_observation(os_k920_handle(),user.owner,|backend,bundle| { let root=bundle.root.as_ref().ok_or(os_k920_exception::Error::Root)?; let translation=backend.tree.walk(root,entry).map_err(|_|os_k920_exception::Error::Entry)?; let code=bundle.pages[0].as_ref().ok_or(os_k920_exception::Error::Root)?.address(); if translation.memory!=os_k820_descriptor::Memory::Normal||!translation.permissions.user ||!translation.permissions.execute||translation.permissions.write ||translation.physical&!4095!=code{return Err(os_k920_exception::Error::Entry);}Ok(()) }).map_err(|_|os_k920_exception::Error::Owner)? } fn writable_stack(&self,user:os_k920_exception::User,sp:u64)->Result<(),os_k920_exception::Error> { os_k920_set_state(26,os_k920_state(26)+1); let spaces=unsafe{&*core::ptr::addr_of!(OS_K920_SPACES).cast::>()}; spaces.with_root_observation(os_k920_handle(),user.owner,|backend,bundle| { let root=bundle.root.as_ref().ok_or(os_k920_exception::Error::Root)?; let address=sp.checked_sub(1).ok_or(os_k920_exception::Error::Stack)?; let translation=backend.tree.walk(root,address).map_err(|_|os_k920_exception::Error::Stack)?; let page=bundle.pages[2].as_ref().ok_or(os_k920_exception::Error::Root)?.address(); if translation.memory!=os_k820_descriptor::Memory::Normal||!translation.permissions.user ||!translation.permissions.write||translation.permissions.execute ||translation.physical&!4095!=page{return Err(os_k920_exception::Error::Stack);}Ok(()) }).map_err(|_|os_k920_exception::Error::Owner)? } } fn os_k920_candidate_gates(frame:&mut os_k920_exception::Frame,address:u64) { let original=*frame; let spaces=unsafe{&*core::ptr::addr_of!(OS_K920_SPACES).cast::>()}; spaces.with_backend_observation(|backend|backend.authority(|authority|os_k920_capture_inventory(authority,core::ptr::addr_of_mut!(OS_K920_CONTROL_AUTHORITY_BEFORE).cast()))); os_k920_word(47,unsafe{read_volatile(core::ptr::addr_of!(OS_K920_PAGE_EVENT_COUNT))}); let expected=[1,2,3,4,5,7,2,6]; for index in 0..8 { let mut candidate=original; let mut user=os_k920_user(); match index {0=>candidate.spsr=(candidate.spsr&!31)|5,1=>candidate.sp_el0|=1, 2=>candidate.canary^=1,3=>candidate.current_el=8,4=>user.root+=4096, 5=>candidate.elr=core::ptr::addr_of!(OS_K920_EVIDENCE) as u64, 6=>candidate.sp_el0=mmu::USER_DATA_VA+4096,_=>user.generation+=1} // Mutate the actual saved CPU frame with IRQ masked. The gate must // refuse before any ERET; restore every byte before normal egress. *frame=candidate; let before=[os_k920_state(24),os_k920_state(25),os_k920_state(26)]; let error=os_k920_exception::authorize_user_return(frame,address,os_k920_stack().unwrap_or_else(|_|park()),user,&OsK920ReturnAuthority) .err().unwrap_or_else(||park()); for(field,value)in [address,frame.elr,frame.spsr,frame.sp_el0,frame.canary,frame.current_el,frame.vector_slot].into_iter().enumerate(){ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_CANDIDATES).cast::().add(index*7+field),value)}; } let after=[os_k920_state(24),os_k920_state(25),os_k920_state(26)]; *frame=original; let increment=match index{0..=3=>[0,0,0],4|7=>[1,0,0],5=>[1,1,0],_=>[1,1,1]}; if *frame!=original||os_k920_error(error)!=expected[index] ||(0..3).any(|field|after[field]!=before[field]+increment[field]){park();} for(field,value)in [address,os_k920_error(error),user.root,user.owner,user.generation,user.asid as u64,os_k920_registers().0, before[0],before[1],before[2],after[0],after[1],after[2],u64::from(*frame==original)].into_iter().enumerate(){ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_CANDIDATE_RESULTS).cast::().add(index*14+field),value)}; } } spaces.with_backend_observation(|backend|backend.authority(|authority|os_k920_capture_inventory(authority,core::ptr::addr_of_mut!(OS_K920_CONTROL_AUTHORITY_AFTER).cast()))); os_k920_word(48,unsafe{read_volatile(core::ptr::addr_of!(OS_K920_PAGE_EVENT_COUNT))}); for index in 0..328{if unsafe{read_volatile(core::ptr::addr_of!(OS_K920_CONTROL_AUTHORITY_BEFORE).cast::().add(index))} !=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_CONTROL_AUTHORITY_AFTER).cast::().add(index))}{park();}} if unsafe{read_volatile(core::ptr::addr_of!(OS_K920_EVIDENCE).cast::().add(47))} !=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_EVIDENCE).cast::().add(48))}{park();} os_k920_word(30,8); } fn os_k920_arm_timer()->u64 { let(now,frequency):(u64,u64);unsafe{asm!("mrs {now}, CNTPCT_EL0","mrs {frequency}, CNTFRQ_EL0",now=out(reg)now,frequency=out(reg)frequency,options(nomem,nostack,preserves_flags))}; let ticks=turkos_kernel::irq::timer_ticks_for_ns(frequency,1_000_000).unwrap_or_else(||park()); let deadline=now.checked_add(ticks).unwrap_or_else(||park());os_k920_set_state(21,deadline); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_IRQ_SEEN),0);asm!("msr CNTP_CVAL_EL0, {deadline}","msr CNTP_CTL_EL0, {enable}","isb",deadline=in(reg)deadline,enable=in(reg)1u64,options(nostack,preserves_flags))};deadline } fn os_k920_irq(frame:&os_k920_exception::Frame,address:u64)->os_k920_exception::KernelIrqReturn { let expected_root=if os_k920_state(1)==2{os_k920_state(2)}else{os_k920_state(3)|(os_k920_state(4)<<48)}; if os_k920_registers().0!=expected_root{park();} unsafe extern "C"{static _text_start:u8;static _text_end:u8;} let gate=os_k920_kernel_irq_return(frame,address,os_k920_stack().unwrap_or_else(|_|park()), core::ptr::addr_of!(_text_start) as u64,core::ptr::addr_of!(_text_end) as u64,__os_k920_kernel_tco_observed()).unwrap_or_else(|_|park()); let(iar,now):(u64,u64);unsafe{asm!("mrs {iar}, ICC_IAR1_EL1","msr CNTP_CTL_EL0, xzr","isb","mrs {now}, CNTPCT_EL0",iar=out(reg)iar,now=out(reg)now,options(nomem,nostack,preserves_flags))}; if !matches!(turkos_kernel::irq::classify_iar(iar,30),turkos_kernel::irq::IarClass::Expected)||now!;}unsafe{__os_k920_irq_fatal()}} unsafe{asm!("msr ICC_EOIR1_EL1, {iar}","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags));let control:u64;asm!("mrs {control}, ICC_CTLR_EL1",control=out(reg)control,options(nomem,nostack,preserves_flags));if control&2!=0{asm!("msr ICC_DIR_EL1, {iar}","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags));}write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),0);write_volatile(core::ptr::addr_of_mut!(OS_K920_IRQ_SEEN),1)}; gate } struct OsK920FatalSink { polls:usize } impl os_k920_exception::FatalSink for OsK920FatalSink { fn try_byte(&mut self,_:u8)->bool { self.polls+=1; // The fault control enters while the real serial line lock is held. // Its bounded sink observes that lock; it never acquires a lock or // borrows Runtime, the central authority, Tree or Registry. if !SERIAL_LINE_LOCK.is_locked(){park();}false } } fn os_k920_fatal(frame:&os_k920_exception::Frame,error:os_k920_exception::Error)->! { unsafe extern "C"{fn __os_k920_irq_fatal()->!;fn __os_k920_fatal_again()->!;} let first=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_FATAL).cast::())}==0; let expected_pc=if first{__os_k920_irq_fatal as *const () as u64}else{__os_k920_fatal_again as *const () as u64}; let expected_error=if first{os_k920_exception::Error::Nesting}else{os_k920_exception::Error::Fatal}; if os_k920_state(1)!=5||error!=expected_error||frame.vector_slot!=4 ||frame.esr>>26!=0x3c||frame.esr&0xffff!=(if first{0x911}else{0x912}) ||frame.elr!=expected_pc||!SERIAL_LINE_LOCK.is_locked(){park();} let controller=core::ptr::addr_of_mut!(OS_K920_CONTROLLER).cast::>(); let action=unsafe{(&mut*controller).begin_fatal(4096)}; let index=usize::from(!first); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_FATAL_FRAMES).cast::().add(index),*frame)}; let snapshot=unsafe{(&*controller).snapshot()}; let output=core::ptr::addr_of_mut!(OS_K920_FATAL).cast::(); for(field,value)in [frame.vector_slot,frame.esr,frame.elr,frame.spsr,frame.interrupted_sp, frame.sp_el0,frame.current_el,os_k920_registers().0,snapshot.depth as u64, snapshot.issued,snapshot.completed,snapshot.fatal_entries as u64].into_iter().enumerate(){ unsafe{write_volatile(output.add(8+index*12+field),value)}; } if first { if action!=(os_k920_exception::FatalAction::Report{polls:4096}){park();} let mut sink=OsK920FatalSink{polls:0};let emission=os_k920_exception::emit_bounded(&mut sink,b"K920 fatal",4096); if emission.polls!=4096||emission.bytes!=0||emission.complete||sink.polls!=4096{park();} unsafe{write_volatile(output.add(1),emission.polls as u64);write_volatile(output.add(2),sink.polls as u64); write_volatile(output.add(3),emission.bytes as u64);write_volatile(output.add(4),u64::from(SERIAL_LINE_LOCK.is_locked())); write_volatile(output.add(0),1)}; unsafe{__os_k920_fatal_again()} } if action!=os_k920_exception::FatalAction::StopOnce||snapshot.depth!=2 ||snapshot.fatal_entries!=2||snapshot.issued!=4||snapshot.completed!=2{park();} os_k920_diagnostics(3); unsafe{write_volatile(output.add(5),1); let active=read_volatile(core::ptr::addr_of!(OS_K920_EVIDENCE).cast::().add(50)); write_volatile(output.add(6),read_volatile(active as *const u32) as u64);write_volatile(output.add(7),active); asm!("dsb sy",options(nostack,preserves_flags));write_volatile(output.add(0),2)}; park() } fn os_k920_leave(token:&mut os_k920_exception::EntryToken,index:usize,frame:u64,slot:u64) { let controller=core::ptr::addr_of_mut!(OS_K920_CONTROLLER).cast::>(); unsafe{(&mut*controller).leave(token)}.unwrap_or_else(|_|park()); os_k920_snapshot(index*2+1,1,frame,slot); } #[unsafe(no_mangle)] extern "C" fn os_k920_dispatch(pointer:*mut os_k920_exception::Frame)->u64 { if os_k920_state(0)!=1||os_k920_registers().2!=os_k920_state(14){park();} let address=pointer as u64; // A CPU supplied pointer is checked against the trusted stack interval // before the first Rust reference to its complete 848-byte frame. let stack=os_k920_stack().unwrap_or_else(|_|park()); stack.frame_address(address).unwrap_or_else(|_|park()); let frame=unsafe{&mut*pointer}; let index=os_k920_save_frame(frame,address); os_k920_exception::validate_ingress(frame,address,stack).unwrap_or_else(|error|{os_k920_set_state(31,os_k920_error(error));park()}); let controller=core::ptr::addr_of_mut!(OS_K920_CONTROLLER).cast::>(); let mut token=match unsafe{(&mut*controller).enter(frame,address)} { Ok(token)=>token,Err(error)=>os_k920_fatal(frame,error), }; os_k920_snapshot(index*2,0,address,frame.vector_slot); unsafe extern "C"{fn __os_k920_clobber_witness();static __os_k920_sp0_brk:u8;static __os_k920_sp0_after:u8;} if frame.vector_slot==0&&os_k920_state(1)==1 { if frame.esr>>26!=0x3c||frame.esr&0xffff!=0x920 ||frame.elr!=core::ptr::addr_of!(__os_k920_sp0_brk) as u64 ||frame.spsr&!(0xf0000000|0x3c0|4|OS_K920_KERNEL_TCO)!=0 ||frame.sp_el0.checked_add(8192)!=Some(frame.interrupted_sp){park();} // This narrow trusted continuation is a real EL1t BRK probe. Its // source mode is already checked; it cannot be selected by EL0. os_k920_set_state(30,frame.spsr&OS_K920_KERNEL_TCO); frame.elr=core::ptr::addr_of!(__os_k920_sp0_after) as u64; os_k920_leave(&mut token,index,address,0);return 0; } if frame.vector_slot==5&&matches!(os_k920_state(1),2|3|5) { let gate=os_k920_irq(frame,address); if gate.frame_address()!=address||gate.entry()!=frame.elr{park();} os_k920_leave(&mut token,index,address,5); unsafe{__os_k920_clobber_witness()};return 0; } if frame.vector_slot!=8||!matches!(os_k920_state(1),3|4|5){park();} let original=*frame; if frame.esr>>26==0x15&&frame.esr&0xffff==0x920&&matches!(os_k920_state(1),3|5) { if frame.elr!=mmu::USER_CODE_VA+4{park();} os_k920_candidate_gates(frame,address); unsafe{(&*core::ptr::addr_of!(OS_K920_SPACES).cast::>()).with_backend_observation(|backend|backend.capture())}.unwrap_or_else(|_|park()); os_k920_capture(1).unwrap_or_else(|_|park()); let _held=if os_k920_state(1)==5{Some(SERIAL_LINE_LOCK.lock())}else{None}; os_k920_arm_timer(); unsafe{asm!("msr DAIFClr, #2",options(nostack,preserves_flags))}; let frequency:u64;unsafe{asm!("mrs {frequency}, CNTFRQ_EL0",frequency=out(reg)frequency,options(nomem,nostack,preserves_flags))}; while unsafe{read_volatile(core::ptr::addr_of!(OS_K920_IRQ_SEEN))}!=1 { let now:u64;unsafe{asm!("mrs {now}, CNTPCT_EL0",now=out(reg)now,options(nomem,nostack,preserves_flags))}; if now>os_k920_state(21)+frequency{park();}core::hint::spin_loop(); } unsafe{asm!("msr DAIFSet, #2","isb",options(nostack,preserves_flags))}; if *frame!=original{park();}os_k920_word(46,1); }else if frame.esr>>26==0x24&&os_k920_state(1)==4 { unsafe extern "C"{static __os_k920_user_code:u8;static __os_k920_user_fault:u8;} let offset=core::ptr::addr_of!(__os_k920_user_fault) as u64-core::ptr::addr_of!(__os_k920_user_code) as u64; if frame.far!=0x80002000||frame.esr&63!=7||frame.esr&(1<<6|1<<7|1<<10)!=0 ||frame.elr!=mmu::USER_CODE_VA+offset+8{park();} frame.elr=frame.elr.checked_add(4).unwrap_or_else(||park()); }else if frame.esr>>26==0x15&&frame.esr&0xffff==0x921&&matches!(os_k920_state(1),3|4) { if frame.elr!=mmu::USER_CODE_VA+0x30{park();} let gate=os_k920_exception::authorize_user_return(frame,address,stack,os_k920_user(),&OsK920ReturnAuthority).unwrap_or_else(|_|park()); if gate.frame_address()!=address{park();} os_k920_leave(&mut token,index,address,8); unsafe{(&mut*core::ptr::addr_of_mut!(OS_K920_SPACES).cast::>()).restore_kernel()}.unwrap_or_else(|_|park()); return 1; }else{park();} let gate=os_k920_exception::authorize_user_return(frame,address,stack,os_k920_user(),&OsK920ReturnAuthority).unwrap_or_else(|_|park()); if gate.frame_address()!=address||gate.entry()!=frame.elr||gate.stack_pointer()!=frame.sp_el0{park();} os_k920_leave(&mut token,index,address,8); unsafe{__os_k920_clobber_witness()};0 } fn os_k920_fixture(boot:&BootData)->Result { let blob=unsafe{core::slice::from_raw_parts(boot.dtb_address as *const u8,boot.dtb_size)}; let mode=turkos_kernel::runtime::platform::chosen_u32(blob,unsafe{&*core::ptr::addr_of!(OS_K700_WORKSPACE)},b"turkos,exceptions-mode") .map_err(|_|"exception-fixture")?.unwrap_or(0); if mode>1||mode==1&&cfg!(feature="os-runtime-product"){return Err("exception-fixture-mode");}Ok(mode) } fn os_k920_inventory_equal(left:*const u64,right:*const u64)->bool { (0..328).all(|index|unsafe{read_volatile(left.add(index))==read_volatile(right.add(index))}) } const OS_K920_REGISTER_WITNESS:[u64;6]=[0x9200000000000000,0x9200000000000030, 0x9200000000000010,0x9200000000000011,0x9200000000003100,0x9200000000003101]; fn os_k920_user_witness(data:u64,index:usize)->Result<(),&'static str> { let frame=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_FRAMES).cast::().add(index))}; let expected=[OS_K920_REGISTER_WITNESS[0],OS_K920_REGISTER_WITNESS[1],OS_K920_REGISTER_WITNESS[2], OS_K920_REGISTER_WITNESS[3],OS_K920_REGISTER_WITNESS[4],OS_K920_REGISTER_WITNESS[5], mmu::USER_STACK_TOP,frame.fpcr as u64,frame.fpsr as u64]; for(index,value)in expected.into_iter().enumerate(){if unsafe{read_volatile((data as *const u64).add(index))}!=value{return Err("exception-real-eret-register-witness");}} for offset in 72..4096{if unsafe{read_volatile((data as *const u8).add(offset))}!=0{return Err("exception-user-witness-span");}}Ok(()) } fn os_k920_owned_kernel_leaf(tables:&mmu::Tables,address:u64)->Result<(u64,u64),&'static str> { let mut table=tables.root;let mut visited=[0u64;3]; for(level,shift)in [30,21,12].into_iter().enumerate(){ if table&4095!=0||table==0||visited[..level].contains(&table) ||!tables.tagged_frames().any(|owned|owned.address==table){return Err("exception-image-table-owner");} visited[level]=table; let slot=table+((address>>shift)&511)*8;let raw=unsafe{read_volatile(slot as *const u64)}; if level==2{return Ok((slot,raw));} let next=raw&OS_K920_PAGE_PA_MASK; if raw!=(next|3){return Err("exception-image-table-descriptor");}table=next; }Err("exception-image-table-depth") } fn os_k920_image_span(tables:&mmu::Tables)->Result<(),&'static str> { let start=core::ptr::addr_of!(_image_start) as u64; let end=core::ptr::addr_of!(_image_end) as u64; let guard=end.checked_add(4095).ok_or("exception-image-guard-overflow")?&!4095; let last=end.checked_sub(1).ok_or("exception-image-empty")?&!4095; let guard_method=tables.metadata_stack_guard_descriptor_address(guard).ok_or("exception-image-guard-owner")?; let(guard_slot,guard_raw)=os_k920_owned_kernel_leaf(tables,guard)?; let(last_slot,last_raw)=os_k920_owned_kernel_leaf(tables,last)?; if end<=start||end-start>8*1024*1024||guard_method!=guard_slot||guard_raw!=0 ||last_raw&3!=3||last_raw&OS_K920_PAGE_PA_MASK!=last||last_raw&(1<<6|1<<7)!=0 ||last_raw&(1<<53|1<<54)!=(1<<53|1<<54)||last_raw>>2&7!=0{return Err("exception-image-leaf-policy");} let(before_ttbr,before_tcr,_,_)=os_k920_registers();let(guard_par,last_par):(u64,u64); unsafe{asm!("at s1e1r, {guard}","isb","mrs {par}, PAR_EL1",guard=in(reg)guard,par=out(reg)guard_par,options(nostack,preserves_flags)); asm!("at s1e1r, {last}","isb","mrs {par}, PAR_EL1",last=in(reg)last,par=out(reg)last_par,options(nostack,preserves_flags))}; let(after_ttbr,after_tcr,_,_)=os_k920_registers(); if before_ttbr!=tables.root||after_ttbr!=before_ttbr||after_tcr!=before_tcr ||guard_par&1!=1||guard_par>>1&63!=7||last_par&1!=0||last_par&OS_K920_PAGE_PA_MASK!=last{return Err("exception-image-cpu-translation");} for(index,value)in [0x4b3932305350414e,start,end,guard,guard_slot,guard_raw,last,last_slot,last_raw, before_ttbr,before_tcr,guard_par,last_par,after_ttbr,after_tcr,tables.tagged_frames().count() as u64, (guard>>21)-(start>>21)+1,guard_slot&!4095,last_slot&!4095,end-start,guard_method,0,0,0].into_iter().enumerate(){ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_IMAGE_SPAN).cast::().add(index),value)}; }Ok(()) } #[inline(never)] fn os_k920_initialize<'a>(destination:*mut OsK920Spaces<'a>,authority:&'a mut os_k720_frame::Allocator, tables:&'a mmu::Tables,configuration:os_k920_spaces::Configuration,tcr:u64)->Result { let page=OsK920PageBackend::new(authority,tables)?;let identity=page.identity(); let geometry=os_k820_descriptor::Geometry::new(64-(tcr&63) as u8,match tcr>>32&7{0=>32,1=>36,2=>40,3=>42,4=>44,5=>48,_=>return Err("exception-pa-width")}) .map_err(|_|"exception-geometry")?; let tree=os_k820_tree::Tree::new(identity,geometry,page).map_err(|_|"exception-tree")?; let backend=OsK920Backend{tree,kernel_root:tables.root,configuration,quarantined:None,requests:0,selections:0,last_new_root:0,last_new_data:0}; unsafe{destination.write(os_k920_spaces::Spaces::new(identity,configuration,backend).map_err(|_|"exception-spaces")?)};Ok(identity) } fn os_k920_enter(handle:os_k920_spaces::Handle,owner:u64,entry:u64,mode:u64)->Result<(),&'static str> { let spaces=core::ptr::addr_of_mut!(OS_K920_SPACES).cast::>(); let observed=unsafe{(&mut*spaces).activate(handle,owner)}.map_err(|_|"exception-space-select")?; if observed.root!=os_k920_state(3)||observed.asid as u64!=os_k920_state(4) ||observed.generation!=os_k920_state(6){return Err("exception-space-binding");} os_k920_set_state(1,mode); unsafe extern "C"{fn __os_k920_enter_user(entry:u64,stack:u64);} unsafe{__os_k920_enter_user(entry,mmu::USER_STACK_TOP)}; if os_k920_registers().0!=os_k920_state(2){return Err("exception-kernel-return-root");}Ok(()) } fn os_k920_product(tables:&mmu::Tables)->Result<(),&'static str> { os_k920_image_span(tables)?;os_k920_irq_addresses(); os_k920_diagnostics(0);os_k920_diagnostics(3); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_PRE_IRQ_COUNT),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))); write_volatile(core::ptr::addr_of_mut!(OS_K920_PRE_IRQ_ACTIVE),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) as u64); write_volatile(core::ptr::addr_of_mut!(OS_K920_PRE_IRQ_LAST),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)))}; unsafe extern "C"{static __os_k920_vectors:u8;static __os_k920_user_code:u8;static __os_k920_user_code_end:u8;} for pointer in [__os_k920_kernel_tco_observed as *const u8,core::ptr::addr_of!(__os_k920_vectors),core::ptr::addr_of!(__os_k920_user_code),core::ptr::addr_of!(__os_k920_user_code_end).wrapping_sub(1), core::ptr::addr_of!(OS_K920_EVIDENCE).cast(),core::ptr::addr_of!(OS_K920_STATE).cast(),core::ptr::addr_of!(OS_K920_FRAMES).cast(), core::ptr::addr_of!(OS_K920_FRAME_ADDRESSES).cast(),core::ptr::addr_of!(OS_K920_NESTING).cast(),core::ptr::addr_of!(OS_K920_IRQ_WITNESS).cast(), core::ptr::addr_of!(OS_K920_CANDIDATES).cast(),core::ptr::addr_of!(OS_K920_CANDIDATE_RESULTS).cast(), core::ptr::addr_of!(OS_K920_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K920_AUTHORITY_AFTER).cast(),core::ptr::addr_of!(OS_K920_AUTHORITY_LIVE).cast(), core::ptr::addr_of!(OS_K920_CONTROL_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K920_CONTROL_AUTHORITY_AFTER).cast(), core::ptr::addr_of!(OS_K920_OWNED).cast(),core::ptr::addr_of!(OS_K920_REGISTRY).cast(),core::ptr::addr_of!(OS_K920_CAPTURE_GATE).cast(), core::ptr::addr_of!(OS_K920_FATAL).cast(),core::ptr::addr_of!(OS_K920_FATAL_FRAMES).cast(),core::ptr::addr_of!(OS_K920_PAGE_EVENTS).cast(), core::ptr::addr_of!(OS_K920_PAGE_EVENT_COUNT).cast(),core::ptr::addr_of!(OS_K920_IMAGE_SPAN).cast()] {let _=unsafe{read_volatile(pointer)};} write_line("TURKOS/OS/K920: PRODUCT-CONTROL EXCEPTIONS-PROBE-ABSENT");Ok(()) } fn os_k920_work(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData,guard:u64,fatal:bool)->Result<(),&'static str> { if !cfg!(feature="os-runtime-acceptance") && !os_k1100_product_probe_enabled(boot)? {return os_k920_product(tables);} let(ttbr,tcr,vbar,daif)=os_k920_registers(); let(current_el,mmfr0,mpidr,sp):(u64,u64,u64,u64);unsafe{asm!("mrs {el}, CurrentEL","mrs {mmfr0}, ID_AA64MMFR0_EL1","mrs {mpidr}, MPIDR_EL1","mov {sp}, sp",el=out(reg)current_el,mmfr0=out(reg)mmfr0,mpidr=out(reg)mpidr,sp=out(reg)sp,options(nomem,nostack,preserves_flags))}; let low=guard.checked_add(4096).ok_or("exception-guard-overflow")?; let high=low.checked_add(65536).ok_or("exception-stack-overflow")?; let stack=os_k920_exception::Stack::new(low,high).map_err(|_|"exception-stack")?; if current_el!=4||ttbr!=tables.root||daif&128==0||!stack.contains_sp(sp) ||!tables.task_stack_guard_unmapped(guard)||lowcore::ptr::addr_of!(_image_end) as u64{return Err("exception-context");} let configuration=os_k920_spaces::Configuration::from_registers(mmfr0,tcr).map_err(|_|"exception-configuration")?; os_k920_image_span(tables)?;os_k920_irq_addresses();os_k920_set_state(10,1); let before_count=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}; let before_active=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))} as u64; let expected_irq_count=16; if before_count!=expected_irq_count||before_active!=0{return Err("exception-prior-irq");} os_k920_diagnostics(0); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_PRE_IRQ_COUNT),before_count); write_volatile(core::ptr::addr_of_mut!(OS_K920_PRE_IRQ_ACTIVE),before_active); write_volatile(core::ptr::addr_of_mut!(OS_K920_PRE_IRQ_LAST),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)))}; unsafe extern "C"{static __os_k920_vectors:u8;fn __os_k760_workspace_identity()->u64;fn __os_k920_sp0_probe();fn __os_k920_irq_probe();} let vector=core::ptr::addr_of!(__os_k920_vectors) as u64; if vector&2047!=0{return Err("exception-vector-alignment");} let identity=unsafe{__os_k760_workspace_identity()}; unsafe{core::ptr::addr_of_mut!(OS_K920_CONTROLLER).cast::>().write(os_k920_exception::Controller::new(identity,stack).map_err(|_|"exception-controller")?)}; let gic_active=boot.interrupts.gic_redistributor.checked_add(0x10300).ok_or("exception-gic-active")?; for(index,value)in [(0,0x4b39323045584350),(1,ttbr),(2,tcr),(3,vbar),(4,vector),(5,daif),(6,current_el),(7,mmfr0), (8,before_count),(10,sp),(12,low),(13,high),(40,before_active),(44,unsafe{read_volatile(low as *const u64)}), (45,unsafe{read_volatile((high-8) as *const u64)}),(49,identity),(50,gic_active), (51,unsafe{read_volatile(gic_active as *const u32)} as u64),(54,mpidr),(55,u64::from(fatal))]{os_k920_word(index,value);} for(index,value)in [(0,1),(1,1),(2,ttbr),(11,low),(12,high),(13,vbar),(14,vector)]{os_k920_set_state(index,value);} unsafe{asm!("msr VBAR_EL1, {vector}","isb",vector=in(reg)vector,options(nostack,preserves_flags));__os_k920_sp0_probe()};os_k920_set_state(10,2); os_k920_set_state(1,2);os_k920_arm_timer();unsafe{__os_k920_irq_probe()}; if unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}!=before_count+1{return Err("exception-standalone-irq");} let irq_frame=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_FRAMES).cast::().add(1))}; for(index,value)in OS_K920_REGISTER_WITNESS.into_iter().enumerate(){if unsafe{read_volatile(core::ptr::addr_of!(OS_K920_IRQ_WITNESS).cast::().add(index))}!=value{return Err("exception-real-irq-register-witness");}} for(index,value)in [(6,irq_frame.interrupted_sp),(7,irq_frame.fpcr as u64),(8,irq_frame.fpsr as u64),(9,daif)]{ if unsafe{read_volatile(core::ptr::addr_of!(OS_K920_IRQ_WITNESS).cast::().add(index))}!=value{return Err("exception-real-irq-status-witness");}} os_k920_set_state(10,3); let(_,memory,_,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?; let task=tasks.id.ok_or("exception-task")?; if tasks.scheduler.running()!=Some(task){return Err("exception-selected-task");} os_k920_word(14,task.slot as u64);os_k920_word(15,task.generation); os_k920_capture_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K920_AUTHORITY_BEFORE).cast()); let pool=memory.allocator.os_authority().free_regions().iter().rev().find(|row|row.end-row.start>=80*4096).ok_or("exception-pool")?.end-76*4096; os_k920_word(23,pool);os_k920_word(24,pool+131072); os_k920_capture(0)?; let storage=core::ptr::addr_of_mut!(OS_K920_SPACES).cast::>(); let registry=os_k920_initialize(storage,memory.allocator.os_authority_mut(),tables,configuration,tcr)?;os_k920_word(53,registry); let handle=unsafe{(&mut*storage).create(task.generation)}.map_err(|_|"exception-create-space")?; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K920_HANDLE),Some(handle))}; let observed=unsafe{(&*storage).observe(handle,task.generation)}.map_err(|_|"exception-space-observe")?; for(index,value)in [(3,observed.root),(4,observed.asid as u64),(5,task.generation),(6,observed.generation),(7,task.slot as u64)]{os_k920_set_state(index,value);} for(index,value)in [(16,observed.generation),(17,observed.root),(18,observed.asid as u64),(19,observed.asid_generation)]{os_k920_word(index,value);} let pages=unsafe{(&*storage).with_root_observation(handle,task.generation,|_,bundle|core::array::from_fn::<_,3,_>(|index|bundle.pages[index].as_ref().map_or(0,|page|page.address())))}.map_err(|_|"exception-pages")?; for(index,address)in pages.into_iter().enumerate(){os_k920_word(20+index,address);} os_k920_enter(handle,task.generation,mmu::USER_CODE_VA,if fatal{5}else{3})?; // Fatal controls never reach this return, any cleanup or acceptance line. if fatal{return Err("exception-fatal-returned");} os_k920_user_witness(pages[1],2)?;os_k920_word(31,1); unsafe{(&*storage).with_backend_observation(|backend|backend.capture())}.map_err(|_|"exception-after-eret-inventory")?; os_k920_capture(2)?; unsafe extern "C"{static __os_k920_user_code:u8;static __os_k920_user_fault:u8;} let fault=core::ptr::addr_of!(__os_k920_user_fault) as u64-core::ptr::addr_of!(__os_k920_user_code) as u64; os_k920_enter(handle,task.generation,mmu::USER_CODE_VA+fault,4)?; os_k920_user_witness(pages[1],5)?;os_k920_word(31,2); unsafe{(&*storage).with_backend_observation(|backend|backend.capture())}.map_err(|_|"exception-after-fault-inventory")?; os_k920_capture(3)?; let mut returned_addresses=[0u64;8];let returned_count=unsafe{(&*storage).with_backend_observation(|backend|{ let mut count=0;for row in backend.tree.owned_records(){if count>=8{park();}returned_addresses[count]=row.address;count+=1;}count })};if returned_count!=6{return Err("exception-owned-retirement-count");} unsafe{(&mut*storage).release(handle,task.generation)}.map_err(|_|"exception-retirement")?; unsafe{(&*storage).audit()}.map_err(|_|"exception-empty-audit")?; if unsafe{(&*storage).owned_records().count()}!=0{return Err("exception-owned-id-leak");} unsafe{(&*storage).with_backend_observation(|backend|backend.capture())}.map_err(|_|"exception-empty-table-audit")?; unsafe{(&*storage).with_backend_observation(|backend|backend.authority(|authority|os_k920_capture_inventory(authority,core::ptr::addr_of_mut!(OS_K920_AUTHORITY_AFTER).cast())))}; if !os_k920_inventory_equal(core::ptr::addr_of!(OS_K920_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K920_AUTHORITY_AFTER).cast()){ return Err("exception-authority-restoration");} let backend=unsafe{storage.read()}.into_backend().map_err(|_|"exception-extract-empty-backend")?; if backend.tree.owned_records().count()!=0||backend.quarantined.is_some() ||backend.tree.with_backend_observation(|page|page.registry_records().count())!=0{return Err("exception-owned-page-leak");}drop(backend); for address in returned_addresses[..returned_count].iter().copied(){for offset in 0..4096{if unsafe{read_volatile((address as *const u8).add(offset))}!=0{return Err("exception-returned-owned-page-not-erased");}}} os_k920_set_state(0,0); unsafe{asm!("msr VBAR_EL1, {vbar}","isb",vbar=in(reg)vbar,options(nostack,preserves_flags))}; let after=os_k920_registers();let after_sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack,preserves_flags))}; let snapshot=unsafe{(&*core::ptr::addr_of!(OS_K920_CONTROLLER).cast::>()).snapshot()}; let after_count=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}; for(index,value)in [(9,after_count),(11,after_sp),(25,os_k920_state(9)),(26,snapshot.high_water as u64),(27,snapshot.issued), (28,snapshot.completed),(29,snapshot.depth as u64),(38,after.2),(39,u64::from(after.2==vbar)), (41,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))} as u64),(52,unsafe{read_volatile(gic_active as *const u32)} as u64)]{os_k920_word(index,value);} os_k920_diagnostics(3); if after_count!=before_count+2||snapshot.depth!=0||snapshot.issued!=7||snapshot.completed!=7||snapshot.high_water!=2 ||after!=(ttbr,tcr,vbar,daif)||after_sp!=sp||tasks.scheduler.running()!=Some(task) ||unsafe{read_volatile(low as *const u64)}!=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_EVIDENCE).cast::().add(44))} ||unsafe{read_volatile((high-8) as *const u64)}!=unsafe{read_volatile(core::ptr::addr_of!(OS_K920_EVIDENCE).cast::().add(45))}{return Err("exception-final-context");} write_line("TURKOS/OS/K905: ACTUAL-VECTOR-EL-STACK SOURCES-VALIDATED PASS"); write_line("TURKOS/OS/K910: FULL-FRAME GP-SIMD ACTUAL-ERET-WITNESSES PASS"); write_line("TURKOS/OS/K915: ACTUAL-NESTED-TIMER BOUNDED-NESTING FATAL-CONTROLS-SEPARATE PASS"); write_line("TURKOS/OS/K920: LOCAL-GUEST PASS VECTORS=16 FRAME=848 GP-SIMD-ERET NESTED-TIMER=1 RETURN-REFUSALS=8 IRQ=18 OWNERS=RESTORED SCOPE=ONE-CPU-BOOTSTRAP");Ok(()) } fn os_k920_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str> { if os_k920_fixture(boot)?==1 { write_line("TURKOS/OS/K920: FATAL-CONTROL-DEFERRED AFTER-PAGER-SP-RESTORATION");return Ok(()); } os_k920_work(runtime,tables,boot,core::ptr::addr_of!(OS_K900_STACK) as u64,false) } fn os_k920_negative_work(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str> { os_k920_work(runtime,tables,boot,core::ptr::addr_of!(OS_K920_STACK) as u64,true) } fn os_k920_fatal_if_requested(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData) { if os_k920_fixture(boot).unwrap_or_else(|_|park())==1 { if let Err(reason)=os_k920_on_stack(runtime,tables,boot){write_str("TURKOS/OS/EXCEPTIONS: REJECTED reason=");write_line(reason);park();} park(); } } // The previous wrapper has returned. This module takes a sequentially exclusive // lease of its kernel-owned Image-BSS stack, retaining the actual old canaries. unsafe extern "C" { static OS_K920_STACK: u8; } global_asm!(".global OS_K920_STACK\n.set OS_K920_STACK, OS_K900_STACK\n.size OS_K920_STACK, 69632"); #[unsafe(no_mangle)] static mut OS_K920_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K920_ERROR: Option<&'static str> = None; const OS_K920_LOW_CANARY: u64 = 0x4b38_3230_4c4f_5721; const OS_K920_HIGH_CANARY: u64 = 0x4b38_3230_4849_4748; fn os_k920_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K920_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } fn os_k920_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { unsafe extern "C" { static __stack_start: u8; static __stack_top: u8; } let guard = core::ptr::addr_of!(OS_K920_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_low = core::ptr::addr_of!(__stack_start) as u64; let original_top = core::ptr::addr_of!(__stack_top) as u64; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("exceptions-fatal-original-stack-range"); } let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_low + 8) as *const u64) }; if original_canary != 0x4b36_3335_5354_414b || original_second != !0x4b36_3335_5354_414bu64 { return Err("exceptions-fatal-original-stack-canaries"); } // Compare a volatile saved address. LLVM treats distinct extern globals as // different objects; the alias is independently checked in the ELF too. let previous_guard=unsafe{read_volatile(core::ptr::addr_of!(OS_K900_STACK_EVIDENCE).cast::().add(13))}; if guard!=previous_guard||guard%4096!=0||guardcore::ptr::addr_of!(_image_end) as u64||end>0x4020_0000 ||unsafe{read_volatile(low as *const u64)}!=OS_K920_LOW_CANARY ||unsafe{read_volatile(high as *const u64)}!=OS_K920_HIGH_CANARY { return Err("exceptions-fatal-returned-stack"); } if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("exceptions-fatal-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K920_LOW_CANARY); write_volatile(high as *mut u64, OS_K920_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K920_LOW_CANARY), (9, OS_K920_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("exceptions-fatal-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("exceptions-fatal-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k920_stack_word(slot, value); } unsafe extern "C" { fn __os_k920_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k920_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K920_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K920_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K920_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_low + 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("exceptions-fatal-stack-restoration"); } os_k920_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k920_stack_word(17, unsafe { read_volatile((original_low + 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K920: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); Ok(()) } else { Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K920_ERROR)) } .unwrap_or("exceptions-fatal-stack-return")) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k920_accept_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k920_stack_word(4, sp); let result = os_k920_guarded_work( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k920_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K920_ERROR), Some(reason)) }; 1 } } } fn os_k920_guarded_work(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str>{ os_k920_negative_work(runtime,tables,boot) } global_asm!( r#" .section .text.os_k920_stack,"ax" .balign 4 .global __os_k920_on_stack __os_k920_on_stack: adrp x9, OS_K920_STACK_EVIDENCE add x9, x9, :lo12:OS_K920_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K920_STACK add x9, x9, :lo12:OS_K920_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K920_STACK_EVIDENCE add x9, x9, :lo12:OS_K920_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k920_accept_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K920_STACK_EVIDENCE add x9, x9, :lo12:OS_K920_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret "# ); struct OsK940Frames<'a> { authority: &'a mut os_k720_frame::Allocator, leases: [Option; 5], addresses: [u64; 5], } impl OsK940Frames<'_> { fn validates(&self, address: u64) -> Result<(), os_k740_metadata::Error> { let lease = self .leases .iter() .flatten() .find(|lease| lease.address() == address) .ok_or(os_k740_metadata::Error::BadAddress)?; self.authority .validate_lease(lease) .map_err(|_| os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK940Frames<'_> { fn allocate(&mut self) -> Result { let slot = self .leases .iter() .position(Option::is_none) .ok_or(os_k740_metadata::Error::Capacity)?; let lease = self .authority .allocate_at(self.addresses[slot]) .map_err(|_| os_k740_metadata::Error::NoMemory)?; let address = lease.address(); self.leases[slot] = Some(lease); Ok(address) } fn release(&mut self, address: u64) -> Result<(), os_k740_metadata::Error> { let slot = self .leases .iter() .position(|lease| { lease .as_ref() .is_some_and(|lease| lease.address() == address) }) .ok_or(os_k740_metadata::Error::BadAddress)?; self.zero_page(address)?; self.authority .release( self.leases[slot] .as_ref() .ok_or(os_k740_metadata::Error::Backend)?, ) .map_err(|_| os_k740_metadata::Error::Backend)?; self.leases[slot] = None; Ok(()) } fn zero_page(&mut self, address: u64) -> Result<(), os_k740_metadata::Error> { self.validates(address)?; for offset in 0..4096 { unsafe { write_volatile((address as *mut u8).add(offset), 0) } } Ok(()) } fn read_byte(&self, address: u64, offset: usize) -> Result { if offset >= 4096 { return Err(os_k740_metadata::Error::BadAddress); } self.validates(address)?; Ok(unsafe { read_volatile((address as *const u8).add(offset)) }) } fn available_pages(&self) -> u64 { self.leases.iter().filter(|lease| lease.is_none()).count() as u64 } } // Trusted single-CPU boundary for the frozen no_std Gic core. Every resource // owns actual private Registry handles and nonclone pins; IRQ code uses the // controller's backend exclusively and never reaches around its records. use turkos_kernel::runtime::gic as os_k940_gic; #[unsafe(no_mangle)] static mut OS_K940_EVENTS: [u64; 768] = [0; 768]; #[unsafe(no_mangle)] static mut OS_K940_EVENT_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K940_NONCE: u64 = 1; fn os_k940_event(code: u64, a: u64, b: u64) { let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K940_EVENT_COUNT)) }; if count >= 256 { park(); } unsafe { let out = core::ptr::addr_of_mut!(OS_K940_EVENTS) .cast::() .add(count as usize * 3); write_volatile(out, code); write_volatile(out.add(1), a); write_volatile(out.add(2), b); write_volatile(core::ptr::addr_of_mut!(OS_K940_EVENT_COUNT), count + 1); } } #[unsafe(no_mangle)] extern "C" fn __os_k940_gic_identity() -> u64 { let daif: u64; unsafe { asm!("mrs {daif}, DAIF", daif=out(reg)daif, options(nomem,nostack,preserves_flags)) }; if daif & 0x80 == 0 { return 0; } let previous = unsafe { read_volatile(core::ptr::addr_of!(OS_K940_NONCE)) }; let Some(next) = previous.checked_add(1) else { return 0; }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K940_NONCE), next) }; previous } fn os_k940_time() -> u64 { let value: u64; unsafe { asm!("mrs {value}, CNTPCT_EL0", value=out(reg)value, options(nomem,nostack,preserves_flags)) }; value } unsafe fn os_k940_read(base: u64, offset: u64) -> u32 { unsafe { read_volatile((base + offset) as *const u32) } } unsafe fn os_k940_write(base: u64, offset: u64, value: u32) { unsafe { write_volatile((base + offset) as *mut u32, value) } } unsafe fn os_k940_write64(base: u64, offset: u64, value: u64) { unsafe { os_k940_write(base, offset, value as u32); os_k940_write(base, offset + 4, (value >> 32) as u32); } } struct OsK940Resource { intid: u32, owner: u64, handles: [Option; 4], pins: [Option; 4], count: usize, consumed: u16, epoch: u8, } impl OsK940Resource { fn address(&self, index: usize) -> u64 { self.handles .get(index) .and_then(|h| *h) .unwrap_or_else(|| park()) .address() } } struct OsK940Backend<'a> { registry: os_k740_metadata::Registry, 5>, distributor: u64, redistributor: u64, route: u64, net: u64, net_spi: u32, net_trigger: u32, boot_id: u64, suppress_unknown_mask: bool, epoch: u8, } impl OsK940Backend<'_> { fn registers(&self, intid: u32) -> (u64, u64, u64, u32) { if intid < 32 { (self.redistributor, 0x1_0180, 0x1_0100, 1u32 << intid) } else { ( self.distributor, 0x180 + 4 * (intid as u64 / 32), 0x100 + 4 * (intid as u64 / 32), 1u32 << (intid % 32), ) } } fn wait_rwp(&self, private: bool) -> bool { let (base, bit) = if private { (self.redistributor, 8) } else { (self.distributor, 1 << 31) }; for _ in 0..1_000_000 { if unsafe { os_k940_read(base, 0) } & bit == 0 { return true; } } false } fn net_setup(&self, resource: &OsK940Resource) { let base = self.net; unsafe { os_k940_write(base, 0x70, 0); asm!("dsb sy", options(nostack, preserves_flags)); if os_k940_read(base, 0x70) != 0 { park(); } os_k940_write(base, 0x70, 3); os_k940_write(base, 0x14, 0); let low = os_k940_read(base, 0x10); os_k940_write(base, 0x14, 1); let high = os_k940_read(base, 0x10); if low & 32 == 0 || high & 1 == 0 { park(); } os_k940_write(base, 0x24, 0); os_k940_write(base, 0x20, 32); os_k940_write(base, 0x24, 1); os_k940_write(base, 0x20, 1); os_k940_write(base, 0x70, 11); if os_k940_read(base, 0x70) != 11 { park(); } os_k940_write(base, 0x30, 0); if os_k940_read(base, 0x34) < 32 || os_k940_read(base, 0x44) != 0 { park(); } os_k940_write(base, 0x38, 32); os_k940_write64(base, 0x80, resource.address(0)); os_k940_write64(base, 0x90, resource.address(1)); os_k940_write64(base, 0xa0, resource.address(2)); for index in 0..32u64 { write_volatile( (resource.address(0) + index * 16) as *mut turkos_kernel::virtio_ring::Descriptor, turkos_kernel::virtio_ring::Descriptor { addr_le: (resource.address(3) + index * 96).to_le(), len_le: 96u32.to_le(), flags_le: 2u16.to_le(), next_le: 0, }, ); write_volatile( (resource.address(1) + 4 + index * 2) as *mut u16, (index as u16).to_le(), ); for offset in 72..96u64 { write_volatile((resource.address(3) + index * 96 + offset) as *mut u8, 0xa5); } } asm!("dmb oshst", options(nostack, preserves_flags)); os_k940_write(base, 0x44, 1); os_k940_write(base, 0x70, 15); if os_k940_read(base, 0x44) != 1 || os_k940_read(base, 0x70) != 15 { park(); } write_volatile((resource.address(1) + 2) as *mut u16, 32u16.to_le()); asm!("dmb oshst", options(nostack, preserves_flags)); os_k940_write(base, 0x50, 0); } os_k940_event(0x20, base, 32); } fn validate_packet(&self, resource: &OsK940Resource, id: u32) { let payload = resource.address(3) + id as u64 * 96; let expected = os_k940_packet(self.boot_id, id as usize, resource.epoch); for (index, byte) in expected.iter().enumerate() { if unsafe { read_volatile((payload + 12 + index as u64) as *const u8) } != *byte { park(); } } for offset in 72..96u64 { if unsafe { read_volatile((payload + offset) as *const u8) } != 0xa5 { park(); } } } } fn os_k940_packet(boot: u64, index: usize, epoch: u8) -> [u8; 60] { let mut packet = [0u8; 60]; packet[..6].copy_from_slice(&[0x52, 0x54, 0, 0x12, 0x34, 0x56]); packet[6..12].copy_from_slice(&[2, 0, 0, 0, 0, 1]); packet[12..14].copy_from_slice(&[0x88, 0xb6]); packet[14..22].copy_from_slice(&boot.to_be_bytes()); packet[22] = index as u8; packet[23] = epoch; for offset in 24..60 { packet[offset] = (index * 7 + offset) as u8; } packet } // SAFETY: admitted Device windows and Normal pages come from the verified // kernel tree and central authority. The controller reference is formed only // while IRQs are masked. All callbacks and RWP polls are bounded, and backend // faults park with the physical resource and exception frame still retained. unsafe impl os_k940_gic::Backend for OsK940Backend<'_> { type Resource = OsK940Resource; fn acquire( &mut self, intid: u32, owner: u64, cpu: u16, ) -> Result { if cpu != 0 || intid != 30 && intid != self.net_spi { return Err(os_k940_gic::Error::Invalid); } let count = if intid == 30 { 1 } else { 4 }; let snapshot = self.registry.snapshot(); if snapshot.free_metadata < count || snapshot.backend_free_pages < count as u64 { return Err(os_k940_gic::Error::NoMemory); } if intid == self.net_spi { self.epoch = self.epoch.checked_add(1).unwrap_or_else(|| park()); } let mut resource = OsK940Resource { intid, owner, handles: [None; 4], pins: core::array::from_fn(|_| None), count, consumed: 0, epoch: self.epoch, }; for index in 0..count { let handle = self .registry .allocate_zeroed(owner) .unwrap_or_else(|_| park()); let pin = self.registry.pin(handle, owner).unwrap_or_else(|_| park()); os_k940_event(1, handle.address(), handle.generation()); resource.handles[index] = Some(handle); resource.pins[index] = Some(pin); } if intid == self.net_spi { self.net_setup(&resource); } Ok(resource) } fn validate( &self, r: &Self::Resource, intid: u32, owner: u64, cpu: u16, ) -> Result<(), os_k940_gic::Error> { if cpu != 0 || (r.intid, r.owner) != (intid, owner) || r.count != if intid == 30 { 1 } else { 4 } { return Err(os_k940_gic::Error::Backing); } for index in 0..r.count { let h = r.handles[index].ok_or(os_k940_gic::Error::Backing)?; if r.pins[index].is_none() { return Err(os_k940_gic::Error::Backing); } // A live pin and the private registry retain this already validated // central lease. IRQ paths inspect these records without borrowing // the central allocator; main validates the physical lease itself. if unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } == 0 { self.registry .read_byte(h, owner, 0) .map_err(|_| os_k940_gic::Error::Backing)?; } if !self.registry.owned_records().any(|record| { record.address == h.address() && record.owner == owner && record.generation == h.generation() && record.pins == 1 && !record.quarantined }) { return Err(os_k940_gic::Error::Backing); } } Ok(()) } fn configure(&mut self, _: &mut Self::Resource, intid: u32, cpu: u16) { if cpu != 0 { park(); } let (base, _, _, bit) = self.registers(intid); let group = if intid < 32 { 0x1_0080 } else { 0x80 + 4 * (intid as u64 / 32) }; let priority = if intid < 32 { 0x1_0400 + intid as u64 } else { 0x400 + intid as u64 }; unsafe { os_k940_write(base, group, os_k940_read(base, group) | bit); write_volatile((base + priority) as *mut u8, 0x80); if intid >= 32 { let offset = 0xc00 + 4 * (intid as u64 / 16); let shift = 2 * (intid % 16); let trigger = if intid == self.net_spi && self.net_trigger == 1 { 2 << shift } else { 0 }; os_k940_write( base, offset, (os_k940_read(base, offset) & !(3 << shift)) | trigger, ); write_volatile( (self.distributor + 0x6000 + 8 * intid as u64) as *mut u64, self.route, ); } asm!("dsb sy", options(nostack, preserves_flags)); if os_k940_read(base, group) & bit == 0 || read_volatile((base + priority) as *const u8) != 0x80 || intid >= 32 && read_volatile((self.distributor + 0x6000 + 8 * intid as u64) as *const u64) != self.route { park(); } } os_k940_event(4, intid as u64, self.route); } fn mask_readback(&mut self, intid: u32, cpu: u16) -> bool { if cpu != 0 { return false; } let (base, mask, enable, bit) = self.registers(intid); unsafe { if !self.suppress_unknown_mask || intid == 30 || intid == self.net_spi { os_k940_write(base, mask, bit); } asm!("dsb sy", options(nostack, preserves_flags)); } let settled = self.wait_rwp(intid < 32); let read = unsafe { os_k940_read(base, enable) }; os_k940_event(2, intid as u64, read as u64); settled && read & bit == 0 } fn enable_readback(&mut self, intid: u32, cpu: u16) -> bool { if cpu != 0 { return false; } let (base, _, enable, bit) = self.registers(intid); unsafe { os_k940_write(base, enable, bit); asm!("dsb sy", options(nostack, preserves_flags)); } let settled = self.wait_rwp(intid < 32); let read = unsafe { os_k940_read(base, enable) }; os_k940_event(3, intid as u64, read as u64); settled && read & bit != 0 } fn acknowledge(&mut self) -> u64 { let (iar, daif): (u64, u64); unsafe { asm!("mrs {iar}, ICC_IAR1_EL1","mrs {daif}, DAIF",iar=out(reg)iar,daif=out(reg)daif,options(nomem,nostack,preserves_flags)) }; if daif & 0x80 == 0 { park(); } os_k940_event(5, iar, daif); iar } fn next_completion( &mut self, r: &mut Self::Resource, owner: u64, ) -> Result { if owner != r.owner { return Err(os_k940_gic::Error::Foreign); } if r.intid == 30 { let storm = unsafe { read_volatile(core::ptr::addr_of!(OS_K940_TIMER_STORM)) } == 1; if storm { let control: u64; unsafe { asm!("mrs {control}, CNTP_CTL_EL0",control=out(reg)control,options(nomem,nostack,preserves_flags)) }; if control & 7 != 5 { return Err(os_k940_gic::Error::Backing); } // Each completion is one bounded physical counter operation. // Withholding disable deliberately leaves the real expired // level timer asserted until main gets its deferral handoff. unsafe { write_volatile(r.address(0) as *mut u64, os_k940_time()) }; r.consumed = r.consumed.checked_add(1).unwrap_or_else(|| park()); os_k940_event(0x16, 30, control); return Ok(os_k940_gic::Completion::Done { more: true }); } unsafe { asm!( "msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack, preserves_flags) ); write_volatile(r.address(0) as *mut u64, os_k940_time()); } r.consumed = r.consumed.checked_add(1).unwrap_or_else(|| park()); os_k940_event(6, 30, r.consumed as u64); return Ok(os_k940_gic::Completion::Done { more: false }); } let used = unsafe { u16::from_le(read_volatile((r.address(2) + 2) as *const u16)) }; if used > 32 || r.consumed > used { return Err(os_k940_gic::Error::Backing); } unsafe { asm!("dmb oshld", options(nostack, preserves_flags)) }; if r.consumed == used { let status = unsafe { os_k940_read(self.net, 0x60) }; if status & 1 != 0 { unsafe { os_k940_write(self.net, 0x64, status & 1); asm!("dsb sy", options(nostack, preserves_flags)) }; } return Ok(os_k940_gic::Completion::Idle); } let element = r.address(2) + 4 + r.consumed as u64 * 8; let id = unsafe { u32::from_le(read_volatile(element as *const u32)) }; let length = unsafe { u32::from_le(read_volatile((element + 4) as *const u32)) }; if id != r.consumed as u32 || id >= 32 || length != 72 { return Err(os_k940_gic::Error::Backing); } self.validate_packet(r, id); r.consumed += 1; os_k940_event(6, r.intid as u64, r.consumed as u64); let more = r.consumed < used; if !more { let status = unsafe { os_k940_read(self.net, 0x60) }; unsafe { os_k940_write(self.net, 0x64, status & 1); asm!("dsb sy", options(nostack, preserves_flags)) }; let after = unsafe { os_k940_read(self.net, 0x60) }; os_k940_event(7, status as u64, after as u64); if after & 1 != 0 { return Err(os_k940_gic::Error::Backing); } } Ok(os_k940_gic::Completion::Done { more }) } fn eoi(&mut self, intid: u32) { let control: u64; unsafe { asm!("mrs {control}, ICC_CTLR_EL1","msr ICC_EOIR1_EL1, {id}",control=out(reg)control,id=in(reg)intid as u64,options(nomem,nostack,preserves_flags)) }; os_k940_event(8, intid as u64, control); } fn deactivate(&mut self, intid: u32) { unsafe { asm!("msr ICC_DIR_EL1, {id}",id=in(reg)intid as u64,options(nomem,nostack,preserves_flags)) }; os_k940_event(9, intid as u64, 0); } fn return_barrier(&mut self) { unsafe { asm!("dsb sy", "isb", options(nostack, preserves_flags)) }; os_k940_event(10, os_k940_time(), 0); } fn release( &mut self, mut r: Self::Resource, ) -> Result<(), (os_k940_gic::Error, Self::Resource)> { let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) }; if active != 0 { return Err((os_k940_gic::Error::Retirement, r)); } os_k940_event(15, r.intid as u64, active as u64); if r.intid == self.net_spi { unsafe { os_k940_write(self.net, 0x70, 0); asm!("dsb sy", options(nostack, preserves_flags)); os_k940_write(self.net, 0x30, 0); } let status = unsafe { os_k940_read(self.net, 0x70) }; let ready = unsafe { os_k940_read(self.net, 0x44) }; os_k940_event(12, status as u64, ready as u64); if status != 0 || ready != 0 { return Err((os_k940_gic::Error::Retirement, r)); } } else { unsafe { asm!( "msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack, preserves_flags) ) }; } for index in 0..r.count { if let Some(pin) = r.pins[index].as_mut() { self.registry.unpin(pin, r.owner).unwrap_or_else(|_| park()); r.pins[index] = None; } let handle = r.handles[index].unwrap_or_else(|| park()); self.registry .release(handle, r.owner) .unwrap_or_else(|_| park()); os_k940_event(11, handle.address(), handle.generation()); r.handles[index] = None; } Ok(()) } } // Actual GIC ingress and foreground work. Raw singleton storage permits only // scoped loans while DAIF.I is masked; no &mut controller survives IRQ unmask. type OsK940Controller<'a> = os_k940_gic::Gic, 2>; static mut OS_K940_CONTROLLER: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K940_REGISTRATION: Option = None; static mut OS_K940_TIMER_REGISTRATION: Option = None; static mut OS_K940_HANDOFF: Option = None; static mut OS_K940_USER: Option = None; static mut OS_K940_TASKS: *const OsK660Tasks = core::ptr::null(); #[unsafe(no_mangle)] static mut OS_K940_EVIDENCE: [u64; 96] = [0; 96]; #[unsafe(no_mangle)] static mut OS_K940_STATE: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K940_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K940_AUTHORITY_LIVE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K940_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K940_REGISTRY: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K940_ROUTES: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K940_FRAMES: [os_k920_exception::Frame; 9] = [os_k920_exception::Frame { current_el: 0, canary: 0, ..os_k920_exception::Frame::EMPTY }; 9]; #[unsafe(no_mangle)] static mut OS_K940_FRAME_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K940_TASK_PROGRESS: [u64; 30] = [0; 30]; #[unsafe(no_mangle)] static mut OS_K940_USER_FRAMES: [os_k920_exception::Frame; 5] = [os_k920_exception::Frame {current_el:0,canary:0,..os_k920_exception::Frame::EMPTY};5]; #[unsafe(no_mangle)] static mut OS_K940_TIMER_STORM: u64 = 0; #[unsafe(no_mangle)] static mut OS_K940_CAPTURE_GATE: u8 = 0; #[unsafe(no_mangle)] static mut OS_K940_KERNEL_FRAME: u64 = 0; #[unsafe(no_mangle)] static mut OS_K940_IRQ_SEEN: u64 = 0; #[unsafe(no_mangle)] static mut OS_K940_LAST_TIMER: u64 = 0; #[unsafe(no_mangle)] static mut OS_K940_DIAGNOSTICS: [u64; 6] = [0; 6]; #[unsafe(no_mangle)] static mut OS_K940_PRE_IRQ_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K940_PRE_IRQ_LAST: u64 = 0; #[unsafe(no_mangle)] static mut OS_K940_PRE_IRQ_ACTIVE: u32 = 0; #[unsafe(no_mangle)] static mut OS_K940_MMIO: [u64; 16] = [0; 16]; fn os_k940_word(index: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_EVIDENCE) .cast::() .add(index), value, ) }; } fn os_k940_state(index: usize) -> u64 { unsafe { read_volatile(core::ptr::addr_of!(OS_K940_STATE).cast::().add(index)) } } fn os_k940_set(index: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_STATE) .cast::() .add(index), value, ) }; } fn os_k940_controller() -> *mut OsK940Controller<'static> { core::ptr::addr_of_mut!(OS_K940_CONTROLLER).cast() } struct OsK940Installation<'a> { pointer: *mut OsK940Controller<'a>, ownership: core::marker::PhantomData>, } impl<'a> OsK940Installation<'a> { // SAFETY: the one boot CPU owns this empty BSS storage. The returned // lifetime marker keeps the authority loan alive until controller drop; // IRQs only use short raw reborrows while this installation is retained. unsafe fn install(controller: OsK940Controller<'a>) -> Self { let pointer = core::ptr::addr_of_mut!(OS_K940_CONTROLLER).cast::>(); unsafe { pointer.write(controller) }; Self { pointer, ownership: core::marker::PhantomData, } } } impl Drop for OsK940Installation<'_> { fn drop(&mut self) { let daif = os_k940_registers().3; // Unexpected early return is contained with every owner retained. It // cannot make the raw installed IRQ callback outlive its authority. if daif & 0x80 == 0 || os_k940_state(0) != 0 { park(); } unsafe { core::ptr::drop_in_place(self.pointer) }; } } fn os_k940_user() -> &'static mmu::UserSpace { unsafe { (&*core::ptr::addr_of!(OS_K940_USER)).as_ref() }.unwrap_or_else(|| park()) } fn os_k940_run_user(tasks:&OsK660Tasks,root:u64) { unsafe extern "C"{fn __os_k940_enter_user(entry:u64,sp:u64);} if !unsafe{read_volatile(core::ptr::addr_of!(OS_K940_TASKS))}.is_null(){park();} // Derive this pointer from the current immutable loan after scheduler // selection. It is cleared before main can mutate that scheduler again. unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K940_TASKS),tasks as *const OsK660Tasks)}; os_k940_set(1,3); unsafe{ asm!("dsb ishst","msr TTBR0_EL1, {root}","tlbi vmalle1","dsb ish","isb",root=in(reg)(root|(1<<48)),options(nostack,preserves_flags)); __os_k940_enter_user(mmu::USER_CODE_VA,mmu::USER_STACK_TOP); write_volatile(core::ptr::addr_of_mut!(OS_K940_TASKS),core::ptr::null()); } } fn os_k940_core_error(_: os_k940_gic::Error) -> &'static str { "gic-core-refusal" } fn os_k940_stack() -> os_k920_exception::Stack { os_k920_exception::Stack::new(os_k940_state(11), os_k940_state(12)).unwrap_or_else(|_| park()) } fn os_k940_registers() -> (u64, u64, u64, u64) { let (ttbr, tcr, vbar, daif): (u64, u64, u64, u64); unsafe { asm!("mrs {ttbr}, TTBR0_EL1","mrs {tcr}, TCR_EL1","mrs {vbar}, VBAR_EL1","mrs {daif}, DAIF",ttbr=out(reg)ttbr,tcr=out(reg)tcr,vbar=out(reg)vbar,daif=out(reg)daif,options(nomem,nostack,preserves_flags)) }; (ttbr, tcr, vbar, daif) } fn os_k940_inventory(authority: &os_k720_frame::Allocator, out: *mut u64) { for (index, value) in os_k740_authority_inventory(authority) .words .into_iter() .enumerate() { unsafe { write_volatile(out.add(index), value) }; } } fn os_k940_diagnostics(offset: usize) { let log = unsafe { &*core::ptr::addr_of!(OS_K680_LOG) } .as_ref() .unwrap_or_else(|| park()); for (index, value) in [log.boot().value(), log.next_sequence(), log.len() as u64] .into_iter() .enumerate() { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_DIAGNOSTICS) .cast::() .add(offset + index), value, ) }; } } fn os_k940_snapshot(stage: u64) { let controller = unsafe { &*os_k940_controller() }; controller.audit().unwrap_or_else(|_| park()); let counter = controller.counters(); let (ttbr, tcr, vbar, daif) = os_k940_registers(); for (index, value) in [ (7, stage), (20, counter.issued), (21, counter.returned), (22, counter.completions), (23, ttbr), (24, tcr), (25, vbar), (26, daif), ] { os_k940_word(index, value); } controller.with_backend_observation(|backend| { if unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } == 0 { backend.registry.with_backend_observation(|frames| { os_k940_inventory( frames.authority, core::ptr::addr_of_mut!(OS_K940_AUTHORITY_LIVE).cast(), ) }); } let snapshot = backend.registry.snapshot(); os_k940_word(27, snapshot.live_pages as u64); os_k940_word(28, snapshot.pins as u64); unsafe { core::ptr::write_bytes( core::ptr::addr_of_mut!(OS_K940_REGISTRY).cast::(), 0, 32, ) }; for (index, record) in backend.registry.owned_records().enumerate() { for (field, value) in [ record.address, record.owner, record.generation, record.pins as u64, u64::from(record.quarantined), ] .into_iter() .enumerate() { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_REGISTRY) .cast::() .add(index * 5 + field), value, ) }; } } os_k940_word(29, unsafe { os_k940_read(backend.net, 0x60) } as u64); }); unsafe { core::ptr::write_bytes(core::ptr::addr_of_mut!(OS_K940_ROUTES).cast::(), 0, 32) }; for (index, record) in controller.owned_records().enumerate() { let state = match record.state { os_k940_gic::State::Open => 1, os_k940_gic::State::Suspended => 2, os_k940_gic::State::Closing => 3, os_k940_gic::State::Quarantined => 4, }; for (field, value) in [ record.slot as u64, record.generation, record.owner, record.intid as u64, record.cpu as u64, state, u64::from(record.active), u64::from(record.serviced), u64::from(record.masked), record.callbacks, ] .into_iter() .enumerate() { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_ROUTES) .cast::() .add(index * 10 + field), value, ) }; } } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_LAST_TIMER), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)), ) }; os_k940_word(30, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }); } fn os_k940_capture(stage: u64) { if os_k940_state(0)==1 {os_k940_snapshot(stage);} else { // The controller is already dropped and its authority loan has ended. // Retain its measured final counters while capturing current registers // and the actual post-release ledger, without reading dropped storage. let(ttbr,tcr,vbar,daif)=os_k940_registers(); for(index,value)in[(7,stage),(23,ttbr),(24,tcr),(25,vbar),(26,daif)]{os_k940_word(index,value);} unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K940_AUTHORITY_LIVE),read_volatile(core::ptr::addr_of!(OS_K940_AUTHORITY_AFTER)));} } os_k940_mmio_snapshot(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K940_CAPTURE_GATE), 0) }; write_str("TURKOS/OS/K940: CAPTURE STAGE=0x"); write_hex(stage as usize); write_str(" GATE=0x"); write_hex(core::ptr::addr_of!(OS_K940_CAPTURE_GATE) as usize); write_line(" READY"); let frequency: u64; unsafe { asm!("mrs {frequency}, CNTFRQ_EL0",frequency=out(reg)frequency,options(nomem,nostack,preserves_flags)) }; let deadline = os_k940_time() .checked_add(frequency.checked_mul(90).unwrap_or_else(|| park())) .unwrap_or_else(|| park()); while unsafe { read_volatile(core::ptr::addr_of!(OS_K940_CAPTURE_GATE)) } != 1 { if os_k940_time() >= deadline { park(); } core::hint::spin_loop(); } } fn os_k940_mmio_snapshot() { // Debugger physical MMIO reads are not CPU-context accesses under HVF. // These are actual volatile CPU Device reads, physically captured as BSS; // the host writes only the distinct single-byte continuation gate. let evidence=core::ptr::addr_of!(OS_K940_EVIDENCE).cast::(); let (net,dist,redist,intid)=unsafe{(read_volatile(evidence.add(1)),read_volatile(evidence.add(2)),read_volatile(evidence.add(3)),read_volatile(evidence.add(4)))}; let word=4*(intid/32); let(timer,deadline,control,daif):(u64,u64,u64,u64); unsafe{asm!("mrs {timer}, CNTP_CTL_EL0","mrs {deadline}, CNTP_CVAL_EL0","mrs {control}, ICC_CTLR_EL1","mrs {daif}, DAIF",timer=out(reg)timer,deadline=out(reg)deadline,control=out(reg)control,daif=out(reg)daif,options(nomem,nostack,preserves_flags))}; if daif&0x80==0{park();} let values=unsafe{[ os_k940_read(redist,0x1_0100)as u64,os_k940_read(redist,0x1_0300)as u64, os_k940_read(dist,0x100+word)as u64,os_k940_read(dist,0x200+word)as u64, os_k940_read(dist,0x300+word)as u64,os_k940_read(dist,0x80+word)as u64, read_volatile((dist+0x400+intid)as *const u8)as u64, read_volatile((dist+0x6000+8*intid)as *const u64), os_k940_read(dist,0xc00+4*(intid/16))as u64, os_k940_read(net,0x70)as u64,os_k940_read(net,0x44)as u64,os_k940_read(net,0x60)as u64, timer,deadline,control,daif, ]}; for(index,value)in values.into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K940_MMIO).cast::().add(index),value)};} } fn os_k940_wait_irq() { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K940_IRQ_SEEN), 0) }; let frequency: u64; unsafe { asm!("mrs {frequency}, CNTFRQ_EL0",frequency=out(reg)frequency,options(nomem,nostack,preserves_flags)) }; let deadline = os_k940_time() .checked_add(frequency * 5) .unwrap_or_else(|| park()); unsafe { asm!("msr DAIFClr, #2", "isb", options(nostack, preserves_flags)) }; while unsafe { read_volatile(core::ptr::addr_of!(OS_K940_IRQ_SEEN)) } != 1 { if os_k940_time() >= deadline { park(); } core::hint::spin_loop(); } unsafe { asm!("msr DAIFSet, #2", "isb", options(nostack, preserves_flags)) }; } fn os_k940_wait_frames(epoch: u64) { write_str("TURKOS/OS/K935: RX-BURST EPOCH=0x"); write_hex(epoch as usize); write_line(" WAITING-32-HOST-FRAMES"); let controller = unsafe { &*os_k940_controller() }; let used = controller.with_backend_observation(|backend| { let row = backend .registry .owned_records() .filter(|r| r.owner == 0x94002) .nth(2) .unwrap_or_else(|| park()); row.address }); let frequency: u64; unsafe { asm!("mrs {frequency}, CNTFRQ_EL0",frequency=out(reg)frequency,options(nomem,nostack,preserves_flags)) }; let deadline = os_k940_time() + frequency * 10; while unsafe { u16::from_le(read_volatile((used + 2) as *const u16)) } != 32 { if os_k940_time() >= deadline { park(); } core::hint::spin_loop(); } unsafe { asm!("dmb oshld", options(nostack, preserves_flags)) }; } struct OsK940UserAuthority; impl os_k920_exception::ReturnAuthority for OsK940UserAuthority { fn owner_root(&self, user: os_k920_exception::User) -> Result<(), os_k920_exception::Error> { let actual = os_k940_user(); let ttbr = os_k940_registers().0; if user.owner != os_k940_state(5) || user.generation != os_k940_state(6) || user.root != actual.root || user.asid != 1 || ttbr != actual.root | (1 << 48) { return Err(os_k920_exception::Error::Owner); } Ok(()) } fn executable( &self, _: os_k920_exception::User, entry: u64, ) -> Result<(), os_k920_exception::Error> { let user = os_k940_user(); let descriptor = user.descriptor(0).ok_or(os_k920_exception::Error::Entry)?; if !(mmu::USER_CODE_VA..mmu::USER_CODE_VA + 4096).contains(&entry) || entry % 4 != 0 || descriptor & 3 != 3 || descriptor & 0x0000_ffff_ffff_f000 != user.code_frame() || descriptor & (1 << 6 | 1 << 7 | 1 << 11 | 1 << 53) != (1 << 6 | 1 << 7 | 1 << 11 | 1 << 53) || descriptor & 1 << 54 != 0 { return Err(os_k920_exception::Error::Entry); } Ok(()) } fn writable_stack( &self, _: os_k920_exception::User, sp: u64, ) -> Result<(), os_k920_exception::Error> { let user = os_k940_user(); let descriptor = user.descriptor(3).ok_or(os_k920_exception::Error::Stack)?; if sp != mmu::USER_STACK_TOP || descriptor & 0x0000_ffff_ffff_f000 != user.stack_frame() || descriptor & 3 != 3 || descriptor & (3 << 6) != (1 << 6) || descriptor & (1 << 53 | 1 << 54) != (1 << 53 | 1 << 54) || user.descriptor(2) != Some(0) { return Err(os_k920_exception::Error::Stack); } Ok(()) } } #[unsafe(no_mangle)] extern "C" fn os_k940_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { if os_k940_state(0) != 1 || os_k940_registers().2 != os_k940_state(14) { park(); } let address = pointer as u64; let stack = os_k940_stack(); stack.frame_address(address).unwrap_or_else(|_| park()); let frame = unsafe { &mut *pointer }; os_k920_exception::validate_ingress(frame, address, stack).unwrap_or_else(|_| park()); if frame.vector_slot == 8 && os_k940_state(1) == 3 { if frame.esr >> 26 != 0x15 || frame.esr & 0xffff != 0x940 || frame.elr != mmu::USER_CODE_VA + 24 { park(); } let owner = os_k920_exception::User { owner: os_k940_state(5), generation: os_k940_state(6), root: os_k940_user().root, asid: 1, }; os_k920_exception::authorize_user_return( frame, address, stack, owner, &OsK940UserAuthority, ) .unwrap_or_else(|_| park()); let tasks = unsafe { read_volatile(core::ptr::addr_of!(OS_K940_TASKS)) }; if tasks.is_null() { park(); } let id = unsafe { (&*tasks).scheduler.running() }.unwrap_or_else(|| park()); if id.slot as u64 + 1 != os_k940_state(5) || id.generation as u64 != os_k940_state(6) { park(); } let progress = unsafe { read_volatile(os_k940_user().data_frame() as *const u64) }; let index = os_k940_state(10) as usize; if index >= 5 || progress != index as u64 + 1 || frame.x[0] != progress { park(); } unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K940_USER_FRAMES).cast::().add(index),*frame)}; for (field, value) in [ id.slot as u64, id.generation as u64, progress, frame.elr, frame.interrupted_sp, os_k940_registers().0, ] .into_iter() .enumerate() { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_TASK_PROGRESS) .cast::() .add(index * 6 + field), value, ) }; } os_k940_event(13, id.slot as u64, progress); os_k940_set(10, index as u64 + 1); unsafe { asm!("dsb ishst","msr TTBR0_EL1, {root}","tlbi vmalle1","dsb ish","isb",root=in(reg)os_k940_state(2),options(nostack,preserves_flags)) }; return 1; } if frame.vector_slot != 5 { park(); } os_k920_kernel_irq_return( frame, address, stack, core::ptr::addr_of!(_text_start) as u64, core::ptr::addr_of!(_text_end) as u64, __os_k920_kernel_tco_observed(), ) .unwrap_or_else(|_| park()); let index = unsafe { read_volatile(core::ptr::addr_of!(OS_K940_FRAME_COUNT)) }; if index >= 9 { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_FRAMES) .cast::() .add(index as usize), *frame, ); write_volatile(core::ptr::addr_of_mut!(OS_K940_FRAME_COUNT), index + 1); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE), 1); write_volatile( core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) + 1, ); write_volatile( core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER), os_k940_time(), ); } // The interrupted foreground has no live controller reference. This loan // ends before assembly restores the complete checked exception frame. let begin = unsafe { (&mut *os_k940_controller()).begin(0) }; match begin { Ok(os_k940_gic::Begin::Dispatch(mut token)) => { let work = unsafe { (&mut *os_k940_controller()).service(&token, 8) } .unwrap_or_else(|_| park()); if os_k940_state(1) == 6 { if token.intid() != 30 || work.completed != 8 || !work.deferred || unsafe { (&mut *os_k940_controller()).service(&token, 8) } != Err(os_k940_gic::Error::AlreadyServiced) { park(); } os_k940_word(55, 1); } if os_k940_state(1) == 4 { let registration = unsafe { (&*core::ptr::addr_of!(OS_K940_REGISTRATION)).as_ref() } .unwrap_or_else(|| park()); if token.intid() as u64 != os_k940_state(4) || work.completed != 8 || !work.deferred || unsafe { (&mut *os_k940_controller()).request_close(registration, 0x94002) } != Ok(os_k940_gic::Close::RetainedActive) { park(); } os_k940_event(14, token.intid() as u64, unsafe { (&*os_k940_controller()) .with_backend_observation(|b| b.registry.snapshot().pins as u64) }); // End the mutable loan before this separately borrowed read-only // checkpoint; the nonclone dispatch still retains actual pages. os_k940_capture(7); } let permit = unsafe { (&mut *os_k940_controller()).finish_deferred(&mut token) } .unwrap_or_else(|_| park()); os_k940_word(31, permit.observation().1); os_k940_word(32, u64::from(permit.observation().5)); if os_k940_state(1) == 4 { if unsafe { (&*core::ptr::addr_of!(OS_K940_HANDOFF)).is_some() } { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K940_HANDOFF), Some(permit)) }; } } Ok(os_k940_gic::Begin::Masked { intid, reason }) => { if reason != os_k940_gic::MaskReason::Unknown { park(); } if intid == 127 { os_k940_word(33, 1); } else if intid as u64 == os_k940_state(4) && os_k940_state(1) == 7 { os_k940_word(54, 1); } else { park(); } } Ok(os_k940_gic::Begin::Spurious(_)) => { park(); } Err(os_k940_gic::Error::MaskReadback) if os_k940_state(15) == 1 => { os_k940_snapshot(13); os_k940_word(34, 1); unsafe { asm!("dsb sy", options(nostack, preserves_flags)) }; write_line( "TURKOS/OS/K930: REAL-UNKNOWN-SPI MASK-WRITE-WITHHELD READBACK-ENABLED NO-EOI FRAME-RETAINED PARKED", ); park(); } Err(_) => park(), } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K940_IRQ_SEEN), 1) }; 0 } fn os_k940_accept( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { unsafe extern "C" { static __os_k940_vectors: u8; static __os_k940_user_code: u8; static __os_k940_user_code_end: u8; fn __os_k940_enter_user(entry: u64, sp: u64); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K940_PRE_IRQ_COUNT),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))); write_volatile(core::ptr::addr_of_mut!(OS_K940_PRE_IRQ_LAST),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER))); write_volatile(core::ptr::addr_of_mut!(OS_K940_PRE_IRQ_ACTIVE),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))); } if cfg!(feature = "os-runtime-product") && !os_k1100_product_probe_enabled(boot)? { os_k940_product_configuration(); write_line("TURKOS/OS/K940: PRODUCT-CONTROL NO-GIC-PROBE"); return Ok(()); } let blob = unsafe { core::slice::from_raw_parts(boot.dtb_address as *const u8, boot.dtb_size) }; let scratch = unsafe { &*core::ptr::addr_of!(OS_K700_WORKSPACE) }; let mode = turkos_kernel::runtime::platform::chosen_u32(blob, scratch, b"turkos,gic-mode") .map_err(|_| "gic-fixture")? .unwrap_or(0); if mode > 1 { return Err("gic-fixture-mode"); } let (old_ttbr, old_tcr, old_vbar, old_daif) = os_k940_registers(); if old_daif & 0x80 == 0 || old_ttbr != tables.root { return Err("gic-entry-control"); } let previous_count = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; let guard = unsafe { read_volatile( core::ptr::addr_of!(OS_K900_STACK_EVIDENCE) .cast::() .add(13), ) }; let low = guard + 4096; let top = low + 65536 - 16; let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags)) }; if sp <= low + 848 || sp > top || !tables.task_stack_guard_unmapped(guard) || unsafe { read_volatile(low as *const u64) } != OS_K900_LOW_CANARY || unsafe { read_volatile((top + 8) as *const u64) } != OS_K900_HIGH_CANARY { return Err("gic-returned-stack"); } os_k940_diagnostics(0); let mut net = None; for node in dtb::discover_virtio_mmio(blob) .map_err(|_| "gic-devices")? .as_slice() { if unsafe { os_k940_read(node.base, 0) } != 0x7472_6976 { return Err("gic-mmio-magic"); } if unsafe { os_k940_read(node.base, 8) } == 1 { if unsafe { os_k940_read(node.base, 4) } != 2 || net.replace(node.base).is_some() { return Err("gic-net-admission"); } } } let net = net.ok_or("gic-no-net")?; let (net_spi, net_reg_offset, net_irq_offset, net_trigger) = turkos_kernel::runtime::platform::os_k940_virtio_spi( blob, scratch, net, boot.interrupts.gic_distributor, ) .map_err(|_| "gic-dtb-spi")?; if net_spi == 127 { return Err("gic-source-collision"); } for address in [ net, boot.interrupts.gic_distributor, boot.interrupts.gic_redistributor, boot.interrupts.gic_redistributor + 65536, ] { let descriptor = os_k700_translation(tables, address)?; if descriptor & 3 != 3 || descriptor & (7 << 2) != 1 << 2 || descriptor & (1 << 6 | 1 << 53 | 1 << 54) != (1 << 53 | 1 << 54) { return Err("gic-device-mapping"); } } let (count, split_eoi, mpidr): (u32, bool, u64); unsafe { count = ((os_k940_read(boot.interrupts.gic_distributor, 4) & 31) + 1) * 32; let control: u64; asm!("mrs {control}, ICC_CTLR_EL1","mrs {mpidr}, MPIDR_EL1",control=out(reg)control,mpidr=out(reg)mpidr,options(nomem,nostack,preserves_flags)); split_eoi = control & 2 != 0; } let route = (mpidr & 0x00ff_ffff) | (mpidr & 0xff00_0000_00); if count <= 127 || net_spi >= count || boot.interrupts.nonsecure_phys_timer_ppi != 30 { return Err("gic-intid-range"); } let dist = boot.interrupts.gic_distributor; let net_bit = 1u32 << (net_spi % 32); let net_word = 4 * (net_spi as u64 / 32); let (net_enabled, net_group, net_priority, net_route, net_config, net_pending): ( u32, u32, u8, u64, u32, u32, ); unsafe { if os_k940_read(dist, 0x300 + net_word) & net_bit != 0 { return Err("gic-net-already-active"); } net_enabled = os_k940_read(dist, 0x100 + net_word) & net_bit; net_group = os_k940_read(dist, 0x80 + net_word); net_priority = read_volatile((dist + 0x400 + net_spi as u64) as *const u8); net_route = read_volatile((dist + 0x6000 + 8 * net_spi as u64) as *const u64); net_config = os_k940_read(dist, 0xc00 + 4 * (net_spi as u64 / 16)); net_pending = os_k940_read(dist, 0x200 + net_word) & net_bit; os_k940_write(dist, 0x180 + net_word, net_bit); os_k940_write(dist, 0x280 + net_word, net_bit); asm!("dsb sy", options(nostack, preserves_flags)); if os_k940_read(dist, 0x100 + net_word) & net_bit != 0 || os_k940_read(dist, 0x200 + net_word) & net_bit != 0 { return Err("gic-net-initial-mask"); } } let configuration = os_k940_gic::Configuration::new(count.min(1020), 1, 8, split_eoi) .map_err(os_k940_core_error)?; let (_, memory, irq, tasks) = runtime.running().map_err(os_k660_policy::Error::tag)?; let previous_binding=::ownership(irq).irq_bindings as u64; os_k940_word(71,previous_binding); let irq_transfer=os_k940_transfer_irq(irq,boot.interrupts.gic_redistributor)?; os_k940_word(72,irq_transfer.predecessor_bindings()); if previous_binding!=1||irq_transfer.predecessor_bindings()!=0{return Err("gic-IRQ-transfer-binding");} os_k940_inventory( memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K940_AUTHORITY_BEFORE).cast(), ); let user = unsafe { tables.build_user_space( &mut memory.allocator, boot.hardware.ram_base, boot.hardware.ram_size, ) } .map_err(|_| "gic-user-space")?; let program = core::ptr::addr_of!(__os_k940_user_code) as u64; let program_end = core::ptr::addr_of!(__os_k940_user_code_end) as u64; if program_end <= program || program_end - program > 4096 { return Err("gic-user-program"); } for offset in 0..program_end - program { unsafe { write_volatile( (user.code_frame() + offset) as *mut u8, read_volatile((program + offset) as *const u8), ) }; } unsafe { asm!("dc cvau, {code}","dsb ish","ic ivau, {code}","dsb ish","isb",code=in(reg)user.code_frame(),options(nostack,preserves_flags)) }; let user_root = user.root; let user_code = user.code_frame(); let user_data = user.data_frame(); let user_stack = user.stack_frame(); for(index,address)in user.owned_frames().enumerate(){os_k940_word(65+index,address);} if !user.contract_valid(tables.root) { return Err("gic-user-descriptors"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K940_USER), Some(user)); } let alpha = tasks.id.ok_or("gic-alpha")?; if tasks.scheduler.running() != Some(alpha) { return Err("gic-alpha-running"); } let beta = tasks.scheduler.spawn().map_err(|_| "gic-beta")?; let pool = memory .allocator .os_authority() .free_regions() .iter() .rev() .find(|r| r.end - r.start >= 96 * 4096) .ok_or("gic-resource-pool")? .end - 84 * 4096; for address in (0..5).map(|i| pool + i * 4096) { if !tables.normal_shareable_identity(address) { return Err("gic-resource-normal-memory"); } } let frames = OsK940Frames { authority: memory.allocator.os_authority_mut(), leases: core::array::from_fn(|_| None), addresses: core::array::from_fn(|i| pool + i as u64 * 4096), }; let backend = OsK940Backend { registry: os_k740_metadata::Registry::new(0x940, frames).map_err(|_| "gic-registry")?, distributor: boot.interrupts.gic_distributor, redistributor: boot.interrupts.gic_redistributor, route, net, net_spi, net_trigger, boot_id: unsafe { read_volatile(core::ptr::addr_of!(OS_K680_EVIDENCE).cast::()) }, suppress_unknown_mask: mode == 1, epoch: 0, }; let controller = os_k940_gic::Gic::new(0x940, configuration, backend).map_err(os_k940_core_error)?; let installation = unsafe { OsK940Installation::install(controller) }; for (index, value) in [ (0, 1), (2, old_ttbr), (3, old_tcr), (4, net_spi as u64), (5, beta.slot as u64 + 1), (6, beta.generation), (11, low), (12, top), (14, core::ptr::addr_of!(__os_k940_vectors) as u64), (15, mode as u64), ] { os_k940_set(index, value); } for (index, value) in [ (0, 0x4b39343047494321), (1, net), (2, boot.interrupts.gic_distributor), (3, boot.interrupts.gic_redistributor), (4, net_spi as u64), (5, previous_count), (6, old_ttbr), (8, old_tcr), (9, old_vbar), (10, old_daif), (11, pool), (12, user_root), (13, user_code), (14, user_data), (15, user_stack), (16, net_reg_offset as u64), (17, net_irq_offset as u64), (18, route), (19, u64::from(split_eoi)), (35, guard), (36, low), (37, top), (38, sp), (39, mode as u64), (40, alpha.slot as u64), (41, alpha.generation), (42, beta.slot as u64), (43, beta.generation), (50, net_trigger as u64), (74, __os_k920_kernel_tco_observed()), ] { os_k940_word(index, value); } for (index, value) in [ (56, net_enabled as u64), (57, net_group as u64), (58, net_priority as u64), (59, net_route), (60, net_config as u64), (61, net_pending as u64), ] { os_k940_word(index, value); } // A quiescent hardware IAR read returns 1023 without an EOI/DIR/barrier. let spurious = unsafe { (&mut *os_k940_controller()).begin(0) }.map_err(os_k940_core_error)?; if !matches!(spurious, os_k940_gic::Begin::Spurious(1023)) { return Err("gic-real-spurious"); } os_k940_word(44, 1023); let timer = unsafe { (&mut *os_k940_controller()).register(30, 0x94001, 0) } .map_err(|_| "gic-timer-route")?; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_TIMER_REGISTRATION), Some(timer), ); asm!("msr VBAR_EL1, {vbar}","isb",vbar=in(reg)os_k940_state(14),options(nomem,nostack,preserves_flags)) }; os_k940_set(1, 1); let frequency: u64; unsafe { asm!("mrs {frequency}, CNTFRQ_EL0",frequency=out(reg)frequency,options(nomem,nostack,preserves_flags)) }; let deadline = os_k940_time() + turkos_kernel::irq::timer_ticks_for_ns(frequency, 1_000_000).ok_or("gic-timer-period")?; os_k940_word(45, deadline); unsafe { asm!("msr CNTP_CVAL_EL0, {deadline}","msr CNTP_CTL_EL0, {enable}","isb",deadline=in(reg)deadline,enable=in(reg)1u64,options(nomem,nostack,preserves_flags)) }; os_k940_wait_irq(); os_k940_capture(1); let registration = unsafe { (&mut *os_k940_controller()).register(net_spi, 0x94002, 0) } .map_err(|_| "gic-net-route")?; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_REGISTRATION), Some(registration), ) }; let event_before = unsafe { read_volatile(core::ptr::addr_of!(OS_K940_EVENT_COUNT)) }; for owner in [0x94002, 0x94003] { let refusal = unsafe { (&mut *os_k940_controller()).register(net_spi, owner, 0) }.unwrap_err(); if refusal.error != os_k940_gic::Error::Conflict || refusal.retained.is_some() { return Err("gic-duplicate-route"); } } if unsafe { read_volatile(core::ptr::addr_of!(OS_K940_EVENT_COUNT)) } != event_before { return Err("gic-duplicate-hardware-effects"); } os_k940_word(46, event_before); os_k940_wait_frames(1); os_k940_capture(2); for batch in 0..4 { os_k940_set(1, 2); os_k940_wait_irq(); os_k940_capture(3 + batch); if tasks.scheduler.yield_current(false) != Ok(Some(beta)) { return Err("gic-beta-dispatch"); } os_k940_run_user(tasks,user_root); if tasks.scheduler.yield_current(false) != Ok(Some(alpha)) { return Err("gic-alpha-restore"); } tasks.progress = tasks .progress .checked_add(1) .ok_or("gic-task-progress-overflow")?; os_k940_word(47, batch + 1); if batch < 3 { let registration = unsafe { (&*core::ptr::addr_of!(OS_K940_REGISTRATION)).as_ref() } .ok_or("gic-registration")?; unsafe { (&mut *os_k940_controller()).resume(registration, 0x94002) } .map_err(os_k940_core_error)?; if net_trigger == 1 { // The real edge is already acknowledged. Its actual used ring // still contains work; main explicitly continues that owned // SPI only after the healthy foreground/task opportunity. unsafe { os_k940_write( boot.interrupts.gic_distributor, 0x200 + 4 * (net_spi as u64 / 32), 1 << (net_spi % 32), ); asm!("dsb sy", options(nostack, preserves_flags)) }; os_k940_event(0x17, net_spi as u64, batch + 1); } } } let registration = unsafe { (&*core::ptr::addr_of!(OS_K940_REGISTRATION)).as_ref() } .ok_or("gic-registration")?; if unsafe { (&mut *os_k940_controller()).request_close(registration, 0x94002) } != Ok(os_k940_gic::Close::Released) { return Err("gic-first-net-close"); } let replacement = unsafe { (&mut *os_k940_controller()).register(net_spi, 0x94002, 0) } .map_err(|_| "gic-second-net-route")?; if unsafe { (&mut *os_k940_controller()).request_close(registration, 0x94002) } != Err(os_k940_gic::Error::Stale) { return Err("gic-stale-close"); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K940_REGISTRATION), Some(replacement), ) }; os_k940_wait_frames(2); os_k940_snapshot(7); os_k940_set(1, 4); os_k940_wait_irq(); os_k940_capture(8); let handoff = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K940_HANDOFF)).take() } .ok_or("gic-IRQ-handoff")?; let registration = unsafe { (&*core::ptr::addr_of!(OS_K940_REGISTRATION)).as_ref() } .ok_or("gic-returned-registration")?; if unsafe { (&mut *os_k940_controller()).retire_returned(registration, 0x94002, handoff) } .map_err(|_| "gic-main-retirement")? != os_k940_gic::Close::Released { return Err("gic-main-retirement-result"); } os_k940_capture(9); if unsafe { (&*os_k940_controller()) .owned_records() .any(|r| r.intid == net_spi) } { return Err("gic-dead-net-route"); } let completed_before_late = unsafe { (&*os_k940_controller()).counters().completions }; os_k940_set(1, 7); unsafe { os_k940_write(dist, 0x100 + net_word, net_bit); os_k940_write(dist, 0x200 + net_word, net_bit); asm!("dsb sy", options(nostack, preserves_flags)) }; os_k940_wait_irq(); if unsafe { (&*os_k940_controller()).counters().completions } != completed_before_late { return Err("gic-late-callback"); } os_k940_word(62, completed_before_late); write_line( "TURKOS/OS/K940: CLOSED-SPI REAL-REASSERTION MASKED NO-DEAD-CALLBACK RESOURCES-UNCHANGED", ); // Actual continuous level source: an expired timer remains enabled. Eight // bounded physical reads hit the completion budget, mask PPI30 and return; // main and another actual EL0 task run while ISTATUS still remains set. unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K940_TIMER_STORM), 1) }; os_k940_set(1, 6); let storm_deadline = os_k940_time() + turkos_kernel::irq::timer_ticks_for_ns(frequency, 1_000_000).ok_or("gic-storm-period")?; unsafe { asm!("msr CNTP_CVAL_EL0, {deadline}","msr CNTP_CTL_EL0, {enable}","isb",deadline=in(reg)storm_deadline,enable=in(reg)1u64,options(nomem,nostack,preserves_flags)) }; os_k940_wait_irq(); let storm_control: u64; unsafe { asm!("mrs {control}, CNTP_CTL_EL0",control=out(reg)storm_control,options(nomem,nostack,preserves_flags)) }; os_k940_word(51, storm_control); os_k940_word(52, storm_deadline); if storm_control & 7 != 5 { return Err("gic-storm-not-asserted"); } os_k940_capture(10); if tasks.scheduler.yield_current(false) != Ok(Some(beta)) { return Err("gic-storm-beta"); } os_k940_run_user(tasks,user_root); if tasks.scheduler.yield_current(false) != Ok(Some(alpha)) { return Err("gic-storm-alpha"); } tasks.progress = tasks.progress.checked_add(1).ok_or("gic-storm-progress")?; os_k940_word(47, 5); let still_asserted: u64; unsafe { asm!("mrs {control}, CNTP_CTL_EL0",control=out(reg)still_asserted,options(nomem,nostack,preserves_flags)) }; os_k940_word(53, still_asserted); if still_asserted & 7 != 5 { return Err("gic-storm-task-source-cleared"); } os_k940_capture(11); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K940_TIMER_STORM), 0); asm!( "msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack, preserves_flags) ) }; // Own an otherwise disabled standard SPI as a deliberate unregistered // source. Hardware pending, IAR, mask readback and completion are real. let intid = 127u32; let bit = 1 << (intid % 32); let word = 4 * (intid as u64 / 32); let (group, priority, route_before, configuration_before): (u32, u8, u64, u32); unsafe { if os_k940_read(dist, 0x100 + word) & bit != 0 || os_k940_read(dist, 0x200 + word) & bit != 0 || os_k940_read(dist, 0x300 + word) & bit != 0 { return Err("gic-unknown-source-in-use"); } group = os_k940_read(dist, 0x80 + word); priority = read_volatile((dist + 0x400 + intid as u64) as *const u8); route_before = read_volatile((dist + 0x6000 + 8 * intid as u64) as *const u64); configuration_before = os_k940_read(dist, 0xc00 + 4 * (intid as u64 / 16)); os_k940_write(dist, 0x80 + word, group | bit); write_volatile((dist + 0x400 + intid as u64) as *mut u8, 0x80); os_k940_write( dist, 0xc00 + 4 * (intid as u64 / 16), configuration_before & !(3 << (2 * (intid % 16))), ); write_volatile((dist + 0x6000 + 8 * intid as u64) as *mut u64, route); os_k940_write(dist, 0x100 + word, bit); asm!("dsb sy", options(nostack, preserves_flags)); os_k940_write(dist, 0x200 + word, bit); asm!("dsb sy", options(nostack, preserves_flags)); } os_k940_set(1, 5); os_k940_wait_irq(); unsafe { os_k940_write(dist, 0x280 + word, bit); os_k940_write(dist, 0x80 + word, group); write_volatile((dist + 0x400 + intid as u64) as *mut u8, priority); write_volatile((dist + 0x6000 + 8 * intid as u64) as *mut u64, route_before); os_k940_write(dist, 0xc00 + 4 * (intid as u64 / 16), configuration_before); asm!("dsb sy", options(nostack, preserves_flags)); } let timer = unsafe { (&*core::ptr::addr_of!(OS_K940_TIMER_REGISTRATION)).as_ref() } .ok_or("gic-timer-registration")?; if unsafe { (&mut *os_k940_controller()).request_close(timer, 0x94001) } != Ok(os_k940_gic::Close::Released) { return Err("gic-timer-close"); } os_k940_capture(12); if unsafe { (&*os_k940_controller()).owned_records().next().is_some() || (&*os_k940_controller()) .with_backend_observation(|b| b.registry.snapshot().live_pages != 0) } { return Err("gic-live-teardown"); } unsafe { asm!("msr VBAR_EL1, {vbar}","isb",vbar=in(reg)old_vbar,options(nomem,nostack,preserves_flags)); os_k940_set(0, 0); write_volatile(core::ptr::addr_of_mut!(OS_K940_TASKS), core::ptr::null()); } drop(installation); unsafe { os_k940_write(dist, 0x280 + net_word, net_bit); os_k940_write(dist, 0x80 + net_word, net_group); write_volatile((dist + 0x400 + net_spi as u64) as *mut u8, net_priority); write_volatile((dist + 0x6000 + 8 * net_spi as u64) as *mut u64, net_route); os_k940_write(dist, 0xc00 + 4 * (net_spi as u64 / 16), net_config); if net_pending != 0 { os_k940_write(dist, 0x200 + net_word, net_bit); } if net_enabled != 0 { os_k940_write(dist, 0x100 + net_word, net_bit); } asm!("dsb sy", options(nostack, preserves_flags)); let actual_enabled = os_k940_read(dist, 0x100 + net_word) & net_bit; let actual_pending = os_k940_read(dist, 0x200 + net_word) & net_bit; os_k940_word(63, actual_enabled as u64); os_k940_word(64, actual_pending as u64); if actual_enabled != net_enabled || actual_pending != net_pending || os_k940_read(dist, 0x80 + net_word) != net_group || read_volatile((dist + 0x400 + net_spi as u64) as *const u8) != net_priority || read_volatile((dist + 0x6000 + 8 * net_spi as u64) as *const u64) != net_route || os_k940_read(dist, 0xc00 + 4 * (net_spi as u64 / 16)) != net_config { return Err("gic-net-route-restoration"); } } if tasks.scheduler.yield_current(false) != Ok(Some(beta)) || tasks.scheduler.exit_current(0x940) != Ok(Some(alpha)) || tasks.scheduler.reap(beta) != Ok(0x940) { return Err("gic-task-teardown"); } let user = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K940_USER)).take() }.ok_or("gic-user-owner")?; unsafe { asm!( "dsb ishst", "tlbi vmalle1", "dsb ish", "isb", options(nostack, preserves_flags) ); user.release_after_tlbi(&mut memory.allocator, tables.root) } .map_err(|_| "gic-user-release")?; // The predecessor keeps its timer route enabled. Restore that exact live // service route only after this controller has relinquished all resources. unsafe { os_k940_write(boot.interrupts.gic_redistributor, 0x1_0100, 1 << 30); asm!("dsb sy", options(nostack, preserves_flags)) }; drop(irq_transfer); os_k940_word(73,::ownership(irq).irq_bindings as u64); os_k940_inventory( memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K940_AUTHORITY_AFTER).cast(), ); for index in 0..328 { if unsafe { read_volatile( core::ptr::addr_of!(OS_K940_AUTHORITY_BEFORE) .cast::() .add(index), ) } != unsafe { read_volatile( core::ptr::addr_of!(OS_K940_AUTHORITY_AFTER) .cast::() .add(index), ) } { return Err("gic-authority-restoration"); } } let (after_ttbr, after_tcr, after_vbar, after_daif) = os_k940_registers(); if (after_ttbr, after_tcr, after_vbar, after_daif) != (old_ttbr, old_tcr, old_vbar, old_daif) { return Err("gic-control-restoration"); } os_k940_diagnostics(3); os_k940_word(48, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }); os_k940_word(49, 1); os_k940_capture(14); write_line("TURKOS/OS/K925: PASS"); write_line("TURKOS/OS/K930: PASS"); write_line("TURKOS/OS/K935: PASS"); write_line("TURKOS/OS/K940: PASS"); Ok(()) } #[inline(never)] fn os_k940_product_configuration() { unsafe extern "C" { static __os_k940_vectors: u8; static __os_k940_user_code: u8; } for pointer in [ core::ptr::addr_of!(OS_K940_EVIDENCE).cast::(), core::ptr::addr_of!(OS_K940_STATE).cast(), core::ptr::addr_of!(OS_K940_AUTHORITY_BEFORE).cast(), core::ptr::addr_of!(OS_K940_AUTHORITY_LIVE).cast(), core::ptr::addr_of!(OS_K940_AUTHORITY_AFTER).cast(), core::ptr::addr_of!(OS_K940_REGISTRY).cast(), core::ptr::addr_of!(OS_K940_ROUTES).cast(), core::ptr::addr_of!(OS_K940_FRAMES).cast(), core::ptr::addr_of!(OS_K940_FRAME_COUNT).cast(), core::ptr::addr_of!(OS_K940_TASK_PROGRESS).cast(), core::ptr::addr_of!(OS_K940_USER_FRAMES).cast(), core::ptr::addr_of!(OS_K940_MMIO).cast(), core::ptr::addr_of!(OS_K940_CAPTURE_GATE).cast(), core::ptr::addr_of!(OS_K940_KERNEL_FRAME).cast(), core::ptr::addr_of!(OS_K940_IRQ_SEEN).cast(), core::ptr::addr_of!(OS_K940_LAST_TIMER).cast(), core::ptr::addr_of!(OS_K940_DIAGNOSTICS).cast(), core::ptr::addr_of!(OS_K940_EVENTS).cast(), core::ptr::addr_of!(OS_K940_EVENT_COUNT).cast(), core::ptr::addr_of!(OS_K940_TIMER_STORM).cast(), ] { if unsafe { read_volatile(pointer) } != 0 { park(); } } if unsafe { read_volatile(core::ptr::addr_of!(OS_K940_NONCE)) } != 1 { park(); } let _ = unsafe { read_volatile(core::ptr::addr_of!(__os_k940_vectors)); read_volatile(core::ptr::addr_of!(__os_k940_user_code)) }; } // A nonclone loan of the actual predecessor service. This scoped transfer // carries exclusive access to its physical PPI binding across the temporary // vector/controller interval. It allocates nothing and is never used in IRQ. struct OsK940IrqTransfer<'a> { predecessor: &'a mut OsK660Irq, original_vbar: u64, redistributor: u64, } impl OsK940IrqTransfer<'_> { fn predecessor_bindings(&self) -> u64 { ::ownership(self.predecessor).irq_bindings as u64 } } fn os_k940_transfer_irq( predecessor: &mut OsK660Irq, redistributor: u64, ) -> Result, &'static str> { let (vbar,daif):(u64,u64); unsafe{asm!("mrs {vbar}, VBAR_EL1","mrs {daif}, DAIF",vbar=out(reg)vbar,daif=out(reg)daif,options(nomem,nostack,preserves_flags))}; if daif&0x80==0 || !predecessor.installed || predecessor.gicr!=redistributor || unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}!=0 {return Err("irq-transfer-predecessor-not-idle");} // IRQs remain masked until the new owner installs its vector/controller. predecessor.installed=false; Ok(OsK940IrqTransfer{predecessor,original_vbar:vbar,redistributor}) } impl Drop for OsK940IrqTransfer<'_> { fn drop(&mut self) { let(vbar,daif,timer):(u64,u64,u64); unsafe{asm!("mrs {vbar}, VBAR_EL1","mrs {daif}, DAIF","mrs {timer}, CNTP_CTL_EL0",vbar=out(reg)vbar,daif=out(reg)daif,timer=out(reg)timer,options(nomem,nostack,preserves_flags))}; // The new owner must finish EOI/DIR/return barriers, retire all of its // registrations/backing, drop its controller and restore the old VBAR // before dropping this transfer. Errors park with the loan retained. let mut ready=false; for _ in 0..1_000_000 {if unsafe{read_volatile(self.redistributor as *const u32)}&8==0 {ready=true;break;}} let enabled=unsafe{read_volatile((self.redistributor+0x1_0100) as *const u32)}; if daif&0x80==0 || vbar!=self.original_vbar || self.predecessor.installed || timer&1!=0 || self.predecessor.gicr!=self.redistributor || !ready || enabled&(1<<30)==0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}!=0 {park();} self.predecessor.installed=true; } } const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k940_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); use turkos_kernel::runtime::clock as os_k960_clock; #[unsafe(no_mangle)] static mut OS_K960_NONCE: u64 = 1; #[unsafe(no_mangle)] static mut OS_K960_EVENTS: [u64; 1024] = [0; 1024]; #[unsafe(no_mangle)] static mut OS_K960_EVENT_COUNT: u64 = 0; fn os_k960_counter() -> u64 { let value: u64; unsafe { asm!("isb", "mrs {value}, CNTPCT_EL0", value=out(reg)value, options(nomem,nostack,preserves_flags)) }; value } fn os_k960_frequency() -> u64 { let value: u64; unsafe { asm!("mrs {value}, CNTFRQ_EL0", value=out(reg)value, options(nomem,nostack,preserves_flags)) }; value } fn os_k960_timer() -> (u64, u64) { let (cval, ctl): (u64, u64); unsafe { asm!("mrs {cval}, CNTP_CVAL_EL0", "mrs {ctl}, CNTP_CTL_EL0", cval=out(reg)cval,ctl=out(reg)ctl,options(nomem,nostack,preserves_flags)) }; (cval, ctl) } fn os_k960_event(code: u64, a: u64, b: u64) { let index = unsafe { read_volatile(core::ptr::addr_of!(OS_K960_EVENT_COUNT)) } as usize; if index >= 256 { park(); } for (field, value) in [code, os_k960_counter(), a, b].into_iter().enumerate() { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_EVENTS).cast::().add(index*4+field), value) }; } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_EVENT_COUNT), index as u64+1) }; } #[unsafe(no_mangle)] extern "C" fn __os_k960_clock_identity() -> u64 { if os_k940_registers().3 & 0x80 == 0 { return 0; } let value = unsafe { read_volatile(core::ptr::addr_of!(OS_K960_NONCE)) }; let Some(next) = value.checked_add(1) else { return 0; }; if value == 0 { return 0; } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_NONCE), next) }; value } // The live backend retains the actual predecessor's nonclone binding loan. // IRQ programming accesses only physical system registers, never Runtime, // memory, scheduler or the borrowed predecessor service. struct OsK960Backend<'a> { transfer: OsK940IrqTransfer<'a>, redistributor: u64, programs: u64, } unsafe impl os_k960_clock::Backend for OsK960Backend<'_> { fn masked(&self) -> bool { os_k940_registers().3 & 0x80 != 0 } fn frequency(&self) -> u64 { os_k960_frequency() } fn counter(&self) -> u64 { os_k960_counter() } fn program(&mut self, deadline: Option) -> Result<(), os_k960_clock::Error> { if !self.masked() { return Err(os_k960_clock::Error::Unmasked); } let now = os_k960_counter(); // Conservative physical horizon below the architectural minimum // counter width; full u64 overflow handling remains in the core. // This bootstrap backend deliberately does not infer a 64-bit width. if now >= 1u64 << 55 || deadline.is_some_and(|v| v >= 1u64 << 55) { return Err(os_k960_clock::Error::Overflow); } let next = self.programs.checked_add(1).ok_or(os_k960_clock::Error::Overflow)?; unsafe { asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem,nostack,preserves_flags)); if let Some(cval) = deadline { asm!("msr CNTP_CVAL_EL0, {cval}", "isb", "msr CNTP_CTL_EL0, {enable}", "isb", cval=in(reg)cval,enable=in(reg)1u64,options(nomem,nostack,preserves_flags)); } } let (actual, control) = os_k960_timer(); if control & 3 != u64::from(deadline.is_some()) || deadline.is_some_and(|v| actual != v) { return Err(os_k960_clock::Error::Backend); } self.programs = next; os_k960_event(if deadline.is_some() { 1 } else { 2 }, actual, control); Ok(()) } } type OsK960Bank<'a> = os_k960_clock::Bank, 4>; static mut OS_K960_BANK: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K960_HANDOFF: Option = None; static mut OS_K960_USERS: [Option; 2] = [None, None]; static mut OS_K960_TASKS: *const OsK660Tasks = core::ptr::null(); #[unsafe(no_mangle)] static mut OS_K960_EVIDENCE: [u64; 120] = [0; 120]; #[unsafe(no_mangle)] static mut OS_K960_STATE: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K960_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K960_AUTHORITY_LIVE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K960_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K960_OWNED: [u64; 12] = [0; 12]; #[unsafe(no_mangle)] static mut OS_K960_SLOT_RECORDS: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K960_EXPIRIES: [u64; 50] = [0; 50]; #[unsafe(no_mangle)] static mut OS_K960_TIMER_FRAMES: [os_k920_exception::Frame; 5] = [OS_K920_ZERO_FRAME; 5]; #[unsafe(no_mangle)] static mut OS_K960_USER_FRAMES: [os_k920_exception::Frame; 6] = [OS_K920_ZERO_FRAME; 6]; #[unsafe(no_mangle)] static mut OS_K960_TASK_PROGRESS: [u64; 60] = [0; 60]; #[unsafe(no_mangle)] static mut OS_K960_DELIVERIES: [u64; 40] = [0; 40]; #[unsafe(no_mangle)] static mut OS_K960_DIAGNOSTICS: [u64; 6] = [0; 6]; #[unsafe(no_mangle)] static mut OS_K960_KERNEL_FRAME: u64 = 0; #[unsafe(no_mangle)] static mut OS_K960_IRQ_SEEN: u64 = 0; #[unsafe(no_mangle)] static mut OS_K960_CAPTURE_GATE: u8 = 0; #[unsafe(no_mangle)] static mut OS_K960_LAST_TIMER: u64 = 0; #[unsafe(no_mangle)] static mut OS_K960_PRE_IRQ_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K960_PRE_IRQ_LAST: u64 = 0; #[unsafe(no_mangle)] static mut OS_K960_PRE_IRQ_ACTIVE: u32 = 0; fn os_k960_bank() -> *mut OsK960Bank<'static> { core::ptr::addr_of_mut!(OS_K960_BANK).cast() } fn os_k960_state(index: usize) -> u64 { unsafe { read_volatile(core::ptr::addr_of!(OS_K960_STATE).cast::().add(index)) } } fn os_k960_set(index: usize, value: u64) { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_STATE).cast::().add(index), value) }; } fn os_k960_word(index: usize, value: u64) { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_EVIDENCE).cast::().add(index), value) }; } struct OsK960Installation<'a> { pointer: *mut OsK960Bank<'a>, active: bool, ownership: core::marker::PhantomData>, } impl<'a> OsK960Installation<'a> { unsafe fn install(bank: OsK960Bank<'a>) -> Self { let pointer = core::ptr::addr_of_mut!(OS_K960_BANK).cast::>(); unsafe { pointer.write(bank) }; Self { pointer, active: true, ownership: core::marker::PhantomData } } fn retire(mut self) -> OsK960Backend<'a> { if os_k940_registers().3 & 0x80 == 0 || os_k960_state(0) != 0 || unsafe { (&*core::ptr::addr_of!(OS_K960_HANDOFF)).is_some() } { park(); } let bank = unsafe { self.pointer.read() }; let backend = bank.into_backend().unwrap_or_else(|_| park()); self.active = false; backend } } impl Drop for OsK960Installation<'_> { fn drop(&mut self) { if self.active { park(); } } } fn os_k960_diagnostics(offset: usize) { let log = unsafe { (&*core::ptr::addr_of!(OS_K680_LOG)).as_ref() }.unwrap_or_else(|| park()); for (index, value) in [log.boot().value(), log.next_sequence(), log.len() as u64].into_iter().enumerate() { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_DIAGNOSTICS).cast::().add(offset+index), value) }; } } fn os_k960_snapshot(stage: u64) { let (ttbr,tcr,vbar,daif) = os_k940_registers(); let before_counter = os_k960_counter(); let (cval,ctl) = os_k960_timer(); for (index,value) in [(5,stage),(6,ttbr),(7,tcr),(8,vbar),(9,daif),(10,os_k960_counter()),(11,cval),(12,ctl)] { os_k960_word(index,value); } os_k960_word(49,before_counter); let gic_active=unsafe{read_volatile(os_k960_state_address() as *const u32)}; os_k960_word(71,gic_active as u64); if os_k960_state(0) == 1 { let bank = unsafe { &*os_k960_bank() }; let snapshot = bank.snapshot(); for(index,row)in bank.os_k960_slot_records().into_iter().enumerate(){ for(field,value)in row.into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_SLOT_RECORDS).cast::().add(index*8+field),value)};} } for (index,value) in [(13,snapshot.live as u64),(14,snapshot.armed as u64), (15,snapshot.pending as u64),(16,snapshot.last_counter),(17,snapshot.statistics.deliveries), (18,snapshot.statistics.elapsed),(19,snapshot.statistics.missed), (20,snapshot.statistics.cancelled_expiries),(21,snapshot.faulted as u64)] { os_k960_word(index,value); } bank.observe(|backend| { os_k960_word(22,backend.programs); // This main-only observation traverses the retained Runtime loan. // IRQ captures retain the previously measured binding count0. if unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}==0{ os_k960_word(23,backend.transfer.predecessor_bindings()); } let enabled = unsafe { read_volatile((backend.redistributor+0x1_0100) as *const u32) }; os_k960_word(24,enabled as u64); }); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_LAST_TIMER), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER))) }; os_k960_word(25,unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }); os_k960_word(26,os_k960_state(10)); os_k960_word(27,os_k960_state(18)); os_k960_word(28,unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } as u64); os_k960_word(48,os_k960_state(19)); } fn os_k960_state_address()->u64{ unsafe{read_volatile(core::ptr::addr_of!(OS_K960_EVIDENCE).cast::().add(70))} } fn os_k960_capture(stage: u64) { os_k960_snapshot(stage); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_CAPTURE_GATE),0) }; write_str("TURKOS/OS/K960: CAPTURE STAGE=0x"); write_hex(stage as usize); write_str(" GATE=0x"); write_hex(core::ptr::addr_of!(OS_K960_CAPTURE_GATE) as usize); write_line(" READY"); let deadline = os_k960_counter().checked_add(os_k960_frequency().checked_mul(90).unwrap_or_else(|| park())).unwrap_or_else(|| park()); while unsafe { read_volatile(core::ptr::addr_of!(OS_K960_CAPTURE_GATE)) } != 1 { if os_k960_counter() >= deadline { park(); } core::hint::spin_loop(); } } fn os_k960_wait_irq() -> os_k960_clock::Expiry { if unsafe { (&*core::ptr::addr_of!(OS_K960_HANDOFF)).is_some() } { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_IRQ_SEEN),0) }; let deadline = os_k960_counter().checked_add(os_k960_frequency().checked_mul(5).unwrap_or_else(|| park())).unwrap_or_else(|| park()); unsafe { asm!("msr DAIFClr, #2", "isb", options(nostack,preserves_flags)) }; while unsafe { read_volatile(core::ptr::addr_of!(OS_K960_IRQ_SEEN)) } != 1 { if os_k960_counter() >= deadline { park(); } core::hint::spin_loop(); } unsafe { asm!("msr DAIFSet, #2", "isb", options(nostack,preserves_flags)) }; unsafe { (&mut *core::ptr::addr_of_mut!(OS_K960_HANDOFF)).take() }.unwrap_or_else(|| park()) } fn os_k960_user(index: usize) -> &'static mmu::UserSpace { if index >= 2 { park(); } unsafe { (&*core::ptr::addr_of!(OS_K960_USERS)).get(index).and_then(Option::as_ref) }.unwrap_or_else(|| park()) } struct OsK960ReturnAuthority; impl os_k920_exception::ReturnAuthority for OsK960ReturnAuthority { fn owner_root(&self, user: os_k920_exception::User) -> Result<(),os_k920_exception::Error> { let space = os_k960_user(os_k960_state(1) as usize); if user.owner != os_k960_state(5) || user.generation != os_k960_state(6) || user.root != space.root || user.asid != 1 || os_k940_registers().0 != space.root | (1<<48) { return Err(os_k920_exception::Error::Owner); } Ok(()) } fn executable(&self,_:os_k920_exception::User,entry:u64)->Result<(),os_k920_exception::Error> { let space=os_k960_user(os_k960_state(1) as usize); let pte=space.descriptor(0).ok_or(os_k920_exception::Error::Entry)?; if !(mmu::USER_CODE_VA..mmu::USER_CODE_VA+4096).contains(&entry) || entry%4!=0 || pte&0x0000_ffff_ffff_f000!=space.code_frame() || pte&3!=3 || pte&(1<<6|1<<7|1<<11|1<<53)!=(1<<6|1<<7|1<<11|1<<53) || pte&1<<54!=0 { return Err(os_k920_exception::Error::Entry); } Ok(()) } fn writable_stack(&self,_:os_k920_exception::User,sp:u64)->Result<(),os_k920_exception::Error> { let space=os_k960_user(os_k960_state(1) as usize); let pte=space.descriptor(3).ok_or(os_k920_exception::Error::Stack)?; if sp!=mmu::USER_STACK_TOP || pte&0x0000_ffff_ffff_f000!=space.stack_frame() || pte&3!=3 || pte&(3<<6)!=1<<6 || pte&(1<<53|1<<54)!=(1<<53|1<<54) || space.descriptor(2)!=Some(0) { return Err(os_k920_exception::Error::Stack); } Ok(()) } } #[unsafe(no_mangle)] extern "C" fn os_k960_dispatch(pointer:*mut os_k920_exception::Frame)->u64 { if os_k960_state(0)!=1 || os_k940_registers().2!=os_k960_state(14) { park(); } let address=pointer as u64; let stack=os_k920_exception::Stack::new(os_k960_state(11),os_k960_state(12)).unwrap_or_else(|_|park()); stack.frame_address(address).unwrap_or_else(|_|park()); let frame=unsafe{&mut *pointer}; os_k920_exception::validate_ingress(frame,address,stack).unwrap_or_else(|_|park()); if frame.vector_slot==8 { let index=os_k960_state(1) as usize; let space=os_k960_user(index); if frame.esr!=0x56000960 || frame.elr!=mmu::USER_CODE_VA+36 { park(); } let user=os_k920_exception::User{owner:os_k960_state(5),generation:os_k960_state(6),root:space.root,asid:1}; os_k920_exception::authorize_user_return(frame,address,stack,user,&OsK960ReturnAuthority).unwrap_or_else(|_|park()); let tasks=unsafe{read_volatile(core::ptr::addr_of!(OS_K960_TASKS))}; if tasks.is_null(){park();} let selected=unsafe{(&*tasks).scheduler.running()}.unwrap_or_else(||park()); if selected.slot as u64+1!=user.owner || selected.generation!=user.generation {park();} let n=os_k960_state(10) as usize; let progress=unsafe{read_volatile(space.data_frame() as *const u64)}; let stack_value=unsafe{read_volatile((space.stack_frame()+4080) as *const u64)}; if n>=6 || progress!=os_k960_state(20+index)+1 || progress!=frame.x[0] || progress!=stack_value {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_USER_FRAMES).cast::().add(n),*frame)}; for(field,value)in [index as u64,selected.slot as u64,selected.generation,progress,space.root, space.code_frame(),space.data_frame(),space.stack_frame(),frame.elr,frame.sp_el0].into_iter().enumerate(){ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_TASK_PROGRESS).cast::().add(n*10+field),value)}; } os_k960_set(10,n as u64+1);os_k960_set(20+index,progress); os_k960_event(0x20,user.owner,progress); unsafe{asm!("dsb ishst","msr TTBR0_EL1, {root}","tlbi vmalle1","dsb ish","isb",root=in(reg)os_k960_state(2),options(nostack,preserves_flags))}; return 1; } if frame.vector_slot!=5 {park();} os_k920_kernel_irq_return(frame,address,stack,core::ptr::addr_of!(_text_start) as u64, core::ptr::addr_of!(_text_end) as u64,__os_k920_kernel_tco_observed()).unwrap_or_else(|_|park()); let n=os_k960_state(18) as usize; if n>=5 || unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}!=0 {park();} let iar:u64;unsafe{asm!("mrs {iar}, ICC_IAR1_EL1",iar=out(reg)iar,options(nomem,nostack,preserves_flags))}; let(cval,ctl)=os_k960_timer(); if iar&0x00ff_ffff!=30 || ctl&7!=5 {park();} os_k960_word(72+n,ctl);os_k960_word(77+n,cval); os_k960_word(82+n,os_k960_counter());os_k960_word(87+n,iar); let active_address=unsafe{(&*os_k960_bank()).observe(|b|b.redistributor+0x1_0300)}; let active=unsafe{read_volatile(active_address as *const u32)}; if active&(1<<30)==0{park();}os_k960_word(96+n,active as u64); unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),1); write_volatile(core::ptr::addr_of_mut!(OS_K960_TIMER_FRAMES).cast::().add(n),*frame); } os_k960_event(0x10,iar,cval); // One bounded bank operation hands an owned expiry to main. No callback, // scheduler loan, allocation or frame retirement takes place in this IRQ. let expiry=unsafe{(&mut *os_k960_bank()).take_due()}.unwrap_or_else(|_|park()).unwrap_or_else(||park()); for(field,value)in expiry.os_k960_record().into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_EXPIRIES).cast::().add(n*10+field),value)};} if n==3{os_k960_word(93,expiry.delivery().scheduled);os_k960_word(94,expiry.delivery().observed);os_k960_word(103,address);} if unsafe{(&*core::ptr::addr_of!(OS_K960_HANDOFF)).is_some()}{park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_HANDOFF),Some(expiry));} let split:u64;unsafe{asm!("mrs {split}, ICC_CTLR_EL1",split=out(reg)split,options(nomem,nostack,preserves_flags)); asm!("dsb sy","msr ICC_EOIR1_EL1, {iar}","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags)); if split&2!=0{asm!("msr ICC_DIR_EL1, {iar}","dsb sy","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags));} } os_k960_word(92,split&2); os_k960_event(0x11,iar,split&2); let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}.checked_add(1).unwrap_or_else(||park()); unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT),count); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER),os_k960_counter()); } os_k960_set(18,n as u64+1); if n==3{os_k960_capture(5);} unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),0); write_volatile(core::ptr::addr_of_mut!(OS_K960_IRQ_SEEN),1); } 0 } fn os_k960_owner(id: turkos_kernel::task::TaskId) -> os_k960_clock::Owner { os_k960_clock::Owner { id: id.slot as u64+1, generation: id.generation } } fn os_k960_channel(index: usize) -> u64 { 0x9601+index as u64 } fn os_k960_checkpoint(stage:u64,memory:&OsK660Memory) { os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K960_AUTHORITY_LIVE).cast()); os_k960_capture(stage); } fn os_k960_run_user(tasks:&mut OsK660Tasks,index:usize,id:turkos_kernel::task::TaskId,woken:bool)->Result<(),&'static str> { unsafe extern "C" { fn __os_k960_enter_user(entry:u64,sp:u64); } if tasks.scheduler.running()!=Some(id) {return Err("clock-user-selection");} if woken && tasks.scheduler.take_wake_cause(id)!=Ok(Some(turkos_kernel::task::WakeCause::TimedOut)) { return Err("clock-user-wake-cause"); } let user=os_k960_user(index); os_k960_set(1,index as u64);os_k960_set(5,id.slot as u64+1);os_k960_set(6,id.generation); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_TASKS),tasks as *const OsK660Tasks); asm!("dsb ishst","msr TTBR0_EL1, {root}","tlbi vmalle1","dsb ish","isb",root=in(reg)(user.root|(1<<48)),options(nostack,preserves_flags)); __os_k960_enter_user(mmu::USER_CODE_VA,mmu::USER_STACK_TOP) }; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_TASKS),core::ptr::null());} if os_k940_registers().0!=os_k960_state(2) {return Err("clock-user-root-return");} Ok(()) } fn os_k960_dispatch_main(expiry:os_k960_clock::Expiry,tasks:&mut OsK660Tasks,ids:[turkos_kernel::task::TaskId;2]) -> Result<(),&'static str> { let bank=unsafe{&mut *os_k960_bank()}; let redistributor=bank.observe(|backend|backend.redistributor); bank.dispatch(expiry,|delivery|{ let active=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}; let gic_active=unsafe{read_volatile((redistributor+0x1_0300) as *const u32)}; if active!=0 || gic_active&(1<<30)!=0 || os_k940_registers().3&0x80==0 {park();} let index=ids.iter().position(|id|os_k960_owner(*id)==delivery.owner).unwrap_or_else(||park()); let id=ids[index]; if tasks.scheduler.wake_blocked(id,os_k960_channel(index),turkos_kernel::task::WaitKind::Receive, turkos_kernel::task::WakeCause::TimedOut)!=Ok(()) {park();} let n=os_k960_state(19) as usize;if n>=4{park();} for(field,value)in [delivery.owner.id,delivery.owner.generation,delivery.scheduled,delivery.observed, delivery.elapsed,delivery.missed,os_k940_registers().3,active as u64,os_k960_counter(),gic_active as u64].into_iter().enumerate(){ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_DELIVERIES).cast::().add(n*10+field),value)}; } os_k960_set(19,n as u64+1);os_k960_event(0x30,delivery.owner.id,delivery.elapsed); }).map_err(|_|"clock-main-dispatch") } fn os_k960_block(tasks:&mut OsK660Tasks,index:usize,expected:turkos_kernel::task::TaskId)->Result<(),&'static str>{ if tasks.scheduler.block_current(os_k960_channel(index),turkos_kernel::task::WaitKind::Receive,None,os_k960_counter())!=Ok(Some(expected)){ return Err("clock-task-wait-transition"); }Ok(()) } fn os_k960_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str>{ unsafe extern "C" {static __os_k960_vectors:u8;static __os_k960_user_code:u8;static __os_k960_user_code_end:u8;} unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K960_PRE_IRQ_COUNT),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))); write_volatile(core::ptr::addr_of_mut!(OS_K960_PRE_IRQ_LAST),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER))); write_volatile(core::ptr::addr_of_mut!(OS_K960_PRE_IRQ_ACTIVE),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))); } if cfg!(feature="os-runtime-product") && !os_k1100_product_probe_enabled(boot)? { os_k960_product_configuration();write_line("TURKOS/OS/K960: PRODUCT-CONTROL NO-CLOCK-PROBE");return Ok(()); } let(old_ttbr,old_tcr,old_vbar,old_daif)=os_k940_registers(); if old_ttbr!=tables.root || old_daif&0x80==0 || os_k960_timer().1&1!=0 {return Err("clock-predecessor-control");} let frequency=os_k960_clock::Frequency::new(os_k960_frequency()).map_err(|_|"clock-frequency")?; let guard=unsafe{read_volatile(core::ptr::addr_of!(OS_K900_STACK_EVIDENCE).cast::().add(13))}; let low=guard+4096;let top=low+65536-16;let sp:u64; unsafe{asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags))}; if sp<=low+848 || sp>top || !tables.task_stack_guard_unmapped(guard) || unsafe{read_volatile(low as *const u64)}!=OS_K900_LOW_CANARY || unsafe{read_volatile((top+8) as *const u64)}!=OS_K900_HIGH_CANARY {return Err("clock-guarded-stack");} let vectors=core::ptr::addr_of!(__os_k960_vectors) as u64; let (_,memory,irq,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?; let alpha=tasks.id.ok_or("clock-alpha")?; if tasks.scheduler.running()!=Some(alpha){return Err("clock-alpha-running");} os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K960_AUTHORITY_BEFORE).cast()); os_k960_diagnostics(0); let before_binding=::ownership(irq).irq_bindings as u64; let before_count=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}; let transfer=os_k940_transfer_irq(irq,boot.interrupts.gic_redistributor)?; let backend=OsK960Backend{transfer,redistributor:boot.interrupts.gic_redistributor,programs:0}; let bank=OsK960Bank::new(backend).map_err(|_|"clock-bank-constructor")?; let installation=unsafe{OsK960Installation::install(bank)}; for(index,value)in [(0,1),(2,old_ttbr),(3,old_tcr),(4,old_vbar),(11,low),(12,top+16),(14,vectors)]{os_k960_set(index,value);} for(index,value)in [(0,0x4b393630434c4f43),(1,old_ttbr),(2,old_tcr),(3,old_vbar),(4,frequency.hz()), (29,before_count),(30,before_binding),(31,low),(32,top),(33,guard),(34,old_daif)] {os_k960_word(index,value);} os_k960_word(68,core::ptr::addr_of!(OS_K660_IRQ_ACTIVE) as u64); os_k960_word(69,core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER) as u64); os_k960_word(70,boot.interrupts.gic_redistributor+0x1_0300); os_k960_word(95,__os_k920_kernel_tco_observed()); if before_binding!=1 {return Err("clock-transfer-owner");} let program=core::ptr::addr_of!(__os_k960_user_code) as u64; let program_end=core::ptr::addr_of!(__os_k960_user_code_end) as u64; if program_end<=program || program_end-program>4096 {return Err("clock-user-code");} for index in 0..2 { let user=unsafe{tables.build_user_space(&mut memory.allocator,boot.hardware.ram_base,boot.hardware.ram_size)}.map_err(|_|"clock-user-allocation")?; if !user.contract_valid(tables.root){return Err("clock-user-descriptors");} for offset in 0..program_end-program{unsafe{write_volatile((user.code_frame()+offset) as *mut u8,read_volatile((program+offset) as *const u8))};} let ctr:u64;unsafe{asm!("mrs {ctr}, CTR_EL0",ctr=out(reg)ctr,options(nomem,nostack,preserves_flags))}; let line=4u64.checked_shl(((ctr>>16)&15) as u32).ok_or("clock-cache-line")?; let code_end=user.code_frame().checked_add(program_end-program).ok_or("clock-cache-span")?; let mut address=user.code_frame()&!(line-1); while address().add(index*6+field),address)};} os_k960_word(50+index*4,user.root);os_k960_word(51+index*4,user.code_frame()); os_k960_word(52+index*4,user.data_frame());os_k960_word(53+index*4,user.stack_frame()); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_USERS).cast::>().add(index),Some(user))}; } let ids=[tasks.scheduler.spawn().map_err(|_|"clock-beta")?,tasks.scheduler.spawn().map_err(|_|"clock-gamma")?]; unsafe{asm!("msr VBAR_EL1, {vectors}","isb",vectors=in(reg)vectors,options(nomem,nostack,preserves_flags));} for(index,id)in ids.iter().enumerate(){os_k960_word(58+index*2,id.slot as u64);os_k960_word(59+index*2,id.generation);} if tasks.scheduler.yield_current(false)!=Ok(Some(ids[0])){return Err("clock-initial-beta");} os_k960_run_user(tasks,0,ids[0],false)?;os_k960_block(tasks,0,ids[1])?; os_k960_run_user(tasks,1,ids[1],false)?;os_k960_block(tasks,1,alpha)?; // Actual frequency conversions and explicit software boundaries execute // before arming. These refusals cannot alter the physical timer program. let writes_before=unsafe{(&*os_k960_bank()).observe(|b|b.programs)}; let known=frequency.ticks_for_ns(1_000_000).map_err(|_|"clock-conversion")?; if known==0 || frequency.ns_for_ticks(known).map_err(|_|"clock-inverse")?<1_000_000 || os_k960_clock::Frequency::new(0)!=Err(os_k960_clock::Error::Frequency) || frequency.ticks_for_ns(u64::MAX)!=Err(os_k960_clock::Error::Overflow) || frequency.after(u64::MAX,1_000_000)!=Err(os_k960_clock::Error::Overflow) || unsafe{(&mut *os_k960_bank()).arm(os_k960_owner(ids[0]),0,None)}.unwrap_err()!=os_k960_clock::Error::Past {return Err("clock-boundary-refusals");} let writes_after=unsafe{(&*os_k960_bank()).observe(|b|b.programs)}; if writes_before!=writes_after{return Err("clock-refusal-hardware-mutation");} os_k960_word(35,known);os_k960_word(36,writes_before);os_k960_word(37,writes_after); os_k960_checkpoint(0,memory); let now=os_k960_counter(); let late=frequency.after(now,200_000_000).map_err(|_|"clock-late-deadline")?; let early=frequency.after(now,100_000_000).map_err(|_|"clock-early-deadline")?; let beta=unsafe{(&mut *os_k960_bank()).arm(os_k960_owner(ids[0]),late,None)}.map_err(|_|"clock-beta-arm")?; let gamma=unsafe{(&mut *os_k960_bank()).arm(os_k960_owner(ids[1]),early,None)}.map_err(|_|"clock-gamma-arm")?; os_k960_word(38,late);os_k960_word(39,early);os_k960_word(40,beta.slot() as u64);os_k960_word(41,gamma.slot() as u64); os_k960_checkpoint(1,memory); let first=os_k960_wait_irq(); if first.delivery().owner!=os_k960_owner(ids[1]) {return Err("clock-deadline-order-first");} os_k960_dispatch_main(first,tasks,ids)?; if tasks.scheduler.yield_current(false)!=Ok(Some(ids[1])){return Err("clock-gamma-wake-selection");} os_k960_run_user(tasks,1,ids[1],true)?;os_k960_block(tasks,1,alpha)?; os_k960_checkpoint(2,memory); let second=os_k960_wait_irq(); if second.delivery().owner!=os_k960_owner(ids[0]) {return Err("clock-deadline-order-second");} os_k960_dispatch_main(second,tasks,ids)?; if tasks.scheduler.yield_current(false)!=Ok(Some(ids[0])){return Err("clock-beta-wake-selection");} os_k960_run_user(tasks,0,ids[0],true)?;os_k960_block(tasks,0,alpha)?; unsafe{(&mut *os_k960_bank()).cancel(beta,os_k960_owner(ids[0]))}.map_err(|_|"clock-beta-cancel")?; unsafe{(&mut *os_k960_bank()).cancel(gamma,os_k960_owner(ids[1]))}.map_err(|_|"clock-gamma-cancel")?; os_k960_checkpoint(3,memory); let period=frequency.ticks_for_ns(20_000_000).map_err(|_|"clock-period")?; let deadline=os_k960_counter().checked_add(period).ok_or("clock-period-deadline")?; let periodic=unsafe{(&mut *os_k960_bank()).arm(os_k960_owner(ids[1]),deadline,Some(period))}.map_err(|_|"clock-period-arm")?; let release=deadline.checked_add(period.checked_mul(3).ok_or("clock-delay-overflow")?).and_then(|v|v.checked_add(period/2)).ok_or("clock-delay-overflow")?; while os_k960_counter()::ownership(irq).irq_bindings as u64); for index in 0..2{ let user=unsafe{(&mut *core::ptr::addr_of_mut!(OS_K960_USERS)).get_mut(index).and_then(Option::take)}.ok_or("clock-user-release-owner")?; unsafe{asm!("dsb ishst","tlbi vmalle1","dsb ish","isb",options(nostack,preserves_flags)); user.release_after_tlbi(&mut memory.allocator,tables.root)}.map_err(|_|"clock-user-release")?; } unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K960_TASKS),core::ptr::null());} os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K960_AUTHORITY_AFTER).cast()); if !os_k920_inventory_equal(core::ptr::addr_of!(OS_K960_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K960_AUTHORITY_AFTER).cast()) || os_k940_registers()!=(old_ttbr,old_tcr,old_vbar,old_daif) || os_k960_timer().1&1!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}!=before_count+5 || os_k960_state(10)!=6 || os_k960_state(19)!=4 || os_k960_state(18)!=5 || unsafe{read_volatile(low as *const u64)}!=OS_K900_LOW_CANARY || unsafe{read_volatile((top+8) as *const u64)}!=OS_K900_HIGH_CANARY {return Err("clock-owner-restoration");} os_k960_diagnostics(3);os_k960_word(47,1);os_k960_checkpoint(8,memory); write_line("TURKOS/OS/K945: PASS");write_line("TURKOS/OS/K950: PASS"); write_line("TURKOS/OS/K955: PASS");write_line("TURKOS/OS/K960: PASS IRQ=5 EL0=6 CALLBACKS=4 OWNERS=RESTORED");Ok(()) } #[inline(never)] fn os_k960_product_configuration(){ unsafe extern "C"{static __os_k960_vectors:u8;static __os_k960_user_code:u8;} for pointer in [core::ptr::addr_of!(OS_K960_EVIDENCE).cast::(),core::ptr::addr_of!(OS_K960_STATE).cast(), core::ptr::addr_of!(OS_K960_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K960_AUTHORITY_LIVE).cast(), core::ptr::addr_of!(OS_K960_AUTHORITY_AFTER).cast(),core::ptr::addr_of!(OS_K960_OWNED).cast(), core::ptr::addr_of!(OS_K960_SLOT_RECORDS).cast(),core::ptr::addr_of!(OS_K960_EXPIRIES).cast(), core::ptr::addr_of!(OS_K960_TIMER_FRAMES).cast(),core::ptr::addr_of!(OS_K960_USER_FRAMES).cast(), core::ptr::addr_of!(OS_K960_TASK_PROGRESS).cast(),core::ptr::addr_of!(OS_K960_DELIVERIES).cast(), core::ptr::addr_of!(OS_K960_DIAGNOSTICS).cast(),core::ptr::addr_of!(OS_K960_KERNEL_FRAME).cast(), core::ptr::addr_of!(OS_K960_IRQ_SEEN).cast(),core::ptr::addr_of!(OS_K960_CAPTURE_GATE).cast(), core::ptr::addr_of!(OS_K960_LAST_TIMER).cast(),core::ptr::addr_of!(OS_K960_EVENTS).cast(),core::ptr::addr_of!(OS_K960_EVENT_COUNT).cast()] {if unsafe{read_volatile(pointer)}!=0{park();}} if unsafe{read_volatile(core::ptr::addr_of!(OS_K960_NONCE))}!=1{park();} for address in [core::ptr::addr_of!(__os_k960_vectors) as u64,core::ptr::addr_of!(__os_k960_user_code) as u64]{ unsafe{asm!("ldrb wzr, [{address}]",address=in(reg)address,options(nostack,preserves_flags))}; } } const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k960_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); // Read-only continuation API. This exact raw bit25 state was acquired only // from the trusted real EL1t frame before the normal timer/IRQ probes. use turkos_kernel::runtime::locking as os_k980_locking; use os_k980_locking::{irq as os_k980_irq,mutex as os_k980_mutex,spin as os_k980_spin,ordering as os_k980_ordering}; fn os_k980_cpu()->u16{ let mpidr:u64;unsafe{asm!("mrs {mpidr}, MPIDR_EL1",mpidr=out(reg)mpidr,options(nomem,nostack,preserves_flags));} if mpidr&0xff00_ffff_ff!=0{u16::MAX}else{0} } struct OsK980Registers; unsafe impl os_k980_irq::Backend for OsK980Registers{ fn cpu(&self)->u16{os_k980_cpu()} fn read_daif(&self)->u64{os_k940_registers().3} fn write_daif(&mut self,value:u64){unsafe{asm!("msr DAIF, {value}","isb",value=in(reg)value,options(nostack,preserves_flags));}} } // Explicitly one CPU. This backend has no Sync implementation and makes no // physical SMP contention claim. The frozen native host CAS stays unchanged. struct OsK980OwnerWord{word:*mut u32} unsafe impl os_k980_spin::OwnerWord for OsK980OwnerWord{ fn load(&self)->u32{unsafe{read_volatile(self.word)}} fn acquire(&self,owner:u32)->Result<(),u32>{ let old=self.load();if os_k980_cpu()!=0 || os_k940_registers().3&0x80==0 || old!=0{return Err(old);} unsafe{write_volatile(self.word,owner)};core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::Acquire);Ok(()) } fn release(&self,owner:u32)->Result<(),u32>{ let old=self.load();if os_k980_cpu()!=0 || os_k940_registers().3&0x80==0 || old!=owner{return Err(old);} core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::Release); unsafe{asm!("dsb ish",options(nostack,preserves_flags));write_volatile(self.word,0)};Ok(()) } } struct OsK980Scheduler<'a>{tasks:&'a mut OsK660Tasks,controller:turkos_kernel::task::TaskId,parks:u64,wakes:u64,cancels:u64} impl OsK980Scheduler<'_>{ fn id(task:os_k980_locking::Task)->Result{ let slot=task.id.checked_sub(1).and_then(|v|u8::try_from(v).ok()).ok_or(os_k980_mutex::Failure{error:os_k980_locking::Error::Invalid,quarantined:false})?; Ok(turkos_kernel::task::TaskId{slot,generation:task.generation}) } fn failure(_:turkos_kernel::task::Error)->os_k980_mutex::Failure{os_k980_mutex::Failure{error:os_k980_locking::Error::Backend,quarantined:false}} } unsafe impl os_k980_mutex::Backend for OsK980Scheduler<'_>{ type Park=turkos_kernel::task::OsK980PreparedScheduler;type Wake=turkos_kernel::task::OsK980PreparedScheduler;type Cancel=turkos_kernel::task::OsK980PreparedScheduler; fn identity(&self)->u64{0x980} fn observation(&self)->os_k980_mutex::Observation{ let in_irq=os_k980_state(2)!=0; // Actual IRQ refusal never observes or mutably borrows the scheduler. let task=if in_irq{None}else{self.tasks.scheduler.running().map(|id|os_k980_locking::Task{id:id.slot as u64+1,generation:id.generation})}; os_k980_mutex::Observation{cpu:os_k980_cpu(),task,in_irq,daif:os_k940_registers().3} } fn validate_task(&self,task:os_k980_locking::Task)->bool{Self::id(task).ok().is_some_and(|id|self.tasks.scheduler.state(id).is_ok_and(|state|!matches!(state,turkos_kernel::task::State::Empty|turkos_kernel::task::State::Exited{..})))} fn prepare_park(&mut self,ticket:os_k980_mutex::Ticket)->Result{ self.parks.checked_add(1).ok_or(os_k980_mutex::Failure{error:os_k980_locking::Error::Overflow,quarantined:false})?; self.tasks.scheduler.os_k980_prepare_park(Self::id(ticket.task)?,ticket.scope).map_err(Self::failure) } fn commit_park(&mut self,prepared:Self::Park){self.tasks.scheduler.os_k980_commit(prepared);self.parks+=1;} fn prepare_wake(&mut self,ticket:os_k980_mutex::Ticket)->Result{ self.wakes.checked_add(1).ok_or(os_k980_mutex::Failure{error:os_k980_locking::Error::Overflow,quarantined:false})?; self.tasks.scheduler.os_k980_prepare_wake(Self::id(ticket.task)?,ticket.scope,turkos_kernel::task::WakeCause::MessageAvailable).map_err(Self::failure) } fn commit_wake(&mut self,prepared:Self::Wake){self.tasks.scheduler.os_k980_commit(prepared);self.wakes+=1;} fn authorize_cancel(&self,caller:os_k980_locking::Task,waiter:os_k980_locking::Task)->bool{caller==waiter || Self::id(caller).ok()==Some(self.controller)} fn prepare_cancel(&mut self,ticket:os_k980_mutex::Ticket)->Result{self.cancels.checked_add(1).ok_or(os_k980_mutex::Failure{error:os_k980_locking::Error::Overflow,quarantined:false})?;self.tasks.scheduler.os_k980_prepare_wake(Self::id(ticket.task)?,ticket.scope,turkos_kernel::task::WakeCause::WaitCancelled).map_err(Self::failure)} fn commit_cancel(&mut self,prepared:Self::Cancel){self.tasks.scheduler.os_k980_commit(prepared);self.cancels+=1;} } type OsK980Mutex<'a>=os_k980_mutex::Mutex,2,u64>; static mut OS_K980_MUTEX:core::mem::MaybeUninit>=core::mem::MaybeUninit::uninit(); static mut OS_K980_USERS:[Option;2]=[None,None]; static mut OS_K980_TASK_LOAN:*const OsK660Tasks=core::ptr::null(); #[unsafe(no_mangle)]static mut OS_K980_STATE:[u64;32]=[0;32]; #[unsafe(no_mangle)]static mut OS_K980_EVIDENCE:[u64;96]=[0;96]; #[unsafe(no_mangle)]static mut OS_K980_AUTHORITY_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)]static mut OS_K980_AUTHORITY_LIVE:[u64;328]=[0;328]; #[unsafe(no_mangle)]static mut OS_K980_AUTHORITY_AFTER:[u64;328]=[0;328]; #[unsafe(no_mangle)]static mut OS_K980_OWNED:[u64;12]=[0;12]; #[unsafe(no_mangle)]static mut OS_K980_MUTEX_RECORDS:[u64;32]=[0;32]; #[unsafe(no_mangle)]static mut OS_K980_IRQ_MUTEX_BEFORE:[u64;32]=[0;32]; #[unsafe(no_mangle)]static mut OS_K980_IRQ_MUTEX_AFTER:[u64;32]=[0;32]; #[unsafe(no_mangle)]static mut OS_K980_SCHEDULER:[u64;48]=[0;48]; #[unsafe(no_mangle)]static mut OS_K980_SCHEDULER_BEFORE:[u64;48]=[0;48]; #[unsafe(no_mangle)]static mut OS_K980_DIAGNOSTICS:[u64;6]=[0;6]; #[unsafe(no_mangle)]static mut OS_K980_TIMER_FRAMES:[os_k920_exception::Frame;2]=[OS_K920_ZERO_FRAME;2]; #[unsafe(no_mangle)]static mut OS_K980_USER_FRAMES:[os_k920_exception::Frame;4]=[OS_K920_ZERO_FRAME;4]; #[unsafe(no_mangle)]static mut OS_K980_TASK_PROGRESS:[u64;48]=[0;48]; #[unsafe(no_mangle)]static mut OS_K980_FRAME_ADDRESSES:[u64;6]=[0;6]; #[unsafe(no_mangle)]static mut OS_K980_KERNEL_FRAME:u64=0; #[unsafe(no_mangle)]static mut OS_K980_IRQ_SEEN:u64=0; #[unsafe(no_mangle)]static mut OS_K980_PRE_IRQ_COUNT:u64=0; #[unsafe(no_mangle)]static mut OS_K980_PRE_IRQ_LAST:u64=0; #[unsafe(no_mangle)]static mut OS_K980_PRE_IRQ_ACTIVE:u32=0; #[unsafe(no_mangle)]static mut OS_K980_LAST_TIMER:u64=0; #[unsafe(no_mangle)]static mut OS_K980_CAPTURE_GATE:u8=0; #[unsafe(no_mangle)]static mut OS_K980_SPIN_MEMORY_WORD:u32=0; #[unsafe(no_mangle)]static mut OS_K980_SPIN_PROCESS_WORD:u32=0; fn os_k980_state(index:usize)->u64{unsafe{read_volatile(core::ptr::addr_of!(OS_K980_STATE).cast::().add(index))}} fn os_k980_set(index:usize,value:u64){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_STATE).cast::().add(index),value)}} fn os_k980_word(index:usize,value:u64){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_EVIDENCE).cast::().add(index),value)}} fn os_k980_mutex()->*mut OsK980Mutex<'static>{core::ptr::addr_of_mut!(OS_K980_MUTEX).cast()} fn os_k980_store(target:*mut u64,value:[u64;N]){for(i,v)in value.into_iter().enumerate(){unsafe{write_volatile(target.add(i),v)}}} struct OsK980Installation<'a>{pointer:*mut OsK980Mutex<'a>,active:bool,ownership:core::marker::PhantomData>} impl<'a>OsK980Installation<'a>{ unsafe fn install(value:OsK980Mutex<'a>)->Self{ if os_k980_state(3)!=0{park();}let pointer=core::ptr::addr_of_mut!(OS_K980_MUTEX).cast::>();unsafe{pointer.write(value)};os_k980_set(3,1);Self{pointer,active:true,ownership:core::marker::PhantomData} } fn retire(mut self)->(OsK980Scheduler<'a>,u64){ if os_k960_timer().1&1!=0 || os_k980_state(2)!=0{park();} let value=unsafe{self.pointer.read()};let result=value.into_resources().unwrap_or_else(|_|park());os_k980_set(3,0);self.active=false;result } } impl Drop for OsK980Installation<'_>{fn drop(&mut self){if self.active{park();}}} fn os_k980_diagnostics(offset:usize){ let log=unsafe{(&*core::ptr::addr_of!(OS_K680_LOG)).as_ref()}.unwrap_or_else(||park()); for(i,v)in[log.boot().value(),log.next_sequence(),log.len() as u64].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_DIAGNOSTICS).cast::().add(offset+i),v)}} } fn os_k980_capture(stage:u64){ let(ttbr,tcr,vbar,daif)=os_k940_registers();let(cval,ctl)=os_k960_timer(); for(i,v)in[(5,stage),(6,ttbr),(7,tcr),(8,vbar),(9,daif),(10,os_k960_counter()),(11,cval),(12,ctl), (13,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}),(14,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))} as u64), (15,os_k980_state(10)),(16,os_k980_state(18)),(17,unsafe{read_volatile(core::ptr::addr_of!(OS_K980_SPIN_MEMORY_WORD))} as u64), (18,unsafe{read_volatile(core::ptr::addr_of!(OS_K980_SPIN_PROCESS_WORD))} as u64)]{os_k980_word(i,v);} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_LAST_TIMER),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)))}; if os_k980_state(3)==1{ let bank=unsafe{&*os_k980_mutex()};os_k980_store(core::ptr::addr_of_mut!(OS_K980_MUTEX_RECORDS).cast(),bank.os_k980_observation()); // IRQ capture retains the already measured main scheduler inventory. // It traverses no predecessor Runtime, memory or scheduler borrow. if os_k980_state(2)==0{bank.with_backend(|b|os_k980_store(core::ptr::addr_of_mut!(OS_K980_SCHEDULER).cast(),b.tasks.scheduler.os_k980_inventory()));} } unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_CAPTURE_GATE),0)}; write_str("TURKOS/OS/K980: CAPTURE STAGE=0x");write_hex(stage as usize);write_str(" GATE=0x");write_hex(core::ptr::addr_of!(OS_K980_CAPTURE_GATE) as usize);write_line(" READY"); let end=os_k960_counter().checked_add(os_k960_frequency().checked_mul(90).unwrap_or_else(||park())).unwrap_or_else(||park()); while unsafe{read_volatile(core::ptr::addr_of!(OS_K980_CAPTURE_GATE))}!=1{if os_k960_counter()>=end{park();}core::hint::spin_loop();} } fn os_k980_context()->os_k980_spin::Context{ let observed=unsafe{(&*os_k980_mutex()).with_backend(os_k980_mutex::Backend::observation)}; unsafe{os_k980_spin::Context::new(observed.cpu,observed.task,observed.in_irq)}.unwrap_or_else(|_|park()) } fn os_k980_arm_timer(delay_ns:u64){ if os_k940_registers().3&0x80==0 || os_k960_timer().1&1!=0{park();} let ticks=os_k960_clock::Frequency::new(os_k960_frequency()).and_then(|hz|hz.ticks_for_ns(delay_ns)).unwrap_or_else(|_|park()); let cval=os_k960_counter().checked_add(ticks).unwrap_or_else(||park());os_k980_word(40+os_k980_state(18) as usize,cval); if cval>=1u64<<55{park();} unsafe{asm!("msr CNTP_CVAL_EL0, {cval}","isb","msr CNTP_CTL_EL0, {enable}","isb",cval=in(reg)cval,enable=in(reg)1u64,options(nostack,preserves_flags));write_volatile(core::ptr::addr_of_mut!(OS_K980_IRQ_SEEN),0)}; let(actual,ctl)=os_k960_timer();if actual!=cval||ctl&3!=1{park();} } fn os_k980_wait_irq(){ let end=os_k960_counter().checked_add(os_k960_frequency().checked_mul(5).unwrap_or_else(||park())).unwrap_or_else(||park()); while unsafe{read_volatile(core::ptr::addr_of!(OS_K980_IRQ_SEEN))}!=1{if os_k960_counter()>=end{park();}core::hint::spin_loop();} } fn os_k980_user(index:usize)->&'static mmu::UserSpace{if index>=2{park();}unsafe{(&*core::ptr::addr_of!(OS_K980_USERS)).get(index).and_then(Option::as_ref)}.unwrap_or_else(||park())} struct OsK980ReturnAuthority; impl os_k920_exception::ReturnAuthority for OsK980ReturnAuthority{ fn owner_root(&self,user:os_k920_exception::User)->Result<(),os_k920_exception::Error>{ let space=os_k980_user(os_k980_state(1) as usize); if user.owner!=os_k980_state(20) || user.generation!=os_k980_state(21) || user.root!=space.root || user.asid!=1 || os_k940_registers().0!=space.root|(1<<48){return Err(os_k920_exception::Error::Owner);}Ok(()) } fn executable(&self,_:os_k920_exception::User,entry:u64)->Result<(),os_k920_exception::Error>{ let space=os_k980_user(os_k980_state(1) as usize);let pte=space.descriptor(0).ok_or(os_k920_exception::Error::Entry)?; if !(mmu::USER_CODE_VA..mmu::USER_CODE_VA+4096).contains(&entry)||entry%4!=0||pte&0x0000_ffff_ffff_f000!=space.code_frame()||pte&3!=3||pte&(1<<6|1<<7|1<<11|1<<53)!=(1<<6|1<<7|1<<11|1<<53)||pte&1<<54!=0{return Err(os_k920_exception::Error::Entry);}Ok(()) } fn writable_stack(&self,_:os_k920_exception::User,sp:u64)->Result<(),os_k920_exception::Error>{ let space=os_k980_user(os_k980_state(1) as usize);let pte=space.descriptor(3).ok_or(os_k920_exception::Error::Stack)?; if sp!=mmu::USER_STACK_TOP||pte&0x0000_ffff_ffff_f000!=space.stack_frame()||pte&3!=3||pte&(3<<6)!=1<<6||pte&(1<<53|1<<54)!=(1<<53|1<<54)||space.descriptor(2)!=Some(0){return Err(os_k920_exception::Error::Stack);}Ok(()) } } #[unsafe(no_mangle)]extern "C" fn os_k980_dispatch(pointer:*mut os_k920_exception::Frame)->u64{ let address=pointer as u64;let stack=os_k920_exception::Stack::new(os_k980_state(11),os_k980_state(12)).unwrap_or_else(|_|park()); stack.frame_address(address).unwrap_or_else(|_|park());let frame=unsafe{&mut*pointer};os_k920_exception::validate_ingress(frame,address,stack).unwrap_or_else(|_|park()); if os_k980_state(0)!=1 || os_k940_registers().2!=os_k980_state(14){park();} if frame.vector_slot==8{ let index=os_k980_state(1) as usize;let space=os_k980_user(index);if frame.esr!=0x56000980 || frame.elr!=mmu::USER_CODE_VA+36{park();} let user=os_k920_exception::User{owner:os_k980_state(20),generation:os_k980_state(21),root:space.root,asid:1};os_k920_exception::authorize_user_return(frame,address,stack,user,&OsK980ReturnAuthority).unwrap_or_else(|_|park()); let loan=unsafe{read_volatile(core::ptr::addr_of!(OS_K980_TASK_LOAN))};if loan.is_null(){park();} let selected=unsafe{(&*loan).scheduler.running()}.unwrap_or_else(||park());if selected.slot as u64+1!=user.owner || selected.generation!=user.generation{park();} let n=os_k980_state(10) as usize;let value=unsafe{read_volatile(space.data_frame() as *const u64)}; if n>=4 || value!=os_k980_state(22+index)+1 || value!=frame.x[0] || value!=unsafe{read_volatile((space.stack_frame()+4080) as *const u64)}{park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_USER_FRAMES).cast::().add(n),*frame);write_volatile(core::ptr::addr_of_mut!(OS_K980_FRAME_ADDRESSES).cast::().add(2+n),address)}; for(field,value)in[index as u64,selected.slot as u64+1,selected.generation,value,space.root,space.code_frame(),space.data_frame(),space.stack_frame(),frame.elr,frame.sp_el0,os_k940_registers().0,os_k960_counter()].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_TASK_PROGRESS).cast::().add(n*12+field),value)}} os_k980_set(22+index,value);os_k980_set(10,n as u64+1); unsafe{asm!("dsb ishst","msr TTBR0_EL1, {root}","tlbi vmalle1","dsb ish","isb",root=in(reg)os_k980_state(4),options(nostack,preserves_flags));}return 1; } if frame.vector_slot!=5{park();} os_k920_kernel_irq_return(frame,address,stack,core::ptr::addr_of!(_text_start) as u64,core::ptr::addr_of!(_text_end) as u64,__os_k920_kernel_tco_observed()).unwrap_or_else(|_|park()); let n=os_k980_state(18) as usize;if n>=2 || unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}!=0{park();} let iar:u64;unsafe{asm!("mrs {iar}, ICC_IAR1_EL1",iar=out(reg)iar,options(nomem,nostack,preserves_flags));} let(cval,ctl)=os_k960_timer();let active=unsafe{read_volatile((os_k980_state(13)+0x1_0300) as *const u32)}; if iar&0xffffff!=30 || ctl&7!=5 || active&(1<<30)==0{park();} os_k980_word(42+n,cval);os_k980_word(44+n,ctl);os_k980_word(46+n,iar);os_k980_word(48+n,active as u64); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),1);write_volatile(core::ptr::addr_of_mut!(OS_K980_TIMER_FRAMES).cast::().add(n),*frame);write_volatile(core::ptr::addr_of_mut!(OS_K980_FRAME_ADDRESSES).cast::().add(n),address); asm!("msr CNTP_CTL_EL0, xzr","isb",options(nostack,preserves_flags));} if n==1{ os_k980_set(2,1);let bank=unsafe{&mut*os_k980_mutex()};let before=bank.os_k980_observation();os_k980_store(core::ptr::addr_of_mut!(OS_K980_IRQ_MUTEX_BEFORE).cast(),before); let context=unsafe{os_k980_spin::Context::new(os_k980_cpu(),None,true)}.unwrap_or_else(|_|park()); if !matches!(bank.request(&context),Err(os_k980_locking::Error::Context)){park();} let after=bank.os_k980_observation();if before!=after{park();}os_k980_store(core::ptr::addr_of_mut!(OS_K980_IRQ_MUTEX_AFTER).cast(),after);os_k980_word(50,1); } let split:u64;unsafe{asm!("mrs {split}, ICC_CTLR_EL1",split=out(reg)split,options(nomem,nostack,preserves_flags));asm!("dsb sy","msr ICC_EOIR1_EL1, {iar}","isb",iar=in(reg)iar,options(nostack,preserves_flags));if split&2!=0{asm!("msr ICC_DIR_EL1, {iar}","dsb sy","isb",iar=in(reg)iar,options(nostack,preserves_flags));}} let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}.checked_add(1).unwrap_or_else(||park()); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT),count);write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER),os_k960_counter())}; os_k980_set(18,n as u64+1); if n==1{os_k980_word(90,address);os_k980_capture(4);} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),0);write_volatile(core::ptr::addr_of_mut!(OS_K980_IRQ_SEEN),1)};os_k980_set(2,0);0 } fn os_k980_backend_mut(callback:impl FnOnce(&mut OsK980Scheduler<'_>)->R)->R{ if os_k960_timer().1&1!=0 || os_k980_state(2)!=0{park();} unsafe{(&mut*os_k980_mutex()).os_k980_with_backend_mut(callback)} } fn os_k980_checkpoint(stage:u64,memory:&OsK660Memory){os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K980_AUTHORITY_LIVE).cast());os_k980_capture(stage);} fn os_k980_run_user(index:usize,id:turkos_kernel::task::TaskId)->Result<(),&'static str>{ unsafe extern "C"{fn __os_k980_enter_user(entry:u64,sp:u64);} let (running,pointer)=unsafe{(&*os_k980_mutex()).with_backend(|b|(b.tasks.scheduler.running(),b.tasks as *const OsK660Tasks))}; if running!=Some(id)||os_k960_timer().1&1!=0{return Err("locking-user-selection");} let space=os_k980_user(index);let daif=os_k940_registers().3;os_k980_set(1,index as u64);os_k980_set(20,id.slot as u64+1);os_k980_set(21,id.generation); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_TASK_LOAN),pointer);asm!("dsb ishst","msr TTBR0_EL1, {root}","tlbi vmalle1","dsb ish","isb",root=in(reg)(space.root|(1<<48)),options(nostack,preserves_flags));__os_k980_enter_user(mmu::USER_CODE_VA,mmu::USER_STACK_TOP);write_volatile(core::ptr::addr_of_mut!(OS_K980_TASK_LOAN),core::ptr::null());asm!("msr DAIF, {daif}","isb",daif=in(reg)daif,options(nostack,preserves_flags));} if os_k940_registers().0!=os_k980_state(4){return Err("locking-user-return-root");}Ok(()) } fn os_k980_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str>{ unsafe extern "C"{static __os_k980_vectors:u8;static __os_k980_user_code:u8;static __os_k980_user_code_end:u8;} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_PRE_IRQ_COUNT),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)));write_volatile(core::ptr::addr_of_mut!(OS_K980_PRE_IRQ_LAST),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)));write_volatile(core::ptr::addr_of_mut!(OS_K980_PRE_IRQ_ACTIVE),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)));} if cfg!(feature="os-runtime-product") && !os_k1100_product_probe_enabled(boot)? {os_k980_product_configuration();write_line("TURKOS/OS/K980: PRODUCT-CONTROL NO-LOCKING-PROBE");return Ok(());} let(old_ttbr,old_tcr,old_vbar,old_daif)=os_k940_registers();let before_count=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}; if os_k980_cpu()!=0||old_ttbr!=tables.root||old_daif&0x80==0||os_k960_timer().1&1!=0{return Err("locking-predecessor-control");} let guard=unsafe{read_volatile(core::ptr::addr_of!(OS_K900_STACK_EVIDENCE).cast::().add(13))};let low=guard+4096;let top=low+65536-16;let sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags));} if sp<=low+848||sp>top||!tables.task_stack_guard_unmapped(guard)||unsafe{read_volatile(low as *const u64)}!=OS_K900_LOW_CANARY||unsafe{read_volatile((top+8) as *const u64)}!=OS_K900_HIGH_CANARY{return Err("locking-stack-guard");} let vectors=core::ptr::addr_of!(__os_k980_vectors) as u64;let(_,memory,irq,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?;let alpha=tasks.id.ok_or("locking-alpha")?; if tasks.scheduler.running()!=Some(alpha){return Err("locking-alpha-running");} os_k980_store(core::ptr::addr_of_mut!(OS_K980_SCHEDULER_BEFORE).cast(),tasks.scheduler.os_k980_inventory()); os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K980_AUTHORITY_BEFORE).cast());os_k980_diagnostics(0); let transfer=os_k940_transfer_irq(irq,boot.interrupts.gic_redistributor)?; if unsafe{read_volatile((boot.interrupts.gic_redistributor+0x1_0100) as *const u32)}!=1<<30{return Err("locking-only-timer-source");} for(i,v)in[(0,1),(4,old_ttbr),(11,low),(12,top+16),(13,boot.interrupts.gic_redistributor),(14,vectors)]{os_k980_set(i,v);} for(i,v)in[(0,0x4b3938304c4f434b),(1,old_ttbr),(2,old_tcr),(3,old_vbar),(4,old_daif),(19,before_count),(20,low),(21,top),(22,guard),(23,boot.interrupts.gic_redistributor)]{os_k980_word(i,v);} unsafe{asm!("msr VBAR_EL1, {vectors}","isb",vectors=in(reg)vectors,options(nostack,preserves_flags));} os_k980_checkpoint(0,memory); // No timer is enabled when entering the initial unmasked task context. unsafe{asm!("msr DAIFClr, #2","isb",options(nostack,preserves_flags));} let mut irq_state=os_k980_irq::IrqState::new(OsK980Registers).map_err(|_|"locking-irq-constructor")?; { let mut outer=irq_state.enter().map_err(|_|"locking-outer-irq")?;os_k980_word(24,outer.saved_daif()); os_k980_arm_timer(10_000_000); {let inner=outer.enter().map_err(|_|"locking-inner-irq")?;os_k980_word(25,inner.saved_daif());if inner.observation().2!=2{return Err("locking-nested-depth");}} if outer.observation().1&0x80==0{return Err("locking-inner-early-unmask");} let end=os_k960_counter().checked_add(os_k960_frequency()).ok_or("locking-pending-timeout")?; while os_k960_timer().1&4==0{if os_k960_counter()>=end{return Err("locking-pending-timeout");}core::hint::spin_loop();} if unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}!=before_count{return Err("locking-premature-irq");} os_k980_word(26,outer.observation().1);os_k980_checkpoint(1,memory);outer.finish().map_err(|_|"locking-outer-restore")?; } os_k980_wait_irq();if irq_state.observation().1!=old_daif&!0x80||irq_state.observation().2!=0{return Err("locking-exact-daif-restore");} os_k980_word(27,irq_state.observation().1);os_k980_checkpoint(2,memory); { let mask=irq_state.enter().map_err(|_|"locking-spin-mask")?;let mut context=unsafe{os_k980_spin::Context::new(0,Some(os_k980_locking::Task{id:alpha.slot as u64+1,generation:alpha.generation}),false)}.map_err(|_|"locking-spin-context")?; let memory_lock=os_k980_spin::SpinLock::with_owner(0x9801,17u64,OsK980OwnerWord{word:core::ptr::addr_of_mut!(OS_K980_SPIN_MEMORY_WORD)}).map_err(|_|"locking-spin-memory")?; let process_lock=os_k980_spin::SpinLock::with_owner(0x9802,19u64,OsK980OwnerWord{word:core::ptr::addr_of_mut!(OS_K980_SPIN_PROCESS_WORD)}).map_err(|_|"locking-spin-process")?; let mut graph=os_k980_ordering::Graph::<2,2>::new(0x980).map_err(|_|"locking-graph")?;graph.register(memory_lock.id()).map_err(|_|"locking-memory-id")?;graph.register(process_lock.id()).map_err(|_|"locking-process-id")?;let mut ordering=os_k980_ordering::Context::default(); {let(mut memory_order,mut memory_guard)=graph.acquire(&mut ordering,memory_lock.id(),||memory_lock.try_lock(&mut context,&mask)).map_err(|_|"locking-memory-acquire")?; if !matches!(memory_guard.nested(&memory_lock),Err(os_k980_locking::Error::Recursive))||memory_guard.may_sleep()!=Err(os_k980_locking::Error::Context){return Err("locking-recursion-sleep-refusal");} *memory_guard=29;os_k980_word(28,*memory_guard);os_k980_word(29,1); {let(process_order,mut process_guard)=memory_order.acquire(process_lock.id(),||memory_guard.nested(&process_lock)).map_err(|_|"locking-memory-process-order")?;*process_guard=31;drop(process_guard);drop(process_order);} os_k980_checkpoint(3,memory);drop(memory_guard);drop(memory_order); } {let(mut process_order,mut process_guard)=graph.acquire(&mut ordering,process_lock.id(),||process_lock.try_lock(&mut context,&mask)).map_err(|_|"locking-process-acquire")?; let mut attempted=0u64; if !matches!(process_order.acquire(memory_lock.id(),||{attempted+=1;process_guard.nested(&memory_lock)}),Err(os_k980_locking::Error::Cycle)){return Err("locking-inversion-refusal");} os_k980_word(35,unsafe{read_volatile(core::ptr::addr_of!(OS_K980_SPIN_PROCESS_WORD))} as u64);os_k980_word(36,unsafe{read_volatile(core::ptr::addr_of!(OS_K980_SPIN_MEMORY_WORD))} as u64);os_k980_word(37,attempted); if attempted!=0{return Err("locking-inversion-acquired");} let diagnostic=process_order.last_diagnostic().ok_or("locking-diagnostic")?;for(i,v)in[(30,diagnostic.graph),(31,diagnostic.held),(32,diagnostic.requested)]{os_k980_word(i,v);}drop(process_guard);drop(process_order); } let edges=graph.edges();os_k980_word(33,edges[0]);os_k980_word(34,edges[1]); if context.spin_depth()!=0||memory_lock.owner_cpu().is_some()||process_lock.owner_cpu().is_some(){return Err("locking-spin-release");} drop(mask); } let code=core::ptr::addr_of!(__os_k980_user_code) as u64;let code_end=core::ptr::addr_of!(__os_k980_user_code_end) as u64;if code_end<=code||code_end-code>4096{return Err("locking-user-template");} for index in 0..2{ let user=unsafe{tables.build_user_space(&mut memory.allocator,boot.hardware.ram_base,boot.hardware.ram_size)}.map_err(|_|"locking-user-allocation")?;if !user.contract_valid(tables.root){return Err("locking-user-descriptors");} for offset in 0..code_end-code{unsafe{write_volatile((user.code_frame()+offset) as *mut u8,read_volatile((code+offset) as *const u8))};} let ctr:u64;unsafe{asm!("mrs {ctr}, CTR_EL0",ctr=out(reg)ctr,options(nomem,nostack,preserves_flags));} let line=4u64.checked_shl(((ctr>>16)&15) as u32).ok_or("locking-cache-line")?;let mut address=user.code_frame()&!(line-1);while address().add(index*6+field),address)}} for(field,address)in[user.root,user.code_frame(),user.data_frame(),user.stack_frame()].into_iter().enumerate(){os_k980_word(60+index*4+field,address);} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_USERS).cast::>().add(index),Some(user))}; } let beta=tasks.scheduler.spawn().map_err(|_|"locking-beta")?;let gamma=tasks.scheduler.spawn().map_err(|_|"locking-gamma")?; for(i,id)in[beta,gamma].into_iter().enumerate(){os_k980_word(68+i*2,id.slot as u64+1);os_k980_word(69+i*2,id.generation);} let backend=OsK980Scheduler{tasks,controller:alpha,parks:0,wakes:0,cancels:0};let bank=OsK980Mutex::with_value(backend,0).map_err(|_|"locking-mutex-constructor")?;let installation=unsafe{OsK980Installation::install(bank)}; if os_k980_backend_mut(|b|b.tasks.scheduler.yield_current(false))!=Ok(Some(beta)){return Err("locking-beta-selection");} os_k980_run_user(0,beta)?;let ctx=os_k980_context();let beta_lease=match unsafe{(&mut*os_k980_mutex()).request(&ctx)}.map_err(|_|"locking-beta-request")?{os_k980_mutex::Request::Owned(lease)=>lease,_=>return Err("locking-beta-not-owner")}; unsafe{(&mut*os_k980_mutex()).with_lease(&ctx,&beta_lease,|value|{if *value!=0{park();}*value=1;})}.map_err(|_|"locking-beta-resource")?; if os_k980_backend_mut(|b|b.tasks.scheduler.yield_current(false))!=Ok(Some(gamma)){return Err("locking-gamma-selection");} os_k980_run_user(1,gamma)?;let ctx=os_k980_context();let gamma_wait=match unsafe{(&mut*os_k980_mutex()).request(&ctx)}.map_err(|_|"locking-gamma-wait")?{os_k980_mutex::Request::Waiting(wait)=>wait,_=>return Err("locking-gamma-barged")}; if unsafe{(&*os_k980_mutex()).with_backend(|b|b.tasks.scheduler.running())}!=Some(alpha){return Err("locking-wait-selected-control");} // Actual IRQ attempts the mutex only after every main reborrow has ended. os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K980_AUTHORITY_LIVE).cast()); unsafe{(&*os_k980_mutex()).with_backend(|b|os_k980_store(core::ptr::addr_of_mut!(OS_K980_SCHEDULER).cast(),b.tasks.scheduler.os_k980_inventory()))}; unsafe{asm!("msr DAIFSet, #2","isb",options(nostack,preserves_flags));}os_k980_arm_timer(10_000_000);unsafe{asm!("msr DAIFClr, #2","isb",options(nostack,preserves_flags));}os_k980_wait_irq(); if os_k980_backend_mut(|b|b.tasks.scheduler.yield_current(false))!=Ok(Some(beta)){return Err("locking-owner-resume");} os_k980_run_user(0,beta)?;let ctx=os_k980_context();unsafe{(&mut*os_k980_mutex()).release(&ctx,beta_lease)}.unwrap_or_else(|_|park()); if os_k980_backend_mut(|b|b.tasks.scheduler.exit_current(0x980))!=Ok(Some(alpha))||os_k980_backend_mut(|b|b.tasks.scheduler.reap(beta))!=Ok(0x980){return Err("locking-beta-retire");} if os_k980_backend_mut(|b|b.tasks.scheduler.yield_current(false))!=Ok(Some(gamma)){return Err("locking-handoff-selection");} if os_k980_backend_mut(|b|b.tasks.scheduler.take_wake_cause(gamma))!=Ok(Some(turkos_kernel::task::WakeCause::MessageAvailable)){return Err("locking-real-wake-cause");} os_k980_run_user(1,gamma)?;let ctx=os_k980_context();let gamma_lease=unsafe{(&mut*os_k980_mutex()).claim(&ctx,gamma_wait)}.unwrap_or_else(|_|park()); unsafe{(&mut*os_k980_mutex()).with_lease(&ctx,&gamma_lease,|value|{if *value!=1{park();}*value=2;})}.map_err(|_|"locking-gamma-resource")?;unsafe{(&mut*os_k980_mutex()).release(&ctx,gamma_lease)}.unwrap_or_else(|_|park()); if os_k980_backend_mut(|b|b.tasks.scheduler.exit_current(0x981))!=Ok(Some(alpha))||os_k980_backend_mut(|b|b.tasks.scheduler.reap(gamma))!=Ok(0x981){return Err("locking-gamma-retire");} os_k980_store(core::ptr::addr_of_mut!(OS_K980_MUTEX_RECORDS).cast(),unsafe{(&*os_k980_mutex()).os_k980_observation()}); let(backend,value)=installation.retire();if value!=2||backend.parks!=1||backend.wakes!=1||backend.cancels!=0{return Err("locking-fair-mutex-accounting");} os_k980_store(core::ptr::addr_of_mut!(OS_K980_SCHEDULER).cast(),backend.tasks.scheduler.os_k980_inventory());os_k980_word(51,value);os_k980_word(52,backend.parks);os_k980_word(53,backend.wakes);drop(backend); unsafe{asm!("msr DAIF, {old}","msr VBAR_EL1, {vbar}","isb",old=in(reg)old_daif,vbar=in(reg)old_vbar,options(nostack,preserves_flags));}os_k980_set(0,0);drop(transfer); for index in 0..2{ let user=unsafe{(&mut*core::ptr::addr_of_mut!(OS_K980_USERS)).get_mut(index).and_then(Option::take)}.ok_or("locking-user-retire-token")?; unsafe{asm!("dsb ishst","tlbi vmalle1","dsb ish","isb",options(nostack,preserves_flags));} if let Err((_,user))=unsafe{user.release_after_tlbi(&mut memory.allocator,tables.root)}{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K980_USERS).cast::>().add(index),Some(user))};return Err("locking-user-retire-retained");} } os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K980_AUTHORITY_AFTER).cast());os_k980_diagnostics(3); if !os_k920_inventory_equal(core::ptr::addr_of!(OS_K980_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K980_AUTHORITY_AFTER).cast())||os_k940_registers()!=(old_ttbr,old_tcr,old_vbar,old_daif)||unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}!=before_count+2||os_k980_state(10)!=4||os_k980_state(18)!=2||unsafe{read_volatile(low as *const u64)}!=OS_K900_LOW_CANARY||unsafe{read_volatile((top+8) as *const u64)}!=OS_K900_HIGH_CANARY{return Err("locking-owner-restoration");} os_k980_checkpoint(5,memory);write_line("TURKOS/OS/K965: PASS");write_line("TURKOS/OS/K970: SINGLE-CPU-RECURSION-PROBE ACTUAL-TWO-CPU-CONTENTION-PENDING");write_line("TURKOS/OS/K975: PASS");write_line("TURKOS/OS/K980: PASS IRQ=2 EL0=4 PAYLOAD=2 OWNERS=RESTORED SMP-CONTENTION-PENDING");Ok(()) } #[inline(never)]fn os_k980_product_configuration(){ unsafe extern "C"{static __os_k980_vectors:u8;static __os_k980_user_code:u8;} for pointer in[core::ptr::addr_of!(OS_K980_EVIDENCE).cast::(),core::ptr::addr_of!(OS_K980_STATE).cast(),core::ptr::addr_of!(OS_K980_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K980_AUTHORITY_LIVE).cast(),core::ptr::addr_of!(OS_K980_AUTHORITY_AFTER).cast(),core::ptr::addr_of!(OS_K980_OWNED).cast(),core::ptr::addr_of!(OS_K980_MUTEX_RECORDS).cast(),core::ptr::addr_of!(OS_K980_IRQ_MUTEX_BEFORE).cast(),core::ptr::addr_of!(OS_K980_IRQ_MUTEX_AFTER).cast(),core::ptr::addr_of!(OS_K980_SCHEDULER).cast(),core::ptr::addr_of!(OS_K980_SCHEDULER_BEFORE).cast(),core::ptr::addr_of!(OS_K980_DIAGNOSTICS).cast(),core::ptr::addr_of!(OS_K980_TIMER_FRAMES).cast(),core::ptr::addr_of!(OS_K980_USER_FRAMES).cast(),core::ptr::addr_of!(OS_K980_TASK_PROGRESS).cast(),core::ptr::addr_of!(OS_K980_FRAME_ADDRESSES).cast(),core::ptr::addr_of!(OS_K980_LAST_TIMER).cast(),core::ptr::addr_of!(OS_K980_IRQ_SEEN).cast(),core::ptr::addr_of!(OS_K980_KERNEL_FRAME).cast(),core::ptr::addr_of!(OS_K980_CAPTURE_GATE).cast(),core::ptr::addr_of!(OS_K980_SPIN_MEMORY_WORD).cast(),core::ptr::addr_of!(OS_K980_SPIN_PROCESS_WORD).cast()]{if unsafe{read_volatile(pointer)}!=0{park();}} for address in[core::ptr::addr_of!(__os_k980_vectors) as u64,core::ptr::addr_of!(__os_k980_user_code) as u64]{unsafe{asm!("ldrb wzr, [{address}]",address=in(reg)address,options(nostack,preserves_flags));}} } const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k980_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); struct OsK1000Frames<'a> { authority: &'a mut os_k720_frame::Allocator, leases: [Option; 24], addresses: [u64; 24], } impl OsK1000Frames<'_> { fn validates(&self, address: u64) -> Result<(), os_k740_metadata::Error> { let lease = self .leases .iter() .flatten() .find(|lease| lease.address() == address) .ok_or(os_k740_metadata::Error::BadAddress)?; self.authority .validate_lease(lease) .map_err(|_| os_k740_metadata::Error::Backend) } } impl os_k740_metadata::Backend for OsK1000Frames<'_> { fn allocate(&mut self) -> Result { let slot = self .leases .iter() .position(Option::is_none) .ok_or(os_k740_metadata::Error::Capacity)?; let lease = self .authority .allocate_at(self.addresses[slot]) .map_err(|_| os_k740_metadata::Error::NoMemory)?; let address = lease.address(); os_k1000_event(1,address,lease.generation(),0); self.leases[slot] = Some(lease); Ok(address) } fn release(&mut self, address: u64) -> Result<(), os_k740_metadata::Error> { let slot = self .leases .iter() .position(|lease| { lease .as_ref() .is_some_and(|lease| lease.address() == address) }) .ok_or(os_k740_metadata::Error::BadAddress)?; self.zero_page(address)?; self.authority .release( self.leases[slot] .as_ref() .ok_or(os_k740_metadata::Error::Backend)?, ) .map_err(|_| os_k740_metadata::Error::Backend)?; os_k1000_event(0x52,address,0,0); self.leases[slot] = None; Ok(()) } fn zero_page(&mut self, address: u64) -> Result<(), os_k740_metadata::Error> { self.validates(address)?; for offset in 0..4096 { unsafe { write_volatile((address as *mut u8).add(offset), 0) } } Ok(()) } fn read_byte(&self, address: u64, offset: usize) -> Result { if offset >= 4096 { return Err(os_k740_metadata::Error::BadAddress); } self.validates(address)?; Ok(unsafe { read_volatile((address as *const u8).add(offset)) }) } fn available_pages(&self) -> u64 { self.leases.iter().filter(|lease| lease.is_none()).count() as u64 } } use turkos_kernel::runtime::deferred_work as os_k1000_work; type OsK1000Registry<'a> = os_k740_metadata::Registry,24>; type OsK1000Queue<'a> = os_k1000_work::Queue,8,2>; const OS_K1000_PAGE_COUNT:usize=19; struct OsK1000Source { id:u64,binding:os_k1000_work::Binding,context:[os_k740_metadata::Handle;2], pins:[Option;2], } struct OsK1000Resource { id:u64,binding:os_k1000_work::Binding,buffer:usize,source_pin:usize,start:u16, page:os_k740_metadata::Handle,context:os_k740_metadata::Handle, buffer_pin:Option,context_pin:Option, producer:u64,tx:[u64;3],callback:u64,boot_id:u64, } impl os_k1000_work::WorkResource for OsK1000Resource { fn observed_context(&self)->os_k1000_work::ObservedContext{os_k1000_observed()} fn identity(&self)->u64{self.id} fn binding(&self)->os_k1000_work::Binding{self.binding} fn service(&mut self,count:u32)->Result<(),os_k1000_work::Error>{ if self.buffer_pin.is_none()||self.context_pin.is_none()||count==0||count>32{return Err(os_k1000_work::Error::Ownership);} let expected=if self.binding.epoch==1{os_k1000_worker_epoch_one as *const () as u64}else if self.binding.epoch==2{os_k1000_worker_epoch_two as *const () as u64}else{return Err(os_k1000_work::Error::Binding)}; if self.callback!=expected||unsafe{read_volatile((self.context.address()+16)as*const u64)}!=self.binding.epoch{return Err(os_k1000_work::Error::Binding);} if self.binding.epoch==1{os_k1000_worker_epoch_one(self,count)}else{os_k1000_worker_epoch_two(self,count)} } fn recover(&mut self)->Result<(),os_k1000_work::Error>{Err(os_k1000_work::Error::Quarantined)} } struct OsK1000Backend<'a>{ registry:OsK1000Registry<'a>,handles:[Option;OS_K1000_PAGE_COUNT], source_free:[Option;2],source_issued:[bool;2], source_pins:[[Option;8];2],buffer_pins:[Option;8],ring_pins:[Option;7], sources:[Option<(u64,os_k1000_work::Binding)>;2],jobs:[Option<(u64,os_k1000_work::Binding)>;8], net:u64,owner:u64,boot_id:u64,next:u64,retains:u64,returns:u64,source_returns:u64, } impl OsK1000Backend<'_>{ fn handle(&self,index:usize)->os_k740_metadata::Handle{self.handles.get(index).and_then(|h|*h).unwrap_or_else(||park())} fn address(&self,index:usize)->u64{self.handle(index).address()} fn unpin_observed(registry:&mut OsK1000Registry<'_>,pin:&mut os_k740_metadata::Pin,owner:u64,handle:os_k740_metadata::Handle)->Result<(),&'static str>{ registry.unpin(pin,owner).map_err(|_|"deferred-observed-unpin")?; let record=registry.owned_records().find(|r|r.address==handle.address()).ok_or("deferred-unpin-record")?; os_k1000_event(0x53,record.address,record.pins as u64,record.generation);Ok(()) } fn cleanup(&mut self)->Result<(),&'static str>{ if os_k1000_observed().kind!=os_k1000_work::Context::Task||os_k940_registers().3&0x80==0||self.sources.iter().any(Option::is_some)||self.jobs.iter().any(Option::is_some){return Err("deferred-cleanup-active");} for source in self.source_free.iter_mut().flatten(){for(i,pin)in source.pins.iter_mut().enumerate(){if let Some(pin)=pin{Self::unpin_observed(&mut self.registry,pin,self.owner,source.context[i])?;}}} for(epoch,row)in self.source_pins.iter_mut().enumerate(){for(i,pin)in row.iter_mut().enumerate(){if let Some(pin)=pin{let handle=self.handles[7+epoch*2+usize::from(i==7)].ok_or("deferred-context-handle")?;Self::unpin_observed(&mut self.registry,pin,self.owner,handle)?;}}} for(i,pin)in self.buffer_pins.iter_mut().enumerate(){if let Some(pin)=pin{let handle=self.handles[11+i].ok_or("deferred-buffer-handle")?;Self::unpin_observed(&mut self.registry,pin,self.owner,handle)?;}} for(i,pin)in self.ring_pins.iter_mut().enumerate(){if let Some(pin)=pin{let handle=self.handles[i].ok_or("deferred-ring-handle")?;Self::unpin_observed(&mut self.registry,pin,self.owner,handle)?;}} for index in 0..OS_K1000_PAGE_COUNT{ let handle=self.handle(index);os_k1000_event(0x51,handle.address(),handle.generation(),0); self.registry.release(handle,self.owner).map_err(|_|"deferred-page-release")?;self.handles[index]=None; }Ok(()) } } impl os_k1000_work::Backend for OsK1000Backend<'_>{ type Source=OsK1000Source;type Work=OsK1000Resource; fn observed_context(&self)->os_k1000_work::ObservedContext{os_k1000_observed()} fn acquire(&mut self,binding:os_k1000_work::Binding)->Result{ if binding.device!=self.net||binding.owner!=self.owner||binding.cpu!=0||!(1..=2).contains(&binding.epoch){return Err(os_k1000_work::Error::Binding);} let index=binding.epoch as usize-1;if self.source_issued[index]{return Err(os_k1000_work::Error::Stale);} let source=self.source_free[index].take().ok_or(os_k1000_work::Error::Ownership)?; if source.binding!=binding{self.source_free[index]=Some(source);return Err(os_k1000_work::Error::Binding);} self.source_issued[index]=true;self.sources[index]=Some((source.id,binding));os_k1000_event(2,source.id,binding.epoch,binding.owner);Ok(source) } fn source_identity(&self,source:&Self::Source)->u64{source.id} fn source_binding(&self,source:&Self::Source)->os_k1000_work::Binding{source.binding} fn retain(&mut self,source:&Self::Source)->Result{ // IRQ uses only these private preallocated slots and detached Pins. // No Registry, central allocator, heap, Runtime or predecessor access. let epoch=source.binding.epoch as usize-1; if epoch>=2||self.sources[epoch]!=Some((source.id,source.binding)){return Err(os_k1000_work::Error::Binding);} let buffer=self.buffer_pins.iter().position(Option::is_some).ok_or(os_k1000_work::Error::Full)?; let source_pin=self.source_pins[epoch].iter().position(Option::is_some).ok_or(os_k1000_work::Error::Full)?; let next=self.next.checked_add(1).ok_or(os_k1000_work::Error::Generation)?; let retains=self.retains.checked_add(1).ok_or(os_k1000_work::Error::Overflow)?; let context=source.context[usize::from(source_pin==7)]; let resource=OsK1000Resource{id:self.next,binding:source.binding,buffer,source_pin,start:os_k1000_state(8)as u16, page:self.handle(11+buffer),context,buffer_pin:self.buffer_pins[buffer].take(),context_pin:self.source_pins[epoch][source_pin].take(), producer:self.address(6),tx:[self.address(3),self.address(4),self.address(5)],callback:if epoch==0{os_k1000_worker_epoch_one as *const () as u64}else{os_k1000_worker_epoch_two as *const () as u64},boot_id:self.boot_id}; self.jobs[buffer]=Some((resource.id,resource.binding));self.next=next;self.retains=retains;os_k1000_event(3,resource.id,source.binding.epoch,resource.page.address());Ok(resource) } fn return_work(&mut self,mut work:Self::Work)->Result<(),(os_k1000_work::Error,Self::Work)>{ if os_k1000_observed().kind!=os_k1000_work::Context::Task||os_k940_registers().3&0x80==0||self.jobs.get(work.buffer)!=Some(&Some((work.id,work.binding)))||work.buffer_pin.is_none()||work.context_pin.is_none(){return Err((os_k1000_work::Error::Ownership,work));} let Some(next)=self.returns.checked_add(1)else{return Err((os_k1000_work::Error::Overflow,work));}; for offset in 0..4096{unsafe{write_volatile((work.page.address()+offset)as*mut u8,0)}} self.buffer_pins[work.buffer]=work.buffer_pin.take();self.source_pins[work.binding.epoch as usize-1][work.source_pin]=work.context_pin.take();self.jobs[work.buffer]=None;self.returns=next;os_k1000_event(4,work.id,work.binding.epoch,work.page.address());Ok(()) } fn return_source(&mut self,source:Self::Source)->Result<(),(os_k1000_work::Error,Self::Source)>{ let epoch=source.binding.epoch as usize-1; if epoch>=2||self.sources[epoch]!=Some((source.id,source.binding))||self.jobs.iter().flatten().any(|(_,b)|*b==source.binding){return Err((os_k1000_work::Error::Busy,source));} let Some(next)=self.source_returns.checked_add(1)else{return Err((os_k1000_work::Error::Overflow,source));}; os_k1000_event(5,source.id,source.binding.epoch,0);self.sources[epoch]=None;self.source_free[epoch]=Some(source);self.source_returns=next;Ok(()) } fn audit_owned(&self,sources:&[os_k1000_work::SourceRecord],jobs:&[os_k1000_work::JobRecord])->Result<(),os_k1000_work::Error>{ // An independent exact private ledger includes detached running leases. let source_count=sources.iter().filter(|s|s.state!=os_k1000_work::SourceState::Empty).count(); let job_count=jobs.iter().filter(|j|j.state!=os_k1000_work::JobState::Empty).count(); if source_count!=self.sources.iter().flatten().count()||job_count!=self.jobs.iter().flatten().count(){return Err(os_k1000_work::Error::Ownership);} for record in sources.iter().filter(|s|s.state!=os_k1000_work::SourceState::Empty){if !self.sources.iter().flatten().any(|r|*r==(record.resource,record.binding)){return Err(os_k1000_work::Error::Ownership);}} for record in jobs.iter().filter(|j|j.state!=os_k1000_work::JobState::Empty){if !self.jobs.iter().flatten().any(|r|*r==(record.resource,record.binding)){return Err(os_k1000_work::Error::Ownership);}}Ok(()) } } impl os_k1000_work::OsK1000MainBacking for OsK1000Backend<'_>{ fn os_k1000_pinned_refusal(&mut self)->Result<(),os_k1000_work::Error>{ self.registry.with_backend_observation(|f|os_k940_inventory(f.authority,core::ptr::addr_of_mut!(OS_K1000_PIN_REFUSAL_BEFORE).cast())); let before=self.registry.snapshot();let handle=self.handle(11); if self.registry.release(handle,self.owner)!=Err(os_k740_metadata::Error::Pinned){return Err(os_k1000_work::Error::Ownership);} self.registry.with_backend_observation(|f|os_k940_inventory(f.authority,core::ptr::addr_of_mut!(OS_K1000_PIN_REFUSAL_AFTER).cast())); if self.registry.snapshot()!=before||!os_k920_inventory_equal(core::ptr::addr_of!(OS_K1000_PIN_REFUSAL_BEFORE).cast(),core::ptr::addr_of!(OS_K1000_PIN_REFUSAL_AFTER).cast()){return Err(os_k1000_work::Error::Ownership);} os_k1000_event(0x90,handle.address(),handle.generation(),1);Ok(()) } fn os_k1000_retire_backing(&mut self)->Result<(),os_k1000_work::Error>{ if unsafe{os_k940_read(self.net,0x70)}!=0{return Err(os_k1000_work::Error::Busy);} self.cleanup().map_err(|_|os_k1000_work::Error::Ownership) } } #[unsafe(no_mangle)]static mut OS_K1000_FACTORY_BACKEND:core::mem::MaybeUninit>=core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)]static mut OS_K1000_QUEUE:core::mem::MaybeUninit>=core::mem::MaybeUninit::uninit(); static mut OS_K1000_REGISTRATIONS:[Option;2]=[None,None]; #[unsafe(no_mangle)]static mut OS_K1000_WORKS:[Option>;8]=[const{None};8]; #[unsafe(no_mangle)]static mut OS_K1000_STATE:[u64;32]=[0;32]; #[unsafe(no_mangle)]static mut OS_K1000_EVIDENCE:[u64;128]=[0;128]; #[unsafe(no_mangle)]static mut OS_K1000_AUTHORITY_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)]static mut OS_K1000_AUTHORITY_LIVE:[u64;328]=[0;328]; #[unsafe(no_mangle)]static mut OS_K1000_AUTHORITY_AFTER:[u64;328]=[0;328]; #[unsafe(no_mangle)]static mut OS_K1000_PIN_REFUSAL_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)]static mut OS_K1000_PIN_REFUSAL_AFTER:[u64;328]=[0;328]; #[unsafe(no_mangle)]static mut OS_K1000_OWNED:[u64;19]=[0;19]; #[unsafe(no_mangle)]static mut OS_K1000_REGISTRY:[u64;19*6]=[0;19*6]; #[unsafe(no_mangle)]static mut OS_K1000_SOURCE_RECORDS:[u64;2*9]=[0;2*9]; #[unsafe(no_mangle)]static mut OS_K1000_JOB_RECORDS:[u64;8*12]=[0;8*12]; #[unsafe(no_mangle)]static mut OS_K1000_WORK_ASSETS:[u64;8*12]=[0;8*12]; #[unsafe(no_mangle)]static mut OS_K1000_IRQ_FRAMES:[os_k920_exception::Frame;16]=[OS_K920_ZERO_FRAME;16]; #[unsafe(no_mangle)]static mut OS_K1000_IRQ_RECORDS:[u64;16*16]=[0;16*16]; #[unsafe(no_mangle)]static mut OS_K1000_FRAME_ADDRESSES:[u64;16]=[0;16]; #[unsafe(no_mangle)]static mut OS_K1000_IO_RECORDS:[u64;13*16]=[0;13*16]; #[unsafe(no_mangle)]static mut OS_K1000_EVENTS:[u64;256*4]=[0;256*4]; #[unsafe(no_mangle)]static mut OS_K1000_EVENT_COUNT:u64=0; #[unsafe(no_mangle)]static mut OS_K1000_NONCE:u64=1; #[unsafe(no_mangle)]static mut OS_K1000_SOURCE_NONCE:u64=1; #[unsafe(no_mangle)]static mut OS_K1000_PRE_IRQ_COUNT:u64=0; #[unsafe(no_mangle)]static mut OS_K1000_PRE_IRQ_LAST:u64=0; #[unsafe(no_mangle)]static mut OS_K1000_PRE_IRQ_ACTIVE:u32=0; #[unsafe(no_mangle)]static mut OS_K1000_LAST_IRQ:u64=0; #[unsafe(no_mangle)]static mut OS_K1000_DIAGNOSTICS:[u64;6]=[0;6]; #[unsafe(no_mangle)]static mut OS_K1000_KERNEL_FRAME:u64=0; #[unsafe(no_mangle)]static OS_K1000_QUEUE_LAYOUT_EXPECTED:[u64;41]=OsK1000Queue::os_k1000_physical_layout(); #[unsafe(no_mangle)]static OS_K1000_BACKEND_LAYOUT_EXPECTED:[u64;3]=[core::mem::offset_of!(OsK1000Backend,registry)as u64,core::mem::offset_of!(OsK1000Frames,authority)as u64,core::mem::size_of::()as u64]; #[unsafe(no_mangle)]static OS_K1000_REGISTRY_LAYOUT_EXPECTED:[u64;12]=OsK1000Registry::os_k1000_physical_layout(); #[unsafe(no_mangle)]static mut OS_K1000_AUTHORITY_LAYOUT:[u64;24]=[0;24]; #[unsafe(no_mangle)]static mut OS_K1000_MMIO:[u64;32]=[0;32]; #[unsafe(no_mangle)]static mut OS_K1000_CAPTURE_GATE:u8=0; fn os_k1000_state(i:usize)->u64{unsafe{read_volatile(core::ptr::addr_of!(OS_K1000_STATE).cast::().add(i))}} fn os_k1000_set(i:usize,v:u64){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_STATE).cast::().add(i),v)}} fn os_k1000_word(i:usize,v:u64){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_EVIDENCE).cast::().add(i),v)}} fn os_k1000_observed()->os_k1000_work::ObservedContext{ let(el,mpidr,daif):(u64,u64,u64);unsafe{asm!("mrs {el}, CurrentEL","mrs {mpidr}, MPIDR_EL1","mrs {daif}, DAIF",el=out(reg)el,mpidr=out(reg)mpidr,daif=out(reg)daif,options(nomem,nostack,preserves_flags));} os_k1000_work::ObservedContext{cpu:if el==4&&mpidr&0xff00_ffff_ff==0{0}else{u32::MAX},kind:if os_k1000_state(1)==0{os_k1000_work::Context::Task}else{os_k1000_work::Context::Interrupt},irq_masked:daif&0x80!=0} } #[unsafe(no_mangle)]extern "C" fn __os_k1000_deferred_identity()->u64{ let observed=os_k1000_observed();if observed.cpu!=0||observed.kind!=os_k1000_work::Context::Task||!observed.irq_masked{return 0;} let prior=unsafe{read_volatile(core::ptr::addr_of!(OS_K1000_NONCE))};let Some(next)=prior.checked_add(1)else{return 0;};if prior==0{return 0;} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_NONCE),next)};prior } fn os_k1000_event(code:u64,a:u64,b:u64,c:u64){let i=unsafe{read_volatile(core::ptr::addr_of!(OS_K1000_EVENT_COUNT))}as usize;if i>=256{park();}for(n,v)in[code,a,b,c].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_EVENTS).cast::().add(i*4+n),v)}}unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_EVENT_COUNT),i as u64+1)};} fn os_k1000_queue()->*mut OsK1000Queue<'static>{core::ptr::addr_of_mut!(OS_K1000_QUEUE).cast()} struct OsK1000Installation<'a>{active:bool,ownership:core::marker::PhantomData>} impl<'a>OsK1000Installation<'a>{ #[inline(never)]unsafe fn install_initialized()->Self{ if os_k1000_state(0)!=0||os_k1000_state(28)!=0{park();}os_k1000_set(0,1);Self{active:true,ownership:core::marker::PhantomData} } #[inline(never)]fn retire(mut self){ if os_k1000_observed().kind!=os_k1000_work::Context::Task||os_k940_registers().3&0x80==0{park();} let q=unsafe{core::ptr::addr_of_mut!(OS_K1000_QUEUE).cast::>().read()}; let backend=match q.into_backend(){Ok(backend)=>backend,Err(actual_queue)=>{unsafe{core::ptr::addr_of_mut!(OS_K1000_QUEUE).cast::>().write(actual_queue)};park()}};os_k1000_set(0,0);self.active=false;backend.registry.with_backend_observation(|f|os_k940_inventory(f.authority,core::ptr::addr_of_mut!(OS_K1000_AUTHORITY_AFTER).cast()));drop(backend); } } impl Drop for OsK1000Installation<'_>{fn drop(&mut self){if self.active{park();}}} fn os_k1000_hold_failed_backend(backend:OsK1000Backend<'_>)->!{ const _:()=assert!(core::mem::size_of::>()>=core::mem::size_of::>()); const _:()=assert!(core::mem::align_of::>()>=core::mem::align_of::>()); // Queue storage is still uninitialized. Retain exact error ownership in // the same aligned BSS rather than dropping the exclusive actual backend. if os_k1000_state(0)!=0{park();}unsafe{core::ptr::addr_of_mut!(OS_K1000_QUEUE).cast::>().write(backend)};os_k1000_set(0,2);park() } fn os_k1000_registration(epoch:usize)->&'static os_k1000_work::Registration{if epoch>=2{park();}unsafe{(&*core::ptr::addr_of!(OS_K1000_REGISTRATIONS)).get(epoch).and_then(Option::as_ref)}.unwrap_or_else(||park())} fn os_k1000_queue_mut(operation:os_k1000_work::Operation,callback:impl FnOnce(&mut OsK1000Queue<'_>,os_k1000_work::ObservedContext)->T)->T{ let actual=os_k1000_observed();os_k1000_work::context(0,actual,operation).unwrap_or_else(|_|park());if os_k1000_state(0)!=1{park();}unsafe{callback(&mut*os_k1000_queue(),actual)} } fn os_k1000_diagnostics(offset:usize){let log=unsafe{(&*core::ptr::addr_of!(OS_K680_LOG)).as_ref()}.unwrap_or_else(||park());for(i,v)in[log.boot().value(),log.next_sequence(),log.len()as u64].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_DIAGNOSTICS).cast::().add(offset+i),v)}}} fn os_k1000_assets(slot:usize,r:&OsK1000Resource){ for(i,v)in[r.id,r.binding.device,r.binding.epoch,r.binding.owner,r.buffer as u64,r.source_pin as u64,r.start as u64,r.page.address(),r.page.generation(),r.context.address(),r.context.generation(),r.callback].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_WORK_ASSETS).cast::().add(slot*12+i),v)}} } fn os_k1000_snapshot(main:bool){ let q=unsafe{&*os_k1000_queue()};let s=q.snapshot(); for(i,v)in[(27,q.identity()),(30,s.queued as u64),(31,s.running as u64),(32,s.open as u64),(33,s.draining as u64),(34,s.quarantined as u64),(35,s.submitted),(36,s.completed),(37,s.coalesced_jobs),(38,s.coalesced_completions),(39,s.lost_jobs),(40,s.lost_completions)]{os_k1000_word(i,v);} for(i,r)in q.source_records().into_iter().enumerate(){for(n,v)in[r.state as u64,r.slot as u64,r.generation,r.binding.device,r.binding.epoch,r.binding.owner,r.binding.cpu as u64,r.resource,q.identity()].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_SOURCE_RECORDS).cast::().add(i*9+n),v)}}} for(i,r)in q.job_records().into_iter().enumerate(){for(n,v)in[r.state as u64,r.slot as u64,r.generation,r.registration_slot as u64,r.registration_generation,r.binding.device,r.binding.epoch,r.binding.owner,r.binding.cpu as u64,r.resource,r.completions as u64,r.sequence].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_JOB_RECORDS).cast::().add(i*12+n),v)}}} q.with_backend_observation(|b|{ for(i,v)in[(41,b.retains),(42,b.returns),(43,b.source_returns)]{os_k1000_word(i,v);} if main{ b.registry.with_backend_observation(|frames|{os_k940_inventory(frames.authority,core::ptr::addr_of_mut!(OS_K1000_AUTHORITY_LIVE).cast());os_k1000_word(52,frames.authority as*const os_k720_frame::Allocator as u64);os_k1000_word(53,core::mem::size_of::()as u64);for(i,v)in frames.authority.os_k1000_physical_layout().into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_AUTHORITY_LAYOUT).cast::().add(i),v)}}}); os_k1000_word(54,&b.registry as*const OsK1000Registry<'_>as u64);os_k1000_word(55,core::mem::size_of::>()as u64); for i in 0..114{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_REGISTRY).cast::().add(i),0)}} for(i,r)in b.registry.owned_records().enumerate(){let lease_generation=b.registry.with_backend_observation(|frames|{let lease=frames.leases.iter().flatten().find(|l|l.address()==r.address).unwrap_or_else(||park());frames.authority.validate_lease(lease).unwrap_or_else(|_|park());lease.generation()});for(n,v)in[r.address,r.owner,r.generation,r.pins as u64,r.quarantined as u64,lease_generation].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_REGISTRY).cast::().add(i*6+n),v)}}} } }); // Never observe a Work that a preempted task owns mutably during its I/O. if main{for i in 0..8{ for n in 0..12{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_WORK_ASSETS).cast::().add(i*12+n),0)}} q.os_k1000_job_observation(i,|_,resource|{if let Some(r)=resource{os_k1000_assets(i,r)}}).unwrap_or_else(|_|park()); if let Some(work)=unsafe{(&*core::ptr::addr_of!(OS_K1000_WORKS)).get(i).and_then(Option::as_ref)}{work.os_k1000_resource_observation(|r|os_k1000_assets(i,r)).unwrap_or_else(|_|park());} }os_k1000_custody_snapshot(q);} } fn os_k1000_capture(stage:u64,main:bool){ if main&&(os_k1000_observed().kind!=os_k1000_work::Context::Task||os_k940_registers().3&0x80==0){park();} if os_k1000_state(0)==1{os_k1000_snapshot(main);} let(t,tcr,vbar,daif)=os_k940_registers();let sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags));} for(i,v)in[(5,stage),(6,t),(7,tcr),(8,vbar),(9,daif),(10,os_k960_counter()),(12,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}),(13,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}as u64),(44,os_k1000_state(5)),(45,os_k1000_state(9)),(49,os_k1000_state(6)),(50,os_k1000_state(2)),(56,sp),(75,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER))})]{os_k1000_word(i,v);} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_LAST_IRQ),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)));write_volatile(core::ptr::addr_of_mut!(OS_K1000_CAPTURE_GATE),0)}; write_str("TURKOS/OS/K1000: CAPTURE STAGE=0x");write_hex(stage as usize);write_str(" GATE=0x");write_hex(core::ptr::addr_of!(OS_K1000_CAPTURE_GATE)as usize);write_line(" READY"); let end=os_k960_counter().checked_add(os_k960_frequency().checked_mul(90).unwrap_or_else(||park())).unwrap_or_else(||park()); while unsafe{read_volatile(core::ptr::addr_of!(OS_K1000_CAPTURE_GATE))}!=1{if os_k960_counter()>=end{park();}core::hint::spin_loop();} } fn os_k1000_packet(boot:u64,id:usize,epoch:u64,reply:bool)->[u8;60]{ let mut packet=[0;60];let nic=[0x52,0x54,0,0x12,0x34,0x56];let peer=[2,0,0,0,0,1]; packet[..6].copy_from_slice(if reply{&peer}else{&nic});packet[6..12].copy_from_slice(if reply{&nic}else{&peer});packet[12..14].copy_from_slice(if reply{&[0x88,0xb8]}else{&[0x88,0xb7]}); packet[14..22].copy_from_slice(&boot.to_be_bytes());packet[22]=id as u8;packet[23]=epoch as u8;for n in 24..60{packet[n]=(id*11+n)as u8;}packet } #[unsafe(no_mangle)]#[inline(never)]fn os_k1000_factory_setup_and_install<'a>(authority:&'a mut os_k720_frame::Allocator,net:u64,owner:u64,boot:u64)->Result,&'static str>{ let factory_sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)factory_sp,options(nomem,nostack,preserves_flags));}os_k1000_word(87,factory_sp); if factory_sp<=os_k1000_state(11)+848{park();} let region=authority.free_regions().iter().rev().find(|r|r.end-r.start>=160*4096).ok_or("deferred-interior-pool")?; let start=region.end-140*4096;os_k1000_word(20,start);os_k1000_word(21,start+32*4096); let identity=unsafe{__os_k760_workspace_identity()};if identity==0{return Err("deferred-registry-identity");}os_k1000_word(72,identity); let frames=OsK1000Frames{authority,leases:core::array::from_fn(|_|None),addresses:core::array::from_fn(|i|start+i as u64*4096)}; let registry=OsK1000Registry::new(identity,frames).map_err(|_|"deferred-registry")?; let storage=core::ptr::addr_of_mut!(OS_K1000_FACTORY_BACKEND).cast::>(); if os_k1000_state(28)!=0||os_k1000_state(0)!=0{park();} unsafe{storage.write(OsK1000Backend{registry,handles:[None;19],source_free:core::array::from_fn(|_|None),source_issued:[false;2],source_pins:core::array::from_fn(|_|core::array::from_fn(|_|None)),buffer_pins:core::array::from_fn(|_|None),ring_pins:core::array::from_fn(|_|None),sources:[None;2],jobs:[None;8],net,owner,boot_id:boot,next:1,retains:0,returns:0,source_returns:0});} // Every acquisition is written immediately into this retained BSS owner. // Failure parks with the actual Registry/lease/Pin inventory still owned. os_k1000_set(28,1);let b=unsafe{&mut*storage}; for i in 0..19{let h=b.registry.allocate_zeroed(owner).unwrap_or_else(|_|park());b.handles[i]=Some(h);unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_OWNED).cast::().add(i),h.address())};os_k1000_event(0x40,h.address(),h.generation(),owner);} for i in 0..7{b.ring_pins[i]=Some(b.registry.pin(b.handle(i),owner).unwrap_or_else(|_|park()));} for i in 0..8{b.buffer_pins[i]=Some(b.registry.pin(b.handle(11+i),owner).unwrap_or_else(|_|park()));} for epoch in 0..2{ let prior=unsafe{read_volatile(core::ptr::addr_of!(OS_K1000_SOURCE_NONCE))};let next=prior.checked_add(1).unwrap_or_else(||park()); if prior!=epoch as u64+1{park();}unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_SOURCE_NONCE),next)}; let context=[b.handle(7+epoch*2),b.handle(8+epoch*2)];let callback=if epoch==0{os_k1000_worker_epoch_one as*const()as u64}else{os_k1000_worker_epoch_two as*const()as u64}; for h in context{for offset in 0..4096{unsafe{write_volatile((h.address()+offset)as*mut u8,0x91+epoch as u8)}} for(i,v)in[0x4b31303030535243,net,prior,owner,h.address(),h.generation(),callback,b.address(6),b.address(0),b.address(3),0,os_k1000_state(20),os_k1000_state(21)].into_iter().enumerate(){unsafe{write_volatile((h.address()+i as u64*8)as*mut u64,v)}}} b.source_free[epoch]=Some(OsK1000Source{id:context[0].address(),binding:os_k1000_work::Binding{device:net,epoch:prior,owner,cpu:0},context,pins:core::array::from_fn(|_|None)}); for i in 0..2{let pin=b.registry.pin(context[i],owner).unwrap_or_else(|_|park());b.source_free[epoch].as_mut().unwrap_or_else(||park()).pins[i]=Some(pin);} for i in 0..8{b.source_pins[epoch][i]=Some(b.registry.pin(context[usize::from(i==7)],owner).unwrap_or_else(|_|park()));} for(i,v)in[(64+epoch*4,context[0].address()),(65+epoch*4,context[0].generation()),(66+epoch*4,prior),(67+epoch*4,callback)]{os_k1000_word(i,v);} os_k1000_event(0x60,prior,context[0].address(),callback); } let snapshot=b.registry.snapshot();if snapshot.live_pages!=19||snapshot.pins!=35{park();} for r in b.registry.owned_records(){os_k1000_event(0x50,r.address,r.pins as u64,r.generation);} os_k1000_word(25,19);os_k1000_word(26,35);os_k1000_set(24,b.address(2));os_k1000_set(26,b.address(6));os_k1000_set(27,boot); if os_k1000_setup(b).is_err(){park();} let destination=core::ptr::addr_of_mut!(OS_K1000_QUEUE).cast::>(); if unsafe{OsK1000Queue::os_k1000_initialize_retained(destination,storage)}.is_err(){park();} os_k1000_set(28,0);Ok(unsafe{OsK1000Installation::install_initialized()}) } fn os_k1000_setup(b:&OsK1000Backend<'_>)->Result<(),&'static str>{ let base=b.net;unsafe{ os_k940_write(base,0x70,0);asm!("dsb sy",options(nostack,preserves_flags));if os_k940_read(base,0x70)!=0{return Err("deferred-net-reset");} os_k940_write(base,0x70,3);os_k940_write(base,0x14,0);let low=os_k940_read(base,0x10);os_k940_write(base,0x14,1);let high=os_k940_read(base,0x10);if low&32==0||high&1==0{return Err("deferred-net-features");} os_k940_write(base,0x24,0);os_k940_write(base,0x20,32);os_k940_write(base,0x24,1);os_k940_write(base,0x20,1);os_k940_write(base,0x70,11);if os_k940_read(base,0x70)!=11{return Err("deferred-features-ok");} for queue in 0..2usize{ os_k940_write(base,0x30,queue as u32);let count=if queue==0{32}else{8};if os_k940_read(base,0x34)=end{park();}core::hint::spin_loop();} unsafe{asm!("dmb oshld","msr DAIFClr, #2","isb",options(nostack,preserves_flags));} while os_k1000_state(5)!=target{if os_k960_counter()>=end{park();}core::hint::spin_loop();} unsafe{asm!("msr DAIFSet, #2","isb",options(nostack,preserves_flags));} } fn os_k1000_take(slot:usize){ let work=os_k1000_queue_mut(os_k1000_work::Operation::Take,|q,c|q.take(c)).unwrap_or_else(|_|park()).unwrap_or_else(||park()); let pointer=unsafe{core::ptr::addr_of_mut!(OS_K1000_WORKS).cast::>>().add(slot)};if unsafe{(&*pointer).is_some()}{park();}unsafe{pointer.write(Some(work))}; } fn os_k1000_finish(slot:usize){ let entry_sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)entry_sp,options(nomem,nostack,preserves_flags));} os_k1000_word(84,entry_sp);os_k1000_word(85,slot as u64+1); os_k1000_set(7,slot as u64+1);unsafe{asm!("msr DAIFClr, #2","isb",options(nostack,preserves_flags));} // A reference to only this element is formed. IRQ may test distinct slot7, // and accesses Queue after the main Queue loan has completely ended. let pointer=unsafe{core::ptr::addr_of_mut!(OS_K1000_WORKS).cast::>>().add(slot)}; let work=unsafe{(&mut*pointer).as_mut()}.unwrap_or_else(||park());work.service(os_k1000_observed()).unwrap_or_else(|_|park()); unsafe{asm!("msr DAIFSet, #2","isb",options(nostack,preserves_flags));}os_k1000_set(7,0); if slot==0&&os_k1000_state(9)==3{os_k1000_capture(5,true);} if slot==0&&os_k1000_state(9)==12{os_k1000_source_release(1,true);os_k1000_capture(8,true);} let work=unsafe{(&mut*pointer).take()}.unwrap_or_else(||park());os_k1000_queue_mut(os_k1000_work::Operation::Complete,|q,c|q.complete(work,c)).unwrap_or_else(|_|park()); } #[unsafe(no_mangle)]#[inline(never)]fn os_k1000_worker_epoch_one(r:&mut OsK1000Resource,count:u32)->Result<(),os_k1000_work::Error>{os_k1000_io(r,count,1)} #[unsafe(no_mangle)]#[inline(never)]fn os_k1000_worker_epoch_two(r:&mut OsK1000Resource,count:u32)->Result<(),os_k1000_work::Error>{os_k1000_io(r,count,2)} #[unsafe(no_mangle)]#[inline(never)]fn os_k1000_io(r:&mut OsK1000Resource,count:u32,epoch:u64)->Result<(),os_k1000_work::Error>{ let observed=os_k1000_observed();if observed.cpu!=0||observed.kind!=os_k1000_work::Context::Task||observed.irq_masked||r.binding.epoch!=epoch{return Err(os_k1000_work::Error::Context);} for n in 0..count{ let id=r.start as usize+n as usize;let expected=os_k1000_packet(r.boot_id,id,epoch,false);let payload=r.producer+id as u64*96; for(i,v)in expected.into_iter().enumerate(){if unsafe{read_volatile((payload+12+i as u64)as*const u8)}!=v{return Err(os_k1000_work::Error::Binding);}} for offset in 72..96{if unsafe{read_volatile((payload+offset)as*const u8)}!=0xa5{return Err(os_k1000_work::Error::Ownership);}} let response=os_k1000_packet(r.boot_id,id,epoch,true);let out=r.page.address()+n as u64*96; for(i,v)in response.into_iter().enumerate(){unsafe{write_volatile((out+12+i as u64)as*mut u8,v)}}for offset in 72..96{unsafe{write_volatile((out+offset)as*mut u8,0xa5)}} if id==0{write_line("TURKOS/OS/K1000: WORKER-IO READY NINTH-RX");let end=os_k960_counter().checked_add(os_k960_frequency()*15).ok_or(os_k1000_work::Error::Overflow)?;while os_k1000_state(5)!=11{if os_k960_counter()>=end{return Err(os_k1000_work::Error::Backend);}core::hint::spin_loop();}} let index=os_k1000_state(9)as u16;let descriptor=index as usize%8; unsafe{ write_volatile((r.tx[0]+descriptor as u64*16)as*mut turkos_kernel::virtio_ring::Descriptor,turkos_kernel::virtio_ring::Descriptor{addr_le:out.to_le(),len_le:72u32.to_le(),flags_le:0,next_le:0}); write_volatile((r.tx[1]+4+descriptor as u64*2)as*mut u16,(descriptor as u16).to_le());asm!("dmb oshst",options(nostack,preserves_flags));write_volatile((r.tx[1]+2)as*mut u16,(index+1).to_le());asm!("dmb oshst",options(nostack,preserves_flags));os_k940_write(r.binding.device,0x50,1); } let end=os_k960_counter().checked_add(os_k960_frequency()*15).ok_or(os_k1000_work::Error::Overflow)?; while unsafe{u16::from_le(read_volatile((r.tx[2]+2)as*const u16))}!=index+1{if os_k960_counter()>=end{return Err(os_k1000_work::Error::Backend);}core::hint::spin_loop();} unsafe{asm!("dmb oshld",options(nostack,preserves_flags));} let used=r.tx[2]+4+descriptor as u64*8;let used_id=unsafe{u32::from_le(read_volatile(used as*const u32))};let used_len=unsafe{u32::from_le(read_volatile((used+4)as*const u32))}; if used_id!=descriptor as u32||used_len!=0{return Err(os_k1000_work::Error::Backend);} let sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags));} for(i,v)in[r.id,epoch,id as u64,r.start as u64,count as u64,r.page.address(),r.context.address(),r.callback,observed.cpu as u64,os_k940_registers().3,sp,os_k960_counter(),index as u64+1,used_id as u64,used_len as u64,os_k1000_state(7)].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_IO_RECORDS).cast::().add(index as usize*16+i),v)}} os_k1000_set(9,index as u64+1);os_k1000_event(0x70,r.id,epoch,id as u64); }Ok(()) } #[unsafe(no_mangle)]extern "C" fn os_k1000_dispatch(pointer:*mut os_k920_exception::Frame)->u64{ let address=pointer as u64;let stack=os_k920_exception::Stack::new(os_k1000_state(11),os_k1000_state(12)).unwrap_or_else(|_|park());stack.frame_address(address).unwrap_or_else(|_|park());let frame=unsafe{&mut*pointer};os_k920_exception::validate_ingress(frame,address,stack).unwrap_or_else(|_|park()); if os_k1000_state(0)!=1||frame.vector_slot!=5||os_k1000_state(1)!=0||os_k940_registers().2!=os_k1000_state(14){park();} os_k920_kernel_irq_return(frame,address,stack,core::ptr::addr_of!(_text_start)as u64,core::ptr::addr_of!(_text_end)as u64,__os_k920_kernel_tco_observed()).unwrap_or_else(|_|park()); let n=os_k1000_state(6)as usize;if n>=16||unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}!=0{park();} let iar:u64;unsafe{asm!("mrs {iar}, ICC_IAR1_EL1",iar=out(reg)iar,options(nomem,nostack,preserves_flags));} if iar&0xffffff!=os_k1000_state(4){park();}os_k1000_set(1,1);unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),1);write_volatile(core::ptr::addr_of_mut!(OS_K1000_IRQ_FRAMES).cast::().add(n),*frame);write_volatile(core::ptr::addr_of_mut!(OS_K1000_FRAME_ADDRESSES).cast::().add(n),address)}; let used=unsafe{u16::from_le(read_volatile((os_k1000_state(24)+2)as*const u16))};let before=os_k1000_state(5)as u16;let epoch=os_k1000_state(2);if used<=before||used>32{park();}unsafe{asm!("dmb oshld",options(nostack,preserves_flags));} for id in before..used{ let entry=os_k1000_state(24)+4+id as u64*8;let actual_id=unsafe{u32::from_le(read_volatile(entry as*const u32))};let actual_length=unsafe{u32::from_le(read_volatile((entry+4)as*const u32))}; if actual_id!=id as u32||actual_length!=72{park();} let payload=os_k1000_state(26)+id as u64*96;let expected=os_k1000_packet(os_k1000_state(27),id as usize,epoch,false); for(i,v)in expected.into_iter().enumerate(){if unsafe{read_volatile((payload+12+i as u64)as*const u8)}!=v{park();}} for offset in 72..96{if unsafe{read_volatile((payload+offset)as*const u8)}!=0xa5{park();}} os_k1000_event(0x84,id as u64,actual_length as u64,epoch); os_k1000_set(8,id as u64);let result=os_k1000_queue_mut(os_k1000_work::Operation::Enqueue,|q,c|q.enqueue(os_k1000_registration(epoch as usize-1),1,c)); match result{ Ok(os_k1000_work::Enqueued::New(r))=>os_k1000_event(0x80,id as u64,r.resource,epoch), Ok(os_k1000_work::Enqueued::Coalesced(r))=>os_k1000_event(0x81,id as u64,r.resource,epoch), Err(os_k1000_work::Error::Full)=>{os_k1000_word(48,1);os_k1000_event(0x82,id as u64,epoch,os_k1000_state(7));}, Err(os_k1000_work::Error::Draining)=>{let p=core::ptr::addr_of_mut!(OS_K1000_EVIDENCE).cast::();let count=unsafe{read_volatile(p.add(47))};unsafe{write_volatile(p.add(47),count+1)};os_k1000_event(0x83,id as u64,epoch,0);}, Err(_)=>park(), } } os_k1000_set(5,used as u64);let status=unsafe{os_k940_read(os_k1000_state(3),0x60)};if status&1==0{park();}unsafe{os_k940_write(os_k1000_state(3),0x64,status&1);asm!("dsb sy",options(nostack,preserves_flags));} let active=unsafe{os_k940_read(os_k1000_state(25),0x300+4*(os_k1000_state(4)/32))}; for(i,v)in[iar,epoch,before as u64,used as u64,status as u64,active as u64,address,frame.elr,frame.spsr,frame.interrupted_sp,os_k940_registers().0,os_k940_registers().3,os_k960_counter(),os_k1000_state(7),0,0].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_IRQ_RECORDS).cast::().add(n*16+i),v)}} if before==10&&used==11{ if os_k1000_state(7)!=1{park();}let p=unsafe{core::ptr::addr_of_mut!(OS_K1000_WORKS).cast::>>().add(7)}; // IRQ owns this distinct idle work token through a bounded metadata // move; no raw &mut Work ingress is formed. The forged TASK then acts // on an owned local value, and its actual observer rejects before any // resource as_mut. Task's active slot0 is never borrowed or inspected. os_k1000_work::context(0,os_k1000_observed(),os_k1000_work::Operation::Enqueue).unwrap_or_else(|_|park()); let mut work=unsafe{core::ptr::replace(p,None)}.unwrap_or_else(||park());let prior=(work.record(),work.state());let false_task=os_k1000_work::ObservedContext{cpu:0,kind:os_k1000_work::Context::Task,irq_masked:true}; if work.service(false_task)!=Err(os_k1000_work::Error::Context)||(work.record(),work.state())!=prior{park();}unsafe{p.write(Some(work))};os_k1000_word(46,1);os_k1000_word(79,n as u64+1);os_k1000_set(13,address);os_k1000_capture(4,false); } let split:u64;unsafe{asm!("mrs {split}, ICC_CTLR_EL1",split=out(reg)split,options(nomem,nostack,preserves_flags));asm!("dsb sy","msr ICC_EOIR1_EL1, {iar}","isb",iar=in(reg)iar,options(nostack,preserves_flags));if split&2!=0{asm!("msr ICC_DIR_EL1, {iar}","dsb sy","isb",iar=in(reg)iar,options(nostack,preserves_flags));}} let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}.checked_add(1).unwrap_or_else(||park());unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT),count);write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER),os_k960_counter());write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),0)};os_k1000_set(6,n as u64+1);os_k1000_set(1,0);0 } fn os_k1000_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str>{ unsafe extern "C"{static __os_k1000_vectors:u8;} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_PRE_IRQ_COUNT),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)));write_volatile(core::ptr::addr_of_mut!(OS_K1000_PRE_IRQ_LAST),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)));write_volatile(core::ptr::addr_of_mut!(OS_K1000_PRE_IRQ_ACTIVE),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)));} let backend_layout=core::ptr::addr_of!(OS_K1000_BACKEND_LAYOUT_EXPECTED)as u64;unsafe{asm!("ldrb wzr, [{address}]",address=in(reg)backend_layout,options(nostack,preserves_flags));} let queue_layout=core::ptr::addr_of!(OS_K1000_QUEUE_LAYOUT_EXPECTED)as u64;unsafe{asm!("ldrb wzr, [{address}]",address=in(reg)queue_layout,options(nostack,preserves_flags));} let registry_layout=core::ptr::addr_of!(OS_K1000_REGISTRY_LAYOUT_EXPECTED)as u64;unsafe{asm!("ldrb wzr, [{address}]",address=in(reg)registry_layout,options(nostack,preserves_flags));} for address in[core::ptr::addr_of!(OS_K1000_PIN_LAYOUT_EXPECTED)as u64,core::ptr::addr_of!(OS_K1000_DETACHED_LAYOUT_EXPECTED)as u64,core::ptr::addr_of!(OS_K1000_RESOURCE_LAYOUT_EXPECTED)as u64,core::ptr::addr_of!(OS_K1000_JOB_RESOURCE_LAYOUT_EXPECTED)as u64,core::ptr::addr_of!(OS_K1000_PIN_CUSTODY)as u64,core::ptr::addr_of!(OS_K1000_RESOURCE_CUSTODY)as u64]{unsafe{asm!("ldrb wzr, [{address}]",address=in(reg)address,options(nostack,preserves_flags));}} if cfg!(feature="os-runtime-product") && !os_k1100_product_probe_enabled(boot)? {os_k1000_product_configuration();write_line("TURKOS/OS/K1000: PRODUCT-CONTROL NO-DEFERRED-PROBE");return Ok(());} let(old_ttbr,old_tcr,old_vbar,old_daif)=os_k940_registers();if old_ttbr!=tables.root||old_daif&0x80==0||os_k960_timer().1&1!=0{return Err("deferred-predecessor-control");} let guard=unsafe{read_volatile(core::ptr::addr_of!(OS_K900_STACK_EVIDENCE).cast::().add(13))};let low=guard+4096;let high=low+65536;let sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags));} if sp<=low+848||sp>=high-16||!tables.task_stack_guard_unmapped(guard)||unsafe{read_volatile(low as*const u64)}!=OS_K900_LOW_CANARY||unsafe{read_volatile((high-8)as*const u64)}!=OS_K900_HIGH_CANARY{return Err("deferred-stack-guard");} let blob=unsafe{core::slice::from_raw_parts(boot.dtb_address as*const u8,boot.dtb_size)};let mut net=None; for node in dtb::discover_virtio_mmio(blob).map_err(|_|"deferred-dtb-net")?.as_slice(){if unsafe{os_k940_read(node.base,8)}==1{if unsafe{os_k940_read(node.base,4)}!=2||net.replace(node.base).is_some(){return Err("deferred-net-device");}}} let net=net.ok_or("deferred-no-net")?;let(spi,_,_,trigger)=turkos_kernel::runtime::platform::os_k940_virtio_spi(blob,unsafe{&*core::ptr::addr_of!(OS_K700_WORKSPACE)},net,boot.interrupts.gic_distributor).map_err(|_|"deferred-net-spi")?; let (_,memory,irq,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?;let task=tasks.scheduler.running().ok_or("deferred-worker-task")?;if Some(task)!=tasks.id{return Err("deferred-controller-task");} if task.generation>u32::MAX as u64{return Err("deferred-owner-generation");}let owner=(task.generation<<32)|(task.slot as u64+1);let boot_id=unsafe{read_volatile(core::ptr::addr_of!(OS_K680_EVIDENCE).cast::())}; os_k1000_set(20,task.slot as u64+1);os_k1000_set(21,task.generation);os_k1000_set(3,net);os_k1000_set(4,spi as u64);os_k1000_set(11,low);os_k1000_set(12,high);os_k1000_set(14,core::ptr::addr_of!(__os_k1000_vectors)as u64);os_k1000_set(25,boot.interrupts.gic_distributor); for(i,v)in[(0,0x4b31303030574f52),(1,old_ttbr),(2,old_tcr),(3,old_vbar),(4,old_daif),(11,unsafe{read_volatile(core::ptr::addr_of!(OS_K1000_PRE_IRQ_COUNT))}),(14,task.slot as u64+1),(15,task.generation),(16,owner),(17,net),(18,spi as u64),(19,boot_id),(22,low),(23,high-16),(24,guard),(57,sp),(59,OS_K900_LOW_CANARY),(60,OS_K900_HIGH_CANARY),(73,os_k960_frequency()),(74,unsafe{read_volatile(core::ptr::addr_of!(OS_K1000_PRE_IRQ_LAST))})]{os_k1000_word(i,v);} os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1000_AUTHORITY_BEFORE).cast()); let region=memory.allocator.os_authority().free_regions().iter().rev().find(|r|r.end-r.start>=160*4096).ok_or("deferred-initial-pool")?;os_k1000_word(20,region.end-140*4096);os_k1000_word(21,region.end-108*4096);os_k1000_diagnostics(0);os_k1000_capture(0,true); let transfer=os_k940_transfer_irq(irq,boot.interrupts.gic_redistributor)?; let factory_before_sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)factory_before_sp,options(nomem,nostack,preserves_flags));}os_k1000_word(86,factory_before_sp); if factory_before_sp<=low+24576{return Err("deferred-factory-headroom");} let installation=os_k1000_factory_setup_and_install(memory.allocator.os_authority_mut(),net,owner,boot_id)?; let binding=|epoch|os_k1000_work::Binding{device:net,epoch,owner,cpu:0};let a=os_k1000_queue_mut(os_k1000_work::Operation::Register,|q,c|q.register(binding(1),c)).map_err(|_|"deferred-source-one")?;unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_REGISTRATIONS).cast::>(),Some(a))}; let dist=boot.interrupts.gic_distributor;let word=4*(spi as u64/32);let bit=1u32<<(spi%32);let route=(os_k1000_mpidr()&0x00ff_ffff)|(os_k1000_mpidr()&0xff00_0000_00); let old=[unsafe{os_k940_read(dist,0x100+word)&bit}as u64,unsafe{os_k940_read(dist,0x80+word)}as u64,unsafe{read_volatile((dist+0x400+spi as u64)as*const u8)}as u64,unsafe{read_volatile((dist+0x6000+8*spi as u64)as*const u64)},unsafe{os_k940_read(dist,0xc00+4*(spi as u64/16))}as u64]; if old[0]!=0||unsafe{os_k940_read(dist,0x300+word)|os_k940_read(dist,0x200+word)}&bit!=0{return Err("deferred-source-not-idle");} for(i,v)in old.into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_MMIO).cast::().add(i),v)}} unsafe{os_k940_write(dist,0x80+word,old[1]as u32|bit);write_volatile((dist+0x400+spi as u64)as*mut u8,0x80);write_volatile((dist+0x6000+8*spi as u64)as*mut u64,route);let shift=2*(spi%16);os_k940_write(dist,0xc00+4*(spi as u64/16),(old[4]as u32&!(3<>().add(1),Some(b))}; os_k1000_wait_rx(4,2);os_k1000_capture(2,true);os_k1000_take(0);os_k1000_take(1); for target in 5..=10{os_k1000_wait_rx(target,1);os_k1000_take(target as usize-3);} os_k1000_capture(3,true);os_k1000_finish(0); os_k1000_queue_mut(os_k1000_work::Operation::Drain,|q,c|q.drain(os_k1000_registration(0),c)).map_err(|_|"deferred-drain-one")?; os_k1000_wait_rx(12,1);os_k1000_wait_rx(15,2); os_k1000_queue_mut(os_k1000_work::Operation::Drain,|q,c|q.drain(os_k1000_registration(1),c)).map_err(|_|"deferred-drain-two")?;os_k1000_wait_rx(16,2); os_k1000_queue_mut(os_k1000_work::Operation::Release,|q,c|q.with_backend_observation(|b|{b.registry.with_backend_observation(|f|os_k940_inventory(f.authority,core::ptr::addr_of_mut!(OS_K1000_PIN_REFUSAL_BEFORE).cast()));let _=c;})); os_k1000_source_release(0,true);os_k1000_pin_refusal();os_k1000_capture(6,true); for slot in 2..8{os_k1000_finish(slot);}os_k1000_source_release(0,false);os_k1000_capture(7,true); os_k1000_take(0);os_k1000_finish(0);os_k1000_finish(1);os_k1000_source_release(1,false); if os_k1000_state(5)!=16||os_k1000_state(9)!=13||os_k1000_state(6)!=12{return Err("deferred-actual-completion-counts");} unsafe{os_k940_write(dist,0x180+word,bit);asm!("dsb sy",options(nostack,preserves_flags));os_k940_write(net,0x70,0);asm!("dsb sy",options(nostack,preserves_flags));os_k940_write(net,0x30,0);if os_k940_read(net,0x70)!=0||os_k940_read(net,0x44)!=0{return Err("deferred-net-retire-reset");}os_k940_write(net,0x30,1);if os_k940_read(net,0x44)!=0{return Err("deferred-net-retire-tx");}} os_k1000_mmio(dist,net,spi,16);if unsafe{os_k940_read(dist,0x100+word)|os_k940_read(dist,0x200+word)|os_k940_read(dist,0x300+word)}&bit!=0||unsafe{os_k940_read(net,0x60)}!=0{return Err("deferred-source-not-quiescent");} os_k1000_event(0x91,os_k1000_state(9),os_k1000_state(6),0); os_k1000_queue_mut(os_k1000_work::Operation::Release,|q,_|q.os_k1000_retire_backing()).map_err(|_|"deferred-backing-retirement")?;os_k1000_snapshot(true); installation.retire(); unsafe{os_k940_write(dist,0x80+word,old[1]as u32);write_volatile((dist+0x400+spi as u64)as*mut u8,old[2]as u8);write_volatile((dist+0x6000+8*spi as u64)as*mut u64,old[3]);os_k940_write(dist,0xc00+4*(spi as u64/16),old[4]as u32);asm!("msr VBAR_EL1, {vbar}","msr DAIF, {daif}","dsb sy","isb",vbar=in(reg)old_vbar,daif=in(reg)old_daif,options(nostack,preserves_flags));} drop(transfer);os_k1000_diagnostics(3);if !os_k920_inventory_equal(core::ptr::addr_of!(OS_K1000_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K1000_AUTHORITY_AFTER).cast())||os_k940_registers()!=(old_ttbr,old_tcr,old_vbar,old_daif){return Err("deferred-owner-restoration");} let post_sp:u64;unsafe{asm!("mov {sp}, sp",sp=out(reg)post_sp,options(nomem,nostack,preserves_flags));}os_k1000_word(58,post_sp);os_k1000_mmio(dist,net,spi,24);os_k1000_word(81,unsafe{os_k940_read(dist,0xc00+4*(spi as u64/16))}as u64); if post_sp<=low+848||post_sp>=high-16||unsafe{read_volatile(low as*const u64)}!=OS_K900_LOW_CANARY||unsafe{read_volatile((high-8)as*const u64)}!=OS_K900_HIGH_CANARY||unsafe{os_k940_read(dist,0x80+word)}!=old[1]as u32||unsafe{read_volatile((dist+0x400+spi as u64)as*const u8)}!=old[2]as u8||unsafe{read_volatile((dist+0x6000+8*spi as u64)as*const u64)}!=old[3]||unsafe{os_k940_read(dist,0xc00+4*(spi as u64/16))}!=old[4]as u32{return Err("deferred-final-stack-source");} os_k1000_capture(9,true);write_line("TURKOS/OS/K985: PASS JOBS=8 NINTH=REFUSED LOST=1");write_line("TURKOS/OS/K990: PASS IRQ=12 TASK-TX=13 DETACHED-WORK");write_line("TURKOS/OS/K995: PASS SOURCE-EPOCHS=2 EXACT-COALESCING");write_line("TURKOS/OS/K1000: PASS DRAINED SOURCE-PINS=RESTORED");Ok(()) } fn os_k1000_mmio(dist:u64,net:u64,spi:u32,offset:usize){let word=4*(spi as u64/32);for(i,v)in[unsafe{os_k940_read(dist,0x100+word)}as u64,unsafe{os_k940_read(dist,0x200+word)}as u64,unsafe{os_k940_read(dist,0x300+word)}as u64,unsafe{os_k940_read(dist,0x80+word)}as u64,unsafe{read_volatile((dist+0x400+spi as u64)as*const u8)}as u64,unsafe{read_volatile((dist+0x6000+8*spi as u64)as*const u64)},unsafe{os_k940_read(net,0x70)}as u64,unsafe{os_k940_read(net,0x60)}as u64].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_MMIO).cast::().add(offset+i),v)}}} fn os_k1000_mpidr()->u64{let value:u64;unsafe{asm!("mrs {value}, MPIDR_EL1",value=out(reg)value,options(nomem,nostack,preserves_flags));}value} fn os_k1000_source_release(epoch:usize,busy:bool){let pointer=unsafe{core::ptr::addr_of_mut!(OS_K1000_REGISTRATIONS).cast::>().add(epoch)};let registration=unsafe{(&mut*pointer).take()}.unwrap_or_else(||park());let result=os_k1000_queue_mut(os_k1000_work::Operation::Release,|q,c|q.release(registration,c));match(result,busy){(Ok(()),false)=>{},(Err((os_k1000_work::Error::Busy,registration)),true)=>unsafe{pointer.write(Some(registration))},_=>park()}} fn os_k1000_pin_refusal(){ os_k1000_queue_mut(os_k1000_work::Operation::Release,|q,_|q.os_k1000_pin_refusal()).unwrap_or_else(|_|park()); } #[inline(never)]fn os_k1000_product_configuration(){ unsafe extern "C"{static __os_k1000_vectors:u8;} for address in[ core::ptr::addr_of!(OS_K1000_QUEUE)as u64,core::ptr::addr_of!(OS_K1000_WORKS)as u64,core::ptr::addr_of!(OS_K1000_FACTORY_BACKEND)as u64,os_k1000_factory_setup_and_install as*const()as u64, core::ptr::addr_of!(OS_K1000_STATE)as u64,core::ptr::addr_of!(OS_K1000_EVIDENCE)as u64, core::ptr::addr_of!(OS_K1000_AUTHORITY_BEFORE)as u64,core::ptr::addr_of!(OS_K1000_AUTHORITY_LIVE)as u64,core::ptr::addr_of!(OS_K1000_AUTHORITY_AFTER)as u64, core::ptr::addr_of!(OS_K1000_PIN_REFUSAL_BEFORE)as u64,core::ptr::addr_of!(OS_K1000_PIN_REFUSAL_AFTER)as u64, core::ptr::addr_of!(OS_K1000_OWNED)as u64,core::ptr::addr_of!(OS_K1000_REGISTRY)as u64, core::ptr::addr_of!(OS_K1000_SOURCE_RECORDS)as u64,core::ptr::addr_of!(OS_K1000_JOB_RECORDS)as u64,core::ptr::addr_of!(OS_K1000_WORK_ASSETS)as u64, core::ptr::addr_of!(OS_K1000_IRQ_FRAMES)as u64,core::ptr::addr_of!(OS_K1000_IRQ_RECORDS)as u64,core::ptr::addr_of!(OS_K1000_FRAME_ADDRESSES)as u64,core::ptr::addr_of!(OS_K1000_IO_RECORDS)as u64, core::ptr::addr_of!(OS_K1000_EVENTS)as u64,core::ptr::addr_of!(OS_K1000_EVENT_COUNT)as u64,core::ptr::addr_of!(OS_K1000_NONCE)as u64,core::ptr::addr_of!(OS_K1000_SOURCE_NONCE)as u64, core::ptr::addr_of!(OS_K1000_LAST_IRQ)as u64,core::ptr::addr_of!(OS_K1000_DIAGNOSTICS)as u64,core::ptr::addr_of!(OS_K1000_KERNEL_FRAME)as u64,core::ptr::addr_of!(OS_K1000_CAPTURE_GATE)as u64,core::ptr::addr_of!(OS_K1000_MMIO)as u64,core::ptr::addr_of!(OS_K1000_AUTHORITY_LAYOUT)as u64,core::ptr::addr_of!(OS_K1000_REGISTRY_LAYOUT_EXPECTED)as u64,core::ptr::addr_of!(os_k720_frame::OS_K1000_AUTHORITY_LAYOUT_EXPECTED)as u64, core::ptr::addr_of!(__os_k1000_vectors)as u64,os_k1000_worker_epoch_one as*const()as u64,os_k1000_worker_epoch_two as*const()as u64,os_k1000_io as*const()as u64,__os_k1000_deferred_identity as*const()as u64, ]{unsafe{asm!("ldrb wzr, [{address}]",address=in(reg)address,options(nostack,preserves_flags));}} } #[unsafe(no_mangle)]static mut OS_K1000_PIN_CUSTODY:[u64;35*7]=[0;35*7]; #[unsafe(no_mangle)]static mut OS_K1000_RESOURCE_CUSTODY:[u64;8*5]=[0;8*5]; #[unsafe(no_mangle)]static OS_K1000_PIN_LAYOUT_EXPECTED:[u64;9]=os_k740_metadata::Pin::os_k1000_pin_layout(); #[unsafe(no_mangle)]static OS_K1000_DETACHED_LAYOUT_EXPECTED:[u64;8]=os_k1000_work::Work::::os_k1000_detached_layout(); #[unsafe(no_mangle)]static OS_K1000_JOB_RESOURCE_LAYOUT_EXPECTED:[u64;2]=OsK1000Queue::os_k1000_job_resource_layout(); #[unsafe(no_mangle)]static OS_K1000_RESOURCE_LAYOUT_EXPECTED:[u64;17]=[ core::mem::size_of::()as u64, core::mem::offset_of!(OsK1000Resource,id)as u64,core::mem::offset_of!(OsK1000Resource,binding)as u64, core::mem::offset_of!(OsK1000Resource,buffer)as u64,core::mem::offset_of!(OsK1000Resource,source_pin)as u64, core::mem::offset_of!(OsK1000Resource,start)as u64,core::mem::offset_of!(OsK1000Resource,page)as u64, core::mem::offset_of!(OsK1000Resource,context)as u64,core::mem::offset_of!(OsK1000Resource,buffer_pin)as u64, core::mem::offset_of!(OsK1000Resource,context_pin)as u64,core::mem::size_of::>()as u64, core::mem::offset_of!(OsK1000Resource,producer)as u64,core::mem::offset_of!(OsK1000Resource,tx)as u64, core::mem::offset_of!(OsK1000Resource,callback)as u64,core::mem::offset_of!(OsK1000Resource,boot_id)as u64, core::mem::size_of::>>()as u64, core::mem::size_of::>()as u64, ]; fn os_k1000_pin_custody(pin:&os_k740_metadata::Pin,count:&mut usize){ let observation=pin.os_k1000_pin_observation();if observation[5]!=0{return;} if *count>=35{park();} for(i,v)in core::iter::once(pin as*const os_k740_metadata::Pin as u64).chain(observation).enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_PIN_CUSTODY).cast::().add(*count*7+i),v)}}*count+=1; } fn os_k1000_resource_custody(slot:usize,resource:&OsK1000Resource,work_address:u64,count:&mut usize){ let buffer=resource.buffer_pin.as_ref().unwrap_or_else(||park());let context=resource.context_pin.as_ref().unwrap_or_else(||park()); for(i,v)in[resource.id,resource as*const OsK1000Resource as u64,work_address,buffer as*const os_k740_metadata::Pin as u64,context as*const os_k740_metadata::Pin as u64].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_RESOURCE_CUSTODY).cast::().add(slot*5+i),v)}} os_k1000_pin_custody(buffer,count);os_k1000_pin_custody(context,count); } fn os_k1000_custody_snapshot(q:&OsK1000Queue<'_>){ for i in 0..245{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_PIN_CUSTODY).cast::().add(i),0)}} for i in 0..40{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1000_RESOURCE_CUSTODY).cast::().add(i),0)}} let mut count=0; q.with_backend_observation(|b|{ for pin in b.ring_pins.iter().chain(b.buffer_pins.iter()).flatten(){os_k1000_pin_custody(pin,&mut count);} for pin in b.source_pins.iter().flat_map(|r|r.iter()).flatten(){os_k1000_pin_custody(pin,&mut count);} for source in b.source_free.iter().flatten(){for pin in source.pins.iter().flatten(){os_k1000_pin_custody(pin,&mut count);}} }); for slot in 0..2{q.os_k1000_source_observation(slot,|source|{if let Some(source)=source{for pin in source.pins.iter().flatten(){os_k1000_pin_custody(pin,&mut count);}}}).unwrap_or_else(|_|park());} for slot in 0..8{ q.os_k1000_job_observation(slot,|_,resource|{if let Some(resource)=resource{os_k1000_resource_custody(slot,resource,0,&mut count);}}).unwrap_or_else(|_|park()); if let Some(work)=unsafe{(&*core::ptr::addr_of!(OS_K1000_WORKS)).get(slot).and_then(Option::as_ref)}{ work.os_k1000_resource_observation(|resource|os_k1000_resource_custody(slot,resource,work as*const os_k1000_work::Workas u64,&mut count)).unwrap_or_else(|_|park()); } } os_k1000_word(83,count as u64); } const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k1000_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); use turkos_kernel::runtime::threads as os_k1020_thread; const OS_K1020_STACK_COUNT: usize = 5; const OS_K1020_STACK_PAGES: usize = 4; const OS_K1020_OWNED_PAGES: usize = OS_K1020_STACK_COUNT * (OS_K1020_STACK_PAGES + 1); struct OsK1020Stack { ordinal: usize, start: u64, leases: [Option; 8], guard: Option, initial_observation: core::cell::Cell, } impl OsK1020Stack { fn layout(&self) -> os_k1020_thread::StackLayout { os_k1020_thread::StackLayout { base: self.start + 4096, physical: self.start + 4096, bytes: OS_K1020_STACK_PAGES as u64 * 4096 } } } struct OsK1020Backend<'a> { memory: &'a mut OsK660Memory, tables: &'a mmu::Tables, trace: bool } fn os_k1020_execution() -> os_k1020_thread::Execution { let (cpu, daif, sp): (u64, u64, u64); unsafe { asm!("mrs {cpu}, MPIDR_EL1", "mrs {daif}, DAIF", "mov {sp}, sp", cpu=out(reg)cpu, daif=out(reg)daif, sp=out(reg)sp, options(nomem,nostack,preserves_flags)); } os_k1020_thread::Execution { cpu: (cpu & 0xff) as u32, irq_masked: daif & 0x80 != 0, in_irq: unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0, sp } } impl OsK1020Backend<'_> { fn validate(&self, stack: &OsK1020Stack) -> Result<(), os_k1020_thread::Error> { let e = os_k1020_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || os_k940_registers().0 != self.tables.root { return Err(os_k1020_thread::Error::Context); } if stack.ordinal >= OS_K1020_STACK_COUNT || stack.start % 4096 != 0 || stack.leases[OS_K1020_STACK_PAGES+1..].iter().any(Option::is_some) { return Err(os_k1020_thread::Error::Layout); } for index in 0..=OS_K1020_STACK_PAGES { let lease=stack.leases[index].as_ref().ok_or(os_k1020_thread::Error::Backend)?; if lease.address()!=stack.start+index as u64*4096 { return Err(os_k1020_thread::Error::Layout); } self.memory.allocator.os_authority().validate_lease(lease) .map_err(|_|os_k1020_thread::Error::Backend)?; } Ok(()) } } unsafe impl os_k1020_thread::Backend for OsK1020Backend<'_> { type Stack = OsK1020Stack; fn execution(&self) -> os_k1020_thread::Execution { os_k1020_execution() } fn layout(&self, stack: &OsK1020Stack) -> Result { self.validate(stack)?; Ok(stack.layout()) } fn entry_executable(&self, entry: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } entry.is_multiple_of(4) && entry >= core::ptr::addr_of!(_text_start) as u64 && entry < core::ptr::addr_of!(_text_end) as u64 } fn prepare(&mut self, stack: &mut OsK1020Stack) -> Result<(), os_k1020_thread::Error> { self.validate(stack)?; let layout=stack.layout();let e=os_k1020_execution(); if e.sp>=stack.start && e.sp<=layout.base+layout.bytes || stack.guard.is_some() { return Err(os_k1020_thread::Error::ActiveStack); } let guard=unsafe { self.tables.os_k1020_acquire_stack_guard(stack.start, layout.base+layout.bytes) } .map_err(|_|os_k1020_thread::Error::Guard)?; stack.guard=Some(guard); for offset in 0..layout.bytes/8 { unsafe { write_volatile((layout.base as *mut u64).add(offset as usize),0); } } unsafe { write_volatile(layout.base as *mut u64,os_k1020_thread::LOW_CANARY); write_volatile((layout.base+layout.bytes-8) as *mut u64,os_k1020_thread::HIGH_CANARY); asm!("dsb ish",options(nostack,preserves_flags)); } stack.initial_observation.set(true); if self.trace {os_k1020_event(2,stack.ordinal as u64,stack.start,layout.bytes);} Ok(()) } fn observe(&self, stack: &OsK1020Stack) -> Result { self.validate(stack)?;let layout=stack.layout();let token=stack.guard.as_ref().ok_or(os_k1020_thread::Error::Guard)?; let descriptor=self.tables.os_k1020_stack_guard_descriptor(token).map_err(|_|os_k1020_thread::Error::Guard)?; if token.guard()!=stack.start || token.end()!=layout.base+layout.bytes {return Err(os_k1020_thread::Error::Layout);} // Initial full erasure is observed once before publication. Later // selection/exit checks read the actual guard and canaries without // walking the active or previously used Rust stack contents. let zeroed=stack.initial_observation.get()&&(1..layout.bytes/8-1).all(|offset|unsafe {read_volatile((layout.base as *const u64).add(offset as usize))}==0); if zeroed {stack.initial_observation.set(false);} Ok(os_k1020_thread::StackObservation {layout,guard_descriptor:descriptor, low_canary:unsafe {read_volatile(layout.base as *const u64)}, high_canary:unsafe {read_volatile((layout.base+layout.bytes-8) as *const u64)},initially_zeroed:zeroed}) } fn release(&mut self, mut stack: OsK1020Stack) -> Result<(), (os_k1020_thread::Error, OsK1020Stack)> { if let Err(error)=self.validate(&stack) {return Err((error,stack));} let layout=stack.layout();let e=os_k1020_execution(); if e.sp>=stack.start && e.sp<=layout.base+layout.bytes {return Err((os_k1020_thread::Error::ActiveStack,stack));} let token=match stack.guard.take(){Some(token)=>token,None=>return Err((os_k1020_thread::Error::Guard,stack))}; if let Err((_error,token))=unsafe {self.tables.os_k1020_restore_stack_guard(token)} { stack.guard=Some(token);return Err((os_k1020_thread::Error::Guard,stack)); } for offset in 0..(OS_K1020_STACK_PAGES+1)*4096/8 {unsafe {write_volatile((stack.start as *mut u64).add(offset),0);}} unsafe {asm!("dsb ish",options(nostack,preserves_flags));} if !(0..(OS_K1020_STACK_PAGES+1)*4096/8).all(|offset|unsafe {read_volatile((stack.start as *const u64).add(offset))}==0) { return Err((os_k1020_thread::Error::Backend,stack)); } let refs=core::array::from_fn::<_,5,_>(|index|stack.leases[index].as_ref().unwrap()); if self.memory.allocator.os_authority_mut().release_many_leases(&refs).is_err() { return Err((os_k1020_thread::Error::Backend,stack)); } if self.trace {os_k1020_event(3,stack.ordinal as u64,stack.start,5);} for lease in &mut stack.leases { *lease=None; } Ok(()) } } fn os_k1020_allocate_stack(memory:&mut OsK660Memory,tables:&mmu::Tables,ordinal:usize) -> Result { let authority=memory.allocator.os_authority();let mut selected=None;let mut checked=0; for region in authority.free_regions() { let mut address=region.start; while address.checked_add(5*4096).is_some_and(|end|end<=region.end) && checked<512 { checked+=1; if tables.os_k1020_stack_candidate(address,address+5*4096) {selected=Some(address);break;} address=address.checked_add(4096).ok_or("thread-stack-candidate-overflow")?; } if selected.is_some(){break;} } let start=selected.ok_or("thread-stack-no-owned-L3-span")?; let leases=memory.allocator.os_authority_mut().allocate_many_at(start,5).map_err(|_|"thread-stack-lease-block")?; // Every byte starts poisoned; core prepare must erase the whole private // backing before publishing Ready. The guard is erased after restoration. for offset in 0..5*4096/8 {unsafe {write_volatile((start as *mut u64).add(offset),0xa5a5a5a5a5a5a5a5);}} for index in 0..5 {let lease=leases[index].as_ref().ok_or("thread-stack-lease-prefix")?; unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K1020_OWNED).cast::().add(ordinal*5+index),lease.address()); write_volatile(core::ptr::addr_of_mut!(OS_K1020_LEASE_GENERATIONS).cast::().add(ordinal*5+index),lease.generation());} os_k1020_event(1,ordinal as u64,lease.address(),lease.generation()); } Ok(OsK1020Stack {ordinal,start,leases,guard:None,initial_observation:core::cell::Cell::new(false)}) } type OsK1020Table<'a> = os_k1020_thread::Table,3>; static mut OS_K1020_TABLE: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K1020_FAILED_BACKEND: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K1020_SPARES: [Option;5] = [const {None};5]; static mut OS_K1020_IDS: [Option;5] = [None;5]; static mut OS_K1020_WAITS: [Option;3] = [const {None};3]; #[unsafe(no_mangle)] static mut OS_K1020_EVIDENCE: [u64;96] = [0;96]; #[unsafe(no_mangle)] static mut OS_K1020_SLOT_RECORDS: [u64;36] = [0;36]; #[unsafe(no_mangle)] static mut OS_K1020_IDENTITIES: [u64;15] = [0;15]; #[unsafe(no_mangle)] static mut OS_K1020_AUTHORITY_BEFORE: [u64;328] = [0;328]; #[unsafe(no_mangle)] static mut OS_K1020_AUTHORITY_LIVE: [u64;328] = [0;328]; #[unsafe(no_mangle)] static mut OS_K1020_AUTHORITY_AFTER: [u64;328] = [0;328]; #[unsafe(no_mangle)] static mut OS_K1020_OWNED: [u64;OS_K1020_OWNED_PAGES] = [0;OS_K1020_OWNED_PAGES]; #[unsafe(no_mangle)] static mut OS_K1020_LEASE_GENERATIONS: [u64;OS_K1020_OWNED_PAGES] = [0;OS_K1020_OWNED_PAGES]; #[unsafe(no_mangle)] static mut OS_K1020_KERNEL_TABLES: [u64;32] = [0;32]; #[unsafe(no_mangle)] static mut OS_K1020_TRACES: [u64;140] = [0;140]; #[unsafe(no_mangle)] static mut OS_K1020_EVENTS: [u64;1024] = [0;1024]; #[unsafe(no_mangle)] static mut OS_K1020_EVENT_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1020_NONCE: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1020_CAPTURE_GATE: u8 = 0; #[unsafe(no_mangle)] static mut OS_K1020_KERNEL_FRAME: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1020_IRQ_SEEN: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1020_PRE_IRQ: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1020_PRE_LAST: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1020_PRE_ACTIVE: u32 = 0; #[unsafe(no_mangle)] static mut OS_K1020_DIAGNOSTICS: [u64;6] = [0;6]; #[unsafe(no_mangle)] static mut OS_K1020_FATAL_FRAME: os_k920_exception::Frame = OS_K920_ZERO_FRAME; #[unsafe(no_mangle)] static mut OS_K1020_FATAL: [u64;32] = [0;32]; fn os_k1020_table() -> *mut OsK1020Table<'static> {core::ptr::addr_of_mut!(OS_K1020_TABLE).cast()} fn os_k1020_word(index:usize,value:u64){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_EVIDENCE).cast::().add(index),value);}} fn os_k1020_read(index:usize)->u64{unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_EVIDENCE).cast::().add(index))}} fn os_k1020_event(kind:u64,a:u64,b:u64,c:u64){ let n=unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_EVENT_COUNT))} as usize;if n>=256{park();} for(i,value)in[kind,a,b,c].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_EVENTS).cast::().add(n*4+i),value);}} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_EVENT_COUNT),n as u64+1);} } #[unsafe(no_mangle)] extern "C" fn __os_k1020_threads_identity()->u64{ let e=os_k1020_execution();if e.cpu!=0||!e.irq_masked||e.in_irq{return 0;} let n=unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_NONCE))}; let Some(next)=n.checked_add(1)else{return 0;};if n==0{return 0;} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_NONCE),next);}n } struct OsK1020Installation<'a>{pointer:*mut OsK1020Table<'a>,active:bool,ownership:core::marker::PhantomData>} impl<'a> OsK1020Installation<'a>{ unsafe fn install(table:OsK1020Table<'a>)->Self{let pointer=core::ptr::addr_of_mut!(OS_K1020_TABLE).cast::>();unsafe{pointer.write(table);} Self{pointer,active:true,ownership:core::marker::PhantomData}} fn retire(mut self)->OsK1020Backend<'a>{let e=os_k1020_execution();if !e.irq_masked||e.in_irq||os_k1020_read(17)!=0{park();} let table=unsafe{self.pointer.read()};let backend=match table.into_backend(){Ok(backend)=>backend,Err((_error,table))=>{ unsafe{self.pointer.write(table);}park();}};self.active=false;backend} } impl Drop for OsK1020Installation<'_>{fn drop(&mut self){if self.active{park();}}} fn os_k1020_id(index:usize)->os_k1020_thread::ThreadId{ unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_IDS).cast::>().add(index))}.unwrap_or_else(||park()) } fn os_k1020_diagnostics(offset:usize){let log=unsafe{(&*core::ptr::addr_of!(OS_K680_LOG)).as_ref()}.unwrap_or_else(||park()); for(i,value)in[log.boot().value(),log.next_sequence(),log.len() as u64].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_DIAGNOSTICS).cast::().add(offset+i),value);}} } fn os_k1020_snapshot(stage:u64){ let(r,t,v,d)=os_k940_registers();let e=os_k1020_execution();let(ctl,spsel,mpidr):(u64,u64,u64); unsafe{asm!("mrs {ctl}, CNTP_CTL_EL0","mrs {s}, SPSel","mrs {m}, MPIDR_EL1",ctl=out(reg)ctl,s=out(reg)spsel,m=out(reg)mpidr,options(nomem,nostack,preserves_flags));} for(i,value)in[(5,stage),(6,r),(7,t),(8,v),(9,d),(10,e.cpu as u64),(11,e.sp), (12,os_k960_counter()),(13,u64::from(e.in_irq)),(14,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}),(16,ctl), (30,spsel),(31,mpidr)]{os_k1020_word(i,value);} if os_k1020_read(17)!=0 { let table=unsafe{&*os_k1020_table()};let s=table.snapshot(); for(i,value)in[(60,s.live as u64),(61,s.ready as u64),(62,s.running as u64),(63,s.sleeping as u64), (64,s.dead as u64),(65,s.quarantined as u64),(66,s.pages),(67,s.retired as u64)]{os_k1020_word(i,value);} for(index,value)in table.os_k1020_slot_words().into_iter().flatten().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_SLOT_RECORDS).cast::().add(index),value);}} table.os_k1020_observe_backend(|backend|{ os_k1020_word(70,backend.memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64); os_k1020_word(71,core::mem::size_of::() as u64); os_k940_inventory(backend.memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1020_AUTHORITY_LIVE).cast()); for ordinal in 0..5{let address=os_k1020_read(40+ordinal); if let Some(slot)=backend.tables.metadata_stack_guard_descriptor_address(address){os_k1020_word(50+ordinal,slot);} let slot=os_k1020_read(50+ordinal);if slot!=0{os_k1020_word(32+ordinal,unsafe{read_volatile(slot as *const u64)});} } }); } } #[unsafe(no_mangle)] extern "C" fn os_k1020_dispatch(frame:*mut os_k920_exception::Frame)->u64{ if frame.is_null(){park();}let captured=unsafe{read_volatile(frame)}; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_FATAL_FRAME),captured); write_volatile(core::ptr::addr_of_mut!(OS_K1020_FATAL).cast::().add(21),frame as u64);} os_k1020_fatal_dispatch(&captured,frame as u64) } fn os_k1020_capture(stage:u64){os_k1020_snapshot(stage);unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_CAPTURE_GATE),0);} write_str("TURKOS/OS/K1020: CAPTURE STAGE=");write_hex(stage as usize);write_line(""); while unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_CAPTURE_GATE))}!=1{core::hint::spin_loop();} unsafe{asm!("dsb ish","isb",options(nostack,preserves_flags));} } fn os_k1020_create(ordinal:usize)->Result{ let stack=unsafe{(&mut*core::ptr::addr_of_mut!(OS_K1020_SPARES).cast::>().add(ordinal)).take()}.ok_or("thread-stack-token")?; let table=unsafe{&mut*os_k1020_table()};let id=match table.create(stack,os_k1020_thread_step as *const () as u64,ordinal as u64){ Ok(id)=>id,Err((_error,stack))=>{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_SPARES).cast::>().add(ordinal),Some(stack));}return Err("thread-create");}}; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_IDS).cast::>().add(ordinal),Some(id));} for(i,value)in[id.table(),id.slot() as u64,id.generation()].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_IDENTITIES).cast::().add(ordinal*3+i),value);}} os_k1020_event(4,ordinal as u64,id.slot() as u64,id.generation());Ok(id) } fn os_k1020_refused(){os_k1020_word(23,os_k1020_read(23).checked_add(1).unwrap_or_else(||park()));} #[unsafe(no_mangle)] extern "C" fn os_k1020_thread_step(argument:u64){ let ordinal=argument as usize;if ordinal>=5{park();}let id=os_k1020_id(ordinal); unsafe{(&mut*os_k1020_table()).activate(id)}.unwrap_or_else(|_|park()); let entry=unsafe{(&*os_k1020_table()).record(id)}.unwrap_or_else(|_|park());let e=os_k1020_execution(); if entry.state!=os_k1020_thread::State::Running||e.spentry.layout.usable_top(){park();} let n=os_k1020_read(27) as usize;if n>=7{park();} let op=os_k1020_read(26);let mut refusal=0; if unsafe{(&mut*os_k1020_table()).reap(id)}!=Err(os_k1020_thread::Error::State){park();}os_k1020_refused();refusal+=1; if ordinal==0&&n==0 {os_k1020_capture(1);} match op{ 1=>{let wait=unsafe{(&mut*os_k1020_table()).sleep_current(id)}.unwrap_or_else(|_|park()); let slot=if ordinal==2{2}else{0};let target=unsafe{&mut*core::ptr::addr_of_mut!(OS_K1020_WAITS).cast::>().add(slot)}; if target.is_some(){park();}*target=Some(wait);} 2=>unsafe{(&mut*os_k1020_table()).yield_current(id)}.unwrap_or_else(|_|park()), 3=>{unsafe{(&mut*os_k1020_table()).exit_current(id)}.unwrap_or_else(|_|park()); if unsafe{(&mut*os_k1020_table()).activate(id)}!=Err(os_k1020_thread::Error::State) || unsafe{(&*os_k1020_table()).ready_context(id)}!=Err(os_k1020_thread::Error::State){park();} os_k1020_refused();os_k1020_refused();refusal+=2; if unsafe{(&mut*os_k1020_table()).reap(id)}!=Err(os_k1020_thread::Error::ActiveStack){park();}os_k1020_refused();refusal+=1; if ordinal==4{os_k1020_capture(7);}} _=>park(), } let after=unsafe{(&*os_k1020_table()).record(id)}.unwrap_or_else(|_|park()); for(i,value)in[ordinal as u64,id.table(),id.slot() as u64,id.generation(),op,e.sp,entry.layout.base, entry.layout.bytes,entry.state as u64,after.state as u64,after.wait_epoch,e.cpu as u64,u64::from(e.irq_masked), u64::from(e.in_irq),os_k940_registers().0,os_k960_counter(),refusal,os_k1020_thread::LOW_CANARY, os_k1020_thread::HIGH_CANARY,0].into_iter().enumerate(){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_TRACES).cast::().add(n*20+i),value);}} os_k1020_word(27,n as u64+1);os_k1020_word(22,n as u64+1); os_k1020_event(5,ordinal as u64,op,after.state as u64); } fn os_k1020_step(ordinal:usize,operation:u64)->Result<(),&'static str>{ unsafe extern "C"{fn __os_k1020_run_on_stack(sp:u64,entry:u64,argument:u64);} let context=unsafe{(&*os_k1020_table()).ready_context(os_k1020_id(ordinal))}.map_err(|_|"thread-ready-context")?; if context.entry!=os_k1020_thread_step as *const () as u64||context.argument!=ordinal as u64{return Err("thread-entry-owner");} os_k1020_word(26,operation); unsafe{__os_k1020_run_on_stack(context.initial_sp,context.entry,context.argument);} Ok(()) } fn os_k1020_wake(index:usize)->Result<(),&'static str>{let wait=unsafe{(&mut*core::ptr::addr_of_mut!(OS_K1020_WAITS).cast::>().add(index)).take()}.ok_or("thread-wait-custody")?; match unsafe{(&mut*os_k1020_table()).wake(wait)}{Ok(_)=>Ok(()),Err((_error,wait))=>{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_WAITS).cast::>().add(index),Some(wait));}Err("thread-wake")}} } fn os_k1020_reap(ordinal:usize)->Result<(),&'static str>{unsafe{(&mut*os_k1020_table()).reap(os_k1020_id(ordinal))}.map_err(|_|"thread-reap")?; os_k1020_word(24,os_k1020_read(24)+1);Ok(())} fn os_k1020_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str>{ let mode=os_k1020_fixture(boot)?; let pre=unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_PRE_IRQ),pre); write_volatile(core::ptr::addr_of_mut!(OS_K1020_PRE_LAST),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER))); write_volatile(core::ptr::addr_of_mut!(OS_K1020_PRE_ACTIVE),read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)));} os_k1020_retain_configuration(); if cfg!(feature="os-runtime-product") && !os_k1100_product_probe_enabled(boot)? {write_line("TURKOS/OS/K1020: PRODUCT-CONTROL NO-THREAD-PROBE");return Ok(());} let(root,tcr,vbar,daif)=os_k940_registers();let e=os_k1020_execution();let ctl:u64; unsafe{asm!("mrs {ctl}, CNTP_CTL_EL0",ctl=out(reg)ctl,options(nomem,nostack,preserves_flags));} if root!=tables.root||!e.irq_masked||e.in_irq||e.cpu!=0||ctl&1!=0{return Err("thread-predecessor-context");} let guard=unsafe{read_volatile(core::ptr::addr_of!(OS_K900_STACK_EVIDENCE).cast::().add(13))}; let low=guard+4096;let top=low+65536-16; if e.sp<=low+1024||e.sp>top||!tables.task_stack_guard_unmapped(guard){return Err("thread-primary-stack");} let(_,memory,_,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?; if tasks.scheduler.running()!=tasks.id{return Err("thread-alpha-running");} os_k940_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1020_AUTHORITY_BEFORE).cast());os_k1020_diagnostics(0); for(index,address)in tables.os_k1020_retained_table_words().map_err(|_|"thread-kernel-table-owners")?.into_iter().enumerate(){ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_KERNEL_TABLES).cast::().add(index),address);}} for(i,value)in[(0,0x4b31303230544852),(1,root),(2,tcr),(3,vbar),(4,daif),(15,pre),(18,low),(19,top),(20,guard),(21,25),(68,mode as u64)]{os_k1020_word(i,value);} for ordinal in 0..5{let stack=os_k1020_allocate_stack(memory,tables,ordinal)?;os_k1020_word(40+ordinal,stack.start); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_SPARES).cast::>().add(ordinal),Some(stack));}} let duplicate_before=os_k740_authority_inventory(memory.allocator.os_authority()); if !matches!(memory.allocator.os_authority_mut().allocate_many_at(os_k1020_read(40),5),Err(os_k720_frame::Error::NotFree)) { return Err("thread-shared-backing-acquisition-admitted"); } if duplicate_before.words!=os_k740_authority_inventory(memory.allocator.os_authority()).words { return Err("thread-shared-backing-refusal-mutated-authority"); } os_k1020_refused();os_k1020_word(29,1); let backend=OsK1020Backend{memory,tables,trace:true};let table=match OsK1020Table::new(backend){Ok(table)=>table,Err((_error,backend))=>{ unsafe{core::ptr::addr_of_mut!(OS_K1020_FAILED_BACKEND).cast::>().write(backend);}return Err("thread-table-constructor");}}; let installation=unsafe{OsK1020Installation::install(table)};os_k1020_word(17,1); unsafe extern "C"{static __os_k1020_vectors:u8;} let vectors=core::ptr::addr_of!(__os_k1020_vectors) as u64; unsafe{asm!("msr VBAR_EL1, {vectors}","isb",vectors=in(reg)vectors,options(nomem,nostack,preserves_flags));} os_k1020_word(28,vectors); for ordinal in 0..3{os_k1020_create(ordinal)?;}os_k1020_capture(0); os_k1020_step(0,1)?;os_k1020_capture(2); os_k1020_step(1,2)?;os_k1020_step(2,1)?;os_k1020_capture(3); os_k1020_wake(0)?;os_k1020_step(0,1)?; unsafe{(&mut*os_k1020_table()).cancel(os_k1020_id(0))}.map_err(|_|"thread-cancel-first")?; os_k1020_reap(0)?;os_k1020_capture(4); let second=os_k1020_create(3)?; if second.slot()!=os_k1020_id(0).slot()||second.generation()!=2{return Err("thread-second-generation");} let old=unsafe{(&mut*core::ptr::addr_of_mut!(OS_K1020_WAITS).cast::>()).take()}.ok_or("thread-stale-wait")?; match unsafe{(&mut*os_k1020_table()).wake(old)}{Err((os_k1020_thread::Error::Stale,wait))=>{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_WAITS).cast::>(),Some(wait));}os_k1020_refused();},_=>return Err("thread-stale-wait-admitted")} if unsafe{(&*os_k1020_table()).ready_context(os_k1020_id(0))}.is_ok(){return Err("thread-old-ready-admitted");}os_k1020_refused();os_k1020_capture(5); os_k1020_step(3,2)?;unsafe{(&mut*os_k1020_table()).cancel(second)}.map_err(|_|"thread-cancel-second")?;os_k1020_reap(3)?; let third=os_k1020_create(4)?;if third.slot()!=second.slot()||third.generation()!=3{return Err("thread-third-generation");} if unsafe{(&*os_k1020_table()).record(second)}.is_ok(){return Err("thread-old-id-admitted");}os_k1020_refused();os_k1020_capture(6); os_k1020_step(4,3)?;os_k1020_wake(2)?;os_k1020_step(2,3)?; os_k1020_reap(4)?;os_k1020_reap(2)?; unsafe{(&mut*os_k1020_table()).cancel(os_k1020_id(1))}.map_err(|_|"thread-cancel-ready")?;os_k1020_reap(1)?; if unsafe{(&*os_k1020_table()).snapshot()}.live!=0||os_k1020_read(22)!=7||os_k1020_read(24)!=5 || os_k1020_read(23)!=17{return Err("thread-final-accounting");} os_k1020_capture(8);os_k1020_word(17,0); unsafe{asm!("msr VBAR_EL1, {vbar}","isb",vbar=in(reg)vbar,options(nomem,nostack,preserves_flags));} let backend=installation.retire(); os_k940_inventory(backend.memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1020_AUTHORITY_AFTER).cast()); if !os_k920_inventory_equal(core::ptr::addr_of!(OS_K1020_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K1020_AUTHORITY_AFTER).cast()) || os_k940_registers()!=(root,tcr,vbar,daif)||unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}!=pre {return Err("thread-full-authority-return");} os_k1020_word(68,mode as u64);os_k1020_diagnostics(3);os_k1020_capture(9); write_line("TURKOS/OS/K1020: PASS SLOTS=3 STACKS=5 ENTRIES=7 REFUSALS=17 OWNERS=RESTORED"); for line in ["TURKOS/OS/K1005: PASS DEVELOPMENT-GUEST-PARTIAL","TURKOS/OS/K1010: PASS DEVELOPMENT-GUEST-PARTIAL", "TURKOS/OS/K1015: PASS DEVELOPMENT-GUEST-PARTIAL","TURKOS/OS/K1020: PASS DEVELOPMENT-GUEST-PARTIAL"]{write_line(line);}Ok(()) } #[inline(never)]fn os_k1020_retain_configuration(){ let bootstrap=os_k1020_fatal_bootstrap_entry as *const () as u64; unsafe{asm!("ldrb wzr, [{p}]",p=in(reg)bootstrap,options(nostack,preserves_flags,readonly));} unsafe extern "C"{static __os_k1020_vectors:u8;fn __os_k1020_run_on_stack(sp:u64,entry:u64,argument:u64);} for address in [core::ptr::addr_of!(OS_K1020_TABLE) as u64,core::ptr::addr_of!(OS_K1020_EVIDENCE) as u64, core::ptr::addr_of!(OS_K1020_SPARES) as u64,core::ptr::addr_of!(OS_K1020_IDS) as u64,core::ptr::addr_of!(OS_K1020_WAITS) as u64, core::ptr::addr_of!(OS_K1020_SLOT_RECORDS) as u64,core::ptr::addr_of!(OS_K1020_IDENTITIES) as u64, core::ptr::addr_of!(OS_K1020_AUTHORITY_BEFORE) as u64,core::ptr::addr_of!(OS_K1020_AUTHORITY_LIVE) as u64, core::ptr::addr_of!(OS_K1020_AUTHORITY_AFTER) as u64,core::ptr::addr_of!(OS_K1020_OWNED) as u64, core::ptr::addr_of!(OS_K1020_LEASE_GENERATIONS) as u64,core::ptr::addr_of!(OS_K1020_TRACES) as u64, core::ptr::addr_of!(OS_K1020_KERNEL_TABLES) as u64, core::ptr::addr_of!(OS_K1020_EVENTS) as u64,core::ptr::addr_of!(OS_K1020_EVENT_COUNT) as u64, core::ptr::addr_of!(OS_K1020_NONCE) as u64,core::ptr::addr_of!(OS_K1020_CAPTURE_GATE) as u64, core::ptr::addr_of!(OS_K1020_KERNEL_FRAME) as u64,core::ptr::addr_of!(OS_K1020_IRQ_SEEN) as u64, core::ptr::addr_of!(OS_K1020_DIAGNOSTICS) as u64,core::ptr::addr_of!(OS_K1020_FATAL_FRAME) as u64, core::ptr::addr_of!(OS_K1020_FATAL) as u64,core::ptr::addr_of!(__os_k1020_vectors) as u64, core::ptr::addr_of!(OS_K1020_FATAL_OWNED) as u64,core::ptr::addr_of!(OS_K1020_FATAL_LEASE_GENERATIONS) as u64, core::ptr::addr_of!(OS_K1020_FATAL_AUTHORITY_BEFORE) as u64,core::ptr::addr_of!(OS_K1020_FATAL_AUTHORITY_AFTER) as u64, core::ptr::addr_of!(OS_K1020_FATAL_GATE) as u64, __os_k1020_run_on_stack as *const () as u64,os_k1020_thread_step as *const () as u64, os_k1020_fatal_entry as *const () as u64, __os_k1020_threads_identity as *const () as u64] {unsafe{asm!("ldrb wzr, [{address}]",address=in(reg)address,options(nostack,preserves_flags));}} } static mut OS_K1020_FATAL_TABLE: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K1020_FATAL_ID: Option = None; static mut OS_K1020_FATAL_FAILED_STACK: Option = None; static mut OS_K1020_FATAL_FAILED_BACKEND: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K1020_FATAL_ARGS: [u64;3] = [0;3]; #[unsafe(no_mangle)] static mut OS_K1020_FATAL_OWNED: [u64;5] = [0;5]; #[unsafe(no_mangle)] static mut OS_K1020_FATAL_LEASE_GENERATIONS: [u64;5] = [0;5]; #[unsafe(no_mangle)] static mut OS_K1020_FATAL_AUTHORITY_BEFORE: [u64;328] = [0;328]; #[unsafe(no_mangle)] static mut OS_K1020_FATAL_AUTHORITY_AFTER: [u64;328] = [0;328]; #[unsafe(no_mangle)] static mut OS_K1020_FATAL_GATE: u8 = 0; fn os_k1020_fatal_word(index:usize,value:u64){unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_FATAL).cast::().add(index),value);}} fn os_k1020_fatal_read(index:usize)->u64{unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_FATAL).cast::().add(index))}} fn os_k1020_fixture(boot:&BootData)->Result{ let blob=unsafe{core::slice::from_raw_parts(boot.dtb_address as *const u8,boot.dtb_size)}; let mode=turkos_kernel::runtime::platform::chosen_u32(blob,unsafe{&*core::ptr::addr_of!(OS_K700_WORKSPACE)},b"turkos,threads-mode") .map_err(|_|"thread-fixture")?.unwrap_or(0); if mode>1||mode==1&&cfg!(feature="os-runtime-product"){return Err("thread-fixture-mode");}Ok(mode) } fn os_k1020_fatal_capture(stage:u64){os_k1020_fatal_word(1,stage);unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_FATAL_GATE),0);} write_str("TURKOS/OS/K1020: FATAL CAPTURE STAGE=");write_hex(stage as usize);write_line(""); while unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_FATAL_GATE))}!=1{core::hint::spin_loop();} unsafe{asm!("dsb ish","isb",options(nostack,preserves_flags));} } #[unsafe(no_mangle)] extern "C" fn os_k1020_fatal_entry(guard:u64){ let pointer=core::ptr::addr_of_mut!(OS_K1020_FATAL_TABLE).cast::>(); let id=unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_FATAL_ID))}.unwrap_or_else(||park()); unsafe{(&mut*pointer).activate(id)}.unwrap_or_else(|_|park()); let record=unsafe{(&*pointer).record(id)}.unwrap_or_else(|_|park());let e=os_k1020_execution(); let(root,tcr,vbar,daif)=os_k940_registers();let spsel:u64;unsafe{asm!("mrs {s}, SPSel",s=out(reg)spsel,options(nomem,nostack,preserves_flags));} if guard!=record.layout.guard()||record.state!=os_k1020_thread::State::Running||e.in_irq||!e.irq_masked{park();} for(i,value)in[(5,e.cpu as u64),(6,e.sp),(9,root),(10,tcr),(11,vbar),(12,daif),(13,id.table()),(14,id.slot() as u64), (15,id.generation()),(16,record.state as u64),(17,unsafe{(&*pointer).snapshot()}.pages),(18,1),(19,spsel), (20,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))})]{os_k1020_fatal_word(i,value);} os_k1020_fatal_capture(0); unsafe extern "C"{fn __os_k1020_guard_fault_instruction(target:u64)->u64;} unsafe{__os_k1020_guard_fault_instruction(guard);} park(); } fn os_k1020_fatal_dispatch(frame:&os_k920_exception::Frame,address:u64)->!{ if os_k1020_fatal_read(0)!=0x4b31303230464154 {write_line("TURKOS/OS/THREADS: UNEXPECTED-FATAL RETAINED-FRAME");park();} unsafe extern "C"{fn __os_k1020_guard_fault_instruction(target:u64)->u64;} if frame.elr!=__os_k1020_guard_fault_instruction as *const () as u64||frame.far!=os_k1020_fatal_read(2) || frame.esr!=0x96000007||frame.current_el!=4||frame.vector_slot!=if frame.spsr&31==5{4}else{0} || ![4,5].contains(&(frame.spsr&31)) || frame.canary!=os_k920_exception::FRAME_CANARY||frame.padding!=0||address%16!=0 {write_line("TURKOS/OS/THREADS: REJECTED FATAL-INGRESS");park();} let table=unsafe{&*core::ptr::addr_of!(OS_K1020_FATAL_TABLE).cast::>()}; table.os_k1020_observe_backend(|backend|{ os_k1020_fatal_word(27,backend.memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64); os_k1020_fatal_word(28,core::mem::size_of::() as u64); os_k940_inventory(backend.memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1020_FATAL_AUTHORITY_AFTER).cast()); }); if !os_k920_inventory_equal(core::ptr::addr_of!(OS_K1020_FATAL_AUTHORITY_BEFORE).cast(),core::ptr::addr_of!(OS_K1020_FATAL_AUTHORITY_AFTER).cast()){park();} let id=unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_FATAL_ID))}.unwrap_or_else(||park()); if table.record(id).unwrap_or_else(|_|park()).state!=os_k1020_thread::State::Running {park();} os_k1020_fatal_word(22,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))} as u64); os_k1020_fatal_word(23,1);os_k1020_fatal_word(24,os_k1020_thread::LOW_CANARY);os_k1020_fatal_word(25,os_k1020_thread::HIGH_CANARY); os_k1020_fatal_capture(1);write_line("TURKOS/OS/THREADS: GUARD-FAULT RETAINED-STACK NO-RESTORE");park(); } #[inline(never)] fn os_k1020_fatal_if_requested(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData){ let mode=os_k1020_fixture(boot).unwrap_or_else(|_|park()); if mode==0{return;}if mode!=1||cfg!(feature="os-runtime-product"){park();} let e=os_k1020_execution();let(root,_,_,_)=os_k940_registers(); let guard=os_k1020_read(20);let low=guard+4096;let top=low+65536-16; if !e.irq_masked||e.in_irq||e.cpu!=0||root!=tables.root||e.sp>=guard&&e.sp<=top || !tables.task_stack_guard_unmapped(guard) || unsafe{read_volatile(low as *const u64)}!=OS_K920_LOW_CANARY || unsafe{read_volatile((top+8) as *const u64)}!=OS_K920_HIGH_CANARY {park();} // The predecessor has returned from this Image-owned guarded64KiB stack. // Constructor and central batch validation are sequentially loaned that // stack; their compiled frames do not fit the post-K640 ordinary boot SP. // The real five-page thread stack remains a separately owned allocation. os_k1020_fatal_word(29,e.sp);os_k1020_fatal_word(31,top); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1020_FATAL_ARGS),[ runtime as *mut OsK660Runtime as u64,tables as *const mmu::Tables as u64,boot as *const BootData as u64]);} unsafe extern "C"{fn __os_k1020_run_on_stack(sp:u64,entry:u64,argument:u64);} unsafe{__os_k1020_run_on_stack(top,os_k1020_fatal_bootstrap_entry as *const () as u64, core::ptr::addr_of!(OS_K1020_FATAL_ARGS) as u64);} park(); } #[inline(never)] #[unsafe(no_mangle)] extern "C" fn os_k1020_fatal_bootstrap_entry(argument:u64){ if argument!=core::ptr::addr_of!(OS_K1020_FATAL_ARGS) as u64 {park();} let e=os_k1020_execution();let low=os_k1020_read(20)+4096;let top=os_k1020_fatal_read(31); if !e.irq_masked||e.in_irq||e.cpu!=0||e.sp<=low+16384||e.sp>top {park();} os_k1020_fatal_word(30,e.sp); let arguments=unsafe{read_volatile(core::ptr::addr_of!(OS_K1020_FATAL_ARGS))}; unsafe{os_k1020_fatal_body(&mut*(arguments[0] as *mut OsK660Runtime), &*(arguments[1] as *const mmu::Tables),&*(arguments[2] as *const BootData));} park(); } #[inline(never)] fn os_k1020_fatal_body(runtime:&mut OsK660Runtime,tables:&mmu::Tables,_boot:&BootData){ let(_,memory,_,_)=runtime.running().unwrap_or_else(|_|park()); let start=os_k1020_read(40);let end=start+5*4096; if !tables.os_k1020_stack_candidate(start,end){park();} let leases=memory.allocator.os_authority_mut().allocate_many_at(start,5).unwrap_or_else(|_|park()); for(i,lease)in leases[..5].iter().enumerate(){let lease=lease.as_ref().unwrap_or_else(||park());unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1020_FATAL_OWNED).cast::().add(i),lease.address()); write_volatile(core::ptr::addr_of_mut!(OS_K1020_FATAL_LEASE_GENERATIONS).cast::().add(i),lease.generation());}} let stack=OsK1020Stack{ordinal:0,start,leases,guard:None,initial_observation:core::cell::Cell::new(false)};let backend=OsK1020Backend{memory,tables,trace:false}; let mut table=match OsK1020Table::new(backend){Ok(table)=>table,Err((_error,backend))=>{ unsafe{core::ptr::addr_of_mut!(OS_K1020_FATAL_FAILED_BACKEND).cast::>().write(backend); core::ptr::addr_of_mut!(OS_K1020_FATAL_FAILED_STACK).write(Some(stack));}park();}}; let id=match table.create(stack,os_k1020_fatal_entry as *const () as u64,start){Ok(id)=>id,Err((_error,stack))=>{ unsafe{core::ptr::addr_of_mut!(OS_K1020_FATAL_TABLE).cast::>().write(table); core::ptr::addr_of_mut!(OS_K1020_FATAL_FAILED_STACK).write(Some(stack));}park();}}; let context=table.ready_context(id).unwrap_or_else(|_|park()); table.os_k1020_observe_backend(|backend|{ os_k940_inventory(backend.memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1020_FATAL_AUTHORITY_BEFORE).cast()); os_k1020_fatal_word(27,backend.memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64); os_k1020_fatal_word(28,core::mem::size_of::() as u64); }); let slot=tables.metadata_stack_guard_descriptor_address(start).unwrap_or_else(||park()); let original=table.os_k1020_observe_stack(id,|stack|stack.guard.as_ref().unwrap_or_else(||park()).original_descriptor()).unwrap_or_else(|_|park()); unsafe extern "C"{static __os_k1020_vectors:u8;fn __os_k1020_run_on_stack(sp:u64,entry:u64,argument:u64);} let vector=core::ptr::addr_of!(__os_k1020_vectors) as u64; for(i,value)in[(0,0x4b31303230464154),(2,start),(3,end),(4,tables.root),(7,slot),(8,original),(26,vector)]{os_k1020_fatal_word(i,value);} unsafe{core::ptr::addr_of_mut!(OS_K1020_FATAL_TABLE).cast::>().write(table); write_volatile(core::ptr::addr_of_mut!(OS_K1020_FATAL_ID),Some(id)); asm!("msr VBAR_EL1, {v}","isb",v=in(reg)vector,options(nomem,nostack,preserves_flags)); __os_k1020_run_on_stack(context.initial_sp,context.entry,context.argument);} park(); } global_asm!(r#" .section .text.os_k1020_guard_fault,"ax" .balign 16 .global __os_k1020_guard_fault_instruction .type __os_k1020_guard_fault_instruction, %function __os_k1020_guard_fault_instruction: ldr x0, [x0] ret .size __os_k1020_guard_fault_instruction, .-__os_k1020_guard_fault_instruction "#); const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k1020_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); global_asm!(include_str!("os_k1020_stack.S")); const OS_K1040_PA_MASK: u64 = 0x0000_ffff_ffff_f000; /// Owns only the new root page. Kernel branches remain separately owned by the /// inherited Tables and are validated as EL0 inaccessible before sharing. struct OsK1040Root { lease: Option, bootstrap: u64, branches: [u64; 2], } impl OsK1040Root { fn address(&self) -> u64 { self.lease.as_ref().map_or(0, |l| l.address()) } fn generation(&self) -> u64 { self.lease.as_ref().map_or(0, |l| l.generation()) } fn validate(&self, memory: &OsK660Memory, tables: &mmu::Tables) -> Result<(), &'static str> { let lease = self.lease.as_ref().ok_or("context-root-token")?; if self.bootstrap != tables.root || lease.address() == tables.root || lease.address() % 4096 != 0 || lease.generation() == 0 { return Err("context-root-owner"); } memory .allocator .os_authority() .validate_lease(lease) .map_err(|_| "context-root-generation")?; os_k1040_kernel_branches(tables, self.branches)?; os_k1040_root_words(lease.address(), self.branches) } } fn os_k1040_shared_walk( tables: &mmu::Tables, address: u64, level: u8, visited: &mut [u64; 4], count: usize, ) -> Result<(), &'static str> { if !(1..=3).contains(&level) || count >= 4 || visited[..count].contains(&address) || !tables.tagged_frames().any(|frame| frame.address == address) { return Err("context-shared-kernel-owner"); } visited[count] = address; for index in 0..512 { let raw = unsafe { read_volatile((address as *const u64).add(index)) }; if raw == 0 { continue; } match raw & 3 { 3 if level < 3 => { if raw != (raw & OS_K1040_PA_MASK) | 3 { return Err("context-shared-kernel-table-flags"); } os_k1040_shared_walk( tables, raw & OS_K1040_PA_MASK, level + 1, visited, count + 1, )?; } 1 if level < 3 => { if raw & (1 << 6) != 0 { return Err("context-shared-kernel-user-block"); } } 3 if level == 3 => { if raw & (1 << 6) != 0 { return Err("context-shared-kernel-user-page"); } } _ => return Err("context-shared-kernel-descriptor"), } } Ok(()) } fn os_k1040_root_words(address: u64, branches: [u64; 2]) -> Result<(), &'static str> { if address == 0 || address & !OS_K1040_PA_MASK != 0 { return Err("context-root-shape-address"); } // The caller has already checked the live typed root owner (or the // immutable pin established from that owner before IRQ unmask). Check the // entire page before accepting any later observation or root selection. for index in 0..512 { let expected = branches.get(index).copied().unwrap_or(0); if unsafe { read_volatile((address as *const u64).add(index)) } != expected { return Err("context-root-shape"); } } Ok(()) } fn os_k1040_kernel_branches(tables: &mmu::Tables, branches: [u64; 2]) -> Result<(), &'static str> { // Establish the inherited root ownership before forming its physical // pointer. Child table ownership is established independently at each // recursive step, before its pointer is formed too. if !tables .tagged_frames() .any(|frame| frame.address == tables.root) { return Err("context-shared-root-owner"); } for (index, raw) in branches.into_iter().enumerate() { if unsafe { read_volatile((tables.root as *const u64).add(index)) } != raw { return Err("context-shared-root-changed"); } match raw & 3 { 0 if raw == 0 => {} 3 => { if raw != (raw & OS_K1040_PA_MASK) | 3 { return Err("context-shared-root-table-flags"); } os_k1040_shared_walk(tables, raw & OS_K1040_PA_MASK, 2, &mut [0; 4], 0)?; } 1 if raw & (1 << 6) == 0 => {} _ => return Err("context-shared-root-descriptor"), } } Ok(()) } fn os_k1040_create_root( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result { let (root, tcr, _, daif) = os_k940_registers(); if root != tables.root || tcr & 63 != 25 || daif & 0x80 == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { return Err("context-root-bootstrap-context"); } // Only kernel L1 branches0/1 are inherited; user branch2 is absent. if !tables.tagged_frames().any(|frame| frame.address == root) { return Err("context-shared-root-owner"); } let mut branches = [0u64; 2]; for (index, raw) in branches.iter_mut().enumerate() { *raw = unsafe { read_volatile((root as *const u64).add(index)) }; } os_k1040_kernel_branches(tables, branches)?; let address = memory .allocator .os_authority() .free_regions() .iter() .rev() .find_map(|region| { region .end .checked_sub(4096) .filter(|address| *address >= region.start) }) .ok_or("context-root-ENOSPC")?; let lease = memory .allocator .os_authority_mut() .allocate_at(address) .map_err(|_| "context-root-acquire")?; let owned = OsK1040Root { lease: Some(lease), bootstrap: root, branches, }; for index in 0..512 { unsafe { write_volatile((address as *mut u64).add(index), 0); } } for (index, raw) in branches.into_iter().enumerate() { unsafe { write_volatile((address as *mut u64).add(index), raw); } } unsafe { asm!("dsb ishst", options(nostack, preserves_flags)); } os_k1040_event(0x41, address, owned.generation(), root); Ok(owned) } fn os_k1040_select_root(root: u64) -> Result<(), &'static str> { let (_, tcr, _, daif) = os_k940_registers(); if root == 0 || root % 4096 != 0 || root & !OS_K1040_PA_MASK != 0 || tcr & 63 != 25 || daif & 0x80 == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { return Err("context-root-select-context"); } unsafe { asm!("dsb ishst", options(nostack, preserves_flags)); } os_k1040_event(0x43, root, 0, 0); unsafe { asm!("msr TTBR0_EL1, {root}",root=in(reg)root,options(nostack,preserves_flags)); } os_k1040_event(0x44, root, 0, 0); unsafe { asm!("tlbi vmalle1is", options(nostack, preserves_flags)); } os_k1040_event(0x45, root, 0, 0); unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } os_k1040_event(0x46, root, 0, 0); unsafe { asm!("isb", options(nostack, preserves_flags)); } os_k1040_event(0x47, root, 0, 0); if os_k940_registers().0 != root { return Err("context-root-readback"); } Ok(()) } fn os_k1040_erase_inactive_root( root: &OsK1040Root, tables: &mmu::Tables, ) -> Result<(), &'static str> { let actual = os_k940_registers(); if actual.0 != tables.root || actual.3 & 0x80 == 0 || root.bootstrap != tables.root || root.address() == actual.0 || root.address() == 0 { return Err("context-root-active-free-refusal"); } // Bootstrap select completed a full TLBI before this operation. Erasure // alone returns no authority: StackBackend publishes one atomic six-lease // return only after its guard restoration and full stack erasure too. for index in 0..512 { unsafe { write_volatile((root.address() as *mut u64).add(index), 0); } } unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } if !(0..512) .all(|index| unsafe { read_volatile((root.address() as *const u64).add(index)) } == 0) { return Err("context-root-erasure"); } os_k1040_event(0x48, root.address(), root.generation(), 0); Ok(()) } use turkos_kernel::runtime::context_threads as os_k1040_thread; const OS_K1040_STACK_COUNT: usize = 2; const OS_K1040_STACK_PAGES: usize = 4; const OS_K1040_OWNED_PAGES: usize = OS_K1040_STACK_COUNT * (OS_K1040_STACK_PAGES + 2); struct OsK1040Stack { root: OsK1040Root, ordinal: usize, start: u64, leases: [Option; 8], guard: Option, initial_observation: core::cell::Cell, } impl OsK1040Stack { fn layout(&self) -> os_k1040_thread::StackLayout { os_k1040_thread::StackLayout { base: self.start + 4096, physical: self.start + 4096, bytes: OS_K1040_STACK_PAGES as u64 * 4096, } } } fn os_k1040_cpu_root_allowed(root: u64, tables: &mmu::Tables) -> bool { root==tables.root||(0..2).any(|slot|unsafe{read_volatile(core::ptr::addr_of!(OS_K1040_ROOT_ADDRESSES).cast::().add(slot))}==root&&root!=0&&unsafe{read_volatile(core::ptr::addr_of!(OS_K1040_ROOT_LIVE).cast::().add(slot))}==1) } struct OsK1040StackBackend<'a> { memory: &'a mut OsK660Memory, tables: &'a mmu::Tables, trace: bool, } fn os_k1040_execution() -> os_k1040_thread::Execution { let (cpu, daif, sp): (u64, u64, u64); unsafe { asm!("mrs {cpu}, MPIDR_EL1", "mrs {daif}, DAIF", "mov {sp}, sp", cpu=out(reg)cpu, daif=out(reg)daif, sp=out(reg)sp, options(nomem,nostack,preserves_flags)); } os_k1040_thread::Execution { cpu: (cpu & 0xff) as u32, irq_masked: daif & 0x80 != 0, in_irq: unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0, sp, } } fn os_k1040_allocate_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ordinal: usize, root: OsK1040Root, ) -> Result { if ordinal >= 2 || root.validate(memory, tables).is_err() { return Err(("context-stack-root-owner", root)); } let authority = memory.allocator.os_authority(); let mut selected = None; let mut checked = 0; for region in authority.free_regions() { let mut address = region.start; while address .checked_add(5 * 4096) .is_some_and(|end| end <= region.end) && checked < 512 { checked += 1; if tables.os_k1020_stack_candidate(address, address + 5 * 4096) { selected = Some(address); break; } let Some(next) = address.checked_add(4096) else { return Err(("context-stack-candidate-overflow", root)); }; address = next; } if selected.is_some() { break; } } let Some(start) = selected else { return Err(("context-stack-no-owned-L3-span", root)); }; let leases = match memory .allocator .os_authority_mut() .allocate_many_at(start, 5) { Ok(leases) => leases, Err(_) => return Err(("context-stack-acquire", root)), }; // The five live stack leases are immediately inside this actual owner; // failure cannot discard a prefix or the independently retained root. let stack = OsK1040Stack { root, ordinal, start, leases, guard: None, initial_observation: core::cell::Cell::new(false), }; for offset in 0..5 * 4096 / 8 { unsafe { write_volatile((start as *mut u64).add(offset), 0xa5a5a5a5a5a5a5a5); } } for index in 0..6 { let lease = if index < 5 { stack.leases[index].as_ref().unwrap_or_else(|| park()) } else { stack.root.lease.as_ref().unwrap_or_else(|| park()) }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_OWNED) .cast::() .add(ordinal * 6 + index), lease.address(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_LEASE_GENERATIONS) .cast::() .add(ordinal * 6 + index), lease.generation(), ); } os_k1040_event(1, ordinal as u64, lease.address(), lease.generation()); } Ok(stack) } impl OsK1040StackBackend<'_> { fn validate(&self, stack: &OsK1040Stack) -> Result<(), os_k1040_thread::Error> { let e = os_k1040_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || !os_k1040_cpu_root_allowed(os_k940_registers().0, self.tables) { return Err(os_k1040_thread::Error::Context); } if stack.ordinal >= OS_K1040_STACK_COUNT || stack.start % 4096 != 0 || stack.leases[OS_K1040_STACK_PAGES + 1..] .iter() .any(Option::is_some) { return Err(os_k1040_thread::Error::Layout); } for index in 0..=OS_K1040_STACK_PAGES { let lease = stack.leases[index] .as_ref() .ok_or(os_k1040_thread::Error::Backend)?; if lease.address() != stack.start + index as u64 * 4096 { return Err(os_k1040_thread::Error::Layout); } self.memory .allocator .os_authority() .validate_lease(lease) .map_err(|_| os_k1040_thread::Error::Backend)?; } stack .root .validate(self.memory, self.tables) .map_err(|_| os_k1040_thread::Error::Backend)?; Ok(()) } } unsafe impl os_k1040_thread::Backend for OsK1040StackBackend<'_> { type Stack = OsK1040Stack; fn execution(&self) -> os_k1040_thread::Execution { os_k1040_execution() } fn layout( &self, stack: &OsK1040Stack, ) -> Result { self.validate(stack)?; Ok(stack.layout()) } fn entry_executable(&self, entry: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } entry.is_multiple_of(4) && entry >= core::ptr::addr_of!(_text_start) as u64 && entry < core::ptr::addr_of!(_text_end) as u64 } fn prepare(&mut self, stack: &mut OsK1040Stack) -> Result<(), os_k1040_thread::Error> { self.validate(stack)?; let layout = stack.layout(); let e = os_k1040_execution(); if e.sp >= stack.start && e.sp <= layout.base + layout.bytes || stack.guard.is_some() || os_k940_registers().0 != self.tables.root { return Err(os_k1040_thread::Error::ActiveStack); } let guard = unsafe { self.tables .os_k1020_acquire_stack_guard(stack.start, layout.base + layout.bytes) } .map_err(|_| os_k1040_thread::Error::Guard)?; stack.guard = Some(guard); for offset in 0..layout.bytes / 8 { unsafe { write_volatile((layout.base as *mut u64).add(offset as usize), 0); } } unsafe { write_volatile(layout.base as *mut u64, os_k1040_thread::LOW_CANARY); write_volatile( (layout.base + layout.bytes - 8) as *mut u64, os_k1040_thread::HIGH_CANARY, ); asm!("dsb ish", options(nostack, preserves_flags)); } stack.initial_observation.set(true); if self.trace { os_k1040_event(2, stack.ordinal as u64, stack.start, layout.bytes); } Ok(()) } fn observe( &self, stack: &OsK1040Stack, ) -> Result { self.validate(stack)?; let layout = stack.layout(); let token = stack.guard.as_ref().ok_or(os_k1040_thread::Error::Guard)?; let descriptor = self .tables .os_k1020_stack_guard_descriptor(token) .map_err(|_| os_k1040_thread::Error::Guard)?; if token.guard() != stack.start || token.end() != layout.base + layout.bytes { return Err(os_k1040_thread::Error::Layout); } // Initial full erasure is observed once before publication. Later // selection/exit checks read the actual guard and canaries without // walking the active or previously used Rust stack contents. let zeroed=stack.initial_observation.get()&&(1..layout.bytes/8-1).all(|offset|unsafe {read_volatile((layout.base as *const u64).add(offset as usize))}==0); if zeroed { stack.initial_observation.set(false); } Ok(os_k1040_thread::StackObservation { layout, guard_descriptor: descriptor, low_canary: unsafe { read_volatile(layout.base as *const u64) }, high_canary: unsafe { read_volatile((layout.base + layout.bytes - 8) as *const u64) }, initially_zeroed: zeroed, }) } fn release( &mut self, mut stack: OsK1040Stack, ) -> Result<(), (os_k1040_thread::Error, OsK1040Stack)> { if let Err(error) = self.validate(&stack) { return Err((error, stack)); } let layout = stack.layout(); let e = os_k1040_execution(); if e.sp >= stack.start && e.sp <= layout.base + layout.bytes { return Err((os_k1040_thread::Error::ActiveStack, stack)); } if os_k1040_erase_inactive_root(&stack.root, self.tables).is_err() { return Err((os_k1040_thread::Error::Backend, stack)); } let token = match stack.guard.take() { Some(token) => token, None => return Err((os_k1040_thread::Error::Guard, stack)), }; if let Err((_error, token)) = unsafe { self.tables.os_k1020_restore_stack_guard(token) } { stack.guard = Some(token); return Err((os_k1040_thread::Error::Guard, stack)); } for offset in 0..(OS_K1040_STACK_PAGES + 1) * 4096 / 8 { unsafe { write_volatile((stack.start as *mut u64).add(offset), 0); } } unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } if !(0..(OS_K1040_STACK_PAGES + 1) * 4096 / 8) .all(|offset| unsafe { read_volatile((stack.start as *const u64).add(offset)) } == 0) { return Err((os_k1040_thread::Error::Backend, stack)); } let refs = core::array::from_fn::<_, 6, _>(|index| { if index < 5 { stack.leases[index].as_ref().unwrap() } else { stack.root.lease.as_ref().unwrap() } }); if self .memory .allocator .os_authority_mut() .release_many_leases(&refs) .is_err() { return Err((os_k1040_thread::Error::Backend, stack)); } if self.trace { os_k1040_event(3, stack.ordinal as u64, stack.start, 5); } for lease in &mut stack.leases { *lease = None; } stack.root.lease = None; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_ROOT_LIVE) .cast::() .add(stack.ordinal), 0, ); } Ok(()) } } use turkos_kernel::runtime::context_switch as os_k1040_context; type OsK1040Table<'a> = os_k1040_thread::Table, 2>; /// Private nonclone pin for one real Table stack/root owner. The actual six /// central leases stay in Table::Stack until this pin is consumed by release. struct OsK1040Binding { id: os_k1040_thread::ThreadId, ordinal: usize, root: u64, root_generation: u64, branches: [u64; 2], } struct OsK1040Prepared { source: Option, target: os_k1040_thread::ThreadId, nonce: u64, departed: bool, } struct OsK1040ReturnPrepared { source: os_k1040_thread::ThreadId, bootstrap_root: u64, bootstrap: os_k1040_context::Cooperative, address: u64, nonce: u64, } struct OsK1040IdlePrepared { epoch: u64, wait: Option, } struct OsK1040ContextBackend<'a> { table: OsK1040Table<'a>, bootstrap: u64, current: Option, pins: [bool; 2], records: [Option; 2], observations: [Option; 2], guards: [u64; 2], sleep_wait: Option, sequence: u64, } impl OsK1040ContextBackend<'_> { fn validate_stack_owner( &self, id: os_k1040_thread::ThreadId, ) -> Result<(), os_k1040_context::Error> { if os_k1040_execution().in_irq { return Err(os_k1040_context::Error::Context); } self.table .os_k1020_observe_backend(|backend| { self.table .os_k1020_observe_stack(id, |stack| backend.validate(stack)) }) .map_err(|_| os_k1040_context::Error::Stack)? .map_err(|_| os_k1040_context::Error::Backend) } fn valid_pin(&self, b: &OsK1040Binding) -> Result<(), os_k1040_context::Error> { if b.ordinal >= 2 || !self.pins[b.ordinal] || b.root == 0 || b.root_generation == 0 { return Err(os_k1040_context::Error::Stale); } let r = self.records[b.ordinal].ok_or(os_k1040_context::Error::Stale)?; if r.id != b.id { return Err(os_k1040_context::Error::Stale); } let root = unsafe { read_volatile( core::ptr::addr_of!(OS_K1040_ROOT_ADDRESSES) .cast::() .add(b.ordinal), ) }; let generation = unsafe { read_volatile( core::ptr::addr_of!(OS_K1040_ROOT_GENERATIONS) .cast::() .add(b.ordinal), ) }; if root != b.root || generation != b.root_generation || unsafe { read_volatile( core::ptr::addr_of!(OS_K1040_ROOT_LIVE) .cast::() .add(b.ordinal), ) } != 1 { return Err(os_k1040_context::Error::Stale); } // Immutable IRQ pin: the typed central/root/stack owner was validated // by Table in main context before this pin was published. IRQ reads // only these bounded physical words and its private pin, never Table // or the allocator. Main observations additionally revalidate Table. os_k1040_root_words(b.root, b.branches).map_err(|_| os_k1040_context::Error::Backend)?; if !os_k1040_execution().in_irq { self.validate_stack_owner(b.id)?; } Ok(()) } fn refresh(&mut self) -> Result<(), os_k1040_context::Error> { if os_k1040_execution().in_irq { return Err(os_k1040_context::Error::Context); } for ordinal in 0..2 { if let Some(previous) = self.records[ordinal] { if !self.pins[ordinal] { continue; } let record = self .table .record(previous.id) .map_err(|_| os_k1040_context::Error::State)?; self.validate_stack_owner(previous.id)?; let observation = self .table .os_k1020_observe_stack(previous.id, |stack| { let guard = stack .guard .as_ref() .map_or(0, |guard| guard.descriptor_address()); ( stack.layout(), guard, stack.root.address(), stack.root.generation(), ) }) .map_err(|_| os_k1040_context::Error::Stack)?; if observation.1 == 0 { return Err(os_k1040_context::Error::Guard); } self.records[ordinal] = Some(record); self.guards[ordinal] = observation.1; self.observations[ordinal] = Some(os_k1040_thread::StackObservation { layout: observation.0, guard_descriptor: unsafe { read_volatile(observation.1 as *const u64) }, low_canary: unsafe { read_volatile(observation.0.base as *const u64) }, high_canary: unsafe { read_volatile((observation.0.base + observation.0.bytes - 8) as *const u64) }, initially_zeroed: false, }); } } Ok(()) } fn pin( &mut self, id: os_k1040_thread::ThreadId, ordinal: usize, ) -> Result { if ordinal >= 2 || self.pins[ordinal] || os_k1040_execution().in_irq { return Err(os_k1040_context::Error::State); } let record = self .table .record(id) .map_err(|_| os_k1040_context::Error::State)?; self.validate_stack_owner(id)?; let (root, generation, branches) = self .table .os_k1020_observe_stack(id, |stack| { ( stack.root.address(), stack.root.generation(), stack.root.branches, ) }) .map_err(|_| os_k1040_context::Error::Stack)?; let before_records = self.records; let before_observations = self.observations; let before_guards = self.guards; self.records[ordinal] = Some(record); self.pins[ordinal] = true; if let Err(error) = self.refresh() { self.pins[ordinal] = false; self.records = before_records; self.observations = before_observations; self.guards = before_guards; return Err(error); } Ok(OsK1040Binding { id, ordinal, root, root_generation: generation, branches, }) } fn next_prepared(&mut self) -> Result { let next = self .sequence .checked_add(1) .ok_or(os_k1040_context::Error::Exhausted)?; self.sequence = next; Ok(next) } } unsafe impl os_k1040_context::Backend for OsK1040ContextBackend<'_> { type Binding = OsK1040Binding; type Prepared = OsK1040Prepared; type IdlePrepared = OsK1040IdlePrepared; fn execution(&self) -> os_k1040_context::Execution { let e = os_k1040_execution(); let (root, _, _, daif) = os_k940_registers(); let cpacr: u64; unsafe { asm!("mrs {fp}, CPACR_EL1",fp=out(reg)cpacr,options(nomem,nostack,preserves_flags)); } os_k1040_context::Execution { cpu: e.cpu, daif, in_irq: e.in_irq, sp: e.sp, root, thread: self.current, fp_enabled: cpacr & (3 << 20) == 3 << 20, critical_depth: unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_CRITICAL)) } as u32, safe_point: unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_SAFE_POINT)) } == 1, tco: if e.in_irq { unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_IRQ_TCO)) } } else { 0 }, } } fn record( &self, b: &Self::Binding, ) -> Result { self.valid_pin(b)?; // IRQ uses the last real main-only Table transition, an immutable pin // and actual guard/canaries. It never borrows Table/Memory/authority. if os_k1040_execution().in_irq { return self.records[b.ordinal].ok_or(os_k1040_context::Error::Stale); } self.table .record(b.id) .map_err(|_| os_k1040_context::Error::State) } fn stack( &self, b: &Self::Binding, ) -> Result { self.valid_pin(b)?; let cached = self.observations[b.ordinal].ok_or(os_k1040_context::Error::Stack)?; if !os_k1040_execution().in_irq { self.table .record(b.id) .map_err(|_| os_k1040_context::Error::State)?; self.table .os_k1020_observe_stack(b.id, |stack| { stack.root.address() == b.root && stack.root.generation() == b.root_generation }) .map_err(|_| os_k1040_context::Error::Stack)? .then_some(()) .ok_or(os_k1040_context::Error::Stale)?; } Ok(os_k1040_thread::StackObservation { guard_descriptor: unsafe { read_volatile(self.guards[b.ordinal] as *const u64) }, low_canary: unsafe { read_volatile(cached.layout.base as *const u64) }, high_canary: unsafe { read_volatile((cached.layout.base + cached.layout.bytes - 8) as *const u64) }, ..cached }) } fn root(&self, b: &Self::Binding) -> Result { self.valid_pin(b)?; if !os_k1040_execution().in_irq { self.table .record(b.id) .map_err(|_| os_k1040_context::Error::State)?; } Ok(b.root) } fn executable(&self, pc: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } pc % 4 == 0 && pc >= core::ptr::addr_of!(_text_start) as u64 && pc < core::ptr::addr_of!(_text_end) as u64 } fn initial_return_pc(&self, _: &Self::Binding) -> Result { unsafe extern "C" { fn __os_k1040_bad_return(); } Ok(__os_k1040_bad_return as *const () as u64) } fn capture_matches(&self, b: &Self::Binding, c: &os_k1040_context::Capture) -> bool { if self.valid_pin(b).is_err() { return false; } let Some(observation) = self.observations[b.ordinal] else { return false; }; let (address, episode, bytes) = match c.image { os_k1040_context::Image::Cooperative(_) => ( unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LIVE_CAPTURE_ADDRESS)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_CAPTURE_EPISODE)) }, 640u64, ), os_k1040_context::Image::Timer(_) => ( unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_IRQ_CAPTURE_ADDRESS)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_IRQ_CAPTURE_EPISODE)) }, 848u64, ), _ => return false, }; if c.address != address || c.episode != episode || episode == 0 || address % 16 != 0 || address < observation.layout.usable_bottom() || !address .checked_add(bytes) .is_some_and(|end| end <= observation.layout.usable_top()) { return false; } match c.image { os_k1040_context::Image::Cooperative(f) => { (unsafe { read_volatile(address as *const os_k1040_context::Cooperative) }) == f } os_k1040_context::Image::Timer(f) => { (unsafe { read_volatile(address as *const os_k920_exception::Frame) }) == f } _ => false, } } fn registers(&self) -> os_k1040_context::Registers { let f = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LANDING_FRAME)) }; os_k1040_context::Registers { sp: f.sp, x19_x30: f.x19_x30, q: f.q, fpcr: f.fpcr, fpsr: f.fpsr, pc: unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LANDING_PC)) }, argument: unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LANDING_ARGUMENT)) }, timer: None, } } fn prepare( &mut self, source: Option<&Self::Binding>, target: &Self::Binding, _: &os_k1040_context::Image, _: os_k1040_context::Disposition, ) -> Result { self.valid_pin(target)?; if let Some(source) = source { self.valid_pin(source)?; } Ok(OsK1040Prepared { source: source.map(|s| s.id), target: target.id, nonce: self.next_prepared()?, departed: false, }) } fn commit( &mut self, _: &mut Self::Prepared, _: Option<&Self::Binding>, _: &Self::Binding, _: &os_k1040_context::Image, _: os_k1040_context::Disposition, ) -> Result<(), os_k1040_context::Failure> { // A synchronous bank loan must never span a suspended continuation. Err(os_k1040_context::Failure { error: os_k1040_context::Error::Context, phase: os_k1040_context::FailurePhase::BeforeSwitch, }) } fn finish( &mut self, p: Self::Prepared, ) -> Result<(), (os_k1040_context::Error, Self::Prepared)> { if !p.departed { return Err((os_k1040_context::Error::State, p)); } Ok(()) } fn abort( &mut self, p: Self::Prepared, ) -> Result<(), (os_k1040_context::Error, Self::Prepared)> { if p.departed { return Err((os_k1040_context::Error::State, p)); } Ok(()) } fn release( &mut self, b: Self::Binding, ) -> Result<(), (os_k1040_context::Error, Self::Binding)> { if let Err(e) = self.valid_pin(&b) { return Err((e, b)); } let record = match self.table.record(b.id) { Ok(r) => r, Err(_) => return Err((os_k1040_context::Error::State, b)), }; if record.state != os_k1040_thread::State::Dead || os_k940_registers().0 != self.bootstrap { return Err((os_k1040_context::Error::Active, b)); } if self.table.reap(b.id).is_err() { return Err((os_k1040_context::Error::Backend, b)); } self.pins[b.ordinal] = false; self.records[b.ordinal] = None; self.observations[b.ordinal] = None; self.guards[b.ordinal] = 0; Ok(()) } fn ready_empty(&self) -> bool { let s = self.table.snapshot(); s.ready == 0 && s.running == 0 } fn wake_epoch(&self) -> u64 { unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_WAKE_EPOCH)) } } fn idle_prepare(&mut self, epoch: u64) -> Result { let wait = self .sleep_wait .take() .ok_or(os_k1040_context::Error::Wake)?; Ok(OsK1040IdlePrepared { epoch, wait: Some(wait), }) } fn idle_commit( &mut self, p: &mut Self::IdlePrepared, epoch: u64, ) -> Result { os_k1040_idle_atomic(self, p, epoch) } fn idle_finish( &mut self, p: Self::IdlePrepared, ) -> Result<(), (os_k1040_context::Error, Self::IdlePrepared)> { if p.wait.is_some() { return Err((os_k1040_context::Error::Wake, p)); } Ok(()) } fn idle_abort( &mut self, mut p: Self::IdlePrepared, ) -> Result<(), (os_k1040_context::Error, Self::IdlePrepared)> { if self.sleep_wait.is_some() { return Err((os_k1040_context::Error::Wake, p)); } self.sleep_wait = p.wait.take(); Ok(()) } } unsafe impl os_k1040_context::HandoffBackend for OsK1040ContextBackend<'_> { fn depart( &mut self, p: &mut Self::Prepared, source: Option<&Self::Binding>, nonce: u64, d: os_k1040_context::Disposition, ) -> Result<(), os_k1040_context::Failure> { let before = |error| os_k1040_context::Failure { error, phase: os_k1040_context::FailurePhase::BeforeSwitch, }; if p.source != source.map(|s| s.id) || p.departed || p.nonce == 0 || nonce == 0 { return Err(before(os_k1040_context::Error::Stale)); } self.validate_stack_owner(p.target).map_err(before)?; if let Some(source) = source { self.valid_pin(source).map_err(before)?; match d { os_k1040_context::Disposition::Ready => self .table .yield_current(source.id) .map_err(|_| before(os_k1040_context::Error::State))?, os_k1040_context::Disposition::Sleeping => { if self.sleep_wait.is_some() { return Err(before(os_k1040_context::Error::Wake)); } self.sleep_wait = Some( self.table .sleep_current(source.id) .map_err(|_| before(os_k1040_context::Error::State))?, ); } } } p.departed = true; self.current = None; if let Err(e) = self.refresh() { return Err(os_k1040_context::Failure { error: e, phase: os_k1040_context::FailurePhase::AfterSwitch, }); } let root = match self .table .os_k1020_observe_stack(p.target, |stack| stack.root.address()) { Ok(root) => root, Err(_) => { return Err(os_k1040_context::Failure { error: os_k1040_context::Error::Stack, phase: os_k1040_context::FailurePhase::AfterSwitch, }); } }; if os_k1040_select_root(root).is_err() { return Err(os_k1040_context::Failure { error: os_k1040_context::Error::Backend, phase: os_k1040_context::FailurePhase::AfterSwitch, }); } Ok(()) } fn arrive( &mut self, p: &mut Self::Prepared, target: &Self::Binding, _: &os_k1040_context::Image, nonce: u64, ) -> Result<(), os_k1040_context::Error> { self.valid_pin(target)?; if !p.departed || p.target != target.id || nonce != unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_DEPARTED_NONCE)) } || os_k940_registers().0 != target.root { return Err(os_k1040_context::Error::Capture); } self.table .activate(target.id) .map_err(|_| os_k1040_context::Error::State)?; self.current = Some(target.id); self.refresh() } } unsafe impl os_k1040_context::CompletionBackend for OsK1040ContextBackend<'_> { type ReturnPrepared = OsK1040ReturnPrepared; fn prepare_return( &mut self, source: &Self::Binding, _: &os_k1040_context::Capture, ) -> Result { self.valid_pin(source)?; let address = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_BOOT_CAPTURE_ADDRESS)) }; if !os_k1040_boot_frame_range(address) { return Err(os_k1040_context::Error::Stack); } let bootstrap = unsafe { read_volatile(address as *const os_k1040_context::Cooperative) }; if bootstrap.sp != address + 640 || bootstrap.canary != os_k1040_context::CONTEXT_CANARY || bootstrap.padding != 0 { return Err(os_k1040_context::Error::Capture); } Ok(OsK1040ReturnPrepared { source: source.id, bootstrap_root: self.bootstrap, bootstrap, address, nonce: self.next_prepared()?, }) } fn leave_return( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Binding, ) -> Result<(), os_k1040_context::Failure> { let before = |error| os_k1040_context::Failure { error, phase: os_k1040_context::FailurePhase::BeforeSwitch, }; self.valid_pin(source).map_err(before)?; if p.source != source.id || p.nonce == 0 { return Err(before(os_k1040_context::Error::Stale)); } self.table .exit_current(source.id) .map_err(|_| before(os_k1040_context::Error::State))?; self.current = None; if self.refresh().is_err() || os_k1040_select_root(p.bootstrap_root).is_err() { return Err(os_k1040_context::Failure { error: os_k1040_context::Error::Backend, phase: os_k1040_context::FailurePhase::AfterSwitch, }); } Ok(()) } fn returned(&self, p: &Self::ReturnPrepared) -> Result<(), os_k1040_context::Error> { unsafe extern "C" { fn __os_k1040_boot_landed(); } let actual = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LANDING_FRAME)) }; let stacks = os_k1040_stack_registers(); let original = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_BOOT_STACK_REGISTERS)) }; if actual != p.bootstrap || !os_k1040_boot_frame_range(p.address) || actual.sp != p.address + 640 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LANDING_PC)) } != __os_k1040_boot_landed as *const () as u64 || stacks[0] != original[0] || stacks[if original[0] == 0 { 2 } else { 1 }] != original[if original[0] == 0 { 2 } else { 1 }] { return Err(os_k1040_context::Error::Capture); } Ok(()) } fn return_root(&self, p: &Self::ReturnPrepared) -> Result { if p.bootstrap_root != self.bootstrap { return Err(os_k1040_context::Error::Stack); } Ok(p.bootstrap_root) } fn finish_return( &mut self, _: Self::ReturnPrepared, ) -> Result<(), (os_k1040_context::Error, Self::ReturnPrepared)> { Ok(()) } fn abort_return( &mut self, _: Self::ReturnPrepared, ) -> Result<(), (os_k1040_context::Error, Self::ReturnPrepared)> { Ok(()) } } unsafe impl os_k1040_context::InactiveBackend for OsK1040ContextBackend<'_> { fn cancel_inactive( &mut self, binding: &Self::Binding, ) -> Result<(), os_k1040_context::Failure> { let before = |error| os_k1040_context::Failure { error, phase: os_k1040_context::FailurePhase::BeforeSwitch, }; self.valid_pin(binding).map_err(before)?; if self.current.is_some() || os_k940_registers().0 != self.bootstrap { return Err(before(os_k1040_context::Error::Context)); } self.table .cancel(binding.id) .map_err(|_| before(os_k1040_context::Error::State))?; // Any retained sleeping authorization was invalidated by actual // Table.cancel. It is never reused to manufacture a wake after cancel. if self .sleep_wait .as_ref() .is_some_and(|wait| wait.thread() == binding.id) { self.sleep_wait = None; } self.refresh().map_err(|error| os_k1040_context::Failure { error, phase: os_k1040_context::FailurePhase::AfterSwitch, }) } } #[repr(C, align(4096))] struct OsK1040PrimaryStack { guard: [u8; 4096], usable: [u8; 65536], } #[unsafe(no_mangle)] static mut OS_K1040_PRIMARY_STACK: OsK1040PrimaryStack = OsK1040PrimaryStack { guard: [0; 4096], usable: [0; 65536], }; #[unsafe(no_mangle)] static mut OS_K1040_PRIMARY_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K1040_PRIMARY_ERROR: Option<&'static str> = None; const OS_K1040_PRIMARY_LOW_CANARY: u64 = 0x4b31_3034_4c4f_5721; const OS_K1040_PRIMARY_HIGH_CANARY: u64 = 0x4b31_3034_4849_4748; fn os_k1040_primary_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_PRIMARY_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } #[inline(never)] fn os_k1040_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { if cfg!(feature = "os-runtime-product") && !os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K1040: PRODUCT-CONTROL NO-CONTEXT-PROBE"); return Ok(()); } let guard = core::ptr::addr_of!(OS_K1040_PRIMARY_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_guard = core::ptr::addr_of!(OS_K900_STACK) as u64; let original_low = original_guard + 4096; let original_top = original_low + 65536; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("context-primary-original-stack-range"); } let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_top - 8) as *const u64) }; if original_canary != 0x4b38_3230_4c4f_5721 || original_second != 0x4b38_3230_4849_4748u64 { return Err("context-primary-original-stack-canaries"); } unsafe { tables.unmap_metadata_stack_guard( guard, low, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64, ) } .map_err(|_| "context-primary-stack-guard")?; if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("context-primary-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K1040_PRIMARY_LOW_CANARY); write_volatile(high as *mut u64, OS_K1040_PRIMARY_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K1040_PRIMARY_LOW_CANARY), (9, OS_K1040_PRIMARY_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("context-primary-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("context-primary-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k1040_primary_stack_word(slot, value); } unsafe extern "C" { fn __os_k1040_primary_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k1040_primary_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K1040_PRIMARY_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K1040_PRIMARY_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K1040_PRIMARY_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_top - 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("context-primary-stack-restoration"); } os_k1040_primary_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k1040_primary_stack_word(17, unsafe { read_volatile((original_top - 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K1040: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); os_k1040_capture(15); Ok(()) } else { Err( unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_PRIMARY_ERROR)) } .unwrap_or("context-primary-stack-return"), ) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1040_primary_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k1040_primary_stack_word(4, sp); let result = os_k1040_accept( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k1040_primary_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_PRIMARY_ERROR), Some(reason), ) }; 1 } } } global_asm!( r#" .section .text.os_k1040_primary_stack,"ax" .balign 4 .global __os_k1040_primary_on_stack .type __os_k1040_primary_on_stack, %function __os_k1040_primary_on_stack: adrp x9, OS_K1040_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1040_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K1040_PRIMARY_STACK add x9, x9, :lo12:OS_K1040_PRIMARY_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K1040_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1040_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k1040_primary_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K1040_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1040_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret .size __os_k1040_primary_on_stack, .-__os_k1040_primary_on_stack "# ); type OsK1040Bank<'a> = os_k1040_context::CompletionBank, 2>; #[unsafe(no_mangle)] static mut OS_K1040_BANK: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K1040_SPARES: [Option; 2] = [const { None }; 2]; static mut OS_K1040_FAILED_ROOTS: [Option; 2] = [const { None }; 2]; static mut OS_K1040_BINDINGS: [Option; 2] = [const { None }; 2]; static mut OS_K1040_HANDLES: [Option; 2] = [None; 2]; static mut OS_K1040_HANDOFF: Option = None; static mut OS_K1040_RETURN: Option = None; // A single masked main continuation owns this inert Copy observation until // staging ends. IRQ code never reads or writes it; no lease or bank reference // is stored here. Keeping the two sequential ABI checks off the caller stack // leaves room for their actual bounded bank validation on a 16 KiB fibre. static mut OS_K1040_CAPTURE_SCRATCH: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); #[repr(C, align(16))] #[derive(Clone, Copy)] struct OsK1040Wire { frame: os_k1040_context::Cooperative, pc: u64, argument: u64, nonce: u64, kind: u64, } #[unsafe(no_mangle)] static mut OS_K1040_WIRE: OsK1040Wire = OsK1040Wire { frame: os_k1040_context::Cooperative::EMPTY, pc: 0, argument: 0, nonce: 0, kind: 0, }; #[unsafe(no_mangle)] static mut OS_K1040_ROOT_ADDRESSES: [u64; 2] = [0; 2]; #[unsafe(no_mangle)] static mut OS_K1040_ROOT_GENERATIONS: [u64; 2] = [0; 2]; #[unsafe(no_mangle)] static mut OS_K1040_ROOT_LIVE: [u64; 2] = [0; 2]; #[unsafe(no_mangle)] static mut OS_K1040_OWNED: [u64; 12] = [0; 12]; #[unsafe(no_mangle)] static mut OS_K1040_LEASE_GENERATIONS: [u64; 12] = [0; 12]; #[unsafe(no_mangle)] static mut OS_K1040_EVIDENCE: [u64; 96] = [0; 96]; #[unsafe(no_mangle)] static mut OS_K1040_CONTEXT_WORDS: [[u64; 16]; 4] = [[0; 16]; 4]; #[unsafe(no_mangle)] static OS_K1040_CONTEXT_LAYOUT_EXPECTED: [u64; 40] = os_k1040_context::os_k1040_layout::, 2>(); #[unsafe(no_mangle)] static OS_K1040_DEFERRED_LAYOUT_EXPECTED: [u64; 20] = os_k1040_context::os_k1040_deferred_layout(); #[unsafe(no_mangle)] static OS_K1040_THREAD_ID_LAYOUT_EXPECTED: [u64; 4] = os_k1040_thread::ThreadId::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1040_THREAD_TABLE_LAYOUT_EXPECTED: [u64; 20] = OsK1040Table::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1040_LEASE_LAYOUT_EXPECTED: [u64; 7] = os_k720_frame::Lease::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1040_WAIT_LAYOUT_EXPECTED: [u64; 5] = os_k1040_thread::Wait::os_k1040_layout(); #[unsafe(no_mangle)] static mut OS_K1040_TABLE_ADDRESSES: [[u64; 3]; 2] = [[0; 3]; 2]; #[unsafe(no_mangle)] static mut OS_K1040_TABLE_WORDS: [[u64; 12]; 2] = [[0; 12]; 2]; #[unsafe(no_mangle)] static mut OS_K1040_STACK_WORDS: [[u64; 20]; 2] = [[0; 20]; 2]; #[unsafe(no_mangle)] static OS_K1040_BACKEND_LAYOUT_EXPECTED: [u64; 30] = [ core::mem::size_of::>() as u64, core::mem::offset_of!(OsK1040ContextBackend<'static>, table) as u64, core::mem::offset_of!(OsK1040ContextBackend<'static>, bootstrap) as u64, core::mem::offset_of!(OsK1040ContextBackend<'static>, pins) as u64, core::mem::offset_of!(OsK1040ContextBackend<'static>, sequence) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1040Binding, id) as u64, core::mem::offset_of!(OsK1040Binding, ordinal) as u64, core::mem::offset_of!(OsK1040Binding, root) as u64, core::mem::offset_of!(OsK1040Binding, root_generation) as u64, core::mem::offset_of!(OsK1040Binding, branches) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1040Stack, root) as u64, core::mem::offset_of!(OsK1040Stack, ordinal) as u64, core::mem::offset_of!(OsK1040Stack, start) as u64, core::mem::offset_of!(OsK1040Stack, leases) as u64, core::mem::offset_of!(OsK1040Stack, guard) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1040Root, lease) as u64, core::mem::offset_of!(OsK1040Root, bootstrap) as u64, core::mem::offset_of!(OsK1040Root, branches) as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::size_of::>() as u64, core::mem::offset_of!(OsK1040ContextBackend<'static>, current) as u64, core::mem::offset_of!(OsK1040ContextBackend<'static>, sleep_wait) as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1040IdlePrepared, epoch) as u64, core::mem::offset_of!(OsK1040IdlePrepared, wait) as u64, ]; #[unsafe(no_mangle)] static mut OS_K1040_EVENTS: [u64; 32768] = [0; 32768]; #[unsafe(no_mangle)] static mut OS_K1040_EVENT_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LOOP_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LOOP_REMAINING: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LOOP_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LOOP_PENDING: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LOOP_SOURCE: [os_k1040_context::Cooperative; 1000] = [os_k1040_context::Cooperative::EMPTY; 1000]; #[unsafe(no_mangle)] static mut OS_K1040_LOOP_LANDING: [os_k1040_context::Cooperative; 1000] = [os_k1040_context::Cooperative::EMPTY; 1000]; #[unsafe(no_mangle)] static mut OS_K1040_LOOP_WORDS: [u64; 16000] = [0; 16000]; #[unsafe(no_mangle)] static mut OS_K1040_REFUSALS: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K1040_CAPTURE_EPISODE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LIVE_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LANDING_ARGUMENT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LANDING_PC: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_LANDING_FRAME: os_k1040_context::Cooperative = os_k1040_context::Cooperative::EMPTY; #[unsafe(no_mangle)] static mut OS_K1040_LANDING_FRAMES: [os_k1040_context::Cooperative; 8] = [os_k1040_context::Cooperative::EMPTY; 8]; #[unsafe(no_mangle)] static mut OS_K1040_SOURCE_FRAMES: [os_k1040_context::Cooperative; 6] = [os_k1040_context::Cooperative::EMPTY; 6]; #[unsafe(no_mangle)] static mut OS_K1040_SOURCE_EPISODES: [u64; 6] = [0; 6]; #[unsafe(no_mangle)] static mut OS_K1040_LANDING_PCS: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K1040_LANDING_ARGUMENTS: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K1040_BOOT_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_BOOT_STACK_REGISTERS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1040_STACK_REGISTERS_BEFORE: [u64; 4] = [0; 4]; #[unsafe(no_mangle)] static mut OS_K1040_STACK_REGISTERS_AFTER: [u64; 4] = [0; 4]; #[unsafe(no_mangle)] static mut OS_K1040_IDLE_STACK_REGISTERS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1040_IDLE_BOUNDARY: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K1040_IDLE_RECEIPTS: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K1040_ROOT_REFUSAL: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K1040_BOOT_FRAME: os_k1040_context::Cooperative = os_k1040_context::Cooperative::EMPTY; #[unsafe(no_mangle)] static mut OS_K1040_BOOT_FRAMES: [os_k1040_context::Cooperative; 2] = [os_k1040_context::Cooperative::EMPTY; 2]; #[unsafe(no_mangle)] static mut OS_K1040_BOOT_STACK_HISTORY: [[u64; 4]; 2] = [[0; 4]; 2]; #[unsafe(no_mangle)] static mut OS_K1040_RETURN_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_DEPARTED_NONCE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_CRITICAL: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_SAFE_POINT: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1040_IRQ_TCO: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_KERNEL_FRAME: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_IRQ_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_IRQ_CAPTURE_EPISODE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_IRQ_FRAMES: [os_k920_exception::Frame; 3] = [OS_K920_ZERO_FRAME; 3]; #[unsafe(no_mangle)] static mut OS_K1040_IRQ_PHYSICAL: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1040_WAKE_EPOCH: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_CAPTURE_GATE: u8 = 0; #[unsafe(no_mangle)] static mut OS_K1040_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1040_THREADS_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1040_INTERVENING_SUM: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_IRQ_SEEN: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_PRE_IRQ: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_PRE_LAST: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1040_PRE_ACTIVE: u32 = 0; #[unsafe(no_mangle)] static mut OS_K1040_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K1040_AUTHORITY_LIVE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K1040_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K1040_DIAGNOSTICS: [u64; 6] = [0; 6]; const _: () = { assert!(core::mem::size_of::() == 672); assert!(core::mem::offset_of!(OsK1040Wire, pc) == 640); assert!(core::mem::offset_of!(OsK1040Wire, argument) == 648); assert!(core::mem::offset_of!(OsK1040Wire, nonce) == 656); assert!(core::mem::offset_of!(OsK1040Wire, kind) == 664); }; fn os_k1040_stack_registers() -> [u64; 4] { unsafe extern "C" { fn __os_k1040_stack_registers(output: *mut u64); } if !os_k1040_execution().irq_masked { park(); } let mut values = [0; 4]; unsafe { __os_k1040_stack_registers(values.as_mut_ptr()); } values } fn os_k1040_word(index: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_EVIDENCE) .cast::() .add(index), value, ); } } fn os_k1040_read(index: usize) -> u64 { unsafe { read_volatile( core::ptr::addr_of!(OS_K1040_EVIDENCE) .cast::() .add(index), ) } } fn os_k1040_event(kind: u64, a: u64, b: u64, c: u64) { let n = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_EVENT_COUNT)) } as usize; if n >= 8192 { park(); } for (index, value) in [kind, a, b, c].into_iter().enumerate() { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_EVENTS) .cast::() .add(n * 4 + index), value, ); } } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_EVENT_COUNT), n as u64 + 1); } } fn os_k1040_bank() -> *mut OsK1040Bank<'static> { core::ptr::addr_of_mut!(OS_K1040_BANK).cast() } #[unsafe(no_mangle)] extern "C" fn __os_k1040_context_identity() -> u64 { let e = os_k1040_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { return 0; } let n = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_IDENTITY)) }; let Some(next) = n.checked_add(1) else { return 0; }; if n == 0 { return 0; } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_IDENTITY), next); } n } #[unsafe(no_mangle)] extern "C" fn __os_k1040_threads_identity() -> u64 { let execution = os_k1040_execution(); if execution.cpu != 0 || !execution.irq_masked || execution.in_irq { return 0; } let value = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_THREADS_IDENTITY)) }; let Some(next) = value.checked_add(1) else { return 0; }; if value == 0 { return 0; } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_THREADS_IDENTITY), next); } value } fn os_k1040_handle(index: usize) -> os_k1040_context::Handle { if index >= 2 { park(); } unsafe { read_volatile( core::ptr::addr_of!(OS_K1040_HANDLES) .cast::>() .add(index), ) } .unwrap_or_else(|| park()) } fn os_k1040_boot_frame_range(address: u64) -> bool { let guard = unsafe { read_volatile( core::ptr::addr_of!(OS_K1040_PRIMARY_STACK_EVIDENCE) .cast::() .add(13), ) }; address % 16 == 0 && address >= guard + 4096 + 1024 && address .checked_add(640) .is_some_and(|end| end <= guard + 4096 + 65536 - 16) } fn os_k1040_set_wire(image: os_k1040_context::Image, nonce: u64) -> *const OsK1040Wire { let (frame, pc, argument, kind) = match image { os_k1040_context::Image::Initial { entry, argument, sp, return_pc, } => { let mut f = os_k1040_context::Cooperative::EMPTY; f.sp = sp; f.x19_x30[11] = return_pc; (f, entry, argument, 0) } os_k1040_context::Image::Cooperative(frame) => (frame, frame.x19_x30[11], 0, 1), os_k1040_context::Image::Timer(_) => park(), // Measured scope is fresh safe-point cooperative restore. }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_WIRE), OsK1040Wire { frame, pc, argument, nonce, kind, }, ); } core::ptr::addr_of!(OS_K1040_WIRE) } #[inline(never)] fn os_k1040_capture_source(address: u64) -> os_k1040_context::Capture { let bank = unsafe { &*os_k1040_bank() }; let execution = bank .handoff() .bank() .with_backend_observation(|backend| os_k1040_context::Backend::execution(backend)); let current = execution.thread.unwrap_or_else(|| park()); let source = (0..2) .find(|index| { bank.handoff() .bank() .record(os_k1040_handle(*index)) .is_ok_and(|record| record.thread == current) }) .unwrap_or_else(|| park()); let record = bank .handoff() .bank() .record(os_k1040_handle(source)) .unwrap_or_else(|_| park()); let layout = bank.handoff().bank().with_backend_observation(|backend| { backend.observations[source] .unwrap_or_else(|| park()) .layout }); if address % 16 != 0 || address < layout.usable_bottom() || !address .checked_add(640) .is_some_and(|end| end <= layout.usable_top()) || unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LIVE_CAPTURE_ADDRESS)) } != address { return park(); } let frame = unsafe { read_volatile(address as *const os_k1040_context::Cooperative) }; let episode = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_CAPTURE_EPISODE)) }; if frame.sp != address + 640 || record.state != os_k1040_context::State::Active || episode == 0 { park(); } let n = os_k1040_read(24) as usize; let in_loop = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_ACTIVE)) } == 1; let step = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_COUNT)) } as usize; if in_loop { if step >= 1000 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_PENDING)) } != 0 { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_LOOP_SOURCE) .cast::() .add(step), frame, ); write_volatile(core::ptr::addr_of_mut!(OS_K1040_LOOP_PENDING), 1); } for (i, value) in [ episode, source as u64, 1 - source as u64, address, 0, execution.root, 0, 0, current.table(), current.slot() as u64, current.generation(), 0, 0, 0, 0, 0, ] .into_iter() .enumerate() { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_LOOP_WORDS) .cast::() .add(step * 16 + i), value, ); } } } let sample = !in_loop || step < 2; if sample && n >= 6 { park(); } if sample { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_SOURCE_FRAMES) .cast::() .add(n), frame, ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_SOURCE_EPISODES) .cast::() .add(n), episode, ); } os_k1040_word(24, n as u64 + 1); } os_k1040_context::Capture { image: os_k1040_context::Image::Cooperative(frame), address, abi: os_k1040_context::Abi::COOPERATIVE, episode, } } #[inline(never)] fn os_k1040_depart(token: os_k1040_context::OwnedHandoff) -> *const OsK1040Wire { let bank = unsafe { &mut *os_k1040_bank() }; let handoff = bank.quiescent().unwrap_or_else(|_| park()); let image = handoff.target_image(&token).unwrap_or_else(|_| park()); let token = handoff.depart(token).unwrap_or_else(|_| park()); let nonce = token.nonce(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_DEPARTED_NONCE), nonce); write_volatile(core::ptr::addr_of_mut!(OS_K1040_HANDOFF), Some(token)); } os_k1040_set_wire(image, nonce) } #[inline(never)] fn os_k1040_store_capture(address: u64) { let capture = os_k1040_capture_source(address); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_CAPTURE_SCRATCH).cast(), capture, ); } } #[inline(never)] fn os_k1040_reject_shifted_abi() { let capture: os_k1040_context::Capture = unsafe { core::ptr::read(core::ptr::addr_of!(OS_K1040_CAPTURE_SCRATCH).cast()) }; let bank = unsafe { &mut *os_k1040_bank() }; let core = bank .quiescent() .unwrap_or_else(|_| park()) .quiescent() .unwrap_or_else(|_| park()); let before = core.snapshot(); let mut bad = capture; bad.abi.offsets[1] += 8; if !matches!( core.cooperate(os_k1040_handle(0), os_k1040_handle(1), bad), Err(os_k1040_context::Error::Abi) ) || core.snapshot() != before { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_REFUSALS).cast::(), 1); } } #[inline(never)] fn os_k1040_stage_cooperative(target: u64) -> os_k1040_context::OwnedHandoff { let capture: os_k1040_context::Capture = unsafe { core::ptr::read(core::ptr::addr_of!(OS_K1040_CAPTURE_SCRATCH).cast()) }; let target = os_k1040_handle(target as usize); let bank = unsafe { &mut *os_k1040_bank() }; let handoff = bank.quiescent().unwrap_or_else(|_| park()); let current = handoff .bank() .with_backend_observation(|b| os_k1040_context::Backend::execution(b)) .thread .unwrap_or_else(|| park()); if current.slot() >= 2 { park(); } let source = os_k1040_handle(current.slot()); let token = if os_k1040_read(24) == 4 { // B's second physical continuation sleeps with its real nonclone // Table Wait while A finishes on its own retained stack. handoff.sleep_to(source, target, capture) } else { handoff.cooperate(source, target, capture) } .unwrap_or_else(|_| park()); token } #[unsafe(no_mangle)] extern "C" fn __os_k1040_prepare_cooperative(address: u64, target: u64) -> *const OsK1040Wire { os_k1040_store_capture(address); if unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_ACTIVE)) } == 1 && unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_COUNT)) } == 0 { os_k1040_reject_shifted_abi(); } let token = os_k1040_stage_cooperative(target); let pointer = os_k1040_depart(token); if unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_ACTIVE)) } == 1 { let step = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_COUNT)) } as usize; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_LOOP_WORDS) .cast::() .add(step * 16 + 6), os_k940_registers().0, ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_LOOP_WORDS) .cast::() .add(step * 16 + 7), read_volatile(core::ptr::addr_of!(OS_K1040_DEPARTED_NONCE)), ); } } pointer } #[inline(never)] fn os_k1040_stage_deferred(capture: os_k1040_context::Capture) -> os_k1040_context::OwnedHandoff { unsafe { (&mut *os_k1040_bank()) .quiescent() .unwrap_or_else(|_| park()) .service_deferred(capture) } .unwrap_or_else(|_| park()) } #[unsafe(no_mangle)] extern "C" fn __os_k1040_prepare_deferred(address: u64) -> *const OsK1040Wire { let capture = os_k1040_capture_source(address); let token = os_k1040_stage_deferred(capture); os_k1040_depart(token) } fn os_k1040_landing_snapshot(address: u64) { let root = os_k940_registers().0; let normal = unsafe { &*os_k1040_bank() }.handoff().bank(); let valid = normal.with_backend_observation(|backend| { if root == backend.bootstrap { os_k1040_boot_frame_range(address) } else { backend.observations.iter().flatten().any(|o| { address >= o.layout.usable_bottom() && address .checked_add(640) .is_some_and(|end| end <= o.layout.usable_top()) }) } }); if !valid || address % 16 != 0 { park(); } let frame = unsafe { read_volatile(address as *const os_k1040_context::Cooperative) }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_LANDING_FRAME), frame); } let pending = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_PENDING)) } == 1; let step = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_COUNT)) } as usize; if pending { if step >= 1000 { park(); } let target = unsafe { read_volatile( core::ptr::addr_of!(OS_K1040_LOOP_WORDS) .cast::() .add(step * 16 + 2), ) } as usize; let record = normal .record(os_k1040_handle(target)) .unwrap_or_else(|_| park()); if root != record.root { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_LOOP_LANDING) .cast::() .add(step), frame, ); } for (index, value) in [ (4, address), (11, record.thread.table()), (12, record.thread.slot() as u64), (13, record.thread.generation()), (14, unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LANDING_PC)) }), (15, os_k1040_stack_registers()[0]), ] { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_LOOP_WORDS) .cast::() .add(step * 16 + index), value, ); } } } let sample = !pending || step == 0 || step == 999; if !sample { return; } let n = os_k1040_read(25) as usize; if n >= 8 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_LANDING_FRAME), frame); write_volatile( core::ptr::addr_of_mut!(OS_K1040_LANDING_FRAMES) .cast::() .add(n), frame, ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_LANDING_PCS) .cast::() .add(n), read_volatile(core::ptr::addr_of!(OS_K1040_LANDING_PC)), ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_LANDING_ARGUMENTS) .cast::() .add(n), read_volatile(core::ptr::addr_of!(OS_K1040_LANDING_ARGUMENT)), ); } os_k1040_word(25, n as u64 + 1); } #[unsafe(no_mangle)] extern "C" fn __os_k1040_landed(address: u64) { os_k1040_landing_snapshot(address); let token = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1040_HANDOFF)).take() } .unwrap_or_else(|| park()); unsafe { (&mut *os_k1040_bank()) .quiescent() .unwrap_or_else(|_| park()) .landed(token) } .unwrap_or_else(|_| park()); if unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_PENDING)) } == 1 { let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_COUNT)) }; let remaining = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_REMAINING)) }; if count >= 1000 || remaining != 1000 - count { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_LOOP_COUNT), count + 1); write_volatile( core::ptr::addr_of_mut!(OS_K1040_LOOP_REMAINING), remaining - 1, ); write_volatile(core::ptr::addr_of_mut!(OS_K1040_LOOP_PENDING), 0); } } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_LIVE_CAPTURE_ADDRESS), 0); } } #[unsafe(no_mangle)] extern "C" fn __os_k1040_prepare_return(address: u64) -> *const OsK1040Wire { let capture = os_k1040_capture_source(address); let bank = unsafe { &mut *os_k1040_bank() }; let source = (0..2) .map(os_k1040_handle) .find(|handle| { bank.handoff() .bank() .record(*handle) .is_ok_and(|r| r.state == os_k1040_context::State::Active) }) .unwrap_or_else(|| park()); let token = bank .prepare_return(source, capture) .unwrap_or_else(|_| park()); let token = bank.depart_return(token).unwrap_or_else(|_| park()); let frame = bank.with_return_observation(|p| p.unwrap_or_else(|| park()).bootstrap); unsafe extern "C" { fn __os_k1040_boot_landed(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_BOOT_FRAME), frame); write_volatile(core::ptr::addr_of_mut!(OS_K1040_RETURN), Some(token)); write_volatile( core::ptr::addr_of_mut!(OS_K1040_WIRE), OsK1040Wire { frame, pc: __os_k1040_boot_landed as *const () as u64, argument: 0, nonce: 0, kind: 2, }, ); } let returned = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_RETURN_COUNT)) } as usize; if returned >= 2 { park(); } let stacks = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_BOOT_STACK_REGISTERS)) }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_BOOT_FRAMES) .cast::() .add(returned), frame, ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_BOOT_STACK_HISTORY) .cast::<[u64; 4]>() .add(returned), [ stacks[0], stacks[1], stacks[2], read_volatile(core::ptr::addr_of!(OS_K1040_BOOT_CAPTURE_ADDRESS)), ], ); } core::ptr::addr_of!(OS_K1040_WIRE) } #[unsafe(no_mangle)] extern "C" fn __os_k1040_return_landed(address: u64) { os_k1040_landing_snapshot(address); let token = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1040_RETURN)).take() } .unwrap_or_else(|| park()); unsafe { (&mut *os_k1040_bank()).return_landed(token) }.unwrap_or_else(|_| park()); unsafe { let count = read_volatile(core::ptr::addr_of!(OS_K1040_RETURN_COUNT)); if count >= 2 { park(); } write_volatile(core::ptr::addr_of_mut!(OS_K1040_RETURN_COUNT), count + 1); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_LIVE_CAPTURE_ADDRESS), 0); } } fn os_k1040_arm_idle(pending: bool) -> Result<(), os_k1040_context::Error> { if !os_k1040_execution().irq_masked || os_k1040_execution().in_irq { return Err(os_k1040_context::Error::Context); } let counter = os_k960_counter(); let deadline = if pending { counter.checked_sub(1) } else { counter.checked_add((os_k960_frequency() / 1000).max(1)) } .ok_or(os_k1040_context::Error::Exhausted)?; unsafe { asm!("msr CNTP_CVAL_EL0, {deadline}","mov x9, #1","msr CNTP_CTL_EL0, x9","isb", deadline=in(reg)deadline,out("x9")_,options(nomem,nostack,preserves_flags)); } if pending { while os_k960_timer().1 & 4 == 0 { core::hint::spin_loop(); } } os_k1040_event(0x61, u64::from(pending), counter, deadline); Ok(()) } fn os_k1040_idle_atomic( backend: &mut OsK1040ContextBackend<'_>, prepared: &mut OsK1040IdlePrepared, epoch: u64, ) -> Result { let before = |error| os_k1040_context::Failure { error, phase: os_k1040_context::FailurePhase::BeforeSwitch, }; let after = |error| os_k1040_context::Failure { error, phase: os_k1040_context::FailurePhase::AfterSwitch, }; if prepared.epoch != epoch || prepared.wait.is_none() || backend.current.is_some() || !os_k1040_context::Backend::ready_empty(backend) || os_k940_registers().0 != backend.bootstrap || os_k1040_context::Backend::wake_epoch(backend) != epoch { return Err(before(os_k1040_context::Error::Wake)); } let wait = prepared .wait .as_ref() .ok_or_else(|| before(os_k1040_context::Error::Wake))?; let actual = backend .table .record(wait.thread()) .map_err(|_| before(os_k1040_context::Error::State))?; if actual.state != os_k1040_thread::State::Sleeping || actual.wait_epoch != wait.epoch() { return Err(before(os_k1040_context::Error::Stale)); } backend.validate_stack_owner(actual.id).map_err(before)?; let final_expected = epoch .checked_add(2) .ok_or_else(|| before(os_k1040_context::Error::Exhausted))?; unsafe extern "C" { fn __os_k1040_idle_wait(epoch: u64) -> u64; } os_k1040_word(86, prepared as *const OsK1040IdlePrepared as u64); os_k1040_word(87, core::mem::size_of::() as u64); os_k1040_word( 88, prepared .wait .as_ref() .map_or(0, |w| w as *const os_k1040_thread::Wait as u64), ); // No IRQ callback observes this paused bank/backend/Table loan. Both // handlers access only independent scalars and actual848 frame buffers. os_k1040_arm_idle(false).map_err(before)?; let executed = unsafe { __os_k1040_idle_wait(epoch) }; let boundary = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_IDLE_BOUNDARY)) }; let wake = os_k1040_context::Backend::wake_epoch(backend); if executed != 1 || boundary[0] != epoch || boundary[1] & 7 != 1 || boundary[3] != 1 || boundary[4] < boundary[2] || wake != epoch + 1 { return Err(after(os_k1040_context::Error::Wake)); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_IDLE_RECEIPTS), [boundary[0], 0, executed, wake, 0, 0, 0, 0], ); } // The same still-retained Wait remains asleep for the pending-at-boundary // experiment. CPU ISTATUS is already1 while IRQs remain masked. The naked // final boundary must skip WFI; a second realIRQ consumes its own wake // epoch before the actual Table.wake below consumes this Wait once. os_k1040_arm_idle(true).map_err(after)?; let skipped = unsafe { __os_k1040_idle_wait(wake) }; let pending_boundary = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_IDLE_BOUNDARY)) }; let final_wake = os_k1040_context::Backend::wake_epoch(backend); if skipped != 0 || pending_boundary[0] != wake || pending_boundary[1] & 7 != 5 || pending_boundary[5] != wake || pending_boundary[6] & 7 != 5 || pending_boundary[7] != 1 || final_wake != final_expected { return Err(after(os_k1040_context::Error::Wake)); } for (index, value) in [pending_boundary[0], 1, skipped, final_wake] .into_iter() .enumerate() { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_IDLE_RECEIPTS) .cast::() .add(4 + index), value, ); } } let wait = prepared .wait .take() .ok_or_else(|| after(os_k1040_context::Error::Wake))?; match backend.table.wake(wait) { Ok(id) if id == actual.id => {} Ok(_) => return Err(after(os_k1040_context::Error::Stale)), Err((_error, wait)) => { prepared.wait = Some(wait); return Err(after(os_k1040_context::Error::Wake)); } } backend.refresh().map_err(after)?; os_k1040_event(0x62, actual.id.generation(), epoch, final_wake); Ok(os_k1040_context::IdleReceipt { checked_epoch: boundary[0], work_at_handoff: false, executed_wfi: true, wake_epoch: final_wake, }) } #[unsafe(no_mangle)] extern "C" fn __os_k1040_arm_critical() { let e = os_k1040_execution(); if !e.irq_masked || e.in_irq || e.cpu != 0 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_CRITICAL), 1); write_volatile(core::ptr::addr_of_mut!(OS_K1040_IRQ_SEEN), 0); } let frequency = os_k960_frequency(); let counter = os_k960_counter(); let deadline = counter .checked_add((frequency / 1000).max(1)) .unwrap_or_else(|| park()); os_k1040_word(40, deadline); os_k1040_word(41, counter); unsafe { asm!("msr CNTP_CVAL_EL0, {deadline}","mov x9, #1","msr CNTP_CTL_EL0, x9","isb",deadline=in(reg)deadline,out("x9")_,options(nomem,nostack,preserves_flags)); } } #[unsafe(no_mangle)] extern "C" fn os_k1040_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { if unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_ACTIVE)) } != 1 || pointer.is_null() { park(); } let address = pointer as u64; let own_irq = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_IRQ_SEEN)) }; let frame_number = os_k1040_read(26) as usize; if frame_number >= 3 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { park(); } let (low, high) = if frame_number == 0 { // The root/stack cache is private and fixed before IRQ unmask. No // Runtime, allocator, Table or central observer loan exists here. let bank = unsafe { &*os_k1040_bank() }; bank.handoff().bank().with_backend_observation(|backend| { let o = backend.observations[0].unwrap_or_else(|| park()); (o.layout.base, o.layout.base + o.layout.bytes) }) } else { let guard = unsafe { read_volatile( core::ptr::addr_of!(OS_K1040_PRIMARY_STACK_EVIDENCE) .cast::() .add(13), ) }; (guard + 4096, guard + 4096 + 65536) }; let stack = os_k920_exception::Stack::new(low, high).unwrap_or_else(|_| park()); stack.frame_address(address).unwrap_or_else(|_| park()); let frame = unsafe { read_volatile(pointer) }; os_k920_kernel_irq_return( &frame, address, stack, core::ptr::addr_of!(_text_start) as u64, core::ptr::addr_of!(_text_end) as u64, __os_k920_kernel_tco_observed(), ) .unwrap_or_else(|_| park()); if frame.vector_slot != 5 || frame.spsr & 31 != 5 || frame.elr & 3 != 0 || frame.interrupted_sp != address + 848 { park(); } let iar: u64; unsafe { asm!("mrs {iar}, ICC_IAR1_EL1",iar=out(reg)iar,options(nomem,nostack,preserves_flags)); } let (cval, ctl) = os_k960_timer(); let counter = os_k960_counter(); if iar & 0x00ff_ffff != 30 || ctl & 7 != 5 || counter < cval { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE), 1); write_volatile( core::ptr::addr_of_mut!(OS_K1040_IRQ_FRAMES) .cast::() .add(frame_number), frame, ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_IRQ_PHYSICAL) .cast::() .add(frame_number), address, ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_IRQ_TCO), frame.spsr & (1 << 25), ); } for (index, value) in [counter, cval, ctl, iar, os_k940_registers().0, address] .into_iter() .enumerate() { os_k1040_word(48 + frame_number * 6 + index, value); } if frame_number == 0 { if unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_CRITICAL)) } != 1 || own_irq != 0 { park(); } let episode = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_CAPTURE_EPISODE)) } .checked_add(1) .unwrap_or_else(|| park()); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_CAPTURE_EPISODE), episode); write_volatile( core::ptr::addr_of_mut!(OS_K1040_IRQ_CAPTURE_EPISODE), episode, ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_IRQ_CAPTURE_ADDRESS), address, ); } let capture = os_k1040_context::Capture { image: os_k1040_context::Image::Timer(frame), address, abi: os_k1040_context::Abi::COOPERATIVE, episode, }; let bank = unsafe { &mut *os_k1040_bank() }; let core = bank .quiescent() .unwrap_or_else(|_| park()) .quiescent() .unwrap_or_else(|_| park()); if !matches!( core.timer(os_k1040_handle(0), os_k1040_handle(1), capture), Err(os_k1040_context::Error::Context) ) { park(); } let epoch = core .defer_timer(os_k1040_handle(0), os_k1040_handle(1), capture) .unwrap_or_else(|_| park()); os_k1040_word(27, epoch); os_k1040_word(28, 1); } else { let wake = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_WAKE_EPOCH)) } .checked_add(1) .unwrap_or_else(|| park()); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_WAKE_EPOCH), wake); } } unsafe { asm!( "msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack, preserves_flags) ); } let split: u64; unsafe { asm!("mrs {split}, ICC_CTLR_EL1",split=out(reg)split,options(nomem,nostack,preserves_flags)); asm!("dsb sy","msr ICC_EOIR1_EL1, {iar}","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags)); if split & 2 != 0 { asm!("msr ICC_DIR_EL1, {iar}","dsb sy","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags)); } } let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) } .checked_add(1) .unwrap_or_else(|| park()); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT), count); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER), counter); } os_k1040_word(26, frame_number as u64 + 1); os_k1040_event(0x60, iar, counter, cval); os_k1040_capture_irq(match frame_number { 0 => 5, 1 => 9, _ => 10, }); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1040_IRQ_SEEN), own_irq + 1); } 0 } fn os_k1040_observe_backend( backend: &OsK1040ContextBackend<'_>, ids: [Option; 2], ) { if os_k1040_execution().in_irq { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_TABLE_ADDRESSES), backend.table.os_k1040_addresses(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_TABLE_WORDS), backend.table.os_k1020_slot_words(), ); } for ordinal in 0..2 { let mut words = [0; 20]; if let Some(id) = ids[ordinal] { backend .table .os_k1020_observe_stack(id, |stack| { words[..8].copy_from_slice(&[ stack.start, stack.root.address(), stack.root.generation(), stack.root.bootstrap, stack.root.branches[0], stack.root.branches[1], stack.ordinal as u64, stack.guard.as_ref().map_or(0, |g| g.descriptor_address()), ]); for index in 0..6 { let lease = if index < 5 { stack.leases[index].as_ref() } else { stack.root.lease.as_ref() }; words[8 + index] = lease.map_or(0, |v| v as *const os_k720_frame::Lease as u64); } let layout = stack.layout(); words[14] = layout.base; words[15] = layout.bytes; }) .unwrap_or_else(|_| park()); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_STACK_WORDS) .cast::<[u64; 20]>() .add(ordinal), words, ); } } os_k1040_word(84, backend as *const OsK1040ContextBackend<'_> as u64); os_k1040_word(85, core::mem::size_of::>() as u64); } #[inline(never)] fn os_k1040_snapshot_registers(stage: u64) -> os_k1040_thread::Execution { let (root, tcr, vbar, daif) = os_k940_registers(); let (mair, sctlr, current_el): (u64, u64, u64); unsafe { asm!("mrs {mair}, MAIR_EL1", "mrs {sctlr}, SCTLR_EL1", "mrs {current_el}, CurrentEL", mair=out(reg)mair, sctlr=out(reg)sctlr, current_el=out(reg)current_el, options(nomem,nostack,preserves_flags)); } os_k1040_word(93, mair); os_k1040_word(94, sctlr); os_k1040_word(95, current_el); let e = os_k1040_execution(); let stacks = os_k1040_stack_registers(); for (index, value) in stacks.into_iter().enumerate() { os_k1040_word(80 + index, value); } for (index, value) in [ (5, stage), (6, root), (7, tcr), (8, vbar), (9, daif), (10, e.cpu as u64), (11, e.sp), (12, os_k960_counter()), (13, u64::from(e.in_irq)), (14, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }), ] { os_k1040_word(index, value); } e } #[inline(never)] fn os_k1040_snapshot_bank() { if os_k1040_read(17) == 1 { let bank = unsafe { &*os_k1040_bank() }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_CONTEXT_WORDS), bank.os_k1040_words(), ); } let core = bank.handoff().bank(); let s = core.snapshot(); for (index, value) in [ (30, s.retained as u64), (31, s.active as u64), (32, s.ready as u64), (33, s.quarantined as u64), (34, s.deferred_epoch.unwrap_or(0)), (35, u64::from(s.faulted)), (36, u64::from(s.retained_switch)), (37, u64::from(s.retained_capture)), ] { os_k1040_word(index, value); } } } #[inline(never)] fn os_k1040_snapshot(stage: u64) { let e = os_k1040_snapshot_registers(stage); if e.in_irq { park(); } os_k1040_snapshot_bank(); if os_k1040_read(17) == 1 { let bank = unsafe { &*os_k1040_bank() }; let core = bank.handoff().bank(); core.with_backend_observation(|backend| { os_k1040_observe_backend( backend, backend.records.map(|record| record.map(|r| r.id)), ); backend.table.os_k1020_observe_backend(|stack_backend| { os_k1040_word( 70, stack_backend.memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64, ); os_k1040_word(71, core::mem::size_of::() as u64); os_k940_inventory( stack_backend.memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K1040_AUTHORITY_LIVE).cast(), ); }) }); } } #[inline(never)] fn os_k1040_snapshot_irq(stage: u64) { let e = os_k1040_snapshot_registers(stage); if !e.in_irq { park(); } // The positive WFI and pending IRQs run with the retained Idle loan. // Only the critical IRQ has an otherwise quiescent bank observation. if os_k1040_read(26) == 1 { os_k1040_snapshot_bank(); } } #[inline(never)] fn os_k1040_capture_gate(stage: u64) { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_CAPTURE_GATE), 0); } write_str("TURKOS/OS/K1040: CAPTURE STAGE=0x"); write_hex(stage as usize); write_line(""); while unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_CAPTURE_GATE)) } != 1 { core::hint::spin_loop(); } unsafe { asm!("dsb ish", "isb", options(nostack, preserves_flags)); } } fn os_k1040_capture(stage: u64) { os_k1040_snapshot(stage); os_k1040_capture_gate(stage); } fn os_k1040_capture_irq(stage: u64) { os_k1040_snapshot_irq(stage); os_k1040_capture_gate(stage); } #[unsafe(no_mangle)] extern "C" fn __os_k1040_fibre_work(phase: u64) { if phase > 5 { park(); } let ordinal = if phase == 1 || phase == 3 || phase == 5 { 1 } else { 0 }; let bank = unsafe { &*os_k1040_bank() }; let record = bank .handoff() .bank() .record(os_k1040_handle(ordinal)) .unwrap_or_else(|_| park()); let execution = bank .handoff() .bank() .with_backend_observation(|b| os_k1040_context::Backend::execution(b)); if record.state != os_k1040_context::State::Active || execution.thread != Some(record.thread) || execution.root != record.root || !execution.safe_point { park(); } unsafe extern "C" { fn __os_k1040_cpu_work() -> u64; } let sum = unsafe { __os_k1040_cpu_work() }; if sum != 500500 { park(); } os_k1040_word(72 + phase as usize, sum); if phase == 5 { let elapsed = os_k960_counter() .checked_sub(os_k1040_read(90)) .unwrap_or_else(|| park()); if elapsed > os_k960_frequency() { park(); } os_k1040_word(91, elapsed); os_k1040_word(92, 1); } if phase == 3 { let deferred = bank .handoff() .bank() .last_deferred_record() .unwrap_or_else(|| park()); if deferred.capture.image != os_k1040_context::Image::Timer(unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_IRQ_FRAMES).cast()) }) || unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_INTERVENING_SUM)) } != 500500 { park(); } let source = bank .handoff() .bank() .record(os_k1040_handle(0)) .unwrap_or_else(|_| park()); if !matches!(source.image, os_k1040_context::Image::Cooperative(_)) { park(); } } os_k1040_capture(match phase { 0 => 2, 1 => 3, 2 => 4, 3 => 6, 4 => 7, _ => 11, }); } struct OsK1040Installation<'a> { pointer: *mut OsK1040Bank<'a>, active: bool, ownership: core::marker::PhantomData>, } impl<'a> OsK1040Installation<'a> { unsafe fn install(bank: OsK1040Bank<'a>) -> Self { let pointer = core::ptr::addr_of_mut!(OS_K1040_BANK).cast::>(); unsafe { pointer.write(bank); } Self { pointer, active: true, ownership: core::marker::PhantomData, } } fn extract(mut self) -> OsK1040ContextBackend<'a> { let mut destination = core::mem::MaybeUninit::uninit(); // Installation is the sole typed owner of this BSS bank. Successful // extraction seals it before returning the uniquely moved backend. unsafe { OsK1040Bank::os_k1040_take_backend_in_place(self.pointer, &mut destination) } .unwrap_or_else(|_| park()); self.active = false; unsafe { destination.assume_init() } } } impl Drop for OsK1040Installation<'_> { fn drop(&mut self) { if self.active { park(); } } } fn os_k1040_accept( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { os_k1040_retain_configuration(); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_DIAGNOSTICS), read_volatile(core::ptr::addr_of!(OS_K1020_DIAGNOSTICS)), ); } let (root, tcr, vbar, daif) = os_k940_registers(); let pre = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_PRE_IRQ), pre); write_volatile( core::ptr::addr_of_mut!(OS_K1040_PRE_LAST), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)), ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_PRE_ACTIVE), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)), ); } if cfg!(feature = "os-runtime-product") && !os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K1040: PRODUCT-CONTROL NO-CONTEXT-PROBE"); return Ok(()); } if root != tables.root || tcr & 63 != 25 || daif & 0x80 == 0 || os_k960_timer().1 & 1 != 0 { return Err("context-predecessor-root"); } let before_stacks = os_k1040_stack_registers(); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_STACK_REGISTERS_BEFORE), before_stacks, ); } let (_, memory, _, tasks) = runtime.running().map_err(os_k660_policy::Error::tag)?; if tasks.scheduler.running() != tasks.id { return Err("context-alpha-running"); } os_k940_inventory( memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K1040_AUTHORITY_BEFORE).cast(), ); for (index, value) in [ (0, 0x4b31303430435458), (1, root), (2, tcr), (3, vbar), (4, daif), (15, pre), (21, 12), ] { os_k1040_word(index, value); } for ordinal in 0..2 { let root = os_k1040_create_root(memory, tables)?; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_ROOT_ADDRESSES) .cast::() .add(ordinal), root.address(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_ROOT_GENERATIONS) .cast::() .add(ordinal), root.generation(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1040_ROOT_LIVE) .cast::() .add(ordinal), 1, ); } let stack = match os_k1040_allocate_stack(memory, tables, ordinal, root) { Ok(stack) => stack, Err((e, root)) => { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_FAILED_ROOTS) .cast::>() .add(ordinal), Some(root), ); } return Err(e); } }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_SPARES) .cast::>() .add(ordinal), Some(stack), ); } } let backend = OsK1040StackBackend { memory, tables, trace: true, }; let mut table = OsK1040Table::new(backend).unwrap_or_else(|_| park()); unsafe extern "C" { fn __os_k1040_a_entry(); fn __os_k1040_b_entry(); fn __os_k1040_run(wire: *const OsK1040Wire); static __os_k1040_vectors: u8; } let mut ids = [None; 2]; for ordinal in 0..2 { let stack = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1040_SPARES) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); let entry = if ordinal == 0 { __os_k1040_a_entry as *const () as u64 } else { __os_k1040_b_entry as *const () as u64 }; match table.create(stack, entry, ordinal as u64) { Ok(id) => ids[ordinal] = Some(id), Err((_error, stack)) => { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_SPARES) .cast::>() .add(ordinal), Some(stack), ); } return Err("context-table-create"); } } } let mut backend = OsK1040ContextBackend { table, bootstrap: root, current: None, pins: [false; 2], records: [None; 2], observations: [None; 2], guards: [0; 2], sleep_wait: None, sequence: 0, }; // A coherent root address/generation does not authorize corrupt physical // descriptors. Measure a real pin refusal with all twelve typed owners // still held, before restoring the exact root word for normal execution. let rejected_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_ROOT_ADDRESSES).cast::()) }; let original = unsafe { read_volatile((rejected_root as *const u64).add(2)) }; if original != 0 { return Err("context-private-root-user-branch"); } let corrupt = root | 3; unsafe { write_volatile((rejected_root as *mut u64).add(2), corrupt); asm!("dsb ish", options(nostack, preserves_flags)); } let refused = backend.pin(ids[0].unwrap(), 0); if !matches!(refused, Err(os_k1040_context::Error::Backend)) || backend.pins != [false; 2] { if let Ok(pin) = refused { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_BINDINGS).cast::>(), Some(pin), ); } } return Err("context-corrupt-root-admitted"); } backend.table.os_k1020_observe_backend(|s| { os_k940_inventory( s.memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K1040_AUTHORITY_LIVE).cast(), ); os_k1040_word( 70, s.memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64, ); os_k1040_word(71, core::mem::size_of::() as u64); }); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_ROOT_REFUSAL), [rejected_root, 2, original, corrupt, 1, 12, 0, 0], ); } os_k1040_observe_backend(&backend, ids); os_k1040_capture(0); unsafe { write_volatile((rejected_root as *mut u64).add(2), original); asm!("dsb ish", options(nostack, preserves_flags)); } backend .validate_stack_owner(ids[0].unwrap()) .map_err(|_| "context-root-refusal-restore")?; for ordinal in 0..2 { let pin = backend .pin(ids[ordinal].unwrap(), ordinal) .unwrap_or_else(|_| park()); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_BINDINGS) .cast::>() .add(ordinal), Some(pin), ); } } let mut core = os_k1040_context::ContextBank::<_, 2>::new(backend).unwrap_or_else(|_| park()); for ordinal in 0..2 { let pin = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1040_BINDINGS) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); let handle = match core.attach(pin) { Ok(h) => h, Err((_error, pin)) => { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_BINDINGS) .cast::>() .add(ordinal), Some(pin), ); } return Err("context-attach"); } }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_HANDLES) .cast::>() .add(ordinal), Some(handle), ); } } let async_bank = os_k1040_context::AsyncBank::new(core).unwrap_or_else(|_| park()); let completion = os_k1040_context::CompletionBank::new(async_bank).unwrap_or_else(|_| park()); let installation = unsafe { OsK1040Installation::install(completion) }; os_k1040_word(17, 1); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1040_ACTIVE), 1); write_volatile(core::ptr::addr_of_mut!(OS_K1040_LOOP_ACTIVE), 1); write_volatile(core::ptr::addr_of_mut!(OS_K1040_LOOP_REMAINING), 1000); } let vectors = core::ptr::addr_of!(__os_k1040_vectors) as u64; unsafe { asm!("msr VBAR_EL1, {vectors}","isb",vectors=in(reg)vectors,options(nomem,nostack,preserves_flags)); } os_k1040_capture(1); let token = unsafe { (&mut *os_k1040_bank()) .quiescent() .unwrap_or_else(|_| park()) .start(os_k1040_handle(0)) } .unwrap_or_else(|_| park()); let wire = os_k1040_depart(token); unsafe { __os_k1040_run(wire); } let after_stacks = os_k1040_stack_registers(); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_STACK_REGISTERS_AFTER), after_stacks, ); } let inactive = if before_stacks[0] == 0 { 2 } else { 1 }; if after_stacks[0] != before_stacks[0] || after_stacks[inactive] != before_stacks[inactive] { return Err("context-bootstrap-stack-register-return"); } os_k1040_capture(8); let bank = unsafe { &mut *os_k1040_bank() }; let core = bank .quiescent() .unwrap_or_else(|_| park()) .quiescent() .unwrap_or_else(|_| park()); let receipt = match core.idle().map_err(|_| "context-idle-prepare")?.complete() { Ok(receipt) => receipt, Err(_) => return Err("context-idle-handoff"), }; if !receipt.executed_wfi || receipt.work_at_handoff || receipt.wake_epoch != receipt.checked_epoch + 2 { return Err("context-idle-receipt"); } for index in 86..=88 { os_k1040_word(index, 0); } os_k1040_word(90, os_k960_counter()); let wake_token = unsafe { (&mut *os_k1040_bank()) .quiescent() .unwrap_or_else(|_| park()) .start(os_k1040_handle(1)) } .unwrap_or_else(|_| park()); let wake_wire = os_k1040_depart(wake_token); // The short bank/backend loans above end before the saved B continuation // runs. B supplies an actual CPU landing and exit before this call returns. unsafe { __os_k1040_run(wake_wire); } let wake_after_stacks = os_k1040_stack_registers(); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_STACK_REGISTERS_AFTER), wake_after_stacks, ); } if wake_after_stacks[0] != before_stacks[0] || wake_after_stacks[inactive] != before_stacks[inactive] || os_k1040_read(92) != 1 { return Err("context-woken-continuation-return"); } os_k1040_capture(12); let core = unsafe { (&mut *os_k1040_bank()) .quiescent() .unwrap_or_else(|_| park()) .quiescent() .unwrap_or_else(|_| park()) }; core.retire(os_k1040_handle(0)) .map_err(|_| "context-retire-a")?; // B has now run its real woken continuation and exited on its own stack. let before = core.snapshot(); let cpu = os_k940_registers(); if !matches!( core.start(os_k1040_handle(1)), Err(os_k1040_context::Error::State) ) || core.snapshot() != before || os_k940_registers() != cpu { return Err("context-dead-simd-owner-selection"); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1040_REFUSALS) .cast::() .add(1), 1, ); } core.retire(os_k1040_handle(1)) .map_err(|_| "context-retire-b")?; os_k1040_capture(13); os_k1040_word(17, 0); let backend = installation.extract(); let stack_backend = backend.table.into_backend().unwrap_or_else(|_| park()); os_k940_inventory( stack_backend.memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K1040_AUTHORITY_AFTER).cast(), ); unsafe { asm!("msr VBAR_EL1, {vbar}","isb",vbar=in(reg)vbar,options(nomem,nostack,preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1040_ACTIVE), 0); } if !os_k920_inventory_equal( core::ptr::addr_of!(OS_K1040_AUTHORITY_BEFORE).cast(), core::ptr::addr_of!(OS_K1040_AUTHORITY_AFTER).cast(), ) || os_k940_registers() != (root, tcr, vbar, daif) { return Err("context-owner-return"); } os_k1040_capture(14); if unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_COUNT)) } != 1000 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_REMAINING)) } != 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_LOOP_ACTIVE)) } != 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_CAPTURE_EPISODE)) } != 1005 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1040_RETURN_COUNT)) } != 2 { return Err("context-thousand-switch-accounting"); } write_line( "TURKOS/OS/K1040: PASS COOPERATIVE-SWITCHES=1000 SAMPLED-LANDINGS=8 IRQ=3 SAFEPOINT-DEFERRED WFI=1 PENDING-SKIP=1 ACTUAL-WAKE-RUN=1 OWNERS=RESTORED", ); for line in [ "TURKOS/OS/K1025: PASS DEVELOPMENT-GUEST-PARTIAL", "TURKOS/OS/K1030: PASS DEVELOPMENT-GUEST-PARTIAL", "TURKOS/OS/K1035: PASS DEVELOPMENT-GUEST-PARTIAL SAFEPOINT-DEFERRED", "TURKOS/OS/K1040: PASS DEVELOPMENT-GUEST-PARTIAL", ] { write_line(line); } Ok(()) } #[inline(never)] fn os_k1040_retain_configuration() { unsafe extern "C" { static __os_k1040_vectors: u8; fn __os_k1040_run(wire: *const OsK1040Wire); fn __os_k1040_actual_wfi(); fn __os_k1040_stack_registers(output: *mut u64); fn __os_k1040_idle_wait(epoch: u64) -> u64; fn __os_k1040_primary_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } for address in [ __os_k1040_primary_on_stack as *const () as u64, core::ptr::addr_of!(OS_K1040_PRIMARY_STACK) as u64 + 4096, core::ptr::addr_of!(OS_K1040_PRIMARY_STACK_EVIDENCE) as u64, __os_k1040_primary_body as *const () as u64, __os_k1040_context_identity as *const () as u64, __os_k1040_threads_identity as *const () as u64, core::ptr::addr_of!(OS_K1040_BANK) as u64, core::ptr::addr_of!(OS_K1040_EVIDENCE) as u64, core::ptr::addr_of!(OS_K1040_CONTEXT_WORDS) as u64, core::ptr::addr_of!(OS_K1040_CONTEXT_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1040_DEFERRED_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1040_THREAD_ID_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1040_THREAD_TABLE_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1040_LEASE_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1040_WAIT_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1040_TABLE_ADDRESSES) as u64, core::ptr::addr_of!(OS_K1040_TABLE_WORDS) as u64, core::ptr::addr_of!(OS_K1040_STACK_WORDS) as u64, core::ptr::addr_of!(OS_K1040_BACKEND_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1040_WIRE) as u64, core::ptr::addr_of!(OS_K1040_ROOT_ADDRESSES) as u64, core::ptr::addr_of!(OS_K1040_ROOT_GENERATIONS) as u64, core::ptr::addr_of!(OS_K1040_ROOT_LIVE) as u64, core::ptr::addr_of!(OS_K1040_OWNED) as u64, core::ptr::addr_of!(OS_K1040_LEASE_GENERATIONS) as u64, core::ptr::addr_of!(OS_K1040_EVENTS) as u64, core::ptr::addr_of!(OS_K1040_EVENT_COUNT) as u64, core::ptr::addr_of!(OS_K1040_LOOP_ACTIVE) as u64, core::ptr::addr_of!(OS_K1040_LOOP_REMAINING) as u64, core::ptr::addr_of!(OS_K1040_LOOP_COUNT) as u64, core::ptr::addr_of!(OS_K1040_LOOP_PENDING) as u64, core::ptr::addr_of!(OS_K1040_LOOP_SOURCE) as u64, core::ptr::addr_of!(OS_K1040_LOOP_LANDING) as u64, core::ptr::addr_of!(OS_K1040_LOOP_WORDS) as u64, core::ptr::addr_of!(OS_K1040_REFUSALS) as u64, core::ptr::addr_of!(OS_K1040_CAPTURE_EPISODE) as u64, core::ptr::addr_of!(OS_K1040_LIVE_CAPTURE_ADDRESS) as u64, core::ptr::addr_of!(OS_K1040_LANDING_ARGUMENT) as u64, core::ptr::addr_of!(OS_K1040_LANDING_PC) as u64, core::ptr::addr_of!(OS_K1040_LANDING_FRAME) as u64, core::ptr::addr_of!(OS_K1040_LANDING_FRAMES) as u64, core::ptr::addr_of!(OS_K1040_SOURCE_FRAMES) as u64, core::ptr::addr_of!(OS_K1040_SOURCE_EPISODES) as u64, core::ptr::addr_of!(OS_K1040_LANDING_PCS) as u64, core::ptr::addr_of!(OS_K1040_LANDING_ARGUMENTS) as u64, core::ptr::addr_of!(OS_K1040_BOOT_CAPTURE_ADDRESS) as u64, core::ptr::addr_of!(OS_K1040_BOOT_FRAME) as u64, core::ptr::addr_of!(OS_K1040_BOOT_FRAMES) as u64, core::ptr::addr_of!(OS_K1040_BOOT_STACK_HISTORY) as u64, core::ptr::addr_of!(OS_K1040_RETURN_COUNT) as u64, core::ptr::addr_of!(OS_K1040_BOOT_STACK_REGISTERS) as u64, core::ptr::addr_of!(OS_K1040_STACK_REGISTERS_BEFORE) as u64, core::ptr::addr_of!(OS_K1040_STACK_REGISTERS_AFTER) as u64, core::ptr::addr_of!(OS_K1040_IDLE_STACK_REGISTERS) as u64, core::ptr::addr_of!(OS_K1040_IDLE_BOUNDARY) as u64, core::ptr::addr_of!(OS_K1040_IDLE_RECEIPTS) as u64, core::ptr::addr_of!(OS_K1040_ROOT_REFUSAL) as u64, core::ptr::addr_of!(OS_K1040_DEPARTED_NONCE) as u64, core::ptr::addr_of!(OS_K1040_CRITICAL) as u64, core::ptr::addr_of!(OS_K1040_SAFE_POINT) as u64, core::ptr::addr_of!(OS_K1040_IRQ_TCO) as u64, core::ptr::addr_of!(OS_K1040_IRQ_CAPTURE_ADDRESS) as u64, core::ptr::addr_of!(OS_K1040_IRQ_CAPTURE_EPISODE) as u64, core::ptr::addr_of!(OS_K1040_IRQ_FRAMES) as u64, core::ptr::addr_of!(OS_K1040_IRQ_PHYSICAL) as u64, core::ptr::addr_of!(OS_K1040_WAKE_EPOCH) as u64, core::ptr::addr_of!(OS_K1040_CAPTURE_GATE) as u64, core::ptr::addr_of!(OS_K1040_IDENTITY) as u64, core::ptr::addr_of!(OS_K1040_THREADS_IDENTITY) as u64, core::ptr::addr_of!(OS_K1040_INTERVENING_SUM) as u64, core::ptr::addr_of!(OS_K1040_IRQ_SEEN) as u64, core::ptr::addr_of!(OS_K1040_ACTIVE) as u64, core::ptr::addr_of!(OS_K1040_PRE_IRQ) as u64, core::ptr::addr_of!(OS_K1040_PRE_LAST) as u64, core::ptr::addr_of!(OS_K1040_PRE_ACTIVE) as u64, core::ptr::addr_of!(OS_K1040_AUTHORITY_BEFORE) as u64, core::ptr::addr_of!(OS_K1040_AUTHORITY_LIVE) as u64, core::ptr::addr_of!(OS_K1040_AUTHORITY_AFTER) as u64, core::ptr::addr_of!(OS_K1040_DIAGNOSTICS) as u64, core::ptr::addr_of!(__os_k1040_vectors) as u64, __os_k1040_run as *const () as u64, __os_k1040_actual_wfi as *const () as u64, __os_k1040_stack_registers as *const () as u64, __os_k1040_idle_wait as *const () as u64, ] { unsafe { asm!("ldrb wzr, [{address}]",address=in(reg)address,options(nostack,preserves_flags)); } } } const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k1040_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); global_asm!(include_str!("os_k1040_context.S")); const OS_K1060_PA_MASK: u64 = 0x0000_ffff_ffff_f000; /// Owns only the new root page. Kernel branches remain separately owned by the /// inherited Tables and are validated as EL0 inaccessible before sharing. struct OsK1060Root { lease: Option, bootstrap: u64, branches: [u64; 2], } impl OsK1060Root { fn address(&self) -> u64 { self.lease.as_ref().map_or(0, |l| l.address()) } fn generation(&self) -> u64 { self.lease.as_ref().map_or(0, |l| l.generation()) } fn validate(&self, memory: &OsK660Memory, tables: &mmu::Tables) -> Result<(), &'static str> { let lease = self.lease.as_ref().ok_or("context-root-token")?; if self.bootstrap != tables.root || lease.address() == tables.root || lease.address() % 4096 != 0 || lease.generation() == 0 { return Err("context-root-owner"); } memory .allocator .os_authority() .validate_lease(lease) .map_err(|_| "context-root-generation")?; os_k1060_kernel_branches(tables, self.branches)?; os_k1060_root_words(lease.address(), self.branches) } } fn os_k1060_shared_walk( tables: &mmu::Tables, address: u64, level: u8, visited: &mut [u64; 4], count: usize, ) -> Result<(), &'static str> { if !(1..=3).contains(&level) || count >= 4 || visited[..count].contains(&address) || !tables.tagged_frames().any(|frame| frame.address == address) { return Err("context-shared-kernel-owner"); } visited[count] = address; for index in 0..512 { let raw = unsafe { read_volatile((address as *const u64).add(index)) }; if raw == 0 { continue; } match raw & 3 { 3 if level < 3 => { if raw != (raw & OS_K1060_PA_MASK) | 3 { return Err("context-shared-kernel-table-flags"); } os_k1060_shared_walk( tables, raw & OS_K1060_PA_MASK, level + 1, visited, count + 1, )?; } 1 if level < 3 => { if raw & (1 << 6) != 0 { return Err("context-shared-kernel-user-block"); } } 3 if level == 3 => { if raw & (1 << 6) != 0 { return Err("context-shared-kernel-user-page"); } } _ => return Err("context-shared-kernel-descriptor"), } } Ok(()) } fn os_k1060_root_words(address: u64, branches: [u64; 2]) -> Result<(), &'static str> { if address == 0 || address & !OS_K1060_PA_MASK != 0 { return Err("context-root-shape-address"); } // The caller has already checked the live typed root owner (or the // immutable pin established from that owner before IRQ unmask). Check the // entire page before accepting any later observation or root selection. for index in 0..512 { let expected = branches.get(index).copied().unwrap_or(0); if unsafe { read_volatile((address as *const u64).add(index)) } != expected { return Err("context-root-shape"); } } Ok(()) } fn os_k1060_kernel_branches(tables: &mmu::Tables, branches: [u64; 2]) -> Result<(), &'static str> { // Establish the inherited root ownership before forming its physical // pointer. Child table ownership is established independently at each // recursive step, before its pointer is formed too. if !tables .tagged_frames() .any(|frame| frame.address == tables.root) { return Err("context-shared-root-owner"); } for (index, raw) in branches.into_iter().enumerate() { if unsafe { read_volatile((tables.root as *const u64).add(index)) } != raw { return Err("context-shared-root-changed"); } match raw & 3 { 0 if raw == 0 => {} 3 => { if raw != (raw & OS_K1060_PA_MASK) | 3 { return Err("context-shared-root-table-flags"); } os_k1060_shared_walk(tables, raw & OS_K1060_PA_MASK, 2, &mut [0; 4], 0)?; } 1 if raw & (1 << 6) == 0 => {} _ => return Err("context-shared-root-descriptor"), } } Ok(()) } fn os_k1060_create_root( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result { let (root, tcr, _, daif) = os_k940_registers(); if root != tables.root || tcr & 63 != 25 || daif & 0x80 == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { return Err("context-root-bootstrap-context"); } // Only kernel L1 branches0/1 are inherited; user branch2 is absent. if !tables.tagged_frames().any(|frame| frame.address == root) { return Err("context-shared-root-owner"); } let mut branches = [0u64; 2]; for (index, raw) in branches.iter_mut().enumerate() { *raw = unsafe { read_volatile((root as *const u64).add(index)) }; } os_k1060_kernel_branches(tables, branches)?; let address = memory .allocator .os_authority() .free_regions() .iter() .rev() .find_map(|region| { region .end .checked_sub(4096) .filter(|address| *address >= region.start) }) .ok_or("context-root-ENOSPC")?; let lease = memory .allocator .os_authority_mut() .allocate_at(address) .map_err(|_| "context-root-acquire")?; let owned = OsK1060Root { lease: Some(lease), bootstrap: root, branches, }; for index in 0..512 { unsafe { write_volatile((address as *mut u64).add(index), 0); } } for (index, raw) in branches.into_iter().enumerate() { unsafe { write_volatile((address as *mut u64).add(index), raw); } } unsafe { asm!("dsb ishst", options(nostack, preserves_flags)); } os_k1060_event(0x41, address, owned.generation(), root); Ok(owned) } fn os_k1060_select_root(root: u64) -> Result<(), &'static str> { let (_, tcr, _, daif) = os_k940_registers(); if root == 0 || root % 4096 != 0 || root & !OS_K1060_PA_MASK != 0 || tcr & 63 != 25 || daif & 0x80 == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { return Err("context-root-select-context"); } unsafe { asm!("dsb ishst", options(nostack, preserves_flags)); } os_k1060_event(0x43, root, 0, 0); unsafe { asm!("msr TTBR0_EL1, {root}",root=in(reg)root,options(nostack,preserves_flags)); } os_k1060_event(0x44, root, 0, 0); unsafe { asm!("tlbi vmalle1is", options(nostack, preserves_flags)); } os_k1060_event(0x45, root, 0, 0); unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } os_k1060_event(0x46, root, 0, 0); unsafe { asm!("isb", options(nostack, preserves_flags)); } os_k1060_event(0x47, root, 0, 0); if os_k940_registers().0 != root { return Err("context-root-readback"); } Ok(()) } fn os_k1060_erase_inactive_root( root: &OsK1060Root, tables: &mmu::Tables, ) -> Result<(), &'static str> { let actual = os_k940_registers(); if actual.0 != tables.root || actual.3 & 0x80 == 0 || root.bootstrap != tables.root || root.address() == actual.0 || root.address() == 0 { return Err("context-root-active-free-refusal"); } // Bootstrap select completed a full TLBI before this operation. Erasure // alone returns no authority: StackBackend publishes one atomic six-lease // return only after its guard restoration and full stack erasure too. for index in 0..512 { unsafe { write_volatile((root.address() as *mut u64).add(index), 0); } } unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } if !(0..512) .all(|index| unsafe { read_volatile((root.address() as *const u64).add(index)) } == 0) { return Err("context-root-erasure"); } os_k1060_event(0x48, root.address(), root.generation(), 0); Ok(()) } use turkos_kernel::runtime::runqueue_threads as os_k1060_thread; const OS_K1060_STACK_COUNT: usize = 3; const OS_K1060_STACK_PAGES: usize = 7; const OS_K1060_OWNED_PAGES: usize = OS_K1060_STACK_COUNT * (OS_K1060_STACK_PAGES + 2); struct OsK1060Stack { root: OsK1060Root, ordinal: usize, start: u64, leases: [Option; 8], guard: Option, initial_observation: core::cell::Cell, } impl OsK1060Stack { fn layout(&self) -> os_k1060_thread::StackLayout { os_k1060_thread::StackLayout { base: self.start + 4096, physical: self.start + 4096, bytes: OS_K1060_STACK_PAGES as u64 * 4096, } } } fn os_k1060_cpu_root_allowed(root: u64, tables: &mmu::Tables) -> bool { root==tables.root||(0..3).any(|slot|unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_ROOT_ADDRESSES).cast::().add(slot))}==root&&root!=0&&unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_ROOT_LIVE).cast::().add(slot))}==1) } struct OsK1060StackBackend<'a> { memory: &'a mut OsK660Memory, tables: &'a mmu::Tables, trace: bool, } fn os_k1060_execution() -> os_k1060_thread::Execution { let (cpu, daif, sp): (u64, u64, u64); unsafe { asm!("mrs {cpu}, MPIDR_EL1", "mrs {daif}, DAIF", "mov {sp}, sp", cpu=out(reg)cpu, daif=out(reg)daif, sp=out(reg)sp, options(nomem,nostack,preserves_flags)); } os_k1060_thread::Execution { cpu: (cpu & 0xff) as u32, irq_masked: daif & 0x80 != 0, in_irq: unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0, sp, } } fn os_k1060_allocate_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ordinal: usize, root: OsK1060Root, ) -> Result { if ordinal >= 3 || root.validate(memory, tables).is_err() { return Err(("context-stack-root-owner", root)); } let authority = memory.allocator.os_authority(); let mut selected = None; let mut checked = 0; for region in authority.free_regions() { let mut address = region.start; while address .checked_add(8 * 4096) .is_some_and(|end| end <= region.end) && checked < 512 { checked += 1; if tables.os_k1020_stack_candidate(address, address + 8 * 4096) { selected = Some(address); break; } let Some(next) = address.checked_add(4096) else { return Err(("context-stack-candidate-overflow", root)); }; address = next; } if selected.is_some() { break; } } let Some(start) = selected else { return Err(("context-stack-no-owned-L3-span", root)); }; let leases = match memory .allocator .os_authority_mut() .allocate_many_at(start, 8) { Ok(leases) => leases, Err(_) => return Err(("context-stack-acquire", root)), }; // The eight live stack leases are immediately inside this actual owner; // failure cannot discard a prefix or the independently retained root. let stack = OsK1060Stack { root, ordinal, start, leases, guard: None, initial_observation: core::cell::Cell::new(false), }; for offset in 0..8 * 4096 / 8 { unsafe { write_volatile((start as *mut u64).add(offset), 0xa5a5a5a5a5a5a5a5); } } for index in 0..9 { let lease = if index < 8 { stack.leases[index].as_ref().unwrap_or_else(|| park()) } else { stack.root.lease.as_ref().unwrap_or_else(|| park()) }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_OWNED) .cast::() .add(ordinal * 9 + index), lease.address(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_LEASE_GENERATIONS) .cast::() .add(ordinal * 9 + index), lease.generation(), ); } os_k1060_event(1, ordinal as u64, lease.address(), lease.generation()); } Ok(stack) } impl OsK1060StackBackend<'_> { fn validate(&self, stack: &OsK1060Stack) -> Result<(), os_k1060_thread::Error> { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || !os_k1060_cpu_root_allowed(os_k940_registers().0, self.tables) { return Err(os_k1060_thread::Error::Context); } if stack.ordinal >= OS_K1060_STACK_COUNT || stack.start % 4096 != 0 || stack.leases[OS_K1060_STACK_PAGES + 1..] .iter() .any(Option::is_some) { return Err(os_k1060_thread::Error::Layout); } for index in 0..=OS_K1060_STACK_PAGES { let lease = stack.leases[index] .as_ref() .ok_or(os_k1060_thread::Error::Backend)?; if lease.address() != stack.start + index as u64 * 4096 { return Err(os_k1060_thread::Error::Layout); } self.memory .allocator .os_authority() .validate_lease(lease) .map_err(|_| os_k1060_thread::Error::Backend)?; } stack .root .validate(self.memory, self.tables) .map_err(|_| os_k1060_thread::Error::Backend)?; Ok(()) } } unsafe impl os_k1060_thread::Backend for OsK1060StackBackend<'_> { type Stack = OsK1060Stack; fn execution(&self) -> os_k1060_thread::Execution { os_k1060_execution() } fn layout( &self, stack: &OsK1060Stack, ) -> Result { self.validate(stack)?; Ok(stack.layout()) } fn entry_executable(&self, entry: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } entry.is_multiple_of(4) && entry >= core::ptr::addr_of!(_text_start) as u64 && entry < core::ptr::addr_of!(_text_end) as u64 } fn prepare(&mut self, stack: &mut OsK1060Stack) -> Result<(), os_k1060_thread::Error> { self.validate(stack)?; let layout = stack.layout(); let e = os_k1060_execution(); if e.sp >= stack.start && e.sp <= layout.base + layout.bytes || stack.guard.is_some() || os_k940_registers().0 != self.tables.root { return Err(os_k1060_thread::Error::ActiveStack); } let guard = unsafe { self.tables .os_k1020_acquire_stack_guard(stack.start, layout.base + layout.bytes) } .map_err(|_| os_k1060_thread::Error::Guard)?; stack.guard = Some(guard); for offset in 0..layout.bytes / 8 { unsafe { write_volatile((layout.base as *mut u64).add(offset as usize), 0); } } unsafe { write_volatile(layout.base as *mut u64, os_k1060_thread::LOW_CANARY); write_volatile( (layout.base + layout.bytes - 8) as *mut u64, os_k1060_thread::HIGH_CANARY, ); asm!("dsb ish", options(nostack, preserves_flags)); } stack.initial_observation.set(true); if self.trace { os_k1060_event(2, stack.ordinal as u64, stack.start, layout.bytes); } Ok(()) } fn observe( &self, stack: &OsK1060Stack, ) -> Result { self.validate(stack)?; let layout = stack.layout(); let token = stack.guard.as_ref().ok_or(os_k1060_thread::Error::Guard)?; let descriptor = self .tables .os_k1020_stack_guard_descriptor(token) .map_err(|_| os_k1060_thread::Error::Guard)?; if token.guard() != stack.start || token.end() != layout.base + layout.bytes { return Err(os_k1060_thread::Error::Layout); } // Initial full erasure is observed once before publication. Later // selection/exit checks read the actual guard and canaries without // walking the active or previously used Rust stack contents. let zeroed=stack.initial_observation.get()&&(1..layout.bytes/8-1).all(|offset|unsafe {read_volatile((layout.base as *const u64).add(offset as usize))}==0); if zeroed { stack.initial_observation.set(false); } Ok(os_k1060_thread::StackObservation { layout, guard_descriptor: descriptor, low_canary: unsafe { read_volatile(layout.base as *const u64) }, high_canary: unsafe { read_volatile((layout.base + layout.bytes - 8) as *const u64) }, initially_zeroed: zeroed, }) } fn release( &mut self, mut stack: OsK1060Stack, ) -> Result<(), (os_k1060_thread::Error, OsK1060Stack)> { if let Err(error) = self.validate(&stack) { return Err((error, stack)); } let layout = stack.layout(); let e = os_k1060_execution(); if e.sp >= stack.start && e.sp <= layout.base + layout.bytes { return Err((os_k1060_thread::Error::ActiveStack, stack)); } if os_k1060_erase_inactive_root(&stack.root, self.tables).is_err() { return Err((os_k1060_thread::Error::Backend, stack)); } let token = match stack.guard.take() { Some(token) => token, None => return Err((os_k1060_thread::Error::Guard, stack)), }; if let Err((_error, token)) = unsafe { self.tables.os_k1020_restore_stack_guard(token) } { stack.guard = Some(token); return Err((os_k1060_thread::Error::Guard, stack)); } for offset in 0..(OS_K1060_STACK_PAGES + 1) * 4096 / 8 { unsafe { write_volatile((stack.start as *mut u64).add(offset), 0); } } unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } if !(0..(OS_K1060_STACK_PAGES + 1) * 4096 / 8) .all(|offset| unsafe { read_volatile((stack.start as *const u64).add(offset)) } == 0) { return Err((os_k1060_thread::Error::Backend, stack)); } let refs = core::array::from_fn::<_, 9, _>(|index| { if index < 8 { stack.leases[index].as_ref().unwrap() } else { stack.root.lease.as_ref().unwrap() } }); if self .memory .allocator .os_authority_mut() .release_many_leases(&refs) .is_err() { return Err((os_k1060_thread::Error::Backend, stack)); } if self.trace { os_k1060_event(3, stack.ordinal as u64, stack.start, 8); } for lease in &mut stack.leases { *lease = None; } stack.root.lease = None; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_ROOT_LIVE) .cast::() .add(stack.ordinal), 0, ); } Ok(()) } } use turkos_kernel::runtime::runqueue_context as os_k1060_context; type OsK1060Table<'a> = os_k1060_thread::Table, 3>; /// Private nonclone pin for one real Table stack/root owner. The actual nine /// central leases stay in Table::Stack until this pin is consumed by release. struct OsK1060Binding { id: os_k1060_thread::ThreadId, ordinal: usize, root: u64, root_generation: u64, branches: [u64; 2], } struct OsK1060Prepared { source: Option, target: os_k1060_thread::ThreadId, nonce: u64, departed: bool, } struct OsK1060ReturnPrepared { source: os_k1060_thread::ThreadId, bootstrap_root: u64, bootstrap: os_k1060_context::Cooperative, address: u64, nonce: u64, } struct OsK1060IdlePrepared { epoch: u64, wait: Option, } struct OsK1060ContextBackend<'a> { table: OsK1060Table<'a>, bootstrap: u64, current: Option, pins: [bool; 3], records: [Option; 3], observations: [Option; 3], guards: [u64; 3], sleep_wait: Option, sequence: u64, } impl OsK1060ContextBackend<'_> { fn validate_stack_owner( &self, id: os_k1060_thread::ThreadId, ) -> Result<(), os_k1060_context::Error> { if os_k1060_execution().in_irq { return Err(os_k1060_context::Error::Context); } self.table .os_k1020_observe_backend(|backend| { self.table .os_k1020_observe_stack(id, |stack| backend.validate(stack)) }) .map_err(|_| os_k1060_context::Error::Stack)? .map_err(|_| os_k1060_context::Error::Backend) } fn valid_pin(&self, b: &OsK1060Binding) -> Result<(), os_k1060_context::Error> { if b.ordinal >= 3 || !self.pins[b.ordinal] || b.root == 0 || b.root_generation == 0 { return Err(os_k1060_context::Error::Stale); } let r = self.records[b.ordinal].ok_or(os_k1060_context::Error::Stale)?; if r.id != b.id { return Err(os_k1060_context::Error::Stale); } let root = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_ROOT_ADDRESSES) .cast::() .add(b.ordinal), ) }; let generation = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_ROOT_GENERATIONS) .cast::() .add(b.ordinal), ) }; if root != b.root || generation != b.root_generation || unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_ROOT_LIVE) .cast::() .add(b.ordinal), ) } != 1 { return Err(os_k1060_context::Error::Stale); } // Immutable IRQ pin: the typed central/root/stack owner was validated // by Table in main context before this pin was published. IRQ reads // only these bounded physical words and its private pin, never Table // or the allocator. Main observations additionally revalidate Table. os_k1060_root_words(b.root, b.branches).map_err(|_| os_k1060_context::Error::Backend)?; if !os_k1060_execution().in_irq { self.validate_stack_owner(b.id)?; } Ok(()) } fn refresh(&mut self) -> Result<(), os_k1060_context::Error> { if os_k1060_execution().in_irq { return Err(os_k1060_context::Error::Context); } for ordinal in 0..3 { if let Some(previous) = self.records[ordinal] { if !self.pins[ordinal] { continue; } let record = self .table .record(previous.id) .map_err(|_| os_k1060_context::Error::State)?; self.validate_stack_owner(previous.id)?; let observation = self .table .os_k1020_observe_stack(previous.id, |stack| { let guard = stack .guard .as_ref() .map_or(0, |guard| guard.descriptor_address()); ( stack.layout(), guard, stack.root.address(), stack.root.generation(), ) }) .map_err(|_| os_k1060_context::Error::Stack)?; if observation.1 == 0 { return Err(os_k1060_context::Error::Guard); } self.records[ordinal] = Some(record); self.guards[ordinal] = observation.1; self.observations[ordinal] = Some(os_k1060_thread::StackObservation { layout: observation.0, guard_descriptor: unsafe { read_volatile(observation.1 as *const u64) }, low_canary: unsafe { read_volatile(observation.0.base as *const u64) }, high_canary: unsafe { read_volatile((observation.0.base + observation.0.bytes - 8) as *const u64) }, initially_zeroed: false, }); } } Ok(()) } fn pin( &mut self, id: os_k1060_thread::ThreadId, ordinal: usize, ) -> Result { if ordinal >= 3 || self.pins[ordinal] || os_k1060_execution().in_irq { return Err(os_k1060_context::Error::State); } let record = self .table .record(id) .map_err(|_| os_k1060_context::Error::State)?; self.validate_stack_owner(id)?; let (root, generation, branches) = self .table .os_k1020_observe_stack(id, |stack| { ( stack.root.address(), stack.root.generation(), stack.root.branches, ) }) .map_err(|_| os_k1060_context::Error::Stack)?; let before_records = self.records; let before_observations = self.observations; let before_guards = self.guards; self.records[ordinal] = Some(record); self.pins[ordinal] = true; if let Err(error) = self.refresh() { self.pins[ordinal] = false; self.records = before_records; self.observations = before_observations; self.guards = before_guards; return Err(error); } Ok(OsK1060Binding { id, ordinal, root, root_generation: generation, branches, }) } fn next_prepared(&mut self) -> Result { let next = self .sequence .checked_add(1) .ok_or(os_k1060_context::Error::Exhausted)?; self.sequence = next; Ok(next) } } unsafe impl os_k1060_context::Backend for OsK1060ContextBackend<'_> { type Binding = OsK1060Binding; type Prepared = OsK1060Prepared; type IdlePrepared = OsK1060IdlePrepared; fn execution(&self) -> os_k1060_context::Execution { let e = os_k1060_execution(); let (root, _, _, daif) = os_k940_registers(); let cpacr: u64; unsafe { asm!("mrs {fp}, CPACR_EL1",fp=out(reg)cpacr,options(nomem,nostack,preserves_flags)); } os_k1060_context::Execution { cpu: e.cpu, daif, in_irq: e.in_irq, sp: e.sp, root, thread: self.current, fp_enabled: cpacr & (3 << 20) == 3 << 20, critical_depth: unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_CRITICAL)) } as u32, safe_point: unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_SAFE_POINT)) } == 1, tco: if e.in_irq { unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_TCO)) } } else { 0 }, } } fn record( &self, b: &Self::Binding, ) -> Result { self.valid_pin(b)?; // IRQ uses the last real main-only Table transition, an immutable pin // and actual guard/canaries. It never borrows Table/Memory/authority. if os_k1060_execution().in_irq { return self.records[b.ordinal].ok_or(os_k1060_context::Error::Stale); } self.table .record(b.id) .map_err(|_| os_k1060_context::Error::State) } fn stack( &self, b: &Self::Binding, ) -> Result { self.valid_pin(b)?; let cached = self.observations[b.ordinal].ok_or(os_k1060_context::Error::Stack)?; if !os_k1060_execution().in_irq { self.table .record(b.id) .map_err(|_| os_k1060_context::Error::State)?; self.table .os_k1020_observe_stack(b.id, |stack| { stack.root.address() == b.root && stack.root.generation() == b.root_generation }) .map_err(|_| os_k1060_context::Error::Stack)? .then_some(()) .ok_or(os_k1060_context::Error::Stale)?; } Ok(os_k1060_thread::StackObservation { guard_descriptor: unsafe { read_volatile(self.guards[b.ordinal] as *const u64) }, low_canary: unsafe { read_volatile(cached.layout.base as *const u64) }, high_canary: unsafe { read_volatile((cached.layout.base + cached.layout.bytes - 8) as *const u64) }, ..cached }) } fn root(&self, b: &Self::Binding) -> Result { self.valid_pin(b)?; if !os_k1060_execution().in_irq { self.table .record(b.id) .map_err(|_| os_k1060_context::Error::State)?; } Ok(b.root) } fn executable(&self, pc: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } pc % 4 == 0 && pc >= core::ptr::addr_of!(_text_start) as u64 && pc < core::ptr::addr_of!(_text_end) as u64 } fn initial_return_pc(&self, _: &Self::Binding) -> Result { unsafe extern "C" { fn __os_k1060_bad_return(); } Ok(__os_k1060_bad_return as *const () as u64) } fn capture_matches(&self, b: &Self::Binding, c: &os_k1060_context::Capture) -> bool { if self.valid_pin(b).is_err() { return false; } let Some(observation) = self.observations[b.ordinal] else { return false; }; let (address, episode, bytes) = match c.image { os_k1060_context::Image::Cooperative(_) => ( unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_LIVE_CAPTURE_ADDRESS)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_CAPTURE_EPISODE)) }, 640u64, ), os_k1060_context::Image::Timer(_) => ( unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_CAPTURE_ADDRESS)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_CAPTURE_EPISODE)) }, 848u64, ), _ => return false, }; if c.address != address || c.episode != episode || episode == 0 || address % 16 != 0 || address < observation.layout.usable_bottom() || !address .checked_add(bytes) .is_some_and(|end| end <= observation.layout.usable_top()) { return false; } match c.image { os_k1060_context::Image::Cooperative(f) => { (unsafe { read_volatile(address as *const os_k1060_context::Cooperative) }) == f } os_k1060_context::Image::Timer(f) => { (unsafe { read_volatile(address as *const os_k920_exception::Frame) }) == f } _ => false, } } fn registers(&self) -> os_k1060_context::Registers { let f = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_FRAME)) }; os_k1060_context::Registers { sp: f.sp, x19_x30: f.x19_x30, q: f.q, fpcr: f.fpcr, fpsr: f.fpsr, pc: os_k1060_observed_landing_pc(&f), argument: unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_ARGUMENT)) }, timer: None, } } fn prepare( &mut self, source: Option<&Self::Binding>, target: &Self::Binding, _: &os_k1060_context::Image, _: os_k1060_context::Disposition, ) -> Result { self.valid_pin(target)?; if let Some(source) = source { self.valid_pin(source)?; } Ok(OsK1060Prepared { source: source.map(|s| s.id), target: target.id, nonce: self.next_prepared()?, departed: false, }) } fn commit( &mut self, _: &mut Self::Prepared, _: Option<&Self::Binding>, _: &Self::Binding, _: &os_k1060_context::Image, _: os_k1060_context::Disposition, ) -> Result<(), os_k1060_context::Failure> { // A synchronous bank loan must never span a suspended continuation. Err(os_k1060_context::Failure { error: os_k1060_context::Error::Context, phase: os_k1060_context::FailurePhase::BeforeSwitch, }) } fn finish( &mut self, p: Self::Prepared, ) -> Result<(), (os_k1060_context::Error, Self::Prepared)> { if !p.departed { return Err((os_k1060_context::Error::State, p)); } Ok(()) } fn abort( &mut self, p: Self::Prepared, ) -> Result<(), (os_k1060_context::Error, Self::Prepared)> { if p.departed { return Err((os_k1060_context::Error::State, p)); } Ok(()) } fn release( &mut self, b: Self::Binding, ) -> Result<(), (os_k1060_context::Error, Self::Binding)> { if let Err(e) = self.valid_pin(&b) { return Err((e, b)); } let record = match self.table.record(b.id) { Ok(r) => r, Err(_) => return Err((os_k1060_context::Error::State, b)), }; if record.state != os_k1060_thread::State::Dead || os_k940_registers().0 != self.bootstrap { return Err((os_k1060_context::Error::Active, b)); } if self.table.reap(b.id).is_err() { return Err((os_k1060_context::Error::Backend, b)); } self.pins[b.ordinal] = false; self.records[b.ordinal] = None; self.observations[b.ordinal] = None; self.guards[b.ordinal] = 0; Ok(()) } fn ready_empty(&self) -> bool { let s = self.table.snapshot(); s.ready == 0 && s.running == 0 } fn wake_epoch(&self) -> u64 { unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_WAKE_EPOCH)) } } fn idle_prepare(&mut self, epoch: u64) -> Result { let wait = self .sleep_wait .take() .ok_or(os_k1060_context::Error::Wake)?; Ok(OsK1060IdlePrepared { epoch, wait: Some(wait), }) } fn idle_commit( &mut self, p: &mut Self::IdlePrepared, epoch: u64, ) -> Result { os_k1060_idle_atomic(self, p, epoch) } fn idle_finish( &mut self, p: Self::IdlePrepared, ) -> Result<(), (os_k1060_context::Error, Self::IdlePrepared)> { if p.wait.is_some() { return Err((os_k1060_context::Error::Wake, p)); } Ok(()) } fn idle_abort( &mut self, mut p: Self::IdlePrepared, ) -> Result<(), (os_k1060_context::Error, Self::IdlePrepared)> { if self.sleep_wait.is_some() { return Err((os_k1060_context::Error::Wake, p)); } self.sleep_wait = p.wait.take(); Ok(()) } } unsafe impl os_k1060_context::HandoffBackend for OsK1060ContextBackend<'_> { fn depart( &mut self, p: &mut Self::Prepared, source: Option<&Self::Binding>, nonce: u64, d: os_k1060_context::Disposition, ) -> Result<(), os_k1060_context::Failure> { let before = |error| os_k1060_context::Failure { error, phase: os_k1060_context::FailurePhase::BeforeSwitch, }; if p.source != source.map(|s| s.id) || p.departed || p.nonce == 0 || nonce == 0 { return Err(before(os_k1060_context::Error::Stale)); } self.validate_stack_owner(p.target).map_err(before)?; if let Some(source) = source { self.valid_pin(source).map_err(before)?; match d { os_k1060_context::Disposition::Ready => self .table .yield_current(source.id) .map_err(|_| before(os_k1060_context::Error::State))?, os_k1060_context::Disposition::Sleeping => { if self.sleep_wait.is_some() { return Err(before(os_k1060_context::Error::Wake)); } self.sleep_wait = Some( self.table .sleep_current(source.id) .map_err(|_| before(os_k1060_context::Error::State))?, ); } } } p.departed = true; self.current = None; if let Err(e) = self.refresh() { return Err(os_k1060_context::Failure { error: e, phase: os_k1060_context::FailurePhase::AfterSwitch, }); } let root = match self .table .os_k1020_observe_stack(p.target, |stack| stack.root.address()) { Ok(root) => root, Err(_) => { return Err(os_k1060_context::Failure { error: os_k1060_context::Error::Stack, phase: os_k1060_context::FailurePhase::AfterSwitch, }); } }; if os_k1060_select_root(root).is_err() { return Err(os_k1060_context::Failure { error: os_k1060_context::Error::Backend, phase: os_k1060_context::FailurePhase::AfterSwitch, }); } Ok(()) } fn arrive( &mut self, p: &mut Self::Prepared, target: &Self::Binding, _: &os_k1060_context::Image, nonce: u64, ) -> Result<(), os_k1060_context::Error> { self.valid_pin(target)?; if !p.departed || p.target != target.id || nonce != unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_DEPARTED_NONCE)) } || os_k940_registers().0 != target.root { return Err(os_k1060_context::Error::Capture); } self.table .activate(target.id) .map_err(|_| os_k1060_context::Error::State)?; self.current = Some(target.id); self.refresh() } } unsafe impl os_k1060_context::CompletionBackend for OsK1060ContextBackend<'_> { type ReturnPrepared = OsK1060ReturnPrepared; fn prepare_return( &mut self, source: &Self::Binding, _: &os_k1060_context::Capture, ) -> Result { self.valid_pin(source)?; let address = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_BOOT_CAPTURE_ADDRESS)) }; if !os_k1060_boot_frame_range(address) { return Err(os_k1060_context::Error::Stack); } let bootstrap = unsafe { read_volatile(address as *const os_k1060_context::Cooperative) }; if bootstrap.sp != address + 640 || bootstrap.canary != os_k1060_context::CONTEXT_CANARY || bootstrap.padding != 0 { return Err(os_k1060_context::Error::Capture); } Ok(OsK1060ReturnPrepared { source: source.id, bootstrap_root: self.bootstrap, bootstrap, address, nonce: self.next_prepared()?, }) } fn leave_return( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Binding, ) -> Result<(), os_k1060_context::Failure> { let before = |error| os_k1060_context::Failure { error, phase: os_k1060_context::FailurePhase::BeforeSwitch, }; self.valid_pin(source).map_err(before)?; if p.source != source.id || p.nonce == 0 { return Err(before(os_k1060_context::Error::Stale)); } self.table .exit_current(source.id) .map_err(|_| before(os_k1060_context::Error::State))?; self.current = None; if self.refresh().is_err() || os_k1060_select_root(p.bootstrap_root).is_err() { return Err(os_k1060_context::Failure { error: os_k1060_context::Error::Backend, phase: os_k1060_context::FailurePhase::AfterSwitch, }); } Ok(()) } fn returned(&self, p: &Self::ReturnPrepared) -> Result<(), os_k1060_context::Error> { unsafe extern "C" { fn __os_k1060_boot_landed(); } let actual = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_FRAME)) }; let stacks = os_k1060_stack_registers(); let original = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_BOOT_STACK_REGISTERS)) }; if actual != p.bootstrap || !os_k1060_boot_frame_range(p.address) || actual.sp != p.address + 640 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_PC)) } != __os_k1060_boot_landed as *const () as u64 || stacks[0] != original[0] || stacks[if original[0] == 0 { 2 } else { 1 }] != original[if original[0] == 0 { 2 } else { 1 }] { return Err(os_k1060_context::Error::Capture); } Ok(()) } fn return_root(&self, p: &Self::ReturnPrepared) -> Result { if p.bootstrap_root != self.bootstrap { return Err(os_k1060_context::Error::Stack); } Ok(p.bootstrap_root) } fn finish_return( &mut self, _: Self::ReturnPrepared, ) -> Result<(), (os_k1060_context::Error, Self::ReturnPrepared)> { Ok(()) } fn abort_return( &mut self, _: Self::ReturnPrepared, ) -> Result<(), (os_k1060_context::Error, Self::ReturnPrepared)> { Ok(()) } } unsafe impl os_k1060_context::InactiveBackend for OsK1060ContextBackend<'_> { fn cancel_inactive( &mut self, binding: &Self::Binding, ) -> Result<(), os_k1060_context::Failure> { let before = |error| os_k1060_context::Failure { error, phase: os_k1060_context::FailurePhase::BeforeSwitch, }; self.valid_pin(binding).map_err(before)?; if self.current.is_some() || os_k940_registers().0 != self.bootstrap { return Err(before(os_k1060_context::Error::Context)); } self.table .cancel(binding.id) .map_err(|_| before(os_k1060_context::Error::State))?; // Any retained sleeping authorization was invalidated by actual // Table.cancel. It is never reused to manufacture a wake after cancel. if self .sleep_wait .as_ref() .is_some_and(|wait| wait.thread() == binding.id) { self.sleep_wait = None; } self.refresh().map_err(|error| os_k1060_context::Failure { error, phase: os_k1060_context::FailurePhase::AfterSwitch, }) } } #[unsafe(no_mangle)] static OS_K1060_THREAD_ID_LAYOUT_EXPECTED: [u64; 4] = os_k1060_thread::ThreadId::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1060_THREAD_TABLE_LAYOUT_EXPECTED: [u64; 20] = OsK1060Table::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1060_LEASE_LAYOUT_EXPECTED: [u64; 7] = os_k720_frame::Lease::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1060_WAIT_LAYOUT_EXPECTED: [u64; 5] = os_k1060_thread::Wait::os_k1040_layout(); #[unsafe(no_mangle)] static mut OS_K1060_TABLE_ADDRESSES: [[u64; 3]; 3] = [[0; 3]; 3]; #[unsafe(no_mangle)] static mut OS_K1060_TABLE_WORDS: [[u64; 12]; 3] = [[0; 12]; 3]; #[unsafe(no_mangle)] static mut OS_K1060_STACK_WORDS: [[u64; 23]; 3] = [[0; 23]; 3]; #[unsafe(no_mangle)] static OS_K1060_BACKEND_LAYOUT_EXPECTED: [u64; 30] = [ core::mem::size_of::>() as u64, core::mem::offset_of!(OsK1060ContextBackend<'static>, table) as u64, core::mem::offset_of!(OsK1060ContextBackend<'static>, bootstrap) as u64, core::mem::offset_of!(OsK1060ContextBackend<'static>, pins) as u64, core::mem::offset_of!(OsK1060ContextBackend<'static>, sequence) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1060Binding, id) as u64, core::mem::offset_of!(OsK1060Binding, ordinal) as u64, core::mem::offset_of!(OsK1060Binding, root) as u64, core::mem::offset_of!(OsK1060Binding, root_generation) as u64, core::mem::offset_of!(OsK1060Binding, branches) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1060Stack, root) as u64, core::mem::offset_of!(OsK1060Stack, ordinal) as u64, core::mem::offset_of!(OsK1060Stack, start) as u64, core::mem::offset_of!(OsK1060Stack, leases) as u64, core::mem::offset_of!(OsK1060Stack, guard) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1060Root, lease) as u64, core::mem::offset_of!(OsK1060Root, bootstrap) as u64, core::mem::offset_of!(OsK1060Root, branches) as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::size_of::>() as u64, core::mem::offset_of!(OsK1060ContextBackend<'static>, current) as u64, core::mem::offset_of!(OsK1060ContextBackend<'static>, sleep_wait) as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1060IdlePrepared, epoch) as u64, core::mem::offset_of!(OsK1060IdlePrepared, wait) as u64, ]; fn os_k1060_observe_backend( backend: &OsK1060ContextBackend<'_>, ids: [Option; 3], ) { let actual=os_k1060_execution(); if actual.cpu!=0 || !actual.irq_masked || actual.in_irq { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_TABLE_ADDRESSES), backend.table.os_k1040_addresses(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_TABLE_WORDS), backend.table.os_k1020_slot_words(), ); } for ordinal in 0..3 { let mut words = [0; 23]; if let Some(id) = ids[ordinal] { backend .table .os_k1020_observe_stack(id, |stack| { words[..8].copy_from_slice(&[ stack.start, stack.root.address(), stack.root.generation(), stack.root.bootstrap, stack.root.branches[0], stack.root.branches[1], stack.ordinal as u64, stack.guard.as_ref().map_or(0, |g| g.descriptor_address()), ]); for index in 0..9 { let lease = if index < 8 { stack.leases[index].as_ref() } else { stack.root.lease.as_ref() }; words[8 + index] = lease.map_or(0, |v| v as *const os_k720_frame::Lease as u64); } let layout = stack.layout(); words[17] = layout.base; words[18] = layout.bytes; }) .unwrap_or_else(|_| park()); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_STACK_WORDS) .cast::<[u64; 23]>() .add(ordinal), words, ); } } os_k1060_word(84, backend as *const OsK1060ContextBackend<'_> as u64); os_k1060_word(85, core::mem::size_of::>() as u64); } // NEW reusable three-member backend. The real typed bindings/backing owners // remain exclusively inside CompletionBank; no binding is reconstructed. use turkos_kernel::runtime::runqueue as os_k1060_queue; struct OsK1060Member { identity: u64, context: os_k1060_context::Handle, thread: os_k1060_thread::ThreadId, ordinal: usize, retired: bool, } struct OsK1060SelectionPrepared { handoff: Option, epoch: u64, nonce: u64, arrived: bool, } struct OsK1060QueueReturnPrepared { token: Option, source: os_k1060_context::Handle, nonce: u64, observed: bool, } struct OsK1060QueueBackend { bank: os_k1060_context::CompletionBank, identity: u64, members: [Option; 3], failed_tick: Option, retirement_quarantined: bool, } impl OsK1060QueueBackend { #[allow(clippy::result_large_err)] fn new( bank: os_k1060_context::CompletionBank, ) -> Result< Self, ( os_k1060_queue::Error, os_k1060_context::CompletionBank, ), > { let snapshot = bank.handoff().bank().snapshot(); if bank.quarantined() || bank.observation().is_some() || bank.handoff().quarantined() || bank.handoff().stage().is_some() || snapshot.retained != 3 || snapshot.active != 0 || snapshot.ready != 3 || snapshot.quarantined != 0 || snapshot.faulted || snapshot.retained_switch || snapshot.retained_capture { return Err((os_k1060_queue::Error::State, bank)); } let e = bank .handoff() .bank() .with_backend_observation(|b| os_k1060_context::Backend::execution(b)); if e.cpu != 0 || e.daif & 128 == 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point || e.thread.is_some() { return Err((os_k1060_queue::Error::Context, bank)); } unsafe extern "C" { fn __os_k1060_backend_identity() -> u64; } let identity = unsafe { __os_k1060_backend_identity() }; if identity == 0 { return Err((os_k1060_queue::Error::Exhausted, bank)); } Ok(Self { bank, identity, members: [None; 3], failed_tick: None, retirement_quarantined: false, }) } fn issue( &mut self, handle: os_k1060_context::Handle, ordinal: usize, ) -> Result { if self.retirement_quarantined { return Err(os_k1060_queue::Error::Quarantined); } if ordinal >= 3 || self.members[ordinal].is_some() || self.members.contains(&Some(handle)) { return Err(os_k1060_queue::Error::Duplicate); } let o = self .bank .os_k1060_observe_member(handle) .map_err(|_| os_k1060_queue::Error::Backend)?; if o.thread.state != os_k1060_thread::State::Ready || o.context.state != os_k1060_context::State::Ready { return Err(os_k1060_queue::Error::State); } self.members[ordinal] = Some(handle); Ok(OsK1060Member { identity: self.identity, context: handle, thread: o.thread.id, ordinal, retired: false, }) } fn member( &self, member: &OsK1060Member, ) -> Result { if self.retirement_quarantined { return Err(os_k1060_queue::Error::Quarantined); } if member.retired { return Err(os_k1060_queue::Error::State); } if member.identity != self.identity || member.ordinal >= 3 || self.members[member.ordinal] != Some(member.context) { return Err(os_k1060_queue::Error::Stale); } let o = self .bank .os_k1060_observe_member(member.context) .map_err(|_| os_k1060_queue::Error::Backend)?; if o.thread.id != member.thread || o.context.thread != member.thread { return Err(os_k1060_queue::Error::Stale); } Ok(o) } } unsafe impl os_k1060_queue::Backend for OsK1060QueueBackend { type Member = OsK1060Member; type Prepared = OsK1060SelectionPrepared; type Tick = B::QueueTick; fn execution(&self) -> os_k1060_queue::Execution { let e = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1060_context::Backend::execution(b)); os_k1060_queue::Execution { cpu: e.cpu, masked: e.daif & 128 != 0, in_irq: e.in_irq, thread: e.thread, root: e.root, } } fn observe( &self, m: &Self::Member, ) -> Result { self.member(m) } fn tick( &self, t: &Self::Tick, ) -> Result { self.bank.os_k1060_tick(t) } fn consume_tick(&mut self, t: Self::Tick) { if let Err((_e, actual)) = self.bank.os_k1060_consume_tick(t) { self.failed_tick = Some(actual); park(); } } fn prepare( &mut self, source: Option<&Self::Member>, target: &Self::Member, epoch: u64, d: os_k1060_queue::Disposition, ) -> Result { self.member(target)?; if let Some(source) = source { self.member(source)?; } let capture = if source.is_some() { Some( self.bank .os_k1060_capture() .map_err(|_| os_k1060_queue::Error::Backend)?, ) } else { None }; let bank = self .bank .quiescent() .map_err(|_| os_k1060_queue::Error::Busy)?; let handoff = match source { None => bank.start(target.context), Some(source) => match d { os_k1060_queue::Disposition::Ready => { bank.cooperate(source.context, target.context, capture.unwrap()) } os_k1060_queue::Disposition::Sleeping => { bank.sleep_to(source.context, target.context, capture.unwrap()) } }, } .map_err(|_| os_k1060_queue::Error::Backend)?; Ok(OsK1060SelectionPrepared { handoff: Some(handoff), epoch, nonce: 0, arrived: false, }) } fn commit( &mut self, _: &mut Self::Prepared, _: Option<&Self::Member>, _: &Self::Member, _: u64, _: os_k1060_queue::Disposition, ) -> Result<(), os_k1060_queue::Failure> { Err(os_k1060_queue::Failure { error: os_k1060_queue::Error::Context, phase: os_k1060_context::FailurePhase::BeforeSwitch, }) } fn finish(&mut self, p: Self::Prepared) -> Result<(), (os_k1060_queue::Error, Self::Prepared)> { if !p.arrived || p.handoff.is_some() { Err((os_k1060_queue::Error::State, p)) } else { Ok(()) } } fn abort( &mut self, mut p: Self::Prepared, ) -> Result<(), (os_k1060_queue::Error, Self::Prepared)> { let b = match self.bank.quiescent() { Ok(b) => b, Err(_) => return Err((os_k1060_queue::Error::Busy, p)), }; let t = p.handoff.take().unwrap(); match b.abort(t) { Ok(()) => Ok(()), Err(os_k1060_context::HandoffFailure::Before(_, t)) => { p.handoff = Some(t); Err((os_k1060_queue::Error::Backend, p)) } Err(os_k1060_context::HandoffFailure::Quarantined(_)) => { Err((os_k1060_queue::Error::Backend, p)) } } } } unsafe impl os_k1060_queue::SelectionHandoffBackend for OsK1060QueueBackend { fn depart( &mut self, p: &mut Self::Prepared, _: Option<&Self::Member>, _: &Self::Member, epoch: u64, _: os_k1060_queue::Disposition, nonce: u64, ) -> Result<(), os_k1060_queue::Failure> { if p.epoch != epoch || p.nonce != 0 || nonce == 0 { return Err(os_k1060_queue::Failure { error: os_k1060_queue::Error::Stale, phase: os_k1060_context::FailurePhase::BeforeSwitch, }); } let b = match self.bank.quiescent() { Ok(b) => b, Err(_) => { return Err(os_k1060_queue::Failure { error: os_k1060_queue::Error::Busy, phase: os_k1060_context::FailurePhase::BeforeSwitch, }); } }; let t = p.handoff.take().unwrap(); match b.depart(t) { Ok(t) => { p.handoff = Some(t); p.nonce = nonce; Ok(()) } Err(os_k1060_context::HandoffFailure::Before(_, t)) => { p.handoff = Some(t); Err(os_k1060_queue::Failure { error: os_k1060_queue::Error::Backend, phase: os_k1060_context::FailurePhase::BeforeSwitch, }) } Err(os_k1060_context::HandoffFailure::Quarantined(_)) => Err(os_k1060_queue::Failure { error: os_k1060_queue::Error::Backend, phase: os_k1060_context::FailurePhase::AfterSwitch, }), } } fn arrive( &mut self, p: &mut Self::Prepared, target: &Self::Member, epoch: u64, nonce: u64, ) -> Result<(), os_k1060_queue::Error> { if p.epoch != epoch || p.nonce != nonce { return Err(os_k1060_queue::Error::Stale); } let b = self .bank .quiescent() .map_err(|_| os_k1060_queue::Error::Busy)?; let t = p.handoff.take().unwrap(); match b.landed(t) { Ok(()) => { p.arrived = true; self.member(target)?; Ok(()) } Err(os_k1060_context::HandoffFailure::Before(_, t)) => { p.handoff = Some(t); Err(os_k1060_queue::Error::Backend) } Err(os_k1060_context::HandoffFailure::Quarantined(_)) => { Err(os_k1060_queue::Error::Backend) } } } } unsafe impl os_k1060_queue::QueueCompletionBackend for OsK1060QueueBackend { type ReturnPrepared = OsK1060QueueReturnPrepared; fn prepare_exit( &mut self, source: &Self::Member, capture: os_k1060_context::Capture, ) -> Result { self.member(source)?; let token = self .bank .prepare_return(source.context, capture) .map_err(|_| os_k1060_queue::Error::Backend)?; Ok(OsK1060QueueReturnPrepared { token: Some(token), source: source.context, nonce: 0, observed: false, }) } fn depart_exit( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Member, nonce: u64, ) -> Result<(), os_k1060_queue::Failure> { if p.source != source.context || nonce == 0 || p.nonce != 0 { return Err(os_k1060_queue::Failure { error: os_k1060_queue::Error::Stale, phase: os_k1060_context::FailurePhase::BeforeSwitch, }); } let t = p.token.take().unwrap(); match self.bank.depart_return(t) { Ok(t) => { p.token = Some(t); p.nonce = nonce; Ok(()) } Err(os_k1060_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err(os_k1060_queue::Failure { error: os_k1060_queue::Error::Backend, phase: os_k1060_context::FailurePhase::BeforeSwitch, }) } Err(os_k1060_context::ReturnFailure::Quarantined(_)) => Err(os_k1060_queue::Failure { error: os_k1060_queue::Error::Backend, phase: os_k1060_context::FailurePhase::AfterSwitch, }), } } fn returned_exit( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Member, nonce: u64, ) -> Result<(), os_k1060_queue::Error> { if p.source != source.context || p.nonce != nonce { return Err(os_k1060_queue::Error::Stale); } self.bank .os_k1060_check_return_landing(p.token.as_ref().ok_or(os_k1060_queue::Error::State)?) .map_err(|_| os_k1060_queue::Error::Backend)?; p.observed = true; Ok(()) } fn finish_exit( &mut self, mut p: Self::ReturnPrepared, ) -> Result<(), (os_k1060_queue::Error, Self::ReturnPrepared)> { if !p.observed { return Err((os_k1060_queue::Error::State, p)); } let t = p.token.take().unwrap(); match self.bank.return_landed(t) { Ok(()) => Ok(()), Err(os_k1060_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err((os_k1060_queue::Error::Backend, p)) } Err(os_k1060_context::ReturnFailure::Quarantined(_)) => { Err((os_k1060_queue::Error::Backend, p)) } } } fn abort_exit( &mut self, mut p: Self::ReturnPrepared, ) -> Result<(), (os_k1060_queue::Error, Self::ReturnPrepared)> { let t = p.token.take().unwrap(); match self.bank.abort_return(t) { Ok(()) => Ok(()), Err(os_k1060_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err((os_k1060_queue::Error::Backend, p)) } Err(os_k1060_context::ReturnFailure::Quarantined(_)) => { Err((os_k1060_queue::Error::Backend, p)) } } } } impl OsK1060QueueBackend { fn retire_member( &mut self, mut m: OsK1060Member, ) -> Result { let observation = match self.member(&m) { Ok(o) => o, Err(e) => return Err((e, m)), }; let before = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1060_context::Backend::execution(b)); let core = match self.bank.quiescent().and_then(|b| b.quiescent()) { Ok(core) => core, Err(_) => return Err((os_k1060_queue::Error::Busy, m)), }; if observation.thread.state != os_k1060_thread::State::Dead && core.os_k1040_cancel_inactive(m.context).is_err() { return Err((os_k1060_queue::Error::Backend, m)); } if core.retire(m.context).is_err() { return Err((os_k1060_queue::Error::Backend, m)); } // Frozen retirement has consumed the real Binding/Stack. A later // context failure retains only this explicitly retired logical member // and the exclusive bank; it never reconstructs a physical binding. m.retired = true; let after = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1060_context::Backend::execution(b)); if after != before { self.retirement_quarantined = true; return Err((os_k1060_queue::Error::Quarantined, m)); } self.members[m.ordinal] = None; Ok(m) } } unsafe impl os_k1060_queue::QueueRetirementBackend for OsK1060QueueBackend { fn retire_withdrawn( &mut self, member: Self::Member, ) -> Result { self.retire_member(member) } } impl OsK1060QueueBackend { fn owner_words(&self) -> [u64; 17] { [ self.identity, &self.bank as *const _ as u64, core::mem::size_of_val(&self.bank) as u64, self.members.as_ptr() as u64, self.members.iter().filter(|m| m.is_some()).count() as u64, self.failed_tick .as_ref() .map_or(0, |t| t as *const B::QueueTick as u64), self.members[0].map_or(0, |h| h.bank()), self.members[0].map_or(0, |h| h.slot() as u64), self.members[0].map_or(0, |h| h.generation()), self.members[1].map_or(0, |h| h.bank()), self.members[1].map_or(0, |h| h.slot() as u64), self.members[1].map_or(0, |h| h.generation()), self.members[2].map_or(0, |h| h.bank()), self.members[2].map_or(0, |h| h.slot() as u64), self.members[2].map_or(0, |h| h.generation()), core::mem::size_of::() as u64, u64::from(self.retirement_quarantined), ] } } const fn os_k1060_backend_layout() -> [u64; 24] { use core::mem::{offset_of, size_of}; [ size_of::>() as u64, offset_of!(OsK1060QueueBackend, bank) as u64, offset_of!(OsK1060QueueBackend, identity) as u64, offset_of!(OsK1060QueueBackend, members) as u64, offset_of!(OsK1060QueueBackend, failed_tick) as u64, size_of::>() as u64, size_of::() as u64, offset_of!(OsK1060Member, identity) as u64, offset_of!(OsK1060Member, context) as u64, offset_of!(OsK1060Member, thread) as u64, offset_of!(OsK1060Member, ordinal) as u64, size_of::() as u64, offset_of!(OsK1060SelectionPrepared, handoff) as u64, offset_of!(OsK1060SelectionPrepared, epoch) as u64, offset_of!(OsK1060SelectionPrepared, nonce) as u64, offset_of!(OsK1060SelectionPrepared, arrived) as u64, size_of::() as u64, offset_of!(OsK1060QueueReturnPrepared, token) as u64, offset_of!(OsK1060QueueReturnPrepared, source) as u64, offset_of!(OsK1060QueueReturnPrepared, nonce) as u64, offset_of!(OsK1060QueueReturnPrepared, observed) as u64, size_of::>() as u64, offset_of!(OsK1060QueueBackend, retirement_quarantined) as u64, offset_of!(OsK1060Member, retired) as u64, ] } // NEW implementation after the frozen private three-member backend bytes. unsafe impl os_k1060_queue::QueueWakeBackend for OsK1060QueueBackend { fn wake_member(&mut self, m: &Self::Member) -> Result<(), os_k1060_queue::Failure> { self.member(m).map_err(|error| os_k1060_queue::Failure { error, phase: os_k1060_context::FailurePhase::BeforeSwitch, })?; self.bank .os_k1060_wake_queue_member(m.context) .map_err(|failure| os_k1060_queue::Failure { error: if failure.phase == os_k1060_context::FailurePhase::AfterSwitch { os_k1060_queue::Error::Quarantined } else { os_k1060_queue::Error::Backend }, phase: failure.phase, }) } } unsafe impl os_k1060_context::QueueWakeContextBackend for OsK1060ContextBackend<'_> { fn wake_queue_binding(&mut self, b: &Self::Binding) -> Result<(), os_k1060_context::Failure> { let before = |error| os_k1060_context::Failure { error, phase: os_k1060_context::FailurePhase::BeforeSwitch, }; self.valid_pin(b).map_err(before)?; if !self.sleep_wait.as_ref().is_some_and(|w| w.thread() == b.id) { return Err(before(os_k1060_context::Error::Wake)); } let wait = self.sleep_wait.take().unwrap(); match self.table.wake(wait) { Ok(id) if id == b.id => {} Ok(_) => { return Err(os_k1060_context::Failure { error: os_k1060_context::Error::Wake, phase: os_k1060_context::FailurePhase::AfterSwitch, }); } Err((_, wait)) => { self.sleep_wait = Some(wait); return Err(before(os_k1060_context::Error::Wake)); } } self.refresh().map_err(|error| os_k1060_context::Failure { error, phase: os_k1060_context::FailurePhase::AfterSwitch, }) } } type OsK1060Bank<'a> = os_k1060_context::CompletionBank, 3>; type OsK1060Backend<'a> = OsK1060QueueBackend>; type OsK1060Queue<'a> = os_k1060_queue::ReturnQueue, 3>; #[unsafe(no_mangle)] static mut OS_K1060_QUEUE: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K1060_SPARES: [Option; 3] = [const { None }; 3]; static mut OS_K1060_FAILED_ROOTS: [Option; 3] = [const { None }; 3]; static mut OS_K1060_BINDINGS: [Option; 3] = [const { None }; 3]; static mut OS_K1060_MEMBERS: [Option; 3] = [const { None }; 3]; static mut OS_K1060_FAILED_TABLE: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); static mut OS_K1060_FAILURE_CUSTODY: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1060_ROOT_ADDRESSES: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1060_ROOT_GENERATIONS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1060_ROOT_LIVE: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1060_OWNED: [u64; 27] = [0; 27]; #[unsafe(no_mangle)] static mut OS_K1060_LEASE_GENERATIONS: [u64; 27] = [0; 27]; #[unsafe(no_mangle)] static mut OS_K1060_CONTEXT_HANDLES: [Option; 3] = [None; 3]; #[unsafe(no_mangle)] static mut OS_K1060_QUEUE_HANDLES: [Option; 3] = [None; 3]; #[unsafe(no_mangle)] static mut OS_K1060_EVIDENCE: [u64; 128] = [0; 128]; #[unsafe(no_mangle)] static mut OS_K1060_EVENTS: [u64; 4096] = [0; 4096]; #[unsafe(no_mangle)] static mut OS_K1060_EVENT_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_CAPTURE_GATE: u8 = 0; #[unsafe(no_mangle)] static mut OS_K1060_CRITICAL: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_SAFE_POINT: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1060_IRQ_TCO: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_WAKE_EPOCH: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_CAPTURE_EPISODE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_LIVE_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_IRQ_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_IRQ_CAPTURE_EPISODE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_BOOT_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_BOOT_STACK_REGISTERS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1060_DEPARTED_NONCE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_LANDING_ARGUMENT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_LANDING_PC: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_LANDING_FRAME: os_k1060_context::Cooperative = os_k1060_context::Cooperative::EMPTY; #[unsafe(no_mangle)] static mut OS_K1060_CURRENT_CAPTURE: Option = None; #[unsafe(no_mangle)] static mut OS_K1060_HANDOFF: Option = None; #[unsafe(no_mangle)] static mut OS_K1060_RETURN: Option = None; #[unsafe(no_mangle)] static mut OS_K1060_TASK_PROGRESS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1060_TASK_ENTRIES: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1060_CONTEXT_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1060_THREADS_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1060_BACKEND_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1060_QUEUE_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1060_AUTHORITY_BEFORE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K1060_AUTHORITY_LIVE: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K1060_AUTHORITY_AFTER: [u64; 328] = [0; 328]; #[unsafe(no_mangle)] static mut OS_K1060_QUEUE_WORDS: [u64; 16] = [0; 16]; #[unsafe(no_mangle)] static mut OS_K1060_MEMBER_WORDS: [[u64; 16]; 3] = [[0; 16]; 3]; #[unsafe(no_mangle)] static mut OS_K1060_BACKEND_WORDS: [u64; 17] = [0; 17]; #[unsafe(no_mangle)] static mut OS_K1060_CONTEXT_WORDS: [[u64; 16]; 5] = [[0; 16]; 5]; #[unsafe(no_mangle)] static OS_K1060_RETURN_LAYOUT: [u64; 40] = os_k1060_queue::os_k1060_return_layout::, 3>(); #[unsafe(no_mangle)] static OS_K1060_QUEUE_BACKEND_LAYOUT: [u64; 24] = os_k1060_backend_layout::>(); #[unsafe(no_mangle)] static OS_K1060_CONTEXT_LAYOUT: [u64; 40] = os_k1060_context::os_k1040_layout::, 3>(); #[unsafe(no_mangle)] static OS_K1060_THREAD_LAYOUT: [u64; 20] = OsK1060Table::os_k1040_layout(); #[repr(C, align(16))] #[derive(Clone, Copy)] struct OsK1060Wire { frame: os_k1060_context::Cooperative, pc: u64, argument: u64, nonce: u64, kind: u64, } #[unsafe(no_mangle)] static mut OS_K1060_WIRE: OsK1060Wire = OsK1060Wire { frame: os_k1060_context::Cooperative::EMPTY, pc: 0, argument: 0, nonce: 0, kind: 0, }; const _: () = { assert!(core::mem::size_of::() == 672); assert!(core::mem::offset_of!(OsK1060Wire, pc) == 640); assert!(core::mem::offset_of!(OsK1060Wire, argument) == 648); assert!(core::mem::offset_of!(OsK1060Wire, kind) == 664); }; fn os_k1060_queue() -> *mut OsK1060Queue<'static> { core::ptr::addr_of_mut!(OS_K1060_QUEUE).cast() } fn os_k1060_word(i: usize, v: u64) { if i >= 128 { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_EVIDENCE) .cast::() .add(i), v, ); } } fn os_k1060_event(kind: u64, a: u64, b: u64, c: u64) { let n = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_EVENT_COUNT)) } as usize; if n >= 1024 { park(); } for (i, v) in [kind, a, b, c].into_iter().enumerate() { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_EVENTS) .cast::() .add(n * 4 + i), v, ); } } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_EVENT_COUNT), n as u64 + 1); } } fn os_k1060_nonce(p: *mut u64) -> u64 { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { return 0; } let n = unsafe { read_volatile(p) }; let Some(next) = n.checked_add(1) else { return 0; }; if n == 0 { return 0; } unsafe { write_volatile(p, next); } n } #[unsafe(no_mangle)] extern "C" fn __os_k1060_context_identity() -> u64 { os_k1060_nonce(core::ptr::addr_of_mut!(OS_K1060_CONTEXT_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1060_threads_identity() -> u64 { os_k1060_nonce(core::ptr::addr_of_mut!(OS_K1060_THREADS_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1060_backend_identity() -> u64 { os_k1060_nonce(core::ptr::addr_of_mut!(OS_K1060_BACKEND_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1060_runqueue_identity() -> u64 { os_k1060_nonce(core::ptr::addr_of_mut!(OS_K1060_QUEUE_IDENTITY)) } fn os_k1060_stack_registers() -> [u64; 4] { unsafe extern "C" { fn __os_k1060_stack_registers(p: *mut u64); } let mut r = [0; 4]; if !os_k1060_execution().irq_masked { park(); } unsafe { __os_k1060_stack_registers(r.as_mut_ptr()); } r } fn os_k1060_boot_frame_range(p: u64) -> bool { let guard = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_PRIMARY_STACK_EVIDENCE) .cast::() .add(13), ) }; p % 16 == 0 && p >= guard + 4096 + 1024 && p.checked_add(640) .is_some_and(|e| e <= guard + 4096 + 65536 - 16) } fn os_k1060_idle_atomic( _: &mut OsK1060ContextBackend<'_>, _: &mut OsK1060IdlePrepared, _: u64, ) -> Result { Err(os_k1060_context::Failure { error: os_k1060_context::Error::Context, phase: os_k1060_context::FailurePhase::BeforeSwitch, }) } struct OsK1060Tick { sequence: u64, observation: os_k1060_queue::TickObservation, } #[unsafe(no_mangle)] static mut OS_K1060_TICK_LIVE: u64 = 0; unsafe impl os_k1060_context::QueueServices for OsK1060ContextBackend<'_> { type QueueTick = OsK1060Tick; fn queue_capture(&self) -> Option { unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_CURRENT_CAPTURE)) } } fn queue_tick( &self, t: &OsK1060Tick, ) -> Result { if t.sequence == 0 || t.sequence != unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_TICK_LIVE)) } || t.sequence > 30 { return Err(os_k1060_queue::Error::Tick); } let r = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_IRQ_RECORDS) .cast::<[u64; 16]>() .add(t.sequence as usize - 1), ) }; if r[0] != t.sequence || t.observation.cpu != 0 || t.observation.source != 30 || t.observation.sequence != t.sequence || t.observation.counter != r[6] || t.observation.thread.slot() as u64 != r[2] || t.observation.thread.generation() != r[3] || t.observation.slice != r[5] { return Err(os_k1060_queue::Error::Tick); } Ok(t.observation) } fn queue_consume_tick(&mut self, t: OsK1060Tick) { if t.sequence != unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_TICK_LIVE)) } { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_TICK_LIVE), 0); } } } fn os_k1060_capture(stage: u64) { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq != ((10..40).contains(&stage)) { park(); } let (root, tcr, vbar, daif) = os_k940_registers(); let (cval, ctl) = os_k960_timer(); for (i, v) in [ (5, stage), (6, root), (7, tcr), (8, vbar), (9, daif), (10, e.cpu as u64), (11, e.sp), (12, os_k960_counter()), (13, u64::from(e.in_irq)), (14, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }), ( 15, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } as u64, ), (16, cval), (17, ctl), (18, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)) }), (19, unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_COUNT)) }), ] { os_k1060_word(i, v); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_CAPTURE_GATE), 0, ); } write_str("TURKOS/OS/K1060: CAPTURE STAGE=0x"); write_hex(stage as usize); write_str(" GATE=0x"); write_hex(core::ptr::addr_of!(OS_K1060_CAPTURE_GATE) as usize); write_line(" READY"); let gate = core::ptr::addr_of!(OS_K1060_CAPTURE_GATE); while unsafe { read_volatile(gate) } != 1 { core::hint::spin_loop(); } unsafe { asm!("dsb ish", "isb", options(nostack, preserves_flags)); } } fn os_k1060_observe() { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let q = unsafe { &*os_k1060_queue() }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_QUEUE_WORDS), q.os_k1060_owner_words(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_MEMBER_WORDS), q.os_k1060_member_words(), ); } q.queue().with_backend_observation(|b| { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_BACKEND_WORDS), b.owner_words(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONTEXT_WORDS), b.bank.os_k1060_words(), ); } b.bank.handoff().bank().with_backend_observation(|b| { os_k1060_observe_backend(b, b.records.map(|record| record.map(|r| r.id))); b.table.os_k1020_observe_backend(|s| { os_k1060_word( 70, s.memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64, ); os_k1060_word(71, core::mem::size_of::() as u64); os_k940_inventory( s.memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K1060_AUTHORITY_LIVE).cast(), ) }) }); }); } fn os_k1060_capture_source(address: u64) -> os_k1060_context::Capture { let actual = os_k1060_execution(); if actual.cpu != 0 || !actual.irq_masked || actual.in_irq { park(); } let q = unsafe { &*os_k1060_queue() }; let e = q .queue() .with_backend_observation(|b| os_k1060_queue::Backend::execution(b)); let id = e.thread.unwrap_or_else(|| park()); let layout = q.queue().with_backend_observation(|b| { b.bank.handoff().bank().with_backend_observation(|c| { if c.current != Some(id) { park(); } let ordinal = c .records .iter() .position(|r| r.is_some_and(|r| r.id == id)) .unwrap_or_else(|| park()); if !c.pins[ordinal] || b.members[ordinal].is_none() || c.table.record(id).unwrap_or_else(|_| park()).state != os_k1060_thread::State::Running { park(); } c.validate_stack_owner(id).unwrap_or_else(|_| park()); c.table .os_k1020_observe_stack(id, |s| { let layout = s.layout(); if s.ordinal != ordinal || s.root.address() != e.root || s.root.generation() == 0 || unsafe { read_volatile(layout.base as *const u64) } != os_k1060_thread::LOW_CANARY || unsafe { read_volatile((layout.base + layout.bytes - 8) as *const u64) } != os_k1060_thread::HIGH_CANARY { park(); } layout }) .unwrap_or_else(|_| park()) }) }); let episode = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_CAPTURE_EPISODE)) }; if address % 16 != 0 || address < layout.usable_bottom() || !address .checked_add(640) .is_some_and(|e| e <= layout.usable_top()) || episode == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_LIVE_CAPTURE_ADDRESS)) } != address { park(); } let f = unsafe { read_volatile(address as *const os_k1060_context::Cooperative) }; if f.sp != address + 640 || f.canary != os_k1060_context::CONTEXT_CANARY || f.padding != 0 { park(); } let c = os_k1060_context::Capture { image: os_k1060_context::Image::Cooperative(f), address, abi: os_k1060_context::Abi::COOPERATIVE, episode, }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_CURRENT_CAPTURE), Some(c)); } os_k1060_event(0x60, id.slot() as u64, address, episode); os_k1060_word(103, id.slot() as u64); c } fn os_k1060_set_wire(image: os_k1060_context::Image, nonce: u64) -> *const OsK1060Wire { let (frame, pc, argument, kind) = match image { os_k1060_context::Image::Initial { entry, argument, sp, return_pc, } => { let mut f = os_k1060_context::Cooperative::EMPTY; f.sp = sp; f.x19_x30[11] = return_pc; (f, entry, argument, 0) } os_k1060_context::Image::Cooperative(f) => (f, f.x19_x30[11], 0, 1), _ => park(), }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_WIRE), OsK1060Wire { frame, pc, argument, nonce, kind, }, ); } core::ptr::addr_of!(OS_K1060_WIRE) } fn os_k1060_depart(t: os_k1060_queue::OwnedSelection) -> *const OsK1060Wire { let q = unsafe { &mut *os_k1060_queue() }; let t = q.depart(t).unwrap_or_else(|_| park()); let (image, nonce) = q.queue().with_retained_observation(|p| { q.queue().with_backend_observation(|b| { let t = p .unwrap_or_else(|| park()) .handoff .as_ref() .unwrap_or_else(|| park()); ( b.bank.handoff().target_image(t).unwrap_or_else(|_| park()), t.nonce(), ) }) }); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_DEPARTED_NONCE), nonce); write_volatile(core::ptr::addr_of_mut!(OS_K1060_HANDOFF), Some(t)); } let wire=os_k1060_set_wire(image, nonce); os_k1060_publish_trace_owners(); wire } #[unsafe(no_mangle)] extern "C" fn __os_k1060_prepare_cooperative(address: u64, requested: u64) -> *const OsK1060Wire { os_k1060_capture_source(address); let disposition = match requested { 0 => os_k1060_queue::Disposition::Ready, 1 => os_k1060_queue::Disposition::Sleeping, _ => park(), }; let t = unsafe { (&mut *os_k1060_queue()).select(disposition) }.unwrap_or_else(|_| park()); let wire=os_k1060_depart(t); __os_k1060_source_birth(address,1); wire } fn os_k1060_landing_snapshot(address: u64) { if address % 16 != 0 { park(); } let f = unsafe { read_volatile(address as *const os_k1060_context::Cooperative) }; if f.sp != address + 640 || f.canary != os_k1060_context::CONTEXT_CANARY || f.padding != 0 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_LANDING_FRAME), f); } os_k1060_event(0x61, address, os_k940_registers().0, unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_PC)) }); os_k1060_landing_birth(address); } #[unsafe(no_mangle)] extern "C" fn __os_k1060_landed(address: u64) { os_k1060_landing_snapshot(address); let t = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1060_HANDOFF)).take() } .unwrap_or_else(|| park()); let h = unsafe { (&mut *os_k1060_queue()).landed(t) }.unwrap_or_else(|_| park()); os_k1060_event( 0x62, h.slot() as u64, h.generation(), unsafe { (&*os_k1060_queue()).queue().record(h) } .unwrap_or_else(|_| park()) .slice, ); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_LIVE_CAPTURE_ADDRESS), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1060_CURRENT_CAPTURE), None); } } #[unsafe(no_mangle)] extern "C" fn __os_k1060_prepare_return(address: u64) -> *const OsK1060Wire { let c = os_k1060_capture_source(address); let q = unsafe { &mut *os_k1060_queue() }; let t = q.prepare_return(c).unwrap_or_else(|_| park()); let t = q.depart_return(t).unwrap_or_else(|_| park()); let f = q.queue().with_backend_observation(|b| { b.bank .with_return_observation(|p| p.unwrap_or_else(|| park()).bootstrap) }); unsafe extern "C" { fn __os_k1060_boot_landed(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_RETURN), Some(t)); write_volatile( core::ptr::addr_of_mut!(OS_K1060_WIRE), OsK1060Wire { frame: f, pc: __os_k1060_boot_landed as *const () as u64, argument: 0, nonce: 0, kind: 2, }, ); } os_k1060_publish_trace_owners(); __os_k1060_source_birth(address,2); core::ptr::addr_of!(OS_K1060_WIRE) } #[unsafe(no_mangle)] extern "C" fn __os_k1060_return_landed(address: u64) { os_k1060_landing_snapshot(address); let t = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1060_RETURN)).take() } .unwrap_or_else(|| park()); unsafe { (&mut *os_k1060_queue()).return_landed(t) }.unwrap_or_else(|_| park()); os_k1060_event(0x63, address, os_k940_registers().0, 0); } // NEW phased BSS construction. No whole Queue/Bank/Backend result returns // through the factory caller. Frozen constructors are invoked unchanged. #[repr(C, align(16))] struct OsK1060ConstructionSlot { bytes: core::mem::MaybeUninit<[u8; core::mem::size_of::>()]>, } #[unsafe(no_mangle)] static mut OS_K1060_CONSTRUCTION: [OsK1060ConstructionSlot; 2] = [const { OsK1060ConstructionSlot { bytes: core::mem::MaybeUninit::uninit(), } }; 2]; #[unsafe(no_mangle)] static mut OS_K1060_CONSTRUCTION_STATES: [u64; 8] = [0; 8]; struct OsK1060Factory<'a> { slot: usize, kind: u64, ownership: core::marker::PhantomData>, } impl<'a> OsK1060Factory<'a> { unsafe fn start(backend: OsK1060ContextBackend<'a>) -> Self { let mut stage = Self { slot: 0, kind: 1, ownership: core::marker::PhantomData, }; stage.gate(); if unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_CONSTRUCTION_STATES).cast::()) } != 0 { os_k1060_retain_failure(backend, 20); } let pointer = stage.pointer::>(0); unsafe { pointer.write(backend); } stage.publish(1); stage } fn gate(&self) { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || os_k960_timer().1 & 1 != 0 { park(); } } fn pointer(&self, slot: usize) -> *mut T { if slot >= 2 || core::mem::size_of::() > core::mem::size_of::() || core::mem::align_of::() > 16 { park(); } unsafe { core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION) .cast::() .add(slot) .cast::() } } fn source(&self, kind: u64) -> *mut T { self.gate(); if self.kind != kind || unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(self.slot), ) } != kind { park(); } self.pointer(self.slot) } fn destination(&self) -> *mut T { let slot = 1 - self.slot; if unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(slot), ) } != 0 { park(); } self.pointer(slot) } fn publish(&mut self, kind: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(self.slot), kind, ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(2), self.slot as u64, ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(3), kind, ); } } fn commit(&mut self, kind: u64) { let old = self.slot; self.slot = 1 - old; self.kind = kind; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(old), 0, ); } self.publish(kind); } } macro_rules! os_k1060_constructor_stage { ($name:ident,$from:ty,$to:ty,$before:expr,$after:expr,$constructor:path) => { impl<'a> OsK1060Factory<'a> { #[inline(never)] fn $name(&mut self) { let source = self.source::<$from>($before); let destination = self.destination::<$to>(); // Destination validation and the actual CPU gate precede the // sole read. The Rust local owns source until one BSS write. let actual = unsafe { source.read() }; match $constructor(actual) { Ok(actual) => { unsafe { destination.write(actual); } self.commit($after); } Err((_, actual)) => { unsafe { source.write(actual); } os_k1060_word(122, $before); park(); } } } } }; } os_k1060_constructor_stage!( context, OsK1060ContextBackend<'a>, os_k1060_context::ContextBank, 3>, 1, 2, os_k1060_context::ContextBank::, 3>::new ); os_k1060_constructor_stage!( asynchronous, os_k1060_context::ContextBank, 3>, os_k1060_context::AsyncBank, 3>, 2, 3, os_k1060_context::AsyncBank::, 3>::new ); os_k1060_constructor_stage!( completion, os_k1060_context::AsyncBank, 3>, OsK1060Bank<'a>, 3, 4, OsK1060Bank::new ); os_k1060_constructor_stage!( members, OsK1060Bank<'a>, OsK1060Backend<'a>, 4, 5, OsK1060Backend::new ); os_k1060_constructor_stage!( policy, OsK1060Backend<'a>, os_k1060_queue::Queue, 3>, 5, 6, os_k1060_queue::Queue::, 3>::new ); os_k1060_constructor_stage!( selection, os_k1060_queue::Queue, 3>, os_k1060_queue::AsyncQueue, 3>, 6, 7, os_k1060_queue::AsyncQueue::, 3>::new ); impl<'a> OsK1060Factory<'a> { #[inline(never)] fn attach(&mut self) { let core = unsafe { &mut *self.source::, 3>>(2) }; for ordinal in 0..3 { let binding = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1060_BINDINGS) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); let h = match core.attach(binding) { Ok(h) => h, Err((_, binding)) => { unsafe { core::ptr::addr_of_mut!(OS_K1060_BINDINGS) .cast::>() .add(ordinal) .write(Some(binding)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1060_CONTEXT_HANDLES) .cast::>() .add(ordinal) .write(Some(h)); } } } #[inline(never)] fn issue(&mut self) { let backend = unsafe { &mut *self.source::>(5) }; for ordinal in 0..3 { let h = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_CONTEXT_HANDLES) .cast::>() .add(ordinal), ) } .unwrap_or_else(|| park()); let m = backend.issue(h, ordinal).unwrap_or_else(|_| park()); unsafe { core::ptr::addr_of_mut!(OS_K1060_MEMBERS) .cast::>() .add(ordinal) .write(Some(m)); } } } #[inline(never)] fn enqueue(&mut self) { let queue = unsafe { &mut *self.source::, 3>>(6) }; for ordinal in 0..3 { let m = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1060_MEMBERS) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); let class = if ordinal == 1 { os_k1060_queue::Class::Low } else { os_k1060_queue::Class::Normal }; let h = match queue.enqueue(m, class) { Ok(h) => h, Err((_, m)) => { unsafe { core::ptr::addr_of_mut!(OS_K1060_MEMBERS) .cast::>() .add(ordinal) .write(Some(m)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1060_QUEUE_HANDLES) .cast::>() .add(ordinal) .write(Some(h)); } } } #[inline(never)] fn install(mut self) -> OsK1060Installation<'a> { let source = self.source::, 3>>(7); let pointer = core::ptr::addr_of_mut!(OS_K1060_QUEUE).cast::>(); let actual = unsafe { source.read() }; match OsK1060Queue::new(actual) { Ok(actual) => { unsafe { pointer.write(actual); write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(self.slot), 0, ); } self.kind = 8; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(2), 2, ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(3), 8, ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_CONSTRUCTION_STATES) .cast::() .add(4), pointer as u64, ); } OsK1060Installation { pointer, active: true, ownership: core::marker::PhantomData, } } Err((_, actual)) => { unsafe { source.write(actual); } os_k1060_word(122, 7); park() } } } } // Own IRQ accesses only these prepublished immutable owner scalars and bounded // frame/receipt buffers. No Queue, Context, Table, Runtime or allocator loan. #[unsafe(no_mangle)] static mut OS_K1060_KERNEL_FRAME: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_TIMER_OWNER: [u64; 16] = [0; 16]; #[unsafe(no_mangle)] static mut OS_K1060_IRQ_FRAMES: [os_k920_exception::Frame; 30] = [OS_K920_ZERO_FRAME; 30]; #[unsafe(no_mangle)] static mut OS_K1060_IRQ_ADDRESSES: [u64; 30] = [0; 30]; #[unsafe(no_mangle)] static mut OS_K1060_IRQ_RECORDS: [[u64; 16]; 30] = [[0; 16]; 30]; #[unsafe(no_mangle)] static mut OS_K1060_IRQ_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_IRQ_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_PRE_IRQ: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1060_REPLAY: Option = None; #[unsafe(no_mangle)] static mut OS_K1060_QUANTUM: [[u64; 8]; 3] = [[0; 8]; 3]; #[unsafe(no_mangle)] static mut OS_K1060_CPU_ITERATIONS: [u64; 3] = [0; 3]; #[inline(never)] fn os_k1060_duplicate_refusal() { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let q = unsafe { &mut *os_k1060_queue() }; let queue = q.quiescent().unwrap_or_else(|_| park()); let before = queue.snapshot(); let order = queue.ready_order(); if before.members != 3 || before.queued != 3 || before.active != 0 { park(); } for ordinal in 0..3 { let h = os_k1060_queue_handle(ordinal); if queue.requeue(h) != Err(os_k1060_queue::Error::Duplicate) || queue.snapshot() != before || queue.ready_order() != order { park(); } } os_k1060_word(100, 3); os_k1060_event(0x71, 3, before.epoch, 0); } #[inline(never)] fn os_k1060_sleep_refusal() { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let q = unsafe { &mut *os_k1060_queue() }; let queue = q.quiescent().unwrap_or_else(|_| park()); let h = os_k1060_queue_handle(0); let before = queue.snapshot(); let order = queue.ready_order(); if queue.requeue(h) != Err(os_k1060_queue::Error::State) || queue.snapshot() != before || queue.ready_order() != order { park(); } queue.with_backend_observation(|b| { let o = b .bank .os_k1060_observe_member(b.members[0].unwrap_or_else(|| park())) .unwrap_or_else(|_| park()); if o.thread.state != os_k1060_thread::State::Sleeping || o.context.state != os_k1060_context::State::Ready { park(); } b.bank.handoff().bank().with_backend_observation(|c| { if !c .sleep_wait .as_ref() .is_some_and(|w| w.thread() == o.thread.id) { park(); } }); }); os_k1060_word(101, 1); os_k1060_observe(); os_k1060_capture(4); } #[inline(never)] fn os_k1060_wake_actual() { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let q = unsafe { &mut *os_k1060_queue() }; let queue = q.quiescent().unwrap_or_else(|_| park()); let h = os_k1060_queue_handle(0); queue.os_k1060_wake(h).unwrap_or_else(|_| park()); let order = queue.ready_order(); if order.iter().flatten().filter(|v| **v == h).count() != 1 { park(); } os_k1060_word(102, 1); os_k1060_observe(); os_k1060_capture(5); } fn os_k1060_queue_handle(ordinal: usize) -> os_k1060_queue::Handle { if ordinal >= 3 { park(); } unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_QUEUE_HANDLES) .cast::>() .add(ordinal), ) } .unwrap_or_else(|| park()) } fn os_k1060_irq_word(index: usize) -> u64 { unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_TIMER_OWNER) .cast::() .add(index), ) } } #[inline(never)] fn os_k1060_arm_tick(ordinal: usize) -> u64 { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || ordinal >= 3 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_ACTIVE)) } != 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_TICK_LIVE)) } != 0 || os_k960_timer().1 & 1 != 0 { park(); } let q = unsafe { &*os_k1060_queue() }; let h = os_k1060_queue_handle(ordinal); let record = q.queue().record(h).unwrap_or_else(|_| park()); if !record.active || record.remaining == 0 { park(); } let (root, base, bytes, context_slot, context_generation) = q.queue().with_backend_observation(|b| { b.bank.handoff().bank().with_backend_observation(|c| { let id = c.current.unwrap_or_else(|| park()); if id != record.thread || !c.pins[ordinal] { park(); } c.validate_stack_owner(id).unwrap_or_else(|_| park()); c.table .os_k1020_observe_stack(id, |s| { if s.ordinal != ordinal || s.root.address() != os_k940_registers().0 { park(); } let l = s.layout(); let handle = b.members[ordinal].unwrap_or_else(|| park()); ( s.root.address(), l.base, l.bytes, handle.slot() as u64, handle.generation(), ) }) .unwrap_or_else(|_| park()) }) }); let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_COUNT)) }; let sequence = count .checked_add(1) .filter(|s| *s <= 30) .unwrap_or_else(|| park()); let counter = os_k960_counter(); let deadline = counter .checked_add((os_k960_frequency() / 10000).max(1)) .unwrap_or_else(|| park()); let words = [ 1, ordinal as u64, record.thread.slot() as u64, record.thread.generation(), root, base, bytes, record.slice, sequence, counter, deadline, record.remaining as u64, h.slot() as u64, h.generation(), context_slot, context_generation, ]; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_TIMER_OWNER), words); asm!("dsb ish","msr CNTP_CVAL_EL0, {deadline}","mov x9, #1","msr CNTP_CTL_EL0, x9","isb",deadline=in(reg)deadline,out("x9")_,options(nomem,nostack,preserves_flags)); } sequence } #[unsafe(no_mangle)] extern "C" fn os_k1060_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { let address = pointer as u64; let e = os_k1060_execution(); if pointer.is_null() || e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } if os_k1060_priority_state(0) == 1 { let guard = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_PRIMARY_STACK_EVIDENCE) .cast::() .add(13), ) }; os_k920_exception::Stack::new(guard + 4096, guard + 4096 + 65536) .and_then(|s| s.frame_address(address)) .unwrap_or_else(|_| park()); return os_k1060_priority_dispatch(pointer); } if pointer.is_null() || e.cpu != 0 || !e.irq_masked || e.in_irq || unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_ACTIVE)) } != 0 || os_k1060_irq_word(0) != 1 { park(); } let number = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_COUNT)) } as usize; if number >= 30 || os_k1060_irq_word(8) != number as u64 + 1 { park(); } let low = os_k1060_irq_word(5); let high = low .checked_add(os_k1060_irq_word(6)) .unwrap_or_else(|| park()); let stack = os_k920_exception::Stack::new(low, high).unwrap_or_else(|_| park()); stack.frame_address(address).unwrap_or_else(|_| park()); let frame = unsafe { read_volatile(pointer) }; os_k920_kernel_irq_return( &frame, address, stack, core::ptr::addr_of!(_text_start) as u64, core::ptr::addr_of!(_text_end) as u64, __os_k920_kernel_tco_observed(), ) .unwrap_or_else(|_| park()); unsafe extern "C" { static __os_k1060_timer_work_loop: u8; static __os_k1060_timer_work_mask: u8; } if frame.vector_slot != 5 || frame.spsr & 31 != 5 || frame.elr & 3 != 0 || frame.elr < core::ptr::addr_of!(__os_k1060_timer_work_loop) as u64 || frame.elr >= core::ptr::addr_of!(__os_k1060_timer_work_mask) as u64 || frame.interrupted_sp != address + 848 || os_k940_registers().0 != os_k1060_irq_word(4) { park(); } let iar: u64; unsafe { asm!("mrs {iar}, ICC_IAR1_EL1",iar=out(reg)iar,options(nomem,nostack,preserves_flags)); } let (cval, ctl) = os_k960_timer(); let counter = os_k960_counter(); if iar & 0x00ff_ffff != 30 || ctl & 7 != 5 || cval != os_k1060_irq_word(10) || counter < cval { park(); } let total = unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) } .checked_add(1) .unwrap_or_else(|| park()); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_IRQ_ACTIVE), 1); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE), 1); write_volatile( core::ptr::addr_of_mut!(OS_K1060_IRQ_FRAMES) .cast::() .add(number), frame, ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_IRQ_ADDRESSES) .cast::() .add(number), address, ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_IRQ_RECORDS) .cast::<[u64; 16]>() .add(number), [ number as u64 + 1, os_k1060_irq_word(1), os_k1060_irq_word(2), os_k1060_irq_word(3), os_k1060_irq_word(4), os_k1060_irq_word(7), counter, cval, ctl, iar, address, total, os_k1060_irq_word(11), os_k1060_irq_word(12), os_k1060_irq_word(13), frame.spsr, ], ); asm!( "msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack, preserves_flags) ); } let split: u64; unsafe { asm!("mrs {split}, ICC_CTLR_EL1",split=out(reg)split,options(nomem,nostack,preserves_flags)); asm!("dsb sy","msr ICC_EOIR1_EL1, {iar}","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags)); if split & 2 != 0 { asm!("msr ICC_DIR_EL1, {iar}","dsb sy","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags)); } write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT), total); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER), counter); write_volatile( core::ptr::addr_of_mut!(OS_K1060_IRQ_COUNT), number as u64 + 1, ); } os_k1060_capture(10 + number as u64); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1060_IRQ_ACTIVE), 0); write_volatile( core::ptr::addr_of_mut!(OS_K1060_TIMER_OWNER).cast::(), 0, ); } 0 } #[inline(never)] fn os_k1060_charge_actual(ordinal: usize, sequence: u64) -> bool { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || sequence == 0 || sequence > 30 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_COUNT)) } != sequence || unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_TICK_LIVE)) } != 0 { park(); } let r = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_IRQ_RECORDS) .cast::<[u64; 16]>() .add(sequence as usize - 1), ) }; let handle = os_k1060_queue_handle(ordinal); let q = unsafe { &mut *os_k1060_queue() }; let queue = q.quiescent().unwrap_or_else(|_| park()); let old = queue.record(handle).unwrap_or_else(|_| park()); if r[0] != sequence || r[1] != ordinal as u64 || r[2] != old.thread.slot() as u64 || r[3] != old.thread.generation() || r[4] != os_k940_registers().0 || r[5] != old.slice || r[9] & 0x00ff_ffff != 30 { park(); } let observation = os_k1060_queue::TickObservation { cpu: 0, source: 30, sequence, counter: r[6], thread: old.thread, slice: old.slice, }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_TICK_LIVE), sequence); } let charge = queue .charge(OsK1060Tick { sequence, observation, }) .unwrap_or_else(|(_, actual)| { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_REPLAY), Some(actual)); } park() }); if charge.remaining.checked_add(1) != Some(old.remaining) || charge.sequence != sequence { park(); } // Named replay fault injection: an inert copied receipt has no live IRQ // authority after consumption. The returned invalid token is retained. let snapshot = queue.snapshot(); let record = queue.record(handle).unwrap_or_else(|_| park()); if sequence == 1 { let replay = match queue.charge(OsK1060Tick { sequence, observation, }) { Err((os_k1060_queue::Error::Tick, actual)) => actual, Err((_, actual)) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_REPLAY), Some(actual)); } park() } Ok(_) => park(), }; if queue.snapshot() != snapshot || queue.record(handle).unwrap_or_else(|_| park()) != record { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_REPLAY), Some(replay)); } } unsafe { let row = core::ptr::addr_of_mut!(OS_K1060_QUANTUM) .cast::<[u64; 8]>() .add(ordinal); let mut counters = read_volatile(row); counters[0] += 1; counters[1] += u64::from(sequence == 1); counters[2] = charge.remaining as u64; counters[3] += u64::from(charge.reschedule); counters[4] = sequence; counters[5] = r[6]; counters[6] = old.slice; counters[7] = old.class as u64; write_volatile(row, counters); } os_k1060_event(0x70, sequence, ordinal as u64, charge.remaining as u64); charge.reschedule } // Actual scheduler-selected legacy EL0 consumer requests unavailable class2. // Pure Class ingress refuses it before any raw mutable Queue/Context loan. static mut OS_K1060_PRIORITY_SPACE: Option = None; static mut OS_K1060_PRIORITY_TASKS: *const OsK660Tasks = core::ptr::null(); #[unsafe(no_mangle)] static mut OS_K1060_PRIORITY_STATE: [u64; 24] = [0; 24]; #[unsafe(no_mangle)] static mut OS_K1060_PRIORITY_FRAMES: [os_k920_exception::Frame; 2] = [OS_K920_ZERO_FRAME; 2]; #[unsafe(no_mangle)] static mut OS_K1060_PRIORITY_OWNED: [u64; 6] = [0; 6]; #[unsafe(no_mangle)] static mut OS_K1060_PRIORITY_SCHEDULER: [u64; 48] = [0; 48]; #[unsafe(no_mangle)] static mut OS_K1060_PRIORITY_BRIDGE: [u64; 12] = [0; 12]; fn os_k1060_priority_state(i: usize) -> u64 { unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_PRIORITY_STATE) .cast::() .add(i), ) } } fn os_k1060_priority_word(i: usize, v: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_STATE) .cast::() .add(i), v, ); } } fn os_k1060_priority_space() -> &'static mmu::UserSpace { unsafe { (&*core::ptr::addr_of!(OS_K1060_PRIORITY_SPACE)).as_ref() }.unwrap_or_else(|| park()) } struct OsK1060PriorityReturn; impl os_k920_exception::ReturnAuthority for OsK1060PriorityReturn { fn owner_root(&self, user: os_k920_exception::User) -> Result<(), os_k920_exception::Error> { let space = os_k1060_priority_space(); let p = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_PRIORITY_TASKS)) }; if p.is_null() { return Err(os_k920_exception::Error::Owner); } let selected = unsafe { (&*p).scheduler.running() }.ok_or(os_k920_exception::Error::Owner)?; if user.owner != selected.slot as u64 + 1 || user.generation != selected.generation || user.owner != os_k1060_priority_state(2) || user.generation != os_k1060_priority_state(3) || user.root != space.root || user.asid != 1 || os_k940_registers().0 != space.root | (1 << 48) { return Err(os_k920_exception::Error::Owner); } Ok(()) } fn executable( &self, _: os_k920_exception::User, entry: u64, ) -> Result<(), os_k920_exception::Error> { let s = os_k1060_priority_space(); let p = s.descriptor(0).ok_or(os_k920_exception::Error::Entry)?; if entry != mmu::USER_CODE_VA + 8 || p & 0x0000_ffff_ffff_f000 != s.code_frame() || p & 3 != 3 || p & (1 << 6 | 1 << 7 | 1 << 11 | 1 << 53) != (1 << 6 | 1 << 7 | 1 << 11 | 1 << 53) || p & 1 << 54 != 0 { return Err(os_k920_exception::Error::Entry); } Ok(()) } fn writable_stack( &self, _: os_k920_exception::User, sp: u64, ) -> Result<(), os_k920_exception::Error> { let s = os_k1060_priority_space(); let p = s.descriptor(3).ok_or(os_k920_exception::Error::Stack)?; if sp != mmu::USER_STACK_TOP || p & 0x0000_ffff_ffff_f000 != s.stack_frame() || p & 3 != 3 || p & (3 << 6) != 1 << 6 || p & (1 << 53 | 1 << 54) != (1 << 53 | 1 << 54) || s.descriptor(2) != Some(0) { return Err(os_k920_exception::Error::Stack); } Ok(()) } } #[inline(never)] fn os_k1060_priority_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { let address = pointer as u64; let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || os_k1060_priority_state(0) != 1 || pointer.is_null() { park(); } let guard = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_PRIMARY_STACK_EVIDENCE) .cast::() .add(13), ) }; let stack = os_k920_exception::Stack::new(guard + 4096, guard + 4096 + 65536) .unwrap_or_else(|_| park()); stack.frame_address(address).unwrap_or_else(|_| park()); let frame = unsafe { read_volatile(pointer) }; os_k920_exception::validate_ingress(&frame, address, stack).unwrap_or_else(|_| park()); if frame.vector_slot != 8 || frame.esr != 0x56001060 || frame.elr != mmu::USER_CODE_VA + 8 || frame.x[0] != 2 || frame.spsr & 31 != 0 { park(); } let user = os_k920_exception::User { owner: os_k1060_priority_state(2), generation: os_k1060_priority_state(3), root: os_k1060_priority_space().root, asid: 1, }; os_k920_exception::authorize_user_return(&frame, address, stack, user, &OsK1060PriorityReturn) .unwrap_or_else(|_| park()); if os_k1060_queue::Class::request(frame.x[0] as u8) != Err(os_k1060_queue::Error::Priority) { park(); } let kernel_address = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_KERNEL_FRAME)) }; stack .frame_address(kernel_address) .unwrap_or_else(|_| park()); let kernel = unsafe { read_volatile(kernel_address as *const os_k920_exception::Frame) }; let trusted = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_PRIORITY_BRIDGE)) }; unsafe extern "C" { static __os_k1060_priority_bridge_return: u8; } if kernel.elr != trusted[6] || kernel.spsr != trusted[7] || kernel.interrupted_sp != kernel_address + 848 || kernel.x[0] != mmu::USER_CODE_VA || kernel.x[1] != mmu::USER_STACK_TOP || kernel.x[30] < core::ptr::addr_of!(_text_start) as u64 || kernel.x[30] >= core::ptr::addr_of!(_text_end) as u64 || kernel.x[30] != core::ptr::addr_of!(__os_k1060_priority_bridge_return) as u64 || kernel.canary != os_k920_exception::FRAME_CANARY || kernel.padding != 0 { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_FRAMES).cast::(), frame, ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_FRAMES) .cast::() .add(1), kernel, ); } os_k1060_priority_word(4, address); os_k1060_priority_word(5, kernel_address); os_k1060_priority_word(6, os_k940_registers().0); os_k1060_priority_word(7, 1); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_BRIDGE) .cast::() .add(11), kernel_address, ); } os_k1060_capture(2); unsafe { asm!("dsb ishst","msr TTBR0_EL1, {root}","tlbi vmalle1","dsb ish","isb",root=in(reg)os_k1060_priority_state(1),options(nostack,preserves_flags)); } 1 } fn os_k1060_priority_prepare( memory: &mut OsK660Memory, tables: &mmu::Tables, boot: &BootData, tasks: &OsK660Tasks, ) -> Result<(), &'static str> { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { return Err("runqueue-priority-prepare-context"); } let alpha = tasks.id.ok_or("runqueue-priority-alpha")?; if tasks.scheduler.running() != Some(alpha) { return Err("runqueue-priority-selected"); } let space = unsafe { tables.build_user_space( &mut memory.allocator, boot.hardware.ram_base, boot.hardware.ram_size, ) } .map_err(|_| "runqueue-priority-space")?; if !space.contract_valid(tables.root) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_SPACE), Some(space), ); } return Err("runqueue-priority-custody"); } unsafe extern "C" { static __os_k1060_priority_user_code: u8; static __os_k1060_priority_user_code_end: u8; } let start = core::ptr::addr_of!(__os_k1060_priority_user_code) as u64; let bytes = core::ptr::addr_of!(__os_k1060_priority_user_code_end) as u64 - start; if bytes != 12 { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_SPACE), Some(space), ); } return Err("runqueue-priority-template"); } for i in 0..bytes { unsafe { write_volatile( (space.code_frame() + i) as *mut u8, read_volatile((start + i) as *const u8), ); } } unsafe { asm!("dc cvau, {code}","dsb ish","ic iallu","dsb ish","isb",code=in(reg)space.code_frame(),options(nostack,preserves_flags)); write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_SCHEDULER), tasks.scheduler.os_k980_inventory(), ); for (i, p) in space.owned_frames().enumerate() { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_OWNED) .cast::() .add(i), p, ); } write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_TASKS), tasks as *const OsK660Tasks, ); } os_k1060_priority_word(1, tables.root); os_k1060_priority_word(2, alpha.slot as u64 + 1); os_k1060_priority_word(3, alpha.generation); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_SPACE), Some(space), ); } Ok(()) } #[inline(never)] fn os_k1060_priority_run() -> Result<(), &'static str> { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || os_k960_timer().1 & 1 != 0 { return Err("runqueue-priority-entry-context"); } let snapshot = unsafe { (&*os_k1060_queue()).queue().snapshot() }; let order = unsafe { (&*os_k1060_queue()).queue().ready_order() }; let root = os_k1060_priority_space().root; os_k1060_priority_word(0, 1); unsafe extern "C" { fn __os_k1060_priority_bridge(entry: u64, stack: u64); } unsafe { asm!("dsb ishst","msr TTBR0_EL1, {root}","tlbi vmalle1","dsb ish","isb",root=in(reg)(root|(1<<48)),options(nostack,preserves_flags)); __os_k1060_priority_bridge(mmu::USER_CODE_VA, mmu::USER_STACK_TOP); } if os_k940_registers().0 != os_k1060_priority_state(1) || os_k1060_priority_state(7) != 1 { return Err("runqueue-priority-return-root"); } let registers = unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_PRIORITY_BRIDGE)) }; if registers[0] > 1 || registers[0] != registers[5] || registers[3] != registers[4] || registers[8] != 1 || registers[9] != registers[1] + if registers[0] == 0 { 16 } else { 0 } || registers[10] != registers[2] + if registers[0] == 1 { 16 } else { 0 } || registers[11] != os_k1060_priority_state(5) { return Err("runqueue-priority-stack-register-restoration"); } os_k1060_priority_word(0, 0); let q = unsafe { &mut *os_k1060_queue() }; let queue = q.quiescent().map_err(|_| "runqueue-priority-quiescent")?; if queue.request_priority(os_k1060_queue_handle(0), 2) != Err(os_k1060_queue::Error::Priority) || queue.snapshot() != snapshot || queue.ready_order() != order { return Err("runqueue-priority-unchanged"); } os_k1060_priority_word(8, 1); Ok(()) } fn os_k1060_priority_retire( memory: &mut OsK660Memory, kernel_root: u64, ) -> Result<(), &'static str> { let e = os_k1060_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { return Err("runqueue-priority-retire-context"); } if os_k1060_priority_state(0) != 0 || os_k940_registers().0 != kernel_root || os_k960_timer().1 & 1 != 0 { return Err("runqueue-priority-active-retirement"); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_TASKS), core::ptr::null(), ); asm!( "dsb ishst", "tlbi vmalle1", "dsb ish", "isb", options(nostack, preserves_flags) ); } let space = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1060_PRIORITY_SPACE)).take() } .ok_or("runqueue-priority-owner")?; if let Err((_, space)) = unsafe { space.release_after_tlbi(&mut memory.allocator, kernel_root) } { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIORITY_SPACE), Some(space), ); } return Err("runqueue-priority-retained-release"); } os_k1060_priority_word(9, 1); Ok(()) } // Actual birth checkpoints. The callback reads only immutable Wire and its // own CPU stack frame; all Queue/Context/Table loans already ended. #[unsafe(no_mangle)] static mut OS_K1060_SOURCE_FRAMES: [os_k1060_context::Cooperative; 21] = [os_k1060_context::Cooperative::EMPTY; 21]; #[unsafe(no_mangle)] static mut OS_K1060_LANDING_FRAMES: [os_k1060_context::Cooperative; 21] = [os_k1060_context::Cooperative::EMPTY; 21]; #[unsafe(no_mangle)] static mut OS_K1060_TRACE_WIRES: [OsK1060Wire; 21] = [OsK1060Wire { frame: os_k1060_context::Cooperative::EMPTY, pc: 0, argument: 0, nonce: 0, kind: 0, }; 21]; #[unsafe(no_mangle)] static mut OS_K1060_TRACE_WORDS: [[u64; 16]; 21] = [[0; 16]; 21]; #[unsafe(no_mangle)] static mut OS_K1060_SOURCE_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_LANDING_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1060_TRACE_OWNERS: [[u64;32];21] = [[0;32];21]; #[unsafe(no_mangle)] static OS_K1060_TRACE_QUEUE_LAYOUT: [u64;24]=os_k1060_queue::os_k1060_trace_queue_layout::,3>(); #[unsafe(no_mangle)] static OS_K1060_TRACE_CONTEXT_LAYOUT: [u64;38]=os_k1060_context::os_k1060_trace_context_layout::>(); #[unsafe(no_mangle)] static OS_K1060_TRACE_BACKEND_LAYOUT: [u64;16]=[ core::mem::size_of::() as u64, core::mem::offset_of!(OsK1060Prepared,source) as u64, core::mem::offset_of!(OsK1060Prepared,target) as u64, core::mem::offset_of!(OsK1060Prepared,nonce) as u64, core::mem::offset_of!(OsK1060Prepared,departed) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1060ReturnPrepared,source) as u64, core::mem::offset_of!(OsK1060ReturnPrepared,bootstrap_root) as u64, core::mem::offset_of!(OsK1060ReturnPrepared,bootstrap) as u64, core::mem::offset_of!(OsK1060ReturnPrepared,address) as u64, core::mem::offset_of!(OsK1060ReturnPrepared,nonce) as u64, 0,0,0,0,0, ]; // Immutable actual compiler representations identify the private Image // variant and field offsets without reading Rust enum padding as authority. #[unsafe(no_mangle)] static OS_K1060_IMAGE_LAYOUT_TEMPLATES: [os_k1060_context::Image; 2] = [ os_k1060_context::Image::Initial { entry: 0x4b31303630494e31, argument: 0x4b31303630494e32, sp: 0x4b31303630494e33, return_pc: 0x4b31303630494e34, }, os_k1060_context::Image::Cooperative(os_k1060_context::Cooperative::EMPTY), ]; #[unsafe(no_mangle)] static OS_K1060_MEMBER_NONE_LAYOUT_TEMPLATE: Option = None; // The trampoline executes before the saved Rust continuation. Frozen // completion observes that exact upcoming x30 PC; raw LANDING_PC continues // to record the independently executed trampoline, and RET restores x30. fn os_k1060_observed_landing_pc(frame: &os_k1060_context::Cooperative) -> u64 { let e=os_k1060_execution(); if e.cpu!=0 || !e.irq_masked || e.in_irq {park();} let actual=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_PC))}; unsafe extern "C" {fn __os_k1060_cooperative_landed();} if actual!=__os_k1060_cooperative_landed as *const () as u64 {return actual;} let wire=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_WIRE))}; if wire.kind!=1 || wire.nonce==0 || wire.pc!=frame.x19_x30[11] || wire.nonce!=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_DEPARTED_NONCE))} || wire.frame!=*frame {park();} frame.x19_x30[11] } fn os_k1060_publish_trace_owners() { let e=os_k1060_execution(); if e.cpu!=0 || !e.irq_masked || e.in_irq {park();} let index=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_SOURCE_COUNT))} as usize; if index>=21 {park();} let q=unsafe{&*os_k1060_queue()}; let mut words=[0;32]; words[..8].copy_from_slice(&q.os_k1060_trace_owners().unwrap_or_else(|_|park())); q.queue().with_backend_observation(|b|{ words[8..16].copy_from_slice(&b.bank.os_k1060_trace_owners().unwrap_or_else(|_|park())); words[16..24].copy_from_slice(&b.bank.os_k1060_trace_parts(false).unwrap_or_else(|_|park())); words[24..32].copy_from_slice(&b.bank.os_k1060_trace_parts(true).unwrap_or_else(|_|park())); }); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1060_TRACE_OWNERS).cast::<[u64;32]>().add(index),words);} } #[unsafe(no_mangle)] extern "C" fn __os_k1060_source_birth(address: u64, kind: u64) { let e=os_k1060_execution(); if e.cpu!=0 || !e.irq_masked || e.in_irq || kind>2 || address%16!=0 { park(); } let index=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_SOURCE_COUNT))} as usize; if index>=21 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_COUNT))}!=index as u64 { park(); } let source=if kind==0 {3} else {unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_EVIDENCE).cast::().add(103))}}; if !os_k1060_trace_range(address,source) {park();} let frame=unsafe{read_volatile(address as *const os_k1060_context::Cooperative)}; let wire=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_WIRE))}; if frame.sp!=address+640 || frame.canary!=os_k1060_context::CONTEXT_CANARY || frame.padding!=0 { park(); } let root=os_k940_registers().0; let mut target=3; for ordinal in 0..3 { if root==unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_ROOT_ADDRESSES).cast::().add(ordinal))} { target=ordinal as u64; } } let episode=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_CAPTURE_EPISODE))}; let lookup=if kind==2 { unsafe{(&*core::ptr::addr_of!(OS_K1060_RETURN)).as_ref()}.unwrap_or_else(||park()).nonce() } else { unsafe{(&*core::ptr::addr_of!(OS_K1060_HANDOFF)).as_ref()}.unwrap_or_else(||park()).nonce() }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_SOURCE_FRAMES).cast::().add(index),frame); write_volatile(core::ptr::addr_of_mut!(OS_K1060_TRACE_WIRES).cast::().add(index),wire); write_volatile(core::ptr::addr_of_mut!(OS_K1060_TRACE_WORDS).cast::<[u64;16]>().add(index),[ address,kind,source,target,root,wire.pc,wire.argument,wire.nonce,wire.kind, episode,frame.sp,frame.x19_x30[11],lookup,0,0,0, ]); write_volatile(core::ptr::addr_of_mut!(OS_K1060_SOURCE_COUNT),index as u64+1); } os_k1060_capture(40+index as u64*2); } fn os_k1060_landing_birth(address: u64) { let e=os_k1060_execution(); if e.cpu!=0 || !e.irq_masked || e.in_irq || address%16!=0 {park();} let index=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_COUNT))} as usize; if index>=21 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_SOURCE_COUNT))}!=index as u64+1 {park();} let words=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_TRACE_WORDS).cast::<[u64;16]>().add(index))}; if !os_k1060_trace_range(address,words[3]) {park();} let frame=unsafe{read_volatile(address as *const os_k1060_context::Cooperative)}; if frame.sp!=address+640 || frame.canary!=os_k1060_context::CONTEXT_CANARY || frame.padding!=0 {park();} unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1060_LANDING_FRAMES).cast::().add(index),frame); let row=core::ptr::addr_of_mut!(OS_K1060_TRACE_WORDS).cast::<[u64;16]>().add(index); let mut words=read_volatile(row); if !os_k1060_trace_range(address,words[3]) {park();} words[13]=address;words[14]=os_k940_registers().0; words[15]=read_volatile(core::ptr::addr_of!(OS_K1060_LANDING_PC)); write_volatile(row,words); write_volatile(core::ptr::addr_of_mut!(OS_K1060_LANDING_COUNT),index as u64+1); } os_k1060_capture(41+index as u64*2); } fn os_k1060_trace_range(address:u64,ordinal:u64)->bool { let (low,high)=if ordinal==3 { let guard=core::ptr::addr_of!(OS_K1060_PRIMARY_STACK) as u64; (guard+4096,guard+4096+65536) } else if ordinal<3 { let guard=unsafe{read_volatile(core::ptr::addr_of!(OS_K1060_OWNED).cast::().add(ordinal as usize*9))}; (guard+4096,guard+8*4096) } else {return false;}; address%16==0 && address>=low+16 && address.checked_add(640).is_some_and(|end|end<=high-16) } fn os_k1060_retain_failure(actual: T, kind: u64) -> ! { if core::mem::size_of::() > core::mem::size_of::() || core::mem::align_of::() > core::mem::align_of::() { park(); } unsafe { core::ptr::addr_of_mut!(OS_K1060_FAILURE_CUSTODY) .cast::() .write(actual); } os_k1060_word(120, kind); os_k1060_word(121, core::mem::size_of::() as u64); park() } struct OsK1060Installation<'a> { pointer: *mut OsK1060Queue<'a>, active: bool, ownership: core::marker::PhantomData>, } impl<'a> OsK1060Installation<'a> { unsafe fn install(q: OsK1060Queue<'a>) -> Self { let pointer = core::ptr::addr_of_mut!(OS_K1060_QUEUE).cast::>(); unsafe { pointer.write(q); } Self { pointer, active: true, ownership: core::marker::PhantomData, } } fn extract(mut self) -> OsK1060ContextBackend<'a> { { let q = unsafe { &mut *self.pointer }; q.quiescent() .unwrap_or_else(|_| park()) .audit() .unwrap_or_else(|_| park()); if q.queue().snapshot().members != 0 || q.queue().snapshot().active != 0 || q.return_stage().is_some() || q.quarantined() { park(); } } // The root raw pointer comes from this sole typed BSS Installation, // not from a shared backend observation. Compiler offsets derive the // exact nested CB subobject; all Queue and Context loans ended above. let layout = os_k1060_queue::os_k1060_return_layout::(); let offset = layout[1] + layout[6] + layout[11] + core::mem::offset_of!(OsK1060Backend, bank) as u64; let source = unsafe { self.pointer.cast::().add(offset as usize) }.cast::>(); let mut destination = core::mem::MaybeUninit::uninit(); unsafe { OsK1060Bank::os_k1040_take_backend_in_place(source, &mut destination) } .unwrap_or_else(|_| park()); self.active = false; os_k1060_word(96, 0); unsafe { destination.assume_init() } } } impl Drop for OsK1060Installation<'_> { fn drop(&mut self) { if self.active { park(); } } } #[inline(never)] fn os_k1060_factory<'a>( memory: &'a mut OsK660Memory, tables: &'a mmu::Tables, bootstrap: u64, ) -> OsK1060Installation<'a> { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags)); } os_k1060_word(112, sp); unsafe extern "C" { fn __os_k1060_a_entry(); fn __os_k1060_b_entry(); fn __os_k1060_c_entry(); } for ordinal in 0..3 { let root = os_k1060_create_root(memory, tables).unwrap_or_else(|_| park()); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_ROOT_ADDRESSES) .cast::() .add(ordinal), root.address(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_ROOT_GENERATIONS) .cast::() .add(ordinal), root.generation(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1060_ROOT_LIVE) .cast::() .add(ordinal), 1, ); } let stack = match os_k1060_allocate_stack(memory, tables, ordinal, root) { Ok(s) => s, Err((_, root)) => { unsafe { core::ptr::addr_of_mut!(OS_K1060_FAILED_ROOTS) .cast::>() .add(ordinal) .write(Some(root)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1060_SPARES) .cast::>() .add(ordinal) .write(Some(stack)); } } let mut table = match OsK1060Table::new(OsK1060StackBackend { memory, tables, trace: true, }) { Ok(t) => t, Err((_, b)) => os_k1060_retain_failure(b, 1), }; let entries = [ __os_k1060_a_entry as *const () as u64, __os_k1060_b_entry as *const () as u64, __os_k1060_c_entry as *const () as u64, ]; let mut ids = [None; 3]; for ordinal in 0..3 { let stack = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1060_SPARES) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); ids[ordinal] = Some( match table.create(stack, entries[ordinal], ordinal as u64) { Ok(id) => id, Err((_, stack)) => { unsafe { core::ptr::addr_of_mut!(OS_K1060_SPARES) .cast::>() .add(ordinal) .write(Some(stack)); } os_k1060_retain_failure(table, 2) } }, ); } let mut backend = OsK1060ContextBackend { table, bootstrap, current: None, pins: [false; 3], records: [None; 3], observations: [None; 3], guards: [0; 3], sleep_wait: None, sequence: 0, }; for ordinal in 0..3 { let binding = match backend.pin(ids[ordinal].unwrap(), ordinal) { Ok(b) => b, Err(_) => os_k1060_retain_failure(backend, 3), }; unsafe { core::ptr::addr_of_mut!(OS_K1060_BINDINGS) .cast::>() .add(ordinal) .write(Some(binding)); } } let mut factory = unsafe { OsK1060Factory::start(backend) }; factory.context(); factory.attach(); factory.asynchronous(); factory.completion(); factory.members(); factory.issue(); factory.policy(); factory.enqueue(); factory.selection(); factory.install() } #[unsafe(no_mangle)] extern "C" fn __os_k1060_task_work(ordinal: u64) { if ordinal >= 3 { park(); } let progress = unsafe { core::ptr::addr_of_mut!(OS_K1060_TASK_PROGRESS) .cast::() .add(ordinal as usize) }; let entries = unsafe { core::ptr::addr_of_mut!(OS_K1060_TASK_ENTRIES) .cast::() .add(ordinal as usize) }; unsafe { write_volatile(entries, read_volatile(entries) + 1); } unsafe extern "C" { fn __os_k1060_cpu_work() -> u64; fn __os_k1060_cooperate(target: u64); fn __os_k1060_timer_work(sequence: u64) -> u64; } // Each real initial Ready member lands and makes progress before the // controlled sleep episode. The first source's actual Wait stays private. let sum = unsafe { __os_k1060_cpu_work() }; if sum != 500500 { park(); } unsafe { write_volatile(progress, read_volatile(progress) + sum); __os_k1060_cooperate(0); } if ordinal == 0 { os_k1060_observe(); os_k1060_capture(3); unsafe { __os_k1060_cooperate(1); } os_k1060_event(0x72, 0, unsafe { read_volatile(progress) }, 1); } else if ordinal == 1 { os_k1060_sleep_refusal(); unsafe { __os_k1060_cooperate(0); } } else { os_k1060_wake_actual(); unsafe { __os_k1060_cooperate(0); } } for _ in 0..3 { let sum = unsafe { __os_k1060_cpu_work() }; if sum != 500500 { park(); } unsafe { write_volatile(progress, read_volatile(progress) + sum); } let quantum = if ordinal == 1 { 2 } else { 4 }; for left in (0..quantum).rev() { let sequence = os_k1060_arm_tick(ordinal as usize); let iterations = unsafe { __os_k1060_timer_work(sequence) }; if iterations == 0 { park(); } let expired = os_k1060_charge_actual(ordinal as usize, sequence); if expired != (left == 0) { park(); } unsafe { let row = core::ptr::addr_of_mut!(OS_K1060_CPU_ITERATIONS) .cast::() .add(ordinal as usize); write_volatile( row, read_volatile(row) .checked_add(iterations) .unwrap_or_else(|| park()), ); } } unsafe { __os_k1060_cooperate(0); } } } fn os_k1060_accept( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { os_k1060_retain_configuration(); if cfg!(feature = "os-runtime-product") && !os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K1060: PRODUCT-CONTROL NO-RUNQUEUE-PROBE"); return Ok(()); } let (root, tcr, vbar, daif) = os_k940_registers(); if root != tables.root || tcr & 63 != 25 || daif & 0x80 == 0 || os_k960_timer().1 & 1 != 0 { return Err("runqueue-predecessor-control"); } for (i, v) in [ (0, 0x4b31303630515545), (1, root), (2, tcr), (3, vbar), (4, daif), (20, unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)) }), (21, 27), (22, 6), ] { os_k1060_word(i, v); } let (_, memory, _, tasks) = runtime.running().map_err(os_k660_policy::Error::tag)?; if tasks.scheduler.running() != tasks.id { return Err("runqueue-alpha-owner"); } os_k940_inventory( memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K1060_AUTHORITY_BEFORE).cast(), ); os_k1060_word( 70, memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64, ); os_k1060_word(71, core::mem::size_of::() as u64); let image_guard = (core::ptr::addr_of!(_image_end) as u64) .checked_add(4095) .ok_or("runqueue-image-guard-overflow")? & !4095; let guard_slot = tables .metadata_stack_guard_descriptor_address(image_guard) .ok_or("runqueue-image-guard-owned-leaf")?; let (par, mair, sctlr, current_el): (u64, u64, u64, u64); unsafe { asm!("at s1e1r, {guard}", "isb", "mrs {par}, PAR_EL1", "mrs {mair}, MAIR_EL1", "mrs {sctlr}, SCTLR_EL1", "mrs {current_el}, CurrentEL", guard=in(reg)image_guard, par=out(reg)par, mair=out(reg)mair, sctlr=out(reg)sctlr, current_el=out(reg)current_el, options(nostack,preserves_flags)); } for (i, value) in [(90, image_guard), (91, guard_slot), (92, par), (93, mair), (94, sctlr), (95, current_el)] { os_k1060_word(i, value); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRE_IRQ), [ read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) as u64, ], ); } os_k1060_capture(0); os_k1060_priority_prepare(memory, tables, boot, tasks)?; let before: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before,options(nomem,nostack,preserves_flags)); } os_k1060_word(111, before); let installation = os_k1060_factory(memory, tables, root); os_k1060_word(96, 1); os_k1060_observe(); os_k1060_duplicate_refusal(); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRE_IRQ), [ read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_LAST_COUNTER)), read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) as u64, ], ); } unsafe extern "C" { static __os_k1060_vectors: u8; } unsafe { asm!("msr VBAR_EL1, {vectors}","isb",vectors=in(reg)core::ptr::addr_of!(__os_k1060_vectors) as u64,options(nomem,nostack,preserves_flags)); } os_k1060_capture(1); os_k1060_priority_run()?; unsafe extern "C" { fn __os_k1060_run(wire: *const OsK1060Wire); } for _ in 0..3 { let t = unsafe { (&mut *os_k1060_queue()).select(os_k1060_queue::Disposition::Ready) } .map_err(|_| "runqueue-initial-selection")?; let wire = os_k1060_depart(t); unsafe { __os_k1060_run(wire); } os_k1060_observe(); } if unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_IRQ_COUNT)) } != 30 || os_k960_timer().1 & 1 != 0 { return Err("runqueue-actual-quantum-count"); } unsafe { asm!("msr VBAR_EL1, {vectors}","isb",vectors=in(reg)vbar,options(nomem,nostack,preserves_flags)); } os_k1060_capture(6); for ordinal in 0..3 { let h = unsafe { read_volatile( core::ptr::addr_of!(OS_K1060_QUEUE_HANDLES) .cast::>() .add(ordinal), ) } .ok_or("runqueue-owned-handle")?; let q = unsafe { &mut *os_k1060_queue() }; let q = q.quiescent().map_err(|_| "runqueue-retire-quiescent")?; let m = q.withdraw(h).map_err(|_| "runqueue-withdraw")?; q.os_k1060_probe_withdrawn_ready_slot(h) .map_err(|_| "runqueue-stale-ready-injection")?; if q.record(h) != Err(os_k1060_queue::Error::Stale) || q.requeue(h) != Err(os_k1060_queue::Error::Stale) { return Err("runqueue-stale-withdrawn-slot"); } os_k1060_event(0x73, h.slot() as u64, h.generation(), ordinal as u64); if let Err((_, m)) = q.os_k1060_retire_withdrawn(m) { unsafe { core::ptr::addr_of_mut!(OS_K1060_MEMBERS) .cast::>() .add(ordinal) .write(Some(m)); } return Err("runqueue-retire-retained"); } } os_k1060_observe(); os_k1060_capture(7); let backend = installation.extract(); let stack = match backend.table.into_backend() { Ok(b) => b, Err((_, table)) => { unsafe { core::ptr::addr_of_mut!(OS_K1060_FAILED_TABLE) .cast::() .write(table); } return Err("runqueue-table-retained"); } }; os_k1060_priority_retire(stack.memory, tables.root)?; os_k940_inventory( stack.memory.allocator.os_authority(), core::ptr::addr_of_mut!(OS_K1060_AUTHORITY_AFTER).cast(), ); os_k1060_capture(8); if !os_k920_inventory_equal( core::ptr::addr_of!(OS_K1060_AUTHORITY_BEFORE).cast(), core::ptr::addr_of!(OS_K1060_AUTHORITY_AFTER).cast(), ) || os_k940_registers() != (root, tcr, vbar, daif) { return Err("runqueue-owner-restoration"); } // These guest markers follow the complete actual ownership restoration. // The external report additionally verifies every physical birth/IRQ. write_line("TURKOS/OS/K1045: PASS"); write_line("TURKOS/OS/K1050: PASS"); write_line("TURKOS/OS/K1055: PASS"); write_line("TURKOS/OS/K1060: PASS"); Ok(()) } #[inline(never)] fn os_k1060_retain_configuration() { unsafe extern "C" { fn __os_k1060_run(wire: *const OsK1060Wire); fn __os_k1060_a_entry(); fn __os_k1060_b_entry(); fn __os_k1060_c_entry(); fn __os_k1060_primary_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; static __os_k1060_vectors: u8; fn os_k1060_dispatch(pointer: *mut os_k920_exception::Frame) -> u64; } for p in [ __os_k1060_run as *const () as u64, __os_k1060_a_entry as *const () as u64, __os_k1060_b_entry as *const () as u64, __os_k1060_c_entry as *const () as u64, __os_k1060_primary_on_stack as *const () as u64, __os_k1060_primary_body as *const () as u64, os_k1060_factory as *const () as u64, // Retain the symbol through a mapped byte; the guard is already // deliberately unmapped before acceptance enters this function. core::ptr::addr_of!(OS_K1060_PRIMARY_STACK) as u64 + 4096, core::ptr::addr_of!(OS_K1060_PRIMARY_STACK_EVIDENCE) as u64, core::ptr::addr_of!(OS_K1060_QUEUE) as u64, core::ptr::addr_of!(OS_K1060_RETURN_LAYOUT) as u64, core::ptr::addr_of!(OS_K1060_QUEUE_BACKEND_LAYOUT) as u64, core::ptr::addr_of!(OS_K1060_CONTEXT_LAYOUT) as u64, core::ptr::addr_of!(OS_K1060_THREAD_LAYOUT) as u64, core::ptr::addr_of!(__os_k1060_vectors) as u64, os_k1060_dispatch as *const () as u64, core::ptr::addr_of!(OS_K1060_IRQ_FRAMES) as u64, core::ptr::addr_of!(OS_K1060_IRQ_RECORDS) as u64, core::ptr::addr_of!(OS_K1060_PRIORITY_FRAMES) as u64, core::ptr::addr_of!(OS_K1060_PRIORITY_BRIDGE) as u64, core::ptr::addr_of!(OS_K1060_PRIORITY_OWNED) as u64, core::ptr::addr_of!(OS_K1060_PRIORITY_SCHEDULER) as u64, core::ptr::addr_of!(OS_K1060_STACK_WORDS) as u64, core::ptr::addr_of!(OS_K1060_TABLE_WORDS) as u64, core::ptr::addr_of!(OS_K1060_BACKEND_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1060_THREAD_ID_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1060_LEASE_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1060_WAIT_LAYOUT_EXPECTED) as u64, core::ptr::addr_of!(OS_K1060_TRACE_QUEUE_LAYOUT) as u64, core::ptr::addr_of!(OS_K1060_TRACE_CONTEXT_LAYOUT) as u64, core::ptr::addr_of!(OS_K1060_TRACE_BACKEND_LAYOUT) as u64, core::ptr::addr_of!(OS_K1060_IMAGE_LAYOUT_TEMPLATES) as u64, core::ptr::addr_of!(OS_K1060_MEMBER_NONE_LAYOUT_TEMPLATE) as u64, core::ptr::addr_of!(OS_K1060_AUTHORITY_BEFORE) as u64, core::ptr::addr_of!(OS_K1060_AUTHORITY_AFTER) as u64, core::ptr::addr_of!(OS_K1060_PRE_IRQ) as u64, ] { unsafe { asm!("ldrb wzr, [{p}]",p=in(reg)p,options(nostack,preserves_flags)); } } } #[repr(C, align(4096))] struct OsK1060PrimaryStack { guard: [u8; 4096], usable: [u8; 65536], } #[unsafe(no_mangle)] static mut OS_K1060_PRIMARY_STACK: OsK1060PrimaryStack = OsK1060PrimaryStack { guard: [0; 4096], usable: [0; 65536], }; #[unsafe(no_mangle)] static mut OS_K1060_PRIMARY_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K1060_PRIMARY_ERROR: Option<&'static str> = None; const OS_K1060_PRIMARY_LOW_CANARY: u64 = 0x4b31_3034_4c4f_5721; const OS_K1060_PRIMARY_HIGH_CANARY: u64 = 0x4b31_3034_4849_4748; fn os_k1060_primary_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIMARY_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } #[inline(never)] fn os_k1060_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { if cfg!(feature = "os-runtime-product") && !os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K1060: PRODUCT-CONTROL NO-RUNQUEUE-PROBE"); return Ok(()); } let guard = core::ptr::addr_of!(OS_K1060_PRIMARY_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_guard = core::ptr::addr_of!(OS_K900_STACK) as u64; let original_low = original_guard + 4096; let original_top = original_low + 65536; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("context-primary-original-stack-range"); } if before_sp - (original_low + 8) < 6144 { return Err("runqueue-primary-original-stack-headroom"); } os_k1060_primary_stack_word(22, 6144); os_k1060_primary_stack_word(23, before_sp); let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_top - 8) as *const u64) }; if original_canary != 0x4b38_3230_4c4f_5721 || original_second != 0x4b38_3230_4849_4748u64 { return Err("context-primary-original-stack-canaries"); } unsafe { tables.unmap_metadata_stack_guard( guard, low, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64, ) } .map_err(|_| "context-primary-stack-guard")?; if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("context-primary-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K1060_PRIMARY_LOW_CANARY); write_volatile(high as *mut u64, OS_K1060_PRIMARY_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K1060_PRIMARY_LOW_CANARY), (9, OS_K1060_PRIMARY_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("context-primary-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("context-primary-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k1060_primary_stack_word(slot, value); } unsafe extern "C" { fn __os_k1060_primary_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k1060_primary_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K1060_PRIMARY_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K1060_PRIMARY_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K1060_PRIMARY_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_top - 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("context-primary-stack-restoration"); } os_k1060_primary_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k1060_primary_stack_word(17, unsafe { read_volatile((original_top - 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K1060: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); os_k1060_capture(9); Ok(()) } else { Err( unsafe { read_volatile(core::ptr::addr_of!(OS_K1060_PRIMARY_ERROR)) } .unwrap_or("context-primary-stack-return"), ) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1060_primary_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k1060_primary_stack_word(4, sp); let result = os_k1060_accept( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k1060_primary_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1060_PRIMARY_ERROR), Some(reason), ) }; 1 } } } global_asm!( r#" .section .text.os_k1060_primary_stack,"ax" .balign 4 .global __os_k1060_primary_on_stack .type __os_k1060_primary_on_stack, %function __os_k1060_primary_on_stack: adrp x9, OS_K1060_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1060_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K1060_PRIMARY_STACK add x9, x9, :lo12:OS_K1060_PRIMARY_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K1060_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1060_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k1060_primary_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K1060_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1060_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret .size __os_k1060_primary_on_stack, .-__os_k1060_primary_on_stack "# ); global_asm!(include_str!("os_k1060_runqueue.S")); const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k1060_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); use turkos_kernel::runtime::{wait_core as os_k1080_wait,wait_bridge as os_k1080_bridge}; const OS_K1080_PA_MASK: u64 = 0x0000_ffff_ffff_f000; /// Owns only the new root page. Kernel branches remain separately owned by the /// inherited Tables and are validated as EL0 inaccessible before sharing. struct OsK1080Root { lease: Option, bootstrap: u64, branches: [u64; 2], } impl OsK1080Root { fn address(&self) -> u64 { self.lease.as_ref().map_or(0, |l| l.address()) } fn generation(&self) -> u64 { self.lease.as_ref().map_or(0, |l| l.generation()) } fn validate(&self, memory: &OsK660Memory, tables: &mmu::Tables) -> Result<(), &'static str> { let lease = self.lease.as_ref().ok_or("context-root-token")?; if self.bootstrap != tables.root || lease.address() == tables.root || lease.address() % 4096 != 0 || lease.generation() == 0 { return Err("context-root-owner"); } memory .allocator .os_authority() .validate_lease(lease) .map_err(|_| "context-root-generation")?; os_k1080_kernel_branches(tables, self.branches)?; os_k1080_root_words(lease.address(), self.branches) } } fn os_k1080_shared_walk( tables: &mmu::Tables, address: u64, level: u8, visited: &mut [u64; 4], count: usize, ) -> Result<(), &'static str> { if !(1..=3).contains(&level) || count >= 4 || visited[..count].contains(&address) || !tables.tagged_frames().any(|frame| frame.address == address) { return Err("context-shared-kernel-owner"); } visited[count] = address; for index in 0..512 { let raw = unsafe { read_volatile((address as *const u64).add(index)) }; if raw == 0 { continue; } match raw & 3 { 3 if level < 3 => { if raw != (raw & OS_K1080_PA_MASK) | 3 { return Err("context-shared-kernel-table-flags"); } os_k1080_shared_walk( tables, raw & OS_K1080_PA_MASK, level + 1, visited, count + 1, )?; } 1 if level < 3 => { if raw & (1 << 6) != 0 { return Err("context-shared-kernel-user-block"); } } 3 if level == 3 => { if raw & (1 << 6) != 0 { return Err("context-shared-kernel-user-page"); } } _ => return Err("context-shared-kernel-descriptor"), } } Ok(()) } fn os_k1080_root_words(address: u64, branches: [u64; 2]) -> Result<(), &'static str> { if address == 0 || address & !OS_K1080_PA_MASK != 0 { return Err("context-root-shape-address"); } // The caller has already checked the live typed root owner (or the // immutable pin established from that owner before IRQ unmask). Check the // entire page before accepting any later observation or root selection. for index in 0..512 { let expected = branches.get(index).copied().unwrap_or(0); if unsafe { read_volatile((address as *const u64).add(index)) } != expected { return Err("context-root-shape"); } } Ok(()) } fn os_k1080_kernel_branches(tables: &mmu::Tables, branches: [u64; 2]) -> Result<(), &'static str> { // Establish the inherited root ownership before forming its physical // pointer. Child table ownership is established independently at each // recursive step, before its pointer is formed too. if !tables .tagged_frames() .any(|frame| frame.address == tables.root) { return Err("context-shared-root-owner"); } for (index, raw) in branches.into_iter().enumerate() { if unsafe { read_volatile((tables.root as *const u64).add(index)) } != raw { return Err("context-shared-root-changed"); } match raw & 3 { 0 if raw == 0 => {} 3 => { if raw != (raw & OS_K1080_PA_MASK) | 3 { return Err("context-shared-root-table-flags"); } os_k1080_shared_walk(tables, raw & OS_K1080_PA_MASK, 2, &mut [0; 4], 0)?; } 1 if raw & (1 << 6) == 0 => {} _ => return Err("context-shared-root-descriptor"), } } Ok(()) } fn os_k1080_create_root( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result { let (root, tcr, _, daif) = os_k940_registers(); if root != tables.root || tcr & 63 != 25 || daif & 0x80 == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { return Err("context-root-bootstrap-context"); } // Only kernel L1 branches0/1 are inherited; user branch2 is absent. if !tables.tagged_frames().any(|frame| frame.address == root) { return Err("context-shared-root-owner"); } let mut branches = [0u64; 2]; for (index, raw) in branches.iter_mut().enumerate() { *raw = unsafe { read_volatile((root as *const u64).add(index)) }; } os_k1080_kernel_branches(tables, branches)?; let address = memory .allocator .os_authority() .free_regions() .iter() .rev() .find_map(|region| { region .end .checked_sub(4096) .filter(|address| *address >= region.start) }) .ok_or("context-root-ENOSPC")?; let lease = memory .allocator .os_authority_mut() .allocate_at(address) .map_err(|_| "context-root-acquire")?; let owned = OsK1080Root { lease: Some(lease), bootstrap: root, branches, }; for index in 0..512 { unsafe { write_volatile((address as *mut u64).add(index), 0); } } for (index, raw) in branches.into_iter().enumerate() { unsafe { write_volatile((address as *mut u64).add(index), raw); } } unsafe { asm!("dsb ishst", options(nostack, preserves_flags)); } os_k1080_event(0x41, address, owned.generation(), root); Ok(owned) } fn os_k1080_select_root(root: u64) -> Result<(), &'static str> { let (_, tcr, _, daif) = os_k940_registers(); if root == 0 || root % 4096 != 0 || root & !OS_K1080_PA_MASK != 0 || tcr & 63 != 25 || daif & 0x80 == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { return Err("context-root-select-context"); } unsafe { asm!("dsb ishst", options(nostack, preserves_flags)); } os_k1080_event(0x43, root, 0, 0); unsafe { asm!("msr TTBR0_EL1, {root}",root=in(reg)root,options(nostack,preserves_flags)); } os_k1080_event(0x44, root, 0, 0); unsafe { asm!("tlbi vmalle1is", options(nostack, preserves_flags)); } os_k1080_event(0x45, root, 0, 0); unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } os_k1080_event(0x46, root, 0, 0); unsafe { asm!("isb", options(nostack, preserves_flags)); } os_k1080_event(0x47, root, 0, 0); if os_k940_registers().0 != root { return Err("context-root-readback"); } Ok(()) } fn os_k1080_erase_inactive_root( root: &OsK1080Root, tables: &mmu::Tables, ) -> Result<(), &'static str> { let actual = os_k940_registers(); if actual.0 != tables.root || actual.3 & 0x80 == 0 || root.bootstrap != tables.root || root.address() == actual.0 || root.address() == 0 { return Err("context-root-active-free-refusal"); } // Bootstrap select completed a full TLBI before this operation. Erasure // alone returns no authority: StackBackend publishes one atomic six-lease // return only after its guard restoration and full stack erasure too. for index in 0..512 { unsafe { write_volatile((root.address() as *mut u64).add(index), 0); } } unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } if !(0..512) .all(|index| unsafe { read_volatile((root.address() as *const u64).add(index)) } == 0) { return Err("context-root-erasure"); } os_k1080_event(0x48, root.address(), root.generation(), 0); Ok(()) } use turkos_kernel::runtime::wait_threads as os_k1080_thread; const OS_K1080_STACK_COUNT: usize = 3; const OS_K1080_STACK_PAGES: usize = 7; const OS_K1080_OWNED_PAGES: usize = OS_K1080_STACK_COUNT * (OS_K1080_STACK_PAGES + 2); struct OsK1080Stack { root: OsK1080Root, ordinal: usize, start: u64, leases: [Option; 8], guard: Option, initial_observation: core::cell::Cell, } impl OsK1080Stack { fn layout(&self) -> os_k1080_thread::StackLayout { os_k1080_thread::StackLayout { base: self.start + 4096, physical: self.start + 4096, bytes: OS_K1080_STACK_PAGES as u64 * 4096, } } } fn os_k1080_cpu_root_allowed(root: u64, tables: &mmu::Tables) -> bool { root==tables.root||(0..3).any(|slot|unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_ROOT_ADDRESSES).cast::().add(slot))}==root&&root!=0&&unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_ROOT_LIVE).cast::().add(slot))}==1) } struct OsK1080StackBackend<'a> { memory: &'a mut OsK660Memory, tables: &'a mmu::Tables, trace: bool, } fn os_k1080_execution() -> os_k1080_thread::Execution { let (cpu, daif, sp): (u64, u64, u64); unsafe { asm!("mrs {cpu}, MPIDR_EL1", "mrs {daif}, DAIF", "mov {sp}, sp", cpu=out(reg)cpu, daif=out(reg)daif, sp=out(reg)sp, options(nomem,nostack,preserves_flags)); } os_k1080_thread::Execution { cpu: (cpu & 0xff) as u32, irq_masked: daif & 0x80 != 0, in_irq: unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0, sp, } } fn os_k1080_allocate_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ordinal: usize, root: OsK1080Root, ) -> Result { if ordinal >= 3 || root.validate(memory, tables).is_err() { return Err(("context-stack-root-owner", root)); } let authority = memory.allocator.os_authority(); let mut selected = None; let mut checked = 0; for region in authority.free_regions() { let mut address = region.start; while address .checked_add(8 * 4096) .is_some_and(|end| end <= region.end) && checked < 512 { checked += 1; if tables.os_k1020_stack_candidate(address, address + 8 * 4096) { selected = Some(address); break; } let Some(next) = address.checked_add(4096) else { return Err(("context-stack-candidate-overflow", root)); }; address = next; } if selected.is_some() { break; } } let Some(start) = selected else { return Err(("context-stack-no-owned-L3-span", root)); }; let leases = match memory .allocator .os_authority_mut() .allocate_many_at(start, 8) { Ok(leases) => leases, Err(_) => return Err(("context-stack-acquire", root)), }; // The eight live stack leases are immediately inside this actual owner; // failure cannot discard a prefix or the independently retained root. let stack = OsK1080Stack { root, ordinal, start, leases, guard: None, initial_observation: core::cell::Cell::new(false), }; for offset in 0..8 * 4096 / 8 { unsafe { write_volatile((start as *mut u64).add(offset), 0xa5a5a5a5a5a5a5a5); } } for index in 0..9 { let lease = if index < 8 { stack.leases[index].as_ref().unwrap_or_else(|| park()) } else { stack.root.lease.as_ref().unwrap_or_else(|| park()) }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_OWNED) .cast::() .add(ordinal * 9 + index), lease.address(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1080_LEASE_GENERATIONS) .cast::() .add(ordinal * 9 + index), lease.generation(), ); } os_k1080_event(1, ordinal as u64, lease.address(), lease.generation()); } Ok(stack) } impl OsK1080StackBackend<'_> { fn validate(&self, stack: &OsK1080Stack) -> Result<(), os_k1080_thread::Error> { let e = os_k1080_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || !os_k1080_cpu_root_allowed(os_k940_registers().0, self.tables) { return Err(os_k1080_thread::Error::Context); } if stack.ordinal >= OS_K1080_STACK_COUNT || stack.start % 4096 != 0 || stack.leases[OS_K1080_STACK_PAGES + 1..] .iter() .any(Option::is_some) { return Err(os_k1080_thread::Error::Layout); } for index in 0..=OS_K1080_STACK_PAGES { let lease = stack.leases[index] .as_ref() .ok_or(os_k1080_thread::Error::Backend)?; if lease.address() != stack.start + index as u64 * 4096 { return Err(os_k1080_thread::Error::Layout); } self.memory .allocator .os_authority() .validate_lease(lease) .map_err(|_| os_k1080_thread::Error::Backend)?; } stack .root .validate(self.memory, self.tables) .map_err(|_| os_k1080_thread::Error::Backend)?; Ok(()) } } unsafe impl os_k1080_thread::Backend for OsK1080StackBackend<'_> { type Stack = OsK1080Stack; fn execution(&self) -> os_k1080_thread::Execution { os_k1080_execution() } fn layout( &self, stack: &OsK1080Stack, ) -> Result { self.validate(stack)?; Ok(stack.layout()) } fn entry_executable(&self, entry: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } entry.is_multiple_of(4) && entry >= core::ptr::addr_of!(_text_start) as u64 && entry < core::ptr::addr_of!(_text_end) as u64 } fn prepare(&mut self, stack: &mut OsK1080Stack) -> Result<(), os_k1080_thread::Error> { self.validate(stack)?; let layout = stack.layout(); let e = os_k1080_execution(); if e.sp >= stack.start && e.sp <= layout.base + layout.bytes || stack.guard.is_some() || os_k940_registers().0 != self.tables.root { return Err(os_k1080_thread::Error::ActiveStack); } let guard = unsafe { self.tables .os_k1020_acquire_stack_guard(stack.start, layout.base + layout.bytes) } .map_err(|_| os_k1080_thread::Error::Guard)?; stack.guard = Some(guard); for offset in 0..layout.bytes / 8 { unsafe { write_volatile((layout.base as *mut u64).add(offset as usize), 0); } } unsafe { write_volatile(layout.base as *mut u64, os_k1080_thread::LOW_CANARY); write_volatile( (layout.base + layout.bytes - 8) as *mut u64, os_k1080_thread::HIGH_CANARY, ); asm!("dsb ish", options(nostack, preserves_flags)); } stack.initial_observation.set(true); if self.trace { os_k1080_event(2, stack.ordinal as u64, stack.start, layout.bytes); } Ok(()) } fn observe( &self, stack: &OsK1080Stack, ) -> Result { self.validate(stack)?; let layout = stack.layout(); let token = stack.guard.as_ref().ok_or(os_k1080_thread::Error::Guard)?; let descriptor = self .tables .os_k1020_stack_guard_descriptor(token) .map_err(|_| os_k1080_thread::Error::Guard)?; if token.guard() != stack.start || token.end() != layout.base + layout.bytes { return Err(os_k1080_thread::Error::Layout); } // Initial full erasure is observed once before publication. Later // selection/exit checks read the actual guard and canaries without // walking the active or previously used Rust stack contents. let zeroed=stack.initial_observation.get()&&(1..layout.bytes/8-1).all(|offset|unsafe {read_volatile((layout.base as *const u64).add(offset as usize))}==0); if zeroed { stack.initial_observation.set(false); } Ok(os_k1080_thread::StackObservation { layout, guard_descriptor: descriptor, low_canary: unsafe { read_volatile(layout.base as *const u64) }, high_canary: unsafe { read_volatile((layout.base + layout.bytes - 8) as *const u64) }, initially_zeroed: zeroed, }) } fn release( &mut self, mut stack: OsK1080Stack, ) -> Result<(), (os_k1080_thread::Error, OsK1080Stack)> { if let Err(error) = self.validate(&stack) { return Err((error, stack)); } let layout = stack.layout(); let e = os_k1080_execution(); if e.sp >= stack.start && e.sp <= layout.base + layout.bytes { return Err((os_k1080_thread::Error::ActiveStack, stack)); } if os_k1080_erase_inactive_root(&stack.root, self.tables).is_err() { return Err((os_k1080_thread::Error::Backend, stack)); } let token = match stack.guard.take() { Some(token) => token, None => return Err((os_k1080_thread::Error::Guard, stack)), }; if let Err((_error, token)) = unsafe { self.tables.os_k1020_restore_stack_guard(token) } { stack.guard = Some(token); return Err((os_k1080_thread::Error::Guard, stack)); } for offset in 0..(OS_K1080_STACK_PAGES + 1) * 4096 / 8 { unsafe { write_volatile((stack.start as *mut u64).add(offset), 0); } } unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } if !(0..(OS_K1080_STACK_PAGES + 1) * 4096 / 8) .all(|offset| unsafe { read_volatile((stack.start as *const u64).add(offset)) } == 0) { return Err((os_k1080_thread::Error::Backend, stack)); } let refs = core::array::from_fn::<_, 9, _>(|index| { if index < 8 { stack.leases[index].as_ref().unwrap() } else { stack.root.lease.as_ref().unwrap() } }); if self .memory .allocator .os_authority_mut() .release_many_leases(&refs) .is_err() { return Err((os_k1080_thread::Error::Backend, stack)); } if self.trace { os_k1080_event(3, stack.ordinal as u64, stack.start, 8); } for lease in &mut stack.leases { *lease = None; } stack.root.lease = None; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_ROOT_LIVE) .cast::() .add(stack.ordinal), 0, ); } Ok(()) } } use turkos_kernel::runtime::wait_context as os_k1080_context; type OsK1080Table<'a> = os_k1080_thread::Table, 3>; /// Private nonclone pin for one real Table stack/root owner. The actual nine /// central leases stay in Table::Stack until this pin is consumed by release. struct OsK1080Binding { id: os_k1080_thread::ThreadId, ordinal: usize, root: u64, root_generation: u64, branches: [u64; 2], } struct OsK1080Prepared { source: Option, target: os_k1080_thread::ThreadId, nonce: u64, departed: bool, } struct OsK1080ReturnPrepared { source: os_k1080_thread::ThreadId, bootstrap_root: u64, bootstrap: os_k1080_context::Cooperative, address: u64, nonce: u64, } struct OsK1080IdlePrepared { epoch: u64, wait: Option, } struct OsK1080ContextBackend<'a> { table: OsK1080Table<'a>, bootstrap: u64, current: Option, pins: [bool; 3], records: [Option; 3], observations: [Option; 3], guards: [u64; 3], sleep_wait: Option, sequence: u64, } impl OsK1080ContextBackend<'_> { fn validate_stack_owner( &self, id: os_k1080_thread::ThreadId, ) -> Result<(), os_k1080_context::Error> { if os_k1080_execution().in_irq { return Err(os_k1080_context::Error::Context); } self.table .os_k1020_observe_backend(|backend| { self.table .os_k1020_observe_stack(id, |stack| backend.validate(stack)) }) .map_err(|_| os_k1080_context::Error::Stack)? .map_err(|_| os_k1080_context::Error::Backend) } fn valid_pin(&self, b: &OsK1080Binding) -> Result<(), os_k1080_context::Error> { if b.ordinal >= 3 || !self.pins[b.ordinal] || b.root == 0 || b.root_generation == 0 { return Err(os_k1080_context::Error::Stale); } let r = self.records[b.ordinal].ok_or(os_k1080_context::Error::Stale)?; if r.id != b.id { return Err(os_k1080_context::Error::Stale); } let root = unsafe { read_volatile( core::ptr::addr_of!(OS_K1080_ROOT_ADDRESSES) .cast::() .add(b.ordinal), ) }; let generation = unsafe { read_volatile( core::ptr::addr_of!(OS_K1080_ROOT_GENERATIONS) .cast::() .add(b.ordinal), ) }; if root != b.root || generation != b.root_generation || unsafe { read_volatile( core::ptr::addr_of!(OS_K1080_ROOT_LIVE) .cast::() .add(b.ordinal), ) } != 1 { return Err(os_k1080_context::Error::Stale); } // Immutable IRQ pin: the typed central/root/stack owner was validated // by Table in main context before this pin was published. IRQ reads // only these bounded physical words and its private pin, never Table // or the allocator. Main observations additionally revalidate Table. os_k1080_root_words(b.root, b.branches).map_err(|_| os_k1080_context::Error::Backend)?; if !os_k1080_execution().in_irq { self.validate_stack_owner(b.id)?; } Ok(()) } fn refresh(&mut self) -> Result<(), os_k1080_context::Error> { if os_k1080_execution().in_irq { return Err(os_k1080_context::Error::Context); } for ordinal in 0..3 { if let Some(previous) = self.records[ordinal] { if !self.pins[ordinal] { continue; } let record = self .table .record(previous.id) .map_err(|_| os_k1080_context::Error::State)?; self.validate_stack_owner(previous.id)?; let observation = self .table .os_k1020_observe_stack(previous.id, |stack| { let guard = stack .guard .as_ref() .map_or(0, |guard| guard.descriptor_address()); ( stack.layout(), guard, stack.root.address(), stack.root.generation(), ) }) .map_err(|_| os_k1080_context::Error::Stack)?; if observation.1 == 0 { return Err(os_k1080_context::Error::Guard); } self.records[ordinal] = Some(record); self.guards[ordinal] = observation.1; self.observations[ordinal] = Some(os_k1080_thread::StackObservation { layout: observation.0, guard_descriptor: unsafe { read_volatile(observation.1 as *const u64) }, low_canary: unsafe { read_volatile(observation.0.base as *const u64) }, high_canary: unsafe { read_volatile((observation.0.base + observation.0.bytes - 8) as *const u64) }, initially_zeroed: false, }); } } Ok(()) } fn pin( &mut self, id: os_k1080_thread::ThreadId, ordinal: usize, ) -> Result { if ordinal >= 3 || self.pins[ordinal] || os_k1080_execution().in_irq { return Err(os_k1080_context::Error::State); } let record = self .table .record(id) .map_err(|_| os_k1080_context::Error::State)?; self.validate_stack_owner(id)?; let (root, generation, branches) = self .table .os_k1020_observe_stack(id, |stack| { ( stack.root.address(), stack.root.generation(), stack.root.branches, ) }) .map_err(|_| os_k1080_context::Error::Stack)?; let before_records = self.records; let before_observations = self.observations; let before_guards = self.guards; self.records[ordinal] = Some(record); self.pins[ordinal] = true; if let Err(error) = self.refresh() { self.pins[ordinal] = false; self.records = before_records; self.observations = before_observations; self.guards = before_guards; return Err(error); } Ok(OsK1080Binding { id, ordinal, root, root_generation: generation, branches, }) } fn next_prepared(&mut self) -> Result { let next = self .sequence .checked_add(1) .ok_or(os_k1080_context::Error::Exhausted)?; self.sequence = next; Ok(next) } } unsafe impl os_k1080_context::Backend for OsK1080ContextBackend<'_> { type Binding = OsK1080Binding; type Prepared = OsK1080Prepared; type IdlePrepared = OsK1080IdlePrepared; fn execution(&self) -> os_k1080_context::Execution { let e = os_k1080_execution(); let (root, _, _, daif) = os_k940_registers(); let cpacr: u64; unsafe { asm!("mrs {fp}, CPACR_EL1",fp=out(reg)cpacr,options(nomem,nostack,preserves_flags)); } os_k1080_context::Execution { cpu: e.cpu, daif, in_irq: e.in_irq, sp: e.sp, root, thread: self.current, fp_enabled: cpacr & (3 << 20) == 3 << 20, critical_depth: unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_CRITICAL)) } as u32, safe_point: unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_SAFE_POINT)) } == 1, tco: if e.in_irq { unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_TCO)) } } else { 0 }, } } fn record( &self, b: &Self::Binding, ) -> Result { self.valid_pin(b)?; // IRQ uses the last real main-only Table transition, an immutable pin // and actual guard/canaries. It never borrows Table/Memory/authority. if os_k1080_execution().in_irq { return self.records[b.ordinal].ok_or(os_k1080_context::Error::Stale); } self.table .record(b.id) .map_err(|_| os_k1080_context::Error::State) } fn stack( &self, b: &Self::Binding, ) -> Result { self.valid_pin(b)?; let cached = self.observations[b.ordinal].ok_or(os_k1080_context::Error::Stack)?; if !os_k1080_execution().in_irq { self.table .record(b.id) .map_err(|_| os_k1080_context::Error::State)?; self.table .os_k1020_observe_stack(b.id, |stack| { stack.root.address() == b.root && stack.root.generation() == b.root_generation }) .map_err(|_| os_k1080_context::Error::Stack)? .then_some(()) .ok_or(os_k1080_context::Error::Stale)?; } Ok(os_k1080_thread::StackObservation { guard_descriptor: unsafe { read_volatile(self.guards[b.ordinal] as *const u64) }, low_canary: unsafe { read_volatile(cached.layout.base as *const u64) }, high_canary: unsafe { read_volatile((cached.layout.base + cached.layout.bytes - 8) as *const u64) }, ..cached }) } fn root(&self, b: &Self::Binding) -> Result { self.valid_pin(b)?; if !os_k1080_execution().in_irq { self.table .record(b.id) .map_err(|_| os_k1080_context::Error::State)?; } Ok(b.root) } fn executable(&self, pc: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } pc % 4 == 0 && pc >= core::ptr::addr_of!(_text_start) as u64 && pc < core::ptr::addr_of!(_text_end) as u64 } fn initial_return_pc(&self, _: &Self::Binding) -> Result { unsafe extern "C" { fn __os_k1080_bad_return(); } Ok(__os_k1080_bad_return as *const () as u64) } fn capture_matches(&self, b: &Self::Binding, c: &os_k1080_context::Capture) -> bool { if self.valid_pin(b).is_err() { return false; } let Some(observation) = self.observations[b.ordinal] else { return false; }; let (address, episode, bytes) = match c.image { os_k1080_context::Image::Cooperative(_) => ( unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LIVE_CAPTURE_ADDRESS)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_CAPTURE_EPISODE)) }, 640u64, ), os_k1080_context::Image::Timer(_) => ( unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_CAPTURE_ADDRESS)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_CAPTURE_EPISODE)) }, 848u64, ), _ => return false, }; if c.address != address || c.episode != episode || episode == 0 || address % 16 != 0 || address < observation.layout.usable_bottom() || !address .checked_add(bytes) .is_some_and(|end| end <= observation.layout.usable_top()) { return false; } match c.image { os_k1080_context::Image::Cooperative(f) => { (unsafe { read_volatile(address as *const os_k1080_context::Cooperative) }) == f } os_k1080_context::Image::Timer(f) => { (unsafe { read_volatile(address as *const os_k920_exception::Frame) }) == f } _ => false, } } fn registers(&self) -> os_k1080_context::Registers { let f = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_FRAME)) }; os_k1080_context::Registers { sp: f.sp, x19_x30: f.x19_x30, q: f.q, fpcr: f.fpcr, fpsr: f.fpsr, pc: os_k1080_observed_landing_pc(&f), argument: unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_ARGUMENT)) }, timer: None, } } fn prepare( &mut self, source: Option<&Self::Binding>, target: &Self::Binding, _: &os_k1080_context::Image, _: os_k1080_context::Disposition, ) -> Result { self.valid_pin(target)?; if let Some(source) = source { self.valid_pin(source)?; } Ok(OsK1080Prepared { source: source.map(|s| s.id), target: target.id, nonce: self.next_prepared()?, departed: false, }) } fn commit( &mut self, _: &mut Self::Prepared, _: Option<&Self::Binding>, _: &Self::Binding, _: &os_k1080_context::Image, _: os_k1080_context::Disposition, ) -> Result<(), os_k1080_context::Failure> { // A synchronous bank loan must never span a suspended continuation. Err(os_k1080_context::Failure { error: os_k1080_context::Error::Context, phase: os_k1080_context::FailurePhase::BeforeSwitch, }) } fn finish( &mut self, p: Self::Prepared, ) -> Result<(), (os_k1080_context::Error, Self::Prepared)> { if !p.departed { return Err((os_k1080_context::Error::State, p)); } Ok(()) } fn abort( &mut self, p: Self::Prepared, ) -> Result<(), (os_k1080_context::Error, Self::Prepared)> { if p.departed { return Err((os_k1080_context::Error::State, p)); } Ok(()) } fn release( &mut self, b: Self::Binding, ) -> Result<(), (os_k1080_context::Error, Self::Binding)> { if let Err(e) = self.valid_pin(&b) { return Err((e, b)); } let record = match self.table.record(b.id) { Ok(r) => r, Err(_) => return Err((os_k1080_context::Error::State, b)), }; if record.state != os_k1080_thread::State::Dead || os_k940_registers().0 != self.bootstrap { return Err((os_k1080_context::Error::Active, b)); } if self.table.reap(b.id).is_err() { return Err((os_k1080_context::Error::Backend, b)); } self.pins[b.ordinal] = false; self.records[b.ordinal] = None; self.observations[b.ordinal] = None; self.guards[b.ordinal] = 0; Ok(()) } fn ready_empty(&self) -> bool { let s = self.table.snapshot(); s.ready == 0 && s.running == 0 } fn wake_epoch(&self) -> u64 { unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_WAKE_EPOCH)) } } fn idle_prepare(&mut self, epoch: u64) -> Result { let wait = self .sleep_wait .take() .ok_or(os_k1080_context::Error::Wake)?; Ok(OsK1080IdlePrepared { epoch, wait: Some(wait), }) } fn idle_commit( &mut self, p: &mut Self::IdlePrepared, epoch: u64, ) -> Result { os_k1080_idle_atomic(self, p, epoch) } fn idle_finish( &mut self, p: Self::IdlePrepared, ) -> Result<(), (os_k1080_context::Error, Self::IdlePrepared)> { if p.wait.is_some() { return Err((os_k1080_context::Error::Wake, p)); } Ok(()) } fn idle_abort( &mut self, mut p: Self::IdlePrepared, ) -> Result<(), (os_k1080_context::Error, Self::IdlePrepared)> { if self.sleep_wait.is_some() { return Err((os_k1080_context::Error::Wake, p)); } self.sleep_wait = p.wait.take(); Ok(()) } } unsafe impl os_k1080_context::HandoffBackend for OsK1080ContextBackend<'_> { fn depart( &mut self, p: &mut Self::Prepared, source: Option<&Self::Binding>, nonce: u64, d: os_k1080_context::Disposition, ) -> Result<(), os_k1080_context::Failure> { let before = |error| os_k1080_context::Failure { error, phase: os_k1080_context::FailurePhase::BeforeSwitch, }; if p.source != source.map(|s| s.id) || p.departed || p.nonce == 0 || nonce == 0 { return Err(before(os_k1080_context::Error::Stale)); } self.validate_stack_owner(p.target).map_err(before)?; if let Some(source) = source { self.valid_pin(source).map_err(before)?; match d { os_k1080_context::Disposition::Ready => self .table .yield_current(source.id) .map_err(|_| before(os_k1080_context::Error::State))?, os_k1080_context::Disposition::Sleeping => { if self.sleep_wait.is_some() { return Err(before(os_k1080_context::Error::Wake)); } self.sleep_wait = Some( self.table .sleep_current(source.id) .map_err(|_| before(os_k1080_context::Error::State))?, ); } } } p.departed = true; self.current = None; if let Err(e) = self.refresh() { return Err(os_k1080_context::Failure { error: e, phase: os_k1080_context::FailurePhase::AfterSwitch, }); } let root = match self .table .os_k1020_observe_stack(p.target, |stack| stack.root.address()) { Ok(root) => root, Err(_) => { return Err(os_k1080_context::Failure { error: os_k1080_context::Error::Stack, phase: os_k1080_context::FailurePhase::AfterSwitch, }); } }; if os_k1080_select_root(root).is_err() { return Err(os_k1080_context::Failure { error: os_k1080_context::Error::Backend, phase: os_k1080_context::FailurePhase::AfterSwitch, }); } Ok(()) } fn arrive( &mut self, p: &mut Self::Prepared, target: &Self::Binding, _: &os_k1080_context::Image, nonce: u64, ) -> Result<(), os_k1080_context::Error> { self.valid_pin(target)?; if !p.departed || p.target != target.id || nonce != unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_DEPARTED_NONCE)) } || os_k940_registers().0 != target.root { return Err(os_k1080_context::Error::Capture); } self.table .activate(target.id) .map_err(|_| os_k1080_context::Error::State)?; self.current = Some(target.id); self.refresh() } } unsafe impl os_k1080_context::CompletionBackend for OsK1080ContextBackend<'_> { type ReturnPrepared = OsK1080ReturnPrepared; fn prepare_return( &mut self, source: &Self::Binding, _: &os_k1080_context::Capture, ) -> Result { self.valid_pin(source)?; let address = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_BOOT_CAPTURE_ADDRESS)) }; if !os_k1080_boot_frame_range(address) { return Err(os_k1080_context::Error::Stack); } let bootstrap = unsafe { read_volatile(address as *const os_k1080_context::Cooperative) }; if bootstrap.sp != address + 640 || bootstrap.canary != os_k1080_context::CONTEXT_CANARY || bootstrap.padding != 0 { return Err(os_k1080_context::Error::Capture); } Ok(OsK1080ReturnPrepared { source: source.id, bootstrap_root: self.bootstrap, bootstrap, address, nonce: self.next_prepared()?, }) } fn leave_return( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Binding, ) -> Result<(), os_k1080_context::Failure> { let before = |error| os_k1080_context::Failure { error, phase: os_k1080_context::FailurePhase::BeforeSwitch, }; self.valid_pin(source).map_err(before)?; if p.source != source.id || p.nonce == 0 { return Err(before(os_k1080_context::Error::Stale)); } self.table .exit_current(source.id) .map_err(|_| before(os_k1080_context::Error::State))?; self.current = None; if self.refresh().is_err() || os_k1080_select_root(p.bootstrap_root).is_err() { return Err(os_k1080_context::Failure { error: os_k1080_context::Error::Backend, phase: os_k1080_context::FailurePhase::AfterSwitch, }); } Ok(()) } fn returned(&self, p: &Self::ReturnPrepared) -> Result<(), os_k1080_context::Error> { unsafe extern "C" { fn __os_k1080_boot_landed(); } let actual = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_FRAME)) }; let stacks = os_k1080_stack_registers(); let original = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_BOOT_STACK_REGISTERS)) }; if actual != p.bootstrap || !os_k1080_boot_frame_range(p.address) || actual.sp != p.address + 640 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_PC)) } != __os_k1080_boot_landed as *const () as u64 || stacks[0] != original[0] || stacks[if original[0] == 0 { 2 } else { 1 }] != original[if original[0] == 0 { 2 } else { 1 }] { return Err(os_k1080_context::Error::Capture); } Ok(()) } fn return_root(&self, p: &Self::ReturnPrepared) -> Result { if p.bootstrap_root != self.bootstrap { return Err(os_k1080_context::Error::Stack); } Ok(p.bootstrap_root) } fn finish_return( &mut self, _: Self::ReturnPrepared, ) -> Result<(), (os_k1080_context::Error, Self::ReturnPrepared)> { Ok(()) } fn abort_return( &mut self, _: Self::ReturnPrepared, ) -> Result<(), (os_k1080_context::Error, Self::ReturnPrepared)> { Ok(()) } } unsafe impl os_k1080_context::InactiveBackend for OsK1080ContextBackend<'_> { fn cancel_inactive( &mut self, binding: &Self::Binding, ) -> Result<(), os_k1080_context::Failure> { let before = |error| os_k1080_context::Failure { error, phase: os_k1080_context::FailurePhase::BeforeSwitch, }; self.valid_pin(binding).map_err(before)?; if self.current.is_some() || os_k940_registers().0 != self.bootstrap { return Err(before(os_k1080_context::Error::Context)); } self.table .cancel(binding.id) .map_err(|_| before(os_k1080_context::Error::State))?; // Any retained sleeping authorization was invalidated by actual // Table.cancel. It is never reused to manufacture a wake after cancel. if self .sleep_wait .as_ref() .is_some_and(|wait| wait.thread() == binding.id) { self.sleep_wait = None; } self.refresh().map_err(|error| os_k1080_context::Failure { error, phase: os_k1080_context::FailurePhase::AfterSwitch, }) } } // NEW reusable three-member backend. The real typed bindings/backing owners // remain exclusively inside CompletionBank; no binding is reconstructed. use turkos_kernel::runtime::wait_queue as os_k1080_queue; struct OsK1080Member { identity: u64, context: os_k1080_context::Handle, thread: os_k1080_thread::ThreadId, ordinal: usize, retired: bool, } struct OsK1080SelectionPrepared { handoff: Option, epoch: u64, nonce: u64, arrived: bool, } struct OsK1080QueueReturnPrepared { token: Option, source: os_k1080_context::Handle, nonce: u64, observed: bool, } struct OsK1080QueueBackend { bank: os_k1080_context::CompletionBank, identity: u64, members: [Option; 3], failed_tick: Option, retirement_quarantined: bool, } impl OsK1080QueueBackend { #[allow(clippy::result_large_err)] fn new( bank: os_k1080_context::CompletionBank, ) -> Result< Self, ( os_k1080_queue::Error, os_k1080_context::CompletionBank, ), > { let snapshot = bank.handoff().bank().snapshot(); if bank.quarantined() || bank.observation().is_some() || bank.handoff().quarantined() || bank.handoff().stage().is_some() || snapshot.retained != 3 || snapshot.active != 0 || snapshot.ready != 3 || snapshot.quarantined != 0 || snapshot.faulted || snapshot.retained_switch || snapshot.retained_capture { return Err((os_k1080_queue::Error::State, bank)); } let e = bank .handoff() .bank() .with_backend_observation(|b| os_k1080_context::Backend::execution(b)); if e.cpu != 0 || e.daif & 128 == 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point || e.thread.is_some() { return Err((os_k1080_queue::Error::Context, bank)); } unsafe extern "C" { fn __os_k1080_backend_identity() -> u64; } let identity = unsafe { __os_k1080_backend_identity() }; if identity == 0 { return Err((os_k1080_queue::Error::Exhausted, bank)); } Ok(Self { bank, identity, members: [None; 3], failed_tick: None, retirement_quarantined: false, }) } fn issue( &mut self, handle: os_k1080_context::Handle, ordinal: usize, ) -> Result { if self.retirement_quarantined { return Err(os_k1080_queue::Error::Quarantined); } if ordinal >= 3 || self.members[ordinal].is_some() || self.members.contains(&Some(handle)) { return Err(os_k1080_queue::Error::Duplicate); } let o = self .bank .os_k1080_observe_member(handle) .map_err(|_| os_k1080_queue::Error::Backend)?; if o.thread.state != os_k1080_thread::State::Ready || o.context.state != os_k1080_context::State::Ready { return Err(os_k1080_queue::Error::State); } self.members[ordinal] = Some(handle); Ok(OsK1080Member { identity: self.identity, context: handle, thread: o.thread.id, ordinal, retired: false, }) } fn member( &self, member: &OsK1080Member, ) -> Result { if self.retirement_quarantined { return Err(os_k1080_queue::Error::Quarantined); } if member.retired { return Err(os_k1080_queue::Error::State); } if member.identity != self.identity || member.ordinal >= 3 || self.members[member.ordinal] != Some(member.context) { return Err(os_k1080_queue::Error::Stale); } let o = self .bank .os_k1080_observe_member(member.context) .map_err(|_| os_k1080_queue::Error::Backend)?; if o.thread.id != member.thread || o.context.thread != member.thread { return Err(os_k1080_queue::Error::Stale); } Ok(o) } } unsafe impl os_k1080_queue::Backend for OsK1080QueueBackend { type Member = OsK1080Member; type Prepared = OsK1080SelectionPrepared; type Tick = B::QueueTick; fn execution(&self) -> os_k1080_queue::Execution { let e = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1080_context::Backend::execution(b)); os_k1080_queue::Execution { cpu: e.cpu, masked: e.daif & 128 != 0, in_irq: e.in_irq, thread: e.thread, root: e.root, } } fn observe( &self, m: &Self::Member, ) -> Result { self.member(m) } fn tick( &self, t: &Self::Tick, ) -> Result { self.bank.os_k1080_tick(t) } fn consume_tick(&mut self, t: Self::Tick) { if let Err((_e, actual)) = self.bank.os_k1080_consume_tick(t) { self.failed_tick = Some(actual); park(); } } fn prepare( &mut self, source: Option<&Self::Member>, target: &Self::Member, epoch: u64, d: os_k1080_queue::Disposition, ) -> Result { self.member(target)?; if let Some(source) = source { self.member(source)?; } let capture = if source.is_some() { Some( self.bank .os_k1080_capture() .map_err(|_| os_k1080_queue::Error::Backend)?, ) } else { None }; let bank = self .bank .quiescent() .map_err(|_| os_k1080_queue::Error::Busy)?; let handoff = match source { None => bank.start(target.context), Some(source) => match d { os_k1080_queue::Disposition::Ready => { bank.cooperate(source.context, target.context, capture.unwrap()) } os_k1080_queue::Disposition::Sleeping => { bank.sleep_to(source.context, target.context, capture.unwrap()) } }, } .map_err(|_| os_k1080_queue::Error::Backend)?; Ok(OsK1080SelectionPrepared { handoff: Some(handoff), epoch, nonce: 0, arrived: false, }) } fn commit( &mut self, _: &mut Self::Prepared, _: Option<&Self::Member>, _: &Self::Member, _: u64, _: os_k1080_queue::Disposition, ) -> Result<(), os_k1080_queue::Failure> { Err(os_k1080_queue::Failure { error: os_k1080_queue::Error::Context, phase: os_k1080_context::FailurePhase::BeforeSwitch, }) } fn finish(&mut self, p: Self::Prepared) -> Result<(), (os_k1080_queue::Error, Self::Prepared)> { if !p.arrived || p.handoff.is_some() { Err((os_k1080_queue::Error::State, p)) } else { Ok(()) } } fn abort( &mut self, mut p: Self::Prepared, ) -> Result<(), (os_k1080_queue::Error, Self::Prepared)> { let b = match self.bank.quiescent() { Ok(b) => b, Err(_) => return Err((os_k1080_queue::Error::Busy, p)), }; let t = p.handoff.take().unwrap(); match b.abort(t) { Ok(()) => Ok(()), Err(os_k1080_context::HandoffFailure::Before(_, t)) => { p.handoff = Some(t); Err((os_k1080_queue::Error::Backend, p)) } Err(os_k1080_context::HandoffFailure::Quarantined(_)) => { Err((os_k1080_queue::Error::Backend, p)) } } } } unsafe impl os_k1080_queue::SelectionHandoffBackend for OsK1080QueueBackend { fn depart( &mut self, p: &mut Self::Prepared, _: Option<&Self::Member>, _: &Self::Member, epoch: u64, _: os_k1080_queue::Disposition, nonce: u64, ) -> Result<(), os_k1080_queue::Failure> { if p.epoch != epoch || p.nonce != 0 || nonce == 0 { return Err(os_k1080_queue::Failure { error: os_k1080_queue::Error::Stale, phase: os_k1080_context::FailurePhase::BeforeSwitch, }); } let b = match self.bank.quiescent() { Ok(b) => b, Err(_) => { return Err(os_k1080_queue::Failure { error: os_k1080_queue::Error::Busy, phase: os_k1080_context::FailurePhase::BeforeSwitch, }); } }; let t = p.handoff.take().unwrap(); match b.depart(t) { Ok(t) => { p.handoff = Some(t); p.nonce = nonce; Ok(()) } Err(os_k1080_context::HandoffFailure::Before(_, t)) => { p.handoff = Some(t); Err(os_k1080_queue::Failure { error: os_k1080_queue::Error::Backend, phase: os_k1080_context::FailurePhase::BeforeSwitch, }) } Err(os_k1080_context::HandoffFailure::Quarantined(_)) => Err(os_k1080_queue::Failure { error: os_k1080_queue::Error::Backend, phase: os_k1080_context::FailurePhase::AfterSwitch, }), } } fn arrive( &mut self, p: &mut Self::Prepared, target: &Self::Member, epoch: u64, nonce: u64, ) -> Result<(), os_k1080_queue::Error> { if p.epoch != epoch || p.nonce != nonce { return Err(os_k1080_queue::Error::Stale); } let b = self .bank .quiescent() .map_err(|_| os_k1080_queue::Error::Busy)?; let t = p.handoff.take().unwrap(); match b.landed(t) { Ok(()) => { p.arrived = true; self.member(target)?; Ok(()) } Err(os_k1080_context::HandoffFailure::Before(_, t)) => { p.handoff = Some(t); Err(os_k1080_queue::Error::Backend) } Err(os_k1080_context::HandoffFailure::Quarantined(_)) => { Err(os_k1080_queue::Error::Backend) } } } } unsafe impl os_k1080_queue::QueueCompletionBackend for OsK1080QueueBackend { type ReturnPrepared = OsK1080QueueReturnPrepared; fn prepare_exit( &mut self, source: &Self::Member, capture: os_k1080_context::Capture, ) -> Result { self.member(source)?; let token = self .bank .prepare_return(source.context, capture) .map_err(|_| os_k1080_queue::Error::Backend)?; Ok(OsK1080QueueReturnPrepared { token: Some(token), source: source.context, nonce: 0, observed: false, }) } fn depart_exit( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Member, nonce: u64, ) -> Result<(), os_k1080_queue::Failure> { if p.source != source.context || nonce == 0 || p.nonce != 0 { return Err(os_k1080_queue::Failure { error: os_k1080_queue::Error::Stale, phase: os_k1080_context::FailurePhase::BeforeSwitch, }); } let t = p.token.take().unwrap(); match self.bank.depart_return(t) { Ok(t) => { p.token = Some(t); p.nonce = nonce; Ok(()) } Err(os_k1080_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err(os_k1080_queue::Failure { error: os_k1080_queue::Error::Backend, phase: os_k1080_context::FailurePhase::BeforeSwitch, }) } Err(os_k1080_context::ReturnFailure::Quarantined(_)) => Err(os_k1080_queue::Failure { error: os_k1080_queue::Error::Backend, phase: os_k1080_context::FailurePhase::AfterSwitch, }), } } fn returned_exit( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Member, nonce: u64, ) -> Result<(), os_k1080_queue::Error> { if p.source != source.context || p.nonce != nonce { return Err(os_k1080_queue::Error::Stale); } self.bank .os_k1080_check_return_landing(p.token.as_ref().ok_or(os_k1080_queue::Error::State)?) .map_err(|_| os_k1080_queue::Error::Backend)?; p.observed = true; Ok(()) } fn finish_exit( &mut self, mut p: Self::ReturnPrepared, ) -> Result<(), (os_k1080_queue::Error, Self::ReturnPrepared)> { if !p.observed { return Err((os_k1080_queue::Error::State, p)); } let t = p.token.take().unwrap(); match self.bank.return_landed(t) { Ok(()) => Ok(()), Err(os_k1080_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err((os_k1080_queue::Error::Backend, p)) } Err(os_k1080_context::ReturnFailure::Quarantined(_)) => { Err((os_k1080_queue::Error::Backend, p)) } } } fn abort_exit( &mut self, mut p: Self::ReturnPrepared, ) -> Result<(), (os_k1080_queue::Error, Self::ReturnPrepared)> { let t = p.token.take().unwrap(); match self.bank.abort_return(t) { Ok(()) => Ok(()), Err(os_k1080_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err((os_k1080_queue::Error::Backend, p)) } Err(os_k1080_context::ReturnFailure::Quarantined(_)) => { Err((os_k1080_queue::Error::Backend, p)) } } } } impl OsK1080QueueBackend { fn retire_member( &mut self, mut m: OsK1080Member, ) -> Result { let observation = match self.member(&m) { Ok(o) => o, Err(e) => return Err((e, m)), }; let before = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1080_context::Backend::execution(b)); let core = match self.bank.quiescent().and_then(|b| b.quiescent()) { Ok(core) => core, Err(_) => return Err((os_k1080_queue::Error::Busy, m)), }; if observation.thread.state != os_k1080_thread::State::Dead && core.os_k1040_cancel_inactive(m.context).is_err() { return Err((os_k1080_queue::Error::Backend, m)); } if core.retire(m.context).is_err() { return Err((os_k1080_queue::Error::Backend, m)); } // Frozen retirement has consumed the real Binding/Stack. A later // context failure retains only this explicitly retired logical member // and the exclusive bank; it never reconstructs a physical binding. m.retired = true; let after = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1080_context::Backend::execution(b)); if after != before { self.retirement_quarantined = true; return Err((os_k1080_queue::Error::Quarantined, m)); } self.members[m.ordinal] = None; Ok(m) } } unsafe impl os_k1080_queue::QueueRetirementBackend for OsK1080QueueBackend { fn retire_withdrawn( &mut self, member: Self::Member, ) -> Result { self.retire_member(member) } } impl OsK1080QueueBackend { fn owner_words(&self) -> [u64; 17] { [ self.identity, &self.bank as *const _ as u64, core::mem::size_of_val(&self.bank) as u64, self.members.as_ptr() as u64, self.members.iter().filter(|m| m.is_some()).count() as u64, self.failed_tick .as_ref() .map_or(0, |t| t as *const B::QueueTick as u64), self.members[0].map_or(0, |h| h.bank()), self.members[0].map_or(0, |h| h.slot() as u64), self.members[0].map_or(0, |h| h.generation()), self.members[1].map_or(0, |h| h.bank()), self.members[1].map_or(0, |h| h.slot() as u64), self.members[1].map_or(0, |h| h.generation()), self.members[2].map_or(0, |h| h.bank()), self.members[2].map_or(0, |h| h.slot() as u64), self.members[2].map_or(0, |h| h.generation()), core::mem::size_of::() as u64, u64::from(self.retirement_quarantined), ] } } const fn os_k1080_backend_layout() -> [u64; 24] { use core::mem::{offset_of, size_of}; [ size_of::>() as u64, offset_of!(OsK1080QueueBackend, bank) as u64, offset_of!(OsK1080QueueBackend, identity) as u64, offset_of!(OsK1080QueueBackend, members) as u64, offset_of!(OsK1080QueueBackend, failed_tick) as u64, size_of::>() as u64, size_of::() as u64, offset_of!(OsK1080Member, identity) as u64, offset_of!(OsK1080Member, context) as u64, offset_of!(OsK1080Member, thread) as u64, offset_of!(OsK1080Member, ordinal) as u64, size_of::() as u64, offset_of!(OsK1080SelectionPrepared, handoff) as u64, offset_of!(OsK1080SelectionPrepared, epoch) as u64, offset_of!(OsK1080SelectionPrepared, nonce) as u64, offset_of!(OsK1080SelectionPrepared, arrived) as u64, size_of::() as u64, offset_of!(OsK1080QueueReturnPrepared, token) as u64, offset_of!(OsK1080QueueReturnPrepared, source) as u64, offset_of!(OsK1080QueueReturnPrepared, nonce) as u64, offset_of!(OsK1080QueueReturnPrepared, observed) as u64, size_of::>() as u64, offset_of!(OsK1080QueueBackend, retirement_quarantined) as u64, offset_of!(OsK1080Member, retired) as u64, ] } // Appended in the private real three-member Queue backend module. unsafe impl os_k1080_queue::WaitQueueServices for OsK1080QueueBackend { fn wait_physical_landing( &self, p: &Self::Prepared, m: &Self::Member, ) -> Result { if p.arrived || p.nonce == 0 || p.handoff.is_none() { return Err(os_k1080_queue::Error::State); } self.bank .handoff() .target_image(p.handoff.as_ref().unwrap()) .map_err(|_| os_k1080_queue::Error::Stale)?; self.member(m)?; self.bank .wait_guest_physical_landing(m.context) .map_err(|_| os_k1080_queue::Error::Context) } fn wait_capture(&self) -> Result { self.bank .os_k1080_capture() .map_err(|_| os_k1080_queue::Error::Context) } fn wait_custody( &mut self, m: &Self::Member, owned: &mut Option, resolve: bool, ) -> Result<(), os_k1080_queue::Failure> { self.member(m).map_err(|error| os_k1080_queue::Failure { error, phase: os_k1080_context::FailurePhase::BeforeSwitch, })?; self.bank .wait_guest_custody(m.context, owned, resolve) .map_err(|f| os_k1080_queue::Failure { error: if f.phase == os_k1080_context::FailurePhase::AfterSwitch { os_k1080_queue::Error::Quarantined } else { os_k1080_queue::Error::Backend }, phase: f.phase, }) } } #[inline(never)]fn os_k1080_inventory(authority:&os_k720_frame::Allocator,out:*mut u64){ for index in 0..328{unsafe{write_volatile(out.add(index),0);}} let free=authority.free_regions();if free.len()>64{park();} unsafe{write_volatile(out,free.len() as u64);write_volatile(out.add(3),authority.remaining_pages());} for(index,region)in free.iter().enumerate(){unsafe{ write_volatile(out.add(4+2*index),region.start);write_volatile(out.add(5+2*index),region.end);}} let mut live=0;for address in authority.live_addresses(){if live>=64{park();} unsafe{write_volatile(out.add(132+live),address);}live+=1;} let mut reserved=0;for region in authority.reserved_regions(){if reserved>=64{park();} unsafe{write_volatile(out.add(196+2*reserved),region.start);write_volatile(out.add(197+2*reserved),region.end);}reserved+=1;} unsafe{write_volatile(out.add(1),live as u64);write_volatile(out.add(2),reserved as u64);asm!("dsb sy",options(nostack,preserves_flags));} } type OsK1080Bank<'a> = os_k1080_context::CompletionBank, 3>; type OsK1080Backend<'a> = OsK1080QueueBackend>; type OsK1080Queue<'a> = os_k1080_queue::ReturnQueue, 3>; type OsK1080Bridge<'a> = os_k1080_bridge::WaitGuestBridge, OsK1080Port>; type OsK1080Core<'a> = os_k1080_wait::WaitSet, 3>; #[unsafe(no_mangle)] static mut OS_K1080_CORE: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1080_ROOT_ADDRESSES: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1080_ROOT_GENERATIONS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1080_ROOT_LIVE: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1080_OWNED: [u64; 27] = [0; 27]; #[unsafe(no_mangle)] static mut OS_K1080_LEASE_GENERATIONS: [u64; 27] = [0; 27]; #[unsafe(no_mangle)] static mut OS_K1080_EVIDENCE: [u64; 128] = [0; 128]; #[unsafe(no_mangle)] static mut OS_K1080_KERNEL_TABLES: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K1080_PRIMARY_GUARD_BEFORE: [u64; 4] = [0; 4]; #[unsafe(no_mangle)] static mut OS_K1080_EVENTS: [u64; 4096] = [0; 4096]; #[unsafe(no_mangle)] static mut OS_K1080_EVENT_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_CRITICAL: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_SAFE_POINT: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1080_WAKE_EPOCH: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_CAPTURE_EPISODE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_BOOT_STACK_REGISTERS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1080_IRQ_TCO: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_LIVE_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_IRQ_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_IRQ_CAPTURE_EPISODE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_BOOT_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_DEPARTED_NONCE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_LANDING_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_LANDING_PC: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_LANDING_ARGUMENT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1080_LANDING_FRAME: os_k1080_context::Cooperative = os_k1080_context::Cooperative::EMPTY; #[unsafe(no_mangle)] static mut OS_K1080_CURRENT_CAPTURE: Option = None; #[unsafe(no_mangle)] static mut OS_K1080_PUBLISHED_CURRENT: Option = None; #[derive(Clone, Copy)] #[repr(C)] struct OsK1080Wire { frame: os_k1080_context::Cooperative, pc: u64, argument: u64, nonce: u64, kind: u64, } #[unsafe(no_mangle)] static mut OS_K1080_WIRE: OsK1080Wire = OsK1080Wire { frame: os_k1080_context::Cooperative::EMPTY, pc: 0, argument: 0, nonce: 0, kind: 0, }; #[unsafe(no_mangle)] static mut OS_K1080_CONTEXT_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1080_THREADS_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1080_BACKEND_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1080_QUEUE_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1080_WAIT_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static OS_K1080_COMPILE_ABI: [u64; 8] = [ core::mem::size_of::>() as u64, core::mem::size_of::>() as u64, core::mem::size_of::>() as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::size_of::() as u64, core::mem::size_of::() as u64, core::mem::size_of::() as u64, ]; fn os_k1080_word(index: usize, value: u64) { if index >= 128 { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_EVIDENCE) .cast::() .add(index), value, ); } } fn os_k1080_event(kind: u64, a: u64, b: u64, c: u64) { let e = os_k1080_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_EVENT_COUNT)) } as usize; if count >= 1024 { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_EVENTS) .cast::<[u64; 4]>() .add(count), [kind, a, b, c], ); write_volatile( core::ptr::addr_of_mut!(OS_K1080_EVENT_COUNT), count as u64 + 1, ); } } fn os_k1080_nonce(pointer: *mut u64) -> u64 { let e = os_k1080_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let current = unsafe { read_volatile(pointer) }; let next = current.checked_add(1).unwrap_or_else(|| park()); unsafe { write_volatile(pointer, next); } current } #[unsafe(no_mangle)] extern "C" fn __os_k1080_context_identity() -> u64 { os_k1080_nonce(core::ptr::addr_of_mut!(OS_K1080_CONTEXT_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1080_threads_identity() -> u64 { os_k1080_nonce(core::ptr::addr_of_mut!(OS_K1080_THREADS_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1080_backend_identity() -> u64 { os_k1080_nonce(core::ptr::addr_of_mut!(OS_K1080_BACKEND_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1080_runqueue_identity() -> u64 { os_k1080_nonce(core::ptr::addr_of_mut!(OS_K1080_QUEUE_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1080_wait_identity() -> u64 { os_k1080_nonce(core::ptr::addr_of_mut!(OS_K1080_WAIT_IDENTITY)) } // Exact namespaced frozen stack-register wrapper ABI: selector,SP0,SP1,activeSP. fn os_k1080_stack_registers() -> [u64; 4] { unsafe extern "C" { fn __os_k1080_stack_registers(output: *mut u64); } if !os_k1080_execution().irq_masked { park(); } let mut values = [0; 4]; unsafe { __os_k1080_stack_registers(values.as_mut_ptr()); } values } fn os_k1080_observed_landing_pc(frame: &os_k1080_context::Cooperative) -> u64 { let e = os_k1080_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let actual = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_PC)) }; unsafe extern "C" { fn __os_k1080_cooperative_landed(); } if actual != __os_k1080_cooperative_landed as *const () as u64 { return actual; } let wire = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_WIRE)) }; if wire.kind != 1 || wire.nonce == 0 || wire.pc != frame.x19_x30[11] || wire.nonce != unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_DEPARTED_NONCE)) } || wire.frame != *frame { park(); } frame.x19_x30[11] } fn os_k1080_boot_frame_range(address: u64) -> bool { let guard = core::ptr::addr_of!(OS_K1080_PRIMARY_STACK) as u64; address % 16 == 0 && address >= guard + 4096 + 1024 && address .checked_add(640) .is_some_and(|end| end <= guard + 69632 - 16) } fn os_k1080_idle_atomic( _: &mut OsK1080ContextBackend<'_>, _: &mut OsK1080IdlePrepared, _: u64, ) -> Result { Err(os_k1080_context::Failure { error: os_k1080_context::Error::Context, phase: os_k1080_context::FailurePhase::BeforeSwitch, }) } unsafe impl os_k1080_context::QueueServices for OsK1080ContextBackend<'_> { type QueueTick = core::convert::Infallible; fn queue_capture(&self) -> Option { unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_CURRENT_CAPTURE)) } } fn queue_tick( &self, t: &Self::QueueTick, ) -> Result { match *t {} } fn queue_consume_tick(&mut self, t: Self::QueueTick) { match t {} } } // Main's source authority owns only private registration claims. IRQ has its // separately owned Producer/Ingress and calls the pure scalar reader below. #[unsafe(no_mangle)] static mut OS_K1080_SOURCE: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K1080_SOURCE_PINS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1080_SOURCE_PIN_SEQUENCE: u64 = 1; struct OsK1080Port; struct OsK1080SignalPin { source: os_k1080_wait::Source, slot: usize, claim: u64, } fn os_k1080_source_word(index: usize) -> u64 { if index >= 8 { park(); } unsafe { read_volatile( core::ptr::addr_of!(OS_K1080_SOURCE) .cast::() .add(index), ) } } fn os_k1080_source_execution() -> os_k1080_wait::Execution { let e = os_k1080_execution(); os_k1080_wait::Execution { cpu: e.cpu, irq_masked: e.irq_masked, in_irq: e.in_irq, current: unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_PUBLISHED_CURRENT)) }, } } fn os_k1080_source_observation() -> Result { let e = os_k1080_source_execution(); if e.cpu != 0 || !e.irq_masked { return Err(os_k1080_wait::Error::Context); } if os_k1080_source_word(0) != 1 { return Err(os_k1080_wait::Error::Source); } Ok(os_k1080_wait::Observation { source: os_k1080_wait::Source { channel: os_k1080_source_word(1), epoch: os_k1080_source_word(2), owner: os_k1080_source_word(3), }, sequence: os_k1080_source_word(4), ready: os_k1080_source_word(5) == 1, now: os_k960_counter(), }) } unsafe impl os_k1080_wait::Producer for OsK1080Port { fn execution(&self) -> os_k1080_wait::Execution { os_k1080_source_execution() } fn observe(&self) -> Result { os_k1080_source_observation() } } unsafe impl os_k1080_bridge::WaitAuthority for OsK1080Port { type Pin = OsK1080SignalPin; fn retain_source( &mut self, source: os_k1080_wait::Source, ) -> Result { os_k1080_wait::context( 0, ::execution(self), os_k1080_wait::Operation::Arm, )?; if ::observe(self)?.source != source { return Err(os_k1080_wait::Error::Source); } let slot=(0..3).find(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_SOURCE_PINS).cast::().add(*i))}==0).ok_or(os_k1080_wait::Error::Full)?; let claim = os_k1080_nonce(core::ptr::addr_of_mut!(OS_K1080_SOURCE_PIN_SEQUENCE)); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_SOURCE_PINS) .cast::() .add(slot), claim, ); } Ok(OsK1080SignalPin { source, slot, claim, }) } fn validate_source( &self, pin: &Self::Pin, source: os_k1080_wait::Source, ) -> Result<(), os_k1080_wait::Error> { os_k1080_wait::context( 0, ::execution(self), os_k1080_wait::Operation::Service, )?; if pin.slot >= 3 || pin.source != source || pin.claim == 0 || ::observe(self)?.source != source || unsafe { read_volatile( core::ptr::addr_of!(OS_K1080_SOURCE_PINS) .cast::() .add(pin.slot), ) } != pin.claim { return Err(os_k1080_wait::Error::Source); } Ok(()) } fn release_source(&mut self, pin: Self::Pin) -> Result<(), (os_k1080_wait::Error, Self::Pin)> { if let Err(e) = self.validate_source(&pin, pin.source) { return Err((e, pin)); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_SOURCE_PINS) .cast::() .add(pin.slot), 0, ); } Ok(()) } } // NEW actual adapter for the independently copied three-root context backend. // Frozen core/services code remains byte exact in the generated wait modules. unsafe impl os_k1080_context::WaitContextServices for OsK1080ContextBackend<'_> { fn physical_wait_landing( &self, binding: &Self::Binding, image: &os_k1080_context::Image, nonce: u64, ) -> Result { let raw = os_k1080_execution(); if raw.cpu != 0 || !raw.irq_masked || raw.in_irq { return Err(os_k1080_context::Error::Context); } let before = ::execution(self); if !before.fp_enabled || before.critical_depth != 0 || !before.safe_point { return Err(os_k1080_context::Error::Context); } self.valid_pin(binding)?; let record = self .table .record(binding.id) .map_err(|_| os_k1080_context::Error::State)?; if record.state != os_k1080_thread::State::Ready || record.id != binding.id || nonce == 0 || os_k940_registers().0 != binding.root { return Err(os_k1080_context::Error::Capture); } let frame_address = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_ADDRESS)) }; let pc = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_PC)) }; let actual_nonce = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_DEPARTED_NONCE)) }; let layout = record.layout; if raw.sp % 16 != 0 || raw.sp < layout.usable_bottom() || raw.sp > layout.usable_top() || frame_address % 16 != 0 || frame_address < layout.usable_bottom() || frame_address .checked_add(640) .is_none_or(|end| end > layout.usable_top()) || actual_nonce != nonce { return Err(os_k1080_context::Error::Capture); } // Validate the actual owned physical address before forming a Frame // reference/copy. The assembly captures these bytes before the BL. let actual = unsafe { read_volatile(frame_address as *const os_k1080_context::Cooperative) }; let retained = unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_FRAME)) }; if actual != retained || actual.sp != image.sp() || actual.canary != os_k1080_context::CONTEXT_CANARY || actual.padding != 0 { return Err(os_k1080_context::Error::Capture); } match image { os_k1080_context::Image::Cooperative(expected) => { unsafe extern "C" { fn __os_k1080_cooperative_landed(); } if actual != *expected || pc != __os_k1080_cooperative_landed as *const () as u64 { return Err(os_k1080_context::Error::Capture); } } os_k1080_context::Image::Initial { entry, argument, sp, return_pc, } => { if pc != *entry || actual.sp != *sp || actual.x19_x30[..11] != [0; 11] || actual.x19_x30[11] != *return_pc || actual.q.iter().any(|q| *q != [0; 2]) || actual.fpcr != 0 || actual.fpsr != 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_ARGUMENT)) } != *argument { return Err(os_k1080_context::Error::Capture); } } os_k1080_context::Image::Timer(_) => return Err(os_k1080_context::Error::Capture), } if ::execution(self) != before { return Err(os_k1080_context::Error::Context); } Ok(os_k1080_wait::Execution { cpu: 0, irq_masked: true, in_irq: false, current: Some(binding.id), }) } fn take_sleep_wait( &mut self, binding: &Self::Binding, owned: &mut Option, ) -> Result<(), os_k1080_context::Failure> { let before = |error| os_k1080_context::Failure { error, phase: os_k1080_context::FailurePhase::BeforeSwitch, }; let e = ::execution(self); if e.cpu != 0 || e.daif & 128 == 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point { return Err(before(os_k1080_context::Error::Context)); } self.valid_pin(binding).map_err(before)?; let r = self .table .record(binding.id) .map_err(|_| before(os_k1080_context::Error::State))?; if r.state != os_k1080_thread::State::Sleeping || owned.is_some() || self .sleep_wait .as_ref() .is_none_or(|w| w.thread() != binding.id || w.epoch() != r.wait_epoch) { return Err(before(os_k1080_context::Error::Wake)); } *owned = self.sleep_wait.take(); Ok(()) } fn resolve_sleep_wait( &mut self, binding: &Self::Binding, owned: &mut Option, ) -> Result<(), os_k1080_context::Failure> { let before = |error| os_k1080_context::Failure { error, phase: os_k1080_context::FailurePhase::BeforeSwitch, }; let after = |error| os_k1080_context::Failure { error, phase: os_k1080_context::FailurePhase::AfterSwitch, }; let e = ::execution(self); if e.cpu != 0 || e.daif & 128 == 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point { return Err(before(os_k1080_context::Error::Context)); } self.valid_pin(binding).map_err(before)?; let r = self .table .record(binding.id) .map_err(|_| before(os_k1080_context::Error::State))?; if r.state != os_k1080_thread::State::Sleeping || owned .as_ref() .is_none_or(|w| w.thread() != binding.id || w.epoch() != r.wait_epoch) { return Err(before(os_k1080_context::Error::Wake)); } match self.table.wake(owned.take().unwrap()) { Ok(id) if id == binding.id => {} Ok(_) => return Err(after(os_k1080_context::Error::Wake)), Err((_, actual)) => { *owned = Some(actual); return Err(before(os_k1080_context::Error::Wake)); } } // Actual Wait has been consumed. Never access Table/stack again after // drift or falsely reconstruct a consumed nonclone token. if ::execution(self) != e { return Err(after(os_k1080_context::Error::Context)); } self.refresh().map_err(after) } } unsafe impl os_k1080_bridge::K1080MainBackend for OsK1080QueueBackend> { fn k1080_execution(&self)->os_k1080_context::Execution { self.bank.handoff().bank().with_backend_observation(|b|os_k1080_context::Backend::execution(b)) } fn k1080_target_wire(&self, p: &Self::Prepared) -> Result<(os_k1080_context::Image, u64), os_k1080_wait::Error> { let e = ::k1080_execution(self); if e.cpu != 0 || e.daif&128==0 || e.in_irq || !e.fp_enabled || e.critical_depth!=0 || !e.safe_point || e.thread.is_some() { return Err(os_k1080_wait::Error::Context); } let t = p.handoff.as_ref().ok_or(os_k1080_wait::Error::State)?; if p.arrived || p.nonce == 0 || p.nonce != t.nonce() { return Err(os_k1080_wait::Error::State); } let image = self.bank.handoff().target_image(t).map_err(|_| os_k1080_wait::Error::Stale)?; if ::k1080_execution(self) != e { return Err(os_k1080_wait::Error::Context); } Ok((image, t.nonce())) } fn k1080_bootstrap_wire(&self, p: &Self::ReturnPrepared) -> Result { let e = ::k1080_execution(self); if e.cpu != 0 || e.daif&128==0 || e.in_irq || !e.fp_enabled || e.critical_depth!=0 || !e.safe_point || e.thread.is_some() { return Err(os_k1080_wait::Error::Context); } let t = p.token.as_ref().ok_or(os_k1080_wait::Error::State)?; if p.nonce == 0 || p.observed || t.nonce() == 0 { return Err(os_k1080_wait::Error::State); } let actual = self.bank.with_return_observation(|r| r.map(|r| r.bootstrap)) .ok_or(os_k1080_wait::Error::State)?; if ::k1080_execution(self) != e { return Err(os_k1080_wait::Error::Context); } Ok(actual) } } // NEW phased BSS construction. No whole Queue/Bank/Backend result returns // through the factory caller. Frozen constructors are invoked unchanged. #[repr(C, align(16))] struct OsK1080ConstructionSlot { bytes: core::mem::MaybeUninit<[u8; core::mem::size_of::>()]>, } #[unsafe(no_mangle)] static mut OS_K1080_CONSTRUCTION: [OsK1080ConstructionSlot; 2] = [const { OsK1080ConstructionSlot { bytes: core::mem::MaybeUninit::uninit(), } }; 2]; #[unsafe(no_mangle)] static mut OS_K1080_CONSTRUCTION_STATES: [u64; 8] = [0; 8]; struct OsK1080Factory<'a> { slot: usize, kind: u64, ownership: core::marker::PhantomData>, } impl<'a> OsK1080Factory<'a> { unsafe fn start(backend: OsK1080ContextBackend<'a>) -> Self { let mut stage = Self { slot: 0, kind: 1, ownership: core::marker::PhantomData, }; stage.gate(); if unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_CONSTRUCTION_STATES).cast::()) } != 0 { os_k1080_retain_failure(backend, 20); } let pointer = stage.pointer::>(0); unsafe { pointer.write(backend); } stage.publish(1); stage } fn gate(&self) { let e = os_k1080_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || os_k960_timer().1 & 1 != 0 { park(); } } fn pointer(&self, slot: usize) -> *mut T { if slot >= 2 || core::mem::size_of::() > core::mem::size_of::() || core::mem::align_of::() > 16 { park(); } unsafe { core::ptr::addr_of_mut!(OS_K1080_CONSTRUCTION) .cast::() .add(slot) .cast::() } } fn source(&self, kind: u64) -> *mut T { self.gate(); if self.kind != kind || unsafe { read_volatile( core::ptr::addr_of!(OS_K1080_CONSTRUCTION_STATES) .cast::() .add(self.slot), ) } != kind { park(); } self.pointer(self.slot) } fn destination(&self) -> *mut T { let slot = 1 - self.slot; if unsafe { read_volatile( core::ptr::addr_of!(OS_K1080_CONSTRUCTION_STATES) .cast::() .add(slot), ) } != 0 { park(); } self.pointer(slot) } fn publish(&mut self, kind: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_CONSTRUCTION_STATES) .cast::() .add(self.slot), kind, ); write_volatile( core::ptr::addr_of_mut!(OS_K1080_CONSTRUCTION_STATES) .cast::() .add(2), self.slot as u64, ); write_volatile( core::ptr::addr_of_mut!(OS_K1080_CONSTRUCTION_STATES) .cast::() .add(3), kind, ); } } fn commit(&mut self, kind: u64) { let old = self.slot; self.slot = 1 - old; self.kind = kind; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_CONSTRUCTION_STATES) .cast::() .add(old), 0, ); } self.publish(kind); } } macro_rules! os_k1080_constructor_stage { ($name:ident,$from:ty,$to:ty,$before:expr,$after:expr,$constructor:path) => { impl<'a> OsK1080Factory<'a> { #[inline(never)] fn $name(&mut self) { let source = self.source::<$from>($before); let destination = self.destination::<$to>(); // Destination validation and the actual CPU gate precede the // sole read. The Rust local owns source until one BSS write. let actual = unsafe { source.read() }; match $constructor(actual) { Ok(actual) => { unsafe { destination.write(actual); } self.commit($after); } Err((_, actual)) => { unsafe { source.write(actual); } os_k1080_word(122, $before); park(); } } } } }; } os_k1080_constructor_stage!( context, OsK1080ContextBackend<'a>, os_k1080_context::ContextBank, 3>, 1, 2, os_k1080_context::ContextBank::, 3>::new ); os_k1080_constructor_stage!( asynchronous, os_k1080_context::ContextBank, 3>, os_k1080_context::AsyncBank, 3>, 2, 3, os_k1080_context::AsyncBank::, 3>::new ); os_k1080_constructor_stage!( completion, os_k1080_context::AsyncBank, 3>, OsK1080Bank<'a>, 3, 4, OsK1080Bank::new ); os_k1080_constructor_stage!( members, OsK1080Bank<'a>, OsK1080Backend<'a>, 4, 5, OsK1080Backend::new ); os_k1080_constructor_stage!( policy, OsK1080Backend<'a>, os_k1080_queue::Queue, 3>, 5, 6, os_k1080_queue::Queue::, 3>::new ); os_k1080_constructor_stage!( selection, os_k1080_queue::Queue, 3>, os_k1080_queue::AsyncQueue, 3>, 6, 7, os_k1080_queue::AsyncQueue::, 3>::new ); impl<'a> OsK1080Factory<'a> { #[inline(never)] fn attach(&mut self) { let core = unsafe { &mut *self.source::, 3>>(2) }; for ordinal in 0..3 { let binding = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1080_BINDINGS) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); let h = match core.attach(binding) { Ok(h) => h, Err((_, binding)) => { unsafe { core::ptr::addr_of_mut!(OS_K1080_BINDINGS) .cast::>() .add(ordinal) .write(Some(binding)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1080_CONTEXT_HANDLES) .cast::>() .add(ordinal) .write(Some(h)); } } } #[inline(never)] fn issue(&mut self) { let backend = unsafe { &mut *self.source::>(5) }; for ordinal in 0..3 { let h = unsafe { read_volatile( core::ptr::addr_of!(OS_K1080_CONTEXT_HANDLES) .cast::>() .add(ordinal), ) } .unwrap_or_else(|| park()); let m = backend.issue(h, ordinal).unwrap_or_else(|_| park()); unsafe { core::ptr::addr_of_mut!(OS_K1080_MEMBERS) .cast::>() .add(ordinal) .write(Some(m)); } } } #[inline(never)] fn enqueue(&mut self) { let queue = unsafe { &mut *self.source::, 3>>(6) }; for ordinal in 0..3 { let m = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1080_MEMBERS) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); let class = os_k1080_queue::Class::Normal; let h = match queue.enqueue(m, class) { Ok(h) => h, Err((_, m)) => { unsafe { core::ptr::addr_of_mut!(OS_K1080_MEMBERS) .cast::>() .add(ordinal) .write(Some(m)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1080_QUEUE_HANDLES) .cast::>() .add(ordinal) .write(Some(h)); } } } } static mut OS_K1080_SPARES: [Option; 3] = [const { None }; 3]; static mut OS_K1080_FAILED_ROOTS: [Option; 3] = [const { None }; 3]; static mut OS_K1080_BINDINGS: [Option; 3] = [const { None }; 3]; static mut OS_K1080_MEMBERS: [Option; 3] = [const { None }; 3]; #[unsafe(no_mangle)] static mut OS_K1080_CONTEXT_HANDLES: [Option; 3] = [None; 3]; #[unsafe(no_mangle)] static mut OS_K1080_QUEUE_HANDLES: [Option; 3] = [None; 3]; static mut OS_K1080_FAILURE_CUSTODY: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); fn os_k1080_retain_failure(actual: T, kind: u64) -> ! { if core::mem::size_of::() > core::mem::size_of::() || core::mem::align_of::() > core::mem::align_of::() { park(); } unsafe { core::ptr::addr_of_mut!(OS_K1080_FAILURE_CUSTODY).cast::().write(actual); } os_k1080_word(120, kind); os_k1080_word(121, core::mem::size_of::() as u64); park() } struct OsK1080Installation<'a> { pointer: *mut OsK1080Core<'a>, active: bool, ownership: core::marker::PhantomData>, } impl<'a> OsK1080Installation<'a> { fn extract(mut self)->OsK1080ContextBackend<'a> { os_k1080_main_gate(); unsafe{(&*self.pointer).os_k1080_cleanup_ready()}.unwrap_or_else(|_|park()); let q=os_k1080_queue::os_k1080_return_layout::(); let offset=OsK1080Core::os_k1080_backend_offset()+OsK1080Bridge::os_k1080_queue_offset() +q[1]+q[6]+q[11]+core::mem::offset_of!(OsK1080Backend,bank) as u64; let source=unsafe{self.pointer.cast::().add(offset as usize)}.cast::>(); let mut destination=core::mem::MaybeUninit::uninit(); unsafe{OsK1080Bank::os_k1040_take_backend_in_place(source,&mut destination)}.unwrap_or_else(|_|park()); self.active=false;os_k1080_word(96,0); unsafe{destination.assume_init()} } } impl Drop for OsK1080Installation<'_> { fn drop(&mut self) { if self.active { park(); } } } os_k1080_constructor_stage!( return_queue, os_k1080_queue::AsyncQueue,3>, OsK1080Queue<'a>,7,8,OsK1080Queue::new ); impl<'a> OsK1080Factory<'a> { #[inline(never)] fn bridge(&mut self) { let source = self.source::>(8); let destination = self.destination::>(); let handles = core::array::from_fn(|i| unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_QUEUE_HANDLES) .cast::>().add(i)) }.unwrap_or_else(|| park())); let queue = unsafe { source.read() }; match OsK1080Bridge::new(queue, handles, OsK1080Port) { Ok(actual) => { unsafe { destination.write(actual); } self.commit(9); } Err((_, actual, _authority)) => { unsafe { source.write(actual); } os_k1080_word(122,8); park(); } } } } os_k1080_constructor_stage!(wait_core,OsK1080Bridge<'a>,OsK1080Core<'a>,9,10,OsK1080Core::new); impl<'a> OsK1080Factory<'a> { #[inline(never)] fn install(mut self) -> OsK1080Installation<'a> { let source = self.source::>(10); let pointer = core::ptr::addr_of_mut!(OS_K1080_CORE).cast::>(); let actual = unsafe { source.read() }; unsafe { pointer.write(actual); write_volatile(core::ptr::addr_of_mut!(OS_K1080_CONSTRUCTION_STATES).cast::().add(self.slot),0); } self.kind=11; self.publish(11); os_k1080_word(96,1); OsK1080Installation {pointer,active:true,ownership:core::marker::PhantomData} } } fn os_k1080_core() -> *mut OsK1080Core<'static> { let e=os_k1080_execution(); if e.cpu!=0 || !e.irq_masked || e.in_irq || unsafe {read_volatile(core::ptr::addr_of!(OS_K1080_EVIDENCE).cast::().add(96))}!=1 { park(); } core::ptr::addr_of_mut!(OS_K1080_CORE).cast() } #[inline(never)] fn os_k1080_factory<'a>( memory: &'a mut OsK660Memory, tables: &'a mmu::Tables, bootstrap: u64, ) -> OsK1080Installation<'a> { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags)); } os_k1080_word(112, sp); unsafe extern "C" { fn __os_k1080_a_entry(); fn __os_k1080_b_entry(); fn __os_k1080_c_entry(); } for ordinal in 0..3 { let root = os_k1080_create_root(memory, tables).unwrap_or_else(|_| park()); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_ROOT_ADDRESSES) .cast::() .add(ordinal), root.address(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1080_ROOT_GENERATIONS) .cast::() .add(ordinal), root.generation(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1080_ROOT_LIVE) .cast::() .add(ordinal), 1, ); } let stack = match os_k1080_allocate_stack(memory, tables, ordinal, root) { Ok(s) => s, Err((_, root)) => { unsafe { core::ptr::addr_of_mut!(OS_K1080_FAILED_ROOTS) .cast::>() .add(ordinal) .write(Some(root)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1080_SPARES) .cast::>() .add(ordinal) .write(Some(stack)); } } let mut table = match OsK1080Table::new(OsK1080StackBackend { memory, tables, trace: true, }) { Ok(t) => t, Err((_, b)) => os_k1080_retain_failure(b, 1), }; let entries = [ __os_k1080_a_entry as *const () as u64, __os_k1080_b_entry as *const () as u64, __os_k1080_c_entry as *const () as u64, ]; let mut ids = [None; 3]; for ordinal in 0..3 { let stack = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1080_SPARES) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); ids[ordinal] = Some( match table.create(stack, entries[ordinal], ordinal as u64) { Ok(id) => id, Err((_, stack)) => { unsafe { core::ptr::addr_of_mut!(OS_K1080_SPARES) .cast::>() .add(ordinal) .write(Some(stack)); } os_k1080_retain_failure(table, 2) } }, ); } let mut backend = OsK1080ContextBackend { table, bootstrap, current: None, pins: [false; 3], records: [None; 3], observations: [None; 3], guards: [0; 3], sleep_wait: None, sequence: 0, }; for ordinal in 0..3 { let binding = match backend.pin(ids[ordinal].unwrap(), ordinal) { Ok(b) => b, Err(_) => os_k1080_retain_failure(backend, 3), }; unsafe { core::ptr::addr_of_mut!(OS_K1080_BINDINGS) .cast::>() .add(ordinal) .write(Some(binding)); } } let mut factory = unsafe { OsK1080Factory::start(backend) }; factory.context(); factory.attach(); factory.asynchronous(); factory.completion(); factory.members(); factory.issue(); factory.policy(); factory.enqueue(); factory.selection(); factory.return_queue(); factory.bridge(); factory.wait_core(); factory.install() } // The IRQ owns this ingress separately from WaitSet/Bridge/Queue/Table. // No pointer to those owners, Runtime or allocator is published here. type OsK1080Ingress = os_k1080_wait::Ingress; #[unsafe(no_mangle)] static mut OS_K1080_KERNEL_FRAME:u64=0; struct OsK1080IrqPort; unsafe impl os_k1080_wait::Producer for OsK1080IrqPort { fn execution(&self) -> os_k1080_wait::Execution { os_k1080_source_execution() } fn observe(&self) -> Result { os_k1080_source_observation() } } #[unsafe(no_mangle)] static mut OS_K1080_INGRESS: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1080_INGRESS_LIVE: u64=0; #[unsafe(no_mangle)] static mut OS_K1080_REDIS: u64=0; #[unsafe(no_mangle)] static mut OS_K1080_IRQ_COUNT: u64=0; #[unsafe(no_mangle)] static mut OS_K1080_IRQ_ACTIVE: u64=0; #[unsafe(no_mangle)] static mut OS_K1080_TIMER_OWNER: [u64;16]=[0;16]; #[unsafe(no_mangle)] static mut OS_K1080_IRQ_FRAMES: [os_k920_exception::Frame;2]=[OS_K920_ZERO_FRAME;2]; #[unsafe(no_mangle)] static mut OS_K1080_IRQ_ADDRESSES: [u64;2]=[0;2]; #[unsafe(no_mangle)] static mut OS_K1080_IRQ_RECORDS: [[u64;20];2]=[[0;20];2]; #[unsafe(no_mangle)] static mut OS_K1080_HARDWARE_OBSERVATIONS: [[u64;8];128]=[[0;8];128]; #[unsafe(no_mangle)] static mut OS_K1080_HARDWARE_COUNT: u64=0; fn os_k1080_main_gate() { let e=os_k1080_execution(); if e.cpu!=0 || !e.irq_masked || e.in_irq {park();} } fn os_k1080_ingress() -> *mut OsK1080Ingress { os_k1080_main_gate(); if unsafe {read_volatile(core::ptr::addr_of!(OS_K1080_INGRESS_LIVE))}!=1 {park();} core::ptr::addr_of_mut!(OS_K1080_INGRESS).cast() } fn os_k1080_gic_state(redistributor:u64)->[u64;3] { if redistributor==0 {park();} unsafe {[ read_volatile((redistributor+0x1_0100) as *const u32) as u64, read_volatile((redistributor+0x1_0200) as *const u32) as u64, read_volatile((redistributor+0x1_0300) as *const u32) as u64, ]} } fn os_k1080_quiesce(reason:u64) { os_k1080_main_gate(); // Clear ENABLE with actual CPU readback. No pending/active GIC bit is // written or relabelled; uncertain hardware retains all owners masked. unsafe {asm!("msr CNTP_CTL_EL0, xzr","isb",options(nomem,nostack,preserves_flags));} let (cval,ctl)=os_k960_timer(); let redistributor=unsafe {read_volatile(core::ptr::addr_of!(OS_K1080_REDIS))}; let [enabled,pending,active]=os_k1080_gic_state(redistributor); let count=unsafe {read_volatile(core::ptr::addr_of!(OS_K1080_HARDWARE_COUNT))} as usize; if count>=128 {park();} let row=[reason,os_k960_counter(),cval,ctl,enabled,pending,active,os_k940_registers().0]; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1080_HARDWARE_OBSERVATIONS).cast::<[u64;8]>().add(count),row); write_volatile(core::ptr::addr_of_mut!(OS_K1080_HARDWARE_COUNT),count as u64+1); } if ctl&3!=0 || enabled&(1<<30)==0 || pending&(1<<30)!=0 || active&(1<<30)!=0 || unsafe {read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_ACTIVE))}!=0 || unsafe {read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}!=0 {park();} } struct OsK1080SourceLease<'a> { transfer: OsK940IrqTransfer<'a>, epoch:u64, owner:u64, } impl<'a> OsK1080SourceLease<'a> { fn new(irq:&'a mut OsK660Irq,redistributor:u64)->Result { os_k1080_main_gate(); if os_k1080_source_word(0)!=0 || unsafe {read_volatile(core::ptr::addr_of!(OS_K1080_INGRESS_LIVE))}!=0 || (0..3).any(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_SOURCE_PINS).cast::().add(i))}!=0) {return Err("wait-source-already-owned");} unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K1080_REDIS),redistributor);} os_k1080_quiesce(0); let transfer=os_k940_transfer_irq(irq,redistributor)?; let epoch=os_k1080_nonce(core::ptr::addr_of_mut!(OS_K1080_WAIT_IDENTITY)); let owner=os_k1080_nonce(core::ptr::addr_of_mut!(OS_K1080_WAIT_IDENTITY)); unsafe {write_volatile(core::ptr::addr_of_mut!(OS_K1080_SOURCE),[1,30,epoch,owner,0,0,0,0]);} let ingress=match OsK1080Ingress::new(OsK1080IrqPort) { Ok(i)=>i, // A constructor failure retains the actual predecessor transfer; // do not drop it while source publication is uncertain. Err(_)=>park(), }; unsafe { core::ptr::addr_of_mut!(OS_K1080_INGRESS).cast::().write(ingress); write_volatile(core::ptr::addr_of_mut!(OS_K1080_INGRESS_LIVE),1); } Ok(Self{transfer,epoch,owner}) } } impl Drop for OsK1080SourceLease<'_> { fn drop(&mut self) { os_k1080_main_gate(); os_k1080_quiesce(0xff); if self.transfer.predecessor_bindings()!=0 || os_k1080_source_word(2)!=self.epoch || os_k1080_source_word(3)!=self.owner || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_EVIDENCE).cast::().add(96))}!=0 || unsafe{(&*os_k1080_ingress()).pending()}!=0 || (0..3).any(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_SOURCE_PINS).cast::().add(i))}!=0) {park();} let actual=unsafe{core::ptr::addr_of_mut!(OS_K1080_INGRESS).cast::().read()}; match actual.into_producer() {Ok(_)=>{},Err(actual)=>{ unsafe{core::ptr::addr_of_mut!(OS_K1080_INGRESS).cast::().write(actual);} park(); }} unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1080_INGRESS_LIVE),0); write_volatile(core::ptr::addr_of_mut!(OS_K1080_SOURCE).cast::(),0); } // The contained actual transfer performs its frozen VBAR/timer/GIC // readback restoration checks after this main-only Drop completes. } } fn os_k1080_timer_owner_word(index:usize)->u64 { if index>=16 {park();} unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_TIMER_OWNER).cast::().add(index))} } #[unsafe(no_mangle)] extern "C" fn os_k1080_dispatch(pointer:*mut os_k920_exception::Frame)->u64 { let raw=os_k1080_execution(); let address=pointer as u64; if pointer.is_null() || raw.cpu!=0 || !raw.irq_masked || raw.in_irq || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_ACTIVE))}!=0 || os_k1080_timer_owner_word(0)!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_INGRESS_LIVE))}!=1 {park();} let stack=os_k920_exception::Stack::new(os_k1080_timer_owner_word(5), os_k1080_timer_owner_word(5).checked_add(os_k1080_timer_owner_word(6)).unwrap_or_else(||park())) .unwrap_or_else(|_|park()); stack.frame_address(address).unwrap_or_else(|_|park()); let frame=unsafe{read_volatile(pointer)}; os_k920_kernel_irq_return(&frame,address,stack,core::ptr::addr_of!(_text_start) as u64, core::ptr::addr_of!(_text_end) as u64,__os_k920_kernel_tco_observed()).unwrap_or_else(|_|park()); unsafe extern "C" {static __os_k1080_timer_wait_loop:u8;static __os_k1080_timer_wait_mask:u8;} if frame.vector_slot!=5 || frame.spsr&31!=5 || frame.elr&3!=0 || frame.elr=core::ptr::addr_of!(__os_k1080_timer_wait_mask) as u64 || frame.interrupted_sp!=address+848 || os_k940_registers().0!=os_k1080_timer_owner_word(4) {park();} let iar:u64; unsafe{asm!("mrs {iar}, ICC_IAR1_EL1",iar=out(reg)iar,options(nomem,nostack,preserves_flags));} if iar&0x00ff_ffff>=1020 {return 0;} // Spurious is never EOI/DIR. if iar&0x00ff_ffff!=30 {park();} // Retain uncertain actual acknowledgement. let number=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_COUNT))} as usize; let (cval,ctl)=os_k960_timer();let counter=os_k960_counter(); if number>=2 || os_k1080_timer_owner_word(8)!=number as u64+1 || cval!=os_k1080_timer_owner_word(10) || ctl&7!=5 || counter().add(number),frame); write_volatile(core::ptr::addr_of_mut!(OS_K1080_IRQ_ADDRESSES).cast::().add(number),address); asm!("msr CNTP_CTL_EL0, xzr","isb",options(nomem,nostack,preserves_flags)); } let split:u64; unsafe{ asm!("mrs {s}, ICC_CTLR_EL1",s=out(reg)split,options(nomem,nostack,preserves_flags)); asm!("dsb sy","msr ICC_EOIR1_EL1, {iar}","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags)); if split&2!=0 {asm!("msr ICC_DIR_EL1, {iar}","dsb sy","isb",iar=in(reg)iar,options(nomem,nostack,preserves_flags));} write_volatile(core::ptr::addr_of_mut!(OS_K1080_SOURCE).cast::().add(4),sequence); write_volatile(core::ptr::addr_of_mut!(OS_K1080_SOURCE).cast::().add(5),1); } let produced=unsafe{turkos_kernel::runtime::wait_ingress::wait_guest_irq_poll( core::ptr::addr_of_mut!(OS_K1080_INGRESS).cast::(),||{ ::execution(&OsK1080IrqPort) })}.unwrap_or_else(|_|park()); if !produced {park();} let [enabled,pending,active]=os_k1080_gic_state(unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_REDIS))}); let row=[number as u64+1,os_k1080_timer_owner_word(1),os_k1080_timer_owner_word(2), os_k1080_timer_owner_word(3),os_k1080_timer_owner_word(4),counter,cval,ctl,iar,address,total, sequence,frame.spsr,split,enabled,pending,active,os_k1080_source_word(2),os_k1080_source_word(3),1]; unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_IRQ_RECORDS).cast::<[u64;20]>().add(number),row); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT),total); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER),counter); write_volatile(core::ptr::addr_of_mut!(OS_K1080_IRQ_COUNT),number as u64+1); } os_k1080_checkpoint(0x80+number as u64,true); unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),0); write_volatile(core::ptr::addr_of_mut!(OS_K1080_IRQ_ACTIVE),0); write_volatile(core::ptr::addr_of_mut!(OS_K1080_TIMER_OWNER).cast::(),0); } 0 } #[unsafe(no_mangle)] static mut OS_K1080_CAPTURE_GATE:u8=0; #[unsafe(no_mangle)] static mut OS_K1080_HANDOFF:Option=None; #[unsafe(no_mangle)] static mut OS_K1080_TRANSFER:Option=None; #[unsafe(no_mangle)] static mut OS_K1080_RETURN:Option=None; #[unsafe(no_mangle)] static mut OS_K1080_TOKENS:[Option;3]=[const{None};3]; #[unsafe(no_mangle)] static mut OS_K1080_TOKEN_LOOKUPS:[[u64;3];3]=[[0;3];3]; #[unsafe(no_mangle)] static mut OS_K1080_OLD_LOOKUPS:[Option;3]=[None;3]; #[unsafe(no_mangle)] static mut OS_K1080_SWITCH_MODE:u64=0; #[unsafe(no_mangle)] static mut OS_K1080_SLEEP_SOURCE:u64=0; #[unsafe(no_mangle)] static mut OS_K1080_DEADLINES:[u64;3]=[0;3]; #[unsafe(no_mangle)] static mut OS_K1080_TASK_PROGRESS:[u64;3]=[0;3]; #[unsafe(no_mangle)] static mut OS_K1080_TASK_ENTRIES:[u64;3]=[0;3]; #[unsafe(no_mangle)] static mut OS_K1080_AUTHORITY_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K1080_AUTHORITY_AFTER:[u64;328]=[0;328]; fn os_k1080_checkpoint(stage:u64,in_irq:bool) { let e=os_k1080_execution(); if e.cpu!=0 || !e.irq_masked || e.in_irq!=in_irq {park();} let (root,tcr,vbar,daif)=os_k940_registers(); for(i,v) in [(5,stage),(6,root),(7,tcr),(8,vbar),(9,daif),(10,e.cpu as u64), (11,e.sp),(12,os_k960_counter()),(13,u64::from(in_irq)), (14,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))}), (15,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))} as u64), (16,os_k960_timer().0),(17,os_k960_timer().1)] { os_k1080_word(i,v); } if !in_irq { os_k1080_word(18,core::ptr::addr_of!(OS_K1080_CORE) as u64); os_k1080_word(19,core::mem::size_of::() as u64); } // Checkpoint evidence in IRQ is exclusively the dedicated producer/frame // buffers. Main harness may capture the actual owner graph after I=1. unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_CAPTURE_GATE),0);} write_str("TURKOS/OS/K1080: DRAFT-CAPTURE STAGE=0x");write_hex(stage as usize); write_line(" READY"); while unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_CAPTURE_GATE))}!=1 {core::hint::spin_loop();} unsafe{asm!("dsb ish","isb",options(nostack,preserves_flags));} } fn os_k1080_capture_source(address:u64)->os_k1080_context::Capture { os_k1080_main_gate(); let (e,layout)=unsafe{(&*os_k1080_core()).with_backend_observation(|bridge| { bridge.with_queue_observation(|q| { let e=q.queue().with_backend_observation(|b|os_k1080_queue::Backend::execution(b)); let id=e.thread.unwrap_or_else(||park()); let layout=q.queue().with_backend_observation(|b| { b.bank.handoff().bank().with_backend_observation(|c| { if c.current!=Some(id) {park();} let ordinal=c.records.iter().position(|r|r.is_some_and(|r|r.id==id)).unwrap_or_else(||park()); if !c.pins[ordinal] || b.members[ordinal].is_none() || c.table.record(id).unwrap_or_else(|_|park()).state!=os_k1080_thread::State::Running {park();} c.validate_stack_owner(id).unwrap_or_else(|_|park()); c.table.os_k1020_observe_stack(id,|s| { let l=s.layout(); if s.ordinal!=ordinal || s.root.address()!=e.root || s.root.generation()==0 || unsafe{read_volatile(l.base as *const u64)}!=os_k1080_thread::LOW_CANARY || unsafe{read_volatile((l.base+l.bytes-8) as *const u64)}!=os_k1080_thread::HIGH_CANARY {park();} l }).unwrap_or_else(|_|park()) }) }); (e,layout) }) })}; let episode=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_CAPTURE_EPISODE))}; if address%16!=0 || addresslayout.usable_top()) || episode==0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_LIVE_CAPTURE_ADDRESS))}!=address {park();} let frame=unsafe{read_volatile(address as *const os_k1080_context::Cooperative)}; if frame.sp!=address+640 || frame.canary!=os_k1080_context::CONTEXT_CANARY || frame.padding!=0 {park();} let capture=os_k1080_context::Capture{image:os_k1080_context::Image::Cooperative(frame),address, abi:os_k1080_context::Abi::COOPERATIVE,episode}; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_CURRENT_CAPTURE),Some(capture));} os_k1080_event(0x60,e.thread.unwrap().slot() as u64,address,episode); capture } fn os_k1080_set_wire(image:os_k1080_context::Image,nonce:u64)->*const OsK1080Wire { let (frame,pc,argument,kind)=match image { os_k1080_context::Image::Initial{entry,argument,sp,return_pc}=>{ let mut f=os_k1080_context::Cooperative::EMPTY;f.sp=sp;f.x19_x30[11]=return_pc; (f,entry,argument,0) }, os_k1080_context::Image::Cooperative(f)=>(f,f.x19_x30[11],0,1), _=>park(), }; unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_DEPARTED_NONCE),nonce); write_volatile(core::ptr::addr_of_mut!(OS_K1080_WIRE),OsK1080Wire{frame,pc,argument,nonce,kind}); } core::ptr::addr_of!(OS_K1080_WIRE) } fn os_k1080_depart_normal(token:os_k1080_queue::OwnedSelection)->*const OsK1080Wire { os_k1080_quiesce(1); let result=unsafe{(&mut *os_k1080_core()).os_k1080_main_depart(token)}; let (actual,image,nonce)=match result { Ok(v)=>v, Err(os_k1080_queue::HandoffSelectionError::Before(_,actual))=>{ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_HANDOFF),Some(actual));}park(); },Err(os_k1080_queue::HandoffSelectionError::Quarantined(_))=>park(), }; unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_HANDOFF),Some(actual)); write_volatile(core::ptr::addr_of_mut!(OS_K1080_SWITCH_MODE),0); write_volatile(core::ptr::addr_of_mut!(OS_K1080_PUBLISHED_CURRENT),None); } let wire=os_k1080_set_wire(image,nonce); os_k1080_checkpoint(0x20,false); wire } fn os_k1080_token_store(ordinal:usize,token:os_k1080_wait::WaitToken) { os_k1080_main_gate(); if ordinal>=3 {park();} let pointer=unsafe{core::ptr::addr_of_mut!(OS_K1080_TOKENS).cast::>().add(ordinal)}; if unsafe{(&*pointer).is_some()} {os_k1080_retain_failure(token,0x30);} unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_TOKEN_LOOKUPS).cast::<[u64;3]>().add(ordinal), [token.slot() as u64,token.generation(),ordinal as u64]); pointer.write(Some(token)); } } fn os_k1080_token_take(ordinal:usize)->os_k1080_wait::WaitToken { os_k1080_main_gate();if ordinal>=3 {park();} unsafe{(&mut *core::ptr::addr_of_mut!(OS_K1080_TOKENS).cast::>().add(ordinal)).take()} .unwrap_or_else(||park()) } fn os_k1080_retire_token(ordinal:usize,expected:os_k1080_wait::Outcome) { let token=os_k1080_token_take(ordinal); let core=unsafe{&mut *os_k1080_core()}; let records=core.records();let winners=core.winners(); if core.cancel(&token)!=Err(os_k1080_wait::Error::Duplicate) || core.records()!=records || core.winners()!=winners {park();} let old=core.os_k1080_inert_lookup(&token).unwrap_or_else(|_|park()); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_OLD_LOOKUPS) .cast::>().add(ordinal),Some(old));} match core.retire(token) { Ok(c) if c.outcome==expected=>os_k1080_event(0x74,ordinal as u64,c.generation,c.outcome as u64), Ok(_)=>park(), Err((_,actual))=>{os_k1080_token_store(ordinal,actual);park();} } } fn os_k1080_service(expected:usize) { os_k1080_main_gate();os_k1080_quiesce(2); let count={ let core=unsafe{&mut *os_k1080_core()}; let ingress=unsafe{&mut *os_k1080_ingress()}; core.service(ingress).unwrap_or_else(|_|park()) }; if count!=expected {park();} if unsafe{(&*os_k1080_ingress()).pending()}==0 { unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_SOURCE).cast::().add(5),0);} } os_k1080_event(0x75,count as u64,os_k1080_source_word(4),os_k960_counter()); } #[unsafe(no_mangle)] extern "C" fn __os_k1080_prepare_cooperative(address:u64,requested:u64)->*const OsK1080Wire { let capture=os_k1080_capture_source(address); if requested==0 { let token=unsafe{(&mut *os_k1080_core()).os_k1080_main_select()}.unwrap_or_else(|_|park()); return os_k1080_depart_normal(token); } let current=unsafe{(&*os_k1080_core()).with_backend_observation(|b|os_k1080_wait::Backend::execution(b))}.current.unwrap_or_else(||park()); let ordinal=(0..3).find(|i|unsafe{(&*os_k1080_core()).os_k1080_thread(*i)}.ok()==Some(current)).unwrap_or_else(||park()); let target_ordinal=match(requested,ordinal){(1,0)=>1,(2,1)=>2,(2,0)=>2,(3,2)=>1,_=>park()}; let target=unsafe{(&*os_k1080_core()).os_k1080_thread(target_ordinal)}.unwrap_or_else(|_|park()); os_k1080_quiesce(3); let deadline=if requested==3 || (requested==2&&ordinal==0) { Some(os_k960_counter().checked_add((os_k960_frequency()/10).max(1)).unwrap_or_else(||park())) } else {None}; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_DEADLINES).cast::().add(ordinal),deadline.unwrap_or(0));} let token={ let core=unsafe{&mut *os_k1080_core()};let ingress=unsafe{&*os_k1080_ingress()}; match core.arm(ingress,current,target,capture,deadline) { Ok(t)=>t,Err(f)=>{if let Some(t)=f.retained {os_k1080_token_store(ordinal,t);}park();} } }; os_k1080_token_store(ordinal,token); if requested==1 { os_k1080_checkpoint(0x30,false); let sequence=os_k1080_arm_irq(0); unsafe extern "C" {fn __os_k1080_timer_wait(sequence:u64)->u64;} let iterations=unsafe{__os_k1080_timer_wait(sequence)}; if iterations==0 {park();} os_k1080_quiesce(4); os_k1080_event(0x76,ordinal as u64,sequence,iterations); } let token=os_k1080_token_take(ordinal); let departure=match unsafe{(&mut *os_k1080_core()).depart(token)} { Ok(d)=>d,Err((_,actual))=>{os_k1080_token_store(ordinal,actual);park();} }; match departure { os_k1080_wait::Departure::Ready(actual)=>{ if requested!=1 {os_k1080_token_store(ordinal,actual);park();} os_k1080_token_store(ordinal,actual); os_k1080_retire_token(ordinal,os_k1080_wait::Outcome::Signal); os_k1080_service(0); os_k1080_checkpoint(0x31,false); let token=unsafe{(&mut *os_k1080_core()).os_k1080_main_select()}.unwrap_or_else(|_|park()); os_k1080_depart_normal(token) }, os_k1080_wait::Departure::Switch(actual)=>{ if requested==1 {os_k1080_retain_failure(actual,0x31);} let (image,nonce)=unsafe{(&*os_k1080_core()).os_k1080_wait_wire(&actual)}.unwrap_or_else(|_|park()); unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_TRANSFER),Some(actual)); write_volatile(core::ptr::addr_of_mut!(OS_K1080_SLEEP_SOURCE),ordinal as u64); write_volatile(core::ptr::addr_of_mut!(OS_K1080_SWITCH_MODE),1); write_volatile(core::ptr::addr_of_mut!(OS_K1080_PUBLISHED_CURRENT),None); } let wire=os_k1080_set_wire(image,nonce); os_k1080_checkpoint(0x32+ordinal as u64,false); wire } } } fn os_k1080_landing_snapshot(address:u64,bootstrap:bool) { os_k1080_main_gate(); if bootstrap { if !os_k1080_boot_frame_range(address) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_BOOT_CAPTURE_ADDRESS))}!=address {park();} } else { let layout=unsafe{(&*os_k1080_core()).os_k1080_landing_bounds()}.unwrap_or_else(|_|park()); if address%16!=0 || addresslayout.usable_top()) {park();} } let frame=unsafe{read_volatile(address as *const os_k1080_context::Cooperative)}; if frame.sp!=address+640 || frame.canary!=os_k1080_context::CONTEXT_CANARY || frame.padding!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_LANDING_ADDRESS))}!=address {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_LANDING_FRAME),frame);} os_k1080_checkpoint(0x40,false); } #[unsafe(no_mangle)] extern "C" fn __os_k1080_landed(address:u64) { os_k1080_landing_snapshot(address,false); let actual=unsafe{(&*os_k1080_core()).os_k1080_physical_execution()}.unwrap_or_else(|_|park()); // Only the checked actual root/SP/640/PC/nonce callback supplies this ID. unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_PUBLISHED_CURRENT),actual.current);} match unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_SWITCH_MODE))} { 0=>{ let token=unsafe{(&mut *core::ptr::addr_of_mut!(OS_K1080_HANDOFF)).take()}.unwrap_or_else(||park()); match unsafe{(&mut *os_k1080_core()).os_k1080_main_landed(token)} { Ok(_)=>{},Err(os_k1080_queue::HandoffSelectionError::Before(_,t))=>{ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_HANDOFF),Some(t));}park(); },Err(_)=>park(), } }, 1=>{ let source=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_SLEEP_SOURCE))} as usize; let transfer=unsafe{(&mut *core::ptr::addr_of_mut!(OS_K1080_TRANSFER)).take()}.unwrap_or_else(||park()); match unsafe{(&mut *os_k1080_core()).landed(transfer)} { Ok(token)=>os_k1080_token_store(source,token),Err((_,actual))=>{ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_TRANSFER),Some(actual));}park(); } } },_=>park(), } unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_LIVE_CAPTURE_ADDRESS),0); write_volatile(core::ptr::addr_of_mut!(OS_K1080_CURRENT_CAPTURE),None); } os_k1080_event(0x62,address,actual.current.unwrap().slot() as u64,actual.current.unwrap().generation()); os_k1080_checkpoint(0x41,false); } #[unsafe(no_mangle)] extern "C" fn __os_k1080_prepare_return(address:u64)->*const OsK1080Wire { let capture=os_k1080_capture_source(address);os_k1080_quiesce(5); let token=unsafe{(&mut *os_k1080_core()).os_k1080_main_return(capture)}.unwrap_or_else(|_|park()); let result=unsafe{(&mut *os_k1080_core()).os_k1080_main_return_depart(token)}; let (token,frame)=match result { Ok(v)=>v, Err(os_k1080_bridge::K1080MainReturnFailure::Before(_,t)|os_k1080_bridge::K1080MainReturnFailure::After(_,t))=>{ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_RETURN),Some(t));}park(); },Err(_)=>park(), }; unsafe extern "C" {fn __os_k1080_boot_landed();} unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_RETURN),Some(token)); write_volatile(core::ptr::addr_of_mut!(OS_K1080_SWITCH_MODE),2); write_volatile(core::ptr::addr_of_mut!(OS_K1080_PUBLISHED_CURRENT),None); write_volatile(core::ptr::addr_of_mut!(OS_K1080_WIRE),OsK1080Wire{ frame,pc:__os_k1080_boot_landed as *const () as u64,argument:0,nonce:0,kind:2}); } os_k1080_checkpoint(0x50,false); core::ptr::addr_of!(OS_K1080_WIRE) } #[unsafe(no_mangle)] extern "C" fn __os_k1080_return_landed(address:u64) { os_k1080_landing_snapshot(address,true); let token=unsafe{(&mut *core::ptr::addr_of_mut!(OS_K1080_RETURN)).take()}.unwrap_or_else(||park()); match unsafe{(&mut *os_k1080_core()).os_k1080_main_return_landed(token)} { Ok(())=>{}, Err(os_k1080_bridge::K1080MainReturnFailure::Before(_,t)|os_k1080_bridge::K1080MainReturnFailure::After(_,t))=>{ unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_RETURN),Some(t));}park(); },Err(_)=>park(), } unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_LIVE_CAPTURE_ADDRESS),0); write_volatile(core::ptr::addr_of_mut!(OS_K1080_CURRENT_CAPTURE),None); } os_k1080_event(0x63,address,os_k940_registers().0,0); os_k1080_checkpoint(0x51,false); } // NEW readonly pre-mutation observer. The frozen descriptor accessor belongs // to already-unmapped guards; this independently checks the mapped leaf. #[inline(never)] fn os_k1080_primary_guard_leaf(tables:&mmu::Tables,guard:u64)->Result<(u64,u64),&'static str> { os_k1080_main_gate(); const MASK:u64=0x0000_ffff_ffff_f000; let image_start=core::ptr::addr_of!(_image_start) as u64; let image_end=core::ptr::addr_of!(_image_end) as u64; if guard%4096!=0 || guardimage_end) || guard<0x4000_0000 || guard>=0x8000_0000 || os_k940_registers().0!=tables.root || tables.frame_count()<5 || tables.frame_count()>31 || tables.root%4096!=0 || !tables.owns_table_frame(tables.root) {return Err("wait-mapped-guard-owner");} let first=unsafe{read_volatile((tables.root as *const u64).add(1))}; let l2=first&MASK; if first!=l2|3 || l2==0 || l2==tables.root || !tables.owns_table_frame(l2) { return Err("wait-mapped-guard-l2"); } let second=unsafe{read_volatile((l2 as *const u64).add(((guard>>21)&511) as usize))}; let l3=second&MASK; if second!=l3|3 || l3==0 || l3==tables.root || l3==l2 || !tables.owns_table_frame(l3) { return Err("wait-mapped-guard-l3"); } let slot=l3+((guard>>12)&511)*8; let descriptor=unsafe{read_volatile(slot as *const u64)}; Ok((slot,descriptor)) } fn os_k1080_mapped_guard_slot(tables:&mmu::Tables,guard:u64)->Result<(u64,u64),&'static str> { let (slot,descriptor)=os_k1080_primary_guard_leaf(tables,guard)?; if descriptor!=(guard|0x0060_0000_0000_0703) {return Err("wait-mapped-guard-leaf");} Ok((slot,descriptor)) } #[inline(never)] fn os_k1080_restore_primary_guard(tables:&mmu::Tables,guard:u64)->Result<(),&'static str> { os_k1080_main_gate(); let e=os_k1080_execution(); let low=core::ptr::addr_of!(OS_K900_STACK) as u64+4096; let saved=unsafe{core::ptr::addr_of!(OS_K1080_EVIDENCE).cast::()}; let controls=unsafe{(read_volatile(saved.add(1)),read_volatile(saved.add(2)), read_volatile(saved.add(3)),read_volatile(saved.add(4)))}; if e.sp<=low+16 || e.sp>low+65536-16 || e.sp%16!=0 || os_k940_registers()!=controls || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_EVIDENCE).cast::().add(96))}!=0 || unsafe{(&*core::ptr::addr_of!(OS_K1080_HANDOFF)).is_some()} || unsafe{(&*core::ptr::addr_of!(OS_K1080_TRANSFER)).is_some()} || unsafe{(&*core::ptr::addr_of!(OS_K1080_RETURN)).is_some()} || (0..3).any(|i|unsafe{(&*core::ptr::addr_of!(OS_K1080_TOKENS) .cast::>().add(i)).is_some()}) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_ACTIVE))}!=0 || os_k1080_timer_owner_word(0)!=0 || os_k1080_source_word(0)!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_INGRESS_LIVE))}!=0 || (0..3).any(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_ROOT_LIVE).cast::().add(i))}!=0) || (0..3).any(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_SOURCE_PINS).cast::().add(i))}!=0) || os_k960_timer().1&3!=0 {return Err("wait-primary-return-owner");} let original=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_PRIMARY_GUARD_BEFORE))}; let (slot,descriptor)=os_k1080_primary_guard_leaf(tables,guard)?; if original!=[slot,guard|0x0060_0000_0000_0703,guard,guard+4096] || descriptor!=0 { return Err("wait-primary-return-leaf"); } let [_,pending,active]=os_k1080_gic_state(unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_REDIS))}); if pending&(1<<30)!=0 || active&(1<<30)!=0 {return Err("wait-primary-return-source");} let par:u64; unsafe{ write_volatile(slot as *mut u64,original[1]); asm!("dsb ishst","tlbi vmalle1","dsb ish","isb","at s1e1r, {guard}","isb","mrs {par}, PAR_EL1", guard=in(reg)guard,par=out(reg)par,options(nostack,preserves_flags)); } if par&1!=0 || par&0x0000_ffff_ffff_f000!=guard {return Err("wait-primary-return-translation");} os_k1080_word(118,1);os_k1080_word(119,par); Ok(()) } #[inline(never)] fn os_k1080_arm_irq(minimum_deadline:u64)->u64 { os_k1080_quiesce(6); if os_k1080_timer_owner_word(0)!=0 || os_k1080_source_word(5)!=0 || unsafe{(&*os_k1080_ingress()).pending()}!=0 {park();} let [ordinal,slot,generation,root,base,bytes]=unsafe{ (&*os_k1080_core()).with_backend_observation(|bridge|bridge.with_queue_observation(|q| { let e=q.queue().with_backend_observation(|b|os_k1080_queue::Backend::execution(b)); let id=e.thread.unwrap_or_else(||park()); q.queue().with_backend_observation(|b|b.bank.handoff().bank().with_backend_observation(|c| { if c.current!=Some(id) {park();} let ordinal=c.records.iter().position(|r|r.is_some_and(|r|r.id==id)).unwrap_or_else(||park()); let record=c.table.record(id).unwrap_or_else(|_|park()); if record.state!=os_k1080_thread::State::Running || !c.pins[ordinal] {park();} c.validate_stack_owner(id).unwrap_or_else(|_|park()); c.table.os_k1020_observe_stack(id,|s| { let l=s.layout();if s.ordinal!=ordinal || s.root.address()!=e.root {park();} [ordinal as u64,id.slot() as u64,id.generation(),e.root,l.base,l.bytes] }).unwrap_or_else(|_|park()) })) })) }; let sequence=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_COUNT))} .checked_add(1).filter(|s|*s<=2).unwrap_or_else(||park()); let now=os_k960_counter();let deadline=now.checked_add((os_k960_frequency()/10000).max(1)) .unwrap_or_else(||park()).max(minimum_deadline); unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_TIMER_OWNER), [1,ordinal,slot,generation,root,base,bytes,os_k1080_source_word(2),sequence,now,deadline, os_k1080_source_word(3),0,0,0,0]); asm!("dsb ish","msr CNTP_CVAL_EL0, {v}","isb","msr CNTP_CTL_EL0, {one}","isb", v=in(reg)deadline,one=in(reg)1u64,options(nomem,nostack,preserves_flags)); } let (actual,ctl)=os_k960_timer();if actual!=deadline || ctl&3!=1 {park();} os_k1080_event(0x77,ordinal,sequence,deadline); sequence } fn os_k1080_useful_work(ordinal:usize) { os_k1080_main_gate();if ordinal>=3 {park();} unsafe extern "C" {fn __os_k1080_cpu_work()->u64;} let sum=unsafe{__os_k1080_cpu_work()};if sum!=500500 {park();} let pointer=unsafe{core::ptr::addr_of_mut!(OS_K1080_TASK_PROGRESS).cast::().add(ordinal)}; let before=unsafe{read_volatile(pointer)}; unsafe{write_volatile(pointer,before.checked_add(sum).unwrap_or_else(||park()));} os_k1080_event(0x78,ordinal as u64,sum,before+sum); } #[unsafe(no_mangle)] extern "C" fn __os_k1080_task_work(ordinal:u64) { os_k1080_main_gate();if ordinal>=3 {park();} let ordinal=ordinal as usize; let e=unsafe{(&*os_k1080_core()).with_backend_observation(|b|os_k1080_wait::Backend::execution(b))}; if e.current!=Some(unsafe{(&*os_k1080_core()).os_k1080_thread(ordinal)}.unwrap_or_else(|_|park())) {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_TASK_ENTRIES).cast::().add(ordinal),1);} unsafe extern "C" {fn __os_k1080_cooperate(request:u64);fn __os_k1080_timer_wait(sequence:u64)->u64;} os_k1080_useful_work(ordinal); unsafe{__os_k1080_cooperate(0);} match ordinal { 0=>{ unsafe{__os_k1080_cooperate(1);} // Actual early IRQ, no sleep publication. unsafe{__os_k1080_cooperate(2);} // A joins B's separately retained Sleep. os_k1080_retire_token(0,os_k1080_wait::Outcome::Signal); os_k1080_useful_work(0); unsafe{__os_k1080_cooperate(0);} os_k1080_useful_work(0); unsafe{__os_k1080_cooperate(0);} }, 1=>{ unsafe{__os_k1080_cooperate(2);} // B->C first actual Sleep. os_k1080_retire_token(1,os_k1080_wait::Outcome::Cancelled); os_k1080_useful_work(1); unsafe{__os_k1080_cooperate(0);} // C is actually asleep with a real absolute counter deadline. // The timer remains stopped: this episode exercises Timeout, // independently of the earlier actual IRQ signal winner. let deadline=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_DEADLINES).cast::().add(2))}; if deadline==0 {park();} let old=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_OLD_LOOKUPS) .cast::>().add(1))}.unwrap_or_else(||park()); let records=unsafe{(&*os_k1080_core()).records()}; let winners=unsafe{(&*os_k1080_core()).winners()}; if unsafe{(&*os_k1080_core()).os_k1080_check_old_lookup(&old)}!=Err(os_k1080_wait::Error::Stale) || unsafe{(&*os_k1080_core()).records()}!=records || unsafe{(&*os_k1080_core()).winners()}!=winners {park();} os_k1080_event(0x7a,1,0,0); while os_k960_counter(){ unsafe{__os_k1080_cooperate(0);} // Select A while B's Wait stays owned. let records={let core=unsafe{&*os_k1080_core()};core.records()}; if records.iter().filter(|r|r.phase==os_k1080_wait::Phase::Sleeping).count()!=2 {park();} if records.iter().filter(|r|r.signal_link).count()!=2 {park();} // The logical core read above ends before any later ASM/unmask. os_k1080_useful_work(2); os_k1080_checkpoint(0x60,false); let token=os_k1080_token_take(1); let before=unsafe{(&*os_k1080_core()).winners()}; unsafe{(&mut *os_k1080_core()).cancel(&token)}.unwrap_or_else(|_|park()); os_k1080_token_store(1,token); if unsafe{(&*os_k1080_core()).winners()}!=before+1 {park();} let deadline=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_DEADLINES).cast::().add(0))}; let sequence=os_k1080_arm_irq(deadline); let iterations=unsafe{__os_k1080_timer_wait(sequence)};if iterations==0 {park();} os_k1080_service(1); // The signal was observed at/after A's deadline. It won once; // servicing again cannot deliver a second wake or FIFO member. let winners=unsafe{(&*os_k1080_core()).winners()}; os_k1080_service(0); if unsafe{(&*os_k1080_core()).winners()}!=winners {park();} os_k1080_checkpoint(0x61,false); os_k1080_useful_work(2); unsafe{__os_k1080_cooperate(0);} unsafe{__os_k1080_cooperate(3);} // C->B actual timeout Wait. os_k1080_retire_token(2,os_k1080_wait::Outcome::Timeout); os_k1080_useful_work(2); },_=>park(), } // Each task returns only through the actual captured completion/assembly // bootstrap landing. No owner reference lives beyond this Rust callback. os_k1080_event(0x79,ordinal as u64,os_k960_counter(),0); } #[inline(never)] fn os_k1080_retain_configuration() { os_k1080_retain_layouts(); unsafe extern "C" { fn __os_k1080_run(wire:*const OsK1080Wire); fn __os_k1080_a_entry();fn __os_k1080_b_entry();fn __os_k1080_c_entry(); fn __os_k1080_primary_on_stack(runtime:*mut u8,tables:*const u8,boot:*const u8)->u64; static __os_k1080_vectors:u8; } for address in [__os_k1080_run as *const () as u64, __os_k1080_a_entry as *const () as u64,__os_k1080_b_entry as *const () as u64, __os_k1080_c_entry as *const () as u64,os_k1080_factory as *const () as u64, os_k1080_dispatch as *const () as u64,core::ptr::addr_of!(__os_k1080_vectors) as u64, // Read only a mapped usable byte; the new primary guard is unmapped. core::ptr::addr_of!(OS_K1080_PRIMARY_STACK) as u64+4096, core::ptr::addr_of!(OS_K1080_CORE) as u64,core::ptr::addr_of!(OS_K1080_COMPILE_ABI) as u64, core::ptr::addr_of!(OS_K1080_PRIMARY_STACK_EVIDENCE) as u64, core::ptr::addr_of!(OS_K1080_KERNEL_TABLES) as u64, core::ptr::addr_of!(OS_K1080_PRIMARY_GUARD_BEFORE) as u64, core::ptr::addr_of!(OS_K1080_AUTHORITY_BEFORE) as u64, core::ptr::addr_of!(OS_K1080_AUTHORITY_AFTER) as u64, core::ptr::addr_of!(OS_K1080_WAKE_EPOCH) as u64, os_k1080_deferred_batch_gate as *const () as u64, core::ptr::addr_of!(OS_K1080_BATCH_GATES) as u64, core::ptr::addr_of!(OS_K1080_BATCH_EVIDENCE) as u64, core::ptr::addr_of!(OS_K1080_BATCH_COUNT) as u64, __os_k1080_primary_on_stack as *const () as u64, __os_k1080_primary_body as *const () as u64, __os_k1080_backend_identity as *const () as u64] { // Retain source/ELF addresses without reading an unmapped guard or an // absent/uninitialized owner payload. No typed owner is borrowed. core::hint::black_box(address); } } #[inline(never)] fn os_k1080_preflight(runtime:&mut OsK660Runtime,tables:&mmu::Tables)->Result<(),&'static str> { os_k1080_main_gate(); let (root,tcr,vbar,daif)=os_k940_registers(); if root!=tables.root || tcr&63!=25 || daif&128==0 || os_k960_timer().1&1!=0 { return Err("wait-predecessor-control"); } // The genuine pre-allocation checkpoint runs on the restored K900 stack, // before our primary guard, IRQ transfer or owner construction changes. // The allocator loan ends before the checkpoint and its ASM handoff. { let (_,memory,_,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?; if tasks.scheduler.running()!=tasks.id {return Err("wait-alpha-owner");} os_k1080_inventory(memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1080_AUTHORITY_BEFORE).cast()); os_k1080_word(22,memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64); os_k1080_word(23,core::mem::size_of::() as u64); } let guard=core::ptr::addr_of!(OS_K1080_PRIMARY_STACK) as u64; let (slot,descriptor)=os_k1080_mapped_guard_slot(tables,guard)?; unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1080_PRIMARY_GUARD_BEFORE),[slot,descriptor,guard,guard+4096]); // The addresses come from the retained actual kernel table inventory. // The harness captures their current physical bytes independently. for i in 0..32 { write_volatile(core::ptr::addr_of_mut!(OS_K1080_KERNEL_TABLES).cast::().add(i), read_volatile(core::ptr::addr_of!(OS_K1020_KERNEL_TABLES).cast::().add(i))); } } for(i,v)in [(0,0x4b31303830574149),(1,root),(2,tcr),(3,vbar),(4,daif),(20,27), (21,unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT))})] {os_k1080_word(i,v);} os_k1080_checkpoint(0,false); Ok(()) } fn os_k1100_product_probe_mode(boot:&BootData)->Result{ let blob=unsafe{core::slice::from_raw_parts(boot.dtb_address as *const u8,boot.dtb_size)}; let scratch=unsafe{&mut *core::ptr::addr_of_mut!(OS_K700_WORKSPACE)}; let mode=turkos_kernel::runtime::platform::chosen_u32(blob,scratch,b"turkos,policy-probe-mode") .map_err(|_|"policy-probe-mode")?.unwrap_or(0); if mode>128 {return Err("policy-probe-mode");} Ok(mode) } fn os_k1100_product_probe_enabled(boot:&BootData)->Result{ Ok(os_k1100_product_probe_mode(boot)?!=0) } #[inline(never)] fn os_k1080_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str> { os_k1080_retain_configuration(); if cfg!(feature="os-runtime-product") && !os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K1080: PRODUCT-CONTROL NO-WAIT-PROBE");return Ok(()); } let (root,tcr,vbar,daif)=os_k940_registers(); if root!=tables.root || tcr&63!=25 || daif&128==0 || os_k960_timer().1&1!=0 { return Err("wait-predecessor-control"); } let (_,memory,irq,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?; if tasks.scheduler.running()!=tasks.id {return Err("wait-alpha-owner");} if unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_EVIDENCE).cast::().add(22))} !=memory.allocator.os_authority() as *const os_k720_frame::Allocator as u64 { return Err("wait-preflight-authority"); } let source=OsK1080SourceLease::new(irq,boot.interrupts.gic_redistributor)?; let installation=os_k1080_factory(memory,tables,root); unsafe extern "C" {static __os_k1080_vectors:u8;fn __os_k1080_run(wire:*const OsK1080Wire);} unsafe{asm!("msr VBAR_EL1, {v}","isb",v=in(reg)core::ptr::addr_of!(__os_k1080_vectors) as u64,options(nomem,nostack,preserves_flags));} os_k1080_checkpoint(1,false); for _ in 0..3 { let actual=unsafe{(&mut *os_k1080_core()).os_k1080_main_select()}.unwrap_or_else(|_|park()); let wire=os_k1080_depart_normal(actual); unsafe{__os_k1080_run(wire);} os_k1080_checkpoint(2,false); } os_k1080_quiesce(7); let winners=unsafe{(&*os_k1080_core()).winners()}; if winners!=4 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_IRQ_COUNT))}!=2 || unsafe{(&*os_k1080_ingress()).pending()}!=0 || (0..3).any(|i|unsafe{(&*core::ptr::addr_of!(OS_K1080_TOKENS).cast::>().add(i)).is_some()}) || (0..3).any(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_TASK_ENTRIES).cast::().add(i))}!=1) {park();} os_k1080_word(24,winners); os_k1080_checkpoint(3,false); for ordinal in 0..3 { os_k1080_quiesce(8); match unsafe{(&mut *os_k1080_core()).os_k1080_main_retire(ordinal)} { Ok(())=>{},Err((_,actual))=>{ if let Some(actual)=actual {unsafe{core::ptr::addr_of_mut!(OS_K1080_MEMBERS) .cast::>().add(ordinal).write(Some(actual));}} park(); } } os_k1080_event(0x73,ordinal as u64,0,0); } os_k1080_checkpoint(4,false); let backend=installation.extract(); let stack=match backend.table.into_backend() {Ok(b)=>b,Err((_,actual))=>os_k1080_retain_failure(actual,0x41)}; os_k1080_inventory(stack.memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1080_AUTHORITY_AFTER).cast()); if !os_k920_inventory_equal(core::ptr::addr_of!(OS_K1080_AUTHORITY_BEFORE).cast(), core::ptr::addr_of!(OS_K1080_AUTHORITY_AFTER).cast()) {park();} unsafe{asm!("msr VBAR_EL1, {v}","isb",v=in(reg)vbar,options(nomem,nostack,preserves_flags));} drop(stack); drop(source); // Actual nonclone predecessor IRQ transfer restores only now. if os_k940_registers()!=(root,tcr,vbar,daif) {park();} os_k1080_checkpoint(5,false); write_line("TURKOS/OS/K1080: DRAFT-OWNERS-RESTORED NO-CANONICAL-CLAIM"); Ok(()) } #[unsafe(no_mangle)] static OS_K1080_CONTEXT_LAYOUT_EXPECTED: [u64; 40] = os_k1080_context::os_k1040_layout::, 3>(); #[unsafe(no_mangle)] static OS_K1080_THREAD_ID_LAYOUT_EXPECTED: [u64; 4] = os_k1080_thread::ThreadId::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1080_THREAD_TABLE_LAYOUT_EXPECTED: [u64; 20] = OsK1080Table::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1080_LEASE_LAYOUT_EXPECTED: [u64; 7] = os_k720_frame::Lease::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1080_WAIT_LAYOUT_EXPECTED: [u64; 5] = os_k1080_thread::Wait::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1080_BACKEND_LAYOUT_EXPECTED: [u64; 30] = [ core::mem::size_of::>() as u64, core::mem::offset_of!(OsK1080ContextBackend<'static>, table) as u64, core::mem::offset_of!(OsK1080ContextBackend<'static>, bootstrap) as u64, core::mem::offset_of!(OsK1080ContextBackend<'static>, pins) as u64, core::mem::offset_of!(OsK1080ContextBackend<'static>, sequence) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1080Binding, id) as u64, core::mem::offset_of!(OsK1080Binding, ordinal) as u64, core::mem::offset_of!(OsK1080Binding, root) as u64, core::mem::offset_of!(OsK1080Binding, root_generation) as u64, core::mem::offset_of!(OsK1080Binding, branches) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1080Stack, root) as u64, core::mem::offset_of!(OsK1080Stack, ordinal) as u64, core::mem::offset_of!(OsK1080Stack, start) as u64, core::mem::offset_of!(OsK1080Stack, leases) as u64, core::mem::offset_of!(OsK1080Stack, guard) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1080Root, lease) as u64, core::mem::offset_of!(OsK1080Root, bootstrap) as u64, core::mem::offset_of!(OsK1080Root, branches) as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::size_of::>() as u64, core::mem::offset_of!(OsK1080ContextBackend<'static>, current) as u64, core::mem::offset_of!(OsK1080ContextBackend<'static>, sleep_wait) as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1080IdlePrepared, epoch) as u64, core::mem::offset_of!(OsK1080IdlePrepared, wait) as u64, ]; #[unsafe(no_mangle)] static OS_K1080_TRACE_BACKEND_LAYOUT: [u64;16]=[ core::mem::size_of::() as u64, core::mem::offset_of!(OsK1080Prepared,source) as u64, core::mem::offset_of!(OsK1080Prepared,target) as u64, core::mem::offset_of!(OsK1080Prepared,nonce) as u64, core::mem::offset_of!(OsK1080Prepared,departed) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1080ReturnPrepared,source) as u64, core::mem::offset_of!(OsK1080ReturnPrepared,bootstrap_root) as u64, core::mem::offset_of!(OsK1080ReturnPrepared,bootstrap) as u64, core::mem::offset_of!(OsK1080ReturnPrepared,address) as u64, core::mem::offset_of!(OsK1080ReturnPrepared,nonce) as u64, 0,0,0,0,0, ]; #[unsafe(no_mangle)] static OS_K1080_WAIT_GRAPH_LAYOUT: [u64; 48] = os_k1080_wait::os_k1080_wait_graph_layout::>(); #[unsafe(no_mangle)] static OS_K1080_INGRESS_GRAPH_LAYOUT: [u64; 24] = os_k1080_wait::os_k1080_ingress_graph_layout::(); #[unsafe(no_mangle)] static OS_K1080_BRIDGE_GRAPH_LAYOUT: [u64; 32] = os_k1080_bridge::os_k1080_wait_bridge_layout::, OsK1080Port>(); #[unsafe(no_mangle)] static OS_K1080_RETURN_LAYOUT: [u64; 40] = os_k1080_queue::os_k1080_return_layout::, 3>(); #[unsafe(no_mangle)] static OS_K1080_QUEUE_BACKEND_LAYOUT: [u64; 24] = os_k1080_backend_layout::>(); #[unsafe(no_mangle)] static OS_K1080_TRACE_QUEUE_LAYOUT: [u64; 24] = os_k1080_queue::os_k1080_trace_queue_layout::, 3>(); #[unsafe(no_mangle)] static OS_K1080_TRACE_CONTEXT_LAYOUT: [u64; 38] = os_k1080_context::os_k1080_trace_context_layout::>(); #[unsafe(no_mangle)] static OS_K1080_SIGNAL_PIN_LAYOUT: [u64; 4] = [ core::mem::size_of::() as u64, core::mem::offset_of!(OsK1080SignalPin, source) as u64, core::mem::offset_of!(OsK1080SignalPin, slot) as u64, core::mem::offset_of!(OsK1080SignalPin, claim) as u64, ]; #[unsafe(no_mangle)] static OS_K1080_CONTEXT_OPTION_TAGS: [u64; 20] = os_k1080_context::os_k1080_context_option_tags::>(); #[unsafe(no_mangle)] static OS_K1080_QUEUE_OPTION_TAGS: [u64; 12] = os_k1080_queue::os_k1080_queue_option_tags::>(); const fn os_k1080_main_none_word(value: T, offset: usize) -> u64 { let value=core::mem::ManuallyDrop::new(value); unsafe {core::ptr::addr_of!(value).cast::().add(offset).cast::().read_unaligned()} } #[unsafe(no_mangle)] static OS_K1080_BACKEND_OPTION_TAGS:[u64;4]=[ os_k1080_main_none_word(None::,0), os_k1080_main_none_word(None::,0), os_k1080_main_none_word(None::,0), os_k1080_main_none_word(None::,0), ]; #[unsafe(no_mangle)] static OS_K1080_CAPTURE_ABI_TEMPLATE: os_k1080_context::Abi = os_k1080_context::Abi::COOPERATIVE; #[unsafe(no_mangle)] static OS_K1080_RETAINED_NONE_TEMPLATE: Option< os_k1080_wait::Retained, os_k1080_bridge::WaitPrepared>, > = None; #[unsafe(no_mangle)] static OS_K1080_WAIT_PREPARED_NONE_TEMPLATE: Option = None; #[unsafe(no_mangle)] static OS_K1080_SIGNAL_PIN_NONE_TEMPLATE: Option = None; #[unsafe(no_mangle)] static OS_K1080_MEMBER_NONE_TEMPLATE: Option = None; #[unsafe(no_mangle)] static OS_K1080_IMAGE_TEMPLATES: [os_k1080_context::Image; 2] = [ os_k1080_context::Image::Initial { entry: 0x4b31303830494e31, argument: 0x4b31303830494e32, sp: 0x4b31303830494e33, return_pc: 0x4b31303830494e34, }, os_k1080_context::Image::Cooperative(os_k1080_context::Cooperative::EMPTY), ]; fn os_k1080_retain_layouts() { unsafe extern "C" {static OS_K1080_INGRESS_EVENT_NONE_TEMPLATE:u8;} for address in [core::ptr::addr_of!(OS_K1080_WAIT_GRAPH_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_INGRESS_GRAPH_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_BRIDGE_GRAPH_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_RETURN_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_QUEUE_BACKEND_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_TRACE_QUEUE_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_TRACE_CONTEXT_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_SIGNAL_PIN_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_CONTEXT_OPTION_TAGS) as u64,core::ptr::addr_of!(OS_K1080_QUEUE_OPTION_TAGS) as u64,core::ptr::addr_of!(OS_K1080_BACKEND_OPTION_TAGS) as u64,core::ptr::addr_of!(OS_K1080_CAPTURE_ABI_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1080_RETAINED_NONE_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1080_WAIT_PREPARED_NONE_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1080_SIGNAL_PIN_NONE_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1080_MEMBER_NONE_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1080_IMAGE_TEMPLATES) as u64,core::ptr::addr_of!(OS_K1080_CONTEXT_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1080_THREAD_ID_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1080_THREAD_TABLE_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1080_LEASE_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1080_WAIT_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1080_BACKEND_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1080_TRACE_BACKEND_LAYOUT) as u64,core::ptr::addr_of!(OS_K1080_INGRESS_EVENT_NONE_TEMPLATE) as u64] {core::hint::black_box(address);} } #[repr(C, align(4096))] struct OsK1080PrimaryStack { guard: [u8; 4096], usable: [u8; 65536], } #[unsafe(no_mangle)] static mut OS_K1080_PRIMARY_STACK: OsK1080PrimaryStack = OsK1080PrimaryStack { guard: [0; 4096], usable: [0; 65536], }; #[unsafe(no_mangle)] static mut OS_K1080_PRIMARY_STACK_EVIDENCE: [u64; 24] = [0; 24]; static mut OS_K1080_PRIMARY_ERROR: Option<&'static str> = None; const OS_K1080_PRIMARY_LOW_CANARY: u64 = 0x4b31_3034_4c4f_5721; const OS_K1080_PRIMARY_HIGH_CANARY: u64 = 0x4b31_3034_4849_4748; fn os_k1080_primary_stack_word(slot: usize, value: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_PRIMARY_STACK_EVIDENCE) .cast::() .add(slot), value, ) }; } #[inline(never)] fn os_k1080_on_stack( runtime: &mut OsK660Runtime, tables: &mmu::Tables, boot: &BootData, ) -> Result<(), &'static str> { if cfg!(feature = "os-runtime-product") && !os_k1100_product_probe_enabled(boot)? { write_line("TURKOS/OS/K1080: PRODUCT-CONTROL NO-WAIT-PROBE"); return Ok(()); } let guard = core::ptr::addr_of!(OS_K1080_PRIMARY_STACK) as u64; let low = guard + 4096; let end = low + 65536; let high = end - 8; let before_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)before_sp,options(nomem,nostack)); } let original_guard = core::ptr::addr_of!(OS_K900_STACK) as u64; let original_low = original_guard + 4096; let original_top = original_low + 65536; if before_sp <= original_low + 8 || before_sp >= original_top { return Err("context-primary-original-stack-range"); } if before_sp - (original_low + 8) < 6144 { return Err("runqueue-primary-original-stack-headroom"); } os_k1080_primary_stack_word(22, 6144); os_k1080_primary_stack_word(23, before_sp); let original_canary = unsafe { read_volatile(original_low as *const u64) }; let original_second = unsafe { read_volatile((original_top - 8) as *const u64) }; if original_canary != 0x4b38_3230_4c4f_5721 || original_second != 0x4b38_3230_4849_4748u64 { return Err("context-primary-original-stack-canaries"); } os_k1080_preflight(runtime,tables)?; unsafe { tables.unmap_metadata_stack_guard( guard, low, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64, ) } .map_err(|_| "context-primary-stack-guard")?; if !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("context-primary-stack-guard-descriptor"); } unsafe { write_volatile(low as *mut u64, OS_K1080_PRIMARY_LOW_CANARY); write_volatile(high as *mut u64, OS_K1080_PRIMARY_HIGH_CANARY) }; for (slot, value) in [ (1, low), (2, end - 16), (7, 0), (8, OS_K1080_PRIMARY_LOW_CANARY), (9, OS_K1080_PRIMARY_HIGH_CANARY), (10, original_low), (11, original_canary), (13, guard), (14, low), (15, high), (16, original_second), ( 18, tables .metadata_stack_guard_descriptor_address(guard) .ok_or("context-primary-guard-slot")?, ), ( 19, tables .metadata_stack_guard_descriptor_address(original_low - 4096) .ok_or("context-primary-boot-guard-slot")?, ), (20, original_low), (21, original_top), ] { os_k1080_primary_stack_word(slot, value); } unsafe extern "C" { fn __os_k1080_primary_on_stack(runtime: *mut u8, tables: *const u8, boot: *const u8) -> u64; } let result = unsafe { __os_k1080_primary_on_stack( (runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(), (boot as *const BootData).cast(), ) }; let after_sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)after_sp,options(nomem,nostack)); } let state = unsafe { &*core::ptr::addr_of!(OS_K1080_PRIMARY_STACK_EVIDENCE) }; if before_sp != after_sp || state[0] != state[6] || state[3] != end - 16 || state[4] <= low + 8 || state[4] > end - 16 || state[5] <= low + 8 || state[5] > end - 16 || unsafe { read_volatile(low as *const u64) } != OS_K1080_PRIMARY_LOW_CANARY || unsafe { read_volatile(high as *const u64) } != OS_K1080_PRIMARY_HIGH_CANARY || unsafe { read_volatile(original_low as *const u64) } != original_canary || unsafe { read_volatile((original_top - 8) as *const u64) } != original_second || !tables.task_stack_guard_unmapped(guard) || !tables.task_stack_guard_unmapped(original_low - 4096) { return Err("context-primary-stack-restoration"); } os_k1080_primary_stack_word(12, unsafe { read_volatile(original_low as *const u64) }); os_k1080_primary_stack_word(17, unsafe { read_volatile((original_top - 8) as *const u64) }); if result == 0 { write_line( "TURKOS/OS/K1080: ISOLATED-64K-STACK GUARD-UNMAPPED TWO-CANARIES ORIGINAL-SP-RESTORED PASS", ); os_k1080_restore_primary_guard(tables,guard)?; os_k1080_checkpoint(0x71,false); Ok(()) } else { Err( unsafe { read_volatile(core::ptr::addr_of!(OS_K1080_PRIMARY_ERROR)) } .unwrap_or("context-primary-stack-return"), ) } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1080_primary_body( runtime: *mut u8, tables: *const u8, boot: *const u8, ) -> u64 { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k1080_primary_stack_word(4, sp); let result = os_k1080_accept( unsafe { &mut *runtime.cast::() }, unsafe { &*tables.cast::() }, unsafe { &*boot.cast::() }, ); let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack)); } os_k1080_primary_stack_word(5, sp); match result { Ok(()) => 0, Err(reason) => { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1080_PRIMARY_ERROR), Some(reason), ) }; 1 } } } global_asm!( r#" .section .text.os_k1080_primary_stack,"ax" .balign 4 .global __os_k1080_primary_on_stack .type __os_k1080_primary_on_stack, %function __os_k1080_primary_on_stack: adrp x9, OS_K1080_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1080_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9] stp x19, x30, [sp, #-16]! mov x19, sp adrp x9, OS_K1080_PRIMARY_STACK add x9, x9, :lo12:OS_K1080_PRIMARY_STACK add x9, x9, #4096 mov x10, #65536 add x9, x9, x10 sub x9, x9, #16 mov sp, x9 adrp x9, OS_K1080_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1080_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9, #24] bl __os_k1080_primary_body mov sp, x19 ldp x19, x30, [sp], #16 adrp x9, OS_K1080_PRIMARY_STACK_EVIDENCE add x9, x9, :lo12:OS_K1080_PRIMARY_STACK_EVIDENCE mov x10, sp str x10, [x9, #48] ret .size __os_k1080_primary_on_stack, .-__os_k1080_primary_on_stack "# ); global_asm!(include_str!("os_k1080_wait.S")); const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k1080_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); // NEW observational readiness barrier. Frozen Deferred owns all resources; // this function owns only its private gate byte and scalar observations. #[unsafe(no_mangle)] static mut OS_K1080_BATCH_GATES: [u8; 11] = [0; 11]; #[unsafe(no_mangle)] static mut OS_K1080_BATCH_EVIDENCE: [[u64; 48]; 11] = [[0; 48]; 11]; #[unsafe(no_mangle)] static mut OS_K1080_BATCH_COUNT: u64 = 0; const OS_K1080_BATCH_TARGETS: [u64; 11] = [3,4,5,6,7,8,9,10,12,15,16]; const OS_K1080_BATCH_BEFORE: [u64; 11] = [0,3,4,5,6,7,8,9,11,12,15]; const OS_K1080_BATCH_EPOCHS: [u64; 11] = [1,2,1,1,1,1,1,1,1,2,2]; const OS_K1080_BATCH_IRQS: [u64; 11] = [0,1,2,3,4,5,6,7,9,10,11]; #[inline(always)] fn os_k1080_deferred_batch_snapshot(ordinal:usize,target:u64,epoch:u64)->[u64;24] { let (el,mpidr,daif,sp):(u64,u64,u64,u64); unsafe {asm!("mrs {el}, CurrentEL","mrs {mpidr}, MPIDR_EL1","mrs {daif}, DAIF","mov {sp}, sp", el=out(reg)el,mpidr=out(reg)mpidr,daif=out(reg)daif,sp=out(reg)sp, options(nomem,nostack,preserves_flags));} // Pure hardware gate precedes all retained source scalar/address reads. if el!=4 || mpidr&0xff00_ffff_ff!=0 || daif&0x80==0 {park();} if unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}!=0 || os_k1000_state(1)!=0 || os_k1000_state(0)!=1 || os_k1000_state(7)!=0 {park();} let (root,tcr,vbar,observed_daif)=os_k940_registers(); if observed_daif!=daif || vbar!=os_k1000_state(14) || sp<=os_k1000_state(11)+848 || sp>=os_k1000_state(12)-16 {park();} let (used,payload)=(os_k1000_state(24),os_k1000_state(26)); if used==0 || used%4096!=0 || payload==0 || payload%4096!=0 || os_k1000_state(2)!=epoch || os_k1000_state(5)!=OS_K1080_BATCH_BEFORE[ordinal] || os_k1000_state(6)!=OS_K1080_BATCH_IRQS[ordinal] {park();} let actual_used=unsafe{u16::from_le(read_volatile((used+2) as *const u16))} as u64; [0x4b31303830424154,ordinal as u64,target,epoch,el,mpidr,daif,sp, root,tcr,vbar,used,payload,os_k1000_state(5),os_k1000_state(6),actual_used, os_k1000_state(1),unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))} as u64, os_k1000_state(7),os_k960_frequency(),os_k960_counter(),os_k1000_state(3), os_k1000_state(4),os_k1000_state(27)] } #[unsafe(no_mangle)] #[inline(never)] extern "C" fn os_k1080_deferred_batch_gate(target:u64,epoch:u64) { // There is no owner pointer/reference or callback across this gate. let ordinal=unsafe{read_volatile(core::ptr::addr_of!(OS_K1080_BATCH_COUNT))} as usize; if ordinal>=11 || target!=OS_K1080_BATCH_TARGETS[ordinal] || epoch!=OS_K1080_BATCH_EPOCHS[ordinal] {park();} let entry=os_k1080_deferred_batch_snapshot(ordinal,target,epoch); if entry[15]!=OS_K1080_BATCH_BEFORE[ordinal] {park();} let gate=unsafe{core::ptr::addr_of_mut!(OS_K1080_BATCH_GATES).cast::().add(ordinal)}; if unsafe{read_volatile(gate)}!=0 {park();} for (i,value) in entry.into_iter().enumerate() { unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_BATCH_EVIDENCE) .cast::().add(ordinal*48+i),value);} } write_str("TURKOS/OS/K1080-BATCH: READY ORDINAL=0x");write_hex(ordinal); write_str(" TARGET=0x");write_hex(target as usize); write_str(" EPOCH=0x");write_hex(epoch as usize);write_line(" I1"); let end=os_k960_counter().checked_add(os_k960_frequency().checked_mul(30) .unwrap_or_else(||park())).unwrap_or_else(||park()); loop { let observed=unsafe{read_volatile(gate)}; if observed==1 {break;} if observed!=0 || os_k960_counter()>=end {park();} core::hint::spin_loop(); } unsafe{asm!("dmb oshld",options(nostack,preserves_flags));} let exit=os_k1080_deferred_batch_snapshot(ordinal,target,epoch); if exit[15]!=target || exit[20]().add(ordinal*48+24+i),value);} } unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1080_BATCH_COUNT),ordinal as u64+1);} // Original wait_rx subsequently unmasks and handles the real interrupt. } use turkos_kernel::runtime::{policy as os_k1100_policy,policy_owner as os_k1100_owner,policy_wait_core as os_k1100_wait,policy_wait_bridge as os_k1100_bridge}; // Actual architecture gate for the separately installed policy episode. // All published values below are private scalar copies from retained owners. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(C)] struct OsK1100Registers { cpu: u64, daif: u64, current_el: u64, ttbr: u64, tcr: u64, sctlr: u64, spsel: u64, sp: u64, cpacr: u64, irq_active: bool, } #[derive(Clone, Copy)] #[repr(C)] struct OsK1100ExpectedExecution { identity: u64, stack_owner: u64, stack_low: u64, stack_high: u64, registers: OsK1100Registers, current: Option, } #[unsafe(no_mangle)] static mut OS_K1100_EXPECTED_EXECUTION: core::mem::MaybeUninit = core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_EXPECTED_VALID: u64 = 0; fn os_k1100_registers() -> OsK1100Registers { let (mpidr, daif, current_el, ttbr, tcr, sctlr, spsel, sp, cpacr): (u64,u64,u64,u64,u64,u64,u64,u64,u64); unsafe { asm!( "mrs {mpidr}, MPIDR_EL1", "mrs {daif}, DAIF", "mrs {current_el}, CurrentEL", "mrs {ttbr}, TTBR0_EL1", "mrs {tcr}, TCR_EL1", "mrs {sctlr}, SCTLR_EL1", "mrs {spsel}, SPSel", "mov {sp}, sp", "mrs {cpacr}, CPACR_EL1", mpidr=out(reg)mpidr,daif=out(reg)daif,current_el=out(reg)current_el, ttbr=out(reg)ttbr,tcr=out(reg)tcr,sctlr=out(reg)sctlr,spsel=out(reg)spsel, sp=out(reg)sp,cpacr=out(reg)cpacr, options(nomem,nostack,preserves_flags)); } OsK1100Registers { cpu: (mpidr & 0x00ff_ffff) | (((mpidr >> 32) & 255) << 24), daif,current_el,ttbr,tcr,sctlr,spsel,sp,cpacr, irq_active: unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0, } } fn os_k1100_gate_execution(irq: bool) -> Result { os_k1100_gate_kind(irq,false) } fn os_k1100_gate_kind(irq: bool,vector_entry:bool) -> Result { // No owner, expected-record pointer, or ingress loan precedes this gate. let r=os_k1100_registers(); if r.cpu!=0 || r.current_el!=4 || r.daif&128==0 || r.irq_active!=irq || r.cpacr&(3<<20)!=(3<<20) || !r.sp.is_multiple_of(16) { return Err(os_k1100_owner::Error::Context); } if unsafe {read_volatile(core::ptr::addr_of!(OS_K1100_EXPECTED_VALID))}!=1 { return Err(os_k1100_owner::Error::Context); } let expected=unsafe {read_volatile(core::ptr::addr_of!(OS_K1100_EXPECTED_EXECUTION).cast::())}; let old=expected.registers; // The source-bound timer wait moves the same retained primary stack to // SP_EL1 before unmasking. Task waits already use SP_EL1. No owner pointer // is read to authorize this IRQ-only architectural difference. let wanted_spsel=if irq||vector_entry { if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_STACK_VALID))}!=1 { return Err(os_k1100_owner::Error::Context); } 1 } else {old.spsel}; if expected.identity==0 || expected.stack_owner==0 || expected.stack_low>=expected.stack_high || r.spexpected.stack_high || r.ttbr!=old.ttbr || r.tcr!=old.tcr || r.sctlr!=old.sctlr || r.spsel!=wanted_spsel || r.daif!=old.daif || r.cpacr!=old.cpacr { return Err(os_k1100_owner::Error::Context); } Ok(expected) } struct OsK1100MainGate; unsafe impl os_k1100_owner::Gate for OsK1100MainGate { fn validate(&self)->Result<(),os_k1100_owner::Error> { os_k1100_gate_execution(false).map(|_|()) } fn execution(&self)->os_k1100_policy::Execution { match os_k1100_gate_execution(false) { Ok(e)=>os_k1100_policy::Execution {cpu:0,masked:true,in_irq:false,current:e.current}, Err(_)=>os_k1100_policy::Execution {cpu:u32::MAX,masked:false,in_irq:true,current:None}, } } fn identity(&self)->u64 { os_k1100_gate_execution(false).map_or(0,|e|e.identity) } } /// # Safety /// binding came from Driver::k1100_target while it retained the actual owned /// Queue selection. No asynchronous owner loan or actual departure is active. unsafe fn os_k1100_publish_target(binding:turkos_kernel::runtime::policy_main::TargetBinding) ->Result<(),&'static str> { let current=os_k1100_gate_execution(false).map_err(|_|"policy-target-context")?; let stack=binding.stack; if binding.root==0 || stack.physical!=stack.base || stack.bytes==0 || stack.usable_bottom()>=stack.usable_top() || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_VALID))}!=0 { return Err("policy-target-owner"); } let mut registers=current.registers;registers.ttbr=binding.root;registers.spsel=1; let expected=OsK1100ExpectedExecution{identity:current.identity,stack_owner:stack.physical, stack_low:stack.usable_bottom(),stack_high:stack.usable_top(),registers,current:None}; unsafe{ core::ptr::addr_of_mut!(OS_K1100_PENDING_EXECUTION).cast::().write(expected); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PENDING_THREAD),Some(binding.thread)); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PENDING_VALID),1); } Ok(()) } fn os_k1100_root_departure(root:u64)->Result<(),&'static str> { let mut current=os_k1100_gate_execution(false).map_err(|_|"policy-departure-context")?; if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_VALID))}!=1 { return Err("policy-departure-owner"); } let target=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_EXECUTION).cast::())}; let thread=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_THREAD))}; let boot=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_NATIVE_ROLE))}==2; if target.identity!=current.identity || target.registers.ttbr!=root || thread.is_none() && !boot { return Err("policy-departure-binding"); } // Called only by the actual Context departure after Table.yield/sleep has // completed. The original caller stack remains live until ASM restores // the real retained target image; no requested Thread becomes current. current.registers.ttbr=root;current.current=None; unsafe{core::ptr::addr_of_mut!(OS_K1100_EXPECTED_EXECUTION) .cast::().write(current);} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_PUBLISHED_CURRENT),None);} Ok(()) } fn os_k1100_root_changed(root:u64)->Result<(),&'static str> { if os_k1100_registers().ttbr!=root {return Err("policy-departure-readback");} os_k1100_gate_execution(false).map(|_|()).map_err(|_|"policy-departure-post") } // Entry checks pure immutable pending scalars and physical address bounds // before any owner reference or dereference of the incoming640-byte frame. fn os_k1100_admit_landing(address:u64)->Result<(),&'static str> { let raw=os_k1100_registers(); if raw.cpu!=0 || raw.current_el!=4 || raw.daif&128==0 || raw.irq_active || raw.cpacr&(3<<20)!=(3<<20) || !raw.sp.is_multiple_of(16) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_VALID))}!=1 { return Err("policy-landing-context"); } let target=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_EXECUTION).cast::())}; let old=target.registers; if target.identity==0 || target.stack_owner==0 || target.current.is_some() || raw.ttbr!=old.ttbr || raw.tcr!=old.tcr || raw.sctlr!=old.sctlr || raw.daif!=old.daif || raw.spsel!=old.spsel || raw.cpacr!=old.cpacr || raw.sptarget.stack_high || address%16!=0 || addresstarget.stack_high) { return Err("policy-landing-owner"); } unsafe{core::ptr::addr_of_mut!(OS_K1100_EXPECTED_EXECUTION) .cast::().write(target);} os_k1100_gate_execution(false).map(|_|()).map_err(|_|"policy-landing-post") } fn os_k1100_publish_activated(actual:os_k1100_thread::ThreadId)->Result<(),os_k1100_context::Error> { let mut expected=os_k1100_gate_execution(false).map_err(|_|os_k1100_context::Error::Context)?; if expected.current.is_some() || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_THREAD))}!=Some(actual) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_VALID))}!=1 { return Err(os_k1100_context::Error::Context); } // Actual physical validation and Table.activate succeeded before this // call. This is the full identity from that real binding, never a hint. expected.current=Some(actual); unsafe{ core::ptr::addr_of_mut!(OS_K1100_EXPECTED_EXECUTION).cast::().write(expected); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PUBLISHED_CURRENT),Some(actual)); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PENDING_VALID),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PENDING_THREAD),None); } Ok(()) } // Distinct private hardware custody. None of these pointers names a scheduler. #[derive(Clone,Copy)] #[repr(C)] struct OsK1100SourceStamp { issuer:u64, sequence:u64, counter:u64, queue_epoch:u64, slice:u64, current:Option, root:u64, stack_owner:u64, stack_low:u64, stack_high:u64, deadline:u64, } struct OsK1100QueueTick { issuer:u64, observation:turkos_kernel::runtime::policy_wait_queue::TickObservation, } struct OsK1100CompletedDelivery { stamp:OsK1100SourceStamp, frame:os_k920_exception::Frame, frame_address:u64, iar:u64, split:u64, queue_tick:Option, } #[unsafe(no_mangle)] static mut OS_K1100_SOURCE_PENDING:Option=None; #[unsafe(no_mangle)] static mut OS_K1100_KERNEL_FRAME:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_QUEUE_TICK:Option=None; #[unsafe(no_mangle)] static mut OS_K1100_QUEUE_TICK_LIVE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_SOURCE_ISSUER:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_SOURCE_LIVE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_REDIS:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_IRQ_COUNT:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_IRQ_ACTIVE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_TIMER_ARMED:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_TIMER_STAMP:core::mem::MaybeUninit=core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_LAST_IRQ_FRAME:os_k920_exception::Frame=OS_K920_ZERO_FRAME; #[unsafe(no_mangle)] static mut OS_K1100_LAST_IRQ_ADDRESS:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_SOURCE_RETIRED:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_SOURCE_FAILED:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_DELIVERY_CLAIMS:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_REGISTRATION_CLAIM:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_OWNERS_LIVE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_IRQ_STACK_VALID:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_RETAINED_REDIS:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_ARCHIVE_HARDWARE:[u64;22]=[0;22]; // The first three words preserve the caller's stack selector and pointers. // The remaining words are bounded live timer diagnostics for a stalled wait. #[unsafe(no_mangle)] static mut OS_K1100_TIMER_STACK_REGISTERS:[u64;10]=[0;10]; struct OsK1100RegistrationClaim<'a> { issuer:u64, // Lifetimes prevent retiring the main registration while this claim lives; // no Runtime, source-lease, Queue or allocator pointer is stored in IRQ. registration:core::marker::PhantomData<&'a ()>, } type OsK1100MainOwner<'a>=turkos_kernel::runtime::policy_main::MainOwner< OsK1100Backend<'a>,OsK1100Port,OsK1100Port>; type OsK1100OwnerCell<'a>=os_k1100_owner::OwnerCell,OsK1100MainGate>; #[unsafe(no_mangle)] static mut OS_K1100_OWNER:core::mem::MaybeUninit>= core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_PENDING_EXECUTION:core::mem::MaybeUninit= core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_PENDING_THREAD:Option=None; #[unsafe(no_mangle)] static mut OS_K1100_PENDING_VALID:u64=0; #[derive(Clone,Copy,PartialEq,Eq)] struct OsK1100SourceSnapshot { registers:OsK1100Registers, identity:u64, stack_owner:u64, stack_low:u64, stack_high:u64, current:Option, } fn os_k1100_source_snapshot()->Result { let r=os_k1100_registers(); let expected=os_k1100_gate_execution(r.irq_active).map_err(|_|turkos_kernel::runtime::policy_ingress::Error::Context)?; let mut registers=os_k1100_registers(); if registers.cpu!=r.cpu || registers.irq_active!=r.irq_active || registers.daif!=r.daif || registers.ttbr!=r.ttbr || registers.tcr!=r.tcr || registers.sctlr!=r.sctlr || registers.spsel!=r.spsel || registers.cpacr!=r.cpacr || registers.current_el!=r.current_el || registers.spexpected.stack_high || !registers.sp.is_multiple_of(16) { return Err(turkos_kernel::runtime::policy_ingress::Error::Context); } // Actual SP was validated above. Call-depth differences are not custody // changes; the retained bounds/owner remain in this complete snapshot. registers.sp=0; Ok(OsK1100SourceSnapshot{registers,identity:expected.identity,stack_owner:expected.stack_owner, stack_low:expected.stack_low,stack_high:expected.stack_high,current:expected.current}) } const OS_K1100_PA_MASK: u64 = 0x0000_ffff_ffff_f000; /// Owns only the new root page. Kernel branches remain separately owned by the /// inherited Tables and are validated as EL0 inaccessible before sharing. struct OsK1100Root { lease: Option, bootstrap: u64, branches: [u64; 2], } impl OsK1100Root { fn address(&self) -> u64 { self.lease.as_ref().map_or(0, |l| l.address()) } fn generation(&self) -> u64 { self.lease.as_ref().map_or(0, |l| l.generation()) } fn validate(&self, memory: &OsK660Memory, tables: &mmu::Tables) -> Result<(), &'static str> { let lease = self.lease.as_ref().ok_or("context-root-token")?; if self.bootstrap != tables.root || lease.address() == tables.root || lease.address() % 4096 != 0 || lease.generation() == 0 { return Err("context-root-owner"); } memory .allocator .os_authority() .validate_lease(lease) .map_err(|_| "context-root-generation")?; os_k1100_kernel_branches(tables, self.branches)?; os_k1100_root_words(lease.address(), self.branches) } } fn os_k1100_shared_walk( tables: &mmu::Tables, address: u64, level: u8, visited: &mut [u64; 4], count: usize, ) -> Result<(), &'static str> { if !(1..=3).contains(&level) || count >= 4 || visited[..count].contains(&address) || !tables.tagged_frames().any(|frame| frame.address == address) { return Err("context-shared-kernel-owner"); } visited[count] = address; for index in 0..512 { let raw = unsafe { read_volatile((address as *const u64).add(index)) }; if raw == 0 { continue; } match raw & 3 { 3 if level < 3 => { if raw != (raw & OS_K1100_PA_MASK) | 3 { return Err("context-shared-kernel-table-flags"); } os_k1100_shared_walk( tables, raw & OS_K1100_PA_MASK, level + 1, visited, count + 1, )?; } 1 if level < 3 => { if raw & (1 << 6) != 0 { return Err("context-shared-kernel-user-block"); } } 3 if level == 3 => { if raw & (1 << 6) != 0 { return Err("context-shared-kernel-user-page"); } } _ => return Err("context-shared-kernel-descriptor"), } } Ok(()) } fn os_k1100_root_words(address: u64, branches: [u64; 2]) -> Result<(), &'static str> { if address == 0 || address & !OS_K1100_PA_MASK != 0 { return Err("context-root-shape-address"); } // The caller has already checked the live typed root owner (or the // immutable pin established from that owner before IRQ unmask). Check the // entire page before accepting any later observation or root selection. for index in 0..512 { let expected = branches.get(index).copied().unwrap_or(0); if unsafe { read_volatile((address as *const u64).add(index)) } != expected { return Err("context-root-shape"); } } Ok(()) } fn os_k1100_kernel_branches(tables: &mmu::Tables, branches: [u64; 2]) -> Result<(), &'static str> { // Establish the inherited root ownership before forming its physical // pointer. Child table ownership is established independently at each // recursive step, before its pointer is formed too. if !tables .tagged_frames() .any(|frame| frame.address == tables.root) { return Err("context-shared-root-owner"); } for (index, raw) in branches.into_iter().enumerate() { if unsafe { read_volatile((tables.root as *const u64).add(index)) } != raw { return Err("context-shared-root-changed"); } match raw & 3 { 0 if raw == 0 => {} 3 => { if raw != (raw & OS_K1100_PA_MASK) | 3 { return Err("context-shared-root-table-flags"); } os_k1100_shared_walk(tables, raw & OS_K1100_PA_MASK, 2, &mut [0; 4], 0)?; } 1 if raw & (1 << 6) == 0 => {} _ => return Err("context-shared-root-descriptor"), } } Ok(()) } fn os_k1100_create_root( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result { let (root, tcr, _, daif) = os_k940_registers(); if root != tables.root || tcr & 63 != 25 || daif & 0x80 == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { return Err("context-root-bootstrap-context"); } // Only kernel L1 branches0/1 are inherited; user branch2 is absent. if !tables.tagged_frames().any(|frame| frame.address == root) { return Err("context-shared-root-owner"); } let mut branches = [0u64; 2]; for (index, raw) in branches.iter_mut().enumerate() { *raw = unsafe { read_volatile((root as *const u64).add(index)) }; } os_k1100_kernel_branches(tables, branches)?; let address = memory .allocator .os_authority() .free_regions() .iter() .rev() .find_map(|region| { region .end .checked_sub(4096) .filter(|address| *address >= region.start) }) .ok_or("context-root-ENOSPC")?; let lease = memory .allocator .os_authority_mut() .allocate_at(address) .map_err(|_| "context-root-acquire")?; let owned = OsK1100Root { lease: Some(lease), bootstrap: root, branches, }; for index in 0..512 { unsafe { write_volatile((address as *mut u64).add(index), 0); } } for (index, raw) in branches.into_iter().enumerate() { unsafe { write_volatile((address as *mut u64).add(index), raw); } } unsafe { asm!("dsb ishst", options(nostack, preserves_flags)); } os_k1100_event(0x41, address, owned.generation(), root); Ok(owned) } fn os_k1100_select_root(root: u64) -> Result<(), &'static str> { let (_, tcr, _, daif) = os_k940_registers(); if root == 0 || root % 4096 != 0 || root & !OS_K1100_PA_MASK != 0 || tcr & 63 != 25 || daif & 0x80 == 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE)) } != 0 { return Err("context-root-select-context"); } unsafe { asm!("dsb ishst", options(nostack, preserves_flags)); } os_k1100_event(0x43, root, 0, 0); os_k1100_root_departure(root)?; unsafe { asm!("msr TTBR0_EL1, {root}",root=in(reg)root,options(nostack,preserves_flags)); } os_k1100_root_changed(root)?; os_k1100_event(0x44, root, 0, 0); unsafe { asm!("tlbi vmalle1is", options(nostack, preserves_flags)); } os_k1100_event(0x45, root, 0, 0); unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } os_k1100_event(0x46, root, 0, 0); unsafe { asm!("isb", options(nostack, preserves_flags)); } os_k1100_event(0x47, root, 0, 0); if os_k940_registers().0 != root { return Err("context-root-readback"); } Ok(()) } fn os_k1100_erase_inactive_root( root: &OsK1100Root, tables: &mmu::Tables, ) -> Result<(), &'static str> { let actual = os_k940_registers(); if actual.0 != tables.root || actual.3 & 0x80 == 0 || root.bootstrap != tables.root || root.address() == actual.0 || root.address() == 0 { return Err("context-root-active-free-refusal"); } // Bootstrap select completed a full TLBI before this operation. Erasure // alone returns no authority: StackBackend publishes one atomic six-lease // return only after its guard restoration and full stack erasure too. for index in 0..512 { unsafe { write_volatile((root.address() as *mut u64).add(index), 0); } } unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } if !(0..512) .all(|index| unsafe { read_volatile((root.address() as *const u64).add(index)) } == 0) { return Err("context-root-erasure"); } os_k1100_event(0x48, root.address(), root.generation(), 0); Ok(()) } use turkos_kernel::runtime::policy_wait_threads as os_k1100_thread; const OS_K1100_STACK_COUNT: usize = 3; const OS_K1100_STACK_PAGES: usize = 7; const OS_K1100_OWNED_PAGES: usize = OS_K1100_STACK_COUNT * (OS_K1100_STACK_PAGES + 2); struct OsK1100Stack { root: OsK1100Root, ordinal: usize, start: u64, leases: [Option; 8], guard: Option, initial_observation: core::cell::Cell, } impl OsK1100Stack { fn layout(&self) -> os_k1100_thread::StackLayout { os_k1100_thread::StackLayout { base: self.start + 4096, physical: self.start + 4096, bytes: OS_K1100_STACK_PAGES as u64 * 4096, } } } fn os_k1100_cpu_root_allowed(root: u64, tables: &mmu::Tables) -> bool { root==tables.root||(0..3).any(|slot|unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_ROOT_ADDRESSES).cast::().add(slot))}==root&&root!=0&&unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_ROOT_LIVE).cast::().add(slot))}==1) } struct OsK1100StackBackend<'a> { memory: &'a mut OsK660Memory, tables: &'a mmu::Tables, trace: bool, } fn os_k1100_execution()->os_k1100_thread::Execution { let raw=os_k1100_registers(); let valid=os_k1100_gate_execution(raw.irq_active).is_ok(); os_k1100_thread::Execution{cpu:if valid {0}else{u32::MAX},irq_masked:valid, in_irq:raw.irq_active,sp:raw.sp} } fn os_k1100_allocate_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ordinal: usize, root: OsK1100Root, ) -> Result { if ordinal >= 3 || root.validate(memory, tables).is_err() { return Err(("context-stack-root-owner", root)); } let authority = memory.allocator.os_authority(); let mut selected = None; let mut checked = 0; for region in authority.free_regions() { let mut address = region.start; while address .checked_add(8 * 4096) .is_some_and(|end| end <= region.end) && checked < 512 { checked += 1; if tables.os_k1020_stack_candidate(address, address + 8 * 4096) { selected = Some(address); break; } let Some(next) = address.checked_add(4096) else { return Err(("context-stack-candidate-overflow", root)); }; address = next; } if selected.is_some() { break; } } let Some(start) = selected else { return Err(("context-stack-no-owned-L3-span", root)); }; let leases = match memory .allocator .os_authority_mut() .allocate_many_at(start, 8) { Ok(leases) => leases, Err(_) => return Err(("context-stack-acquire", root)), }; // The eight live stack leases are immediately inside this actual owner; // failure cannot discard a prefix or the independently retained root. let stack = OsK1100Stack { root, ordinal, start, leases, guard: None, initial_observation: core::cell::Cell::new(false), }; for offset in 0..8 * 4096 / 8 { unsafe { write_volatile((start as *mut u64).add(offset), 0xa5a5a5a5a5a5a5a5); } } for index in 0..9 { let lease = if index < 8 { stack.leases[index].as_ref().unwrap_or_else(|| park()) } else { stack.root.lease.as_ref().unwrap_or_else(|| park()) }; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_OWNED) .cast::() .add(ordinal * 9 + index), lease.address(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1100_LEASE_GENERATIONS) .cast::() .add(ordinal * 9 + index), lease.generation(), ); } os_k1100_event(1, ordinal as u64, lease.address(), lease.generation()); } Ok(stack) } impl OsK1100StackBackend<'_> { fn validate(&self, stack: &OsK1100Stack) -> Result<(), os_k1100_thread::Error> { let e = os_k1100_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || !os_k1100_cpu_root_allowed(os_k940_registers().0, self.tables) { return Err(os_k1100_thread::Error::Context); } if stack.ordinal >= OS_K1100_STACK_COUNT || stack.start % 4096 != 0 || stack.leases[OS_K1100_STACK_PAGES + 1..] .iter() .any(Option::is_some) { return Err(os_k1100_thread::Error::Layout); } for index in 0..=OS_K1100_STACK_PAGES { let lease = stack.leases[index] .as_ref() .ok_or(os_k1100_thread::Error::Backend)?; if lease.address() != stack.start + index as u64 * 4096 { return Err(os_k1100_thread::Error::Layout); } self.memory .allocator .os_authority() .validate_lease(lease) .map_err(|_| os_k1100_thread::Error::Backend)?; } stack .root .validate(self.memory, self.tables) .map_err(|_| os_k1100_thread::Error::Backend)?; Ok(()) } } unsafe impl os_k1100_thread::Backend for OsK1100StackBackend<'_> { type Stack = OsK1100Stack; fn execution(&self) -> os_k1100_thread::Execution { os_k1100_execution() } fn layout( &self, stack: &OsK1100Stack, ) -> Result { self.validate(stack)?; Ok(stack.layout()) } fn entry_executable(&self, entry: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } entry.is_multiple_of(4) && entry >= core::ptr::addr_of!(_text_start) as u64 && entry < core::ptr::addr_of!(_text_end) as u64 } fn prepare(&mut self, stack: &mut OsK1100Stack) -> Result<(), os_k1100_thread::Error> { self.validate(stack)?; let layout = stack.layout(); let e = os_k1100_execution(); if e.sp >= stack.start && e.sp <= layout.base + layout.bytes || stack.guard.is_some() || os_k940_registers().0 != self.tables.root { return Err(os_k1100_thread::Error::ActiveStack); } let guard = unsafe { self.tables .os_k1020_acquire_stack_guard(stack.start, layout.base + layout.bytes) } .map_err(|_| os_k1100_thread::Error::Guard)?; stack.guard = Some(guard); for offset in 0..layout.bytes / 8 { unsafe { write_volatile((layout.base as *mut u64).add(offset as usize), 0); } } unsafe { write_volatile(layout.base as *mut u64, os_k1100_thread::LOW_CANARY); write_volatile( (layout.base + layout.bytes - 8) as *mut u64, os_k1100_thread::HIGH_CANARY, ); asm!("dsb ish", options(nostack, preserves_flags)); } stack.initial_observation.set(true); if self.trace { os_k1100_event(2, stack.ordinal as u64, stack.start, layout.bytes); } Ok(()) } fn observe( &self, stack: &OsK1100Stack, ) -> Result { self.validate(stack)?; let layout = stack.layout(); let token = stack.guard.as_ref().ok_or(os_k1100_thread::Error::Guard)?; let descriptor = self .tables .os_k1020_stack_guard_descriptor(token) .map_err(|_| os_k1100_thread::Error::Guard)?; if token.guard() != stack.start || token.end() != layout.base + layout.bytes { return Err(os_k1100_thread::Error::Layout); } // Initial full erasure is observed once before publication. Later // selection/exit checks read the actual guard and canaries without // walking the active or previously used Rust stack contents. let zeroed=stack.initial_observation.get()&&(1..layout.bytes/8-1).all(|offset|unsafe {read_volatile((layout.base as *const u64).add(offset as usize))}==0); if zeroed { stack.initial_observation.set(false); } Ok(os_k1100_thread::StackObservation { layout, guard_descriptor: descriptor, low_canary: unsafe { read_volatile(layout.base as *const u64) }, high_canary: unsafe { read_volatile((layout.base + layout.bytes - 8) as *const u64) }, initially_zeroed: zeroed, }) } fn release( &mut self, mut stack: OsK1100Stack, ) -> Result<(), (os_k1100_thread::Error, OsK1100Stack)> { if let Err(error) = self.validate(&stack) { return Err((error, stack)); } let layout = stack.layout(); let e = os_k1100_execution(); if e.sp >= stack.start && e.sp <= layout.base + layout.bytes { return Err((os_k1100_thread::Error::ActiveStack, stack)); } if os_k1100_erase_inactive_root(&stack.root, self.tables).is_err() { return Err((os_k1100_thread::Error::Backend, stack)); } let token = match stack.guard.take() { Some(token) => token, None => return Err((os_k1100_thread::Error::Guard, stack)), }; if let Err((_error, token)) = unsafe { self.tables.os_k1020_restore_stack_guard(token) } { stack.guard = Some(token); return Err((os_k1100_thread::Error::Guard, stack)); } for offset in 0..(OS_K1100_STACK_PAGES + 1) * 4096 / 8 { unsafe { write_volatile((stack.start as *mut u64).add(offset), 0); } } unsafe { asm!("dsb ish", options(nostack, preserves_flags)); } if !(0..(OS_K1100_STACK_PAGES + 1) * 4096 / 8) .all(|offset| unsafe { read_volatile((stack.start as *const u64).add(offset)) } == 0) { return Err((os_k1100_thread::Error::Backend, stack)); } let refs = core::array::from_fn::<_, 9, _>(|index| { if index < 8 { stack.leases[index].as_ref().unwrap() } else { stack.root.lease.as_ref().unwrap() } }); if self .memory .allocator .os_authority_mut() .release_many_leases(&refs) .is_err() { return Err((os_k1100_thread::Error::Backend, stack)); } if self.trace { os_k1100_event(3, stack.ordinal as u64, stack.start, 8); } for lease in &mut stack.leases { *lease = None; } stack.root.lease = None; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_ROOT_LIVE) .cast::() .add(stack.ordinal), 0, ); } Ok(()) } } use turkos_kernel::runtime::policy_wait_context as os_k1100_context; type OsK1100Table<'a> = os_k1100_thread::Table, 3>; /// Private nonclone pin for one real Table stack/root owner. The actual nine /// central leases stay in Table::Stack until this pin is consumed by release. struct OsK1100Binding { id: os_k1100_thread::ThreadId, ordinal: usize, root: u64, root_generation: u64, branches: [u64; 2], } struct OsK1100Prepared { source: Option, target: os_k1100_thread::ThreadId, nonce: u64, departed: bool, } struct OsK1100ReturnPrepared { source: os_k1100_thread::ThreadId, bootstrap_root: u64, bootstrap: os_k1100_context::Cooperative, address: u64, nonce: u64, } struct OsK1100IdlePrepared { epoch: u64, wait: Option, } struct OsK1100ContextBackend<'a> { table: OsK1100Table<'a>, bootstrap: u64, current: Option, pins: [bool; 3], records: [Option; 3], observations: [Option; 3], guards: [u64; 3], sleep_wait: Option, sequence: u64, } impl OsK1100ContextBackend<'_> { fn validate_stack_owner( &self, id: os_k1100_thread::ThreadId, ) -> Result<(), os_k1100_context::Error> { if os_k1100_execution().in_irq { return Err(os_k1100_context::Error::Context); } self.table .os_k1020_observe_backend(|backend| { self.table .os_k1020_observe_stack(id, |stack| backend.validate(stack)) }) .map_err(|_| os_k1100_context::Error::Stack)? .map_err(|_| os_k1100_context::Error::Backend) } fn valid_pin(&self, b: &OsK1100Binding) -> Result<(), os_k1100_context::Error> { if b.ordinal >= 3 || !self.pins[b.ordinal] || b.root == 0 || b.root_generation == 0 { return Err(os_k1100_context::Error::Stale); } let r = self.records[b.ordinal].ok_or(os_k1100_context::Error::Stale)?; if r.id != b.id { return Err(os_k1100_context::Error::Stale); } let root = unsafe { read_volatile( core::ptr::addr_of!(OS_K1100_ROOT_ADDRESSES) .cast::() .add(b.ordinal), ) }; let generation = unsafe { read_volatile( core::ptr::addr_of!(OS_K1100_ROOT_GENERATIONS) .cast::() .add(b.ordinal), ) }; if root != b.root || generation != b.root_generation || unsafe { read_volatile( core::ptr::addr_of!(OS_K1100_ROOT_LIVE) .cast::() .add(b.ordinal), ) } != 1 { return Err(os_k1100_context::Error::Stale); } // Immutable IRQ pin: the typed central/root/stack owner was validated // by Table in main context before this pin was published. IRQ reads // only these bounded physical words and its private pin, never Table // or the allocator. Main observations additionally revalidate Table. os_k1100_root_words(b.root, b.branches).map_err(|_| os_k1100_context::Error::Backend)?; if !os_k1100_execution().in_irq { self.validate_stack_owner(b.id)?; } Ok(()) } fn refresh(&mut self) -> Result<(), os_k1100_context::Error> { if os_k1100_execution().in_irq { return Err(os_k1100_context::Error::Context); } for ordinal in 0..3 { if let Some(previous) = self.records[ordinal] { if !self.pins[ordinal] { continue; } let record = self .table .record(previous.id) .map_err(|_| os_k1100_context::Error::State)?; self.validate_stack_owner(previous.id)?; let observation = self .table .os_k1020_observe_stack(previous.id, |stack| { let guard = stack .guard .as_ref() .map_or(0, |guard| guard.descriptor_address()); ( stack.layout(), guard, stack.root.address(), stack.root.generation(), ) }) .map_err(|_| os_k1100_context::Error::Stack)?; if observation.1 == 0 { return Err(os_k1100_context::Error::Guard); } self.records[ordinal] = Some(record); self.guards[ordinal] = observation.1; self.observations[ordinal] = Some(os_k1100_thread::StackObservation { layout: observation.0, guard_descriptor: unsafe { read_volatile(observation.1 as *const u64) }, low_canary: unsafe { read_volatile(observation.0.base as *const u64) }, high_canary: unsafe { read_volatile((observation.0.base + observation.0.bytes - 8) as *const u64) }, initially_zeroed: false, }); } } Ok(()) } fn pin( &mut self, id: os_k1100_thread::ThreadId, ordinal: usize, ) -> Result { if ordinal >= 3 || self.pins[ordinal] || os_k1100_execution().in_irq { return Err(os_k1100_context::Error::State); } let record = self .table .record(id) .map_err(|_| os_k1100_context::Error::State)?; self.validate_stack_owner(id)?; let (root, generation, branches) = self .table .os_k1020_observe_stack(id, |stack| { ( stack.root.address(), stack.root.generation(), stack.root.branches, ) }) .map_err(|_| os_k1100_context::Error::Stack)?; let before_records = self.records; let before_observations = self.observations; let before_guards = self.guards; self.records[ordinal] = Some(record); self.pins[ordinal] = true; if let Err(error) = self.refresh() { self.pins[ordinal] = false; self.records = before_records; self.observations = before_observations; self.guards = before_guards; return Err(error); } Ok(OsK1100Binding { id, ordinal, root, root_generation: generation, branches, }) } fn next_prepared(&mut self) -> Result { let next = self .sequence .checked_add(1) .ok_or(os_k1100_context::Error::Exhausted)?; self.sequence = next; Ok(next) } } unsafe impl os_k1100_context::Backend for OsK1100ContextBackend<'_> { type Binding = OsK1100Binding; type Prepared = OsK1100Prepared; type IdlePrepared = OsK1100IdlePrepared; fn execution(&self) -> os_k1100_context::Execution { let e = os_k1100_execution(); let (root, _, _, daif) = os_k940_registers(); let cpacr: u64; unsafe { asm!("mrs {fp}, CPACR_EL1",fp=out(reg)cpacr,options(nomem,nostack,preserves_flags)); } os_k1100_context::Execution { cpu: e.cpu, daif, in_irq: e.in_irq, sp: e.sp, root, thread: self.current, fp_enabled: cpacr & (3 << 20) == 3 << 20, critical_depth: unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_CRITICAL)) } as u32, safe_point: unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_SAFE_POINT)) } == 1, tco: if e.in_irq { unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_TCO)) } } else { 0 }, } } fn record( &self, b: &Self::Binding, ) -> Result { self.valid_pin(b)?; // IRQ uses the last real main-only Table transition, an immutable pin // and actual guard/canaries. It never borrows Table/Memory/authority. if os_k1100_execution().in_irq { return self.records[b.ordinal].ok_or(os_k1100_context::Error::Stale); } self.table .record(b.id) .map_err(|_| os_k1100_context::Error::State) } fn stack( &self, b: &Self::Binding, ) -> Result { self.valid_pin(b)?; let cached = self.observations[b.ordinal].ok_or(os_k1100_context::Error::Stack)?; if !os_k1100_execution().in_irq { self.table .record(b.id) .map_err(|_| os_k1100_context::Error::State)?; self.table .os_k1020_observe_stack(b.id, |stack| { stack.root.address() == b.root && stack.root.generation() == b.root_generation }) .map_err(|_| os_k1100_context::Error::Stack)? .then_some(()) .ok_or(os_k1100_context::Error::Stale)?; } Ok(os_k1100_thread::StackObservation { guard_descriptor: unsafe { read_volatile(self.guards[b.ordinal] as *const u64) }, low_canary: unsafe { read_volatile(cached.layout.base as *const u64) }, high_canary: unsafe { read_volatile((cached.layout.base + cached.layout.bytes - 8) as *const u64) }, ..cached }) } fn root(&self, b: &Self::Binding) -> Result { self.valid_pin(b)?; if !os_k1100_execution().in_irq { self.table .record(b.id) .map_err(|_| os_k1100_context::Error::State)?; } Ok(b.root) } fn executable(&self, pc: u64) -> bool { unsafe extern "C" { static _text_start: u8; static _text_end: u8; } pc % 4 == 0 && pc >= core::ptr::addr_of!(_text_start) as u64 && pc < core::ptr::addr_of!(_text_end) as u64 } fn initial_return_pc(&self, _: &Self::Binding) -> Result { unsafe extern "C" { fn __os_k1100_bad_return(); } Ok(__os_k1100_bad_return as *const () as u64) } fn capture_matches(&self, b: &Self::Binding, c: &os_k1100_context::Capture) -> bool { if self.valid_pin(b).is_err() { return false; } let Some(observation) = self.observations[b.ordinal] else { return false; }; let (address, episode, bytes) = match c.image { os_k1100_context::Image::Cooperative(_) => ( unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LIVE_CAPTURE_ADDRESS)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_CAPTURE_EPISODE)) }, 640u64, ), os_k1100_context::Image::Timer(_) => ( unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_CAPTURE_ADDRESS)) }, unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_CAPTURE_EPISODE)) }, 848u64, ), _ => return false, }; if c.address != address || c.episode != episode || episode == 0 || address % 16 != 0 || address < observation.layout.usable_bottom() || !address .checked_add(bytes) .is_some_and(|end| end <= observation.layout.usable_top()) { return false; } match c.image { os_k1100_context::Image::Cooperative(f) => { (unsafe { read_volatile(address as *const os_k1100_context::Cooperative) }) == f } os_k1100_context::Image::Timer(f) => { (unsafe { read_volatile(address as *const os_k920_exception::Frame) }) == f } _ => false, } } fn registers(&self) -> os_k1100_context::Registers { let f = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_FRAME)) }; os_k1100_context::Registers { sp: f.sp, x19_x30: f.x19_x30, q: f.q, fpcr: f.fpcr, fpsr: f.fpsr, pc: os_k1100_observed_landing_pc(&f), argument: unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_ARGUMENT)) }, timer: None, } } fn prepare( &mut self, source: Option<&Self::Binding>, target: &Self::Binding, _: &os_k1100_context::Image, _: os_k1100_context::Disposition, ) -> Result { self.valid_pin(target)?; if let Some(source) = source { self.valid_pin(source)?; } Ok(OsK1100Prepared { source: source.map(|s| s.id), target: target.id, nonce: self.next_prepared()?, departed: false, }) } fn commit( &mut self, _: &mut Self::Prepared, _: Option<&Self::Binding>, _: &Self::Binding, _: &os_k1100_context::Image, _: os_k1100_context::Disposition, ) -> Result<(), os_k1100_context::Failure> { // A synchronous bank loan must never span a suspended continuation. Err(os_k1100_context::Failure { error: os_k1100_context::Error::Context, phase: os_k1100_context::FailurePhase::BeforeSwitch, }) } fn finish( &mut self, p: Self::Prepared, ) -> Result<(), (os_k1100_context::Error, Self::Prepared)> { if !p.departed { return Err((os_k1100_context::Error::State, p)); } Ok(()) } fn abort( &mut self, p: Self::Prepared, ) -> Result<(), (os_k1100_context::Error, Self::Prepared)> { if p.departed { return Err((os_k1100_context::Error::State, p)); } Ok(()) } fn release( &mut self, b: Self::Binding, ) -> Result<(), (os_k1100_context::Error, Self::Binding)> { if let Err(e) = self.valid_pin(&b) { return Err((e, b)); } let record = match self.table.record(b.id) { Ok(r) => r, Err(_) => return Err((os_k1100_context::Error::State, b)), }; if record.state != os_k1100_thread::State::Dead || os_k940_registers().0 != self.bootstrap { return Err((os_k1100_context::Error::Active, b)); } if self.table.reap(b.id).is_err() { return Err((os_k1100_context::Error::Backend, b)); } self.pins[b.ordinal] = false; self.records[b.ordinal] = None; self.observations[b.ordinal] = None; self.guards[b.ordinal] = 0; Ok(()) } fn ready_empty(&self) -> bool { let s = self.table.snapshot(); s.ready == 0 && s.running == 0 } fn wake_epoch(&self) -> u64 { unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_WAKE_EPOCH)) } } fn idle_prepare(&mut self, epoch: u64) -> Result { let wait = self .sleep_wait .take() .ok_or(os_k1100_context::Error::Wake)?; Ok(OsK1100IdlePrepared { epoch, wait: Some(wait), }) } fn idle_commit( &mut self, p: &mut Self::IdlePrepared, epoch: u64, ) -> Result { os_k1100_idle_atomic(self, p, epoch) } fn idle_finish( &mut self, p: Self::IdlePrepared, ) -> Result<(), (os_k1100_context::Error, Self::IdlePrepared)> { if p.wait.is_some() { return Err((os_k1100_context::Error::Wake, p)); } Ok(()) } fn idle_abort( &mut self, mut p: Self::IdlePrepared, ) -> Result<(), (os_k1100_context::Error, Self::IdlePrepared)> { if self.sleep_wait.is_some() { return Err((os_k1100_context::Error::Wake, p)); } self.sleep_wait = p.wait.take(); Ok(()) } } unsafe impl os_k1100_context::HandoffBackend for OsK1100ContextBackend<'_> { fn depart( &mut self, p: &mut Self::Prepared, source: Option<&Self::Binding>, nonce: u64, d: os_k1100_context::Disposition, ) -> Result<(), os_k1100_context::Failure> { let before = |error| os_k1100_context::Failure { error, phase: os_k1100_context::FailurePhase::BeforeSwitch, }; if p.source != source.map(|s| s.id) || p.departed || p.nonce == 0 || nonce == 0 { return Err(before(os_k1100_context::Error::Stale)); } self.validate_stack_owner(p.target).map_err(before)?; if let Some(source) = source { self.valid_pin(source).map_err(before)?; match d { os_k1100_context::Disposition::Ready => self .table .yield_current(source.id) .map_err(|_| before(os_k1100_context::Error::State))?, os_k1100_context::Disposition::Sleeping => { if self.sleep_wait.is_some() { return Err(before(os_k1100_context::Error::Wake)); } self.sleep_wait = Some( self.table .sleep_current(source.id) .map_err(|_| before(os_k1100_context::Error::State))?, ); } } } p.departed = true; self.current = None; if let Err(e) = self.refresh() { return Err(os_k1100_context::Failure { error: e, phase: os_k1100_context::FailurePhase::AfterSwitch, }); } let root = match self .table .os_k1020_observe_stack(p.target, |stack| stack.root.address()) { Ok(root) => root, Err(_) => { return Err(os_k1100_context::Failure { error: os_k1100_context::Error::Stack, phase: os_k1100_context::FailurePhase::AfterSwitch, }); } }; if os_k1100_select_root(root).is_err() { return Err(os_k1100_context::Failure { error: os_k1100_context::Error::Backend, phase: os_k1100_context::FailurePhase::AfterSwitch, }); } Ok(()) } fn arrive( &mut self, p: &mut Self::Prepared, target: &Self::Binding, _: &os_k1100_context::Image, nonce: u64, ) -> Result<(), os_k1100_context::Error> { self.valid_pin(target)?; if !p.departed || p.target != target.id || nonce != unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_DEPARTED_NONCE)) } || os_k940_registers().0 != target.root { return Err(os_k1100_context::Error::Capture); } self.table .activate(target.id) .map_err(|_| os_k1100_context::Error::State)?; self.current = Some(target.id); os_k1100_publish_activated(target.id)?; self.refresh() } } unsafe impl os_k1100_context::CompletionBackend for OsK1100ContextBackend<'_> { type ReturnPrepared = OsK1100ReturnPrepared; fn prepare_return( &mut self, source: &Self::Binding, _: &os_k1100_context::Capture, ) -> Result { self.valid_pin(source)?; let address = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_BOOT_CAPTURE_ADDRESS)) }; if !os_k1100_boot_frame_range(address) { return Err(os_k1100_context::Error::Stack); } let bootstrap = unsafe { read_volatile(address as *const os_k1100_context::Cooperative) }; if bootstrap.sp != address + 640 || bootstrap.canary != os_k1100_context::CONTEXT_CANARY || bootstrap.padding != 0 { return Err(os_k1100_context::Error::Capture); } Ok(OsK1100ReturnPrepared { source: source.id, bootstrap_root: self.bootstrap, bootstrap, address, nonce: self.next_prepared()?, }) } fn leave_return( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Binding, ) -> Result<(), os_k1100_context::Failure> { let before = |error| os_k1100_context::Failure { error, phase: os_k1100_context::FailurePhase::BeforeSwitch, }; self.valid_pin(source).map_err(before)?; if p.source != source.id || p.nonce == 0 { return Err(before(os_k1100_context::Error::Stale)); } self.table .exit_current(source.id) .map_err(|_| before(os_k1100_context::Error::State))?; self.current = None; if self.refresh().is_err() || os_k1100_select_root(p.bootstrap_root).is_err() { return Err(os_k1100_context::Failure { error: os_k1100_context::Error::Backend, phase: os_k1100_context::FailurePhase::AfterSwitch, }); } Ok(()) } fn returned(&self, p: &Self::ReturnPrepared) -> Result<(), os_k1100_context::Error> { unsafe extern "C" { fn __os_k1100_boot_landed(); } let actual = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_FRAME)) }; let stacks = os_k1100_stack_registers(); let original = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_BOOT_STACK_REGISTERS)) }; if actual != p.bootstrap || !os_k1100_boot_frame_range(p.address) || actual.sp != p.address + 640 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_PC)) } != __os_k1100_boot_landed as *const () as u64 || stacks[0] != original[0] || stacks[if original[0] == 0 { 2 } else { 1 }] != original[if original[0] == 0 { 2 } else { 1 }] { return Err(os_k1100_context::Error::Capture); } Ok(()) } fn return_root(&self, p: &Self::ReturnPrepared) -> Result { if p.bootstrap_root != self.bootstrap { return Err(os_k1100_context::Error::Stack); } Ok(p.bootstrap_root) } fn finish_return( &mut self, _: Self::ReturnPrepared, ) -> Result<(), (os_k1100_context::Error, Self::ReturnPrepared)> { Ok(()) } fn abort_return( &mut self, _: Self::ReturnPrepared, ) -> Result<(), (os_k1100_context::Error, Self::ReturnPrepared)> { Ok(()) } } unsafe impl os_k1100_context::InactiveBackend for OsK1100ContextBackend<'_> { fn cancel_inactive( &mut self, binding: &Self::Binding, ) -> Result<(), os_k1100_context::Failure> { let before = |error| os_k1100_context::Failure { error, phase: os_k1100_context::FailurePhase::BeforeSwitch, }; self.valid_pin(binding).map_err(before)?; if self.current.is_some() || os_k940_registers().0 != self.bootstrap { return Err(before(os_k1100_context::Error::Context)); } self.table .cancel(binding.id) .map_err(|_| before(os_k1100_context::Error::State))?; // Any retained sleeping authorization was invalidated by actual // Table.cancel. It is never reused to manufacture a wake after cancel. if self .sleep_wait .as_ref() .is_some_and(|wait| wait.thread() == binding.id) { self.sleep_wait = None; } self.refresh().map_err(|error| os_k1100_context::Failure { error, phase: os_k1100_context::FailurePhase::AfterSwitch, }) } } type OsK1100Bank<'a> = os_k1100_context::CompletionBank, 3>; type OsK1100Backend<'a> = OsK1100QueueBackend>; type OsK1100Queue<'a> = os_k1100_queue::ReturnQueue, 3>; type OsK1100Bridge<'a> = os_k1100_bridge::WaitGuestBridge, OsK1100Port>; type OsK1100Core<'a> = os_k1100_wait::WaitSet, 3>; #[unsafe(no_mangle)] static mut OS_K1100_CORE: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_ROOT_ADDRESSES: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1100_ROOT_GENERATIONS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1100_ROOT_LIVE: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1100_OWNED: [u64; 27] = [0; 27]; #[unsafe(no_mangle)] static mut OS_K1100_LEASE_GENERATIONS: [u64; 27] = [0; 27]; #[unsafe(no_mangle)] static mut OS_K1100_EVIDENCE: [u64; 128] = [0; 128]; #[unsafe(no_mangle)] static mut OS_K1100_KERNEL_TABLES: [u64; 32] = [0; 32]; #[unsafe(no_mangle)] static mut OS_K1100_PRIMARY_GUARD_BEFORE: [u64; 4] = [0; 4]; #[unsafe(no_mangle)] static mut OS_K1100_EVENTS: [u64; 4096] = [0; 4096]; #[unsafe(no_mangle)] static mut OS_K1100_EVENT_COUNT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_CRITICAL: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_SAFE_POINT: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1100_WAKE_EPOCH: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_CAPTURE_EPISODE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_BOOT_STACK_REGISTERS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1100_IRQ_TCO: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_LIVE_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_IRQ_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_IRQ_CAPTURE_EPISODE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_BOOT_CAPTURE_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_DEPARTED_NONCE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_LANDING_ADDRESS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_LANDING_PC: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_LANDING_ARGUMENT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1100_LANDING_FRAME: os_k1100_context::Cooperative = os_k1100_context::Cooperative::EMPTY; #[unsafe(no_mangle)] static mut OS_K1100_CURRENT_CAPTURE: Option = None; #[unsafe(no_mangle)] static mut OS_K1100_PUBLISHED_CURRENT: Option = None; #[derive(Clone, Copy)] #[repr(C)] struct OsK1100Wire { frame: os_k1100_context::Cooperative, pc: u64, argument: u64, nonce: u64, kind: u64, } #[unsafe(no_mangle)] static mut OS_K1100_WIRE: OsK1100Wire = OsK1100Wire { frame: os_k1100_context::Cooperative::EMPTY, pc: 0, argument: 0, nonce: 0, kind: 0, }; #[unsafe(no_mangle)] static mut OS_K1100_CONTEXT_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1100_THREADS_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1100_BACKEND_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1100_QUEUE_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static mut OS_K1100_WAIT_IDENTITY: u64 = 1; #[unsafe(no_mangle)] static OS_K1100_COMPILE_ABI: [u64; 8] = [ core::mem::size_of::>() as u64, core::mem::size_of::>() as u64, core::mem::size_of::>() as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::size_of::() as u64, core::mem::size_of::() as u64, core::mem::size_of::() as u64, ]; fn os_k1100_word(index: usize, value: u64) { if index >= 128 { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_EVIDENCE) .cast::() .add(index), value, ); } } fn os_k1100_event(kind: u64, a: u64, b: u64, c: u64) { let e = os_k1100_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let count = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_EVENT_COUNT)) } as usize; if count >= 1024 { park(); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_EVENTS) .cast::<[u64; 4]>() .add(count), [kind, a, b, c], ); write_volatile( core::ptr::addr_of_mut!(OS_K1100_EVENT_COUNT), count as u64 + 1, ); } } fn os_k1100_nonce(pointer: *mut u64) -> u64 { let e = os_k1100_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let current = unsafe { read_volatile(pointer) }; let next = current.checked_add(1).unwrap_or_else(|| park()); unsafe { write_volatile(pointer, next); } current } #[unsafe(no_mangle)] extern "C" fn __os_k1100_context_identity() -> u64 { os_k1100_nonce(core::ptr::addr_of_mut!(OS_K1100_CONTEXT_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1100_threads_identity() -> u64 { os_k1100_nonce(core::ptr::addr_of_mut!(OS_K1100_THREADS_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1100_backend_identity() -> u64 { os_k1100_nonce(core::ptr::addr_of_mut!(OS_K1100_BACKEND_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1100_runqueue_identity() -> u64 { os_k1100_nonce(core::ptr::addr_of_mut!(OS_K1100_QUEUE_IDENTITY)) } #[unsafe(no_mangle)] extern "C" fn __os_k1100_wait_identity() -> u64 { os_k1100_nonce(core::ptr::addr_of_mut!(OS_K1100_WAIT_IDENTITY)) } // Exact namespaced frozen stack-register wrapper ABI: selector,SP0,SP1,activeSP. fn os_k1100_stack_registers() -> [u64; 4] { unsafe extern "C" { fn __os_k1100_stack_registers(output: *mut u64); } if !os_k1100_execution().irq_masked { park(); } let mut values = [0; 4]; unsafe { __os_k1100_stack_registers(values.as_mut_ptr()); } values } fn os_k1100_observed_landing_pc(frame: &os_k1100_context::Cooperative) -> u64 { let e = os_k1100_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq { park(); } let actual = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_PC)) }; unsafe extern "C" { fn __os_k1100_cooperative_landed(); } if actual != __os_k1100_cooperative_landed as *const () as u64 { return actual; } let wire = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_WIRE)) }; if wire.kind != 1 || wire.nonce == 0 || wire.pc != frame.x19_x30[11] || wire.nonce != unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_DEPARTED_NONCE)) } || wire.frame != *frame { park(); } frame.x19_x30[11] } fn os_k1100_boot_frame_range(address: u64) -> bool { let guard = core::ptr::addr_of!(OS_K1100_PRIMARY_STACK) as u64; address % 16 == 0 && address >= guard + 4096 + 1024 && address .checked_add(640) .is_some_and(|end| end <= guard + 69632 - 16) } fn os_k1100_idle_atomic( _: &mut OsK1100ContextBackend<'_>, _: &mut OsK1100IdlePrepared, _: u64, ) -> Result { Err(os_k1100_context::Failure { error: os_k1100_context::Error::Context, phase: os_k1100_context::FailurePhase::BeforeSwitch, }) } // Main's source authority owns only private registration claims. IRQ has its // separately owned Producer/Ingress and calls the pure scalar reader below. #[unsafe(no_mangle)] static mut OS_K1100_WAIT_SIGNAL: [u64; 8] = [0; 8]; #[unsafe(no_mangle)] static mut OS_K1100_WAIT_SIGNAL_PINS: [u64; 3] = [0; 3]; #[unsafe(no_mangle)] static mut OS_K1100_WAIT_SIGNAL_PIN_SEQUENCE: u64 = 1; struct OsK1100Port; struct OsK1100SignalPin { source: os_k1100_wait::Source, slot: usize, claim: u64, } fn os_k1100_wait_signal_word(index: usize) -> u64 { if index >= 8 { park(); } unsafe { read_volatile( core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL) .cast::() .add(index), ) } } fn os_k1100_wait_signal_execution() -> os_k1100_wait::Execution { let e = os_k1100_execution(); os_k1100_wait::Execution { cpu: e.cpu, irq_masked: e.irq_masked, in_irq: e.in_irq, current: unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_PUBLISHED_CURRENT)) }, } } fn os_k1100_wait_signal_observation() -> Result { let e = os_k1100_wait_signal_execution(); if e.cpu != 0 || !e.irq_masked { return Err(os_k1100_wait::Error::Context); } if os_k1100_wait_signal_word(0) != 1 { return Err(os_k1100_wait::Error::Source); } Ok(os_k1100_wait::Observation { source: os_k1100_wait::Source { channel: os_k1100_wait_signal_word(1), epoch: os_k1100_wait_signal_word(2), owner: os_k1100_wait_signal_word(3), }, sequence: os_k1100_wait_signal_word(4), ready: os_k1100_wait_signal_word(5) == 1, now: os_k960_counter(), }) } unsafe impl os_k1100_wait::Producer for OsK1100Port { fn execution(&self) -> os_k1100_wait::Execution { os_k1100_wait_signal_execution() } fn observe(&self) -> Result { os_k1100_wait_signal_observation() } } unsafe impl os_k1100_bridge::WaitAuthority for OsK1100Port { type Pin = OsK1100SignalPin; fn retain_source( &mut self, source: os_k1100_wait::Source, ) -> Result { os_k1100_wait::context( 0, ::execution(self), os_k1100_wait::Operation::Arm, )?; if ::observe(self)?.source != source { return Err(os_k1100_wait::Error::Source); } let slot=(0..3).find(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL_PINS).cast::().add(*i))}==0).ok_or(os_k1100_wait::Error::Full)?; let claim = os_k1100_nonce(core::ptr::addr_of_mut!(OS_K1100_WAIT_SIGNAL_PIN_SEQUENCE)); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_WAIT_SIGNAL_PINS) .cast::() .add(slot), claim, ); } Ok(OsK1100SignalPin { source, slot, claim, }) } fn validate_source( &self, pin: &Self::Pin, source: os_k1100_wait::Source, ) -> Result<(), os_k1100_wait::Error> { os_k1100_wait::context( 0, ::execution(self), os_k1100_wait::Operation::Service, )?; if pin.slot >= 3 || pin.source != source || pin.claim == 0 || ::observe(self)?.source != source || unsafe { read_volatile( core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL_PINS) .cast::() .add(pin.slot), ) } != pin.claim { return Err(os_k1100_wait::Error::Source); } Ok(()) } fn release_source(&mut self, pin: Self::Pin) -> Result<(), (os_k1100_wait::Error, Self::Pin)> { if let Err(e) = self.validate_source(&pin, pin.source) { return Err((e, pin)); } unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_WAIT_SIGNAL_PINS) .cast::() .add(pin.slot), 0, ); } Ok(()) } } // NEW reusable three-member backend. The real typed bindings/backing owners // remain exclusively inside CompletionBank; no binding is reconstructed. use turkos_kernel::runtime::policy_wait_queue as os_k1100_queue; struct OsK1100Member { identity: u64, context: os_k1100_context::Handle, thread: os_k1100_thread::ThreadId, ordinal: usize, retired: bool, } struct OsK1100SelectionPrepared { handoff: Option, epoch: u64, nonce: u64, arrived: bool, } struct OsK1100QueueReturnPrepared { token: Option, source: os_k1100_context::Handle, nonce: u64, observed: bool, } struct OsK1100QueueBackend { bank: os_k1100_context::CompletionBank, identity: u64, members: [Option; 3], failed_tick: Option, retirement_quarantined: bool, } impl OsK1100QueueBackend { #[allow(clippy::result_large_err)] fn new( bank: os_k1100_context::CompletionBank, ) -> Result< Self, ( os_k1100_queue::Error, os_k1100_context::CompletionBank, ), > { let snapshot = bank.handoff().bank().snapshot(); if bank.quarantined() || bank.observation().is_some() || bank.handoff().quarantined() || bank.handoff().stage().is_some() || snapshot.retained != 3 || snapshot.active != 0 || snapshot.ready != 3 || snapshot.quarantined != 0 || snapshot.faulted || snapshot.retained_switch || snapshot.retained_capture { return Err((os_k1100_queue::Error::State, bank)); } let e = bank .handoff() .bank() .with_backend_observation(|b| os_k1100_context::Backend::execution(b)); if e.cpu != 0 || e.daif & 128 == 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point || e.thread.is_some() { return Err((os_k1100_queue::Error::Context, bank)); } unsafe extern "C" { fn __os_k1100_backend_identity() -> u64; } let identity = unsafe { __os_k1100_backend_identity() }; if identity == 0 { return Err((os_k1100_queue::Error::Exhausted, bank)); } Ok(Self { bank, identity, members: [None; 3], failed_tick: None, retirement_quarantined: false, }) } fn issue( &mut self, handle: os_k1100_context::Handle, ordinal: usize, ) -> Result { if self.retirement_quarantined { return Err(os_k1100_queue::Error::Quarantined); } if ordinal >= 3 || self.members[ordinal].is_some() || self.members.contains(&Some(handle)) { return Err(os_k1100_queue::Error::Duplicate); } let o = self .bank .os_k1100_observe_member(handle) .map_err(|_| os_k1100_queue::Error::Backend)?; if o.thread.state != os_k1100_thread::State::Ready || o.context.state != os_k1100_context::State::Ready { return Err(os_k1100_queue::Error::State); } self.members[ordinal] = Some(handle); Ok(OsK1100Member { identity: self.identity, context: handle, thread: o.thread.id, ordinal, retired: false, }) } fn member( &self, member: &OsK1100Member, ) -> Result { if self.retirement_quarantined { return Err(os_k1100_queue::Error::Quarantined); } if member.retired { return Err(os_k1100_queue::Error::State); } if member.identity != self.identity || member.ordinal >= 3 || self.members[member.ordinal] != Some(member.context) { return Err(os_k1100_queue::Error::Stale); } let o = self .bank .os_k1100_observe_member(member.context) .map_err(|_| os_k1100_queue::Error::Backend)?; if o.thread.id != member.thread || o.context.thread != member.thread { return Err(os_k1100_queue::Error::Stale); } Ok(o) } } unsafe impl os_k1100_queue::Backend for OsK1100QueueBackend { type Member = OsK1100Member; type Prepared = OsK1100SelectionPrepared; type Tick = B::QueueTick; fn execution(&self) -> os_k1100_queue::Execution { let e = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1100_context::Backend::execution(b)); os_k1100_queue::Execution { cpu: e.cpu, masked: e.daif & 128 != 0, in_irq: e.in_irq, thread: e.thread, root: e.root, } } fn observe( &self, m: &Self::Member, ) -> Result { self.member(m) } fn tick( &self, t: &Self::Tick, ) -> Result { self.bank.os_k1100_tick(t) } fn consume_tick(&mut self, t: Self::Tick) { if let Err((_e, actual)) = self.bank.os_k1100_consume_tick(t) { self.failed_tick = Some(actual); park(); } } fn prepare( &mut self, source: Option<&Self::Member>, target: &Self::Member, epoch: u64, d: os_k1100_queue::Disposition, ) -> Result { self.member(target)?; if let Some(source) = source { self.member(source)?; } let capture = if source.is_some() { Some( self.bank .os_k1100_capture() .map_err(|_| os_k1100_queue::Error::Backend)?, ) } else { None }; let bank = self .bank .quiescent() .map_err(|_| os_k1100_queue::Error::Busy)?; let handoff = match source { None => bank.start(target.context), Some(source) => match d { os_k1100_queue::Disposition::Ready => { bank.cooperate(source.context, target.context, capture.unwrap()) } os_k1100_queue::Disposition::Sleeping => { bank.sleep_to(source.context, target.context, capture.unwrap()) } }, } .map_err(|_| os_k1100_queue::Error::Backend)?; Ok(OsK1100SelectionPrepared { handoff: Some(handoff), epoch, nonce: 0, arrived: false, }) } fn commit( &mut self, _: &mut Self::Prepared, _: Option<&Self::Member>, _: &Self::Member, _: u64, _: os_k1100_queue::Disposition, ) -> Result<(), os_k1100_queue::Failure> { Err(os_k1100_queue::Failure { error: os_k1100_queue::Error::Context, phase: os_k1100_context::FailurePhase::BeforeSwitch, }) } fn finish(&mut self, p: Self::Prepared) -> Result<(), (os_k1100_queue::Error, Self::Prepared)> { if !p.arrived || p.handoff.is_some() { Err((os_k1100_queue::Error::State, p)) } else { Ok(()) } } fn abort( &mut self, mut p: Self::Prepared, ) -> Result<(), (os_k1100_queue::Error, Self::Prepared)> { let b = match self.bank.quiescent() { Ok(b) => b, Err(_) => return Err((os_k1100_queue::Error::Busy, p)), }; let t = p.handoff.take().unwrap(); match b.abort(t) { Ok(()) => Ok(()), Err(os_k1100_context::HandoffFailure::Before(_, t)) => { p.handoff = Some(t); Err((os_k1100_queue::Error::Backend, p)) } Err(os_k1100_context::HandoffFailure::Quarantined(_)) => { Err((os_k1100_queue::Error::Backend, p)) } } } } unsafe impl os_k1100_queue::SelectionHandoffBackend for OsK1100QueueBackend { fn depart( &mut self, p: &mut Self::Prepared, _: Option<&Self::Member>, _: &Self::Member, epoch: u64, _: os_k1100_queue::Disposition, nonce: u64, ) -> Result<(), os_k1100_queue::Failure> { if p.epoch != epoch || p.nonce != 0 || nonce == 0 { return Err(os_k1100_queue::Failure { error: os_k1100_queue::Error::Stale, phase: os_k1100_context::FailurePhase::BeforeSwitch, }); } let b = match self.bank.quiescent() { Ok(b) => b, Err(_) => { return Err(os_k1100_queue::Failure { error: os_k1100_queue::Error::Busy, phase: os_k1100_context::FailurePhase::BeforeSwitch, }); } }; let t = p.handoff.take().unwrap(); match b.depart(t) { Ok(t) => { p.handoff = Some(t); p.nonce = nonce; Ok(()) } Err(os_k1100_context::HandoffFailure::Before(_, t)) => { p.handoff = Some(t); Err(os_k1100_queue::Failure { error: os_k1100_queue::Error::Backend, phase: os_k1100_context::FailurePhase::BeforeSwitch, }) } Err(os_k1100_context::HandoffFailure::Quarantined(_)) => Err(os_k1100_queue::Failure { error: os_k1100_queue::Error::Backend, phase: os_k1100_context::FailurePhase::AfterSwitch, }), } } fn arrive( &mut self, p: &mut Self::Prepared, target: &Self::Member, epoch: u64, nonce: u64, ) -> Result<(), os_k1100_queue::Error> { if p.epoch != epoch || p.nonce != nonce { os_k1100_backend_arrival_diagnostic(0x401);return Err(os_k1100_queue::Error::Stale); } let b = self .bank .quiescent() .map_err(|_| {os_k1100_backend_arrival_diagnostic(0x402);os_k1100_queue::Error::Busy})?; let t = p.handoff.take().unwrap(); match b.landed(t) { Ok(()) => { p.arrived = true; self.member(target).map_err(|error| {os_k1100_backend_arrival_diagnostic(0x404+error as u64);error})?; Ok(()) } Err(os_k1100_context::HandoffFailure::Before(error, t)) => { os_k1100_backend_arrival_diagnostic(0x500+error as u64);p.handoff = Some(t); Err(os_k1100_queue::Error::Backend) } Err(os_k1100_context::HandoffFailure::Quarantined(error)) => { os_k1100_backend_arrival_diagnostic(0x600+error as u64);Err(os_k1100_queue::Error::Backend) } } } } unsafe impl os_k1100_queue::QueueCompletionBackend for OsK1100QueueBackend { type ReturnPrepared = OsK1100QueueReturnPrepared; fn prepare_exit( &mut self, source: &Self::Member, capture: os_k1100_context::Capture, ) -> Result { self.member(source)?; let token = self .bank .prepare_return(source.context, capture) .map_err(|_| os_k1100_queue::Error::Backend)?; Ok(OsK1100QueueReturnPrepared { token: Some(token), source: source.context, nonce: 0, observed: false, }) } fn depart_exit( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Member, nonce: u64, ) -> Result<(), os_k1100_queue::Failure> { if p.source != source.context || nonce == 0 || p.nonce != 0 { return Err(os_k1100_queue::Failure { error: os_k1100_queue::Error::Stale, phase: os_k1100_context::FailurePhase::BeforeSwitch, }); } let t = p.token.take().unwrap(); match self.bank.depart_return(t) { Ok(t) => { p.token = Some(t); p.nonce = nonce; Ok(()) } Err(os_k1100_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err(os_k1100_queue::Failure { error: os_k1100_queue::Error::Backend, phase: os_k1100_context::FailurePhase::BeforeSwitch, }) } Err(os_k1100_context::ReturnFailure::Quarantined(_)) => Err(os_k1100_queue::Failure { error: os_k1100_queue::Error::Backend, phase: os_k1100_context::FailurePhase::AfterSwitch, }), } } fn returned_exit( &mut self, p: &mut Self::ReturnPrepared, source: &Self::Member, nonce: u64, ) -> Result<(), os_k1100_queue::Error> { if p.source != source.context || p.nonce != nonce { return Err(os_k1100_queue::Error::Stale); } self.bank .os_k1100_check_return_landing(p.token.as_ref().ok_or(os_k1100_queue::Error::State)?) .map_err(|_| os_k1100_queue::Error::Backend)?; p.observed = true; Ok(()) } fn finish_exit( &mut self, mut p: Self::ReturnPrepared, ) -> Result<(), (os_k1100_queue::Error, Self::ReturnPrepared)> { if !p.observed { return Err((os_k1100_queue::Error::State, p)); } let t = p.token.take().unwrap(); match self.bank.return_landed(t) { Ok(()) => Ok(()), Err(os_k1100_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err((os_k1100_queue::Error::Backend, p)) } Err(os_k1100_context::ReturnFailure::Quarantined(_)) => { Err((os_k1100_queue::Error::Backend, p)) } } } fn abort_exit( &mut self, mut p: Self::ReturnPrepared, ) -> Result<(), (os_k1100_queue::Error, Self::ReturnPrepared)> { let t = p.token.take().unwrap(); match self.bank.abort_return(t) { Ok(()) => Ok(()), Err(os_k1100_context::ReturnFailure::Before(_, t)) => { p.token = Some(t); Err((os_k1100_queue::Error::Backend, p)) } Err(os_k1100_context::ReturnFailure::Quarantined(_)) => { Err((os_k1100_queue::Error::Backend, p)) } } } } impl OsK1100QueueBackend { fn retire_member( &mut self, mut m: OsK1100Member, ) -> Result { let observation = match self.member(&m) { Ok(o) => o, Err(e) => return Err((e, m)), }; let before = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1100_context::Backend::execution(b)); let core = match self.bank.quiescent().and_then(|b| b.quiescent()) { Ok(core) => core, Err(_) => return Err((os_k1100_queue::Error::Busy, m)), }; if observation.thread.state != os_k1100_thread::State::Dead && core.os_k1040_cancel_inactive(m.context).is_err() { return Err((os_k1100_queue::Error::Backend, m)); } if core.retire(m.context).is_err() { return Err((os_k1100_queue::Error::Backend, m)); } // Frozen retirement has consumed the real Binding/Stack. A later // context failure retains only this explicitly retired logical member // and the exclusive bank; it never reconstructs a physical binding. m.retired = true; let after = self .bank .handoff() .bank() .with_backend_observation(|b| os_k1100_context::Backend::execution(b)); if after != before { self.retirement_quarantined = true; return Err((os_k1100_queue::Error::Quarantined, m)); } self.members[m.ordinal] = None; Ok(m) } } unsafe impl os_k1100_queue::QueueRetirementBackend for OsK1100QueueBackend { fn retire_withdrawn( &mut self, member: Self::Member, ) -> Result { self.retire_member(member) } } impl OsK1100QueueBackend { fn owner_words(&self) -> [u64; 17] { [ self.identity, &self.bank as *const _ as u64, core::mem::size_of_val(&self.bank) as u64, self.members.as_ptr() as u64, self.members.iter().filter(|m| m.is_some()).count() as u64, self.failed_tick .as_ref() .map_or(0, |t| t as *const B::QueueTick as u64), self.members[0].map_or(0, |h| h.bank()), self.members[0].map_or(0, |h| h.slot() as u64), self.members[0].map_or(0, |h| h.generation()), self.members[1].map_or(0, |h| h.bank()), self.members[1].map_or(0, |h| h.slot() as u64), self.members[1].map_or(0, |h| h.generation()), self.members[2].map_or(0, |h| h.bank()), self.members[2].map_or(0, |h| h.slot() as u64), self.members[2].map_or(0, |h| h.generation()), core::mem::size_of::() as u64, u64::from(self.retirement_quarantined), ] } } const fn os_k1100_backend_layout() -> [u64; 24] { use core::mem::{offset_of, size_of}; [ size_of::>() as u64, offset_of!(OsK1100QueueBackend, bank) as u64, offset_of!(OsK1100QueueBackend, identity) as u64, offset_of!(OsK1100QueueBackend, members) as u64, offset_of!(OsK1100QueueBackend, failed_tick) as u64, size_of::>() as u64, size_of::() as u64, offset_of!(OsK1100Member, identity) as u64, offset_of!(OsK1100Member, context) as u64, offset_of!(OsK1100Member, thread) as u64, offset_of!(OsK1100Member, ordinal) as u64, size_of::() as u64, offset_of!(OsK1100SelectionPrepared, handoff) as u64, offset_of!(OsK1100SelectionPrepared, epoch) as u64, offset_of!(OsK1100SelectionPrepared, nonce) as u64, offset_of!(OsK1100SelectionPrepared, arrived) as u64, size_of::() as u64, offset_of!(OsK1100QueueReturnPrepared, token) as u64, offset_of!(OsK1100QueueReturnPrepared, source) as u64, offset_of!(OsK1100QueueReturnPrepared, nonce) as u64, offset_of!(OsK1100QueueReturnPrepared, observed) as u64, size_of::>() as u64, offset_of!(OsK1100QueueBackend, retirement_quarantined) as u64, offset_of!(OsK1100Member, retired) as u64, ] } // Appended in the private real three-member Queue backend module. unsafe impl os_k1100_queue::WaitQueueServices for OsK1100QueueBackend { fn wait_physical_landing( &self, p: &Self::Prepared, m: &Self::Member, ) -> Result { if p.arrived || p.nonce == 0 || p.handoff.is_none() { return Err(os_k1100_queue::Error::State); } self.bank .handoff() .target_image(p.handoff.as_ref().unwrap()) .map_err(|_| os_k1100_queue::Error::Stale)?; self.member(m)?; self.bank .wait_guest_physical_landing(m.context) .map_err(|_| os_k1100_queue::Error::Context) } fn wait_capture(&self) -> Result { self.bank .os_k1100_capture() .map_err(|_| os_k1100_queue::Error::Context) } fn wait_custody( &mut self, m: &Self::Member, owned: &mut Option, resolve: bool, ) -> Result<(), os_k1100_queue::Failure> { self.member(m).map_err(|error| os_k1100_queue::Failure { error, phase: os_k1100_context::FailurePhase::BeforeSwitch, })?; self.bank .wait_guest_custody(m.context, owned, resolve) .map_err(|f| os_k1100_queue::Failure { error: if f.phase == os_k1100_context::FailurePhase::AfterSwitch { os_k1100_queue::Error::Quarantined } else { os_k1100_queue::Error::Backend }, phase: f.phase, }) } } // NEW actual wire binding after the frozen private three-member backend. unsafe impl os_k1100_queue::PolicyQueueBackend for OsK1100QueueBackend { fn policy_wire( &self, p: &Self::Prepared, target: &Self::Member, ) -> Result { let observed = self.member(target)?; let actual = p.handoff.as_ref().ok_or(os_k1100_queue::Error::State)?; let image = self .bank .handoff() .target_image(actual) .map_err(|_| os_k1100_queue::Error::Backend)?; Ok(os_k1100_queue::PolicyWire { image, root: observed.context.root, nonce: actual.nonce(), thread: observed.thread.id, }) } } fn os_k1100_backend_arrival_diagnostic(code:u64){unsafe extern "C" {fn __os_k1100_landing_error(code:u64);} unsafe{__os_k1100_landing_error(code);}} // NEW actual adapter for the independently copied three-root context backend. // Frozen core/services code remains byte exact in the generated wait modules. unsafe impl os_k1100_context::WaitContextServices for OsK1100ContextBackend<'_> { fn physical_wait_landing( &self, binding: &Self::Binding, image: &os_k1100_context::Image, nonce: u64, ) -> Result { let raw = os_k1100_execution(); if raw.cpu != 0 || !raw.irq_masked || raw.in_irq { return Err(os_k1100_context::Error::Context); } let before = ::execution(self); if !before.fp_enabled || before.critical_depth != 0 || !before.safe_point { return Err(os_k1100_context::Error::Context); } self.valid_pin(binding)?; let record = self .table .record(binding.id) .map_err(|_| os_k1100_context::Error::State)?; if record.state != os_k1100_thread::State::Ready || record.id != binding.id || nonce == 0 || os_k940_registers().0 != binding.root { return Err(os_k1100_context::Error::Capture); } let frame_address = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_ADDRESS)) }; let pc = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_PC)) }; let actual_nonce = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_DEPARTED_NONCE)) }; let layout = record.layout; if raw.sp % 16 != 0 || raw.sp < layout.usable_bottom() || raw.sp > layout.usable_top() || frame_address % 16 != 0 || frame_address < layout.usable_bottom() || frame_address .checked_add(640) .is_none_or(|end| end > layout.usable_top()) || actual_nonce != nonce { return Err(os_k1100_context::Error::Capture); } // Validate the actual owned physical address before forming a Frame // reference/copy. The assembly captures these bytes before the BL. let actual = unsafe { read_volatile(frame_address as *const os_k1100_context::Cooperative) }; let retained = unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_FRAME)) }; if actual != retained || actual.sp != image.sp() || actual.canary != os_k1100_context::CONTEXT_CANARY || actual.padding != 0 { return Err(os_k1100_context::Error::Capture); } match image { os_k1100_context::Image::Cooperative(expected) => { unsafe extern "C" { fn __os_k1100_cooperative_landed(); } if actual != *expected || pc != __os_k1100_cooperative_landed as *const () as u64 { return Err(os_k1100_context::Error::Capture); } } os_k1100_context::Image::Initial { entry, argument, sp, return_pc, } => { if pc != *entry || actual.sp != *sp || actual.x19_x30[..11] != [0; 11] || actual.x19_x30[11] != *return_pc || actual.q.iter().any(|q| *q != [0; 2]) || actual.fpcr != 0 || actual.fpsr != 0 || unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_ARGUMENT)) } != *argument { return Err(os_k1100_context::Error::Capture); } } os_k1100_context::Image::Timer(_) => return Err(os_k1100_context::Error::Capture), } if ::execution(self) != before { return Err(os_k1100_context::Error::Context); } Ok(os_k1100_wait::Execution { cpu: 0, irq_masked: true, in_irq: false, current: Some(binding.id), }) } fn take_sleep_wait( &mut self, binding: &Self::Binding, owned: &mut Option, ) -> Result<(), os_k1100_context::Failure> { let before = |error| os_k1100_context::Failure { error, phase: os_k1100_context::FailurePhase::BeforeSwitch, }; let e = ::execution(self); if e.cpu != 0 || e.daif & 128 == 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point { return Err(before(os_k1100_context::Error::Context)); } self.valid_pin(binding).map_err(before)?; let r = self .table .record(binding.id) .map_err(|_| before(os_k1100_context::Error::State))?; if r.state != os_k1100_thread::State::Sleeping || owned.is_some() || self .sleep_wait .as_ref() .is_none_or(|w| w.thread() != binding.id || w.epoch() != r.wait_epoch) { return Err(before(os_k1100_context::Error::Wake)); } *owned = self.sleep_wait.take(); Ok(()) } fn resolve_sleep_wait( &mut self, binding: &Self::Binding, owned: &mut Option, ) -> Result<(), os_k1100_context::Failure> { let before = |error| os_k1100_context::Failure { error, phase: os_k1100_context::FailurePhase::BeforeSwitch, }; let after = |error| os_k1100_context::Failure { error, phase: os_k1100_context::FailurePhase::AfterSwitch, }; let e = ::execution(self); if e.cpu != 0 || e.daif & 128 == 0 || e.in_irq || !e.fp_enabled || e.critical_depth != 0 || !e.safe_point { return Err(before(os_k1100_context::Error::Context)); } self.valid_pin(binding).map_err(before)?; let r = self .table .record(binding.id) .map_err(|_| before(os_k1100_context::Error::State))?; if r.state != os_k1100_thread::State::Sleeping || owned .as_ref() .is_none_or(|w| w.thread() != binding.id || w.epoch() != r.wait_epoch) { return Err(before(os_k1100_context::Error::Wake)); } match self.table.wake(owned.take().unwrap()) { Ok(id) if id == binding.id => {} Ok(_) => return Err(after(os_k1100_context::Error::Wake)), Err((_, actual)) => { *owned = Some(actual); return Err(before(os_k1100_context::Error::Wake)); } } // Actual Wait has been consumed. Never access Table/stack again after // drift or falsely reconstruct a consumed nonclone token. if ::execution(self) != e { return Err(after(os_k1100_context::Error::Context)); } self.refresh().map_err(after) } } unsafe impl os_k1100_bridge::K1100MainBackend for OsK1100QueueBackend> { fn k1100_execution(&self)->os_k1100_context::Execution { self.bank.handoff().bank().with_backend_observation(|b|os_k1100_context::Backend::execution(b)) } fn k1100_target_wire(&self, p: &Self::Prepared) -> Result<(os_k1100_context::Image, u64), os_k1100_wait::Error> { let e = ::k1100_execution(self); if e.cpu != 0 || e.daif&128==0 || e.in_irq || !e.fp_enabled || e.critical_depth!=0 || !e.safe_point || e.thread.is_some() { return Err(os_k1100_wait::Error::Context); } let t = p.handoff.as_ref().ok_or(os_k1100_wait::Error::State)?; if p.arrived || p.nonce == 0 || p.nonce != t.nonce() { return Err(os_k1100_wait::Error::State); } let image = self.bank.handoff().target_image(t).map_err(|_| os_k1100_wait::Error::Stale)?; if ::k1100_execution(self) != e { return Err(os_k1100_wait::Error::Context); } Ok((image, t.nonce())) } fn k1100_bootstrap_wire(&self, p: &Self::ReturnPrepared) -> Result { let e = ::k1100_execution(self); if e.cpu != 0 || e.daif&128==0 || e.in_irq || !e.fp_enabled || e.critical_depth!=0 || !e.safe_point || e.thread.is_some() { return Err(os_k1100_wait::Error::Context); } let t = p.token.as_ref().ok_or(os_k1100_wait::Error::State)?; if p.nonce == 0 || p.observed || t.nonce() == 0 { return Err(os_k1100_wait::Error::State); } let actual = self.bank.with_return_observation(|r| r.map(|r| r.bootstrap)) .ok_or(os_k1100_wait::Error::State)?; if ::k1100_execution(self) != e { return Err(os_k1100_wait::Error::Context); } Ok(actual) } } // The actual typed Context/Table/root/stack definitions above are whole // namespaced copies of the accepted predecessor. These are NEW adapters. #[repr(C,align(4096))] struct OsK1100PrimaryStack {guard:[u8;4096],usable:[u8;65536]} #[unsafe(no_mangle)] static mut OS_K1100_PRIMARY_STACK:OsK1100PrimaryStack= OsK1100PrimaryStack{guard:[0;4096],usable:[0;65536]}; unsafe impl os_k1100_context::QueueServices for OsK1100ContextBackend<'_> { type QueueTick=OsK1100QueueTick; fn queue_capture(&self)->Option { os_k1100_gate_execution(false).ok()?; unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CURRENT_CAPTURE))} } fn queue_tick(&self,tick:&Self::QueueTick) ->Result { let before=os_k1100_source_snapshot().map_err(|_|os_k1100_queue::Error::Context)?; if before.registers.irq_active || tick.issuer==0 || tick.issuer!=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_ISSUER))} || tick.observation.sequence!=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_QUEUE_TICK_LIVE))} || tick.observation.sequence!=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_RETIRED))} || Some(tick.observation.thread)!=self.current || self.current!=before.current || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DELIVERY_CLAIMS))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_FAILED))}!=0 { return Err(os_k1100_queue::Error::Tick); } let actual=self.table.record(tick.observation.thread).map_err(|_|os_k1100_queue::Error::State)?; if actual.state!=os_k1100_thread::State::Running || os_k1100_source_snapshot()!=Ok(before) {return Err(os_k1100_queue::Error::Context);} Ok(tick.observation) } fn queue_consume_tick(&mut self,tick:Self::QueueTick) { let before=os_k1100_source_snapshot().unwrap_or_else(|_|park()); ::queue_tick(self,&tick).unwrap_or_else(|_|park()); if unsafe{(&*core::ptr::addr_of!(OS_K1100_QUEUE_TICK)).is_some()} {park();} // Queue has consumed the actual Tick. Clear its separate source claim // before any postcheck; never recreate a retired hardware delivery. unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_QUEUE_TICK_LIVE),0);} if os_k1100_source_snapshot()!=Ok(before) { unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_SOURCE_FAILED),1);} park(); } } } unsafe impl turkos_kernel::runtime::policy_main::ActualBackend for OsK1100QueueBackend> { fn actual_publish_sleep_target(&self,binding:turkos_kernel::runtime::policy_main::TargetBinding) ->Result<(),os_k1100_owner::Error> { let before=os_k1100_source_snapshot().map_err(|_|os_k1100_owner::Error::Context)?; unsafe{os_k1100_publish_target(binding)}.map_err(|_|os_k1100_owner::Error::Context)?; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_NATIVE_ROLE),1);} if os_k1100_source_snapshot()!=Ok(before) {return Err(os_k1100_owner::Error::Context);} Ok(()) } fn actual_capture(&self)->Result { let before=os_k1100_source_snapshot().map_err(|_|os_k1100_owner::Error::Context)?; if before.registers.irq_active {return Err(os_k1100_owner::Error::Context);} let actual=self.bank.handoff().bank().with_backend_observation(|backend| { let capture=::queue_capture(backend) .ok_or(os_k1100_owner::Error::Context)?; let current=backend.current.ok_or(os_k1100_owner::Error::Context)?; let ordinal=backend.records.iter().position(|r|r.is_some_and(|r|r.id==current)) .ok_or(os_k1100_owner::Error::Stale)?; let record=backend.table.record(current).map_err(|_|os_k1100_owner::Error::Stale)?; let stack=backend.observations[ordinal].ok_or(os_k1100_owner::Error::Stale)?.layout; if current!=before.current.ok_or(os_k1100_owner::Error::Context)? || record.state!=os_k1100_thread::State::Running || !backend.pins[ordinal] || capture.addressstack.usable_top()) || capture.episode==0 || capture.abi!=os_k1100_context::Abi::COOPERATIVE { return Err(os_k1100_owner::Error::Context); } backend.validate_stack_owner(current).map_err(|_|os_k1100_owner::Error::Backend)?; Ok(capture) }); if os_k1100_source_snapshot()!=Ok(before) {return Err(os_k1100_owner::Error::Context);} actual } } // Native source hardware adapter. Registration and the workload entry remain // preparation only until the predecessor and complete driver are accepted. struct OsK1100TimerHardware; type OsK1100Timer<'a>=turkos_kernel::runtime::policy_timer::TimerCell< turkos_kernel::runtime::policy_source::Source>, OsK1100TimerHardware,8>; #[unsafe(no_mangle)] static mut OS_K1100_TIMER:core::mem::MaybeUninit>= core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_TIMER_LIVE:u64=0; unsafe impl turkos_kernel::runtime::policy_timer::Hardware for OsK1100TimerHardware { fn execution(&self)->os_k1100_policy::Execution { match os_k1100_source_snapshot() { Ok(s)=>os_k1100_policy::Execution{cpu:0,masked:true,in_irq:s.registers.irq_active,current:s.current}, Err(_)=>os_k1100_policy::Execution{cpu:u32::MAX,masked:false,in_irq:true,current:None}, } } fn validate(&self,irq:bool)->Result<(),turkos_kernel::runtime::policy_ingress::Error> { let snapshot=os_k1100_source_snapshot()?; if snapshot.registers.irq_active!=irq {return Err(turkos_kernel::runtime::policy_ingress::Error::Context);} os_k1100_native_source_gate(unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_ISSUER))})?; Ok(()) } fn counter(&self)->u64 {os_k960_counter()} fn halt(&self,_:turkos_kernel::runtime::policy_ingress::Error)->! {park()} } struct OsK1100NativeSource<'a> { // The nonclone claim is issued only by the retained main registration. claim:OsK1100RegistrationClaim<'a>, issuer:u64, source:u64, frequency:u64, } fn os_k1100_native_source_gate(issuer:u64)->Result { let snapshot=os_k1100_source_snapshot()?; if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_LIVE))}!=1 || issuer==0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_ISSUER))}!=issuer || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_REGISTRATION_CLAIM))}!=issuer || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_FAILED))}!=0 { return Err(turkos_kernel::runtime::policy_ingress::Error::Quarantined); } Ok(snapshot) } fn os_k1100_native_gic()->Result<[u64;3],turkos_kernel::runtime::policy_ingress::Error> { os_k1100_source_snapshot()?; let address=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_REDIS))}; if address==0 {return Err(turkos_kernel::runtime::policy_ingress::Error::Source);} Ok(unsafe{[ read_volatile((address+0x1_0100) as *const u32) as u64, read_volatile((address+0x1_0200) as *const u32) as u64, read_volatile((address+0x1_0300) as *const u32) as u64, ]}) } unsafe impl turkos_kernel::runtime::policy_source::Hardware for OsK1100NativeSource<'_> { type Snapshot=OsK1100SourceSnapshot; type Delivery=OsK1100CompletedDelivery; fn snapshot(&self)->Result { let snapshot=os_k1100_native_source_gate(self.issuer)?; if self.claim.issuer!=self.issuer {return Err(turkos_kernel::runtime::policy_ingress::Error::Foreign);} Ok(snapshot) } fn execution(&self)->os_k1100_policy::Execution { match os_k1100_source_snapshot() { Ok(s)=>os_k1100_policy::Execution{cpu:0,masked:true,in_irq:s.registers.irq_active,current:s.current}, Err(_)=>os_k1100_policy::Execution{cpu:u32::MAX,masked:false,in_irq:true,current:None}, } } fn identity(&self)->u64 {self.issuer} fn source(&self)->u64 {self.source} fn frequency(&self)->u64 {self.frequency} fn stop(&mut self)->Result<(),turkos_kernel::runtime::policy_ingress::Error> { let before=os_k1100_native_source_gate(self.issuer)?; unsafe{asm!("msr CNTP_CTL_EL0, xzr","isb","dsb sy","isb",options(nomem,nostack,preserves_flags));} let (_,control)=os_k960_timer(); if os_k1100_native_source_gate(self.issuer)?!=before || control&3!=0 { return Err(turkos_kernel::runtime::policy_ingress::Error::Context); } unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_TIMER_ARMED),0);} Ok(()) } fn quiescent(&self)->Result { let before=os_k1100_native_source_gate(self.issuer)?; if before.registers.irq_active {return Err(turkos_kernel::runtime::policy_ingress::Error::Context);} let (_,control)=os_k960_timer();let [enabled,pending,active]=os_k1100_native_gic()?; if os_k1100_native_source_gate(self.issuer)?!=before { return Err(turkos_kernel::runtime::policy_ingress::Error::Context); } let completed=unsafe{(&*core::ptr::addr_of!(OS_K1100_SOURCE_PENDING)).is_some()}; Ok(control&3==0 && enabled&(1<<30)!=0 && pending&(1<<30)==0 && active&(1<<30)==0 && !completed && unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_ARMED))}==0) } fn pending(&self)->Result,turkos_kernel::runtime::policy_ingress::Error> { os_k1100_native_source_gate(self.issuer)?; let actual=unsafe{&*core::ptr::addr_of!(OS_K1100_SOURCE_PENDING)}; match actual.as_ref() { None=>Ok(None), Some(actual) if actual.stamp.issuer==self.issuer=>Ok(Some(os_k1100_policy::TickObservation{ cpu:0,source:self.source,sequence:actual.stamp.sequence,counter:actual.stamp.counter, queue_epoch:actual.stamp.queue_epoch,current:actual.stamp.current})), Some(_)=>Err(turkos_kernel::runtime::policy_ingress::Error::Foreign), } } fn take(&mut self)->Result { let before=os_k1100_native_source_gate(self.issuer)?; if !before.registers.irq_active {return Err(turkos_kernel::runtime::policy_ingress::Error::Context);} let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DELIVERY_CLAIMS))}; let next=count.checked_add(1).ok_or(turkos_kernel::runtime::policy_ingress::Error::Exhausted)?; let actual=unsafe{(&mut *core::ptr::addr_of_mut!(OS_K1100_SOURCE_PENDING)).take()} .ok_or(turkos_kernel::runtime::policy_ingress::Error::Pending)?; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_DELIVERY_CLAIMS),next);} Ok(actual) } fn inspect(&self,actual:&Self::Delivery)->Result { let before=os_k1100_native_source_gate(self.issuer)?; if actual.stamp.issuer!=self.issuer || actual.stamp.root!=before.registers.ttbr || actual.stamp.stack_owner!=before.stack_owner || actual.stamp.current!=before.current || actual.stamp.stack_low!=before.stack_low || actual.stamp.stack_high!=before.stack_high || actual.stamp.sequence==0 || actual.stamp.counterbefore.stack_high) || actual.frame.interrupted_sp!=actual.frame_address+848 || actual.iar&0x00ff_ffff!=30 { return Err(turkos_kernel::runtime::policy_ingress::Error::Source); } match (actual.queue_tick.as_ref(),actual.stamp.current) { (None,None)=>{}, (Some(tick),Some(thread)) if tick.issuer==self.issuer && tick.observation.cpu==0 && tick.observation.source==self.source && tick.observation.thread==thread && tick.observation.slice==actual.stamp.slice && tick.observation.sequence==actual.stamp.sequence && tick.observation.counter==actual.stamp.counter=>{}, _=>return Err(turkos_kernel::runtime::policy_ingress::Error::Source), } Ok(os_k1100_policy::TickObservation{cpu:0,source:self.source,sequence:actual.stamp.sequence, counter:actual.stamp.counter,queue_epoch:actual.stamp.queue_epoch,current:actual.stamp.current}) } fn retire(&mut self,mut actual:Self::Delivery)->Result<(),(turkos_kernel::runtime::policy_ingress::Error,Self::Delivery)> { let before=match os_k1100_native_source_gate(self.issuer){Ok(s)=>s,Err(e)=>return Err((e,actual))}; if before.registers.irq_active || self.inspect(&actual).is_err() || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_QUEUE_TICK_LIVE))}!=0 || unsafe{(&*core::ptr::addr_of!(OS_K1100_QUEUE_TICK)).is_some()} { return Err((turkos_kernel::runtime::policy_ingress::Error::Pending,actual)); } let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DELIVERY_CLAIMS))}; if count==0 {return Err((turkos_kernel::runtime::policy_ingress::Error::Foreign,actual));} if let Some(tick)=actual.queue_tick.take() { unsafe{ core::ptr::addr_of_mut!(OS_K1100_QUEUE_TICK).write(Some(tick)); write_volatile(core::ptr::addr_of_mut!(OS_K1100_QUEUE_TICK_LIVE),actual.stamp.sequence); } } unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1100_SOURCE_RETIRED),actual.stamp.sequence); write_volatile(core::ptr::addr_of_mut!(OS_K1100_DELIVERY_CLAIMS),count-1); } // This actual token has now been consumed. Do not reconstruct it if // a complete fresh snapshot detects drift after the mailbox transfer. if os_k1100_native_source_gate(self.issuer)!=Ok(before) { unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_SOURCE_FAILED),1);} } Ok(()) } } impl Drop for OsK1100NativeSource<'_> { fn drop(&mut self) { use turkos_kernel::runtime::policy_source::Hardware as _; let before=os_k1100_native_source_gate(self.issuer).unwrap_or_else(|_|park()); if before.registers.irq_active || self.quiescent()!=Ok(true) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_LIVE))}!=0 || self.claim.issuer!=self.issuer || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_REGISTRATION_CLAIM))}!=self.issuer || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DELIVERY_CLAIMS))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_QUEUE_TICK_LIVE))}!=0 || unsafe{(&*core::ptr::addr_of!(OS_K1100_QUEUE_TICK)).is_some()} || os_k1100_native_source_gate(self.issuer)!=Ok(before) {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_REGISTRATION_CLAIM),0);} // The separate main registration still owns the actual predecessor // transfer and restores it only after this true claim is consumed. } } // Only this independent TimerCell pointer is available to IRQ. Its Producer // retains completed hardware receipts; its Ingress has no scheduler backend, // Runtime, allocator, Table, mutex, Policy or subscription pointer. #[unsafe(no_mangle)] extern "C" fn os_k1100_dispatch(pointer:*mut os_k920_exception::Frame)->u64 { let expected=os_k1100_gate_kind(false,true).unwrap_or_else(|_|park()); let raw=os_k1100_registers();let address=pointer as u64; if raw.irq_active || raw.cpu!=0 || raw.daif&128==0 || raw.spsel!=1 || pointer.is_null() || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_LIVE))}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_ARMED))}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_STACK_VALID))}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_ACTIVE))}!=0 || unsafe{(&*core::ptr::addr_of!(OS_K1100_SOURCE_PENDING)).is_some()} {park();} let stamp=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_STAMP).cast::())}; if stamp.issuer==0 || stamp.issuer!=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_ISSUER))} || stamp.current!=expected.current || stamp.root!=raw.ttbr || stamp.stack_owner!=expected.stack_owner || stamp.stack_low!=expected.stack_low || stamp.stack_high!=expected.stack_high {park();} // The retained execution bounds exclude the low/high canaries. The // inherited exception frame verifier takes aligned backing page bounds. let page_low=stamp.stack_low&!4095; let page_high=stamp.stack_high.checked_add(16).unwrap_or_else(||park()); if !page_high.is_multiple_of(4096) || page_lowpage_high) {park();} let stack=os_k920_exception::Stack::new(page_low,page_high).unwrap_or_else(|_|park()); stack.frame_address(address).unwrap_or_else(|_|park()); let frame=unsafe{read_volatile(pointer)}; os_k920_kernel_irq_return(&frame,address,stack,core::ptr::addr_of!(_text_start) as u64, core::ptr::addr_of!(_text_end) as u64,__os_k920_kernel_tco_observed()).unwrap_or_else(|_|park()); unsafe extern "C" {static __os_k1100_timer_wait_loop:u8;static __os_k1100_timer_wait_mask:u8;} if frame.vector_slot!=5 || frame.spsr&31!=5 || frame.elr&3!=0 || frame.elr=core::ptr::addr_of!(__os_k1100_timer_wait_mask) as u64 || frame.interrupted_sp!=address+848 {park();} let iar:u64; unsafe{asm!("mrs {iar}, ICC_IAR1_EL1",iar=out(reg)iar,options(nomem,nostack,preserves_flags));} if iar&0x00ff_ffff>=1020 {return 0;} // Refusal preserves a genuinely acknowledged unexpected interrupt; no // synthetic completion or EOI/DIR occurs on this uncertain branch. if iar&0x00ff_ffff!=30 {park();} let (cval,ctl)=os_k960_timer();let counter=os_k960_counter(); let previous=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_COUNT))}; let sequence=previous.checked_add(1).unwrap_or_else(||park()); if sequence!=stamp.sequence || cval!=stamp.deadline || ctl&7!=5 || counter()).irq()} .unwrap_or_else(|_|park()); if !produced {park();} unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1100_LAST_IRQ_FRAME),frame); write_volatile(core::ptr::addr_of_mut!(OS_K1100_LAST_IRQ_ADDRESS),address); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_COUNT),total); write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_LAST_COUNTER),counter); write_volatile(core::ptr::addr_of_mut!(OS_K1100_IRQ_COUNT),sequence); } os_k1100_checkpoint_irq(address,counter); unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K660_IRQ_ACTIVE),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_IRQ_ACTIVE),0); } 0 } // The Mutex window is opened only after the timer is genuinely quiescent and // all scheduler loans have ended. Its metadata contains no owner reference. #[derive(Clone,Copy)] struct OsK1100MutexWindow { expected:OsK1100ExpectedExecution, original_daif:u64, } #[unsafe(no_mangle)] static mut OS_K1100_MUTEX_WINDOW:core::mem::MaybeUninit=core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_MUTEX_WINDOW_LIVE:u64=0; struct OsK1100MutexMask; fn os_k1100_mutex_snapshot()->Result<(OsK1100MutexWindow,OsK1100Registers),turkos_kernel::runtime::policy_locking::Error> { let r=os_k1100_registers(); if r.cpu!=0 || r.current_el!=4 || r.irq_active || r.cpacr&(3<<20)!=(3<<20) || !r.sp.is_multiple_of(16) || r.daif&!0x3c0!=0 { return Err(turkos_kernel::runtime::policy_locking::Error::Context); } if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_MUTEX_WINDOW_LIVE))}!=1 { return Err(turkos_kernel::runtime::policy_locking::Error::Context); } let window=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_MUTEX_WINDOW).cast::())}; let e=window.expected;let old=e.registers; if e.current.is_none() || e.identity==0 || e.stack_owner==0 || r.spe.stack_high || r.ttbr!=old.ttbr || r.tcr!=old.tcr || r.sctlr!=old.sctlr || r.spsel!=old.spsel || r.cpacr!=old.cpacr || r.daif&!128!=window.original_daif&!128 { return Err(turkos_kernel::runtime::policy_locking::Error::Context); } Ok((window,r)) } fn os_k1100_same_mutex_execution(before:(OsK1100MutexWindow,OsK1100Registers),after:(OsK1100MutexWindow,OsK1100Registers),wanted_daif:u64)->bool { let (b,br)=before;let (a,ar)=after; b.expected.identity==a.expected.identity && b.expected.current==a.expected.current && b.expected.stack_owner==a.expected.stack_owner && b.expected.stack_low==a.expected.stack_low && b.expected.stack_high==a.expected.stack_high && b.original_daif==a.original_daif && br.cpu==ar.cpu && br.ttbr==ar.ttbr && br.tcr==ar.tcr && br.sctlr==ar.sctlr && br.spsel==ar.spsel && br.cpacr==ar.cpacr && br.current_el==ar.current_el && !ar.irq_active && ar.daif==wanted_daif } unsafe impl turkos_kernel::runtime::policy_mutex::Mask for OsK1100MutexMask { fn cpu(&self)->u16 {u16::try_from(os_k1100_registers().cpu).unwrap_or(u16::MAX)} fn thread(&self)->Option { os_k1100_mutex_snapshot().ok().and_then(|(w,_)|w.expected.current) } fn in_irq(&self)->bool {os_k1100_registers().irq_active} fn daif(&self)->u64 {os_k1100_registers().daif} fn identity(&self)->u64 {os_k1100_mutex_snapshot().map_or(0,|(w,_)|w.expected.identity)} fn validate(&self)->bool {os_k1100_mutex_snapshot().is_ok()} fn mask(&self)->Result { let before=os_k1100_mutex_snapshot()?; if before.1.daif&128!=0 {return Err(turkos_kernel::runtime::policy_locking::Error::Context);} unsafe{asm!("msr DAIFSet, #2","isb",options(nomem,nostack,preserves_flags));} let after=os_k1100_mutex_snapshot()?; if !os_k1100_same_mutex_execution(before,after,before.1.daif|128) { return Err(turkos_kernel::runtime::policy_locking::Error::Quarantined); } Ok(before.1.daif) } fn restore(&self,saved:u64)->Result<(),turkos_kernel::runtime::policy_locking::Error> { let before=os_k1100_mutex_snapshot()?; if saved!=before.0.original_daif || saved&128!=0 || before.1.daif!=(saved|128) { return Err(turkos_kernel::runtime::policy_locking::Error::Quarantined); } unsafe{asm!("msr DAIF, {saved}","isb",saved=in(reg)saved,options(nomem,nostack,preserves_flags));} let after=os_k1100_mutex_snapshot()?; if !os_k1100_same_mutex_execution(before,after,saved) { return Err(turkos_kernel::runtime::policy_locking::Error::Quarantined); } Ok(()) } fn halt(&self,_:turkos_kernel::runtime::policy_locking::Error)->! {park()} } /// # Safety /// The actual independent timer is stopped/drained/inactive/nonpending. The /// published current was obtained after genuine physical landing and actual /// Table activation. No Policy/WaitSet/Queue/Bank/Allocator loan remains live. unsafe fn os_k1100_open_mutex_window()->Result<(),turkos_kernel::runtime::policy_locking::Error> { let e=os_k1100_gate_execution(false).map_err(|_|turkos_kernel::runtime::policy_locking::Error::Context)?; if e.current.is_none() || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_MUTEX_WINDOW_LIVE))}!=0 { return Err(turkos_kernel::runtime::policy_locking::Error::Context); } let r=os_k1100_registers(); unsafe{ core::ptr::addr_of_mut!(OS_K1100_MUTEX_WINDOW).cast::() .write(OsK1100MutexWindow{expected:e,original_daif:r.daif&!128}); write_volatile(core::ptr::addr_of_mut!(OS_K1100_MUTEX_WINDOW_LIVE),1); } Ok(()) } fn os_k1100_close_mutex_window()->Result<(),turkos_kernel::runtime::policy_locking::Error> { let (_,r)=os_k1100_mutex_snapshot()?; if r.daif&128==0 {return Err(turkos_kernel::runtime::policy_locking::Error::Context);} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_MUTEX_WINDOW_LIVE),0);} os_k1100_gate_execution(false).map(|_|()).map_err(|_|turkos_kernel::runtime::policy_locking::Error::Quarantined) } fn os_k1100_close_departed_mutex_window()->Result<(),turkos_kernel::runtime::policy_locking::Error> { let current=os_k1100_gate_execution(false).map_err(|_|turkos_kernel::runtime::policy_locking::Error::Context)?; if current.current.is_some() || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_NATIVE_ROLE))}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_VALID))}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_MUTEX_WINDOW_LIVE))}!=1 { return Err(turkos_kernel::runtime::policy_locking::Error::Quarantined); } let old=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_MUTEX_WINDOW).cast::())}; let target=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_EXECUTION).cast::())}; let r=os_k1100_registers(); if old.expected.current.is_none() || old.expected.identity!=current.identity || old.expected.stack_owner!=current.stack_owner || old.expected.stack_low!=current.stack_low || old.expected.stack_high!=current.stack_high || r.daif!=old.original_daif|128 || target.registers.ttbr!=r.ttbr || target.identity!=current.identity { return Err(turkos_kernel::runtime::policy_locking::Error::Quarantined); } // Actual commit moved Table.current to None and selected the retained // target root. Retire the old task's window while its caller stack remains // live; restore/unmask is forbidden across this authorized departure. unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_MUTEX_WINDOW_LIVE),0);} Ok(()) } // NEW phased BSS construction. No whole Queue/Bank/Backend result returns // through the factory caller. Frozen constructors are invoked unchanged. #[repr(C, align(16))] struct OsK1100ConstructionSlot { bytes: core::mem::MaybeUninit<[u8; core::mem::size_of::>()]>, } #[unsafe(no_mangle)] static mut OS_K1100_CONSTRUCTION: [OsK1100ConstructionSlot; 2] = [const { OsK1100ConstructionSlot { bytes: core::mem::MaybeUninit::uninit(), } }; 2]; #[unsafe(no_mangle)] static mut OS_K1100_CONSTRUCTION_STATES: [u64; 8] = [0; 8]; struct OsK1100Factory<'a> { slot: usize, kind: u64, ownership: core::marker::PhantomData>, } impl<'a> OsK1100Factory<'a> { unsafe fn start(backend: OsK1100ContextBackend<'a>) -> Self { let mut stage = Self { slot: 0, kind: 1, ownership: core::marker::PhantomData, }; stage.gate(); if unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_CONSTRUCTION_STATES).cast::()) } != 0 { os_k1100_retain_failure(backend, 20); } let pointer = stage.pointer::>(0); unsafe { pointer.write(backend); } stage.publish(1); stage } fn gate(&self) { let e = os_k1100_execution(); if e.cpu != 0 || !e.irq_masked || e.in_irq || os_k960_timer().1 & 1 != 0 { park(); } } fn pointer(&self, slot: usize) -> *mut T { if slot >= 2 || core::mem::size_of::() > core::mem::size_of::() || core::mem::align_of::() > 16 { park(); } unsafe { core::ptr::addr_of_mut!(OS_K1100_CONSTRUCTION) .cast::() .add(slot) .cast::() } } fn source(&self, kind: u64) -> *mut T { self.gate(); if self.kind != kind || unsafe { read_volatile( core::ptr::addr_of!(OS_K1100_CONSTRUCTION_STATES) .cast::() .add(self.slot), ) } != kind { park(); } self.pointer(self.slot) } fn destination(&self) -> *mut T { let slot = 1 - self.slot; if unsafe { read_volatile( core::ptr::addr_of!(OS_K1100_CONSTRUCTION_STATES) .cast::() .add(slot), ) } != 0 { park(); } self.pointer(slot) } fn publish(&mut self, kind: u64) { unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_CONSTRUCTION_STATES) .cast::() .add(self.slot), kind, ); write_volatile( core::ptr::addr_of_mut!(OS_K1100_CONSTRUCTION_STATES) .cast::() .add(2), self.slot as u64, ); write_volatile( core::ptr::addr_of_mut!(OS_K1100_CONSTRUCTION_STATES) .cast::() .add(3), kind, ); } } fn commit(&mut self, kind: u64) { let old = self.slot; self.slot = 1 - old; self.kind = kind; unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_CONSTRUCTION_STATES) .cast::() .add(old), 0, ); } self.publish(kind); } } macro_rules! os_k1100_constructor_stage { ($name:ident,$from:ty,$to:ty,$before:expr,$after:expr,$constructor:path) => { impl<'a> OsK1100Factory<'a> { #[inline(never)] fn $name(&mut self) { let source = self.source::<$from>($before); let destination = self.destination::<$to>(); // Destination validation and the actual CPU gate precede the // sole read. The Rust local owns source until one BSS write. let actual = unsafe { source.read() }; match $constructor(actual) { Ok(actual) => { unsafe { destination.write(actual); } self.commit($after); } Err((_, actual)) => { unsafe { source.write(actual); } os_k1100_word(122, $before); park(); } } } } }; } os_k1100_constructor_stage!( context, OsK1100ContextBackend<'a>, os_k1100_context::ContextBank, 3>, 1, 2, os_k1100_context::ContextBank::, 3>::new ); os_k1100_constructor_stage!( asynchronous, os_k1100_context::ContextBank, 3>, os_k1100_context::AsyncBank, 3>, 2, 3, os_k1100_context::AsyncBank::, 3>::new ); os_k1100_constructor_stage!( completion, os_k1100_context::AsyncBank, 3>, OsK1100Bank<'a>, 3, 4, OsK1100Bank::new ); os_k1100_constructor_stage!( members, OsK1100Bank<'a>, OsK1100Backend<'a>, 4, 5, OsK1100Backend::new ); os_k1100_constructor_stage!( policy, OsK1100Backend<'a>, os_k1100_queue::Queue, 3>, 5, 6, os_k1100_queue::Queue::, 3>::new ); os_k1100_constructor_stage!( selection, os_k1100_queue::Queue, 3>, os_k1100_queue::AsyncQueue, 3>, 6, 7, os_k1100_queue::AsyncQueue::, 3>::new ); impl<'a> OsK1100Factory<'a> { #[inline(never)] fn attach(&mut self) { let core = unsafe { &mut *self.source::, 3>>(2) }; for ordinal in 0..3 { let binding = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1100_BINDINGS) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); let h = match core.attach(binding) { Ok(h) => h, Err((_, binding)) => { unsafe { core::ptr::addr_of_mut!(OS_K1100_BINDINGS) .cast::>() .add(ordinal) .write(Some(binding)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1100_CONTEXT_HANDLES) .cast::>() .add(ordinal) .write(Some(h)); } } } #[inline(never)] fn issue(&mut self) { let backend = unsafe { &mut *self.source::>(5) }; for ordinal in 0..3 { let h = unsafe { read_volatile( core::ptr::addr_of!(OS_K1100_CONTEXT_HANDLES) .cast::>() .add(ordinal), ) } .unwrap_or_else(|| park()); let m = backend.issue(h, ordinal).unwrap_or_else(|_| park()); unsafe { core::ptr::addr_of_mut!(OS_K1100_MEMBERS) .cast::>() .add(ordinal) .write(Some(m)); } } } #[inline(never)] fn enqueue(&mut self) { let queue = unsafe { &mut *self.source::, 3>>(6) }; for ordinal in 0..3 { let m = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1100_MEMBERS) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); let class = if ordinal == 1 { os_k1100_queue::Class::Low } else { os_k1100_queue::Class::Normal }; let h = match queue.enqueue(m, class) { Ok(h) => h, Err((_, m)) => { unsafe { core::ptr::addr_of_mut!(OS_K1100_MEMBERS) .cast::>() .add(ordinal) .write(Some(m)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1100_QUEUE_HANDLES) .cast::>() .add(ordinal) .write(Some(h)); } } } } static mut OS_K1100_SPARES: [Option; 3] = [const { None }; 3]; static mut OS_K1100_FAILED_ROOTS: [Option; 3] = [const { None }; 3]; static mut OS_K1100_BINDINGS: [Option; 3] = [const { None }; 3]; static mut OS_K1100_MEMBERS: [Option; 3] = [const { None }; 3]; #[unsafe(no_mangle)] static mut OS_K1100_CONTEXT_HANDLES: [Option; 3] = [None; 3]; #[unsafe(no_mangle)] static mut OS_K1100_QUEUE_HANDLES: [Option; 3] = [None; 3]; static mut OS_K1100_FAILURE_CUSTODY: core::mem::MaybeUninit> = core::mem::MaybeUninit::uninit(); fn os_k1100_retain_failure(actual: T, kind: u64) -> ! { if core::mem::size_of::() > core::mem::size_of::() || core::mem::align_of::() > core::mem::align_of::() { park(); } unsafe { core::ptr::addr_of_mut!(OS_K1100_FAILURE_CUSTODY).cast::().write(actual); } os_k1100_word(120, kind); os_k1100_word(121, core::mem::size_of::() as u64); park() } os_k1100_constructor_stage!( return_queue, os_k1100_queue::AsyncQueue,3>, OsK1100Queue<'a>,7,8,OsK1100Queue::new ); impl<'a> OsK1100Factory<'a> { #[inline(never)] fn bridge(&mut self) { let source = self.source::>(8); let destination = self.destination::>(); let handles = core::array::from_fn(|i| unsafe { read_volatile(core::ptr::addr_of!(OS_K1100_QUEUE_HANDLES) .cast::>().add(i)) }.unwrap_or_else(|| park())); let queue = unsafe { source.read() }; match OsK1100Bridge::new(queue, handles, OsK1100Port) { Ok(actual) => { unsafe { destination.write(actual); } self.commit(9); } Err((_, actual, _authority)) => { unsafe { source.write(actual); } os_k1100_word(122,8); park(); } } } } os_k1100_constructor_stage!(policy_wait_core,OsK1100Bridge<'a>,OsK1100Core<'a>,9,10,OsK1100Core::new); // Main retains the actual predecessor IRQ transfer. The independent producer // receives a nonclone scalar claim with a Rust lifetime, never this pointer. struct OsK1100SourceLease<'a> { transfer:OsK940IrqTransfer<'a>, issuer:u64, identity:u64, claimed:bool, } static mut OS_K1100_FAILED_SOURCE_LEASE:core::mem::MaybeUninit>= core::mem::MaybeUninit::uninit(); impl<'a> OsK1100SourceLease<'a> { /// # Safety /// irq and redistributor are the genuine still-owned accepted predecessor /// controller, with no old registration binding or completed delivery. /// The primary expected record is already bound to its retained guarded /// Image stack. This method never accepts a user-supplied MMIO address. unsafe fn new(irq:&'a mut OsK660Irq,redistributor:u64,issuer:u64)->Result { let before=os_k1100_source_snapshot().map_err(|_|"policy-source-context")?; if before.registers.irq_active || issuer==0 || redistributor==0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_LIVE))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_REGISTRATION_CLAIM))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL))}!=[0;8] || (0..3).any(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL_PINS) .cast::().add(i))}!=0) || unsafe{(&*core::ptr::addr_of!(OS_K1100_SOURCE_PENDING)).is_some()} || unsafe{(&*core::ptr::addr_of!(OS_K1100_QUEUE_TICK)).is_some()} { return Err("policy-source-already-owned"); } unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1100_REDIS),redistributor); asm!("msr CNTP_CTL_EL0, xzr","isb","dsb sy","isb",options(nomem,nostack,preserves_flags)); } let (_,control)=os_k960_timer();let [enabled,pending,active]=os_k1100_native_gic().map_err(|_|"policy-source-gic")?; if control&3!=0 || enabled&(1<<30)==0 || pending&(1<<30)!=0 || active&(1<<30)!=0 || os_k1100_source_snapshot()!=Ok(before) {return Err("policy-source-not-quiescent");} let transfer=os_k940_transfer_irq(irq,redistributor)?; // Actual transfer acquisition precedes publication. No constructor // failure may discard it after an uncertain callback/context change. if os_k1100_source_snapshot()!=Ok(before) { // Retain the real transfer in place. No failed publication invokes // its Drop or replaces the borrowed predecessor with a scalar. unsafe{core::ptr::addr_of_mut!(OS_K1100_FAILED_SOURCE_LEASE) .cast::>().write(Self{transfer,issuer, identity:before.identity,claimed:false});} park(); } unsafe{ // The Wait ingress has its own published source. Its epoch comes // from the retained IRQ claim and its owner from the verified // execution identity, so a fresh producer cannot inherit K1080. write_volatile(core::ptr::addr_of_mut!(OS_K1100_WAIT_SIGNAL), [1,30,issuer,before.identity,0,0,0,0]); write_volatile(core::ptr::addr_of_mut!(OS_K1100_SOURCE_ISSUER),issuer); write_volatile(core::ptr::addr_of_mut!(OS_K1100_SOURCE_LIVE),1); } Ok(Self{transfer,issuer,identity:before.identity,claimed:false}) } fn producer<'b>(&'b mut self)->Result,&'static str> { let before=os_k1100_source_snapshot().map_err(|_|"policy-source-context")?; if before.registers.irq_active || before.identity!=self.identity || self.claimed || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_LIVE))}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_ISSUER))}!=self.issuer || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_REGISTRATION_CLAIM))}!=0 { return Err("policy-source-claim"); } let frequency=os_k960_frequency();if frequency==0 {return Err("policy-source-frequency");} if os_k1100_source_snapshot()!=Ok(before) {park();} self.claimed=true; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_REGISTRATION_CLAIM),self.issuer);} Ok(OsK1100NativeSource{claim:OsK1100RegistrationClaim{issuer:self.issuer, registration:core::marker::PhantomData},issuer:self.issuer,source:30,frequency}) } } impl Drop for OsK1100SourceLease<'_> { fn drop(&mut self) { let before=os_k1100_source_snapshot().unwrap_or_else(|_|park()); let (_,control)=os_k960_timer();let [enabled,pending,active]=os_k1100_native_gic().unwrap_or_else(|_|park()); if before.registers.irq_active || before.identity!=self.identity || self.transfer.predecessor_bindings()!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_LIVE))}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_ISSUER))}!=self.issuer || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL).cast::())}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL).cast::().add(1))}!=30 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL).cast::().add(2))}!=self.issuer || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL).cast::().add(3))}!=self.identity || (0..3).any(|i|unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WAIT_SIGNAL_PINS) .cast::().add(i))}!=0) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_REGISTRATION_CLAIM))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DELIVERY_CLAIMS))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_QUEUE_TICK_LIVE))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_FAILED))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_ARMED))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_OWNERS_LIVE))}!=0 || unsafe{(&*core::ptr::addr_of!(OS_K1100_SOURCE_PENDING)).is_some()} || unsafe{(&*core::ptr::addr_of!(OS_K1100_QUEUE_TICK)).is_some()} || control&3!=0 || enabled&(1<<30)==0 || pending&(1<<30)!=0 || active&(1<<30)!=0 || os_k1100_source_snapshot()!=Ok(before) {park();} unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1100_WAIT_SIGNAL),[0;8]); write_volatile(core::ptr::addr_of_mut!(OS_K1100_SOURCE_LIVE),0); } // The actual contained transfer performs its frozen restoration now. } } // The predecessor's complete phased real owner constructors are copied above. // This final stage moves the actual WaitSet into the pinned opaque owner cell. struct OsK1100Installation<'a> { pointer:*mut OsK1100OwnerCell<'a>, active:bool, ownership:core::marker::PhantomData>, } static mut OS_K1100_EXTRACT_OWNER:core::mem::MaybeUninit>= core::mem::MaybeUninit::uninit(); static mut OS_K1100_EXTRACT_CORE:core::mem::MaybeUninit>= core::mem::MaybeUninit::uninit(); impl<'a> OsK1100Installation<'a> { fn pin(&self)->core::pin::Pin<&OsK1100OwnerCell<'a>> { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if !self.active || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_OWNERS_LIVE))}!=1 {park();} unsafe{core::pin::Pin::new_unchecked(&*self.pointer)} } #[inline(never)] fn extract(mut self)->OsK1100ContextBackend<'a> { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); let owner=core::ptr::addr_of_mut!(OS_K1100_EXTRACT_OWNER) .cast::>>(); unsafe{OsK1100OwnerCell::k1100_take_value_in_place(self.pointer,owner)} .unwrap_or_else(|_|park()); let actual=owner.cast::>(); let core=core::ptr::addr_of_mut!(OS_K1100_EXTRACT_CORE) .cast::>>(); unsafe{OsK1100MainOwner::native_take_core_in_place(actual,core)} .unwrap_or_else(|_|park()); let q=os_k1100_queue::os_k1100_return_layout::(); let offset=OsK1100Core::os_k1100_backend_offset()+OsK1100Bridge::os_k1100_queue_offset() +q[1]+q[6]+q[11]+core::mem::offset_of!(OsK1100Backend,bank) as u64; let source=unsafe{core.cast::().add(offset as usize)}.cast::>(); let mut destination=core::mem::MaybeUninit::uninit(); unsafe{OsK1100Bank::os_k1040_take_backend_in_place(source,&mut destination)} .unwrap_or_else(|_|park()); self.active=false; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_OWNERS_LIVE),0);} unsafe{destination.assume_init()} } } impl Drop for OsK1100Installation<'_> { fn drop(&mut self) {if self.active {park();}} } impl<'a> OsK1100Factory<'a> { #[inline(never)] fn install(mut self)->OsK1100Installation<'a> { let source=self.source::>(10); let destination=self.destination::>(); let handles=core::array::from_fn(|i|unsafe{ read_volatile(core::ptr::addr_of!(OS_K1100_QUEUE_HANDLES) .cast::>().add(i))}.unwrap_or_else(||park())); let threads=core::array::from_fn(|i|unsafe{(&*source).os_k1100_thread(i)} .unwrap_or_else(|_|park())); let signal=os_k1100_wait::Ingress::new(OsK1100Port).unwrap_or_else(|_|park()); let initialized=unsafe{OsK1100MainOwner::native_construct_in_place(source,destination,signal,handles,threads)}; self.commit(11);if initialized.is_err(){park();} let source=self.source::>(11); let destination=self.destination::>(); match unsafe{OsK1100OwnerCell::k1100_construct_in_place(source,destination,OsK1100MainGate)} { Ok(())=>self.commit(12), Err((_,moved))=>{if moved {self.commit(12);}park();}, } let source=self.source::>(12); let pointer=core::ptr::addr_of_mut!(OS_K1100_OWNER).cast::>(); unsafe{ core::ptr::copy_nonoverlapping(source,pointer,1); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CONSTRUCTION_STATES) .cast::().add(self.slot),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_OWNERS_LIVE),1); } self.kind=13;self.publish(13); OsK1100Installation{pointer,active:true,ownership:core::marker::PhantomData} } } #[inline(never)] fn os_k1100_factory<'a>( memory: &'a mut OsK660Memory, tables: &'a mmu::Tables, bootstrap: u64, ) -> OsK1100Installation<'a> { let sp: u64; unsafe { asm!("mov {sp}, sp",sp=out(reg)sp,options(nomem,nostack,preserves_flags)); } os_k1100_word(112, sp); unsafe extern "C" { fn __os_k1100_a_entry(); fn __os_k1100_b_entry(); fn __os_k1100_c_entry(); } for ordinal in 0..3 { let root = os_k1100_create_root(memory, tables).unwrap_or_else(|_| park()); unsafe { write_volatile( core::ptr::addr_of_mut!(OS_K1100_ROOT_ADDRESSES) .cast::() .add(ordinal), root.address(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1100_ROOT_GENERATIONS) .cast::() .add(ordinal), root.generation(), ); write_volatile( core::ptr::addr_of_mut!(OS_K1100_ROOT_LIVE) .cast::() .add(ordinal), 1, ); } let stack = match os_k1100_allocate_stack(memory, tables, ordinal, root) { Ok(s) => s, Err((_, root)) => { unsafe { core::ptr::addr_of_mut!(OS_K1100_FAILED_ROOTS) .cast::>() .add(ordinal) .write(Some(root)); } park() } }; unsafe { core::ptr::addr_of_mut!(OS_K1100_SPARES) .cast::>() .add(ordinal) .write(Some(stack)); } } let mut table = match OsK1100Table::new(OsK1100StackBackend { memory, tables, trace: true, }) { Ok(t) => t, Err((_, b)) => os_k1100_retain_failure(b, 1), }; let entries = [ __os_k1100_a_entry as *const () as u64, __os_k1100_b_entry as *const () as u64, __os_k1100_c_entry as *const () as u64, ]; let mut ids = [None; 3]; for ordinal in 0..3 { let stack = unsafe { (&mut *core::ptr::addr_of_mut!(OS_K1100_SPARES) .cast::>() .add(ordinal)) .take() } .unwrap_or_else(|| park()); ids[ordinal] = Some( match table.create(stack, entries[ordinal], ordinal as u64) { Ok(id) => id, Err((_, stack)) => { unsafe { core::ptr::addr_of_mut!(OS_K1100_SPARES) .cast::>() .add(ordinal) .write(Some(stack)); } os_k1100_retain_failure(table, 2) } }, ); } let mut backend = OsK1100ContextBackend { table, bootstrap, current: None, pins: [false; 3], records: [None; 3], observations: [None; 3], guards: [0; 3], sleep_wait: None, sequence: 0, }; for ordinal in 0..3 { let binding = match backend.pin(ids[ordinal].unwrap(), ordinal) { Ok(b) => b, Err(_) => os_k1100_retain_failure(backend, 3), }; unsafe { core::ptr::addr_of_mut!(OS_K1100_BINDINGS) .cast::>() .add(ordinal) .write(Some(binding)); } } let mut factory = unsafe { OsK1100Factory::start(backend) }; factory.context(); factory.attach(); factory.asynchronous(); factory.completion(); factory.members(); factory.issue(); factory.policy(); factory.enqueue(); factory.selection(); factory.return_queue(); factory.bridge(); factory.policy_wait_core(); factory.install() } use turkos_kernel::runtime::{policy_driver as os_k1100_driver,policy_backend as os_k1100_backend, policy_timer as os_k1100_timer,policy_ingress as os_k1100_ingress,policy_source as os_k1100_source}; type OsK1100Driver<'loan,'owners>=os_k1100_driver::Driver<'loan,OsK1100MainOwner<'owners>,OsK1100MainGate>; type OsK1100PolicyBackend<'loan,'owners>=os_k1100_backend::Backend<'loan,OsK1100MainOwner<'owners>,OsK1100MainGate, os_k1100_source::Source>,OsK1100TimerHardware,8>; type OsK1100Policy<'loan,'owners>=os_k1100_policy::Policy,3>; #[unsafe(no_mangle)] static mut OS_K1100_DRIVER:core::mem::MaybeUninit>=core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_POLICY:core::mem::MaybeUninit>=core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_POLICY_HANDLES:[Option;3]=[None;3]; static mut OS_K1100_POLICY_IDENTITY:u64=1; #[unsafe(no_mangle)] static mut OS_K1100_DRIVER_LIVE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_POLICY_LIVE:u64=0; static mut OS_K1100_NATIVE_READY:u64=0; static mut OS_K1100_NATIVE_ROLE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_LANDING_ERROR:u64=0; #[unsafe(no_mangle)] extern "C" fn __os_k1100_landing_error(code:u64) { unsafe{if read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_ERROR))==0 { write_volatile(core::ptr::addr_of_mut!(OS_K1100_LANDING_ERROR),code); }} } static mut OS_K1100_BOOT_EXPECTED:core::mem::MaybeUninit=core::mem::MaybeUninit::uninit(); static mut OS_K1100_BOOT_EXPECTED_LIVE:u64=0; type OsK1100MutexBackend=turkos_kernel::runtime::policy_mutex::Backend<'static, OsK1100MainOwner<'static>,OsK1100MainGate,os_k1100_source::Source>, OsK1100TimerHardware,OsK1100MutexMask,8>; type OsK1100Mutex=turkos_kernel::runtime::policy_locking::mutex::Mutex; #[unsafe(no_mangle)] static mut OS_K1100_MUTEX:core::mem::MaybeUninit=core::mem::MaybeUninit::uninit(); #[unsafe(no_mangle)] static mut OS_K1100_MUTEX_LIVE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_MUTEX_LEASE:Option=None; #[unsafe(no_mangle)] static mut OS_K1100_MUTEX_WAIT:Option=None; #[unsafe(no_mangle)] static mut OS_K1100_INVERSION_PHASE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_C_SLEEP_OBSERVED:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_PHASE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_TASK_ENTRIES:[u64;3]=[0;3]; #[unsafe(no_mangle)] static mut OS_K1100_HOUR_START:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_HOUR_END:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_LONG_RUN:core::mem::MaybeUninit=core::mem::MaybeUninit::uninit(); // The immutable compiled mode chooses a short matrix episode or the separate // genuine hour episode. Host ACK writes cannot change this mode. #[unsafe(no_mangle)] static mut OS_K1100_EPISODE_MODE:u64=if cfg!(feature="os-policy-hour-acceptance") {1}else{0}; #[unsafe(no_mangle)] static mut OS_K1085_NEGATIVE_REENQUEUE_MODE:u64=0; #[unsafe(no_mangle)] static mut OS_K1090_INVERSION_REPORTED:u64=0; #[unsafe(no_mangle)] static mut OS_K1090_NEGATIVE_MODE:u64=0; #[unsafe(no_mangle)] static mut OS_K1090_BUSY_LOOP_REJECTED:u64=0; #[unsafe(no_mangle)] static mut OS_K1095_ACCOUNTING_REPORTED:u64=0; #[unsafe(no_mangle)] static mut OS_K1095_NEGATIVE_MODE:u64=0; #[unsafe(no_mangle)] static mut OS_K1095_DOUBLE_CLASS_REJECTED:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_NEGATIVE_MODE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_LATE_WAKE_REFUSALS:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_LONG_RUN_REPORTED:u64=0; #[unsafe(no_mangle)] static mut OS_K1085_NEGATIVE_REENQUEUE_ATTEMPTS:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_CHECKPOINT:[u64;24]=[0;24]; #[unsafe(no_mangle)] static mut OS_K1100_CHECKPOINT_READY:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_CHECKPOINT_ACK:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_CHECKPOINT_SEQUENCE:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_EVENT_TOTAL:u64=0; #[unsafe(no_mangle)] static mut OS_K1100_AUTHORITY_BEFORE:[u64;328]=[0;328]; #[unsafe(no_mangle)] static mut OS_K1100_AUTHORITY_AFTER:[u64;328]=[0;328]; static mut OS_K1100_LAST_PERIODIC:u64=0; static mut OS_K1100_LAST_PERIODIC_PROGRESS:[u64;3]=[0;3]; static mut OS_K1100_IRQ_ARCHIVE_COUNTER:u64=0; static mut OS_K1100_PUBLISHED_PROGRESS:[u64;8]=[0;8]; #[unsafe(no_mangle)] extern "C" fn __os_k1100_policy_identity()->u64 { let before=os_k1100_source_snapshot().unwrap_or_else(|_|park()); if before.registers.irq_active {park();} let value=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_POLICY_IDENTITY))}; let next=value.checked_add(1).unwrap_or_else(||park()); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_POLICY_IDENTITY),next);} if os_k1100_source_snapshot()!=Ok(before) {park();} value } struct OsK1100TimerInstallation<'a> { pointer:*mut OsK1100Timer<'a>,active:bool, ownership:core::marker::PhantomData>, } impl<'a> OsK1100TimerInstallation<'a> { fn new(actual:OsK1100NativeSource<'a>)->Self { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_LIVE))}!=0 { os_k1100_retain_failure(actual,0x110); } let source=match os_k1100_source::Source::new(actual) { Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x111), }; let ingress=match os_k1100_ingress::Ingress::new(source) { Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x112), }; let timer=match OsK1100Timer::new(ingress,OsK1100TimerHardware) { Ok(actual)=>actual,Err((_,actual,_))=>os_k1100_retain_failure(actual,0x113), }; let pointer=core::ptr::addr_of_mut!(OS_K1100_TIMER).cast::>(); unsafe{pointer.write(timer);write_volatile(core::ptr::addr_of_mut!(OS_K1100_TIMER_LIVE),1);} Self{pointer,active:true,ownership:core::marker::PhantomData} } fn pin(&self)->core::pin::Pin<&OsK1100Timer<'a>> { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if !self.active || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_LIVE))}!=1 {park();} unsafe{core::pin::Pin::new_unchecked(&*self.pointer)} } #[inline(never)] fn finish(mut self) { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); let mut actual=core::mem::MaybeUninit::uninit(); unsafe{OsK1100Timer::k1100_take_producer_in_place(self.pointer,&mut actual)} .unwrap_or_else(|_|park()); // NativeSource::Drop is authorized only after the real consumer and // ingress have been extracted; predecessor transfer remains main-owned. unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_TIMER_LIVE),0);} self.active=false; let source=unsafe{actual.assume_init()}; let hardware=match source.into_hardware() { Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x118), }; drop(hardware); } } impl Drop for OsK1100TimerInstallation<'_> {fn drop(&mut self){if self.active {park();}}} // This main-only token retains both installation lifetimes. No reference to a // scheduler value is put in IRQ; IRQ reaches only the independent TimerCell. struct OsK1100PolicyInstallation<'loan,'owners> { owner:core::marker::PhantomData<&'loan OsK1100Installation<'owners>>, timer:core::marker::PhantomData<&'loan OsK1100TimerInstallation<'owners>>, active:bool, } impl<'loan,'owners> OsK1100PolicyInstallation<'loan,'owners> { fn new(owner:&'loan OsK1100Installation<'owners>,timer:&'loan OsK1100TimerInstallation<'owners>)->Self { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DRIVER_LIVE))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_POLICY_LIVE))}!=0 {park();} let owner=owner.pin();let timer=timer.pin(); timer.drained().unwrap_or_else(|_|park()); let driver=OsK1100Driver::new(owner).unwrap_or_else(|_|park()); unsafe{core::ptr::addr_of_mut!(OS_K1100_DRIVER).cast::>().write(driver); write_volatile(core::ptr::addr_of_mut!(OS_K1100_DRIVER_LIVE),1);} let consumer=timer.consumer().unwrap_or_else(|_|park()); let backend=match OsK1100PolicyBackend::new(owner,consumer) { Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x114), }; let policy=match OsK1100Policy::new(backend) { Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x115), }; let pointer=core::ptr::addr_of_mut!(OS_K1100_POLICY).cast::>(); unsafe{pointer.write(policy);write_volatile(core::ptr::addr_of_mut!(OS_K1100_POLICY_LIVE),1);} for i in 0..3 { let handle=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_QUEUE_HANDLES) .cast::>().add(i))}.unwrap_or_else(||park()); let actual=owner.subscribe(handle).unwrap_or_else(|_|park()); let registered=match unsafe{(&mut *pointer).register(actual)} { Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x116), }; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_POLICY_HANDLES) .cast::>().add(i),Some(registered));} } unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_NATIVE_READY),1);} Self{owner:core::marker::PhantomData,timer:core::marker::PhantomData,active:true} } #[inline(never)] fn finish(mut self,owner:&OsK1100Installation<'owners>,timer:&OsK1100TimerInstallation<'owners>) { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); let cell=owner.pin();let timer=timer.pin(); timer.quiesce().unwrap_or_else(|_|park());timer.drained().unwrap_or_else(|_|park()); let policy=core::ptr::addr_of_mut!(OS_K1100_POLICY).cast::>(); for i in 0..3 { let handle=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_POLICY_HANDLES) .cast::>().add(i))}.unwrap_or_else(||park()); let actual=unsafe{(&mut *policy).unregister(handle)}.unwrap_or_else(|_|park()); // The subscription was actually consumed by unregister before its // slot is cleared. A release refusal preserves the returned token. unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_POLICY_HANDLES) .cast::>().add(i),None);} match cell.release(actual) { Ok(())=>{},Err((_,actual))=>os_k1100_retain_failure(actual,0x119), } } let actual=unsafe{policy.read()}; let backend=match actual.into_backend() { Ok(actual)=>actual,Err(actual)=>{ unsafe{policy.write(actual);}park(); }, }; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_POLICY_LIVE),0);} let consumer=backend.into_consumer(); match timer.release_consumer(consumer) { Ok(())=>{},Err((_,actual))=>os_k1100_retain_failure(actual,0x11a), } let driver=core::ptr::addr_of_mut!(OS_K1100_DRIVER).cast::>(); for ordinal in 0..3 { unsafe{(&*driver).k1100_retire(timer,ordinal)}.unwrap_or_else(|_|park()); } match unsafe{driver.read()}.finish() { Ok(())=>{},Err((_,actual))=>{unsafe{driver.write(actual);}park();}, } unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_DRIVER_LIVE),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_NATIVE_READY),0);} self.active=false; } } impl Drop for OsK1100PolicyInstallation<'_,'_> {fn drop(&mut self){if self.active {park();}}} fn os_k1100_timer_pin()->core::pin::Pin<&'static OsK1100Timer<'static>> { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_LIVE))}!=1 {park();} unsafe{core::pin::Pin::new_unchecked(&*core::ptr::addr_of!(OS_K1100_TIMER).cast::())} } fn os_k1100_driver_pointer()->*const OsK1100Driver<'static,'static> { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DRIVER_LIVE))}!=1 {park();} core::ptr::addr_of!(OS_K1100_DRIVER).cast() } fn os_k1100_policy_pointer()->*mut OsK1100Policy<'static,'static> { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_POLICY_LIVE))}!=1 {park();} core::ptr::addr_of_mut!(OS_K1100_POLICY).cast() } #[inline(never)] fn os_k1100_charge_pending()->Option { let timer=os_k1100_timer_pin(); // IRQ completed the actual disable/DSB/EOI/DIR before publishing this // receipt. Ingress::quiesce refuses a still-pending receipt, so validate // physical quiescence first, then let Policy consume the real receipt. let before=os_k1100_source_snapshot().unwrap_or_else(|_|park()); let (_,control)=os_k960_timer();let [enabled,pending,active]=os_k1100_native_gic().unwrap_or_else(|_|park()); if before.registers.irq_active || control&3!=0 || enabled&(1<<30)==0 || pending&(1<<30)!=0 || active&(1<<30)!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_ARMED))}!=0 || os_k1100_source_snapshot()!=Ok(before) {park();} let policy=os_k1100_policy_pointer(); let charged=match unsafe{(&mut *policy).charge_pending()} { Ok(actual)=>actual,Err((_,actual))=>{ if let Some(actual)=actual {os_k1100_retain_failure(actual,0x117);}park(); }, }; if charged.is_none() {park();} if cfg!(feature="os-runtime-product") && unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_INVERSION_PHASE))}>=2 && unsafe{read_volatile(core::ptr::addr_of!(OS_K1090_INVERSION_REPORTED))}==0 { let policy=unsafe{&*policy}; let handles=core::ptr::addr_of!(OS_K1100_POLICY_HANDLES) .cast::>(); let waiter=unsafe{read_volatile(handles)}.unwrap_or_else(||park()); let owner=unsafe{read_volatile(handles.add(1))}.unwrap_or_else(||park()); let waiter=policy.record(waiter).unwrap_or_else(|_|park()); let owner=policy.record(owner).unwrap_or_else(|_|park()); if waiter.inversion_ticks>0 && waiter.chain_len>0 { let report=os_k1100_policy::os_scheduler_policy_inversion(&waiter,&owner) .unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1090_NEGATIVE_MODE))}==1 { let before=policy.snapshot(); let mut fault_waiter=waiter; let mut fault_owner=owner; fault_waiter.observation.state=os_k1100_thread::State::Running; fault_owner.observation.state=os_k1100_thread::State::Sleeping; if os_k1100_policy::os_scheduler_policy_inversion(&fault_waiter,&fault_owner) !=Err(os_k1100_policy::Error::BusyLoop) || policy.snapshot()!=before || policy.record(waiter.handle)!=Ok(waiter) || policy.record(owner.handle)!=Ok(owner) {park();} write_line("TURKOS/OS/K1090: INJECTED-BUSYLOOP-REJECTED LIVE-RECORDS-UNCHANGED"); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1090_BUSY_LOOP_REJECTED),1);} } write_str("TURKOS/OS/K1090: DELAY=0x");write_hex(report.delay_ticks as usize); write_str(" WAITER=0x");write_hex(report.waiter.slot()); write_str(" OWNER=0x");write_hex(report.owner.slot()); write_str(" CHAIN=0x");write_hex(report.chain_len);write_line(""); write_line("TURKOS/OS/K1090: PASS"); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1090_INVERSION_REPORTED),1);} } } if cfg!(feature="os-runtime-product") && unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_INVERSION_PHASE))}>=2 && unsafe{read_volatile(core::ptr::addr_of!(OS_K1095_ACCOUNTING_REPORTED))}==0 { let policy=unsafe{&*policy}; let handles=core::ptr::addr_of!(OS_K1100_POLICY_HANDLES) .cast::>(); let records=core::array::from_fn(|i|{ let handle=unsafe{read_volatile(handles.add(i))}.unwrap_or_else(||park()); policy.record(handle).unwrap_or_else(|_|park()) }); let before=policy.snapshot(); let report=os_k1100_policy::os_scheduler_policy_accounting(before,&records) .unwrap_or_else(|_|park()); if report.ticks==0 || report.cpu_running+report.cpu_idle!=report.ticks {park();} if unsafe{read_volatile(core::ptr::addr_of!(OS_K1095_NEGATIVE_MODE))}==1 { let mut fault=before; fault.cpu_running=fault.cpu_running.checked_add(1).unwrap_or_else(||park()); if os_k1100_policy::os_scheduler_policy_accounting(fault,&records) !=Err(os_k1100_policy::Error::State) || policy.snapshot()!=before || records.iter().enumerate().any(|(i,record)|{ let handle=unsafe{read_volatile(handles.add(i))}.unwrap_or_else(||park()); policy.record(handle)!=Ok(*record) }) {park();} write_line("TURKOS/OS/K1095: INJECTED-DOUBLE-CLASS-REJECTED LIVE-RECORDS-UNCHANGED"); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1095_DOUBLE_CLASS_REJECTED),1);} } write_str("TURKOS/OS/K1095: TICKS=0x");write_hex(report.ticks as usize); write_str(" CPU-RUNNING=0x");write_hex(report.cpu_running as usize); write_str(" CPU-IDLE=0x");write_hex(report.cpu_idle as usize); write_str(" MEMBER-TICKS=0x");write_hex(report.member_ticks as usize); write_str(" SLEEPING=0x");write_hex(report.sleeping_ticks as usize);write_line(""); write_line("TURKOS/OS/K1095: PASS"); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1095_ACCOUNTING_REPORTED),1);} } if cfg!(feature="os-runtime-product") && unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_NEGATIVE_MODE))}==1 && unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_INVERSION_PHASE))}>=2 && unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_LATE_WAKE_REFUSALS))}==0 { let policy=unsafe{&*policy}; let handle=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_POLICY_HANDLES) .cast::>())}.unwrap_or_else(||park()); let record=policy.record(handle).unwrap_or_else(|_|park()); let before=policy.snapshot(); let stale=record.thread.generation().checked_add(1).unwrap_or_else(||park()); for _ in 0..1000 { if os_k1100_policy::os_scheduler_policy_long_run( record.thread,record.thread,stale)!=Err(os_k1100_policy::Error::Resource) {park();} } if policy.snapshot()!=before || policy.record(handle)!=Ok(record) {park();} write_line("TURKOS/OS/K1100: LATE-GENERATION-WAKE-STORM-REJECTED ATTEMPTS=0x00000000000003e8 LIVE-RECORDS-UNCHANGED"); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_LATE_WAKE_REFUSALS),1000);} } timer.quiesce().unwrap_or_else(|_|park()); timer.drained().unwrap_or_else(|_|park()); // Policy really consumed its completed physical delivery before this // actual queue token is moved. Every early refusal restores the same token. let tick=unsafe{(&mut *core::ptr::addr_of_mut!(OS_K1100_QUEUE_TICK)).take()}; let Some(tick)=tick else { let snapshot=charged.unwrap_or_else(||park()); os_k1100_event(0x93,snapshot.last_sequence,snapshot.cpu_idle,snapshot.ticks);return None; }; let observation=tick.observation; match unsafe{(&*os_k1100_driver_pointer()).k1100_charge(timer,tick)} { Ok(actual)=>{os_k1100_event(0x91,actual.sequence,observation.thread.slot() as u64,observation.slice);Some(actual)},Err((_,actual))=>{ if let Some(actual)=actual {unsafe{core::ptr::addr_of_mut!(OS_K1100_QUEUE_TICK).write(Some(actual));}} park(); }, } } #[inline(never)] fn os_k1100_arm_native(hz:u64)->u64 { let timer=os_k1100_timer_pin();timer.quiesce().unwrap_or_else(|_|park()); timer.drained().unwrap_or_else(|_|park()); let before=os_k1100_source_snapshot().unwrap_or_else(|_|park()); if hz==0 || before.registers.irq_active || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_VALID))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_QUEUE_TICK_LIVE))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DELIVERY_CLAIMS))}!=0 || unsafe{(&*core::ptr::addr_of!(OS_K1100_QUEUE_TICK)).is_some()} || unsafe{(&*core::ptr::addr_of!(OS_K1100_SOURCE_PENDING)).is_some()} {park();} let (epoch,current)=unsafe{(&*os_k1100_driver_pointer()).k1100_tick_binding(timer)}.unwrap_or_else(|_|park()); let (progress,transitions)=unsafe{(&*os_k1100_driver_pointer()).k1100_progress(timer)}.unwrap_or_else(|_|park()); let policy=unsafe{(&*os_k1100_policy_pointer()).snapshot()}; if current.map(|(thread,_)|thread)!=before.current || os_k1100_source_snapshot()!=Ok(before) {park();} let sequence=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_COUNT))} .checked_add(1).unwrap_or_else(||park()); let counter=os_k960_counter();let period=(os_k960_frequency()/hz).max(1); let deadline=counter.checked_add(period).unwrap_or_else(||park()); let stamp=OsK1100SourceStamp{issuer:unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_ISSUER))}, sequence,counter,queue_epoch:epoch,slice:current.map_or(0,|(_,slice)|slice),current:before.current, root:before.registers.ttbr,stack_owner:before.stack_owner,stack_low:before.stack_low, stack_high:before.stack_high,deadline}; // All actual owner observations and loans ended above. Only source scalars // and genuine timer registers are accessed after enabling this deadline. unsafe{ write_volatile(core::ptr::addr_of_mut!(OS_K1100_PUBLISHED_PROGRESS),[ progress[0],progress[1],progress[2],transitions,policy.ticks,policy.last_sequence, policy.last_counter,policy.queue_epoch]); core::ptr::addr_of_mut!(OS_K1100_TIMER_STAMP).cast::().write(stamp); write_volatile(core::ptr::addr_of_mut!(OS_K1100_IRQ_STACK_VALID),1); write_volatile(core::ptr::addr_of_mut!(OS_K1100_TIMER_ARMED),1); asm!("dsb ish","msr CNTP_CVAL_EL0, {deadline}","isb","msr CNTP_CTL_EL0, {one}","isb", deadline=in(reg)deadline,one=in(reg)1u64,options(nomem,nostack,preserves_flags)); } let (actual,control)=os_k960_timer(); if actual!=deadline || control&3!=1 || os_k1100_source_snapshot()!=Ok(before) {park();} sequence } #[inline(never)] fn os_k1100_capture_native(address:u64) { let expected=os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if expected.current.is_none() || address%16!=0 || addressexpected.stack_high) {park();} let episode=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CAPTURE_EPISODE))}; if episode==0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_LIVE_CAPTURE_ADDRESS))}!=address {park();} let frame=unsafe{read_volatile(address as *const os_k1100_context::Cooperative)}; if frame.sp!=address+640 || frame.canary!=os_k1100_context::CONTEXT_CANARY || frame.padding!=0 {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_CURRENT_CAPTURE),Some(os_k1100_context::Capture{ image:os_k1100_context::Image::Cooperative(frame),address,abi:os_k1100_context::Abi::COOPERATIVE,episode}));} } #[inline(never)] fn os_k1100_set_native_wire(wire:os_k1100_queue::PolicyWire)->*const OsK1100Wire { let (frame,pc,argument,kind)=match wire.image { os_k1100_context::Image::Initial{entry,argument,sp,return_pc}=>{ let mut frame=os_k1100_context::Cooperative::EMPTY;frame.sp=sp;frame.x19_x30[11]=return_pc; (frame,entry,argument,0) },os_k1100_context::Image::Cooperative(frame)=>(frame,frame.x19_x30[11],0,1),_=>park(), }; if wire.nonce==0 {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_DEPARTED_NONCE),wire.nonce); write_volatile(core::ptr::addr_of_mut!(OS_K1100_WIRE),OsK1100Wire{frame,pc,argument,nonce:wire.nonce,kind});} core::ptr::addr_of!(OS_K1100_WIRE) } #[inline(never)] fn os_k1100_prepare_ready_native()->*const OsK1100Wire { let timer=os_k1100_timer_pin();timer.drained().unwrap_or_else(|_|park()); let driver=os_k1100_driver_pointer(); let wire=unsafe{(&*driver).prepare_ready(timer)}.unwrap_or_else(|_|park()); let target=unsafe{(&*driver).k1100_target(timer)}.unwrap_or_else(|_|park()); unsafe{os_k1100_publish_target(target)}.unwrap_or_else(|_|park()); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_NATIVE_ROLE),0);} unsafe{(&*driver).depart_ready(timer)}.unwrap_or_else(|_|park()); os_k1100_set_native_wire(wire) } #[unsafe(no_mangle)] extern "C" fn __os_k1100_prepare_return(address:u64)->*const OsK1100Wire { os_k1100_capture_native(address); let capture=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CURRENT_CAPTURE))}.unwrap_or_else(||park()); let timer=os_k1100_timer_pin();timer.quiesce().unwrap_or_else(|_|park()); timer.drained().unwrap_or_else(|_|park());let driver=os_k1100_driver_pointer(); unsafe{(&*driver).k1100_prepare_return(timer,capture)}.unwrap_or_else(|_|park()); let current=os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_BOOT_EXPECTED_LIVE))}!=1 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PENDING_VALID))}!=0 {park();} let boot=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_BOOT_EXPECTED).cast::())}; if boot.identity!=current.identity || boot.current.is_some() {park();} unsafe{core::ptr::addr_of_mut!(OS_K1100_PENDING_EXECUTION).cast::().write(boot); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PENDING_THREAD),None); write_volatile(core::ptr::addr_of_mut!(OS_K1100_NATIVE_ROLE),2); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PENDING_VALID),1);} let frame=unsafe{(&*driver).k1100_depart_return(timer)}.unwrap_or_else(|_|park()); unsafe extern "C" {fn __os_k1100_boot_landed();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_WIRE),OsK1100Wire{ frame,pc:__os_k1100_boot_landed as *const () as u64,argument:0,nonce:0,kind:2});} core::ptr::addr_of!(OS_K1100_WIRE) } #[unsafe(no_mangle)] extern "C" fn __os_k1100_return_landed(address:u64) { os_k1100_admit_landing(address).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_NATIVE_ROLE))}!=2 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_ADDRESS))}!=address {park();} let frame=unsafe{read_volatile(address as *const os_k1100_context::Cooperative)}; let wire=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_WIRE))}; let original=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_BOOT_CAPTURE_ADDRESS))}; if wire.kind!=2 || wire.frame!=frame || address!=original || frame.sp!=address+640 || frame.canary!=os_k1100_context::CONTEXT_CANARY || frame.padding!=0 {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_LANDING_FRAME),frame);} let timer=os_k1100_timer_pin(); unsafe{(&*os_k1100_driver_pointer()).k1100_landed_return(timer)}.unwrap_or_else(|_|park()); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_CURRENT_CAPTURE),None); write_volatile(core::ptr::addr_of_mut!(OS_K1100_LIVE_CAPTURE_ADDRESS),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PENDING_VALID),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_NATIVE_ROLE),0);} } #[unsafe(no_mangle)] extern "C" fn __os_k1100_prepare_cooperative(address:u64,request:u64)->*const OsK1100Wire { if request!=0 {park();}os_k1100_capture_native(address);os_k1100_prepare_ready_native() } #[unsafe(no_mangle)] extern "C" fn __os_k1100_landed(address:u64) { os_k1100_admit_landing(address).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_ADDRESS))}!=address {park();} let frame=unsafe{read_volatile(address as *const os_k1100_context::Cooperative)}; if frame.sp!=address+640 || frame.canary!=os_k1100_context::CONTEXT_CANARY || frame.padding!=0 {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_LANDING_FRAME),frame);} let timer=os_k1100_timer_pin();let driver=os_k1100_driver_pointer(); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_NATIVE_ROLE))}==1 { match turkos_kernel::runtime::policy_mutex::land(unsafe{&*driver},timer) { Ok(())=>{}, Err(error)=>{if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_LANDING_ERROR))}==0 { unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_LANDING_ERROR), 0x100+error as u64);} }park();}, } } else { match unsafe{(&*driver).land_ready(timer)} { Ok(())=>{}, Err(error)=>{unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_LANDING_ERROR), error as u64+1);}park();}, } } unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_CURRENT_CAPTURE),None); write_volatile(core::ptr::addr_of_mut!(OS_K1100_LIVE_CAPTURE_ADDRESS),0);} os_k1100_log_completed_landing(); } #[inline(never)] fn os_k1100_log_completed_landing() { let current=os_k1100_gate_execution(false).unwrap_or_else(|_|park()).current.unwrap_or_else(||park()); let timer=os_k1100_timer_pin(); let (_,completed)=unsafe{(&*os_k1100_driver_pointer()).k1100_progress(timer)}.unwrap_or_else(|_|park()); os_k1100_event(0x90,current.slot() as u64,current.generation(),completed); } fn os_k1100_mutex_pointer()->*mut OsK1100Mutex { os_k1100_gate_execution(false).unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_MUTEX_LIVE))}!=1 {park();} core::ptr::addr_of_mut!(OS_K1100_MUTEX).cast() } fn os_k1100_mutex_context()->turkos_kernel::runtime::policy_locking::spin::Context { let (window,raw)=os_k1100_mutex_snapshot().unwrap_or_else(|_|park()); if raw.daif&128!=0 {park();} let thread=window.expected.current.unwrap_or_else(||park()); let task=turkos_kernel::runtime::policy_locking::Task{id:thread.slot() as u64+1,generation:thread.generation()}; unsafe{turkos_kernel::runtime::policy_locking::spin::Context::new(0,Some(task),false)}.unwrap_or_else(|_|park()) } #[inline(never)] fn os_k1100_mutex_init() { let timer=os_k1100_timer_pin();timer.quiesce().unwrap_or_else(|_|park());timer.drained().unwrap_or_else(|_|park()); unsafe{os_k1100_open_mutex_window()}.unwrap_or_else(|_|park()); let backend=match OsK1100MutexBackend::new(unsafe{&*os_k1100_driver_pointer()},timer,OsK1100MutexMask) { Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x11b), }; let actual=match OsK1100Mutex::new(backend) { Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x11c), }; unsafe{core::ptr::addr_of_mut!(OS_K1100_MUTEX).cast::().write(actual); write_volatile(core::ptr::addr_of_mut!(OS_K1100_MUTEX_LIVE),1);} turkos_kernel::runtime::policy_mutex::synchronize(unsafe{&*os_k1100_mutex_pointer()}).unwrap_or_else(|_|park()); os_k1100_close_mutex_window().unwrap_or_else(|_|park()); } #[inline(never)] fn os_k1100_mutex_begin_request(address:u64) { os_k1100_capture_native(address); let timer=os_k1100_timer_pin();timer.quiesce().unwrap_or_else(|_|park());timer.drained().unwrap_or_else(|_|park()); unsafe{os_k1100_open_mutex_window()}.unwrap_or_else(|_|park()); } #[inline(never)] fn os_k1100_mutex_request_retained()->bool { let pointer=os_k1100_mutex_pointer(); unsafe{asm!("msr DAIFClr, #2","isb",options(nomem,nostack,preserves_flags));} let actual=unsafe{(&mut *pointer).request(&os_k1100_mutex_context())}.unwrap_or_else(|_|park()); // Fast acquisition has no scheduler callback, so mask explicitly. Waiting // returned only after the real backend committed Sleep and kept I=1. unsafe{asm!("msr DAIFSet, #2","isb",options(nomem,nostack,preserves_flags));} match actual { turkos_kernel::runtime::policy_locking::mutex::Request::Owned(actual)=>{ if unsafe{(&*core::ptr::addr_of!(OS_K1100_MUTEX_LEASE)).is_some()} {os_k1100_retain_failure(actual,0x11d);} unsafe{core::ptr::addr_of_mut!(OS_K1100_MUTEX_LEASE).write(Some(actual));} false }, turkos_kernel::runtime::policy_locking::mutex::Request::Waiting(actual)=>{ if unsafe{(&*core::ptr::addr_of!(OS_K1100_MUTEX_WAIT)).is_some()} {os_k1100_retain_failure(actual,0x11e);} unsafe{core::ptr::addr_of_mut!(OS_K1100_MUTEX_WAIT).write(Some(actual)); write_volatile(core::ptr::addr_of_mut!(OS_K1100_INVERSION_PHASE),2);} true }, } } #[inline(never)] fn os_k1100_mutex_finish_request(waiting:bool)->*const OsK1100Wire { let pointer=os_k1100_mutex_pointer(); turkos_kernel::runtime::policy_mutex::synchronize(unsafe{&*pointer}).unwrap_or_else(|_|park()); if waiting { let timer=os_k1100_timer_pin(); let wire=unsafe{(&*os_k1100_driver_pointer()).k1100_sleep_wire(timer)}.unwrap_or_else(|_|park()); os_k1100_owned_scalar_checkpoint(0x22); os_k1100_close_departed_mutex_window().unwrap_or_else(|_|park()); os_k1100_set_native_wire(wire) } else { os_k1100_close_mutex_window().unwrap_or_else(|_|park());core::ptr::null() } } #[unsafe(no_mangle)] extern "C" fn __os_k1100_prepare_mutex(address:u64)->*const OsK1100Wire { os_k1100_mutex_begin_request(address); let waiting=os_k1100_mutex_request_retained(); os_k1100_mutex_finish_request(waiting) } #[inline(never)] fn os_k1100_mutex_release(claim:bool) { let timer=os_k1100_timer_pin();timer.quiesce().unwrap_or_else(|_|park());timer.drained().unwrap_or_else(|_|park()); unsafe{os_k1100_open_mutex_window()}.unwrap_or_else(|_|park()); let pointer=os_k1100_mutex_pointer(); unsafe{asm!("msr DAIFClr, #2","isb",options(nomem,nostack,preserves_flags));} let ctx=os_k1100_mutex_context(); let lease=if claim { let actual=unsafe{(&mut *core::ptr::addr_of_mut!(OS_K1100_MUTEX_WAIT)).take()}.unwrap_or_else(||park()); match unsafe{(&mut *pointer).claim(&ctx,actual)} { Ok(actual)=>actual,Err((_,actual))=>{ unsafe{core::ptr::addr_of_mut!(OS_K1100_MUTEX_WAIT).write(Some(actual));}park(); }, } } else {unsafe{(&mut *core::ptr::addr_of_mut!(OS_K1100_MUTEX_LEASE)).take()}.unwrap_or_else(||park())}; match unsafe{(&mut *pointer).release(&ctx,lease)} { Ok(())=>{},Err((_,actual))=>{ unsafe{core::ptr::addr_of_mut!(OS_K1100_MUTEX_LEASE).write(Some(actual));}park(); }, } unsafe{asm!("msr DAIFSet, #2","isb",options(nomem,nostack,preserves_flags));} turkos_kernel::runtime::policy_mutex::synchronize(unsafe{&*pointer}).unwrap_or_else(|_|park()); os_k1100_close_mutex_window().unwrap_or_else(|_|park()); } #[inline(never)] fn os_k1100_useful_tick(hz:u64)->os_k1100_queue::Charge { let timer=os_k1100_timer_pin(); unsafe{(&*os_k1100_driver_pointer()).work(timer)}.unwrap_or_else(|_|park()); os_k1100_log_completed_work(); let sequence=os_k1100_arm_native(hz); unsafe extern "C" {fn __os_k1100_timer_wait(sequence:u64)->u64;} if unsafe{__os_k1100_timer_wait(sequence)}==0 {park();} os_k1100_charge_pending().unwrap_or_else(||park()) } #[inline(never)] fn os_k1100_log_completed_work() { let current=os_k1100_gate_execution(false).unwrap_or_else(|_|park()).current.unwrap_or_else(||park()); let timer=os_k1100_timer_pin(); let (progress,_)=unsafe{(&*os_k1100_driver_pointer()).k1100_progress(timer)}.unwrap_or_else(|_|park()); os_k1100_event(0x92,current.slot() as u64,current.generation(),progress[current.slot()]); } #[unsafe(no_mangle)] extern "C" fn __os_k1100_task_work(ordinal:u64) { let current=os_k1100_gate_execution(false).unwrap_or_else(|_|park()).current.unwrap_or_else(||park()); if ordinal>=3 || unsafe{(&*core::ptr::addr_of!(OS_K1100_CONTEXT_HANDLES))}[ordinal as usize].is_none() || current.slot()!=ordinal as usize {park();} let entries=unsafe{core::ptr::addr_of_mut!(OS_K1100_TASK_ENTRIES).cast::().add(ordinal as usize)}; if unsafe{read_volatile(entries)}!=0 {park();}unsafe{write_volatile(entries,1);} unsafe extern "C" {fn __os_k1100_cooperate(request:u64);fn __os_k1100_mutex_request();} if ordinal==0 { os_k1100_mutex_init();os_k1100_useful_tick(2000);unsafe{__os_k1100_cooperate(0);} unsafe{__os_k1100_mutex_request();} // Normal A really Sleeps behind Low B. os_k1100_mutex_release(true); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_INVERSION_PHASE),4);} os_k1100_native_checkpoint(0x23); } else if ordinal==1 { unsafe{__os_k1100_mutex_request();write_volatile(core::ptr::addr_of_mut!(OS_K1100_INVERSION_PHASE),1);} os_k1100_native_checkpoint(0x21); while unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_C_SLEEP_OBSERVED))}==0 { os_k1100_useful_tick(2000);unsafe{__os_k1100_cooperate(0);} } os_k1100_mutex_release(false); os_k1100_native_checkpoint(0x25); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_INVERSION_PHASE),3);} } else { while unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_INVERSION_PHASE))}<2 { os_k1100_useful_tick(2000);unsafe{__os_k1100_cooperate(0);} } os_k1100_useful_tick(2000); os_k1100_native_checkpoint(0x24); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_C_SLEEP_OBSERVED),1);__os_k1100_cooperate(0);} } while unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_INVERSION_PHASE))}<4 { os_k1100_useful_tick(2000);unsafe{__os_k1100_cooperate(0);} } loop { if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PHASE))}==3 {return;} let hz=if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PHASE))}<2 {2000}else{32}; let charge=os_k1100_useful_tick(hz); os_k1100_maybe_long_run(); if charge.reschedule {unsafe{__os_k1100_cooperate(0);}} } } #[inline(never)] fn os_k1100_maybe_long_run() { let timer=os_k1100_timer_pin();let driver=os_k1100_driver_pointer(); let (progress,transitions)=unsafe{(&*driver).k1100_progress(timer)}.unwrap_or_else(|_|park()); let phase=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PHASE))}; if cfg!(feature="os-runtime-product") && phase<2 && unsafe{read_volatile(core::ptr::addr_of!(OS_K1085_NEGATIVE_REENQUEUE_MODE))}==1 && unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_INVERSION_PHASE))}==4 { unsafe{(&*driver).k1100_negative_duplicate_requeue(timer)}.unwrap_or_else(|_|park()); let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K1085_NEGATIVE_REENQUEUE_ATTEMPTS))}; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1085_NEGATIVE_REENQUEUE_ATTEMPTS), count.checked_add(1).unwrap_or_else(||park()));} } if phase<2 && transitions>=10_000 { if progress.iter().any(|p|*p==0) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_INVERSION_PHASE))}!=4 {park();} if cfg!(feature="os-runtime-product") && unsafe{read_volatile(core::ptr::addr_of!(OS_K1085_NEGATIVE_REENQUEUE_MODE))}==0 { let policy=unsafe{&*os_k1100_policy_pointer()}; let mut slices=[0u64;3];let mut useful=[0u64;3]; for index in 0..3 { let handle=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_POLICY_HANDLES) .cast::>().add(index))}.unwrap_or_else(||park()); let record=policy.record(handle).unwrap_or_else(|_|park()); slices[index]=record.observation.slices; useful[index]=record.observation.progress; } if os_k1100_policy::os_scheduler_policy_fairness( slices[0],slices[1],slices[2],useful[0],useful[1],useful[2])!=1 {park();} write_line("TURKOS/OS/K1085: PASS"); } if cfg!(feature="os-runtime-product") && unsafe{read_volatile(core::ptr::addr_of!(OS_K1085_NEGATIVE_REENQUEUE_MODE))}==1 { let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K1085_NEGATIVE_REENQUEUE_ATTEMPTS))}; if count<1000 {park();} write_str("TURKOS/OS/K1085: DIAGNOSTIC DUPLICATE-REENQUEUE-REJECTED ATTEMPTS=0x"); write_hex(count as usize);write_line(""); } if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_EPISODE_MODE))}==0 { unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_PHASE),3);} os_k1100_native_checkpoint(0x1095); return; } os_k1100_native_checkpoint(0x3600); // Host starts its independent monotonic clock before ACK. let policy=os_k1100_policy_pointer(); let start=os_k960_counter(); let actual=unsafe{(&mut *policy).begin_long_run()}.unwrap_or_else(|_|park()); unsafe{core::ptr::addr_of_mut!(OS_K1100_LONG_RUN).cast::().write(actual); write_volatile(core::ptr::addr_of_mut!(OS_K1100_HOUR_START),start); write_volatile(core::ptr::addr_of_mut!(OS_K1100_LAST_PERIODIC),start); write_volatile(core::ptr::addr_of_mut!(OS_K1100_IRQ_ARCHIVE_COUNTER),start); write_volatile(core::ptr::addr_of_mut!(OS_K1100_LAST_PERIODIC_PROGRESS),progress); write_volatile(core::ptr::addr_of_mut!(OS_K1100_PHASE),2);} } else if phase==2 { let now=os_k960_counter();let frequency=os_k960_frequency(); let start=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_HOUR_START))}; let last=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_LAST_PERIODIC))}; // Validate the old elapsed gap before publishing a new checkpoint time. if now.checked_sub(last).is_none_or(|gap|gap>frequency.checked_mul(60).unwrap_or_else(||park())) {park();} if now-last>=frequency.checked_mul(30).unwrap_or_else(||park()) { let previous=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_LAST_PERIODIC_PROGRESS))}; if (0..3).any(|i|progress[i]<=previous[i]) {park();} let run=unsafe{&*core::ptr::addr_of!(OS_K1100_LONG_RUN).cast::()}; let status=unsafe{(&*os_k1100_policy_pointer()).workload_status(run)}.unwrap_or_else(|_|park()); if status.failed.is_some() || status.maximum_progress_gap>frequency*60 {park();} os_k1100_native_checkpoint(0x30); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_LAST_PERIODIC),now); write_volatile(core::ptr::addr_of_mut!(OS_K1100_LAST_PERIODIC_PROGRESS),progress);} } if now.checked_sub(start).is_some_and(|elapsed|elapsed>=frequency.checked_mul(3601).unwrap_or_else(||park())) { let run=unsafe{&*core::ptr::addr_of!(OS_K1100_LONG_RUN).cast::()}; let finished=unsafe{(&*os_k1100_policy_pointer()).finish_long_run(run)} .unwrap_or_else(|_|park()); if finished.cpu_running.checked_add(finished.cpu_idle)!=Some(finished.ticks) || now.checked_sub(start).is_none_or(|elapsed|elapsed=768 {os_k1100_native_checkpoint(0x10000);} } // Called only after Producer::poll ended its loan and before the actual848 // frame is restored. This IRQ path reads dedicated scalars and physical source // bytes; it never forms an OwnerCell/Policy/Queue/Bank/allocator reference. fn os_k1100_checkpoint_irq(address:u64,counter:u64) { let before=os_k1100_source_snapshot().unwrap_or_else(|_|park()); if !before.registers.irq_active {park();} let frequency=os_k960_frequency(); let last=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_ARCHIVE_COUNTER))}; let phase=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PHASE))}; let idle=phase==0&&before.current.is_none() &&unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_COUNT))}==1; if !idle&&(phase!=2 || counter.checked_sub(last).is_some_and(|gap|last!=0&&gap=core::mem::MaybeUninit::uninit(); static mut OS_K1100_PRIMARY_GUARD:[u64;2]=[0;2]; static mut OS_K1100_PRIMARY_ERROR:Option<&'static str>=None; const OS_K1100_PRIMARY_LOW_CANARY:u64=0x4b31_3130_4c4f_5721; const OS_K1100_PRIMARY_HIGH_CANARY:u64=0x4b31_3130_4849_4748; #[inline(never)] fn os_k1100_on_stack(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str> { if cfg!(feature="os-runtime-product") && !os_k1100_product_probe_enabled(boot)? {write_line("TURKOS/OS/K1100: PRODUCT-CONTROL NO-POLICY-PROBE");return Ok(());} let raw=os_k1100_registers(); let original=core::ptr::addr_of!(OS_K900_STACK) as u64; if raw.cpu!=0 || raw.current_el!=4 || raw.irq_active || raw.daif&128==0 || raw.cpacr&(3<<20)!=(3<<20) || raw.sporiginal+69632-16 || !raw.sp.is_multiple_of(16) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_EXPECTED_VALID))}!=0 { return Err("policy-primary-raw-context"); } // Retained accepted predecessor controls bind the fresh TTBR/ASID/TCR/DAIF // before any runtime.running() loan or allocator/IRQ owner access. let prior=unsafe{core::ptr::addr_of!(OS_K1080_EVIDENCE).cast::()}; if raw.ttbr!=unsafe{read_volatile(prior.add(1))} || raw.tcr!=unsafe{read_volatile(prior.add(2))} || raw.daif!=unsafe{read_volatile(prior.add(4))} || raw.ttbr!=tables.root || os_k960_timer().1&3!=0 || !tables.task_stack_guard_unmapped(original) || unsafe{read_volatile((original+4096) as *const u64)}!=0x4b38_3230_4c4f_5721 || unsafe{read_volatile((original+69632-8) as *const u64)}!=0x4b38_3230_4849_4748 { return Err("policy-primary-predecessor"); } let expected=OsK1100ExpectedExecution{identity:core::ptr::addr_of!(OS_K1100_EXPECTED_EXECUTION) as u64, stack_owner:original,stack_low:original+4096+8,stack_high:original+69632-16,registers:raw,current:None}; unsafe{core::ptr::addr_of_mut!(OS_K1100_EXPECTED_EXECUTION).cast::().write(expected); core::ptr::addr_of_mut!(OS_K1100_ORIGINAL_EXPECTED).cast::().write(expected); write_volatile(core::ptr::addr_of_mut!(OS_K1100_EXPECTED_VALID),1);} os_k1100_gate_execution(false).map_err(|_|"policy-primary-published")?; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_RETAINED_REDIS),boot.interrupts.gic_redistributor);} // The predecessor's retained real kernel inventory is complete; this // episode adds three private roots and edits four existing guard leaves. let kernel=unsafe{core::ptr::addr_of!(OS_K1080_KERNEL_TABLES).cast::()}; let count=unsafe{read_volatile(kernel)}; if !(5..=31).contains(&count) || unsafe{read_volatile(kernel.add(1))}!=tables.root {return Err("policy-kernel-inventory");} for index in 0..32 {unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_KERNEL_TABLES).cast::().add(index),read_volatile(kernel.add(index)));}} os_k1100_word(22,unsafe{read_volatile(prior.add(22))});os_k1100_word(23,unsafe{read_volatile(prior.add(23))}); os_k1100_unowned_checkpoint(0x10); let guard=core::ptr::addr_of!(OS_K1100_PRIMARY_STACK) as u64;let low=guard+4096;let end=low+65536; let (slot,descriptor)=os_k1080_mapped_guard_slot(tables,guard)?; unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_PRIMARY_GUARD),[slot,descriptor]); tables.unmap_metadata_stack_guard(guard,low,end,core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64).map_err(|_|"policy-primary-unmap")?; write_volatile(low as *mut u64,OS_K1100_PRIMARY_LOW_CANARY); write_volatile((end-8) as *mut u64,OS_K1100_PRIMARY_HIGH_CANARY);} if !tables.task_stack_guard_unmapped(guard) {return Err("policy-primary-guard");} unsafe extern "C" {fn __os_k1100_primary_on_stack(runtime:*mut u8,tables:*const u8,boot:*const u8)->u64;} let result=unsafe{__os_k1100_primary_on_stack((runtime as *mut OsK660Runtime).cast(), (tables as *const mmu::Tables).cast(),(boot as *const BootData).cast())}; let after=os_k1100_registers(); if after.cpu!=raw.cpu || after.ttbr!=raw.ttbr || after.tcr!=raw.tcr || after.sctlr!=raw.sctlr || after.spsel!=raw.spsel || after.daif!=raw.daif || after.cpacr!=raw.cpacr || after.sp!=raw.sp || after.irq_active || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_OWNERS_LIVE))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_LIVE))}!=0 || unsafe{read_volatile(low as *const u64)}!=OS_K1100_PRIMARY_LOW_CANARY || unsafe{read_volatile((end-8) as *const u64)}!=OS_K1100_PRIMARY_HIGH_CANARY {return Err("policy-primary-return");} unsafe{core::ptr::addr_of_mut!(OS_K1100_EXPECTED_EXECUTION).cast::().write(expected);} os_k1100_gate_execution(false).map_err(|_|"policy-primary-original-gate")?; let (actual,leaf)=os_k1080_primary_guard_leaf(tables,guard)?; if actual!=slot || leaf!=0 || descriptor!=guard|0x0060_0000_0000_0703 {return Err("policy-primary-restore-slot");} let par:u64; unsafe{write_volatile(slot as *mut u64,descriptor); asm!("dsb ishst","tlbi vmalle1","dsb ish","isb","at s1e1r, {guard}","isb","mrs {par}, PAR_EL1", guard=in(reg)guard,par=out(reg)par,options(nostack,preserves_flags));} if par&1!=0 || par&0x0000_ffff_ffff_f000!=guard {return Err("policy-primary-restored-translation");} os_k1100_unowned_checkpoint(0x400); if result==0 {Ok(())}else{Err(unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PRIMARY_ERROR))}.unwrap_or("policy-primary-body"))} } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1100_primary_body(runtime:*mut u8,tables:*const u8,boot:*const u8)->u64 { let raw=os_k1100_registers(); if raw.cpu!=0 || raw.current_el!=4 || raw.irq_active || raw.daif&128==0 || !raw.sp.is_multiple_of(16) {park();} let retained=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_ORIGINAL_EXPECTED).cast::())}; let guard=core::ptr::addr_of!(OS_K1100_PRIMARY_STACK) as u64; if raw.ttbr!=retained.registers.ttbr || raw.tcr!=retained.registers.tcr || raw.sctlr!=retained.registers.sctlr || raw.daif!=retained.registers.daif || raw.cpacr!=retained.registers.cpacr || raw.spsel!=retained.registers.spsel || raw.spguard+69632-16 {park();} let expected=OsK1100ExpectedExecution{stack_owner:guard,stack_low:guard+4096+8, stack_high:guard+69632-16,registers:raw,..retained}; unsafe{core::ptr::addr_of_mut!(OS_K1100_EXPECTED_EXECUTION).cast::().write(expected); core::ptr::addr_of_mut!(OS_K1100_BOOT_EXPECTED).cast::().write(expected); write_volatile(core::ptr::addr_of_mut!(OS_K1100_BOOT_EXPECTED_LIVE),1);} os_k1100_gate_execution(false).unwrap_or_else(|_|park()); // Full hardware gate precedes forming these trusted predecessor loans. let result=os_k1100_accept(unsafe{&mut *runtime.cast::()}, unsafe{&*tables.cast::()},unsafe{&*boot.cast::()}); match result {Ok(())=>0,Err(error)=>{write_str("TURKOS/OS/K1105: PROBE-ERROR ");write_line(error);unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_PRIMARY_ERROR),Some(error));}1}} } #[inline(never)] fn os_k1100_accept(runtime:&mut OsK660Runtime,tables:&mmu::Tables,boot:&BootData)->Result<(),&'static str> { let expected=os_k1100_gate_execution(false).map_err(|_|"policy-accept-context")?; if cfg!(feature="os-runtime-product") { let mode=os_k1100_product_probe_mode(boot)?; let hour=u64::from(mode==3); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_EPISODE_MODE),hour); write_volatile(core::ptr::addr_of_mut!(OS_K1085_NEGATIVE_REENQUEUE_MODE),u64::from(mode==4)); write_volatile(core::ptr::addr_of_mut!(OS_K1090_NEGATIVE_MODE),u64::from(mode==5)); write_volatile(core::ptr::addr_of_mut!(OS_K1095_NEGATIVE_MODE),u64::from(mode==6)); write_volatile(core::ptr::addr_of_mut!(OS_K1100_NEGATIVE_MODE),u64::from(mode==7));} } os_k1100_retain_policy_layouts(); if expected.current.is_some() || expected.registers.ttbr!=tables.root {return Err("policy-accept-root");} let (root,tcr,vbar,daif)=os_k940_registers(); let (_,memory,irq,tasks)=runtime.running().map_err(os_k660_policy::Error::tag)?; if tasks.scheduler.running()!=tasks.id {return Err("policy-predecessor-task");} let authority=memory.allocator.os_authority() as *const os_k720_frame::Allocator; os_k1080_inventory(unsafe{&*authority},core::ptr::addr_of_mut!(OS_K1100_AUTHORITY_BEFORE).cast()); let issuer=__os_k1100_policy_identity(); let mut source=unsafe{OsK1100SourceLease::new(irq,boot.interrupts.gic_redistributor,issuer)}?; let installation=os_k1100_factory(memory,tables,root); let mut timer=Some(OsK1100TimerInstallation::new(source.producer()?)); let policy=OsK1100PolicyInstallation::new(&installation,timer.as_ref().unwrap()); if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==2 { let pointer=os_k1100_policy_pointer(); let before=unsafe{(&*pointer).snapshot()}; let run=unsafe{(&mut *pointer).begin_long_run()}.unwrap_or_else(|_|park()); let refused=unsafe{(&*pointer).finish_long_run(&run)}; if refused!=Err(os_k1100_policy::Error::Duration) || unsafe{(&*pointer).snapshot()}!=before {park();} write_line("TURKOS/OS/K1100: NEGATIVE EARLY-HOUR DURATION-REJECTED SNAPSHOT-UNCHANGED"); park(); } os_k1100_owned_scalar_checkpoint(0x12); unsafe extern "C" {static __os_k1100_vectors:u8;fn __os_k1100_run(wire:*const OsK1100Wire);fn __os_k1100_timer_wait(sequence:u64)->u64;} unsafe{asm!("msr VBAR_EL1, {v}","isb",v=in(reg)core::ptr::addr_of!(__os_k1100_vectors) as u64,options(nomem,nostack,preserves_flags));} // Actual3Ready/currentNone on the genuine retained primary, before __run. let sequence=os_k1100_arm_native(32); if unsafe{__os_k1100_timer_wait(sequence)}==0 || os_k1100_charge_pending().is_some() {park();} os_k1100_owned_scalar_checkpoint(0x13); for _ in 0..3 { let wire=os_k1100_prepare_ready_native();unsafe{__os_k1100_run(wire);} } os_k1100_timer_pin().quiesce().unwrap_or_else(|_|park()); if unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_PHASE))}!=3 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_INVERSION_PHASE))}!=4 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TASK_ENTRIES))}!=[1;3] || unsafe{(&*core::ptr::addr_of!(OS_K1100_MUTEX_LEASE)).is_some()} || unsafe{(&*core::ptr::addr_of!(OS_K1100_MUTEX_WAIT)).is_some()} {park();} let actual=unsafe{os_k1100_mutex_pointer().read()}; let backend=match actual.into_backend(){Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x11f)}; drop(backend);unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_MUTEX_LIVE),0);} policy.finish(&installation,timer.as_ref().unwrap()); os_k1100_unowned_checkpoint(0x201); let backend=installation.extract(); let stack=match backend.table.into_backend(){Ok(actual)=>actual,Err((_,actual))=>os_k1100_retain_failure(actual,0x120)}; let mut stack=stack; // This mode's owner checks and page-table writes outlast the K1100 timer // interval. Finish the already-quiesced source before changing stacks. if cfg!(feature="os-runtime-product") && matches!(os_k1100_product_probe_mode(boot)?,34|35|36|37|38) { timer.take().unwrap().finish(); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==8 { if let Err(error)=os_k1105_run_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1105: PROBE-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==9 { if let Err(error)=os_k1110_run_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1110: PROBE-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==10 { if let Err(error)=os_k1115_run_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1115: PROBE-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==11 { if let Err(error)=os_k1120_run_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1120: PROBE-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==12 { if let Err(error)=os_k1125_run_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1125-PREFLIGHT-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==13 { if let Err(error)=os_k1125_launch_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1125-LAUNCH-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==14 { if let Err(error)=os_k1125_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1125-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==15 { if let Err(error)=os_k1130_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1130-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==16 { if let Err(error)=os_k1135_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1135-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==17 { if let Err(error)=os_k1140_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1140-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==18 { if let Err(error)=os_k1145_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1145-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==19 { if let Err(error)=os_k1150_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1150-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==20 { if let Err(error)=os_k1155_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1155-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==21 { if let Err(error)=os_k1160_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1160-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==22 { if let Err(error)=os_k1165_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1165-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==23 { if let Err(error)=os_k1170_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1170-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==24 { if let Err(error)=os_k1175_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1175-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==25 { if let Err(error)=os_k1180_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1180-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==26 { if let Err(error)=os_k1185_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1185-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==27 { if let Err(error)=os_k1185i_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1185I-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==28 { if let Err(error)=os_k1190w_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1190W-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==29 { if let Err(error)=os_k1195o_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1195O-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==30 { unsafe { core::ptr::write_volatile(core::ptr::addr_of_mut!(OS_K1200_ACTIVE), true); } if let Err(error)=os_k1195o_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1200-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==31 { if let Err(error)=os_k1205_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1205-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==32 { if let Err(error)=os_k1210_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1210-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==33 { if let Err(error)=os_k1210s_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1210S-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==34 { write_line("TURKOS/OS/K1215A-TRACE: MODE34-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1215A-EL0-ERROR ");write_line(error);return Err(error); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==35 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); } write_line("TURKOS/OS/K1220M-TRACE: MODE35-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1220M-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==36 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); } write_line("TURKOS/OS/K1225T-TRACE: MODE36-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1225T-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==37 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1230_IO_PROBE), true); } write_line("TURKOS/OS/K1230I-TRACE: MODE37-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1230I-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1230_IO_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==38 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1230_IO_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1235_HANDLE_PROBE), true); } write_line("TURKOS/OS/K1235H-TRACE: MODE38-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1235H-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1230_IO_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1235_HANDLE_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==39 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1240_COMPAT_PROBE), true); } write_line("TURKOS/OS/K1240C-TRACE: MODE39-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1240C-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1240_COMPAT_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==40 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1240_COMPAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1245_MANIFEST_PROBE), true); } write_line("TURKOS/OS/K1245M-TRACE: MODE40-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1245M-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1240_COMPAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1245_MANIFEST_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==41 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1250_STARTUP_PROBE), true); } write_line("TURKOS/OS/K1250S-TRACE: MODE41-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1250S-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1250_STARTUP_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==42 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1255_RESTART_PROBE), true); } write_line("TURKOS/OS/K1255R-TRACE: MODE42-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1255R-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1255_RESTART_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==43 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1255_RESTART_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1260_REAPER_PROBE), true); } write_line("TURKOS/OS/K1260R-TRACE: MODE43-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1260R-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1255_RESTART_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1260_REAPER_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==44 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1265_WORDS_PROBE), true); } write_line("TURKOS/OS/K1265W-TRACE: MODE44-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1265W-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1225_TYPED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1265_WORDS_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==45 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1270_DISPATCH_PROBE), true); } write_line("TURKOS/OS/K1270D-TRACE: MODE45-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1270D-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1270_DISPATCH_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==46 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1270_DISPATCH_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1275_FOREGROUND_PROBE), true); } write_line("TURKOS/OS/K1275F-TRACE: MODE46-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1275F-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1270_DISPATCH_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1275_FOREGROUND_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==47 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1280_INPUT_PROBE), true); } write_line("TURKOS/OS/K1280I-TRACE: MODE47-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1280I-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1280_INPUT_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==48 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1280_INPUT_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1285_ECHO_PROBE), true); } write_line("TURKOS/OS/K1285E-TRACE: MODE48-ENTER"); if let Err(error)=os_k1215a_el0_on_work_stack(&mut *stack.memory,tables) { write_str("TURKOS/OS/K1285E-EL0-ERROR ");write_line(error);return Err(error); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1220_FORMAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1280_INPUT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1285_ECHO_PROBE), false); } } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==49 { let mut cat = k1290_cat_disk::CatDisk::load( boot, &mut stack.memory.allocator)?; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1290_CAT_DISK), &mut cat); write_volatile(core::ptr::addr_of_mut!(OS_K1290_CAT_PROBE), true); } write_line("TURKOS/OS/K1290C-TRACE: DISK-CACHED-DMA-RELEASED"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1290_CAT_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1290_CAT_DISK), core::ptr::null_mut()); } guest?; if !cat.owner_released() { return Err("cat fd owner leak"); } write_line("TURKOS/OS/K1290C-EL0: README0 ONE1 TAIL513 ENOENT EBADF OWNER=RELEASED"); write_line("TURKOS/OS/K1290: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==50 { let mut listing = k1295_ls_disk::LsDisk::load( boot, &mut stack.memory.allocator)?; if !listing.child_verified() { return Err("ls child cache"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1295_LS_DISK), &mut listing); write_volatile(core::ptr::addr_of_mut!(OS_K1295_LS_PROBE), true); } write_line("TURKOS/OS/K1295L-TRACE: TWO-DIRECTORIES-CACHED-DMA-RELEASED"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1295_LS_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1295_LS_DISK), core::ptr::null_mut()); } guest?; if !listing.owner_released() { return Err("ls fd owner leak"); } write_line("TURKOS/OS/K1295L-EL0: ROOT-LISTED SUB-DENIED STALE-REJECTED OWNER=RELEASED"); write_line("TURKOS/OS/K1295: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==51 { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1300_PROCESS_PROBE), true); } write_line("TURKOS/OS/K1300P-TRACE: MODE51-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1300_PROCESS_PROBE), false); } guest?; write_line("TURKOS/OS/K1300P-EL0: THREE-PROCESSES MONOTONIC-UPTIME FOREIGN-ADDRESS-DENIED OWNER=RELEASED"); write_line("TURKOS/OS/K1300: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==52 { k1305_handles_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1305_HANDLES_PROBE), true); } write_line("TURKOS/OS/K1305H-TRACE: MODE52-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1305_HANDLES_PROBE), false); } guest?; k1305_handles_dispatch::finish()?; write_line("TURKOS/OS/K1305H-EL0: THREE-TYPED-HANDLES WRONG-KIND-EBADF OWNER=RELEASED"); write_line("TURKOS/OS/K1305: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==53 { k1310_generation_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1310_GENERATION_PROBE), true); } write_line("TURKOS/OS/K1310G-TRACE: MODE53-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1310_GENERATION_PROBE), false); } guest?; k1310_generation_dispatch::finish()?; write_line("TURKOS/OS/K1310G-EL0: SAME-SLOT-THREE-GENERATIONS STALE-EBADF OVERFLOW-DENIED OWNER=RELEASED"); write_line("TURKOS/OS/K1310: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==54 { k1315_rights_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1315_RIGHTS_PROBE), true); } write_line("TURKOS/OS/K1315R-TRACE: MODE54-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1315_RIGHTS_PROBE), false); } guest?; k1315_rights_dispatch::finish()?; write_line("TURKOS/OS/K1315R-EL0: RW-TO-RO-COPY WIDEN-DENIED OWNER=RELEASED"); write_line("TURKOS/OS/K1315: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==55 { k1320_lifetime_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1320_LIFETIME_PROBE), true); } write_line("TURKOS/OS/K1320L-TRACE: MODE55-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1320_LIFETIME_PROBE), false); } guest?; k1320_lifetime_dispatch::finish()?; write_line("TURKOS/OS/K1320L-EL0: TWO-PROCESSES-TWO-REFERENCES LAST-CLOSE-WAITER-CLOSED OWNER=RELEASED"); write_line("TURKOS/OS/K1320: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==56 { k1325_queue_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1325_QUEUE_PROBE), true); } write_line("TURKOS/OS/K1325Q-TRACE: MODE56-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1325_QUEUE_PROBE), false); } guest?; k1325_queue_dispatch::finish()?; write_line("TURKOS/OS/K1325Q-EL0: BOUNDED-FIFO-LIMITS-FAULT-ROLLBACK OWNER=RELEASED"); write_line("TURKOS/OS/K1325: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==57 { k1330_owned_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1330_OWNED_PROBE), true); } write_line("TURKOS/OS/K1330O-TRACE: MODE57-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1330_OWNED_PROBE), false); } guest?; k1330_owned_dispatch::finish()?; write_line("TURKOS/OS/K1330O-EL0: SENDER-EXIT-PAYLOAD-COPY-PRESERVED OWNER=RELEASED"); write_line("TURKOS/OS/K1330: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==58 { k1335_transfer_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1335_TRANSFER_PROBE), true); } write_line("TURKOS/OS/K1335T-TRACE: MODE58-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1335_TRANSFER_PROBE), false); } guest?; k1335_transfer_dispatch::finish()?; write_line("TURKOS/OS/K1335T-EL0: MESSAGE-HANDLE-TRANSFER-ESCROW-RELEASED OWNER=RELEASED"); write_line("TURKOS/OS/K1335: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==59 { k1340_peer_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1340_PEER_PROBE), true); } write_line("TURKOS/OS/K1340P-TRACE: MODE59-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1340_PEER_PROBE), false); } guest?; k1340_peer_dispatch::finish()?; write_line("TURKOS/OS/K1340P-EL0: PEER-DEATH-DRAIN-CLOSED-EPIPE OWNER=RELEASED"); write_line("TURKOS/OS/K1340: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==60 { k1345_waitset_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1345_WAITSET_PROBE), true); } write_line("TURKOS/OS/K1345W-TRACE: MODE60-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1345_WAITSET_PROBE), false); } guest?; k1345_waitset_dispatch::finish()?; write_line("TURKOS/OS/K1345W-EL0: TYPED-REGISTRATION-STALE-DENIAL OWNER=RELEASED"); write_line("TURKOS/OS/K1345: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==61 { k1350_readiness_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1350_READINESS_PROBE), true); } write_line("TURKOS/OS/K1350R-TRACE: MODE61-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1350_READINESS_PROBE), false); } guest?; k1350_readiness_dispatch::finish()?; write_line("TURKOS/OS/K1350R-EL0: LEVEL-REPEATED-CLEAR-FILTER-STALE OWNER=RELEASED"); write_line("TURKOS/OS/K1350: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==62 { k1355_deadlines_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1355_DEADLINES_PROBE), true); } write_line("TURKOS/OS/K1355D-TRACE: MODE62-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1355_DEADLINES_PROBE), false); } guest?; k1355_deadlines_dispatch::finish()?; write_line("TURKOS/OS/K1355D-EL0: SINGLE-COMPLETION-STALE-TIMER-REJECTED OWNER=RELEASED"); write_line("TURKOS/OS/K1355: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==63 { k1360_teardown_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1360_TEARDOWN_PROBE), true); } write_line("TURKOS/OS/K1360C-TRACE: MODE63-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1360_TEARDOWN_PROBE), false); } guest?; k1360_teardown_dispatch::finish()?; write_line("TURKOS/OS/K1360C-EL0: THREE-SUBSCRIPTIONS-CLEARED-RACE-REJECTED OWNER=RELEASED"); write_line("TURKOS/OS/K1360: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==64 { k1365_shared_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1365_SHARED_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1300_PROCESS_PROBE), true); } write_line("TURKOS/OS/K1365S-TRACE: MODE64-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1365_SHARED_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1300_PROCESS_PROBE), false); } guest?; k1365_shared_dispatch::finish()?; write_line("TURKOS/OS/K1365S-EL0: TWO-PHYSICAL-PAGES-SHARED-C-DENIED OWNER=RELEASED"); write_line("TURKOS/OS/K1365: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==65 { k1370_seal_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1370_SEAL_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1300_PROCESS_PROBE), true); } write_line("TURKOS/OS/K1370S-TRACE: MODE65-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1370_SEAL_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1300_PROCESS_PROBE), false); } guest?; k1370_seal_dispatch::finish()?; write_line("TURKOS/OS/K1370S-EL0: OWNER-RO-RECIPIENT-RO-POST-SEAL-RW-DENIED OWNER=RELEASED"); write_line("TURKOS/OS/K1370: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==66 { k1375_lifetime_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1375_LIFETIME_PROBE), true); write_volatile(core::ptr::addr_of_mut!(OS_K1300_PROCESS_PROBE), true); } write_line("TURKOS/OS/K1375L-TRACE: MODE66-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1375_LIFETIME_PROBE), false); write_volatile(core::ptr::addr_of_mut!(OS_K1300_PROCESS_PROBE), false); } guest?; k1375_lifetime_dispatch::finish()?; write_line("TURKOS/OS/K1375L-EL0: HANDLE-CLOSE-MAPS-LIVE-EARLY-FREE-DENIED-ONCE-RELEASED"); write_line("TURKOS/OS/K1375: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==67 { k1380_ring_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1380_RING_PROBE), true); } write_line("TURKOS/OS/K1380R-TRACE: MODE67-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1380_RING_PROBE), false); } guest?; k1380_ring_dispatch::finish()?; write_line("TURKOS/OS/K1380R-EL0: TWO-PROCESS-1000-MESSAGES-BAD-INDEX-REJECTED OWNER=RELEASED"); write_line("TURKOS/OS/K1380: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==68 { k1385_registry_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1385_REGISTRY_PROBE), true); } write_line("TURKOS/OS/K1385S-TRACE: MODE68-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1385_REGISTRY_PROBE), false); } guest?; k1385_registry_dispatch::finish()?; write_line("TURKOS/OS/K1385S-EL0: TWO-PROCESS-REGISTRY-OWNERSHIP-AND-CLEANUP"); write_line("TURKOS/OS/K1385: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==69 { k1390_protocol_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1390_PROTOCOL_PROBE), true); } write_line("TURKOS/OS/K1390V-TRACE: MODE69-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1390_PROTOCOL_PROBE), false); } guest?; k1390_protocol_dispatch::finish()?; write_line("TURKOS/OS/K1390V-EL0: V1-REQUEST-RESPONSE-V255-AND-TRUNCATED-REJECTED"); write_line("TURKOS/OS/K1390: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==70 { k1395_restart_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1395_RESTART_PROBE), true); } write_line("TURKOS/OS/K1395G-TRACE: MODE70-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1395_RESTART_PROBE), false); } guest?; k1395_restart_dispatch::finish()?; write_line("TURKOS/OS/K1395G-EL0: OLD-SESSION-REPLAY-REJECTED-NEW-PID-SESSION-ACQUIRED"); write_line("TURKOS/OS/K1395: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==71 { k1400_discovery_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1400_DISCOVERY_PROBE), true); } write_line("TURKOS/OS/K1400D-TRACE: MODE71-ENTER"); let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1400_DISCOVERY_PROBE), false); } guest?; k1400_discovery_dispatch::finish()?; write_line("TURKOS/OS/K1400D-EL0: TWO-PUBLIC-NAMES-MANAGEMENT-HIDDEN-SAME-AS-UNKNOWN"); write_line("TURKOS/OS/K1400: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==72 { k1405_device_registry::run(boot)?; write_line("TURKOS/OS/K1405: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==73 { k1410_device_resources::run(boot)?; write_line("TURKOS/OS/K1410: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==74 { k1415_device_driver_match::run(boot)?; write_line("TURKOS/OS/K1415: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==75 { let free_before = stack.memory.allocator.remaining_pages(); k1420_device_safe_detach::run(boot)?; if stack.memory.allocator.remaining_pages() != free_before { return Err("k1420-empty-free-changed"); } write_line("TURKOS/OS/K1420: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==76 { k1425_pci_ecam::run(boot)?; write_line("TURKOS/OS/K1425: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==77 { k1430_pci_headers::run(boot)?; write_line("TURKOS/OS/K1430: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==78 { k1435_pci_capabilities::run(boot)?; write_line("TURKOS/OS/K1435: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==79 { k1440_pci_bridges::run(boot)?; write_line("TURKOS/OS/K1440: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==80 { k1440_pci_bridges::run(boot)?; k1445_pci_bar_size::run(boot)?; write_line("TURKOS/OS/K1445: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==81 { k1440_pci_bridges::run(boot)?; k1445_pci_bar_size::run(boot)?; k1450_pci_device_map::run(boot)?; write_line("TURKOS/OS/K1450: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==82 { k1440_pci_bridges::run(boot)?; k1445_pci_bar_size::run(boot)?; k1455_pci_intx::run(boot)?; write_line("TURKOS/OS/K1455: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==83 { k1460_pci_msix::run(boot)?; write_line("TURKOS/OS/K1460: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==84 { k1465_virtio_mmio_v2::run(boot, tables, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1465: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==85 { k1470_virtio_pci_common::run(boot, tables, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1470: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==86 { k1475_virtio_features::run(boot, tables, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1475: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==87 { k1480_virtio_status::run(boot, tables, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1480: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==88 { k1485_virtio_descriptors::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1485: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==89 { k1490_virtio_indices::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1490: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==90 { k1495_virtio_completions::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1495: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==91 { k1500_virtio_backpressure::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1500: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==92 { k1505_block_geometry::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1505: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==93 { k1510_block_requests::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1510: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==94 { k1515_block_timeouts::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1515: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==95 { k1520_block_errors::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1520: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==96 { k1525_partitions_mbr::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1525: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==97 { k1530_partitions_gpt::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1530: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==98 { k1535_partitions_view::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1535: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==99 { k1540_partitions_recovery::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1540: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==100 { k1545_vfs_paths::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1545: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==101 { k1550_vfs_mount_cross::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1550: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==102 { k1555_vfs_lookup::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1555: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==103 { k1560_vfs_symlink::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1560: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==104 { k1565_vfs_ro_open::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1565: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==105 { k1570_vfs_offset::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1570: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==106 { k1575_stat_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1575_STAT_PROBE), true); } let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1575_STAT_PROBE), false); } guest?; k1575_stat_dispatch::finish()?; write_line("TURKOS/OS/K1575: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==107 { k1580_dup_dispatch::begin(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1580_DUP_PROBE), true); } let guest = os_k1215a_el0_on_work_stack(&mut *stack.memory, tables); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1580_DUP_PROBE), false); } guest?; k1580_dup_dispatch::finish()?; write_line("TURKOS/OS/K1580: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==108 { k1585_vfs_traits::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1585: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==109 { k1590_vfs_tmpfs::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1590: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==110 { k1595_vfs_ro_cache::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1595: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==111 { k1600_vfs_unmount::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1600: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==112 { k1605_vfs_superblock::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1605: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==113 { k1610_vfs_inode_table::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1610: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==114 { k1615_turkfs_reader::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1615: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==115 { k1620_turkfs_directory::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1620: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==116 { k1625_turkfs_allocation::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1625: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==117 { k1630_turkfs_growth::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1630: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==118 { k1635_turkfs_inode::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1635: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==119 { k1640_turkfs_truncate::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1640: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==120 { k1645_turkfs_create::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1645: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==121 { k1650_turkfs_rename::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1650: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==122 { k1655_turkfs_unlink::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1655: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==123 { k1660_turkfs_links::run(&mut stack.memory.allocator)?; write_line("TURKOS/OS/K1660: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==124 { k1665_turkfs_journal::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1665: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==125 { k1670_turkfs_ordering::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1670: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==126 { k1675_turkfs_reservation::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1675: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==127 { k1680_turkfs_checkpoint::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1680: PASS"); } if cfg!(feature="os-runtime-product") && os_k1100_product_probe_mode(boot)?==128 { k1685_turkfs_replay::run(boot, &mut stack.memory.allocator)?; write_line("TURKOS/OS/K1685: PASS"); } os_k1080_inventory(stack.memory.allocator.os_authority(),core::ptr::addr_of_mut!(OS_K1100_AUTHORITY_AFTER).cast()); if !os_k920_inventory_equal(core::ptr::addr_of!(OS_K1100_AUTHORITY_BEFORE).cast(), core::ptr::addr_of!(OS_K1100_AUTHORITY_AFTER).cast()) {park();} unsafe{asm!("msr VBAR_EL1, {v}","isb",v=in(reg)vbar,options(nomem,nostack,preserves_flags));} drop(stack);if let Some(timer)=timer.take() {timer.finish();}drop(timer);drop(source); if os_k940_registers()!=(root,tcr,vbar,daif) {park();} write_line("TURKOS/OS/K1100: DRAFT-ACTUAL-POLICY-RETURN NO-CANONICAL-CLAIM"); Ok(()) } #[inline(never)] fn os_k1100_native_checkpoint(code:u64) { unsafe extern "C" {fn __os_k1100_checkpoint(code:u64);} unsafe{__os_k1100_checkpoint(code);} } #[unsafe(no_mangle)] extern "C" fn __os_k1100_checkpoint_body(address:u64,code:u64) { os_k1100_capture_native(address); let before=os_k1100_source_snapshot().unwrap_or_else(|_|park()); let timer=os_k1100_timer_pin();timer.drained().unwrap_or_else(|_|park()); let (progress,transitions)=unsafe{(&*os_k1100_driver_pointer()).k1100_progress(timer)}.unwrap_or_else(|_|park()); let snapshot=unsafe{(&*os_k1100_policy_pointer()).snapshot()}; let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_EVENT_COUNT))}; let total=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_EVENT_TOTAL))}.checked_add(count).unwrap_or_else(||park()); let sequence=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CHECKPOINT_SEQUENCE))}.checked_add(1).unwrap_or_else(||park()); let thread=before.current.unwrap_or_else(||park()); let words=[sequence,code,os_k960_counter(),os_k960_frequency(),address,640, before.registers.ttbr,before.stack_owner,before.stack_low,before.stack_high, thread.slot() as u64,thread.generation(),progress[0],progress[1],progress[2],transitions, snapshot.ticks,snapshot.last_sequence,snapshot.last_counter,snapshot.queue_epoch, count,total,unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_COUNT))}, unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CAPTURE_EPISODE))}]; if os_k1100_source_snapshot()!=Ok(before) {park();} // Every short actual owner loan ended before this no-owner archive barrier. // Host captures these exact physical frame/event/owner bytes before ACK. os_k1100_publish_archive(words); if os_k1100_source_snapshot()!=Ok(before) {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_EVENT_TOTAL),total); write_volatile(core::ptr::addr_of_mut!(OS_K1100_EVENT_COUNT),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CHECKPOINT_SEQUENCE),sequence); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CHECKPOINT_READY),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CURRENT_CAPTURE),None); write_volatile(core::ptr::addr_of_mut!(OS_K1100_LIVE_CAPTURE_ADDRESS),0);} } #[inline(never)] fn os_k1100_publish_archive(words:[u64;24]) { let irq=os_k1100_registers().irq_active; os_k1100_gate_execution(irq).unwrap_or_else(|_|park()); let previous=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CHECKPOINT_SEQUENCE))}; if words[0]!=previous.checked_add(1).unwrap_or_else(||park()) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CHECKPOINT_READY))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CHECKPOINT_ACK))}!=previous {park();} os_k1100_archive_hardware(); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_CHECKPOINT),words); asm!("dsb sy",options(nomem,nostack,preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CHECKPOINT_READY),words[0]);} // UART is an independent scalar MMIO write. No owner/producer reference // survives here; this fresh marker tells the host to pause before reading. write_line("TURKOS/OS/K1100: ARCHIVE READY"); while unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CHECKPOINT_ACK))}!=words[0] { core::hint::spin_loop(); if os_k960_counter().checked_sub(words[2]).is_none_or(|gap|gap>words[3]*45) {park();} } os_k1100_gate_execution(irq).unwrap_or_else(|_|park()); } #[inline(never)] fn os_k1100_archive_hardware() { let raw=os_k1100_registers(); os_k1100_gate_execution(raw.irq_active).unwrap_or_else(|_|park()); let redis=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_RETAINED_REDIS))}; if redis==0 || !redis.is_multiple_of(65536) {park();} let (cval,control)=os_k960_timer();let vbar:u64; unsafe{asm!("mrs {vbar}, VBAR_EL1",vbar=out(reg)vbar,options(nomem,nostack,preserves_flags));} let [enabled,pending,active]=unsafe{[ read_volatile((redis+0x1_0100) as *const u32) as u64, read_volatile((redis+0x1_0200) as *const u32) as u64, read_volatile((redis+0x1_0300) as *const u32) as u64]}; if control&3!=0 || pending&(1<<30)!=0 || active&(1<<30)!=0 {park();} os_k1100_gate_execution(raw.irq_active).unwrap_or_else(|_|park()); unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_ARCHIVE_HARDWARE),[ raw.cpu,raw.daif,raw.current_el,raw.ttbr,raw.tcr,raw.sctlr,raw.spsel,raw.sp,raw.cpacr,u64::from(raw.irq_active), os_k960_counter(),os_k960_frequency(),cval,control,redis,enabled,pending,active,vbar, read_volatile(core::ptr::addr_of!(OS_K660_IRQ_COUNT)),read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_COUNT)), read_volatile(core::ptr::addr_of!(OS_K1100_SOURCE_ISSUER))]);} } #[inline(never)] fn os_k1100_unowned_checkpoint(code:u64) { let before=os_k1100_gate_execution(false).unwrap_or_else(|_|park()); let retired=code==0x201; if before.current.is_some() || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_OWNERS_LIVE))}!=u64::from(retired) || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_POLICY_LIVE))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_DRIVER_LIVE))}!=0 || unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_TIMER_LIVE))}!=u64::from(retired) {park();} let sequence=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CHECKPOINT_SEQUENCE))}.checked_add(1).unwrap_or_else(||park()); let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_EVENT_COUNT))}; let total=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_EVENT_TOTAL))}.checked_add(count).unwrap_or_else(||park()); os_k1100_publish_archive([sequence,code,os_k960_counter(),os_k960_frequency(),0,0, before.registers.ttbr,before.stack_owner,before.stack_low,before.stack_high,u64::MAX,0, 0,0,0,0,0,0,0,0,count,total,unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_COUNT))},0]); if os_k1100_gate_execution(false).unwrap_or_else(|_|park()).registers!=before.registers {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_EVENT_TOTAL),total); write_volatile(core::ptr::addr_of_mut!(OS_K1100_EVENT_COUNT),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CHECKPOINT_SEQUENCE),sequence); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CHECKPOINT_READY),0);} } #[inline(never)] fn os_k1100_owned_scalar_checkpoint(code:u64) { let before=os_k1100_source_snapshot().unwrap_or_else(|_|park()); if before.registers.irq_active {park();} let timer=os_k1100_timer_pin();timer.drained().unwrap_or_else(|_|park()); let (progress,completed)=unsafe{(&*os_k1100_driver_pointer()).k1100_progress(timer)}.unwrap_or_else(|_|park()); let snapshot=unsafe{(&*os_k1100_policy_pointer()).snapshot()}; let sequence=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CHECKPOINT_SEQUENCE))}.checked_add(1).unwrap_or_else(||park()); let count=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_EVENT_COUNT))}; let total=unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_EVENT_TOTAL))}.checked_add(count).unwrap_or_else(||park()); let address=if code==0x22 {unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_LIVE_CAPTURE_ADDRESS))}}else{0}; if code==0x22&&address==0 {park();} let (slot,generation)=before.current.map_or((u64::MAX,0),|t|(t.slot() as u64,t.generation())); os_k1100_publish_archive([sequence,code,os_k960_counter(),os_k960_frequency(),address,if address==0 {0}else{640}, before.registers.ttbr,before.stack_owner,before.stack_low,before.stack_high,slot,generation, progress[0],progress[1],progress[2],completed,snapshot.ticks,snapshot.last_sequence,snapshot.last_counter, snapshot.queue_epoch,count,total,unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_IRQ_COUNT))}, unsafe{read_volatile(core::ptr::addr_of!(OS_K1100_CAPTURE_EPISODE))}]); if os_k1100_source_snapshot()!=Ok(before) {park();} unsafe{write_volatile(core::ptr::addr_of_mut!(OS_K1100_EVENT_TOTAL),total); write_volatile(core::ptr::addr_of_mut!(OS_K1100_EVENT_COUNT),0); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CHECKPOINT_SEQUENCE),sequence); write_volatile(core::ptr::addr_of_mut!(OS_K1100_CHECKPOINT_READY),0);} } #[unsafe(no_mangle)] static OS_K1100_CONTEXT_LAYOUT_EXPECTED: [u64; 40] = os_k1100_context::os_k1040_layout::, 3>(); #[unsafe(no_mangle)] static OS_K1100_THREAD_ID_LAYOUT_EXPECTED: [u64; 4] = os_k1100_thread::ThreadId::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1100_THREAD_TABLE_LAYOUT_EXPECTED: [u64; 20] = OsK1100Table::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1100_LEASE_LAYOUT_EXPECTED: [u64; 7] = os_k720_frame::Lease::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1100_WAIT_LAYOUT_EXPECTED: [u64; 5] = os_k1100_thread::Wait::os_k1040_layout(); #[unsafe(no_mangle)] static OS_K1100_BACKEND_LAYOUT_EXPECTED: [u64; 30] = [ core::mem::size_of::>() as u64, core::mem::offset_of!(OsK1100ContextBackend<'static>, table) as u64, core::mem::offset_of!(OsK1100ContextBackend<'static>, bootstrap) as u64, core::mem::offset_of!(OsK1100ContextBackend<'static>, pins) as u64, core::mem::offset_of!(OsK1100ContextBackend<'static>, sequence) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1100Binding, id) as u64, core::mem::offset_of!(OsK1100Binding, ordinal) as u64, core::mem::offset_of!(OsK1100Binding, root) as u64, core::mem::offset_of!(OsK1100Binding, root_generation) as u64, core::mem::offset_of!(OsK1100Binding, branches) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1100Stack, root) as u64, core::mem::offset_of!(OsK1100Stack, ordinal) as u64, core::mem::offset_of!(OsK1100Stack, start) as u64, core::mem::offset_of!(OsK1100Stack, leases) as u64, core::mem::offset_of!(OsK1100Stack, guard) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1100Root, lease) as u64, core::mem::offset_of!(OsK1100Root, bootstrap) as u64, core::mem::offset_of!(OsK1100Root, branches) as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::size_of::>() as u64, core::mem::offset_of!(OsK1100ContextBackend<'static>, current) as u64, core::mem::offset_of!(OsK1100ContextBackend<'static>, sleep_wait) as u64, core::mem::size_of::>() as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1100IdlePrepared, epoch) as u64, core::mem::offset_of!(OsK1100IdlePrepared, wait) as u64, ]; #[unsafe(no_mangle)] static OS_K1100_TRACE_BACKEND_LAYOUT: [u64;16]=[ core::mem::size_of::() as u64, core::mem::offset_of!(OsK1100Prepared,source) as u64, core::mem::offset_of!(OsK1100Prepared,target) as u64, core::mem::offset_of!(OsK1100Prepared,nonce) as u64, core::mem::offset_of!(OsK1100Prepared,departed) as u64, core::mem::size_of::() as u64, core::mem::offset_of!(OsK1100ReturnPrepared,source) as u64, core::mem::offset_of!(OsK1100ReturnPrepared,bootstrap_root) as u64, core::mem::offset_of!(OsK1100ReturnPrepared,bootstrap) as u64, core::mem::offset_of!(OsK1100ReturnPrepared,address) as u64, core::mem::offset_of!(OsK1100ReturnPrepared,nonce) as u64, 0,0,0,0,0, ]; #[unsafe(no_mangle)] static OS_K1100_WAIT_GRAPH_LAYOUT: [u64; 48] = os_k1100_wait::os_k1100_wait_graph_layout::>(); #[unsafe(no_mangle)] static OS_K1100_INGRESS_GRAPH_LAYOUT: [u64; 24] = os_k1100_wait::os_k1100_ingress_graph_layout::(); #[unsafe(no_mangle)] static OS_K1100_BRIDGE_GRAPH_LAYOUT: [u64; 32] = os_k1100_bridge::os_k1100_wait_bridge_layout::, OsK1100Port>(); #[unsafe(no_mangle)] static OS_K1100_RETURN_LAYOUT: [u64; 40] = os_k1100_queue::os_k1100_return_layout::, 3>(); #[unsafe(no_mangle)] static OS_K1100_QUEUE_BACKEND_LAYOUT: [u64; 24] = os_k1100_backend_layout::>(); #[unsafe(no_mangle)] static OS_K1100_TRACE_QUEUE_LAYOUT: [u64; 24] = os_k1100_queue::os_k1100_trace_queue_layout::, 3>(); #[unsafe(no_mangle)] static OS_K1100_TRACE_CONTEXT_LAYOUT: [u64; 38] = os_k1100_context::os_k1100_trace_context_layout::>(); #[unsafe(no_mangle)] static OS_K1100_SIGNAL_PIN_LAYOUT: [u64; 4] = [ core::mem::size_of::() as u64, core::mem::offset_of!(OsK1100SignalPin, source) as u64, core::mem::offset_of!(OsK1100SignalPin, slot) as u64, core::mem::offset_of!(OsK1100SignalPin, claim) as u64, ]; #[unsafe(no_mangle)] static OS_K1100_CONTEXT_OPTION_TAGS: [u64; 20] = os_k1100_context::os_k1100_context_option_tags::>(); #[unsafe(no_mangle)] static OS_K1100_QUEUE_OPTION_TAGS: [u64; 12] = os_k1100_queue::os_k1100_queue_option_tags::>(); const fn os_k1100_main_none_word(value: T, offset: usize) -> u64 { let value=core::mem::ManuallyDrop::new(value); unsafe {core::ptr::addr_of!(value).cast::().add(offset).cast::().read_unaligned()} } #[unsafe(no_mangle)] static OS_K1100_BACKEND_OPTION_TAGS:[u64;4]=[ os_k1100_main_none_word(None::,0), os_k1100_main_none_word(None::,0), os_k1100_main_none_word(None::,0), os_k1100_main_none_word(None::,0), ]; #[unsafe(no_mangle)] static OS_K1100_CAPTURE_ABI_TEMPLATE: os_k1100_context::Abi = os_k1100_context::Abi::COOPERATIVE; #[unsafe(no_mangle)] static OS_K1100_RETAINED_NONE_TEMPLATE: Option< os_k1100_wait::Retained, os_k1100_bridge::WaitPrepared>, > = None; #[unsafe(no_mangle)] static OS_K1100_WAIT_PREPARED_NONE_TEMPLATE: Option = None; #[unsafe(no_mangle)] static OS_K1100_SIGNAL_PIN_NONE_TEMPLATE: Option = None; #[unsafe(no_mangle)] static OS_K1100_MEMBER_NONE_TEMPLATE: Option = None; #[unsafe(no_mangle)] static OS_K1100_IMAGE_TEMPLATES: [os_k1100_context::Image; 2] = [ os_k1100_context::Image::Initial { entry: 0x4b31303830494e31, argument: 0x4b31303830494e32, sp: 0x4b31303830494e33, return_pc: 0x4b31303830494e34, }, os_k1100_context::Image::Cooperative(os_k1100_context::Cooperative::EMPTY), ]; fn os_k1100_retain_layouts() { unsafe extern "C" {static OS_K1100_INGRESS_EVENT_NONE_TEMPLATE:u8;} for address in [core::ptr::addr_of!(OS_K1100_WAIT_GRAPH_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_INGRESS_GRAPH_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_BRIDGE_GRAPH_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_RETURN_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_QUEUE_BACKEND_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_TRACE_QUEUE_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_TRACE_CONTEXT_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_SIGNAL_PIN_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_CONTEXT_OPTION_TAGS) as u64,core::ptr::addr_of!(OS_K1100_QUEUE_OPTION_TAGS) as u64,core::ptr::addr_of!(OS_K1100_BACKEND_OPTION_TAGS) as u64,core::ptr::addr_of!(OS_K1100_CAPTURE_ABI_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1100_RETAINED_NONE_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1100_WAIT_PREPARED_NONE_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1100_SIGNAL_PIN_NONE_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1100_MEMBER_NONE_TEMPLATE) as u64,core::ptr::addr_of!(OS_K1100_IMAGE_TEMPLATES) as u64,core::ptr::addr_of!(OS_K1100_CONTEXT_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1100_THREAD_ID_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1100_THREAD_TABLE_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1100_LEASE_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1100_WAIT_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1100_BACKEND_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1100_TRACE_BACKEND_LAYOUT) as u64,core::ptr::addr_of!(OS_K1100_INGRESS_EVENT_NONE_TEMPLATE) as u64] {core::hint::black_box(address);} } #[unsafe(no_mangle)] static OS_K1100_LAYOUT_OWNER_CELL:[u64;12]=turkos_kernel::runtime::policy_owner::os_k1100_layout_owner_cell::,OsK1100MainGate>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_MAIN:[u64;22]=turkos_kernel::runtime::policy_main::os_k1100_layout_main::,OsK1100Port,OsK1100Port>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_SUBSCRIPTION:[u64;8]=turkos_kernel::runtime::policy_owner::os_k1100_layout_subscription::,OsK1100MainGate>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_PRODUCER_INGRESS:[u64;20]=turkos_kernel::runtime::policy_ingress::os_k1100_layout_producer_ingress::>,8>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_TIMER:[u64;10]=turkos_kernel::runtime::policy_timer::os_k1100_layout_timer::>,OsK1100TimerHardware,8>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_SOURCE:[u64;13]=turkos_kernel::runtime::policy_source::os_k1100_layout_source::>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_POLICY_BACKEND:[u64;4]=turkos_kernel::runtime::policy_backend::os_k1100_layout_policy_backend::,OsK1100MainGate,turkos_kernel::runtime::policy_source::Source>,OsK1100TimerHardware,8>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_POLICY:[u64;21]=turkos_kernel::runtime::policy::os_k1100_layout_policy::,3>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_POLICY_RECORD:[u64;12]=turkos_kernel::runtime::policy::os_k1100_layout_policy_record(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_POLICY_HANDLE:[u64;5]=turkos_kernel::runtime::policy::os_k1100_layout_policy_handle(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_POLICY_OBSERVATION:[u64;15]=turkos_kernel::runtime::policy::os_k1100_layout_policy_observation(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_POLICY_SNAPSHOT:[u64;10]=turkos_kernel::runtime::policy::os_k1100_layout_policy_snapshot(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_TASK_TIME:[u64;6]=turkos_kernel::runtime::policy::os_k1100_layout_task_time(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_TICK_OBSERVATION:[u64;8]=turkos_kernel::runtime::policy::os_k1100_layout_tick_observation(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_QUEUE_TICK_OBSERVATION:[u64;8]=turkos_kernel::runtime::policy_wait_queue::os_k1100_layout_queue_tick_observation(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_QUEUE_HANDLE:[u64;5]=turkos_kernel::runtime::policy_wait_queue::os_k1100_layout_queue_handle(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_QUEUE:[u64;16]=turkos_kernel::runtime::policy_wait_queue::os_k1100_layout_queue::,3>(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_WORKLOAD_MEMBER:[u64;8]=turkos_kernel::runtime::policy::os_k1100_layout_workload_member(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_LONG_RUN:[u64;19]=turkos_kernel::runtime::policy::os_k1100_layout_long_run(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_MUTEX:[u64;23]=turkos_kernel::runtime::policy_locking::mutex::os_k1100_layout_mutex::(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_MUTEX_LEASE:[u64;5]=turkos_kernel::runtime::policy_locking::mutex::os_k1100_layout_mutex_lease(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_MUTEX_WAIT:[u64;6]=turkos_kernel::runtime::policy_locking::mutex::os_k1100_layout_mutex_wait(); #[unsafe(no_mangle)] static OS_K1100_LAYOUT_REGISTERS:[u64;12]=[core::mem::size_of::() as u64,core::mem::align_of::() as u64,core::mem::offset_of!(OsK1100Registers,cpu) as u64,core::mem::offset_of!(OsK1100Registers,daif) as u64,core::mem::offset_of!(OsK1100Registers,current_el) as u64,core::mem::offset_of!(OsK1100Registers,ttbr) as u64,core::mem::offset_of!(OsK1100Registers,tcr) as u64,core::mem::offset_of!(OsK1100Registers,sctlr) as u64,core::mem::offset_of!(OsK1100Registers,spsel) as u64,core::mem::offset_of!(OsK1100Registers,sp) as u64,core::mem::offset_of!(OsK1100Registers,cpacr) as u64,core::mem::offset_of!(OsK1100Registers,irq_active) as u64]; #[unsafe(no_mangle)] static OS_K1100_LAYOUT_EXPECTED:[u64;8]=[core::mem::size_of::() as u64,core::mem::align_of::() as u64,core::mem::offset_of!(OsK1100ExpectedExecution,identity) as u64,core::mem::offset_of!(OsK1100ExpectedExecution,stack_owner) as u64,core::mem::offset_of!(OsK1100ExpectedExecution,stack_low) as u64,core::mem::offset_of!(OsK1100ExpectedExecution,stack_high) as u64,core::mem::offset_of!(OsK1100ExpectedExecution,registers) as u64,core::mem::offset_of!(OsK1100ExpectedExecution,current) as u64]; #[unsafe(no_mangle)] static OS_K1100_LAYOUT_SOURCE_STAMP:[u64;13]=[core::mem::size_of::() as u64,core::mem::align_of::() as u64,core::mem::offset_of!(OsK1100SourceStamp,issuer) as u64,core::mem::offset_of!(OsK1100SourceStamp,sequence) as u64,core::mem::offset_of!(OsK1100SourceStamp,counter) as u64,core::mem::offset_of!(OsK1100SourceStamp,queue_epoch) as u64,core::mem::offset_of!(OsK1100SourceStamp,slice) as u64,core::mem::offset_of!(OsK1100SourceStamp,current) as u64,core::mem::offset_of!(OsK1100SourceStamp,root) as u64,core::mem::offset_of!(OsK1100SourceStamp,stack_owner) as u64,core::mem::offset_of!(OsK1100SourceStamp,stack_low) as u64,core::mem::offset_of!(OsK1100SourceStamp,stack_high) as u64,core::mem::offset_of!(OsK1100SourceStamp,deadline) as u64]; #[unsafe(no_mangle)] static OS_K1100_LAYOUT_QUEUE_TICK:[u64;4]=[core::mem::size_of::() as u64,core::mem::align_of::() as u64,core::mem::offset_of!(OsK1100QueueTick,issuer) as u64,core::mem::offset_of!(OsK1100QueueTick,observation) as u64]; #[unsafe(no_mangle)] static OS_K1100_LAYOUT_COMPLETED_DELIVERY:[u64;8]=[core::mem::size_of::() as u64,core::mem::align_of::() as u64,core::mem::offset_of!(OsK1100CompletedDelivery,stamp) as u64,core::mem::offset_of!(OsK1100CompletedDelivery,frame) as u64,core::mem::offset_of!(OsK1100CompletedDelivery,frame_address) as u64,core::mem::offset_of!(OsK1100CompletedDelivery,iar) as u64,core::mem::offset_of!(OsK1100CompletedDelivery,split) as u64,core::mem::offset_of!(OsK1100CompletedDelivery,queue_tick) as u64]; #[unsafe(no_mangle)] static OS_K1100_LAYOUT_NATIVE_SOURCE:[u64;6]=[core::mem::size_of::>() as u64,core::mem::align_of::>() as u64,core::mem::offset_of!(OsK1100NativeSource<'static>,claim) as u64,core::mem::offset_of!(OsK1100NativeSource<'static>,issuer) as u64,core::mem::offset_of!(OsK1100NativeSource<'static>,source) as u64,core::mem::offset_of!(OsK1100NativeSource<'static>,frequency) as u64]; const OS_K1100_DELIVERY_TAG_OFFSET:usize=(OS_K1100_LAYOUT_SOURCE[10]+OS_K1100_LAYOUT_COMPLETED_DELIVERY[7]) as usize; #[unsafe(no_mangle)] static OS_K1100_OPTION_TAGS:[u64;16]=[ turkos_kernel::runtime::policy::os_k1100_policy_none_tags::,3>()[0], turkos_kernel::runtime::policy::os_k1100_policy_none_tags::,3>()[1], turkos_kernel::runtime::policy::os_k1100_policy_none_tags::,3>()[2], turkos_kernel::runtime::policy::os_k1100_policy_none_tags::,3>()[3], turkos_kernel::runtime::policy_main::os_k1100_none_word::(0), turkos_kernel::runtime::policy_main::os_k1100_main_parked_none(), turkos_kernel::runtime::policy_main::os_k1100_none_word::(0), turkos_kernel::runtime::policy_main::os_k1100_none_word::(0), turkos_kernel::runtime::policy_source::os_k1100_none_word::>(OS_K1100_DELIVERY_TAG_OFFSET), turkos_kernel::runtime::policy_ingress::os_k1100_none_word::>>((OS_K1100_LAYOUT_PRODUCER_INGRESS[17] as usize)+OS_K1100_DELIVERY_TAG_OFFSET), turkos_kernel::runtime::policy_source::os_k1100_none_word::(OS_K1100_LAYOUT_TICK_OBSERVATION[7] as usize), turkos_kernel::runtime::policy_locking::mutex::os_k1100_mutex_owner_none(), turkos_kernel::runtime::policy_locking::mutex::os_k1100_none_word::(0), turkos_kernel::runtime::policy_locking::mutex::os_k1100_none_word::(0), turkos_kernel::runtime::policy_main::os_k1100_none_word::(OS_K1100_LAYOUT_COMPLETED_DELIVERY[7] as usize), OS_K1100_DELIVERY_TAG_OFFSET as u64]; fn os_k1100_retain_policy_layouts(){os_k1100_retain_layouts();for address in [core::ptr::addr_of!(OS_K1100_LAYOUT_OWNER_CELL) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_MAIN) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_SUBSCRIPTION) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_PRODUCER_INGRESS) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_TIMER) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_SOURCE) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_POLICY_BACKEND) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_POLICY) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_POLICY_RECORD) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_POLICY_HANDLE) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_POLICY_OBSERVATION) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_POLICY_SNAPSHOT) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_TASK_TIME) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_TICK_OBSERVATION) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_QUEUE_TICK_OBSERVATION) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_QUEUE_HANDLE) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_QUEUE) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_WORKLOAD_MEMBER) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_LONG_RUN) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_MUTEX) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_MUTEX_LEASE) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_MUTEX_WAIT) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_REGISTERS) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_EXPECTED) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_SOURCE_STAMP) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_QUEUE_TICK) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_COMPLETED_DELIVERY) as u64,core::ptr::addr_of!(OS_K1100_LAYOUT_NATIVE_SOURCE) as u64,core::ptr::addr_of!(OS_K1100_OPTION_TAGS) as u64] {core::hint::black_box(address);}} global_asm!(include_str!("os_k1100_policy-tasks.S")); const _: () = assert!(os_k920_exception::FRAME_BYTES == 848); const _: () = assert!(os_k920_exception::FRAME_ALIGNMENT == 16); global_asm!(include_str!("os_k1100_policy-vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); // NEW ARM adapter. This compile draft is never called from a guest entry hook. // It has no switching/ERET method and does not emit an acceptance PASS marker. use turkos_kernel::runtime::process_guest as os_k1120_owners; struct OsK1120Hardware<'a> { tables: &'a mmu::Tables } const OS_K1120_PA_MASK: u64 = 0x0000_ffff_ffff_f000; fn os_k1120_registers() -> os_k1120_owners::Registers { let (mpidr,daif,el,ttbr,tcr,sctlr,sp,spsel):(u64,u64,u64,u64,u64,u64,u64,u64); unsafe { asm!("mrs {mpidr}, MPIDR_EL1", "mrs {daif}, DAIF", "mrs {el}, CurrentEL", "mrs {ttbr}, TTBR0_EL1", "mrs {tcr}, TCR_EL1", "mrs {sctlr}, SCTLR_EL1", "mov {sp}, sp", "mrs {spsel}, SPSel",mpidr=out(reg)mpidr,daif=out(reg)daif, el=out(reg)el,ttbr=out(reg)ttbr,tcr=out(reg)tcr,sctlr=out(reg)sctlr, sp=out(reg)sp,spsel=out(reg)spsel,options(nomem,nostack,preserves_flags)); } os_k1120_owners::Registers { cpu:(mpidr&0x00ff_ffff)|(((mpidr>>32)&255)<<24), daif,current_el:el,irq_active:unsafe{read_volatile(core::ptr::addr_of!(OS_K660_IRQ_ACTIVE))}!=0, ttbr,tcr,sctlr,sp,spsel } } fn os_k1120_shared_walk(tables:&mmu::Tables,address:u64,level:u8,visited:&mut[u64;4],depth:usize) ->Result<(),os_k1120_owners::Error> { if !(1..=3).contains(&level)||depth>=4||visited[..depth].contains(&address) ||!tables.tagged_frames().any(|frame|frame.address==address){return Err(os_k1120_owners::Error::Owner);} visited[depth]=address; for index in 0..512 { let raw=unsafe{read_volatile((address as *const u64).add(index))}; if raw==0 {continue;} match raw&3 { 3 if level<3 => { if raw!=(raw&OS_K1120_PA_MASK)|3 {return Err(os_k1120_owners::Error::Owner);} os_k1120_shared_walk(tables,raw&OS_K1120_PA_MASK,level+1,visited,depth+1)?; } 1 if level<3 => {if raw&(1<<6)!=0 {return Err(os_k1120_owners::Error::Owner);}} 3 if level==3 => {if raw&(1<<6)!=0 {return Err(os_k1120_owners::Error::Owner);}} _=>return Err(os_k1120_owners::Error::Owner), } } Ok(()) } unsafe impl os_k1120_owners::Hardware for OsK1120Hardware<'_> { type Guard=mmu::OsK1020StackGuard; fn registers(&self)->os_k1120_owners::Registers {os_k1120_registers()} fn kernel_branches(&self)->Result<[u64;2],os_k1120_owners::Error>{ let r=os_k1120_registers(); // Pure CPU/root gate comes before the separately owned table graph. if r.cpu!=0||r.current_el!=4||r.daif&128==0||r.irq_active||r.ttbr!=self.tables.root ||!self.tables.tagged_frames().any(|frame|frame.address==self.tables.root){return Err(os_k1120_owners::Error::Context);} let mut branches=[0;2];for(index,value)in branches.iter_mut().enumerate(){*value=unsafe{read_volatile((self.tables.root as *const u64).add(index))}; match *value&3 {0 if *value==0=>{},3=>{if *value!=(*value&OS_K1120_PA_MASK)|3{return Err(os_k1120_owners::Error::Owner);}os_k1120_shared_walk(self.tables,*value&OS_K1120_PA_MASK,2,&mut[0;4],0)?;},1 if *value&(1<<6)==0=>{},_=>return Err(os_k1120_owners::Error::Owner)} }Ok(branches) } fn kernel_entry(&self,address:u64)->bool {unsafe extern "C"{static _text_start:u8;static _text_end:u8;}address.is_multiple_of(4)&&address>=core::ptr::addr_of!(_text_start) as u64&&addressbool {self.tables.os_k1020_stack_candidate(guard,end)} fn acquire_guard(&self,guard:u64,end:u64)->Result{unsafe{self.tables.os_k1020_acquire_stack_guard(guard,end)}.map_err(|_|os_k1120_owners::Error::Guard)} fn guard_descriptor(&self,guard:&Self::Guard)->Result{self.tables.os_k1020_stack_guard_descriptor(guard).map_err(|_|os_k1120_owners::Error::Guard)} fn restore_guard(&self,guard:Self::Guard)->Result<(),(os_k1120_owners::Error,Self::Guard)>{unsafe{self.tables.os_k1020_restore_stack_guard(guard)}.map_err(|(_,guard)|(os_k1120_owners::Error::Guard,guard))} unsafe fn read(&self,address:u64)->u8 {unsafe{read_volatile(address as *const u8)}} unsafe fn write(&self,address:u64,value:u8){unsafe{write_volatile(address as *mut u8,value)}} fn synchronize_code(&self,address:u64)->Result<(),os_k1120_owners::Error>{let ctr:u64;unsafe{asm!("mrs {ctr}, CTR_EL0",ctr=out(reg)ctr,options(nomem,nostack,preserves_flags));} let line=4u64.checked_shl(((ctr>>16)&15) as u32).ok_or(os_k1120_owners::Error::Backing)?; let end=address.checked_add(4096).ok_or(os_k1120_owners::Error::Backing)?;let mut at=address&!(line-1); while atResult<(),os_k1120_owners::Error>{let r=os_k1120_registers();if r.cpu!=0||r.daif&128==0||r.current_el!=4||r.irq_active||r.ttbr!=self.tables.root||root!=0&&r.ttbr&OS_K1120_PA_MASK==root{return Err(os_k1120_owners::Error::Context);} unsafe{asm!("dsb ish",options(nostack,preserves_flags));if asid==0{asm!("tlbi vmalle1is",options(nostack,preserves_flags));}else{asm!("tlbi aside1is, {operand}",operand=in(reg)(asid as u64)<<48,options(nostack,preserves_flags));}asm!("dsb ish","isb",options(nostack,preserves_flags));}Ok(()) } } struct OsK1120PreparedImage<'a>(turkos_kernel::runtime::process_image_recipe::Recipe<'a>); unsafe impl os_k1120_owners::PreparedImage for OsK1120PreparedImage<'_> { fn entry(&self)->u64 {self.0.start().pc} fn user_sp(&self)->u64 {self.0.start().sp} fn expected_byte(&self,page:usize,offset:usize)->Result{use turkos_kernel::runtime::process_image_recipe::Kind;let kind=match page{0=>Kind::Code,1=>Kind::Data,2=>Kind::Stack,_=>return Err(os_k1120_owners::Error::Image)};self.0.expected_byte(kind,offset).map_err(|_|os_k1120_owners::Error::Image)} } #[unsafe(no_mangle)] static mut OS_K1120_THREADS_IDENTITY:u64=1; #[unsafe(no_mangle)] static mut OS_K1120_PROCESS_IDENTITY:u64=1; fn os_k1120_identity(pointer:*mut u64)->u64 {let r=os_k1120_registers();if r.cpu!=0||r.daif&128==0||r.current_el!=4||r.irq_active{return 0;}let value=unsafe{read_volatile(pointer)};let Some(next)=value.checked_add(1)else{return 0};if value==0{return 0;}unsafe{write_volatile(pointer,next)};value} #[unsafe(no_mangle)] extern "C" fn __os_k1120_threads_identity()->u64 {os_k1120_identity(core::ptr::addr_of_mut!(OS_K1120_THREADS_IDENTITY))} #[unsafe(no_mangle)] extern "C" fn __os_k1120_process_identity()->u64 {os_k1120_identity(core::ptr::addr_of_mut!(OS_K1120_PROCESS_IDENTITY))} // K1105 development guest episode, after the K1100 owner has retired. // It publishes and retires genuine frozen Threads/Spaces/Registry owners. // No process stack switch or EL0 execution is claimed here. #[unsafe(no_mangle)] static OS_K1105_TRUSTED_ELF: [u8; 124] = os_k1105_trusted_elf(); const fn os_k1105_trusted_elf() -> [u8; 124] { let mut b = [0u8; 124]; b[0] = 0x7f; b[1] = b'E'; b[2] = b'L'; b[3] = b'F'; b[4] = 2; b[5] = 1; b[6] = 1; b[16] = 2; b[18] = 183; b[20] = 1; b[27] = 0x80; // ELF entry 0x8000_0000. b[32] = 64; b[52] = 64; b[54] = 56; b[56] = 1; b[64] = 1; b[68] = 5; b[72] = 120; b[83] = 0x80; b[96] = 4; b[104] = 64; b[112] = 1; b[120] = 1; b[123] = 0xd4; b } #[unsafe(no_mangle)] extern "C" fn __os_k1105_kernel_trampoline(_owner: u64) { // K1105 retires Ready processes from the actual bootstrap parent. // Reaching this entry would mean an unrequested context switch occurred. park(); } type OsK1105Table<'a> = turkos_kernel::runtime::process_table::Table< os_k1120_owners::Owners<'a, OsK1120Hardware<'a>, OsK1120PreparedImage<'a>>, 4>; type OsK1105Backend<'a> = os_k1120_owners::Owners<'a, OsK1120Hardware<'a>, OsK1120PreparedImage<'a>>; #[repr(align(64))] struct OsK1105TableStorage([u8; core::mem::size_of::>()]); // The masked single CPU owns this slot for the complete mode-8 episode. A // failed retirement leaves the complete typed owner graph here for inspection. #[unsafe(no_mangle)] static mut OS_K1105_TABLE_STORAGE: OsK1105TableStorage = OsK1105TableStorage([0; core::mem::size_of::>()]); #[repr(C, align(4096))] struct OsK1105WorkStack { guard: [u8; 4096], usable: [u8; 262144] } #[unsafe(no_mangle)] static mut OS_K1105_WORK_STACK: OsK1105WorkStack = OsK1105WorkStack { guard: [0; 4096], usable: [0; 262144] }; static mut OS_K1105_PROBE_ERROR: Option<&'static str> = None; const OS_K1105_LOW_CANARY: u64 = 0x4b31_3130_354c_4f57; const OS_K1105_HIGH_CANARY: u64 = 0x4b31_3130_3548_4947; #[inline(never)] fn os_k1105_run_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("process-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "process-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1105_PROBE_ERROR), None); } unsafe extern "C" { fn __os_k1105_probe_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1105_probe_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; // Return the descriptor before reporting any probe failure. The work // stack no longer has a live frame when this restoration runs. unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "process-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("process-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1105_PROBE_ERROR)) } .unwrap_or("process-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1105_probe_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1105_guest_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1105_PROBE_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1105_install_table<'a>( gate: &'a os_k1120_owners::Gate<'a, OsK1120Hardware<'a>>, authority: &'a os_k1120_owners::AuthorityCell<'a>, store: &'a os_k1120_owners::FrameStore<'a, OsK1120Hardware<'a>>, image: &'a OsK1120PreparedImage<'a>, config: turkos_kernel::runtime::address_spaces::Configuration, ) -> Result<*mut OsK1105Table<'a>, &'static str> { let backend = os_k1120_owners::Owners::new(gate, authority, store, image, 0x1105_0000, config, __os_k1105_kernel_trampoline as *const () as u64) .map_err(|_| "process-owners")?; let ptr = unsafe { core::ptr::addr_of_mut!(OS_K1105_TABLE_STORAGE.0) } .cast::>(); unsafe { OsK1105Table::new_in_place(backend, ptr) } .map_err(|_| "process-table")?; Ok(ptr) } #[inline(never)] fn os_k1105_finish_table<'a>(ptr: *mut OsK1105Table<'a>) -> Result, &'static str> { match unsafe { ptr.read() }.into_backend() { Ok(backend) => Ok(backend), Err((_, table)) => { unsafe { ptr.write(table); } Err("process-table-release") } } } #[inline(never)] fn os_k1105_guest_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe as image_recipe; use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::process_threads as threads; let before = memory.allocator.os_authority().audit() .map_err(|_| "process-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel, }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "process-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1105, &authority, &gate) .map_err(|_| "process-frame-store")?; let trusted = unsafe { image_recipe::TrustedImage::from_kernel(&OS_K1105_TRUSTED_ELF, OS_K1105_TRUSTED_ELF.len()) }.map_err(|_| "process-trusted-image")?; let recipe = image_recipe::Recipe::new(trusted, &[b"init", b"raw\xff"], &[b"A=1"]) .map_err(|_| "process-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "process-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: image.0.start().pc, user_stack_top: guest::STACK_TOP }; let mut previous = None; for generation in 1..=3u64 { let reservation = table.begin(request).map_err(|_| "process-begin")?; let prepared = table.prepare(reservation).map_err(|_| "process-prepare")?; let pid = table.publish(prepared).map_err(|_| "process-publish")?; if pid.slot() != 0 || pid.generation() != generation || table.published().count() != 1 { return Err("process-pid-generation"); } if let Some(old) = previous { if table.record(old) != Err(process::Error::Stale) || table.terminate_child(old, -1) != Err(process::Error::Stale) { return Err("process-stale-pid"); } } let record = table.record(pid).map_err(|_| "process-record")?; if record.resources.state != threads::State::Ready || record.resources.pages != (os_k1120_owners::STACK_LEASES as u64 + 9) || record.resources.spaces.iter().flatten().count() != 2 || record.resources.handles.iter().flatten().count() != 3 { return Err("process-resource-owners"); } table.terminate_child(pid, generation as i32) .map_err(|_| "process-terminate")?; if table.wait_child(pid) != Ok(generation as i32) || table.published().count() != 0 { return Err("process-wait-release"); } previous = Some(pid); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "process-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("process-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "process-after-gate")? .map_err(|_| "process-after-inventory")? != before { return Err("process-physical-rollback"); } write_line("TURKOS/OS/K1105: PASS PID-SLOT=0 GENERATIONS=1,2,3 PHYSICAL-OWNERS-RELEASED"); Ok(()) } // K1110 diagnostic: the parent authority comes from three actual stack // landings, not from a caller-supplied PID or a synthetic execution value. static mut OS_K1110_PROBE_ERROR: Option<&'static str> = None; struct OsK1110Episode { parent: turkos_kernel::runtime::process_table::ProcessId, sibling: Option, child: Option, } #[inline(never)] fn os_k1110_run_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("parent-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "parent-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1110_PROBE_ERROR), None); } unsafe extern "C" { fn __os_k1110_probe_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1110_probe_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "parent-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("parent-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1110_PROBE_ERROR)) } .unwrap_or("parent-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1110_probe_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1110_guest_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1110_PROBE_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1110_spawn(table: &mut OsK1105Table<'_>) -> Result { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::process_guest as guest; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "parent-begin")?; let prepared = table.prepare(reservation).map_err(|_| "parent-prepare")?; table.publish(prepared).map_err(|_| "parent-publish") } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1110_parent_body( table: *mut u8, state: *mut OsK1110Episode, phase: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table = unsafe { &mut *table.cast::>() }; let state = unsafe { &mut *state }; let result = (|| -> Result<(), &'static str> { unsafe { table.register_actual_landing( if phase == 2 { state.sibling.ok_or("parent-sibling-missing")? } else { state.parent }) } .map_err(|_| "parent-actual-landing")?; match phase { 1 => { let sibling = os_k1110_spawn(table)?; let child = os_k1110_spawn(table)?; if table.record(sibling).map_err(|_| "parent-sibling-record")?.parent != Some(state.parent) || table.record(child).map_err(|_| "parent-child-record")?.parent != Some(state.parent) { return Err("parent-identity"); } table.terminate_child(child, 42).map_err(|_| "parent-child-exit")?; state.sibling = Some(sibling); state.child = Some(child); } 2 => { let child = state.child.ok_or("parent-child-missing")?; let before = table.snapshot(); if table.wait_child(child) != Err(process::Error::Foreign) || table.terminate_child(child, 9) != Err(process::Error::Foreign) || table.snapshot() != before || table.record(child).map_err(|_| "parent-foreign-record")?.parent != Some(state.parent) { return Err("parent-foreign-authority"); } } 3 => { let sibling = state.sibling.ok_or("parent-sibling-missing")?; let child = state.child.ok_or("parent-child-missing")?; if table.wait_child(child) != Ok(42) || table.wait_child(child) != Err(process::Error::NoChild) { return Err("parent-child-result"); } table.terminate_child(sibling, 17) .map_err(|_| "parent-sibling-exit")?; if table.wait_child(sibling) != Ok(17) { return Err("parent-sibling-result"); } } _ => return Err("parent-phase"), } Ok(()) })(); if unsafe { table.release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1110_PROBE_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1110_on_parent_stack( table: *mut OsK1105Table<'_>, state: &mut OsK1110Episode, phase: u64, ) -> Result<(), &'static str> { let pid = if phase == 2 { state.sibling.ok_or("parent-sibling-missing")? } else { state.parent }; let (_, top) = unsafe { &*table }.landing_stack(pid) .map_err(|_| "parent-landing-layout")?; unsafe extern "C" { fn __os_k1110_enter_parent_stack( table: *mut u8, state: *mut u8, stack_top: u64, phase: u64) -> u64; } let status = unsafe { __os_k1110_enter_parent_stack( table.cast(), (state as *mut OsK1110Episode).cast(), top, phase) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1110_PROBE_ERROR)) }.unwrap_or("parent-stack-phase")); } Ok(()) } #[inline(never)] fn os_k1110_guest_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe as image_recipe; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "parent-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "parent-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1110, &authority, &gate) .map_err(|_| "parent-frame-store")?; let trusted = unsafe { image_recipe::TrustedImage::from_kernel(&OS_K1105_TRUSTED_ELF, OS_K1105_TRUSTED_ELF.len()) } .map_err(|_| "parent-trusted-image")?; let recipe = image_recipe::Recipe::new(trusted, &[b"init", b"raw\xff"], &[b"A=1"]) .map_err(|_| "parent-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "parent-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let parent = os_k1110_spawn(unsafe { &mut *table_ptr })?; let mut episode = OsK1110Episode { parent, sibling: None, child: None }; os_k1110_on_parent_stack(table_ptr, &mut episode, 1)?; if episode.sibling.is_none() || episode.child.is_none() { return Err("parent-two-children"); } let foreign_before = authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "parent-foreign-before-gate")? .map_err(|_| "parent-foreign-before-inventory")?; os_k1110_on_parent_stack(table_ptr, &mut episode, 2)?; let foreign_after = authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "parent-foreign-after-gate")? .map_err(|_| "parent-foreign-after-inventory")?; if foreign_after != foreign_before { return Err("parent-foreign-resource-change"); } os_k1110_on_parent_stack(table_ptr, &mut episode, 3)?; let table = unsafe { &mut *table_ptr }; table.terminate_child(parent, 0).map_err(|_| "parent-exit")?; if table.wait_child(parent) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("parent-final-result"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "parent-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("parent-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "parent-after-gate")? .map_err(|_| "parent-after-inventory")? != before { return Err("parent-physical-rollback"); } write_line("TURKOS/OS/K1110: PASS PARENT-CHILDREN=2 FOREIGN-WAIT=REJECTED RESULTS=42,17 OWNERS-RELEASED"); Ok(()) } // K1115 development guest: actual metadata capacity runs out after one image // page is acquired. Partial process ownership must return before publication. static mut OS_K1115_PROBE_ERROR: Option<&'static str> = None; #[inline(never)] fn os_k1115_run_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("resources-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "resources-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1115_PROBE_ERROR), None); } unsafe extern "C" { fn __os_k1115_probe_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1115_probe_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "resources-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("resources-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1115_PROBE_ERROR)) } .unwrap_or("resources-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1115_probe_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1115_guest_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1115_PROBE_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1115_guest_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe as image_recipe; use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::process_threads as threads; let before = memory.allocator.os_authority().audit() .map_err(|_| "resources-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "resources-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1115, &authority, &gate) .map_err(|_| "resources-frame-store")?; let trusted = unsafe { image_recipe::TrustedImage::from_kernel(&OS_K1105_TRUSTED_ELF, OS_K1105_TRUSTED_ELF.len()) } .map_err(|_| "resources-trusted-image")?; let recipe = image_recipe::Recipe::new(trusted, &[b"init", b"raw\xff"], &[b"A=1"]) .map_err(|_| "resources-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "resources-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let mut pressure = store.k1115_fill_to_one_free(0x1115_ffff) .map_err(|_| "resources-pressure-acquire")?; let pressure_snapshot = store.snapshot() .map_err(|_| "resources-pressure-snapshot")?; if pressure_snapshot.free_metadata != 1 || pressure_snapshot.live_pages != guest::FRAME_CAPACITY - 1 { return Err("resources-pressure-capacity"); } let pressure_audit = authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "resources-pressure-gate")? .map_err(|_| "resources-pressure-inventory")?; let reservation = table.begin(request).map_err(|_| "resources-begin-negative")?; let reservation = match table.prepare(reservation) { Err((process::Error::Resource, reservation)) => reservation, Err(_) => return Err("resources-wrong-failure"), Ok(_) => return Err("resources-failure-not-observed"), }; let after_failure = store.snapshot() .map_err(|_| "resources-failure-snapshot")?; let after_audit = authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "resources-failure-gate")? .map_err(|_| "resources-failure-inventory")?; if after_failure != pressure_snapshot || after_audit != pressure_audit || table.snapshot() != (process::Snapshot { published: 0, constructing: 1, zombies: 0, quarantined: 0, }) { return Err("resources-partial-rollback"); } table.abort(reservation).map_err(|_| "resources-abort-negative")?; if table.snapshot() != process::Snapshot::default() { return Err("resources-hidden-reservation"); } store.k1115_release_pressure(&mut pressure) .map_err(|_| "resources-pressure-release")?; let clear = store.snapshot().map_err(|_| "resources-clear-snapshot")?; if clear.live_pages != 0 || clear.free_metadata != guest::FRAME_CAPACITY || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "resources-clear-gate")? .map_err(|_| "resources-clear-inventory")? != before { return Err("resources-pressure-leak"); } let reservation = table.begin(request).map_err(|_| "resources-begin-positive")?; let prepared = table.prepare(reservation).map_err(|_| "resources-prepare-positive")?; let pid = table.publish(prepared).map_err(|_| "resources-publish-positive")?; let record = table.record(pid).map_err(|_| "resources-record-positive")?; let observed = record.resources; if observed.process != pid || observed.resource_owner == 0 || observed.state != threads::State::Ready || !observed.image_ready || !observed.stack_ready || !observed.published || observed.spaces.iter().flatten().count() != 2 || observed.handles.iter().flatten().count() != 3 || observed.pages != guest::STACK_LEASES as u64 + 9 { return Err("resources-positive-owners"); } table.terminate_child(pid, 15).map_err(|_| "resources-positive-exit")?; if table.wait_child(pid) != Ok(15) || table.snapshot() != process::Snapshot::default() { return Err("resources-positive-result"); } let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "resources-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("resources-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "resources-after-gate")? .map_err(|_| "resources-after-inventory")? != before { return Err("resources-physical-rollback"); } write_line("TURKOS/OS/K1115: PASS SPACES=2 HANDLES=3 MID-BAG-EXHAUSTION=ROLLED-BACK HEALTHY-EXIT=15 OWNERS-RELEASED"); Ok(()) } // K1120 development guest: invalid stack configuration never reaches a public // PID or ready thread list; a complete image and stack then publish normally. static mut OS_K1120_PROBE_ERROR: Option<&'static str> = None; #[inline(never)] fn os_k1120_run_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("publish-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "publish-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1120_PROBE_ERROR), None); } unsafe extern "C" { fn __os_k1120_probe_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1120_probe_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "publish-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("publish-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1120_PROBE_ERROR)) } .unwrap_or("publish-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1120_probe_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1120_guest_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1120_PROBE_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1120_guest_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe as image_recipe; use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::process_threads as threads; let before = memory.allocator.os_authority().audit() .map_err(|_| "publish-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "publish-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1120, &authority, &gate) .map_err(|_| "publish-frame-store")?; let trusted = unsafe { image_recipe::TrustedImage::from_kernel(&OS_K1105_TRUSTED_ELF, OS_K1105_TRUSTED_ELF.len()) } .map_err(|_| "publish-trusted-image")?; let recipe = image_recipe::Recipe::new(trusted, &[b"init", b"raw\xff"], &[b"A=1"]) .map_err(|_| "publish-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "publish-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let invalid = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP + 16 }; let reservation = table.begin(invalid).map_err(|_| "publish-begin-negative")?; let reservation = match table.prepare(reservation) { Err((process::Error::Preparation, reservation)) => reservation, Err(_) => return Err("publish-wrong-prepare-error"), Ok(_) => return Err("publish-invalid-stack-prepared"), }; if table.published().count() != 0 || table.snapshot() != (process::Snapshot { published: 0, constructing: 1, zombies: 0, quarantined: 0, }) { return Err("publish-negative-visible"); } let reservation = match table.publish(reservation) { Err((process::Error::Preparation, reservation)) => reservation, Err(_) => return Err("publish-wrong-enqueue-error"), Ok(_) => return Err("publish-unprepared-enqueued"), }; if table.published().count() != 0 { return Err("publish-failed-pid-listed"); } table.abort(reservation).map_err(|_| "publish-abort-negative")?; if table.snapshot() != process::Snapshot::default() || store.snapshot().map_err(|_| "publish-negative-store")?.live_pages != 0 || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "publish-negative-gate")? .map_err(|_| "publish-negative-inventory")? != before { return Err("publish-negative-rollback"); } let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "publish-negative-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("publish-negative-owner-list"); } drop(backend); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "publish-positive-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let valid = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(valid).map_err(|_| "publish-begin-positive")?; let prepared = table.prepare(reservation).map_err(|_| "publish-prepare-positive")?; if table.published().count() != 0 || table.snapshot().constructing != 1 { return Err("publish-premature-visibility"); } let pid = table.publish(prepared).map_err(|_| "publish-positive")?; let record = table.record(pid).map_err(|_| "publish-positive-record")?; let observed = record.resources; if pid.slot() != 0 || pid.generation() != 1 || table.published().count() != 1 || observed.state != threads::State::Ready || !observed.image_ready || !observed.stack_ready || !observed.published || observed.spaces.iter().flatten().count() != 2 || observed.handles.iter().flatten().count() != 3 { return Err("publish-positive-state"); } table.terminate_child(pid, 29).map_err(|_| "publish-positive-exit")?; if table.wait_child(pid) != Ok(29) || table.snapshot() != process::Snapshot::default() { return Err("publish-positive-result"); } let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "publish-positive-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("publish-positive-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "publish-after-gate")? .map_err(|_| "publish-after-inventory")? != before { return Err("publish-physical-rollback"); } write_line("TURKOS/OS/K1120: PASS BAD-STACK=REJECTED FAILED-PID=ABSENT READY=1 RESULT=29 OWNERS-RELEASED"); Ok(()) } // K1125 preparation episode. This uses the linked frozen ABI decoder and a // trusted two-page user ELF on a guarded kernel stack. It does not enter EL0. static mut OS_K1125_PREFLIGHT_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static OS_K1125_USER_ELF: [u8; 8808] = *include_bytes!("k1125-user.elf"); #[inline(never)] fn os_k1125_run_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("abi-preflight-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "abi-preflight-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1125_PREFLIGHT_ERROR), None); } unsafe extern "C" { fn __os_k1125_preflight_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1125_preflight_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "abi-preflight-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("abi-preflight-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_PREFLIGHT_ERROR)) } .unwrap_or("abi-preflight-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1125_preflight_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1125_guest_preflight( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1125_PREFLIGHT_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1125_guest_preflight( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::process_image_recipe::{Kind, Recipe, TrustedImage}; use turkos_kernel::runtime::syscall_abi as abi; use turkos_os_exceptions::exceptions::Frame; let before = memory.allocator.os_authority().audit() .map_err(|_| "abi-preflight-before-inventory")?; if os_k1120_registers().ttbr != tables.root { return Err("abi-preflight-root"); } let trusted = unsafe { TrustedImage::from_kernel(&OS_K1125_USER_ELF, OS_K1125_USER_ELF.len()) } .map_err(|_| "abi-preflight-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "abi-preflight-recipe")?; let start = recipe.start(); if start.pc != 0x8000_0000 || start.spsr != 0x3c0 || recipe.expected_byte(Kind::Code, 16).ok() != Some(1) || recipe.expected_byte(Kind::Code, 19).ok() != Some(0xd4) || recipe.expected_byte(Kind::Data, 0).ok() != Some(0) { return Err("abi-preflight-image-bytes"); } let mut frame = Frame::EMPTY; frame.x[7] = abi::ABI_VERSION; frame.x[8] = abi::CALL_VERSION; if abi::decode_call(&frame) != Ok(abi::Call::Version) { return Err("abi-preflight-version"); } frame.x[8] = u64::MAX; if abi::decode_call(&frame) != Err(abi::Errno::NoSys) { return Err("abi-preflight-full-number"); } frame.x[7] = abi::ABI_VERSION + 1; frame.x[8] = abi::CALL_VERSION; if abi::decode_call(&frame) != Err(abi::Errno::NoSys) { return Err("abi-preflight-unknown-version"); } if memory.allocator.os_authority().audit() .map_err(|_| "abi-preflight-after-inventory")? != before { return Err("abi-preflight-inventory"); } write_line("TURKOS/OS/K1125-PREFLIGHT: ELF=OK ABI-DECODE=OK NO-SVC-NO-ERET"); Ok(()) } // Mode 13 is a process-owner launch diagnostic. It never enters EL0. static mut OS_K1125_LAUNCH_ERROR: Option<&'static str> = None; #[inline(never)] fn os_k1125_launch_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("launch-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "launch-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1125_LAUNCH_ERROR), None); } unsafe extern "C" { fn __os_k1125_launch_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1125_launch_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "launch-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("launch-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_LAUNCH_ERROR)) } .unwrap_or("launch-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1125_launch_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1125_launch_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1125_LAUNCH_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1125_launch_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table = unsafe { &mut *table.cast::>() }; let pid = unsafe { *pid.cast::() }; if unsafe { table.register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = table.current_launch(pid).map_err(|_| "launch-owner-snapshot")?; let registers = os_k1120_registers(); if launch.process != pid || launch.kernel_root != kernel_root || launch.user_root == kernel_root || launch.user_root == 0 || launch.user_asid == 0 || launch.resource_owner == 0 || launch.user_pc != 0x8000_0000 || launch.user_sp != 0x8000_4000 - 512 || launch.user_spsr != 0x3c0 || launch.code_frame == launch.data_frame || launch.code_frame == launch.stack_frame || launch.data_frame == launch.stack_frame || launch.code_frame == 0 || launch.data_frame == 0 || launch.stack_frame == 0 || registers.sp <= launch.kernel_stack_low || registers.sp >= launch.kernel_stack_high || registers.ttbr != kernel_root { return Err("launch-owner-values"); } Ok(()) })(); if unsafe { table.release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1125_LAUNCH_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1125_launch_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "launch-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "launch-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1125, &authority, &gate) .map_err(|_| "launch-frame-store")?; let trusted = unsafe { TrustedImage::from_kernel(&OS_K1125_USER_ELF, OS_K1125_USER_ELF.len()) } .map_err(|_| "launch-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "launch-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "launch-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "launch-begin")?; let prepared = table.prepare(reservation).map_err(|_| "launch-prepare")?; let pid = table.publish(prepared).map_err(|_| "launch-publish")?; let (_, stack_top) = table.landing_stack(pid).map_err(|_| "launch-landing-stack")?; unsafe extern "C" { fn __os_k1125_enter_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1125_enter_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_LAUNCH_ERROR)) } .unwrap_or("launch-process-body")); } table.terminate_child(pid, 0).map_err(|_| "launch-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("launch-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "launch-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("launch-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "launch-after-gate")? .map_err(|_| "launch-after-inventory")? != before { return Err("launch-physical-rollback"); } write_line("TURKOS/OS/K1125-LAUNCH: OWNER=OK ROOT=RETAINED FRAMES=RETAINED PROCESS-STACK=REAL NO-EL0-NO-SVC-NO-ERET"); Ok(()) } global_asm!(include_str!("os_k1125_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); // Mode 14 executes the trusted K1125 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fifth SVC. static mut OS_K1125_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1125_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1125_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1125_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1125_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1125_KERNEL_FRAME: u64 = 0; struct OsK1125El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, old_vbar: u64, } #[inline(never)] fn os_k1125_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1125_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1125_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1125_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1125_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1125_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1125_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1125_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1125-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1125-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1125El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1125-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1125-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1125-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [0x14u64, 0x2c, 0x50, 0x6c, 0x80] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1125-TRACE: FRAME"); let expected = match episode.calls { 0 | 3 | 4 => Ok(abi::Call::Version), 1 | 2 => Err(abi::Errno::NoSys), _ => park(), }; let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if episode.calls == 1 && (frame.x[7] != abi::ABI_VERSION || frame.x[8] != u64::MAX) || episode.calls == 2 && (frame.x[7] != abi::ABI_VERSION + 1 || frame.x[8] != abi::CALL_VERSION) || abi::decode_call(&abi_frame) != expected { park(); } if episode.calls == 4 { for (index, expected) in [1u64, 2, 3, 4, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } episode.calls = 5; write_line("TURKOS/OS/K1125-TRACE: COMPLETE"); return 1; } let result = if episode.calls == 1 || episode.calls == 2 { abi::AbiResult::Err(abi::Errno::NoSys) } else { abi::AbiResult::Ok(abi::Success::new(abi::ABI_VERSION).unwrap()) }; frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1125-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1125_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1125-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1125El0Episode {table, pid, launch, calls: 0, old_vbar}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1125_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1125_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1125_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1125_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1125_ACTIVE), (&mut episode as *mut OsK1125El0Episode) as u64); } unsafe extern "C" { fn __os_k1125_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1125_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1125-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1125_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1125_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1125_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1125_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1125_KERNEL_FRAME), 0); } if episode.calls != 5 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1125_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1125_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1126, &authority, &gate) .map_err(|_| "el0-frame-store")?; let trusted = unsafe { TrustedImage::from_kernel(&OS_K1125_USER_ELF, OS_K1125_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1125-TRACE: PUBLISHED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1125_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1125_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1125_EL0_ERROR)) } .unwrap_or("el0-process-body")); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("el0-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1125-EL0: FIVE-SVC=OK ABI-V1=OK ENOSYS=OK ERET=4 OWNER=RELEASED"); write_line("TURKOS/OS/K1125: PASS"); Ok(()) } global_asm!(include_str!("os_k1130_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1130_USER_ELF: [u8; 8992] = *include_bytes!("k1130-user.elf"); // Mode 15 executes the trusted K1130 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fourth SVC. static mut OS_K1130_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1130_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1130_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1130_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1130_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1130_KERNEL_FRAME: u64 = 0; struct OsK1130El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, old_vbar: u64, } #[inline(never)] fn os_k1130_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1130_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1130_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1130_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1130_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1130_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1130_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1130_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1130_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1130-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1130_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1130_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1130_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1130_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1130-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1130El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1130-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1130-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1130-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [0x2cu64, 0x68, 0x9c, 0xc0] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1130-TRACE: FRAME"); let canonical = [1u64, 2, (-3i64) as u64, 4, 5, (-6i64) as u64]; let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } episode.calls = 4; write_line("TURKOS/OS/K1130-TRACE: COMPLETE"); return 1; } if frame.x[8] != abi::CALL_ARGUMENTS { park(); } for (index, expected) in canonical.into_iter().enumerate() { let expected = if episode.calls == 1 && index == 1 { 0x0000_1234_0000_0002 } else { expected }; if frame.x[index] != expected { park(); } } let normalized = abi::Arguments::normalize(canonical, abi::ARGUMENT_WIDTHS); let expected_call = if episode.calls == 1 { Err(abi::Errno::Invalid) } else { Ok(abi::Call::Arguments(normalized)) }; if abi::decode_call(&abi_frame) != expected_call { park(); } let result = if episode.calls == 1 { abi::AbiResult::Err(abi::Errno::Invalid) } else { abi::AbiResult::Ok(normalized.checksum()) }; frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1130-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1130_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1130-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1130El0Episode {table, pid, launch, calls: 0, old_vbar}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1130_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1130_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1130_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1130_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1130_ACTIVE), (&mut episode as *mut OsK1130El0Episode) as u64); } unsafe extern "C" { fn __os_k1130_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1130_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1130-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1130_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1130_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1130_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1130_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1130_KERNEL_FRAME), 0); } if episode.calls != 4 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1130_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1130_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1131, &authority, &gate) .map_err(|_| "el0-frame-store")?; let trusted = unsafe { TrustedImage::from_kernel(&OS_K1130_USER_ELF, OS_K1130_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1130-TRACE: PUBLISHED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1130_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1130_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1130_EL0_ERROR)) } .unwrap_or("el0-process-body")); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("el0-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1130-EL0: FOUR-SVC=OK SIX-ARGS=OK EINVAL=OK ERET=3 OWNER=RELEASED"); write_line("TURKOS/OS/K1130: PASS"); Ok(()) } global_asm!(include_str!("os_k1135_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1135_USER_ELF: [u8; 9176] = *include_bytes!("k1135-user.elf"); // Mode 16 executes the trusted K1135 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fifth SVC. static mut OS_K1135_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1135_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1135_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1135_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1135_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1135_KERNEL_FRAME: u64 = 0; struct OsK1135El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, write_budget: u64, old_vbar: u64, } #[inline(never)] fn os_k1135_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1135_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1135_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1135_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1135_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1135_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1135_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1135_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1135_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1135-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1135_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1135_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1135_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1135_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1135-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1135El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1135-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1135-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1135-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [0xe8u64, 0xe8, 0xe8, 0xb8, 0xcc] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1135-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 4 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) { park(); } for (index, expected) in [1u64, 2, 3, 4, 5, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } episode.calls = 5; write_line("TURKOS/OS/K1135-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); let result = if episode.calls == 3 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) { park(); } abi::AbiResult::Ok(abi::Success::new(abi::ABI_VERSION).unwrap()) } else { if frame.x[8] != abi::CALL_WRITE || frame.x[2] != 8 { park(); } let decoded = abi::decode_call(&abi_frame); match episode.calls { 0 => { let Ok(abi::Call::Write(write)) = decoded else { park(); }; if write.fd() != 3 || write.address() != 0x8000_1020 || write.bytes() != 8 { park(); } abi::AbiResult::Err(abi::Errno::BadFd) } 1 => { if frame.x[0] != 1 || frame.x[1] != u64::MAX || decoded != Err(abi::Errno::Fault) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } 2 => { let Ok(abi::Call::Write(write)) = decoded else { park(); }; if write.fd() != 1 || write.address() != 0x8000_1020 || write.bytes() != 8 || episode.write_budget != 0 || unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 0 { park(); } // The bounded diagnostic service refuses preparation before // any copy or output while its retained budget is exhausted. abi::AbiResult::Err(abi::Errno::NoMem) } _ => park(), } }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1135-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1135_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1135-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1135El0Episode {table, pid, launch, calls: 0, write_budget: 0, old_vbar}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1135_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1135_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1135_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1135_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1135_ACTIVE), (&mut episode as *mut OsK1135El0Episode) as u64); } unsafe extern "C" { fn __os_k1135_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1135_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1135-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1135_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1135_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1135_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1135_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1135_KERNEL_FRAME), 0); } if episode.calls != 5 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1135_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1135_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1136, &authority, &gate) .map_err(|_| "el0-frame-store")?; let trusted = unsafe { TrustedImage::from_kernel(&OS_K1135_USER_ELF, OS_K1135_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1135-TRACE: PUBLISHED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1135_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1135_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1135_EL0_ERROR)) } .unwrap_or("el0-process-body")); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("el0-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1135-EL0: FIVE-SVC=OK EBADF=OK EFAULT=OK ENOMEM=OK WRAPPER=TAGGED ERET=4 OWNER=RELEASED"); write_line("TURKOS/OS/K1135: PASS"); Ok(()) } global_asm!(include_str!("os_k1140_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1140_USER_ELF: [u8; 9256] = *include_bytes!("k1140-user.elf"); // Mode 17 executes the trusted K1140 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fourth SVC. static mut OS_K1140_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1140_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1140_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1140_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1140_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1140_KERNEL_FRAME: u64 = 0; struct OsK1140El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, } #[inline(never)] fn os_k1140_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1140_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1140_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1140_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1140_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1140_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1140_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1140_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1140_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1140-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1140_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1140_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1140_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1140_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1140-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1140El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1140-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1140-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1140-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [0xb8u64, 0xb8, 0xb8, 0x9c] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1140-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } episode.calls = 4; write_line("TURKOS/OS/K1140-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1140-A\n" } else { b"K1140-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1140-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1140_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1140-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1140El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1140_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1140_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1140_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1140_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1140_ACTIVE), (&mut episode as *mut OsK1140El0Episode) as u64); } unsafe extern "C" { fn __os_k1140_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1140_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1140-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1140_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1140_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1140_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1140_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1140_KERNEL_FRAME), 0); } if episode.calls != 4 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1140_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1140_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1141, &authority, &gate) .map_err(|_| "el0-frame-store")?; let trusted = unsafe { TrustedImage::from_kernel(&OS_K1140_USER_ELF, OS_K1140_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1140-TRACE: PUBLISHED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1140_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1140_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1140_EL0_ERROR)) } .unwrap_or("el0-process-body")); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("el0-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1140-EL0: FOUR-SVC=OK WRITE=16 EFAULT=OK NO-PARTIAL-OUTPUT=OK ERET=3 OWNER=RELEASED"); write_line("TURKOS/OS/K1140: PASS"); Ok(()) } global_asm!(include_str!("os_k1145_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1145_USER_ELF: [u8; 9256] = *include_bytes!("k1145-user.elf"); // Mode 18 executes the trusted K1145 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fourth SVC. static mut OS_K1145_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1145_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1145_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1145_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1145_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1145_KERNEL_FRAME: u64 = 0; struct OsK1145El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, } #[inline(never)] fn os_k1145_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1145_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1145_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1145_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1145_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1145_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1145_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1145_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1145_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1145-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1145_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1145_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1145_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1145_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1145-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1145El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1145-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1145-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1145-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [0xb8u64, 0xb8, 0xb8, 0x9c] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1145-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } episode.calls = 4; write_line("TURKOS/OS/K1145-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1145-A\n" } else { b"K1145-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1145-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1145_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1145-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1145El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1145_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1145_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1145_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1145_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1145_ACTIVE), (&mut episode as *mut OsK1145El0Episode) as u64); } unsafe extern "C" { fn __os_k1145_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1145_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1145-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1145_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1145_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1145_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1145_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1145_KERNEL_FRAME), 0); } if episode.calls != 4 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1145_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1145_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1146, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1145_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1145_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1145_USER_ELF, OS_K1145_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1145-TRACE: TRUNCATED-HEADER-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1145_USER_ELF, OS_K1145_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1145-TRACE: PUBLISHED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1145_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1145_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1145_EL0_ERROR)) } .unwrap_or("el0-process-body")); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("el0-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1145-EL0: HEADER-REJECTED=6 PID=0 FRAME=0 INIT=OK FOUR-SVC=OK ERET=3 OWNER=RELEASED"); write_line("TURKOS/OS/K1145: PASS"); Ok(()) } global_asm!(include_str!("os_k1150_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1150_USER_ELF: [u8; 9312] = *include_bytes!("k1150-user.elf"); // Mode 19 executes the trusted K1150 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fourth SVC. static mut OS_K1150_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1150_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1150_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1150_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1150_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1150_KERNEL_FRAME: u64 = 0; struct OsK1150El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, } #[inline(never)] fn os_k1150_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1150_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1150_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1150_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1150_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1150_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1150_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1150_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1150_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1150-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1150_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1150_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1150_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1150_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1150-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1150El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1150-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1150-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1150-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [0x1b0u64, 0x1b0, 0x1b0, 0x160] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1150-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } if unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 4 { park(); } episode.calls = 4; write_line("TURKOS/OS/K1150-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1150-A\n" } else { b"K1150-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1150-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1150_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1150-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1150El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1150_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1150_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1150_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1150_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1150_ACTIVE), (&mut episode as *mut OsK1150El0Episode) as u64); } unsafe extern "C" { fn __os_k1150_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1150_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1150-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1150_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1150_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1150_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1150_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1150_KERNEL_FRAME), 0); } if episode.calls != 4 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1150_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1150_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1151, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1150_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1150_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1150_USER_ELF, OS_K1150_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1150-TRACE: TRUNCATED-HEADER-REJECTED"); let long_a = [b'A'; 63]; let long_b = [b'B'; 63]; let long_c = [b'C'; 63]; let long_d = [b'D'; 64]; let max_argv: [&[u8]; 2] = [&long_a, &long_b]; let max_env: [&[u8]; 2] = [&long_c, &long_c]; let over_env: [&[u8]; 2] = [&long_c, &long_d]; let bound_image = unsafe { TrustedImage::from_kernel(&OS_K1150_USER_ELF, OS_K1150_USER_ELF.len()) }.map_err(|_| "argv-bound-image")?; let bound = Recipe::new(bound_image, &max_argv, &max_env) .map_err(|_| "argv-exact-limit")?; if bound.start().argc != 2 || bound.start().envc != 2 { return Err("argv-exact-count"); } let over_image = unsafe { TrustedImage::from_kernel(&OS_K1150_USER_ELF, OS_K1150_USER_ELF.len()) }.map_err(|_| "argv-over-image")?; if !matches!(Recipe::new(over_image, &max_argv, &over_env), Err(ImageError::Arguments)) { return Err("argv-over-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "argv-after-gate")? .map_err(|_| "argv-after-inventory")? != before { return Err("argv-resource-change"); } write_line("TURKOS/OS/K1150-TRACE: TEXT-LIMIT-PLUS-ONE-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1150_USER_ELF, OS_K1150_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let argv: [&[u8]; 2] = [b"init", b"alpha"]; let env: [&[u8]; 2] = [b"MODE=1", b"LANG=tr"]; let recipe = Recipe::new(trusted, &argv, &env) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1150-TRACE: PUBLISHED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1150_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1150_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1150_EL0_ERROR)) } .unwrap_or("el0-process-body")); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("el0-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1150-EL0: TEXT-LIMIT+1=REJECTED ARGV=2 ENV=2 STACK=PRIVATE FOUR-SVC=OK ERET=3 OWNER=RELEASED"); write_line("TURKOS/OS/K1150: PASS"); Ok(()) } global_asm!(include_str!("os_k1155_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1155_USER_ELF: [u8; 9312] = *include_bytes!("k1155-user.elf"); static OS_K1155_BAD_ENTRY_ELF: [u8; 9312] = *include_bytes!("k1155-bad-entry.elf"); // Mode 20 executes the trusted K1155 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fourth SVC. static mut OS_K1155_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1155_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1155_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1155_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1155_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1155_KERNEL_FRAME: u64 = 0; struct OsK1155El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, } #[inline(never)] fn os_k1155_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1155_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1155_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1155_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1155_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1155_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1155_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1155_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1155_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1155-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1155_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1155_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1155_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1155_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1155-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1155El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1155-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1155-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1155-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [0x1b0u64, 0x1b0, 0x1b0, 0x160] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1155-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } if unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 4 { park(); } episode.calls = 4; write_line("TURKOS/OS/K1155-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1155-A\n" } else { b"K1155-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1155-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1155_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1155-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } use turkos_kernel::runtime::process_guest as guest; let mask = 0x0000_ffff_ffff_f000u64; let l1 = unsafe { read_volatile((launch.user_root as *const u64) .add(((guest::CODE >> 30) & 511) as usize)) }; if l1 & 3 != 3 { return Err("entry-l1"); } let l2 = unsafe { read_volatile(((l1 & mask) as *const u64) .add(((guest::CODE >> 21) & 511) as usize)) }; if l2 & 3 != 3 { return Err("entry-l2"); } let l3 = l2 & mask; let leaf = |va: u64| unsafe { read_volatile((l3 as *const u64) .add(((va >> 12) & 511) as usize)) }; let code = leaf(guest::CODE); let stack = leaf(guest::STACK); let access = 3u64 | (1 << 6) | (1 << 10) | (1 << 11) | (1 << 53); if code & mask != launch.code_frame || code & access != access || code & (1 << 7) == 0 || code & (1 << 54) != 0 || stack & mask != launch.stack_frame || stack & access != access || stack & (1 << 7) != 0 || stack & (1 << 54) == 0 || leaf(guest::DATA + guest::PAGE) != 0 || launch.user_pc != guest::CODE || launch.user_sp != guest::STACK_TOP - 512 || launch.user_spsr != 0x3c0 { return Err("entry-rx-stack-rw-spsr"); } write_line("TURKOS/OS/K1155-TRACE: ENTRY-RX-STACK-RW-SPSR-EL0"); let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1155El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1155_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1155_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1155_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1155_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1155_ACTIVE), (&mut episode as *mut OsK1155El0Episode) as u64); } unsafe extern "C" { fn __os_k1155_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1155_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1155-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1155_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1155_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1155_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1155_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1155_KERNEL_FRAME), 0); } if episode.calls != 4 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1155_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1155_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1156, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1155_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1155_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1155_USER_ELF, OS_K1155_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1155-TRACE: TRUNCATED-HEADER-REJECTED"); let long_a = [b'A'; 63]; let long_b = [b'B'; 63]; let long_c = [b'C'; 63]; let long_d = [b'D'; 64]; let max_argv: [&[u8]; 2] = [&long_a, &long_b]; let max_env: [&[u8]; 2] = [&long_c, &long_c]; let over_env: [&[u8]; 2] = [&long_c, &long_d]; let bound_image = unsafe { TrustedImage::from_kernel(&OS_K1155_USER_ELF, OS_K1155_USER_ELF.len()) }.map_err(|_| "argv-bound-image")?; let bound = Recipe::new(bound_image, &max_argv, &max_env) .map_err(|_| "argv-exact-limit")?; if bound.start().argc != 2 || bound.start().envc != 2 { return Err("argv-exact-count"); } let over_image = unsafe { TrustedImage::from_kernel(&OS_K1155_USER_ELF, OS_K1155_USER_ELF.len()) }.map_err(|_| "argv-over-image")?; if !matches!(Recipe::new(over_image, &max_argv, &over_env), Err(ImageError::Arguments)) { return Err("argv-over-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "argv-after-gate")? .map_err(|_| "argv-after-inventory")? != before { return Err("argv-resource-change"); } write_line("TURKOS/OS/K1155-TRACE: TEXT-LIMIT-PLUS-ONE-REJECTED"); let bad_image = unsafe { TrustedImage::from_kernel(&OS_K1155_BAD_ENTRY_ELF, OS_K1155_BAD_ENTRY_ELF.len()) }.map_err(|_| "bad-entry-trust")?; if !matches!(Recipe::new(bad_image, &[b"init"], &[]), Err(ImageError::Entry)) { return Err("kernel-entry-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "entry-after-gate")? .map_err(|_| "entry-after-inventory")? != before { return Err("entry-resource-change"); } write_line("TURKOS/OS/K1155-TRACE: KERNEL-ENTRY-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1155_USER_ELF, OS_K1155_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let argv: [&[u8]; 2] = [b"init", b"alpha"]; let env: [&[u8]; 2] = [b"MODE=1", b"LANG=tr"]; let recipe = Recipe::new(trusted, &argv, &env) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1155-TRACE: PUBLISHED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1155_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1155_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1155_EL0_ERROR)) } .unwrap_or("el0-process-body")); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("el0-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1155-EL0: KERNEL-ENTRY=REJECTED RX=OK STACK-RW=OK SPSR-EL0=OK FOUR-SVC=OK ERET=3 OWNER=RELEASED"); write_line("TURKOS/OS/K1155: PASS"); Ok(()) } global_asm!(include_str!("os_k1160_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1160_USER_ELF: [u8; 9312] = *include_bytes!("k1160-user.elf"); static OS_K1160_BAD_ENTRY_ELF: [u8; 9312] = *include_bytes!("k1160-bad-entry.elf"); #[inline(never)] fn os_k1160_install_fail_table<'a>( gate: &'a os_k1120_owners::Gate<'a, OsK1120Hardware<'a>>, authority: &'a os_k1120_owners::AuthorityCell<'a>, store: &'a os_k1120_owners::FrameStore<'a, OsK1120Hardware<'a>>, image: &'a OsK1120PreparedImage<'a>, config: turkos_kernel::runtime::address_spaces::Configuration, stage: u8, ) -> Result<*mut OsK1105Table<'a>, &'static str> { let mut backend = os_k1120_owners::Owners::new(gate, authority, store, image, 0x1160_0000, config, __os_k1105_kernel_trampoline as *const () as u64) .map_err(|_| "unwind-owners")?; backend.k1160_arm_failure(stage); let ptr = unsafe { core::ptr::addr_of_mut!(OS_K1105_TABLE_STORAGE.0) } .cast::>(); unsafe { OsK1105Table::new_in_place(backend, ptr) } .map_err(|_| "unwind-table")?; Ok(ptr) } // Mode 21 executes the trusted K1160 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fourth SVC. static mut OS_K1160_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1160_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1160_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1160_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1160_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1160_KERNEL_FRAME: u64 = 0; struct OsK1160El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, } #[inline(never)] fn os_k1160_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1160_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1160_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1160_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1160_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1160_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1160_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1160_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1160_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1160-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1160_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1160_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1160_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1160_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1160-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1160El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1160-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1160-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1160-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [0x1b0u64, 0x1b0, 0x1b0, 0x160] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1160-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } if unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 4 { park(); } episode.calls = 4; write_line("TURKOS/OS/K1160-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1160-A\n" } else { b"K1160-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1160-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1160_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1160-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } use turkos_kernel::runtime::process_guest as guest; let mask = 0x0000_ffff_ffff_f000u64; let l1 = unsafe { read_volatile((launch.user_root as *const u64) .add(((guest::CODE >> 30) & 511) as usize)) }; if l1 & 3 != 3 { return Err("entry-l1"); } let l2 = unsafe { read_volatile(((l1 & mask) as *const u64) .add(((guest::CODE >> 21) & 511) as usize)) }; if l2 & 3 != 3 { return Err("entry-l2"); } let l3 = l2 & mask; let leaf = |va: u64| unsafe { read_volatile((l3 as *const u64) .add(((va >> 12) & 511) as usize)) }; let code = leaf(guest::CODE); let stack = leaf(guest::STACK); let access = 3u64 | (1 << 6) | (1 << 10) | (1 << 11) | (1 << 53); if code & mask != launch.code_frame || code & access != access || code & (1 << 7) == 0 || code & (1 << 54) != 0 || stack & mask != launch.stack_frame || stack & access != access || stack & (1 << 7) != 0 || stack & (1 << 54) == 0 || leaf(guest::DATA + guest::PAGE) != 0 || launch.user_pc != guest::CODE || launch.user_sp != guest::STACK_TOP - 512 || launch.user_spsr != 0x3c0 { return Err("entry-rx-stack-rw-spsr"); } write_line("TURKOS/OS/K1160-TRACE: ENTRY-RX-STACK-RW-SPSR-EL0"); let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1160El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1160_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1160_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1160_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1160_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1160_ACTIVE), (&mut episode as *mut OsK1160El0Episode) as u64); } unsafe extern "C" { fn __os_k1160_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1160_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1160-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1160_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1160_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1160_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1160_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1160_KERNEL_FRAME), 0); } if episode.calls != 4 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1160_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1160_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1161, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1160_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1160_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1160_USER_ELF, OS_K1160_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1160-TRACE: TRUNCATED-HEADER-REJECTED"); let long_a = [b'A'; 63]; let long_b = [b'B'; 63]; let long_c = [b'C'; 63]; let long_d = [b'D'; 64]; let max_argv: [&[u8]; 2] = [&long_a, &long_b]; let max_env: [&[u8]; 2] = [&long_c, &long_c]; let over_env: [&[u8]; 2] = [&long_c, &long_d]; let bound_image = unsafe { TrustedImage::from_kernel(&OS_K1160_USER_ELF, OS_K1160_USER_ELF.len()) }.map_err(|_| "argv-bound-image")?; let bound = Recipe::new(bound_image, &max_argv, &max_env) .map_err(|_| "argv-exact-limit")?; if bound.start().argc != 2 || bound.start().envc != 2 { return Err("argv-exact-count"); } let over_image = unsafe { TrustedImage::from_kernel(&OS_K1160_USER_ELF, OS_K1160_USER_ELF.len()) }.map_err(|_| "argv-over-image")?; if !matches!(Recipe::new(over_image, &max_argv, &over_env), Err(ImageError::Arguments)) { return Err("argv-over-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "argv-after-gate")? .map_err(|_| "argv-after-inventory")? != before { return Err("argv-resource-change"); } write_line("TURKOS/OS/K1160-TRACE: TEXT-LIMIT-PLUS-ONE-REJECTED"); let bad_image = unsafe { TrustedImage::from_kernel(&OS_K1160_BAD_ENTRY_ELF, OS_K1160_BAD_ENTRY_ELF.len()) }.map_err(|_| "bad-entry-trust")?; if !matches!(Recipe::new(bad_image, &[b"init"], &[]), Err(ImageError::Entry)) { return Err("kernel-entry-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "entry-after-gate")? .map_err(|_| "entry-after-inventory")? != before { return Err("entry-resource-change"); } write_line("TURKOS/OS/K1160-TRACE: KERNEL-ENTRY-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1160_USER_ELF, OS_K1160_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let argv: [&[u8]; 2] = [b"init", b"alpha"]; let env: [&[u8]; 2] = [b"MODE=1", b"LANG=tr"]; let recipe = Recipe::new(trusted, &argv, &env) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; for stage in 0u8..11 { let fail_ptr = os_k1160_install_fail_table(&gate, &authority, &store, &image, config, stage)?; let fail_table = unsafe { &mut *fail_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let ticket = fail_table.begin(request).map_err(|_| "unwind-begin")?; let (error, ticket) = fail_table.prepare(ticket).err().ok_or("unwind-accepted")?; if error != process::Error::Resource || fail_table.snapshot().published != 0 || fail_table.snapshot().quarantined != 0 { return Err("unwind-failure-class"); } fail_table.abort(ticket).map_err(|_| "unwind-abort")?; if fail_table.snapshot() != process::Snapshot::default() { return Err("unwind-table-state"); } let backend = os_k1105_finish_table(fail_ptr)?; if backend.inventory().map_err(|_| "unwind-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("unwind-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "unwind-after-gate")? .map_err(|_| "unwind-after-inventory")? != before { return Err("unwind-physical-leak"); } write_str("TURKOS/OS/K1160-TRACE: UNWIND-STAGE="); write_hex(stage as usize); write_line(" PASS"); } let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1160-TRACE: PUBLISHED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1160_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1160_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1160_EL0_ERROR)) } .unwrap_or("el0-process-body")); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) || table.snapshot() != process::Snapshot::default() { return Err("el0-reap"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1160-EL0: UNWIND=11 RESTART=BASELINE FOUR-SVC=OK ERET=3 OWNER=RELEASED"); write_line("TURKOS/OS/K1160: PASS"); Ok(()) } global_asm!(include_str!("os_k1165_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1165_USER_ELF: [u8; 9360] = *include_bytes!("k1165-user.elf"); static OS_K1165_BAD_ENTRY_ELF: [u8; 9360] = *include_bytes!("k1165-bad-entry.elf"); use turkos_kernel::runtime::process_rights as os_k1165_rights; // Mode 22 executes the trusted K1165 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the fifth SVC. static mut OS_K1165_EL0_ERROR: Option<&'static str> = None; static mut OS_K1165_RIGHTS: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1165_KERNEL_FRAME: u64 = 0; struct OsK1165El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, rights: *mut turkos_kernel::runtime::process_rights::Registry, } #[inline(never)] fn os_k1165_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1165_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1165_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1165_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1165_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1165_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1165_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1165-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1165-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1165El0Episode) }; let table = unsafe { &*episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1165-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1165-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1165-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [460u64, 460, 460, 356, 380] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1165-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_PRINCIPAL_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::PrincipalRequest(0)) || episode.output_bytes != 16 { park(); } let rights = unsafe { &*episode.rights }; let before_rights = rights.snapshot(); let before_table = table.snapshot(); if rights.request_user_change(episode.pid, 0) != Err(os_k1165_rights::Error::Access) || rights.snapshot() != before_rights || table.snapshot() != before_table { park(); } frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 4; write_line("TURKOS/OS/K1165-TRACE: PRINCIPAL-ZERO-DENIED"); write_line("TURKOS/OS/K1165-TRACE: RETURN"); return 0; } if episode.calls == 4 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } if unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 4 || unsafe { read_volatile((launch.data_frame + 40) as *const u64) } != 1 { park(); } episode.calls = 5; write_line("TURKOS/OS/K1165-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1165-A\n" } else { b"K1165-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1165-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1165_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1165-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } use turkos_kernel::runtime::process_guest as guest; let mask = 0x0000_ffff_ffff_f000u64; let l1 = unsafe { read_volatile((launch.user_root as *const u64) .add(((guest::CODE >> 30) & 511) as usize)) }; if l1 & 3 != 3 { return Err("entry-l1"); } let l2 = unsafe { read_volatile(((l1 & mask) as *const u64) .add(((guest::CODE >> 21) & 511) as usize)) }; if l2 & 3 != 3 { return Err("entry-l2"); } let l3 = l2 & mask; let leaf = |va: u64| unsafe { read_volatile((l3 as *const u64) .add(((va >> 12) & 511) as usize)) }; let code = leaf(guest::CODE); let stack = leaf(guest::STACK); let access = 3u64 | (1 << 6) | (1 << 10) | (1 << 11) | (1 << 53); if code & mask != launch.code_frame || code & access != access || code & (1 << 7) == 0 || code & (1 << 54) != 0 || stack & mask != launch.stack_frame || stack & access != access || stack & (1 << 7) != 0 || stack & (1 << 54) == 0 || leaf(guest::DATA + guest::PAGE) != 0 || launch.user_pc != guest::CODE || launch.user_sp != guest::STACK_TOP - 512 || launch.user_spsr != 0x3c0 { return Err("entry-rx-stack-rw-spsr"); } write_line("TURKOS/OS/K1165-TRACE: ENTRY-RX-STACK-RW-SPSR-EL0"); let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1165El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar, rights: unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_RIGHTS)) } as *mut _}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1165_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1165_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_ACTIVE), (&mut episode as *mut OsK1165El0Episode) as u64); } unsafe extern "C" { fn __os_k1165_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1165_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1165-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1165_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1165_KERNEL_FRAME), 0); } if episode.calls != 5 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1165_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1166, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, OS_K1165_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1165-TRACE: TRUNCATED-HEADER-REJECTED"); let long_a = [b'A'; 63]; let long_b = [b'B'; 63]; let long_c = [b'C'; 63]; let long_d = [b'D'; 64]; let max_argv: [&[u8]; 2] = [&long_a, &long_b]; let max_env: [&[u8]; 2] = [&long_c, &long_c]; let over_env: [&[u8]; 2] = [&long_c, &long_d]; let bound_image = unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, OS_K1165_USER_ELF.len()) }.map_err(|_| "argv-bound-image")?; let bound = Recipe::new(bound_image, &max_argv, &max_env) .map_err(|_| "argv-exact-limit")?; if bound.start().argc != 2 || bound.start().envc != 2 { return Err("argv-exact-count"); } let over_image = unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, OS_K1165_USER_ELF.len()) }.map_err(|_| "argv-over-image")?; if !matches!(Recipe::new(over_image, &max_argv, &over_env), Err(ImageError::Arguments)) { return Err("argv-over-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "argv-after-gate")? .map_err(|_| "argv-after-inventory")? != before { return Err("argv-resource-change"); } write_line("TURKOS/OS/K1165-TRACE: TEXT-LIMIT-PLUS-ONE-REJECTED"); let bad_image = unsafe { TrustedImage::from_kernel(&OS_K1165_BAD_ENTRY_ELF, OS_K1165_BAD_ENTRY_ELF.len()) }.map_err(|_| "bad-entry-trust")?; if !matches!(Recipe::new(bad_image, &[b"init"], &[]), Err(ImageError::Entry)) { return Err("kernel-entry-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "entry-after-gate")? .map_err(|_| "entry-after-inventory")? != before { return Err("entry-resource-change"); } write_line("TURKOS/OS/K1165-TRACE: KERNEL-ENTRY-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1165_USER_ELF, OS_K1165_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let argv: [&[u8]; 2] = [b"init", b"alpha"]; let env: [&[u8]; 2] = [b"MODE=1", b"LANG=tr"]; let recipe = Recipe::new(trusted, &argv, &env) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; for stage in 0u8..11 { let fail_ptr = os_k1160_install_fail_table(&gate, &authority, &store, &image, config, stage)?; let fail_table = unsafe { &mut *fail_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let ticket = fail_table.begin(request).map_err(|_| "unwind-begin")?; let (error, ticket) = fail_table.prepare(ticket).err().ok_or("unwind-accepted")?; if error != process::Error::Resource || fail_table.snapshot().published != 0 || fail_table.snapshot().quarantined != 0 { return Err("unwind-failure-class"); } fail_table.abort(ticket).map_err(|_| "unwind-abort")?; if fail_table.snapshot() != process::Snapshot::default() { return Err("unwind-table-state"); } let backend = os_k1105_finish_table(fail_ptr)?; if backend.inventory().map_err(|_| "unwind-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("unwind-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "unwind-after-gate")? .map_err(|_| "unwind-after-inventory")? != before { return Err("unwind-physical-leak"); } write_str("TURKOS/OS/K1165-TRACE: UNWIND-STAGE="); write_hex(stage as usize); write_line(" PASS"); } let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1165-TRACE: PUBLISHED"); let mut identities = os_k1165_rights::Registry::::new(); let first = identities.bind_kernel(pid).map_err(|_| "rights-first-bind")?; if first.id() != 1 || identities.principal(pid) != Ok(first) { return Err("rights-first-principal"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_RIGHTS), (&mut identities as *mut _) as u64); } write_line("TURKOS/OS/K1165-TRACE: FIRST-PRINCIPAL-KERNEL-ASSIGNED"); let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1165_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1165_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1165_EL0_ERROR)) } .unwrap_or("el0-process-body")); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1165_RIGHTS), 0); } if identities.principal(pid) != Ok(first) { return Err("rights-first-changed"); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) { return Err("el0-reap"); } identities.retire_kernel(pid).map_err(|_| "rights-first-retire")?; let second_ticket = table.begin(request).map_err(|_| "rights-second-begin")?; let second_ticket = table.prepare(second_ticket).map_err(|_| "rights-second-prepare")?; let second_pid = table.publish(second_ticket).map_err(|_| "rights-second-publish")?; let second = identities.bind_kernel(second_pid).map_err(|_| "rights-second-bind")?; if second.id() != 2 || second == first || identities.principal(second_pid) != Ok(second) || identities.principal(pid) != Err(os_k1165_rights::Error::Unknown) { return Err("rights-second-principal"); } write_line("TURKOS/OS/K1165-TRACE: SECOND-PRINCIPAL-KERNEL-ASSIGNED"); table.terminate_child(second_pid, 0).map_err(|_| "rights-second-terminate")?; if table.wait_child(second_pid) != Ok(0) { return Err("rights-second-reap"); } identities.retire_kernel(second_pid).map_err(|_| "rights-second-retire")?; if table.snapshot() != process::Snapshot::default() || identities.snapshot() != (3, [None, None]) { return Err("rights-final-state"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1165-EL0: PRINCIPALS=2 ZERO=EACCES FIVE-SVC=OK ERET=4 OWNER=RELEASED"); write_line("TURKOS/OS/K1165: PASS"); Ok(()) } global_asm!(include_str!("os_k1170_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1170_USER_ELF: [u8; 9408] = *include_bytes!("k1170-user.elf"); static OS_K1170_BAD_ENTRY_ELF: [u8; 9408] = *include_bytes!("k1170-bad-entry.elf"); use turkos_kernel::runtime::process_rights as os_k1170_rights; // Mode 23 executes the trusted K1170 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the sixth SVC. static mut OS_K1170_EL0_ERROR: Option<&'static str> = None; static mut OS_K1170_RIGHTS: u64 = 0; static mut OS_K1170_HANDLES: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1170_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1170_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1170_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1170_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1170_KERNEL_FRAME: u64 = 0; struct OsK1170El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, rights: *mut turkos_kernel::runtime::process_rights::Registry, handles: *mut turkos_kernel::runtime::process_rights::HandleSet, } #[inline(never)] fn os_k1170_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1170_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1170_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1170_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1170_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1170_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1170_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1170_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1170_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1170-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1170_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1170_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1170_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1170_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1170-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1170El0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1170-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1170-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1170-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [488u64, 488, 488, 356, 384, 408] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1170-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_PRINCIPAL_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::PrincipalRequest(0)) || episode.output_bytes != 16 { park(); } let rights = unsafe { &*episode.rights }; let before_rights = rights.snapshot(); let before_table = table.snapshot(); if rights.request_user_change(episode.pid, 0) != Err(os_k1170_rights::Error::Access) || rights.snapshot() != before_rights || table.snapshot() != before_table { park(); } frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 4; write_line("TURKOS/OS/K1170-TRACE: PRINCIPAL-ZERO-DENIED"); write_line("TURKOS/OS/K1170-TRACE: RETURN"); return 0; } if episode.calls == 4 { use turkos_kernel::runtime::process_guest as guest; if frame.x[8] != abi::CALL_INHERIT_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::InheritRequest(2)) || episode.output_bytes != 16 { park(); } let rights = unsafe { &mut *episode.rights }; let handles = unsafe { &mut *episode.handles }; let parent_before = table.snapshot(); let spawn = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let child_ticket = table.begin(spawn).unwrap_or_else(|_| park()); let child_ticket = table.prepare(child_ticket).ok().unwrap_or_else(|| park()); let child = table.publish(child_ticket).ok().unwrap_or_else(|| park()); let relationship = table.record(child).unwrap_or_else(|_| park()).parent; if relationship != Some(episode.pid) { park(); } let child_principal = rights.bind_kernel(child).unwrap_or_else(|_| park()); if child_principal.id() != 2 || handles.lookup(child, 1) != Err(os_k1170_rights::Error::Unknown) { park(); } let child_file = handles.inherit_explicit(rights, episode.pid, child, relationship, 1).unwrap_or_else(|_| park()); if child_file.id() != 3 || handles.lookup(child, child_file.id()) != Ok( os_k1170_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: false }) { park(); } write_line("TURKOS/OS/K1170-TRACE: CHILD-RO-FILE-INHERITED"); let before_denial = handles.snapshot(); if handles.inherit_explicit(rights, episode.pid, child, relationship, frame.x[0]) != Err(os_k1170_rights::Error::Access) || handles.snapshot() != before_denial || handles.lookup(child, frame.x[0]) != Err(os_k1170_rights::Error::Unknown) || table.record(child).unwrap_or_else(|_| park()).parent != Some(episode.pid) { park(); } write_line("TURKOS/OS/K1170-TRACE: KERNEL-DEVICE-INHERIT-DENIED"); if table.terminate_child(child, 0).is_err() || table.wait_child(child) != Ok(0) || handles.close_owner(child) != 1 || rights.retire_kernel(child).is_err() || table.snapshot() != parent_before { park(); } frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 5; write_line("TURKOS/OS/K1170-TRACE: RETURN"); return 0; } if episode.calls == 5 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } if unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 4 || unsafe { read_volatile((launch.data_frame + 40) as *const u64) } != 1 || unsafe { read_volatile((launch.data_frame + 48) as *const u64) } != 1 { park(); } episode.calls = 6; write_line("TURKOS/OS/K1170-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1170-A\n" } else { b"K1170-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1170-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1170_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1170-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } use turkos_kernel::runtime::process_guest as guest; let mask = 0x0000_ffff_ffff_f000u64; let l1 = unsafe { read_volatile((launch.user_root as *const u64) .add(((guest::CODE >> 30) & 511) as usize)) }; if l1 & 3 != 3 { return Err("entry-l1"); } let l2 = unsafe { read_volatile(((l1 & mask) as *const u64) .add(((guest::CODE >> 21) & 511) as usize)) }; if l2 & 3 != 3 { return Err("entry-l2"); } let l3 = l2 & mask; let leaf = |va: u64| unsafe { read_volatile((l3 as *const u64) .add(((va >> 12) & 511) as usize)) }; let code = leaf(guest::CODE); let stack = leaf(guest::STACK); let access = 3u64 | (1 << 6) | (1 << 10) | (1 << 11) | (1 << 53); if code & mask != launch.code_frame || code & access != access || code & (1 << 7) == 0 || code & (1 << 54) != 0 || stack & mask != launch.stack_frame || stack & access != access || stack & (1 << 7) != 0 || stack & (1 << 54) == 0 || leaf(guest::DATA + guest::PAGE) != 0 || launch.user_pc != guest::CODE || launch.user_sp != guest::STACK_TOP - 512 || launch.user_spsr != 0x3c0 { return Err("entry-rx-stack-rw-spsr"); } write_line("TURKOS/OS/K1170-TRACE: ENTRY-RX-STACK-RW-SPSR-EL0"); let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1170El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar, rights: unsafe { read_volatile(core::ptr::addr_of!(OS_K1170_RIGHTS)) } as *mut _, handles: unsafe { read_volatile(core::ptr::addr_of!(OS_K1170_HANDLES)) } as *mut _}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1170_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1170_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1170_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1170_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1170_ACTIVE), (&mut episode as *mut OsK1170El0Episode) as u64); } unsafe extern "C" { fn __os_k1170_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1170_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1170-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1170_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1170_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1170_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1170_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1170_KERNEL_FRAME), 0); } if episode.calls != 6 || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1170_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1170_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1171, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1170_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1170_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1170_USER_ELF, OS_K1170_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1170-TRACE: TRUNCATED-HEADER-REJECTED"); let long_a = [b'A'; 63]; let long_b = [b'B'; 63]; let long_c = [b'C'; 63]; let long_d = [b'D'; 64]; let max_argv: [&[u8]; 2] = [&long_a, &long_b]; let max_env: [&[u8]; 2] = [&long_c, &long_c]; let over_env: [&[u8]; 2] = [&long_c, &long_d]; let bound_image = unsafe { TrustedImage::from_kernel(&OS_K1170_USER_ELF, OS_K1170_USER_ELF.len()) }.map_err(|_| "argv-bound-image")?; let bound = Recipe::new(bound_image, &max_argv, &max_env) .map_err(|_| "argv-exact-limit")?; if bound.start().argc != 2 || bound.start().envc != 2 { return Err("argv-exact-count"); } let over_image = unsafe { TrustedImage::from_kernel(&OS_K1170_USER_ELF, OS_K1170_USER_ELF.len()) }.map_err(|_| "argv-over-image")?; if !matches!(Recipe::new(over_image, &max_argv, &over_env), Err(ImageError::Arguments)) { return Err("argv-over-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "argv-after-gate")? .map_err(|_| "argv-after-inventory")? != before { return Err("argv-resource-change"); } write_line("TURKOS/OS/K1170-TRACE: TEXT-LIMIT-PLUS-ONE-REJECTED"); let bad_image = unsafe { TrustedImage::from_kernel(&OS_K1170_BAD_ENTRY_ELF, OS_K1170_BAD_ENTRY_ELF.len()) }.map_err(|_| "bad-entry-trust")?; if !matches!(Recipe::new(bad_image, &[b"init"], &[]), Err(ImageError::Entry)) { return Err("kernel-entry-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "entry-after-gate")? .map_err(|_| "entry-after-inventory")? != before { return Err("entry-resource-change"); } write_line("TURKOS/OS/K1170-TRACE: KERNEL-ENTRY-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1170_USER_ELF, OS_K1170_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let argv: [&[u8]; 2] = [b"init", b"alpha"]; let env: [&[u8]; 2] = [b"MODE=1", b"LANG=tr"]; let recipe = Recipe::new(trusted, &argv, &env) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; for stage in 0u8..11 { let fail_ptr = os_k1160_install_fail_table(&gate, &authority, &store, &image, config, stage)?; let fail_table = unsafe { &mut *fail_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let ticket = fail_table.begin(request).map_err(|_| "unwind-begin")?; let (error, ticket) = fail_table.prepare(ticket).err().ok_or("unwind-accepted")?; if error != process::Error::Resource || fail_table.snapshot().published != 0 || fail_table.snapshot().quarantined != 0 { return Err("unwind-failure-class"); } fail_table.abort(ticket).map_err(|_| "unwind-abort")?; if fail_table.snapshot() != process::Snapshot::default() { return Err("unwind-table-state"); } let backend = os_k1105_finish_table(fail_ptr)?; if backend.inventory().map_err(|_| "unwind-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("unwind-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "unwind-after-gate")? .map_err(|_| "unwind-after-inventory")? != before { return Err("unwind-physical-leak"); } write_str("TURKOS/OS/K1170-TRACE: UNWIND-STAGE="); write_hex(stage as usize); write_line(" PASS"); } let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1170-TRACE: PUBLISHED"); let mut identities = os_k1170_rights::Registry::::new(); let first = identities.bind_kernel(pid).map_err(|_| "rights-first-bind")?; if first.id() != 1 || identities.principal(pid) != Ok(first) { return Err("rights-first-principal"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1170_RIGHTS), (&mut identities as *mut _) as u64); } write_line("TURKOS/OS/K1170-TRACE: FIRST-PRINCIPAL-KERNEL-ASSIGNED"); let mut handles = os_k1170_rights::HandleSet::::new(); let file = handles.issue_kernel(&identities, pid, os_k1170_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: true }) .map_err(|_| "inherit-parent-file")?; let device = handles.issue_kernel(&identities, pid, os_k1170_rights::Capability::KernelDevice { object: 0xfee }) .map_err(|_| "inherit-parent-device")?; if file.id() != 1 || device.id() != 2 { return Err("inherit-parent-handles"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1170_HANDLES), (&mut handles as *mut _) as u64); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1170_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1170_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1170_EL0_ERROR)) } .unwrap_or("el0-process-body")); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1170_RIGHTS), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1170_HANDLES), 0); } if identities.principal(pid) != Ok(first) || identities.snapshot().0 != 3 || handles.lookup(pid, file.id()) != Ok( os_k1170_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: true }) || handles.lookup(pid, device.id()) != Ok( os_k1170_rights::Capability::KernelDevice { object: 0xfee }) { return Err("inherit-parent-after-el0"); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) { return Err("el0-reap"); } if handles.close_owner(pid) != 2 { return Err("inherit-parent-close"); } identities.retire_kernel(pid).map_err(|_| "rights-first-retire")?; if table.snapshot() != process::Snapshot::default() || identities.snapshot() != (3, [None, None]) || handles.snapshot() != (4, [None, None, None, None]) { return Err("rights-final-state"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1170-EL0: CHILD-RO=INHERITED DEVICE=EACCES SIX-SVC=OK ERET=5 OWNER=RELEASED"); write_line("TURKOS/OS/K1170: PASS"); Ok(()) } global_asm!(include_str!("os_k1175_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1175_USER_ELF: [u8; 9456] = *include_bytes!("k1175-user.elf"); static OS_K1175_BAD_ENTRY_ELF: [u8; 9456] = *include_bytes!("k1175-bad-entry.elf"); use turkos_kernel::runtime::process_rights as os_k1175_rights; // Mode 24 executes the trusted K1175 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the seventh SVC. static mut OS_K1175_EL0_ERROR: Option<&'static str> = None; static mut OS_K1175_RIGHTS: u64 = 0; static mut OS_K1175_HANDLES: u64 = 0; static mut OS_K1175_INSPECT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1175_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1175_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1175_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1175_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1175_KERNEL_FRAME: u64 = 0; struct OsK1175El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, rights: *mut turkos_kernel::runtime::process_rights::Registry, handles: *mut turkos_kernel::runtime::process_rights::HandleSet, inspect: *mut turkos_kernel::runtime::process_rights::InspectSet, child: Option, parent_baseline: turkos_kernel::runtime::process_table::Snapshot, } #[inline(never)] fn os_k1175_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1175_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1175_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1175_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1175_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1175_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1175_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1175_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1175-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1175-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1175El0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1175-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1175-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1175-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [516u64, 516, 516, 356, 384, 412, 436] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1175-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_PRINCIPAL_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::PrincipalRequest(0)) || episode.output_bytes != 16 { park(); } let rights = unsafe { &*episode.rights }; let before_rights = rights.snapshot(); let before_table = table.snapshot(); if rights.request_user_change(episode.pid, 0) != Err(os_k1175_rights::Error::Access) || rights.snapshot() != before_rights || table.snapshot() != before_table { park(); } frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 4; write_line("TURKOS/OS/K1175-TRACE: PRINCIPAL-ZERO-DENIED"); write_line("TURKOS/OS/K1175-TRACE: RETURN"); return 0; } if episode.calls == 4 { use turkos_kernel::runtime::process_guest as guest; if frame.x[8] != abi::CALL_INHERIT_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::InheritRequest(2)) || episode.output_bytes != 16 { park(); } let rights = unsafe { &mut *episode.rights }; let handles = unsafe { &mut *episode.handles }; if table.snapshot() != episode.parent_baseline || episode.child.is_some() { park(); } let spawn = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let child_ticket = table.begin(spawn).unwrap_or_else(|_| park()); let child_ticket = table.prepare(child_ticket).ok().unwrap_or_else(|| park()); let child = table.publish(child_ticket).ok().unwrap_or_else(|| park()); let relationship = table.record(child).unwrap_or_else(|_| park()).parent; if relationship != Some(episode.pid) { park(); } let child_principal = rights.bind_kernel(child).unwrap_or_else(|_| park()); if child_principal.id() != 2 || handles.lookup(child, 1) != Err(os_k1175_rights::Error::Unknown) { park(); } let child_file = handles.inherit_explicit(rights, episode.pid, child, relationship, 1).unwrap_or_else(|_| park()); if child_file.id() != 3 || handles.lookup(child, child_file.id()) != Ok( os_k1175_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: false }) { park(); } write_line("TURKOS/OS/K1175-TRACE: CHILD-RO-FILE-INHERITED"); let before_denial = handles.snapshot(); if handles.inherit_explicit(rights, episode.pid, child, relationship, frame.x[0]) != Err(os_k1175_rights::Error::Access) || handles.snapshot() != before_denial || handles.lookup(child, frame.x[0]) != Err(os_k1175_rights::Error::Unknown) || table.record(child).unwrap_or_else(|_| park()).parent != Some(episode.pid) { park(); } write_line("TURKOS/OS/K1175-TRACE: KERNEL-DEVICE-INHERIT-DENIED"); let inspect = unsafe { &mut *episode.inspect }; let permit = inspect.issue_kernel(rights, episode.pid, child) .unwrap_or_else(|_| park()); if permit.id() != 1 { park(); } episode.child = Some(child); frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 5; write_line("TURKOS/OS/K1175-TRACE: RETURN"); return 0; } if episode.calls == 5 { if frame.x[8] != abi::CALL_INSPECT_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::InspectRequest(1)) || episode.output_bytes != 16 { park(); } let child = episode.child.unwrap_or_else(|| park()); let rights = unsafe { &mut *episode.rights }; let handles = unsafe { &mut *episode.handles }; let inspect = unsafe { &mut *episode.inspect }; let record = table.record(child).unwrap_or_else(|_| park()); if record.parent != Some(episode.pid) || !record.resources.published || record.resources.pages == 0 { park(); } let metadata = os_k1175_rights::MapMetadata { resource_owner: record.resources.resource_owner, pages: record.resources.pages, published: record.resources.published, }; if inspect.inspect_metadata(rights, episode.pid, child, frame.x[0], metadata) != Ok(metadata) { park(); } write_line("TURKOS/OS/K1175-TRACE: AUTHORIZED-METADATA-OBSERVED"); let target_before = table.target_digest(child).unwrap_or_else(|_| park()); let table_before = table.snapshot(); let rights_before = rights.snapshot(); let handles_before = handles.snapshot(); let inspect_before = inspect.snapshot(); if inspect.read_foreign_map(rights, episode.pid, child, frame.x[0]) != Err(os_k1175_rights::Error::Access) || inspect.snapshot() != inspect_before || handles.snapshot() != handles_before || rights.snapshot() != rights_before || table.snapshot() != table_before || table.record(child) != Ok(record) || table.target_digest(child) != Ok(target_before) { park(); } write_line("TURKOS/OS/K1175-TRACE: FOREIGN-MAP-DENIED-TARGET-HASH-STABLE"); if inspect.close_observer(episode.pid) != 1 || table.terminate_child(child, 0).is_err() || table.wait_child(child) != Ok(0) || handles.close_owner(child) != 1 || rights.retire_kernel(child).is_err() || table.snapshot() != episode.parent_baseline || inspect.snapshot() != (2, [None, None]) { park(); } episode.child = None; frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 6; write_line("TURKOS/OS/K1175-TRACE: RETURN"); return 0; } if episode.calls == 6 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 16 { park(); } for (index, expected) in [1u64, 2, 3, 0].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } if unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 4 || unsafe { read_volatile((launch.data_frame + 40) as *const u64) } != 1 || unsafe { read_volatile((launch.data_frame + 48) as *const u64) } != 1 || unsafe { read_volatile((launch.data_frame + 56) as *const u64) } != 1 { park(); } episode.calls = 7; write_line("TURKOS/OS/K1175-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = if episode.calls == 0 { b"K1175-A\n" } else { b"K1175-B\n" }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1175-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1175_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1175-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } use turkos_kernel::runtime::process_guest as guest; let mask = 0x0000_ffff_ffff_f000u64; let l1 = unsafe { read_volatile((launch.user_root as *const u64) .add(((guest::CODE >> 30) & 511) as usize)) }; if l1 & 3 != 3 { return Err("entry-l1"); } let l2 = unsafe { read_volatile(((l1 & mask) as *const u64) .add(((guest::CODE >> 21) & 511) as usize)) }; if l2 & 3 != 3 { return Err("entry-l2"); } let l3 = l2 & mask; let leaf = |va: u64| unsafe { read_volatile((l3 as *const u64) .add(((va >> 12) & 511) as usize)) }; let code = leaf(guest::CODE); let stack = leaf(guest::STACK); let access = 3u64 | (1 << 6) | (1 << 10) | (1 << 11) | (1 << 53); if code & mask != launch.code_frame || code & access != access || code & (1 << 7) == 0 || code & (1 << 54) != 0 || stack & mask != launch.stack_frame || stack & access != access || stack & (1 << 7) != 0 || stack & (1 << 54) == 0 || leaf(guest::DATA + guest::PAGE) != 0 || launch.user_pc != guest::CODE || launch.user_sp != guest::STACK_TOP - 512 || launch.user_spsr != 0x3c0 { return Err("entry-rx-stack-rw-spsr"); } write_line("TURKOS/OS/K1175-TRACE: ENTRY-RX-STACK-RW-SPSR-EL0"); let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1175El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar, rights: unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_RIGHTS)) } as *mut _, handles: unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_HANDLES)) } as *mut _, inspect: unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_INSPECT)) } as *mut _, child: None, parent_baseline: unsafe { &*table_ptr }.snapshot()}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1175_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1175_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1175_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1175_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1175_ACTIVE), (&mut episode as *mut OsK1175El0Episode) as u64); } unsafe extern "C" { fn __os_k1175_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1175_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1175-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1175_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1175_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1175_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1175_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1175_KERNEL_FRAME), 0); } if episode.calls != 7 || episode.child.is_some() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1175_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1175_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1176, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1175_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1175_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1175_USER_ELF, OS_K1175_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1175-TRACE: TRUNCATED-HEADER-REJECTED"); let long_a = [b'A'; 63]; let long_b = [b'B'; 63]; let long_c = [b'C'; 63]; let long_d = [b'D'; 64]; let max_argv: [&[u8]; 2] = [&long_a, &long_b]; let max_env: [&[u8]; 2] = [&long_c, &long_c]; let over_env: [&[u8]; 2] = [&long_c, &long_d]; let bound_image = unsafe { TrustedImage::from_kernel(&OS_K1175_USER_ELF, OS_K1175_USER_ELF.len()) }.map_err(|_| "argv-bound-image")?; let bound = Recipe::new(bound_image, &max_argv, &max_env) .map_err(|_| "argv-exact-limit")?; if bound.start().argc != 2 || bound.start().envc != 2 { return Err("argv-exact-count"); } let over_image = unsafe { TrustedImage::from_kernel(&OS_K1175_USER_ELF, OS_K1175_USER_ELF.len()) }.map_err(|_| "argv-over-image")?; if !matches!(Recipe::new(over_image, &max_argv, &over_env), Err(ImageError::Arguments)) { return Err("argv-over-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "argv-after-gate")? .map_err(|_| "argv-after-inventory")? != before { return Err("argv-resource-change"); } write_line("TURKOS/OS/K1175-TRACE: TEXT-LIMIT-PLUS-ONE-REJECTED"); let bad_image = unsafe { TrustedImage::from_kernel(&OS_K1175_BAD_ENTRY_ELF, OS_K1175_BAD_ENTRY_ELF.len()) }.map_err(|_| "bad-entry-trust")?; if !matches!(Recipe::new(bad_image, &[b"init"], &[]), Err(ImageError::Entry)) { return Err("kernel-entry-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "entry-after-gate")? .map_err(|_| "entry-after-inventory")? != before { return Err("entry-resource-change"); } write_line("TURKOS/OS/K1175-TRACE: KERNEL-ENTRY-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1175_USER_ELF, OS_K1175_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let argv: [&[u8]; 2] = [b"init", b"alpha"]; let env: [&[u8]; 2] = [b"MODE=1", b"LANG=tr"]; let recipe = Recipe::new(trusted, &argv, &env) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; for stage in 0u8..11 { let fail_ptr = os_k1160_install_fail_table(&gate, &authority, &store, &image, config, stage)?; let fail_table = unsafe { &mut *fail_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let ticket = fail_table.begin(request).map_err(|_| "unwind-begin")?; let (error, ticket) = fail_table.prepare(ticket).err().ok_or("unwind-accepted")?; if error != process::Error::Resource || fail_table.snapshot().published != 0 || fail_table.snapshot().quarantined != 0 { return Err("unwind-failure-class"); } fail_table.abort(ticket).map_err(|_| "unwind-abort")?; if fail_table.snapshot() != process::Snapshot::default() { return Err("unwind-table-state"); } let backend = os_k1105_finish_table(fail_ptr)?; if backend.inventory().map_err(|_| "unwind-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("unwind-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "unwind-after-gate")? .map_err(|_| "unwind-after-inventory")? != before { return Err("unwind-physical-leak"); } write_str("TURKOS/OS/K1175-TRACE: UNWIND-STAGE="); write_hex(stage as usize); write_line(" PASS"); } let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1175-TRACE: PUBLISHED"); let mut identities = os_k1175_rights::Registry::::new(); let first = identities.bind_kernel(pid).map_err(|_| "rights-first-bind")?; if first.id() != 1 || identities.principal(pid) != Ok(first) { return Err("rights-first-principal"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1175_RIGHTS), (&mut identities as *mut _) as u64); } write_line("TURKOS/OS/K1175-TRACE: FIRST-PRINCIPAL-KERNEL-ASSIGNED"); let mut handles = os_k1175_rights::HandleSet::::new(); let file = handles.issue_kernel(&identities, pid, os_k1175_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: true }) .map_err(|_| "inherit-parent-file")?; let device = handles.issue_kernel(&identities, pid, os_k1175_rights::Capability::KernelDevice { object: 0xfee }) .map_err(|_| "inherit-parent-device")?; if file.id() != 1 || device.id() != 2 { return Err("inherit-parent-handles"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1175_HANDLES), (&mut handles as *mut _) as u64); } let mut inspect = os_k1175_rights::InspectSet::::new(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1175_INSPECT), (&mut inspect as *mut _) as u64); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1175_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1175_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1175_EL0_ERROR)) } .unwrap_or("el0-process-body")); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1175_RIGHTS), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1175_HANDLES), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1175_INSPECT), 0); } if identities.principal(pid) != Ok(first) || identities.snapshot().0 != 3 || handles.lookup(pid, file.id()) != Ok( os_k1175_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: true }) || handles.lookup(pid, device.id()) != Ok( os_k1175_rights::Capability::KernelDevice { object: 0xfee }) || inspect.snapshot() != (2, [None, None]) { return Err("inherit-parent-after-el0"); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) { return Err("el0-reap"); } if handles.close_owner(pid) != 2 { return Err("inherit-parent-close"); } identities.retire_kernel(pid).map_err(|_| "rights-first-retire")?; if table.snapshot() != process::Snapshot::default() || identities.snapshot() != (3, [None, None]) || handles.snapshot() != (4, [None, None, None, None]) || inspect.snapshot() != (2, [None, None]) { return Err("rights-final-state"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1175-EL0: INSPECT-META=OK FOREIGN-MAP=EACCES TARGET-HASH=STABLE SEVEN-SVC=OK ERET=6 OWNER=RELEASED"); write_line("TURKOS/OS/K1175: PASS"); Ok(()) } global_asm!(include_str!("os_k1180_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1180_USER_ELF: [u8; 9608] = *include_bytes!("k1180-user.elf"); static OS_K1180_BAD_ENTRY_ELF: [u8; 9608] = *include_bytes!("k1180-bad-entry.elf"); use turkos_kernel::runtime::process_rights as os_k1180_rights; // Mode 25 executes the trusted K1180 ELF from its retained ProcessTable root. // A single masked CPU owns all globals from activation until the ninth SVC. static mut OS_K1180_EL0_ERROR: Option<&'static str> = None; static mut OS_K1180_RIGHTS: u64 = 0; static mut OS_K1180_HANDLES: u64 = 0; static mut OS_K1180_INSPECT: u64 = 0; static mut OS_K1180_QUOTA: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1180_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1180_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1180_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1180_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1180_KERNEL_FRAME: u64 = 0; struct OsK1180El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, calls: u64, output_bytes: u64, old_vbar: u64, rights: *mut turkos_kernel::runtime::process_rights::Registry, handles: *mut turkos_kernel::runtime::process_rights::HandleSet, inspect: *mut turkos_kernel::runtime::process_rights::InspectSet, quota: *mut turkos_kernel::runtime::process_rights::QuotaSet, child: Option, parent_baseline: turkos_kernel::runtime::process_table::Snapshot, } #[inline(never)] fn os_k1180_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("el0-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "el0-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1180_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1180_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1180_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "el0-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("el0-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_EL0_ERROR)) } .unwrap_or("el0-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1180_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1180_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1180_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1180_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; write_line("TURKOS/OS/K1180-TRACE: DISPATCH"); let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } write_line("TURKOS/OS/K1180-TRACE: ROOT"); let episode = unsafe { &mut *(active as *mut OsK1180El0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1180-TRACE: OWNER"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1180-TRACE: STACK"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1180-TRACE: ESR=0x"); write_hex(frame.esr as usize); write_str(" ELR=0x"); write_hex(frame.elr as usize); write_line(""); let expected_elr = [580u64, 580, 580, 356, 384, 412, 440, 580, 500] .get(episode.calls as usize).copied().unwrap_or_else(|| park()); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.esr != abi::SVC_ESR || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + expected_elr { park(); } write_line("TURKOS/OS/K1180-TRACE: FRAME"); let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.x[7] != abi::ABI_VERSION { park(); } if episode.calls == 3 { if frame.x[8] != abi::CALL_PRINCIPAL_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::PrincipalRequest(0)) || episode.output_bytes != 16 { park(); } let rights = unsafe { &*episode.rights }; let before_rights = rights.snapshot(); let before_table = table.snapshot(); if rights.request_user_change(episode.pid, 0) != Err(os_k1180_rights::Error::Access) || rights.snapshot() != before_rights || table.snapshot() != before_table { park(); } frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 4; write_line("TURKOS/OS/K1180-TRACE: PRINCIPAL-ZERO-DENIED"); write_line("TURKOS/OS/K1180-TRACE: RETURN"); return 0; } if episode.calls == 4 { use turkos_kernel::runtime::process_guest as guest; if frame.x[8] != abi::CALL_INHERIT_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::InheritRequest(2)) || episode.output_bytes != 16 { park(); } let rights = unsafe { &mut *episode.rights }; let handles = unsafe { &mut *episode.handles }; if table.snapshot() != episode.parent_baseline || episode.child.is_some() { park(); } let spawn = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let child_ticket = table.begin(spawn).unwrap_or_else(|_| park()); let child_ticket = table.prepare(child_ticket).ok().unwrap_or_else(|| park()); let child = table.publish(child_ticket).ok().unwrap_or_else(|| park()); let relationship = table.record(child).unwrap_or_else(|_| park()).parent; if relationship != Some(episode.pid) { park(); } let child_principal = rights.bind_kernel(child).unwrap_or_else(|_| park()); if child_principal.id() != 2 || handles.lookup(child, 1) != Err(os_k1180_rights::Error::Unknown) { park(); } let child_file = handles.inherit_explicit(rights, episode.pid, child, relationship, 1).unwrap_or_else(|_| park()); if child_file.id() != 3 || handles.lookup(child, child_file.id()) != Ok( os_k1180_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: false }) { park(); } write_line("TURKOS/OS/K1180-TRACE: CHILD-RO-FILE-INHERITED"); let before_denial = handles.snapshot(); if handles.inherit_explicit(rights, episode.pid, child, relationship, frame.x[0]) != Err(os_k1180_rights::Error::Access) || handles.snapshot() != before_denial || handles.lookup(child, frame.x[0]) != Err(os_k1180_rights::Error::Unknown) || table.record(child).unwrap_or_else(|_| park()).parent != Some(episode.pid) { park(); } write_line("TURKOS/OS/K1180-TRACE: KERNEL-DEVICE-INHERIT-DENIED"); let inspect = unsafe { &mut *episode.inspect }; let permit = inspect.issue_kernel(rights, episode.pid, child) .unwrap_or_else(|_| park()); if permit.id() != 1 { park(); } episode.child = Some(child); frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 5; write_line("TURKOS/OS/K1180-TRACE: RETURN"); return 0; } if episode.calls == 5 { if frame.x[8] != abi::CALL_INSPECT_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::InspectRequest(1)) || episode.output_bytes != 16 { park(); } let child = episode.child.unwrap_or_else(|| park()); let rights = unsafe { &mut *episode.rights }; let handles = unsafe { &mut *episode.handles }; let inspect = unsafe { &mut *episode.inspect }; let record = table.record(child).unwrap_or_else(|_| park()); if record.parent != Some(episode.pid) || !record.resources.published || record.resources.pages == 0 { park(); } let metadata = os_k1180_rights::MapMetadata { resource_owner: record.resources.resource_owner, pages: record.resources.pages, published: record.resources.published, }; if inspect.inspect_metadata(rights, episode.pid, child, frame.x[0], metadata) != Ok(metadata) { park(); } write_line("TURKOS/OS/K1180-TRACE: AUTHORIZED-METADATA-OBSERVED"); let target_before = table.target_digest(child).unwrap_or_else(|_| park()); let table_before = table.snapshot(); let rights_before = rights.snapshot(); let handles_before = handles.snapshot(); let inspect_before = inspect.snapshot(); if inspect.read_foreign_map(rights, episode.pid, child, frame.x[0]) != Err(os_k1180_rights::Error::Access) || inspect.snapshot() != inspect_before || handles.snapshot() != handles_before || rights.snapshot() != rights_before || table.snapshot() != table_before || table.record(child) != Ok(record) || table.target_digest(child) != Ok(target_before) { park(); } write_line("TURKOS/OS/K1180-TRACE: FOREIGN-MAP-DENIED-TARGET-HASH-STABLE"); if inspect.close_observer(episode.pid) != 1 || inspect.snapshot() != (2, [None, None]) || table.record(child) != Ok(record) { park(); } frame.x[0] = abi::AbiResult::Err(abi::Errno::Access).encode(); episode.calls = 6; write_line("TURKOS/OS/K1180-TRACE: RETURN"); return 0; } if episode.calls == 6 { if frame.x[8] != abi::CALL_QUOTA_REQUEST || abi::decode_call(&abi_frame) != Ok(abi::Call::QuotaRequest(1)) || episode.output_bytes != 16 { park(); } let child = episode.child.unwrap_or_else(|| park()); let parent_record = table.record(episode.pid).unwrap_or_else(|_| park()); let child_record = table.record(child).unwrap_or_else(|_| park()); let rights = unsafe { &mut *episode.rights }; let handles = unsafe { &mut *episode.handles }; let inspect = unsafe { &mut *episode.inspect }; let quota = unsafe { &mut *episode.quota }; let grants = handles.snapshot().1.iter().flatten() .filter(|(_, owner, _)| *owner == child).count() as u64; if child_record.parent != Some(episode.pid) || !parent_record.resources.published || !child_record.resources.published || parent_record.resources.state != turkos_kernel::runtime::process_threads::State::Running || child_record.resources.state != turkos_kernel::runtime::process_threads::State::Ready || parent_record.resources.handles.iter().flatten().count() != 3 || child_record.resources.handles.iter().flatten().count() != 3 || grants != 1 || table.snapshot().published != 2 || quota.snapshot() != [None] { park(); } let used = os_k1180_rights::Budget { pages: child_record.resources.pages, threads: 1, handles: grants, }; if used.pages == 0 || quota.adopt_kernel(rights, child, used, used).is_err() || quota.usage(child) != Ok((used, used)) { park(); } write_line("TURKOS/OS/K1180-TRACE: CHILD-PAGE-THREAD-HANDLE-AT-LIMIT"); let before_quota = quota.snapshot(); let before_table = table.snapshot(); let before_rights = rights.snapshot(); let before_handles = handles.snapshot(); let before_inspect = inspect.snapshot(); for request in [ os_k1180_rights::Budget { pages: 1, ..os_k1180_rights::Budget::default() }, os_k1180_rights::Budget { threads: frame.x[0], ..os_k1180_rights::Budget::default() }, os_k1180_rights::Budget { handles: 1, ..os_k1180_rights::Budget::default() }, ] { if quota.try_reserve(rights, child, request) != Err(os_k1180_rights::Error::Limit) || quota.snapshot() != before_quota || table.snapshot() != before_table || rights.snapshot() != before_rights || handles.snapshot() != before_handles || inspect.snapshot() != before_inspect || table.record(child) != Ok(child_record) { park(); } } write_line("TURKOS/OS/K1180-TRACE: PLUS-ONE-TASK-DENIED-NO-PARTIAL-STATE"); if table.current_launch(episode.pid) != Ok(launch) || child == episode.pid { park(); } frame.x[0] = abi::AbiResult::Err(abi::Errno::Again).encode(); episode.calls = 7; write_line("TURKOS/OS/K1180-TRACE: RETURN"); return 0; } if episode.calls == 8 { if frame.x[8] != abi::CALL_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Version) || episode.output_bytes != 24 { park(); } for (index, expected) in [1u64, 2, 3, 1].into_iter().enumerate() { if unsafe { read_volatile((launch.data_frame + index as u64 * 8) as *const u64) } != expected { park(); } } if unsafe { read_volatile((launch.data_frame + 32) as *const u64) } != 4 || unsafe { read_volatile((launch.data_frame + 40) as *const u64) } != 1 || unsafe { read_volatile((launch.data_frame + 48) as *const u64) } != 1 || unsafe { read_volatile((launch.data_frame + 56) as *const u64) } != 1 { park(); } let child = episode.child.unwrap_or_else(|| park()); let rights = unsafe { &mut *episode.rights }; let handles = unsafe { &mut *episode.handles }; let quota = unsafe { &mut *episode.quota }; let (used, limit) = quota.usage(child).unwrap_or_else(|_| park()); if used != limit || table.record(child).unwrap_or_else(|_| park()).parent != Some(episode.pid) || quota.release_kernel(child) != Ok(used) || quota.snapshot() != [None] || table.terminate_child(child, 0).is_err() || table.wait_child(child) != Ok(0) || handles.close_owner(child) != 1 || rights.retire_kernel(child).is_err() || table.snapshot() != episode.parent_baseline { park(); } episode.child = None; write_line("TURKOS/OS/K1180-TRACE: OTHER-PROCESS-CONTINUED"); episode.calls = 9; write_line("TURKOS/OS/K1180-TRACE: COMPLETE"); return 1; } let resources_before = table.snapshot(); if frame.x[8] != abi::CALL_WRITE { park(); } let Ok(abi::Call::Write(request)) = abi::decode_call(&abi_frame) else { park(); }; if request.fd() != 1 || request.bytes() != 8 { park(); } let wanted = match episode.calls { 0 => 0x8000_1040u64, 1 => 0x9000_0000u64, 2 => 0x8000_1050u64, 7 => 0x8000_1060u64, _ => park(), }; if request.address() != wanted { park(); } let mut copied = [0u8; abi::MAX_WRITE]; let result = match table.copy_current_user_data(episode.pid, request.address(), &mut copied[..request.bytes()]) { Ok(()) if episode.calls != 1 => { let expected: &[u8; 8] = match episode.calls { 0 => b"K1180-A\n", 2 => b"K1180-B\n", 7 => b"K1180-C\n", _ => park(), }; if &copied[..8] != expected || table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) || os_k1120_registers().ttbr != kernel_root { park(); } for byte in copied[..request.bytes()].iter().copied() { write_byte(byte); } episode.output_bytes += request.bytes() as u64; abi::AbiResult::Ok(abi::Success::new(request.bytes() as u64).unwrap()) } Err(process::GuestCopyError::Fault) if episode.calls == 1 => { if episode.output_bytes != 8 || copied.iter().any(|byte| *byte != 0) { park(); } abi::AbiResult::Err(abi::Errno::Fault) } _ => park(), }; if table.snapshot() != resources_before || table.current_launch(episode.pid) != Ok(launch) { park(); } frame.x[0] = result.encode(); episode.calls += 1; write_line("TURKOS/OS/K1180-TRACE: RETURN"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1180_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "el0-launch-owner")?; write_line("TURKOS/OS/K1180-TRACE: LANDED"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("el0-launch-root"); } use turkos_kernel::runtime::process_guest as guest; let mask = 0x0000_ffff_ffff_f000u64; let l1 = unsafe { read_volatile((launch.user_root as *const u64) .add(((guest::CODE >> 30) & 511) as usize)) }; if l1 & 3 != 3 { return Err("entry-l1"); } let l2 = unsafe { read_volatile(((l1 & mask) as *const u64) .add(((guest::CODE >> 21) & 511) as usize)) }; if l2 & 3 != 3 { return Err("entry-l2"); } let l3 = l2 & mask; let leaf = |va: u64| unsafe { read_volatile((l3 as *const u64) .add(((va >> 12) & 511) as usize)) }; let code = leaf(guest::CODE); let stack = leaf(guest::STACK); let access = 3u64 | (1 << 6) | (1 << 10) | (1 << 11) | (1 << 53); if code & mask != launch.code_frame || code & access != access || code & (1 << 7) == 0 || code & (1 << 54) != 0 || stack & mask != launch.stack_frame || stack & access != access || stack & (1 << 7) != 0 || stack & (1 << 54) == 0 || leaf(guest::DATA + guest::PAGE) != 0 || launch.user_pc != guest::CODE || launch.user_sp != guest::STACK_TOP - 512 || launch.user_spsr != 0x3c0 { return Err("entry-rx-stack-rw-spsr"); } write_line("TURKOS/OS/K1180-TRACE: ENTRY-RX-STACK-RW-SPSR-EL0"); let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1180El0Episode {table, pid, launch, calls: 0, output_bytes: 0, old_vbar, rights: unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_RIGHTS)) } as *mut _, handles: unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_HANDLES)) } as *mut _, inspect: unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_INSPECT)) } as *mut _, quota: unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_QUOTA)) } as *mut _, child: None, parent_baseline: unsafe { &*table_ptr }.snapshot()}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1180_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1180_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1180_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_ACTIVE), (&mut episode as *mut OsK1180El0Episode) as u64); } unsafe extern "C" { fn __os_k1180_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1180_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1180-TRACE: KERNEL-RESUME"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1180_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_KERNEL_FRAME), 0); } if episode.calls != 9 || episode.child.is_some() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.current_launch(pid) != Ok(launch) { return Err("el0-completion-owner"); } Ok(()) })(); if unsafe { (&mut *table_ptr).release_actual_current_to_ready() }.is_err() { park(); } match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1180_EL0_ERROR), Some(error)); } 1 } } } #[inline(never)] fn os_k1180_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Error as ImageError, Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "el0-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "el0-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1181, &authority, &gate) .map_err(|_| "el0-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "short-frame-baseline")?; if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1180_USER_ELF, 0) }, Err(ImageError::Length)) { return Err("empty-image-accepted"); } for cut in [1usize, 16, 63, 64] { let short = unsafe { TrustedImage::from_kernel(&OS_K1180_USER_ELF, cut) } .map_err(|_| "short-prefix")?; if !matches!(Recipe::new(short, &[b"init"], &[]), Err(ImageError::Image)) { return Err("short-header-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "short-after-gate")? .map_err(|_| "short-after-inventory")? != before { return Err("short-resource-change"); } } if !matches!(unsafe { TrustedImage::from_kernel(&OS_K1180_USER_ELF, OS_K1180_USER_ELF.len() + 1) }, Err(ImageError::Length)) || store.snapshot() != Ok(frame_baseline) { return Err("untrusted-length-accepted"); } write_line("TURKOS/OS/K1180-TRACE: TRUNCATED-HEADER-REJECTED"); let long_a = [b'A'; 63]; let long_b = [b'B'; 63]; let long_c = [b'C'; 63]; let long_d = [b'D'; 64]; let max_argv: [&[u8]; 2] = [&long_a, &long_b]; let max_env: [&[u8]; 2] = [&long_c, &long_c]; let over_env: [&[u8]; 2] = [&long_c, &long_d]; let bound_image = unsafe { TrustedImage::from_kernel(&OS_K1180_USER_ELF, OS_K1180_USER_ELF.len()) }.map_err(|_| "argv-bound-image")?; let bound = Recipe::new(bound_image, &max_argv, &max_env) .map_err(|_| "argv-exact-limit")?; if bound.start().argc != 2 || bound.start().envc != 2 { return Err("argv-exact-count"); } let over_image = unsafe { TrustedImage::from_kernel(&OS_K1180_USER_ELF, OS_K1180_USER_ELF.len()) }.map_err(|_| "argv-over-image")?; if !matches!(Recipe::new(over_image, &max_argv, &over_env), Err(ImageError::Arguments)) { return Err("argv-over-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "argv-after-gate")? .map_err(|_| "argv-after-inventory")? != before { return Err("argv-resource-change"); } write_line("TURKOS/OS/K1180-TRACE: TEXT-LIMIT-PLUS-ONE-REJECTED"); let bad_image = unsafe { TrustedImage::from_kernel(&OS_K1180_BAD_ENTRY_ELF, OS_K1180_BAD_ENTRY_ELF.len()) }.map_err(|_| "bad-entry-trust")?; if !matches!(Recipe::new(bad_image, &[b"init"], &[]), Err(ImageError::Entry)) { return Err("kernel-entry-accepted"); } if store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "entry-after-gate")? .map_err(|_| "entry-after-inventory")? != before { return Err("entry-resource-change"); } write_line("TURKOS/OS/K1180-TRACE: KERNEL-ENTRY-REJECTED"); let trusted = unsafe { TrustedImage::from_kernel(&OS_K1180_USER_ELF, OS_K1180_USER_ELF.len()) } .map_err(|_| "el0-trusted-image")?; let argv: [&[u8]; 2] = [b"init", b"alpha"]; let env: [&[u8]; 2] = [b"MODE=1", b"LANG=tr"]; let recipe = Recipe::new(trusted, &argv, &env) .map_err(|_| "el0-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "el0-space-config")?; for stage in 0u8..11 { let fail_ptr = os_k1160_install_fail_table(&gate, &authority, &store, &image, config, stage)?; let fail_table = unsafe { &mut *fail_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let ticket = fail_table.begin(request).map_err(|_| "unwind-begin")?; let (error, ticket) = fail_table.prepare(ticket).err().ok_or("unwind-accepted")?; if error != process::Error::Resource || fail_table.snapshot().published != 0 || fail_table.snapshot().quarantined != 0 { return Err("unwind-failure-class"); } fail_table.abort(ticket).map_err(|_| "unwind-abort")?; if fail_table.snapshot() != process::Snapshot::default() { return Err("unwind-table-state"); } let backend = os_k1105_finish_table(fail_ptr)?; if backend.inventory().map_err(|_| "unwind-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("unwind-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "unwind-after-gate")? .map_err(|_| "unwind-after-inventory")? != before { return Err("unwind-physical-leak"); } write_str("TURKOS/OS/K1180-TRACE: UNWIND-STAGE="); write_hex(stage as usize); write_line(" PASS"); } let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "el0-begin")?; let prepared = table.prepare(reservation).map_err(|_| "el0-prepare")?; let pid = table.publish(prepared).map_err(|_| "el0-publish")?; write_line("TURKOS/OS/K1180-TRACE: PUBLISHED"); let mut identities = os_k1180_rights::Registry::::new(); let first = identities.bind_kernel(pid).map_err(|_| "rights-first-bind")?; if first.id() != 1 || identities.principal(pid) != Ok(first) { return Err("rights-first-principal"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1180_RIGHTS), (&mut identities as *mut _) as u64); } write_line("TURKOS/OS/K1180-TRACE: FIRST-PRINCIPAL-KERNEL-ASSIGNED"); let mut handles = os_k1180_rights::HandleSet::::new(); let file = handles.issue_kernel(&identities, pid, os_k1180_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: true }) .map_err(|_| "inherit-parent-file")?; let device = handles.issue_kernel(&identities, pid, os_k1180_rights::Capability::KernelDevice { object: 0xfee }) .map_err(|_| "inherit-parent-device")?; if file.id() != 1 || device.id() != 2 { return Err("inherit-parent-handles"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1180_HANDLES), (&mut handles as *mut _) as u64); } let mut inspect = os_k1180_rights::InspectSet::::new(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1180_INSPECT), (&mut inspect as *mut _) as u64); } let mut quota = os_k1180_rights::QuotaSet::::new(); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1180_QUOTA), (&mut quota as *mut _) as u64); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "el0-landing-stack")?; unsafe extern "C" { fn __os_k1180_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1180_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1180_EL0_ERROR)) } .unwrap_or("el0-process-body")); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1180_RIGHTS), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_HANDLES), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_INSPECT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1180_QUOTA), 0); } if identities.principal(pid) != Ok(first) || identities.snapshot().0 != 3 || handles.lookup(pid, file.id()) != Ok( os_k1180_rights::Capability::ReadOnlyFile { object: 0x1170, allow_child: true }) || handles.lookup(pid, device.id()) != Ok( os_k1180_rights::Capability::KernelDevice { object: 0xfee }) || inspect.snapshot() != (2, [None, None]) || quota.snapshot() != [None] { return Err("inherit-parent-after-el0"); } table.terminate_child(pid, 0).map_err(|_| "el0-terminate")?; if table.wait_child(pid) != Ok(0) { return Err("el0-reap"); } if handles.close_owner(pid) != 2 { return Err("inherit-parent-close"); } identities.retire_kernel(pid).map_err(|_| "rights-first-retire")?; if table.snapshot() != process::Snapshot::default() || identities.snapshot() != (3, [None, None]) || handles.snapshot() != (4, [None, None, None, None]) || inspect.snapshot() != (2, [None, None]) || quota.snapshot() != [None] { return Err("rights-final-state"); } let backend = os_k1105_finish_table(table_ptr)?; let inventory = backend.inventory().map_err(|_| "el0-owner-inventory")?; if inventory != [0, 0, 0, before.live_pages] { return Err("el0-owner-leak"); } drop(backend); if gate.faulted() || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "el0-after-gate")? .map_err(|_| "el0-after-inventory")? != before { return Err("el0-physical-rollback"); } write_line("TURKOS/OS/K1180-EL0: LIMIT=PAGE-THREAD-HANDLE PLUS-ONE=EAGAIN OTHER=RUNNING NINE-SVC=OK ERET=8 OWNER=RELEASED"); write_line("TURKOS/OS/K1180: PASS"); Ok(()) } global_asm!(include_str!("os_k1185_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1185_ZERO_ELF: [u8; 8824] = *include_bytes!("k1185-zero.elf"); static OS_K1185_FAULT_ELF: [u8; 8824] = *include_bytes!("k1185-fault.elf"); static OS_K1185_FORTY_TWO_ELF: [u8; 8824] = *include_bytes!("k1185-fortyTwo.elf"); // Mode 26 runs three separate retained EL0 processes from the same Image. // INIT is the kernel bootstrap collector until an EL0 parent/wait ABI exists. static mut OS_K1185_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1185_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1185_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1185_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1185_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1185_KERNEL_FRAME: u64 = 0; struct OsK1185El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, case: usize, old_vbar: u64, outcome: Option, } #[inline(never)] fn os_k1185_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("exit-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "exit-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1185_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1185_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1185_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "exit-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("exit-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1185_EL0_ERROR)) } .unwrap_or("exit-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1185_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1185_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1185_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } let episode = unsafe { &mut *(active as *mut OsK1185El0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } let frame = unsafe { &mut *pointer }; let offset = if episode.case == 1 { 8 } else { 16 }; if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + offset || episode.outcome.is_some() { park(); } let outcome = if episode.case == 1 { if frame.esr >> 26 & 0x3f != 0x24 || frame.far != 0x9000_0000 || frame.x[0] != 0x9000_0000 { park(); } let fault = process::Fault::from_lower_el_data_abort(frame.esr, frame.far) .unwrap_or_else(|_| park()); if table.fault_current(fault).is_err() { park(); } write_line("TURKOS/OS/K1185-TRACE: ACTUAL-EL0-FAR-DATA-ABORT"); process::Outcome::Fault(fault) } else { let expected = if episode.case == 0 { 0 } else { 42 }; let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.esr != abi::SVC_ESR || frame.x[8] != abi::CALL_EXIT || frame.x[7] != abi::ABI_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Exit(expected)) { park(); } if table.exit_current(expected as i32).is_err() { park(); } if expected == 0 { write_line("TURKOS/OS/K1185-TRACE: EXIT-0"); } else { write_line("TURKOS/OS/K1185-TRACE: EXIT-42-AFTER-FAULT"); } process::Outcome::Exited(expected as i32) }; if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); 1 // Never ERET after process termination. } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1185_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "exit-launch-owner")?; if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("exit-launch-root"); } let case = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185_CASE)) } as usize; if case >= 3 { return Err("exit-case"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1185El0Episode {table, pid, launch, case, old_vbar, outcome: None}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1185_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1185_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185_ACTIVE), (&mut episode as *mut OsK1185El0Episode) as u64); } unsafe extern "C" { fn __os_k1185_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1185_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1185_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185_KERNEL_FRAME), 0); } if episode.outcome.is_none() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) != Ok(episode.outcome) { return Err("exit-completion-owner"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185_RESULT), episode.outcome); } Ok(()) })(); // exit_current/fault_current already made the thread Dead and cleared current. match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185_EL0_ERROR), Some(error)); } 1 } } } static mut OS_K1185_CASE: u64 = 0; static mut OS_K1185_RESULT: Option = None; #[inline(never)] fn os_k1185_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "exit-before-inventory")?; let images: [&[u8]; 3] = [&OS_K1185_ZERO_ELF, &OS_K1185_FAULT_ELF, &OS_K1185_FORTY_TWO_ELF]; let mut outcomes: [Option; 3] = [None; 3]; let mut ids: [Option; 3] = [None; 3]; for (case, bytes) in images.into_iter().enumerate() { let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "exit-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1185 + case as u64, &authority, &gate) .map_err(|_| "exit-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "exit-frame-baseline")?; let trusted = unsafe { TrustedImage::from_kernel(bytes, bytes.len()) } .map_err(|_| "exit-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "exit-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "exit-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "exit-begin")?; let prepared = table.prepare(reservation).map_err(|_| "exit-prepare")?; let pid = table.publish(prepared).map_err(|_| "exit-publish")?; ids[case] = Some(pid); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185_CASE), case as u64); write_volatile(core::ptr::addr_of_mut!(OS_K1185_RESULT), None); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "exit-stack")?; unsafe extern "C" { fn __os_k1185_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1185_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1185_EL0_ERROR)) } .unwrap_or("exit-process-body")); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185_RESULT)) } .ok_or("exit-missing-result")?; if table.record(pid).map(|r| r.outcome) != Ok(Some(observed)) || table.snapshot().zombies != 1 { return Err("exit-zombie-result"); } if case == 1 { let snapshot = table.snapshot(); let inventory = authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "exit-fault-gate")? .map_err(|_| "exit-fault-inventory")?; if table.wait_child(pid) != Err(process::Error::State) || table.snapshot() != snapshot || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "exit-fault-after-gate")? .map_err(|_| "exit-fault-after-inventory")? != inventory { return Err("exit-fault-misclassified"); } write_line("TURKOS/OS/K1185-TRACE: LEGACY-WAIT-REFUSED-FAULT"); } let reaped = table.wait_child_outcome(pid) .map_err(|_| "exit-typed-wait")?; if reaped != observed || table.snapshot() != process::Snapshot::default() { return Err("exit-reap-result"); } outcomes[case] = Some(reaped); let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "exit-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("exit-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "exit-after-gate")? .map_err(|_| "exit-after-inventory")? != before { return Err("exit-physical-rollback"); } write_line("TURKOS/OS/K1185-TRACE: INIT-RECORDED-RESULT"); } let fault = match outcomes[1] { Some(process::Outcome::Fault(fault)) if fault.far() == 0x9000_0000 => fault, _ => return Err("exit-fault-outcome"), }; if outcomes[0] != Some(process::Outcome::Exited(0)) || outcomes[2] != Some(process::Outcome::Exited(42)) || fault.esr() >> 26 & 0x3f != 0x24 || ids[0] == ids[1] || ids[1] == ids[2] || ids[0] == ids[2] { return Err("exit-init-ledger"); } write_line("TURKOS/OS/K1185-EL0: INIT=EXIT0,FAULT,EXIT42 FAR=0x90000000 LEGACY-WAIT=DENIED OWNER=RELEASED"); write_line("TURKOS/OS/K1185: PASS"); Ok(()) } global_asm!(include_str!("os_k1185i_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1185I_USER_ELF: [u8; 9216] = *include_bytes!("k1185i-init-user.elf"); // Mode 27 runs a real EL0 INIT parent and three retained child episodes. static mut OS_K1185I_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1185I_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1185I_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1185I_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1185I_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1185I_KERNEL_FRAME: u64 = 0; static mut OS_K1185I_INIT_PID: Option = None; struct OsK1185IEl0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, case: usize, calls: u8, children: [Option; 3], old_vbar: u64, outcome: Option, } #[inline(never)] fn os_k1185i_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("exit-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "exit-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1185i_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1185i_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "exit-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("exit-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_EL0_ERROR)) } .unwrap_or("exit-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1185i_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1185i_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185I_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1185i_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { write_line("TURKOS/OS/K1185I-TRACE: DISPATCH-ENTRY"); use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } let episode = unsafe { &mut *(active as *mut OsK1185IEl0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1185I-TRACE: DISPATCH-OWNER"); let init_pid = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_INIT_PID)) } .unwrap_or_else(|| park()); if episode.case == 0 && episode.pid != init_pid || episode.case != 0 && table.record(episode.pid).map(|r| r.parent) != Ok(Some(init_pid)) { park(); } if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1185I-TRACE: FRAME ELR=0x"); write_hex(frame.elr as usize); write_str(" ESR=0x"); write_hex(frame.esr as usize); write_str(" FAR=0x"); write_hex(frame.far as usize); write_line(""); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || episode.outcome.is_some() { park(); } if episode.case == 2 && episode.calls == 1 { write_line("TURKOS/OS/K1185I-TRACE: FAULT-BRANCH"); if episode.calls != 1 || frame.elr != launch.user_pc + 200 { park(); } if frame.esr >> 26 & 0x3f != 0x24 || frame.far != 0x9000_0000 || frame.x[0] != 0x9000_0000 { park(); } let fault = process::Fault::from_lower_el_data_abort(frame.esr, frame.far) .unwrap_or_else(|_| park()); if table.fault_current(fault).is_err() { park(); } write_line("TURKOS/OS/K1185I-TRACE: ACTUAL-EL0-FAR-DATA-ABORT"); let outcome = process::Outcome::Fault(fault); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); return 1; } let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; write_str("TURKOS/OS/K1185I-TRACE: NEXT-SVC ELR=0x"); write_hex(frame.elr as usize); write_str(" CALL=0x"); write_hex(frame.x[8] as usize); write_line(""); if frame.esr != abi::SVC_ESR || frame.x[7] != abi::ABI_VERSION { park(); } let call = abi::decode_call(&abi_frame).unwrap_or_else(|_| park()); if call == abi::Call::Role { if episode.calls != 0 || frame.elr != launch.user_pc + 12 { park(); } frame.x[0] = episode.case as u64; episode.calls = 1; write_line("TURKOS/OS/K1185I-TRACE: EL0-ROLE"); return 0; } if let abi::Call::RunChild(role) = call { write_line("TURKOS/OS/K1185I-TRACE: RUN-CHILD-DECODED"); if episode.case != 0 || role != episode.calls as u64 || !(1..=3).contains(&role) || frame.elr != launch.user_pc + [0, 68, 92, 136][role as usize] { park(); } return os_k1185i_run_child(active as *mut OsK1185IEl0Episode, pointer, role, kernel_root); } let expected = match episode.case { 0 if episode.calls == 4 => 0, 1 if episode.calls == 1 => 0, 3 if episode.calls == 1 => 42, _ => park(), }; let exit_offset = match episode.case {0 => 172, 1 => 188, 3 => 220, _ => park()}; if call != abi::Call::Exit(expected) || frame.elr != launch.user_pc + exit_offset { park(); } if episode.case == 0 { let data = launch.data_frame as *const u64; if unsafe { read_volatile(data) } != 0 || unsafe { read_volatile(data.add(1)) } != 1 || unsafe { read_volatile(data.add(2)) } != 0x9000_0000 || unsafe { read_volatile(data.add(3)) } != 42 || unsafe { read_volatile(data.add(4)) } != 1 || table.snapshot().published != 1 || episode.children.iter().any(Option::is_none) || episode.children[0] == episode.children[1] || episode.children[1] == episode.children[2] || episode.children[0] == episode.children[2] { park(); } write_line("TURKOS/OS/K1185I-TRACE: EL0-INIT-PRIVATE-RESULTS-VERIFIED"); } if table.exit_current(expected as i32).is_err() { park(); } if episode.case == 1 { write_line("TURKOS/OS/K1185I-TRACE: CHILD-EXIT-0"); } else if episode.case == 3 { write_line("TURKOS/OS/K1185I-TRACE: CHILD-EXIT-42-AFTER-FAULT"); } else { write_line("TURKOS/OS/K1185I-TRACE: INIT-EXIT-0"); } let outcome = process::Outcome::Exited(expected as i32); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); 1 // Never ERET after process termination. } fn os_k1185i_run_child( parent_ptr: *mut OsK1185IEl0Episode, frame_ptr: *mut os_k920_exception::Frame, role: u64, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_table as process; let parent = unsafe { &mut *parent_ptr }; let table_ptr = parent.table.cast::>(); write_line("TURKOS/OS/K1185I-TRACE: RUN-CHILD-ENTRY"); let child = { let table = unsafe { &mut *table_ptr }; if table.snapshot().published != 1 || table.snapshot().zombies != 0 || table.record(parent.pid).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Running) { park(); } let spawn = process::Spawn {handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP}; let ticket = table.begin(spawn).unwrap_or_else(|_| park()); let ticket = table.prepare(ticket).ok().unwrap_or_else(|| park()); let child = table.publish(ticket).ok().unwrap_or_else(|| park()); if table.record(child).map(|r| r.parent) != Ok(Some(parent.pid)) || child == parent.pid || table.snapshot().published != 2 { park(); } child }; write_line("TURKOS/OS/K1185I-TRACE: RUN-CHILD-PUBLISHED"); parent.children[(role - 1) as usize] = Some(child); let saved_active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_ACTIVE)) }; let saved_user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_USER_TTBR)) }; let saved_trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_TRAP_TTBR)) }; let saved_frame = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_KERNEL_FRAME)) }; if saved_active != parent_ptr as u64 || saved_frame == 0 || saved_trap_ttbr != saved_user_ttbr { park(); } write_line("TURKOS/OS/K1185I-TRACE: RUN-CHILD-SAVED"); let stack_top = { let table = unsafe { &mut *table_ptr }; if unsafe { table.release_actual_current_to_ready() }.is_err() { park(); } write_line("TURKOS/OS/K1185I-TRACE: RUN-CHILD-YIELDED"); table.landing_stack_from_suspended_parent(parent.pid, child) .unwrap_or_else(|_| park()).1 }; write_line("TURKOS/OS/K1185I-TRACE: RUN-CHILD-STACK"); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185I_CASE), role); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_RESULT), None); } unsafe extern "C" { fn __os_k1185i_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } write_line("TURKOS/OS/K1185I-TRACE: CHILD-LAUNCH"); let status = unsafe { __os_k1185i_enter_el0_process_stack( parent.table, (&child as *const process::ProcessId).cast(), stack_top, kernel_root) }; if status != 0 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185I_ACTIVE), saved_active); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_USER_TTBR), saved_user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_TRAP_TTBR), saved_trap_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_KERNEL_FRAME), saved_frame); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_CASE), 0); } let table = unsafe { &mut *table_ptr }; if unsafe { table.register_actual_landing(parent.pid) }.is_err() { park(); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_RESULT)) } .unwrap_or_else(|| park()); if table.record(child).map(|r| r.outcome) != Ok(Some(observed)) || table.record(parent.pid).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Running) || table.snapshot().zombies != 1 { park(); } if role == 2 { let before = table.snapshot(); if table.wait_child(child) != Err(process::Error::State) || table.snapshot() != before { park(); } write_line("TURKOS/OS/K1185I-TRACE: FAULT-NOT-SUCCESS"); } let result = table.wait_child_outcome(child).unwrap_or_else(|_| park()); if result != observed || table.snapshot().published != 1 || table.snapshot().zombies != 0 { park(); } let frame = unsafe { &mut *frame_ptr }; match (role, result) { (1, process::Outcome::Exited(0)) => { frame.x[0] = 0; frame.x[1] = 0; frame.x[2] = 0; } (2, process::Outcome::Fault(fault)) if fault.far() == 0x9000_0000 => { frame.x[0] = 0; frame.x[1] = 1; frame.x[2] = fault.far(); } (3, process::Outcome::Exited(42)) => { frame.x[0] = 42; frame.x[1] = 0; frame.x[2] = 0; } _ => park(), } parent.calls = role as u8 + 1; write_line("TURKOS/OS/K1185I-TRACE: EL0-INIT-WAIT-RESULT"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1185i_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; write_line("TURKOS/OS/K1185I-TRACE: BODY-ENTRY"); if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } write_line("TURKOS/OS/K1185I-TRACE: BODY-LANDED"); let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "exit-launch-owner")?; write_line("TURKOS/OS/K1185I-TRACE: BODY-LAUNCH-READY"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("exit-launch-root"); } let case = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_CASE)) } as usize; write_str("TURKOS/OS/K1185I-TRACE: BODY-CASE=0x"); write_hex(case); write_line(""); if case >= 4 { return Err("exit-case"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1185IEl0Episode {table, pid, launch, case, calls: 0, children: [None; 3], old_vbar, outcome: None}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185I_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_ACTIVE), (&mut episode as *mut OsK1185IEl0Episode) as u64); } unsafe extern "C" { fn __os_k1185i_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1185i_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1185I-TRACE: BODY-EL0-RETURNED"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_KERNEL_FRAME), 0); } if episode.outcome.is_none() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) != Ok(episode.outcome) { return Err("exit-completion-owner"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185I_RESULT), episode.outcome); } Ok(()) })(); // exit_current/fault_current already made the thread Dead and cleared current. match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185I_EL0_ERROR), Some(error)); } 1 } } } static mut OS_K1185I_CASE: u64 = 0; static mut OS_K1185I_RESULT: Option = None; #[inline(never)] fn os_k1185i_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "exit-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "init-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1186, &authority, &gate) .map_err(|_| "init-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "init-frame-baseline")?; let trusted = unsafe { TrustedImage::from_kernel(&OS_K1185I_USER_ELF, OS_K1185I_USER_ELF.len()) }.map_err(|_| "init-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "init-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "init-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "init-begin")?; let prepared = table.prepare(reservation).map_err(|_| "init-prepare")?; let pid = table.publish(prepared).map_err(|_| "init-publish")?; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185I_CASE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_RESULT), None); write_volatile(core::ptr::addr_of_mut!(OS_K1185I_INIT_PID), Some(pid)); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "init-stack")?; unsafe extern "C" { fn __os_k1185i_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1185i_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_EL0_ERROR)) } .unwrap_or("init-process-body")); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1185I_RESULT)) } .ok_or("init-missing-result")?; if observed != process::Outcome::Exited(0) || table.record(pid).map(|r| r.outcome) != Ok(Some(observed)) || table.snapshot().zombies != 1 || table.snapshot().published != 1 { return Err("init-zombie-result"); } if table.wait_child_outcome(pid) != Ok(process::Outcome::Exited(0)) || table.snapshot() != process::Snapshot::default() { return Err("init-reap-result"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1185I_INIT_PID), None); } let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "init-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("init-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "init-after-gate")? .map_err(|_| "init-after-inventory")? != before { return Err("init-physical-rollback"); } write_line("TURKOS/OS/K1185I-EL0: EL0-INIT=EXIT0,FAULT,EXIT42 CHILDREN=3 FAR=0x90000000 OWNER=RELEASED"); write_line("TURKOS/OS/K1185I: PASS"); Ok(()) } global_asm!(include_str!("os_k1190w_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1190W_USER_ELF: [u8; 9376] = *include_bytes!("k1190w-wait-user.elf"); // Mode 28 runs a real EL0 INIT parent and consumes each child outcome once. static mut OS_K1190W_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1190W_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1190W_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1190W_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1190W_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1190W_KERNEL_FRAME: u64 = 0; static mut OS_K1190W_INIT_PID: Option = None; struct OsK1190WEl0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, case: usize, calls: u8, waits: u8, children: [Option; 3], old_vbar: u64, outcome: Option, } #[inline(never)] fn os_k1190w_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("exit-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "exit-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1190w_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1190w_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "exit-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("exit-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_EL0_ERROR)) } .unwrap_or("exit-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1190w_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1190w_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1190W_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1190w_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { write_line("TURKOS/OS/K1190W-TRACE: DISPATCH-ENTRY"); use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } let episode = unsafe { &mut *(active as *mut OsK1190WEl0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1190W-TRACE: DISPATCH-OWNER"); let init_pid = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_INIT_PID)) } .unwrap_or_else(|| park()); if episode.case == 0 && episode.pid != init_pid || episode.case != 0 && table.record(episode.pid).map(|r| r.parent) != Ok(Some(init_pid)) { park(); } if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1190W-TRACE: FRAME ELR=0x"); write_hex(frame.elr as usize); write_str(" ESR=0x"); write_hex(frame.esr as usize); write_str(" FAR=0x"); write_hex(frame.far as usize); write_line(""); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || episode.outcome.is_some() { park(); } if episode.case == 2 && episode.calls == 1 { write_line("TURKOS/OS/K1190W-TRACE: FAULT-BRANCH"); if episode.calls != 1 || frame.elr != launch.user_pc + 284 { park(); } if frame.esr >> 26 & 0x3f != 0x24 || frame.far != 0x9000_0000 || frame.x[0] != 0x9000_0000 { park(); } let fault = process::Fault::from_lower_el_data_abort(frame.esr, frame.far) .unwrap_or_else(|_| park()); if table.fault_current(fault).is_err() { park(); } write_line("TURKOS/OS/K1190W-TRACE: ACTUAL-EL0-FAR-DATA-ABORT"); let outcome = process::Outcome::Fault(fault); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); return 1; } let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; write_str("TURKOS/OS/K1190W-TRACE: NEXT-SVC ELR=0x"); write_hex(frame.elr as usize); write_str(" CALL=0x"); write_hex(frame.x[8] as usize); write_line(""); if frame.esr != abi::SVC_ESR || frame.x[7] != abi::ABI_VERSION { park(); } let call = abi::decode_call(&abi_frame).unwrap_or_else(|_| park()); if call == abi::Call::Role { if episode.calls != 0 || frame.elr != launch.user_pc + 12 { park(); } frame.x[0] = episode.case as u64; episode.calls = 1; write_line("TURKOS/OS/K1190W-TRACE: EL0-ROLE"); return 0; } if let abi::Call::RunChild(role) = call { write_line("TURKOS/OS/K1190W-TRACE: RUN-CHILD-DECODED"); if episode.case != 0 || !(1..=3).contains(&role) || role != episode.calls as u64 || episode.waits as u64 != role - 1 || frame.elr != launch.user_pc + [0, 76, 124, 192][role as usize] { park(); } return os_k1190w_run_child(active as *mut OsK1190WEl0Episode, pointer, role, kernel_root); } if let abi::Call::WaitAgain(role) = call { if episode.case != 0 || !(1..=3).contains(&role) || episode.calls as u64 != role + 1 || episode.waits as u64 != role - 1 || frame.elr != launch.user_pc + [0, 100, 168, 220][role as usize] { park(); } let child = episode.children[(role - 1) as usize] .unwrap_or_else(|| park()); let before = table.snapshot(); if table.wait_child_outcome(child) != Err(process::Error::NoChild) || table.snapshot() != before || table.record(child) != Err(process::Error::Stale) { park(); } episode.waits += 1; frame.x[0] = abi::AbiResult::Err(abi::Errno::Child).encode(); write_line("TURKOS/OS/K1190W-TRACE: SECOND-WAIT-ECHILD"); return 0; } let expected = match episode.case { 0 if episode.calls == 4 && episode.waits == 3 => 0, 1 if episode.calls == 1 => 0, 3 if episode.calls == 1 => 42, _ => park(), }; let exit_offset = match episode.case {0 => 256, 1 => 272, 3 => 304, _ => park()}; if call != abi::Call::Exit(expected) || frame.elr != launch.user_pc + exit_offset { park(); } if episode.case == 0 { let data = launch.data_frame as *const u64; if unsafe { read_volatile(data) } != 0 || unsafe { read_volatile(data.add(1)) } != 1 || unsafe { read_volatile(data.add(2)) } != 0x9000_0000 || unsafe { read_volatile(data.add(3)) } != 42 || unsafe { read_volatile(data.add(4)) } != 1 || unsafe { read_volatile(data.add(5)) } != 3 || table.snapshot().published != 1 || episode.children.iter().any(Option::is_none) || episode.children[0] == episode.children[1] || episode.children[1] == episode.children[2] || episode.children[0] == episode.children[2] { park(); } write_line("TURKOS/OS/K1190W-TRACE: EL0-INIT-PRIVATE-RESULTS-VERIFIED"); } if table.exit_current(expected as i32).is_err() { park(); } if episode.case == 1 { write_line("TURKOS/OS/K1190W-TRACE: CHILD-EXIT-0"); } else if episode.case == 3 { write_line("TURKOS/OS/K1190W-TRACE: CHILD-EXIT-42-AFTER-FAULT"); } else { write_line("TURKOS/OS/K1190W-TRACE: INIT-EXIT-0"); } let outcome = process::Outcome::Exited(expected as i32); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); 1 // Never ERET after process termination. } fn os_k1190w_run_child( parent_ptr: *mut OsK1190WEl0Episode, frame_ptr: *mut os_k920_exception::Frame, role: u64, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_table as process; let parent = unsafe { &mut *parent_ptr }; let table_ptr = parent.table.cast::>(); write_line("TURKOS/OS/K1190W-TRACE: RUN-CHILD-ENTRY"); let child = { let table = unsafe { &mut *table_ptr }; if table.snapshot().published != 1 || table.snapshot().zombies != 0 || table.record(parent.pid).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Running) { park(); } let spawn = process::Spawn {handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP}; let ticket = table.begin(spawn).unwrap_or_else(|_| park()); let ticket = table.prepare(ticket).ok().unwrap_or_else(|| park()); let child = table.publish(ticket).ok().unwrap_or_else(|| park()); if table.record(child).map(|r| r.parent) != Ok(Some(parent.pid)) || child == parent.pid || table.snapshot().published != 2 { park(); } child }; write_line("TURKOS/OS/K1190W-TRACE: RUN-CHILD-PUBLISHED"); parent.children[(role - 1) as usize] = Some(child); { let table = unsafe { &mut *table_ptr }; let before = table.snapshot(); let child_before = table.record(child).unwrap_or_else(|_| park()); if table.wait_child_outcome(child) != Err(process::Error::Busy) || table.snapshot() != before || table.record(child) != Ok(child_before) { park(); } } write_line("TURKOS/OS/K1190W-TRACE: BEFORE-EXIT-WAIT-BUSY"); let saved_active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_ACTIVE)) }; let saved_user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_USER_TTBR)) }; let saved_trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_TRAP_TTBR)) }; let saved_frame = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_KERNEL_FRAME)) }; if saved_active != parent_ptr as u64 || saved_frame == 0 || saved_trap_ttbr != saved_user_ttbr { park(); } write_line("TURKOS/OS/K1190W-TRACE: RUN-CHILD-SAVED"); let stack_top = { let table = unsafe { &mut *table_ptr }; if unsafe { table.release_actual_current_to_ready() }.is_err() { park(); } write_line("TURKOS/OS/K1190W-TRACE: RUN-CHILD-YIELDED"); table.landing_stack_from_suspended_parent(parent.pid, child) .unwrap_or_else(|_| park()).1 }; write_line("TURKOS/OS/K1190W-TRACE: RUN-CHILD-STACK"); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1190W_CASE), role); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_RESULT), None); } unsafe extern "C" { fn __os_k1190w_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } write_line("TURKOS/OS/K1190W-TRACE: CHILD-LAUNCH"); let status = unsafe { __os_k1190w_enter_el0_process_stack( parent.table, (&child as *const process::ProcessId).cast(), stack_top, kernel_root) }; if status != 0 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1190W_ACTIVE), saved_active); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_USER_TTBR), saved_user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_TRAP_TTBR), saved_trap_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_KERNEL_FRAME), saved_frame); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_CASE), 0); } let table = unsafe { &mut *table_ptr }; if unsafe { table.register_actual_landing(parent.pid) }.is_err() { park(); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_RESULT)) } .unwrap_or_else(|| park()); if table.record(child).map(|r| r.outcome) != Ok(Some(observed)) || table.record(parent.pid).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Running) || table.snapshot().zombies != 1 { park(); } if role == 2 { let before = table.snapshot(); if table.wait_child(child) != Err(process::Error::State) || table.snapshot() != before { park(); } write_line("TURKOS/OS/K1190W-TRACE: FAULT-NOT-SUCCESS"); } let result = table.wait_child_outcome(child).unwrap_or_else(|_| park()); if result != observed || table.snapshot().published != 1 || table.snapshot().zombies != 0 { park(); } let frame = unsafe { &mut *frame_ptr }; match (role, result) { (1, process::Outcome::Exited(0)) => { frame.x[0] = 0; frame.x[1] = 0; frame.x[2] = 0; } (2, process::Outcome::Fault(fault)) if fault.far() == 0x9000_0000 => { frame.x[0] = 0; frame.x[1] = 1; frame.x[2] = fault.far(); } (3, process::Outcome::Exited(42)) => { frame.x[0] = 42; frame.x[1] = 0; frame.x[2] = 0; } _ => park(), } parent.calls = role as u8 + 1; write_line("TURKOS/OS/K1190W-TRACE: EL0-INIT-WAIT-RESULT"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1190w_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; write_line("TURKOS/OS/K1190W-TRACE: BODY-ENTRY"); if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } write_line("TURKOS/OS/K1190W-TRACE: BODY-LANDED"); let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "exit-launch-owner")?; write_line("TURKOS/OS/K1190W-TRACE: BODY-LAUNCH-READY"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("exit-launch-root"); } let case = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_CASE)) } as usize; write_str("TURKOS/OS/K1190W-TRACE: BODY-CASE=0x"); write_hex(case); write_line(""); if case >= 4 { return Err("exit-case"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1190WEl0Episode {table, pid, launch, case, calls: 0, waits: 0, children: [None; 3], old_vbar, outcome: None}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1190W_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_ACTIVE), (&mut episode as *mut OsK1190WEl0Episode) as u64); } unsafe extern "C" { fn __os_k1190w_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1190w_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1190W-TRACE: BODY-EL0-RETURNED"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_KERNEL_FRAME), 0); } if episode.outcome.is_none() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) != Ok(episode.outcome) { return Err("exit-completion-owner"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1190W_RESULT), episode.outcome); } Ok(()) })(); // exit_current/fault_current already made the thread Dead and cleared current. match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1190W_EL0_ERROR), Some(error)); } 1 } } } static mut OS_K1190W_CASE: u64 = 0; static mut OS_K1190W_RESULT: Option = None; #[inline(never)] fn os_k1190w_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "exit-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "init-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1186, &authority, &gate) .map_err(|_| "init-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "init-frame-baseline")?; let trusted = unsafe { TrustedImage::from_kernel(&OS_K1190W_USER_ELF, OS_K1190W_USER_ELF.len()) }.map_err(|_| "init-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "init-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "init-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "init-begin")?; let prepared = table.prepare(reservation).map_err(|_| "init-prepare")?; let pid = table.publish(prepared).map_err(|_| "init-publish")?; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1190W_CASE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_RESULT), None); write_volatile(core::ptr::addr_of_mut!(OS_K1190W_INIT_PID), Some(pid)); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "init-stack")?; unsafe extern "C" { fn __os_k1190w_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1190w_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_EL0_ERROR)) } .unwrap_or("init-process-body")); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1190W_RESULT)) } .ok_or("init-missing-result")?; if observed != process::Outcome::Exited(0) || table.record(pid).map(|r| r.outcome) != Ok(Some(observed)) || table.snapshot().zombies != 1 || table.snapshot().published != 1 { return Err("init-zombie-result"); } if table.wait_child_outcome(pid) != Ok(process::Outcome::Exited(0)) || table.snapshot() != process::Snapshot::default() { return Err("init-reap-result"); } if table.wait_child_outcome(pid) != Err(process::Error::NoChild) || table.snapshot() != process::Snapshot::default() { return Err("init-second-wait"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1190W_INIT_PID), None); } let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "init-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("init-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "init-after-gate")? .map_err(|_| "init-after-inventory")? != before { return Err("init-physical-rollback"); } write_line("TURKOS/OS/K1190W-EL0: RESULT-ONCE=3 SECOND-WAIT=ECHILD BEFORE-EXIT=BUSY FAR=0x90000000 OWNER=RELEASED"); write_line("TURKOS/OS/K1190W: PASS"); Ok(()) } global_asm!(include_str!("os_k1195o_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1195O_USER_ELF: [u8; 9376] = *include_bytes!("k1195o-orphan-user.elf"); // Mode 29: the trusted kernel trap publishes two children for an EL0 parent // immediately before its requested exit. INIT later runs and reaps both. static mut OS_K1195O_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1195O_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1195O_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1195O_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1195O_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1195O_KERNEL_FRAME: u64 = 0; static mut OS_K1195O_INIT_PID: Option = None; static mut OS_K1195O_ORIGINAL_PARENT: Option = None; static mut OS_K1195O_ORPHANS: [Option; 2] = [None; 2]; struct OsK1195OEl0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, case: usize, calls: u8, waits: u8, children: [Option; 3], old_vbar: u64, outcome: Option, } #[inline(never)] fn os_k1195o_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("exit-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "exit-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1195o_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1195o_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "exit-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("exit-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_EL0_ERROR)) } .unwrap_or("exit-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1195o_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1195o_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1195o_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { write_line("TURKOS/OS/K1195O-TRACE: DISPATCH-ENTRY"); use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { write_str("TURKOS/OS/K1195O-TRACE: DISPATCH-GATE-ERROR ROOT=0x"); write_hex(kernel_root as usize); write_str(" USER=0x"); write_hex(user_ttbr as usize); write_str(" TRAP=0x"); write_hex(trap_ttbr as usize); write_str(" ACTIVE=0x"); write_hex(active as usize); write_str(" TTBR=0x"); write_hex(registers.ttbr as usize); write_str(" SPSEL=0x"); write_hex(registers.spsel as usize); write_line(""); park(); } write_line("TURKOS/OS/K1195O-TRACE: DISPATCH-GATE-PASSED"); let episode = unsafe { &mut *(active as *mut OsK1195OEl0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|error| { write_str("TURKOS/OS/K1195O-TRACE: DISPATCH-LAUNCH-ERROR=0x"); write_hex(error as usize); write_line(""); park() }); write_line("TURKOS/OS/K1195O-TRACE: DISPATCH-OWNER"); let init_pid = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_INIT_PID)) } .unwrap_or_else(|| park()); if episode.case == 0 && episode.pid != init_pid || episode.case != 0 && table.record(episode.pid).map(|r| r.parent) != Ok(Some(init_pid)) { park(); } write_line("TURKOS/OS/K1195O-TRACE: OWNER-CHECK-PASSED"); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } write_line("TURKOS/OS/K1195O-TRACE: LAUNCH-CHECK-PASSED"); let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } write_line("TURKOS/OS/K1195O-TRACE: FRAME-BOUNDS-PASSED"); let frame = unsafe { &mut *pointer }; write_str("TURKOS/OS/K1195O-TRACE: FRAME ELR=0x"); write_hex(frame.elr as usize); write_str(" ESR=0x"); write_hex(frame.esr as usize); write_str(" FAR=0x"); write_hex(frame.far as usize); write_line(""); if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || episode.outcome.is_some() { park(); } if episode.case == 2 && episode.calls == 1 { write_line("TURKOS/OS/K1195O-TRACE: FAULT-BRANCH"); if episode.calls != 1 || frame.elr != launch.user_pc + 284 { park(); } if frame.esr >> 26 & 0x3f != 0x24 || frame.far != 0x9000_0000 || frame.x[0] != 0x9000_0000 { park(); } write_line("TURKOS/OS/K1195O-TRACE: FAULT-HEADER-PASSED"); let fault = process::Fault::from_lower_el_data_abort(frame.esr, frame.far) .unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1195O-TRACE: FAULT-TYPED"); if let Err(error) = table.fault_current(fault) { write_str("TURKOS/OS/K1195O-TRACE: FAULT-EXIT-ERROR=0x"); write_hex(error as usize); write_line(""); park(); } write_line("TURKOS/OS/K1195O-TRACE: ACTUAL-EL0-FAR-DATA-ABORT"); let outcome = process::Outcome::Fault(fault); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); return 1; } let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; write_str("TURKOS/OS/K1195O-TRACE: NEXT-SVC ELR=0x"); write_hex(frame.elr as usize); write_str(" CALL=0x"); write_hex(frame.x[8] as usize); write_line(""); if frame.esr != abi::SVC_ESR || frame.x[7] != abi::ABI_VERSION { park(); } let call = abi::decode_call(&abi_frame).unwrap_or_else(|_| park()); if call == abi::Call::Role { if episode.calls != 0 || frame.elr != launch.user_pc + 12 { park(); } frame.x[0] = episode.case as u64; episode.calls = 1; write_line("TURKOS/OS/K1195O-TRACE: EL0-ROLE"); return 0; } if let abi::Call::RunChild(role) = call { write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-DECODED"); if episode.case != 0 || !(1..=3).contains(&role) || role != episode.calls as u64 || episode.waits as u64 != role - 1 || frame.elr != launch.user_pc + [0, 76, 124, 192][role as usize] { park(); } return os_k1195o_run_child(active as *mut OsK1195OEl0Episode, pointer, role, kernel_root); } if let abi::Call::WaitAgain(role) = call { if episode.case != 0 || !(1..=3).contains(&role) || episode.calls as u64 != role + 1 || episode.waits as u64 != role - 1 || frame.elr != launch.user_pc + [0, 100, 168, 220][role as usize] { park(); } let child = episode.children[(role - 1) as usize] .unwrap_or_else(|| park()); let before = table.snapshot(); if table.wait_child_outcome(child) != Err(process::Error::NoChild) || table.snapshot() != before || table.record(child) != Err(process::Error::Stale) { park(); } episode.waits += 1; frame.x[0] = abi::AbiResult::Err(abi::Errno::Child).encode(); write_line("TURKOS/OS/K1195O-TRACE: SECOND-WAIT-ECHILD"); return 0; } let expected = match episode.case { 0 if episode.calls == 4 && episode.waits == 3 => 0, 1 if episode.calls == 1 => 0, 3 if episode.calls == 1 => 42, _ => park(), }; let exit_offset = match episode.case {0 => 256, 1 => 272, 3 => 304, _ => park()}; if call != abi::Call::Exit(expected) || frame.elr != launch.user_pc + exit_offset { park(); } if episode.case == 0 { let data = launch.data_frame as *const u64; if unsafe { read_volatile(data) } != 0 || unsafe { read_volatile(data.add(1)) } != 1 || unsafe { read_volatile(data.add(2)) } != 0x9000_0000 || unsafe { read_volatile(data.add(3)) } != 42 || unsafe { read_volatile(data.add(4)) } != 1 || unsafe { read_volatile(data.add(5)) } != 3 || table.snapshot().published != 1 || episode.children.iter().any(Option::is_none) || episode.children[0] == episode.children[1] || episode.children[1] == episode.children[2] || episode.children[0] == episode.children[2] || table.next_orphan(episode.pid) != Err(process::Error::NoChild) { park(); } write_line("TURKOS/OS/K1195O-TRACE: EL0-INIT-PRIVATE-RESULTS-VERIFIED"); } if episode.case == 1 { use turkos_kernel::runtime::process_guest as guest; write_line("TURKOS/OS/K1195O-TRACE: PARENT-SPAWN-ENTRY"); if table.snapshot().published != 2 || table.snapshot().zombies != 0 || table.record(episode.pid).map(|r| r.parent) != Ok(Some(init_pid)) { park(); } write_line("TURKOS/OS/K1195O-TRACE: PARENT-SPAWN-GATE"); let spawn = process::Spawn {handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP}; let mut orphans = [None; 2]; for slot in &mut orphans { let ticket = table.begin(spawn).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1195O-TRACE: ORPHAN-BEGIN"); let ticket = match table.prepare(ticket) { Ok(ticket) => ticket, Err((error, _)) => { write_str("TURKOS/OS/K1195O-TRACE: ORPHAN-PREPARE-ERROR=0x"); write_hex(error as usize); write_str(" STAGE=0x"); write_hex(unsafe { read_volatile(core::ptr::addr_of!(guest::OS_K1195O_PREP_STAGE)) } as usize); write_line(""); park() } }; write_line("TURKOS/OS/K1195O-TRACE: ORPHAN-PREPARE"); let child = table.publish(ticket).ok().unwrap_or_else(|| park()); write_line("TURKOS/OS/K1195O-TRACE: ORPHAN-PUBLISH"); if table.record(child).map(|r| (r.parent, r.orphaned_from)) != Ok((Some(episode.pid), None)) { park(); } *slot = Some(child); } if orphans[0] == orphans[1] || table.snapshot().published != 4 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ORIGINAL_PARENT), Some(episode.pid)); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ORPHANS), orphans); } write_line("TURKOS/OS/K1195O-TRACE: TWO-CHILDREN-PUBLISHED-UNDER-EL0-PARENT"); } if table.exit_current(expected as i32).is_err() { park(); } if episode.case == 1 { let orphans = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ORPHANS)) }; for child in orphans.into_iter().flatten() { if table.record(child).map(|r| (r.parent, r.orphaned_from)) != Ok((Some(init_pid), Some(episode.pid))) { park(); } } write_line("TURKOS/OS/K1195O-TRACE: EL0-PARENT-EXIT-ADOPTED-TWO"); } else if episode.case == 3 { write_line("TURKOS/OS/K1195O-TRACE: CHILD-EXIT-42-AFTER-FAULT"); } else { write_line("TURKOS/OS/K1195O-TRACE: INIT-EXIT-0"); } let outcome = process::Outcome::Exited(expected as i32); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); 1 // Never ERET after process termination. } fn os_k1195o_run_child( parent_ptr: *mut OsK1195OEl0Episode, frame_ptr: *mut os_k920_exception::Frame, role: u64, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_table as process; let parent = unsafe { &mut *parent_ptr }; let table_ptr = parent.table.cast::>(); write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-ENTRY"); let child = { let table = unsafe { &mut *table_ptr }; if table.snapshot().published != (if role == 1 { 1 } else { 5 - role as usize }) || table.snapshot().zombies != 0 || table.record(parent.pid).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Running) { park(); } if role == 1 { let spawn = process::Spawn {handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP}; let ticket = table.begin(spawn).unwrap_or_else(|_| park()); let ticket = table.prepare(ticket).ok().unwrap_or_else(|| park()); let child = table.publish(ticket).ok().unwrap_or_else(|| park()); if table.record(child).map(|r| r.parent) != Ok(Some(parent.pid)) || child == parent.pid || table.snapshot().published != 2 { park(); } child } else { let orphans = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ORPHANS)) }; let original = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ORIGINAL_PARENT)) } .unwrap_or_else(|| park()); let child = orphans[role as usize - 2].unwrap_or_else(|| park()); if table.next_orphan(parent.pid) != Ok(child) || table.record(child).map(|r| (r.parent, r.orphaned_from)) != Ok((Some(parent.pid), Some(original))) || table.record(original) != Err(process::Error::Stale) { park(); } write_line("TURKOS/OS/K1195O-TRACE: INIT-OLDEST-ORPHAN-SELECTED"); child } }; write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-PUBLISHED"); parent.children[(role - 1) as usize] = Some(child); { let table = unsafe { &mut *table_ptr }; let before = table.snapshot(); let child_before = table.record(child).unwrap_or_else(|_| park()); if table.wait_child_outcome(child) != Err(process::Error::Busy) || table.snapshot() != before || table.record(child) != Ok(child_before) { park(); } } write_line("TURKOS/OS/K1195O-TRACE: BEFORE-EXIT-WAIT-BUSY"); let saved_active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ACTIVE)) }; let saved_user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_USER_TTBR)) }; let saved_trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_TRAP_TTBR)) }; let saved_frame = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_KERNEL_FRAME)) }; if saved_active != parent_ptr as u64 || saved_frame == 0 || saved_trap_ttbr != saved_user_ttbr { park(); } write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-SAVED"); let stack_top = { let table = unsafe { &mut *table_ptr }; if unsafe { table.release_actual_current_to_ready() }.is_err() { park(); } write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-YIELDED"); table.landing_stack_from_suspended_parent(parent.pid, child) .unwrap_or_else(|_| park()).1 }; write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-STACK"); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_CASE), role); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_RESULT), None); } unsafe extern "C" { fn __os_k1195o_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } write_line("TURKOS/OS/K1195O-TRACE: CHILD-LAUNCH"); let status = unsafe { __os_k1195o_enter_el0_process_stack( parent.table, (&child as *const process::ProcessId).cast(), stack_top, kernel_root) }; if status != 0 { write_str("TURKOS/OS/K1195O-TRACE: CHILD-BODY-ERROR="); write_line(unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_EL0_ERROR)) } .unwrap_or("unknown")); park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ACTIVE), saved_active); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_USER_TTBR), saved_user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_TRAP_TTBR), saved_trap_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_FRAME), saved_frame); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_CASE), 0); } let table = unsafe { &mut *table_ptr }; if unsafe { table.register_actual_landing(parent.pid) }.is_err() { park(); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_RESULT)) } .unwrap_or_else(|| park()); if table.record(child).map(|r| r.outcome) != Ok(Some(observed)) || table.record(parent.pid).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Running) || table.snapshot().zombies != 1 || table.snapshot().published != 5 - role as usize { park(); } if role == 2 { let before = table.snapshot(); if table.wait_child(child) != Err(process::Error::State) || table.snapshot() != before { park(); } write_line("TURKOS/OS/K1195O-TRACE: FAULT-NOT-SUCCESS"); } let result = table.wait_child_outcome(child).unwrap_or_else(|_| park()); if result != observed || table.snapshot().published != 4 - role as usize || table.snapshot().zombies != 0 { park(); } if role == 1 { let before = table.snapshot(); let first_orphan = table.next_orphan(parent.pid).unwrap_or_else(|_| park()); let spawn = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let ticket = table.begin(spawn).unwrap_or_else(|_| park()); let ticket = table.prepare(ticket).ok().unwrap_or_else(|| park()); let replacement = table.publish(ticket).ok().unwrap_or_else(|| park()); if replacement.slot() != child.slot() || replacement.generation() == child.generation() || table.record(child) != Err(process::Error::Stale) || table.record(replacement).map(|r| r.parent) != Ok(Some(parent.pid)) || table.next_orphan(parent.pid) != Ok(first_orphan) { park(); } table.terminate_child(replacement, 7).unwrap_or_else(|_| park()); if table.wait_child_outcome(replacement) != Ok(process::Outcome::Exited(7)) || table.snapshot() != before || table.record(replacement) != Err(process::Error::Stale) || table.next_orphan(parent.pid) != Ok(first_orphan) { park(); } write_line("TURKOS/OS/K1195O-TRACE: PARENT-PID-SLOT-REUSED-ORPHANS-UNCHANGED"); } let frame = unsafe { &mut *frame_ptr }; match (role, result) { (1, process::Outcome::Exited(0)) => { frame.x[0] = 0; frame.x[1] = 0; frame.x[2] = 0; } (2, process::Outcome::Fault(fault)) if fault.far() == 0x9000_0000 => { frame.x[0] = 0; frame.x[1] = 1; frame.x[2] = fault.far(); } (3, process::Outcome::Exited(42)) => { frame.x[0] = 42; frame.x[1] = 0; frame.x[2] = 0; } _ => park(), } parent.calls = role as u8 + 1; write_line("TURKOS/OS/K1195O-TRACE: EL0-INIT-WAIT-RESULT"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1195o_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; write_line("TURKOS/OS/K1195O-TRACE: BODY-ENTRY"); if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } write_line("TURKOS/OS/K1195O-TRACE: BODY-LANDED"); let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|error| { write_str("TURKOS/OS/K1195O-TRACE: BODY-LAUNCH-ERROR=0x"); write_hex(error as usize); write_line(""); "exit-launch-owner" })?; write_line("TURKOS/OS/K1195O-TRACE: BODY-LAUNCH-READY"); if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("exit-launch-root"); } let case = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_CASE)) } as usize; write_str("TURKOS/OS/K1195O-TRACE: BODY-CASE=0x"); write_hex(case); write_line(""); if case >= 4 { return Err("exit-case"); } if case == 3 && unsafe { read_volatile(core::ptr::addr_of!(OS_K1200_ACTIVE)) } { use turkos_kernel::runtime::frame_metadata as metadata; let store_address = unsafe { read_volatile(core::ptr::addr_of!(OS_K1200_STORE)) }; if store_address == 0 { return Err("retire-store"); } let store = unsafe { &*(store_address as *const os_k1120_owners::FrameStore<'_, OsK1120Hardware<'_>>) }; let record = unsafe { &*table_ptr }.record(pid) .map_err(|_| "retire-record")?; let page = record.resources.handles[1].ok_or("retire-page")?; let init = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_INIT_PID)) } .ok_or("retire-init")?; let before = store.snapshot().map_err(|_| "retire-inventory-before")?; if store.try_early_release(page, record.resources.resource_owner) != Err(metadata::Error::Pinned) || store.snapshot() != Ok(before) || unsafe { &*table_ptr }.record(pid) != Ok(record) || unsafe { &*table_ptr }.snapshot().published != 2 || unsafe { &*table_ptr }.record(init).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Ready) { return Err("retire-early-free"); } write_line("TURKOS/OS/K1200-EL0: EARLY-FREE=PINNED HEALTHY-2=CONTINUE"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1195OEl0Episode {table, pid, launch, case, calls: 0, waits: 0, children: [None; 3], old_vbar, outcome: None}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ACTIVE), (&mut episode as *mut OsK1195OEl0Episode) as u64); } unsafe extern "C" { fn __os_k1195o_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1195o_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1195O-TRACE: BODY-EL0-RETURNED"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_FRAME), 0); } if episode.outcome.is_none() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) != Ok(episode.outcome) { return Err("exit-completion-owner"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_RESULT), episode.outcome); } Ok(()) })(); // exit_current/fault_current already made the thread Dead and cleared current. match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_EL0_ERROR), Some(error)); } 1 } } } static mut OS_K1195O_CASE: u64 = 0; static mut OS_K1195O_RESULT: Option = None; #[inline(never)] fn os_k1195o_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "exit-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "init-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1186, &authority, &gate) .map_err(|_| "init-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "init-frame-baseline")?; unsafe { core::ptr::write_volatile(core::ptr::addr_of_mut!(OS_K1200_STORE), (&store as *const _) as u64); } let trusted = unsafe { TrustedImage::from_kernel(&OS_K1195O_USER_ELF, OS_K1195O_USER_ELF.len()) }.map_err(|_| "init-trusted-image")?; let recipe = Recipe::new(trusted, &[b"init"], &[]) .map_err(|_| "init-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "init-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "init-begin")?; let prepared = table.prepare(reservation).map_err(|_| "init-prepare")?; let pid = table.publish(prepared).map_err(|_| "init-publish")?; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_CASE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_RESULT), None); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_INIT_PID), Some(pid)); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ORIGINAL_PARENT), None); write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ORPHANS), [None; 2]); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "init-stack")?; unsafe extern "C" { fn __os_k1195o_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1195o_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_EL0_ERROR)) } .unwrap_or("init-process-body")); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_RESULT)) } .ok_or("init-missing-result")?; if observed != process::Outcome::Exited(0) || table.record(pid).map(|r| r.outcome) != Ok(Some(observed)) || table.snapshot().zombies != 1 || table.snapshot().published != 1 { return Err("init-zombie-result"); } if table.wait_child_outcome(pid) != Ok(process::Outcome::Exited(0)) || table.snapshot() != process::Snapshot::default() { return Err("init-reap-result"); } if table.wait_child_outcome(pid) != Err(process::Error::NoChild) || table.snapshot() != process::Snapshot::default() { return Err("init-second-wait"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_INIT_PID), None); } let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "init-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("init-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "init-after-gate")? .map_err(|_| "init-after-inventory")? != before { return Err("init-physical-rollback"); } write_line("TURKOS/OS/K1195O-EL0: PARENT-EXIT=ADOPTED-2 INIT-ORDER=2 FAULT=0x90000000 OWNER=RELEASED"); write_line("TURKOS/OS/K1195: PASS"); if unsafe { read_volatile(core::ptr::addr_of!(OS_K1200_ACTIVE)) } { write_line("TURKOS/OS/K1200: PASS"); } Ok(()) } #[unsafe(no_mangle)] static mut OS_K1200_STORE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1200_ACTIVE: bool = false; global_asm!(include_str!("os_k1205_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1205_USER_ELF: [u8; 11312] = *include_bytes!("k1205-rust-two-argv.elf"); // Mode 31 runs the same static Rust ELF with a rejected unaligned SP and // then a healthy aligned SP. The bootstrap owner collects both outcomes. static mut OS_K1205_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1205_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1205_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1205_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1205_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1205_KERNEL_FRAME: u64 = 0; struct OsK1205El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, case: usize, old_vbar: u64, outcome: Option, } #[inline(never)] fn os_k1205_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("exit-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "exit-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1205_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1205_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1205_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "exit-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("exit-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1205_EL0_ERROR)) } .unwrap_or("exit-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1205_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1205_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1205_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1205_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1205_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1205_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1205_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1205_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } let episode = unsafe { &mut *(active as *mut OsK1205El0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } let frame = unsafe { &mut *pointer }; let offset = 0x58; if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 != launch.user_sp + u64::from(episode.case == 0) || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + offset || episode.outcome.is_some() { park(); } let expected = if episode.case == 0 { 126 } else { 42 }; let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.esr != abi::SVC_ESR || frame.x[8] != abi::CALL_EXIT || frame.x[7] != abi::ABI_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Exit(expected)) { park(); } if table.exit_current(expected as i32).is_err() { park(); } if episode.case == 0 { write_line("TURKOS/OS/K1205-TRACE: UNALIGNED-SP-REJECTED-126"); } else { write_line("TURKOS/OS/K1205-TRACE: RUST-MAIN-ARGV-EXIT-42"); } let outcome = process::Outcome::Exited(expected as i32); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); 1 // Never ERET after process termination. } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1205_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "exit-launch-owner")?; if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("exit-launch-root"); } let case = unsafe { read_volatile(core::ptr::addr_of!(OS_K1205_CASE)) } as usize; if case >= 2 { return Err("exit-case"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1205El0Episode {table, pid, launch, case, old_vbar, outcome: None}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1205_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1205_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1205_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1205_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1205_ACTIVE), (&mut episode as *mut OsK1205El0Episode) as u64); } unsafe extern "C" { fn __os_k1205_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1205_enter_el0(user_ttbr, launch.user_pc, launch.user_sp + u64::from(case == 0), launch.user_spsr); } unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1205_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1205_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1205_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1205_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1205_KERNEL_FRAME), 0); } write_str("TURKOS/OS/K1205-TRACE: RETURN OUTCOME=0x"); write_hex(episode.outcome.is_some() as usize); write_str(" TTBR=0x"); write_hex(os_k1120_registers().ttbr as usize); write_str(" ROOT=0x"); write_hex(kernel_root as usize); write_str(" RECORD=0x"); write_hex((unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) == Ok(episode.outcome)) as usize); write_line(""); if episode.outcome.is_none() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) != Ok(episode.outcome) { return Err("exit-completion-owner"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1205_RESULT), episode.outcome); } Ok(()) })(); // exit_current/fault_current already made the thread Dead and cleared current. match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1205_EL0_ERROR), Some(error)); } 1 } } } static mut OS_K1205_CASE: u64 = 0; static mut OS_K1205_RESULT: Option = None; #[inline(never)] fn os_k1205_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "exit-before-inventory")?; let images: [&[u8]; 2] = [&OS_K1205_USER_ELF, &OS_K1205_USER_ELF]; let mut outcomes: [Option; 2] = [None; 2]; let mut ids: [Option; 2] = [None; 2]; for (case, bytes) in images.into_iter().enumerate() { let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "exit-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1205 + case as u64, &authority, &gate) .map_err(|_| "exit-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "exit-frame-baseline")?; let trusted = unsafe { TrustedImage::from_kernel(bytes, bytes.len()) } .map_err(|_| "exit-trusted-image")?; let recipe = Recipe::new(trusted, &[b"runtime", b"two"], &[]) .map_err(|_| "exit-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "exit-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "exit-begin")?; let prepared = table.prepare(reservation).map_err(|_| "exit-prepare")?; let pid = table.publish(prepared).map_err(|_| "exit-publish")?; ids[case] = Some(pid); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1205_CASE), case as u64); write_volatile(core::ptr::addr_of_mut!(OS_K1205_RESULT), None); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "exit-stack")?; unsafe extern "C" { fn __os_k1205_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1205_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1205_EL0_ERROR)) } .unwrap_or("exit-process-body")); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1205_RESULT)) } .ok_or("exit-missing-result")?; if table.record(pid).map(|r| r.outcome) != Ok(Some(observed)) || table.snapshot().zombies != 1 { return Err("exit-zombie-result"); } let reaped = table.wait_child_outcome(pid) .map_err(|_| "exit-typed-wait")?; if reaped != observed || table.snapshot() != process::Snapshot::default() { return Err("exit-reap-result"); } outcomes[case] = Some(reaped); let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "exit-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("exit-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "exit-after-gate")? .map_err(|_| "exit-after-inventory")? != before { return Err("exit-physical-rollback"); } write_line("TURKOS/OS/K1205-TRACE: INIT-RECORDED-RESULT"); } if outcomes != [Some(process::Outcome::Exited(126)), Some(process::Outcome::Exited(42))] || ids[0] == ids[1] { return Err("rust-start-ledger"); } write_line("TURKOS/OS/K1205-EL0: BAD-SP=EXIT126 RUST-ARGV=2 EXIT42 OWNER=RELEASED"); write_line("TURKOS/OS/K1205: PASS"); Ok(()) } global_asm!(include_str!("os_k1210_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1210_PANIC_ELF: [u8; 11312] = *include_bytes!("k1210-rust-panic.elf"); static OS_K1210_HEALTHY_ELF: [u8; 11312] = *include_bytes!("k1210-rust-healthy.elf"); // Mode 32 runs a Rust panic child and then a healthy Rust child. // The kernel bootstrap collector reaps both outcomes. static mut OS_K1210_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1210_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1210_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1210_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1210_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1210_KERNEL_FRAME: u64 = 0; struct OsK1210El0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, case: usize, old_vbar: u64, outcome: Option, } #[inline(never)] fn os_k1210_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("exit-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "exit-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1210_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1210_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1210_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "exit-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("exit-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1210_EL0_ERROR)) } .unwrap_or("exit-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1210_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1210_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1210_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } let episode = unsafe { &mut *(active as *mut OsK1210El0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } let frame = unsafe { &mut *pointer }; let offset = 0x58; if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 < 0x80003000 || frame.sp_el0 > launch.user_sp || frame.sp_el0 & 15 != 0 || episode.case == 1 && frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || frame.elr != launch.user_pc + offset || episode.outcome.is_some() { park(); } let expected = if episode.case == 0 { 127 } else { 42 }; let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.esr != abi::SVC_ESR || frame.x[8] != abi::CALL_EXIT || frame.x[7] != abi::ABI_VERSION || abi::decode_call(&abi_frame) != Ok(abi::Call::Exit(expected)) { park(); } if table.exit_current(expected as i32).is_err() { park(); } if episode.case == 0 { write_line("TURKOS/OS/K1210-TRACE: RUST-PANIC-EXIT-127"); } else { write_line("TURKOS/OS/K1210-TRACE: HEALTHY-AFTER-PANIC-EXIT-42"); } let outcome = process::Outcome::Exited(expected as i32); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); 1 // Never ERET after process termination. } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1210_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "exit-launch-owner")?; if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("exit-launch-root"); } let case = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210_CASE)) } as usize; if case >= 2 { return Err("exit-case"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1210El0Episode {table, pid, launch, case, old_vbar, outcome: None}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1210_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1210_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210_ACTIVE), (&mut episode as *mut OsK1210El0Episode) as u64); } unsafe extern "C" { fn __os_k1210_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1210_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1210_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210_KERNEL_FRAME), 0); } write_str("TURKOS/OS/K1210-TRACE: RETURN OUTCOME=0x"); write_hex(episode.outcome.is_some() as usize); write_str(" TTBR=0x"); write_hex(os_k1120_registers().ttbr as usize); write_str(" ROOT=0x"); write_hex(kernel_root as usize); write_str(" RECORD=0x"); write_hex((unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) == Ok(episode.outcome)) as usize); write_line(""); if episode.outcome.is_none() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) != Ok(episode.outcome) { return Err("exit-completion-owner"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210_RESULT), episode.outcome); } Ok(()) })(); // exit_current/fault_current already made the thread Dead and cleared current. match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210_EL0_ERROR), Some(error)); } 1 } } } static mut OS_K1210_CASE: u64 = 0; static mut OS_K1210_RESULT: Option = None; #[inline(never)] fn os_k1210_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "exit-before-inventory")?; let images: [&[u8]; 2] = [&OS_K1210_PANIC_ELF, &OS_K1210_HEALTHY_ELF]; let mut outcomes: [Option; 2] = [None; 2]; let mut ids: [Option; 2] = [None; 2]; for (case, bytes) in images.into_iter().enumerate() { let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "exit-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1210 + case as u64, &authority, &gate) .map_err(|_| "exit-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "exit-frame-baseline")?; let trusted = unsafe { TrustedImage::from_kernel(bytes, bytes.len()) } .map_err(|_| "exit-trusted-image")?; let recipe = Recipe::new(trusted, &[b"runtime".as_slice(), if case == 0 { b"panic".as_slice() } else { b"two".as_slice() }], &[]) .map_err(|_| "exit-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "exit-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "exit-begin")?; let prepared = table.prepare(reservation).map_err(|_| "exit-prepare")?; let pid = table.publish(prepared).map_err(|_| "exit-publish")?; ids[case] = Some(pid); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210_CASE), case as u64); write_volatile(core::ptr::addr_of_mut!(OS_K1210_RESULT), None); } let (_, stack_top) = table.landing_stack(pid).map_err(|_| "exit-stack")?; unsafe extern "C" { fn __os_k1210_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1210_enter_el0_process_stack( table_ptr.cast(), (&pid as *const process::ProcessId).cast(), stack_top, tables.root) }; if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1210_EL0_ERROR)) } .unwrap_or("exit-process-body")); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210_RESULT)) } .ok_or("exit-missing-result")?; if table.record(pid).map(|r| r.outcome) != Ok(Some(observed)) || table.snapshot().zombies != 1 { return Err("exit-zombie-result"); } let reaped = table.wait_child_outcome(pid) .map_err(|_| "exit-typed-wait")?; if reaped != observed || table.snapshot() != process::Snapshot::default() { return Err("exit-reap-result"); } outcomes[case] = Some(reaped); let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "exit-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("exit-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "exit-after-gate")? .map_err(|_| "exit-after-inventory")? != before { return Err("exit-physical-rollback"); } write_line("TURKOS/OS/K1210-TRACE: INIT-RECORDED-RESULT"); } if outcomes != [Some(process::Outcome::Exited(127)), Some(process::Outcome::Exited(42))] || ids[0] == ids[1] { return Err("rust-start-ledger"); } write_line("TURKOS/OS/K1210-EL0: PANIC=EXIT127 HEALTHY=EXIT42 OWNER=RELEASED"); write_line("TURKOS/OS/K1210: PASS"); Ok(()) } global_asm!(include_str!("os_k1210s_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1210S_USER_ELF: [u8; 11592] = *include_bytes!("k1210s-user.elf"); // Mode 33 runs an EL0 shell parent and Rust panic child from one immutable ELF. // The shell waits once, rejects a repeated wait and calls Version after panic. static mut OS_K1210S_EL0_ERROR: Option<&'static str> = None; #[unsafe(no_mangle)] static mut OS_K1210S_ACTIVE: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1210S_KERNEL_ROOT: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1210S_USER_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1210S_TRAP_TTBR: u64 = 0; #[unsafe(no_mangle)] static mut OS_K1210S_KERNEL_FRAME: u64 = 0; static mut OS_K1210S_INIT_PID: Option = None; struct OsK1210SEl0Episode { table: *mut u8, pid: turkos_kernel::runtime::process_table::ProcessId, launch: turkos_kernel::runtime::process_table::GuestLaunch, case: usize, calls: u8, waits: u8, child: Option, old_vbar: u64, outcome: Option, } #[inline(never)] fn os_k1210s_el0_on_work_stack( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let low = guard + 4096; let end = low + 131072; let before = os_k1120_registers(); if before.sp >= guard && before.sp < end || before.cpu != 0 || before.daif & 128 == 0 || before.current_el != 4 || before.irq_active || before.ttbr != tables.root { return Err("exit-work-stack-context"); } let (slot, descriptor) = unsafe { tables.os_k1105_acquire_work_guard(guard, end, core::ptr::addr_of!(_image_start) as u64, core::ptr::addr_of!(_image_end) as u64) }.map_err(|_| "exit-work-stack-guard")?; unsafe { write_volatile(low as *mut u64, OS_K1105_LOW_CANARY); write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_EL0_ERROR), None); } unsafe extern "C" { fn __os_k1210s_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64; } let status = unsafe { __os_k1210s_el0_on_stack( (memory as *mut OsK660Memory).cast(), (tables as *const mmu::Tables).cast()) }; let after = os_k1120_registers(); let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY }; unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) } .map_err(|_| "exit-work-stack-restore")?; if after.sp != before.sp || after.spsel != before.spsel || after.ttbr != before.ttbr || after.tcr != before.tcr || after.sctlr != before.sctlr || after.daif != before.daif || !canaries { return Err("exit-work-stack-return"); } if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_EL0_ERROR)) } .unwrap_or("exit-work-stack-body")); } Ok(()) } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1210s_el0_body(memory: *mut u8, tables: *const u8) -> u64 { let result = os_k1210s_el0_probe( unsafe { &mut *memory.cast::() }, unsafe { &*tables.cast::() }); match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210S_EL0_ERROR), Some(error)); } 1 } } } #[unsafe(no_mangle)] extern "C" fn os_k1210s_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 { write_line("TURKOS/OS/K1210S-TRACE: DISPATCH-ENTRY"); use turkos_kernel::runtime::process_table as process; use turkos_kernel::runtime::syscall_abi as abi; let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_KERNEL_ROOT)) }; let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_USER_TTBR)) }; let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_TRAP_TTBR)) }; let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_ACTIVE)) }; let registers = os_k1120_registers(); if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null() || registers.cpu != 0 || registers.current_el != 4 || registers.daif & 128 == 0 || registers.irq_active || registers.ttbr != kernel_root || registers.spsel != 1 || trap_ttbr != user_ttbr { park(); } let episode = unsafe { &mut *(active as *mut OsK1210SEl0Episode) }; let table = unsafe { &mut *episode.table.cast::>() }; let launch = table.current_launch(episode.pid).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1210S-TRACE: DISPATCH-OWNER"); let shell = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_INIT_PID)) } .unwrap_or_else(|| park()); if episode.case == 0 && episode.pid != shell || episode.case == 1 && table.record(episode.pid).map(|r| r.parent) != Ok(Some(shell)) { park(); } if launch != episode.launch || launch.kernel_root != kernel_root || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr { park(); } let address = pointer as u64; if address < launch.kernel_stack_low || address.checked_add(os_k920_exception::FRAME_BYTES as u64) .is_none_or(|end| end > launch.kernel_stack_high) || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64) { park(); } let frame = unsafe { &mut *pointer }; if frame.validate_header().is_err() || frame.vector_slot != 8 || frame.spsr & 0x3ff != launch.user_spsr || frame.sp_el0 < 0x80003000 || frame.sp_el0 > launch.user_sp || frame.sp_el0 & 15 != 0 || episode.case == 0 && frame.sp_el0 != launch.user_sp || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64 || episode.outcome.is_some() { park(); } let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY; abi_frame.x = frame.x; if frame.esr != abi::SVC_ESR || frame.x[7] != abi::ABI_VERSION { park(); } let call = abi::decode_call(&abi_frame).unwrap_or_else(|_| park()); write_str("TURKOS/OS/K1210S-TRACE: SVC=0x"); write_hex(frame.x[8] as usize); write_line(""); if call == abi::Call::Role { if episode.calls != 0 || frame.elr != launch.user_pc + 12 { park(); } frame.x[0] = episode.case as u64; episode.calls = 1; write_line("TURKOS/OS/K1210S-TRACE: ROLE-RETURNED"); return 0; } if episode.case == 0 { if let abi::Call::RunChild(1) = call { if episode.calls != 1 || frame.elr != launch.user_pc + 0x78 { park(); } return os_k1210s_run_child(active as *mut OsK1210SEl0Episode, pointer, kernel_root); } if call == abi::Call::WaitAgain(1) { if episode.calls != 2 || episode.waits != 0 || frame.elr != launch.user_pc + 0x9c { park(); } let child = episode.child.unwrap_or_else(|| park()); let before = table.snapshot(); if table.wait_child_outcome(child) != Err(process::Error::NoChild) || table.snapshot() != before { park(); } episode.waits = 1; frame.x[0] = abi::AbiResult::Err(abi::Errno::Child).encode(); write_line("TURKOS/OS/K1210S-TRACE: SHELL-SECOND-WAIT-ECHILD"); return 0; } if call == abi::Call::Version { if episode.calls != 2 || episode.waits != 1 || frame.elr != launch.user_pc + 0xb8 { park(); } frame.x[0] = abi::ABI_VERSION; write_line("TURKOS/OS/K1210S-TRACE: SHELL-SERVICE-AFTER-PANIC"); return 0; } if call != abi::Call::Exit(0) || episode.calls != 2 || episode.waits != 1 || frame.elr != launch.user_pc + 0xf4 { park(); } let data = launch.data_frame as *const u64; if unsafe { read_volatile(data) } != 127 || unsafe { read_volatile(data.add(1)) } != 1 || unsafe { read_volatile(data.add(2)) } != 1 || unsafe { read_volatile(data.add(3)) } != 0 || table.snapshot().published != 1 { park(); } write_line("TURKOS/OS/K1210S-TRACE: SHELL-PRIVATE-RESULTS-VERIFIED"); } else if call != abi::Call::Exit(127) || episode.calls != 1 || frame.elr != launch.user_pc + 0xf4 { park(); } let code = if episode.case == 0 { 0 } else { 127 }; if table.exit_current(code).is_err() { park(); } if episode.case == 0 { write_line("TURKOS/OS/K1210S-TRACE: SHELL-EXIT-0"); } else { write_line("TURKOS/OS/K1210S-TRACE: RUST-PANIC-EXIT-127"); } let outcome = process::Outcome::Exited(code); if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome)) || table.snapshot().zombies != 1 { park(); } episode.outcome = Some(outcome); 1 // Never ERET after process termination. } fn os_k1210s_run_child( parent_ptr: *mut OsK1210SEl0Episode, frame_ptr: *mut os_k920_exception::Frame, kernel_root: u64, ) -> u64 { write_line("TURKOS/OS/K1210S-TRACE: RUN-CHILD-ENTRY"); use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_table as process; let parent = unsafe { &mut *parent_ptr }; let table_ptr = parent.table.cast::>(); let child = { let table = unsafe { &mut *table_ptr }; let snapshot = table.snapshot(); write_str("TURKOS/OS/K1210S-TRACE: PARENT PUBLISHED=0x"); write_hex(snapshot.published); write_str(" ZOMBIES=0x"); write_hex(snapshot.zombies); write_str(" STATE=0x"); write_hex(table.record(parent.pid).map(|r| r.resources.state as usize) .unwrap_or(usize::MAX)); write_line(""); if table.snapshot().published != 1 || table.snapshot().zombies != 0 || table.record(parent.pid).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Running) { park(); } write_line("TURKOS/OS/K1210S-TRACE: CHILD-OWNER-CHECKED"); let request = process::Spawn {handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP}; let ticket = table.begin(request).unwrap_or_else(|_| park()); write_line("TURKOS/OS/K1210S-TRACE: CHILD-BEGIN"); let ticket = table.prepare(ticket).ok().unwrap_or_else(|| park()); write_line("TURKOS/OS/K1210S-TRACE: CHILD-PREPARED"); let child = table.publish(ticket).ok().unwrap_or_else(|| park()); write_line("TURKOS/OS/K1210S-TRACE: CHILD-PUBLISH-COMMITTED"); if table.record(child).map(|r| r.parent) != Ok(Some(parent.pid)) || table.snapshot().published != 2 { park(); } child }; parent.child = Some(child); write_line("TURKOS/OS/K1210S-TRACE: SHELL-CHILD-PUBLISHED"); let saved_active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_ACTIVE)) }; let saved_user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_USER_TTBR)) }; let saved_trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_TRAP_TTBR)) }; let saved_frame = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_KERNEL_FRAME)) }; if saved_active != parent_ptr as u64 || saved_frame == 0 || saved_trap_ttbr != saved_user_ttbr { park(); } let stack_top = { let table = unsafe { &mut *table_ptr }; if unsafe { table.release_actual_current_to_ready() }.is_err() { park(); } table.landing_stack_from_suspended_parent(parent.pid, child) .unwrap_or_else(|_| park()).1 }; unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210S_CASE), 1); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_RESULT), None); } unsafe extern "C" { fn __os_k1210s_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1210s_enter_el0_process_stack( parent.table, (&child as *const process::ProcessId).cast(), stack_top, kernel_root) }; if status != 0 { park(); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210S_ACTIVE), saved_active); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_USER_TTBR), saved_user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_TRAP_TTBR), saved_trap_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_KERNEL_FRAME), saved_frame); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_CASE), 0); } let table = unsafe { &mut *table_ptr }; if unsafe { table.register_actual_landing(parent.pid) }.is_err() { park(); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_RESULT)) } .unwrap_or_else(|| park()); if observed != process::Outcome::Exited(127) || table.record(child).map(|r| r.outcome) != Ok(Some(observed)) || table.record(parent.pid).map(|r| r.resources.state) != Ok(turkos_kernel::runtime::process_threads::State::Running) || table.snapshot().zombies != 1 { park(); } let outcome = table.wait_child_outcome(child).unwrap_or_else(|_| park()); if outcome != observed || table.snapshot().published != 1 || table.snapshot().zombies != 0 { park(); } let frame = unsafe { &mut *frame_ptr }; frame.x[0] = 127; frame.x[1] = 0; parent.calls = 2; write_line("TURKOS/OS/K1210S-TRACE: SHELL-RESUMES-AFTER-PANIC"); 0 } #[unsafe(no_mangle)] unsafe extern "C" fn __os_k1210s_el0_process_body( table: *mut u8, pid: *const u8, kernel_root: u64, ) -> u64 { write_line("TURKOS/OS/K1210S-TRACE: BODY-ENTRY"); use turkos_kernel::runtime::process_table as process; let table_ptr = table.cast::>(); let pid = unsafe { *pid.cast::() }; if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); } write_line("TURKOS/OS/K1210S-TRACE: BODY-LANDED"); let result = (|| -> Result<(), &'static str> { let launch = unsafe { &*table_ptr }.current_launch(pid) .map_err(|_| "exit-launch-owner")?; if launch.kernel_root != kernel_root || launch.user_asid == 0 || launch.user_root == kernel_root || launch.user_root == 0 { return Err("exit-launch-root"); } let case = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_CASE)) } as usize; if case >= 2 { return Err("exit-case"); } let old_vbar: u64; unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar, options(nomem, nostack, preserves_flags)); } let mut episode = OsK1210SEl0Episode {table, pid, launch, case, calls: 0, waits: 0, child: None, old_vbar, outcome: None}; let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210S_KERNEL_ROOT), kernel_root); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_USER_TTBR), user_ttbr); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_KERNEL_FRAME), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_ACTIVE), (&mut episode as *mut OsK1210SEl0Episode) as u64); } unsafe extern "C" { fn __os_k1210s_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64); } unsafe { __os_k1210s_enter_el0(user_ttbr, launch.user_pc, launch.user_sp, launch.user_spsr); } write_line("TURKOS/OS/K1210S-TRACE: BODY-USER-RETURNED"); unsafe { asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar, options(nostack, preserves_flags)); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_ACTIVE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_KERNEL_ROOT), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_USER_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_TRAP_TTBR), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_KERNEL_FRAME), 0); } write_str("TURKOS/OS/K1210S-TRACE: RETURN OUTCOME=0x"); write_hex(episode.outcome.is_some() as usize); write_str(" TTBR=0x"); write_hex(os_k1120_registers().ttbr as usize); write_str(" ROOT=0x"); write_hex(kernel_root as usize); write_str(" RECORD=0x"); write_hex((unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) == Ok(episode.outcome)) as usize); write_line(""); if episode.outcome.is_none() || os_k1120_registers().ttbr != kernel_root || unsafe { &*table_ptr }.record(pid).map(|r| r.outcome) != Ok(episode.outcome) { return Err("exit-completion-owner"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210S_RESULT), episode.outcome); } Ok(()) })(); // exit_current/fault_current already made the thread Dead and cleared current. match result { Ok(()) => 0, Err(error) => { unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210S_EL0_ERROR), Some(error)); } 1 } } } static mut OS_K1210S_CASE: u64 = 0; static mut OS_K1210S_RESULT: Option = None; #[inline(never)] fn os_k1210s_el0_probe( memory: &mut OsK660Memory, tables: &mmu::Tables, ) -> Result<(), &'static str> { use turkos_kernel::runtime::address_spaces as spaces; use turkos_kernel::runtime::process_guest as guest; use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage}; use turkos_kernel::runtime::process_table as process; let before = memory.allocator.os_authority().audit() .map_err(|_| "exit-before-inventory")?; let hardware = OsK1120Hardware { tables }; let registers = os_k1120_registers(); let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64; let binding = guest::StackBinding { owner: guard, low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16, spsel: registers.spsel }; let gate = unsafe { guest::Gate::new(&hardware, binding) } .map_err(|_| "shell-primary-gate")?; let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut()); let store = guest::FrameStore::new(0x1210_5, &authority, &gate) .map_err(|_| "shell-frame-store")?; let frame_baseline = store.snapshot().map_err(|_| "shell-frame-baseline")?; let trusted = unsafe { TrustedImage::from_kernel(&OS_K1210S_USER_ELF, OS_K1210S_USER_ELF.len()) }.map_err(|_| "shell-trusted-image")?; let recipe = Recipe::new(trusted, &[b"runtime", b"panic"], &[]) .map_err(|_| "shell-image-recipe")?; let image = OsK1120PreparedImage(recipe); let config = spaces::Configuration::from_registers(5, registers.tcr) .map_err(|_| "shell-space-config")?; let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?; write_line("TURKOS/OS/K1210S-TRACE: TABLE-INSTALLED"); let table = unsafe { &mut *table_ptr }; let request = process::Spawn { handles: 3, code_entry: guest::CODE, user_stack_top: guest::STACK_TOP }; let reservation = table.begin(request).map_err(|_| "shell-begin")?; let prepared = table.prepare(reservation).map_err(|_| "shell-prepare")?; let shell = table.publish(prepared).map_err(|_| "shell-publish")?; write_line("TURKOS/OS/K1210S-TRACE: SHELL-PUBLISHED"); unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210S_CASE), 0); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_RESULT), None); write_volatile(core::ptr::addr_of_mut!(OS_K1210S_INIT_PID), Some(shell)); } let (_, stack_top) = table.landing_stack(shell).map_err(|_| "shell-stack")?; unsafe extern "C" { fn __os_k1210s_enter_el0_process_stack( table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64; } let status = unsafe { __os_k1210s_enter_el0_process_stack( table_ptr.cast(), (&shell as *const process::ProcessId).cast(), stack_top, tables.root) }; write_line("TURKOS/OS/K1210S-TRACE: SHELL-RETURNED"); if status != 0 { return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_EL0_ERROR)) } .unwrap_or("shell-process-body")); } let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1210S_RESULT)) } .ok_or("shell-missing-result")?; if observed != process::Outcome::Exited(0) || table.record(shell).map(|r| r.outcome) != Ok(Some(observed)) || table.snapshot().zombies != 1 || table.snapshot().published != 1 { return Err("shell-zombie-result"); } if table.wait_child_outcome(shell) != Ok(process::Outcome::Exited(0)) || table.snapshot() != process::Snapshot::default() { return Err("shell-reap-result"); } unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1210S_INIT_PID), None); } let backend = os_k1105_finish_table(table_ptr)?; if backend.inventory().map_err(|_| "shell-owner-inventory")? != [0, 0, 0, before.live_pages] { return Err("shell-owner-leak"); } drop(backend); if gate.faulted() || store.snapshot() != Ok(frame_baseline) || authority.observe(&gate, |allocator| allocator.audit()) .map_err(|_| "shell-after-gate")? .map_err(|_| "shell-after-inventory")? != before { return Err("shell-physical-rollback"); } write_line("TURKOS/OS/K1210S-EL0: SHELL-CHILD-PANIC127 WAIT-ONCE ECHILD VERSION1 SHELL-EXIT0 OWNER=RELEASED"); write_line("TURKOS/OS/K1210: PASS"); Ok(()) } global_asm!(include_str!("os_k1215a_vectors.S"), frame_bytes=const os_k920_exception::FRAME_BYTES, sp_offset=const os_k920_exception::FRAME_SP_OFFSET, sp_el0_offset=const os_k920_exception::FRAME_SP_EL0_OFFSET, elr_offset=const os_k920_exception::FRAME_ELR_OFFSET, spsr_offset=const os_k920_exception::FRAME_SPSR_OFFSET, esr_offset=const os_k920_exception::FRAME_ESR_OFFSET, far_offset=const os_k920_exception::FRAME_FAR_OFFSET, slot_offset=const os_k920_exception::FRAME_SLOT_OFFSET, q_offset=const os_k920_exception::FRAME_Q_OFFSET, fpcr_offset=const os_k920_exception::FRAME_FPCR_OFFSET, fpsr_offset=const os_k920_exception::FRAME_FPSR_OFFSET, current_el_offset=const os_k920_exception::FRAME_CURRENT_EL_OFFSET, canary_offset=const os_k920_exception::FRAME_CANARY_OFFSET, padding_offset=const os_k920_exception::FRAME_PADDING_OFFSET, ); static OS_K1215A_USER_ELF: &[u8] = include_bytes!("k1215-user.elf"); static OS_K1575_STAT_USER_ELF: &[u8] = include_bytes!("k1575-stat-user.elf"); static OS_K1580_DUP_USER_ELF: &[u8] = include_bytes!("k1580-dup-user.elf"); static OS_K1220_HOST_MACHO: [u8; 466064] = *include_bytes!("k1220-host.macho"); include!("k1215-alloc-guest.rs");