// Mode 29: the trusted kernel trap publishes two children for an EL0 parent
// immediately before its requested exit. INIT later runs and reaps both.
static mut OS_K1195O_EL0_ERROR: Option<&'static str> = None;
#[unsafe(no_mangle)] static mut OS_K1195O_ACTIVE: u64 = 0;
#[unsafe(no_mangle)] static mut OS_K1195O_KERNEL_ROOT: u64 = 0;
#[unsafe(no_mangle)] static mut OS_K1195O_USER_TTBR: u64 = 0;
#[unsafe(no_mangle)] static mut OS_K1195O_TRAP_TTBR: u64 = 0;
#[unsafe(no_mangle)] static mut OS_K1195O_KERNEL_FRAME: u64 = 0;
static mut OS_K1195O_INIT_PID: Option<turkos_kernel::runtime::process_table::ProcessId> = None;
static mut OS_K1195O_ORIGINAL_PARENT: Option<turkos_kernel::runtime::process_table::ProcessId> = None;
static mut OS_K1195O_ORPHANS: [Option<turkos_kernel::runtime::process_table::ProcessId>; 2] = [None; 2];

struct OsK1195OEl0Episode {
    table: *mut u8,
    pid: turkos_kernel::runtime::process_table::ProcessId,
    launch: turkos_kernel::runtime::process_table::GuestLaunch,
    case: usize,
    calls: u8,
    waits: u8,
    children: [Option<turkos_kernel::runtime::process_table::ProcessId>; 3],
    old_vbar: u64,
    outcome: Option<turkos_kernel::runtime::process_table::Outcome>,
}

#[inline(never)]
fn os_k1195o_el0_on_work_stack(
    memory: &mut OsK660Memory, tables: &mmu::Tables,
) -> Result<(), &'static str> {
    let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64;
    let low = guard + 4096;
    let end = low + 131072;
    let before = os_k1120_registers();
    if before.sp >= guard && before.sp < end
        || before.cpu != 0 || before.daif & 128 == 0
        || before.current_el != 4 || before.irq_active
        || before.ttbr != tables.root
    { return Err("exit-work-stack-context"); }
    let (slot, descriptor) = unsafe {
        tables.os_k1105_acquire_work_guard(guard, end,
            core::ptr::addr_of!(_image_start) as u64,
            core::ptr::addr_of!(_image_end) as u64)
    }.map_err(|_| "exit-work-stack-guard")?;
    unsafe {
        write_volatile(low as *mut u64, OS_K1105_LOW_CANARY);
        write_volatile((end - 8) as *mut u64, OS_K1105_HIGH_CANARY);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_EL0_ERROR), None);
    }
    unsafe extern "C" {
        fn __os_k1195o_el0_on_stack(memory: *mut u8, tables: *const u8) -> u64;
    }
    let status = unsafe { __os_k1195o_el0_on_stack(
        (memory as *mut OsK660Memory).cast(),
        (tables as *const mmu::Tables).cast()) };
    let after = os_k1120_registers();
    let canaries = unsafe { read_volatile(low as *const u64) == OS_K1105_LOW_CANARY
        && read_volatile((end - 8) as *const u64) == OS_K1105_HIGH_CANARY };
    unsafe { tables.os_k1105_restore_work_guard(guard, end, slot, descriptor) }
        .map_err(|_| "exit-work-stack-restore")?;
    if after.sp != before.sp || after.spsel != before.spsel
        || after.ttbr != before.ttbr || after.tcr != before.tcr
        || after.sctlr != before.sctlr || after.daif != before.daif
        || !canaries { return Err("exit-work-stack-return"); }
    if status != 0 {
        return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_EL0_ERROR)) }
            .unwrap_or("exit-work-stack-body"));
    }
    Ok(())
}

#[unsafe(no_mangle)]
unsafe extern "C" fn __os_k1195o_el0_body(memory: *mut u8, tables: *const u8) -> u64 {
    let result = os_k1195o_el0_probe(
        unsafe { &mut *memory.cast::<OsK660Memory>() },
        unsafe { &*tables.cast::<mmu::Tables>() });
    match result {
        Ok(()) => 0,
        Err(error) => {
            unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_EL0_ERROR), Some(error)); }
            1
        }
    }
}

#[unsafe(no_mangle)]
extern "C" fn os_k1195o_dispatch(pointer: *mut os_k920_exception::Frame) -> u64 {
    write_line("TURKOS/OS/K1195O-TRACE: DISPATCH-ENTRY");
    use turkos_kernel::runtime::process_table as process;
    use turkos_kernel::runtime::syscall_abi as abi;
    let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_KERNEL_ROOT)) };
    let user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_USER_TTBR)) };
    let trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_TRAP_TTBR)) };
    let active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ACTIVE)) };
    let registers = os_k1120_registers();
    if kernel_root == 0 || user_ttbr == 0 || active == 0 || pointer.is_null()
        || registers.cpu != 0 || registers.current_el != 4
        || registers.daif & 128 == 0 || registers.irq_active
        || registers.ttbr != kernel_root || registers.spsel != 1
        || trap_ttbr != user_ttbr
    {
        write_str("TURKOS/OS/K1195O-TRACE: DISPATCH-GATE-ERROR ROOT=0x");
        write_hex(kernel_root as usize);
        write_str(" USER=0x"); write_hex(user_ttbr as usize);
        write_str(" TRAP=0x"); write_hex(trap_ttbr as usize);
        write_str(" ACTIVE=0x"); write_hex(active as usize);
        write_str(" TTBR=0x"); write_hex(registers.ttbr as usize);
        write_str(" SPSEL=0x"); write_hex(registers.spsel as usize);
        write_line("");
        park();
    }
    write_line("TURKOS/OS/K1195O-TRACE: DISPATCH-GATE-PASSED");
    let episode = unsafe { &mut *(active as *mut OsK1195OEl0Episode) };
    let table = unsafe { &mut *episode.table.cast::<OsK1105Table<'_>>() };
    let launch = table.current_launch(episode.pid).unwrap_or_else(|error| {
        write_str("TURKOS/OS/K1195O-TRACE: DISPATCH-LAUNCH-ERROR=0x");
        write_hex(error as usize);
        write_line("");
        park()
    });
    write_line("TURKOS/OS/K1195O-TRACE: DISPATCH-OWNER");
    let init_pid = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_INIT_PID)) }
        .unwrap_or_else(|| park());
    if episode.case == 0 && episode.pid != init_pid
        || episode.case != 0 && table.record(episode.pid).map(|r| r.parent)
            != Ok(Some(init_pid))
    { park(); }
    write_line("TURKOS/OS/K1195O-TRACE: OWNER-CHECK-PASSED");
    if launch != episode.launch || launch.kernel_root != kernel_root
        || launch.user_root | ((launch.user_asid as u64) << 48) != user_ttbr
    { park(); }
    write_line("TURKOS/OS/K1195O-TRACE: LAUNCH-CHECK-PASSED");
    let address = pointer as u64;
    if address < launch.kernel_stack_low
        || address.checked_add(os_k920_exception::FRAME_BYTES as u64)
            .is_none_or(|end| end > launch.kernel_stack_high)
        || !address.is_multiple_of(os_k920_exception::FRAME_ALIGNMENT as u64)
    { park(); }
    write_line("TURKOS/OS/K1195O-TRACE: FRAME-BOUNDS-PASSED");
    let frame = unsafe { &mut *pointer };
    write_str("TURKOS/OS/K1195O-TRACE: FRAME ELR=0x");
    write_hex(frame.elr as usize);
    write_str(" ESR=0x");
    write_hex(frame.esr as usize);
    write_str(" FAR=0x");
    write_hex(frame.far as usize);
    write_line("");
    if frame.validate_header().is_err() || frame.vector_slot != 8
        || frame.spsr & 0x3ff != launch.user_spsr
        || frame.sp_el0 != launch.user_sp
        || frame.interrupted_sp != address + os_k920_exception::FRAME_BYTES as u64
        || episode.outcome.is_some()
    { park(); }
    if episode.case == 2 && episode.calls == 1 {
        write_line("TURKOS/OS/K1195O-TRACE: FAULT-BRANCH");
        if episode.calls != 1 || frame.elr != launch.user_pc + 284
        { park(); }
        if frame.esr >> 26 & 0x3f != 0x24
            || frame.far != 0x9000_0000 || frame.x[0] != 0x9000_0000
        { park(); }
        write_line("TURKOS/OS/K1195O-TRACE: FAULT-HEADER-PASSED");
        let fault = process::Fault::from_lower_el_data_abort(frame.esr, frame.far)
            .unwrap_or_else(|_| park());
        write_line("TURKOS/OS/K1195O-TRACE: FAULT-TYPED");
        if let Err(error) = table.fault_current(fault) {
            write_str("TURKOS/OS/K1195O-TRACE: FAULT-EXIT-ERROR=0x");
            write_hex(error as usize);
            write_line("");
            park();
        }
        write_line("TURKOS/OS/K1195O-TRACE: ACTUAL-EL0-FAR-DATA-ABORT");
        let outcome = process::Outcome::Fault(fault);
        if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome))
            || table.snapshot().zombies != 1
        { park(); }
        episode.outcome = Some(outcome);
        return 1;
    }
    let mut abi_frame = turkos_os_exceptions::exceptions::Frame::EMPTY;
    abi_frame.x = frame.x;
    write_str("TURKOS/OS/K1195O-TRACE: NEXT-SVC ELR=0x");
    write_hex(frame.elr as usize);
    write_str(" CALL=0x");
    write_hex(frame.x[8] as usize);
    write_line("");
    if frame.esr != abi::SVC_ESR || frame.x[7] != abi::ABI_VERSION
    { park(); }
    let call = abi::decode_call(&abi_frame).unwrap_or_else(|_| park());
    if call == abi::Call::Role {
        if episode.calls != 0 || frame.elr != launch.user_pc + 12
        { park(); }
        frame.x[0] = episode.case as u64;
        episode.calls = 1;
        write_line("TURKOS/OS/K1195O-TRACE: EL0-ROLE");
        return 0;
    }
    if let abi::Call::RunChild(role) = call {
        write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-DECODED");
        if episode.case != 0 || !(1..=3).contains(&role)
            || role != episode.calls as u64
            || episode.waits as u64 != role - 1
            || frame.elr != launch.user_pc + [0, 76, 124, 192][role as usize]
        { park(); }
        return os_k1195o_run_child(active as *mut OsK1195OEl0Episode,
            pointer, role, kernel_root);
    }
    if let abi::Call::WaitAgain(role) = call {
        if episode.case != 0 || !(1..=3).contains(&role)
            || episode.calls as u64 != role + 1
            || episode.waits as u64 != role - 1
            || frame.elr != launch.user_pc + [0, 100, 168, 220][role as usize]
        { park(); }
        let child = episode.children[(role - 1) as usize]
            .unwrap_or_else(|| park());
        let before = table.snapshot();
        if table.wait_child_outcome(child) != Err(process::Error::NoChild)
            || table.snapshot() != before
            || table.record(child) != Err(process::Error::Stale)
        { park(); }
        episode.waits += 1;
        frame.x[0] = abi::AbiResult::Err(abi::Errno::Child).encode();
        write_line("TURKOS/OS/K1195O-TRACE: SECOND-WAIT-ECHILD");
        return 0;
    }
    let expected = match episode.case {
        0 if episode.calls == 4 && episode.waits == 3 => 0,
        1 if episode.calls == 1 => 0,
        3 if episode.calls == 1 => 42,
        _ => park(),
    };
    let exit_offset = match episode.case {0 => 256, 1 => 272, 3 => 304, _ => park()};
    if call != abi::Call::Exit(expected)
        || frame.elr != launch.user_pc + exit_offset
    { park(); }
    if episode.case == 0 {
        let data = launch.data_frame as *const u64;
        if unsafe { read_volatile(data) } != 0
            || unsafe { read_volatile(data.add(1)) } != 1
            || unsafe { read_volatile(data.add(2)) } != 0x9000_0000
            || unsafe { read_volatile(data.add(3)) } != 42
            || unsafe { read_volatile(data.add(4)) } != 1
            || unsafe { read_volatile(data.add(5)) } != 3
            || table.snapshot().published != 1
            || episode.children.iter().any(Option::is_none)
            || episode.children[0] == episode.children[1]
            || episode.children[1] == episode.children[2]
            || episode.children[0] == episode.children[2]
            || table.next_orphan(episode.pid) != Err(process::Error::NoChild)
        { park(); }
        write_line("TURKOS/OS/K1195O-TRACE: EL0-INIT-PRIVATE-RESULTS-VERIFIED");
    }
    if episode.case == 1 {
        use turkos_kernel::runtime::process_guest as guest;
        write_line("TURKOS/OS/K1195O-TRACE: PARENT-SPAWN-ENTRY");
        if table.snapshot().published != 2 || table.snapshot().zombies != 0
            || table.record(episode.pid).map(|r| r.parent) != Ok(Some(init_pid))
        { park(); }
        write_line("TURKOS/OS/K1195O-TRACE: PARENT-SPAWN-GATE");
        let spawn = process::Spawn {handles: 3,
            code_entry: guest::CODE, user_stack_top: guest::STACK_TOP};
        let mut orphans = [None; 2];
        for slot in &mut orphans {
            let ticket = table.begin(spawn).unwrap_or_else(|_| park());
            write_line("TURKOS/OS/K1195O-TRACE: ORPHAN-BEGIN");
            let ticket = match table.prepare(ticket) {
                Ok(ticket) => ticket,
                Err((error, _)) => {
                    write_str("TURKOS/OS/K1195O-TRACE: ORPHAN-PREPARE-ERROR=0x");
                    write_hex(error as usize);
                    write_str(" STAGE=0x");
                    write_hex(unsafe { read_volatile(core::ptr::addr_of!(guest::OS_K1195O_PREP_STAGE)) } as usize);
                    write_line("");
                    park()
                }
            };
            write_line("TURKOS/OS/K1195O-TRACE: ORPHAN-PREPARE");
            let child = table.publish(ticket).ok().unwrap_or_else(|| park());
            write_line("TURKOS/OS/K1195O-TRACE: ORPHAN-PUBLISH");
            if table.record(child).map(|r| (r.parent, r.orphaned_from))
                != Ok((Some(episode.pid), None))
            { park(); }
            *slot = Some(child);
        }
        if orphans[0] == orphans[1] || table.snapshot().published != 4
        { park(); }
        unsafe {
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ORIGINAL_PARENT), Some(episode.pid));
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ORPHANS), orphans);
        }
        write_line("TURKOS/OS/K1195O-TRACE: TWO-CHILDREN-PUBLISHED-UNDER-EL0-PARENT");
    }
    if table.exit_current(expected as i32).is_err() { park(); }
    if episode.case == 1 {
        let orphans = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ORPHANS)) };
        for child in orphans.into_iter().flatten() {
            if table.record(child).map(|r| (r.parent, r.orphaned_from))
                != Ok((Some(init_pid), Some(episode.pid)))
            { park(); }
        }
        write_line("TURKOS/OS/K1195O-TRACE: EL0-PARENT-EXIT-ADOPTED-TWO");
    } else if episode.case == 3 {
        write_line("TURKOS/OS/K1195O-TRACE: CHILD-EXIT-42-AFTER-FAULT");
    } else {
        write_line("TURKOS/OS/K1195O-TRACE: INIT-EXIT-0");
    }
    let outcome = process::Outcome::Exited(expected as i32);
    if table.record(episode.pid).map(|r| r.outcome) != Ok(Some(outcome))
        || table.snapshot().zombies != 1
    { park(); }
    episode.outcome = Some(outcome);
    1 // Never ERET after process termination.
}

fn os_k1195o_run_child(
    parent_ptr: *mut OsK1195OEl0Episode,
    frame_ptr: *mut os_k920_exception::Frame,
    role: u64,
    kernel_root: u64,
) -> u64 {
    use turkos_kernel::runtime::process_guest as guest;
    use turkos_kernel::runtime::process_table as process;
    let parent = unsafe { &mut *parent_ptr };
    let table_ptr = parent.table.cast::<OsK1105Table<'_>>();
    write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-ENTRY");
    let child = {
        let table = unsafe { &mut *table_ptr };
        if table.snapshot().published != (if role == 1 { 1 } else { 5 - role as usize })
            || table.snapshot().zombies != 0
            || table.record(parent.pid).map(|r| r.resources.state)
                != Ok(turkos_kernel::runtime::process_threads::State::Running)
        { park(); }
        if role == 1 {
            let spawn = process::Spawn {handles: 3,
                code_entry: guest::CODE, user_stack_top: guest::STACK_TOP};
            let ticket = table.begin(spawn).unwrap_or_else(|_| park());
            let ticket = table.prepare(ticket).ok().unwrap_or_else(|| park());
            let child = table.publish(ticket).ok().unwrap_or_else(|| park());
            if table.record(child).map(|r| r.parent) != Ok(Some(parent.pid))
                || child == parent.pid || table.snapshot().published != 2
            { park(); }
            child
        } else {
            let orphans = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ORPHANS)) };
            let original = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ORIGINAL_PARENT)) }
                .unwrap_or_else(|| park());
            let child = orphans[role as usize - 2].unwrap_or_else(|| park());
            if table.next_orphan(parent.pid) != Ok(child)
                || table.record(child).map(|r| (r.parent, r.orphaned_from))
                    != Ok((Some(parent.pid), Some(original)))
                || table.record(original) != Err(process::Error::Stale)
            { park(); }
            write_line("TURKOS/OS/K1195O-TRACE: INIT-OLDEST-ORPHAN-SELECTED");
            child
        }
    };
    write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-PUBLISHED");
    parent.children[(role - 1) as usize] = Some(child);
    {
        let table = unsafe { &mut *table_ptr };
        let before = table.snapshot();
        let child_before = table.record(child).unwrap_or_else(|_| park());
        if table.wait_child_outcome(child) != Err(process::Error::Busy)
            || table.snapshot() != before
            || table.record(child) != Ok(child_before)
        { park(); }
    }
    write_line("TURKOS/OS/K1195O-TRACE: BEFORE-EXIT-WAIT-BUSY");
    let saved_active = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_ACTIVE)) };
    let saved_user_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_USER_TTBR)) };
    let saved_trap_ttbr = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_TRAP_TTBR)) };
    let saved_frame = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_KERNEL_FRAME)) };
    if saved_active != parent_ptr as u64 || saved_frame == 0
        || saved_trap_ttbr != saved_user_ttbr
    { park(); }
    write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-SAVED");
    let stack_top = {
        let table = unsafe { &mut *table_ptr };
        if unsafe { table.release_actual_current_to_ready() }.is_err() { park(); }
        write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-YIELDED");
        table.landing_stack_from_suspended_parent(parent.pid, child)
            .unwrap_or_else(|_| park()).1
    };
    write_line("TURKOS/OS/K1195O-TRACE: RUN-CHILD-STACK");
    unsafe {
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_CASE), role);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_RESULT), None);
    }
    unsafe extern "C" {
        fn __os_k1195o_enter_el0_process_stack(
            table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64;
    }
    write_line("TURKOS/OS/K1195O-TRACE: CHILD-LAUNCH");
    let status = unsafe { __os_k1195o_enter_el0_process_stack(
        parent.table, (&child as *const process::ProcessId).cast(),
        stack_top, kernel_root) };
    if status != 0 {
        write_str("TURKOS/OS/K1195O-TRACE: CHILD-BODY-ERROR=");
        write_line(unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_EL0_ERROR)) }
            .unwrap_or("unknown"));
        park();
    }
    unsafe {
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ACTIVE), saved_active);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_ROOT), kernel_root);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_USER_TTBR), saved_user_ttbr);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_TRAP_TTBR), saved_trap_ttbr);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_FRAME), saved_frame);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_CASE), 0);
    }
    let table = unsafe { &mut *table_ptr };
    if unsafe { table.register_actual_landing(parent.pid) }.is_err() { park(); }
    let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_RESULT)) }
        .unwrap_or_else(|| park());
    if table.record(child).map(|r| r.outcome) != Ok(Some(observed))
        || table.record(parent.pid).map(|r| r.resources.state)
            != Ok(turkos_kernel::runtime::process_threads::State::Running)
        || table.snapshot().zombies != 1
        || table.snapshot().published != 5 - role as usize
    { park(); }
    if role == 2 {
        let before = table.snapshot();
        if table.wait_child(child) != Err(process::Error::State)
            || table.snapshot() != before
        { park(); }
        write_line("TURKOS/OS/K1195O-TRACE: FAULT-NOT-SUCCESS");
    }
    let result = table.wait_child_outcome(child).unwrap_or_else(|_| park());
    if result != observed || table.snapshot().published != 4 - role as usize
        || table.snapshot().zombies != 0
    { park(); }
    if role == 1 {
        let before = table.snapshot();
        let first_orphan = table.next_orphan(parent.pid).unwrap_or_else(|_| park());
        let spawn = process::Spawn { handles: 3,
            code_entry: guest::CODE, user_stack_top: guest::STACK_TOP };
        let ticket = table.begin(spawn).unwrap_or_else(|_| park());
        let ticket = table.prepare(ticket).ok().unwrap_or_else(|| park());
        let replacement = table.publish(ticket).ok().unwrap_or_else(|| park());
        if replacement.slot() != child.slot()
            || replacement.generation() == child.generation()
            || table.record(child) != Err(process::Error::Stale)
            || table.record(replacement).map(|r| r.parent) != Ok(Some(parent.pid))
            || table.next_orphan(parent.pid) != Ok(first_orphan)
        { park(); }
        table.terminate_child(replacement, 7).unwrap_or_else(|_| park());
        if table.wait_child_outcome(replacement) != Ok(process::Outcome::Exited(7))
            || table.snapshot() != before
            || table.record(replacement) != Err(process::Error::Stale)
            || table.next_orphan(parent.pid) != Ok(first_orphan)
        { park(); }
        write_line("TURKOS/OS/K1195O-TRACE: PARENT-PID-SLOT-REUSED-ORPHANS-UNCHANGED");
    }
    let frame = unsafe { &mut *frame_ptr };
    match (role, result) {
        (1, process::Outcome::Exited(0)) => {
            frame.x[0] = 0; frame.x[1] = 0; frame.x[2] = 0;
        }
        (2, process::Outcome::Fault(fault)) if fault.far() == 0x9000_0000 => {
            frame.x[0] = 0; frame.x[1] = 1; frame.x[2] = fault.far();
        }
        (3, process::Outcome::Exited(42)) => {
            frame.x[0] = 42; frame.x[1] = 0; frame.x[2] = 0;
        }
        _ => park(),
    }
    parent.calls = role as u8 + 1;
    write_line("TURKOS/OS/K1195O-TRACE: EL0-INIT-WAIT-RESULT");
    0
}

#[unsafe(no_mangle)]
unsafe extern "C" fn __os_k1195o_el0_process_body(
    table: *mut u8, pid: *const u8, kernel_root: u64,
) -> u64 {
    use turkos_kernel::runtime::process_table as process;
    let table_ptr = table.cast::<OsK1105Table<'_>>();
    let pid = unsafe { *pid.cast::<process::ProcessId>() };
    write_line("TURKOS/OS/K1195O-TRACE: BODY-ENTRY");
    if unsafe { (&mut *table_ptr).register_actual_landing(pid) }.is_err() { park(); }
    write_line("TURKOS/OS/K1195O-TRACE: BODY-LANDED");
    let result = (|| -> Result<(), &'static str> {
        let launch = unsafe { &*table_ptr }.current_launch(pid)
            .map_err(|error| {
                write_str("TURKOS/OS/K1195O-TRACE: BODY-LAUNCH-ERROR=0x");
                write_hex(error as usize);
                write_line("");
                "exit-launch-owner"
            })?;
        write_line("TURKOS/OS/K1195O-TRACE: BODY-LAUNCH-READY");
        if launch.kernel_root != kernel_root || launch.user_asid == 0
            || launch.user_root == kernel_root || launch.user_root == 0
        { return Err("exit-launch-root"); }
        let case = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_CASE)) } as usize;
        write_str("TURKOS/OS/K1195O-TRACE: BODY-CASE=0x");
        write_hex(case);
        write_line("");
        if case >= 4 { return Err("exit-case"); }
        let old_vbar: u64;
        unsafe { asm!("mrs {vbar}, VBAR_EL1", vbar=out(reg) old_vbar,
            options(nomem, nostack, preserves_flags)); }
        let mut episode = OsK1195OEl0Episode {table, pid, launch, case,
            calls: 0, waits: 0, children: [None; 3], old_vbar, outcome: None};
        let user_ttbr = launch.user_root | ((launch.user_asid as u64) << 48);
        unsafe {
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_ROOT), kernel_root);
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_USER_TTBR), user_ttbr);
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_TRAP_TTBR), 0);
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_FRAME), 0);
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ACTIVE),
                (&mut episode as *mut OsK1195OEl0Episode) as u64);
        }
        unsafe extern "C" {
            fn __os_k1195o_enter_el0(user_ttbr: u64, pc: u64, sp: u64, spsr: u64);
        }
        unsafe { __os_k1195o_enter_el0(user_ttbr, launch.user_pc, launch.user_sp,
            launch.user_spsr); }
        write_line("TURKOS/OS/K1195O-TRACE: BODY-EL0-RETURNED");
        unsafe {
            asm!("msr VBAR_EL1, {vbar}", "isb", vbar=in(reg) episode.old_vbar,
                options(nostack, preserves_flags));
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ACTIVE), 0);
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_ROOT), 0);
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_USER_TTBR), 0);
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_TRAP_TTBR), 0);
            write_volatile(core::ptr::addr_of_mut!(OS_K1195O_KERNEL_FRAME), 0);
        }
        if episode.outcome.is_none() || os_k1120_registers().ttbr != kernel_root
            || unsafe { &*table_ptr }.record(pid).map(|r| r.outcome)
                != Ok(episode.outcome)
        { return Err("exit-completion-owner"); }
        unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_RESULT),
            episode.outcome); }
        Ok(())
    })();
    // exit_current/fault_current already made the thread Dead and cleared current.
    match result {
        Ok(()) => 0,
        Err(error) => {
            unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_EL0_ERROR), Some(error)); }
            1
        }
    }
}

static mut OS_K1195O_CASE: u64 = 0;
static mut OS_K1195O_RESULT: Option<turkos_kernel::runtime::process_table::Outcome> = None;

#[inline(never)]
fn os_k1195o_el0_probe(
    memory: &mut OsK660Memory, tables: &mmu::Tables,
) -> Result<(), &'static str> {
    use turkos_kernel::runtime::address_spaces as spaces;
    use turkos_kernel::runtime::process_guest as guest;
    use turkos_kernel::runtime::process_image_recipe::{Recipe, TrustedImage};
    use turkos_kernel::runtime::process_table as process;
    let before = memory.allocator.os_authority().audit()
        .map_err(|_| "exit-before-inventory")?;
    let hardware = OsK1120Hardware { tables };
    let registers = os_k1120_registers();
    let guard = core::ptr::addr_of!(OS_K1105_WORK_STACK) as u64;
    let binding = guest::StackBinding { owner: guard,
        low: guard + 4096 + 16, high: guard + 4096 + 131072 - 16,
        spsel: registers.spsel };
    let gate = unsafe { guest::Gate::new(&hardware, binding) }
        .map_err(|_| "init-primary-gate")?;
    let authority = guest::AuthorityCell::new(memory.allocator.os_authority_mut());
    let store = guest::FrameStore::new(0x1186, &authority, &gate)
        .map_err(|_| "init-frame-store")?;
    let frame_baseline = store.snapshot().map_err(|_| "init-frame-baseline")?;
    let trusted = unsafe { TrustedImage::from_kernel(&OS_K1195O_USER_ELF,
        OS_K1195O_USER_ELF.len()) }.map_err(|_| "init-trusted-image")?;
    let recipe = Recipe::new(trusted, &[b"init"], &[])
        .map_err(|_| "init-image-recipe")?;
    let image = OsK1120PreparedImage(recipe);
    let config = spaces::Configuration::from_registers(5, registers.tcr)
        .map_err(|_| "init-space-config")?;
    let table_ptr = os_k1105_install_table(&gate, &authority, &store, &image, config)?;
    let table = unsafe { &mut *table_ptr };
    let request = process::Spawn { handles: 3,
        code_entry: guest::CODE, user_stack_top: guest::STACK_TOP };
    let reservation = table.begin(request).map_err(|_| "init-begin")?;
    let prepared = table.prepare(reservation).map_err(|_| "init-prepare")?;
    let pid = table.publish(prepared).map_err(|_| "init-publish")?;
    unsafe {
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_CASE), 0);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_RESULT), None);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_INIT_PID), Some(pid));
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ORIGINAL_PARENT), None);
        write_volatile(core::ptr::addr_of_mut!(OS_K1195O_ORPHANS), [None; 2]);
    }
    let (_, stack_top) = table.landing_stack(pid).map_err(|_| "init-stack")?;
    unsafe extern "C" {
        fn __os_k1195o_enter_el0_process_stack(
            table: *mut u8, pid: *const u8, stack_top: u64, kernel_root: u64) -> u64;
    }
    let status = unsafe { __os_k1195o_enter_el0_process_stack(
        table_ptr.cast(), (&pid as *const process::ProcessId).cast(),
        stack_top, tables.root) };
    if status != 0 {
        return Err(unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_EL0_ERROR)) }
            .unwrap_or("init-process-body"));
    }
    let observed = unsafe { read_volatile(core::ptr::addr_of!(OS_K1195O_RESULT)) }
        .ok_or("init-missing-result")?;
    if observed != process::Outcome::Exited(0)
        || table.record(pid).map(|r| r.outcome) != Ok(Some(observed))
        || table.snapshot().zombies != 1 || table.snapshot().published != 1
    { return Err("init-zombie-result"); }
    if table.wait_child_outcome(pid) != Ok(process::Outcome::Exited(0))
        || table.snapshot() != process::Snapshot::default()
    { return Err("init-reap-result"); }
    if table.wait_child_outcome(pid) != Err(process::Error::NoChild)
        || table.snapshot() != process::Snapshot::default()
    { return Err("init-second-wait"); }
    unsafe { write_volatile(core::ptr::addr_of_mut!(OS_K1195O_INIT_PID), None); }
    let backend = os_k1105_finish_table(table_ptr)?;
    if backend.inventory().map_err(|_| "init-owner-inventory")?
        != [0, 0, 0, before.live_pages]
    { return Err("init-owner-leak"); }
    drop(backend);
    if gate.faulted() || store.snapshot() != Ok(frame_baseline)
        || authority.observe(&gate, |allocator| allocator.audit())
            .map_err(|_| "init-after-gate")?
            .map_err(|_| "init-after-inventory")? != before
    { return Err("init-physical-rollback"); }
    write_line("TURKOS/OS/K1195O-EL0: PARENT-EXIT=ADOPTED-2 INIT-ORDER=2 FAULT=0x90000000 OWNER=RELEASED");
    write_line("TURKOS/OS/K1195: PASS");
    Ok(())
}