//! Isolated live EL0 syscall ABI experiment. The normal image still parks.
use super::*;
#[cfg(feature = "ipc-user-test")]
use turkos_kernel::ipc::{self, SharedChannel, UserSendError};
use turkos_kernel::syscall::{self, Call, Outcome};
#[cfg(feature = "ipc-user-test")]
use turkos_kernel::task::{Scheduler, TaskId};
#[cfg(any(feature = "user-copy-test", feature = "ipc-user-test"))]
use turkos_kernel::user_copy::{self, UserPages};

const EL0T_MASKED: usize = 0x3c0;
const EL0T_IRQ_ENABLED: usize = 0x340;
const GUARD: u64 = 0x5455_524b_4952_5147;
static mut USER_ROOT: u64 = 0;
static mut USER_CALLS: usize = 0;
#[cfg(feature = "user-el0-irq-test")]
static mut EL0_IRQ_INTID: usize = 0;
#[cfg(feature = "user-el0-irq-test")]
static mut EL0_IRQ_SEEN: usize = 0;
#[cfg(feature = "user-clock-test")]
static mut LAST_CLOCK: u64 = 0;
#[cfg(feature = "user-clock-test")]
static mut CLOCK_FREQUENCY: u64 = 0;
#[cfg(any(feature = "user-copy-test", feature = "ipc-user-test"))]
static mut USER_FRAMES: [u64; 3] = [0; 3];
#[cfg(feature = "ipc-user-test")]
static mut IPC_OWNERS: [TaskId; 4] = [TaskId {
    slot: 0,
    generation: 0,
}; 4];
#[cfg(feature = "ipc-user-test")]
static mut IPC_CHANNEL: Option<SharedChannel> = None;
#[cfg(feature = "user-copy-irq-test")]
static mut COPY_IRQ_SEEN: usize = 0;
#[cfg(feature = "user-copy-irq-test")]
static mut COPY_PHASE: usize = 0;
#[cfg(feature = "user-copy-irq-test")]
static mut COPY_FIRST: usize = 0;

unsafe extern "C" {
    static __syscall_blob_start: u8;
    static __syscall_blob_end: u8;
    #[cfg(feature = "user-el0-irq-test")]
    static __el0_irq_blob_start: u8;
    #[cfg(feature = "user-el0-irq-test")]
    static __el0_irq_blob_end: u8;
}

fn fail(reason: &str) -> ! {
    write_str("TURKOS/SYSCALL-ERROR: ");
    write_line(reason);
    park()
}

#[cfg(feature = "user-el0-irq-test")]
pub(super) fn set_el0_irq_intid(intid: usize) {
    if !(16..32).contains(&intid) {
        fail("EL0 timer PPI outside private range");
    }
    unsafe { write_volatile(core::ptr::addr_of_mut!(EL0_IRQ_INTID), intid) };
}

#[cfg(feature = "user-el0-irq-test")]
fn el0_irq_probe(frame: *mut irq::IrqContext) -> usize {
    let context = unsafe { &*frame };
    let stack_top = core::ptr::addr_of!(__irq_stack_top) as usize;
    let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) };
    let root: u64;
    unsafe {
        asm!("mrs {value}, TTBR0_EL1", value = out(reg) root, options(nomem, nostack, preserves_flags))
    };
    let expected_root = unsafe { read_volatile(core::ptr::addr_of!(USER_ROOT)) };
    let data = unsafe { read_volatile(core::ptr::addr_of!(USER_FRAMES))[1] };
    let valid = frame as usize == stack_top - core::mem::size_of::<irq::IrqContext>()
        && guard == GUARD
        && context.vector_slot == 9
        && context.spsr & 0x8f == 0
        && context.sp_el0 == mmu::USER_STACK_TOP
        && context.elr >= mmu::USER_CODE_VA
        && context.elr < mmu::USER_CODE_VA + memory::PAGE_SIZE
        && context.x[10] == mmu::USER_DATA_VA
        && context.x[19] == 0x1460
        && context.x[20] == 0x2460
        && context.x[21] == 0x3460
        && root == expected_root
        && unsafe { read_volatile(data as *const u32) } == 0
        && unsafe { read_volatile(core::ptr::addr_of!(EL0_IRQ_SEEN)) } == 0;
    let iar: usize;
    unsafe {
        asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags))
    };
    let expected_intid = unsafe { read_volatile(core::ptr::addr_of!(EL0_IRQ_INTID)) };
    if !valid || irq::classify_iar(iar as u64, expected_intid as u32) != irq::IarClass::Expected {
        write_line("TURKOS/EL0-IRQ-ERROR: frame, timer source or user context");
        return 0;
    }
    unsafe {
        asm!("msr CNTP_CTL_EL0, {zero}", "isb", zero = in(reg) 0usize, options(nomem, nostack));
        write_volatile(data as *mut u32, 1);
        asm!("dsb ishst", "msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nostack));
        let control: usize;
        asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags));
        if control & 2 != 0 {
            asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack));
        }
        write_volatile(core::ptr::addr_of_mut!(EL0_IRQ_SEEN), 1);
    }
    write_str("TURKOS/EL0-IRQ: SLOT=0x09 IAR=0x");
    write_hex(iar);
    write_str(" ELR=0x");
    write_hex(context.elr as usize);
    write_str(" SPSR=0x");
    write_hex(context.spsr as usize);
    write_str(" SP_EL0=0x");
    write_hex(context.sp_el0 as usize);
    write_str(" ROOT=0x");
    write_hex(root as usize);
    write_str(" X19=0x");
    write_hex(context.x[19] as usize);
    write_str(" X20=0x");
    write_hex(context.x[20] as usize);
    write_str(" X21=0x");
    write_hex(context.x[21] as usize);
    write_str(" FLAG=0x1\r\n");
    1
}

#[cfg(feature = "user-el0-irq-test")]
fn el0_irq_resume(frame: *mut irq::IrqContext) -> ! {
    let context = unsafe { &*frame };
    let esr: u64;
    let root: u64;
    unsafe {
        asm!("mrs {value}, ESR_EL1", value = out(reg) esr, options(nomem, nostack, preserves_flags));
        asm!("mrs {value}, TTBR0_EL1", value = out(reg) root, options(nomem, nostack, preserves_flags));
    }
    let data = unsafe { read_volatile(core::ptr::addr_of!(USER_FRAMES))[1] };
    if context.vector_slot != 8
        || context.spsr & 0xf != 0
        || (esr >> 26) & 0x3f != 0x15
        || esr & 0xffff != 0x42
        || context.sp_el0 != mmu::USER_STACK_TOP
        || context.x[8] != 0x146
        || context.x[19] != 0x1460
        || context.x[20] != 0x2460
        || context.x[21] != 0x3460
        || root != unsafe { read_volatile(core::ptr::addr_of!(USER_ROOT)) }
        || unsafe { read_volatile(data as *const u32) } != 1
        || unsafe { read_volatile(core::ptr::addr_of!(EL0_IRQ_SEEN)) } != 1
    {
        fail("EL0 timer IRQ did not restore full user context");
    }
    write_str("TURKOS/EL0-IRQ-RESUME: SLOT=0x08 X19=0x");
    write_hex(context.x[19] as usize);
    write_str(" X20=0x");
    write_hex(context.x[20] as usize);
    write_str(" X21=0x");
    write_hex(context.x[21] as usize);
    write_str(" ESR=0x");
    write_hex(esr as usize);
    write_line(" FLAG=0x1 PASS");
    park()
}

#[cfg(feature = "user-copy-irq-test")]
fn copy_four_across_irq(pages: UserPages, address: u64, output: &mut [u8]) -> Result<usize, i64> {
    if output.len() != 4 || address != mmu::USER_DATA_VA {
        fail("IRQ copy contract");
    }
    // Validate the *complete* user range before copying even its first half.
    pages.resolve(address, output.len(), false)?;
    if unsafe { pages.copy_from(address, &mut output[..2]) }? != 2 {
        fail("IRQ copy first half");
    }
    unsafe {
        write_volatile(
            core::ptr::addr_of_mut!(COPY_FIRST),
            u16::from_le_bytes([output[0], output[1]]) as usize,
        );
        write_volatile(core::ptr::addr_of_mut!(COPY_PHASE), 1);
    }
    let mpidr: u64;
    unsafe {
        asm!("mrs {value}, MPIDR_EL1", value = out(reg) mpidr, options(nomem, nostack, preserves_flags))
    };
    let command = irq::sgi1r_for_mpidr(mpidr, 1).unwrap_or_else(|| fail("IRQ SGI affinity"));
    let observed: usize;
    let outer = (core::ptr::addr_of!(__irq_stack_top) as usize
        - core::mem::size_of::<irq::IrqContext>()) as *const irq::IrqContext;
    let before = unsafe { read_volatile(outer) };
    let gp20: u64;
    let gp21: u64;
    let gp22: u64;
    let simd0: u64;
    let simd31: u64;
    // IRQ is unmasked only inside this bounded interval. The pending SGI
    // interrupts the active copy syscall with its outer EL0 frame still live.
    unsafe {
        asm!(
            "movi v0.16b, #0x5a",
            "movi v31.16b, #0xa5",
            "msr ICC_SGI1R_EL1, {sgi}",
            "isb",
            "msr daifclr, #2",
            "2:",
            "ldr {seen}, [{flag}]",
            "cbnz {seen}, 3f",
            "subs {budget}, {budget}, #1",
            "b.ne 2b",
            "3:",
            "msr daifset, #2",
            "umov {simd0}, v0.d[0]",
            "umov {simd31}, v31.d[0]",
            sgi = in(reg) command,
            flag = in(reg) core::ptr::addr_of!(COPY_IRQ_SEEN) as usize,
            seen = out(reg) observed,
            budget = inout(reg) 50_000_000usize => _,
            inout("x20") 0x5350u64 => gp20,
            inout("x21") 0x5351u64 => gp21,
            inout("x22") 0x5352u64 => gp22,
            simd0 = lateout(reg) simd0, simd31 = lateout(reg) simd31,
            out("v0") _, out("v31") _,
            options(nostack)
        );
    }
    let after = unsafe { read_volatile(outer) };
    if (gp20, gp21, gp22) != (0x5350, 0x5351, 0x5352)
        || simd0 != 0x5a5a_5a5a_5a5a_5a5a
        || simd31 != 0xa5a5_a5a5_a5a5_a5a5
        || before.x != after.x
        || before.q != after.q
        || before.fpcr != after.fpcr
        || before.fpsr != after.fpsr
        || before.elr != after.elr
        || before.spsr != after.spsr
        || before.sp_el0 != after.sp_el0
        || before.interrupted_sp != after.interrupted_sp
    {
        fail("IRQ syscall GP/SIMD or outer context changed");
    }
    if observed != 1 || unsafe { read_volatile(core::ptr::addr_of!(COPY_PHASE)) } != 2 {
        fail("IRQ did not resume half copy");
    }
    if unsafe { pages.copy_from(address + 2, &mut output[2..]) }? != 2 {
        fail("IRQ copy second half");
    }
    if output != b"ABCD" {
        fail("IRQ copy not complete");
    }
    unsafe { write_volatile(core::ptr::addr_of_mut!(COPY_PHASE), 3) };
    write_str("TURKOS/USER-COPY-IRQ-RESUME: BYTES=0x");
    write_hex(output.len());
    write_str(" DATA=0x");
    write_hex(u32::from_le_bytes(output.try_into().unwrap()) as usize);
    write_str("\r\n");
    write_line("TURKOS/USER-COPY-IRQ-CONTEXT: GP=3 SIMD=2 OUTER=UNMODIFIED PASS");
    Ok(output.len())
}

#[cfg(feature = "user-copy-irq-test")]
fn copy_irq_probe(frame: *const irq::IrqContext) -> usize {
    let context = unsafe { &*frame };
    let outer_base =
        core::ptr::addr_of!(__irq_stack_top) as usize - core::mem::size_of::<irq::IrqContext>();
    let stack_start = core::ptr::addr_of!(__irq_stack_start) as usize;
    let first = unsafe { read_volatile(core::ptr::addr_of!(COPY_FIRST)) };
    let phase = unsafe { read_volatile(core::ptr::addr_of!(COPY_PHASE)) };
    let seen = unsafe { read_volatile(core::ptr::addr_of!(COPY_IRQ_SEEN)) };
    let count = unsafe { read_volatile(core::ptr::addr_of!(USER_CALLS)) };
    let text_start = core::ptr::addr_of!(_text_start) as u64;
    let text_end = core::ptr::addr_of!(_text_end) as u64;
    let current_sp: usize;
    unsafe {
        asm!("mov {value}, sp", value = out(reg) current_sp, options(nomem, nostack, preserves_flags))
    };
    let valid = context.vector_slot == 5
        && context.x[20] == 0x5350
        && context.x[21] == 0x5351
        && context.x[22] == 0x5352
        && context.q[0] == [0x5a5a_5a5a_5a5a_5a5a; 2]
        && context.q[31] == [0xa5a5_a5a5_a5a5_a5a5; 2]
        && context.spsr & 0x8f == 5
        && frame as usize >= stack_start + 64
        && irq::handler_sp_in_stack(stack_start, frame as usize, current_sp)
        && frame as usize + core::mem::size_of::<irq::IrqContext>() + 32
            == context.interrupted_sp as usize
        && (context.interrupted_sp as usize) < outer_base
        && first == 0x4241
        && context.elr >= text_start
        && context.elr < text_end
        && context.elr & 3 == 0
        && unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) } == GUARD
        && count == 4
        && phase == 1
        && seen == 0;
    let iar: usize;
    unsafe {
        asm!("mrs {value}, ICC_IAR1_EL1", value = out(reg) iar, options(nomem, nostack, preserves_flags))
    };
    if !valid || irq::classify_iar(iar as u64, 1) != irq::IarClass::Expected {
        write_line("TURKOS/USER-COPY-IRQ-ERROR: nested frame or source");
        return 0;
    }
    unsafe {
        asm!("msr ICC_EOIR1_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack));
        let control: usize;
        asm!("mrs {value}, ICC_CTLR_EL1", value = out(reg) control, options(nomem, nostack, preserves_flags));
        if control & 2 != 0 {
            asm!("msr ICC_DIR_EL1, {value}", "isb", value = in(reg) iar, options(nomem, nostack));
        }
        write_volatile(core::ptr::addr_of_mut!(COPY_PHASE), 2);
        write_volatile(core::ptr::addr_of_mut!(COPY_IRQ_SEEN), 1);
    }
    write_str("TURKOS/USER-COPY-IRQ-MID: IAR=0x");
    write_hex(iar);
    write_str(" PHASE=0x");
    write_hex(phase);
    write_str(" FIRST=0x");
    write_hex(first);
    write_str(" FRAME=0x");
    write_hex(frame as usize);
    write_str(" OUTER=0x");
    write_hex(outer_base);
    write_str("\r\n");
    1
}

pub(super) fn smoke_test(
    allocator: &mut memory::FrameAllocator,
    tables: &mmu::Tables,
    layout: mmu::Layout,
) -> ! {
    let user = unsafe { tables.build_user_space(allocator, layout.ram_base, layout.ram_size) }
        .unwrap_or_else(|_| fail("private user root"));
    if !user.contract_valid(tables.root) {
        fail("user address-space contract");
    }
    #[cfg(feature = "ipc-user-test")]
    {
        let mut scheduler = Scheduler::new();
        let owners = [
            scheduler.spawn().unwrap_or_else(|_| fail("IPC producer A")),
            scheduler.spawn().unwrap_or_else(|_| fail("IPC producer B")),
            scheduler.spawn().unwrap_or_else(|_| fail("IPC consumer C")),
            scheduler.spawn().unwrap_or_else(|_| fail("IPC consumer D")),
        ];
        let channel = SharedChannel::new([owners[0], owners[1]], [owners[2], owners[3]])
            .unwrap_or_else(|_| fail("IPC channel owners"));
        // Single CPU, DAIF-masked test image. These IDs bind the one EL0
        // address space to producer A; C observes from EL1 only.
        unsafe {
            write_volatile(core::ptr::addr_of_mut!(IPC_OWNERS), owners);
            core::ptr::write(core::ptr::addr_of_mut!(IPC_CHANNEL), Some(channel));
        }
    }
    #[cfg(not(feature = "user-el0-irq-test"))]
    let start = core::ptr::addr_of!(__syscall_blob_start) as usize;
    #[cfg(not(feature = "user-el0-irq-test"))]
    let end = core::ptr::addr_of!(__syscall_blob_end) as usize;
    #[cfg(feature = "user-el0-irq-test")]
    let start = core::ptr::addr_of!(__el0_irq_blob_start) as usize;
    #[cfg(feature = "user-el0-irq-test")]
    let end = core::ptr::addr_of!(__el0_irq_blob_end) as usize;
    let length = end.checked_sub(start).unwrap_or(0);
    if length == 0 || length > memory::PAGE_SIZE as usize || length & 3 != 0 {
        fail("user blob length");
    }
    unsafe {
        core::ptr::copy_nonoverlapping(start as *const u8, user.code_frame() as *mut u8, length);
    }
    let ctr: usize;
    unsafe {
        asm!("mrs {value}, CTR_EL0", value = out(reg) ctr, options(nomem, nostack, preserves_flags))
    };
    let dline = 4usize << ((ctr >> 16) & 15);
    for offset in (0..length).step_by(dline) {
        unsafe {
            asm!("dc cvau, {addr}", addr = in(reg) (user.code_frame() as usize + offset), options(nostack))
        };
    }
    unsafe { asm!("dsb ish", "ic iallu", "dsb ish", "isb", options(nostack)) };
    unsafe {
        write_volatile(core::ptr::addr_of_mut!(USER_ROOT), user.root);
        #[cfg(any(feature = "user-copy-test", feature = "ipc-user-test"))]
        write_volatile(
            core::ptr::addr_of_mut!(USER_FRAMES),
            [user.code_frame(), user.data_frame(), user.stack_frame()],
        );
        write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), GUARD);
        asm!(
            "dsb ishst", "msr TTBR0_EL1, {root}", "isb",
            "tlbi vmalle1is", "dsb ish", "isb",
            root = in(reg) user.root,
            options(nostack)
        );
    }
    write_str("TURKOS/SYSCALL-ABI: VERSION=0x");
    write_hex(syscall::VERSION as usize);
    write_str(" ROOT=0x");
    write_hex(user.root as usize);
    write_str(" CODE=0x");
    write_hex(mmu::USER_CODE_VA as usize);
    write_str("\r\n");
    write_line("TURKOS/SYSCALL-ENTRY: EL0");
    #[cfg(feature = "user-el0-irq-test")]
    {
        if unsafe { read_volatile(user.data_frame() as *const u32) } != 0
            || unsafe { read_volatile(core::ptr::addr_of!(EL0_IRQ_INTID)) } == 0
        {
            fail("EL0 timer IRQ precondition");
        }
        write_str("TURKOS/EL0-IRQ-ARM: PPI=0x");
        write_hex(unsafe { read_volatile(core::ptr::addr_of!(EL0_IRQ_INTID)) });
        write_line(" TICKS=0x0000000000075300");
        unsafe {
            asm!(
                "msr CNTP_TVAL_EL0, {ticks}", "msr CNTP_CTL_EL0, {enable}", "isb",
                ticks = in(reg) 480_000usize, enable = in(reg) 1usize,
                options(nomem, nostack)
            );
        }
    }
    unsafe {
        asm!(
            "msr SP_EL0, {stack}", "msr ELR_EL1, {entry}",
            "msr SPSR_EL1, {mode}", "isb", "eret",
            stack = in(reg) mmu::USER_STACK_TOP,
            entry = in(reg) mmu::USER_CODE_VA,
            mode = in(reg) if cfg!(feature = "user-el0-irq-test") { EL0T_IRQ_ENABLED } else { EL0T_MASKED },
            options(noreturn)
        )
    }
}

#[cfg_attr(feature = "user-el0-irq-test", allow(unreachable_code))]
#[unsafe(no_mangle)]
pub extern "C" fn kernel_return_probe(frame: *mut irq::IrqContext) -> usize {
    if frame.is_null() {
        fail("null exception frame");
    }
    #[cfg(feature = "user-el0-irq-test")]
    {
        match unsafe { (*frame).vector_slot } {
            9 => return el0_irq_probe(frame),
            8 => el0_irq_resume(frame),
            _ => fail("unexpected EL0 IRQ experiment vector"),
        }
    }
    #[cfg(feature = "user-copy-irq-test")]
    if unsafe { (*frame).vector_slot } == 5 {
        return copy_irq_probe(frame);
    }
    let frame = unsafe { &mut *frame };
    let esr: u64;
    let root: u64;
    unsafe {
        asm!("mrs {value}, ESR_EL1", value = out(reg) esr, options(nomem, nostack, preserves_flags));
        asm!("mrs {value}, TTBR0_EL1", value = out(reg) root, options(nomem, nostack, preserves_flags));
    }
    let count = unsafe { read_volatile(core::ptr::addr_of!(USER_CALLS)) };
    let expected_root = unsafe { read_volatile(core::ptr::addr_of!(USER_ROOT)) };
    let guard = unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) };
    let expected_calls = if cfg!(feature = "user-copy-test") {
        15
    } else if cfg!(feature = "ipc-user-test") {
        13
    } else if cfg!(any(
        feature = "user-clock-test",
        feature = "user-mmio-request-test"
    )) {
        8
    } else {
        5
    };
    if count >= expected_calls
        || guard != GUARD
        || frame.vector_slot != 8
        || (esr >> 26) & 0x3f != 0x15
        || esr & 0xffff != 0x42
        || frame.spsr & 0xf != 0
        || frame.sp_el0 != mmu::USER_STACK_TOP
        || root != expected_root
        || frame.elr < mmu::USER_CODE_VA
        || frame.elr >= mmu::USER_CODE_VA + memory::PAGE_SIZE
        || frame.elr & 3 != 0
    {
        write_str("TURKOS/SYSCALL-DIAG: COUNT=0x");
        write_hex(count);
        write_str(" SLOT=0x");
        write_hex(frame.vector_slot as usize);
        write_str(" ESR=0x");
        write_hex(esr as usize);
        write_str(" SPSR=0x");
        write_hex(frame.spsr as usize);
        write_str(" SP=0x");
        write_hex(frame.sp_el0 as usize);
        write_str(" TTBR=0x");
        write_hex(root as usize);
        write_str(" EXPECTED=0x");
        write_hex(expected_root as usize);
        write_str(" ELR=0x");
        write_hex(frame.elr as usize);
        write_str(" GUARD=0x");
        write_hex(guard as usize);
        write_str(" X0=0x");
        write_hex(frame.x[0] as usize);
        write_str(" X1=0x");
        write_hex(frame.x[1] as usize);
        write_str(" X8=0x");
        write_hex(frame.x[8] as usize);
        write_str("\r\n");
        fail("EL0 exception record or guarded frame");
    }
    let call = Call {
        version: frame.x[7],
        number: frame.x[8],
        args: frame.x[..6].try_into().unwrap(),
    };
    let expected = match count {
        0 => (syscall::VERSION, syscall::ABI_QUERY, Outcome::Return(1)),
        1 => (syscall::VERSION, syscall::ECHO_SIX, Outcome::Return(21)),
        2 => (syscall::VERSION, 0x7fff, Outcome::Return(syscall::ENOSYS)),
        3 => (2, syscall::ABI_QUERY, Outcome::Return(syscall::EINVAL)),
        #[cfg(feature = "user-copy-test")]
        4 => (syscall::VERSION, syscall::READ_USER, Outcome::Return(4)),
        #[cfg(feature = "user-copy-test")]
        5 => (syscall::VERSION, syscall::WRITE_USER, Outcome::Return(4)),
        #[cfg(feature = "user-copy-test")]
        6 => (syscall::VERSION, syscall::READ_USER, Outcome::Return(0)),
        #[cfg(feature = "user-copy-test")]
        7 => (
            syscall::VERSION,
            syscall::READ_USER,
            Outcome::Return(user_copy::EFAULT),
        ),
        #[cfg(feature = "user-copy-test")]
        8 => (
            syscall::VERSION,
            syscall::WRITE_USER,
            Outcome::Return(user_copy::EFAULT),
        ),
        #[cfg(feature = "user-copy-test")]
        9 => (
            syscall::VERSION,
            syscall::WRITE_USER,
            Outcome::Return(user_copy::EFAULT),
        ),
        #[cfg(feature = "user-copy-test")]
        10 => (
            syscall::VERSION,
            syscall::READ_USER,
            Outcome::Return(user_copy::EFAULT),
        ),
        #[cfg(feature = "user-copy-test")]
        11 => (syscall::VERSION, syscall::SERIAL_WRITE, Outcome::Return(4)),
        #[cfg(feature = "user-copy-test")]
        12 => (
            syscall::VERSION,
            syscall::SERIAL_WRITE,
            Outcome::Return(user_copy::EFAULT),
        ),
        #[cfg(feature = "user-copy-test")]
        13 => (
            syscall::VERSION,
            syscall::SERIAL_WRITE,
            Outcome::Return(user_copy::EINVAL),
        ),
        #[cfg(feature = "user-copy-test")]
        14 => (syscall::VERSION, syscall::EXIT, Outcome::Exit(0)),
        #[cfg(feature = "ipc-user-test")]
        4 => (syscall::VERSION, syscall::IPC_SEND, Outcome::Return(1)),
        #[cfg(feature = "ipc-user-test")]
        5 => (syscall::VERSION, syscall::IPC_INSPECT, Outcome::Return(4)),
        #[cfg(feature = "ipc-user-test")]
        6 | 8 => (
            syscall::VERSION,
            syscall::IPC_SEND,
            Outcome::Return(user_copy::EFAULT),
        ),
        #[cfg(feature = "ipc-user-test")]
        7 | 9 | 11 => (syscall::VERSION, syscall::IPC_INSPECT, Outcome::Return(0)),
        #[cfg(feature = "ipc-user-test")]
        10 => (
            syscall::VERSION,
            syscall::IPC_SEND,
            Outcome::Return(user_copy::EINVAL),
        ),
        #[cfg(feature = "ipc-user-test")]
        12 => (syscall::VERSION, syscall::EXIT, Outcome::Exit(0)),
        #[cfg(feature = "user-clock-test")]
        4..=6 => (syscall::VERSION, syscall::CLOCK_READ, Outcome::ClockRead),
        #[cfg(feature = "user-clock-test")]
        7 => (syscall::VERSION, syscall::EXIT, Outcome::Exit(0)),
        #[cfg(feature = "user-mmio-request-test")]
        4..=6 => (
            syscall::VERSION,
            syscall::MMIO_MAP,
            Outcome::Return(syscall::EPERM),
        ),
        #[cfg(feature = "user-mmio-request-test")]
        7 => (syscall::VERSION, syscall::EXIT, Outcome::Exit(0)),
        #[cfg(not(any(
            feature = "user-copy-test",
            feature = "ipc-user-test",
            feature = "user-clock-test",
            feature = "user-mmio-request-test"
        )))]
        4 => (syscall::VERSION, syscall::EXIT, Outcome::Exit(0)),
        _ => unreachable!(),
    };
    if call.version != expected.0
        || call.number != expected.1
        || (count == 1 && call.args != [1, 2, 3, 4, 5, 6])
        || (cfg!(feature = "user-mmio-request-test")
            && (4..=6).contains(&count)
            && call.args
                != [
                    if count == 6 {
                        0xffff
                    } else {
                        (count - 3) as u64
                    },
                    0,
                    0,
                    0,
                    0,
                    0,
                ])
    {
        fail("syscall register ABI or call order");
    }
    #[cfg(feature = "user-copy-test")]
    if matches!(call.number, syscall::READ_USER | syscall::WRITE_USER) {
        write_str("TURKOS/USER-COPY-REQ: INDEX=0x");
        write_hex(count);
        write_str(" PTR=0x");
        write_hex(call.args[0] as usize);
        write_str(" LEN=0x");
        write_hex(call.args[1] as usize);
        write_str(" WRITE=0x");
        write_hex((call.number == syscall::WRITE_USER) as usize);
        write_str("\r\n");
    }
    #[cfg(feature = "user-copy-test")]
    if call.number == syscall::SERIAL_WRITE {
        write_str("TURKOS/USER-SERIAL-REQ: INDEX=0x");
        write_hex(count);
        write_str(" PTR=0x");
        write_hex(call.args[0] as usize);
        write_str(" LEN=0x");
        write_hex(call.args[1] as usize);
        write_str("\r\n");
    }
    #[cfg(feature = "user-copy-test")]
    let result = if call.version == syscall::VERSION
        && matches!(
            call.number,
            syscall::READ_USER | syscall::WRITE_USER | syscall::SERIAL_WRITE
        ) {
        let frames = unsafe { read_volatile(core::ptr::addr_of!(USER_FRAMES)) };
        let pages = UserPages {
            code_frame: frames[0],
            data_frame: frames[1],
            stack_frame: frames[2],
        };
        let address = call.args[0];
        let length = usize::try_from(call.args[1]).unwrap_or(usize::MAX);
        if matches!(call.number, syscall::READ_USER | syscall::SERIAL_WRITE) {
            let mut scratch = [0u8; user_copy::MAX_COPY];
            if length > scratch.len() {
                Outcome::Return(user_copy::EINVAL)
            } else {
                // SAFETY: all three physical frames belong to the still-live
                // private UserSpace and are identity mapped. Scratch is local.
                let copied = if cfg!(feature = "user-copy-irq-test") && count == 4 {
                    #[cfg(feature = "user-copy-irq-test")]
                    {
                        copy_four_across_irq(pages, address, &mut scratch[..length])
                    }
                    #[cfg(not(feature = "user-copy-irq-test"))]
                    {
                        unreachable!()
                    }
                } else {
                    unsafe { pages.copy_from(address, &mut scratch[..length]) }
                };
                match copied {
                    Ok(bytes) if call.number == syscall::SERIAL_WRITE => {
                        match user_copy::serial_text(&scratch[..bytes]) {
                            Ok(text) => {
                                write_str("TURKOS/USER-SERIAL: ");
                                write_line(text);
                                Outcome::Return(bytes as i64)
                            }
                            Err(error) => {
                                write_str("TURKOS/USER-SERIAL-REJECT: INDEX=0x");
                                write_hex(count);
                                write_str(" ERROR=0x");
                                write_hex(error as usize);
                                write_str("\r\n");
                                Outcome::Return(error)
                            }
                        }
                    }
                    Ok(bytes) => {
                        let checksum: usize =
                            scratch[..bytes].iter().map(|byte| *byte as usize).sum();
                        write_str("TURKOS/USER-COPY-FROM: BYTES=0x");
                        write_hex(bytes);
                        write_str(" SUM=0x");
                        write_hex(checksum);
                        write_str("\r\n");
                        Outcome::Return(bytes as i64)
                    }
                    Err(error) => {
                        if call.number == syscall::SERIAL_WRITE {
                            write_str("TURKOS/USER-SERIAL-REJECT: INDEX=0x");
                            write_hex(count);
                            write_str(" ERROR=0x");
                            write_hex(error as usize);
                            write_str("\r\n");
                        }
                        Outcome::Return(error)
                    }
                }
            }
        } else {
            let source = b"WXYZ";
            if length > source.len() {
                Outcome::Return(user_copy::EINVAL)
            } else {
                // SAFETY: the destination page is owned and mapped; the
                // literal does not alias it. resolve checks the whole range.
                match unsafe { pages.copy_to(address, &source[..length]) } {
                    Ok(bytes) => {
                        write_str("TURKOS/USER-COPY-TO: BYTES=0x");
                        write_hex(bytes);
                        write_str("\r\n");
                        Outcome::Return(bytes as i64)
                    }
                    Err(error) => Outcome::Return(error),
                }
            }
        }
    } else {
        syscall::dispatch(call)
    };
    #[cfg(feature = "ipc-user-test")]
    let result = if call.version == syscall::VERSION && call.number == syscall::IPC_SEND {
        let frames = unsafe { read_volatile(core::ptr::addr_of!(USER_FRAMES)) };
        let owners = unsafe { read_volatile(core::ptr::addr_of!(IPC_OWNERS)) };
        let pages = UserPages {
            code_frame: frames[0],
            data_frame: frames[1],
            stack_frame: frames[2],
        };
        let channel = unsafe { (*core::ptr::addr_of_mut!(IPC_CHANNEL)).as_mut() }
            .unwrap_or_else(|| fail("IPC channel missing"));
        let before = channel.queued();
        let length = usize::try_from(call.args[1]).unwrap_or(usize::MAX);
        // SAFETY: frames belong to the private live UserSpace above; no
        // other task or IRQ can replace them in this masked one-CPU probe.
        let sent = unsafe { channel.send_from_user(owners[0], pages, call.args[0], length) };
        let value = match sent {
            Ok(sequence) => sequence as i64,
            Err(UserSendError::Copy(error)) => error,
            Err(UserSendError::Channel(ipc::Error::TooLarge)) => syscall::EINVAL,
            Err(_) => fail("IPC send channel error"),
        };
        let expected_after = before + usize::from(value > 0);
        if channel.queued() != expected_after {
            fail("IPC rejected send changed queue");
        }
        write_str("TURKOS/IPC-USER-SEND: INDEX=0x");
        write_hex(count);
        write_str(" PTR=0x");
        write_hex(call.args[0] as usize);
        write_str(" LEN=0x");
        write_hex(length);
        write_str(" RESULT=0x");
        write_hex(value as usize);
        write_str(" QUEUED=0x");
        write_hex(channel.queued());
        write_str("\r\n");
        Outcome::Return(value)
    } else if call.version == syscall::VERSION && call.number == syscall::IPC_INSPECT {
        let owners = unsafe { read_volatile(core::ptr::addr_of!(IPC_OWNERS)) };
        let channel = unsafe { (*core::ptr::addr_of_mut!(IPC_CHANNEL)).as_mut() }
            .unwrap_or_else(|| fail("IPC channel missing"));
        let value = if count == 5 {
            let message = channel
                .receive(owners[2])
                .unwrap_or_else(|_| fail("IPC receive"));
            if message.sequence != 1 || message.message.bytes() != b"ABCD" || channel.queued() != 0
            {
                fail("IPC snapshot changed after EL0 mutation");
            }
            let frames = unsafe { read_volatile(core::ptr::addr_of!(USER_FRAMES)) };
            let pages = UserPages {
                code_frame: frames[0],
                data_frame: frames[1],
                stack_frame: frames[2],
            };
            let mut source = [0; 4];
            if unsafe { pages.copy_from(mmu::USER_DATA_VA, &mut source) } != Ok(4)
                || &source != b"WXYZ"
            {
                fail("EL0 source mutation not observed");
            }
            write_line("TURKOS/IPC-USER-SNAPSHOT: SEQ=0x1 DATA=ABCD SOURCE=WXYZ");
            4
        } else {
            if channel.receive(owners[2]) != Err(ipc::Error::QueueEmpty) || channel.queued() != 0 {
                fail("IPC invalid send left a message");
            }
            0
        };
        write_str("TURKOS/IPC-USER-INSPECT: INDEX=0x");
        write_hex(count);
        write_str(" QUEUED=0x");
        write_hex(channel.queued());
        write_str(" RESULT=0x");
        write_hex(value);
        write_str("\r\n");
        Outcome::Return(value as i64)
    } else {
        syscall::dispatch(call)
    };
    #[cfg(not(any(feature = "user-copy-test", feature = "ipc-user-test")))]
    let result = syscall::dispatch(call);
    if result != expected.2 {
        fail("syscall result");
    }
    #[cfg(feature = "user-mmio-request-test")]
    if call.number == syscall::MMIO_MAP {
        write_str("TURKOS/USER-MMIO-DENY: INDEX=0x");
        write_hex(count);
        write_str(" DEVICE=0x");
        write_hex(call.args[0] as usize);
        write_str(" RESULT=0x");
        write_hex(syscall::EPERM as usize);
        write_str("\r\n");
    }
    write_str("TURKOS/SYSCALL-CALL: INDEX=0x");
    write_hex(count);
    write_str(" VERSION=0x");
    write_hex(call.version as usize);
    write_str(" NUMBER=0x");
    write_hex(call.number as usize);
    write_str(" ESR=0x");
    write_hex(esr as usize);
    write_str(" ELR=0x");
    write_hex(frame.elr as usize);
    write_str("\r\n");
    unsafe { write_volatile(core::ptr::addr_of_mut!(USER_CALLS), count + 1) };
    match result {
        Outcome::Return(value) => {
            frame.x[0] = value as u64;
            // For AArch64 SVC, ELR_EL1 already names the instruction after SVC.
            // Advancing it again skips the caller's first return-value check.
            write_str("TURKOS/SYSCALL-RET: INDEX=0x");
            write_hex(count);
            write_str(" X0=0x");
            write_hex(value as usize);
            write_str(" NEXT=0x");
            write_hex(frame.elr as usize);
            write_str("\r\n");
            1
        }
        Outcome::Exit(code) => {
            if code != 0 || count + 1 != expected_calls {
                fail("exit status");
            }
            #[cfg(feature = "user-copy-irq-test")]
            if unsafe { read_volatile(core::ptr::addr_of!(COPY_PHASE)) } != 3
                || unsafe { read_volatile(core::ptr::addr_of!(COPY_IRQ_SEEN)) } != 1
            {
                fail("IRQ copy completion");
            }
            #[cfg(feature = "user-copy-irq-test")]
            write_line("TURKOS/USER-COPY-IRQ: PASS");
            #[cfg(feature = "ipc-user-test")]
            {
                let owners = unsafe { read_volatile(core::ptr::addr_of!(IPC_OWNERS)) };
                let channel = unsafe { (*core::ptr::addr_of_mut!(IPC_CHANNEL)).as_mut() }
                    .unwrap_or_else(|| fail("IPC channel missing at exit"));
                if channel.queued() != 0 || channel.send(owners[1], b"NEXT") != Ok(2) {
                    fail("IPC rejected send consumed sequence");
                }
                let next = channel
                    .receive(owners[3])
                    .unwrap_or_else(|_| fail("IPC next receive"));
                if next.sequence != 2 || next.message.bytes() != b"NEXT" || channel.queued() != 0 {
                    fail("IPC next sequence or consumer");
                }
                write_line("TURKOS/IPC-USER-NEXT: SEQ=0x2 DATA=NEXT QUEUED=0x0");
                write_line("TURKOS/IPC-USER: PASS");
            }
            #[cfg(feature = "user-clock-test")]
            write_line("TURKOS/USER-CLOCK: PASS");
            #[cfg(feature = "user-mmio-request-test")]
            write_line("TURKOS/USER-MMIO-DENY: PASS");
            write_line("TURKOS/SYSCALL: PASS");
            park()
        }
        Outcome::Yield => fail("yield needs task scheduler"),
        Outcome::HandleOpen(_) | Outcome::HandleQuery(_) | Outcome::HandleClose(_) => {
            fail("handle call needs task ownership")
        }
        Outcome::ClockRead => {
            #[cfg(feature = "user-clock-test")]
            {
                let tick: u64;
                let frequency: u64;
                unsafe {
                    // Arm Generic Timer permits speculative counter reads.
                    // ISB orders this read after the observed SVC entry.
                    asm!("isb", "mrs {value}, CNTPCT_EL0", value = out(reg) tick, options(nomem, nostack, preserves_flags));
                    asm!("mrs {value}, CNTFRQ_EL0", value = out(reg) frequency, options(nomem, nostack, preserves_flags));
                }
                let previous = unsafe { read_volatile(core::ptr::addr_of!(LAST_CLOCK)) };
                let original_frequency =
                    unsafe { read_volatile(core::ptr::addr_of!(CLOCK_FREQUENCY)) };
                if tick == 0
                    || tick > i64::MAX as u64
                    || tick < previous
                    || frequency == 0
                    || (count > 4 && original_frequency != frequency)
                {
                    fail("physical count or frequency regressed");
                }
                unsafe {
                    write_volatile(core::ptr::addr_of_mut!(LAST_CLOCK), tick);
                    write_volatile(core::ptr::addr_of_mut!(CLOCK_FREQUENCY), frequency);
                }
                frame.x[0] = tick;
                write_str("TURKOS/USER-CLOCK: INDEX=0x");
                write_hex(count);
                write_str(" TICK=0x");
                write_hex(tick as usize);
                write_str(" PREV=0x");
                write_hex(previous as usize);
                write_str(" FREQ=0x");
                write_hex(frequency as usize);
                write_str("\r\nTURKOS/SYSCALL-RET: INDEX=0x");
                write_hex(count);
                write_str(" X0=0x");
                write_hex(tick as usize);
                write_str(" NEXT=0x");
                write_hex(frame.elr as usize);
                write_str("\r\n");
                1
            }
            #[cfg(not(feature = "user-clock-test"))]
            fail("clock read needs clock probe")
        }
    }
}