//! Two EL0 address spaces using identical virtual addresses and disjoint
//! physical frames. The ASID fixture additionally exercises one tagged ASID
//! recycled between the roots; none of these fixtures claim a scheduler.

use super::*;
#[cfg(feature = "user-asid-test")]
use turkos_kernel::asid;

const TOKEN_A: usize = 0xa11a;
const TOKEN_B: usize = 0xb22b;
const PEER_SECRET: usize = 0xdead_beef;
const EL0T_MASKED: usize = 0x3c0;
static mut A_ROOT: u64 = 0;
static mut B_ROOT: u64 = 0;
static mut KERNEL_ROOT: u64 = 0;
static mut FAULT_TARGET: u64 = 0;
static mut A_DATA: u64 = 0;
static mut B_DATA: u64 = 0;
static mut A_STACK: u64 = 0;
static mut B_STACK: u64 = 0;
static mut DUAL_READY: usize = 0;
static mut DUAL_SVC_COUNT: usize = 0;
#[cfg(feature = "user-dual-test")]
static mut A_SPACE: Option<mmu::UserSpace> = None;
#[cfg(feature = "user-dual-test")]
static mut B_SPACE: Option<mmu::UserSpace> = None;
#[cfg(feature = "user-dual-test")]
static mut USER_ALLOCATOR: *mut memory::FrameAllocator = core::ptr::null_mut();
#[cfg(feature = "user-asid-test")]
static mut ASID_POOL: asid::Pool<1> = asid::Pool::new();
#[cfg(feature = "user-asid-test")]
static mut ASID_ACTIVE: Option<asid::Lease> = None;

unsafe extern "C" {
    static __user_blob_start: u8;
    static __user_blob_end: u8;
    #[cfg(feature = "user-dual-stack-fault-test")]
    static __user_stack_fault_blob_start: u8;
    #[cfg(feature = "user-dual-stack-fault-test")]
    static __user_stack_fault_blob_end: u8;
    #[cfg(any(
        feature = "user-dual-fault-test",
        feature = "user-peer-read-test",
        feature = "user-mmio-read-test"
    ))]
    static __user_kernel_read_blob_start: u8;
    #[cfg(any(
        feature = "user-dual-fault-test",
        feature = "user-peer-read-test",
        feature = "user-mmio-read-test"
    ))]
    static __user_kernel_read_blob_end: u8;
}

fn fail(reason: &str) -> ! {
    write_str("TURKOS/USER-DUAL-ERROR: ");
    write_line(reason);
    park()
}

fn physical_value(frame: u64) -> usize {
    unsafe { read_volatile(frame as *const usize) }
}

fn stack_value(frame: u64) -> usize {
    unsafe { read_volatile((frame + memory::PAGE_SIZE - 16) as *const usize) }
}

fn enter(root: u64, token: usize, id: &str) -> ! {
    #[cfg(feature = "user-asid-test")]
    let root = {
        let lease = unsafe { read_volatile(core::ptr::addr_of!(ASID_ACTIVE)) }
            .unwrap_or_else(|| fail("ASID missing at entry"));
        let pool = unsafe { &*core::ptr::addr_of!(ASID_POOL) };
        if !pool.ready(lease) {
            fail("recycled ASID without TLBI");
        }
        lease
            .ttbr0_value(root)
            .unwrap_or_else(|| fail("ASID TTBR encoding"))
    };
    #[cfg(feature = "user-asid-test")]
    unsafe {
        asm!("dsb ishst", "msr TTBR0_EL1, {root}", "isb", root = in(reg) root, options(nostack))
    };
    #[cfg(not(feature = "user-asid-test"))]
    unsafe {
        asm!(
            "dsb ishst",
            "msr TTBR0_EL1, {root}",
            "isb",
            "tlbi vmalle1is",
            "dsb ish",
            "isb",
            root = in(reg) root,
            options(nostack)
        )
    };
    let actual: u64;
    unsafe {
        asm!("mrs {value}, TTBR0_EL1", value = out(reg) actual, options(nomem, nostack, preserves_flags))
    };
    if actual != root {
        fail("TTBR0 readback");
    }
    let fault_target = unsafe { read_volatile(core::ptr::addr_of!(FAULT_TARGET)) };
    write_str("TURKOS/USER-DUAL-ENTRY: ID=");
    write_str(id);
    write_str(" TTBR=0x");
    write_hex(actual as usize);
    write_str(" VA=0x");
    write_hex(mmu::USER_CODE_VA as usize);
    write_str(" SP=0x");
    write_hex(mmu::USER_STACK_TOP as usize);
    write_str(" TOKEN=0x");
    write_hex(token);
    write_str("\r\n");
    unsafe {
        asm!(
            "msr SP_EL0, {stack}",
            "msr ELR_EL1, {entry}",
            "msr SPSR_EL1, {mode}",
            "isb",
            "eret",
            stack = in(reg) mmu::USER_STACK_TOP,
            entry = in(reg) mmu::USER_CODE_VA,
            mode = in(reg) EL0T_MASKED,
            in("x0") token,
            in("x1") fault_target,
            options(noreturn)
        )
    }
}

pub(super) fn smoke_test(
    allocator: &mut memory::FrameAllocator,
    tables: &mmu::Tables,
    layout: mmu::Layout,
) -> ! {
    let before = allocator.remaining_pages();
    let initial_root: u64;
    unsafe {
        asm!("mrs {value}, TTBR0_EL1", value = out(reg) initial_root, options(nomem, nostack, preserves_flags))
    };
    if initial_root != tables.root {
        fail("kernel TTBR0 before user spaces");
    }
    let a = unsafe { tables.build_user_space(allocator, layout.ram_base, layout.ram_size) }
        .unwrap_or_else(|_| fail("A allocation"));
    let b = unsafe { tables.build_user_space(allocator, layout.ram_base, layout.ram_size) }
        .unwrap_or_else(|_| fail("B allocation"));
    let first_root = a.root;
    if !a.contract_valid(tables.root)
        || !b.contract_valid(tables.root)
        || a.root == b.root
        || allocator.remaining_pages() + 12 != before
        || [a.code_frame(), a.data_frame(), a.stack_frame()]
            .iter()
            .any(|frame| [b.code_frame(), b.data_frame(), b.stack_frame()].contains(frame))
        || unsafe { read_volatile((tables.root as *const u64).add(2)) } != 0
    {
        fail("two private roots or frame ownership");
    }
    if cfg!(feature = "user-mmio-read-test") {
        if layout.uart_base != 0x0900_0000 {
            fail("unexpected DTB UART window");
        }
        let windows = [
            (layout.uart_base, memory::PAGE_SIZE),
            (0x080a_0000, 0x2_0000),
            (0x0a00_0000, 4 * memory::PAGE_SIZE),
        ];
        if tables.audit_device_el0(&a, &windows) != Ok(37)
            || tables.audit_device_el0(&b, &windows) != Ok(37)
        {
            fail("EL0 Device mapping audit");
        }
        write_line(
            "TURKOS/USER-MMIO-AUDIT: ROOTS=2 UART=1 GICR=32 VIRTIO=4 PAGES=37 EL0=0 GICD=EL0-DENIED",
        );
    }
    let start = core::ptr::addr_of!(__user_blob_start) as usize;
    let end = core::ptr::addr_of!(__user_blob_end) as usize;
    let length = end.checked_sub(start).unwrap_or(0);
    if length != 36 {
        fail("dual user blob length");
    }
    #[cfg(any(
        feature = "user-dual-fault-test",
        feature = "user-peer-read-test",
        feature = "user-mmio-read-test"
    ))]
    let a_start = {
        let fault_start = core::ptr::addr_of!(__user_kernel_read_blob_start) as usize;
        let fault_end = core::ptr::addr_of!(__user_kernel_read_blob_end) as usize;
        if fault_end.checked_sub(fault_start) != Some(length) {
            fail("kernel-read user blob length");
        }
        fault_start
    };
    #[cfg(feature = "user-dual-stack-fault-test")]
    let a_start = {
        let fault_start = core::ptr::addr_of!(__user_stack_fault_blob_start) as usize;
        let fault_end = core::ptr::addr_of!(__user_stack_fault_blob_end) as usize;
        if fault_end.checked_sub(fault_start) != Some(length) {
            fail("stack-overflow user blob length");
        }
        fault_start
    };
    #[cfg(not(any(
        feature = "user-dual-fault-test",
        feature = "user-dual-stack-fault-test",
        feature = "user-peer-read-test",
        feature = "user-mmio-read-test"
    )))]
    let a_start = start;
    let ctr: usize;
    unsafe {
        asm!("mrs {value}, CTR_EL0", value = out(reg) ctr, options(nomem, nostack, preserves_flags))
    };
    let dline = 4usize << ((ctr >> 16) & 15);
    for (frame, source) in [(a.code_frame(), a_start), (b.code_frame(), start)] {
        unsafe { core::ptr::copy_nonoverlapping(source as *const u8, frame as *mut u8, length) };
        for offset in (0..length).step_by(dline) {
            unsafe {
                asm!("dc cvau, {addr}", addr = in(reg) (frame as usize + offset), options(nostack))
            };
        }
    }
    unsafe { asm!("dsb ish", "ic iallu", "dsb ish", "isb", options(nostack)) };
    if cfg!(feature = "user-peer-read-test") {
        unsafe { write_volatile(b.data_frame() as *mut usize, PEER_SECRET) };
    }
    unsafe {
        write_volatile(core::ptr::addr_of_mut!(A_ROOT), a.root);
        write_volatile(core::ptr::addr_of_mut!(B_ROOT), b.root);
        write_volatile(core::ptr::addr_of_mut!(KERNEL_ROOT), tables.root);
        write_volatile(
            core::ptr::addr_of_mut!(FAULT_TARGET),
            if cfg!(feature = "user-peer-read-test") {
                b.data_frame()
            } else if cfg!(feature = "user-mmio-read-test") {
                layout.uart_base
            } else if cfg!(feature = "user-dual-stack-fault-test") {
                mmu::USER_STACK_VA - 16
            } else {
                tables.root
            },
        );
        write_volatile(core::ptr::addr_of_mut!(A_DATA), a.data_frame());
        write_volatile(core::ptr::addr_of_mut!(B_DATA), b.data_frame());
        write_volatile(core::ptr::addr_of_mut!(A_STACK), a.stack_frame());
        write_volatile(core::ptr::addr_of_mut!(B_STACK), b.stack_frame());
        write_volatile(core::ptr::addr_of_mut!(DUAL_READY), 1);
    }
    write_str("TURKOS/USER-DUAL-WINDOW: RAM-END=0x");
    write_hex((layout.ram_base + layout.ram_size) as usize);
    write_str(" CODE-VA=0x");
    write_hex(mmu::USER_CODE_VA as usize);
    write_str(" DATA-VA=0x");
    write_hex(mmu::USER_DATA_VA as usize);
    write_str(" STACK-VA=0x");
    write_hex(mmu::USER_STACK_VA as usize);
    write_str("\r\nTURKOS/USER-DUAL-ROOTS: KERNEL=0x");
    write_hex(tables.root as usize);
    write_str(" A=0x");
    write_hex(a.root as usize);
    write_str(" B=0x");
    write_hex(b.root as usize);
    write_str(" BEFORE=0x");
    write_hex(before as usize);
    write_str(" AFTER=0x");
    write_hex(allocator.remaining_pages() as usize);
    write_str("\r\nTURKOS/USER-DUAL-TTBR-BEFORE: 0x");
    write_hex(initial_root as usize);
    write_str("\r\nTURKOS/USER-DUAL-L1: KERNEL=0x");
    write_hex(unsafe { read_volatile((tables.root as *const u64).add(2)) } as usize);
    write_str(" A=0x");
    write_hex(unsafe { read_volatile((a.root as *const u64).add(2)) } as usize);
    write_str(" B=0x");
    write_hex(unsafe { read_volatile((b.root as *const u64).add(2)) } as usize);
    write_str("\r\nTURKOS/USER-DUAL-FRAMES: A-CODE=0x");
    write_hex(a.code_frame() as usize);
    write_str(" A-DATA=0x");
    write_hex(a.data_frame() as usize);
    write_str(" A-STACK=0x");
    write_hex(a.stack_frame() as usize);
    write_str(" B-CODE=0x");
    write_hex(b.code_frame() as usize);
    write_str(" B-DATA=0x");
    write_hex(b.data_frame() as usize);
    write_str(" B-STACK=0x");
    write_hex(b.stack_frame() as usize);
    write_str("\r\nTURKOS/USER-DUAL-DESC: A-CODE=0x");
    write_hex(a.descriptor(0).unwrap() as usize);
    write_str(" A-DATA=0x");
    write_hex(a.descriptor(1).unwrap() as usize);
    write_str(" A-GUARD=0x");
    write_hex(a.descriptor(2).unwrap() as usize);
    write_str(" A-STACK=0x");
    write_hex(a.descriptor(3).unwrap() as usize);
    write_str(" B-CODE=0x");
    write_hex(b.descriptor(0).unwrap() as usize);
    write_str(" B-DATA=0x");
    write_hex(b.descriptor(1).unwrap() as usize);
    write_str(" B-GUARD=0x");
    write_hex(b.descriptor(2).unwrap() as usize);
    write_str(" B-STACK=0x");
    write_hex(b.descriptor(3).unwrap() as usize);
    write_str("\r\n");
    #[cfg(feature = "user-dual-test")]
    unsafe {
        core::ptr::write(core::ptr::addr_of_mut!(A_SPACE), Some(a));
        core::ptr::write(core::ptr::addr_of_mut!(B_SPACE), Some(b));
        write_volatile(core::ptr::addr_of_mut!(USER_ALLOCATOR), allocator as *mut _);
    }
    #[cfg(feature = "user-asid-test")]
    {
        let pool = unsafe { &mut *core::ptr::addr_of_mut!(ASID_POOL) };
        let grant = pool
            .acquire()
            .unwrap_or_else(|_| fail("initial ASID allocation"));
        if grant.recycled
            || grant.lease.id != 1
            || grant.lease.generation != 1
            || !pool.ready(grant.lease)
        {
            fail("first ASID ownership");
        }
        unsafe { write_volatile(core::ptr::addr_of_mut!(ASID_ACTIVE), Some(grant.lease)) };
        write_str("TURKOS/USER-ASID-ALLOC: ID=0x");
        write_hex(grant.lease.id as usize);
        write_str(" GENERATION=0x");
        write_hex(grant.lease.generation as usize);
        write_str(" TLBI=0x");
        write_hex(grant.recycled as usize);
        write_str("\r\n");
    }
    enter(first_root, TOKEN_A, "A")
}

#[unsafe(no_mangle)]
pub extern "C" fn kernel_user_dual_exception(
    esr: usize,
    elr: usize,
    user_sp: usize,
    spsr: usize,
    root: usize,
    token: usize,
    far: usize,
) -> ! {
    let count = unsafe { read_volatile(core::ptr::addr_of!(DUAL_SVC_COUNT)) };
    let a_root = unsafe { read_volatile(core::ptr::addr_of!(A_ROOT)) };
    let b_root = unsafe { read_volatile(core::ptr::addr_of!(B_ROOT)) };
    let a_data = unsafe { read_volatile(core::ptr::addr_of!(A_DATA)) };
    let b_data = unsafe { read_volatile(core::ptr::addr_of!(B_DATA)) };
    let a_stack = unsafe { read_volatile(core::ptr::addr_of!(A_STACK)) };
    let b_stack = unsafe { read_volatile(core::ptr::addr_of!(B_STACK)) };
    let asid_tag = if cfg!(feature = "user-asid-test") {
        1usize << 48
    } else {
        0
    };
    let first_stack_fault = cfg!(feature = "user-dual-stack-fault-test") && count == 0;
    if unsafe { read_volatile(core::ptr::addr_of!(DUAL_READY)) } != 1
        || (!first_stack_fault && user_sp != mmu::USER_STACK_TOP as usize)
        || spsr & 15 != 0
    {
        fail("dual EL0 exception origin");
    }
    if count == 0
        && (cfg!(feature = "user-dual-fault-test")
            || cfg!(feature = "user-dual-stack-fault-test")
            || cfg!(feature = "user-peer-read-test")
            || cfg!(feature = "user-mmio-read-test"))
    {
        let fault_target = unsafe { read_volatile(core::ptr::addr_of!(FAULT_TARGET)) };
        let b_before = if cfg!(feature = "user-peer-read-test") {
            PEER_SECRET
        } else {
            0
        };
        let fault_valid = if cfg!(feature = "user-dual-stack-fault-test") {
            (esr >> 26) & 0x3f == 0x24
                && esr & 0x3f == 0x07
                && esr & (1 << 6) != 0
                && user_sp == fault_target as usize
        } else {
            (esr >> 26) & 0x3f == 0x24
                && esr & 0x3f == 0x0f
                && esr & (1 << 6) == 0
                && user_sp == mmu::USER_STACK_TOP as usize
        };
        if root != (a_root as usize | asid_tag)
            || token != TOKEN_A
            || !fault_valid
            || elr != (mmu::USER_CODE_VA + 12) as usize
            || far != fault_target as usize
            || physical_value(a_data) != 0
            || stack_value(a_stack) != 0
            || physical_value(b_data) != b_before
            || stack_value(b_stack) != 0
        {
            fail("A forbidden access or stack guard fault");
        }
        unsafe { write_volatile(core::ptr::addr_of_mut!(DUAL_SVC_COUNT), 1) };
        if cfg!(feature = "user-peer-read-test") {
            write_str("TURKOS/USER-PEER-FAULT: ID=A ROOT=0x");
        } else if cfg!(feature = "user-mmio-read-test") {
            write_str("TURKOS/USER-MMIO-FAULT: ID=A ROOT=0x");
        } else if cfg!(feature = "user-dual-stack-fault-test") {
            write_str("TURKOS/USER-STACK-ISOLATION-FAULT: ID=A ROOT=0x");
        } else {
            write_str("TURKOS/USER-DUAL-FAULT: ID=A ROOT=0x");
        }
        write_hex(root);
        write_str(" ESR=0x");
        write_hex(esr);
        write_str(" ELR=0x");
        write_hex(elr);
        write_str(" FAR=0x");
        write_hex(far);
        write_str(" SP=0x");
        write_hex(user_sp);
        write_str(" X0=0x");
        write_hex(token);
        write_str(" A-DATA=0x");
        write_hex(physical_value(a_data));
        write_str(" B-DATA=0x");
        write_hex(physical_value(b_data));
        write_str("\r\n");
        enter(b_root, TOKEN_B, "B")
    }
    if (esr >> 26) & 0x3f != 0x15
        || esr & 0xffff != 0x42
        || elr != (mmu::USER_CODE_VA + 32) as usize
    {
        fail("dual EL0 SVC record");
    }
    if count == 0 {
        if root != (a_root as usize | asid_tag)
            || token != TOKEN_A
            || physical_value(a_data) != TOKEN_A
            || stack_value(a_stack) != TOKEN_A
            || physical_value(b_data) != 0
            || stack_value(b_stack) != 0
        {
            fail("A isolation witness");
        }
        unsafe { write_volatile(core::ptr::addr_of_mut!(DUAL_SVC_COUNT), 1) };
        write_str("TURKOS/USER-DUAL-SVC: ID=A ROOT=0x");
        write_hex(root);
        write_str(" ESR=0x");
        write_hex(esr);
        write_str(" ELR=0x");
        write_hex(elr);
        write_str(" SP=0x");
        write_hex(user_sp);
        write_str(" X0=0x");
        write_hex(token);
        write_str(" DATA=0x");
        write_hex(physical_value(a_data));
        write_str(" STACK=0x");
        write_hex(stack_value(a_stack));
        write_str(" B-DATA-BEFORE=0x");
        write_hex(physical_value(b_data));
        write_str("\r\n");
        #[cfg(feature = "user-asid-test")]
        {
            let pool = unsafe { &mut *core::ptr::addr_of_mut!(ASID_POOL) };
            let previous = unsafe { read_volatile(core::ptr::addr_of!(ASID_ACTIVE)) }
                .unwrap_or_else(|| fail("A ASID missing"));
            pool.release(previous)
                .unwrap_or_else(|_| fail("A ASID release"));
            let grant = pool.acquire().unwrap_or_else(|_| fail("B ASID recycle"));
            if !grant.recycled
                || grant.lease.id != previous.id
                || grant.lease.generation <= previous.generation
                || pool.ready(grant.lease)
            {
                fail("ASID reuse gate");
            }
            // Stop executing with the retired ASID before invalidating it: an
            // active translation context could otherwise refill old entries.
            let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(KERNEL_ROOT)) };
            unsafe {
                asm!("dsb ish", "msr TTBR0_EL1, {root}", "isb", root = in(reg) kernel_root, options(nostack))
            };
            let switched_root: u64;
            unsafe {
                asm!("mrs {value}, TTBR0_EL1", value = out(reg) switched_root, options(nomem, nostack, preserves_flags))
            };
            if switched_root != kernel_root {
                fail("retired ASID root switch");
            }
            write_str("TURKOS/USER-ASID-KERNEL: TTBR=0x");
            write_hex(switched_root as usize);
            write_str("\r\n");
            let operand = grant.lease.tlbi_operand();
            unsafe {
                asm!("dsb ish", "tlbi aside1is, {operand}", "dsb ish", "isb", operand = in(reg) operand, options(nostack))
            };
            pool.confirm_tlbi(grant.lease)
                .unwrap_or_else(|_| fail("ASID TLBI confirmation"));
            unsafe { write_volatile(core::ptr::addr_of_mut!(ASID_ACTIVE), Some(grant.lease)) };
            write_str("TURKOS/USER-ASID-RECYCLE: ID=0x");
            write_hex(grant.lease.id as usize);
            write_str(" OLD=0x");
            write_hex(previous.generation as usize);
            write_str(" NEW=0x");
            write_hex(grant.lease.generation as usize);
            write_str(" OPERAND=0x");
            write_hex(operand as usize);
            write_str(" TLBI=0x");
            write_hex(grant.recycled as usize);
            write_str("\r\n");
        }
        enter(b_root, TOKEN_B, "B")
    }
    let a_expected = if cfg!(feature = "user-dual-fault-test")
        || cfg!(feature = "user-dual-stack-fault-test")
        || cfg!(feature = "user-peer-read-test")
        || cfg!(feature = "user-mmio-read-test")
    {
        0
    } else {
        TOKEN_A
    };
    if count != 1
        || root != (b_root as usize | asid_tag)
        || token != TOKEN_B
        || physical_value(a_data) != a_expected
        || stack_value(a_stack) != a_expected
        || physical_value(b_data) != TOKEN_B
        || stack_value(b_stack) != TOKEN_B
    {
        #[cfg(feature = "user-asid-test")]
        {
            write_str("TURKOS/USER-ASID-WITNESS-ERROR: COUNT=0x");
            write_hex(count);
            write_str(" ROOT=0x");
            write_hex(root);
            write_str(" TOKEN=0x");
            write_hex(token);
            write_str(" A-DATA=0x");
            write_hex(physical_value(a_data));
            write_str(" A-STACK=0x");
            write_hex(stack_value(a_stack));
            write_str(" B-DATA=0x");
            write_hex(physical_value(b_data));
            write_str(" B-STACK=0x");
            write_hex(stack_value(b_stack));
            write_str("\r\n");
        }
        fail("B isolation witness");
    }
    unsafe { write_volatile(core::ptr::addr_of_mut!(DUAL_SVC_COUNT), 2) };
    if cfg!(feature = "user-peer-read-test") {
        write_str("TURKOS/USER-PEER-HEALTHY: ID=B ROOT=0x");
    } else if cfg!(feature = "user-mmio-read-test") {
        write_str("TURKOS/USER-MMIO-HEALTHY: ID=B ROOT=0x");
    } else if cfg!(feature = "user-dual-fault-test") {
        write_str("TURKOS/USER-DUAL-HEALTHY: ID=B ROOT=0x");
    } else if cfg!(feature = "user-dual-stack-fault-test") {
        write_str("TURKOS/USER-STACK-ISOLATION-HEALTHY: ID=B ROOT=0x");
    } else {
        write_str("TURKOS/USER-DUAL-SVC: ID=B ROOT=0x");
    }
    write_hex(root);
    write_str(" ESR=0x");
    write_hex(esr);
    write_str(" ELR=0x");
    write_hex(elr);
    write_str(" SP=0x");
    write_hex(user_sp);
    write_str(" X0=0x");
    write_hex(token);
    write_str(" A-DATA=0x");
    write_hex(physical_value(a_data));
    write_str(" B-DATA=0x");
    write_hex(physical_value(b_data));
    write_str(" A-STACK=0x");
    write_hex(stack_value(a_stack));
    write_str(" B-STACK=0x");
    write_hex(stack_value(b_stack));
    write_str("\r\n");
    #[cfg(feature = "user-dual-test")]
    {
        let kernel_root = unsafe { read_volatile(core::ptr::addr_of!(KERNEL_ROOT)) };
        let allocator_ptr = unsafe { read_volatile(core::ptr::addr_of!(USER_ALLOCATOR)) };
        if allocator_ptr.is_null() {
            fail("user allocator lost before reclaim");
        }
        let allocator = unsafe { &mut *allocator_ptr };
        let before = allocator.remaining_pages();
        unsafe {
            asm!(
                "dsb ish",
                "msr TTBR0_EL1, {root}",
                "isb",
                "tlbi vmalle1is",
                "dsb ish",
                "isb",
                root = in(reg) kernel_root,
                options(nostack)
            )
        };
        let switched_root: u64;
        unsafe {
            asm!("mrs {value}, TTBR0_EL1", value = out(reg) switched_root, options(nomem, nostack, preserves_flags))
        };
        if switched_root != kernel_root {
            fail("kernel root before user reclaim");
        }
        let a_space = unsafe { core::ptr::replace(core::ptr::addr_of_mut!(A_SPACE), None) }
            .unwrap_or_else(|| fail("A owner missing at reclaim"));
        let b_space = unsafe { core::ptr::replace(core::ptr::addr_of_mut!(B_SPACE), None) }
            .unwrap_or_else(|| fail("B owner missing at reclaim"));
        unsafe { a_space.release_after_tlbi(allocator, kernel_root) }
            .unwrap_or_else(|_| fail("A user frames reclaim"));
        unsafe { b_space.release_after_tlbi(allocator, kernel_root) }
            .unwrap_or_else(|_| fail("B user frames reclaim"));
        let after = allocator.remaining_pages();
        let kernel_user_l1 = unsafe { read_volatile((kernel_root as *const u64).add(2)) };
        if before.checked_add(12) != Some(after) || kernel_user_l1 != 0 {
            fail("user frame reclaim accounting");
        }
        write_str("TURKOS/USER-DUAL-RECLAIM: KERNEL=0x");
        write_hex(switched_root as usize);
        write_str(" BEFORE=0x");
        write_hex(before as usize);
        write_str(" AFTER=0x");
        write_hex(after as usize);
        write_str(" RELEASED=0x");
        write_hex((after - before) as usize);
        write_str(" L1=0x");
        write_hex(kernel_user_l1 as usize);
        write_str("\r\n");
    }
    if cfg!(feature = "user-peer-read-test") {
        write_line("TURKOS/USER-PEER: PASS");
    } else if cfg!(feature = "user-mmio-read-test") {
        write_line("TURKOS/USER-MMIO: PASS");
    } else if cfg!(feature = "user-dual-fault-test") {
        write_line("TURKOS/USER-DUAL-RECOVERY: PASS");
    } else if cfg!(feature = "user-dual-stack-fault-test") {
        write_line("TURKOS/USER-STACK-ISOLATION: PASS");
    } else {
        write_line("TURKOS/USER-DUAL: PASS");
        #[cfg(feature = "user-asid-test")]
        write_line("TURKOS/USER-ASID: PASS");
    }
    park()
}