//! Separate long-running acceptance Image. No default IRQ policy change.
//! 32 100-us deliveries saturate a bounded queue, then 6000 10-ms deliveries
//! measure service, loss/coalescing, deadline lateness and owner/stack guards.
use super::*;

#[repr(align(4096))]
struct ServiceStack([u8; 16 * 1024]);
static mut STACK: ServiceStack = ServiceStack([0; 16 * 1024]);
static mut ACTIVE: bool = false;
static mut ALLOCATOR: *const memory::FrameAllocator = core::ptr::null();
static mut FREE: u64 = 0;
static mut GICR: u64 = 0;
static mut FREQUENCY: u64 = 0;
static mut DEADLINE: u64 = 0;
static mut START: u64 = 0;
static mut SEEN: u64 = 0;
static mut MISSED: u64 = 0;
static mut LATENCY: [u64; 6] = [0; 6];
static mut MAX_LATE: u64 = 0;
static mut HEAD: u64 = 0;
static mut TAIL: u64 = 0;
static mut QUEUE: [u64; irq::SERVICE_QUEUE_CAPACITY] = [0; irq::SERVICE_QUEUE_CAPACITY];
static mut DROPPED: u64 = 0;
static mut MAX_OCCUPIED: u64 = 0;
static mut PROGRESS: u64 = 0;
static mut LAST_PROGRESS: u64 = 0;
static mut STORM_INTERLEAVED: u64 = 0;

fn fail(reason: &str) -> ! {
    write_str("TURKOS/IRQ-SERVICE-ERROR: ");
    write_line(reason);
    park()
}
pub(super) fn active() -> bool {
    unsafe { read_volatile(core::ptr::addr_of!(ACTIVE)) }
}
pub(super) fn run(allocator: &memory::FrameAllocator, gicr: u64) -> ! {
    // The original boot stack is retained, not reused: ALLOCATOR still refers
    // to its live owner. Only the test's execution SP moves to private BSS.
    unsafe {
        ALLOCATOR = allocator;
        FREE = allocator.remaining_pages();
        GICR = gicr;
        ACTIVE = true;
        write_volatile(core::ptr::addr_of_mut!(__irq_stack_guard), IRQ_GUARD);
        let bottom = core::ptr::addr_of_mut!(STACK.0) as usize;
        write_volatile(bottom as *mut u64, IRQ_GUARD);
        asm!("mov sp, {stack}", "b kernel_irq_service_begin", stack = in(reg) (bottom + 16 * 1024), options(noreturn));
    }
}

pub(super) fn handle(frame: *const irq::IrqContext, low: usize, top: usize, sp: usize) -> usize {
    if frame.is_null() {
        fail("null context");
    }
    let context = unsafe { &*frame };
    let bottom = unsafe { core::ptr::addr_of!(STACK.0) } as usize;
    if frame as usize != top - size_of::<irq::IrqContext>()
        || !irq::handler_sp_in_stack(low, frame as usize, sp)
        || context.vector_slot != 5
        || context.spsr & 0x8f != 5
        || context.interrupted_sp < (bottom + 16) as u64
        || context.interrupted_sp > (bottom + 16 * 1024) as u64
        || context.interrupted_sp & 15 != 0
        || context.x[0] != 0x5455_524b_4952_5130
        || context.x[3] != 0x5455_524b_4952_5133
        || context.x[16] != 0x5455_524b_4952_5116
        || context.x[17] != 0x5455_524b_4952_5117
        || context.x[30] != 0x5455_524b_4952_514c
        || context.q[0] != [0x5a5a_5a5a_5a5a_5a5a; 2]
        || context.q[31] != [0xa5a5_a5a5_a5a5_a5a5; 2]
        || unsafe { read_volatile(core::ptr::addr_of!(__irq_stack_guard)) } != IRQ_GUARD
        || unsafe { read_volatile(bottom as *const u64) } != IRQ_GUARD
        || unsafe { (*ALLOCATOR).remaining_pages() } != unsafe { FREE }
    {
        fail("context/stack/frame owner");
    }
    let iar: u64;
    let now: u64;
    unsafe {
        asm!("mrs {iar}, ICC_IAR1_EL1", "mrs {now}, CNTPCT_EL0", iar=out(reg) iar, now=out(reg) now, options(nomem,nostack));
    }
    if irq::classify_iar(iar, 30) != irq::IarClass::Expected {
        fail("unexpected IAR");
    }
    let previous = unsafe { read_volatile(core::ptr::addr_of!(SEEN)) };
    let count = previous
        .checked_add(1)
        .unwrap_or_else(|| fail("count overflow"));
    if count > irq::SERVICE_TICKS as u64 {
        fail("extra tick");
    }
    let deadline = unsafe { DEADLINE };
    let frequency = unsafe { FREQUENCY };
    let bucket = irq::service_latency_bucket(now, deadline, frequency)
        .unwrap_or_else(|| fail("latency range"));
    unsafe {
        let slot = core::ptr::addr_of_mut!(LATENCY).cast::<u64>().add(bucket);
        write_volatile(slot, read_volatile(slot) + 1);
        MAX_LATE = MAX_LATE.max(now - deadline);
    }
    let head = unsafe { read_volatile(core::ptr::addr_of!(HEAD)) };
    let tail = unsafe { read_volatile(core::ptr::addr_of!(TAIL)) };
    match irq::service_queue_admit(head, tail) {
        Some(true) => unsafe {
            write_volatile(
                core::ptr::addr_of_mut!(QUEUE)
                    .cast::<u64>()
                    .add((head % 8) as usize),
                count,
            );
            write_volatile(core::ptr::addr_of_mut!(HEAD), head + 1);
            MAX_OCCUPIED = MAX_OCCUPIED.max(head + 1 - tail);
        },
        Some(false) => unsafe {
            DROPPED += 1;
        },
        None => fail("queue ownership corrupted"),
    }
    let progress = unsafe { read_volatile(core::ptr::addr_of!(PROGRESS)) };
    if count > 1 && count <= 32 && progress > unsafe { LAST_PROGRESS } {
        unsafe {
            STORM_INTERLEAVED += 1;
        }
    }
    unsafe {
        LAST_PROGRESS = progress;
        write_volatile(core::ptr::addr_of_mut!(SEEN), count);
    }
    if count < irq::SERVICE_TICKS as u64 {
        let period_ns = if count < 32 { 100_000 } else { 10_000_000 };
        let period =
            irq::timer_ticks_for_ns(frequency, period_ns).unwrap_or_else(|| fail("period"));
        let (next, missed) = irq::next_timer_deadline(deadline, period, now)
            .unwrap_or_else(|| fail("deadline overflow"));
        unsafe {
            DEADLINE = next;
            MISSED = MISSED
                .checked_add(missed)
                .unwrap_or_else(|| fail("missed overflow"));
            asm!("msr CNTP_CVAL_EL0, {next}", "isb", next=in(reg) next, options(nostack));
        }
    } else {
        unsafe {
            asm!("msr CNTP_CTL_EL0, xzr", "isb", options(nomem, nostack));
        }
    }
    // Clear/rearm the timer source BEFORE dropping its acknowledged priority.
    unsafe {
        asm!("msr ICC_EOIR1_EL1, {iar}", "isb", iar=in(reg) iar, options(nomem,nostack));
        let control: u64;
        asm!("mrs {control}, ICC_CTLR_EL1", control=out(reg) control, options(nomem,nostack));
        if control & 2 != 0 {
            asm!("msr ICC_DIR_EL1, {iar}", "isb", iar=in(reg) iar, options(nomem,nostack));
        }
    }
    1
}

#[unsafe(no_mangle)]
pub extern "C" fn kernel_irq_service_begin() -> ! {
    let frequency: u64;
    let start: u64;
    unsafe {
        asm!("mrs {frequency}, CNTFRQ_EL0", "mrs {start}, CNTPCT_EL0", frequency=out(reg) frequency,start=out(reg) start,options(nomem,nostack));
    }
    let first = start
        .checked_add(
            irq::timer_ticks_for_ns(frequency, 100_000).unwrap_or_else(|| fail("first period")),
        )
        .unwrap_or_else(|| fail("first overflow"));
    unsafe {
        FREQUENCY = frequency;
        START = start;
        DEADLINE = first;
    }
    write_line("TURKOS/IRQ-SERVICE: ARMED STORM=32 LONG=6000 QUEUE=8 NO-ACTIVE-DMA-QUEUE");
    let progress: u64;
    let consumed: u64;
    let last_ticket: u64;
    let x0: u64;
    let x3: u64;
    let x16: u64;
    let x17: u64;
    let x30: u64;
    let q0: u64;
    let q31: u64;
    unsafe {
        asm!(
            "movi v0.16b, #0x5a", "movi v31.16b, #0xa5",
            "msr CNTP_CVAL_EL0, {first}", "msr CNTP_CTL_EL0, {enable}", "isb", "msr daifclr, #2",
            "2:", "add {progress}, {progress}, #1", "str {progress}, [{progress_ptr}]",
            "ldr {observed}, [{seen}]", "cmp {observed}, #32", "b.le 4f",
            // One CPU, producer IRQ masked on entry. The main consumer never
            // acknowledges a head it has not read; an IRQ between these loads
            // can only add a future ticket, not overwrite an unconsumed slot.
            "ldr {head_value}, [{head}]", "ldr {tail_value}, [{tail}]", "cmp {head_value}, {tail_value}", "b.eq 4f",
            "and {ticket}, {tail_value}, #7", "ldr {ticket}, [{queue}, {ticket}, lsl #3]",
            "cmp {ticket}, {last_ticket}", "b.le 5f", "mov {last_ticket}, {ticket}",
            "add {tail_value}, {tail_value}, #1", "str {tail_value}, [{tail}]", "add {consumed}, {consumed}, #1",
            "4:", "cmp {observed}, #1, lsl #12", "b.lt 2b",
            "cmp {observed}, {target}", "b.lt 2b", "b 6f",
            "5:", "mov {consumed}, #-1",
            "6:", "msr daifset, #2", "msr CNTP_CTL_EL0, xzr", "isb",
            "umov {q0}, v0.d[0]", "umov {q31}, v31.d[0]",
            first=in(reg) first, enable=in(reg) 1u64, target=in(reg) irq::SERVICE_TICKS as u64,
            seen=in(reg) core::ptr::addr_of!(SEEN), progress_ptr=in(reg) core::ptr::addr_of_mut!(PROGRESS),
            head=in(reg) core::ptr::addr_of!(HEAD),tail=in(reg) core::ptr::addr_of_mut!(TAIL),queue=in(reg) core::ptr::addr_of!(QUEUE),
            progress=inout(reg) 0u64=>progress,consumed=inout(reg) 0u64=>consumed,last_ticket=inout(reg) 0u64=>last_ticket,
            observed=out(reg) _,head_value=out(reg) _,tail_value=out(reg) _,ticket=out(reg) _,
            q0=lateout(reg) q0,q31=lateout(reg) q31,
            inout("x0") 0x5455_524b_4952_5130u64=>x0,inout("x3") 0x5455_524b_4952_5133u64=>x3,
            inout("x16") 0x5455_524b_4952_5116u64=>x16,inout("x17") 0x5455_524b_4952_5117u64=>x17,
            inout("x30") 0x5455_524b_4952_514cu64=>x30,out("v0") _,out("v31") _,
        );
    }
    let end: u64;
    unsafe {
        asm!("mrs {end}, CNTPCT_EL0",end=out(reg) end,options(nomem,nostack));
    }
    let head = unsafe { HEAD };
    let mut tail = unsafe { TAIL };
    let mut consumed = consumed;
    let mut last_ticket = last_ticket;
    while tail < head {
        let ticket = unsafe {
            read_volatile(
                core::ptr::addr_of!(QUEUE)
                    .cast::<u64>()
                    .add((tail % 8) as usize),
            )
        };
        if ticket <= last_ticket {
            fail("duplicate/reordered ticket");
        }
        last_ticket = ticket;
        consumed += 1;
        tail += 1;
    }
    let gicr = unsafe { GICR };
    let masked;
    unsafe {
        write_volatile((gicr + 0x10180) as *mut u32, 1 << 30);
        asm!("dsb sy", options(nostack));
        for _ in 0..1_000_000 {
            if read_volatile(gicr as *const u32) & 8 == 0 {
                break;
            }
        }
        masked = read_volatile((gicr + 0x10100) as *const u32) & (1 << 30) == 0;
    }
    let histogram = unsafe { LATENCY };
    let dropped = unsafe { DROPPED };
    let ticks = unsafe { SEEN };
    if end - start
        < frequency
            .checked_mul(60)
            .unwrap_or_else(|| fail("duration overflow"))
        || ticks != irq::SERVICE_TICKS as u64
        || histogram.iter().sum::<u64>() != ticks
        || progress <= ticks
        || consumed == u64::MAX
        || consumed + dropped != ticks
        || unsafe { MAX_OCCUPIED } != 8
        || dropped < 24
        || unsafe { STORM_INTERLEAVED } == 0
        || !masked
        || x0 != 0x5455_524b_4952_5130
        || x3 != 0x5455_524b_4952_5133
        || x16 != 0x5455_524b_4952_5116
        || x17 != 0x5455_524b_4952_5117
        || x30 != 0x5455_524b_4952_514c
        || q0 != 0x5a5a_5a5a_5a5a_5a5a
        || q31 != 0xa5a5_a5a5_a5a5_a5a5
        || unsafe { (*ALLOCATOR).remaining_pages() } != unsafe { FREE }
        || unsafe { (*ALLOCATOR).free_regions() }.is_err()
    {
        fail("long acceptance accounting/registers");
    }
    for (label, value) in [
        ("FREQUENCY", frequency),
        ("ELAPSED-TICKS", end - start),
        ("DELIVERED", ticks),
        ("SKIPPED-PERIODS", unsafe { MISSED }),
        ("MAIN-PROGRESS", progress),
        ("QUEUE-CONSUMED", consumed),
        ("QUEUE-DROPPED", dropped),
        ("QUEUE-MAX", unsafe { MAX_OCCUPIED }),
        ("STORM-INTERLEAVED", unsafe { STORM_INTERLEAVED }),
        ("MAX-LATENESS-TICKS", unsafe { MAX_LATE }),
        ("FREE-BEFORE", unsafe { FREE }),
        ("FREE-AFTER", unsafe { (*ALLOCATOR).remaining_pages() }),
    ] {
        write_str("TURKOS/IRQ-SERVICE-");
        write_str(label);
        write_str(": 0x");
        write_hex(value as usize);
        write_str("\r\n");
    }
    for (bucket, count) in histogram.iter().enumerate() {
        write_str("TURKOS/IRQ-SERVICE-LATENCY: BUCKET=0x");
        write_hex(bucket);
        write_str(" COUNT=0x");
        write_hex(*count as usize);
        write_str("\r\n");
    }
    write_line("TURKOS/IRQ-SERVICE: GUARDS=INTACT GP-SIMD=RESTORED PPI=MASKED DMA-PAGES=0 PASS");
    park()
}