.section .text.irq_return,"ax"
.equ IRQ_CONTEXT_SIZE, {ctx_size}
.equ IRQ_SP_OFFSET, {sp_offset}
.equ IRQ_SP_EL0_OFFSET, {sp_el0_offset}
.equ IRQ_ELR_OFFSET, {elr_offset}
.equ IRQ_SPSR_OFFSET, {spsr_offset}
.equ IRQ_Q_OFFSET, {q_offset}
.equ IRQ_FPCR_OFFSET, {fpcr_offset}
.equ IRQ_FPSR_OFFSET, {fpsr_offset}
.equ IRQ_SLOT_OFFSET, {slot_offset}
.balign 128
.global __return_sync_entry
__return_sync_entry:
    // Save scratch registers on the interrupted stack before switching. The
    // isolated EL1 probes use one frame; the user-copy IRQ feature allocates
    // a second frame below its still-active syscall handler stack pointer.
    sub sp, sp, #32
    stp x0, x1, [sp, #0]
    stp x2, x3, [sp, #16]
    mov x2, #4
    b .Lreturn_saved
.global __return_irq_entry
__return_irq_entry:
    sub sp, sp, #32
    stp x0, x1, [sp, #0]
    stp x2, x3, [sp, #16]
    mov x2, #5
    b .Lreturn_saved
.global __user_syscall_sync_entry
__user_syscall_sync_entry:
    // EL0 synchronous exceptions arrive on SP_EL1. Keep every interrupted
    // register, including SIMD/FP state, on the private guarded IRQ stack.
    sub sp, sp, #32
    stp x0, x1, [sp, #0]
    stp x2, x3, [sp, #16]
    mov x2, #8
    b .Lreturn_saved
.global __user_irq_entry
__user_irq_entry:
    // An IRQ from EL0 arrives on SP_EL1. This fixed frame is distinct from
    // the user stack and is restored by the common full GP/SIMD/FP ERET path.
    sub sp, sp, #32
    stp x0, x1, [sp, #0]
    stp x2, x3, [sp, #16]
    mov x2, #9
.Lreturn_saved:
    .if {nested_irq}
    // When an EL1 IRQ interrupts the active EL0 syscall handler, its outer
    // frame already occupies the fixed top slot. Allocate a second frame
    // *below the interrupted handler SP* so neither saved state overlaps.
    cmp x2, #5
    b.ne .Lreturn_fixed_frame
    sub x0, sp, #IRQ_CONTEXT_SIZE
    b .Lreturn_frame_ready
.Lreturn_fixed_frame:
    .endif
    adrp x0, __irq_stack_top
    add x0, x0, :lo12:__irq_stack_top
    sub x0, x0, #IRQ_CONTEXT_SIZE
    .if {nested_irq}
.Lreturn_frame_ready:
    .endif
    str x2, [x0, #IRQ_SLOT_OFFSET]
    ldr x1, [sp, #0]
    str x1, [x0, #0]
    ldr x1, [sp, #8]
    str x1, [x0, #8]
    ldr x1, [sp, #16]
    str x1, [x0, #16]
    ldr x1, [sp, #24]
    str x1, [x0, #24]
    stp x4, x5, [x0, #32]
    stp x6, x7, [x0, #48]
    stp x8, x9, [x0, #64]
    stp x10, x11, [x0, #80]
    stp x12, x13, [x0, #96]
    stp x14, x15, [x0, #112]
    stp x16, x17, [x0, #128]
    stp x18, x19, [x0, #144]
    stp x20, x21, [x0, #160]
    stp x22, x23, [x0, #176]
    stp x24, x25, [x0, #192]
    stp x26, x27, [x0, #208]
    stp x28, x29, [x0, #224]
    str x30, [x0, #240]
    add x1, sp, #32
    str x1, [x0, #IRQ_SP_OFFSET]
    mrs x1, SP_EL0
    str x1, [x0, #IRQ_SP_EL0_OFFSET]
    mrs x1, ELR_EL1
    str x1, [x0, #IRQ_ELR_OFFSET]
    mrs x1, SPSR_EL1
    str x1, [x0, #IRQ_SPSR_OFFSET]
    stp q0, q1, [x0, #IRQ_Q_OFFSET + 0]
    stp q2, q3, [x0, #IRQ_Q_OFFSET + 32]
    stp q4, q5, [x0, #IRQ_Q_OFFSET + 64]
    stp q6, q7, [x0, #IRQ_Q_OFFSET + 96]
    stp q8, q9, [x0, #IRQ_Q_OFFSET + 128]
    stp q10, q11, [x0, #IRQ_Q_OFFSET + 160]
    stp q12, q13, [x0, #IRQ_Q_OFFSET + 192]
    stp q14, q15, [x0, #IRQ_Q_OFFSET + 224]
    stp q16, q17, [x0, #IRQ_Q_OFFSET + 256]
    stp q18, q19, [x0, #IRQ_Q_OFFSET + 288]
    stp q20, q21, [x0, #IRQ_Q_OFFSET + 320]
    stp q22, q23, [x0, #IRQ_Q_OFFSET + 352]
    stp q24, q25, [x0, #IRQ_Q_OFFSET + 384]
    stp q26, q27, [x0, #IRQ_Q_OFFSET + 416]
    stp q28, q29, [x0, #IRQ_Q_OFFSET + 448]
    stp q30, q31, [x0, #IRQ_Q_OFFSET + 480]
    mrs x1, FPCR
    str w1, [x0, #IRQ_FPCR_OFFSET]
    mrs x1, FPSR
    str w1, [x0, #IRQ_FPSR_OFFSET]
    mov sp, x0
    mov x0, sp
    bl kernel_return_probe
    cbz x0, .Lreturn_park

    mov x16, sp
    ldr x17, [x16, #IRQ_ELR_OFFSET]
    msr ELR_EL1, x17
    ldr x17, [x16, #IRQ_SPSR_OFFSET]
    msr SPSR_EL1, x17
    ldr x17, [x16, #IRQ_SP_EL0_OFFSET]
    msr SP_EL0, x17
    ldr w17, [x16, #IRQ_FPCR_OFFSET]
    msr FPCR, x17
    ldr w17, [x16, #IRQ_FPSR_OFFSET]
    msr FPSR, x17
    ldp q0, q1, [x16, #IRQ_Q_OFFSET + 0]
    ldp q2, q3, [x16, #IRQ_Q_OFFSET + 32]
    ldp q4, q5, [x16, #IRQ_Q_OFFSET + 64]
    ldp q6, q7, [x16, #IRQ_Q_OFFSET + 96]
    ldp q8, q9, [x16, #IRQ_Q_OFFSET + 128]
    ldp q10, q11, [x16, #IRQ_Q_OFFSET + 160]
    ldp q12, q13, [x16, #IRQ_Q_OFFSET + 192]
    ldp q14, q15, [x16, #IRQ_Q_OFFSET + 224]
    ldp q16, q17, [x16, #IRQ_Q_OFFSET + 256]
    ldp q18, q19, [x16, #IRQ_Q_OFFSET + 288]
    ldp q20, q21, [x16, #IRQ_Q_OFFSET + 320]
    ldp q22, q23, [x16, #IRQ_Q_OFFSET + 352]
    ldp q24, q25, [x16, #IRQ_Q_OFFSET + 384]
    ldp q26, q27, [x16, #IRQ_Q_OFFSET + 416]
    ldp q28, q29, [x16, #IRQ_Q_OFFSET + 448]
    ldp q30, q31, [x16, #IRQ_Q_OFFSET + 480]
    ldp x0, x1, [x16, #0]
    ldp x2, x3, [x16, #16]
    ldp x4, x5, [x16, #32]
    ldp x6, x7, [x16, #48]
    ldp x8, x9, [x16, #64]
    ldp x10, x11, [x16, #80]
    ldp x12, x13, [x16, #96]
    ldp x14, x15, [x16, #112]
    ldp x18, x19, [x16, #144]
    ldp x20, x21, [x16, #160]
    ldp x22, x23, [x16, #176]
    ldp x24, x25, [x16, #192]
    ldp x26, x27, [x16, #208]
    ldp x28, x29, [x16, #224]
    ldr x30, [x16, #240]
    ldr x17, [x16, #IRQ_SP_OFFSET]
    mov sp, x17
    ldr x17, [x16, #136]
    ldr x16, [x16, #128]
    eret
.Lreturn_park:
    wfe
    b .Lreturn_park