/* * Copyright (c) 2016 Jean-Paul Etienne * Copyright (c) 2018 Foundries.io Ltd * Copyright (c) 2020 BayLibre, SAS * * SPDX-License-Identifier: Apache-2.0 */ #include #include #include #include #include #include #include #include #include #include #include "asm_macros.inc" #ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING #include #endif /* Convenience macro for loading/storing register states. */ #define DO_CALLER_SAVED(op) \ RV_E( op t0, __z_arch_esf_t_t0_OFFSET(sp) );\ RV_E( op t1, __z_arch_esf_t_t1_OFFSET(sp) );\ RV_E( op t2, __z_arch_esf_t_t2_OFFSET(sp) );\ RV_I( op t3, __z_arch_esf_t_t3_OFFSET(sp) );\ RV_I( op t4, __z_arch_esf_t_t4_OFFSET(sp) );\ RV_I( op t5, __z_arch_esf_t_t5_OFFSET(sp) );\ RV_I( op t6, __z_arch_esf_t_t6_OFFSET(sp) );\ RV_E( op a0, __z_arch_esf_t_a0_OFFSET(sp) );\ RV_E( op a1, __z_arch_esf_t_a1_OFFSET(sp) );\ RV_E( op a2, __z_arch_esf_t_a2_OFFSET(sp) );\ RV_E( op a3, __z_arch_esf_t_a3_OFFSET(sp) );\ RV_E( op a4, __z_arch_esf_t_a4_OFFSET(sp) );\ RV_E( op a5, __z_arch_esf_t_a5_OFFSET(sp) );\ RV_I( op a6, __z_arch_esf_t_a6_OFFSET(sp) );\ RV_I( op a7, __z_arch_esf_t_a7_OFFSET(sp) );\ RV_E( op ra, __z_arch_esf_t_ra_OFFSET(sp) ) .macro get_current_cpu dst #if defined(CONFIG_SMP) || defined(CONFIG_USERSPACE) csrr \dst, mscratch #else la \dst, _kernel + ___kernel_t_cpus_OFFSET #endif .endm /* imports */ GDATA(_sw_isr_table) #ifdef CONFIG_RISCV_SOC_EXCEPTION_FROM_IRQ GTEXT(__soc_is_irq) #endif GTEXT(__soc_handle_irq) GTEXT(_Fault) #ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE GTEXT(__soc_save_context) GTEXT(__soc_restore_context) #endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */ GTEXT(z_riscv_fatal_error) GTEXT(z_get_next_switch_handle) GTEXT(z_riscv_switch) GTEXT(z_riscv_thread_start) #ifdef CONFIG_TRACING GTEXT(sys_trace_isr_enter) GTEXT(sys_trace_isr_exit) #endif #ifdef CONFIG_USERSPACE GDATA(_k_syscall_table) #endif #ifdef CONFIG_RISCV_SOC_HAS_CUSTOM_IRQ_HANDLING GTEXT(__soc_handle_all_irqs) #endif /* exports */ GTEXT(_isr_wrapper) /* use ABI name of registers for the sake of simplicity */ /* * Generic architecture-level IRQ handling, along with callouts to * SoC-specific routines. * * Architecture level IRQ handling includes basic context save/restore * of standard registers and calling ISRs registered at Zephyr's driver * level. * * Since RISC-V does not completely prescribe IRQ handling behavior, * implementations vary (some implementations also deviate from * what standard behavior is defined). Hence, the arch level code expects * the following functions to be provided at the SOC level: * * - __soc_is_irq (optional): decide if we're handling an interrupt or an exception * - __soc_handle_irq: handle SoC-specific details for a pending IRQ * (e.g. clear a pending bit in a SoC-specific register) * * If CONFIG_RISCV_SOC_CONTEXT_SAVE=y, calls to SoC-level context save/restore * routines are also made here. For details, see the Kconfig help text. */ /* * Handler called upon each exception/interrupt/fault */ SECTION_FUNC(exception.entry, _isr_wrapper) /* Provide requested alignment, which depends e.g. on MTVEC format */ .balign CONFIG_RISCV_TRAP_HANDLER_ALIGNMENT #ifdef CONFIG_USERSPACE /* retrieve address of _current_cpu preserving s0 */ csrrw s0, mscratch, s0 /* preserve t0 and t1 temporarily */ sr t0, _curr_cpu_arch_user_exc_tmp0(s0) sr t1, _curr_cpu_arch_user_exc_tmp1(s0) /* determine if we come from user space */ csrr t0, mstatus li t1, MSTATUS_MPP and t0, t0, t1 bnez t0, 1f /* in user space we were: switch to our privileged stack */ mv t0, sp lr sp, _curr_cpu_arch_user_exc_sp(s0) /* Save user stack value. Coming from user space, we know this * can't overflow the privileged stack. The esf will be allocated * later but it is safe to store our saved user sp here. */ sr t0, (-__z_arch_esf_t_SIZEOF + __z_arch_esf_t_sp_OFFSET)(sp) /* Make sure tls pointer is sane */ lr t0, ___cpu_t_current_OFFSET(s0) lr tp, _thread_offset_to_tls(t0) /* Clear our per-thread usermode flag */ lui t0, %tprel_hi(is_user_mode) add t0, t0, tp, %tprel_add(is_user_mode) sb zero, %tprel_lo(is_user_mode)(t0) 1: /* retrieve original t0/t1 values */ lr t0, _curr_cpu_arch_user_exc_tmp0(s0) lr t1, _curr_cpu_arch_user_exc_tmp1(s0) /* retrieve original s0 and restore _current_cpu in mscratch */ csrrw s0, mscratch, s0 #endif #ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING SOC_ISR_SW_STACKING #else /* Save caller-saved registers on current thread stack. */ addi sp, sp, -__z_arch_esf_t_SIZEOF DO_CALLER_SAVED(sr) ; #endif /* CONFIG_RISCV_SOC_HAS_ISR_STACKING */ /* Save s0 in the esf and load it with &_current_cpu. */ sr s0, __z_arch_esf_t_s0_OFFSET(sp) get_current_cpu s0 /* Save MEPC register */ csrr t0, mepc sr t0, __z_arch_esf_t_mepc_OFFSET(sp) /* Save MSTATUS register */ csrr t2, mstatus sr t2, __z_arch_esf_t_mstatus_OFFSET(sp) #if defined(CONFIG_FPU_SHARING) /* determine if FPU access was disabled */ li t1, MSTATUS_FS and t1, t1, t2 bnez t1, no_fp /* determine if this is an Illegal Instruction exception */ csrr t2, mcause li t1, 2 /* 2 = illegal instruction */ bne t1, t2, no_fp /* determine if we trapped on an FP instruction. */ csrr t2, mtval /* get faulting instruction */ #ifdef CONFIG_QEMU_TARGET /* * Some implementations may not support MTVAL in this capacity. * Notably QEMU when a CSR instruction is involved. */ bnez t2, 1f lw t2, 0(t0) /* t0 = mepc */ 1: #endif andi t0, t2, 0x7f /* keep only the opcode bits */ /* * Major FP opcodes: * 0000111 = LOAD-FP * 0100111 = STORE-FP * 1000011 = MADD * 1000111 = MSUB * 1001011 = NMSUB * 1001111 = NMADD * 1010011 = OP-FP */ xori t1, t0, 0b1010011 /* OP-FP */ beqz t1, is_fp ori t1, t0, 0b0100000 xori t1, t1, 0b0100111 /* LOAD-FP / STORE-FP */ beqz t1, is_fp ori t1, t0, 0b0001100 xori t1, t1, 0b1001111 /* MADD / MSUB / NMSUB / NMADD */ beqz t1, is_fp /* * The FRCSR, FSCSR, FRRM, FSRM, FSRMI, FRFLAGS, FSFLAGS and FSFLAGSI * are in fact CSR instructions targeting the fcsr, frm and fflags * registers. They should be caught as FPU instructions as well. * * CSR format: csr#[31-20] src[19-15] op[14-12] dst[11-7] SYSTEM[6-0] * SYSTEM = 0b1110011, op = 0b.xx where xx is never 0 * The csr# of interest are: 1=fflags, 2=frm, 3=fcsr */ xori t1, t0, 0b1110011 /* SYSTEM opcode */ bnez t1, 2f /* not a CSR insn */ srli t0, t2, 12 andi t0, t0, 0x3 beqz t0, 2f /* not a CSR insn */ srli t0, t2, 20 /* isolate the csr register number */ beqz t0, 2f /* 0=ustatus */ andi t0, t0, ~0x3 /* 1=fflags, 2=frm, 3=fcsr */ #if !defined(CONFIG_RISCV_ISA_EXT_C) bnez t0, no_fp #else beqz t0, is_fp 2: /* remaining non RVC (0b11) and RVC with 0b01 are not FP instructions */ andi t1, t2, 1 bnez t1, no_fp /* * 001...........00 = C.FLD RV32/64 (RV128 = C.LQ) * 001...........10 = C.FLDSP RV32/64 (RV128 = C.LQSP) * 011...........00 = C.FLW RV32 (RV64/128 = C.LD) * 011...........10 = C.FLWSPP RV32 (RV64/128 = C.LDSP) * 101...........00 = C.FSD RV32/64 (RV128 = C.SQ) * 101...........10 = C.FSDSP RV32/64 (RV128 = C.SQSP) * 111...........00 = C.FSW RV32 (RV64/128 = C.SD) * 111...........10 = C.FSWSP RV32 (RV64/128 = C.SDSP) * * so must be .01............. on RV64 and ..1............. on RV32. */ srli t0, t2, 8 #if defined(CONFIG_64BIT) andi t1, t0, 0b01100000 xori t1, t1, 0b00100000 bnez t1, no_fp #else andi t1, t0, 0b00100000 beqz t1, no_fp #endif #endif /* CONFIG_RISCV_ISA_EXT_C */ is_fp: /* Process the FP trap and quickly return from exception */ la ra, fp_trap_exit mv a0, sp tail z_riscv_fpu_trap 2: no_fp: /* increment _current->arch.exception_depth */ lr t0, ___cpu_t_current_OFFSET(s0) lb t1, _thread_offset_to_exception_depth(t0) add t1, t1, 1 sb t1, _thread_offset_to_exception_depth(t0) /* configure the FPU for exception mode */ call z_riscv_fpu_enter_exc #endif /* CONFIG_FPU_SHARING */ #ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE /* Handle context saving at SOC level. */ addi a0, sp, __z_arch_esf_t_soc_context_OFFSET jal ra, __soc_save_context #endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */ /* * Check if exception is the result of an interrupt or not. * (SOC dependent). Following the RISC-V architecture spec, the MSB * of the mcause register is used to indicate whether an exception * is the result of an interrupt or an exception/fault. But for some * SOCs (like pulpino or riscv-qemu), the MSB is never set to indicate * interrupt. Hence, check for interrupt/exception via the __soc_is_irq * function (that needs to be implemented by each SOC). The result is * returned via register a0 (1: interrupt, 0 exception) */ #ifdef CONFIG_RISCV_SOC_EXCEPTION_FROM_IRQ jal ra, __soc_is_irq bnez a0, is_interrupt #else csrr t0, mcause srli t0, t0, RISCV_MCAUSE_IRQ_POS bnez t0, is_interrupt #endif /* * If the exception is the result of an ECALL, check whether to * perform a context-switch or an IRQ offload. Otherwise call _Fault * to report the exception. */ csrr t0, mcause li t2, CONFIG_RISCV_MCAUSE_EXCEPTION_MASK and t0, t0, t2 /* * If mcause == RISCV_EXC_ECALLM, handle system call from * kernel thread. */ li t1, RISCV_EXC_ECALLM beq t0, t1, is_kernel_syscall #ifdef CONFIG_USERSPACE /* * If mcause == RISCV_EXC_ECALLU, handle system call * for user mode thread. */ li t1, RISCV_EXC_ECALLU beq t0, t1, is_user_syscall #endif /* CONFIG_USERSPACE */ /* * Call _Fault to handle exception. * Stack pointer is pointing to a z_arch_esf_t structure, pass it * to _Fault (via register a0). * If _Fault shall return, set return address to * no_reschedule to restore stack. */ mv a0, sp la ra, no_reschedule tail _Fault is_kernel_syscall: /* * A syscall is the result of an ecall instruction, in which case the * MEPC will contain the address of the ecall instruction. * Increment saved MEPC by 4 to prevent triggering the same ecall * again upon exiting the ISR. * * It's safe to always increment by 4, even with compressed * instructions, because the ecall instruction is always 4 bytes. */ lr t0, __z_arch_esf_t_mepc_OFFSET(sp) addi t0, t0, 4 sr t0, __z_arch_esf_t_mepc_OFFSET(sp) #ifdef CONFIG_PMP_STACK_GUARD /* Re-activate PMP for m-mode */ li t1, MSTATUS_MPP csrc mstatus, t1 li t1, MSTATUS_MPRV csrs mstatus, t1 #endif /* Determine what to do. Operation code is in t0. */ lr t0, __z_arch_esf_t_t0_OFFSET(sp) .if RV_ECALL_RUNTIME_EXCEPT != 0; .err; .endif beqz t0, do_fault #if defined(CONFIG_IRQ_OFFLOAD) li t1, RV_ECALL_IRQ_OFFLOAD beq t0, t1, do_irq_offload #endif #ifdef CONFIG_RISCV_ALWAYS_SWITCH_THROUGH_ECALL li t1, RV_ECALL_SCHEDULE bne t0, t1, skip_schedule lr a0, __z_arch_esf_t_a0_OFFSET(sp) lr a1, __z_arch_esf_t_a1_OFFSET(sp) j reschedule skip_schedule: #endif /* default fault code is K_ERR_KERNEL_OOPS */ li a0, 3 j 1f do_fault: /* Handle RV_ECALL_RUNTIME_EXCEPT. Retrieve reason in a0, esf in A1. */ lr a0, __z_arch_esf_t_a0_OFFSET(sp) 1: mv a1, sp tail z_riscv_fatal_error #if defined(CONFIG_IRQ_OFFLOAD) do_irq_offload: /* * Retrieve provided routine and argument from the stack. * Routine pointer is in saved a0, argument in saved a1 * so we load them with a1/a0 (reversed). */ lr a1, __z_arch_esf_t_a0_OFFSET(sp) lr a0, __z_arch_esf_t_a1_OFFSET(sp) /* Increment _current_cpu->nested */ lw t1, ___cpu_t_nested_OFFSET(s0) addi t2, t1, 1 sw t2, ___cpu_t_nested_OFFSET(s0) bnez t1, 1f /* Switch to interrupt stack */ mv t0, sp lr sp, ___cpu_t_irq_stack_OFFSET(s0) /* Save thread stack pointer on interrupt stack */ addi sp, sp, -16 sr t0, 0(sp) 1: /* Execute provided routine (argument is in a0 already). */ jalr ra, a1, 0 /* Leave through the regular IRQ exit path */ j irq_done #endif /* CONFIG_IRQ_OFFLOAD */ #ifdef CONFIG_USERSPACE is_user_syscall: #ifdef CONFIG_PMP_STACK_GUARD /* * We came from userspace and need to reconfigure the * PMP for kernel mode stack guard. */ lr a0, ___cpu_t_current_OFFSET(s0) call z_riscv_pmp_stackguard_enable #endif /* It is safe to re-enable IRQs now */ csrs mstatus, MSTATUS_IEN /* * Same as for is_kernel_syscall: increment saved MEPC by 4 to * prevent triggering the same ecall again upon exiting the ISR. */ lr t1, __z_arch_esf_t_mepc_OFFSET(sp) addi t1, t1, 4 sr t1, __z_arch_esf_t_mepc_OFFSET(sp) /* Restore argument registers from user stack */ lr a0, __z_arch_esf_t_a0_OFFSET(sp) lr a1, __z_arch_esf_t_a1_OFFSET(sp) lr a2, __z_arch_esf_t_a2_OFFSET(sp) lr a3, __z_arch_esf_t_a3_OFFSET(sp) lr a4, __z_arch_esf_t_a4_OFFSET(sp) lr a5, __z_arch_esf_t_a5_OFFSET(sp) lr t0, __z_arch_esf_t_t0_OFFSET(sp) #if defined(CONFIG_RISCV_ISA_RV32E) /* Stack alignment for RV32E is 4 bytes */ addi sp, sp, -4 mv t1, sp sw t1, 0(sp) #else mv a6, sp #endif /* CONFIG_RISCV_ISA_RV32E */ /* validate syscall limit */ li t1, K_SYSCALL_LIMIT bltu t0, t1, valid_syscall_id /* bad syscall id. Set arg1 to bad id and set call_id to SYSCALL_BAD */ mv a0, t0 li t0, K_SYSCALL_BAD valid_syscall_id: la t2, _k_syscall_table slli t1, t0, RV_REGSHIFT # Determine offset from indice value add t2, t2, t1 # Table addr + offset = function addr lr t2, 0(t2) # Load function address /* Execute syscall function */ jalr ra, t2, 0 #if defined(CONFIG_RISCV_ISA_RV32E) addi sp, sp, 4 #endif /* CONFIG_RISCV_ISA_RV32E */ /* Update a0 (return value) on the stack */ sr a0, __z_arch_esf_t_a0_OFFSET(sp) /* Disable IRQs again before leaving */ csrc mstatus, MSTATUS_IEN j might_have_rescheduled #endif /* CONFIG_USERSPACE */ is_interrupt: #ifdef CONFIG_PMP_STACK_GUARD #ifdef CONFIG_USERSPACE /* * If we came from userspace then we need to reconfigure the * PMP for kernel mode stack guard. */ lr t0, __z_arch_esf_t_mstatus_OFFSET(sp) li t1, MSTATUS_MPP and t0, t0, t1 bnez t0, 1f lr a0, ___cpu_t_current_OFFSET(s0) call z_riscv_pmp_stackguard_enable j 2f #endif /* CONFIG_USERSPACE */ 1: /* Re-activate PMP for m-mode */ li t1, MSTATUS_MPP csrc mstatus, t1 li t1, MSTATUS_MPRV csrs mstatus, t1 2: #endif /* Increment _current_cpu->nested */ lw t1, ___cpu_t_nested_OFFSET(s0) addi t2, t1, 1 sw t2, ___cpu_t_nested_OFFSET(s0) bnez t1, on_irq_stack /* Switch to interrupt stack */ mv t0, sp lr sp, ___cpu_t_irq_stack_OFFSET(s0) /* * Save thread stack pointer on interrupt stack * In RISC-V, stack pointer needs to be 16-byte aligned */ addi sp, sp, -16 sr t0, 0(sp) on_irq_stack: #ifdef CONFIG_RISCV_SOC_HAS_CUSTOM_IRQ_HANDLING call __soc_handle_all_irqs #else #ifdef CONFIG_TRACING_ISR call sys_trace_isr_enter #endif /* Get IRQ causing interrupt */ csrr a0, mcause li t0, CONFIG_RISCV_MCAUSE_EXCEPTION_MASK and a0, a0, t0 /* * Clear pending IRQ generating the interrupt at SOC level * Pass IRQ number to __soc_handle_irq via register a0 */ jal ra, __soc_handle_irq /* * Call corresponding registered function in _sw_isr_table. * (table is 2-word wide, we should shift index accordingly) */ la t0, _sw_isr_table slli a0, a0, (RV_REGSHIFT + 1) add t0, t0, a0 /* Load argument in a0 register */ lr a0, 0(t0) /* Load ISR function address in register t1 */ lr t1, RV_REGSIZE(t0) /* Call ISR function */ jalr ra, t1, 0 #ifdef CONFIG_TRACING_ISR call sys_trace_isr_exit #endif #endif irq_done: /* Decrement _current_cpu->nested */ lw t2, ___cpu_t_nested_OFFSET(s0) addi t2, t2, -1 sw t2, ___cpu_t_nested_OFFSET(s0) bnez t2, no_reschedule /* nested count is back to 0: Return to thread stack */ lr sp, 0(sp) #ifdef CONFIG_STACK_SENTINEL call z_check_stack_sentinel #endif check_reschedule: #ifdef CONFIG_MULTITHREADING /* Get pointer to current thread on this CPU */ lr a1, ___cpu_t_current_OFFSET(s0) /* * Get next thread to schedule with z_get_next_switch_handle(). * We pass it a NULL as we didn't save the whole thread context yet. * If no scheduling is necessary then NULL will be returned. */ addi sp, sp, -16 sr a1, 0(sp) mv a0, zero call z_get_next_switch_handle lr a1, 0(sp) addi sp, sp, 16 beqz a0, no_reschedule reschedule: /* * Perform context switch: * a0 = new thread * a1 = old thread */ call z_riscv_switch z_riscv_thread_start: might_have_rescheduled: /* reload s0 with &_current_cpu as it might have changed or be unset */ get_current_cpu s0 #endif /* CONFIG_MULTITHREADING */ no_reschedule: #ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE /* Restore context at SOC level */ addi a0, sp, __z_arch_esf_t_soc_context_OFFSET jal ra, __soc_restore_context #endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */ #if defined(CONFIG_FPU_SHARING) /* FPU handling upon exception mode exit */ mv a0, sp call z_riscv_fpu_exit_exc /* decrement _current->arch.exception_depth */ lr t0, ___cpu_t_current_OFFSET(s0) lb t1, _thread_offset_to_exception_depth(t0) add t1, t1, -1 sb t1, _thread_offset_to_exception_depth(t0) fp_trap_exit: #endif /* Restore MEPC and MSTATUS registers */ lr t0, __z_arch_esf_t_mepc_OFFSET(sp) lr t2, __z_arch_esf_t_mstatus_OFFSET(sp) csrw mepc, t0 csrw mstatus, t2 #ifdef CONFIG_USERSPACE /* * Check if we are returning to user mode. If so then we must * set is_user_mode to true and preserve our kernel mode stack for * the next exception to come. */ li t1, MSTATUS_MPP and t0, t2, t1 bnez t0, 1f #ifdef CONFIG_PMP_STACK_GUARD /* Remove kernel stack guard and Reconfigure PMP for user mode */ lr a0, ___cpu_t_current_OFFSET(s0) call z_riscv_pmp_usermode_enable #endif /* Set our per-thread usermode flag */ li t1, 1 lui t0, %tprel_hi(is_user_mode) add t0, t0, tp, %tprel_add(is_user_mode) sb t1, %tprel_lo(is_user_mode)(t0) /* preserve stack pointer for next exception entry */ add t0, sp, __z_arch_esf_t_SIZEOF sr t0, _curr_cpu_arch_user_exc_sp(s0) j 2f 1: /* * We are returning to kernel mode. Store the stack pointer to * be re-loaded further down. */ addi t0, sp, __z_arch_esf_t_SIZEOF sr t0, __z_arch_esf_t_sp_OFFSET(sp) 2: #endif /* Restore s0 (it is no longer ours) */ lr s0, __z_arch_esf_t_s0_OFFSET(sp) #ifdef CONFIG_RISCV_SOC_HAS_ISR_STACKING SOC_ISR_SW_UNSTACKING #else /* Restore caller-saved registers from thread stack */ DO_CALLER_SAVED(lr) #ifdef CONFIG_USERSPACE /* retrieve saved stack pointer */ lr sp, __z_arch_esf_t_sp_OFFSET(sp) #else /* remove esf from the stack */ addi sp, sp, __z_arch_esf_t_SIZEOF #endif #endif /* CONFIG_RISCV_SOC_HAS_ISR_STACKING */ mret