/* * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive * for more details. * * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle * Copyright (C) 1999, 2000 Silicon Graphics, Inc. * Copyright (C) 2002, 2007 Maciej W. Rozycki * Copyright (C) 2001, 2012 MIPS Technologies, Inc. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include __INIT /* * General exception vector for all other CPUs. * * Be careful when changing this, it has to be at most 128 bytes * to fit into space reserved for the exception handler. */ NESTED(except_vec3_generic, 0, sp) .set push .set noat mfc0 k1, CP0_CAUSE andi k1, k1, 0x7c #ifdef CONFIG_64BIT dsll k1, k1, 1 #endif PTR_L k0, exception_handlers(k1) jr k0 .set pop END(except_vec3_generic) /* * General exception handler for CPUs with virtual coherency exception. * * Be careful when changing this, it has to be at most 256 (as a special * exception) bytes to fit into space reserved for the exception handler. */ NESTED(except_vec3_r4000, 0, sp) .set push .set arch=r4000 .set noat mfc0 k1, CP0_CAUSE li k0, 31<<2 andi k1, k1, 0x7c .set push .set noreorder .set nomacro beq k1, k0, handle_vced li k0, 14<<2 beq k1, k0, handle_vcei #ifdef CONFIG_64BIT dsll k1, k1, 1 #endif .set pop PTR_L k0, exception_handlers(k1) jr k0 /* * Big shit, we now may have two dirty primary cache lines for the same * physical address. We can safely invalidate the line pointed to by * c0_badvaddr because after return from this exception handler the * load / store will be re-executed. */ handle_vced: MFC0 k0, CP0_BADVADDR li k1, -4 # Is this ... and k0, k1 # ... really needed? mtc0 zero, CP0_TAGLO cache Index_Store_Tag_D, (k0) cache Hit_Writeback_Inv_SD, (k0) #ifdef CONFIG_PROC_FS PTR_LA k0, vced_count lw k1, (k0) addiu k1, 1 sw k1, (k0) #endif eret handle_vcei: MFC0 k0, CP0_BADVADDR cache Hit_Writeback_Inv_SD, (k0) # also cleans pi #ifdef CONFIG_PROC_FS PTR_LA k0, vcei_count lw k1, (k0) addiu k1, 1 sw k1, (k0) #endif eret .set pop END(except_vec3_r4000) __FINIT .align 5 /* 32 byte rollback region */ LEAF(__r4k_wait) .set push .set noreorder /* start of rollback region */ LONG_L t0, TI_FLAGS($28) nop andi t0, _TIF_NEED_RESCHED bnez t0, 1f nop nop nop #ifdef CONFIG_CPU_MICROMIPS nop nop nop nop #endif .set MIPS_ISA_ARCH_LEVEL_RAW wait /* end of rollback region (the region size must be power of two) */ 1: jr ra nop .set pop END(__r4k_wait) .macro BUILD_ROLLBACK_PROLOGUE handler FEXPORT(rollback_\handler) .set push .set noat MFC0 k0, CP0_EPC PTR_LA k1, __r4k_wait ori k0, 0x1f /* 32 byte rollback region */ xori k0, 0x1f bne k0, k1, \handler MTC0 k0, CP0_EPC .set pop .endm .align 5 BUILD_ROLLBACK_PROLOGUE handle_int NESTED(handle_int, PT_SIZE, sp) .cfi_signal_frame #ifdef CONFIG_TRACE_IRQFLAGS /* * Check to see if the interrupted code has just disabled * interrupts and ignore this interrupt for now if so. * * local_irq_disable() disables interrupts and then calls * trace_hardirqs_off() to track the state. If an interrupt is taken * after interrupts are disabled but before the state is updated * it will appear to restore_all that it is incorrectly returning with * interrupts disabled */ .set push .set noat mfc0 k0, CP0_STATUS #if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) and k0, ST0_IEP bnez k0, 1f mfc0 k0, CP0_EPC .set noreorder j k0 rfe #else and k0, ST0_IE bnez k0, 1f eret #endif 1: .set pop #endif SAVE_ALL docfi=1 CLI TRACE_IRQS_OFF LONG_L s0, TI_REGS($28) LONG_S sp, TI_REGS($28) /* * SAVE_ALL ensures we are using a valid kernel stack for the thread. * Check if we are already using the IRQ stack. */ move s1, sp # Preserve the sp /* Get IRQ stack for this CPU */ ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG #if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) lui k1, %hi(irq_stack) #else lui k1, %highest(irq_stack) daddiu k1, %higher(irq_stack) dsll k1, 16 daddiu k1, %hi(irq_stack) dsll k1, 16 #endif LONG_SRL k0, SMP_CPUID_PTRSHIFT LONG_ADDU k1, k0 LONG_L t0, %lo(irq_stack)(k1) # Check if already on IRQ stack PTR_LI t1, ~(_THREAD_SIZE-1) and t1, t1, sp beq t0, t1, 2f /* Switch to IRQ stack */ li t1, _IRQ_STACK_START PTR_ADD sp, t0, t1 /* Save task's sp on IRQ stack so that unwinding can follow it */ LONG_S s1, 0(sp) 2: jal plat_irq_dispatch /* Restore sp */ move sp, s1 j ret_from_irq #ifdef CONFIG_CPU_MICROMIPS nop #endif END(handle_int) __INIT /* * Special interrupt vector for MIPS64 ISA & embedded MIPS processors. * This is a dedicated interrupt exception vector which reduces the * interrupt processing overhead. The jump instruction will be replaced * at the initialization time. * * Be careful when changing this, it has to be at most 128 bytes * to fit into space reserved for the exception handler. */ NESTED(except_vec4, 0, sp) 1: j 1b /* Dummy, will be replaced */ END(except_vec4) /* * EJTAG debug exception handler. * The EJTAG debug exception entry point is 0xbfc00480, which * normally is in the boot PROM, so the boot PROM must do an * unconditional jump to this vector. */ NESTED(except_vec_ejtag_debug, 0, sp) j ejtag_debug_handler #ifdef CONFIG_CPU_MICROMIPS nop #endif END(except_vec_ejtag_debug) __FINIT /* * Vectored interrupt handler. * This prototype is copied to ebase + n*IntCtl.VS and patched * to invoke the handler */ BUILD_ROLLBACK_PROLOGUE except_vec_vi NESTED(except_vec_vi, 0, sp) SAVE_SOME docfi=1 SAVE_AT docfi=1 .set push .set noreorder PTR_LA v1, except_vec_vi_handler FEXPORT(except_vec_vi_lui) lui v0, 0 /* Patched */ jr v1 FEXPORT(except_vec_vi_ori) ori v0, 0 /* Patched */ .set pop END(except_vec_vi) EXPORT(except_vec_vi_end) /* * Common Vectored Interrupt code * Complete the register saves and invoke the handler which is passed in $v0 */ NESTED(except_vec_vi_handler, 0, sp) SAVE_TEMP SAVE_STATIC CLI #ifdef CONFIG_TRACE_IRQFLAGS move s0, v0 TRACE_IRQS_OFF move v0, s0 #endif LONG_L s0, TI_REGS($28) LONG_S sp, TI_REGS($28) /* * SAVE_ALL ensures we are using a valid kernel stack for the thread. * Check if we are already using the IRQ stack. */ move s1, sp # Preserve the sp /* Get IRQ stack for this CPU */ ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG #if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) lui k1, %hi(irq_stack) #else lui k1, %highest(irq_stack) daddiu k1, %higher(irq_stack) dsll k1, 16 daddiu k1, %hi(irq_stack) dsll k1, 16 #endif LONG_SRL k0, SMP_CPUID_PTRSHIFT LONG_ADDU k1, k0 LONG_L t0, %lo(irq_stack)(k1) # Check if already on IRQ stack PTR_LI t1, ~(_THREAD_SIZE-1) and t1, t1, sp beq t0, t1, 2f /* Switch to IRQ stack */ li t1, _IRQ_STACK_START PTR_ADD sp, t0, t1 /* Save task's sp on IRQ stack so that unwinding can follow it */ LONG_S s1, 0(sp) 2: jalr v0 /* Restore sp */ move sp, s1 j ret_from_irq END(except_vec_vi_handler) /* * EJTAG debug exception handler. */ NESTED(ejtag_debug_handler, PT_SIZE, sp) .set push .set noat MTC0 k0, CP0_DESAVE mfc0 k0, CP0_DEBUG sll k0, k0, 30 # Check for SDBBP. bgez k0, ejtag_return #ifdef CONFIG_SMP 1: PTR_LA k0, ejtag_debug_buffer_spinlock ll k0, 0(k0) bnez k0, 1b PTR_LA k0, ejtag_debug_buffer_spinlock sc k0, 0(k0) beqz k0, 1b # ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC sync # endif PTR_LA k0, ejtag_debug_buffer LONG_S k1, 0(k0) ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG PTR_SRL k1, SMP_CPUID_PTRSHIFT PTR_SLL k1, LONGLOG PTR_LA k0, ejtag_debug_buffer_per_cpu PTR_ADDU k0, k1 PTR_LA k1, ejtag_debug_buffer LONG_L k1, 0(k1) LONG_S k1, 0(k0) PTR_LA k0, ejtag_debug_buffer_spinlock sw zero, 0(k0) #else PTR_LA k0, ejtag_debug_buffer LONG_S k1, 0(k0) #endif SAVE_ALL move a0, sp jal ejtag_exception_handler RESTORE_ALL #ifdef CONFIG_SMP ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG PTR_SRL k1, SMP_CPUID_PTRSHIFT PTR_SLL k1, LONGLOG PTR_LA k0, ejtag_debug_buffer_per_cpu PTR_ADDU k0, k1 LONG_L k1, 0(k0) #else PTR_LA k0, ejtag_debug_buffer LONG_L k1, 0(k0) #endif ejtag_return: back_to_back_c0_hazard MFC0 k0, CP0_DESAVE .set mips32 deret .set pop END(ejtag_debug_handler) /* * This buffer is reserved for the use of the EJTAG debug * handler. */ .data EXPORT(ejtag_debug_buffer) .fill LONGSIZE #ifdef CONFIG_SMP EXPORT(ejtag_debug_buffer_spinlock) .fill LONGSIZE EXPORT(ejtag_debug_buffer_per_cpu) .fill LONGSIZE * NR_CPUS #endif .previous __INIT /* * NMI debug exception handler for MIPS reference boards. * The NMI debug exception entry point is 0xbfc00000, which * normally is in the boot PROM, so the boot PROM must do a * unconditional jump to this vector. */ NESTED(except_vec_nmi, 0, sp) j nmi_handler #ifdef CONFIG_CPU_MICROMIPS nop #endif END(except_vec_nmi) __FINIT NESTED(nmi_handler, PT_SIZE, sp) .cfi_signal_frame .set push .set noat /* * Clear ERL - restore segment mapping * Clear BEV - required for page fault exception handler to work */ mfc0 k0, CP0_STATUS ori k0, k0, ST0_EXL li k1, ~(ST0_BEV | ST0_ERL) and k0, k0, k1 mtc0 k0, CP0_STATUS _ehb SAVE_ALL move a0, sp jal nmi_exception_handler /* nmi_exception_handler never returns */ .set pop END(nmi_handler) .macro __build_clear_none .endm .macro __build_clear_sti TRACE_IRQS_ON STI .endm .macro __build_clear_cli CLI TRACE_IRQS_OFF .endm .macro __build_clear_fpe .set push /* gas fails to assemble cfc1 for some archs (octeon).*/ \ .set mips1 SET_HARDFLOAT cfc1 a1, fcr31 .set pop CLI TRACE_IRQS_OFF .endm .macro __build_clear_msa_fpe _cfcmsa a1, MSA_CSR CLI TRACE_IRQS_OFF .endm .macro __build_clear_ade MFC0 t0, CP0_BADVADDR PTR_S t0, PT_BVADDR(sp) KMODE .endm .macro __BUILD_silent exception .endm /* Gas tries to parse the PRINT argument as a string containing string escapes and emits bogus warnings if it believes to recognize an unknown escape code. So make the arguments start with an n and gas will believe \n is ok ... */ .macro __BUILD_verbose nexception LONG_L a1, PT_EPC(sp) #ifdef CONFIG_32BIT PRINT("Got \nexception at %08lx\012") #endif #ifdef CONFIG_64BIT PRINT("Got \nexception at %016lx\012") #endif .endm .macro __BUILD_count exception LONG_L t0,exception_count_\exception LONG_ADDIU t0, 1 LONG_S t0,exception_count_\exception .comm exception_count\exception, 8, 8 .endm .macro __BUILD_HANDLER exception handler clear verbose ext .align 5 NESTED(handle_\exception, PT_SIZE, sp) .cfi_signal_frame .set noat SAVE_ALL FEXPORT(handle_\exception\ext) __build_clear_\clear .set at __BUILD_\verbose \exception move a0, sp jal do_\handler j ret_from_exception END(handle_\exception) .endm .macro BUILD_HANDLER exception handler clear verbose __BUILD_HANDLER \exception \handler \clear \verbose _int .endm BUILD_HANDLER adel ade ade silent /* #4 */ BUILD_HANDLER ades ade ade silent /* #5 */ BUILD_HANDLER ibe be cli silent /* #6 */ BUILD_HANDLER dbe be cli silent /* #7 */ BUILD_HANDLER bp bp sti silent /* #9 */ BUILD_HANDLER ri ri sti silent /* #10 */ BUILD_HANDLER cpu cpu sti silent /* #11 */ BUILD_HANDLER ov ov sti silent /* #12 */ BUILD_HANDLER tr tr sti silent /* #13 */ BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent /* #14 */ #ifdef CONFIG_MIPS_FP_SUPPORT BUILD_HANDLER fpe fpe fpe silent /* #15 */ #endif BUILD_HANDLER ftlb ftlb none silent /* #16 */ BUILD_HANDLER msa msa sti silent /* #21 */ BUILD_HANDLER mdmx mdmx sti silent /* #22 */ #ifdef CONFIG_HARDWARE_WATCHPOINTS /* * For watch, interrupts will be enabled after the watch * registers are read. */ BUILD_HANDLER watch watch cli silent /* #23 */ #else BUILD_HANDLER watch watch sti verbose /* #23 */ #endif BUILD_HANDLER mcheck mcheck cli verbose /* #24 */ BUILD_HANDLER mt mt sti silent /* #25 */ BUILD_HANDLER dsp dsp sti silent /* #26 */ BUILD_HANDLER reserved reserved sti verbose /* others */ .align 5 LEAF(handle_ri_rdhwr_tlbp) .set push .set noat .set noreorder /* check if TLB contains a entry for EPC */ MFC0 k1, CP0_ENTRYHI andi k1, MIPS_ENTRYHI_ASID | MIPS_ENTRYHI_ASIDX MFC0 k0, CP0_EPC PTR_SRL k0, _PAGE_SHIFT + 1 PTR_SLL k0, _PAGE_SHIFT + 1 or k1, k0 MTC0 k1, CP0_ENTRYHI mtc0_tlbw_hazard tlbp tlb_probe_hazard mfc0 k1, CP0_INDEX .set pop bltz k1, handle_ri /* slow path */ /* fall thru */ END(handle_ri_rdhwr_tlbp) LEAF(handle_ri_rdhwr) .set push .set noat .set noreorder /* MIPS32: 0x7c03e83b: rdhwr v1,$29 */ /* microMIPS: 0x007d6b3c: rdhwr v1,$29 */ MFC0 k1, CP0_EPC #if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2) and k0, k1, 1 beqz k0, 1f xor k1, k0 lhu k0, (k1) lhu k1, 2(k1) ins k1, k0, 16, 16 lui k0, 0x007d b docheck ori k0, 0x6b3c 1: lui k0, 0x7c03 lw k1, (k1) ori k0, 0xe83b #else andi k0, k1, 1 bnez k0, handle_ri lui k0, 0x7c03 lw k1, (k1) ori k0, 0xe83b #endif .set reorder docheck: bne k0, k1, handle_ri /* if not ours */ isrdhwr: /* The insn is rdhwr. No need to check CAUSE.BD here. */ get_saved_sp /* k1 := current_thread_info */ .set noreorder MFC0 k0, CP0_EPC #if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) ori k1, _THREAD_MASK xori k1, _THREAD_MASK LONG_L v1, TI_TP_VALUE(k1) LONG_ADDIU k0, 4 jr k0 rfe #else #ifndef CONFIG_CPU_DADDI_WORKAROUNDS LONG_ADDIU k0, 4 /* stall on $k0 */ #else .set at=v1 LONG_ADDIU k0, 4 .set noat #endif MTC0 k0, CP0_EPC /* I hope three instructions between MTC0 and ERET are enough... */ ori k1, _THREAD_MASK xori k1, _THREAD_MASK LONG_L v1, TI_TP_VALUE(k1) .set push .set arch=r4000 eret .set pop #endif .set pop END(handle_ri_rdhwr) #ifdef CONFIG_CPU_R4X00_BUGS64 /* A temporary overflow handler used by check_daddi(). */ __INIT BUILD_HANDLER daddi_ov daddi_ov none silent /* #12 */ #endif