From f05eb1d24eb564488014d90b31824f71984d7ef5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 21 Oct 2021 16:16:47 +0200 Subject: ARM: stackprotector: prefer compiler for TLS based per-task protector Currently, we implement the per-task stack protector for ARM using a GCC plugin, due to lack of native compiler support. However, work is underway to get this implemented in the compiler, which means we will be able to deprecate the GCC plugin at some point. In the meantime, we will need to support both, where the native compiler implementation is obviously preferred. So let's wire this up in Kconfig and the Makefile. Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index f0f9e8bec83a..4e301e4d2da6 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1607,10 +1607,14 @@ config XEN help Say Y if you want to run Linux in a Virtual Machine on Xen on ARM. +config CC_HAVE_STACKPROTECTOR_TLS + def_bool $(cc-option,-mtp=cp15 -mstack-protector-guard=tls -mstack-protector-guard-offset=0) + config STACKPROTECTOR_PER_TASK bool "Use a unique stack canary value for each task" - depends on GCC_PLUGINS && STACKPROTECTOR && THREAD_INFO_IN_TASK && !XIP_DEFLATED_DATA - select GCC_PLUGIN_ARM_SSP_PER_TASK + depends on STACKPROTECTOR && THREAD_INFO_IN_TASK && !XIP_DEFLATED_DATA + depends on GCC_PLUGINS || CC_HAVE_STACKPROTECTOR_TLS + select GCC_PLUGIN_ARM_SSP_PER_TASK if !CC_HAVE_STACKPROTECTOR_TLS default y help Due to the fact that GCC uses an ordinary symbol reference from -- cgit v1.2.3-58-ga151 From d4664b6c987f80338407889c1e3f3abe7e16be94 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:40 +0200 Subject: ARM: implement IRQ stacks Now that we no longer rely on the stack pointer to access the current task struct or thread info, we can implement support for IRQ stacks cleanly as well. Define a per-CPU IRQ stack and switch to this stack when taking an IRQ, provided that we were not already using that stack in the interrupted context. This is never the case for IRQs taken from user space, but ones taken while running in the kernel could fire while one taken from user space has not completed yet. Signed-off-by: Ard Biesheuvel Acked-by: Linus Walleij Tested-by: Keith Packard Acked-by: Nick Desaulniers Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 4 +++ arch/arm/include/asm/assembler.h | 4 +++ arch/arm/kernel/entry-armv.S | 54 +++++++++++++++++++++++++++++++++++++--- arch/arm/kernel/irq.c | 23 +++++++++++++++++ arch/arm/kernel/traps.c | 15 ++++++++++- arch/arm/lib/backtrace-clang.S | 8 ++++++ arch/arm/lib/backtrace.S | 8 ++++++ 7 files changed, 112 insertions(+), 4 deletions(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 4e301e4d2da6..b11846679cc6 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1166,6 +1166,10 @@ config CURRENT_POINTER_IN_TPIDRURO def_bool y depends on SMP && CPU_32v6K && !CPU_V6 +config IRQSTACKS + def_bool y + depends on GENERIC_IRQ_MULTI_HANDLER && THREAD_INFO_IN_TASK + config ARM_CPU_TOPOLOGY bool "Support cpu topology definition" depends on SMP && CPU_V7 diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 870bfaea4318..1b9d4df331aa 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -86,6 +86,10 @@ #define IMM12_MASK 0xfff +/* the frame pointer used for stack unwinding */ +ARM( fpreg .req r11 ) +THUMB( fpreg .req r7 ) + /* * Enable and disable interrupts */ diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index deff286eb5ea..1c7590eef712 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -36,10 +36,58 @@ /* * Interrupt handling. */ - .macro irq_handler + .macro irq_handler, from_user:req #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER mov r0, sp +#ifdef CONFIG_IRQSTACKS + mov_l r2, irq_stack_ptr @ Take base address + mrc p15, 0, r3, c13, c0, 4 @ Get CPU offset +#ifdef CONFIG_UNWINDER_ARM + mov fpreg, sp @ Preserve original SP +#else + mov r8, fp @ Preserve original FP + mov r9, sp @ Preserve original SP +#endif + ldr sp, [r2, r3] @ Load SP from per-CPU var + .if \from_user == 0 +UNWIND( .setfp fpreg, sp ) + @ + @ If we took the interrupt while running in the kernel, we may already + @ be using the IRQ stack, so revert to the original value in that case. + @ + subs r2, sp, r0 @ SP above bottom of IRQ stack? + rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? + movcs sp, r0 @ If so, revert to incoming SP + +#ifndef CONFIG_UNWINDER_ARM + @ + @ Inform the frame pointer unwinder where the next frame lives + @ + movcc lr, pc @ Make LR point into .entry.text so + @ that we will get a dump of the + @ exception stack for this frame. +#ifdef CONFIG_CC_IS_GCC + movcc ip, r0 @ Store the old SP in the frame record. + stmdbcc sp!, {fp, ip, lr, pc} @ Push frame record + addcc fp, sp, #12 +#else + stmdbcc sp!, {fp, lr} @ Push frame record + movcc fp, sp +#endif // CONFIG_CC_IS_GCC +#endif // CONFIG_UNWINDER_ARM + .endif +#endif // CONFIG_IRQSTACKS + bl generic_handle_arch_irq + +#ifdef CONFIG_IRQSTACKS +#ifdef CONFIG_UNWINDER_ARM + mov sp, fpreg @ Restore original SP +#else + mov fp, r8 @ Restore original FP + mov sp, r9 @ Restore original SP +#endif // CONFIG_UNWINDER_ARM +#endif // CONFIG_IRQSTACKS #else arch_irq_handler_default #endif @@ -199,7 +247,7 @@ ENDPROC(__dabt_svc) .align 5 __irq_svc: svc_entry - irq_handler + irq_handler from_user=0 #ifdef CONFIG_PREEMPTION ldr r8, [tsk, #TI_PREEMPT] @ get preempt count @@ -426,7 +474,7 @@ ENDPROC(__dabt_usr) __irq_usr: usr_entry kuser_cmpxchg_check - irq_handler + irq_handler from_user=1 get_thread_info tsk mov why, #0 b ret_to_user_from_irq diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index b79975bd988c..abb0aa679bba 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -43,6 +43,25 @@ unsigned long irq_err_count; +#ifdef CONFIG_IRQSTACKS + +asmlinkage DEFINE_PER_CPU_READ_MOSTLY(u8 *, irq_stack_ptr); + +static void __init init_irq_stacks(void) +{ + u8 *stack; + int cpu; + + for_each_possible_cpu(cpu) { + stack = (u8 *)__get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); + if (WARN_ON(!stack)) + break; + per_cpu(irq_stack_ptr, cpu) = &stack[THREAD_SIZE]; + } +} + +#endif + int arch_show_interrupts(struct seq_file *p, int prec) { #ifdef CONFIG_FIQ @@ -101,6 +120,10 @@ void __init init_IRQ(void) { int ret; +#ifdef CONFIG_IRQSTACKS + init_irq_stacks(); +#endif + if (IS_ENABLED(CONFIG_OF) && !machine_desc->init_irq) irqchip_init(); else diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 89be21ec3b52..b42c446cec9a 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -66,6 +66,19 @@ void dump_backtrace_entry(unsigned long where, unsigned long from, { unsigned long end = frame + 4 + sizeof(struct pt_regs); + if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER) && + IS_ENABLED(CONFIG_CC_IS_GCC) && + end > ALIGN(frame, THREAD_SIZE)) { + /* + * If we are walking past the end of the stack, it may be due + * to the fact that we are on an IRQ or overflow stack. In this + * case, we can load the address of the other stack from the + * frame record. + */ + frame = ((unsigned long *)frame)[-2] - 4; + end = frame + 4 + sizeof(struct pt_regs); + } + #ifdef CONFIG_KALLSYMS printk("%s[<%08lx>] (%ps) from [<%08lx>] (%pS)\n", loglvl, where, (void *)where, from, (void *)from); @@ -278,7 +291,7 @@ static int __die(const char *str, int err, struct pt_regs *regs) if (!user_mode(regs) || in_interrupt()) { dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp, - THREAD_SIZE + (unsigned long)task_stack_page(tsk)); + ALIGN(regs->ARM_sp, THREAD_SIZE)); dump_backtrace(regs, tsk, KERN_EMERG); dump_instr(KERN_EMERG, regs); } diff --git a/arch/arm/lib/backtrace-clang.S b/arch/arm/lib/backtrace-clang.S index 5b4bca85d06d..290c52a60fc6 100644 --- a/arch/arm/lib/backtrace-clang.S +++ b/arch/arm/lib/backtrace-clang.S @@ -197,6 +197,14 @@ finished_setup: cmp sv_fp, frame @ next frame must be mov frame, sv_fp @ above the current frame +#ifdef CONFIG_IRQSTACKS + @ + @ Kernel stacks may be discontiguous in memory. If the next + @ frame is below the previous frame, accept it as long as it + @ lives in kernel memory. + @ + cmpls sv_fp, #PAGE_OFFSET +#endif bhi for_each_frame 1006: adr r0, .Lbad diff --git a/arch/arm/lib/backtrace.S b/arch/arm/lib/backtrace.S index e8408f22d4dc..293a2716bd20 100644 --- a/arch/arm/lib/backtrace.S +++ b/arch/arm/lib/backtrace.S @@ -98,6 +98,14 @@ for_each_frame: tst frame, mask @ Check for address exceptions cmp sv_fp, frame @ next frame must be mov frame, sv_fp @ above the current frame +#ifdef CONFIG_IRQSTACKS + @ + @ Kernel stacks may be discontiguous in memory. If the next + @ frame is below the previous frame, accept it as long as it + @ lives in kernel memory. + @ + cmpls sv_fp, #PAGE_OFFSET +#endif bhi for_each_frame 1006: adr r0, .Lbad -- cgit v1.2.3-58-ga151 From 9974f857768e4ea4f18c4f6eec37178d546365ec Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 5 Oct 2021 09:15:42 +0200 Subject: ARM: run softirqs on the per-CPU IRQ stack Now that we have enabled IRQ stacks, any softIRQs that are handled over the back of a hard IRQ will run from the IRQ stack as well. However, any synchronous softirq processing that happens when re-enabling softIRQs from task context will still execute on that task's stack. Since any call to local_bh_enable() at any level in the task's call stack may trigger a softIRQ processing run, which could potentially cause a task stack overflow if the combined stack footprints exceed the stack's size, let's run these synchronous invocations of do_softirq() on the IRQ stack as well. Signed-off-by: Ard Biesheuvel Reviewed-by: Arnd Bergmann Acked-by: Linus Walleij Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 2 ++ arch/arm/kernel/irq.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index b11846679cc6..41849d5e0bc3 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1169,6 +1169,8 @@ config CURRENT_POINTER_IN_TPIDRURO config IRQSTACKS def_bool y depends on GENERIC_IRQ_MULTI_HANDLER && THREAD_INFO_IN_TASK + select HAVE_IRQ_EXIT_ON_IRQ_STACK + select HAVE_SOFTIRQ_ON_OWN_STACK config ARM_CPU_TOPOLOGY bool "Support cpu topology definition" diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index abb0aa679bba..e05219bca218 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -36,11 +36,14 @@ #include #include #include +#include #include #include #include #include +#include "reboot.h" + unsigned long irq_err_count; #ifdef CONFIG_IRQSTACKS @@ -60,6 +63,17 @@ static void __init init_irq_stacks(void) } } +static void ____do_softirq(void *arg) +{ + __do_softirq(); +} + +void do_softirq_own_stack(void) +{ + call_with_stack(____do_softirq, NULL, + __this_cpu_read(irq_stack_ptr)); +} + #endif int arch_show_interrupts(struct seq_file *p, int prec) -- cgit v1.2.3-58-ga151 From a1c510d0adc604bb143c86052bc5be48cbcfa17c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 23 Sep 2021 09:15:53 +0200 Subject: ARM: implement support for vmap'ed stacks Wire up the generic support for managing task stack allocations via vmalloc, and implement the entry code that detects whether we faulted because of a stack overrun (or future stack overrun caused by pushing the pt_regs array) While this adds a fair amount of tricky entry asm code, it should be noted that it only adds a TST + branch to the svc_entry path. The code implementing the non-trivial handling of the overflow stack is emitted out-of-line into the .text section. Since on ARM, we rely on do_translation_fault() to keep PMD level page table entries that cover the vmalloc region up to date, we need to ensure that we don't hit such a stale PMD entry when accessing the stack. So we do a dummy read from the new stack while still running from the old one on the context switch path, and bump the vmalloc_seq counter when PMD level entries in the vmalloc range are modified, so that the MM switch fetches the latest version of the entries. Note that we need to increase the per-mode stack by 1 word, to gain some space to stash a GPR until we know it is safe to touch the stack. However, due to the cacheline alignment of the struct, this does not actually increase the memory footprint of the struct stack array at all. Signed-off-by: Ard Biesheuvel Tested-by: Keith Packard Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 1 + arch/arm/include/asm/page.h | 4 ++ arch/arm/include/asm/thread_info.h | 8 ++++ arch/arm/kernel/entry-armv.S | 97 ++++++++++++++++++++++++++++++++++++-- arch/arm/kernel/entry-header.S | 37 +++++++++++++++ arch/arm/kernel/irq.c | 9 +++- arch/arm/kernel/setup.c | 8 ++-- arch/arm/kernel/sleep.S | 8 ++++ arch/arm/kernel/traps.c | 80 ++++++++++++++++++++++++++++++- arch/arm/kernel/unwind.c | 3 +- arch/arm/kernel/vmlinux.lds.S | 4 +- 11 files changed, 244 insertions(+), 15 deletions(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 41849d5e0bc3..e2ab72f2bf4a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -127,6 +127,7 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK if CURRENT_POINTER_IN_TPIDRURO + select HAVE_ARCH_VMAP_STACK if MMU && THREAD_INFO_IN_TASK && (!LD_IS_LLD || LLD_VERSION >= 140000) select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h index 11b058a72a5b..7b871ed99ccf 100644 --- a/arch/arm/include/asm/page.h +++ b/arch/arm/include/asm/page.h @@ -149,6 +149,10 @@ extern void copy_page(void *to, const void *from); #include #endif +#ifdef CONFIG_VMAP_STACK +#define ARCH_PAGE_TABLE_SYNC_MASK PGTBL_PMD_MODIFIED +#endif + #endif /* CONFIG_MMU */ typedef struct page *pgtable_t; diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 164e15f26485..004b89d86224 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -25,6 +25,14 @@ #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #define THREAD_START_SP (THREAD_SIZE - 8) +#ifdef CONFIG_VMAP_STACK +#define THREAD_ALIGN (2 * THREAD_SIZE) +#else +#define THREAD_ALIGN THREAD_SIZE +#endif + +#define OVERFLOW_STACK_SIZE SZ_4K + #ifndef __ASSEMBLY__ struct task_struct; diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 207875ac62ff..5fb7465d14d9 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -57,6 +57,10 @@ UNWIND( .setfp fpreg, sp ) @ subs r2, sp, r0 @ SP above bottom of IRQ stack? rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? +#ifdef CONFIG_VMAP_STACK + ldr_l r2, high_memory, cc @ End of the linear region + cmpcc r2, r0 @ Stack pointer was below it? +#endif movcs sp, r0 @ If so, revert to incoming SP #ifndef CONFIG_UNWINDER_ARM @@ -188,13 +192,18 @@ ENDPROC(__und_invalid) #define SPFIX(code...) #endif - .macro svc_entry, stack_hole=0, trace=1, uaccess=1 + .macro svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1 UNWIND(.fnstart ) - UNWIND(.save {r0 - pc} ) sub sp, sp, #(SVC_REGS_SIZE + \stack_hole) + THUMB( add sp, r1 ) @ get SP in a GPR without + THUMB( sub r1, sp, r1 ) @ using a temp register + + .if \overflow_check + UNWIND(.save {r0 - pc} ) + do_overflow_check (SVC_REGS_SIZE + \stack_hole) + .endif + #ifdef CONFIG_THUMB2_KERNEL - add sp, r1 @ get SP in a GPR without - sub r1, sp, r1 @ using a temp register tst r1, #4 @ test stack pointer alignment sub r1, sp, r1 @ restore original R1 sub sp, r1 @ restore original SP @@ -827,12 +836,20 @@ ENTRY(__switch_to) str r7, [r8] #endif mov r0, r5 -#if !defined(CONFIG_THUMB2_KERNEL) +#if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK) set_current r7 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously #else mov r1, r7 ldmia r4, {r4 - sl, fp, ip, lr} @ Load all regs saved previously +#ifdef CONFIG_VMAP_STACK + @ + @ Do a dummy read from the new stack while running from the old one so + @ that we can rely on do_translation_fault() to fix up any stale PMD + @ entries covering the vmalloc region. + @ + ldr r2, [ip] +#endif @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what @ effectuates the task switch, as that is what causes the observable @@ -849,6 +866,76 @@ ENTRY(__switch_to) UNWIND(.fnend ) ENDPROC(__switch_to) +#ifdef CONFIG_VMAP_STACK + .text + .align 2 +__bad_stack: + @ + @ We've just detected an overflow. We need to load the address of this + @ CPU's overflow stack into the stack pointer register. We have only one + @ scratch register so let's use a sequence of ADDs including one + @ involving the PC, and decorate them with PC-relative group + @ relocations. As these are ARM only, switch to ARM mode first. + @ + @ We enter here with IP clobbered and its value stashed on the mode + @ stack. + @ +THUMB( bx pc ) +THUMB( nop ) +THUMB( .arm ) + mrc p15, 0, ip, c13, c0, 4 @ Get per-CPU offset + + .globl overflow_stack_ptr + .reloc 0f, R_ARM_ALU_PC_G0_NC, overflow_stack_ptr + .reloc 1f, R_ARM_ALU_PC_G1_NC, overflow_stack_ptr + .reloc 2f, R_ARM_LDR_PC_G2, overflow_stack_ptr + add ip, ip, pc +0: add ip, ip, #-4 +1: add ip, ip, #0 +2: ldr ip, [ip, #4] + + str sp, [ip, #-4]! @ Preserve original SP value + mov sp, ip @ Switch to overflow stack + pop {ip} @ Original SP in IP + +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + mov ip, ip @ mov expected by unwinder + push {fp, ip, lr, pc} @ GCC flavor frame record +#else + str ip, [sp, #-8]! @ store original SP + push {fpreg, lr} @ Clang flavor frame record +#endif +UNWIND( ldr ip, [r0, #4] ) @ load exception LR +UNWIND( str ip, [sp, #12] ) @ store in the frame record + ldr ip, [r0, #12] @ reload IP + + @ Store the original GPRs to the new stack. + svc_entry uaccess=0, overflow_check=0 + +UNWIND( .save {sp, pc} ) +UNWIND( .save {fpreg, lr} ) +UNWIND( .setfp fpreg, sp ) + + ldr fpreg, [sp, #S_SP] @ Add our frame record + @ to the linked list +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC) + ldr r1, [fp, #4] @ reload SP at entry + add fp, fp, #12 +#else + ldr r1, [fpreg, #8] +#endif + str r1, [sp, #S_SP] @ store in pt_regs + + @ Stash the regs for handle_bad_stack + mov r0, sp + + @ Time to die + bl handle_bad_stack + nop +UNWIND( .fnend ) +ENDPROC(__bad_stack) +#endif + __INIT /* diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index ae24dd54e9ef..81df2a3561ca 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -423,3 +423,40 @@ scno .req r7 @ syscall number tbl .req r8 @ syscall table pointer why .req r8 @ Linux syscall (!= 0) tsk .req r9 @ current thread_info + + .macro do_overflow_check, frame_size:req +#ifdef CONFIG_VMAP_STACK + @ + @ Test whether the SP has overflowed. Task and IRQ stacks are aligned + @ so that SP & BIT(THREAD_SIZE_ORDER + PAGE_SHIFT) should always be + @ zero. + @ +ARM( tst sp, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT) ) +THUMB( tst r1, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT) ) +THUMB( it ne ) + bne .Lstack_overflow_check\@ + + .pushsection .text +.Lstack_overflow_check\@: + @ + @ The stack pointer is not pointing to a valid vmap'ed stack, but it + @ may be pointing into the linear map instead, which may happen if we + @ are already running from the overflow stack. We cannot detect overflow + @ in such cases so just carry on. + @ + str ip, [r0, #12] @ Stash IP on the mode stack + ldr_l ip, high_memory @ Start of VMALLOC space +ARM( cmp sp, ip ) @ SP in vmalloc space? +THUMB( cmp r1, ip ) +THUMB( itt lo ) + ldrlo ip, [r0, #12] @ Restore IP + blo .Lout\@ @ Carry on + +THUMB( sub r1, sp, r1 ) @ Restore original R1 +THUMB( sub sp, r1 ) @ Restore original SP + add sp, sp, #\frame_size @ Undo svc_entry's SP change + b __bad_stack @ Handle VMAP stack overflow + .popsection +.Lout\@: +#endif + .endm diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index e05219bca218..5deb40f39999 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -56,7 +56,14 @@ static void __init init_irq_stacks(void) int cpu; for_each_possible_cpu(cpu) { - stack = (u8 *)__get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); + if (!IS_ENABLED(CONFIG_VMAP_STACK)) + stack = (u8 *)__get_free_pages(GFP_KERNEL, + THREAD_SIZE_ORDER); + else + stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, + THREADINFO_GFP, NUMA_NO_NODE, + __builtin_return_address(0)); + if (WARN_ON(!stack)) break; per_cpu(irq_stack_ptr, cpu) = &stack[THREAD_SIZE]; diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 284a80c0b6e1..039feb7cd590 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -141,10 +141,10 @@ EXPORT_SYMBOL(outer_cache); int __cpu_architecture __read_mostly = CPU_ARCH_UNKNOWN; struct stack { - u32 irq[3]; - u32 abt[3]; - u32 und[3]; - u32 fiq[3]; + u32 irq[4]; + u32 abt[4]; + u32 und[4]; + u32 fiq[4]; } ____cacheline_aligned; #ifndef CONFIG_CPU_V7M diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index 43077e11dafd..803b51e5cba0 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -67,6 +67,14 @@ ENTRY(__cpu_suspend) ldr r4, =cpu_suspend_size #endif mov r5, sp @ current virtual SP +#ifdef CONFIG_VMAP_STACK + @ Run the suspend code from the overflow stack so we don't have to rely + @ on vmalloc-to-phys conversions anywhere in the arch suspend code. + @ The original SP value captured in R5 will be restored on the way out. + mov_l r6, overflow_stack_ptr @ Base pointer + mrc p15, 0, r7, c13, c0, 4 @ Get per-CPU offset + ldr sp, [r6, r7] @ Address of this CPU's overflow stack +#endif add r4, r4, #12 @ Space for pgd, virt sp, phys resume fn sub sp, sp, r4 @ allocate CPU state on stack ldr r3, =sleep_save_sp diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index b42c446cec9a..b28a705c49cb 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -121,7 +121,8 @@ void dump_backtrace_stm(u32 *stack, u32 instruction, const char *loglvl) static int verify_stack(unsigned long sp) { if (sp < PAGE_OFFSET || - (sp > (unsigned long)high_memory && high_memory != NULL)) + (!IS_ENABLED(CONFIG_VMAP_STACK) && + sp > (unsigned long)high_memory && high_memory != NULL)) return -EFAULT; return 0; @@ -291,7 +292,8 @@ static int __die(const char *str, int err, struct pt_regs *regs) if (!user_mode(regs) || in_interrupt()) { dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp, - ALIGN(regs->ARM_sp, THREAD_SIZE)); + ALIGN(regs->ARM_sp - THREAD_SIZE, THREAD_ALIGN) + + THREAD_SIZE); dump_backtrace(regs, tsk, KERN_EMERG); dump_instr(KERN_EMERG, regs); } @@ -838,3 +840,77 @@ void __init early_trap_init(void *vectors_base) */ #endif } + +#ifdef CONFIG_VMAP_STACK + +DECLARE_PER_CPU(u8 *, irq_stack_ptr); + +asmlinkage DEFINE_PER_CPU(u8 *, overflow_stack_ptr); + +static int __init allocate_overflow_stacks(void) +{ + u8 *stack; + int cpu; + + for_each_possible_cpu(cpu) { + stack = (u8 *)__get_free_page(GFP_KERNEL); + if (WARN_ON(!stack)) + return -ENOMEM; + per_cpu(overflow_stack_ptr, cpu) = &stack[OVERFLOW_STACK_SIZE]; + } + return 0; +} +early_initcall(allocate_overflow_stacks); + +asmlinkage void handle_bad_stack(struct pt_regs *regs) +{ + unsigned long tsk_stk = (unsigned long)current->stack; + unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); + unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr); + + console_verbose(); + pr_emerg("Insufficient stack space to handle exception!"); + + pr_emerg("Task stack: [0x%08lx..0x%08lx]\n", + tsk_stk, tsk_stk + THREAD_SIZE); + pr_emerg("IRQ stack: [0x%08lx..0x%08lx]\n", + irq_stk - THREAD_SIZE, irq_stk); + pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n", + ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk); + + die("kernel stack overflow", regs, 0); +} + +/* + * Normally, we rely on the logic in do_translation_fault() to update stale PMD + * entries covering the vmalloc space in a task's page tables when it first + * accesses the region in question. Unfortunately, this is not sufficient when + * the task stack resides in the vmalloc region, as do_translation_fault() is a + * C function that needs a stack to run. + * + * So we need to ensure that these PMD entries are up to date *before* the MM + * switch. As we already have some logic in the MM switch path that takes care + * of this, let's trigger it by bumping the counter every time the core vmalloc + * code modifies a PMD entry in the vmalloc region. + */ +void arch_sync_kernel_mappings(unsigned long start, unsigned long end) +{ + if (start > VMALLOC_END || end < VMALLOC_START) + return; + + /* + * This hooks into the core vmalloc code to receive notifications of + * any PMD level changes that have been made to the kernel page tables. + * This means it should only be triggered once for every MiB worth of + * vmalloc space, given that we don't support huge vmalloc/vmap on ARM, + * and that kernel PMD level table entries are rarely (if ever) + * updated. + * + * This means that the counter is going to max out at ~250 for the + * typical case. If it overflows, something entirely unexpected has + * occurred so let's throw a warning if that happens. + */ + WARN_ON(++init_mm.context.vmalloc_seq == UINT_MAX); +} + +#endif diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index e8d729975f12..c5ea328c428d 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -389,7 +389,8 @@ int unwind_frame(struct stackframe *frame) /* store the highest address on the stack to avoid crossing it*/ ctrl.sp_low = frame->sp; - ctrl.sp_high = ALIGN(ctrl.sp_low, THREAD_SIZE); + ctrl.sp_high = ALIGN(ctrl.sp_low - THREAD_SIZE, THREAD_ALIGN) + + THREAD_SIZE; pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, frame->pc, frame->lr, frame->sp); diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index f02d617e3359..aa12b65a7fd6 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -138,12 +138,12 @@ SECTIONS #ifdef CONFIG_STRICT_KERNEL_RWX . = ALIGN(1< Date: Tue, 30 Nov 2021 08:19:13 +0100 Subject: ARM: riscpc: use GENERIC_IRQ_MULTI_HANDLER This is one of the last platforms using the old entry path. While this code path is spread over a few files, it is fairly straightforward to convert it into an equivalent C version, leaving the existing algorithm and all the priority handling the same. Unlike most irqchip drivers, this means reading the status register(s) in a loop and always handling the highest-priority irq first. The IOMD_IRQREQC and IOMD_IRQREQD registers are not actaully used here, but I left the code in place for the time being, to keep the conversion as direct as possible. It could be removed in a cleanup on top. Signed-off-by: Arnd Bergmann [ardb: drop obsolete IOMD_IRQREQC/IOMD_IRQREQD handling] Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 1 + arch/arm/include/asm/hardware/entry-macro-iomd.S | 84 --------------------- arch/arm/mach-rpc/fiq.S | 5 +- arch/arm/mach-rpc/include/mach/entry-macro.S | 13 ---- arch/arm/mach-rpc/irq.c | 95 ++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 99 deletions(-) delete mode 100644 arch/arm/include/asm/hardware/entry-macro-iomd.S delete mode 100644 arch/arm/mach-rpc/include/mach/entry-macro.S (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e2ab72f2bf4a..25f1868e5703 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -444,6 +444,7 @@ config ARCH_RPC select ARM_HAS_SG_CHAIN select CPU_SA110 select FIQ + select GENERIC_IRQ_MULTI_HANDLER select HAVE_PATA_PLATFORM select ISA_DMA_API select LEGACY_TIMER_TICK diff --git a/arch/arm/include/asm/hardware/entry-macro-iomd.S b/arch/arm/include/asm/hardware/entry-macro-iomd.S deleted file mode 100644 index 81441dfa5282..000000000000 --- a/arch/arm/include/asm/hardware/entry-macro-iomd.S +++ /dev/null @@ -1,84 +0,0 @@ -/* - * arch/arm/include/asm/hardware/entry-macro-iomd.S - * - * Low-level IRQ helper macros for IOC/IOMD based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ - -/* IOC / IOMD based hardware */ -#include - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - ldrb \irqstat, [\base, #IOMD_IRQREQB] @ get high priority first - ldr \tmp, =irq_prio_h - teq \irqstat, #0 -#ifdef IOMD_BASE - ldrbeq \irqstat, [\base, #IOMD_DMAREQ] @ get dma - addeq \tmp, \tmp, #256 @ irq_prio_h table size - teqeq \irqstat, #0 - bne 2406f -#endif - ldrbeq \irqstat, [\base, #IOMD_IRQREQA] @ get low priority - addeq \tmp, \tmp, #256 @ irq_prio_d table size - teqeq \irqstat, #0 -2406: ldrbne \irqnr, [\tmp, \irqstat] @ get IRQ number - .endm - -/* - * Interrupt table (incorporates priority). Please note that we - * rely on the order of these tables (see above code). - */ - .align 5 -irq_prio_h: .byte 0, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 12, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 - .byte 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10 -#ifdef IOMD_BASE -irq_prio_d: .byte 0,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 20,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 - .byte 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16 -#endif -irq_prio_l: .byte 0, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 4, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 - .byte 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 diff --git a/arch/arm/mach-rpc/fiq.S b/arch/arm/mach-rpc/fiq.S index 0de83e9b0b39..087bdf4bc093 100644 --- a/arch/arm/mach-rpc/fiq.S +++ b/arch/arm/mach-rpc/fiq.S @@ -2,10 +2,11 @@ #include #include #include -#include - .text + .equ ioc_base_high, IOC_BASE & 0xff000000 + .equ ioc_base_low, IOC_BASE & 0x00ff0000 + .text .global rpc_default_fiq_end ENTRY(rpc_default_fiq_start) mov r12, #ioc_base_high diff --git a/arch/arm/mach-rpc/include/mach/entry-macro.S b/arch/arm/mach-rpc/include/mach/entry-macro.S deleted file mode 100644 index a6d1a9f4bb79..000000000000 --- a/arch/arm/mach-rpc/include/mach/entry-macro.S +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include -#include - - .equ ioc_base_high, IOC_BASE & 0xff000000 - .equ ioc_base_low, IOC_BASE & 0x00ff0000 - - .macro get_irqnr_preamble, base, tmp - mov \base, #ioc_base_high @ point at IOC - .if ioc_base_low - orr \base, \base, #ioc_base_low - .endif - .endm diff --git a/arch/arm/mach-rpc/irq.c b/arch/arm/mach-rpc/irq.c index 803aeb126f0e..dc29384b6ef8 100644 --- a/arch/arm/mach-rpc/irq.c +++ b/arch/arm/mach-rpc/irq.c @@ -14,6 +14,99 @@ #define CLR 0x04 #define MASK 0x08 +static const u8 irq_prio_h[256] = { + 0, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10, + 12, 8, 9, 8,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10, + 14,14,14,14,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 15,15,15,15,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, + 13,13,13,13,10,10,10,10,11,11,11,11,10,10,10,10, +}; + +static const u8 irq_prio_d[256] = { + 0,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 20,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 23,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 22,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, + 21,16,17,16,18,16,17,16,19,16,17,16,18,16,17,16, +}; + +static const u8 irq_prio_l[256] = { + 0, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 0, 1, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3, + 6, 6, 6, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3, 3, 3, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, +}; + +static int iomd_get_irq_nr(void) +{ + int irq; + u8 reg; + + /* get highest priority first */ + reg = readb(IOC_BASE + IOMD_IRQREQB); + irq = irq_prio_h[reg]; + if (irq) + return irq; + + /* get DMA */ + reg = readb(IOC_BASE + IOMD_DMAREQ); + irq = irq_prio_d[reg]; + if (irq) + return irq; + + /* get low priority */ + reg = readb(IOC_BASE + IOMD_IRQREQA); + irq = irq_prio_l[reg]; + if (irq) + return irq; + return 0; +} + +static void iomd_handle_irq(struct pt_regs *regs) +{ + int irq; + + do { + irq = iomd_get_irq_nr(); + if (irq) + generic_handle_irq(irq); + } while (irq); +} + static void __iomem *iomd_get_base(struct irq_data *d) { void *cd = irq_data_get_irq_chip_data(d); @@ -82,6 +175,8 @@ void __init rpc_init_irq(void) set_fiq_handler(&rpc_default_fiq_start, &rpc_default_fiq_end - &rpc_default_fiq_start); + set_handle_irq(iomd_handle_irq); + for (irq = 0; irq < NR_IRQS; irq++) { clr = IRQ_NOREQUEST; set = 0; -- cgit v1.2.3-58-ga151 From 90890f17ccd2aa96350abd1f4d37d4667e09027f Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 29 Nov 2021 14:43:14 +0100 Subject: ARM: footbridge: use GENERIC_IRQ_MULTI_HANDLER Footbridge still uses the classic IRQ entry path in assembler, but this is easily converted into an equivalent C version. In this case, the correlation between IRQ numbers and bits in the status register is non-obvious, and the priorities are handled by manually checking each bit in a static order, re-reading the status register after each handled event. I moved the code into the new file and edited the syntax without changing this sequence to keep the behavior as close as possible to what it traditionally did. Signed-off-by: Arnd Bergmann Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M Reviewed-by: Linus Walleij --- arch/arm/Kconfig | 1 + arch/arm/mach-footbridge/common.c | 87 +++++++++++++++++ .../arm/mach-footbridge/include/mach/entry-macro.S | 107 --------------------- 3 files changed, 88 insertions(+), 107 deletions(-) delete mode 100644 arch/arm/mach-footbridge/include/mach/entry-macro.S (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 25f1868e5703..a0cc9ca66ae0 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -362,6 +362,7 @@ config ARCH_FOOTBRIDGE select FOOTBRIDGE select NEED_MACH_IO_H if !MMU select NEED_MACH_MEMORY_H + select GENERIC_IRQ_MULTI_HANDLER help Support for systems based on the DC21285 companion chip ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder. diff --git a/arch/arm/mach-footbridge/common.c b/arch/arm/mach-footbridge/common.c index eee095f0e2f6..322495df271d 100644 --- a/arch/arm/mach-footbridge/common.c +++ b/arch/arm/mach-footbridge/common.c @@ -27,6 +27,91 @@ #include "common.h" +#include +#include +#include + +static int dc21285_get_irq(void) +{ + void __iomem *irqstatus = (void __iomem *)CSR_IRQ_STATUS; + u32 mask = readl(irqstatus); + + if (mask & IRQ_MASK_SDRAMPARITY) + return IRQ_SDRAMPARITY; + + if (mask & IRQ_MASK_UART_RX) + return IRQ_CONRX; + + if (mask & IRQ_MASK_DMA1) + return IRQ_DMA1; + + if (mask & IRQ_MASK_DMA2) + return IRQ_DMA2; + + if (mask & IRQ_MASK_IN0) + return IRQ_IN0; + + if (mask & IRQ_MASK_IN1) + return IRQ_IN1; + + if (mask & IRQ_MASK_IN2) + return IRQ_IN2; + + if (mask & IRQ_MASK_IN3) + return IRQ_IN3; + + if (mask & IRQ_MASK_PCI) + return IRQ_PCI; + + if (mask & IRQ_MASK_DOORBELLHOST) + return IRQ_DOORBELLHOST; + + if (mask & IRQ_MASK_I2OINPOST) + return IRQ_I2OINPOST; + + if (mask & IRQ_MASK_TIMER1) + return IRQ_TIMER1; + + if (mask & IRQ_MASK_TIMER2) + return IRQ_TIMER2; + + if (mask & IRQ_MASK_TIMER3) + return IRQ_TIMER3; + + if (mask & IRQ_MASK_UART_TX) + return IRQ_CONTX; + + if (mask & IRQ_MASK_PCI_ABORT) + return IRQ_PCI_ABORT; + + if (mask & IRQ_MASK_PCI_SERR) + return IRQ_PCI_SERR; + + if (mask & IRQ_MASK_DISCARD_TIMER) + return IRQ_DISCARD_TIMER; + + if (mask & IRQ_MASK_PCI_DPERR) + return IRQ_PCI_DPERR; + + if (mask & IRQ_MASK_PCI_PERR) + return IRQ_PCI_PERR; + + return 0; +} + +static void dc21285_handle_irq(struct pt_regs *regs) +{ + int irq; + do { + irq = dc21285_get_irq(); + if (!irq) + break; + + generic_handle_irq(irq); + } while (1); +} + + unsigned int mem_fclk_21285 = 50000000; EXPORT_SYMBOL(mem_fclk_21285); @@ -108,6 +193,8 @@ static void __init __fb_init_irq(void) void __init footbridge_init_irq(void) { + set_handle_irq(dc21285_handle_irq); + __fb_init_irq(); if (!footbridge_cfn_mode()) diff --git a/arch/arm/mach-footbridge/include/mach/entry-macro.S b/arch/arm/mach-footbridge/include/mach/entry-macro.S deleted file mode 100644 index dabbd5c54a78..000000000000 --- a/arch/arm/mach-footbridge/include/mach/entry-macro.S +++ /dev/null @@ -1,107 +0,0 @@ -/* - * arch/arm/mach-footbridge/include/mach/entry-macro.S - * - * Low-level IRQ helper macros for footbridge-based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ -#include -#include -#include - - .equ dc21285_high, ARMCSR_BASE & 0xff000000 - .equ dc21285_low, ARMCSR_BASE & 0x00ffffff - - .macro get_irqnr_preamble, base, tmp - mov \base, #dc21285_high - .if dc21285_low - orr \base, \base, #dc21285_low - .endif - .endm - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - ldr \irqstat, [\base, #0x180] @ get interrupts - - mov \irqnr, #IRQ_SDRAMPARITY - tst \irqstat, #IRQ_MASK_SDRAMPARITY - bne 1001f - - tst \irqstat, #IRQ_MASK_UART_RX - movne \irqnr, #IRQ_CONRX - bne 1001f - - tst \irqstat, #IRQ_MASK_DMA1 - movne \irqnr, #IRQ_DMA1 - bne 1001f - - tst \irqstat, #IRQ_MASK_DMA2 - movne \irqnr, #IRQ_DMA2 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN0 - movne \irqnr, #IRQ_IN0 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN1 - movne \irqnr, #IRQ_IN1 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN2 - movne \irqnr, #IRQ_IN2 - bne 1001f - - tst \irqstat, #IRQ_MASK_IN3 - movne \irqnr, #IRQ_IN3 - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI - movne \irqnr, #IRQ_PCI - bne 1001f - - tst \irqstat, #IRQ_MASK_DOORBELLHOST - movne \irqnr, #IRQ_DOORBELLHOST - bne 1001f - - tst \irqstat, #IRQ_MASK_I2OINPOST - movne \irqnr, #IRQ_I2OINPOST - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER1 - movne \irqnr, #IRQ_TIMER1 - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER2 - movne \irqnr, #IRQ_TIMER2 - bne 1001f - - tst \irqstat, #IRQ_MASK_TIMER3 - movne \irqnr, #IRQ_TIMER3 - bne 1001f - - tst \irqstat, #IRQ_MASK_UART_TX - movne \irqnr, #IRQ_CONTX - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_ABORT - movne \irqnr, #IRQ_PCI_ABORT - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_SERR - movne \irqnr, #IRQ_PCI_SERR - bne 1001f - - tst \irqstat, #IRQ_MASK_DISCARD_TIMER - movne \irqnr, #IRQ_DISCARD_TIMER - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_DPERR - movne \irqnr, #IRQ_PCI_DPERR - bne 1001f - - tst \irqstat, #IRQ_MASK_PCI_PERR - movne \irqnr, #IRQ_PCI_PERR -1001: - .endm - -- cgit v1.2.3-58-ga151 From 6f5d248d05db9c4991366154f1a657a630faa583 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 30 Nov 2021 11:21:49 +0100 Subject: ARM: iop32x: use GENERIC_IRQ_MULTI_HANDLER iop32x uses the entry-macro.S file for both the IRQ entry and for hooking into the arch_ret_to_user code path. This is done because the cp6 registers have to be enabled before accessing any of the interrupt controller registers but have to be disabled when running in user space. There is also a lazy-enable logic in cp6.c, but during a hardirq, we know it has to be enabled. Both the cp6-enable code and the code to read the IRQ status can be lifted into the normal generic_handle_arch_irq() path, but the cp6-disable code has to remain in the user return code. As nothing other than iop32x uses this hook, just open-code it there with an ifdef for the platform that can eventually be removed when iop32x has reached the end of its life. The cp6-enable path in the IRQ entry has an extra cp_wait barrier that the trap version does not have, but it is harmless to do it in both cases to simplify the logic here at the cost of a few extra cycles for the trap. Signed-off-by: Arnd Bergmann Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 5 +--- arch/arm/kernel/entry-common.S | 16 +++++++------ arch/arm/mach-iop32x/cp6.c | 10 +++++++- arch/arm/mach-iop32x/include/mach/entry-macro.S | 31 ------------------------- arch/arm/mach-iop32x/iop3xx.h | 1 + arch/arm/mach-iop32x/irq.c | 23 ++++++++++++++++++ 6 files changed, 43 insertions(+), 43 deletions(-) delete mode 100644 arch/arm/mach-iop32x/include/mach/entry-macro.S (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index a0cc9ca66ae0..d9ba6961b295 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -227,9 +227,6 @@ config GENERIC_ISA_DMA config FIQ bool -config NEED_RET_TO_USER - bool - config ARCH_MTD_XIP bool @@ -371,9 +368,9 @@ config ARCH_IOP32X bool "IOP32x-based" depends on MMU select CPU_XSCALE + select GENERIC_IRQ_MULTI_HANDLER select GPIO_IOP select GPIOLIB - select NEED_RET_TO_USER select FORCE_PCI select PLAT_IOP help diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index ac86c34682bb..c928d6b04cce 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -16,12 +16,14 @@ .equ NR_syscalls, __NR_syscalls -#ifdef CONFIG_NEED_RET_TO_USER -#include -#else - .macro arch_ret_to_user, tmp1, tmp2 - .endm + .macro arch_ret_to_user, tmp +#ifdef CONFIG_ARCH_IOP32X + mrc p15, 0, \tmp, c15, c1, 0 + tst \tmp, #(1 << 6) + bicne \tmp, \tmp, #(1 << 6) + mcrne p15, 0, \tmp, c15, c1, 0 @ Disable cp6 access #endif + .endm #include "entry-header.S" @@ -55,7 +57,7 @@ __ret_fast_syscall: /* perform architecture specific actions before user return */ - arch_ret_to_user r1, lr + arch_ret_to_user r1 restore_user_regs fast = 1, offset = S_OFF UNWIND(.fnend ) @@ -128,7 +130,7 @@ no_work_pending: asm_trace_hardirqs_on save = 0 /* perform architecture specific actions before user return */ - arch_ret_to_user r1, lr + arch_ret_to_user r1 ct_user_enter save = 0 restore_user_regs fast = 0, offset = 0 diff --git a/arch/arm/mach-iop32x/cp6.c b/arch/arm/mach-iop32x/cp6.c index ec74b07fb7e3..2882674a1c39 100644 --- a/arch/arm/mach-iop32x/cp6.c +++ b/arch/arm/mach-iop32x/cp6.c @@ -7,7 +7,7 @@ #include #include -static int cp6_trap(struct pt_regs *regs, unsigned int instr) +void iop_enable_cp6(void) { u32 temp; @@ -16,7 +16,15 @@ static int cp6_trap(struct pt_regs *regs, unsigned int instr) "mrc p15, 0, %0, c15, c1, 0\n\t" "orr %0, %0, #(1 << 6)\n\t" "mcr p15, 0, %0, c15, c1, 0\n\t" + "mrc p15, 0, %0, c15, c1, 0\n\t" + "mov %0, %0\n\t" + "sub pc, pc, #4 @ cp_wait\n\t" : "=r"(temp)); +} + +static int cp6_trap(struct pt_regs *regs, unsigned int instr) +{ + iop_enable_cp6(); return 0; } diff --git a/arch/arm/mach-iop32x/include/mach/entry-macro.S b/arch/arm/mach-iop32x/include/mach/entry-macro.S deleted file mode 100644 index 341e5d9a6616..000000000000 --- a/arch/arm/mach-iop32x/include/mach/entry-macro.S +++ /dev/null @@ -1,31 +0,0 @@ -/* - * arch/arm/mach-iop32x/include/mach/entry-macro.S - * - * Low-level IRQ helper macros for IOP32x-based platforms - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - */ - .macro get_irqnr_preamble, base, tmp - mrc p15, 0, \tmp, c15, c1, 0 - orr \tmp, \tmp, #(1 << 6) - mcr p15, 0, \tmp, c15, c1, 0 @ Enable cp6 access - mrc p15, 0, \tmp, c15, c1, 0 - mov \tmp, \tmp - sub pc, pc, #4 @ cp_wait - .endm - - .macro get_irqnr_and_base, irqnr, irqstat, base, tmp - mrc p6, 0, \irqstat, c8, c0, 0 @ Read IINTSRC - cmp \irqstat, #0 - clzne \irqnr, \irqstat - rsbne \irqnr, \irqnr, #32 - .endm - - .macro arch_ret_to_user, tmp1, tmp2 - mrc p15, 0, \tmp1, c15, c1, 0 - ands \tmp2, \tmp1, #(1 << 6) - bicne \tmp1, \tmp1, #(1 << 6) - mcrne p15, 0, \tmp1, c15, c1, 0 @ Disable cp6 access - .endm diff --git a/arch/arm/mach-iop32x/iop3xx.h b/arch/arm/mach-iop32x/iop3xx.h index 46b4b34a4ad2..a6ec7ebadb35 100644 --- a/arch/arm/mach-iop32x/iop3xx.h +++ b/arch/arm/mach-iop32x/iop3xx.h @@ -225,6 +225,7 @@ extern int iop3xx_get_init_atu(void); #include void iop3xx_map_io(void); +void iop_enable_cp6(void); void iop_init_cp6_handler(void); void iop_init_time(unsigned long tickrate); void iop3xx_restart(enum reboot_mode, const char *); diff --git a/arch/arm/mach-iop32x/irq.c b/arch/arm/mach-iop32x/irq.c index d1e8824cbd82..b820839eaae8 100644 --- a/arch/arm/mach-iop32x/irq.c +++ b/arch/arm/mach-iop32x/irq.c @@ -29,6 +29,15 @@ static void intstr_write(u32 val) asm volatile("mcr p6, 0, %0, c4, c0, 0" : : "r" (val)); } +static u32 iintsrc_read(void) +{ + int irq; + + asm volatile("mrc p6, 0, %0, c8, c0, 0" : "=r" (irq)); + + return irq; +} + static void iop32x_irq_mask(struct irq_data *d) { @@ -50,11 +59,25 @@ struct irq_chip ext_chip = { .irq_unmask = iop32x_irq_unmask, }; +void iop_handle_irq(struct pt_regs *regs) +{ + u32 mask; + + iop_enable_cp6(); + + do { + mask = iintsrc_read(); + if (mask) + generic_handle_irq(fls(mask)); + } while (mask); +} + void __init iop32x_init_irq(void) { int i; iop_init_cp6_handler(); + set_handle_irq(iop_handle_irq); intctl_write(0); intstr_write(0); -- cgit v1.2.3-58-ga151 From 54f481a2308efab49d2b14c3f8263b34fdb1c65e Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 26 Nov 2021 22:36:06 +0100 Subject: ARM: remove old-style irq entry The last user of arch_irq_handler_default is gone now, so the entry-macro-multi.S file and all references to mach/entry-macro.S can be removed, as well as the asm_do_IRQ() entrypoint into the interrupt handling routines implemented in C. Note: The ARMv7-M entry still uses its own top-level IRQ entry, calling nvic_handle_irq() from assembly. This could be changed to go through generic_handle_arch_irq() as well, but it's unclear to me if there are any benefits. Signed-off-by: Arnd Bergmann [ardb: keep irq_handler macro as it carries all the IRQ stack handling] Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M Reviewed-by: Linus Walleij --- arch/arm/Kconfig | 14 ++------------ arch/arm/include/asm/entry-macro-multi.S | 16 ---------------- arch/arm/include/asm/irq.h | 1 - arch/arm/include/asm/mach/arch.h | 2 -- arch/arm/kernel/entry-armv.S | 8 -------- arch/arm/kernel/irq.c | 17 ----------------- 6 files changed, 2 insertions(+), 56 deletions(-) delete mode 100644 arch/arm/include/asm/entry-macro-multi.S (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index d9ba6961b295..b0e403076227 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -58,6 +58,7 @@ config ARM select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP + select GENERIC_IRQ_MULTI_HANDLER if MMU select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL @@ -320,7 +321,6 @@ config ARCH_MULTIPLATFORM select AUTO_ZRELADDR select TIMER_OF select COMMON_CLK - select GENERIC_IRQ_MULTI_HANDLER select HAVE_PCI select PCI_DOMAINS_GENERIC if PCI select SPARSE_IRQ @@ -344,7 +344,6 @@ config ARCH_EP93XX select ARM_AMBA imply ARM_PATCH_PHYS_VIRT select ARM_VIC - select GENERIC_IRQ_MULTI_HANDLER select AUTO_ZRELADDR select CLKSRC_MMIO select CPU_ARM920T @@ -359,7 +358,6 @@ config ARCH_FOOTBRIDGE select FOOTBRIDGE select NEED_MACH_IO_H if !MMU select NEED_MACH_MEMORY_H - select GENERIC_IRQ_MULTI_HANDLER help Support for systems based on the DC21285 companion chip ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder. @@ -368,7 +366,6 @@ config ARCH_IOP32X bool "IOP32x-based" depends on MMU select CPU_XSCALE - select GENERIC_IRQ_MULTI_HANDLER select GPIO_IOP select GPIOLIB select FORCE_PCI @@ -384,7 +381,6 @@ config ARCH_IXP4XX select ARCH_SUPPORTS_BIG_ENDIAN select CPU_XSCALE select DMABOUNCE if PCI - select GENERIC_IRQ_MULTI_HANDLER select GPIO_IXP4XX select GPIOLIB select HAVE_PCI @@ -400,7 +396,6 @@ config ARCH_IXP4XX config ARCH_DOVE bool "Marvell Dove" select CPU_PJ4 - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select HAVE_PCI select MVEBU_MBUS @@ -423,7 +418,6 @@ config ARCH_PXA select CLKSRC_MMIO select TIMER_OF select CPU_XSCALE if !CPU_XSC3 - select GENERIC_IRQ_MULTI_HANDLER select GPIO_PXA select GPIOLIB select IRQ_DOMAIN @@ -442,7 +436,6 @@ config ARCH_RPC select ARM_HAS_SG_CHAIN select CPU_SA110 select FIQ - select GENERIC_IRQ_MULTI_HANDLER select HAVE_PATA_PLATFORM select ISA_DMA_API select LEGACY_TIMER_TICK @@ -463,7 +456,6 @@ config ARCH_SA1100 select COMMON_CLK select CPU_FREQ select CPU_SA1100 - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select IRQ_DOMAIN select ISA @@ -478,7 +470,6 @@ config ARCH_S3C24XX select CLKSRC_SAMSUNG_PWM select GPIO_SAMSUNG select GPIOLIB - select GENERIC_IRQ_MULTI_HANDLER select HAVE_S3C2410_I2C if I2C select NEED_MACH_IO_H select S3C2410_WATCHDOG @@ -497,7 +488,6 @@ config ARCH_OMAP1 select ARCH_OMAP select CLKSRC_MMIO select GENERIC_IRQ_CHIP - select GENERIC_IRQ_MULTI_HANDLER select GPIOLIB select HAVE_LEGACY_CLK select IRQ_DOMAIN @@ -1168,7 +1158,7 @@ config CURRENT_POINTER_IN_TPIDRURO config IRQSTACKS def_bool y - depends on GENERIC_IRQ_MULTI_HANDLER && THREAD_INFO_IN_TASK + depends on THREAD_INFO_IN_TASK select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_SOFTIRQ_ON_OWN_STACK diff --git a/arch/arm/include/asm/entry-macro-multi.S b/arch/arm/include/asm/entry-macro-multi.S deleted file mode 100644 index 24486dad9e19..000000000000 --- a/arch/arm/include/asm/entry-macro-multi.S +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include - -/* - * Interrupt handling. Preserves r7, r8, r9 - */ - .macro arch_irq_handler_default - get_irqnr_preamble r6, lr -1: get_irqnr_and_base r0, r2, r6, lr - movne r1, sp - @ - @ routine called with r0 = irq number, r1 = struct pt_regs * - @ - badrne lr, 1b - bne asm_do_IRQ - .endm diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index 1cbcc462b07e..a7c2337b0c7d 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -26,7 +26,6 @@ struct irqaction; struct pt_regs; -extern void asm_do_IRQ(unsigned int, struct pt_regs *); void handle_IRQ(unsigned int, struct pt_regs *); void init_IRQ(void); diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index eec0c0bda766..9349e7a82c9c 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -56,9 +56,7 @@ struct machine_desc { void (*init_time)(void); void (*init_machine)(void); void (*init_late)(void); -#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER void (*handle_irq)(struct pt_regs *); -#endif void (*restart)(enum reboot_mode, const char *); }; diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 5fb7465d14d9..9744d087ee9f 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -19,9 +19,6 @@ #include #include #include -#ifndef CONFIG_GENERIC_IRQ_MULTI_HANDLER -#include -#endif #include #include #include @@ -30,14 +27,12 @@ #include #include "entry-header.S" -#include #include /* * Interrupt handling. */ .macro irq_handler, from_user:req -#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER mov r0, sp #ifdef CONFIG_IRQSTACKS mov_l r2, irq_stack_ptr @ Take base address @@ -92,9 +87,6 @@ UNWIND( .setfp fpreg, sp ) mov sp, r9 @ Restore original SP #endif // CONFIG_UNWINDER_ARM #endif // CONFIG_IRQSTACKS -#else - arch_irq_handler_default -#endif .endm .macro pabt_helper diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index 5deb40f39999..5c6f8d11a3ce 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -120,23 +120,6 @@ void handle_IRQ(unsigned int irq, struct pt_regs *regs) ack_bad_irq(irq); } -/* - * asm_do_IRQ is the interface to be used from assembly code. - */ -asmlinkage void __exception_irq_entry -asm_do_IRQ(unsigned int irq, struct pt_regs *regs) -{ - struct pt_regs *old_regs; - - irq_enter(); - old_regs = set_irq_regs(regs); - - handle_IRQ(irq, regs); - - set_irq_regs(old_regs); - irq_exit(); -} - void __init init_IRQ(void) { int ret; -- cgit v1.2.3-58-ga151 From b87cf9118e03d8a83039c88fa348165148f5d487 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 26 Nov 2021 10:13:06 +0100 Subject: ARM: use TLS register for 'current' on !SMP as well Enable the use of the TLS register to hold the 'current' pointer also on non-SMP configurations that target v6k or later CPUs. This will permit the use of THREAD_INFO_IN_TASK as well as IRQ stacks and vmap'ed stacks for such configurations. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Acked-by: Arnd Bergmann Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index b0e403076227..5ad2151c43dd 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1154,7 +1154,7 @@ config SMP_ON_UP config CURRENT_POINTER_IN_TPIDRURO def_bool y - depends on SMP && CPU_32v6K && !CPU_V6 + depends on CPU_32v6K && !CPU_V6 config IRQSTACKS def_bool y -- cgit v1.2.3-58-ga151 From 9c46929e7989efacc1dd0a1dd662a839897ea2b6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 24 Nov 2021 14:08:11 +0100 Subject: ARM: implement THREAD_INFO_IN_TASK for uniprocessor systems On UP systems, only a single task can be 'current' at the same time, which means we can use a global variable to track it. This means we can also enable THREAD_INFO_IN_TASK for those systems, as in that case, thread_info is accessed via current rather than the other way around, removing the need to store thread_info at the base of the task stack. This, in turn, permits us to enable IRQ stacks and vmap'ed stacks on UP systems as well. To partially mitigate the performance overhead of this arrangement, use a ADD/ADD/LDR sequence with the appropriate PC-relative group relocations to load the value of current when needed. This means that accessing current will still only require a single load as before, avoiding the need for a literal to carry the address of the global variable in each function. However, accessing thread_info will now require this load as well. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 8 ++-- arch/arm/include/asm/assembler.h | 83 +++++++++++++++++++++++++------------- arch/arm/include/asm/current.h | 37 ++++++++++------- arch/arm/include/asm/switch_to.h | 3 +- arch/arm/include/asm/thread_info.h | 27 ------------- arch/arm/kernel/asm-offsets.c | 3 -- arch/arm/kernel/entry-armv.S | 11 ++--- arch/arm/kernel/entry-v7m.S | 10 +++-- arch/arm/kernel/head-common.S | 4 +- arch/arm/kernel/process.c | 7 +++- arch/arm/kernel/smp.c | 11 +++++ arch/arm/kernel/traps.c | 4 ++ 12 files changed, 116 insertions(+), 92 deletions(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 5ad2151c43dd..50ae5286f59b 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -127,8 +127,8 @@ config ARM select PERF_USE_VMALLOC select RTC_LIB select SYS_SUPPORTS_APM_EMULATION - select THREAD_INFO_IN_TASK if CURRENT_POINTER_IN_TPIDRURO - select HAVE_ARCH_VMAP_STACK if MMU && THREAD_INFO_IN_TASK && (!LD_IS_LLD || LLD_VERSION >= 140000) + select THREAD_INFO_IN_TASK + select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. @@ -1158,7 +1158,7 @@ config CURRENT_POINTER_IN_TPIDRURO config IRQSTACKS def_bool y - depends on THREAD_INFO_IN_TASK + depends on MMU select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_SOFTIRQ_ON_OWN_STACK @@ -1608,7 +1608,7 @@ config CC_HAVE_STACKPROTECTOR_TLS config STACKPROTECTOR_PER_TASK bool "Use a unique stack canary value for each task" - depends on STACKPROTECTOR && THREAD_INFO_IN_TASK && !XIP_DEFLATED_DATA + depends on STACKPROTECTOR && CURRENT_POINTER_IN_TPIDRURO && !XIP_DEFLATED_DATA depends on GCC_PLUGINS || CC_HAVE_STACKPROTECTOR_TLS select GCC_PLUGIN_ARM_SSP_PER_TASK if !CC_HAVE_STACKPROTECTOR_TLS default y diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index f9b3dd0e9ef5..59d7b9e81934 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -203,41 +203,12 @@ THUMB( fpreg .req r7 ) .endm .endr - .macro get_current, rd -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register -#else - get_thread_info \rd - ldr \rd, [\rd, #TI_TASK] -#endif - .endm - - .macro set_current, rn -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register -#endif - .endm - - .macro reload_current, t1:req, t2:req -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - ldr_this_cpu \t1, __entry_task, \t1, \t2 - mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO -#endif - .endm - /* * Get current thread_info. */ .macro get_thread_info, rd -#ifdef CONFIG_THREAD_INFO_IN_TASK /* thread_info is the first member of struct task_struct */ get_current \rd -#else - ARM( mov \rd, sp, lsr #THREAD_SIZE_ORDER + PAGE_SHIFT ) - THUMB( mov \rd, sp ) - THUMB( lsr \rd, \rd, #THREAD_SIZE_ORDER + PAGE_SHIFT ) - mov \rd, \rd, lsl #THREAD_SIZE_ORDER + PAGE_SHIFT -#endif .endm /* @@ -330,6 +301,60 @@ ALT_UP_B(.L1_\@) #endif .endm + /* + * set_current - store the task pointer of this CPU's current task + */ + .macro set_current, rn:req, tmp:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +9998: mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L0_\@) + .subsection 1 +.L0_\@: str_va \rn, __current, \tmp + b .L1_\@ + .previous +.L1_\@: +#endif +#else + str_va \rn, __current, \tmp +#endif + .endm + + /* + * get_current - load the task pointer of this CPU's current task + */ + .macro get_current, rd:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +9998: mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L0_\@) + .subsection 1 +.L0_\@: ldr_va \rd, __current + b .L1_\@ + .previous +.L1_\@: +#endif +#else + ldr_va \rd, __current +#endif + .endm + + /* + * reload_current - reload the task pointer of this CPU's current task + * into the TLS register + */ + .macro reload_current, t1:req, t2:req +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) +#ifdef CONFIG_CPU_V6 +ALT_SMP(nop) +ALT_UP_B(.L0_\@) +#endif + ldr_this_cpu \t1, __entry_task, \t1, \t2 + mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO +.L0_\@: +#endif + .endm + /* * Instruction barrier */ diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 6bf0aad672c3..69ecf4c6c725 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -8,25 +8,18 @@ #define _ASM_ARM_CURRENT_H #ifndef __ASSEMBLY__ +#include struct task_struct; -static inline void set_current(struct task_struct *cur) -{ - if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) - return; - - /* Set TPIDRURO */ - asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); -} +extern struct task_struct *__current; -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - -static inline struct task_struct *get_current(void) +static inline __attribute_const__ struct task_struct *get_current(void) { struct task_struct *cur; #if __has_builtin(__builtin_thread_pointer) && \ + defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && \ !(defined(CONFIG_THUMB2_KERNEL) && \ defined(CONFIG_CC_IS_CLANG) && CONFIG_CLANG_VERSION < 130001) /* @@ -39,16 +32,30 @@ static inline struct task_struct *get_current(void) * https://github.com/ClangBuiltLinux/linux/issues/1485 */ cur = __builtin_thread_pointer(); +#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) + asm("0: mrc p15, 0, %0, c13, c0, 3 \n\t" +#ifdef CONFIG_CPU_V6 + "1: \n\t" + " .subsection 1 \n\t" + "2: " LOAD_SYM_ARMV6(%0, __current) " \n\t" + " b 1b \n\t" + " .previous \n\t" + " .pushsection \".alt.smp.init\", \"a\" \n\t" + " .long 0b - . \n\t" + " b . + (2b - 0b) \n\t" + " .popsection \n\t" +#endif + : "=r"(cur)); +#elif __LINUX_ARM_ARCH__>=7 || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) + cur = __current; #else - asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur)); + asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); #endif return cur; } #define current get_current() -#else -#include -#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */ #endif /* __ASSEMBLY__ */ diff --git a/arch/arm/include/asm/switch_to.h b/arch/arm/include/asm/switch_to.h index b55c7b2755e4..a482c99934ff 100644 --- a/arch/arm/include/asm/switch_to.h +++ b/arch/arm/include/asm/switch_to.h @@ -40,7 +40,8 @@ static inline void set_ti_cpu(struct task_struct *p) do { \ __complete_pending_tlbi(); \ set_ti_cpu(next); \ - if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) \ + if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || \ + IS_ENABLED(CONFIG_SMP)) \ __this_cpu_write(__entry_task, next); \ last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ } while (0) diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 004b89d86224..aecc403b2880 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -62,9 +62,6 @@ struct cpu_context_save { struct thread_info { unsigned long flags; /* low level flags */ int preempt_count; /* 0 => preemptable, <0 => bug */ -#ifndef CONFIG_THREAD_INFO_IN_TASK - struct task_struct *task; /* main task structure */ -#endif __u32 cpu; /* cpu */ __u32 cpu_domain; /* cpu domain */ struct cpu_context_save cpu_context; /* cpu context */ @@ -80,39 +77,15 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ - INIT_THREAD_INFO_TASK(tsk) \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ } -#ifdef CONFIG_THREAD_INFO_IN_TASK -#define INIT_THREAD_INFO_TASK(tsk) - static inline struct task_struct *thread_task(struct thread_info* ti) { return (struct task_struct *)ti; } -#else -#define INIT_THREAD_INFO_TASK(tsk) .task = &(tsk), - -static inline struct task_struct *thread_task(struct thread_info* ti) -{ - return ti->task; -} - -/* - * how to get the thread information struct from C - */ -static inline struct thread_info *current_thread_info(void) __attribute_const__; - -static inline struct thread_info *current_thread_info(void) -{ - return (struct thread_info *) - (current_stack_pointer & ~(THREAD_SIZE - 1)); -} -#endif - #define thread_saved_pc(tsk) \ ((unsigned long)(task_thread_info(tsk)->cpu_context.pc)) #define thread_saved_sp(tsk) \ diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 645845e4982a..2c8d76fd7c66 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -43,9 +43,6 @@ int main(void) BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); -#ifndef CONFIG_THREAD_INFO_IN_TASK - DEFINE(TI_TASK, offsetof(struct thread_info, task)); -#endif DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 43d917f0d9a9..b58bda51e4b8 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -807,12 +807,13 @@ ENTRY(__switch_to) switch_tls r1, r4, r5, r3, r7 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) && \ !defined(CONFIG_STACKPROTECTOR_PER_TASK) - ldr r9, [r2, #TI_TASK] ldr r8, =__stack_chk_guard .if (TSK_STACK_CANARY > IMM12_MASK) - add r9, r9, #TSK_STACK_CANARY & ~IMM12_MASK - .endif + add r9, r2, #TSK_STACK_CANARY & ~IMM12_MASK ldr r9, [r9, #TSK_STACK_CANARY & IMM12_MASK] + .else + ldr r9, [r2, #TSK_STACK_CANARY & IMM12_MASK] + .endif #endif mov r7, r2 @ Preserve 'next' #ifdef CONFIG_CPU_USE_DOMAINS @@ -829,7 +830,7 @@ ENTRY(__switch_to) #endif mov r0, r5 #if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK) - set_current r7 + set_current r7, r8 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously #else mov r1, r7 @@ -851,7 +852,7 @@ ENTRY(__switch_to) @ switches us to another stack, with few other side effects. In order @ to prevent this distinction from causing any inconsistencies, let's @ keep the 'set_current' call as close as we can to the update of SP. - set_current r1 + set_current r1, r2 mov sp, ip ret lr #endif diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index 520dd43e7e08..4e0d318b67c6 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -97,15 +97,17 @@ ENTRY(__switch_to) str sp, [ip], #4 str lr, [ip], #4 mov r5, r0 + mov r6, r2 @ Preserve 'next' add r4, r2, #TI_CPU_SAVE ldr r0, =thread_notify_head mov r1, #THREAD_NOTIFY_SWITCH bl atomic_notifier_call_chain - mov ip, r4 mov r0, r5 - ldmia ip!, {r4 - r11} @ Load all regs saved previously - ldr sp, [ip] - ldr pc, [ip, #4]! + mov r1, r6 + ldmia r4, {r4 - r12, lr} @ Load all regs saved previously + set_current r1, r2 + mov sp, ip + bx lr .fnend ENDPROC(__switch_to) diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S index da18e0a17dc2..42cae73fcc19 100644 --- a/arch/arm/kernel/head-common.S +++ b/arch/arm/kernel/head-common.S @@ -105,10 +105,8 @@ __mmap_switched: mov r1, #0 bl __memset @ clear .bss -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO adr_l r0, init_task @ get swapper task_struct - set_current r0 -#endif + set_current r0, r1 ldmia r4, {r0, r1, r2, r3} str r9, [r0] @ Save processor ID diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index d47159f3791c..0617af11377f 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -36,7 +36,7 @@ #include "signal.h" -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO +#if defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) DEFINE_PER_CPU(struct task_struct *, __entry_task); #endif @@ -46,6 +46,11 @@ unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); #endif +#ifndef CONFIG_CURRENT_POINTER_IN_TPIDRURO +asmlinkage struct task_struct *__current; +EXPORT_SYMBOL(__current); +#endif + static const char *processor_modes[] __maybe_unused = { "USER_26", "FIQ_26" , "IRQ_26" , "SVC_26" , "UK4_26" , "UK5_26" , "UK6_26" , "UK7_26" , "UK8_26" , "UK9_26" , "UK10_26", "UK11_26", "UK12_26", "UK13_26", "UK14_26", "UK15_26", diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 9c55ca915ba4..951559e5bea3 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -403,6 +403,17 @@ static void smp_store_cpu_info(unsigned int cpuid) check_cpu_icache_size(cpuid); } +static void set_current(struct task_struct *cur) +{ + if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && !is_smp()) { + __current = cur; + return; + } + + /* Set TPIDRURO */ + asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); +} + /* * This is the secondary CPU boot entry. We're using this CPUs * idle thread stack, but a set of temporary page tables. diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index b28a705c49cb..3f38357efc46 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -865,7 +865,9 @@ early_initcall(allocate_overflow_stacks); asmlinkage void handle_bad_stack(struct pt_regs *regs) { unsigned long tsk_stk = (unsigned long)current->stack; +#ifdef CONFIG_IRQSTACKS unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); +#endif unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr); console_verbose(); @@ -873,8 +875,10 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) pr_emerg("Task stack: [0x%08lx..0x%08lx]\n", tsk_stk, tsk_stk + THREAD_SIZE); +#ifdef CONFIG_IRQSTACKS pr_emerg("IRQ stack: [0x%08lx..0x%08lx]\n", irq_stk - THREAD_SIZE, irq_stk); +#endif pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n", ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk); -- cgit v1.2.3-58-ga151 From cafc0eab168917ec9c0cd47d530a40cd40eb2928 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 2 Dec 2021 09:36:59 +0100 Subject: ARM: v7m: enable support for IRQ stacks Enable support for IRQ stacks on !MMU, and add the code to the IRQ entry path to switch to the IRQ stack if not running from it already. Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/Kconfig | 1 - arch/arm/kernel/entry-v7m.S | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 50ae5286f59b..359a3b85c8b3 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1158,7 +1158,6 @@ config CURRENT_POINTER_IN_TPIDRURO config IRQSTACKS def_bool y - depends on MMU select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_SOFTIRQ_ON_OWN_STACK diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index 4e0d318b67c6..de8a60363c85 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -40,11 +40,24 @@ __irq_entry: @ Invoke the IRQ handler @ mov r0, sp - stmdb sp!, {lr} + ldr_this_cpu sp, irq_stack_ptr, r1, r2 + + @ + @ If we took the interrupt while running in the kernel, we may already + @ be using the IRQ stack, so revert to the original value in that case. + @ + subs r2, sp, r0 @ SP above bottom of IRQ stack? + rsbscs r2, r2, #THREAD_SIZE @ ... and below the top? + movcs sp, r0 + + push {r0, lr} @ preserve LR and original SP + @ routine called with r0 = struct pt_regs * bl generic_handle_arch_irq - pop {lr} + pop {r0, lr} + mov sp, r0 + @ @ Check for any pending work if returning to user @ -- cgit v1.2.3-58-ga151 From 23d9a9280efea105852de358f21d69231992ae73 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 5 Jan 2022 15:45:26 +0100 Subject: ARM: 9177/1: disable vmap'ed stacks on suspend-capable SMP configs There are several reports about the new vmap'ed stacks code breaking suspend/resume on Exynos, Renesas and Tegra SMP platforms. While this is under investigation, let's disable the vmap'ed stacks feature for the time being for SMP configurations that have suspend/resume enabled. [0] https://lore.kernel.org/linux-arm-kernel/20211122092816.2865873-8-ardb@kernel.org/ Cc: Marek Szyprowski Cc: Geert Uytterhoeven Cc: Jon Hunter Signed-off-by: Ard Biesheuvel Signed-off-by: Russell King (Oracle) --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 359a3b85c8b3..c32b79453ddf 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -128,7 +128,7 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK - select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) + select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) && !PM_SLEEP_SMP select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. -- cgit v1.2.3-58-ga151 From 8b806b82bc804ce8d254ea87d48eaa390451eac4 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 19:51:58 +0100 Subject: ARM: mm: switch to swapper_pg_dir early for vmap'ed stack When onlining a CPU, switch to swapper_pg_dir as soon as possible so that it is guaranteed that the vmap'ed stack is mapped before it is used. Signed-off-by: Ard Biesheuvel --- arch/arm/Kconfig | 2 +- arch/arm/kernel/head.S | 7 +++++++ arch/arm/kernel/sleep.S | 7 +++++++ 3 files changed, 15 insertions(+), 1 deletion(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index c32b79453ddf..359a3b85c8b3 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -128,7 +128,7 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK - select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) && !PM_SLEEP_SMP + select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index c04dd94630c7..500612d3da2e 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -424,6 +424,13 @@ ENDPROC(secondary_startup) ENDPROC(secondary_startup_arm) ENTRY(__secondary_switched) +#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE) + @ Before using the vmap'ed stack, we have to switch to swapper_pg_dir + @ as the ID map does not cover the vmalloc region. + mrc p15, 0, ip, c2, c0, 1 @ read TTBR1 + mcr p15, 0, ip, c2, c0, 0 @ set TTBR0 + instr_sync +#endif adr_l r7, secondary_data + 12 @ get secondary_data.stack ldr sp, [r7] ldr r0, [r7, #4] @ get secondary_data.task diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index f909baf17912..a86a1d4f3461 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -119,6 +119,13 @@ ENTRY(cpu_resume_mmu) ENDPROC(cpu_resume_mmu) .popsection cpu_resume_after_mmu: +#if defined(CONFIG_VMAP_STACK) && !defined(CONFIG_ARM_LPAE) + @ Before using the vmap'ed stack, we have to switch to swapper_pg_dir + @ as the ID map does not cover the vmalloc region. + mrc p15, 0, ip, c2, c0, 1 @ read TTBR1 + mcr p15, 0, ip, c2, c0, 0 @ set TTBR0 + instr_sync +#endif bl cpu_init @ restore the und/abt/irq banked regs mov r0, #0 @ return zero on success ldmfd sp!, {r4 - r11, pc} -- cgit v1.2.3-58-ga151 From d6905849f87596f94c2778c8337697df486de43c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 24 Jan 2022 19:16:58 +0100 Subject: ARM: assembler: define a Kconfig symbol for group relocation support Nathan reports the group relocations go out of range in pathological cases such as allyesconfig kernels, which have little chance of actually booting but are still used in validation. So add a Kconfig symbol for this feature, and make it depend on !COMPILE_TEST. Signed-off-by: Ard Biesheuvel --- arch/arm/Kconfig | 13 ++++++++++++- arch/arm/include/asm/assembler.h | 8 ++++---- arch/arm/include/asm/current.h | 8 ++++---- arch/arm/include/asm/percpu.h | 4 ++-- arch/arm/kernel/module.c | 7 ++++++- 5 files changed, 28 insertions(+), 12 deletions(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 359a3b85c8b3..70ab8d807032 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -128,7 +128,7 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK - select HAVE_ARCH_VMAP_STACK if MMU && (!LD_IS_LLD || LLD_VERSION >= 140000) + select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. @@ -140,6 +140,17 @@ config ARM Europe. There is an ARM Linux project with a web page at . +config ARM_HAS_GROUP_RELOCS + def_bool y + depends on !LD_IS_LLD || LLD_VERSION >= 140000 + depends on !COMPILE_TEST + help + Whether or not to use R_ARM_ALU_PC_Gn or R_ARM_LDR_PC_Gn group + relocations, which have been around for a long time, but were not + supported in LLD until version 14. The combined range is -/+ 256 MiB, + which is usually sufficient, but not for allyesconfig, so we disable + this feature when doing compile testing. + config ARM_HAS_SG_CHAIN bool diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 59d7b9e81934..9998718a49ca 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -656,8 +656,8 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .macro __ldst_va, op, reg, tmp, sym, cond #if __LINUX_ARM_ARCH__ >= 7 || \ - (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ - (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) mov_l \tmp, \sym, \cond \op\cond \reg, [\tmp] #else @@ -716,8 +716,8 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) */ .macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req #if __LINUX_ARM_ARCH__ >= 7 || \ - (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ - (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) this_cpu_offset \t1 mov_l \t2, \sym ldr \rd, [\t1, \t2] diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 2f9d79214b25..131a89bbec6b 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -37,8 +37,8 @@ static inline __attribute_const__ struct task_struct *get_current(void) #ifdef CONFIG_CPU_V6 "1: \n\t" " .subsection 1 \n\t" -#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ - !(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) +#if defined(CONFIG_ARM_HAS_GROUP_RELOCS) && \ + !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) "2: " LOAD_SYM_ARMV6(%0, __current) " \n\t" " b 1b \n\t" #else @@ -55,8 +55,8 @@ static inline __attribute_const__ struct task_struct *get_current(void) #endif : "=r"(cur)); #elif __LINUX_ARM_ARCH__>= 7 || \ - (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ - (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + !defined(CONFIG_ARM_HAS_GROUP_RELOCS) || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) cur = __current; #else asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); diff --git a/arch/arm/include/asm/percpu.h b/arch/arm/include/asm/percpu.h index 28961d60877d..a09034ae45a1 100644 --- a/arch/arm/include/asm/percpu.h +++ b/arch/arm/include/asm/percpu.h @@ -38,8 +38,8 @@ static inline unsigned long __my_cpu_offset(void) #ifdef CONFIG_CPU_V6 "1: \n\t" " .subsection 1 \n\t" -#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \ - !(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) +#if defined(CONFIG_ARM_HAS_GROUP_RELOCS) && \ + !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) "2: " LOAD_SYM_ARMV6(%0, __per_cpu_offset) " \n\t" " b 1b \n\t" #else diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index 4d33a7acf617..549abcedf795 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -68,6 +68,7 @@ bool module_exit_section(const char *name) strstarts(name, ".ARM.exidx.exit"); } +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS /* * This implements the partitioning algorithm for group relocations as * documented in the ARM AArch32 ELF psABI (IHI 0044). @@ -103,6 +104,7 @@ static u32 get_group_rem(u32 group, u32 *offset) } while (group--); return shift; } +#endif int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -118,7 +120,9 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned long loc; Elf32_Sym *sym; const char *symname; +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS u32 shift, group = 1; +#endif s32 offset; u32 tmp; #ifdef CONFIG_THUMB2_KERNEL @@ -249,6 +253,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, *(u32 *)loc = __opcode_to_mem_arm(tmp); break; +#ifdef CONFIG_ARM_HAS_GROUP_RELOCS case R_ARM_ALU_PC_G0_NC: group = 0; fallthrough; @@ -296,7 +301,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, } *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); break; - +#endif #ifdef CONFIG_THUMB2_KERNEL case R_ARM_THM_CALL: case R_ARM_THM_JUMP24: -- cgit v1.2.3-58-ga151 From 41918ec82eb6f80c8b401422f27ca76c85aa0cb7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 25 Jan 2022 15:55:24 +0100 Subject: ARM: ftrace: enable the graph tracer with the EABI unwinder Enable the function graph tracer in combination with the EABI unwinder, so that Thumb2 builds or Clang ARM builds can make use of it. This involves using the unwinder to locate the return address of an instrumented function on the stack, so that it can be overridden and made to refer to the ftrace handling routines that need to be called at function return. Given that for these builds, it is not guaranteed that the value of the link register is stored on the stack, fall back to the stack slot that will be used by the ftrace exit code to restore LR in the instrumented function's execution context. Signed-off-by: Ard Biesheuvel Reviewed-by: Steven Rostedt (Google) --- arch/arm/Kconfig | 2 +- arch/arm/Kconfig.debug | 2 +- arch/arm/include/asm/ftrace.h | 18 ------------------ arch/arm/kernel/Makefile | 5 +---- arch/arm/kernel/entry-ftrace.S | 28 ++++++++++++++++++++-------- arch/arm/kernel/ftrace.c | 19 +++++++++++++++++-- 6 files changed, 40 insertions(+), 34 deletions(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 70ab8d807032..ec98387367d0 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -91,7 +91,7 @@ config ARM select HAVE_EXIT_THREAD select HAVE_FAST_GUP if ARM_LPAE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL - select HAVE_FUNCTION_GRAPH_TRACER if !THUMB2_KERNEL && !CC_IS_CLANG + select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG) select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index 98436702e0c7..b79dc7fa89bf 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -65,7 +65,7 @@ config UNWINDER_FRAME_POINTER config UNWINDER_ARM bool "ARM EABI stack unwinder" - depends on AEABI && !FUNCTION_GRAPH_TRACER + depends on AEABI # https://github.com/ClangBuiltLinux/linux/issues/732 depends on !LD_IS_LLD || LLD_VERSION >= 110000 select ARM_UNWIND diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index b4f5fab6b04e..5358aad67831 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -35,26 +35,8 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) #ifndef __ASSEMBLY__ -#if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) -/* - * return_address uses walk_stackframe to do it's work. If both - * CONFIG_FRAME_POINTER=y and CONFIG_ARM_UNWIND=y walk_stackframe uses unwind - * information. For this to work in the function tracer many functions would - * have to be marked with __notrace. So for now just depend on - * !CONFIG_ARM_UNWIND. - */ - void *return_address(unsigned int); -#else - -static inline void *return_address(unsigned int level) -{ - return NULL; -} - -#endif - #define ftrace_return_address(n) return_address(n) #define ARCH_HAS_SYSCALL_MATCH_SYM_NAME diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 56511856ff9d..5cebb8d5a1d6 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -25,10 +25,7 @@ obj-y := elf.o entry-common.o irq.o opcodes.o \ KASAN_SANITIZE_stacktrace.o := n KASAN_SANITIZE_traps.o := n -ifneq ($(CONFIG_ARM_UNWIND),y) -obj-$(CONFIG_FRAME_POINTER) += return_address.o -endif - +obj-y += return_address.o obj-$(CONFIG_ATAGS) += atags_parse.o obj-$(CONFIG_ATAGS_PROC) += atags_proc.o obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += atags_compat.o diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index bbfa0954c385..3e7bcaca5e07 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -100,7 +100,8 @@ ftrace_regs_call: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .globl ftrace_graph_regs_call ftrace_graph_regs_call: - mov r0, r0 +ARM( mov r0, r0 ) +THUMB( nop.w ) #endif @ pop saved regs @@ -112,13 +113,18 @@ ftrace_graph_regs_call: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .macro __ftrace_graph_regs_caller - sub r0, fp, #4 @ lr of instrumented routine (parent) +#ifdef CONFIG_UNWINDER_FRAME_POINTER + sub r0, fp, #4 @ lr of instrumented routine (parent) +#else + add r0, sp, #S_LR +#endif @ called from __ftrace_regs_caller - ldr r1, [sp, #S_PC] @ instrumented routine (func) + ldr r1, [sp, #S_PC] @ instrumented routine (func) mcount_adjust_addr r1, r1 - mov r2, fp @ frame pointer + mov r2, fpreg @ frame pointer + add r3, sp, #PT_REGS_SIZE bl prepare_ftrace_return @ pop registers saved in ftrace_regs_caller @@ -149,14 +155,19 @@ ftrace_call\suffix: #ifdef CONFIG_FUNCTION_GRAPH_TRACER .globl ftrace_graph_call\suffix ftrace_graph_call\suffix: - mov r0, r0 +ARM( mov r0, r0 ) +THUMB( nop.w ) #endif mcount_exit .endm .macro __ftrace_graph_caller +#ifdef CONFIG_UNWINDER_FRAME_POINTER sub r0, fp, #4 @ &lr of instrumented routine (&parent) +#else + add r0, sp, #20 +#endif #ifdef CONFIG_DYNAMIC_FTRACE @ called from __ftrace_caller, saved in mcount_enter ldr r1, [sp, #16] @ instrumented routine (func) @@ -165,7 +176,8 @@ ftrace_graph_call\suffix: @ called from __mcount, untouched in lr mcount_adjust_addr r1, lr @ instrumented routine (func) #endif - mov r2, fp @ frame pointer + mov r2, fpreg @ frame pointer + add r3, sp, #24 bl prepare_ftrace_return mcount_exit .endm @@ -244,14 +256,14 @@ ENDPROC(ftrace_graph_regs_caller) .purgem mcount_exit #ifdef CONFIG_FUNCTION_GRAPH_TRACER - .globl return_to_handler -return_to_handler: +ENTRY(return_to_handler) stmdb sp!, {r0-r3} add r0, sp, #16 @ sp at exit of instrumented routine bl ftrace_return_to_handler mov lr, r0 @ r0 has real ret addr ldmia sp!, {r0-r3} ret lr +ENDPROC(return_to_handler) #endif ENTRY(ftrace_stub) diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index ea2396900c7d..83cc068586bc 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -22,6 +22,7 @@ #include #include #include +#include #include /* @@ -224,8 +225,10 @@ int ftrace_make_nop(struct module *mod, #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER +asmlinkage void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, - unsigned long frame_pointer) + unsigned long frame_pointer, + unsigned long stack_pointer) { unsigned long return_hooker = (unsigned long) &return_to_handler; unsigned long old; @@ -236,6 +239,18 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) { /* FP points one word below parent's top of stack */ frame_pointer += 4; + } else { + struct stackframe frame = { + .fp = frame_pointer, + .sp = stack_pointer, + .lr = self_addr, + .pc = self_addr, + }; + if (unwind_frame(&frame) < 0) + return; + if (frame.lr != self_addr) + parent = frame.lr_addr; + frame_pointer = frame.sp; } old = *parent; @@ -258,7 +273,7 @@ static int __ftrace_modify_caller(unsigned long *callsite, unsigned long caller_fn = (unsigned long) func; unsigned long pc = (unsigned long) callsite; unsigned long branch = arm_gen_branch(pc, caller_fn); - unsigned long nop = 0xe1a00000; /* mov r0, r0 */ + unsigned long nop = arm_gen_nop(); unsigned long old = enable ? nop : branch; unsigned long new = enable ? branch : nop; -- cgit v1.2.3-58-ga151 From d6800ca73a7d325627c045c16d7cfdc7465f4333 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 26 Jan 2022 11:43:39 +0100 Subject: Revert "ARM: 9144/1: forbid ftrace with clang and thumb2_kernel" This reverts commit ecb108e3e3f7c692082b7c6fce41779c3835854a. Clang + Thumb2 with ftrace is now supported. Signed-off-by: Ard Biesheuvel Reviewed-by: Nick Desaulniers Reviewed-by: Steven Rostedt (Google) --- arch/arm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index ec98387367d0..5c9a8d3362cd 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -92,7 +92,7 @@ config ARM select HAVE_FAST_GUP if ARM_LPAE select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !(THUMB2_KERNEL && CC_IS_CLANG) + select HAVE_FUNCTION_TRACER if !XIP_KERNEL select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS select HAVE_HW_BREAKPOINT if PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7) -- cgit v1.2.3-58-ga151 From 234a0f202a09a6144fd3c17ac6d018bdab9780bb Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 9 Mar 2022 15:40:47 +0100 Subject: ARM: fix building NOMMU ARMv4/v5 kernels The removal of the old-style irq entry broke obscure NOMMU configurations on machines that have an MMU: ld.lld: error: undefined symbol: generic_handle_arch_irq referenced by kernel/entry-armv.o:(__irq_svc) in archive arch/arm/built-in.a A follow-up patch to convert nvic to the generic_handle_arch_irq() could have fixed this by removing the Kconfig conditional, but did it differently. Change the Kconfig logic so ARM machines now unconditionally enable the feature. I have also submitted a patch to remove support for the configurations that broke, but fixing the regression first is a trivial and correct change. Reported-by: kernel test robot Fixes: 54f481a2308e ("ARM: remove old-style irq entry") Fixes: 52d240871760 ("irqchip: nvic: Use GENERIC_IRQ_MULTI_HANDLER") Signed-off-by: Arnd Bergmann Signed-off-by: Russell King (Oracle) --- arch/arm/Kconfig | 2 +- drivers/irqchip/Kconfig | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'arch/arm/Kconfig') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 5c9a8d3362cd..4b3ac41c6f7a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -58,7 +58,7 @@ config ARM select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP - select GENERIC_IRQ_MULTI_HANDLER if MMU + select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 488eaa14d3a7..7038957f4a77 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -58,7 +58,6 @@ config ARM_NVIC bool select IRQ_DOMAIN_HIERARCHY select GENERIC_IRQ_CHIP - select GENERIC_IRQ_MULTI_HANDLER config ARM_VIC bool -- cgit v1.2.3-58-ga151