summaryrefslogtreecommitdiff
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/arm64/Kconfig1
-rw-r--r--arch/arm64/include/asm/ftrace.h22
-rw-r--r--arch/arm64/kernel/asm-offsets.c13
-rw-r--r--arch/arm64/kernel/entry-ftrace.S27
-rw-r--r--arch/loongarch/Kconfig1
-rw-r--r--arch/loongarch/include/asm/ftrace.h22
-rw-r--r--arch/loongarch/kernel/asm-offsets.c15
-rw-r--r--arch/loongarch/kernel/mcount.S14
-rw-r--r--arch/loongarch/kernel/mcount_dyn.S15
-rw-r--r--arch/riscv/Kconfig1
-rw-r--r--arch/riscv/include/asm/ftrace.h21
-rw-r--r--arch/riscv/kernel/mcount.S7
-rw-r--r--arch/x86/Kconfig1
-rw-r--r--arch/x86/include/asm/ftrace.h20
-rw-r--r--arch/x86/kernel/ftrace_32.S8
-rw-r--r--arch/x86/kernel/ftrace_64.S7
16 files changed, 156 insertions, 39 deletions
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 595028bd9160..891ab530a665 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -202,6 +202,7 @@ config ARM64
select HAVE_FTRACE_MCOUNT_RECORD
select HAVE_FUNCTION_TRACER
select HAVE_FUNCTION_ERROR_INJECTION
+ select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_GCC_PLUGINS
select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && \
diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h
index b87d70b693c6..21ac1c5c71d3 100644
--- a/arch/arm64/include/asm/ftrace.h
+++ b/arch/arm64/include/asm/ftrace.h
@@ -192,4 +192,26 @@ static inline bool arch_syscall_match_sym_name(const char *sym,
}
#endif /* ifndef __ASSEMBLY__ */
+#ifndef __ASSEMBLY__
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+struct fgraph_ret_regs {
+ /* x0 - x7 */
+ unsigned long regs[8];
+
+ unsigned long fp;
+ unsigned long __unused;
+};
+
+static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs)
+{
+ return ret_regs->regs[0];
+}
+
+static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs)
+{
+ return ret_regs->fp;
+}
+#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */
+#endif
+
#endif /* __ASM_FTRACE_H */
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 0996094b0d22..757d01a68ffd 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -200,6 +200,19 @@ int main(void)
#endif
#ifdef CONFIG_FUNCTION_TRACER
DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func));
+#endif
+ BLANK();
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ DEFINE(FGRET_REGS_X0, offsetof(struct fgraph_ret_regs, regs[0]));
+ DEFINE(FGRET_REGS_X1, offsetof(struct fgraph_ret_regs, regs[1]));
+ DEFINE(FGRET_REGS_X2, offsetof(struct fgraph_ret_regs, regs[2]));
+ DEFINE(FGRET_REGS_X3, offsetof(struct fgraph_ret_regs, regs[3]));
+ DEFINE(FGRET_REGS_X4, offsetof(struct fgraph_ret_regs, regs[4]));
+ DEFINE(FGRET_REGS_X5, offsetof(struct fgraph_ret_regs, regs[5]));
+ DEFINE(FGRET_REGS_X6, offsetof(struct fgraph_ret_regs, regs[6]));
+ DEFINE(FGRET_REGS_X7, offsetof(struct fgraph_ret_regs, regs[7]));
+ DEFINE(FGRET_REGS_FP, offsetof(struct fgraph_ret_regs, fp));
+ DEFINE(FGRET_REGS_SIZE, sizeof(struct fgraph_ret_regs));
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
DEFINE(FTRACE_OPS_DIRECT_CALL, offsetof(struct ftrace_ops, direct_call));
#endif
diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S
index 1c38a60575aa..f0c16640ef21 100644
--- a/arch/arm64/kernel/entry-ftrace.S
+++ b/arch/arm64/kernel/entry-ftrace.S
@@ -330,22 +330,23 @@ SYM_FUNC_END(ftrace_stub_graph)
*/
SYM_CODE_START(return_to_handler)
/* save return value regs */
- sub sp, sp, #64
- stp x0, x1, [sp]
- stp x2, x3, [sp, #16]
- stp x4, x5, [sp, #32]
- stp x6, x7, [sp, #48]
+ sub sp, sp, #FGRET_REGS_SIZE
+ stp x0, x1, [sp, #FGRET_REGS_X0]
+ stp x2, x3, [sp, #FGRET_REGS_X2]
+ stp x4, x5, [sp, #FGRET_REGS_X4]
+ stp x6, x7, [sp, #FGRET_REGS_X6]
+ str x29, [sp, #FGRET_REGS_FP] // parent's fp
- mov x0, x29 // parent's fp
- bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
- mov x30, x0 // restore the original return address
+ mov x0, sp
+ bl ftrace_return_to_handler // addr = ftrace_return_to_hander(regs);
+ mov x30, x0 // restore the original return address
/* restore return value regs */
- ldp x0, x1, [sp]
- ldp x2, x3, [sp, #16]
- ldp x4, x5, [sp, #32]
- ldp x6, x7, [sp, #48]
- add sp, sp, #64
+ ldp x0, x1, [sp, #FGRET_REGS_X0]
+ ldp x2, x3, [sp, #FGRET_REGS_X2]
+ ldp x4, x5, [sp, #FGRET_REGS_X4]
+ ldp x6, x7, [sp, #FGRET_REGS_X6]
+ add sp, sp, #FGRET_REGS_SIZE
ret
SYM_CODE_END(return_to_handler)
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index 0c7fb52727ba..e55511af4c77 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -111,6 +111,7 @@ config LOONGARCH
select HAVE_FTRACE_MCOUNT_RECORD
select HAVE_FUNCTION_ARG_ACCESS_API
select HAVE_FUNCTION_ERROR_INJECTION
+ select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_TRACER
select HAVE_GENERIC_VDSO
diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h
index 23e2ba78dcb0..a11996eb5892 100644
--- a/arch/loongarch/include/asm/ftrace.h
+++ b/arch/loongarch/include/asm/ftrace.h
@@ -100,4 +100,26 @@ __arch_ftrace_set_direct_caller(struct pt_regs *regs, unsigned long addr)
#endif /* CONFIG_FUNCTION_TRACER */
+#ifndef __ASSEMBLY__
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+struct fgraph_ret_regs {
+ /* a0 - a1 */
+ unsigned long regs[2];
+
+ unsigned long fp;
+ unsigned long __unused;
+};
+
+static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs)
+{
+ return ret_regs->regs[0];
+}
+
+static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs)
+{
+ return ret_regs->fp;
+}
+#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */
+#endif
+
#endif /* _ASM_LOONGARCH_FTRACE_H */
diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c
index 4bdb203fc66e..505e4bf59603 100644
--- a/arch/loongarch/kernel/asm-offsets.c
+++ b/arch/loongarch/kernel/asm-offsets.c
@@ -12,6 +12,7 @@
#include <asm/cpu-info.h>
#include <asm/ptrace.h>
#include <asm/processor.h>
+#include <asm/ftrace.h>
void output_ptreg_defines(void)
{
@@ -264,7 +265,7 @@ void output_smpboot_defines(void)
#ifdef CONFIG_HIBERNATION
void output_pbe_defines(void)
{
- COMMENT(" Linux struct pbe offsets. ");
+ COMMENT("Linux struct pbe offsets.");
OFFSET(PBE_ADDRESS, pbe, address);
OFFSET(PBE_ORIG_ADDRESS, pbe, orig_address);
OFFSET(PBE_NEXT, pbe, next);
@@ -272,3 +273,15 @@ void output_pbe_defines(void)
BLANK();
}
#endif
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+void output_fgraph_ret_regs_defines(void)
+{
+ COMMENT("LoongArch fgraph_ret_regs offsets.");
+ OFFSET(FGRET_REGS_A0, fgraph_ret_regs, regs[0]);
+ OFFSET(FGRET_REGS_A1, fgraph_ret_regs, regs[1]);
+ OFFSET(FGRET_REGS_FP, fgraph_ret_regs, fp);
+ DEFINE(FGRET_REGS_SIZE, sizeof(struct fgraph_ret_regs));
+ BLANK();
+}
+#endif
diff --git a/arch/loongarch/kernel/mcount.S b/arch/loongarch/kernel/mcount.S
index 8cdc1563cd33..cb8e5803de4b 100644
--- a/arch/loongarch/kernel/mcount.S
+++ b/arch/loongarch/kernel/mcount.S
@@ -79,18 +79,20 @@ SYM_FUNC_START(ftrace_graph_caller)
SYM_FUNC_END(ftrace_graph_caller)
SYM_FUNC_START(return_to_handler)
- PTR_ADDI sp, sp, -2 * SZREG
- PTR_S a0, sp, 0
- PTR_S a1, sp, SZREG
+ PTR_ADDI sp, sp, -FGRET_REGS_SIZE
+ PTR_S a0, sp, FGRET_REGS_A0
+ PTR_S a1, sp, FGRET_REGS_A1
+ PTR_S zero, sp, FGRET_REGS_FP
+ move a0, sp
bl ftrace_return_to_handler
/* Restore the real parent address: a0 -> ra */
move ra, a0
- PTR_L a0, sp, 0
- PTR_L a1, sp, SZREG
- PTR_ADDI sp, sp, 2 * SZREG
+ PTR_L a0, sp, FGRET_REGS_A0
+ PTR_L a1, sp, FGRET_REGS_A1
+ PTR_ADDI sp, sp, FGRET_REGS_SIZE
jr ra
SYM_FUNC_END(return_to_handler)
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S
index c7d961fc72c2..e16ab0b98e5a 100644
--- a/arch/loongarch/kernel/mcount_dyn.S
+++ b/arch/loongarch/kernel/mcount_dyn.S
@@ -136,18 +136,19 @@ SYM_CODE_END(ftrace_graph_caller)
SYM_CODE_START(return_to_handler)
/* Save return value regs */
- PTR_ADDI sp, sp, -2 * SZREG
- PTR_S a0, sp, 0
- PTR_S a1, sp, SZREG
+ PTR_ADDI sp, sp, -FGRET_REGS_SIZE
+ PTR_S a0, sp, FGRET_REGS_A0
+ PTR_S a1, sp, FGRET_REGS_A1
+ PTR_S zero, sp, FGRET_REGS_FP
- move a0, zero
+ move a0, sp
bl ftrace_return_to_handler
move ra, a0
/* Restore return value regs */
- PTR_L a0, sp, 0
- PTR_L a1, sp, SZREG
- PTR_ADDI sp, sp, 2 * SZREG
+ PTR_L a0, sp, FGRET_REGS_A0
+ PTR_L a1, sp, FGRET_REGS_A1
+ PTR_ADDI sp, sp, FGRET_REGS_SIZE
jr ra
SYM_CODE_END(return_to_handler)
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 08f3a01fa1ea..b49793cf34eb 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -152,6 +152,7 @@ config RISCV
select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE
select HAVE_FTRACE_MCOUNT_RECORD if !XIP_KERNEL
select HAVE_FUNCTION_GRAPH_TRACER
+ select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_TRACER if !XIP_KERNEL && !PREEMPTION
config CLANG_SUPPORTS_DYNAMIC_FTRACE
diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h
index d47d87c2d7e3..740a979171e5 100644
--- a/arch/riscv/include/asm/ftrace.h
+++ b/arch/riscv/include/asm/ftrace.h
@@ -111,4 +111,25 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec);
#endif /* CONFIG_DYNAMIC_FTRACE */
+#ifndef __ASSEMBLY__
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+struct fgraph_ret_regs {
+ unsigned long a1;
+ unsigned long a0;
+ unsigned long s0;
+ unsigned long ra;
+};
+
+static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs)
+{
+ return ret_regs->a0;
+}
+
+static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs)
+{
+ return ret_regs->s0;
+}
+#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */
+#endif
+
#endif /* _ASM_RISCV_FTRACE_H */
diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S
index 30102aadc4d7..8a6e5a9e842a 100644
--- a/arch/riscv/kernel/mcount.S
+++ b/arch/riscv/kernel/mcount.S
@@ -65,13 +65,8 @@ ENTRY(return_to_handler)
* So alternatively we check the *old* frame pointer position, that is, the
* value stored in -16(s0) on entry, and the s0 on return.
*/
-#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
- mv t6, s0
-#endif
SAVE_RET_ABI_STATE
-#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
- mv a0, t6
-#endif
+ mv a0, sp
call ftrace_return_to_handler
mv a2, a0
RESTORE_RET_ABI_STATE
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index b3ebf58cf77e..7422db409770 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -217,6 +217,7 @@ config X86
select HAVE_FAST_GUP
select HAVE_FENTRY if X86_64 || DYNAMIC_FTRACE
select HAVE_FTRACE_MCOUNT_RECORD
+ select HAVE_FUNCTION_GRAPH_RETVAL if HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_GRAPH_TRACER if X86_32 || (X86_64 && DYNAMIC_FTRACE)
select HAVE_FUNCTION_TRACER
select HAVE_GCC_PLUGINS
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index b8d4a07f9595..897cf02c20b1 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -150,4 +150,24 @@ static inline bool arch_trace_is_compat_syscall(struct pt_regs *regs)
#endif /* !COMPILE_OFFSETS */
#endif /* !__ASSEMBLY__ */
+#ifndef __ASSEMBLY__
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+struct fgraph_ret_regs {
+ unsigned long ax;
+ unsigned long dx;
+ unsigned long bp;
+};
+
+static inline unsigned long fgraph_ret_regs_return_value(struct fgraph_ret_regs *ret_regs)
+{
+ return ret_regs->ax;
+}
+
+static inline unsigned long fgraph_ret_regs_frame_pointer(struct fgraph_ret_regs *ret_regs)
+{
+ return ret_regs->bp;
+}
+#endif /* ifdef CONFIG_FUNCTION_GRAPH_TRACER */
+#endif
+
#endif /* _ASM_X86_FTRACE_H */
diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index 0d9a14528176..24c1175a47e2 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -187,12 +187,14 @@ SYM_CODE_END(ftrace_graph_caller)
.globl return_to_handler
return_to_handler:
- pushl %eax
+ pushl $0
pushl %edx
- movl $0, %eax
+ pushl %eax
+ movl %esp, %eax
call ftrace_return_to_handler
movl %eax, %ecx
- popl %edx
popl %eax
+ popl %edx
+ addl $4, %esp # skip ebp
JMP_NOSPEC ecx
#endif
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index b8c720b5dab2..945cfa5f7239 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -348,12 +348,13 @@ STACK_FRAME_NON_STANDARD_FP(__fentry__)
SYM_CODE_START(return_to_handler)
UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR
- subq $16, %rsp
+ subq $24, %rsp
/* Save the return values */
movq %rax, (%rsp)
movq %rdx, 8(%rsp)
- movq %rbp, %rdi
+ movq %rbp, 16(%rsp)
+ movq %rsp, %rdi
call ftrace_return_to_handler
@@ -361,7 +362,7 @@ SYM_CODE_START(return_to_handler)
movq 8(%rsp), %rdx
movq (%rsp), %rax
- addq $16, %rsp
+ addq $24, %rsp
/*
* Jump back to the old return address. This cannot be JMP_NOSPEC rdi
* since IBT would demand that contain ENDBR, which simply isn't so for
generated by cgit v1.2.3-58-ga151 (git 2.39.0) at 2024-11-27 06:23:22 +0000