summaryrefslogtreecommitdiff
path: root/arch/riscv
diff options
context:
space:
mode:
Diffstat (limited to 'arch/riscv')
-rw-r--r--arch/riscv/Kconfig12
-rw-r--r--arch/riscv/net/bpf_jit.h51
-rw-r--r--arch/riscv/net/bpf_jit_comp32.c3
-rw-r--r--arch/riscv/net/bpf_jit_comp64.c144
-rw-r--r--arch/riscv/net/bpf_jit_core.c5
5 files changed, 163 insertions, 52 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 0525ee2d63c7..9f38a5ecbee3 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -610,6 +610,18 @@ config TOOLCHAIN_HAS_VECTOR_CRYPTO
def_bool $(as-instr, .option arch$(comma) +v$(comma) +zvkb)
depends on AS_HAS_OPTION_ARCH
+config RISCV_ISA_ZBA
+ bool "Zba extension support for bit manipulation instructions"
+ default y
+ help
+ Add support for enabling optimisations in the kernel when the Zba
+ extension is detected at boot.
+
+ The Zba extension provides instructions to accelerate the generation
+ of addresses that index into arrays of basic data types.
+
+ If you don't know what to do here, say Y.
+
config RISCV_ISA_ZBB
bool "Zbb extension support for bit manipulation instructions"
depends on TOOLCHAIN_HAS_ZBB
diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h
index fdbf88ca8b70..1d1c78d4cff1 100644
--- a/arch/riscv/net/bpf_jit.h
+++ b/arch/riscv/net/bpf_jit.h
@@ -18,6 +18,11 @@ static inline bool rvc_enabled(void)
return IS_ENABLED(CONFIG_RISCV_ISA_C);
}
+static inline bool rvzba_enabled(void)
+{
+ return IS_ENABLED(CONFIG_RISCV_ISA_ZBA) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBA);
+}
+
static inline bool rvzbb_enabled(void)
{
return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB);
@@ -737,6 +742,17 @@ static inline u16 rvc_swsp(u32 imm8, u8 rs2)
return rv_css_insn(0x6, imm, rs2, 0x2);
}
+/* RVZBA instructions. */
+static inline u32 rvzba_sh2add(u8 rd, u8 rs1, u8 rs2)
+{
+ return rv_r_insn(0x10, rs2, rs1, 0x4, rd, 0x33);
+}
+
+static inline u32 rvzba_sh3add(u8 rd, u8 rs1, u8 rs2)
+{
+ return rv_r_insn(0x10, rs2, rs1, 0x6, rd, 0x33);
+}
+
/* RVZBB instructions. */
static inline u32 rvzbb_sextb(u8 rd, u8 rs1)
{
@@ -939,6 +955,14 @@ static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
return rv_css_insn(0x7, imm, rs2, 0x2);
}
+/* RV64-only ZBA instructions. */
+
+static inline u32 rvzba_zextw(u8 rd, u8 rs1)
+{
+ /* add.uw rd, rs1, ZERO */
+ return rv_r_insn(0x04, RV_REG_ZERO, rs1, 0, rd, 0x3b);
+}
+
#endif /* __riscv_xlen == 64 */
/* Helper functions that emit RVC instructions when possible. */
@@ -1082,6 +1106,28 @@ static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
emit(rv_sw(rs1, off, rs2), ctx);
}
+static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
+{
+ if (rvzba_enabled()) {
+ emit(rvzba_sh2add(rd, rs1, rs2), ctx);
+ return;
+ }
+
+ emit_slli(rd, rs1, 2, ctx);
+ emit_add(rd, rd, rs2, ctx);
+}
+
+static inline void emit_sh3add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
+{
+ if (rvzba_enabled()) {
+ emit(rvzba_sh3add(rd, rs1, rs2), ctx);
+ return;
+ }
+
+ emit_slli(rd, rs1, 3, ctx);
+ emit_add(rd, rd, rs2, ctx);
+}
+
/* RV64-only helper functions. */
#if __riscv_xlen == 64
@@ -1161,6 +1207,11 @@ static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx)
static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx)
{
+ if (rvzba_enabled()) {
+ emit(rvzba_zextw(rd, rs), ctx);
+ return;
+ }
+
emit_slli(rd, rs, 32, ctx);
emit_srli(rd, rd, 32, ctx);
}
diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c
index f5ba73bb153d..592dd86fbf81 100644
--- a/arch/riscv/net/bpf_jit_comp32.c
+++ b/arch/riscv/net/bpf_jit_comp32.c
@@ -811,8 +811,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
* if (!prog)
* goto out;
*/
- emit(rv_slli(RV_REG_T0, lo(idx_reg), 2), ctx);
- emit(rv_add(RV_REG_T0, RV_REG_T0, lo(arr_reg)), ctx);
+ emit_sh2add(RV_REG_T0, lo(idx_reg), lo(arr_reg), ctx);
off = offsetof(struct bpf_array, ptrs);
if (is_12b_check(off, insn))
return -1;
diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index 79a001d5533e..0795efdd3519 100644
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -15,7 +15,10 @@
#include <asm/percpu.h>
#include "bpf_jit.h"
+#define RV_MAX_REG_ARGS 8
#define RV_FENTRY_NINSNS 2
+/* imm that allows emit_imm to emit max count insns */
+#define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF
#define RV_REG_TCC RV_REG_A6
#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
@@ -380,8 +383,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
* if (!prog)
* goto out;
*/
- emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx);
- emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx);
+ emit_sh3add(RV_REG_T2, RV_REG_A2, RV_REG_A1, ctx);
off = offsetof(struct bpf_array, ptrs);
if (is_12b_check(off, insn))
return -1;
@@ -537,8 +539,10 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64,
/* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */
case BPF_CMPXCHG:
r0 = bpf_to_rv_reg(BPF_REG_0, ctx);
- emit(is64 ? rv_addi(RV_REG_T2, r0, 0) :
- rv_addiw(RV_REG_T2, r0, 0), ctx);
+ if (is64)
+ emit_mv(RV_REG_T2, r0, ctx);
+ else
+ emit_addiw(RV_REG_T2, r0, 0, ctx);
emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
rv_lr_w(r0, 0, rd, 0, 0), ctx);
jmp_offset = ninsns_rvoff(8);
@@ -689,26 +693,45 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
return ret;
}
-static void store_args(int nregs, int args_off, struct rv_jit_context *ctx)
+static void store_args(int nr_arg_slots, int args_off, struct rv_jit_context *ctx)
{
int i;
- for (i = 0; i < nregs; i++) {
- emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx);
+ for (i = 0; i < nr_arg_slots; i++) {
+ if (i < RV_MAX_REG_ARGS) {
+ emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx);
+ } else {
+ /* skip slots for T0 and FP of traced function */
+ emit_ld(RV_REG_T1, 16 + (i - RV_MAX_REG_ARGS) * 8, RV_REG_FP, ctx);
+ emit_sd(RV_REG_FP, -args_off, RV_REG_T1, ctx);
+ }
args_off -= 8;
}
}
-static void restore_args(int nregs, int args_off, struct rv_jit_context *ctx)
+static void restore_args(int nr_reg_args, int args_off, struct rv_jit_context *ctx)
{
int i;
- for (i = 0; i < nregs; i++) {
+ for (i = 0; i < nr_reg_args; i++) {
emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx);
args_off -= 8;
}
}
+static void restore_stack_args(int nr_stack_args, int args_off, int stk_arg_off,
+ struct rv_jit_context *ctx)
+{
+ int i;
+
+ for (i = 0; i < nr_stack_args; i++) {
+ emit_ld(RV_REG_T1, -(args_off - RV_MAX_REG_ARGS * 8), RV_REG_FP, ctx);
+ emit_sd(RV_REG_FP, -stk_arg_off, RV_REG_T1, ctx);
+ args_off -= 8;
+ stk_arg_off -= 8;
+ }
+}
+
static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off,
int run_ctx_off, bool save_ret, struct rv_jit_context *ctx)
{
@@ -781,8 +804,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
{
int i, ret, offset;
int *branches_off = NULL;
- int stack_size = 0, nregs = m->nr_args;
- int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off;
+ int stack_size = 0, nr_arg_slots = 0;
+ int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off, stk_arg_off;
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
@@ -828,20 +851,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
* FP - sreg_off [ callee saved reg ]
*
* [ pads ] pads for 16 bytes alignment
+ *
+ * [ stack_argN ]
+ * [ ... ]
+ * FP - stk_arg_off [ stack_arg1 ] BPF_TRAMP_F_CALL_ORIG
*/
if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
return -ENOTSUPP;
- /* extra regiters for struct arguments */
- for (i = 0; i < m->nr_args; i++)
- if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
- nregs += round_up(m->arg_size[i], 8) / 8 - 1;
-
- /* 8 arguments passed by registers */
- if (nregs > 8)
+ if (m->nr_args > MAX_BPF_FUNC_ARGS)
return -ENOTSUPP;
+ for (i = 0; i < m->nr_args; i++)
+ nr_arg_slots += round_up(m->arg_size[i], 8) / 8;
+
/* room of trampoline frame to store return address and frame pointer */
stack_size += 16;
@@ -851,7 +875,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
retval_off = stack_size;
}
- stack_size += nregs * 8;
+ stack_size += nr_arg_slots * 8;
args_off = stack_size;
stack_size += 8;
@@ -868,7 +892,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
stack_size += 8;
sreg_off = stack_size;
- stack_size = round_up(stack_size, 16);
+ if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0))
+ stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8;
+
+ stack_size = round_up(stack_size, STACK_ALIGN);
+
+ /* room for args on stack must be at the top of stack */
+ stk_arg_off = stack_size;
if (!is_struct_ops) {
/* For the trampoline called from function entry,
@@ -905,17 +935,17 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx);
}
- emit_li(RV_REG_T1, nregs, ctx);
+ emit_li(RV_REG_T1, nr_arg_slots, ctx);
emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx);
- store_args(nregs, args_off, ctx);
+ store_args(nr_arg_slots, args_off, ctx);
/* skip to actual body of traced function */
if (flags & BPF_TRAMP_F_SKIP_FRAME)
orig_call += RV_FENTRY_NINSNS * 4;
if (flags & BPF_TRAMP_F_CALL_ORIG) {
- emit_imm(RV_REG_A0, (const s64)im, ctx);
+ emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
ret = emit_call((const u64)__bpf_tramp_enter, true, ctx);
if (ret)
return ret;
@@ -948,13 +978,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
- restore_args(nregs, args_off, ctx);
+ restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx);
+ restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx);
ret = emit_call((const u64)orig_call, true, ctx);
if (ret)
goto out;
emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx);
emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx);
- im->ip_after_call = ctx->insns + ctx->ninsns;
+ im->ip_after_call = ctx->ro_insns + ctx->ninsns;
/* 2 nops reserved for auipc+jalr pair */
emit(rv_nop(), ctx);
emit(rv_nop(), ctx);
@@ -975,15 +1006,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
- im->ip_epilogue = ctx->insns + ctx->ninsns;
- emit_imm(RV_REG_A0, (const s64)im, ctx);
+ im->ip_epilogue = ctx->ro_insns + ctx->ninsns;
+ emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
ret = emit_call((const u64)__bpf_tramp_exit, true, ctx);
if (ret)
goto out;
}
if (flags & BPF_TRAMP_F_RESTORE_REGS)
- restore_args(nregs, args_off, ctx);
+ restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx);
if (save_ret) {
emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx);
@@ -1038,31 +1069,52 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
return ret < 0 ? ret : ninsns_rvoff(ctx.ninsns);
}
-int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
- void *image_end, const struct btf_func_model *m,
+void *arch_alloc_bpf_trampoline(unsigned int size)
+{
+ return bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
+}
+
+void arch_free_bpf_trampoline(void *image, unsigned int size)
+{
+ bpf_prog_pack_free(image, size);
+}
+
+int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
+ void *ro_image_end, const struct btf_func_model *m,
u32 flags, struct bpf_tramp_links *tlinks,
void *func_addr)
{
int ret;
+ void *image, *res;
struct rv_jit_context ctx;
+ u32 size = ro_image_end - ro_image;
+
+ image = kvmalloc(size, GFP_KERNEL);
+ if (!image)
+ return -ENOMEM;
ctx.ninsns = 0;
- /*
- * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the
- * JITed instructions and later copies it to a RX region (ctx.ro_insns).
- * It also uses ctx.ro_insns to calculate offsets for jumps etc. As the
- * trampoline image uses the same memory area for writing and execution,
- * both ctx.insns and ctx.ro_insns can be set to image.
- */
ctx.insns = image;
- ctx.ro_insns = image;
+ ctx.ro_insns = ro_image;
ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
if (ret < 0)
- return ret;
+ goto out;
- bpf_flush_icache(ctx.insns, ctx.insns + ctx.ninsns);
+ if (WARN_ON(size < ninsns_rvoff(ctx.ninsns))) {
+ ret = -E2BIG;
+ goto out;
+ }
+
+ res = bpf_arch_text_copy(ro_image, image, size);
+ if (IS_ERR(res)) {
+ ret = PTR_ERR(res);
+ goto out;
+ }
- return ninsns_rvoff(ret);
+ bpf_flush_icache(ro_image, ro_image_end);
+out:
+ kvfree(image);
+ return ret < 0 ? ret : size;
}
int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
@@ -1097,12 +1149,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
/* Load current CPU number in T1 */
emit_ld(RV_REG_T1, offsetof(struct thread_info, cpu),
RV_REG_TP, ctx);
- /* << 3 because offsets are 8 bytes */
- emit_slli(RV_REG_T1, RV_REG_T1, 3, ctx);
/* Load address of __per_cpu_offset array in T2 */
emit_addr(RV_REG_T2, (u64)&__per_cpu_offset, extra_pass, ctx);
- /* Add offset of current CPU to __per_cpu_offset */
- emit_add(RV_REG_T1, RV_REG_T2, RV_REG_T1, ctx);
+ /* Get address of __per_cpu_offset[cpu] in T1 */
+ emit_sh3add(RV_REG_T1, RV_REG_T1, RV_REG_T2, ctx);
/* Load __per_cpu_offset[cpu] in T1 */
emit_ld(RV_REG_T1, 0, RV_REG_T1, ctx);
/* Add the offset to Rd */
@@ -1960,7 +2010,7 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
{
int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
- bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
+ bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN);
if (bpf_stack_adjust)
mark_fp(ctx);
@@ -1982,7 +2032,7 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
if (ctx->arena_vm_start)
stack_adjust += 8;
- stack_adjust = round_up(stack_adjust, 16);
+ stack_adjust = round_up(stack_adjust, STACK_ALIGN);
stack_adjust += bpf_stack_adjust;
store_offset = stack_adjust - 8;
diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index 0a96abdaca65..6de753c667f4 100644
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -178,8 +178,7 @@ skip_init_ctx:
prog->jited_len = prog_size - cfi_get_offset();
if (!prog->is_func || extra_pass) {
- if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,
- jit_data->header))) {
+ if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) {
/* ro_header has been freed */
jit_data->ro_header = NULL;
prog = orig_prog;
@@ -258,7 +257,7 @@ void bpf_jit_free(struct bpf_prog *prog)
* before freeing it.
*/
if (jit_data) {
- bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header);
+ bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header);
kfree(jit_data);
}
hdr = bpf_jit_binary_pack_hdr(prog);
generated by cgit v1.2.3-58-ga151 (git 2.39.0) at 2024-11-21 16:40:56 +0000