diff options
author | Marc Zyngier <maz@kernel.org> | 2023-04-21 14:05:17 +0100 |
---|---|---|
committer | Marc Zyngier <maz@kernel.org> | 2023-04-21 14:05:17 +0100 |
commit | e7b5771aa08746c13bb03ebe0a5053df1498c328 (patch) | |
tree | 04360f32dd2cd16196fc3379ad6bdab1ffbb4dd0 /arch | |
parent | f39157b3c0bb4afdcaea7809b3669de8e0495ed5 (diff) | |
parent | f8415f2def181c63486e93c511b82692e0914d9e (diff) |
Merge branch irq/riscv-ipi into irq/irqchip-next
* irq/riscv-ipi:
: .
: RISC-V IPI rework from Anup Patel:
:
: "This series aims to improve IPI support in Linux RISC-V in following ways:
: 1) Treat IPIs as normal per-CPU interrupts instead of having custom RISC-V
: specific hooks. This also makes Linux RISC-V IPI support aligned with
: other architectures.
: 2) Remote TLB flushes and icache flushes should prefer local IPIs instead
: of SBI calls whenever we have specialized hardware (such as RISC-V AIA
: IMSIC and RISC-V SWI) which allows S-mode software to directly inject
: IPIs without any assistance from M-mode runtime firmware."
: .
irqchip/riscv-intc: Add empty irq_eoi() for chained irq handlers
RISC-V: Use IPIs for remote icache flush when possible
RISC-V: Use IPIs for remote TLB flush when possible
RISC-V: Allow marking IPIs as suitable for remote FENCEs
RISC-V: Treat IPIs as normal Linux IRQs
irqchip/riscv-intc: Allow drivers to directly discover INTC hwnode
RISC-V: Clear SIP bit only when using SBI IPI operations
Signed-off-by: Marc Zyngier <maz@kernel.org>
Diffstat (limited to 'arch')
-rw-r--r-- | arch/riscv/Kconfig | 2 | ||||
-rw-r--r-- | arch/riscv/include/asm/irq.h | 4 | ||||
-rw-r--r-- | arch/riscv/include/asm/sbi.h | 9 | ||||
-rw-r--r-- | arch/riscv/include/asm/smp.h | 49 | ||||
-rw-r--r-- | arch/riscv/kernel/Makefile | 1 | ||||
-rw-r--r-- | arch/riscv/kernel/cpu-hotplug.c | 3 | ||||
-rw-r--r-- | arch/riscv/kernel/irq.c | 21 | ||||
-rw-r--r-- | arch/riscv/kernel/sbi-ipi.c | 77 | ||||
-rw-r--r-- | arch/riscv/kernel/sbi.c | 100 | ||||
-rw-r--r-- | arch/riscv/kernel/smp.c | 171 | ||||
-rw-r--r-- | arch/riscv/kernel/smpboot.c | 5 | ||||
-rw-r--r-- | arch/riscv/mm/cacheflush.c | 5 | ||||
-rw-r--r-- | arch/riscv/mm/tlbflush.c | 93 |
13 files changed, 335 insertions, 205 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 5b182d1c196c..b90be751651f 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -63,6 +63,8 @@ config RISCV select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO select GENERIC_IDLE_POLL_SETUP select GENERIC_IOREMAP if MMU + select GENERIC_IRQ_IPI if SMP + select GENERIC_IRQ_IPI_MUX if SMP select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index e4c435509983..43b9ebfbd943 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -12,6 +12,10 @@ #include <asm-generic/irq.h> +void riscv_set_intc_hwnode_fn(struct fwnode_handle *(*fn)(void)); + +struct fwnode_handle *riscv_get_intc_hwnode(void); + extern void __init init_IRQ(void); #endif /* _ASM_RISCV_IRQ_H */ diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 945b7be249c1..acab4410ef2a 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -271,8 +271,7 @@ long sbi_get_marchid(void); long sbi_get_mimpid(void); void sbi_set_timer(uint64_t stime_value); void sbi_shutdown(void); -void sbi_clear_ipi(void); -int sbi_send_ipi(const struct cpumask *cpu_mask); +void sbi_send_ipi(unsigned int cpu); int sbi_remote_fence_i(const struct cpumask *cpu_mask); int sbi_remote_sfence_vma(const struct cpumask *cpu_mask, unsigned long start, @@ -335,4 +334,10 @@ unsigned long riscv_cached_mvendorid(unsigned int cpu_id); unsigned long riscv_cached_marchid(unsigned int cpu_id); unsigned long riscv_cached_mimpid(unsigned int cpu_id); +#if IS_ENABLED(CONFIG_SMP) && IS_ENABLED(CONFIG_RISCV_SBI) +void sbi_ipi_init(void); +#else +static inline void sbi_ipi_init(void) { } +#endif + #endif /* _ASM_RISCV_SBI_H */ diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index 3831b638ecab..c4b77017ec58 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -15,12 +15,10 @@ struct seq_file; extern unsigned long boot_cpu_hartid; -struct riscv_ipi_ops { - void (*ipi_inject)(const struct cpumask *target); - void (*ipi_clear)(void); -}; - #ifdef CONFIG_SMP + +#include <linux/jump_label.h> + /* * Mapping between linux logical cpu index and hartid. */ @@ -33,9 +31,6 @@ void show_ipi_stats(struct seq_file *p, int prec); /* SMP initialization hook for setup_arch */ void __init setup_smp(void); -/* Called from C code, this handles an IPI. */ -void handle_IPI(struct pt_regs *regs); - /* Hook for the generic smp_call_function_many() routine. */ void arch_send_call_function_ipi_mask(struct cpumask *mask); @@ -44,11 +39,22 @@ void arch_send_call_function_single_ipi(int cpu); int riscv_hartid_to_cpuid(unsigned long hartid); -/* Set custom IPI operations */ -void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops); +/* Enable IPI for CPU hotplug */ +void riscv_ipi_enable(void); + +/* Disable IPI for CPU hotplug */ +void riscv_ipi_disable(void); -/* Clear IPI for current CPU */ -void riscv_clear_ipi(void); +/* Check if IPI interrupt numbers are available */ +bool riscv_ipi_have_virq_range(void); + +/* Set the IPI interrupt numbers for arch (called by irqchip drivers) */ +void riscv_ipi_set_virq_range(int virq, int nr, bool use_for_rfence); + +/* Check if we can use IPIs for remote FENCEs */ +DECLARE_STATIC_KEY_FALSE(riscv_ipi_for_rfence); +#define riscv_use_ipi_for_rfence() \ + static_branch_unlikely(&riscv_ipi_for_rfence) /* Check other CPUs stop or not */ bool smp_crash_stop_failed(void); @@ -85,14 +91,29 @@ static inline unsigned long cpuid_to_hartid_map(int cpu) return boot_cpu_hartid; } -static inline void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops) +static inline void riscv_ipi_enable(void) +{ +} + +static inline void riscv_ipi_disable(void) { } -static inline void riscv_clear_ipi(void) +static inline bool riscv_ipi_have_virq_range(void) +{ + return false; +} + +static inline void riscv_ipi_set_virq_range(int virq, int nr, + bool use_for_rfence) { } +static inline bool riscv_use_ipi_for_rfence(void) +{ + return false; +} + #endif /* CONFIG_SMP */ #if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP) diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 4cf303a779ab..67f542be1bea 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -74,6 +74,7 @@ obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o obj-$(CONFIG_RISCV_SBI) += sbi.o ifeq ($(CONFIG_RISCV_SBI), y) +obj-$(CONFIG_SMP) += sbi-ipi.o obj-$(CONFIG_SMP) += cpu_ops_sbi.o endif obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index f7a832e3a1d1..39235cf50652 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -13,7 +13,7 @@ #include <asm/irq.h> #include <asm/cpu_ops.h> #include <asm/numa.h> -#include <asm/sbi.h> +#include <asm/smp.h> bool cpu_has_hotplug(unsigned int cpu) { @@ -43,6 +43,7 @@ int __cpu_disable(void) remove_cpu_topology(cpu); numa_remove_cpu(cpu); set_cpu_online(cpu, false); + riscv_ipi_disable(); irq_migrate_all_off_this_cpu(); return ret; diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 7207fa08d78f..eb9a68a539e6 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -7,8 +7,26 @@ #include <linux/interrupt.h> #include <linux/irqchip.h> +#include <linux/irqdomain.h> +#include <linux/module.h> #include <linux/seq_file.h> -#include <asm/smp.h> +#include <asm/sbi.h> + +static struct fwnode_handle *(*__get_intc_node)(void); + +void riscv_set_intc_hwnode_fn(struct fwnode_handle *(*fn)(void)) +{ + __get_intc_node = fn; +} + +struct fwnode_handle *riscv_get_intc_hwnode(void) +{ + if (__get_intc_node) + return __get_intc_node(); + + return NULL; +} +EXPORT_SYMBOL_GPL(riscv_get_intc_hwnode); int arch_show_interrupts(struct seq_file *p, int prec) { @@ -21,4 +39,5 @@ void __init init_IRQ(void) irqchip_init(); if (!handle_arch_irq) panic("No interrupt controller found."); + sbi_ipi_init(); } diff --git a/arch/riscv/kernel/sbi-ipi.c b/arch/riscv/kernel/sbi-ipi.c new file mode 100644 index 000000000000..a4559695ce62 --- /dev/null +++ b/arch/riscv/kernel/sbi-ipi.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Multiplex several IPIs over a single HW IPI. + * + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "riscv: " fmt +#include <linux/cpu.h> +#include <linux/init.h> +#include <linux/irq.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <asm/sbi.h> + +static int sbi_ipi_virq; + +static void sbi_ipi_handle(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + + chained_irq_enter(chip, desc); + + csr_clear(CSR_IP, IE_SIE); + ipi_mux_process(); + + chained_irq_exit(chip, desc); +} + +static int sbi_ipi_starting_cpu(unsigned int cpu) +{ + enable_percpu_irq(sbi_ipi_virq, irq_get_trigger_type(sbi_ipi_virq)); + return 0; +} + +void __init sbi_ipi_init(void) +{ + int virq; + struct irq_domain *domain; + + if (riscv_ipi_have_virq_range()) + return; + + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), + DOMAIN_BUS_ANY); + if (!domain) { + pr_err("unable to find INTC IRQ domain\n"); + return; + } + + sbi_ipi_virq = irq_create_mapping(domain, RV_IRQ_SOFT); + if (!sbi_ipi_virq) { + pr_err("unable to create INTC IRQ mapping\n"); + return; + } + + virq = ipi_mux_create(BITS_PER_BYTE, sbi_send_ipi); + if (virq <= 0) { + pr_err("unable to create muxed IPIs\n"); + irq_dispose_mapping(sbi_ipi_virq); + return; + } + + irq_set_chained_handler(sbi_ipi_virq, sbi_ipi_handle); + + /* + * Don't disable IPI when CPU goes offline because + * the masking/unmasking of virtual IPIs is done + * via generic IPI-Mux + */ + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "irqchip/sbi-ipi:starting", + sbi_ipi_starting_cpu, NULL); + + riscv_ipi_set_virq_range(virq, BITS_PER_BYTE, false); + pr_info("providing IPIs using SBI IPI extension\n"); +} diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index 5c87db8fdff2..92b9b759ab3d 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -17,7 +17,7 @@ unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT; EXPORT_SYMBOL(sbi_spec_version); static void (*__sbi_set_timer)(uint64_t stime) __ro_after_init; -static int (*__sbi_send_ipi)(const struct cpumask *cpu_mask) __ro_after_init; +static void (*__sbi_send_ipi)(unsigned int cpu) __ro_after_init; static int (*__sbi_rfence)(int fid, const struct cpumask *cpu_mask, unsigned long start, unsigned long size, unsigned long arg4, unsigned long arg5) __ro_after_init; @@ -131,17 +131,6 @@ void sbi_shutdown(void) EXPORT_SYMBOL(sbi_shutdown); /** - * sbi_clear_ipi() - Clear any pending IPIs for the calling hart. - * - * Return: None - */ -void sbi_clear_ipi(void) -{ - sbi_ecall(SBI_EXT_0_1_CLEAR_IPI, 0, 0, 0, 0, 0, 0, 0); -} -EXPORT_SYMBOL(sbi_clear_ipi); - -/** * __sbi_set_timer_v01() - Program the timer for next timer event. * @stime_value: The value after which next timer event should fire. * @@ -157,17 +146,12 @@ static void __sbi_set_timer_v01(uint64_t stime_value) #endif } -static int __sbi_send_ipi_v01(const struct cpumask *cpu_mask) +static void __sbi_send_ipi_v01(unsigned int cpu) { - unsigned long hart_mask; - - if (!cpu_mask || cpumask_empty(cpu_mask)) - cpu_mask = cpu_online_mask; - hart_mask = __sbi_v01_cpumask_to_hartmask(cpu_mask); - + unsigned long hart_mask = + __sbi_v01_cpumask_to_hartmask(cpumask_of(cpu)); sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)(&hart_mask), 0, 0, 0, 0, 0); - return 0; } static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, @@ -216,12 +200,10 @@ static void __sbi_set_timer_v01(uint64_t stime_value) sbi_major_version(), sbi_minor_version()); } -static int __sbi_send_ipi_v01(const struct cpumask *cpu_mask) +static void __sbi_send_ipi_v01(unsigned int cpu) { pr_warn("IPI extension is not available in SBI v%lu.%lu\n", sbi_major_version(), sbi_minor_version()); - - return 0; } static int __sbi_rfence_v01(int fid, const struct cpumask *cpu_mask, @@ -248,55 +230,18 @@ static void __sbi_set_timer_v02(uint64_t stime_value) #endif } -static int __sbi_send_ipi_v02(const struct cpumask *cpu_mask) +static void __sbi_send_ipi_v02(unsigned int cpu) { - unsigned long hartid, cpuid, hmask = 0, hbase = 0, htop = 0; - struct sbiret ret = {0}; int result; + struct sbiret ret = {0}; - if (!cpu_mask || cpumask_empty(cpu_mask)) - cpu_mask = cpu_online_mask; - - for_each_cpu(cpuid, cpu_mask) { - hartid = cpuid_to_hartid_map(cpuid); - if (hmask) { - if (hartid + BITS_PER_LONG <= htop || - hbase + BITS_PER_LONG <= hartid) { - ret = sbi_ecall(SBI_EXT_IPI, - SBI_EXT_IPI_SEND_IPI, hmask, - hbase, 0, 0, 0, 0); - if (ret.error) - goto ecall_failed; - hmask = 0; - } else if (hartid < hbase) { - /* shift the mask to fit lower hartid */ - hmask <<= hbase - hartid; - hbase = hartid; - } - } - if (!hmask) { - hbase = hartid; - htop = hartid; - } else if (hartid > htop) { - htop = hartid; - } - hmask |= BIT(hartid - hbase); - } - - if (hmask) { - ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, - hmask, hbase, 0, 0, 0, 0); - if (ret.error) - goto ecall_failed; + ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, + 1UL, cpuid_to_hartid_map(cpu), 0, 0, 0, 0); + if (ret.error) { + result = sbi_err_map_linux_errno(ret.error); + pr_err("%s: hbase = [%lu] failed (error [%d])\n", + __func__, cpuid_to_hartid_map(cpu), result); } - - return 0; - -ecall_failed: - result = sbi_err_map_linux_errno(ret.error); - pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", - __func__, hbase, hmask, result); - return result; } static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask, @@ -410,13 +355,11 @@ void sbi_set_timer(uint64_t stime_value) /** * sbi_send_ipi() - Send an IPI to any hart. - * @cpu_mask: A cpu mask containing all the target harts. - * - * Return: 0 on success, appropriate linux error code otherwise. + * @cpu: Logical id of the target CPU. */ -int sbi_send_ipi(const struct cpumask *cpu_mask) +void sbi_send_ipi(unsigned int cpu) { - return __sbi_send_ipi(cpu_mask); + __sbi_send_ipi(cpu); } EXPORT_SYMBOL(sbi_send_ipi); @@ -641,15 +584,6 @@ long sbi_get_mimpid(void) } EXPORT_SYMBOL_GPL(sbi_get_mimpid); -static void sbi_send_cpumask_ipi(const struct cpumask *target) -{ - sbi_send_ipi(target); -} - -static const struct riscv_ipi_ops sbi_ipi_ops = { - .ipi_inject = sbi_send_cpumask_ipi -}; - void __init sbi_init(void) { int ret; @@ -696,6 +630,4 @@ void __init sbi_init(void) __sbi_send_ipi = __sbi_send_ipi_v01; __sbi_rfence = __sbi_rfence_v01; } - - riscv_set_ipi_ops(&sbi_ipi_ops); } diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 8c3b59f1f9b8..5f985a197eff 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -13,14 +13,15 @@ #include <linux/interrupt.h> #include <linux/module.h> #include <linux/kexec.h> +#include <linux/percpu.h> #include <linux/profile.h> #include <linux/smp.h> #include <linux/sched.h> #include <linux/seq_file.h> #include <linux/delay.h> +#include <linux/irq.h> #include <linux/irq_work.h> -#include <asm/sbi.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> #include <asm/cpu_ops.h> @@ -44,11 +45,10 @@ void __init smp_setup_processor_id(void) cpuid_to_hartid_map(0) = boot_cpu_hartid; } -/* A collection of single bit ipi messages. */ -static struct { - unsigned long stats[IPI_MAX] ____cacheline_aligned; - unsigned long bits ____cacheline_aligned; -} ipi_data[NR_CPUS] __cacheline_aligned; +static DEFINE_PER_CPU_READ_MOSTLY(int, ipi_dummy_dev); +static int ipi_virq_base __ro_after_init; +static int nr_ipi __ro_after_init = IPI_MAX; +static struct irq_desc *ipi_desc[IPI_MAX] __read_mostly; int riscv_hartid_to_cpuid(unsigned long hartid) { @@ -100,48 +100,14 @@ static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) } #endif -static const struct riscv_ipi_ops *ipi_ops __ro_after_init; - -void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops) -{ - ipi_ops = ops; -} -EXPORT_SYMBOL_GPL(riscv_set_ipi_ops); - -void riscv_clear_ipi(void) -{ - if (ipi_ops && ipi_ops->ipi_clear) - ipi_ops->ipi_clear(); - - csr_clear(CSR_IP, IE_SIE); -} -EXPORT_SYMBOL_GPL(riscv_clear_ipi); - static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op) { - int cpu; - - smp_mb__before_atomic(); - for_each_cpu(cpu, mask) - set_bit(op, &ipi_data[cpu].bits); - smp_mb__after_atomic(); - - if (ipi_ops && ipi_ops->ipi_inject) - ipi_ops->ipi_inject(mask); - else - pr_warn("SMP: IPI inject method not available\n"); + __ipi_send_mask(ipi_desc[op], mask); } static void send_ipi_single(int cpu, enum ipi_message_type op) { - smp_mb__before_atomic(); - set_bit(op, &ipi_data[cpu].bits); - smp_mb__after_atomic(); - - if (ipi_ops && ipi_ops->ipi_inject) - ipi_ops->ipi_inject(cpumask_of(cpu)); - else - pr_warn("SMP: IPI inject method not available\n"); + __ipi_send_mask(ipi_desc[op], cpumask_of(cpu)); } #ifdef CONFIG_IRQ_WORK @@ -151,59 +117,98 @@ void arch_irq_work_raise(void) } #endif -void handle_IPI(struct pt_regs *regs) +static irqreturn_t handle_IPI(int irq, void *data) { - unsigned int cpu = smp_processor_id(); - unsigned long *pending_ipis = &ipi_data[cpu].bits; - unsigned long *stats = ipi_data[cpu].stats; + int ipi = irq - ipi_virq_base; + + switch (ipi) { + case IPI_RESCHEDULE: + scheduler_ipi(); + break; + case IPI_CALL_FUNC: + generic_smp_call_function_interrupt(); + break; + case IPI_CPU_STOP: + ipi_stop(); + break; + case IPI_CPU_CRASH_STOP: + ipi_cpu_crash_stop(smp_processor_id(), get_irq_regs()); + break; + case IPI_IRQ_WORK: + irq_work_run(); + break; +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST + case IPI_TIMER: + tick_receive_broadcast(); + break; +#endif + default: + pr_warn("CPU%d: unhandled IPI%d\n", smp_processor_id(), ipi); + break; + } - riscv_clear_ipi(); + return IRQ_HANDLED; +} - while (true) { - unsigned long ops; +void riscv_ipi_enable(void) +{ + int i; - /* Order bit clearing and data access. */ - mb(); + if (WARN_ON_ONCE(!ipi_virq_base)) + return; - ops = xchg(pending_ipis, 0); - if (ops == 0) - return; + for (i = 0; i < nr_ipi; i++) + enable_percpu_irq(ipi_virq_base + i, 0); +} - if (ops & (1 << IPI_RESCHEDULE)) { - stats[IPI_RESCHEDULE]++; - scheduler_ipi(); - } +void riscv_ipi_disable(void) +{ + int i; - if (ops & (1 << IPI_CALL_FUNC)) { - stats[IPI_CALL_FUNC]++; - generic_smp_call_function_interrupt(); - } + if (WARN_ON_ONCE(!ipi_virq_base)) + return; - if (ops & (1 << IPI_CPU_STOP)) { - stats[IPI_CPU_STOP]++; - ipi_stop(); - } + for (i = 0; i < nr_ipi; i++) + disable_percpu_irq(ipi_virq_base + i); +} - if (ops & (1 << IPI_CPU_CRASH_STOP)) { - ipi_cpu_crash_stop(cpu, get_irq_regs()); - } +bool riscv_ipi_have_virq_range(void) +{ + return (ipi_virq_base) ? true : false; +} - if (ops & (1 << IPI_IRQ_WORK)) { - stats[IPI_IRQ_WORK]++; - irq_work_run(); - } +DEFINE_STATIC_KEY_FALSE(riscv_ipi_for_rfence); +EXPORT_SYMBOL_GPL(riscv_ipi_for_rfence); -#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST - if (ops & (1 << IPI_TIMER)) { - stats[IPI_TIMER]++; - tick_receive_broadcast(); - } -#endif - BUG_ON((ops >> IPI_MAX) != 0); +void riscv_ipi_set_virq_range(int virq, int nr, bool use_for_rfence) +{ + int i, err; + + if (WARN_ON(ipi_virq_base)) + return; + + WARN_ON(nr < IPI_MAX); + nr_ipi = min(nr, IPI_MAX); + ipi_virq_base = virq; - /* Order data access and bit testing. */ - mb(); + /* Request IPIs */ + for (i = 0; i < nr_ipi; i++) { + err = request_percpu_irq(ipi_virq_base + i, handle_IPI, + "IPI", &ipi_dummy_dev); + WARN_ON(err); + + ipi_desc[i] = irq_to_desc(ipi_virq_base + i); + irq_set_status_flags(ipi_virq_base + i, IRQ_HIDDEN); } + + /* Enabled IPIs for boot CPU immediately */ + riscv_ipi_enable(); + + /* Update RFENCE static key */ + if (use_for_rfence) + static_branch_enable(&riscv_ipi_for_rfence); + else + static_branch_disable(&riscv_ipi_for_rfence); } static const char * const ipi_names[] = { @@ -223,7 +228,7 @@ void show_ipi_stats(struct seq_file *p, int prec) seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); for_each_online_cpu(cpu) - seq_printf(p, "%10lu ", ipi_data[cpu].stats[i]); + seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu)); seq_printf(p, " %s\n", ipi_names[i]); } } diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index ddb2afba6d25..00b53913d4c6 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -30,7 +30,6 @@ #include <asm/numa.h> #include <asm/tlbflush.h> #include <asm/sections.h> -#include <asm/sbi.h> #include <asm/smp.h> #include "head.h" @@ -158,12 +157,12 @@ asmlinkage __visible void smp_callin(void) struct mm_struct *mm = &init_mm; unsigned int curr_cpuid = smp_processor_id(); - riscv_clear_ipi(); - /* All kernel threads share the same mm context. */ mmgrab(mm); current->active_mm = mm; + riscv_ipi_enable(); + store_cpu_topology(curr_cpuid); notify_cpu_starting(curr_cpuid); numa_add_cpu(curr_cpuid); diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index fcd6145fbead..20cec5e7cdbf 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -19,7 +19,7 @@ void flush_icache_all(void) { local_flush_icache_all(); - if (IS_ENABLED(CONFIG_RISCV_SBI)) + if (IS_ENABLED(CONFIG_RISCV_SBI) && !riscv_use_ipi_for_rfence()) sbi_remote_fence_i(NULL); else on_each_cpu(ipi_remote_fence_i, NULL, 1); @@ -67,7 +67,8 @@ void flush_icache_mm(struct mm_struct *mm, bool local) * with flush_icache_deferred(). */ smp_mb(); - } else if (IS_ENABLED(CONFIG_RISCV_SBI)) { + } else if (IS_ENABLED(CONFIG_RISCV_SBI) && + !riscv_use_ipi_for_rfence()) { sbi_remote_fence_i(&others); } else { on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index ef701fa83f36..77be59aadc73 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -23,14 +23,62 @@ static inline void local_flush_tlb_page_asid(unsigned long addr, : "memory"); } +static inline void local_flush_tlb_range(unsigned long start, + unsigned long size, unsigned long stride) +{ + if (size <= stride) + local_flush_tlb_page(start); + else + local_flush_tlb_all(); +} + +static inline void local_flush_tlb_range_asid(unsigned long start, + unsigned long size, unsigned long stride, unsigned long asid) +{ + if (size <= stride) + local_flush_tlb_page_asid(start, asid); + else + local_flush_tlb_all_asid(asid); +} + +static void __ipi_flush_tlb_all(void *info) +{ + local_flush_tlb_all(); +} + void flush_tlb_all(void) { - sbi_remote_sfence_vma(NULL, 0, -1); + if (riscv_use_ipi_for_rfence()) + on_each_cpu(__ipi_flush_tlb_all, NULL, 1); + else + sbi_remote_sfence_vma(NULL, 0, -1); +} + +struct flush_tlb_range_data { + unsigned long asid; + unsigned long start; + unsigned long size; + unsigned long stride; +}; + +static void __ipi_flush_tlb_range_asid(void *info) +{ + struct flush_tlb_range_data *d = info; + + local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); +} + +static void __ipi_flush_tlb_range(void *info) +{ + struct flush_tlb_range_data *d = info; + + local_flush_tlb_range(d->start, d->size, d->stride); } -static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, - unsigned long size, unsigned long stride) +static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, + unsigned long size, unsigned long stride) { + struct flush_tlb_range_data ftd; struct cpumask *cmask = mm_cpumask(mm); unsigned int cpuid; bool broadcast; @@ -45,19 +93,34 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, unsigned long asid = atomic_long_read(&mm->context.id) & asid_mask; if (broadcast) { - sbi_remote_sfence_vma_asid(cmask, start, size, asid); - } else if (size <= stride) { - local_flush_tlb_page_asid(start, asid); + if (riscv_use_ipi_for_rfence()) { + ftd.asid = asid; + ftd.start = start; + ftd.size = size; + ftd.stride = stride; + on_each_cpu_mask(cmask, + __ipi_flush_tlb_range_asid, + &ftd, 1); + } else + sbi_remote_sfence_vma_asid(cmask, + start, size, asid); } else { - local_flush_tlb_all_asid(asid); + local_flush_tlb_range_asid(start, size, stride, asid); } } else { if (broadcast) { - sbi_remote_sfence_vma(cmask, start, size); - } else if (size <= stride) { - local_flush_tlb_page(start); + if (riscv_use_ipi_for_rfence()) { + ftd.asid = 0; + ftd.start = start; + ftd.size = size; + ftd.stride = stride; + on_each_cpu_mask(cmask, + __ipi_flush_tlb_range, + &ftd, 1); + } else + sbi_remote_sfence_vma(cmask, start, size); } else { - local_flush_tlb_all(); + local_flush_tlb_range(start, size, stride); } } @@ -66,23 +129,23 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, void flush_tlb_mm(struct mm_struct *mm) { - __sbi_tlb_flush_range(mm, 0, -1, PAGE_SIZE); + __flush_tlb_range(mm, 0, -1, PAGE_SIZE); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) { - __sbi_tlb_flush_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); + __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); } void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PAGE_SIZE); + __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PMD_SIZE); + __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE); } #endif |