From c5905afb0ee6550b42c49213da1c22d67316c194 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 24 Feb 2012 08:31:31 +0100 Subject: static keys: Introduce 'struct static_key', static_key_true()/false() and static_key_slow_[inc|dec]() So here's a boot tested patch on top of Jason's series that does all the cleanups I talked about and turns jump labels into a more intuitive to use facility. It should also address the various misconceptions and confusions that surround jump labels. Typical usage scenarios: #include struct static_key key = STATIC_KEY_INIT_TRUE; if (static_key_false(&key)) do unlikely code else do likely code Or: if (static_key_true(&key)) do likely code else do unlikely code The static key is modified via: static_key_slow_inc(&key); ... static_key_slow_dec(&key); The 'slow' prefix makes it abundantly clear that this is an expensive operation. I've updated all in-kernel code to use this everywhere. Note that I (intentionally) have not pushed through the rename blindly through to the lowest levels: the actual jump-label patching arch facility should be named like that, so we want to decouple jump labels from the static-key facility a bit. On non-jump-label enabled architectures static keys default to likely()/unlikely() branches. Signed-off-by: Ingo Molnar Acked-by: Jason Baron Acked-by: Steven Rostedt Cc: a.p.zijlstra@chello.nl Cc: mathieu.desnoyers@efficios.com Cc: davem@davemloft.net Cc: ddaney.cavm@gmail.com Cc: Linus Torvalds Link: http://lkml.kernel.org/r/20120222085809.GA26397@elte.hu Signed-off-by: Ingo Molnar --- arch/Kconfig | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) (limited to 'arch/Kconfig') diff --git a/arch/Kconfig b/arch/Kconfig index 4f55c736be11..5b448a74d0f7 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -47,18 +47,29 @@ config KPROBES If in doubt, say "N". config JUMP_LABEL - bool "Optimize trace point call sites" + bool "Optimize very unlikely/likely branches" depends on HAVE_ARCH_JUMP_LABEL help + This option enables a transparent branch optimization that + makes certain almost-always-true or almost-always-false branch + conditions even cheaper to execute within the kernel. + + Certain performance-sensitive kernel code, such as trace points, + scheduler functionality, networking code and KVM have such + branches and include support for this optimization technique. + If it is detected that the compiler has support for "asm goto", - the kernel will compile trace point locations with just a - nop instruction. When trace points are enabled, the nop will - be converted to a jump to the trace function. This technique - lowers overhead and stress on the branch prediction of the - processor. - - On i386, options added to the compiler flags may increase - the size of the kernel slightly. + the kernel will compile such branches with just a nop + instruction. When the condition flag is toggled to true, the + nop will be converted to a jump instruction to execute the + conditional block of instructions. + + This technique lowers overhead and stress on the branch prediction + of the processor and generally makes the kernel faster. The update + of the condition is slower, but those are always very rare. + + ( On 32-bit x86, the necessary options added to the compiler + flags may increase the size of the kernel slightly. ) config OPTPROBES def_bool y -- cgit v1.2.3-58-ga151 From 48b25c43e6eebb6c0edf72935e8720385beca76b Mon Sep 17 00:00:00 2001 From: Chris Metcalf Date: Thu, 15 Mar 2012 13:13:38 -0400 Subject: [PATCH v3] ipc: provide generic compat versions of IPC syscalls When using the "compat" APIs, architectures will generally want to be able to make direct syscalls to msgsnd(), shmctl(), etc., and in the kernel we would want them to be handled directly by compat_sys_xxx() functions, as is true for other compat syscalls. However, for historical reasons, several of the existing compat IPC syscalls do not do this. semctl() expects a pointer to the fourth argument, instead of the fourth argument itself. msgsnd(), msgrcv() and shmat() expect arguments in different order. This change adds an ARCH_WANT_OLD_COMPAT_IPC config option that can be set to preserve this behavior for ports that use it (x86, sparc, powerpc, s390, and mips). No actual semantics are changed for those architectures, and there is only a minimal amount of code refactoring in ipc/compat.c. Newer architectures like tile (and perhaps future architectures such as arm64 and unicore64) should not select this option, and thus can avoid having any IPC-specific code at all in their architecture-specific compat layer. In the same vein, if this option is not selected, IPC_64 mode is assumed, since that's what the headers expect. The workaround code in "tile" for msgsnd() and msgrcv() is removed with this change; it also fixes the bug that shmat() and semctl() were not being properly handled. Reviewed-by: Arnd Bergmann Signed-off-by: Chris Metcalf --- arch/Kconfig | 3 ++ arch/mips/Kconfig | 1 + arch/powerpc/Kconfig | 1 + arch/s390/Kconfig | 1 + arch/sparc/Kconfig | 1 + arch/tile/include/asm/compat.h | 11 ------- arch/tile/kernel/compat.c | 43 -------------------------- arch/x86/Kconfig | 1 + include/linux/compat.h | 12 +++++++- ipc/compat.c | 70 ++++++++++++++++++++++++++++++++++++++---- 10 files changed, 83 insertions(+), 61 deletions(-) (limited to 'arch/Kconfig') diff --git a/arch/Kconfig b/arch/Kconfig index 4f55c736be11..b37f8f3ffa54 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -199,4 +199,7 @@ config HAVE_CMPXCHG_LOCAL config HAVE_CMPXCHG_DOUBLE bool +config ARCH_WANT_OLD_COMPAT_IPC + bool + source "kernel/gcov/Kconfig" diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 5ab6e89603c5..4bbbb40f352a 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -2456,6 +2456,7 @@ config MIPS32_COMPAT config COMPAT bool depends on MIPS32_COMPAT + select ARCH_WANT_OLD_COMPAT_IPC default y config SYSVIPC_COMPAT diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 1919634a9b32..48ab0bb38924 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -152,6 +152,7 @@ config COMPAT bool default y if PPC64 select COMPAT_BINFMT_ELF + select ARCH_WANT_OLD_COMPAT_IPC config SYSVIPC_COMPAT bool diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 6d99a5fcc090..0ff53e350092 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -218,6 +218,7 @@ config COMPAT prompt "Kernel support for 31 bit emulation" depends on 64BIT select COMPAT_BINFMT_ELF + select ARCH_WANT_OLD_COMPAT_IPC help Select this option if you want to enable your system kernel to handle system-calls from ELF binaries for 31 bit ESA. This option diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index ca5580e4d813..64e1a8e7cab3 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -576,6 +576,7 @@ config COMPAT depends on SPARC64 default y select COMPAT_BINFMT_ELF + select ARCH_WANT_OLD_COMPAT_IPC config SYSVIPC_COMPAT bool diff --git a/arch/tile/include/asm/compat.h b/arch/tile/include/asm/compat.h index bf95f55b82b0..4b4b28969a65 100644 --- a/arch/tile/include/asm/compat.h +++ b/arch/tile/include/asm/compat.h @@ -242,17 +242,6 @@ long compat_sys_fallocate(int fd, int mode, long compat_sys_sched_rr_get_interval(compat_pid_t pid, struct compat_timespec __user *interval); -/* Versions of compat functions that differ from generic Linux. */ -struct compat_msgbuf; -long tile_compat_sys_msgsnd(int msqid, - struct compat_msgbuf __user *msgp, - size_t msgsz, int msgflg); -long tile_compat_sys_msgrcv(int msqid, - struct compat_msgbuf __user *msgp, - size_t msgsz, long msgtyp, int msgflg); -long tile_compat_sys_ptrace(compat_long_t request, compat_long_t pid, - compat_long_t addr, compat_long_t data); - /* Tilera Linux syscalls that don't have "compat" versions. */ #define compat_sys_flush_cache sys_flush_cache diff --git a/arch/tile/kernel/compat.c b/arch/tile/kernel/compat.c index bf5e9d70266c..d67459b9ac2a 100644 --- a/arch/tile/kernel/compat.c +++ b/arch/tile/kernel/compat.c @@ -16,7 +16,6 @@ #define __SYSCALL_COMPAT #include -#include #include #include #include @@ -95,52 +94,10 @@ long compat_sys_sched_rr_get_interval(compat_pid_t pid, return ret; } -/* - * The usual compat_sys_msgsnd() and _msgrcv() seem to be assuming - * some different calling convention than our normal 32-bit tile code. - */ - -/* Already defined in ipc/compat.c, but we need it here. */ -struct compat_msgbuf { - compat_long_t mtype; - char mtext[1]; -}; - -long tile_compat_sys_msgsnd(int msqid, - struct compat_msgbuf __user *msgp, - size_t msgsz, int msgflg) -{ - compat_long_t mtype; - - if (get_user(mtype, &msgp->mtype)) - return -EFAULT; - return do_msgsnd(msqid, mtype, msgp->mtext, msgsz, msgflg); -} - -long tile_compat_sys_msgrcv(int msqid, - struct compat_msgbuf __user *msgp, - size_t msgsz, long msgtyp, int msgflg) -{ - long err, mtype; - - err = do_msgrcv(msqid, &mtype, msgp->mtext, msgsz, msgtyp, msgflg); - if (err < 0) - goto out; - - if (put_user(mtype, &msgp->mtype)) - err = -EFAULT; - out: - return err; -} - /* Provide the compat syscall number to call mapping. */ #undef __SYSCALL #define __SYSCALL(nr, call) [nr] = (call), -/* The generic versions of these don't work for Tile. */ -#define compat_sys_msgrcv tile_compat_sys_msgrcv -#define compat_sys_msgsnd tile_compat_sys_msgsnd - /* See comments in sys.c */ #define compat_sys_fadvise64_64 sys32_fadvise64_64 #define compat_sys_readahead sys32_readahead diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5bed94e189fa..cde163dc6058 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2178,6 +2178,7 @@ config IA32_AOUT config COMPAT def_bool y depends on IA32_EMULATION + select ARCH_WANT_OLD_COMPAT_IPC config COMPAT_FOR_U64_ALIGNMENT def_bool COMPAT diff --git a/include/linux/compat.h b/include/linux/compat.h index 7e05fcee75a1..35c2dbf2448a 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -224,6 +224,7 @@ struct compat_sysinfo; struct compat_sysctl_args; struct compat_kexec_segment; struct compat_mq_attr; +struct compat_msgbuf; extern void compat_exit_robust_list(struct task_struct *curr); @@ -234,13 +235,22 @@ asmlinkage long compat_sys_get_robust_list(int pid, compat_uptr_t __user *head_ptr, compat_size_t __user *len_ptr); +#ifdef CONFIG_ARCH_WANT_OLD_COMPAT_IPC long compat_sys_semctl(int first, int second, int third, void __user *uptr); long compat_sys_msgsnd(int first, int second, int third, void __user *uptr); long compat_sys_msgrcv(int first, int second, int msgtyp, int third, int version, void __user *uptr); -long compat_sys_msgctl(int first, int second, void __user *uptr); long compat_sys_shmat(int first, int second, compat_uptr_t third, int version, void __user *uptr); +#else +long compat_sys_semctl(int semid, int semnum, int cmd, int arg); +long compat_sys_msgsnd(int msqid, struct compat_msgbuf __user *msgp, + size_t msgsz, int msgflg); +long compat_sys_msgrcv(int msqid, struct compat_msgbuf __user *msgp, + size_t msgsz, long msgtyp, int msgflg); +long compat_sys_shmat(int shmid, compat_uptr_t shmaddr, int shmflg); +#endif +long compat_sys_msgctl(int first, int second, void __user *uptr); long compat_sys_shmctl(int first, int second, void __user *uptr); long compat_sys_semtimedop(int semid, struct sembuf __user *tsems, unsigned nsems, const struct compat_timespec __user *timeout); diff --git a/ipc/compat.c b/ipc/compat.c index 845a28738d3a..a6df704f521e 100644 --- a/ipc/compat.c +++ b/ipc/compat.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -117,6 +118,7 @@ extern int sem_ctls[]; static inline int compat_ipc_parse_version(int *cmd) { +#ifdef CONFIG_ARCH_WANT_OLD_COMPAT_IPC int version = *cmd & IPC_64; /* this is tricky: architectures that have support for the old @@ -128,6 +130,10 @@ static inline int compat_ipc_parse_version(int *cmd) *cmd &= ~IPC_64; #endif return version; +#else + /* With the asm-generic APIs, we always use the 64-bit versions. */ + return IPC_64; +#endif } static inline int __get_compat_ipc64_perm(struct ipc64_perm *p64, @@ -232,10 +238,9 @@ static inline int put_compat_semid_ds(struct semid64_ds *s, return err; } -long compat_sys_semctl(int first, int second, int third, void __user *uptr) +static long do_compat_semctl(int first, int second, int third, u32 pad) { union semun fourth; - u32 pad; int err, err2; struct semid64_ds s64; struct semid64_ds __user *up64; @@ -243,10 +248,6 @@ long compat_sys_semctl(int first, int second, int third, void __user *uptr) memset(&s64, 0, sizeof(s64)); - if (!uptr) - return -EINVAL; - if (get_user(pad, (u32 __user *) uptr)) - return -EFAULT; if ((third & (~IPC_64)) == SETVAL) fourth.val = (int) pad; else @@ -305,6 +306,18 @@ long compat_sys_semctl(int first, int second, int third, void __user *uptr) return err; } +#ifdef CONFIG_ARCH_WANT_OLD_COMPAT_IPC +long compat_sys_semctl(int first, int second, int third, void __user *uptr) +{ + u32 pad; + + if (!uptr) + return -EINVAL; + if (get_user(pad, (u32 __user *) uptr)) + return -EFAULT; + return do_compat_semctl(first, second, third, pad); +} + long compat_sys_msgsnd(int first, int second, int third, void __user *uptr) { struct compat_msgbuf __user *up = uptr; @@ -353,6 +366,37 @@ long compat_sys_msgrcv(int first, int second, int msgtyp, int third, out: return err; } +#else +long compat_sys_semctl(int semid, int semnum, int cmd, int arg) +{ + return do_compat_semctl(semid, semnum, cmd, arg); +} + +long compat_sys_msgsnd(int msqid, struct compat_msgbuf __user *msgp, + size_t msgsz, int msgflg) +{ + compat_long_t mtype; + + if (get_user(mtype, &msgp->mtype)) + return -EFAULT; + return do_msgsnd(msqid, mtype, msgp->mtext, msgsz, msgflg); +} + +long compat_sys_msgrcv(int msqid, struct compat_msgbuf __user *msgp, + size_t msgsz, long msgtyp, int msgflg) +{ + long err, mtype; + + err = do_msgrcv(msqid, &mtype, msgp->mtext, msgsz, msgtyp, msgflg); + if (err < 0) + goto out; + + if (put_user(mtype, &msgp->mtype)) + err = -EFAULT; + out: + return err; +} +#endif static inline int get_compat_msqid64(struct msqid64_ds *m64, struct compat_msqid64_ds __user *up64) @@ -470,6 +514,7 @@ long compat_sys_msgctl(int first, int second, void __user *uptr) return err; } +#ifdef CONFIG_ARCH_WANT_OLD_COMPAT_IPC long compat_sys_shmat(int first, int second, compat_uptr_t third, int version, void __user *uptr) { @@ -485,6 +530,19 @@ long compat_sys_shmat(int first, int second, compat_uptr_t third, int version, uaddr = compat_ptr(third); return put_user(raddr, uaddr); } +#else +long compat_sys_shmat(int shmid, compat_uptr_t shmaddr, int shmflg) +{ + unsigned long ret; + long err; + + err = do_shmat(shmid, compat_ptr(shmaddr), shmflg, &ret); + if (err) + return err; + force_successful_syscall_return(); + return (long)ret; +} +#endif static inline int get_compat_shmid64_ds(struct shmid64_ds *s64, struct compat_shmid64_ds __user *up64) -- cgit v1.2.3-58-ga151 From d314d74c695f967e10598467a326f41c78ed1e20 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 23 Mar 2012 15:01:51 -0700 Subject: nmi watchdog: do not use cpp symbol in Kconfig ARCH_HAS_NMI_WATCHDOG is a macro defined by arch, but config HARDLOCKUP_DETECTOR depends on it. This is wrong, ARCH_HAS_NMI_WATCHDOG has to be a Kconfig config, and arch's need it should select it explicitly. Signed-off-by: WANG Cong Acked-by: Don Zickus Acked-by: Mike Frysinger Cc: David Howells Cc: David Miller Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/Kconfig | 3 +++ arch/blackfin/Kconfig | 1 + arch/blackfin/include/asm/irq.h | 4 ---- arch/mn10300/Kconfig | 1 + arch/mn10300/include/asm/reset-regs.h | 4 ---- arch/sparc/Kconfig | 1 + arch/sparc/include/asm/irq_64.h | 1 - include/linux/nmi.h | 2 +- lib/Kconfig.debug | 2 +- 9 files changed, 8 insertions(+), 11 deletions(-) (limited to 'arch/Kconfig') diff --git a/arch/Kconfig b/arch/Kconfig index 5b448a74d0f7..a6f14f622d13 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -120,6 +120,9 @@ config HAVE_KRETPROBES config HAVE_OPTPROBES bool + +config HAVE_NMI_WATCHDOG + bool # # An arch should select this if it provides all these things: # diff --git a/arch/blackfin/Kconfig b/arch/blackfin/Kconfig index abe5a9e85148..c1269a1085e1 100644 --- a/arch/blackfin/Kconfig +++ b/arch/blackfin/Kconfig @@ -36,6 +36,7 @@ config BLACKFIN select GENERIC_ATOMIC64 select GENERIC_IRQ_PROBE select IRQ_PER_CPU if SMP + select HAVE_NMI_WATCHDOG if NMI_WATCHDOG config GENERIC_CSUM def_bool y diff --git a/arch/blackfin/include/asm/irq.h b/arch/blackfin/include/asm/irq.h index 12f4060a31b0..89de539ed010 100644 --- a/arch/blackfin/include/asm/irq.h +++ b/arch/blackfin/include/asm/irq.h @@ -38,8 +38,4 @@ #include -#ifdef CONFIG_NMI_WATCHDOG -# define ARCH_HAS_NMI_WATCHDOG -#endif - #endif /* _BFIN_IRQ_H_ */ diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig index 8f1c40d5817e..3aa3de017159 100644 --- a/arch/mn10300/Kconfig +++ b/arch/mn10300/Kconfig @@ -5,6 +5,7 @@ config MN10300 select GENERIC_IRQ_SHOW select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_KGDB + select HAVE_NMI_WATCHDOG if MN10300_WD_TIMER config AM33_2 def_bool n diff --git a/arch/mn10300/include/asm/reset-regs.h b/arch/mn10300/include/asm/reset-regs.h index 10c7502a113f..8ca2a42d365b 100644 --- a/arch/mn10300/include/asm/reset-regs.h +++ b/arch/mn10300/include/asm/reset-regs.h @@ -17,10 +17,6 @@ #ifdef __KERNEL__ -#ifdef CONFIG_MN10300_WD_TIMER -#define ARCH_HAS_NMI_WATCHDOG /* See include/linux/nmi.h */ -#endif - /* * watchdog timer registers */ diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index ca5580e4d813..1666de84d477 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -29,6 +29,7 @@ config SPARC select GENERIC_IRQ_SHOW select USE_GENERIC_SMP_HELPERS if SMP select GENERIC_PCI_IOMAP + select HAVE_NMI_WATCHDOG if SPARC64 config SPARC32 def_bool !64BIT diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index 16dcae6d56e7..abf6afe82ca8 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -95,7 +95,6 @@ void arch_trigger_all_cpu_backtrace(void); extern void *hardirq_stack[NR_CPUS]; extern void *softirq_stack[NR_CPUS]; #define __ARCH_HAS_DO_SOFTIRQ -#define ARCH_HAS_NMI_WATCHDOG #define NO_IRQ 0xffffffff diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 2d304efc89df..db50840e6355 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -14,7 +14,7 @@ * may be used to reset the timeout - for code which intentionally * disables interrupts for a long time. This call is stateless. */ -#if defined(ARCH_HAS_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR) +#if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR) #include extern void touch_nmi_watchdog(void); #else diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 05037dc9bde7..391003f7ab46 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -184,7 +184,7 @@ config LOCKUP_DETECTOR config HARDLOCKUP_DETECTOR def_bool LOCKUP_DETECTOR && PERF_EVENTS && HAVE_PERF_EVENTS_NMI && \ - !ARCH_HAS_NMI_WATCHDOG + !HAVE_NMI_WATCHDOG config BOOTPARAM_HARDLOCKUP_PANIC bool "Panic (Reboot) On Hard Lockups" -- cgit v1.2.3-58-ga151