From 114276ac0a3beb9c391a410349bd770653e185ce Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:32:13 +0300 Subject: mm, sched: Drop voluntary schedule from might_fault() might_fault() is called from functions like copy_to_user() which most callers expect to be very fast, like a couple of instructions. So functions like memcpy_toiovec() call them many times in a loop. But might_fault() calls might_sleep() and with CONFIG_PREEMPT_VOLUNTARY this results in a function call. Let's not do this - just call __might_sleep() that produces a diagnostic for sleep within atomic, but drop might_preempt(). Here's a test sending traffic between the VM and the host, host is built with CONFIG_PREEMPT_VOLUNTARY: before: incoming: 7122.77 Mb/s outgoing: 8480.37 Mb/s after: incoming: 8619.24 Mb/s outgoing: 9455.42 Mb/s As a side effect, this fixes an issue pointed out by Ingo: might_fault might schedule differently depending on PROVE_LOCKING. Now there's no preemption point in both cases, so it's consistent. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-10-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar --- mm/memory.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mm/memory.c') diff --git a/mm/memory.c b/mm/memory.c index 6dc1882fbd72..c1f190f51f6f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4222,7 +4222,8 @@ void might_fault(void) if (segment_eq(get_fs(), KERNEL_DS)) return; - might_sleep(); + __might_sleep(__FILE__, __LINE__, 0); + /* * it would be nicer only to annotate paths which are not under * pagefault_disable, however that requires a larger audit and -- cgit v1.2.3-58-ga151 From 662bbcb2747c2422cf98d3d97619509379eee466 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:32:23 +0300 Subject: mm, sched: Allow uaccess in atomic with pagefault_disable() This changes might_fault() so that it does not trigger a false positive diagnostic for e.g. the following sequence: spin_lock_irqsave() pagefault_disable() copy_to_user() pagefault_enable() spin_unlock_irqrestore() In particular vhost wants to do this, to call socket ops from under a lock. There are 3 cases to consider: - CONFIG_PROVE_LOCKING - might_fault is non-inline so it's easy to move the in_atomic test to fix up the false positive warning. - CONFIG_DEBUG_ATOMIC_SLEEP - might_fault is currently inline, but we are calling a non-inline __might_sleep anyway, so let's use the non-line version of might_fault that does the right thing. - !CONFIG_DEBUG_ATOMIC_SLEEP && !CONFIG_PROVE_LOCKING __might_sleep is a nop so might_fault is a nop. Make this explicit. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-11-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar --- include/linux/kernel.h | 7 ++----- mm/memory.c | 11 +++++++---- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'mm/memory.c') diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 24719eaa1209..4c7e2e508766 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -193,13 +193,10 @@ extern int _cond_resched(void); (__x < 0) ? -__x : __x; \ }) -#ifdef CONFIG_PROVE_LOCKING +#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) void might_fault(void); #else -static inline void might_fault(void) -{ - __might_sleep(__FILE__, __LINE__, 0); -} +static inline void might_fault(void) { } #endif extern struct atomic_notifier_head panic_notifier_list; diff --git a/mm/memory.c b/mm/memory.c index c1f190f51f6f..d7d54a114773 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4210,7 +4210,7 @@ void print_vma_addr(char *prefix, unsigned long ip) up_read(&mm->mmap_sem); } -#ifdef CONFIG_PROVE_LOCKING +#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) void might_fault(void) { /* @@ -4222,14 +4222,17 @@ void might_fault(void) if (segment_eq(get_fs(), KERNEL_DS)) return; - __might_sleep(__FILE__, __LINE__, 0); - /* * it would be nicer only to annotate paths which are not under * pagefault_disable, however that requires a larger audit and * providing helpers like get_user_atomic. */ - if (!in_atomic() && current->mm) + if (in_atomic()) + return; + + __might_sleep(__FILE__, __LINE__, 0); + + if (current->mm) might_lock_read(¤t->mm->mmap_sem); } EXPORT_SYMBOL(might_fault); -- cgit v1.2.3-58-ga151