From 24ce66c04a06a678f156cf575128246f3d214b4a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 7 Sep 2020 07:58:18 +0200 Subject: uaccess: provide a generic TASK_SIZE_MAX definition Define TASK_SIZE_MAX as TASK_SIZE if not otherwise defined. Signed-off-by: Christoph Hellwig Signed-off-by: Palmer Dabbelt --- include/linux/uaccess.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 70073c802b48..d0e43761c708 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -31,6 +31,10 @@ typedef struct { /* empty dummy */ } mm_segment_t; +#ifndef TASK_SIZE_MAX +#define TASK_SIZE_MAX TASK_SIZE +#endif + #define uaccess_kernel() (false) #define user_addr_max() (TASK_SIZE_MAX) -- cgit v1.2.3-58-ga151 From 931de11f5a374748f1d620ee17810dedf13c9f69 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 7 Sep 2020 07:58:19 +0200 Subject: asm-generic: improve the nommu {get,put}_user handling Instead of reusing raw_{copy,to}_from_user implement separate handlers using {get,put}_unaligned. This ensures unaligned access is handled correctly, and avoid the need for the small constant size optimization in raw_{copy,to}_from_user. Signed-off-by: Christoph Hellwig Acked-by: Arnd Bergmann Signed-off-by: Palmer Dabbelt --- include/asm-generic/uaccess.h | 91 ++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index ba68ee4dabfa..6de5f524e9e6 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -10,28 +10,60 @@ #include #ifdef CONFIG_UACCESS_MEMCPY -static inline __must_check unsigned long -raw_copy_from_user(void *to, const void __user * from, unsigned long n) +#include + +static inline int __get_user_fn(size_t size, const void __user *from, void *to) { - if (__builtin_constant_p(n)) { - switch(n) { - case 1: - *(u8 *)to = *(u8 __force *)from; - return 0; - case 2: - *(u16 *)to = *(u16 __force *)from; - return 0; - case 4: - *(u32 *)to = *(u32 __force *)from; - return 0; -#ifdef CONFIG_64BIT - case 8: - *(u64 *)to = *(u64 __force *)from; - return 0; -#endif - } + BUILD_BUG_ON(!__builtin_constant_p(size)); + + switch (size) { + case 1: + *(u8 *)to = get_unaligned((u8 __force *)from); + return 0; + case 2: + *(u16 *)to = get_unaligned((u16 __force *)from); + return 0; + case 4: + *(u32 *)to = get_unaligned((u32 __force *)from); + return 0; + case 8: + *(u64 *)to = get_unaligned((u64 __force *)from); + return 0; + default: + BUILD_BUG(); + return 0; + } + +} +#define __get_user_fn(sz, u, k) __get_user_fn(sz, u, k) + +static inline int __put_user_fn(size_t size, void __user *to, void *from) +{ + BUILD_BUG_ON(!__builtin_constant_p(size)); + + switch (size) { + case 1: + put_unaligned(*(u8 *)from, (u8 __force *)to); + return 0; + case 2: + put_unaligned(*(u16 *)from, (u16 __force *)to); + return 0; + case 4: + put_unaligned(*(u32 *)from, (u32 __force *)to); + return 0; + case 8: + put_unaligned(*(u64 *)from, (u64 __force *)to); + return 0; + default: + BUILD_BUG(); + return 0; } +} +#define __put_user_fn(sz, u, k) __put_user_fn(sz, u, k) +static inline __must_check unsigned long +raw_copy_from_user(void *to, const void __user * from, unsigned long n) +{ memcpy(to, (const void __force *)from, n); return 0; } @@ -39,27 +71,6 @@ raw_copy_from_user(void *to, const void __user * from, unsigned long n) static inline __must_check unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - if (__builtin_constant_p(n)) { - switch(n) { - case 1: - *(u8 __force *)to = *(u8 *)from; - return 0; - case 2: - *(u16 __force *)to = *(u16 *)from; - return 0; - case 4: - *(u32 __force *)to = *(u32 *)from; - return 0; -#ifdef CONFIG_64BIT - case 8: - *(u64 __force *)to = *(u64 *)from; - return 0; -#endif - default: - break; - } - } - memcpy((void __force *)to, from, n); return 0; } -- cgit v1.2.3-58-ga151 From 2d2d255467d4ea927fbf606c8b584512edee0f7b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 7 Sep 2020 07:58:20 +0200 Subject: asm-generic: add nommu implementations of __{get,put}_kernel_nofault Add native implementations of __{get,put}_kernel_nofault using {get,put}_unaligned, just like the {get,put}_user implementations. Signed-off-by: Christoph Hellwig Acked-by: Arnd Bergmann Signed-off-by: Palmer Dabbelt --- include/asm-generic/uaccess.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index 6de5f524e9e6..b367f339be1a 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -61,6 +61,22 @@ static inline int __put_user_fn(size_t size, void __user *to, void *from) } #define __put_user_fn(sz, u, k) __put_user_fn(sz, u, k) +#define __get_kernel_nofault(dst, src, type, err_label) \ +do { \ + *((type *)dst) = get_unaligned((type *)(src)); \ + if (0) /* make sure the label looks used to the compiler */ \ + goto err_label; \ +} while (0) + +#define __put_kernel_nofault(dst, src, type, err_label) \ +do { \ + put_unaligned(*((type *)src), (type *)(dst)); \ + if (0) /* make sure the label looks used to the compiler */ \ + goto err_label; \ +} while (0) + +#define HAVE_GET_KERNEL_NOFAULT 1 + static inline __must_check unsigned long raw_copy_from_user(void *to, const void __user * from, unsigned long n) { -- cgit v1.2.3-58-ga151 From 3c57fa13f6bf3906643034c57736c778ce63fa55 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 7 Sep 2020 07:58:21 +0200 Subject: asm-generic: make the set_fs implementation optional Put all the set_fs related code under CONFIG_SET_FS so that asm-generic/uaccess.h can be used for set_fs-less builds. Signed-off-by: Christoph Hellwig Acked-by: Arnd Bergmann Signed-off-by: Palmer Dabbelt --- include/asm-generic/uaccess.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index b367f339be1a..45f9872fd747 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -94,6 +94,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) #define INLINE_COPY_TO_USER #endif /* CONFIG_UACCESS_MEMCPY */ +#ifdef CONFIG_SET_FS #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) #ifndef KERNEL_DS @@ -116,6 +117,7 @@ static inline void set_fs(mm_segment_t fs) #ifndef uaccess_kernel #define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) #endif +#endif /* CONFIG_SET_FS */ #define access_ok(addr, size) __access_ok((unsigned long)(addr),(size)) -- cgit v1.2.3-58-ga151 From 11129e8ed4d91a062c5062d80f476adc7fbedbac Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 7 Sep 2020 07:58:22 +0200 Subject: riscv: use memcpy based uaccess for nommu again This reverts commit adccfb1a805ea84d2db38eb53032533279bdaa97. Now that the generic uaccess by mempcy code handles unaligned addresses the generic code can be used for all RISC-V CPUs. Signed-off-by: Christoph Hellwig Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/uaccess.h | 36 ++++++++++++++++++------------------ arch/riscv/lib/Makefile | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 08c48bb6f6e2..1046ee7c0ccf 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -88,6 +88,7 @@ config RISCV select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK select SET_FS + select UACCESS_MEMCPY if !MMU config ARCH_MMAP_RND_BITS_MIN default 18 if 64BIT diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index f56c66b3f5fe..e8eedf22e907 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -13,24 +13,6 @@ /* * User space memory access functions */ - -extern unsigned long __must_check __asm_copy_to_user(void __user *to, - const void *from, unsigned long n); -extern unsigned long __must_check __asm_copy_from_user(void *to, - const void __user *from, unsigned long n); - -static inline unsigned long -raw_copy_from_user(void *to, const void __user *from, unsigned long n) -{ - return __asm_copy_from_user(to, from, n); -} - -static inline unsigned long -raw_copy_to_user(void __user *to, const void *from, unsigned long n) -{ - return __asm_copy_to_user(to, from, n); -} - #ifdef CONFIG_MMU #include #include @@ -385,6 +367,24 @@ do { \ -EFAULT; \ }) + +unsigned long __must_check __asm_copy_to_user(void __user *to, + const void *from, unsigned long n); +unsigned long __must_check __asm_copy_from_user(void *to, + const void __user *from, unsigned long n); + +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return __asm_copy_from_user(to, from, n); +} + +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return __asm_copy_to_user(to, from, n); +} + extern long strncpy_from_user(char *dest, const char __user *src, long count); extern long __must_check strlen_user(const char __user *str); diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 0d0db80800c4..47e7a8204460 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -2,5 +2,5 @@ lib-y += delay.o lib-y += memcpy.o lib-y += memset.o -lib-y += uaccess.o +lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o -- cgit v1.2.3-58-ga151 From f289a34811d8fff29f304e14f7edd4576ed9b598 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 7 Sep 2020 07:58:23 +0200 Subject: riscv: refactor __get_user and __put_user Add new __get_user_nocheck and __put_user_nocheck that switch on the size and call the actual inline assembly helpers, and move the uaccess enable / disable into the actual __get_user and __put_user. This prepares for natively implementing __get_kernel_nofault and __put_kernel_nofault. Also don't bother with the deprecated register keyword for the error return. Signed-off-by: Christoph Hellwig Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/uaccess.h | 94 ++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 42 deletions(-) diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index e8eedf22e907..b67d1c616ec3 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -107,7 +107,6 @@ static inline int __access_ok(unsigned long addr, unsigned long size) do { \ uintptr_t __tmp; \ __typeof__(x) __x; \ - __enable_user_access(); \ __asm__ __volatile__ ( \ "1:\n" \ " " insn " %1, %3\n" \ @@ -125,7 +124,6 @@ do { \ " .previous" \ : "+r" (err), "=&r" (__x), "=r" (__tmp) \ : "m" (*(ptr)), "i" (-EFAULT)); \ - __disable_user_access(); \ (x) = __x; \ } while (0) @@ -138,7 +136,6 @@ do { \ u32 __user *__ptr = (u32 __user *)(ptr); \ u32 __lo, __hi; \ uintptr_t __tmp; \ - __enable_user_access(); \ __asm__ __volatile__ ( \ "1:\n" \ " lw %1, %4\n" \ @@ -162,12 +159,30 @@ do { \ "=r" (__tmp) \ : "m" (__ptr[__LSW]), "m" (__ptr[__MSW]), \ "i" (-EFAULT)); \ - __disable_user_access(); \ (x) = (__typeof__(x))((__typeof__((x)-(x)))( \ (((u64)__hi << 32) | __lo))); \ } while (0) #endif /* CONFIG_64BIT */ +#define __get_user_nocheck(x, __gu_ptr, __gu_err) \ +do { \ + switch (sizeof(*__gu_ptr)) { \ + case 1: \ + __get_user_asm("lb", (x), __gu_ptr, __gu_err); \ + break; \ + case 2: \ + __get_user_asm("lh", (x), __gu_ptr, __gu_err); \ + break; \ + case 4: \ + __get_user_asm("lw", (x), __gu_ptr, __gu_err); \ + break; \ + case 8: \ + __get_user_8((x), __gu_ptr, __gu_err); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ +} while (0) /** * __get_user: - Get a simple variable from user space, with less checking. @@ -191,25 +206,15 @@ do { \ */ #define __get_user(x, ptr) \ ({ \ - register long __gu_err = 0; \ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ + long __gu_err = 0; \ + \ __chk_user_ptr(__gu_ptr); \ - switch (sizeof(*__gu_ptr)) { \ - case 1: \ - __get_user_asm("lb", (x), __gu_ptr, __gu_err); \ - break; \ - case 2: \ - __get_user_asm("lh", (x), __gu_ptr, __gu_err); \ - break; \ - case 4: \ - __get_user_asm("lw", (x), __gu_ptr, __gu_err); \ - break; \ - case 8: \ - __get_user_8((x), __gu_ptr, __gu_err); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ + \ + __enable_user_access(); \ + __get_user_nocheck(x, __gu_ptr, __gu_err); \ + __disable_user_access(); \ + \ __gu_err; \ }) @@ -243,7 +248,6 @@ do { \ do { \ uintptr_t __tmp; \ __typeof__(*(ptr)) __x = x; \ - __enable_user_access(); \ __asm__ __volatile__ ( \ "1:\n" \ " " insn " %z3, %2\n" \ @@ -260,7 +264,6 @@ do { \ " .previous" \ : "+r" (err), "=r" (__tmp), "=m" (*(ptr)) \ : "rJ" (__x), "i" (-EFAULT)); \ - __disable_user_access(); \ } while (0) #ifdef CONFIG_64BIT @@ -272,7 +275,6 @@ do { \ u32 __user *__ptr = (u32 __user *)(ptr); \ u64 __x = (__typeof__((x)-(x)))(x); \ uintptr_t __tmp; \ - __enable_user_access(); \ __asm__ __volatile__ ( \ "1:\n" \ " sw %z4, %2\n" \ @@ -294,10 +296,28 @@ do { \ "=m" (__ptr[__LSW]), \ "=m" (__ptr[__MSW]) \ : "rJ" (__x), "rJ" (__x >> 32), "i" (-EFAULT)); \ - __disable_user_access(); \ } while (0) #endif /* CONFIG_64BIT */ +#define __put_user_nocheck(x, __gu_ptr, __pu_err) \ +do { \ + switch (sizeof(*__gu_ptr)) { \ + case 1: \ + __put_user_asm("sb", (x), __gu_ptr, __pu_err); \ + break; \ + case 2: \ + __put_user_asm("sh", (x), __gu_ptr, __pu_err); \ + break; \ + case 4: \ + __put_user_asm("sw", (x), __gu_ptr, __pu_err); \ + break; \ + case 8: \ + __put_user_8((x), __gu_ptr, __pu_err); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ +} while (0) /** * __put_user: - Write a simple value into user space, with less checking. @@ -320,25 +340,15 @@ do { \ */ #define __put_user(x, ptr) \ ({ \ - register long __pu_err = 0; \ __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ + long __pu_err = 0; \ + \ __chk_user_ptr(__gu_ptr); \ - switch (sizeof(*__gu_ptr)) { \ - case 1: \ - __put_user_asm("sb", (x), __gu_ptr, __pu_err); \ - break; \ - case 2: \ - __put_user_asm("sh", (x), __gu_ptr, __pu_err); \ - break; \ - case 4: \ - __put_user_asm("sw", (x), __gu_ptr, __pu_err); \ - break; \ - case 8: \ - __put_user_8((x), __gu_ptr, __pu_err); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ + \ + __enable_user_access(); \ + __put_user_nocheck(x, __gu_ptr, __pu_err); \ + __disable_user_access(); \ + \ __pu_err; \ }) -- cgit v1.2.3-58-ga151 From d464118cdc4156c3f356507ea0f5ea0dc5be6c3d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 7 Sep 2020 07:58:24 +0200 Subject: riscv: implement __get_kernel_nofault and __put_user_nofault Implement the non-faulting kernel access helpers directly instead of abusing the uaccess routines under set_fs(KERNEL_DS). Signed-off-by: Christoph Hellwig Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/uaccess.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index b67d1c616ec3..264e52fb62b1 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -486,6 +486,26 @@ unsigned long __must_check clear_user(void __user *to, unsigned long n) __ret; \ }) +#define HAVE_GET_KERNEL_NOFAULT + +#define __get_kernel_nofault(dst, src, type, err_label) \ +do { \ + long __kr_err; \ + \ + __get_user_nocheck(*((type *)(dst)), (type *)(src), __kr_err); \ + if (unlikely(__kr_err)) \ + goto err_label; \ +} while (0) + +#define __put_kernel_nofault(dst, src, type, err_label) \ +do { \ + long __kr_err; \ + \ + __put_user_nocheck(*((type *)(dst)), (type *)(src), __kr_err); \ + if (unlikely(__kr_err)) \ + goto err_label; \ +} while (0) + #else /* CONFIG_MMU */ #include #endif /* CONFIG_MMU */ -- cgit v1.2.3-58-ga151 From e8d444d3e98c255f91d228984abc46cfdfaf48b4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 7 Sep 2020 07:58:25 +0200 Subject: riscv: remove address space overrides using set_fs() Stop providing the possibility to override the address space using set_fs() now that there is no need for that any more. Signed-off-by: Christoph Hellwig Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 - arch/riscv/include/asm/thread_info.h | 6 ------ arch/riscv/include/asm/uaccess.h | 27 +-------------------------- arch/riscv/kernel/process.c | 1 - 4 files changed, 1 insertion(+), 34 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 1046ee7c0ccf..12642bab4700 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -87,7 +87,6 @@ config RISCV select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK - select SET_FS select UACCESS_MEMCPY if !MMU config ARCH_MMAP_RND_BITS_MIN diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 464a2bbc97ea..a390711129de 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -24,10 +24,6 @@ #include #include -typedef struct { - unsigned long seg; -} mm_segment_t; - /* * low level task data that entry.S needs immediate access to * - this struct should fit entirely inside of one cache line @@ -39,7 +35,6 @@ typedef struct { struct thread_info { unsigned long flags; /* low level flags */ int preempt_count; /* 0=>preemptible, <0=>BUG */ - mm_segment_t addr_limit; /* * These stack pointers are overwritten on every system call or * exception. SP is also saved to the stack it can be recovered when @@ -59,7 +54,6 @@ struct thread_info { { \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .addr_limit = KERNEL_DS, \ } #endif /* !__ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 264e52fb62b1..c47e6b35c551 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -26,29 +26,6 @@ #define __disable_user_access() \ __asm__ __volatile__ ("csrc sstatus, %0" : : "r" (SR_SUM) : "memory") -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - */ - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - -#define KERNEL_DS MAKE_MM_SEG(~0UL) -#define USER_DS MAKE_MM_SEG(TASK_SIZE) - -#define get_fs() (current_thread_info()->addr_limit) - -static inline void set_fs(mm_segment_t fs) -{ - current_thread_info()->addr_limit = fs; -} - -#define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) -#define user_addr_max() (get_fs().seg) - /** * access_ok: - Checks if a user space pointer is valid * @addr: User space pointer to start of block to check @@ -76,9 +53,7 @@ static inline void set_fs(mm_segment_t fs) */ static inline int __access_ok(unsigned long addr, unsigned long size) { - const mm_segment_t fs = get_fs(); - - return size <= fs.seg && addr <= fs.seg - size; + return size <= TASK_SIZE && addr <= TASK_SIZE - size; } /* diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 2b97c493427c..19225ec65db6 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -84,7 +84,6 @@ void start_thread(struct pt_regs *regs, unsigned long pc, } regs->epc = pc; regs->sp = sp; - set_fs(USER_DS); } void flush_thread(void) -- cgit v1.2.3-58-ga151