From 5e6039d8a307d8411422c154f3d446b44fa32b6d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 18:00:15 -0500 Subject: [PATCH 01/96] uaccess: move VERIFY_{READ,WRITE} definitions to linux/uaccess.h Signed-off-by: Al Viro --- arch/alpha/include/asm/uaccess.h | 3 --- arch/arm/include/asm/uaccess.h | 3 --- arch/arm64/include/asm/uaccess.h | 3 --- arch/avr32/include/asm/uaccess.h | 3 --- arch/blackfin/include/asm/uaccess.h | 3 --- arch/cris/include/asm/uaccess.h | 3 --- arch/frv/include/asm/uaccess.h | 3 --- arch/hexagon/include/asm/uaccess.h | 2 -- arch/ia64/include/asm/uaccess.h | 3 --- arch/m32r/include/asm/uaccess.h | 3 --- arch/m68k/include/asm/uaccess_mm.h | 3 --- arch/m68k/include/asm/uaccess_no.h | 3 --- arch/metag/include/asm/uaccess.h | 3 --- arch/microblaze/include/asm/uaccess.h | 3 --- arch/mips/include/asm/uaccess.h | 3 --- arch/mn10300/include/asm/uaccess.h | 3 --- arch/nios2/include/asm/uaccess.h | 3 --- arch/openrisc/include/asm/uaccess.h | 3 --- arch/parisc/include/asm/uaccess.h | 3 --- arch/powerpc/include/asm/uaccess.h | 3 --- arch/s390/include/asm/uaccess.h | 3 --- arch/score/include/asm/uaccess.h | 3 --- arch/sh/include/asm/uaccess.h | 3 --- arch/sparc/include/asm/uaccess_32.h | 3 --- arch/sparc/include/asm/uaccess_64.h | 3 --- arch/tile/include/asm/uaccess.h | 3 --- arch/x86/include/asm/uaccess.h | 3 --- arch/xtensa/include/asm/asm-uaccess.h | 3 --- arch/xtensa/include/asm/uaccess.h | 3 --- include/asm-generic/uaccess.h | 3 --- include/linux/uaccess.h | 4 ++++ 31 files changed, 4 insertions(+), 89 deletions(-) diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index 94f587535dee..a37267a5d399 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -20,9 +20,6 @@ #define KERNEL_DS ((mm_segment_t) { 0UL }) #define USER_DS ((mm_segment_t) { -0x40000000000UL }) -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define get_fs() (current_thread_info()->addr_limit) #define get_ds() (KERNEL_DS) #define set_fs(x) (current_thread_info()->addr_limit = (x)) diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index b7e0125c0bbf..a13f39b3e9f8 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -26,9 +26,6 @@ #define __put_user_unaligned __put_user #endif -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The exception table consists of pairs of addresses: the first is the * address of an instruction that is allowed to fault, and the second is diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 5308d696311b..f5e1e090b4d2 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -36,9 +36,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The exception table consists of pairs of relative offsets: the first * is the relative offset to an instruction that is allowed to fault, diff --git a/arch/avr32/include/asm/uaccess.h b/arch/avr32/include/asm/uaccess.h index b1ec1fa06463..1c7f234385fc 100644 --- a/arch/avr32/include/asm/uaccess.h +++ b/arch/avr32/include/asm/uaccess.h @@ -11,9 +11,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - typedef struct { unsigned int is_user_space; } mm_segment_t; diff --git a/arch/blackfin/include/asm/uaccess.h b/arch/blackfin/include/asm/uaccess.h index 0eff88aa6d6a..d9a91108964f 100644 --- a/arch/blackfin/include/asm/uaccess.h +++ b/arch/blackfin/include/asm/uaccess.h @@ -29,9 +29,6 @@ static inline void set_fs(mm_segment_t fs) #define segment_eq(a, b) ((a) == (b)) -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define access_ok(type, addr, size) _access_ok((unsigned long)(addr), (size)) /* diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h index 56c7d5750abd..82bfcb5e2c9b 100644 --- a/arch/cris/include/asm/uaccess.h +++ b/arch/cris/include/asm/uaccess.h @@ -21,9 +21,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/frv/include/asm/uaccess.h b/arch/frv/include/asm/uaccess.h index c0f4057eab60..9e01bd798a03 100644 --- a/arch/frv/include/asm/uaccess.h +++ b/arch/frv/include/asm/uaccess.h @@ -22,9 +22,6 @@ #define __ptr(x) ((unsigned long __force *)(x)) -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * check that a range of addresses falls within the current address limit */ diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index f61cfb28e9f2..21f63593e2b6 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -50,8 +50,6 @@ * reasonably simple and not *too* slow. After all, we've got the * MMU for backup. */ -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 #define __access_ok(addr, size) \ ((get_fs().seg == KERNEL_DS.seg) || \ diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index 471044be2a3b..c60ff6cc8dbd 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -48,9 +48,6 @@ #define KERNEL_DS ((mm_segment_t) { ~0UL }) /* cf. access_ok() */ #define USER_DS ((mm_segment_t) { TASK_SIZE-1 }) /* cf. access_ok() */ -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) diff --git a/arch/m32r/include/asm/uaccess.h b/arch/m32r/include/asm/uaccess.h index 6f8982157a75..7d993a837e39 100644 --- a/arch/m32r/include/asm/uaccess.h +++ b/arch/m32r/include/asm/uaccess.h @@ -16,9 +16,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h index d228601b3afc..fa84e9c6e8f4 100644 --- a/arch/m68k/include/asm/uaccess_mm.h +++ b/arch/m68k/include/asm/uaccess_mm.h @@ -10,9 +10,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* We let the MMU do all checking */ static inline int access_ok(int type, const void __user *addr, unsigned long size) diff --git a/arch/m68k/include/asm/uaccess_no.h b/arch/m68k/include/asm/uaccess_no.h index 36deeb36503b..fab489a25b95 100644 --- a/arch/m68k/include/asm/uaccess_no.h +++ b/arch/m68k/include/asm/uaccess_no.h @@ -10,9 +10,6 @@ #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define access_ok(type,addr,size) _access_ok((unsigned long)(addr),(size)) /* diff --git a/arch/metag/include/asm/uaccess.h b/arch/metag/include/asm/uaccess.h index 273e61225c27..46c1f6c54103 100644 --- a/arch/metag/include/asm/uaccess.h +++ b/arch/metag/include/asm/uaccess.h @@ -6,9 +6,6 @@ */ #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index 253a67e275ad..70cf5f3dfae3 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -24,9 +24,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * On Microblaze the fs value is actually the top of the corresponding * address space. diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index 5347cfe15af2..a058c04b8dd4 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -71,9 +71,6 @@ extern u64 __ua_limit; #define USER_DS ((mm_segment_t) { __UA_LIMIT }) #endif -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) diff --git a/arch/mn10300/include/asm/uaccess.h b/arch/mn10300/include/asm/uaccess.h index 2eedf6f46a57..3e16850c4ccd 100644 --- a/arch/mn10300/include/asm/uaccess.h +++ b/arch/mn10300/include/asm/uaccess.h @@ -19,9 +19,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index 0ab82324c817..07fc68c3e23c 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -19,9 +19,6 @@ #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The exception table consists of pairs of addresses: the first is the * address of an instruction that is allowed to fault, and the second is diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index 140faa16685a..6f88cf8bd112 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -28,9 +28,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index fb4382c28259..598b52e5aa03 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -13,9 +13,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define KERNEL_DS ((mm_segment_t){0}) #define USER_DS ((mm_segment_t){1}) diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 0e6add3187bc..81307633c33f 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -11,9 +11,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 136932ff4250..0e1f515d239b 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -17,9 +17,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be diff --git a/arch/score/include/asm/uaccess.h b/arch/score/include/asm/uaccess.h index db58ab98ec4b..51914244e867 100644 --- a/arch/score/include/asm/uaccess.h +++ b/arch/score/include/asm/uaccess.h @@ -6,9 +6,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define get_ds() (KERNEL_DS) #define get_fs() (current_thread_info()->addr_limit) #define segment_eq(a, b) ((a).seg == (b).seg) diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index c4f0fee812c3..6b66d67c21d2 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -5,9 +5,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define __addr_ok(addr) \ ((unsigned long __force)(addr) < current_thread_info()->addr_limit.seg) diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index ea55f86d7ccd..d8857f5fafad 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -30,9 +30,6 @@ #define KERNEL_DS ((mm_segment_t) { 0 }) #define USER_DS ((mm_segment_t) { -1 }) -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define get_ds() (KERNEL_DS) #define get_fs() (current->thread.current_ds) #define set_fs(val) ((current->thread.current_ds) = (val)) diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 5373136c412b..619223dc9022 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -36,9 +36,6 @@ #define KERNEL_DS ((mm_segment_t) { ASI_P }) #define USER_DS ((mm_segment_t) { ASI_AIUS }) /* har har har */ -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define get_fs() ((mm_segment_t){(current_thread_info()->current_ds)}) #define get_ds() (KERNEL_DS) diff --git a/arch/tile/include/asm/uaccess.h b/arch/tile/include/asm/uaccess.h index a77369e91e54..730073326b46 100644 --- a/arch/tile/include/asm/uaccess.h +++ b/arch/tile/include/asm/uaccess.h @@ -24,9 +24,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index ea148313570f..8dffb8b1d328 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -13,9 +13,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/xtensa/include/asm/asm-uaccess.h b/arch/xtensa/include/asm/asm-uaccess.h index a7a110039786..dfdf9fae1f84 100644 --- a/arch/xtensa/include/asm/asm-uaccess.h +++ b/arch/xtensa/include/asm/asm-uaccess.h @@ -19,9 +19,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #include #include #include diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 848a3d736bcb..dd6b13649aad 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -20,9 +20,6 @@ #include #include -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #include /* diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index cc6bb319e464..d7c17bfd4601 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -35,9 +35,6 @@ static inline void set_fs(mm_segment_t fs) #define segment_eq(a, b) ((a).seg == (b).seg) #endif -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - #define access_ok(type, addr, size) __access_ok((unsigned long)(addr),(size)) /* diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index f30c187ed785..b660f37beaf5 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -2,6 +2,10 @@ #define __LINUX_UACCESS_H__ #include + +#define VERIFY_READ 0 +#define VERIFY_WRITE 1 + #include static __always_inline void pagefault_disabled_inc(void) From af1d5b37d6211c814fac0d5d0b71ec695618054a Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 18:14:09 -0500 Subject: [PATCH 02/96] uaccess: drop duplicate includes from asm/uaccess.h Signed-off-by: Al Viro --- arch/alpha/include/asm/uaccess.h | 4 ---- arch/arc/include/asm/uaccess.h | 2 -- arch/arm/include/asm/uaccess.h | 2 -- arch/arm64/include/asm/uaccess.h | 2 -- arch/avr32/include/asm/uaccess.h | 3 --- arch/blackfin/include/asm/uaccess.h | 1 - arch/cris/include/asm/uaccess.h | 2 -- arch/frv/include/asm/uaccess.h | 1 - arch/hexagon/include/asm/uaccess.h | 1 - arch/ia64/include/asm/uaccess.h | 2 -- arch/m32r/include/asm/uaccess.h | 2 -- arch/m68k/include/asm/uaccess_mm.h | 2 -- arch/m68k/include/asm/uaccess_no.h | 1 - arch/metag/include/asm/uaccess.h | 1 - arch/microblaze/include/asm/uaccess.h | 2 -- arch/mips/include/asm/uaccess.h | 2 -- arch/mn10300/include/asm/uaccess.h | 2 -- arch/nios2/include/asm/uaccess.h | 2 -- arch/openrisc/include/asm/uaccess.h | 2 -- arch/parisc/include/asm/uaccess.h | 2 -- arch/powerpc/include/asm/uaccess.h | 2 -- arch/s390/include/asm/uaccess.h | 2 -- arch/score/include/asm/uaccess.h | 2 -- arch/sh/include/asm/uaccess.h | 2 -- arch/sparc/include/asm/uaccess_32.h | 2 -- arch/sparc/include/asm/uaccess_64.h | 2 -- arch/tile/include/asm/uaccess.h | 1 - arch/um/include/asm/uaccess.h | 1 - arch/unicore32/include/asm/uaccess.h | 3 --- arch/x86/include/asm/uaccess.h | 2 -- arch/x86/include/asm/uaccess_32.h | 2 -- arch/x86/include/asm/uaccess_64.h | 1 - arch/xtensa/include/asm/uaccess.h | 3 --- include/asm-generic/uaccess.h | 1 - include/linux/uaccess.h | 1 + 35 files changed, 1 insertion(+), 64 deletions(-) diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index a37267a5d399..77c55ce89936 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -1,10 +1,6 @@ #ifndef __ALPHA_UACCESS_H #define __ALPHA_UACCESS_H -#include -#include - - /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index 41faf17cd28d..0431f5668354 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -24,8 +24,6 @@ #ifndef _ASM_ARC_UACCESS_H #define _ASM_ARC_UACCESS_H -#include -#include #include /* for generic string functions */ diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index a13f39b3e9f8..9677a7cf7987 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -12,8 +12,6 @@ * User space memory access functions */ #include -#include -#include #include #include #include diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index f5e1e090b4d2..7c514e10a08e 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -28,11 +28,9 @@ #include #include #include -#include #include #include -#include #include #include diff --git a/arch/avr32/include/asm/uaccess.h b/arch/avr32/include/asm/uaccess.h index 1c7f234385fc..7ca5cb33369b 100644 --- a/arch/avr32/include/asm/uaccess.h +++ b/arch/avr32/include/asm/uaccess.h @@ -8,9 +8,6 @@ #ifndef __ASM_AVR32_UACCESS_H #define __ASM_AVR32_UACCESS_H -#include -#include - typedef struct { unsigned int is_user_space; } mm_segment_t; diff --git a/arch/blackfin/include/asm/uaccess.h b/arch/blackfin/include/asm/uaccess.h index d9a91108964f..c9fedc3be30c 100644 --- a/arch/blackfin/include/asm/uaccess.h +++ b/arch/blackfin/include/asm/uaccess.h @@ -12,7 +12,6 @@ /* * User space memory access functions */ -#include #include #include diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h index 82bfcb5e2c9b..bb3004a2b2f7 100644 --- a/arch/cris/include/asm/uaccess.h +++ b/arch/cris/include/asm/uaccess.h @@ -16,8 +16,6 @@ #define _CRIS_UACCESS_H #ifndef __ASSEMBLY__ -#include -#include #include #include diff --git a/arch/frv/include/asm/uaccess.h b/arch/frv/include/asm/uaccess.h index 9e01bd798a03..55b3a69c6c53 100644 --- a/arch/frv/include/asm/uaccess.h +++ b/arch/frv/include/asm/uaccess.h @@ -15,7 +15,6 @@ /* * User space memory access functions */ -#include #include #include #include diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index 21f63593e2b6..3a7f818e5ef7 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -23,7 +23,6 @@ /* * User space memory access functions */ -#include #include #include #include diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index c60ff6cc8dbd..d471d1a1afd0 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -33,8 +33,6 @@ */ #include -#include -#include #include #include diff --git a/arch/m32r/include/asm/uaccess.h b/arch/m32r/include/asm/uaccess.h index 7d993a837e39..96b0efdb5f22 100644 --- a/arch/m32r/include/asm/uaccess.h +++ b/arch/m32r/include/asm/uaccess.h @@ -11,8 +11,6 @@ /* * User space memory access functions */ -#include -#include #include #include diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h index fa84e9c6e8f4..fb72b710759e 100644 --- a/arch/m68k/include/asm/uaccess_mm.h +++ b/arch/m68k/include/asm/uaccess_mm.h @@ -5,9 +5,7 @@ * User space memory access functions */ #include -#include #include -#include #include /* We let the MMU do all checking */ diff --git a/arch/m68k/include/asm/uaccess_no.h b/arch/m68k/include/asm/uaccess_no.h index fab489a25b95..e77ce66c14d5 100644 --- a/arch/m68k/include/asm/uaccess_no.h +++ b/arch/m68k/include/asm/uaccess_no.h @@ -4,7 +4,6 @@ /* * User space memory access functions */ -#include #include #include diff --git a/arch/metag/include/asm/uaccess.h b/arch/metag/include/asm/uaccess.h index 46c1f6c54103..7fc5277ae71f 100644 --- a/arch/metag/include/asm/uaccess.h +++ b/arch/metag/include/asm/uaccess.h @@ -4,7 +4,6 @@ /* * User space memory access functions */ -#include /* * The fs value determines whether argument validity checking should be diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index 70cf5f3dfae3..a3c0a06d7848 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -15,8 +15,6 @@ #ifndef __ASSEMBLY__ #include -#include -#include /* RLIMIT_FSIZE */ #include #include diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index a058c04b8dd4..dd25b312c973 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -12,8 +12,6 @@ #define _ASM_UACCESS_H #include -#include -#include #include #include #include diff --git a/arch/mn10300/include/asm/uaccess.h b/arch/mn10300/include/asm/uaccess.h index 3e16850c4ccd..2da7b0fed4aa 100644 --- a/arch/mn10300/include/asm/uaccess.h +++ b/arch/mn10300/include/asm/uaccess.h @@ -14,10 +14,8 @@ /* * User space memory access functions */ -#include #include #include -#include /* * The fs value determines whether argument validity checking should be diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index 07fc68c3e23c..198bbf15f644 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -13,8 +13,6 @@ #ifndef _ASM_NIOS2_UACCESS_H #define _ASM_NIOS2_UACCESS_H -#include -#include #include #include diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index 6f88cf8bd112..0b0f60444b76 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -22,8 +22,6 @@ /* * User space memory access functions */ -#include -#include #include #include #include diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index 598b52e5aa03..a0b461336b6a 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -6,12 +6,10 @@ */ #include #include -#include #include #include #include -#include #define KERNEL_DS ((mm_segment_t){0}) #define USER_DS ((mm_segment_t){1}) diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 81307633c33f..2ec70aa1cc5d 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -4,8 +4,6 @@ #ifdef __KERNEL__ #ifndef __ASSEMBLY__ -#include -#include #include #include #include diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 0e1f515d239b..9e9a5e8d6cf6 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -12,8 +12,6 @@ /* * User space memory access functions */ -#include -#include #include #include diff --git a/arch/score/include/asm/uaccess.h b/arch/score/include/asm/uaccess.h index 51914244e867..7a6c6982420a 100644 --- a/arch/score/include/asm/uaccess.h +++ b/arch/score/include/asm/uaccess.h @@ -2,8 +2,6 @@ #define __SCORE_UACCESS_H #include -#include -#include #include #define get_ds() (KERNEL_DS) diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index 6b66d67c21d2..89a28dfbabfa 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -1,8 +1,6 @@ #ifndef __ASM_SH_UACCESS_H #define __ASM_SH_UACCESS_H -#include -#include #include #define __addr_ok(addr) \ diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index d8857f5fafad..b10f7d626f0e 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -9,9 +9,7 @@ #ifdef __KERNEL__ #include -#include #include -#include #endif #ifndef __ASSEMBLY__ diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 619223dc9022..d76362cad80f 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -6,10 +6,8 @@ */ #ifdef __KERNEL__ -#include #include #include -#include #include #include #include diff --git a/arch/tile/include/asm/uaccess.h b/arch/tile/include/asm/uaccess.h index 730073326b46..14ea3d1ca2c7 100644 --- a/arch/tile/include/asm/uaccess.h +++ b/arch/tile/include/asm/uaccess.h @@ -18,7 +18,6 @@ /* * User space memory access functions */ -#include #include #include #include diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h index 3705620ca298..076bdcb0c2ad 100644 --- a/arch/um/include/asm/uaccess.h +++ b/arch/um/include/asm/uaccess.h @@ -7,7 +7,6 @@ #ifndef __UM_UACCESS_H #define __UM_UACCESS_H -#include #include #define __under_task_size(addr, size) \ diff --git a/arch/unicore32/include/asm/uaccess.h b/arch/unicore32/include/asm/uaccess.h index 897e11ad8124..1622f37a0514 100644 --- a/arch/unicore32/include/asm/uaccess.h +++ b/arch/unicore32/include/asm/uaccess.h @@ -12,9 +12,6 @@ #ifndef __UNICORE_UACCESS_H__ #define __UNICORE_UACCESS_H__ -#include -#include - #include #define __copy_from_user __copy_from_user diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 8dffb8b1d328..0522d88a7f90 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -3,10 +3,8 @@ /* * User space memory access functions */ -#include #include #include -#include #include #include #include diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h index 7d3bdd1ed697..5268ecceea96 100644 --- a/arch/x86/include/asm/uaccess_32.h +++ b/arch/x86/include/asm/uaccess_32.h @@ -4,8 +4,6 @@ /* * User space memory access functions */ -#include -#include #include #include #include diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 673059a109fe..8ddadd93639e 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -5,7 +5,6 @@ * User space memory access functions */ #include -#include #include #include #include diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index dd6b13649aad..bd8861c811ef 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -16,12 +16,9 @@ #ifndef _XTENSA_UACCESS_H #define _XTENSA_UACCESS_H -#include #include #include -#include - /* * The fs value determines whether argument validity checking should * be performed or not. If get_fs() == USER_DS, checking is diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index d7c17bfd4601..d20955e495b3 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -6,7 +6,6 @@ * on any machine that has kernel and user data in the same * address space, e.g. all NOMMU machines. */ -#include #include #include diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index b660f37beaf5..b786ca2419b4 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -2,6 +2,7 @@ #define __LINUX_UACCESS_H__ #include +#include #define VERIFY_READ 0 #define VERIFY_WRITE 1 From 444f02c458db00bd6049cc1bfe4254e80f57459e Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 18:19:09 -0500 Subject: [PATCH 03/96] uaccess: drop pointless ifdefs None of those file is ever included from uapi stuff, so __KERNEL__ is always defined. None of them is ever included from assembler (they are only pulled from linux/uaccess.h, which _can't_ be included from assembler), so __ASSEMBLY__ is never defined. Signed-off-by: Al Viro --- arch/cris/include/asm/uaccess.h | 3 --- arch/microblaze/include/asm/uaccess.h | 6 ------ arch/powerpc/include/asm/uaccess.h | 6 ------ arch/sparc/include/asm/uaccess_32.h | 6 ------ arch/sparc/include/asm/uaccess_64.h | 6 ------ 5 files changed, 27 deletions(-) diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h index bb3004a2b2f7..c2462ef04eaf 100644 --- a/arch/cris/include/asm/uaccess.h +++ b/arch/cris/include/asm/uaccess.h @@ -15,7 +15,6 @@ #ifndef _CRIS_UACCESS_H #define _CRIS_UACCESS_H -#ifndef __ASSEMBLY__ #include #include @@ -411,6 +410,4 @@ __generic_clear_user_nocheck(void __user *to, unsigned long n) #define strlen_user(str) strnlen_user((str), 0x7ffffffe) -#endif /* __ASSEMBLY__ */ - #endif /* _CRIS_UACCESS_H */ diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index a3c0a06d7848..b132cd301e66 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -11,9 +11,6 @@ #ifndef _ASM_MICROBLAZE_UACCESS_H #define _ASM_MICROBLAZE_UACCESS_H -#ifdef __KERNEL__ -#ifndef __ASSEMBLY__ - #include #include @@ -417,7 +414,4 @@ static inline long strnlen_user(const char __user *src, long n) return __strnlen_user(src, n); } -#endif /* __ASSEMBLY__ */ -#endif /* __KERNEL__ */ - #endif /* _ASM_MICROBLAZE_UACCESS_H */ diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 2ec70aa1cc5d..3904040a3542 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -1,9 +1,6 @@ #ifndef _ARCH_POWERPC_UACCESS_H #define _ARCH_POWERPC_UACCESS_H -#ifdef __KERNEL__ -#ifndef __ASSEMBLY__ - #include #include #include @@ -417,7 +414,4 @@ extern long strncpy_from_user(char *dst, const char __user *src, long count); extern __must_check long strlen_user(const char __user *str); extern __must_check long strnlen_user(const char __user *str, long n); -#endif /* __ASSEMBLY__ */ -#endif /* __KERNEL__ */ - #endif /* _ARCH_POWERPC_UACCESS_H */ diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index b10f7d626f0e..952d512a64f2 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -7,12 +7,8 @@ #ifndef _ASM_UACCESS_H #define _ASM_UACCESS_H -#ifdef __KERNEL__ #include #include -#endif - -#ifndef __ASSEMBLY__ #include @@ -307,6 +303,4 @@ static inline unsigned long clear_user(void __user *addr, unsigned long n) __must_check long strlen_user(const char __user *str); __must_check long strnlen_user(const char __user *str, long n); -#endif /* __ASSEMBLY__ */ - #endif /* _ASM_UACCESS_H */ diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index d76362cad80f..7afb4f64553f 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -5,16 +5,12 @@ * User space memory access functions */ -#ifdef __KERNEL__ #include #include #include #include #include #include -#endif - -#ifndef __ASSEMBLY__ #include @@ -229,6 +225,4 @@ unsigned long compute_effective_address(struct pt_regs *, unsigned int insn, unsigned int rd); -#endif /* __ASSEMBLY__ */ - #endif /* _ASM_UACCESS_H */ From aaa2e7ac80f679230faf28a8e12e8d68dbe977eb Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 01:22:09 -0500 Subject: [PATCH 04/96] add asm-generic/extable.h ... and make the users of generic uaccess.h use that. Signed-off-by: Al Viro --- arch/arc/include/asm/Kbuild | 1 + arch/arc/include/asm/uaccess.h | 2 -- arch/c6x/include/asm/Kbuild | 1 + arch/h8300/include/asm/Kbuild | 1 + arch/hexagon/include/asm/Kbuild | 1 + arch/um/include/asm/Kbuild | 1 + arch/unicore32/include/asm/Kbuild | 1 + arch/unicore32/include/asm/uaccess.h | 2 -- include/asm-generic/extable.h | 26 ++++++++++++++++++++++++++ include/asm-generic/uaccess.h | 20 ++------------------ 10 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 include/asm-generic/extable.h diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild index 63a04013d05a..7bee4e4799fd 100644 --- a/arch/arc/include/asm/Kbuild +++ b/arch/arc/include/asm/Kbuild @@ -6,6 +6,7 @@ generic-y += device.h generic-y += div64.h generic-y += emergency-restart.h generic-y += errno.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += ftrace.h diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index 0431f5668354..d837a53c6e59 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -750,6 +750,4 @@ extern long arc_strnlen_user_noinline(const char __user *src, long n); #include -extern int fixup_exception(struct pt_regs *regs); - #endif diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild index 82619c32d25b..f0eaf0475e7e 100644 --- a/arch/c6x/include/asm/Kbuild +++ b/arch/c6x/include/asm/Kbuild @@ -12,6 +12,7 @@ generic-y += dma.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += futex.h diff --git a/arch/h8300/include/asm/Kbuild b/arch/h8300/include/asm/Kbuild index 341740c3581c..f14e4e6ccbde 100644 --- a/arch/h8300/include/asm/Kbuild +++ b/arch/h8300/include/asm/Kbuild @@ -13,6 +13,7 @@ generic-y += dma.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += ftrace.h diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild index 797b64a4b80b..a2036bfda8af 100644 --- a/arch/hexagon/include/asm/Kbuild +++ b/arch/hexagon/include/asm/Kbuild @@ -11,6 +11,7 @@ generic-y += device.h generic-y += div64.h generic-y += emergency-restart.h generic-y += errno.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += ftrace.h diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index e9d42aab76dc..50a32c33d729 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -6,6 +6,7 @@ generic-y += delay.h generic-y += device.h generic-y += emergency-restart.h generic-y += exec.h +generic-y += extable.h generic-y += ftrace.h generic-y += futex.h generic-y += hardirq.h diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild index 84205fe1cd79..e9ad511c1043 100644 --- a/arch/unicore32/include/asm/Kbuild +++ b/arch/unicore32/include/asm/Kbuild @@ -10,6 +10,7 @@ generic-y += div64.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += ftrace.h diff --git a/arch/unicore32/include/asm/uaccess.h b/arch/unicore32/include/asm/uaccess.h index 1622f37a0514..f60fab718b59 100644 --- a/arch/unicore32/include/asm/uaccess.h +++ b/arch/unicore32/include/asm/uaccess.h @@ -38,6 +38,4 @@ __strnlen_user(const char __user *s, long n); #include -extern int fixup_exception(struct pt_regs *regs); - #endif /* __UNICORE_UACCESS_H__ */ diff --git a/include/asm-generic/extable.h b/include/asm-generic/extable.h new file mode 100644 index 000000000000..ca14c6664027 --- /dev/null +++ b/include/asm-generic/extable.h @@ -0,0 +1,26 @@ +#ifndef __ASM_GENERIC_EXTABLE_H +#define __ASM_GENERIC_EXTABLE_H + +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry +{ + unsigned long insn, fixup; +}; + + +struct pt_regs; +extern int fixup_exception(struct pt_regs *regs); + +#endif diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index d20955e495b3..4b4f4a7f22c6 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -47,24 +47,6 @@ static inline int __access_ok(unsigned long addr, unsigned long size) } #endif -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - /* * architectures with an MMU should override these two */ @@ -344,4 +326,6 @@ clear_user(void __user *to, unsigned long n) return __clear_user(to, n); } +#include + #endif /* __ASM_GENERIC_UACCESS_H */ From db68ce10c4f0a27c1ff9fa0e789e5c41f8c4ea63 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 20 Mar 2017 21:08:07 -0400 Subject: [PATCH 05/96] new helper: uaccess_kernel() Signed-off-by: Al Viro --- arch/alpha/include/asm/uaccess.h | 2 +- arch/arc/include/asm/uaccess.h | 2 +- arch/arm/include/asm/uaccess.h | 2 +- arch/arm/lib/uaccess_with_memcpy.c | 4 ++-- arch/blackfin/kernel/process.c | 2 +- arch/c6x/kernel/sys_c6x.c | 2 +- arch/cris/include/asm/uaccess.h | 2 +- arch/m68k/include/asm/uaccess_mm.h | 2 +- arch/metag/include/asm/uaccess.h | 2 +- arch/mips/include/asm/checksum.h | 4 ++-- arch/mips/include/asm/r4kcache.h | 4 ++-- arch/mips/include/asm/uaccess.h | 2 +- arch/mips/kernel/unaligned.c | 10 +++++----- arch/openrisc/include/asm/uaccess.h | 2 +- arch/parisc/include/asm/futex.h | 2 +- arch/parisc/lib/memcpy.c | 2 +- arch/s390/include/asm/uaccess.h | 2 +- arch/sparc/include/asm/uaccess.h | 2 +- arch/sparc/include/asm/uaccess_32.h | 2 +- arch/um/include/asm/uaccess.h | 2 +- arch/um/kernel/skas/uaccess.c | 10 +++++----- arch/unicore32/include/asm/uaccess.h | 2 +- arch/unicore32/kernel/process.c | 2 +- arch/xtensa/include/asm/uaccess.h | 2 +- block/bsg.c | 2 +- drivers/scsi/sg.c | 2 +- include/linux/uaccess.h | 2 ++ include/rdma/ib.h | 2 +- kernel/trace/bpf_trace.c | 2 +- lib/iov_iter.c | 2 +- mm/memory.c | 2 +- security/tomoyo/network.c | 2 +- 32 files changed, 44 insertions(+), 42 deletions(-) diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index 77c55ce89936..b2f62b98e290 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -425,7 +425,7 @@ clear_user(void __user *to, long len) #undef __module_call #define user_addr_max() \ - (segment_eq(get_fs(), USER_DS) ? TASK_SIZE : ~0UL) + (uaccess_kernel() ? ~0UL : TASK_SIZE) extern long strncpy_from_user(char *dest, const char __user *src, long count); extern __must_check long strlen_user(const char __user *str); diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index d837a53c6e59..ffd14e630aca 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -27,7 +27,7 @@ #include /* for generic string functions */ -#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) +#define __kernel_ok (uaccess_kernel()) /* * Algorithmically, for __user_ok() we want do: diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 9677a7cf7987..b63527359d52 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -266,7 +266,7 @@ static inline void set_fs(mm_segment_t fs) #define access_ok(type, addr, size) (__range_ok(addr, size) == 0) #define user_addr_max() \ - (segment_eq(get_fs(), KERNEL_DS) ? ~0UL : get_fs()) + (uaccess_kernel() ? ~0UL : get_fs()) /* * The "__xxx" versions of the user access functions do not verify the diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index 6bd1089b07e0..9b4ed1728616 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -90,7 +90,7 @@ __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) unsigned long ua_flags; int atomic; - if (unlikely(segment_eq(get_fs(), KERNEL_DS))) { + if (uaccess_kernel()) { memcpy((void *)to, from, n); return 0; } @@ -162,7 +162,7 @@ __clear_user_memset(void __user *addr, unsigned long n) { unsigned long ua_flags; - if (unlikely(segment_eq(get_fs(), KERNEL_DS))) { + if (uaccess_kernel()) { memset((void *)addr, 0, n); return 0; } diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c index 89d5162d4ca6..89814850b08b 100644 --- a/arch/blackfin/kernel/process.c +++ b/arch/blackfin/kernel/process.c @@ -370,7 +370,7 @@ int _access_ok(unsigned long addr, unsigned long size) /* Check that things do not wrap around */ if (addr > ULONG_MAX - size) return 0; - if (segment_eq(get_fs(), KERNEL_DS)) + if (uaccess_kernel()) return 1; #ifdef CONFIG_MTD_UCLINUX if (1) diff --git a/arch/c6x/kernel/sys_c6x.c b/arch/c6x/kernel/sys_c6x.c index 3e9bdfbee8ad..a742ae259239 100644 --- a/arch/c6x/kernel/sys_c6x.c +++ b/arch/c6x/kernel/sys_c6x.c @@ -23,7 +23,7 @@ int _access_ok(unsigned long addr, unsigned long size) if (!addr || addr > (0xffffffffUL - (size - 1))) goto _bad_access; - if (segment_eq(get_fs(), KERNEL_DS)) + if (uaccess_kernel()) return 1; if (memory_start <= addr && (addr + size - 1) < memory_end) diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h index c2462ef04eaf..5f5b8f53d2d7 100644 --- a/arch/cris/include/asm/uaccess.h +++ b/arch/cris/include/asm/uaccess.h @@ -43,7 +43,7 @@ #define segment_eq(a, b) ((a).seg == (b).seg) -#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) +#define __kernel_ok (uaccess_kernel()) #define __user_ok(addr, size) \ (((size) <= TASK_SIZE) && ((addr) <= TASK_SIZE-(size))) #define __access_ok(addr, size) (__kernel_ok || __user_ok((addr), (size))) diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h index fb72b710759e..14054a4e4216 100644 --- a/arch/m68k/include/asm/uaccess_mm.h +++ b/arch/m68k/include/asm/uaccess_mm.h @@ -375,7 +375,7 @@ __constant_copy_to_user(void __user *to, const void *from, unsigned long n) #define copy_to_user(to, from, n) __copy_to_user(to, from, n) #define user_addr_max() \ - (segment_eq(get_fs(), USER_DS) ? TASK_SIZE : ~0UL) + (uaccess_kernel() ? ~0UL : TASK_SIZE) extern long strncpy_from_user(char *dst, const char __user *src, long count); extern __must_check long strlen_user(const char __user *str); diff --git a/arch/metag/include/asm/uaccess.h b/arch/metag/include/asm/uaccess.h index 7fc5277ae71f..83de7554d2b3 100644 --- a/arch/metag/include/asm/uaccess.h +++ b/arch/metag/include/asm/uaccess.h @@ -24,7 +24,7 @@ #define segment_eq(a, b) ((a).seg == (b).seg) -#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) +#define __kernel_ok (uaccess_kernel()) /* * Explicitly allow NULL pointers here. Parts of the kernel such * as readv/writev use access_ok to validate pointers, but want diff --git a/arch/mips/include/asm/checksum.h b/arch/mips/include/asm/checksum.h index c8b574f7e0cc..77cad232a1c6 100644 --- a/arch/mips/include/asm/checksum.h +++ b/arch/mips/include/asm/checksum.h @@ -50,7 +50,7 @@ __wsum csum_partial_copy_from_user(const void __user *src, void *dst, int len, __wsum sum, int *err_ptr) { might_fault(); - if (segment_eq(get_fs(), get_ds())) + if (uaccess_kernel()) return __csum_partial_copy_kernel((__force void *)src, dst, len, sum, err_ptr); else @@ -82,7 +82,7 @@ __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len, { might_fault(); if (access_ok(VERIFY_WRITE, dst, len)) { - if (segment_eq(get_fs(), get_ds())) + if (uaccess_kernel()) return __csum_partial_copy_kernel(src, (__force void *)dst, len, sum, err_ptr); diff --git a/arch/mips/include/asm/r4kcache.h b/arch/mips/include/asm/r4kcache.h index 55fd94e6cd0b..7f12d7e27c94 100644 --- a/arch/mips/include/asm/r4kcache.h +++ b/arch/mips/include/asm/r4kcache.h @@ -20,7 +20,7 @@ #include #include #include -#include /* for segment_eq() */ +#include /* for uaccess_kernel() */ extern void (*r4k_blast_dcache)(void); extern void (*r4k_blast_icache)(void); @@ -714,7 +714,7 @@ static inline void protected_blast_##pfx##cache##_range(unsigned long start,\ \ __##pfx##flush_prologue \ \ - if (segment_eq(get_fs(), USER_DS)) { \ + if (!uaccess_kernel()) { \ while (1) { \ protected_cachee_op(hitop, addr); \ if (addr == aend) \ diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index dd25b312c973..70ca8eee166a 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -88,7 +88,7 @@ static inline bool eva_kernel_access(void) if (!IS_ENABLED(CONFIG_EVA)) return false; - return segment_eq(get_fs(), get_ds()); + return uaccess_kernel(); } /* diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index 7ed98354fe9d..f806ee56e639 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -1026,7 +1026,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, goto sigbus; if (IS_ENABLED(CONFIG_EVA)) { - if (segment_eq(get_fs(), get_ds())) + if (uaccess_kernel()) LoadHW(addr, value, res); else LoadHWE(addr, value, res); @@ -1045,7 +1045,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, goto sigbus; if (IS_ENABLED(CONFIG_EVA)) { - if (segment_eq(get_fs(), get_ds())) + if (uaccess_kernel()) LoadW(addr, value, res); else LoadWE(addr, value, res); @@ -1064,7 +1064,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, goto sigbus; if (IS_ENABLED(CONFIG_EVA)) { - if (segment_eq(get_fs(), get_ds())) + if (uaccess_kernel()) LoadHWU(addr, value, res); else LoadHWUE(addr, value, res); @@ -1132,7 +1132,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, value = regs->regs[insn.i_format.rt]; if (IS_ENABLED(CONFIG_EVA)) { - if (segment_eq(get_fs(), get_ds())) + if (uaccess_kernel()) StoreHW(addr, value, res); else StoreHWE(addr, value, res); @@ -1152,7 +1152,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, value = regs->regs[insn.i_format.rt]; if (IS_ENABLED(CONFIG_EVA)) { - if (segment_eq(get_fs(), get_ds())) + if (uaccess_kernel()) StoreW(addr, value, res); else StoreWE(addr, value, res); diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index 0b0f60444b76..227af1acb8bd 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -292,7 +292,7 @@ clear_user(void *addr, unsigned long size) } #define user_addr_max() \ - (segment_eq(get_fs(), USER_DS) ? TASK_SIZE : ~0UL) + (uaccess_kernel() ? ~0UL : TASK_SIZE) extern long strncpy_from_user(char *dest, const char __user *src, long count); diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h index ac8bd586ace8..0ba14300cd8e 100644 --- a/arch/parisc/include/asm/futex.h +++ b/arch/parisc/include/asm/futex.h @@ -109,7 +109,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, /* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is * our gateway page, and causes no end of trouble... */ - if (segment_eq(KERNEL_DS, get_fs()) && !uaddr) + if (uaccess_kernel() && !uaddr) return -EFAULT; if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) diff --git a/arch/parisc/lib/memcpy.c b/arch/parisc/lib/memcpy.c index f82ff10ed974..66f5160136c2 100644 --- a/arch/parisc/lib/memcpy.c +++ b/arch/parisc/lib/memcpy.c @@ -76,7 +76,7 @@ DECLARE_PER_CPU(struct exception_data, exception_data); goto label; \ } while (0) -#define get_user_space() (segment_eq(get_fs(), KERNEL_DS) ? 0 : mfsp(3)) +#define get_user_space() (uaccess_kernel() ? 0 : mfsp(3)) #define get_kernel_space() (0) #define MERGE(w0, sh_1, w1, sh_2) ({ \ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 9e9a5e8d6cf6..7228ed8da67d 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -37,7 +37,7 @@ static inline void set_fs(mm_segment_t fs) { current->thread.mm_segment = fs; - if (segment_eq(fs, KERNEL_DS)) { + if (uaccess_kernel()) { set_cpu_flag(CIF_ASCE_SECONDARY); __ctl_load(S390_lowcore.kernel_asce, 7, 7); } else { diff --git a/arch/sparc/include/asm/uaccess.h b/arch/sparc/include/asm/uaccess.h index bd56c28fff9f..9e068bf9060a 100644 --- a/arch/sparc/include/asm/uaccess.h +++ b/arch/sparc/include/asm/uaccess.h @@ -7,7 +7,7 @@ #endif #define user_addr_max() \ - (segment_eq(get_fs(), USER_DS) ? TASK_SIZE : ~0UL) + (uaccess_kernel() ? ~0UL : TASK_SIZE) long strncpy_from_user(char *dest, const char __user *src, long count); diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 952d512a64f2..a59a1e81986d 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -36,7 +36,7 @@ * large size and address near to PAGE_OFFSET - a fault will break his intentions. */ #define __user_ok(addr, size) ({ (void)(size); (addr) < STACK_TOP; }) -#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) +#define __kernel_ok (uaccess_kernel()) #define __access_ok(addr, size) (__user_ok((addr) & get_fs().seg, (size))) #define access_ok(type, addr, size) \ ({ (void)(type); __access_ok((unsigned long)(addr), size); }) diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h index 076bdcb0c2ad..e992bb57da5a 100644 --- a/arch/um/include/asm/uaccess.h +++ b/arch/um/include/asm/uaccess.h @@ -45,7 +45,7 @@ static inline int __access_ok(unsigned long addr, unsigned long size) return __addr_range_nowrap(addr, size) && (__under_task_size(addr, size) || __access_ok_vsyscall(addr, size) || - segment_eq(get_fs(), KERNEL_DS)); + uaccess_kernel()); } #endif diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c index 85ac8adb069b..22c9f79db8e6 100644 --- a/arch/um/kernel/skas/uaccess.c +++ b/arch/um/kernel/skas/uaccess.c @@ -141,7 +141,7 @@ static int copy_chunk_from_user(unsigned long from, int len, void *arg) long __copy_from_user(void *to, const void __user *from, unsigned long n) { - if (segment_eq(get_fs(), KERNEL_DS)) { + if (uaccess_kernel()) { memcpy(to, (__force void*)from, n); return 0; } @@ -161,7 +161,7 @@ static int copy_chunk_to_user(unsigned long to, int len, void *arg) long __copy_to_user(void __user *to, const void *from, unsigned long n) { - if (segment_eq(get_fs(), KERNEL_DS)) { + if (uaccess_kernel()) { memcpy((__force void *) to, from, n); return 0; } @@ -189,7 +189,7 @@ long __strncpy_from_user(char *dst, const char __user *src, long count) long n; char *ptr = dst; - if (segment_eq(get_fs(), KERNEL_DS)) { + if (uaccess_kernel()) { strncpy(dst, (__force void *) src, count); return strnlen(dst, count); } @@ -210,7 +210,7 @@ static int clear_chunk(unsigned long addr, int len, void *unused) unsigned long __clear_user(void __user *mem, unsigned long len) { - if (segment_eq(get_fs(), KERNEL_DS)) { + if (uaccess_kernel()) { memset((__force void*)mem, 0, len); return 0; } @@ -235,7 +235,7 @@ long __strnlen_user(const void __user *str, long len) { int count = 0, n; - if (segment_eq(get_fs(), KERNEL_DS)) + if (uaccess_kernel()) return strnlen((__force char*)str, len) + 1; n = buffer_op((unsigned long) str, len, 0, strnlen_chunk, &count); diff --git a/arch/unicore32/include/asm/uaccess.h b/arch/unicore32/include/asm/uaccess.h index f60fab718b59..1196c88c2f9b 100644 --- a/arch/unicore32/include/asm/uaccess.h +++ b/arch/unicore32/include/asm/uaccess.h @@ -20,7 +20,7 @@ #define __strnlen_user __strnlen_user #define __clear_user __clear_user -#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) +#define __kernel_ok (uaccess_kernel()) #define __user_ok(addr, size) (((size) <= TASK_SIZE) \ && ((addr) <= TASK_SIZE - (size))) #define __access_ok(addr, size) (__kernel_ok || __user_ok((addr), (size))) diff --git a/arch/unicore32/kernel/process.c b/arch/unicore32/kernel/process.c index d22c1dc7e39e..ddaf78ae6854 100644 --- a/arch/unicore32/kernel/process.c +++ b/arch/unicore32/kernel/process.c @@ -178,7 +178,7 @@ void __show_regs(struct pt_regs *regs) buf, interrupts_enabled(regs) ? "n" : "ff", fast_interrupts_enabled(regs) ? "n" : "ff", processor_modes[processor_mode(regs)], - segment_eq(get_fs(), get_ds()) ? "kernel" : "user"); + uaccess_kernel() ? "kernel" : "user"); { unsigned int ctrl; diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index bd8861c811ef..26512692e28f 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -37,7 +37,7 @@ #define segment_eq(a, b) ((a).seg == (b).seg) -#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) +#define __kernel_ok (uaccess_kernel()) #define __user_ok(addr, size) \ (((size) <= TASK_SIZE)&&((addr) <= TASK_SIZE-(size))) #define __access_ok(addr, size) (__kernel_ok || __user_ok((addr), (size))) diff --git a/block/bsg.c b/block/bsg.c index 74835dbf0c47..69ccb7801a75 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -650,7 +650,7 @@ bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) dprintk("%s: write %zd bytes\n", bd->name, count); - if (unlikely(segment_eq(get_fs(), KERNEL_DS))) + if (unlikely(uaccess_kernel())) return -EINVAL; bsg_set_block(bd, file); diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 29b86505f796..5d9136a345ec 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -581,7 +581,7 @@ sg_write(struct file *filp, const char __user *buf, size_t count, loff_t * ppos) sg_io_hdr_t *hp; unsigned char cmnd[SG_MAX_CDB_SIZE]; - if (unlikely(segment_eq(get_fs(), KERNEL_DS))) + if (unlikely(uaccess_kernel())) return -EINVAL; if ((!(sfp = (Sg_fd *) filp->private_data)) || (!(sdp = sfp->parentdp))) diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index b786ca2419b4..9c3ae8706e9d 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -7,6 +7,8 @@ #define VERIFY_READ 0 #define VERIFY_WRITE 1 +#define uaccess_kernel() segment_eq(get_fs(), KERNEL_DS) + #include static __always_inline void pagefault_disabled_inc(void) diff --git a/include/rdma/ib.h b/include/rdma/ib.h index 9b4c22a36931..66dbed0c146d 100644 --- a/include/rdma/ib.h +++ b/include/rdma/ib.h @@ -100,7 +100,7 @@ struct sockaddr_ib { */ static inline bool ib_safe_file_access(struct file *filp) { - return filp->f_cred == current_cred() && segment_eq(get_fs(), USER_DS); + return filp->f_cred == current_cred() && !uaccess_kernel(); } #endif /* _RDMA_IB_H */ diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index cee9802cf3e0..f806dbd66de9 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -96,7 +96,7 @@ BPF_CALL_3(bpf_probe_write_user, void *, unsafe_ptr, const void *, src, if (unlikely(in_interrupt() || current->flags & (PF_KTHREAD | PF_EXITING))) return -EPERM; - if (unlikely(segment_eq(get_fs(), KERNEL_DS))) + if (unlikely(uaccess_kernel())) return -EPERM; if (!access_ok(VERIFY_WRITE, unsafe_ptr, size)) return -EPERM; diff --git a/lib/iov_iter.c b/lib/iov_iter.c index e68604ae3ced..97db876c6862 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -413,7 +413,7 @@ void iov_iter_init(struct iov_iter *i, int direction, size_t count) { /* It will get better. Eventually... */ - if (segment_eq(get_fs(), KERNEL_DS)) { + if (uaccess_kernel()) { direction |= ITER_KVEC; i->type = direction; i->kvec = (struct kvec *)iov; diff --git a/mm/memory.c b/mm/memory.c index a97a4cec2e1f..e8f4e10e770a 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4136,7 +4136,7 @@ void __might_fault(const char *file, int line) * get paged out, therefore we'll never actually fault, and the * below annotations will generate false positives. */ - if (segment_eq(get_fs(), KERNEL_DS)) + if (uaccess_kernel()) return; if (pagefault_disabled()) return; diff --git a/security/tomoyo/network.c b/security/tomoyo/network.c index 97527710a72a..6c02ac478247 100644 --- a/security/tomoyo/network.c +++ b/security/tomoyo/network.c @@ -608,7 +608,7 @@ static int tomoyo_check_unix_address(struct sockaddr *addr, static bool tomoyo_kernel_service(void) { /* Nothing to do if I am a kernel service. */ - return segment_eq(get_fs(), KERNEL_DS); + return uaccess_kernel(); } /** From 3fb50075284433093eba7c43e75b16eaa36eeabd Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 00:51:43 -0400 Subject: [PATCH 06/96] asm-generic/uaccess.h: don't mess with __copy_{to,from}_user only h8300 actually used those; might as well define them there. Signed-off-by: Al Viro --- arch/c6x/include/asm/uaccess.h | 3 -- arch/h8300/include/asm/Kbuild | 1 - arch/h8300/include/asm/uaccess.h | 52 +++++++++++++++++++++++ arch/um/include/asm/uaccess.h | 2 - arch/unicore32/include/asm/uaccess.h | 2 - include/asm-generic/uaccess.h | 63 ---------------------------- 6 files changed, 52 insertions(+), 71 deletions(-) create mode 100644 arch/h8300/include/asm/uaccess.h diff --git a/arch/c6x/include/asm/uaccess.h b/arch/c6x/include/asm/uaccess.h index 453dd263bee3..79fb6f439c5c 100644 --- a/arch/c6x/include/asm/uaccess.h +++ b/arch/c6x/include/asm/uaccess.h @@ -94,9 +94,6 @@ static inline __must_check long __copy_to_user(void __user *to, return 0; } -#define __copy_to_user __copy_to_user -#define __copy_from_user __copy_from_user - extern int _access_ok(unsigned long addr, unsigned long size); #ifdef CONFIG_ACCESS_CHECK #define __access_ok _access_ok diff --git a/arch/h8300/include/asm/Kbuild b/arch/h8300/include/asm/Kbuild index f14e4e6ccbde..757cdeb24e6e 100644 --- a/arch/h8300/include/asm/Kbuild +++ b/arch/h8300/include/asm/Kbuild @@ -69,7 +69,6 @@ generic-y += tlbflush.h generic-y += trace_clock.h generic-y += topology.h generic-y += types.h -generic-y += uaccess.h generic-y += ucontext.h generic-y += unaligned.h generic-y += vga.h diff --git a/arch/h8300/include/asm/uaccess.h b/arch/h8300/include/asm/uaccess.h new file mode 100644 index 000000000000..c156a416a279 --- /dev/null +++ b/arch/h8300/include/asm/uaccess.h @@ -0,0 +1,52 @@ +#ifndef _ASM_UACCESS_H +#define _ASM_UACCESS_H + +#include + +static inline __must_check long __copy_from_user(void *to, + const void __user * from, unsigned long n) +{ + if (__builtin_constant_p(n)) { + switch(n) { + case 1: + *(u8 *)to = *(u8 __force *)from; + return 0; + case 2: + *(u16 *)to = *(u16 __force *)from; + return 0; + case 4: + *(u32 *)to = *(u32 __force *)from; + return 0; + } + } + + memcpy(to, (const void __force *)from, n); + return 0; +} + +static inline __must_check long __copy_to_user(void __user *to, + const void *from, unsigned long n) +{ + if (__builtin_constant_p(n)) { + switch(n) { + case 1: + *(u8 __force *)to = *(u8 *)from; + return 0; + case 2: + *(u16 __force *)to = *(u16 *)from; + return 0; + case 4: + *(u32 __force *)to = *(u32 *)from; + return 0; + default: + break; + } + } + + memcpy((void __force *)to, from, n); + return 0; +} + +#include + +#endif diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h index e992bb57da5a..9396c8a4e920 100644 --- a/arch/um/include/asm/uaccess.h +++ b/arch/um/include/asm/uaccess.h @@ -31,8 +31,6 @@ static inline int __access_ok(unsigned long addr, unsigned long size); /* Teach asm-generic/uaccess.h that we have C functions for these. */ #define __access_ok __access_ok #define __clear_user __clear_user -#define __copy_to_user __copy_to_user -#define __copy_from_user __copy_from_user #define __strnlen_user __strnlen_user #define __strncpy_from_user __strncpy_from_user #define __copy_to_user_inatomic __copy_to_user diff --git a/arch/unicore32/include/asm/uaccess.h b/arch/unicore32/include/asm/uaccess.h index 1196c88c2f9b..b2bcab69dfee 100644 --- a/arch/unicore32/include/asm/uaccess.h +++ b/arch/unicore32/include/asm/uaccess.h @@ -14,8 +14,6 @@ #include -#define __copy_from_user __copy_from_user -#define __copy_to_user __copy_to_user #define __strncpy_from_user __strncpy_from_user #define __strnlen_user __strnlen_user #define __clear_user __clear_user diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index 4b4f4a7f22c6..b048246a606b 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -47,69 +47,6 @@ static inline int __access_ok(unsigned long addr, unsigned long size) } #endif -/* - * architectures with an MMU should override these two - */ -#ifndef __copy_from_user -static inline __must_check long __copy_from_user(void *to, - const void __user * from, unsigned long n) -{ - if (__builtin_constant_p(n)) { - switch(n) { - case 1: - *(u8 *)to = *(u8 __force *)from; - return 0; - case 2: - *(u16 *)to = *(u16 __force *)from; - return 0; - case 4: - *(u32 *)to = *(u32 __force *)from; - return 0; -#ifdef CONFIG_64BIT - case 8: - *(u64 *)to = *(u64 __force *)from; - return 0; -#endif - default: - break; - } - } - - memcpy(to, (const void __force *)from, n); - return 0; -} -#endif - -#ifndef __copy_to_user -static inline __must_check long __copy_to_user(void __user *to, - const void *from, unsigned long n) -{ - if (__builtin_constant_p(n)) { - switch(n) { - case 1: - *(u8 __force *)to = *(u8 *)from; - return 0; - case 2: - *(u16 __force *)to = *(u16 *)from; - return 0; - case 4: - *(u32 __force *)to = *(u32 *)from; - return 0; -#ifdef CONFIG_64BIT - case 8: - *(u64 __force *)to = *(u64 *)from; - return 0; -#endif - default: - break; - } - } - - memcpy((void __force *)to, from, n); - return 0; -} -#endif - /* * These are the main single-value transfer routines. They automatically * use the right size if we just have the right pointer type. From c1aad8dcc49382399f48541dc47b6e30b0ef1b62 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 01:02:40 -0400 Subject: [PATCH 07/96] asm-generic: zero in __get_user(), not __get_user_fn() Signed-off-by: Al Viro --- include/asm-generic/uaccess.h | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index b048246a606b..bd7a05e9582b 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -86,8 +86,7 @@ static inline int __access_ok(unsigned long addr, unsigned long size) static inline int __put_user_fn(size_t size, void __user *ptr, void *x) { - size = __copy_to_user(ptr, x, size); - return size ? -EFAULT : size; + return unlikely(__copy_to_user(ptr, x, size)) ? -EFAULT : 0; } #define __put_user_fn(sz, u, k) __put_user_fn(sz, u, k) @@ -102,28 +101,28 @@ extern int __put_user_bad(void) __attribute__((noreturn)); __chk_user_ptr(ptr); \ switch (sizeof(*(ptr))) { \ case 1: { \ - unsigned char __x; \ + unsigned char __x = 0; \ __gu_err = __get_user_fn(sizeof (*(ptr)), \ ptr, &__x); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 2: { \ - unsigned short __x; \ + unsigned short __x = 0; \ __gu_err = __get_user_fn(sizeof (*(ptr)), \ ptr, &__x); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 4: { \ - unsigned int __x; \ + unsigned int __x = 0; \ __gu_err = __get_user_fn(sizeof (*(ptr)), \ ptr, &__x); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ break; \ }; \ case 8: { \ - unsigned long long __x; \ + unsigned long long __x = 0; \ __gu_err = __get_user_fn(sizeof (*(ptr)), \ ptr, &__x); \ (x) = *(__force __typeof__(*(ptr)) *) &__x; \ @@ -148,12 +147,7 @@ extern int __put_user_bad(void) __attribute__((noreturn)); #ifndef __get_user_fn static inline int __get_user_fn(size_t size, const void __user *ptr, void *x) { - size_t n = __copy_from_user(x, ptr, size); - if (unlikely(n)) { - memset(x + (size - n), 0, n); - return -EFAULT; - } - return 0; + return unlikely(__copy_from_user(x, ptr, size)) ? -EFAULT : 0; } #define __get_user_fn(sz, u, k) __get_user_fn(sz, u, k) From d597580d373774b1bdab84b3d26ff0b55162b916 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 20 Mar 2017 21:56:06 -0400 Subject: [PATCH 08/96] generic ...copy_..._user primitives provide raw_copy_..._user() and select ARCH_HAS_RAW_COPY_USER to use those. Signed-off-by: Al Viro --- arch/Kconfig | 3 + include/asm-generic/uaccess.h | 11 ++ include/linux/uaccess.h | 187 ++++++++++++++++++++++++++++++++++ lib/Makefile | 2 + lib/usercopy.c | 26 +++++ 5 files changed, 229 insertions(+) create mode 100644 lib/usercopy.c diff --git a/arch/Kconfig b/arch/Kconfig index cd211a14a88f..315d37626ddc 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -847,4 +847,7 @@ config STRICT_MODULE_RWX config ARCH_WANT_RELAX_ORDER bool +config ARCH_HAS_RAW_COPY_USER + bool + source "kernel/gcov/Kconfig" diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index bd7a05e9582b..d65c311eb128 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -86,7 +86,11 @@ static inline int __access_ok(unsigned long addr, unsigned long size) static inline int __put_user_fn(size_t size, void __user *ptr, void *x) { +#ifdef CONFIG_ARCH_HAS_RAW_COPY_USER + return unlikely(raw_copy_to_user(ptr, x, size)) ? -EFAULT : 0; +#else return unlikely(__copy_to_user(ptr, x, size)) ? -EFAULT : 0; +#endif } #define __put_user_fn(sz, u, k) __put_user_fn(sz, u, k) @@ -147,7 +151,11 @@ extern int __put_user_bad(void) __attribute__((noreturn)); #ifndef __get_user_fn static inline int __get_user_fn(size_t size, const void __user *ptr, void *x) { +#ifdef CONFIG_ARCH_HAS_RAW_COPY_USER + return unlikely(raw_copy_from_user(x, ptr, size)) ? -EFAULT : 0; +#else return unlikely(__copy_from_user(x, ptr, size)) ? -EFAULT : 0; +#endif } #define __get_user_fn(sz, u, k) __get_user_fn(sz, u, k) @@ -156,6 +164,8 @@ static inline int __get_user_fn(size_t size, const void __user *ptr, void *x) extern int __get_user_bad(void) __attribute__((noreturn)); +#ifndef CONFIG_ARCH_HAS_RAW_COPY_USER + #ifndef __copy_from_user_inatomic #define __copy_from_user_inatomic __copy_from_user #endif @@ -185,6 +195,7 @@ static inline long copy_to_user(void __user *to, else return n; } +#endif /* * Copy a null terminated string from userspace. diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 9c3ae8706e9d..5f76bc995d96 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -3,6 +3,7 @@ #include #include +#include #define VERIFY_READ 0 #define VERIFY_WRITE 1 @@ -11,6 +12,192 @@ #include +#ifdef CONFIG_ARCH_HAS_RAW_COPY_USER +/* + * Architectures should provide two primitives (raw_copy_{to,from}_user()) + * select ARCH_HAS_RAW_COPY_FROM_USER and get rid of their private instances + * of copy_{to,from}_user() and __copy_{to,from}_user{,_inatomic}(). Once + * all of them switch, this part of linux/uaccess.h will become unconditional. + * + * raw_copy_{to,from}_user(to, from, size) should copy up to size bytes and + * return the amount left to copy. They should assume that access_ok() has + * already been checked (and succeeded); they should *not* zero-pad anything. + * No KASAN or object size checks either - those belong here. + * + * Both of these functions should attempt to copy size bytes starting at from + * into the area starting at to. They must not fetch or store anything + * outside of those areas. Return value must be between 0 (everything + * copied successfully) and size (nothing copied). + * + * If raw_copy_{to,from}_user(to, from, size) returns N, size - N bytes starting + * at to must become equal to the bytes fetched from the corresponding area + * starting at from. All data past to + size - N must be left unmodified. + * + * If copying succeeds, the return value must be 0. If some data cannot be + * fetched, it is permitted to copy less than had been fetched; the only + * hard requirement is that not storing anything at all (i.e. returning size) + * should happen only when nothing could be copied. In other words, you don't + * have to squeeze as much as possible - it is allowed, but not necessary. + * + * For raw_copy_from_user() to always points to kernel memory and no faults + * on store should happen. Interpretation of from is affected by set_fs(). + * For raw_copy_to_user() it's the other way round. + * + * Both can be inlined - it's up to architectures whether it wants to bother + * with that. They should not be used directly; they are used to implement + * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic()) + * that are used instead. Out of those, __... ones are inlined. Plain + * copy_{to,from}_user() might or might not be inlined. If you want them + * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER. + * + * NOTE: only copy_from_user() zero-pads the destination in case of short copy. + * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything + * at all; their callers absolutely must check the return value. + * + * Biarch ones should also provide raw_copy_in_user() - similar to the above, + * but both source and destination are __user pointers (affected by set_fs() + * as usual) and both source and destination can trigger faults. + */ + +static __always_inline unsigned long +__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n) +{ + kasan_check_write(to, n); + check_object_size(to, n, false); + return raw_copy_from_user(to, from, n); +} + +static __always_inline unsigned long +__copy_from_user(void *to, const void __user *from, unsigned long n) +{ + might_fault(); + kasan_check_write(to, n); + check_object_size(to, n, false); + return raw_copy_from_user(to, from, n); +} + +/** + * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking. + * @to: Destination address, in user space. + * @from: Source address, in kernel space. + * @n: Number of bytes to copy. + * + * Context: User context only. + * + * Copy data from kernel space to user space. Caller must check + * the specified block with access_ok() before calling this function. + * The caller should also make sure he pins the user space address + * so that we don't result in page fault and sleep. + */ +static __always_inline unsigned long +__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n) +{ + kasan_check_read(from, n); + check_object_size(from, n, true); + return raw_copy_to_user(to, from, n); +} + +static __always_inline unsigned long +__copy_to_user(void __user *to, const void *from, unsigned long n) +{ + might_fault(); + kasan_check_read(from, n); + check_object_size(from, n, true); + return raw_copy_to_user(to, from, n); +} + +#ifdef INLINE_COPY_FROM_USER +static inline unsigned long +_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + unsigned long res = n; + if (likely(access_ok(VERIFY_READ, from, n))) + res = raw_copy_from_user(to, from, n); + if (unlikely(res)) + memset(to + (n - res), 0, res); + return res; +} +#else +extern unsigned long +_copy_from_user(void *, const void __user *, unsigned long); +#endif + +#ifdef INLINE_COPY_TO_USER +static inline unsigned long +_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + if (access_ok(VERIFY_WRITE, to, n)) + n = raw_copy_to_user(to, from, n); + return n; +} +#else +extern unsigned long +_copy_to_user(void __user *, const void *, unsigned long); +#endif + +extern void __compiletime_error("usercopy buffer size is too small") +__bad_copy_user(void); + +static inline void copy_user_overflow(int size, unsigned long count) +{ + WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); +} + +static __always_inline unsigned long __must_check +copy_from_user(void *to, const void __user *from, unsigned long n) +{ + int sz = __compiletime_object_size(to); + + might_fault(); + kasan_check_write(to, n); + + if (likely(sz < 0 || sz >= n)) { + check_object_size(to, n, false); + n = _copy_from_user(to, from, n); + } else if (!__builtin_constant_p(n)) + copy_user_overflow(sz, n); + else + __bad_copy_user(); + + return n; +} + +static __always_inline unsigned long __must_check +copy_to_user(void __user *to, const void *from, unsigned long n) +{ + int sz = __compiletime_object_size(from); + + kasan_check_read(from, n); + might_fault(); + + if (likely(sz < 0 || sz >= n)) { + check_object_size(from, n, true); + n = _copy_to_user(to, from, n); + } else if (!__builtin_constant_p(n)) + copy_user_overflow(sz, n); + else + __bad_copy_user(); + + return n; +} +#ifdef CONFIG_COMPAT +static __always_inline unsigned long __must_check +__copy_in_user(void __user *to, const void *from, unsigned long n) +{ + might_fault(); + return raw_copy_in_user(to, from, n); +} +static __always_inline unsigned long __must_check +copy_in_user(void __user *to, const void *from, unsigned long n) +{ + might_fault(); + if (access_ok(VERIFY_WRITE, to, n) && access_ok(VERIFY_READ, from, n)) + n = raw_copy_in_user(to, from, n); + return n; +} +#endif +#endif + static __always_inline void pagefault_disabled_inc(void) { current->pagefault_disabled++; diff --git a/lib/Makefile b/lib/Makefile index 320ac46a8725..7d875c389172 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -242,3 +242,5 @@ UBSAN_SANITIZE_ubsan.o := n obj-$(CONFIG_SBITMAP) += sbitmap.o obj-$(CONFIG_PARMAN) += parman.o + +obj-$(CONFIG_ARCH_HAS_RAW_COPY_USER) += usercopy.o diff --git a/lib/usercopy.c b/lib/usercopy.c new file mode 100644 index 000000000000..1b6010a3beb8 --- /dev/null +++ b/lib/usercopy.c @@ -0,0 +1,26 @@ +#include + +/* out-of-line parts */ + +#ifndef INLINE_COPY_FROM_USER +unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n) +{ + unsigned long res = n; + if (likely(access_ok(VERIFY_READ, from, n))) + res = raw_copy_from_user(to, from, n); + if (unlikely(res)) + memset(to + (n - res), 0, res); + return res; +} +EXPORT_SYMBOL(_copy_from_user); +#endif + +#ifndef INLINE_COPY_TO_USER +unsigned long _copy_to_user(void *to, const void __user *from, unsigned long n) +{ + if (likely(access_ok(VERIFY_WRITE, to, n))) + n = raw_copy_to_user(to, from, n); + return n; +} +EXPORT_SYMBOL(_copy_to_user); +#endif From 8525023121de4848b5f0a7d867ffeadbc477774d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 24 Dec 2016 20:26:18 -0500 Subject: [PATCH 09/96] alpha: switch __copy_user() and __do_clean_user() to normal calling conventions They used to need odd calling conventions due to old exception handling mechanism, the last remnants of which had disappeared back in 2002. Signed-off-by: Al Viro --- arch/alpha/include/asm/uaccess.h | 67 ++++---------------- arch/alpha/lib/clear_user.S | 66 ++++++++------------ arch/alpha/lib/copy_user.S | 82 ++++++++++-------------- arch/alpha/lib/ev6-clear_user.S | 84 +++++++++++-------------- arch/alpha/lib/ev6-copy_user.S | 104 +++++++++++++------------------ 5 files changed, 151 insertions(+), 252 deletions(-) diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index b2f62b98e290..0135199c9dda 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -334,45 +334,17 @@ __asm__ __volatile__("1: stb %r2,%1\n" \ * Complex access routines */ -/* This little bit of silliness is to get the GP loaded for a function - that ordinarily wouldn't. Otherwise we could have it done by the macro - directly, which can be optimized the linker. */ -#ifdef MODULE -#define __module_address(sym) "r"(sym), -#define __module_call(ra, arg, sym) "jsr $" #ra ",(%" #arg ")," #sym -#else -#define __module_address(sym) -#define __module_call(ra, arg, sym) "bsr $" #ra "," #sym " !samegp" -#endif +extern long __copy_user(void *to, const void *from, long len); -extern void __copy_user(void); - -extern inline long -__copy_tofrom_user_nocheck(void *to, const void *from, long len) -{ - register void * __cu_to __asm__("$6") = to; - register const void * __cu_from __asm__("$7") = from; - register long __cu_len __asm__("$0") = len; - - __asm__ __volatile__( - __module_call(28, 3, __copy_user) - : "=r" (__cu_len), "=r" (__cu_from), "=r" (__cu_to) - : __module_address(__copy_user) - "0" (__cu_len), "1" (__cu_from), "2" (__cu_to) - : "$1", "$2", "$3", "$4", "$5", "$28", "memory"); - - return __cu_len; -} - -#define __copy_to_user(to, from, n) \ -({ \ - __chk_user_ptr(to); \ - __copy_tofrom_user_nocheck((__force void *)(to), (from), (n)); \ +#define __copy_to_user(to, from, n) \ +({ \ + __chk_user_ptr(to); \ + __copy_user((__force void *)(to), (from), (n)); \ }) -#define __copy_from_user(to, from, n) \ -({ \ - __chk_user_ptr(from); \ - __copy_tofrom_user_nocheck((to), (__force void *)(from), (n)); \ +#define __copy_from_user(to, from, n) \ +({ \ + __chk_user_ptr(from); \ + __copy_user((to), (__force void *)(from), (n)); \ }) #define __copy_to_user_inatomic __copy_to_user @@ -382,7 +354,7 @@ extern inline long copy_to_user(void __user *to, const void *from, long n) { if (likely(__access_ok((unsigned long)to, n, get_fs()))) - n = __copy_tofrom_user_nocheck((__force void *)to, from, n); + n = __copy_user((__force void *)to, from, n); return n; } @@ -397,21 +369,7 @@ copy_from_user(void *to, const void __user *from, long n) return res; } -extern void __do_clear_user(void); - -extern inline long -__clear_user(void __user *to, long len) -{ - register void __user * __cl_to __asm__("$6") = to; - register long __cl_len __asm__("$0") = len; - __asm__ __volatile__( - __module_call(28, 2, __do_clear_user) - : "=r"(__cl_len), "=r"(__cl_to) - : __module_address(__do_clear_user) - "0"(__cl_len), "1"(__cl_to) - : "$1", "$2", "$3", "$4", "$5", "$28", "memory"); - return __cl_len; -} +extern long __clear_user(void __user *to, long len); extern inline long clear_user(void __user *to, long len) @@ -421,9 +379,6 @@ clear_user(void __user *to, long len) return len; } -#undef __module_address -#undef __module_call - #define user_addr_max() \ (uaccess_kernel() ? ~0UL : TASK_SIZE) diff --git a/arch/alpha/lib/clear_user.S b/arch/alpha/lib/clear_user.S index bf5b931866ba..006f469fef73 100644 --- a/arch/alpha/lib/clear_user.S +++ b/arch/alpha/lib/clear_user.S @@ -8,21 +8,6 @@ * right "bytes left to zero" value (and that it is updated only _after_ * a successful copy). There is also some rather minor exception setup * stuff. - * - * NOTE! This is not directly C-callable, because the calling semantics - * are different: - * - * Inputs: - * length in $0 - * destination address in $6 - * exception pointer in $7 - * return address in $28 (exceptions expect it there) - * - * Outputs: - * bytes left to copy in $0 - * - * Clobbers: - * $1,$2,$3,$4,$5,$6 */ #include @@ -38,62 +23,63 @@ .set noreorder .align 4 - .globl __do_clear_user - .ent __do_clear_user - .frame $30, 0, $28 + .globl __clear_user + .ent __clear_user + .frame $30, 0, $26 .prologue 0 $loop: and $1, 3, $4 # e0 : beq $4, 1f # .. e1 : -0: EX( stq_u $31, 0($6) ) # e0 : zero one word +0: EX( stq_u $31, 0($16) ) # e0 : zero one word subq $0, 8, $0 # .. e1 : subq $4, 1, $4 # e0 : - addq $6, 8, $6 # .. e1 : + addq $16, 8, $16 # .. e1 : bne $4, 0b # e1 : unop # : 1: bic $1, 3, $1 # e0 : beq $1, $tail # .. e1 : -2: EX( stq_u $31, 0($6) ) # e0 : zero four words +2: EX( stq_u $31, 0($16) ) # e0 : zero four words subq $0, 8, $0 # .. e1 : - EX( stq_u $31, 8($6) ) # e0 : + EX( stq_u $31, 8($16) ) # e0 : subq $0, 8, $0 # .. e1 : - EX( stq_u $31, 16($6) ) # e0 : + EX( stq_u $31, 16($16) ) # e0 : subq $0, 8, $0 # .. e1 : - EX( stq_u $31, 24($6) ) # e0 : + EX( stq_u $31, 24($16) ) # e0 : subq $0, 8, $0 # .. e1 : subq $1, 4, $1 # e0 : - addq $6, 32, $6 # .. e1 : + addq $16, 32, $16 # .. e1 : bne $1, 2b # e1 : $tail: bne $2, 1f # e1 : is there a tail to do? - ret $31, ($28), 1 # .. e1 : + ret $31, ($26), 1 # .. e1 : -1: EX( ldq_u $5, 0($6) ) # e0 : +1: EX( ldq_u $5, 0($16) ) # e0 : clr $0 # .. e1 : nop # e1 : mskqh $5, $0, $5 # e0 : - EX( stq_u $5, 0($6) ) # e0 : - ret $31, ($28), 1 # .. e1 : + EX( stq_u $5, 0($16) ) # e0 : + ret $31, ($26), 1 # .. e1 : -__do_clear_user: - and $6, 7, $4 # e0 : find dest misalignment +__clear_user: + and $17, $17, $0 + and $16, 7, $4 # e0 : find dest misalignment beq $0, $zerolength # .. e1 : addq $0, $4, $1 # e0 : bias counter and $1, 7, $2 # e1 : number of bytes in tail srl $1, 3, $1 # e0 : beq $4, $loop # .. e1 : - EX( ldq_u $5, 0($6) ) # e0 : load dst word to mask back in + EX( ldq_u $5, 0($16) ) # e0 : load dst word to mask back in beq $1, $oneword # .. e1 : sub-word store? - mskql $5, $6, $5 # e0 : take care of misaligned head - addq $6, 8, $6 # .. e1 : - EX( stq_u $5, -8($6) ) # e0 : + mskql $5, $16, $5 # e0 : take care of misaligned head + addq $16, 8, $16 # .. e1 : + EX( stq_u $5, -8($16) ) # e0 : addq $0, $4, $0 # .. e1 : bytes left -= 8 - misalignment subq $1, 1, $1 # e0 : subq $0, 8, $0 # .. e1 : @@ -101,15 +87,15 @@ __do_clear_user: unop # : $oneword: - mskql $5, $6, $4 # e0 : + mskql $5, $16, $4 # e0 : mskqh $5, $2, $5 # e0 : or $5, $4, $5 # e1 : - EX( stq_u $5, 0($6) ) # e0 : + EX( stq_u $5, 0($16) ) # e0 : clr $0 # .. e1 : $zerolength: $exception: - ret $31, ($28), 1 # .. e1 : + ret $31, ($26), 1 # .. e1 : - .end __do_clear_user - EXPORT_SYMBOL(__do_clear_user) + .end __clear_user + EXPORT_SYMBOL(__clear_user) diff --git a/arch/alpha/lib/copy_user.S b/arch/alpha/lib/copy_user.S index 509f62b65311..159f1b7e6e49 100644 --- a/arch/alpha/lib/copy_user.S +++ b/arch/alpha/lib/copy_user.S @@ -9,21 +9,6 @@ * contains the right "bytes left to copy" value (and that it is updated * only _after_ a successful copy). There is also some rather minor * exception setup stuff.. - * - * NOTE! This is not directly C-callable, because the calling semantics are - * different: - * - * Inputs: - * length in $0 - * destination address in $6 - * source address in $7 - * return address in $28 - * - * Outputs: - * bytes left to copy in $0 - * - * Clobbers: - * $1,$2,$3,$4,$5,$6,$7 */ #include @@ -49,58 +34,59 @@ .ent __copy_user __copy_user: .prologue 0 - and $6,7,$3 + and $18,$18,$0 + and $16,7,$3 beq $0,$35 beq $3,$36 subq $3,8,$3 .align 4 $37: - EXI( ldq_u $1,0($7) ) - EXO( ldq_u $2,0($6) ) - extbl $1,$7,$1 - mskbl $2,$6,$2 - insbl $1,$6,$1 + EXI( ldq_u $1,0($17) ) + EXO( ldq_u $2,0($16) ) + extbl $1,$17,$1 + mskbl $2,$16,$2 + insbl $1,$16,$1 addq $3,1,$3 bis $1,$2,$1 - EXO( stq_u $1,0($6) ) + EXO( stq_u $1,0($16) ) subq $0,1,$0 - addq $6,1,$6 - addq $7,1,$7 + addq $16,1,$16 + addq $17,1,$17 beq $0,$41 bne $3,$37 $36: - and $7,7,$1 + and $17,7,$1 bic $0,7,$4 beq $1,$43 beq $4,$48 - EXI( ldq_u $3,0($7) ) + EXI( ldq_u $3,0($17) ) .align 4 $50: - EXI( ldq_u $2,8($7) ) + EXI( ldq_u $2,8($17) ) subq $4,8,$4 - extql $3,$7,$3 - extqh $2,$7,$1 + extql $3,$17,$3 + extqh $2,$17,$1 bis $3,$1,$1 - EXO( stq $1,0($6) ) - addq $7,8,$7 + EXO( stq $1,0($16) ) + addq $17,8,$17 subq $0,8,$0 - addq $6,8,$6 + addq $16,8,$16 bis $2,$2,$3 bne $4,$50 $48: beq $0,$41 .align 4 $57: - EXI( ldq_u $1,0($7) ) - EXO( ldq_u $2,0($6) ) - extbl $1,$7,$1 - mskbl $2,$6,$2 - insbl $1,$6,$1 + EXI( ldq_u $1,0($17) ) + EXO( ldq_u $2,0($16) ) + extbl $1,$17,$1 + mskbl $2,$16,$2 + insbl $1,$16,$1 bis $1,$2,$1 - EXO( stq_u $1,0($6) ) + EXO( stq_u $1,0($16) ) subq $0,1,$0 - addq $6,1,$6 - addq $7,1,$7 + addq $16,1,$16 + addq $17,1,$17 bne $0,$57 br $31,$41 .align 4 @@ -108,27 +94,27 @@ $43: beq $4,$65 .align 4 $66: - EXI( ldq $1,0($7) ) + EXI( ldq $1,0($17) ) subq $4,8,$4 - EXO( stq $1,0($6) ) - addq $7,8,$7 + EXO( stq $1,0($16) ) + addq $17,8,$17 subq $0,8,$0 - addq $6,8,$6 + addq $16,8,$16 bne $4,$66 $65: beq $0,$41 - EXI( ldq $2,0($7) ) - EXO( ldq $1,0($6) ) + EXI( ldq $2,0($17) ) + EXO( ldq $1,0($16) ) mskql $2,$0,$2 mskqh $1,$0,$1 bis $2,$1,$2 - EXO( stq $2,0($6) ) + EXO( stq $2,0($16) ) bis $31,$31,$0 $41: $35: $exitin: $exitout: - ret $31,($28),1 + ret $31,($26),1 .end __copy_user EXPORT_SYMBOL(__copy_user) diff --git a/arch/alpha/lib/ev6-clear_user.S b/arch/alpha/lib/ev6-clear_user.S index 05bef6b50598..e179e4757ef8 100644 --- a/arch/alpha/lib/ev6-clear_user.S +++ b/arch/alpha/lib/ev6-clear_user.S @@ -9,21 +9,6 @@ * a successful copy). There is also some rather minor exception setup * stuff. * - * NOTE! This is not directly C-callable, because the calling semantics - * are different: - * - * Inputs: - * length in $0 - * destination address in $6 - * exception pointer in $7 - * return address in $28 (exceptions expect it there) - * - * Outputs: - * bytes left to copy in $0 - * - * Clobbers: - * $1,$2,$3,$4,$5,$6 - * * Much of the information about 21264 scheduling/coding comes from: * Compiler Writer's Guide for the Alpha 21264 * abbreviated as 'CWG' in other comments here @@ -56,14 +41,15 @@ .set noreorder .align 4 - .globl __do_clear_user - .ent __do_clear_user - .frame $30, 0, $28 + .globl __clear_user + .ent __clear_user + .frame $30, 0, $26 .prologue 0 # Pipeline info : Slotting & Comments -__do_clear_user: - and $6, 7, $4 # .. E .. .. : find dest head misalignment +__clear_user: + and $17, $17, $0 + and $16, 7, $4 # .. E .. .. : find dest head misalignment beq $0, $zerolength # U .. .. .. : U L U L addq $0, $4, $1 # .. .. .. E : bias counter @@ -75,14 +61,14 @@ __do_clear_user: /* * Head is not aligned. Write (8 - $4) bytes to head of destination - * This means $6 is known to be misaligned + * This means $16 is known to be misaligned */ - EX( ldq_u $5, 0($6) ) # .. .. .. L : load dst word to mask back in + EX( ldq_u $5, 0($16) ) # .. .. .. L : load dst word to mask back in beq $1, $onebyte # .. .. U .. : sub-word store? - mskql $5, $6, $5 # .. U .. .. : take care of misaligned head - addq $6, 8, $6 # E .. .. .. : L U U L + mskql $5, $16, $5 # .. U .. .. : take care of misaligned head + addq $16, 8, $16 # E .. .. .. : L U U L - EX( stq_u $5, -8($6) ) # .. .. .. L : + EX( stq_u $5, -8($16) ) # .. .. .. L : subq $1, 1, $1 # .. .. E .. : addq $0, $4, $0 # .. E .. .. : bytes left -= 8 - misalignment subq $0, 8, $0 # E .. .. .. : U L U L @@ -93,11 +79,11 @@ __do_clear_user: * values upon initial entry to the loop * $1 is number of quadwords to clear (zero is a valid value) * $2 is number of trailing bytes (0..7) ($2 never used...) - * $6 is known to be aligned 0mod8 + * $16 is known to be aligned 0mod8 */ $headalign: subq $1, 16, $4 # .. .. .. E : If < 16, we can not use the huge loop - and $6, 0x3f, $2 # .. .. E .. : Forward work for huge loop + and $16, 0x3f, $2 # .. .. E .. : Forward work for huge loop subq $2, 0x40, $3 # .. E .. .. : bias counter (huge loop) blt $4, $trailquad # U .. .. .. : U L U L @@ -114,21 +100,21 @@ $headalign: beq $3, $bigalign # U .. .. .. : U L U L : Aligned 0mod64 $alignmod64: - EX( stq_u $31, 0($6) ) # .. .. .. L + EX( stq_u $31, 0($16) ) # .. .. .. L addq $3, 8, $3 # .. .. E .. subq $0, 8, $0 # .. E .. .. nop # E .. .. .. : U L U L nop # .. .. .. E subq $1, 1, $1 # .. .. E .. - addq $6, 8, $6 # .. E .. .. + addq $16, 8, $16 # .. E .. .. blt $3, $alignmod64 # U .. .. .. : U L U L $bigalign: /* * $0 is the number of bytes left * $1 is the number of quads left - * $6 is aligned 0mod64 + * $16 is aligned 0mod64 * we know that we'll be taking a minimum of one trip through * CWG Section 3.7.6: do not expect a sustained store rate of > 1/cycle * We are _not_ going to update $0 after every single store. That @@ -145,39 +131,39 @@ $bigalign: nop # E : nop # E : nop # E : - bis $6,$6,$3 # E : U L U L : Initial wh64 address is dest + bis $16,$16,$3 # E : U L U L : Initial wh64 address is dest /* This might actually help for the current trip... */ $do_wh64: wh64 ($3) # .. .. .. L1 : memory subsystem hint subq $1, 16, $4 # .. .. E .. : Forward calculation - repeat the loop? - EX( stq_u $31, 0($6) ) # .. L .. .. + EX( stq_u $31, 0($16) ) # .. L .. .. subq $0, 8, $0 # E .. .. .. : U L U L - addq $6, 128, $3 # E : Target address of wh64 - EX( stq_u $31, 8($6) ) # L : - EX( stq_u $31, 16($6) ) # L : + addq $16, 128, $3 # E : Target address of wh64 + EX( stq_u $31, 8($16) ) # L : + EX( stq_u $31, 16($16) ) # L : subq $0, 16, $0 # E : U L L U nop # E : - EX( stq_u $31, 24($6) ) # L : - EX( stq_u $31, 32($6) ) # L : + EX( stq_u $31, 24($16) ) # L : + EX( stq_u $31, 32($16) ) # L : subq $0, 168, $5 # E : U L L U : two trips through the loop left? /* 168 = 192 - 24, since we've already completed some stores */ subq $0, 16, $0 # E : - EX( stq_u $31, 40($6) ) # L : - EX( stq_u $31, 48($6) ) # L : - cmovlt $5, $6, $3 # E : U L L U : Latency 2, extra mapping cycle + EX( stq_u $31, 40($16) ) # L : + EX( stq_u $31, 48($16) ) # L : + cmovlt $5, $16, $3 # E : U L L U : Latency 2, extra mapping cycle subq $1, 8, $1 # E : subq $0, 16, $0 # E : - EX( stq_u $31, 56($6) ) # L : + EX( stq_u $31, 56($16) ) # L : nop # E : U L U L nop # E : subq $0, 8, $0 # E : - addq $6, 64, $6 # E : + addq $16, 64, $16 # E : bge $4, $do_wh64 # U : U L U L $trailquad: @@ -190,14 +176,14 @@ $trailquad: beq $1, $trailbytes # U .. .. .. : U L U L : Only 0..7 bytes to go $onequad: - EX( stq_u $31, 0($6) ) # .. .. .. L + EX( stq_u $31, 0($16) ) # .. .. .. L subq $1, 1, $1 # .. .. E .. subq $0, 8, $0 # .. E .. .. nop # E .. .. .. : U L U L nop # .. .. .. E nop # .. .. E .. - addq $6, 8, $6 # .. E .. .. + addq $16, 8, $16 # .. E .. .. bgt $1, $onequad # U .. .. .. : U L U L # We have an unknown number of bytes left to go. @@ -211,9 +197,9 @@ $trailbytes: # so we will use $0 as the loop counter # We know for a fact that $0 > 0 zero due to previous context $onebyte: - EX( stb $31, 0($6) ) # .. .. .. L + EX( stb $31, 0($16) ) # .. .. .. L subq $0, 1, $0 # .. .. E .. : - addq $6, 1, $6 # .. E .. .. : + addq $16, 1, $16 # .. E .. .. : bgt $0, $onebyte # U .. .. .. : U L U L $zerolength: @@ -221,6 +207,6 @@ $exception: # Destination for exception recovery(?) nop # .. .. .. E : nop # .. .. E .. : nop # .. E .. .. : - ret $31, ($28), 1 # L0 .. .. .. : L U L U - .end __do_clear_user - EXPORT_SYMBOL(__do_clear_user) + ret $31, ($26), 1 # L0 .. .. .. : L U L U + .end __clear_user + EXPORT_SYMBOL(__clear_user) diff --git a/arch/alpha/lib/ev6-copy_user.S b/arch/alpha/lib/ev6-copy_user.S index be720b518af9..35e6710d0700 100644 --- a/arch/alpha/lib/ev6-copy_user.S +++ b/arch/alpha/lib/ev6-copy_user.S @@ -12,21 +12,6 @@ * only _after_ a successful copy). There is also some rather minor * exception setup stuff.. * - * NOTE! This is not directly C-callable, because the calling semantics are - * different: - * - * Inputs: - * length in $0 - * destination address in $6 - * source address in $7 - * return address in $28 - * - * Outputs: - * bytes left to copy in $0 - * - * Clobbers: - * $1,$2,$3,$4,$5,$6,$7 - * * Much of the information about 21264 scheduling/coding comes from: * Compiler Writer's Guide for the Alpha 21264 * abbreviated as 'CWG' in other comments here @@ -60,10 +45,11 @@ # Pipeline info: Slotting & Comments __copy_user: .prologue 0 - subq $0, 32, $1 # .. E .. .. : Is this going to be a small copy? + andq $18, $18, $0 + subq $18, 32, $1 # .. E .. .. : Is this going to be a small copy? beq $0, $zerolength # U .. .. .. : U L U L - and $6,7,$3 # .. .. .. E : is leading dest misalignment + and $16,7,$3 # .. .. .. E : is leading dest misalignment ble $1, $onebyteloop # .. .. U .. : 1st branch : small amount of data beq $3, $destaligned # .. U .. .. : 2nd (one cycle fetcher stall) subq $3, 8, $3 # E .. .. .. : L U U L : trip counter @@ -73,17 +59,17 @@ __copy_user: * We know we have at least one trip through this loop */ $aligndest: - EXI( ldbu $1,0($7) ) # .. .. .. L : Keep loads separate from stores - addq $6,1,$6 # .. .. E .. : Section 3.8 in the CWG + EXI( ldbu $1,0($17) ) # .. .. .. L : Keep loads separate from stores + addq $16,1,$16 # .. .. E .. : Section 3.8 in the CWG addq $3,1,$3 # .. E .. .. : nop # E .. .. .. : U L U L /* - * the -1 is to compensate for the inc($6) done in a previous quadpack + * the -1 is to compensate for the inc($16) done in a previous quadpack * which allows us zero dependencies within either quadpack in the loop */ - EXO( stb $1,-1($6) ) # .. .. .. L : - addq $7,1,$7 # .. .. E .. : Section 3.8 in the CWG + EXO( stb $1,-1($16) ) # .. .. .. L : + addq $17,1,$17 # .. .. E .. : Section 3.8 in the CWG subq $0,1,$0 # .. E .. .. : bne $3, $aligndest # U .. .. .. : U L U L @@ -92,29 +78,29 @@ $aligndest: * If we arrived via branch, we have a minimum of 32 bytes */ $destaligned: - and $7,7,$1 # .. .. .. E : Check _current_ source alignment + and $17,7,$1 # .. .. .. E : Check _current_ source alignment bic $0,7,$4 # .. .. E .. : number bytes as a quadword loop - EXI( ldq_u $3,0($7) ) # .. L .. .. : Forward fetch for fallthrough code + EXI( ldq_u $3,0($17) ) # .. L .. .. : Forward fetch for fallthrough code beq $1,$quadaligned # U .. .. .. : U L U L /* - * In the worst case, we've just executed an ldq_u here from 0($7) + * In the worst case, we've just executed an ldq_u here from 0($17) * and we'll repeat it once if we take the branch */ /* Misaligned quadword loop - not unrolled. Leave it that way. */ $misquad: - EXI( ldq_u $2,8($7) ) # .. .. .. L : + EXI( ldq_u $2,8($17) ) # .. .. .. L : subq $4,8,$4 # .. .. E .. : - extql $3,$7,$3 # .. U .. .. : - extqh $2,$7,$1 # U .. .. .. : U U L L + extql $3,$17,$3 # .. U .. .. : + extqh $2,$17,$1 # U .. .. .. : U U L L bis $3,$1,$1 # .. .. .. E : - EXO( stq $1,0($6) ) # .. .. L .. : - addq $7,8,$7 # .. E .. .. : + EXO( stq $1,0($16) ) # .. .. L .. : + addq $17,8,$17 # .. E .. .. : subq $0,8,$0 # E .. .. .. : U L L U - addq $6,8,$6 # .. .. .. E : + addq $16,8,$16 # .. .. .. E : bis $2,$2,$3 # .. .. E .. : nop # .. E .. .. : bne $4,$misquad # U .. .. .. : U L U L @@ -125,8 +111,8 @@ $misquad: beq $0,$zerolength # U .. .. .. : U L U L /* We know we have at least one trip through the byte loop */ - EXI ( ldbu $2,0($7) ) # .. .. .. L : No loads in the same quad - addq $6,1,$6 # .. .. E .. : as the store (Section 3.8 in CWG) + EXI ( ldbu $2,0($17) ) # .. .. .. L : No loads in the same quad + addq $16,1,$16 # .. .. E .. : as the store (Section 3.8 in CWG) nop # .. E .. .. : br $31, $dirtyentry # L0 .. .. .. : L U U L /* Do the trailing byte loop load, then hop into the store part of the loop */ @@ -136,8 +122,8 @@ $misquad: * Based upon the usage context, it's worth the effort to unroll this loop * $0 - number of bytes to be moved * $4 - number of bytes to move as quadwords - * $6 is current destination address - * $7 is current source address + * $16 is current destination address + * $17 is current source address */ $quadaligned: subq $4, 32, $2 # .. .. .. E : do not unroll for small stuff @@ -155,29 +141,29 @@ $quadaligned: * instruction memory hint instruction). */ $unroll4: - EXI( ldq $1,0($7) ) # .. .. .. L - EXI( ldq $2,8($7) ) # .. .. L .. + EXI( ldq $1,0($17) ) # .. .. .. L + EXI( ldq $2,8($17) ) # .. .. L .. subq $4,32,$4 # .. E .. .. nop # E .. .. .. : U U L L - addq $7,16,$7 # .. .. .. E - EXO( stq $1,0($6) ) # .. .. L .. - EXO( stq $2,8($6) ) # .. L .. .. + addq $17,16,$17 # .. .. .. E + EXO( stq $1,0($16) ) # .. .. L .. + EXO( stq $2,8($16) ) # .. L .. .. subq $0,16,$0 # E .. .. .. : U L L U - addq $6,16,$6 # .. .. .. E - EXI( ldq $1,0($7) ) # .. .. L .. - EXI( ldq $2,8($7) ) # .. L .. .. + addq $16,16,$16 # .. .. .. E + EXI( ldq $1,0($17) ) # .. .. L .. + EXI( ldq $2,8($17) ) # .. L .. .. subq $4, 32, $3 # E .. .. .. : U U L L : is there enough for another trip? - EXO( stq $1,0($6) ) # .. .. .. L - EXO( stq $2,8($6) ) # .. .. L .. + EXO( stq $1,0($16) ) # .. .. .. L + EXO( stq $2,8($16) ) # .. .. L .. subq $0,16,$0 # .. E .. .. - addq $7,16,$7 # E .. .. .. : U L L U + addq $17,16,$17 # E .. .. .. : U L L U nop # .. .. .. E nop # .. .. E .. - addq $6,16,$6 # .. E .. .. + addq $16,16,$16 # .. E .. .. bgt $3,$unroll4 # U .. .. .. : U L U L nop @@ -186,14 +172,14 @@ $unroll4: beq $4, $noquads $onequad: - EXI( ldq $1,0($7) ) + EXI( ldq $1,0($17) ) subq $4,8,$4 - addq $7,8,$7 + addq $17,8,$17 nop - EXO( stq $1,0($6) ) + EXO( stq $1,0($16) ) subq $0,8,$0 - addq $6,8,$6 + addq $16,8,$16 bne $4,$onequad $noquads: @@ -207,23 +193,23 @@ $noquads: * There's no point in doing a lot of complex alignment calculations to try to * to quadword stuff for a small amount of data. * $0 - remaining number of bytes left to copy - * $6 - current dest addr - * $7 - current source addr + * $16 - current dest addr + * $17 - current source addr */ $onebyteloop: - EXI ( ldbu $2,0($7) ) # .. .. .. L : No loads in the same quad - addq $6,1,$6 # .. .. E .. : as the store (Section 3.8 in CWG) + EXI ( ldbu $2,0($17) ) # .. .. .. L : No loads in the same quad + addq $16,1,$16 # .. .. E .. : as the store (Section 3.8 in CWG) nop # .. E .. .. : nop # E .. .. .. : U L U L $dirtyentry: /* - * the -1 is to compensate for the inc($6) done in a previous quadpack + * the -1 is to compensate for the inc($16) done in a previous quadpack * which allows us zero dependencies within either quadpack in the loop */ - EXO ( stb $2,-1($6) ) # .. .. .. L : - addq $7,1,$7 # .. .. E .. : quadpack as the load + EXO ( stb $2,-1($16) ) # .. .. .. L : + addq $17,1,$17 # .. .. E .. : quadpack as the load subq $0,1,$0 # .. E .. .. : change count _after_ copy bgt $0,$onebyteloop # U .. .. .. : U L U L @@ -233,7 +219,7 @@ $exitout: # Destination for exception recovery(?) nop # .. .. .. E nop # .. .. E .. nop # .. E .. .. - ret $31,($28),1 # L0 .. .. .. : L U L U + ret $31,($26),1 # L0 .. .. .. : L U L U .end __copy_user EXPORT_SYMBOL(__copy_user) From b5478c1b67bcd52d3b7fb276090b985b4b38a7ea Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 24 Dec 2016 21:18:40 -0500 Subject: [PATCH 10/96] alpha: add asm/extable.h Signed-off-by: Al Viro --- arch/alpha/include/asm/extable.h | 55 ++++++++++++++++++++++++++++++++ arch/alpha/include/asm/uaccess.h | 52 +----------------------------- 2 files changed, 56 insertions(+), 51 deletions(-) create mode 100644 arch/alpha/include/asm/extable.h diff --git a/arch/alpha/include/asm/extable.h b/arch/alpha/include/asm/extable.h new file mode 100644 index 000000000000..048e209e524c --- /dev/null +++ b/arch/alpha/include/asm/extable.h @@ -0,0 +1,55 @@ +#ifndef _ASM_EXTABLE_H +#define _ASM_EXTABLE_H + +/* + * About the exception table: + * + * - insn is a 32-bit pc-relative offset from the faulting insn. + * - nextinsn is a 16-bit offset off of the faulting instruction + * (not off of the *next* instruction as branches are). + * - errreg is the register in which to place -EFAULT. + * - valreg is the final target register for the load sequence + * and will be zeroed. + * + * Either errreg or valreg may be $31, in which case nothing happens. + * + * The exception fixup information "just so happens" to be arranged + * as in a MEM format instruction. This lets us emit our three + * values like so: + * + * lda valreg, nextinsn(errreg) + * + */ + +struct exception_table_entry +{ + signed int insn; + union exception_fixup { + unsigned unit; + struct { + signed int nextinsn : 16; + unsigned int errreg : 5; + unsigned int valreg : 5; + } bits; + } fixup; +}; + +/* Returns the new pc */ +#define fixup_exception(map_reg, _fixup, pc) \ +({ \ + if ((_fixup)->fixup.bits.valreg != 31) \ + map_reg((_fixup)->fixup.bits.valreg) = 0; \ + if ((_fixup)->fixup.bits.errreg != 31) \ + map_reg((_fixup)->fixup.bits.errreg) = -EFAULT; \ + (pc) + (_fixup)->fixup.bits.nextinsn; \ +}) + +#define ARCH_HAS_RELATIVE_EXTABLE + +#define swap_ex_entry_fixup(a, b, tmp, delta) \ + do { \ + (a)->fixup.unit = (b)->fixup.unit; \ + (b)->fixup.unit = (tmp).fixup.unit; \ + } while (0) + +#endif diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index 0135199c9dda..b7270a64fe1c 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -386,56 +386,6 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count); extern __must_check long strlen_user(const char __user *str); extern __must_check long strnlen_user(const char __user *str, long n); -/* - * About the exception table: - * - * - insn is a 32-bit pc-relative offset from the faulting insn. - * - nextinsn is a 16-bit offset off of the faulting instruction - * (not off of the *next* instruction as branches are). - * - errreg is the register in which to place -EFAULT. - * - valreg is the final target register for the load sequence - * and will be zeroed. - * - * Either errreg or valreg may be $31, in which case nothing happens. - * - * The exception fixup information "just so happens" to be arranged - * as in a MEM format instruction. This lets us emit our three - * values like so: - * - * lda valreg, nextinsn(errreg) - * - */ - -struct exception_table_entry -{ - signed int insn; - union exception_fixup { - unsigned unit; - struct { - signed int nextinsn : 16; - unsigned int errreg : 5; - unsigned int valreg : 5; - } bits; - } fixup; -}; - -/* Returns the new pc */ -#define fixup_exception(map_reg, _fixup, pc) \ -({ \ - if ((_fixup)->fixup.bits.valreg != 31) \ - map_reg((_fixup)->fixup.bits.valreg) = 0; \ - if ((_fixup)->fixup.bits.errreg != 31) \ - map_reg((_fixup)->fixup.bits.errreg) = -EFAULT; \ - (pc) + (_fixup)->fixup.bits.nextinsn; \ -}) - -#define ARCH_HAS_RELATIVE_EXTABLE - -#define swap_ex_entry_fixup(a, b, tmp, delta) \ - do { \ - (a)->fixup.unit = (b)->fixup.unit; \ - (b)->fixup.unit = (tmp).fixup.unit; \ - } while (0) - +#include #endif /* __ALPHA_UACCESS_H */ From d78d834bed81bf838281270c7ae3790c88fc95aa Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 02:04:09 -0500 Subject: [PATCH 11/96] alpha: get rid of 'segment' argument of __{get,put}_user_check() always equal to get_fs() Signed-off-by: Al Viro --- arch/alpha/include/asm/uaccess.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index b7270a64fe1c..30fa8843e177 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -54,9 +54,9 @@ * (b) require any knowledge of processes at this stage */ #define put_user(x, ptr) \ - __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), get_fs()) + __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) #define get_user(x, ptr) \ - __get_user_check((x), (ptr), sizeof(*(ptr)), get_fs()) + __get_user_check((x), (ptr), sizeof(*(ptr))) /* * The "__xxx" versions do not do address space checking, useful when @@ -93,12 +93,12 @@ extern void __get_user_unknown(void); __gu_err; \ }) -#define __get_user_check(x, ptr, size, segment) \ +#define __get_user_check(x, ptr, size) \ ({ \ long __gu_err = -EFAULT; \ unsigned long __gu_val = 0; \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - if (__access_ok((unsigned long)__gu_addr, size, segment)) { \ + if (__access_ok((unsigned long)__gu_addr, size, get_fs())) { \ __gu_err = 0; \ switch (size) { \ case 1: __get_user_8(__gu_addr); break; \ @@ -208,11 +208,11 @@ extern void __put_user_unknown(void); __pu_err; \ }) -#define __put_user_check(x, ptr, size, segment) \ +#define __put_user_check(x, ptr, size) \ ({ \ long __pu_err = -EFAULT; \ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - if (__access_ok((unsigned long)__pu_addr, size, segment)) { \ + if (__access_ok((unsigned long)__pu_addr, size, get_fs())) { \ __pu_err = 0; \ switch (size) { \ case 1: __put_user_8(x, __pu_addr); break; \ From f14d6b4f52616465fead8c4b6508a6687032660d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 02:23:46 -0500 Subject: [PATCH 12/96] alpha: don't bother with __access_ok() in traps.c we want to check that address is below TASK_SIZE; sure, __access_ok(addr, 0, USER_DS) will do that, but it's more straightforward to just spell it out and that way we can get rid of the damn 'segment' argument of __access_ok(). Signed-off-by: Al Viro --- arch/alpha/kernel/traps.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index b137390e87e7..b724ad88f7e8 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -802,7 +802,7 @@ do_entUnaUser(void __user * va, unsigned long opcode, /* Don't bother reading ds in the access check since we already know that this came from the user. Also rely on the fact that the page at TASK_SIZE is unmapped and so can't be touched anyway. */ - if (!__access_ok((unsigned long)va, 0, USER_DS)) + if ((unsigned long)va >= TASK_SIZE) goto give_sigsegv; ++unaligned[1].count; @@ -1047,7 +1047,7 @@ do_entUnaUser(void __user * va, unsigned long opcode, /* We need to replicate some of the logic in mm/fault.c, since we don't have access to the fault code in the exception handling return path. */ - if (!__access_ok((unsigned long)va, 0, USER_DS)) + if ((unsigned long)va >= TASK_SIZE) info.si_code = SEGV_ACCERR; else { struct mm_struct *mm = current->mm; From c9df6025a6ecc5b4d243d1469a4697e03c371a88 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 02:26:32 -0500 Subject: [PATCH 13/96] alpha: kill the 'segment' argument of __access_ok() always equal to get_fs(). Signed-off-by: Al Viro --- arch/alpha/include/asm/uaccess.h | 82 ++++++++++++++++---------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index 30fa8843e177..54c1d19ef06d 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -32,13 +32,13 @@ * - AND "addr+size" doesn't have any high-bits set * - OR we are in kernel mode. */ -#define __access_ok(addr, size, segment) \ - (((segment).seg & (addr | size | (addr+size))) == 0) +#define __access_ok(addr, size) \ + ((get_fs().seg & (addr | size | (addr+size))) == 0) -#define access_ok(type, addr, size) \ -({ \ - __chk_user_ptr(addr); \ - __access_ok(((unsigned long)(addr)), (size), get_fs()); \ +#define access_ok(type, addr, size) \ +({ \ + __chk_user_ptr(addr); \ + __access_ok(((unsigned long)(addr)), (size)); \ }) /* @@ -93,23 +93,23 @@ extern void __get_user_unknown(void); __gu_err; \ }) -#define __get_user_check(x, ptr, size) \ -({ \ - long __gu_err = -EFAULT; \ - unsigned long __gu_val = 0; \ - const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - if (__access_ok((unsigned long)__gu_addr, size, get_fs())) { \ - __gu_err = 0; \ - switch (size) { \ - case 1: __get_user_8(__gu_addr); break; \ - case 2: __get_user_16(__gu_addr); break; \ - case 4: __get_user_32(__gu_addr); break; \ - case 8: __get_user_64(__gu_addr); break; \ - default: __get_user_unknown(); break; \ - } \ - } \ - (x) = (__force __typeof__(*(ptr))) __gu_val; \ - __gu_err; \ +#define __get_user_check(x, ptr, size) \ +({ \ + long __gu_err = -EFAULT; \ + unsigned long __gu_val = 0; \ + const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ + if (__access_ok((unsigned long)__gu_addr, size)) { \ + __gu_err = 0; \ + switch (size) { \ + case 1: __get_user_8(__gu_addr); break; \ + case 2: __get_user_16(__gu_addr); break; \ + case 4: __get_user_32(__gu_addr); break; \ + case 8: __get_user_64(__gu_addr); break; \ + default: __get_user_unknown(); break; \ + } \ + } \ + (x) = (__force __typeof__(*(ptr))) __gu_val; \ + __gu_err; \ }) struct __large_struct { unsigned long buf[100]; }; @@ -208,21 +208,21 @@ extern void __put_user_unknown(void); __pu_err; \ }) -#define __put_user_check(x, ptr, size) \ -({ \ - long __pu_err = -EFAULT; \ - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - if (__access_ok((unsigned long)__pu_addr, size, get_fs())) { \ - __pu_err = 0; \ - switch (size) { \ - case 1: __put_user_8(x, __pu_addr); break; \ - case 2: __put_user_16(x, __pu_addr); break; \ - case 4: __put_user_32(x, __pu_addr); break; \ - case 8: __put_user_64(x, __pu_addr); break; \ - default: __put_user_unknown(); break; \ - } \ - } \ - __pu_err; \ +#define __put_user_check(x, ptr, size) \ +({ \ + long __pu_err = -EFAULT; \ + __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ + if (__access_ok((unsigned long)__pu_addr, size)) { \ + __pu_err = 0; \ + switch (size) { \ + case 1: __put_user_8(x, __pu_addr); break; \ + case 2: __put_user_16(x, __pu_addr); break; \ + case 4: __put_user_32(x, __pu_addr); break; \ + case 8: __put_user_64(x, __pu_addr); break; \ + default: __put_user_unknown(); break; \ + } \ + } \ + __pu_err; \ }) /* @@ -353,7 +353,7 @@ extern long __copy_user(void *to, const void *from, long len); extern inline long copy_to_user(void __user *to, const void *from, long n) { - if (likely(__access_ok((unsigned long)to, n, get_fs()))) + if (likely(__access_ok((unsigned long)to, n))) n = __copy_user((__force void *)to, from, n); return n; } @@ -362,7 +362,7 @@ extern inline long copy_from_user(void *to, const void __user *from, long n) { long res = n; - if (likely(__access_ok((unsigned long)from, n, get_fs()))) + if (likely(__access_ok((unsigned long)from, n))) res = __copy_from_user_inatomic(to, from, n); if (unlikely(res)) memset(to + (n - res), 0, res); @@ -374,7 +374,7 @@ extern long __clear_user(void __user *to, long len); extern inline long clear_user(void __user *to, long len) { - if (__access_ok((unsigned long)to, len, get_fs())) + if (__access_ok((unsigned long)to, len)) len = __clear_user(to, len); return len; } From ca282f69738161ca1ac2e13b2c1b283be82ffb4b Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 7 Mar 2017 04:08:46 -0500 Subject: [PATCH 14/96] alpha: add a helper for emitting exception table entries Signed-off-by: Al Viro --- arch/alpha/include/asm/futex.h | 16 +--- arch/alpha/include/asm/uaccess.h | 80 +++++----------- arch/alpha/kernel/traps.c | 148 ++++++++--------------------- arch/alpha/lib/csum_partial_copy.c | 10 +- 4 files changed, 68 insertions(+), 186 deletions(-) diff --git a/arch/alpha/include/asm/futex.h b/arch/alpha/include/asm/futex.h index f939794363ac..fb01dfb760c2 100644 --- a/arch/alpha/include/asm/futex.h +++ b/arch/alpha/include/asm/futex.h @@ -19,12 +19,8 @@ "3: .subsection 2\n" \ "4: br 1b\n" \ " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .long 1b-.\n" \ - " lda $31,3b-1b(%1)\n" \ - " .long 2b-.\n" \ - " lda $31,3b-2b(%1)\n" \ - " .previous\n" \ + EXC(1b,3b,%1,$31) \ + EXC(2b,3b,%1,$31) \ : "=&r" (oldval), "=&r"(ret) \ : "r" (uaddr), "r"(oparg) \ : "memory") @@ -101,12 +97,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, "3: .subsection 2\n" "4: br 1b\n" " .previous\n" - " .section __ex_table,\"a\"\n" - " .long 1b-.\n" - " lda $31,3b-1b(%0)\n" - " .long 2b-.\n" - " lda $31,3b-2b(%0)\n" - " .previous\n" + EXC(1b,3b,%0,$31) + EXC(2b,3b,%0,$31) : "+r"(ret), "=&r"(prev), "=&r"(cmp) : "r"(uaddr), "r"((long)(int)oldval), "r"(newval) : "memory"); diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index 54c1d19ef06d..9557c12823a9 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -74,6 +74,11 @@ * more extensive comments with fixup_inline_exception below for * more information. */ +#define EXC(label,cont,res,err) \ + ".section __ex_table,\"a\"\n" \ + " .long "#label"-.\n" \ + " lda "#res","#cont"-"#label"("#err")\n" \ + ".previous\n" extern void __get_user_unknown(void); @@ -118,20 +123,14 @@ struct __large_struct { unsigned long buf[100]; }; #define __get_user_64(addr) \ __asm__("1: ldq %0,%2\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda %0, 2b-1b(%1)\n" \ - ".previous" \ + EXC(1b,2b,%0,%1) \ : "=r"(__gu_val), "=r"(__gu_err) \ : "m"(__m(addr)), "1"(__gu_err)) #define __get_user_32(addr) \ __asm__("1: ldl %0,%2\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda %0, 2b-1b(%1)\n" \ - ".previous" \ + EXC(1b,2b,%0,%1) \ : "=r"(__gu_val), "=r"(__gu_err) \ : "m"(__m(addr)), "1"(__gu_err)) @@ -141,20 +140,14 @@ struct __large_struct { unsigned long buf[100]; }; #define __get_user_16(addr) \ __asm__("1: ldwu %0,%2\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda %0, 2b-1b(%1)\n" \ - ".previous" \ + EXC(1b,2b,%0,%1) \ : "=r"(__gu_val), "=r"(__gu_err) \ : "m"(__m(addr)), "1"(__gu_err)) #define __get_user_8(addr) \ __asm__("1: ldbu %0,%2\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda %0, 2b-1b(%1)\n" \ - ".previous" \ + EXC(1b,2b,%0,%1) \ : "=r"(__gu_val), "=r"(__gu_err) \ : "m"(__m(addr)), "1"(__gu_err)) #else @@ -170,12 +163,8 @@ struct __large_struct { unsigned long buf[100]; }; " extwh %1,%3,%1\n" \ " or %0,%1,%0\n" \ "3:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda %0, 3b-1b(%2)\n" \ - " .long 2b - .\n" \ - " lda %0, 3b-2b(%2)\n" \ - ".previous" \ + EXC(1b,3b,%0,%2) \ + EXC(2b,3b,%0,%2) \ : "=&r"(__gu_val), "=&r"(__gu_tmp), "=r"(__gu_err) \ : "r"(addr), "2"(__gu_err)); \ } @@ -184,10 +173,7 @@ struct __large_struct { unsigned long buf[100]; }; __asm__("1: ldq_u %0,0(%2)\n" \ " extbl %0,%2,%0\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda %0, 2b-1b(%1)\n" \ - ".previous" \ + EXC(1b,2b,%0,%1) \ : "=&r"(__gu_val), "=r"(__gu_err) \ : "r"(addr), "1"(__gu_err)) #endif @@ -233,20 +219,14 @@ extern void __put_user_unknown(void); #define __put_user_64(x, addr) \ __asm__ __volatile__("1: stq %r2,%1\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda $31,2b-1b(%0)\n" \ - ".previous" \ + EXC(1b,2b,$31,%0) \ : "=r"(__pu_err) \ : "m" (__m(addr)), "rJ" (x), "0"(__pu_err)) #define __put_user_32(x, addr) \ __asm__ __volatile__("1: stl %r2,%1\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda $31,2b-1b(%0)\n" \ - ".previous" \ + EXC(1b,2b,$31,%0) \ : "=r"(__pu_err) \ : "m"(__m(addr)), "rJ"(x), "0"(__pu_err)) @@ -256,20 +236,14 @@ __asm__ __volatile__("1: stl %r2,%1\n" \ #define __put_user_16(x, addr) \ __asm__ __volatile__("1: stw %r2,%1\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda $31,2b-1b(%0)\n" \ - ".previous" \ + EXC(1b,2b,$31,%0) \ : "=r"(__pu_err) \ : "m"(__m(addr)), "rJ"(x), "0"(__pu_err)) #define __put_user_8(x, addr) \ __asm__ __volatile__("1: stb %r2,%1\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda $31,2b-1b(%0)\n" \ - ".previous" \ + EXC(1b,2b,$31,%0) \ : "=r"(__pu_err) \ : "m"(__m(addr)), "rJ"(x), "0"(__pu_err)) #else @@ -291,16 +265,10 @@ __asm__ __volatile__("1: stb %r2,%1\n" \ "3: stq_u %2,1(%5)\n" \ "4: stq_u %1,0(%5)\n" \ "5:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda $31, 5b-1b(%0)\n" \ - " .long 2b - .\n" \ - " lda $31, 5b-2b(%0)\n" \ - " .long 3b - .\n" \ - " lda $31, 5b-3b(%0)\n" \ - " .long 4b - .\n" \ - " lda $31, 5b-4b(%0)\n" \ - ".previous" \ + EXC(1b,5b,$31,%0) \ + EXC(2b,5b,$31,%0) \ + EXC(3b,5b,$31,%0) \ + EXC(4b,5b,$31,%0) \ : "=r"(__pu_err), "=&r"(__pu_tmp1), \ "=&r"(__pu_tmp2), "=&r"(__pu_tmp3), \ "=&r"(__pu_tmp4) \ @@ -317,12 +285,8 @@ __asm__ __volatile__("1: stb %r2,%1\n" \ " or %1,%2,%1\n" \ "2: stq_u %1,0(%4)\n" \ "3:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda $31, 3b-1b(%0)\n" \ - " .long 2b - .\n" \ - " lda $31, 3b-2b(%0)\n" \ - ".previous" \ + EXC(1b,3b,$31,%0) \ + EXC(2b,3b,$31,%0) \ : "=r"(__pu_err), \ "=&r"(__pu_tmp1), "=&r"(__pu_tmp2) \ : "r"((unsigned long)(x)), "r"(addr), "0"(__pu_err)); \ diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index b724ad88f7e8..65bb102d985b 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -482,12 +482,8 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg, " extwl %1,%3,%1\n" " extwh %2,%3,%2\n" "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %1,3b-1b(%0)\n" - " .long 2b - .\n" - " lda %2,3b-2b(%0)\n" - ".previous" + EXC(1b,3b,%1,%0) + EXC(2b,3b,%2,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) : "r"(va), "0"(0)); if (error) @@ -502,12 +498,8 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg, " extll %1,%3,%1\n" " extlh %2,%3,%2\n" "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %1,3b-1b(%0)\n" - " .long 2b - .\n" - " lda %2,3b-2b(%0)\n" - ".previous" + EXC(1b,3b,%1,%0) + EXC(2b,3b,%2,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) : "r"(va), "0"(0)); if (error) @@ -522,12 +514,8 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg, " extql %1,%3,%1\n" " extqh %2,%3,%2\n" "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %1,3b-1b(%0)\n" - " .long 2b - .\n" - " lda %2,3b-2b(%0)\n" - ".previous" + EXC(1b,3b,%1,%0) + EXC(2b,3b,%2,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) : "r"(va), "0"(0)); if (error) @@ -551,16 +539,10 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg, "3: stq_u %2,1(%5)\n" "4: stq_u %1,0(%5)\n" "5:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %2,5b-1b(%0)\n" - " .long 2b - .\n" - " lda %1,5b-2b(%0)\n" - " .long 3b - .\n" - " lda $31,5b-3b(%0)\n" - " .long 4b - .\n" - " lda $31,5b-4b(%0)\n" - ".previous" + EXC(1b,5b,%2,%0) + EXC(2b,5b,%1,%0) + EXC(3b,5b,$31,%0) + EXC(4b,5b,$31,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) : "r"(va), "r"(una_reg(reg)), "0"(0)); @@ -581,16 +563,10 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg, "3: stq_u %2,3(%5)\n" "4: stq_u %1,0(%5)\n" "5:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %2,5b-1b(%0)\n" - " .long 2b - .\n" - " lda %1,5b-2b(%0)\n" - " .long 3b - .\n" - " lda $31,5b-3b(%0)\n" - " .long 4b - .\n" - " lda $31,5b-4b(%0)\n" - ".previous" + EXC(1b,5b,%2,%0) + EXC(2b,5b,%1,%0) + EXC(3b,5b,$31,%0) + EXC(4b,5b,$31,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) : "r"(va), "r"(una_reg(reg)), "0"(0)); @@ -611,16 +587,10 @@ do_entUna(void * va, unsigned long opcode, unsigned long reg, "3: stq_u %2,7(%5)\n" "4: stq_u %1,0(%5)\n" "5:\n" - ".section __ex_table,\"a\"\n\t" - " .long 1b - .\n" - " lda %2,5b-1b(%0)\n" - " .long 2b - .\n" - " lda %1,5b-2b(%0)\n" - " .long 3b - .\n" - " lda $31,5b-3b(%0)\n" - " .long 4b - .\n" - " lda $31,5b-4b(%0)\n" - ".previous" + EXC(1b,5b,%2,%0) + EXC(2b,5b,%1,%0) + EXC(3b,5b,$31,%0) + EXC(4b,5b,$31,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) : "r"(va), "r"(una_reg(reg)), "0"(0)); @@ -835,12 +805,8 @@ do_entUnaUser(void __user * va, unsigned long opcode, " extwl %1,%3,%1\n" " extwh %2,%3,%2\n" "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %1,3b-1b(%0)\n" - " .long 2b - .\n" - " lda %2,3b-2b(%0)\n" - ".previous" + EXC(1b,3b,%1,%0) + EXC(2b,3b,%2,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) : "r"(va), "0"(0)); if (error) @@ -855,12 +821,8 @@ do_entUnaUser(void __user * va, unsigned long opcode, " extll %1,%3,%1\n" " extlh %2,%3,%2\n" "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %1,3b-1b(%0)\n" - " .long 2b - .\n" - " lda %2,3b-2b(%0)\n" - ".previous" + EXC(1b,3b,%1,%0) + EXC(2b,3b,%2,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) : "r"(va), "0"(0)); if (error) @@ -875,12 +837,8 @@ do_entUnaUser(void __user * va, unsigned long opcode, " extql %1,%3,%1\n" " extqh %2,%3,%2\n" "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %1,3b-1b(%0)\n" - " .long 2b - .\n" - " lda %2,3b-2b(%0)\n" - ".previous" + EXC(1b,3b,%1,%0) + EXC(2b,3b,%2,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) : "r"(va), "0"(0)); if (error) @@ -895,12 +853,8 @@ do_entUnaUser(void __user * va, unsigned long opcode, " extll %1,%3,%1\n" " extlh %2,%3,%2\n" "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %1,3b-1b(%0)\n" - " .long 2b - .\n" - " lda %2,3b-2b(%0)\n" - ".previous" + EXC(1b,3b,%1,%0) + EXC(2b,3b,%2,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) : "r"(va), "0"(0)); if (error) @@ -915,12 +869,8 @@ do_entUnaUser(void __user * va, unsigned long opcode, " extql %1,%3,%1\n" " extqh %2,%3,%2\n" "3:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %1,3b-1b(%0)\n" - " .long 2b - .\n" - " lda %2,3b-2b(%0)\n" - ".previous" + EXC(1b,3b,%1,%0) + EXC(2b,3b,%2,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2) : "r"(va), "0"(0)); if (error) @@ -944,16 +894,10 @@ do_entUnaUser(void __user * va, unsigned long opcode, "3: stq_u %2,1(%5)\n" "4: stq_u %1,0(%5)\n" "5:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %2,5b-1b(%0)\n" - " .long 2b - .\n" - " lda %1,5b-2b(%0)\n" - " .long 3b - .\n" - " lda $31,5b-3b(%0)\n" - " .long 4b - .\n" - " lda $31,5b-4b(%0)\n" - ".previous" + EXC(1b,5b,%2,%0) + EXC(2b,5b,%1,%0) + EXC(3b,5b,$31,%0) + EXC(4b,5b,$31,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) : "r"(va), "r"(*reg_addr), "0"(0)); @@ -978,16 +922,10 @@ do_entUnaUser(void __user * va, unsigned long opcode, "3: stq_u %2,3(%5)\n" "4: stq_u %1,0(%5)\n" "5:\n" - ".section __ex_table,\"a\"\n" - " .long 1b - .\n" - " lda %2,5b-1b(%0)\n" - " .long 2b - .\n" - " lda %1,5b-2b(%0)\n" - " .long 3b - .\n" - " lda $31,5b-3b(%0)\n" - " .long 4b - .\n" - " lda $31,5b-4b(%0)\n" - ".previous" + EXC(1b,5b,%2,%0) + EXC(2b,5b,%1,%0) + EXC(3b,5b,$31,%0) + EXC(4b,5b,$31,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) : "r"(va), "r"(*reg_addr), "0"(0)); @@ -1012,16 +950,10 @@ do_entUnaUser(void __user * va, unsigned long opcode, "3: stq_u %2,7(%5)\n" "4: stq_u %1,0(%5)\n" "5:\n" - ".section __ex_table,\"a\"\n\t" - " .long 1b - .\n" - " lda %2,5b-1b(%0)\n" - " .long 2b - .\n" - " lda %1,5b-2b(%0)\n" - " .long 3b - .\n" - " lda $31,5b-3b(%0)\n" - " .long 4b - .\n" - " lda $31,5b-4b(%0)\n" - ".previous" + EXC(1b,5b,%2,%0) + EXC(2b,5b,%1,%0) + EXC(3b,5b,$31,%0) + EXC(4b,5b,$31,%0) : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) : "r"(va), "r"(*reg_addr), "0"(0)); diff --git a/arch/alpha/lib/csum_partial_copy.c b/arch/alpha/lib/csum_partial_copy.c index 5dfb7975895f..ab42afba1720 100644 --- a/arch/alpha/lib/csum_partial_copy.c +++ b/arch/alpha/lib/csum_partial_copy.c @@ -45,10 +45,7 @@ __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) __asm__ __volatile__( \ "1: ldq_u %0,%2\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - .\n" \ - " lda %0,2b-1b(%1)\n" \ - ".previous" \ + EXC(1b,2b,%0,%1) \ : "=r"(x), "=r"(__guu_err) \ : "m"(__m(ptr)), "1"(0)); \ __guu_err; \ @@ -60,10 +57,7 @@ __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y)) __asm__ __volatile__( \ "1: stq_u %2,%1\n" \ "2:\n" \ - ".section __ex_table,\"a\"\n" \ - " .long 1b - ." \ - " lda $31,2b-1b(%0)\n" \ - ".previous" \ + EXC(1b,2b,$31,%0) \ : "=r"(__puu_err) \ : "m"(__m(addr)), "rJ"(x), "0"(0)); \ __puu_err; \ From ec022681a4201006a5bcb1a01401591a326b659c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 08:13:02 -0400 Subject: [PATCH 15/96] alpha: switch to RAW_COPY_USER copy_{to,from}_user() uninlined. We can go back to inlined variants, if we want to... Signed-off-by: Al Viro --- arch/alpha/Kconfig | 1 + arch/alpha/include/asm/uaccess.h | 33 ++++++-------------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 0e49d39ea74a..1be5f61dc630 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -26,6 +26,7 @@ config ALPHA select ODD_RT_SIGACTION select OLD_SIGSUSPEND select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67 + select ARCH_HAS_RAW_COPY_USER help The Alpha is a 64-bit general-purpose processor designed and marketed by the Digital Equipment Corporation of blessed memory, diff --git a/arch/alpha/include/asm/uaccess.h b/arch/alpha/include/asm/uaccess.h index 9557c12823a9..7b82dc9a8556 100644 --- a/arch/alpha/include/asm/uaccess.h +++ b/arch/alpha/include/asm/uaccess.h @@ -300,37 +300,16 @@ __asm__ __volatile__("1: stb %r2,%1\n" \ extern long __copy_user(void *to, const void *from, long len); -#define __copy_to_user(to, from, n) \ -({ \ - __chk_user_ptr(to); \ - __copy_user((__force void *)(to), (from), (n)); \ -}) -#define __copy_from_user(to, from, n) \ -({ \ - __chk_user_ptr(from); \ - __copy_user((to), (__force void *)(from), (n)); \ -}) - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - -extern inline long -copy_to_user(void __user *to, const void *from, long n) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long len) { - if (likely(__access_ok((unsigned long)to, n))) - n = __copy_user((__force void *)to, from, n); - return n; + return __copy_user(to, (__force const void *)from, len); } -extern inline long -copy_from_user(void *to, const void __user *from, long n) +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long len) { - long res = n; - if (likely(__access_ok((unsigned long)from, n))) - res = __copy_from_user_inatomic(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; + return __copy_user((__force void *)to, from, len); } extern long __clear_user(void __user *to, long len); From 3a1e37ae007e9d629617a0d5bc07661078defa32 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 19 Mar 2017 01:01:53 -0400 Subject: [PATCH 16/96] arc: get rid of unused declaration Signed-off-by: Al Viro --- arch/arc/include/asm/uaccess.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index ffd14e630aca..657401f3e217 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -394,9 +394,6 @@ __arc_copy_from_user(void *to, const void __user *from, unsigned long n) return res; } -extern unsigned long slowpath_copy_to_user(void __user *to, const void *from, - unsigned long n); - static inline unsigned long __arc_copy_to_user(void __user *to, const void *from, unsigned long n) { From 0f9b38cd79d528c1e19693899d989521778cc245 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 17/96] arm: switch to generic extable.h Signed-off-by: Al Viro --- arch/arm/include/asm/Kbuild | 1 + arch/arm/include/asm/uaccess.h | 20 +------------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild index b14e8c7d71bd..3a36d99ff836 100644 --- a/arch/arm/include/asm/Kbuild +++ b/arch/arm/include/asm/Kbuild @@ -7,6 +7,7 @@ generic-y += early_ioremap.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += ioctl.h generic-y += ipcbuf.h generic-y += irq_regs.h diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index b63527359d52..3a9190bc0ffa 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -24,25 +24,7 @@ #define __put_user_unaligned __put_user #endif -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); +#include /* * These two functions allow hooking accesses to userspace to increase From 4de5b63e76b2e672478e49622dabe2666b7f727f Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 08:23:33 -0400 Subject: [PATCH 18/96] arm: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/arm/Kconfig | 1 + arch/arm/include/asm/uaccess.h | 64 +++++++++------------------------- 2 files changed, 17 insertions(+), 48 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 0d4e71b42c77..6fab7f34739c 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -96,6 +96,7 @@ config ARM select PERF_USE_VMALLOC select RTC_LIB select SYS_SUPPORTS_APM_EMULATION + select ARCH_HAS_RAW_COPY_USER # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. help diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 3a9190bc0ffa..2577405d082d 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -455,7 +455,7 @@ extern unsigned long __must_check arm_copy_from_user(void *to, const void __user *from, unsigned long n); static inline unsigned long __must_check -__arch_copy_from_user(void *to, const void __user *from, unsigned long n) +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { unsigned int __ua_flags; @@ -471,7 +471,7 @@ extern unsigned long __must_check __copy_to_user_std(void __user *to, const void *from, unsigned long n); static inline unsigned long __must_check -__arch_copy_to_user(void __user *to, const void *from, unsigned long n) +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { #ifndef CONFIG_UACCESS_WITH_MEMCPY unsigned int __ua_flags; @@ -499,54 +499,22 @@ __clear_user(void __user *addr, unsigned long n) } #else -#define __arch_copy_from_user(to, from, n) \ - (memcpy(to, (void __force *)from, n), 0) -#define __arch_copy_to_user(to, from, n) \ - (memcpy((void __force *)to, from, n), 0) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + memcpy(to, (const void __force *)from, n); + return 0; +} +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + memcpy((void __force *)to, from, n); + return 0; +} #define __clear_user(addr, n) (memset((void __force *)addr, 0, n), 0) #endif - -static inline unsigned long __must_check -__copy_from_user(void *to, const void __user *from, unsigned long n) -{ - check_object_size(to, n, false); - return __arch_copy_from_user(to, from, n); -} - -static inline unsigned long __must_check -copy_from_user(void *to, const void __user *from, unsigned long n) -{ - unsigned long res = n; - - check_object_size(to, n, false); - - if (likely(access_ok(VERIFY_READ, from, n))) - res = __arch_copy_from_user(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; -} - -static inline unsigned long __must_check -__copy_to_user(void __user *to, const void *from, unsigned long n) -{ - check_object_size(from, n, true); - - return __arch_copy_to_user(to, from, n); -} - -static inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long n) -{ - check_object_size(from, n, true); - - if (access_ok(VERIFY_WRITE, to, n)) - n = __arch_copy_to_user(to, from, n); - return n; -} - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_TO_USER +#define INLINE_COPY_FROM_USER static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { From 46583939b98d5a3f13ac71eed9518c6b7a80ae7e Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 14:00:03 -0500 Subject: [PATCH 19/96] arm64: add extable.h Signed-off-by: Al Viro --- arch/arm64/include/asm/extable.h | 25 +++++++++++++++++++++++++ arch/arm64/include/asm/uaccess.h | 23 +---------------------- 2 files changed, 26 insertions(+), 22 deletions(-) create mode 100644 arch/arm64/include/asm/extable.h diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h new file mode 100644 index 000000000000..42f50f15a44c --- /dev/null +++ b/arch/arm64/include/asm/extable.h @@ -0,0 +1,25 @@ +#ifndef __ASM_EXTABLE_H +#define __ASM_EXTABLE_H + +/* + * The exception table consists of pairs of relative offsets: the first + * is the relative offset to an instruction that is allowed to fault, + * and the second is the relative offset at which the program should + * continue. No registers are modified, so it is entirely up to the + * continuation code to figure out what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry +{ + int insn, fixup; +}; + +#define ARCH_HAS_RELATIVE_EXTABLE + +extern int fixup_exception(struct pt_regs *regs); +#endif diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 7c514e10a08e..5a9d49c71937 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -33,28 +33,7 @@ #include #include #include - -/* - * The exception table consists of pairs of relative offsets: the first - * is the relative offset to an instruction that is allowed to fault, - * and the second is the relative offset at which the program should - * continue. No registers are modified, so it is entirely up to the - * continuation code to figure out what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry -{ - int insn, fixup; -}; - -#define ARCH_HAS_RELATIVE_EXTABLE - -extern int fixup_exception(struct pt_regs *regs); +#include #define KERNEL_DS (-1UL) #define get_ds() (KERNEL_DS) From 5cbe865b0e03982c7886f36716fc149415ee8381 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 20/96] avr32: switch to generic extable.h Signed-off-by: Al Viro --- arch/avr32/include/asm/Kbuild | 1 + arch/avr32/include/asm/uaccess.h | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/avr32/include/asm/Kbuild b/arch/avr32/include/asm/Kbuild index 3d7ef2c17a7c..ff7d7393b25c 100644 --- a/arch/avr32/include/asm/Kbuild +++ b/arch/avr32/include/asm/Kbuild @@ -5,6 +5,7 @@ generic-y += device.h generic-y += div64.h generic-y += emergency-restart.h generic-y += exec.h +generic-y += extable.h generic-y += futex.h generic-y += irq_regs.h generic-y += irq_work.h diff --git a/arch/avr32/include/asm/uaccess.h b/arch/avr32/include/asm/uaccess.h index 7ca5cb33369b..5f3ce85c1ed7 100644 --- a/arch/avr32/include/asm/uaccess.h +++ b/arch/avr32/include/asm/uaccess.h @@ -323,9 +323,6 @@ extern long __strnlen_user(const char __user *__s, long __n); #define strlen_user(s) strnlen_user(s, ~0UL >> 1) -struct exception_table_entry -{ - unsigned long insn, fixup; -}; +#include #endif /* __ASM_AVR32_UACCESS_H */ From 92430dab364019babd4122fef22441ef20bdb39b Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 08:40:57 -0400 Subject: [PATCH 21/96] arm64: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/uaccess.h | 55 +++----------------------------- arch/arm64/kernel/arm64ksyms.c | 2 +- arch/arm64/lib/copy_in_user.S | 4 +-- 4 files changed, 9 insertions(+), 53 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a39029b5414e..b72160ee7c18 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -115,6 +115,7 @@ config ARM64 select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK + select ARCH_HAS_RAW_COPY_USER help ARM 64-bit (AArch64) Linux support. diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 5a9d49c71937..ba497172610d 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -331,58 +331,13 @@ do { \ }) extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); +#define raw_copy_from_user __arch_copy_from_user extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n); -extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n); +#define raw_copy_to_user __arch_copy_to_user +extern unsigned long __must_check raw_copy_in_user(void __user *to, const void __user *from, unsigned long n); extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); - -static inline unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n) -{ - kasan_check_write(to, n); - check_object_size(to, n, false); - return __arch_copy_from_user(to, from, n); -} - -static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n) -{ - kasan_check_read(from, n); - check_object_size(from, n, true); - return __arch_copy_to_user(to, from, n); -} - -static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) -{ - unsigned long res = n; - kasan_check_write(to, n); - check_object_size(to, n, false); - - if (access_ok(VERIFY_READ, from, n)) { - res = __arch_copy_from_user(to, from, n); - } - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; -} - -static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long n) -{ - kasan_check_read(from, n); - check_object_size(from, n, true); - - if (access_ok(VERIFY_WRITE, to, n)) { - n = __arch_copy_to_user(to, from, n); - } - return n; -} - -static inline unsigned long __must_check copy_in_user(void __user *to, const void __user *from, unsigned long n) -{ - if (access_ok(VERIFY_READ, from, n) && access_ok(VERIFY_WRITE, to, n)) - n = __copy_in_user(to, from, n); - return n; -} - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_TO_USER +#define INLINE_COPY_FROM_USER static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index e9c4dc9e0ada..67368c7329c0 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -38,7 +38,7 @@ EXPORT_SYMBOL(clear_page); EXPORT_SYMBOL(__arch_copy_from_user); EXPORT_SYMBOL(__arch_copy_to_user); EXPORT_SYMBOL(__clear_user); -EXPORT_SYMBOL(__copy_in_user); +EXPORT_SYMBOL(raw_copy_in_user); /* physical memory */ EXPORT_SYMBOL(memstart_addr); diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S index 47184c3a97da..b24a830419ad 100644 --- a/arch/arm64/lib/copy_in_user.S +++ b/arch/arm64/lib/copy_in_user.S @@ -64,14 +64,14 @@ .endm end .req x5 -ENTRY(__copy_in_user) +ENTRY(raw_copy_in_user) uaccess_enable_not_uao x3, x4 add end, x0, x2 #include "copy_template.S" uaccess_disable_not_uao x3 mov x0, #0 ret -ENDPROC(__copy_in_user) +ENDPROC(raw_copy_in_user) .section .fixup,"ax" .align 2 From e5c1540030a0e2da0c46b3da91ed3ceafd0e4368 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 16:03:51 -0400 Subject: [PATCH 22/96] avr32: switch to RAW_COPY_USER This one needs profiling; use of asm variant of access_ok() might have been performance-critical. Now copy_{to,from}_user() are using the C variant. I doubt that it's going to have visible effects, but that needs to be experimentally verified. Signed-off-by: Al Viro --- arch/avr32/Kconfig | 1 + arch/avr32/include/asm/uaccess.h | 28 ++++++---------------------- arch/avr32/kernel/avr32_ksyms.c | 2 -- arch/avr32/lib/copy_user.S | 15 --------------- 4 files changed, 7 insertions(+), 39 deletions(-) diff --git a/arch/avr32/Kconfig b/arch/avr32/Kconfig index 7e75d45e20cd..8c349f2a9ebb 100644 --- a/arch/avr32/Kconfig +++ b/arch/avr32/Kconfig @@ -19,6 +19,7 @@ config AVR32 select HAVE_MOD_ARCH_SPECIFIC select MODULES_USE_ELF_RELA select HAVE_NMI + select ARCH_HAS_RAW_COPY_USER help AVR32 is a high-performance 32-bit RISC microprocessor core, designed for cost-sensitive embedded applications, with particular diff --git a/arch/avr32/include/asm/uaccess.h b/arch/avr32/include/asm/uaccess.h index 5f3ce85c1ed7..619d50e843a5 100644 --- a/arch/avr32/include/asm/uaccess.h +++ b/arch/avr32/include/asm/uaccess.h @@ -66,34 +66,18 @@ static inline void set_fs(mm_segment_t s) extern __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n); -extern __kernel_size_t copy_to_user(void __user *to, const void *from, - __kernel_size_t n); -extern __kernel_size_t ___copy_from_user(void *to, const void __user *from, - __kernel_size_t n); - -static inline __kernel_size_t __copy_to_user(void __user *to, const void *from, - __kernel_size_t n) +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { return __copy_user((void __force *)to, from, n); } -static inline __kernel_size_t __copy_from_user(void *to, - const void __user *from, - __kernel_size_t n) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { return __copy_user(to, (const void __force *)from, n); } -static inline __kernel_size_t copy_from_user(void *to, - const void __user *from, - __kernel_size_t n) -{ - size_t res = ___copy_from_user(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; -} - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER /* * put_user: - Write a simple value into user space. diff --git a/arch/avr32/kernel/avr32_ksyms.c b/arch/avr32/kernel/avr32_ksyms.c index 0d05fd095468..0cdae8eb0220 100644 --- a/arch/avr32/kernel/avr32_ksyms.c +++ b/arch/avr32/kernel/avr32_ksyms.c @@ -36,8 +36,6 @@ EXPORT_SYMBOL(copy_page); /* * Userspace access stuff. */ -EXPORT_SYMBOL(___copy_from_user); -EXPORT_SYMBOL(copy_to_user); EXPORT_SYMBOL(__copy_user); EXPORT_SYMBOL(strncpy_from_user); EXPORT_SYMBOL(__strncpy_from_user); diff --git a/arch/avr32/lib/copy_user.S b/arch/avr32/lib/copy_user.S index 075373471da1..f43275796a0c 100644 --- a/arch/avr32/lib/copy_user.S +++ b/arch/avr32/lib/copy_user.S @@ -23,21 +23,6 @@ */ .text .align 1 - .global ___copy_from_user - .type ___copy_from_user, @function -___copy_from_user: - branch_if_kernel r8, __copy_user - ret_if_privileged r8, r11, r10, r10 - rjmp __copy_user - .size ___copy_from_user, . - ___copy_from_user - - .global copy_to_user - .type copy_to_user, @function -copy_to_user: - branch_if_kernel r8, __copy_user - ret_if_privileged r8, r12, r10, r10 - .size copy_to_user, . - copy_to_user - .global __copy_user .type __copy_user, @function __copy_user: From 72d6f65dd7895276f929b48c8397575e272efcca Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 23/96] blackfin: switch to generic extable.h Signed-off-by: Al Viro --- arch/blackfin/include/asm/Kbuild | 1 + arch/blackfin/include/asm/uaccess.h | 17 +---------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/arch/blackfin/include/asm/Kbuild b/arch/blackfin/include/asm/Kbuild index 625db8ac815e..dc4ef9ac1e83 100644 --- a/arch/blackfin/include/asm/Kbuild +++ b/arch/blackfin/include/asm/Kbuild @@ -7,6 +7,7 @@ generic-y += device.h generic-y += div64.h generic-y += emergency-restart.h generic-y += errno.h +generic-y += extable.h generic-y += fb.h generic-y += futex.h generic-y += hw_irq.h diff --git a/arch/blackfin/include/asm/uaccess.h b/arch/blackfin/include/asm/uaccess.h index c9fedc3be30c..9019cccf2aea 100644 --- a/arch/blackfin/include/asm/uaccess.h +++ b/arch/blackfin/include/asm/uaccess.h @@ -42,22 +42,7 @@ static inline int _access_ok(unsigned long addr, unsigned long size) { return 1; extern int _access_ok(unsigned long addr, unsigned long size); #endif -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn, fixup; -}; +#include /* * These are the main single-value transfer routines. They automatically From 50e9ab915a79b3f51f6de8ef9bbdae8279e8fa46 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 00:36:07 -0400 Subject: [PATCH 24/96] bfin: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/blackfin/Kconfig | 1 + arch/blackfin/include/asm/uaccess.h | 26 ++++---------------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/arch/blackfin/Kconfig b/arch/blackfin/Kconfig index 3c1bd640042a..919dad1436f7 100644 --- a/arch/blackfin/Kconfig +++ b/arch/blackfin/Kconfig @@ -41,6 +41,7 @@ config BLACKFIN select MODULES_USE_ELF_RELA select HAVE_DEBUG_STACKOVERFLOW select HAVE_NMI + select ARCH_HAS_RAW_COPY_USER config GENERIC_CSUM def_bool y diff --git a/arch/blackfin/include/asm/uaccess.h b/arch/blackfin/include/asm/uaccess.h index 9019cccf2aea..f54a34f31cea 100644 --- a/arch/blackfin/include/asm/uaccess.h +++ b/arch/blackfin/include/asm/uaccess.h @@ -144,41 +144,23 @@ static inline int bad_user_access_length(void) : "a" (__ptr(ptr))); \ }) -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - static inline unsigned long __must_check -__copy_from_user(void *to, const void __user *from, unsigned long n) +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { memcpy(to, (const void __force *)from, n); return 0; } static inline unsigned long __must_check -__copy_to_user(void __user *to, const void *from, unsigned long n) +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { memcpy((void __force *)to, from, n); SSYNC(); return 0; } -static inline unsigned long __must_check -copy_from_user(void *to, const void __user *from, unsigned long n) -{ - if (likely(access_ok(VERIFY_READ, from, n))) - return __copy_from_user(to, from, n); - memset(to, 0, n); - return n; -} - -static inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long n) -{ - if (likely(access_ok(VERIFY_WRITE, to, n))) - return __copy_to_user(to, from, n); - return n; -} - +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER /* * Copy a null terminated string from userspace. */ From dcecd9369e3dcbb5da3ecdc9e0b5021d6ee6f879 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 14:16:16 -0500 Subject: [PATCH 25/96] c6x: remove duplicate definition of __access_ok Signed-off-by: Al Viro --- arch/c6x/include/asm/uaccess.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/c6x/include/asm/uaccess.h b/arch/c6x/include/asm/uaccess.h index 79fb6f439c5c..174f3bd811b2 100644 --- a/arch/c6x/include/asm/uaccess.h +++ b/arch/c6x/include/asm/uaccess.h @@ -13,10 +13,6 @@ #include #include -#ifdef CONFIG_ACCESS_CHECK -#define __access_ok _access_ok -#endif - /* * __copy_from_user/copy_to_user are based on ones in asm-generic/uaccess.h * From 86944ee1581f4be6af0126e80877b8069e177739 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 01:11:14 -0400 Subject: [PATCH 26/96] c6x: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/c6x/Kconfig | 1 + arch/c6x/include/asm/uaccess.h | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/c6x/Kconfig b/arch/c6x/Kconfig index 5aa8ea8bad2d..3c7bd9a29f90 100644 --- a/arch/c6x/Kconfig +++ b/arch/c6x/Kconfig @@ -18,6 +18,7 @@ config C6X select GENERIC_CLOCKEVENTS select MODULES_USE_ELF_RELA select ARCH_NO_COHERENT_DMA_MMAP + select ARCH_HAS_RAW_COPY_USER config MMU def_bool n diff --git a/arch/c6x/include/asm/uaccess.h b/arch/c6x/include/asm/uaccess.h index 174f3bd811b2..ba6756879f00 100644 --- a/arch/c6x/include/asm/uaccess.h +++ b/arch/c6x/include/asm/uaccess.h @@ -14,12 +14,10 @@ #include /* - * __copy_from_user/copy_to_user are based on ones in asm-generic/uaccess.h - * * C6X supports unaligned 32 and 64 bit loads and stores. */ -static inline __must_check long __copy_from_user(void *to, - const void __user *from, unsigned long n) +static inline __must_check unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { u32 tmp32; u64 tmp64; @@ -54,8 +52,8 @@ static inline __must_check long __copy_from_user(void *to, return 0; } -static inline __must_check long __copy_to_user(void __user *to, - const void *from, unsigned long n) +static inline __must_check unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { u32 tmp32; u64 tmp64; @@ -89,6 +87,8 @@ static inline __must_check long __copy_to_user(void __user *to, memcpy((void __force *)to, from, n); return 0; } +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER extern int _access_ok(unsigned long addr, unsigned long size); #ifdef CONFIG_ACCESS_CHECK From a8be34459ca92d4f1034921bffd09df5441938be Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 27/96] cris: switch to generic extable.h Signed-off-by: Al Viro --- arch/cris/include/asm/Kbuild | 1 + arch/cris/include/asm/uaccess.h | 18 +----------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/arch/cris/include/asm/Kbuild b/arch/cris/include/asm/Kbuild index 0f5132b08896..2890099992a9 100644 --- a/arch/cris/include/asm/Kbuild +++ b/arch/cris/include/asm/Kbuild @@ -9,6 +9,7 @@ generic-y += device.h generic-y += div64.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += emergency-restart.h generic-y += fcntl.h generic-y += futex.h diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h index 5f5b8f53d2d7..f62f546720a9 100644 --- a/arch/cris/include/asm/uaccess.h +++ b/arch/cris/include/asm/uaccess.h @@ -50,23 +50,7 @@ #define access_ok(type, addr, size) __access_ok((unsigned long)(addr), (size)) #include - -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn, fixup; -}; +#include /* * These are the main single-value transfer routines. They automatically From 07f78b308984042dde424191ab8f3237bcf6a0d1 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 19 Mar 2017 15:07:21 -0400 Subject: [PATCH 28/96] cris: don't rely upon __copy_user_zeroing() zeroing the tail we want to get rid of it; unfortunately, it's tangled as hell, so it'll take many steps, more's the pity. Signed-off-by: Al Viro --- arch/cris/include/asm/uaccess.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h index f62f546720a9..c8858e15d709 100644 --- a/arch/cris/include/asm/uaccess.h +++ b/arch/cris/include/asm/uaccess.h @@ -338,14 +338,16 @@ static inline size_t clear_user(void __user *to, size_t n) static inline size_t copy_from_user(void *to, const void __user *from, size_t n) { - if (unlikely(!access_ok(VERIFY_READ, from, n))) { - memset(to, 0, n); - return n; + size_t res = n; + if (likely(access_ok(VERIFY_READ, from, n))) { + if (__builtin_constant_p(n)) + res = __constant_copy_from_user(to, from, n); + else + res = __copy_user_zeroing(to, from, n); } - if (__builtin_constant_p(n)) - return __constant_copy_from_user(to, from, n); - else - return __copy_user_zeroing(to, from, n); + if (unlikely(res)) + memset(to + n - res , 0, res); + return res; } static inline size_t copy_to_user(void __user *to, const void *from, size_t n) From c8313947af458d9081c3adca2a9a9c962cf014f1 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 19 Mar 2017 15:17:25 -0400 Subject: [PATCH 29/96] cris: get rid of zeroing in __asm_copy_from_user_N for N > 4 only one user for those Signed-off-by: Al Viro --- arch/cris/include/arch-v10/arch/uaccess.h | 34 ++++++++---------- arch/cris/include/arch-v32/arch/uaccess.h | 42 ++++++++--------------- 2 files changed, 28 insertions(+), 48 deletions(-) diff --git a/arch/cris/include/arch-v10/arch/uaccess.h b/arch/cris/include/arch-v10/arch/uaccess.h index 65b02d9b605a..f68d3d19df72 100644 --- a/arch/cris/include/arch-v10/arch/uaccess.h +++ b/arch/cris/include/arch-v10/arch/uaccess.h @@ -210,8 +210,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_4x_cont(to, from, ret, \ " move.b [%1+],$r9\n" \ "4: move.b $r9,[%0+]\n", \ - "5: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "5: addq 1,%2\n", \ " .dword 4b,5b\n") #define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -219,7 +218,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.w [%1+],$r9\n" \ "4: move.w $r9,[%0+]\n" COPY, \ "5: addq 2,%2\n" \ - " clear.w [%0+]\n" FIXUP, \ + FIXUP, \ " .dword 4b,5b\n" TENTRY) #define __asm_copy_from_user_6(to, from, ret) \ @@ -229,8 +228,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_6x_cont(to, from, ret, \ " move.b [%1+],$r9\n" \ "6: move.b $r9,[%0+]\n", \ - "7: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "7: addq 1,%2\n", \ " .dword 6b,7b\n") #define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -238,7 +236,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.d [%1+],$r9\n" \ "4: move.d $r9,[%0+]\n" COPY, \ "5: addq 4,%2\n" \ - " clear.d [%0+]\n" FIXUP, \ + FIXUP, \ " .dword 4b,5b\n" TENTRY) #define __asm_copy_from_user_8(to, from, ret) \ @@ -248,8 +246,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_8x_cont(to, from, ret, \ " move.b [%1+],$r9\n" \ "6: move.b $r9,[%0+]\n", \ - "7: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "7: addq 1,%2\n", \ " .dword 6b,7b\n") #define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -257,7 +254,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.w [%1+],$r9\n" \ "6: move.w $r9,[%0+]\n" COPY, \ "7: addq 2,%2\n" \ - " clear.w [%0+]\n" FIXUP, \ + FIXUP, \ " .dword 6b,7b\n" TENTRY) #define __asm_copy_from_user_10(to, from, ret) \ @@ -267,8 +264,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_10x_cont(to, from, ret, \ " move.b [%1+],$r9\n" \ "8: move.b $r9,[%0+]\n", \ - "9: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "9: addq 1,%2\n", \ " .dword 8b,9b\n") #define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -276,7 +272,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.d [%1+],$r9\n" \ "6: move.d $r9,[%0+]\n" COPY, \ "7: addq 4,%2\n" \ - " clear.d [%0+]\n" FIXUP, \ + FIXUP, \ " .dword 6b,7b\n" TENTRY) #define __asm_copy_from_user_12(to, from, ret) \ @@ -286,8 +282,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_12x_cont(to, from, ret, \ " move.b [%1+],$r9\n" \ "8: move.b $r9,[%0+]\n", \ - "9: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "9: addq 1,%2\n", \ " .dword 8b,9b\n") #define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -295,7 +290,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.w [%1+],$r9\n" \ "8: move.w $r9,[%0+]\n" COPY, \ "9: addq 2,%2\n" \ - " clear.w [%0+]\n" FIXUP, \ + FIXUP, \ " .dword 8b,9b\n" TENTRY) #define __asm_copy_from_user_14(to, from, ret) \ @@ -305,8 +300,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_14x_cont(to, from, ret, \ " move.b [%1+],$r9\n" \ "10: move.b $r9,[%0+]\n", \ - "11: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "11: addq 1,%2\n", \ " .dword 10b,11b\n") #define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -314,7 +308,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.d [%1+],$r9\n" \ "8: move.d $r9,[%0+]\n" COPY, \ "9: addq 4,%2\n" \ - " clear.d [%0+]\n" FIXUP, \ + FIXUP, \ " .dword 8b,9b\n" TENTRY) #define __asm_copy_from_user_16(to, from, ret) \ @@ -325,7 +319,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.d [%1+],$r9\n" \ "10: move.d $r9,[%0+]\n" COPY, \ "11: addq 4,%2\n" \ - " clear.d [%0+]\n" FIXUP, \ + FIXUP, \ " .dword 10b,11b\n" TENTRY) #define __asm_copy_from_user_20(to, from, ret) \ @@ -336,7 +330,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.d [%1+],$r9\n" \ "12: move.d $r9,[%0+]\n" COPY, \ "13: addq 4,%2\n" \ - " clear.d [%0+]\n" FIXUP, \ + FIXUP, \ " .dword 12b,13b\n" TENTRY) #define __asm_copy_from_user_24(to, from, ret) \ diff --git a/arch/cris/include/arch-v32/arch/uaccess.h b/arch/cris/include/arch-v32/arch/uaccess.h index 3196019706cb..7a0032a4b9fb 100644 --- a/arch/cris/include/arch-v32/arch/uaccess.h +++ b/arch/cris/include/arch-v32/arch/uaccess.h @@ -224,8 +224,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_4x_cont(to, from, ret, \ "4: move.b [%1+],$acr\n" \ " move.b $acr,[%0+]\n", \ - "5: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "5: addq 1,%2\n", \ " .dword 4b,5b\n") #define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -234,8 +233,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "4: move.w [%1+],$acr\n" \ " move.w $acr,[%0+]\n", \ FIXUP \ - "5: addq 2,%2\n" \ - " clear.w [%0+]\n", \ + "5: addq 2,%2\n", \ TENTRY \ " .dword 4b,5b\n") @@ -246,8 +244,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_6x_cont(to, from, ret, \ "6: move.b [%1+],$acr\n" \ " move.b $acr,[%0+]\n", \ - "7: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "7: addq 1,%2\n", \ " .dword 6b,7b\n") #define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -256,8 +253,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "4: move.d [%1+],$acr\n" \ " move.d $acr,[%0+]\n", \ FIXUP \ - "5: addq 4,%2\n" \ - " clear.d [%0+]\n", \ + "5: addq 4,%2\n", \ TENTRY \ " .dword 4b,5b\n") @@ -268,8 +264,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_8x_cont(to, from, ret, \ "6: move.b [%1+],$acr\n" \ " move.b $acr,[%0+]\n", \ - "7: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "7: addq 1,%2\n", \ " .dword 6b,7b\n") #define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -278,8 +273,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "6: move.w [%1+],$acr\n" \ " move.w $acr,[%0+]\n", \ FIXUP \ - "7: addq 2,%2\n" \ - " clear.w [%0+]\n", \ + "7: addq 2,%2\n", \ TENTRY \ " .dword 6b,7b\n") @@ -290,8 +284,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_10x_cont(to, from, ret, \ "8: move.b [%1+],$acr\n" \ " move.b $acr,[%0+]\n", \ - "9: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "9: addq 1,%2\n", \ " .dword 8b,9b\n") #define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -300,8 +293,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "6: move.d [%1+],$acr\n" \ " move.d $acr,[%0+]\n", \ FIXUP \ - "7: addq 4,%2\n" \ - " clear.d [%0+]\n", \ + "7: addq 4,%2\n", \ TENTRY \ " .dword 6b,7b\n") @@ -312,8 +304,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_12x_cont(to, from, ret, \ "8: move.b [%1+],$acr\n" \ " move.b $acr,[%0+]\n", \ - "9: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "9: addq 1,%2\n", \ " .dword 8b,9b\n") #define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -322,8 +313,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "8: move.w [%1+],$acr\n" \ " move.w $acr,[%0+]\n", \ FIXUP \ - "9: addq 2,%2\n" \ - " clear.w [%0+]\n", \ + "9: addq 2,%2\n", \ TENTRY \ " .dword 8b,9b\n") @@ -334,8 +324,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_14x_cont(to, from, ret, \ "10: move.b [%1+],$acr\n" \ " move.b $acr,[%0+]\n", \ - "11: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "11: addq 1,%2\n", \ " .dword 10b,11b\n") #define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -344,8 +333,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "8: move.d [%1+],$acr\n" \ " move.d $acr,[%0+]\n", \ FIXUP \ - "9: addq 4,%2\n" \ - " clear.d [%0+]\n", \ + "9: addq 4,%2\n", \ TENTRY \ " .dword 8b,9b\n") @@ -358,8 +346,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "10: move.d [%1+],$acr\n" \ " move.d $acr,[%0+]\n", \ FIXUP \ - "11: addq 4,%2\n" \ - " clear.d [%0+]\n", \ + "11: addq 4,%2\n", \ TENTRY \ " .dword 10b,11b\n") @@ -372,8 +359,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "12: move.d [%1+],$acr\n" \ " move.d $acr,[%0+]\n", \ FIXUP \ - "13: addq 4,%2\n" \ - " clear.d [%0+]\n", \ + "13: addq 4,%2\n", \ TENTRY \ " .dword 12b,13b\n") From de09be340d3ef4568b27282f2f6fc44b1cb559ab Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 19 Mar 2017 15:25:35 -0400 Subject: [PATCH 30/96] cris: get rid of zeroing ... the rest of it Signed-off-by: Al Viro --- arch/cris/arch-v10/lib/usercopy.c | 24 ++++++----------------- arch/cris/arch-v32/lib/usercopy.c | 23 ++++++---------------- arch/cris/include/arch-v10/arch/uaccess.h | 12 ++++-------- arch/cris/include/arch-v32/arch/uaccess.h | 12 ++++-------- 4 files changed, 20 insertions(+), 51 deletions(-) diff --git a/arch/cris/arch-v10/lib/usercopy.c b/arch/cris/arch-v10/lib/usercopy.c index 1ba7cc000dfc..80b72199bdc4 100644 --- a/arch/cris/arch-v10/lib/usercopy.c +++ b/arch/cris/arch-v10/lib/usercopy.c @@ -217,19 +217,17 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, { __asm_copy_from_user_1 (dst, src, retn); n--; + if (retn) + goto exception; } if (((unsigned long) src & 2) && n >= 2) { __asm_copy_from_user_2 (dst, src, retn); n -= 2; + if (retn) + goto exception; } - - /* We only need one check after the unalignment-adjustments, because - if both adjustments were done, either both or neither reference - had an exception. */ - if (retn != 0) - goto copy_exception_bytes; } /* Decide which copying method to use. */ @@ -328,7 +326,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, n -= 4; if (retn) - goto copy_exception_bytes; + goto exception; } /* If we get here, there were no memory read faults. */ @@ -356,17 +354,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, bytes. */ return retn; -copy_exception_bytes: - /* We already have "retn" bytes cleared, and need to clear the - remaining "n" bytes. A non-optimized simple byte-for-byte in-line - memset is preferred here, since this isn't speed-critical code and - we'd rather have this a leaf-function than calling memset. */ - { - char *endp; - for (endp = dst + n; dst < endp; dst++) - *dst = 0; - } - +exception: return retn + n; } EXPORT_SYMBOL(__copy_user_zeroing); diff --git a/arch/cris/arch-v32/lib/usercopy.c b/arch/cris/arch-v32/lib/usercopy.c index 05e58dab800d..25f421f98858 100644 --- a/arch/cris/arch-v32/lib/usercopy.c +++ b/arch/cris/arch-v32/lib/usercopy.c @@ -184,19 +184,18 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, { __asm_copy_from_user_1 (dst, src, retn); n--; + if (retn != 0) + goto exception; } if (((unsigned long) src & 2) && n >= 2) { __asm_copy_from_user_2 (dst, src, retn); n -= 2; + if (retn != 0) + goto exception; } - /* We only need one check after the unalignment-adjustments, because - if both adjustments were done, either both or neither reference - had an exception. */ - if (retn != 0) - goto copy_exception_bytes; } /* Movem is dirt cheap. The overheap is low enough to always use the @@ -279,7 +278,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, n -= 4; if (retn) - goto copy_exception_bytes; + goto exception; } /* If we get here, there were no memory read faults. */ @@ -307,17 +306,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, bytes. */ return retn; -copy_exception_bytes: - /* We already have "retn" bytes cleared, and need to clear the - remaining "n" bytes. A non-optimized simple byte-for-byte in-line - memset is preferred here, since this isn't speed-critical code and - we'd rather have this a leaf-function than calling memset. */ - { - char *endp; - for (endp = dst + n; dst < endp; dst++) - *dst = 0; - } - +exception: return retn + n; } EXPORT_SYMBOL(__copy_user_zeroing); diff --git a/arch/cris/include/arch-v10/arch/uaccess.h b/arch/cris/include/arch-v10/arch/uaccess.h index f68d3d19df72..5477c98c2281 100644 --- a/arch/cris/include/arch-v10/arch/uaccess.h +++ b/arch/cris/include/arch-v10/arch/uaccess.h @@ -172,16 +172,14 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_user_cont(to, from, ret, \ " move.b [%1+],$r9\n" \ "2: move.b $r9,[%0+]\n", \ - "3: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "3: addq 1,%2\n", \ " .dword 2b,3b\n") #define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ __asm_copy_user_cont(to, from, ret, \ " move.w [%1+],$r9\n" \ "2: move.w $r9,[%0+]\n" COPY, \ - "3: addq 2,%2\n" \ - " clear.w [%0+]\n" FIXUP, \ + "3: addq 2,%2\n" FIXUP, \ " .dword 2b,3b\n" TENTRY) #define __asm_copy_from_user_2(to, from, ret) \ @@ -191,16 +189,14 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_2x_cont(to, from, ret, \ " move.b [%1+],$r9\n" \ "4: move.b $r9,[%0+]\n", \ - "5: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "5: addq 1,%2\n", \ " .dword 4b,5b\n") #define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ __asm_copy_user_cont(to, from, ret, \ " move.d [%1+],$r9\n" \ "2: move.d $r9,[%0+]\n" COPY, \ - "3: addq 4,%2\n" \ - " clear.d [%0+]\n" FIXUP, \ + "3: addq 4,%2\n" FIXUP, \ " .dword 2b,3b\n" TENTRY) #define __asm_copy_from_user_4(to, from, ret) \ diff --git a/arch/cris/include/arch-v32/arch/uaccess.h b/arch/cris/include/arch-v32/arch/uaccess.h index 7a0032a4b9fb..dc2ce090f624 100644 --- a/arch/cris/include/arch-v32/arch/uaccess.h +++ b/arch/cris/include/arch-v32/arch/uaccess.h @@ -178,8 +178,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) "2: move.b [%1+],$acr\n" \ " move.b $acr,[%0+]\n", \ "3: addq 1,%2\n" \ - " jump 1b\n" \ - " clear.b [%0+]\n", \ + " jump 1b\n", \ " .dword 2b,3b\n") #define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -189,8 +188,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.w $acr,[%0+]\n", \ FIXUP \ "3: addq 2,%2\n" \ - " jump 1b\n" \ - " clear.w [%0+]\n", \ + " jump 1b\n", \ TENTRY \ " .dword 2b,3b\n") @@ -201,8 +199,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) __asm_copy_from_user_2x_cont(to, from, ret, \ "4: move.b [%1+],$acr\n" \ " move.b $acr,[%0+]\n", \ - "5: addq 1,%2\n" \ - " clear.b [%0+]\n", \ + "5: addq 1,%2\n", \ " .dword 4b,5b\n") #define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ @@ -212,8 +209,7 @@ __do_strncpy_from_user(char *dst, const char *src, long count) " move.d $acr,[%0+]\n", \ FIXUP \ "3: addq 4,%2\n" \ - " jump 1b\n" \ - " clear.d [%0+]\n", \ + " jump 1b\n", \ TENTRY \ " .dword 2b,3b\n") From b71f1bf57f995ae4ce0fe2f907604c841ca8493f Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 19 Mar 2017 15:28:30 -0400 Subject: [PATCH 31/96] cris: rename __copy_user_zeroing to __copy_user_in ... now that it doesn't zero anymore Signed-off-by: Al Viro --- arch/cris/arch-v10/lib/usercopy.c | 7 +++---- arch/cris/arch-v32/lib/usercopy.c | 7 +++---- arch/cris/include/asm/uaccess.h | 8 ++++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/arch/cris/arch-v10/lib/usercopy.c b/arch/cris/arch-v10/lib/usercopy.c index 80b72199bdc4..48fa37fe0f9b 100644 --- a/arch/cris/arch-v10/lib/usercopy.c +++ b/arch/cris/arch-v10/lib/usercopy.c @@ -188,11 +188,10 @@ unsigned long __copy_user(void __user *pdst, const void *psrc, unsigned long pn) } EXPORT_SYMBOL(__copy_user); -/* Copy from user to kernel, zeroing the bytes that were inaccessible in - userland. The return-value is the number of bytes that were +/* Copy from user to kernel. The return-value is the number of bytes that were inaccessible. */ -unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, +unsigned long __copy_user_in(void *pdst, const void __user *psrc, unsigned long pn) { /* We want the parameters put in special registers. @@ -357,7 +356,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, exception: return retn + n; } -EXPORT_SYMBOL(__copy_user_zeroing); +EXPORT_SYMBOL(__copy_user_in); /* Zero userspace. */ unsigned long __do_clear_user(void __user *pto, unsigned long pn) diff --git a/arch/cris/arch-v32/lib/usercopy.c b/arch/cris/arch-v32/lib/usercopy.c index 25f421f98858..20b608026913 100644 --- a/arch/cris/arch-v32/lib/usercopy.c +++ b/arch/cris/arch-v32/lib/usercopy.c @@ -156,10 +156,9 @@ unsigned long __copy_user(void __user *pdst, const void *psrc, unsigned long pn) } EXPORT_SYMBOL(__copy_user); -/* Copy from user to kernel, zeroing the bytes that were inaccessible in - userland. The return-value is the number of bytes that were +/* Copy from user to kernel. The return-value is the number of bytes that were inaccessible. */ -unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, +unsigned long __copy_user_in(void *pdst, const void __user *psrc, unsigned long pn) { /* We want the parameters put in special registers. @@ -309,7 +308,7 @@ unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, exception: return retn + n; } -EXPORT_SYMBOL(__copy_user_zeroing); +EXPORT_SYMBOL(__copy_user_in); /* Zero userspace. */ unsigned long __do_clear_user(void __user *pto, unsigned long pn) diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h index c8858e15d709..fc30fdae8786 100644 --- a/arch/cris/include/asm/uaccess.h +++ b/arch/cris/include/asm/uaccess.h @@ -169,7 +169,7 @@ extern long __get_user_bad(void); live in lib/usercopy.c */ extern unsigned long __copy_user(void __user *to, const void *from, unsigned long n); -extern unsigned long __copy_user_zeroing(void *to, const void __user *from, unsigned long n); +extern unsigned long __copy_user_in(void *to, const void __user *from, unsigned long n); extern unsigned long __do_clear_user(void __user *to, unsigned long n); static inline long @@ -236,7 +236,7 @@ __constant_copy_from_user(void *to, const void __user *from, unsigned long n) else if (n == 24) __asm_copy_from_user_24(to, from, ret); else - ret = __copy_user_zeroing(to, from, n); + ret = __copy_user_in(to, from, n); return ret; } @@ -343,7 +343,7 @@ static inline size_t copy_from_user(void *to, const void __user *from, size_t n) if (__builtin_constant_p(n)) res = __constant_copy_from_user(to, from, n); else - res = __copy_user_zeroing(to, from, n); + res = __copy_user_in(to, from, n); } if (unlikely(res)) memset(to + n - res , 0, res); @@ -368,7 +368,7 @@ static inline unsigned long __generic_copy_from_user_nocheck(void *to, const void __user *from, unsigned long n) { - return __copy_user_zeroing(to, from, n); + return __copy_user_in(to, from, n); } static inline unsigned long From 0c7e9a870eb2a5db03780d7d3f83d0df7623a9c7 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 01:18:37 -0400 Subject: [PATCH 32/96] cris: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/cris/Kconfig | 1 + arch/cris/include/asm/uaccess.h | 53 +++++++-------------------------- 2 files changed, 12 insertions(+), 42 deletions(-) diff --git a/arch/cris/Kconfig b/arch/cris/Kconfig index 71b758dc3a96..36f94c45e3f9 100644 --- a/arch/cris/Kconfig +++ b/arch/cris/Kconfig @@ -71,6 +71,7 @@ config CRIS select GENERIC_SCHED_CLOCK if ETRAX_ARCH_V32 select HAVE_DEBUG_BUGVERBOSE if ETRAX_ARCH_V32 select HAVE_NMI + select ARCH_HAS_RAW_COPY_USER config HZ int diff --git a/arch/cris/include/asm/uaccess.h b/arch/cris/include/asm/uaccess.h index fc30fdae8786..0d473aec3066 100644 --- a/arch/cris/include/asm/uaccess.h +++ b/arch/cris/include/asm/uaccess.h @@ -336,64 +336,33 @@ static inline size_t clear_user(void __user *to, size_t n) return __do_clear_user(to, n); } -static inline size_t copy_from_user(void *to, const void __user *from, size_t n) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { - size_t res = n; - if (likely(access_ok(VERIFY_READ, from, n))) { - if (__builtin_constant_p(n)) - res = __constant_copy_from_user(to, from, n); - else - res = __copy_user_in(to, from, n); - } - if (unlikely(res)) - memset(to + n - res , 0, res); - return res; + if (__builtin_constant_p(n)) + return __constant_copy_from_user(to, from, n); + else + return __copy_user_in(to, from, n); } -static inline size_t copy_to_user(void __user *to, const void *from, size_t n) +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - if (unlikely(!access_ok(VERIFY_WRITE, to, n))) - return n; if (__builtin_constant_p(n)) return __constant_copy_to_user(to, from, n); else return __copy_user(to, from, n); } -/* We let the __ versions of copy_from/to_user inline, because they're often - * used in fast paths and have only a small space overhead. - */ +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER static inline unsigned long -__generic_copy_from_user_nocheck(void *to, const void __user *from, - unsigned long n) -{ - return __copy_user_in(to, from, n); -} - -static inline unsigned long -__generic_copy_to_user_nocheck(void __user *to, const void *from, - unsigned long n) -{ - return __copy_user(to, from, n); -} - -static inline unsigned long -__generic_clear_user_nocheck(void __user *to, unsigned long n) +__clear_user(void __user *to, unsigned long n) { return __do_clear_user(to, n); } -/* without checking */ - -#define __copy_to_user(to, from, n) \ - __generic_copy_to_user_nocheck((to), (from), (n)) -#define __copy_from_user(to, from, n) \ - __generic_copy_from_user_nocheck((to), (from), (n)) -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user -#define __clear_user(to, n) __generic_clear_user_nocheck((to), (n)) - #define strlen_user(str) strnlen_user((str), 0x7ffffffe) #endif /* _CRIS_UACCESS_H */ From 9a7513cfa26f5fcce15de6130ce3c27d77c0ce55 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 14:12:18 -0500 Subject: [PATCH 33/96] frv: switch to use of fixup_exception() Massage frv search_exception_table() to a) taking pt_regs pointer as explicit argument b) updating ->pc on success Simplifies callers a bit and allows to convert to generic extable.h, while we are at it. Signed-off-by: Al Viro --- arch/frv/include/asm/Kbuild | 1 + arch/frv/include/asm/uaccess.h | 23 +---------------------- arch/frv/kernel/traps.c | 7 +------ arch/frv/mm/extable.c | 27 +++++++++++++-------------- arch/frv/mm/fault.c | 6 ++---- 5 files changed, 18 insertions(+), 46 deletions(-) diff --git a/arch/frv/include/asm/Kbuild b/arch/frv/include/asm/Kbuild index c33b46715f65..cce3bc3603ea 100644 --- a/arch/frv/include/asm/Kbuild +++ b/arch/frv/include/asm/Kbuild @@ -1,6 +1,7 @@ generic-y += clkdev.h generic-y += exec.h +generic-y += extable.h generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h diff --git a/arch/frv/include/asm/uaccess.h b/arch/frv/include/asm/uaccess.h index 55b3a69c6c53..5bcc57de3c95 100644 --- a/arch/frv/include/asm/uaccess.h +++ b/arch/frv/include/asm/uaccess.h @@ -18,6 +18,7 @@ #include #include #include +#include #define __ptr(x) ((unsigned long __force *)(x)) @@ -59,26 +60,6 @@ static inline int ___range_ok(unsigned long addr, unsigned long size) #define access_ok(type,addr,size) (__range_ok((void __user *)(addr), (size)) == 0) #define __access_ok(addr,size) (__range_ok((addr), (size)) == 0) -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - -/* Returns 0 if exception not found and fixup otherwise. */ -extern unsigned long search_exception_table(unsigned long); - /* * These are the main single-value transfer routines. They automatically @@ -314,6 +295,4 @@ extern long strnlen_user(const char __user *src, long count); #define strlen_user(str) strnlen_user(str, 32767) -extern unsigned long search_exception_table(unsigned long addr); - #endif /* _ASM_UACCESS_H */ diff --git a/arch/frv/kernel/traps.c b/arch/frv/kernel/traps.c index ce29991e4219..fb08ebe0dab4 100644 --- a/arch/frv/kernel/traps.c +++ b/arch/frv/kernel/traps.c @@ -360,13 +360,8 @@ asmlinkage void memory_access_exception(unsigned long esr0, siginfo_t info; #ifdef CONFIG_MMU - unsigned long fixup; - - fixup = search_exception_table(__frame->pc); - if (fixup) { - __frame->pc = fixup; + if (fixup_exception(__frame)) return; - } #endif die_if_kernel("-- Memory Access Exception --\n" diff --git a/arch/frv/mm/extable.c b/arch/frv/mm/extable.c index a0e8b3e03e4c..9198ddd16092 100644 --- a/arch/frv/mm/extable.c +++ b/arch/frv/mm/extable.c @@ -10,40 +10,39 @@ extern const void __memset_end, __memset_user_error_lr, __memset_user_error_hand extern const void __memcpy_end, __memcpy_user_error_lr, __memcpy_user_error_handler; extern spinlock_t modlist_lock; - -/*****************************************************************************/ -/* - * see if there's a fixup handler available to deal with a kernel fault - */ -unsigned long search_exception_table(unsigned long pc) +int fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *extab; + unsigned long pc = regs->pc; /* determine if the fault lay during a memcpy_user or a memset_user */ - if (__frame->lr == (unsigned long) &__memset_user_error_lr && + if (regs->lr == (unsigned long) &__memset_user_error_lr && (unsigned long) &memset <= pc && pc < (unsigned long) &__memset_end ) { /* the fault occurred in a protected memset * - we search for the return address (in LR) instead of the program counter * - it was probably during a clear_user() */ - return (unsigned long) &__memset_user_error_handler; + regs->pc = (unsigned long) &__memset_user_error_handler; + return 1; } - if (__frame->lr == (unsigned long) &__memcpy_user_error_lr && + if (regs->lr == (unsigned long) &__memcpy_user_error_lr && (unsigned long) &memcpy <= pc && pc < (unsigned long) &__memcpy_end ) { /* the fault occurred in a protected memset * - we search for the return address (in LR) instead of the program counter * - it was probably during a copy_to/from_user() */ - return (unsigned long) &__memcpy_user_error_handler; + regs->pc = (unsigned long) &__memcpy_user_error_handler; + return 1; } extab = search_exception_tables(pc); - if (extab) - return extab->fixup; + if (extab) { + regs->pc = extab->fixup; + return 1; + } return 0; - -} /* end search_exception_table() */ +} diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c index 614a46c413d2..179e79e220e5 100644 --- a/arch/frv/mm/fault.c +++ b/arch/frv/mm/fault.c @@ -33,7 +33,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear { struct vm_area_struct *vma; struct mm_struct *mm; - unsigned long _pme, lrai, lrad, fixup; + unsigned long _pme, lrai, lrad; unsigned long flags = 0; siginfo_t info; pgd_t *pge; @@ -201,10 +201,8 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear no_context: /* are we prepared to handle this kernel fault? */ - if ((fixup = search_exception_table(__frame->pc)) != 0) { - __frame->pc = fixup; + if (fixup_exception(__frame)) return; - } /* * Oops. The kernel tried to access some bad page. We'll have to From 48f666c986671ea2021971bfbe4d74a7ebcf0a44 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 08:46:17 -0400 Subject: [PATCH 34/96] frv: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/frv/Kconfig | 1 + arch/frv/include/asm/uaccess.h | 61 ++++++++++++++-------------------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/arch/frv/Kconfig b/arch/frv/Kconfig index eefd9a4ed156..e489fef111dd 100644 --- a/arch/frv/Kconfig +++ b/arch/frv/Kconfig @@ -16,6 +16,7 @@ config FRV select OLD_SIGACTION select HAVE_DEBUG_STACKOVERFLOW select ARCH_NO_COHERENT_DMA_MMAP + select ARCH_HAS_RAW_COPY_USER config ZONE_DMA bool diff --git a/arch/frv/include/asm/uaccess.h b/arch/frv/include/asm/uaccess.h index 5bcc57de3c95..e4e33b4cd3ae 100644 --- a/arch/frv/include/asm/uaccess.h +++ b/arch/frv/include/asm/uaccess.h @@ -233,23 +233,44 @@ do { \ /* * */ + #define ____force(x) (__force void *)(void __user *)(x) #ifdef CONFIG_MMU extern long __memset_user(void *dst, unsigned long count); extern long __memcpy_user(void *dst, const void *src, unsigned long count); #define __clear_user(dst,count) __memset_user(____force(dst), (count)) -#define __copy_from_user_inatomic(to, from, n) __memcpy_user((to), ____force(from), (n)) -#define __copy_to_user_inatomic(to, from, n) __memcpy_user(____force(to), (from), (n)) #else #define __clear_user(dst,count) (memset(____force(dst), 0, (count)), 0) -#define __copy_from_user_inatomic(to, from, n) (memcpy((to), ____force(from), (n)), 0) -#define __copy_to_user_inatomic(to, from, n) (memcpy(____force(to), (from), (n)), 0) #endif +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ +#ifdef CONFIG_MMU + return __memcpy_user(to, (__force const void *)from, n); +#else + memcpy(to, (__force const void *)from, n); + return 0; +#endif +} + +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ +#ifdef CONFIG_MMU + return __memcpy_user((__force void *)to, from, n); +#else + memcpy((__force void *)to, from, n); + return 0; +#endif +} +#define INLINE_COPY_TO_USER +#define INLINE_COPY_FROM_USER + static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { @@ -258,38 +279,6 @@ clear_user(void __user *to, unsigned long n) return n; } -static inline unsigned long __must_check -__copy_to_user(void __user *to, const void *from, unsigned long n) -{ - might_fault(); - return __copy_to_user_inatomic(to, from, n); -} - -static inline unsigned long -__copy_from_user(void *to, const void __user *from, unsigned long n) -{ - might_fault(); - return __copy_from_user_inatomic(to, from, n); -} - -static inline long copy_from_user(void *to, const void __user *from, unsigned long n) -{ - unsigned long ret = n; - - if (likely(__access_ok(from, n))) - ret = __copy_from_user(to, from, n); - - if (unlikely(ret != 0)) - memset(to + (n - ret), 0, ret); - - return ret; -} - -static inline long copy_to_user(void __user *to, const void *from, unsigned long n) -{ - return likely(__access_ok(to, n)) ? __copy_to_user(to, from, n) : n; -} - extern long strncpy_from_user(char *dst, const char __user *src, long count); extern long strnlen_user(const char __user *src, long count); From 33ab2da801debd947bf0f36848d2b9ea20fad835 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 01:22:03 -0400 Subject: [PATCH 35/96] 8300: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/h8300/Kconfig | 1 + arch/h8300/include/asm/uaccess.h | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/h8300/Kconfig b/arch/h8300/Kconfig index 3ae852507e57..473883417004 100644 --- a/arch/h8300/Kconfig +++ b/arch/h8300/Kconfig @@ -22,6 +22,7 @@ config H8300 select HAVE_ARCH_KGDB select HAVE_ARCH_HASH select CPU_NO_EFFICIENT_FFS + select ARCH_HAS_RAW_COPY_USER config RWSEM_GENERIC_SPINLOCK def_bool y diff --git a/arch/h8300/include/asm/uaccess.h b/arch/h8300/include/asm/uaccess.h index c156a416a279..6f6144a240ce 100644 --- a/arch/h8300/include/asm/uaccess.h +++ b/arch/h8300/include/asm/uaccess.h @@ -3,8 +3,8 @@ #include -static inline __must_check long __copy_from_user(void *to, - const void __user * from, unsigned long n) +static inline __must_check unsigned long +raw_copy_from_user(void *to, const void __user * from, unsigned long n) { if (__builtin_constant_p(n)) { switch(n) { @@ -24,8 +24,8 @@ static inline __must_check long __copy_from_user(void *to, return 0; } -static inline __must_check long __copy_to_user(void __user *to, - const void *from, unsigned long n) +static inline __must_check unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { if (__builtin_constant_p(n)) { switch(n) { @@ -46,6 +46,8 @@ static inline __must_check long __copy_to_user(void __user *to, memcpy((void __force *)to, from, n); return 0; } +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER #include From 35f8acd5c660eb9dbf7a8379724821abb823a3d8 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 36/96] m32r: switch to generic extable.h Signed-off-by: Al Viro --- arch/m32r/include/asm/Kbuild | 1 + arch/m32r/include/asm/uaccess.h | 20 +------------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/arch/m32r/include/asm/Kbuild b/arch/m32r/include/asm/Kbuild index deb298777df2..c000ffac8586 100644 --- a/arch/m32r/include/asm/Kbuild +++ b/arch/m32r/include/asm/Kbuild @@ -2,6 +2,7 @@ generic-y += clkdev.h generic-y += current.h generic-y += exec.h +generic-y += extable.h generic-y += irq_work.h generic-y += kvm_para.h generic-y += mcs_spinlock.h diff --git a/arch/m32r/include/asm/uaccess.h b/arch/m32r/include/asm/uaccess.h index 96b0efdb5f22..e47dcc05a0f2 100644 --- a/arch/m32r/include/asm/uaccess.h +++ b/arch/m32r/include/asm/uaccess.h @@ -109,25 +109,7 @@ static inline int access_ok(int type, const void *addr, unsigned long size) } #endif /* CONFIG_MMU */ -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); +#include /* * These are the main single-value transfer routines. They automatically From 8cd920f26785ce42ec6fb807d40de67fc2eb41ce Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 19 Mar 2017 16:11:13 -0400 Subject: [PATCH 37/96] m32r: get rid of zeroing Signed-off-by: Al Viro --- arch/m32r/include/asm/uaccess.h | 66 +-------------------------------- arch/m32r/lib/usercopy.c | 9 +++-- 2 files changed, 6 insertions(+), 69 deletions(-) diff --git a/arch/m32r/include/asm/uaccess.h b/arch/m32r/include/asm/uaccess.h index e47dcc05a0f2..d5c5e68fa2fb 100644 --- a/arch/m32r/include/asm/uaccess.h +++ b/arch/m32r/include/asm/uaccess.h @@ -460,77 +460,13 @@ do { \ : "r14", "memory"); \ } while (0) -#define __copy_user_zeroing(to, from, size) \ -do { \ - unsigned long __dst, __src, __c; \ - __asm__ __volatile__ ( \ - " mv r14, %0\n" \ - " or r14, %1\n" \ - " beq %0, %1, 9f\n" \ - " beqz %2, 9f\n" \ - " and3 r14, r14, #3\n" \ - " bnez r14, 2f\n" \ - " and3 %2, %2, #3\n" \ - " beqz %3, 2f\n" \ - " addi %0, #-4 ; word_copy \n" \ - " .fillinsn\n" \ - "0: ld r14, @%1+\n" \ - " addi %3, #-1\n" \ - " .fillinsn\n" \ - "1: st r14, @+%0\n" \ - " bnez %3, 0b\n" \ - " beqz %2, 9f\n" \ - " addi %0, #4\n" \ - " .fillinsn\n" \ - "2: ldb r14, @%1 ; byte_copy \n" \ - " .fillinsn\n" \ - "3: stb r14, @%0\n" \ - " addi %1, #1\n" \ - " addi %2, #-1\n" \ - " addi %0, #1\n" \ - " bnez %2, 2b\n" \ - " .fillinsn\n" \ - "9:\n" \ - ".section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "5: addi %3, #1\n" \ - " addi %1, #-4\n" \ - " .fillinsn\n" \ - "6: slli %3, #2\n" \ - " add %2, %3\n" \ - " addi %0, #4\n" \ - " .fillinsn\n" \ - "7: ldi r14, #0 ; store zero \n" \ - " .fillinsn\n" \ - "8: addi %2, #-1\n" \ - " stb r14, @%0 ; ACE? \n" \ - " addi %0, #1\n" \ - " bnez %2, 8b\n" \ - " seth r14, #high(9b)\n" \ - " or3 r14, r14, #low(9b)\n" \ - " jmp r14\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .balign 4\n" \ - " .long 0b,6b\n" \ - " .long 1b,5b\n" \ - " .long 2b,7b\n" \ - " .long 3b,7b\n" \ - ".previous\n" \ - : "=&r" (__dst), "=&r" (__src), "=&r" (size), \ - "=&r" (__c) \ - : "0" (to), "1" (from), "2" (size), "3" (size / 4) \ - : "r14", "memory"); \ -} while (0) - - /* We let the __ versions of copy_from/to_user inline, because they're often * used in fast paths and have only a small space overhead. */ static inline unsigned long __generic_copy_from_user_nocheck(void *to, const void __user *from, unsigned long n) { - __copy_user_zeroing(to, from, n); + __copy_user(to, from, n); return n; } diff --git a/arch/m32r/lib/usercopy.c b/arch/m32r/lib/usercopy.c index fd03f2731f20..6aacf5ba0a58 100644 --- a/arch/m32r/lib/usercopy.c +++ b/arch/m32r/lib/usercopy.c @@ -23,12 +23,13 @@ __generic_copy_to_user(void __user *to, const void *from, unsigned long n) unsigned long __generic_copy_from_user(void *to, const void __user *from, unsigned long n) { + unsigned long ret = n; prefetchw(to); if (access_ok(VERIFY_READ, from, n)) - __copy_user_zeroing(to,from,n); - else - memset(to, 0, n); - return n; + ret = __copy_user(to,from,n); + if (unlikely(ret)) + memset(to + n - ret, 0, ret); + return ret; } From 68acfdcb477abdbf875e33e4a950094c8de08f41 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 14:33:17 -0500 Subject: [PATCH 38/96] m68k: switch to generic extable.h Signed-off-by: Al Viro --- arch/m68k/include/asm/Kbuild | 1 + arch/m68k/include/asm/processor.h | 10 ---------- arch/m68k/include/asm/uaccess.h | 1 + arch/m68k/include/asm/uaccess_mm.h | 18 ------------------ arch/m68k/include/asm/uaccess_no.h | 19 ------------------- arch/m68k/kernel/signal.c | 2 +- arch/m68k/kernel/traps.c | 9 +++++++-- arch/m68k/mm/fault.c | 2 +- 8 files changed, 11 insertions(+), 51 deletions(-) diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild index d4f9ccbfa85c..82005d2ff717 100644 --- a/arch/m68k/include/asm/Kbuild +++ b/arch/m68k/include/asm/Kbuild @@ -5,6 +5,7 @@ generic-y += device.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += futex.h generic-y += hw_irq.h generic-y += ioctl.h diff --git a/arch/m68k/include/asm/processor.h b/arch/m68k/include/asm/processor.h index f5f790c31bf8..77239e81379b 100644 --- a/arch/m68k/include/asm/processor.h +++ b/arch/m68k/include/asm/processor.h @@ -122,16 +122,6 @@ static inline void start_thread(struct pt_regs * regs, unsigned long pc, wrusp(usp); } -#ifdef CONFIG_MMU -extern int handle_kernel_fault(struct pt_regs *regs); -#else -static inline int handle_kernel_fault(struct pt_regs *regs) -{ - /* Any fault in kernel is fatal on non-mmu */ - return 0; -} -#endif - /* Forward declaration, a strange C thing */ struct task_struct; diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h index 3fadc4a93d97..67b3481d6020 100644 --- a/arch/m68k/include/asm/uaccess.h +++ b/arch/m68k/include/asm/uaccess.h @@ -4,6 +4,7 @@ #include #endif +#include #ifdef CONFIG_CPU_HAS_NO_UNALIGNED #include #else diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h index 14054a4e4216..23be968a53fd 100644 --- a/arch/m68k/include/asm/uaccess_mm.h +++ b/arch/m68k/include/asm/uaccess_mm.h @@ -31,24 +31,6 @@ static inline int access_ok(int type, const void __user *addr, #define MOVES "move" #endif -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - extern int __put_user_bad(void); extern int __get_user_bad(void); diff --git a/arch/m68k/include/asm/uaccess_no.h b/arch/m68k/include/asm/uaccess_no.h index e77ce66c14d5..3f7f54d5134c 100644 --- a/arch/m68k/include/asm/uaccess_no.h +++ b/arch/m68k/include/asm/uaccess_no.h @@ -22,25 +22,6 @@ static inline int _access_ok(unsigned long addr, unsigned long size) return 1; } -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - - /* * These are the main single-value transfer routines. They automatically * use the right size if we just have the right pointer type. diff --git a/arch/m68k/kernel/signal.c b/arch/m68k/kernel/signal.c index 093b7c42fb85..6f945bb5ffbd 100644 --- a/arch/m68k/kernel/signal.c +++ b/arch/m68k/kernel/signal.c @@ -88,7 +88,7 @@ static inline int frame_extra_sizes(int f) return frame_size_change[f]; } -int handle_kernel_fault(struct pt_regs *regs) +int fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *fixup; struct pt_regs *tregs; diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c index a926d2c88898..c1cc4e99aa94 100644 --- a/arch/m68k/kernel/traps.c +++ b/arch/m68k/kernel/traps.c @@ -1016,8 +1016,13 @@ asmlinkage void trap_c(struct frame *fp) /* traced a trapping instruction on a 68020/30, * real exception will be executed afterwards. */ - } else if (!handle_kernel_fault(&fp->ptregs)) - bad_super_trap(fp); + return; + } +#ifdef CONFIG_MMU + if (fixup_exception(&fp->ptregs)) + return; +#endif + bad_super_trap(fp); return; } diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index bd66a0b20c6b..2795e4ca09d7 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -32,7 +32,7 @@ int send_fault_sig(struct pt_regs *regs) force_sig_info(siginfo.si_signo, &siginfo, current); } else { - if (handle_kernel_fault(regs)) + if (fixup_exception(regs)) return -1; //if (siginfo.si_signo == SIGBUS) From 7cefa5a05dbda1f0bbbd98e9d2861b09a35cc6ea Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 20 Mar 2017 00:55:24 -0400 Subject: [PATCH 39/96] m68k: get rid of zeroing Signed-off-by: Al Viro --- arch/m68k/include/asm/uaccess_mm.h | 57 +++++++++++++++++++++--------- arch/m68k/lib/uaccess.c | 12 ++----- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h index 23be968a53fd..931bbba449e2 100644 --- a/arch/m68k/include/asm/uaccess_mm.h +++ b/arch/m68k/include/asm/uaccess_mm.h @@ -179,39 +179,55 @@ asm volatile ("\n" \ unsigned long __generic_copy_from_user(void *to, const void __user *from, unsigned long n); unsigned long __generic_copy_to_user(void __user *to, const void *from, unsigned long n); -#define __constant_copy_from_user_asm(res, to, from, tmp, n, s1, s2, s3)\ +#define __suffix0 +#define __suffix1 b +#define __suffix2 w +#define __suffix4 l + +#define ____constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3, s1, s2, s3)\ asm volatile ("\n" \ "1: "MOVES"."#s1" (%2)+,%3\n" \ " move."#s1" %3,(%1)+\n" \ + " .ifnc \""#s2"\",\"\"\n" \ "2: "MOVES"."#s2" (%2)+,%3\n" \ " move."#s2" %3,(%1)+\n" \ " .ifnc \""#s3"\",\"\"\n" \ "3: "MOVES"."#s3" (%2)+,%3\n" \ " move."#s3" %3,(%1)+\n" \ " .endif\n" \ + " .endif\n" \ "4:\n" \ " .section __ex_table,\"a\"\n" \ " .align 4\n" \ " .long 1b,10f\n" \ + " .ifnc \""#s2"\",\"\"\n" \ " .long 2b,20f\n" \ " .ifnc \""#s3"\",\"\"\n" \ " .long 3b,30f\n" \ " .endif\n" \ + " .endif\n" \ " .previous\n" \ "\n" \ " .section .fixup,\"ax\"\n" \ " .even\n" \ - "10: clr."#s1" (%1)+\n" \ - "20: clr."#s2" (%1)+\n" \ + "10: addq.l #"#n1",%0\n" \ + " .ifnc \""#s2"\",\"\"\n" \ + "20: addq.l #"#n2",%0\n" \ " .ifnc \""#s3"\",\"\"\n" \ - "30: clr."#s3" (%1)+\n" \ + "30: addq.l #"#n3",%0\n" \ + " .endif\n" \ " .endif\n" \ - " moveq.l #"#n",%0\n" \ " jra 4b\n" \ " .previous\n" \ : "+d" (res), "+&a" (to), "+a" (from), "=&d" (tmp) \ : : "memory") +#define ___constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3, s1, s2, s3)\ + ____constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3, s1, s2, s3) +#define __constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3) \ + ___constant_copy_from_user_asm(res, to, from, tmp, n1, n2, n3, \ + __suffix##n1, __suffix##n2, __suffix##n3) + static __always_inline unsigned long __constant_copy_from_user(void *to, const void __user *from, unsigned long n) { @@ -219,37 +235,37 @@ __constant_copy_from_user(void *to, const void __user *from, unsigned long n) switch (n) { case 1: - __get_user_asm(res, *(u8 *)to, (u8 __user *)from, u8, b, d, 1); + __constant_copy_from_user_asm(res, to, from, tmp, 1, 0, 0); break; case 2: - __get_user_asm(res, *(u16 *)to, (u16 __user *)from, u16, w, r, 2); + __constant_copy_from_user_asm(res, to, from, tmp, 2, 0, 0); break; case 3: - __constant_copy_from_user_asm(res, to, from, tmp, 3, w, b,); + __constant_copy_from_user_asm(res, to, from, tmp, 2, 1, 0); break; case 4: - __get_user_asm(res, *(u32 *)to, (u32 __user *)from, u32, l, r, 4); + __constant_copy_from_user_asm(res, to, from, tmp, 4, 0, 0); break; case 5: - __constant_copy_from_user_asm(res, to, from, tmp, 5, l, b,); + __constant_copy_from_user_asm(res, to, from, tmp, 4, 1, 0); break; case 6: - __constant_copy_from_user_asm(res, to, from, tmp, 6, l, w,); + __constant_copy_from_user_asm(res, to, from, tmp, 4, 2, 0); break; case 7: - __constant_copy_from_user_asm(res, to, from, tmp, 7, l, w, b); + __constant_copy_from_user_asm(res, to, from, tmp, 4, 2, 1); break; case 8: - __constant_copy_from_user_asm(res, to, from, tmp, 8, l, l,); + __constant_copy_from_user_asm(res, to, from, tmp, 4, 4, 0); break; case 9: - __constant_copy_from_user_asm(res, to, from, tmp, 9, l, l, b); + __constant_copy_from_user_asm(res, to, from, tmp, 4, 4, 1); break; case 10: - __constant_copy_from_user_asm(res, to, from, tmp, 10, l, l, w); + __constant_copy_from_user_asm(res, to, from, tmp, 4, 4, 2); break; case 12: - __constant_copy_from_user_asm(res, to, from, tmp, 12, l, l, l); + __constant_copy_from_user_asm(res, to, from, tmp, 4, 4, 4); break; default: /* we limit the inlined version to 3 moves */ @@ -353,7 +369,14 @@ __constant_copy_to_user(void __user *to, const void *from, unsigned long n) #define __copy_to_user_inatomic __copy_to_user #define __copy_from_user_inatomic __copy_from_user -#define copy_from_user(to, from, n) __copy_from_user(to, from, n) +static inline unsigned long +copy_from_user(void *to, const void __user *from, unsigned long n) +{ + unsigned long res = __copy_from_user_inatomic(to, from, n); + if (unlikely(res)) + memset(to + (n - res), 0, res); + return res; +} #define copy_to_user(to, from, n) __copy_to_user(to, from, n) #define user_addr_max() \ diff --git a/arch/m68k/lib/uaccess.c b/arch/m68k/lib/uaccess.c index a76b73abaf64..7646e461aa62 100644 --- a/arch/m68k/lib/uaccess.c +++ b/arch/m68k/lib/uaccess.c @@ -30,19 +30,13 @@ unsigned long __generic_copy_from_user(void *to, const void __user *from, "6:\n" " .section .fixup,\"ax\"\n" " .even\n" - "10: move.l %0,%3\n" - "7: clr.l (%2)+\n" - " subq.l #1,%3\n" - " jne 7b\n" - " lsl.l #2,%0\n" + "10: lsl.l #2,%0\n" " btst #1,%5\n" " jeq 8f\n" - "30: clr.w (%2)+\n" - " addq.l #2,%0\n" + "30: addq.l #2,%0\n" "8: btst #0,%5\n" " jeq 6b\n" - "50: clr.b (%2)+\n" - " addq.l #1,%0\n" + "50: addq.l #1,%0\n" " jra 6b\n" " .previous\n" "\n" From 29be02eb6fef5f26332eb3e13ead868cff0910af Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 08:50:56 -0400 Subject: [PATCH 40/96] m68k: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/m68k/Kconfig | 1 + arch/m68k/include/asm/uaccess_mm.h | 31 +++++++++++++----------------- arch/m68k/include/asm/uaccess_no.h | 20 +++++++++++++------ 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index d140206d5d29..7d345758ea16 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -22,6 +22,7 @@ config M68K select MODULES_USE_ELF_RELA select OLD_SIGSUSPEND3 select OLD_SIGACTION + select ARCH_HAS_RAW_COPY_USER config RWSEM_GENERIC_SPINLOCK bool diff --git a/arch/m68k/include/asm/uaccess_mm.h b/arch/m68k/include/asm/uaccess_mm.h index 931bbba449e2..ef856ffeffdf 100644 --- a/arch/m68k/include/asm/uaccess_mm.h +++ b/arch/m68k/include/asm/uaccess_mm.h @@ -356,28 +356,23 @@ __constant_copy_to_user(void __user *to, const void *from, unsigned long n) return res; } -#define __copy_from_user(to, from, n) \ -(__builtin_constant_p(n) ? \ - __constant_copy_from_user(to, from, n) : \ - __generic_copy_from_user(to, from, n)) - -#define __copy_to_user(to, from, n) \ -(__builtin_constant_p(n) ? \ - __constant_copy_to_user(to, from, n) : \ - __generic_copy_to_user(to, from, n)) - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + if (__builtin_constant_p(n)) + return __constant_copy_from_user(to, from, n); + return __generic_copy_from_user(to, from, n); +} static inline unsigned long -copy_from_user(void *to, const void __user *from, unsigned long n) +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - unsigned long res = __copy_from_user_inatomic(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; + if (__builtin_constant_p(n)) + return __constant_copy_to_user(to, from, n); + return __generic_copy_to_user(to, from, n); } -#define copy_to_user(to, from, n) __copy_to_user(to, from, n) +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER #define user_addr_max() \ (uaccess_kernel() ? ~0UL : TASK_SIZE) diff --git a/arch/m68k/include/asm/uaccess_no.h b/arch/m68k/include/asm/uaccess_no.h index 3f7f54d5134c..e482c3899ff1 100644 --- a/arch/m68k/include/asm/uaccess_no.h +++ b/arch/m68k/include/asm/uaccess_no.h @@ -101,13 +101,21 @@ extern int __get_user_bad(void); : "=d" (x) \ : "m" (*__ptr(ptr))) -#define copy_from_user(to, from, n) (memcpy(to, from, n), 0) -#define copy_to_user(to, from, n) (memcpy(to, from, n), 0) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + memcpy(to, (__force const void *)from, n); + return 0; +} -#define __copy_from_user(to, from, n) copy_from_user(to, from, n) -#define __copy_to_user(to, from, n) copy_to_user(to, from, n) -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + memcpy((__force void *)to, from, n); + return 0; +} +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER /* * Copy a null terminated string from userspace. From 8b9a7e569569f28d21e5628d5e1f8efe089474f7 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 41/96] metag: switch to generic extable.h Signed-off-by: Al Viro --- arch/metag/include/asm/Kbuild | 1 + arch/metag/include/asm/uaccess.h | 18 +----------------- arch/mn10300/include/asm/Kbuild | 1 + 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/arch/metag/include/asm/Kbuild b/arch/metag/include/asm/Kbuild index f9b9df5d6de9..8f940553a579 100644 --- a/arch/metag/include/asm/Kbuild +++ b/arch/metag/include/asm/Kbuild @@ -8,6 +8,7 @@ generic-y += dma.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += futex.h diff --git a/arch/metag/include/asm/uaccess.h b/arch/metag/include/asm/uaccess.h index 83de7554d2b3..92ba79f63e7c 100644 --- a/arch/metag/include/asm/uaccess.h +++ b/arch/metag/include/asm/uaccess.h @@ -52,23 +52,7 @@ static inline int verify_area(int type, const void *addr, unsigned long size) return access_ok(type, addr, size) ? 0 : -EFAULT; } -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ -struct exception_table_entry { - unsigned long insn, fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); +#include /* * These are the main single-value transfer routines. They automatically diff --git a/arch/mn10300/include/asm/Kbuild b/arch/mn10300/include/asm/Kbuild index 97f64c723a0c..ed810e7206e8 100644 --- a/arch/mn10300/include/asm/Kbuild +++ b/arch/mn10300/include/asm/Kbuild @@ -2,6 +2,7 @@ generic-y += barrier.h generic-y += clkdev.h generic-y += exec.h +generic-y += extable.h generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h From 2c3054176e2ef3f49be9ec8b6fd06d6a47156ff5 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 28 Dec 2016 01:26:29 -0500 Subject: [PATCH 42/96] metag: kill verify_area() Deprecated in 2.6.12, killed in 2.6.13. Time to end that depravity, let's bury the body... Signed-off-by: Al Viro --- arch/metag/include/asm/uaccess.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/metag/include/asm/uaccess.h b/arch/metag/include/asm/uaccess.h index 92ba79f63e7c..2613d91bde1a 100644 --- a/arch/metag/include/asm/uaccess.h +++ b/arch/metag/include/asm/uaccess.h @@ -47,11 +47,6 @@ static inline int __access_ok(unsigned long addr, unsigned long size) #define access_ok(type, addr, size) __access_ok((unsigned long)(addr), \ (unsigned long)(size)) -static inline int verify_area(int type, const void *addr, unsigned long size) -{ - return access_ok(type, addr, size) ? 0 : -EFAULT; -} - #include /* From 730132e7e52210cbbfa95af67f5e23a63199c13d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 14:34:44 -0500 Subject: [PATCH 43/96] microblaze: switch to generic extable.h Signed-off-by: Al Viro --- arch/microblaze/include/asm/Kbuild | 1 + arch/microblaze/include/asm/uaccess.h | 17 +---------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild index 1732ec13b211..56830ff65333 100644 --- a/arch/microblaze/include/asm/Kbuild +++ b/arch/microblaze/include/asm/Kbuild @@ -3,6 +3,7 @@ generic-y += barrier.h generic-y += clkdev.h generic-y += device.h generic-y += exec.h +generic-y += extable.h generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index b132cd301e66..2c39375ac871 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -17,6 +17,7 @@ #include #include #include +#include #include /* @@ -47,22 +48,6 @@ # define segment_eq(a, b) ((a).seg == (b).seg) -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ -struct exception_table_entry { - unsigned long insn, fixup; -}; - #ifndef CONFIG_MMU /* Check against bounds of physical memory */ From 6c03905a63165921411fbe91c167186b13879c8d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 44/96] mn10300: switch to generic extable.h Signed-off-by: Al Viro --- arch/mn10300/include/asm/Kbuild | 1 + arch/mn10300/include/asm/uaccess.h | 21 +-------------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/arch/mn10300/include/asm/Kbuild b/arch/mn10300/include/asm/Kbuild index 97f64c723a0c..ed810e7206e8 100644 --- a/arch/mn10300/include/asm/Kbuild +++ b/arch/mn10300/include/asm/Kbuild @@ -2,6 +2,7 @@ generic-y += barrier.h generic-y += clkdev.h generic-y += exec.h +generic-y += extable.h generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h diff --git a/arch/mn10300/include/asm/uaccess.h b/arch/mn10300/include/asm/uaccess.h index 2da7b0fed4aa..2eeaa8c5d92c 100644 --- a/arch/mn10300/include/asm/uaccess.h +++ b/arch/mn10300/include/asm/uaccess.h @@ -66,26 +66,7 @@ static inline int ___range_ok(unsigned long addr, unsigned int size) #define access_ok(type, addr, size) (__range_ok((addr), (size)) == 0) #define __access_ok(addr, size) (__range_ok((addr), (size)) == 0) -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - -/* Returns 0 if exception not found and fixup otherwise. */ -extern int fixup_exception(struct pt_regs *regs); +#include #define put_user(x, ptr) __put_user_check((x), (ptr), sizeof(*(ptr))) #define get_user(x, ptr) __get_user_check((x), (ptr), sizeof(*(ptr))) From 4df3715efe5ed1e77344c6d5dcbfd75017a2f8b5 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 20 Mar 2017 20:45:41 -0400 Subject: [PATCH 45/96] mn10300: get rid of zeroing Signed-off-by: Al Viro --- arch/mn10300/include/asm/uaccess.h | 152 +++-------------------------- arch/mn10300/lib/usercopy.c | 11 ++- 2 files changed, 21 insertions(+), 142 deletions(-) diff --git a/arch/mn10300/include/asm/uaccess.h b/arch/mn10300/include/asm/uaccess.h index 2eeaa8c5d92c..1c35be3d8fef 100644 --- a/arch/mn10300/include/asm/uaccess.h +++ b/arch/mn10300/include/asm/uaccess.h @@ -275,55 +275,19 @@ do { \ } \ } while (0) -#define __copy_user_zeroing(to, from, size) \ -do { \ - if (size) { \ - void *__to = to; \ - const void *__from = from; \ - int w; \ - asm volatile( \ - "0: movbu (%0),%3;\n" \ - "1: movbu %3,(%1);\n" \ - " inc %0;\n" \ - " inc %1;\n" \ - " add -1,%2;\n" \ - " bne 0b;\n" \ - "2:\n" \ - " .section .fixup,\"ax\"\n" \ - "3:\n" \ - " mov %2,%0\n" \ - " clr %3\n" \ - "4: movbu %3,(%1);\n" \ - " inc %1;\n" \ - " add -1,%2;\n" \ - " bne 4b;\n" \ - " mov %0,%2\n" \ - " jmp 2b\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign 4\n" \ - " .long 0b,3b\n" \ - " .long 1b,3b\n" \ - " .previous\n" \ - : "=a"(__from), "=a"(__to), "=r"(size), "=&r"(w)\ - : "0"(__from), "1"(__to), "2"(size) \ - : "cc", "memory"); \ - } \ -} while (0) - /* We let the __ versions of copy_from/to_user inline, because they're often * used in fast paths and have only a small space overhead. */ static inline -unsigned long __generic_copy_from_user_nocheck(void *to, const void *from, +unsigned long __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n) { - __copy_user_zeroing(to, from, n); + __copy_user(to, from, n); return n; } static inline -unsigned long __generic_copy_to_user_nocheck(void *to, const void *from, +unsigned long __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n) { __copy_user(to, from, n); @@ -331,110 +295,24 @@ unsigned long __generic_copy_to_user_nocheck(void *to, const void *from, } -#if 0 -#error "don't use - these macros don't increment to & from pointers" -/* Optimize just a little bit when we know the size of the move. */ -#define __constant_copy_user(to, from, size) \ -do { \ - asm volatile( \ - " mov %0,a0;\n" \ - "0: movbu (%1),d3;\n" \ - "1: movbu d3,(%2);\n" \ - " add -1,a0;\n" \ - " bne 0b;\n" \ - "2:;" \ - ".section .fixup,\"ax\"\n" \ - "3: jmp 2b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .balign 4\n" \ - " .long 0b,3b\n" \ - " .long 1b,3b\n" \ - ".previous" \ - : \ - : "d"(size), "d"(to), "d"(from) \ - : "d3", "a0"); \ -} while (0) - -/* Optimize just a little bit when we know the size of the move. */ -#define __constant_copy_user_zeroing(to, from, size) \ -do { \ - asm volatile( \ - " mov %0,a0;\n" \ - "0: movbu (%1),d3;\n" \ - "1: movbu d3,(%2);\n" \ - " add -1,a0;\n" \ - " bne 0b;\n" \ - "2:;" \ - ".section .fixup,\"ax\"\n" \ - "3: jmp 2b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .balign 4\n" \ - " .long 0b,3b\n" \ - " .long 1b,3b\n" \ - ".previous" \ - : \ - : "d"(size), "d"(to), "d"(from) \ - : "d3", "a0"); \ -} while (0) - -static inline -unsigned long __constant_copy_to_user(void *to, const void *from, - unsigned long n) -{ - if (access_ok(VERIFY_WRITE, to, n)) - __constant_copy_user(to, from, n); - return n; -} - -static inline -unsigned long __constant_copy_from_user(void *to, const void *from, - unsigned long n) -{ - if (access_ok(VERIFY_READ, from, n)) - __constant_copy_user_zeroing(to, from, n); - return n; -} - -static inline -unsigned long __constant_copy_to_user_nocheck(void *to, const void *from, - unsigned long n) -{ - __constant_copy_user(to, from, n); - return n; -} - -static inline -unsigned long __constant_copy_from_user_nocheck(void *to, const void *from, - unsigned long n) -{ - __constant_copy_user_zeroing(to, from, n); - return n; -} -#endif - extern unsigned long __generic_copy_to_user(void __user *, const void *, unsigned long); extern unsigned long __generic_copy_from_user(void *, const void __user *, unsigned long); -#define __copy_to_user_inatomic(to, from, n) \ - __generic_copy_to_user_nocheck((to), (from), (n)) -#define __copy_from_user_inatomic(to, from, n) \ - __generic_copy_from_user_nocheck((to), (from), (n)) +static inline unsigned long __copy_to_user(void __user *to, const void *from, + unsigned long n) +{ + might_fault(); + return __copy_to_user_inatomic(to, from, n); +} -#define __copy_to_user(to, from, n) \ -({ \ - might_fault(); \ - __copy_to_user_inatomic((to), (from), (n)); \ -}) - -#define __copy_from_user(to, from, n) \ -({ \ - might_fault(); \ - __copy_from_user_inatomic((to), (from), (n)); \ -}) +static inline unsigned long __copy_from_user(void *to, const void __user *from, + unsigned long n) +{ + might_fault(); + return __copy_from_user_inatomic(to, from, n); +} #define copy_to_user(to, from, n) __generic_copy_to_user((to), (from), (n)) diff --git a/arch/mn10300/lib/usercopy.c b/arch/mn10300/lib/usercopy.c index ce8899e5e171..b48af8d4f38d 100644 --- a/arch/mn10300/lib/usercopy.c +++ b/arch/mn10300/lib/usercopy.c @@ -22,11 +22,12 @@ __generic_copy_to_user(void *to, const void *from, unsigned long n) unsigned long __generic_copy_from_user(void *to, const void *from, unsigned long n) { - if (access_ok(VERIFY_READ, from, n)) - __copy_user_zeroing(to, from, n); - else - memset(to, 0, n); - return n; + unsigned long res = n; + if (access_ok(VERIFY_READ, from, res)) + __copy_user(to, from, res); + if (unlikely(res)) + memset(to + n - res, 0, res); + return res; } /* From 19dbf70c5acce91c6cacfd0983c2b062644fa7a4 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 12:06:57 -0400 Subject: [PATCH 46/96] mn10300: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/mn10300/Kconfig | 1 + arch/mn10300/include/asm/uaccess.h | 37 ++++------------------------- arch/mn10300/kernel/mn10300_ksyms.c | 2 -- arch/mn10300/lib/usercopy.c | 19 --------------- 4 files changed, 5 insertions(+), 54 deletions(-) diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig index 38e3494bfb63..a96f3dcb0119 100644 --- a/arch/mn10300/Kconfig +++ b/arch/mn10300/Kconfig @@ -16,6 +16,7 @@ config MN10300 select OLD_SIGACTION select HAVE_DEBUG_STACKOVERFLOW select ARCH_NO_COHERENT_DMA_MMAP + select ARCH_HAS_RAW_COPY_USER config AM33_2 def_bool n diff --git a/arch/mn10300/include/asm/uaccess.h b/arch/mn10300/include/asm/uaccess.h index 1c35be3d8fef..c6966474827f 100644 --- a/arch/mn10300/include/asm/uaccess.h +++ b/arch/mn10300/include/asm/uaccess.h @@ -275,49 +275,20 @@ do { \ } \ } while (0) -/* We let the __ versions of copy_from/to_user inline, because they're often - * used in fast paths and have only a small space overhead. - */ -static inline -unsigned long __copy_from_user_inatomic(void *to, const void __user *from, - unsigned long n) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { __copy_user(to, from, n); return n; } -static inline -unsigned long __copy_to_user_inatomic(void __user *to, const void *from, - unsigned long n) +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { __copy_user(to, from, n); return n; } - -extern unsigned long __generic_copy_to_user(void __user *, const void *, - unsigned long); -extern unsigned long __generic_copy_from_user(void *, const void __user *, - unsigned long); - -static inline unsigned long __copy_to_user(void __user *to, const void *from, - unsigned long n) -{ - might_fault(); - return __copy_to_user_inatomic(to, from, n); -} - -static inline unsigned long __copy_from_user(void *to, const void __user *from, - unsigned long n) -{ - might_fault(); - return __copy_from_user_inatomic(to, from, n); -} - - -#define copy_to_user(to, from, n) __generic_copy_to_user((to), (from), (n)) -#define copy_from_user(to, from, n) __generic_copy_from_user((to), (from), (n)) - extern long strncpy_from_user(char *dst, const char __user *src, long count); extern long __strncpy_from_user(char *dst, const char __user *src, long count); extern long strnlen_user(const char __user *str, long n); diff --git a/arch/mn10300/kernel/mn10300_ksyms.c b/arch/mn10300/kernel/mn10300_ksyms.c index ec6c4f8f93a6..5e9f919635f0 100644 --- a/arch/mn10300/kernel/mn10300_ksyms.c +++ b/arch/mn10300/kernel/mn10300_ksyms.c @@ -26,8 +26,6 @@ EXPORT_SYMBOL(strncpy_from_user); EXPORT_SYMBOL(__strncpy_from_user); EXPORT_SYMBOL(clear_user); EXPORT_SYMBOL(__clear_user); -EXPORT_SYMBOL(__generic_copy_from_user); -EXPORT_SYMBOL(__generic_copy_to_user); EXPORT_SYMBOL(strnlen_user); extern u64 __ashrdi3(u64, unsigned); diff --git a/arch/mn10300/lib/usercopy.c b/arch/mn10300/lib/usercopy.c index b48af8d4f38d..cece1799cc32 100644 --- a/arch/mn10300/lib/usercopy.c +++ b/arch/mn10300/lib/usercopy.c @@ -11,25 +11,6 @@ */ #include -unsigned long -__generic_copy_to_user(void *to, const void *from, unsigned long n) -{ - if (access_ok(VERIFY_WRITE, to, n)) - __copy_user(to, from, n); - return n; -} - -unsigned long -__generic_copy_from_user(void *to, const void *from, unsigned long n) -{ - unsigned long res = n; - if (access_ok(VERIFY_READ, from, res)) - __copy_user(to, from, res); - if (unlikely(res)) - memset(to + n - res, 0, res); - return res; -} - /* * Copy a null terminated string from userspace. */ From 981db65b5229edb5ab336e3e23758791a1aa0b03 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 47/96] nios2: switch to generic extable.h Signed-off-by: Al Viro --- arch/nios2/include/asm/Kbuild | 1 + arch/nios2/include/asm/uaccess.h | 19 +------------------ 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/arch/nios2/include/asm/Kbuild b/arch/nios2/include/asm/Kbuild index aaa3c218b56c..87e70f2b463f 100644 --- a/arch/nios2/include/asm/Kbuild +++ b/arch/nios2/include/asm/Kbuild @@ -13,6 +13,7 @@ generic-y += dma.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += ftrace.h diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index 198bbf15f644..ed06750a6c92 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -17,24 +17,7 @@ #include -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ -struct exception_table_entry { - unsigned long insn; - unsigned long fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); +#include /* * Segment stuff From de51d6cc2cf9daa9b5155faefb714f5b477cf160 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 22 Mar 2017 13:08:32 -0400 Subject: [PATCH 48/96] nios2: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/nios2/Kconfig | 1 + arch/nios2/include/asm/uaccess.h | 31 ++++++------------------------- arch/nios2/mm/uaccess.c | 16 ++++++++-------- 3 files changed, 15 insertions(+), 33 deletions(-) diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index 51a56c8b04b4..45b4727e3136 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -16,6 +16,7 @@ config NIOS2 select SPARSE_IRQ select USB_ARCH_HAS_HCD if USB_SUPPORT select CPU_NO_EFFICIENT_FFS + select ARCH_HAS_RAW_COPY_USER config GENERIC_CSUM def_bool y diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index ed06750a6c92..727bd9504899 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -73,36 +73,17 @@ static inline unsigned long __must_check clear_user(void __user *to, return __clear_user(to, n); } -extern long __copy_from_user(void *to, const void __user *from, - unsigned long n); -extern long __copy_to_user(void __user *to, const void *from, unsigned long n); - -static inline long copy_from_user(void *to, const void __user *from, - unsigned long n) -{ - unsigned long res = n; - if (access_ok(VERIFY_READ, from, n)) - res = __copy_from_user(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; -} - -static inline long copy_to_user(void __user *to, const void *from, - unsigned long n) -{ - if (!access_ok(VERIFY_WRITE, to, n)) - return n; - return __copy_to_user(to, from, n); -} +extern unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n); +extern unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n); +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER extern long strncpy_from_user(char *__to, const char __user *__from, long __len); extern long strnlen_user(const char __user *s, long n); -#define __copy_from_user_inatomic __copy_from_user -#define __copy_to_user_inatomic __copy_to_user - /* Optimized macros */ #define __get_user_asm(val, insn, addr, err) \ { \ diff --git a/arch/nios2/mm/uaccess.c b/arch/nios2/mm/uaccess.c index 7663e156ff4f..804983317766 100644 --- a/arch/nios2/mm/uaccess.c +++ b/arch/nios2/mm/uaccess.c @@ -10,9 +10,9 @@ #include #include -asm(".global __copy_from_user\n" - " .type __copy_from_user, @function\n" - "__copy_from_user:\n" +asm(".global raw_copy_from_user\n" + " .type raw_copy_from_user, @function\n" + "raw_copy_from_user:\n" " movi r2,7\n" " mov r3,r4\n" " bge r2,r6,1f\n" @@ -65,12 +65,12 @@ asm(".global __copy_from_user\n" ".word 7b,13b\n" ".previous\n" ); -EXPORT_SYMBOL(__copy_from_user); +EXPORT_SYMBOL(raw_copy_from_user); asm( - " .global __copy_to_user\n" - " .type __copy_to_user, @function\n" - "__copy_to_user:\n" + " .global raw_copy_to_user\n" + " .type raw_copy_to_user, @function\n" + "raw_copy_to_user:\n" " movi r2,7\n" " mov r3,r4\n" " bge r2,r6,1f\n" @@ -127,7 +127,7 @@ asm( ".word 11b,13b\n" ".word 12b,13b\n" ".previous\n"); -EXPORT_SYMBOL(__copy_to_user); +EXPORT_SYMBOL(raw_copy_to_user); long strncpy_from_user(char *__to, const char __user *__from, long __len) { From 6c36a4eb9aea4081a4286afaf23cf8cce30f3e6f Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 14:34:44 -0500 Subject: [PATCH 49/96] openrisc: switch to generic extable.h Signed-off-by: Al Viro --- arch/openrisc/include/asm/Kbuild | 1 + arch/openrisc/include/asm/uaccess.h | 18 +----------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild index fb01873a5aad..df8e2f7bc7dd 100644 --- a/arch/openrisc/include/asm/Kbuild +++ b/arch/openrisc/include/asm/Kbuild @@ -16,6 +16,7 @@ generic-y += dma.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += ftrace.h diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index 227af1acb8bd..92236c9e7c7d 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -25,6 +25,7 @@ #include #include #include +#include /* * The fs value determines whether argument validity checking should be @@ -60,23 +61,6 @@ #define access_ok(type, addr, size) \ __range_ok((unsigned long)addr, (unsigned long)size) -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn, fixup; -}; - /* * These are the main single-value transfer routines. They automatically * use the right size if we just have the right pointer type. From ab89866be3a02008880719f8928ff4fd960f99b6 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 22 Mar 2017 13:13:15 -0400 Subject: [PATCH 50/96] openrisc: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/openrisc/Kconfig | 1 + arch/openrisc/include/asm/uaccess.h | 28 ++++++---------------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index 1e95920b0737..38954181fa96 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -28,6 +28,7 @@ config OPENRISC select OR1K_PIC select CPU_NO_EFFICIENT_FFS if !OPENRISC_HAVE_INST_FF1 select NO_BOOTMEM + select ARCH_HAS_RAW_COPY_USER config MMU def_bool y diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index 92236c9e7c7d..96fc425271c2 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -236,34 +236,18 @@ do { \ extern unsigned long __must_check __copy_tofrom_user(void *to, const void *from, unsigned long size); - -#define __copy_from_user(to, from, size) \ - __copy_tofrom_user(to, from, size) -#define __copy_to_user(to, from, size) \ - __copy_tofrom_user(to, from, size) - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - static inline unsigned long -copy_from_user(void *to, const void *from, unsigned long n) +raw_copy_from_user(void *to, const void __user *from, unsigned long size) { - unsigned long res = n; - - if (likely(access_ok(VERIFY_READ, from, n))) - res = __copy_tofrom_user(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; + return __copy_tofrom_user(to, (__force const void *)from, size); } - static inline unsigned long -copy_to_user(void *to, const void *from, unsigned long n) +raw_copy_to_user(void *to, const void __user *from, unsigned long size) { - if (likely(access_ok(VERIFY_WRITE, to, n))) - n = __copy_tofrom_user(to, from, n); - return n; + return __copy_tofrom_user((__force void *)to, from, size); } +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER extern unsigned long __clear_user(void *addr, unsigned long size); From 527b5baeada26f9660722b4a498d3855d90f276c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 26 Dec 2016 00:50:06 -0500 Subject: [PATCH 51/96] powerpc: switch to extable.h Signed-off-by: Al Viro --- arch/powerpc/include/asm/extable.h | 29 +++++++++++++++++++++++++++++ arch/powerpc/include/asm/uaccess.h | 26 +------------------------- 2 files changed, 30 insertions(+), 25 deletions(-) create mode 100644 arch/powerpc/include/asm/extable.h diff --git a/arch/powerpc/include/asm/extable.h b/arch/powerpc/include/asm/extable.h new file mode 100644 index 000000000000..07cc45cd86d9 --- /dev/null +++ b/arch/powerpc/include/asm/extable.h @@ -0,0 +1,29 @@ +#ifndef _ARCH_POWERPC_EXTABLE_H +#define _ARCH_POWERPC_EXTABLE_H + +/* + * The exception table consists of pairs of relative addresses: the first is + * the address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out what + * to do. + * + * All the routines below use bits of fixup code that are out of line with the + * main instruction path. This means when everything is well, we don't even + * have to jump over them. Further, they do not intrude on our cache or tlb + * entries. + */ + +#define ARCH_HAS_RELATIVE_EXTABLE + +struct exception_table_entry { + int insn; + int fixup; +}; + +static inline unsigned long extable_fixup(const struct exception_table_entry *x) +{ + return (unsigned long)&x->fixup + x->fixup; +} + +#endif diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 3904040a3542..b19883f6273c 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -5,6 +5,7 @@ #include #include #include +#include /* * The fs value determines whether argument validity checking should be @@ -55,31 +56,6 @@ (__chk_user_ptr(addr), \ __access_ok((__force unsigned long)(addr), (size), get_fs())) -/* - * The exception table consists of pairs of relative addresses: the first is - * the address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out what - * to do. - * - * All the routines below use bits of fixup code that are out of line with the - * main instruction path. This means when everything is well, we don't even - * have to jump over them. Further, they do not intrude on our cache or tlb - * entries. - */ - -#define ARCH_HAS_RELATIVE_EXTABLE - -struct exception_table_entry { - int insn; - int fixup; -}; - -static inline unsigned long extable_fixup(const struct exception_table_entry *x) -{ - return (unsigned long)&x->fixup + x->fixup; -} - /* * These are the main single-value transfer routines. They automatically * use the right size if we just have the right pointer type. From e70f1d59be747a959c240bf2fe2ea9489b629496 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 26 Dec 2016 00:48:37 -0500 Subject: [PATCH 52/96] s390: switch to extable.h Signed-off-by: Al Viro --- arch/s390/include/asm/extable.h | 28 ++++++++++++++++++++++++++++ arch/s390/include/asm/uaccess.h | 26 +------------------------- 2 files changed, 29 insertions(+), 25 deletions(-) create mode 100644 arch/s390/include/asm/extable.h diff --git a/arch/s390/include/asm/extable.h b/arch/s390/include/asm/extable.h new file mode 100644 index 000000000000..16cfe2d62eeb --- /dev/null +++ b/arch/s390/include/asm/extable.h @@ -0,0 +1,28 @@ +#ifndef __S390_EXTABLE_H +#define __S390_EXTABLE_H +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry +{ + int insn, fixup; +}; + +static inline unsigned long extable_fixup(const struct exception_table_entry *x) +{ + return (unsigned long)&x->fixup + x->fixup; +} + +#define ARCH_HAS_RELATIVE_EXTABLE + +#endif diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 7228ed8da67d..29f5bf24e5fa 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -14,6 +14,7 @@ */ #include #include +#include /* @@ -59,31 +60,6 @@ static inline int __range_ok(unsigned long addr, unsigned long size) #define access_ok(type, addr, size) __access_ok(addr, size) -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry -{ - int insn, fixup; -}; - -static inline unsigned long extable_fixup(const struct exception_table_entry *x) -{ - return (unsigned long)&x->fixup + x->fixup; -} - -#define ARCH_HAS_RELATIVE_EXTABLE - /** * __copy_from_user: - Copy a block of data from user space, with less checking. * @to: Destination address, in kernel space. From 271abdc367e204821e19d71ae2bbc444059c8496 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 53/96] score: switch to generic extable.h Signed-off-by: Al Viro --- arch/score/include/asm/Kbuild | 1 + arch/score/include/asm/extable.h | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 arch/score/include/asm/extable.h diff --git a/arch/score/include/asm/Kbuild b/arch/score/include/asm/Kbuild index 926943a49ea5..e3a8d0f96652 100644 --- a/arch/score/include/asm/Kbuild +++ b/arch/score/include/asm/Kbuild @@ -4,6 +4,7 @@ header-y += generic-y += barrier.h generic-y += clkdev.h generic-y += current.h +generic-y += extable.h generic-y += irq_work.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h diff --git a/arch/score/include/asm/extable.h b/arch/score/include/asm/extable.h deleted file mode 100644 index c4423ccf830d..000000000000 --- a/arch/score/include/asm/extable.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef _ASM_SCORE_EXTABLE_H -#define _ASM_SCORE_EXTABLE_H - -struct exception_table_entry { - unsigned long insn; - unsigned long fixup; -}; - -struct pt_regs; -extern int fixup_exception(struct pt_regs *regs); -#endif From 4b4554f6d562c15159b053fad4146ced2201bc4c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 14:50:40 -0500 Subject: [PATCH 54/96] score: it's "VERIFY_WRITE", not "VERFITY_WRITE"... ... and the only reason it worked is that access_ok() discards the first argument before parser even gets a chance of looking at it. Still, no point keeping that typo. Signed-off-by: Al Viro --- arch/score/include/asm/uaccess.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/score/include/asm/uaccess.h b/arch/score/include/asm/uaccess.h index 7a6c6982420a..79125fa6ed43 100644 --- a/arch/score/include/asm/uaccess.h +++ b/arch/score/include/asm/uaccess.h @@ -347,7 +347,7 @@ static inline unsigned long copy_in_user(void *to, const void *from, unsigned long len) { if (access_ok(VERIFY_READ, from, len) && - access_ok(VERFITY_WRITE, to, len)) + access_ok(VERIFY_WRITE, to, len)) return __copy_tofrom_user(to, from, len); } From dc1425352325bf8a56f5d8ecd1258671808bcadd Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 22 Mar 2017 13:26:57 -0400 Subject: [PATCH 55/96] score: switch to RAW_COPY_USER ... and kill copy_in_user() - it's not a biarch architecture, to start with Signed-off-by: Al Viro --- arch/score/Kconfig | 1 + arch/score/include/asm/uaccess.h | 54 ++++---------------------------- 2 files changed, 7 insertions(+), 48 deletions(-) diff --git a/arch/score/Kconfig b/arch/score/Kconfig index 507d63181389..4d241cfedc3b 100644 --- a/arch/score/Kconfig +++ b/arch/score/Kconfig @@ -15,6 +15,7 @@ config SCORE select MODULES_USE_ELF_REL select CLONE_BACKWARDS select CPU_NO_EFFICIENT_FFS + select ARCH_HAS_RAW_COPY_USER choice prompt "System type" diff --git a/arch/score/include/asm/uaccess.h b/arch/score/include/asm/uaccess.h index 79125fa6ed43..916e5dbf0bfd 100644 --- a/arch/score/include/asm/uaccess.h +++ b/arch/score/include/asm/uaccess.h @@ -295,61 +295,19 @@ extern void __put_user_unknown(void); extern int __copy_tofrom_user(void *to, const void *from, unsigned long len); static inline unsigned long -copy_from_user(void *to, const void *from, unsigned long len) +raw_copy_from_user(void *to, const void __user *from, unsigned long len) { - unsigned long res = len; - - if (likely(access_ok(VERIFY_READ, from, len))) - res = __copy_tofrom_user(to, from, len); - - if (unlikely(res)) - memset(to + (len - res), 0, res); - - return res; + return __copy_tofrom_user(to, (__force const void *)from, len); } static inline unsigned long -copy_to_user(void *to, const void *from, unsigned long len) +raw_copy_to_user(void __user *to, const void *from, unsigned long len) { - if (likely(access_ok(VERIFY_WRITE, to, len))) - len = __copy_tofrom_user(to, from, len); - - return len; + return __copy_tofrom_user((__force void *)to, from, len); } -static inline unsigned long -__copy_from_user(void *to, const void *from, unsigned long len) -{ - unsigned long left = __copy_tofrom_user(to, from, len); - if (unlikely(left)) - memset(to + (len - left), 0, left); - return left; -} - -#define __copy_to_user(to, from, len) \ - __copy_tofrom_user((to), (from), (len)) - -static inline unsigned long -__copy_to_user_inatomic(void *to, const void *from, unsigned long len) -{ - return __copy_to_user(to, from, len); -} - -static inline unsigned long -__copy_from_user_inatomic(void *to, const void *from, unsigned long len) -{ - return __copy_tofrom_user(to, from, len); -} - -#define __copy_in_user(to, from, len) __copy_tofrom_user(to, from, len) - -static inline unsigned long -copy_in_user(void *to, const void *from, unsigned long len) -{ - if (access_ok(VERIFY_READ, from, len) && - access_ok(VERIFY_WRITE, to, len)) - return __copy_tofrom_user(to, from, len); -} +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER /* * __clear_user: - Zero a block of memory in user space, with less checking. From bcd541d9a2f43fbf08d2e909fb8247bed3dac750 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 14:49:42 -0500 Subject: [PATCH 56/96] sh: switch to extable.h Signed-off-by: Al Viro --- arch/sh/include/asm/extable.h | 10 ++++++++++ arch/sh/include/asm/uaccess.h | 23 +---------------------- 2 files changed, 11 insertions(+), 22 deletions(-) create mode 100644 arch/sh/include/asm/extable.h diff --git a/arch/sh/include/asm/extable.h b/arch/sh/include/asm/extable.h new file mode 100644 index 000000000000..df2ee2fcb8d3 --- /dev/null +++ b/arch/sh/include/asm/extable.h @@ -0,0 +1,10 @@ +#ifndef __ASM_SH_EXTABLE_H +#define __ASM_SH_EXTABLE_H + +#include + +#if defined(CONFIG_SUPERH64) && defined(CONFIG_MMU) +#define ARCH_HAS_SEARCH_EXTABLE +#endif + +#endif diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index 89a28dfbabfa..5cb0c5484efb 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -2,6 +2,7 @@ #define __ASM_SH_UACCESS_H #include +#include #define __addr_ok(addr) \ ((unsigned long __force)(addr) < current_thread_info()->addr_limit.seg) @@ -166,28 +167,6 @@ copy_to_user(void __user *to, const void *from, unsigned long n) return __copy_size; } -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ -struct exception_table_entry { - unsigned long insn, fixup; -}; - -#if defined(CONFIG_SUPERH64) && defined(CONFIG_MMU) -#define ARCH_HAS_SEARCH_EXTABLE -#endif - -int fixup_exception(struct pt_regs *regs); - extern void *set_exception_table_vec(unsigned int vec, void *handler); static inline void *set_exception_table_evt(unsigned int evt, void *handler) From f98f48ee7cd473a077f8500dbf3d0a1f9af73003 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 14:43:13 -0400 Subject: [PATCH 57/96] sh: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/sh/Kconfig | 1 + arch/sh/include/asm/uaccess.h | 36 ++++------------------------------- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index ee086958b2b2..c689645eb8b7 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -48,6 +48,7 @@ config SUPERH select HAVE_ARCH_AUDITSYSCALL select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_NMI + select ARCH_HAS_RAW_COPY_USER help The SuperH is a RISC processor targeted for use in embedded systems and consumer electronics; it was also used in the Sega Dreamcast diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index 5cb0c5484efb..2722b61b2283 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -108,19 +108,18 @@ extern __must_check long strnlen_user(const char __user *str, long n); __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n); static __always_inline unsigned long -__copy_from_user(void *to, const void __user *from, unsigned long n) +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { return __copy_user(to, (__force void *)from, n); } static __always_inline unsigned long __must_check -__copy_to_user(void __user *to, const void *from, unsigned long n) +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { return __copy_user((__force void *)to, from, n); } - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER /* * Clear the area and return remaining number of bytes @@ -140,33 +139,6 @@ __kernel_size_t __clear_user(void *addr, __kernel_size_t size); __cl_size; \ }) -static inline unsigned long -copy_from_user(void *to, const void __user *from, unsigned long n) -{ - unsigned long __copy_from = (unsigned long) from; - __kernel_size_t __copy_size = (__kernel_size_t) n; - - if (__copy_size && __access_ok(__copy_from, __copy_size)) - __copy_size = __copy_user(to, from, __copy_size); - - if (unlikely(__copy_size)) - memset(to + (n - __copy_size), 0, __copy_size); - - return __copy_size; -} - -static inline unsigned long -copy_to_user(void __user *to, const void *from, unsigned long n) -{ - unsigned long __copy_to = (unsigned long) to; - __kernel_size_t __copy_size = (__kernel_size_t) n; - - if (__copy_size && __access_ok(__copy_to, __copy_size)) - return __copy_user(to, from, __copy_size); - - return __copy_size; -} - extern void *set_exception_table_vec(unsigned int vec, void *handler); static inline void *set_exception_table_evt(unsigned int evt, void *handler) From 1333eb785614779b2000e547bd32aef85c22ca7a Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 20:27:33 -0500 Subject: [PATCH 58/96] sparc32: kill __ret_efault() it's unused on 32bit one as well... Signed-off-by: Al Viro --- arch/sparc/include/asm/uaccess_32.h | 2 -- arch/sparc/kernel/head_32.S | 7 ------- 2 files changed, 9 deletions(-) diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index a59a1e81986d..93910845733a 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -71,8 +71,6 @@ struct exception_table_entry /* Returns 0 if exception not found and fixup otherwise. */ unsigned long search_extables_range(unsigned long addr, unsigned long *g2); -void __ret_efault(void); - /* Uh, these should become the main single-value transfer routines.. * They automatically use the right size if we just have the right * pointer type.. diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S index 7bb317b87dde..7274e43ff9be 100644 --- a/arch/sparc/kernel/head_32.S +++ b/arch/sparc/kernel/head_32.S @@ -809,10 +809,3 @@ lvl14_save: .word 0 .word 0 .word t_irq14 - - .section ".fixup",#alloc,#execinstr - .globl __ret_efault -__ret_efault: - ret - restore %g0, -EFAULT, %o0 -EXPORT_SYMBOL(__ret_efault) From c0ea73f18c057b41801106cfd2ffaf9794681444 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 03:33:03 -0500 Subject: [PATCH 59/96] tile: switch to generic extable.h Signed-off-by: Al Viro --- arch/tile/include/asm/Kbuild | 1 + arch/tile/include/asm/uaccess.h | 19 +------------------ 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/arch/tile/include/asm/Kbuild b/arch/tile/include/asm/Kbuild index aa48b6eaff2d..24c44e93804d 100644 --- a/arch/tile/include/asm/Kbuild +++ b/arch/tile/include/asm/Kbuild @@ -7,6 +7,7 @@ generic-y += clkdev.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += fb.h generic-y += fcntl.h generic-y += hw_irq.h diff --git a/arch/tile/include/asm/uaccess.h b/arch/tile/include/asm/uaccess.h index 14ea3d1ca2c7..ef3a1fd07946 100644 --- a/arch/tile/include/asm/uaccess.h +++ b/arch/tile/include/asm/uaccess.h @@ -98,24 +98,7 @@ int __range_ok(unsigned long addr, unsigned long size); likely(__range_ok((unsigned long)(addr), (size)) == 0); \ }) -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn, fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); +#include /* * This is a type: either unsigned long, if the argument fits into From 23504bae7f3edd1484c4d470362f2b12bcd298f9 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 14:27:36 -0400 Subject: [PATCH 60/96] tile: get rid of zeroing, switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/tile/Kconfig | 1 + arch/tile/include/asm/uaccess.h | 143 ++------------------------------ arch/tile/lib/exports.c | 7 +- arch/tile/lib/memcpy_32.S | 41 +++------ arch/tile/lib/memcpy_user_64.c | 15 +--- 5 files changed, 27 insertions(+), 180 deletions(-) diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig index 4583c0320059..27808d86eb57 100644 --- a/arch/tile/Kconfig +++ b/arch/tile/Kconfig @@ -33,6 +33,7 @@ config TILE select USER_STACKTRACE_SUPPORT select USE_PMC if PERF_EVENTS select VIRT_TO_BUS + select ARCH_HAS_RAW_COPY_USER config MMU def_bool y diff --git a/arch/tile/include/asm/uaccess.h b/arch/tile/include/asm/uaccess.h index ef3a1fd07946..a803f6bb4d92 100644 --- a/arch/tile/include/asm/uaccess.h +++ b/arch/tile/include/asm/uaccess.h @@ -313,145 +313,16 @@ extern int __put_user_bad(void) ((x) = 0, -EFAULT); \ }) -/** - * __copy_to_user() - copy data into user space, with less checking. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * An alternate version - __copy_to_user_inatomic() - is designed - * to be called from atomic context, typically bracketed by calls - * to pagefault_disable() and pagefault_enable(). - */ -extern unsigned long __must_check __copy_to_user_inatomic( - void __user *to, const void *from, unsigned long n); - -static inline unsigned long __must_check -__copy_to_user(void __user *to, const void *from, unsigned long n) -{ - might_fault(); - return __copy_to_user_inatomic(to, from, n); -} - -static inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long n) -{ - if (access_ok(VERIFY_WRITE, to, n)) - n = __copy_to_user(to, from, n); - return n; -} - -/** - * __copy_from_user() - copy data from user space, with less checking. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - * - * An alternate version - __copy_from_user_inatomic() - is designed - * to be called from atomic context, typically bracketed by calls - * to pagefault_disable() and pagefault_enable(). This version - * does *NOT* pad with zeros. - */ -extern unsigned long __must_check __copy_from_user_inatomic( - void *to, const void __user *from, unsigned long n); -extern unsigned long __must_check __copy_from_user_zeroing( - void *to, const void __user *from, unsigned long n); - -static inline unsigned long __must_check -__copy_from_user(void *to, const void __user *from, unsigned long n) -{ - might_fault(); - return __copy_from_user_zeroing(to, from, n); -} - -static inline unsigned long __must_check -_copy_from_user(void *to, const void __user *from, unsigned long n) -{ - if (access_ok(VERIFY_READ, from, n)) - n = __copy_from_user(to, from, n); - else - memset(to, 0, n); - return n; -} - -extern void __compiletime_error("usercopy buffer size is too small") -__bad_copy_user(void); - -static inline void copy_user_overflow(int size, unsigned long count) -{ - WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); -} - -static inline unsigned long __must_check copy_from_user(void *to, - const void __user *from, - unsigned long n) -{ - int sz = __compiletime_object_size(to); - - if (likely(sz == -1 || sz >= n)) - n = _copy_from_user(to, from, n); - else if (!__builtin_constant_p(n)) - copy_user_overflow(sz, n); - else - __bad_copy_user(); - - return n; -} +extern unsigned long __must_check +raw_copy_to_user(void __user *to, const void *from, unsigned long n); +extern unsigned long __must_check +raw_copy_from_user(void *to, const void __user *from, unsigned long n); +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER #ifdef __tilegx__ -/** - * __copy_in_user() - copy data within user space, with less checking. - * @to: Destination address, in user space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to user space. Caller must check - * the specified blocks with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -extern unsigned long __copy_in_user_inatomic( +extern unsigned long raw_copy_in_user( void __user *to, const void __user *from, unsigned long n); - -static inline unsigned long __must_check -__copy_in_user(void __user *to, const void __user *from, unsigned long n) -{ - might_fault(); - return __copy_in_user_inatomic(to, from, n); -} - -static inline unsigned long __must_check -copy_in_user(void __user *to, const void __user *from, unsigned long n) -{ - if (access_ok(VERIFY_WRITE, to, n) && access_ok(VERIFY_READ, from, n)) - n = __copy_in_user(to, from, n); - return n; -} #endif diff --git a/arch/tile/lib/exports.c b/arch/tile/lib/exports.c index c5369fe643c7..ecce8e177e3f 100644 --- a/arch/tile/lib/exports.c +++ b/arch/tile/lib/exports.c @@ -38,11 +38,10 @@ EXPORT_SYMBOL(__mcount); /* arch/tile/lib/, various memcpy files */ EXPORT_SYMBOL(memcpy); -EXPORT_SYMBOL(__copy_to_user_inatomic); -EXPORT_SYMBOL(__copy_from_user_inatomic); -EXPORT_SYMBOL(__copy_from_user_zeroing); +EXPORT_SYMBOL(raw_copy_to_user); +EXPORT_SYMBOL(raw_copy_from_user); #ifdef __tilegx__ -EXPORT_SYMBOL(__copy_in_user_inatomic); +EXPORT_SYMBOL(raw_copy_in_user); #endif /* hypervisor glue */ diff --git a/arch/tile/lib/memcpy_32.S b/arch/tile/lib/memcpy_32.S index a2771ae5da53..270f1267cd18 100644 --- a/arch/tile/lib/memcpy_32.S +++ b/arch/tile/lib/memcpy_32.S @@ -24,7 +24,6 @@ #define IS_MEMCPY 0 #define IS_COPY_FROM_USER 1 -#define IS_COPY_FROM_USER_ZEROING 2 #define IS_COPY_TO_USER -1 .section .text.memcpy_common, "ax" @@ -42,40 +41,31 @@ 9 -/* __copy_from_user_inatomic takes the kernel target address in r0, +/* raw_copy_from_user takes the kernel target address in r0, * the user source in r1, and the bytes to copy in r2. * It returns the number of uncopiable bytes (hopefully zero) in r0. */ -ENTRY(__copy_from_user_inatomic) -.type __copy_from_user_inatomic, @function - FEEDBACK_ENTER_EXPLICIT(__copy_from_user_inatomic, \ +ENTRY(raw_copy_from_user) +.type raw_copy_from_user, @function + FEEDBACK_ENTER_EXPLICIT(raw_copy_from_user, \ .text.memcpy_common, \ - .Lend_memcpy_common - __copy_from_user_inatomic) + .Lend_memcpy_common - raw_copy_from_user) { movei r29, IS_COPY_FROM_USER; j memcpy_common } - .size __copy_from_user_inatomic, . - __copy_from_user_inatomic + .size raw_copy_from_user, . - raw_copy_from_user -/* __copy_from_user_zeroing is like __copy_from_user_inatomic, but - * any uncopiable bytes are zeroed in the target. - */ -ENTRY(__copy_from_user_zeroing) -.type __copy_from_user_zeroing, @function - FEEDBACK_REENTER(__copy_from_user_inatomic) - { movei r29, IS_COPY_FROM_USER_ZEROING; j memcpy_common } - .size __copy_from_user_zeroing, . - __copy_from_user_zeroing - -/* __copy_to_user_inatomic takes the user target address in r0, +/* raw_copy_to_user takes the user target address in r0, * the kernel source in r1, and the bytes to copy in r2. * It returns the number of uncopiable bytes (hopefully zero) in r0. */ -ENTRY(__copy_to_user_inatomic) -.type __copy_to_user_inatomic, @function - FEEDBACK_REENTER(__copy_from_user_inatomic) +ENTRY(raw_copy_to_user) +.type raw_copy_to_user, @function + FEEDBACK_REENTER(raw_copy_from_user) { movei r29, IS_COPY_TO_USER; j memcpy_common } - .size __copy_to_user_inatomic, . - __copy_to_user_inatomic + .size raw_copy_to_user, . - raw_copy_to_user ENTRY(memcpy) .type memcpy, @function - FEEDBACK_REENTER(__copy_from_user_inatomic) + FEEDBACK_REENTER(raw_copy_from_user) { movei r29, IS_MEMCPY } .size memcpy, . - memcpy /* Fall through */ @@ -520,12 +510,7 @@ copy_from_user_fixup_loop: { bnzt r2, copy_from_user_fixup_loop } .Lcopy_from_user_fixup_zero_remainder: - { bbs r29, 2f } /* low bit set means IS_COPY_FROM_USER */ - /* byte-at-a-time loop faulted, so zero the rest. */ - { move r3, r2; bz r2, 2f /* should be impossible, but handle it. */ } -1: { sb r0, zero; addi r0, r0, 1; addi r3, r3, -1 } - { bnzt r3, 1b } -2: move lr, r27 + move lr, r27 { move r0, r2; jrp lr } copy_to_user_fixup_loop: diff --git a/arch/tile/lib/memcpy_user_64.c b/arch/tile/lib/memcpy_user_64.c index 97bbb6060b25..a3fea9fd973e 100644 --- a/arch/tile/lib/memcpy_user_64.c +++ b/arch/tile/lib/memcpy_user_64.c @@ -51,7 +51,7 @@ __v; \ }) -#define USERCOPY_FUNC __copy_to_user_inatomic +#define USERCOPY_FUNC raw_copy_to_user #define ST1(p, v) _ST((p), st1, (v)) #define ST2(p, v) _ST((p), st2, (v)) #define ST4(p, v) _ST((p), st4, (v)) @@ -62,7 +62,7 @@ #define LD8 LD #include "memcpy_64.c" -#define USERCOPY_FUNC __copy_from_user_inatomic +#define USERCOPY_FUNC raw_copy_from_user #define ST1 ST #define ST2 ST #define ST4 ST @@ -73,7 +73,7 @@ #define LD8(p) _LD((p), ld) #include "memcpy_64.c" -#define USERCOPY_FUNC __copy_in_user_inatomic +#define USERCOPY_FUNC raw_copy_in_user #define ST1(p, v) _ST((p), st1, (v)) #define ST2(p, v) _ST((p), st2, (v)) #define ST4(p, v) _ST((p), st4, (v)) @@ -83,12 +83,3 @@ #define LD4(p) _LD((p), ld4u) #define LD8(p) _LD((p), ld) #include "memcpy_64.c" - -unsigned long __copy_from_user_zeroing(void *to, const void __user *from, - unsigned long n) -{ - unsigned long rc = __copy_from_user_inatomic(to, from, n); - if (unlikely(rc)) - memset(to + n - rc, 0, rc); - return rc; -} From a668ce3a001fe0564deb6357129bfd81f757743b Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 01:33:09 -0400 Subject: [PATCH 61/96] um: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/um/Kconfig.common | 1 + arch/um/include/asm/uaccess.h | 8 ++++---- arch/um/kernel/skas/uaccess.c | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/arch/um/Kconfig.common b/arch/um/Kconfig.common index fd443852103c..e13f763849b1 100644 --- a/arch/um/Kconfig.common +++ b/arch/um/Kconfig.common @@ -13,6 +13,7 @@ config UML select GENERIC_CLOCKEVENTS select HAVE_GCC_PLUGINS select TTY # Needed for line.c + select ARCH_HAS_RAW_COPY_USER config MMU bool diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h index 9396c8a4e920..cc00fc50768f 100644 --- a/arch/um/include/asm/uaccess.h +++ b/arch/um/include/asm/uaccess.h @@ -21,8 +21,8 @@ #define __addr_range_nowrap(addr, size) \ ((unsigned long) (addr) <= ((unsigned long) (addr) + (size))) -extern long __copy_from_user(void *to, const void __user *from, unsigned long n); -extern long __copy_to_user(void __user *to, const void *from, unsigned long n); +extern unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n); +extern unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n); extern long __strncpy_from_user(char *dst, const char __user *src, long count); extern long __strnlen_user(const void __user *str, long len); extern unsigned long __clear_user(void __user *mem, unsigned long len); @@ -33,8 +33,8 @@ static inline int __access_ok(unsigned long addr, unsigned long size); #define __clear_user __clear_user #define __strnlen_user __strnlen_user #define __strncpy_from_user __strncpy_from_user -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER #include diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c index 22c9f79db8e6..d450797a3a7c 100644 --- a/arch/um/kernel/skas/uaccess.c +++ b/arch/um/kernel/skas/uaccess.c @@ -139,7 +139,7 @@ static int copy_chunk_from_user(unsigned long from, int len, void *arg) return 0; } -long __copy_from_user(void *to, const void __user *from, unsigned long n) +unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { if (uaccess_kernel()) { memcpy(to, (__force void*)from, n); @@ -148,7 +148,7 @@ long __copy_from_user(void *to, const void __user *from, unsigned long n) return buffer_op((unsigned long) from, n, 0, copy_chunk_from_user, &to); } -EXPORT_SYMBOL(__copy_from_user); +EXPORT_SYMBOL(raw_copy_from_user); static int copy_chunk_to_user(unsigned long to, int len, void *arg) { @@ -159,7 +159,7 @@ static int copy_chunk_to_user(unsigned long to, int len, void *arg) return 0; } -long __copy_to_user(void __user *to, const void *from, unsigned long n) +unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { if (uaccess_kernel()) { memcpy((__force void *) to, from, n); @@ -168,7 +168,7 @@ long __copy_to_user(void __user *to, const void *from, unsigned long n) return buffer_op((unsigned long) to, n, 1, copy_chunk_to_user, &from); } -EXPORT_SYMBOL(__copy_to_user); +EXPORT_SYMBOL(raw_copy_to_user); static int strncpy_chunk_from_user(unsigned long from, int len, void *arg) { From 122b05ddf506e637336dcf64b5a129825f7bf6d4 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 25 Mar 2017 18:36:22 -0400 Subject: [PATCH 62/96] amd64: get rid of zeroing Signed-off-by: Al Viro --- arch/x86/include/asm/uaccess.h | 12 ++++++++++++ arch/x86/include/asm/uaccess_64.h | 16 ++++++++-------- arch/x86/lib/usercopy.c | 9 +++++---- arch/x86/lib/usercopy_64.c | 4 ---- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 0522d88a7f90..26410afcb8b0 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -379,6 +379,18 @@ do { \ : "=r" (err), ltype(x) \ : "m" (__m(addr)), "i" (errret), "0" (err)) +#define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret) \ + asm volatile("\n" \ + "1: mov"itype" %2,%"rtype"1\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: mov %3,%0\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r" (err), ltype(x) \ + : "m" (__m(addr)), "i" (errret), "0" (err)) + /* * This doesn't do __uaccess_begin/end - the exception handling * around it must do that. diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 8ddadd93639e..142f0f1230be 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -59,44 +59,44 @@ int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size) switch (size) { case 1: __uaccess_begin(); - __get_user_asm(*(u8 *)dst, (u8 __user *)src, + __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src, ret, "b", "b", "=q", 1); __uaccess_end(); return ret; case 2: __uaccess_begin(); - __get_user_asm(*(u16 *)dst, (u16 __user *)src, + __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src, ret, "w", "w", "=r", 2); __uaccess_end(); return ret; case 4: __uaccess_begin(); - __get_user_asm(*(u32 *)dst, (u32 __user *)src, + __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src, ret, "l", "k", "=r", 4); __uaccess_end(); return ret; case 8: __uaccess_begin(); - __get_user_asm(*(u64 *)dst, (u64 __user *)src, + __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 8); __uaccess_end(); return ret; case 10: __uaccess_begin(); - __get_user_asm(*(u64 *)dst, (u64 __user *)src, + __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 10); if (likely(!ret)) - __get_user_asm(*(u16 *)(8 + (char *)dst), + __get_user_asm_nozero(*(u16 *)(8 + (char *)dst), (u16 __user *)(8 + (char __user *)src), ret, "w", "w", "=r", 2); __uaccess_end(); return ret; case 16: __uaccess_begin(); - __get_user_asm(*(u64 *)dst, (u64 __user *)src, + __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 16); if (likely(!ret)) - __get_user_asm(*(u64 *)(8 + (char *)dst), + __get_user_asm_nozero(*(u64 *)(8 + (char *)dst), (u64 __user *)(8 + (char __user *)src), ret, "q", "", "=r", 8); __uaccess_end(); diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c index c074799bddae..a851f3d199c2 100644 --- a/arch/x86/lib/usercopy.c +++ b/arch/x86/lib/usercopy.c @@ -76,10 +76,11 @@ EXPORT_SYMBOL(_copy_to_user); */ unsigned long _copy_from_user(void *to, const void __user *from, unsigned n) { + unsigned long res = n; if (access_ok(VERIFY_READ, from, n)) - n = __copy_from_user(to, from, n); - else - memset(to, 0, n); - return n; + res = __copy_from_user_inatomic(to, from, n); + if (unlikely(res)) + memset(to + n - res, 0, res); + return res; } EXPORT_SYMBOL(_copy_from_user); diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 69873589c0ba..6c8b6a6c1b38 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -80,9 +80,5 @@ copy_user_handle_tail(char *to, char *from, unsigned len) break; } clac(); - - /* If the destination is a kernel buffer, we always clear the end */ - if (!__addr_ok(to)) - memset(to, 0, len); return len; } From 2ef59f28567f8a5d66dccc0e6a99a1bd655fd605 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 01:37:07 -0400 Subject: [PATCH 63/96] unicore32: get rid of zeroing and switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/unicore32/Kconfig | 1 + arch/unicore32/include/asm/uaccess.h | 6 ++++-- arch/unicore32/kernel/ksyms.c | 4 ++-- arch/unicore32/lib/copy_from_user.S | 16 ++++++---------- arch/unicore32/lib/copy_to_user.S | 6 +++--- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig index 0769066929c6..319519eaed24 100644 --- a/arch/unicore32/Kconfig +++ b/arch/unicore32/Kconfig @@ -18,6 +18,7 @@ config UNICORE32 select ARCH_WANT_FRAME_POINTERS select GENERIC_IOMAP select MODULES_USE_ELF_REL + select ARCH_HAS_RAW_COPY_USER help UniCore-32 is 32-bit Instruction Set Architecture, including a series of low-power-consumption RISC chip diff --git a/arch/unicore32/include/asm/uaccess.h b/arch/unicore32/include/asm/uaccess.h index b2bcab69dfee..1d55f2f83759 100644 --- a/arch/unicore32/include/asm/uaccess.h +++ b/arch/unicore32/include/asm/uaccess.h @@ -24,15 +24,17 @@ #define __access_ok(addr, size) (__kernel_ok || __user_ok((addr), (size))) extern unsigned long __must_check -__copy_from_user(void *to, const void __user *from, unsigned long n); +raw_copy_from_user(void *to, const void __user *from, unsigned long n); extern unsigned long __must_check -__copy_to_user(void __user *to, const void *from, unsigned long n); +raw_copy_to_user(void __user *to, const void *from, unsigned long n); extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); extern unsigned long __must_check __strncpy_from_user(char *to, const char __user *from, unsigned long count); extern unsigned long __strnlen_user(const char __user *s, long n); +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER #include diff --git a/arch/unicore32/kernel/ksyms.c b/arch/unicore32/kernel/ksyms.c index 0323528a80fd..dcc72ee1fcdb 100644 --- a/arch/unicore32/kernel/ksyms.c +++ b/arch/unicore32/kernel/ksyms.c @@ -46,8 +46,8 @@ EXPORT_SYMBOL(__strncpy_from_user); EXPORT_SYMBOL(copy_page); -EXPORT_SYMBOL(__copy_from_user); -EXPORT_SYMBOL(__copy_to_user); +EXPORT_SYMBOL(raw_copy_from_user); +EXPORT_SYMBOL(raw_copy_to_user); EXPORT_SYMBOL(__clear_user); EXPORT_SYMBOL(__ashldi3); diff --git a/arch/unicore32/lib/copy_from_user.S b/arch/unicore32/lib/copy_from_user.S index ab0767ea5dbd..5f80fcbe8631 100644 --- a/arch/unicore32/lib/copy_from_user.S +++ b/arch/unicore32/lib/copy_from_user.S @@ -16,7 +16,7 @@ /* * Prototype: * - * size_t __copy_from_user(void *to, const void *from, size_t n) + * size_t raw_copy_from_user(void *to, const void *from, size_t n) * * Purpose: * @@ -87,22 +87,18 @@ .text -ENTRY(__copy_from_user) +ENTRY(raw_copy_from_user) #include "copy_template.S" -ENDPROC(__copy_from_user) +ENDPROC(raw_copy_from_user) .pushsection .fixup,"ax" .align 0 copy_abort_preamble - ldm.w (r1, r2), [sp]+ - sub r3, r0, r1 - rsub r2, r3, r2 - stw r2, [sp] - mov r1, #0 - b.l memset - ldw.w r0, [sp]+, #4 + ldm.w (r1, r2, r3), [sp]+ + sub r0, r0, r1 + rsub r0, r0, r2 copy_abort_end .popsection diff --git a/arch/unicore32/lib/copy_to_user.S b/arch/unicore32/lib/copy_to_user.S index 6e22151c840d..857c6816ffe7 100644 --- a/arch/unicore32/lib/copy_to_user.S +++ b/arch/unicore32/lib/copy_to_user.S @@ -16,7 +16,7 @@ /* * Prototype: * - * size_t __copy_to_user(void *to, const void *from, size_t n) + * size_t raw_copy_to_user(void *to, const void *from, size_t n) * * Purpose: * @@ -79,11 +79,11 @@ .text -WEAK(__copy_to_user) +WEAK(raw_copy_to_user) #include "copy_template.S" -ENDPROC(__copy_to_user) +ENDPROC(raw_copy_to_user) .pushsection .fixup,"ax" .align 0 From 3f763453e6f27d82fa0ac58f8e1ac4094c1fb1f8 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 25 Mar 2017 18:47:28 -0400 Subject: [PATCH 64/96] kill __copy_from_user_nocache() Signed-off-by: Al Viro --- arch/x86/include/asm/uaccess_32.h | 30 -------- arch/x86/include/asm/uaccess_64.h | 8 -- arch/x86/lib/usercopy_32.c | 118 ------------------------------ include/linux/uaccess.h | 6 -- lib/iov_iter.c | 4 +- 5 files changed, 2 insertions(+), 164 deletions(-) diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h index 5268ecceea96..19e6c050c438 100644 --- a/arch/x86/include/asm/uaccess_32.h +++ b/arch/x86/include/asm/uaccess_32.h @@ -14,8 +14,6 @@ unsigned long __must_check __copy_from_user_ll (void *to, const void __user *from, unsigned long n); unsigned long __must_check __copy_from_user_ll_nozero (void *to, const void __user *from, unsigned long n); -unsigned long __must_check __copy_from_user_ll_nocache - (void *to, const void __user *from, unsigned long n); unsigned long __must_check __copy_from_user_ll_nocache_nozero (void *to, const void __user *from, unsigned long n); @@ -119,34 +117,6 @@ __copy_from_user(void *to, const void __user *from, unsigned long n) return __copy_from_user_ll(to, from, n); } -static __always_inline unsigned long __copy_from_user_nocache(void *to, - const void __user *from, unsigned long n) -{ - might_fault(); - if (__builtin_constant_p(n)) { - unsigned long ret; - - switch (n) { - case 1: - __uaccess_begin(); - __get_user_size(*(u8 *)to, from, 1, ret, 1); - __uaccess_end(); - return ret; - case 2: - __uaccess_begin(); - __get_user_size(*(u16 *)to, from, 2, ret, 2); - __uaccess_end(); - return ret; - case 4: - __uaccess_begin(); - __get_user_size(*(u32 *)to, from, 4, ret, 4); - __uaccess_end(); - return ret; - } - } - return __copy_from_user_ll_nocache(to, from, n); -} - static __always_inline unsigned long __copy_from_user_inatomic_nocache(void *to, const void __user *from, unsigned long n) diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 142f0f1230be..242936b0cb4b 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -260,14 +260,6 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest); -static inline int -__copy_from_user_nocache(void *dst, const void __user *src, unsigned size) -{ - might_fault(); - kasan_check_write(dst, size); - return __copy_user_nocache(dst, src, size, 1); -} - static inline int __copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned size) diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index 1f65ff6540f0..02aa7aa8b9f3 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -293,105 +293,6 @@ __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size) return size; } -/* - * Non Temporal Hint version of __copy_user_zeroing_intel. It is cache aware. - * hyoshiok@miraclelinux.com - */ - -static unsigned long __copy_user_zeroing_intel_nocache(void *to, - const void __user *from, unsigned long size) -{ - int d0, d1; - - __asm__ __volatile__( - " .align 2,0x90\n" - "0: movl 32(%4), %%eax\n" - " cmpl $67, %0\n" - " jbe 2f\n" - "1: movl 64(%4), %%eax\n" - " .align 2,0x90\n" - "2: movl 0(%4), %%eax\n" - "21: movl 4(%4), %%edx\n" - " movnti %%eax, 0(%3)\n" - " movnti %%edx, 4(%3)\n" - "3: movl 8(%4), %%eax\n" - "31: movl 12(%4),%%edx\n" - " movnti %%eax, 8(%3)\n" - " movnti %%edx, 12(%3)\n" - "4: movl 16(%4), %%eax\n" - "41: movl 20(%4), %%edx\n" - " movnti %%eax, 16(%3)\n" - " movnti %%edx, 20(%3)\n" - "10: movl 24(%4), %%eax\n" - "51: movl 28(%4), %%edx\n" - " movnti %%eax, 24(%3)\n" - " movnti %%edx, 28(%3)\n" - "11: movl 32(%4), %%eax\n" - "61: movl 36(%4), %%edx\n" - " movnti %%eax, 32(%3)\n" - " movnti %%edx, 36(%3)\n" - "12: movl 40(%4), %%eax\n" - "71: movl 44(%4), %%edx\n" - " movnti %%eax, 40(%3)\n" - " movnti %%edx, 44(%3)\n" - "13: movl 48(%4), %%eax\n" - "81: movl 52(%4), %%edx\n" - " movnti %%eax, 48(%3)\n" - " movnti %%edx, 52(%3)\n" - "14: movl 56(%4), %%eax\n" - "91: movl 60(%4), %%edx\n" - " movnti %%eax, 56(%3)\n" - " movnti %%edx, 60(%3)\n" - " addl $-64, %0\n" - " addl $64, %4\n" - " addl $64, %3\n" - " cmpl $63, %0\n" - " ja 0b\n" - " sfence \n" - "5: movl %0, %%eax\n" - " shrl $2, %0\n" - " andl $3, %%eax\n" - " cld\n" - "6: rep; movsl\n" - " movl %%eax,%0\n" - "7: rep; movsb\n" - "8:\n" - ".section .fixup,\"ax\"\n" - "9: lea 0(%%eax,%0,4),%0\n" - "16: pushl %0\n" - " pushl %%eax\n" - " xorl %%eax,%%eax\n" - " rep; stosb\n" - " popl %%eax\n" - " popl %0\n" - " jmp 8b\n" - ".previous\n" - _ASM_EXTABLE(0b,16b) - _ASM_EXTABLE(1b,16b) - _ASM_EXTABLE(2b,16b) - _ASM_EXTABLE(21b,16b) - _ASM_EXTABLE(3b,16b) - _ASM_EXTABLE(31b,16b) - _ASM_EXTABLE(4b,16b) - _ASM_EXTABLE(41b,16b) - _ASM_EXTABLE(10b,16b) - _ASM_EXTABLE(51b,16b) - _ASM_EXTABLE(11b,16b) - _ASM_EXTABLE(61b,16b) - _ASM_EXTABLE(12b,16b) - _ASM_EXTABLE(71b,16b) - _ASM_EXTABLE(13b,16b) - _ASM_EXTABLE(81b,16b) - _ASM_EXTABLE(14b,16b) - _ASM_EXTABLE(91b,16b) - _ASM_EXTABLE(6b,9b) - _ASM_EXTABLE(7b,16b) - : "=&c"(size), "=&D" (d0), "=&S" (d1) - : "1"(to), "2"(from), "0"(size) - : "eax", "edx", "memory"); - return size; -} - static unsigned long __copy_user_intel_nocache(void *to, const void __user *from, unsigned long size) { @@ -490,8 +391,6 @@ unsigned long __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size); unsigned long __copy_user_intel(void __user *to, const void *from, unsigned long size); -unsigned long __copy_user_zeroing_intel_nocache(void *to, - const void __user *from, unsigned long size); #endif /* CONFIG_X86_INTEL_USERCOPY */ /* Generic arbitrary sized copy. */ @@ -607,23 +506,6 @@ unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from, } EXPORT_SYMBOL(__copy_from_user_ll_nozero); -unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from, - unsigned long n) -{ - stac(); -#ifdef CONFIG_X86_INTEL_USERCOPY - if (n > 64 && static_cpu_has(X86_FEATURE_XMM2)) - n = __copy_user_zeroing_intel_nocache(to, from, n); - else - __copy_user_zeroing(to, from, n); -#else - __copy_user_zeroing(to, from, n); -#endif - clac(); - return n; -} -EXPORT_SYMBOL(__copy_from_user_ll_nocache); - unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, unsigned long n) { diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 5f76bc995d96..7fc2104b88bc 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -261,12 +261,6 @@ static inline unsigned long __copy_from_user_inatomic_nocache(void *to, return __copy_from_user_inatomic(to, from, n); } -static inline unsigned long __copy_from_user_nocache(void *to, - const void __user *from, unsigned long n) -{ - return __copy_from_user(to, from, n); -} - #endif /* ARCH_HAS_NOCACHE_UACCESS */ /* diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 97db876c6862..672c32f9f960 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -604,7 +604,7 @@ size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) return 0; } iterate_and_advance(i, bytes, v, - __copy_from_user_nocache((to += v.iov_len) - v.iov_len, + __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, v.bv_offset, v.bv_len), @@ -625,7 +625,7 @@ bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) if (unlikely(i->count < bytes)) return false; iterate_all_kinds(i, bytes, v, ({ - if (__copy_from_user_nocache((to += v.iov_len) - v.iov_len, + if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)) return false; 0;}), From 0b46a94e84c1323d54f8b82eacd3143400fb9521 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 14:34:44 -0500 Subject: [PATCH 65/96] xtensa: switch to generic extable.h Signed-off-by: Al Viro --- arch/xtensa/include/asm/Kbuild | 1 + arch/xtensa/include/asm/uaccess.h | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild index f41408c53fe1..cc23e9ecc6bb 100644 --- a/arch/xtensa/include/asm/Kbuild +++ b/arch/xtensa/include/asm/Kbuild @@ -6,6 +6,7 @@ generic-y += dma-contiguous.h generic-y += emergency-restart.h generic-y += errno.h generic-y += exec.h +generic-y += extable.h generic-y += fcntl.h generic-y += hardirq.h generic-y += ioctl.h diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 26512692e28f..0f338774af99 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -18,6 +18,7 @@ #include #include +#include /* * The fs value determines whether argument validity checking should @@ -342,10 +343,4 @@ static inline long strnlen_user(const char *str, long len) return __strnlen_user(str, len); } - -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - #endif /* _XTENSA_UACCESS_H */ From 3a0e75adecc8da026a5befb2c5828d08c999373c Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 22 Mar 2017 13:02:41 -0400 Subject: [PATCH 66/96] xtensa: get rid of zeroing, use RAW_COPY_USER Signed-off-by: Al Viro --- arch/xtensa/Kconfig | 1 + arch/xtensa/include/asm/uaccess.h | 58 +++------------ arch/xtensa/lib/usercopy.S | 116 +++++++++++++----------------- 3 files changed, 59 insertions(+), 116 deletions(-) diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index f4126cf997a4..043d37d45919 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -29,6 +29,7 @@ config XTENSA select NO_BOOTMEM select PERF_USE_VMALLOC select VIRT_TO_BUS + select ARCH_HAS_RAW_COPY_USER help Xtensa processors are 32-bit RISC machines designed by Tensilica primarily for embedded systems. These processors are both diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 0f338774af99..8e93ed8ad1fe 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -234,60 +234,22 @@ __asm__ __volatile__( \ * Copy to/from user space */ -/* - * We use a generic, arbitrary-sized copy subroutine. The Xtensa - * architecture would cause heavy code bloat if we tried to inline - * these functions and provide __constant_copy_* equivalents like the - * i386 versions. __xtensa_copy_user is quite efficient. See the - * .fixup section of __xtensa_copy_user for a discussion on the - * X_zeroing equivalents for Xtensa. - */ - extern unsigned __xtensa_copy_user(void *to, const void *from, unsigned n); -#define __copy_user(to, from, size) __xtensa_copy_user(to, from, size) - static inline unsigned long -__generic_copy_from_user_nocheck(void *to, const void *from, unsigned long n) -{ - return __copy_user(to, from, n); -} - -static inline unsigned long -__generic_copy_to_user_nocheck(void *to, const void *from, unsigned long n) -{ - return __copy_user(to, from, n); -} - -static inline unsigned long -__generic_copy_to_user(void *to, const void *from, unsigned long n) -{ - prefetch(from); - if (access_ok(VERIFY_WRITE, to, n)) - return __copy_user(to, from, n); - return n; -} - -static inline unsigned long -__generic_copy_from_user(void *to, const void *from, unsigned long n) +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { prefetchw(to); - if (access_ok(VERIFY_READ, from, n)) - return __copy_user(to, from, n); - else - memset(to, 0, n); - return n; + return __xtensa_copy_user(to, (__force const void *)from, n); } - -#define copy_to_user(to, from, n) __generic_copy_to_user((to), (from), (n)) -#define copy_from_user(to, from, n) __generic_copy_from_user((to), (from), (n)) -#define __copy_to_user(to, from, n) \ - __generic_copy_to_user_nocheck((to), (from), (n)) -#define __copy_from_user(to, from, n) \ - __generic_copy_from_user_nocheck((to), (from), (n)) -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + prefetchw(from); + return __xtensa_copy_user((__force void *)to, from, n); +} +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER /* * We need to return the number of bytes not cleared. Our memset() diff --git a/arch/xtensa/lib/usercopy.S b/arch/xtensa/lib/usercopy.S index 7ea4dd68893e..d9cd766bde3e 100644 --- a/arch/xtensa/lib/usercopy.S +++ b/arch/xtensa/lib/usercopy.S @@ -102,9 +102,9 @@ __xtensa_copy_user: bltui a4, 7, .Lbytecopy # do short copies byte by byte # copy 1 byte - EX(l8ui, a6, a3, 0, l_fixup) + EX(l8ui, a6, a3, 0, fixup) addi a3, a3, 1 - EX(s8i, a6, a5, 0, s_fixup) + EX(s8i, a6, a5, 0, fixup) addi a5, a5, 1 addi a4, a4, -1 bbci.l a5, 1, .Ldstaligned # if dst is now aligned, then @@ -112,11 +112,11 @@ __xtensa_copy_user: .Ldst2mod4: # dst 16-bit aligned # copy 2 bytes bltui a4, 6, .Lbytecopy # do short copies byte by byte - EX(l8ui, a6, a3, 0, l_fixup) - EX(l8ui, a7, a3, 1, l_fixup) + EX(l8ui, a6, a3, 0, fixup) + EX(l8ui, a7, a3, 1, fixup) addi a3, a3, 2 - EX(s8i, a6, a5, 0, s_fixup) - EX(s8i, a7, a5, 1, s_fixup) + EX(s8i, a6, a5, 0, fixup) + EX(s8i, a7, a5, 1, fixup) addi a5, a5, 2 addi a4, a4, -2 j .Ldstaligned # dst is now aligned, return to main algorithm @@ -135,9 +135,9 @@ __xtensa_copy_user: add a7, a3, a4 # a7 = end address for source #endif /* !XCHAL_HAVE_LOOPS */ .Lnextbyte: - EX(l8ui, a6, a3, 0, l_fixup) + EX(l8ui, a6, a3, 0, fixup) addi a3, a3, 1 - EX(s8i, a6, a5, 0, s_fixup) + EX(s8i, a6, a5, 0, fixup) addi a5, a5, 1 #if !XCHAL_HAVE_LOOPS blt a3, a7, .Lnextbyte @@ -161,15 +161,15 @@ __xtensa_copy_user: add a8, a8, a3 # a8 = end of last 16B source chunk #endif /* !XCHAL_HAVE_LOOPS */ .Loop1: - EX(l32i, a6, a3, 0, l_fixup) - EX(l32i, a7, a3, 4, l_fixup) - EX(s32i, a6, a5, 0, s_fixup) - EX(l32i, a6, a3, 8, l_fixup) - EX(s32i, a7, a5, 4, s_fixup) - EX(l32i, a7, a3, 12, l_fixup) - EX(s32i, a6, a5, 8, s_fixup) + EX(l32i, a6, a3, 0, fixup) + EX(l32i, a7, a3, 4, fixup) + EX(s32i, a6, a5, 0, fixup) + EX(l32i, a6, a3, 8, fixup) + EX(s32i, a7, a5, 4, fixup) + EX(l32i, a7, a3, 12, fixup) + EX(s32i, a6, a5, 8, fixup) addi a3, a3, 16 - EX(s32i, a7, a5, 12, s_fixup) + EX(s32i, a7, a5, 12, fixup) addi a5, a5, 16 #if !XCHAL_HAVE_LOOPS blt a3, a8, .Loop1 @@ -177,31 +177,31 @@ __xtensa_copy_user: .Loop1done: bbci.l a4, 3, .L2 # copy 8 bytes - EX(l32i, a6, a3, 0, l_fixup) - EX(l32i, a7, a3, 4, l_fixup) + EX(l32i, a6, a3, 0, fixup) + EX(l32i, a7, a3, 4, fixup) addi a3, a3, 8 - EX(s32i, a6, a5, 0, s_fixup) - EX(s32i, a7, a5, 4, s_fixup) + EX(s32i, a6, a5, 0, fixup) + EX(s32i, a7, a5, 4, fixup) addi a5, a5, 8 .L2: bbci.l a4, 2, .L3 # copy 4 bytes - EX(l32i, a6, a3, 0, l_fixup) + EX(l32i, a6, a3, 0, fixup) addi a3, a3, 4 - EX(s32i, a6, a5, 0, s_fixup) + EX(s32i, a6, a5, 0, fixup) addi a5, a5, 4 .L3: bbci.l a4, 1, .L4 # copy 2 bytes - EX(l16ui, a6, a3, 0, l_fixup) + EX(l16ui, a6, a3, 0, fixup) addi a3, a3, 2 - EX(s16i, a6, a5, 0, s_fixup) + EX(s16i, a6, a5, 0, fixup) addi a5, a5, 2 .L4: bbci.l a4, 0, .L5 # copy 1 byte - EX(l8ui, a6, a3, 0, l_fixup) - EX(s8i, a6, a5, 0, s_fixup) + EX(l8ui, a6, a3, 0, fixup) + EX(s8i, a6, a5, 0, fixup) .L5: movi a2, 0 # return success for len bytes copied retw @@ -217,7 +217,7 @@ __xtensa_copy_user: # copy 16 bytes per iteration for word-aligned dst and unaligned src and a10, a3, a8 # save unalignment offset for below sub a3, a3, a10 # align a3 (to avoid sim warnings only; not needed for hardware) - EX(l32i, a6, a3, 0, l_fixup) # load first word + EX(l32i, a6, a3, 0, fixup) # load first word #if XCHAL_HAVE_LOOPS loopnez a7, .Loop2done #else /* !XCHAL_HAVE_LOOPS */ @@ -226,19 +226,19 @@ __xtensa_copy_user: add a12, a12, a3 # a12 = end of last 16B source chunk #endif /* !XCHAL_HAVE_LOOPS */ .Loop2: - EX(l32i, a7, a3, 4, l_fixup) - EX(l32i, a8, a3, 8, l_fixup) + EX(l32i, a7, a3, 4, fixup) + EX(l32i, a8, a3, 8, fixup) ALIGN( a6, a6, a7) - EX(s32i, a6, a5, 0, s_fixup) - EX(l32i, a9, a3, 12, l_fixup) + EX(s32i, a6, a5, 0, fixup) + EX(l32i, a9, a3, 12, fixup) ALIGN( a7, a7, a8) - EX(s32i, a7, a5, 4, s_fixup) - EX(l32i, a6, a3, 16, l_fixup) + EX(s32i, a7, a5, 4, fixup) + EX(l32i, a6, a3, 16, fixup) ALIGN( a8, a8, a9) - EX(s32i, a8, a5, 8, s_fixup) + EX(s32i, a8, a5, 8, fixup) addi a3, a3, 16 ALIGN( a9, a9, a6) - EX(s32i, a9, a5, 12, s_fixup) + EX(s32i, a9, a5, 12, fixup) addi a5, a5, 16 #if !XCHAL_HAVE_LOOPS blt a3, a12, .Loop2 @@ -246,39 +246,39 @@ __xtensa_copy_user: .Loop2done: bbci.l a4, 3, .L12 # copy 8 bytes - EX(l32i, a7, a3, 4, l_fixup) - EX(l32i, a8, a3, 8, l_fixup) + EX(l32i, a7, a3, 4, fixup) + EX(l32i, a8, a3, 8, fixup) ALIGN( a6, a6, a7) - EX(s32i, a6, a5, 0, s_fixup) + EX(s32i, a6, a5, 0, fixup) addi a3, a3, 8 ALIGN( a7, a7, a8) - EX(s32i, a7, a5, 4, s_fixup) + EX(s32i, a7, a5, 4, fixup) addi a5, a5, 8 mov a6, a8 .L12: bbci.l a4, 2, .L13 # copy 4 bytes - EX(l32i, a7, a3, 4, l_fixup) + EX(l32i, a7, a3, 4, fixup) addi a3, a3, 4 ALIGN( a6, a6, a7) - EX(s32i, a6, a5, 0, s_fixup) + EX(s32i, a6, a5, 0, fixup) addi a5, a5, 4 mov a6, a7 .L13: add a3, a3, a10 # readjust a3 with correct misalignment bbci.l a4, 1, .L14 # copy 2 bytes - EX(l8ui, a6, a3, 0, l_fixup) - EX(l8ui, a7, a3, 1, l_fixup) + EX(l8ui, a6, a3, 0, fixup) + EX(l8ui, a7, a3, 1, fixup) addi a3, a3, 2 - EX(s8i, a6, a5, 0, s_fixup) - EX(s8i, a7, a5, 1, s_fixup) + EX(s8i, a6, a5, 0, fixup) + EX(s8i, a7, a5, 1, fixup) addi a5, a5, 2 .L14: bbci.l a4, 0, .L15 # copy 1 byte - EX(l8ui, a6, a3, 0, l_fixup) - EX(s8i, a6, a5, 0, s_fixup) + EX(l8ui, a6, a3, 0, fixup) + EX(s8i, a6, a5, 0, fixup) .L15: movi a2, 0 # return success for len bytes copied retw @@ -291,30 +291,10 @@ __xtensa_copy_user: * bytes_copied = a5 - a2 * retval = bytes_not_copied = original len - bytes_copied * retval = a11 - (a5 - a2) - * - * Clearing the remaining pieces of kernel memory plugs security - * holes. This functionality is the equivalent of the *_zeroing - * functions that some architectures provide. */ -.Lmemset: - .word memset -s_fixup: +fixup: sub a2, a5, a2 /* a2 <-- bytes copied */ sub a2, a11, a2 /* a2 <-- bytes not copied */ retw - -l_fixup: - sub a2, a5, a2 /* a2 <-- bytes copied */ - sub a2, a11, a2 /* a2 <-- bytes not copied == return value */ - - /* void *memset(void *s, int c, size_t n); */ - mov a6, a5 /* s */ - movi a7, 0 /* c */ - mov a8, a2 /* n */ - l32r a4, .Lmemset - callx4 a4 - /* Ignore memset return value in a6. */ - /* a2 still contains bytes not copied. */ - retw From 839cc2954cbdb82511d2ecfa07ba06bab4198519 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 08:19:02 -0400 Subject: [PATCH 67/96] arc: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/arc/Kconfig | 1 + arch/arc/include/asm/uaccess.h | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index c9f30f4763ab..7e213ff4f01f 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -44,6 +44,7 @@ config ARC select HAVE_GENERIC_DMA_COHERENT select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZMA + select ARCH_HAS_RAW_COPY_USER config MIGHT_HAVE_PCI bool diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index 657401f3e217..c4d26e8a21b3 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -721,8 +721,8 @@ static inline long __arc_strnlen_user(const char __user *s, long n) } #ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE -#define __copy_from_user(t, f, n) __arc_copy_from_user(t, f, n) -#define __copy_to_user(t, f, n) __arc_copy_to_user(t, f, n) +#define raw_copy_from_user __arc_copy_from_user +#define raw_copy_to_user __arc_copy_to_user #define __clear_user(d, n) __arc_clear_user(d, n) #define __strncpy_from_user(d, s, n) __arc_strncpy_from_user(d, s, n) #define __strnlen_user(s, n) __arc_strnlen_user(s, n) @@ -737,8 +737,8 @@ extern long arc_strncpy_from_user_noinline (char *dst, const char __user *src, long count); extern long arc_strnlen_user_noinline(const char __user *src, long n); -#define __copy_from_user(t, f, n) arc_copy_from_user_noinline(t, f, n) -#define __copy_to_user(t, f, n) arc_copy_to_user_noinline(t, f, n) +#define raw_copy_from_user arc_copy_from_user_noinline +#define raw_copy_to_user arc_copy_to_user_noinline #define __clear_user(d, n) arc_clear_user_noinline(d, n) #define __strncpy_from_user(d, s, n) arc_strncpy_from_user_noinline(d, s, n) #define __strnlen_user(s, n) arc_strnlen_user_noinline(s, n) From a41e0d754240fe8ca9c4f2070bf67e3b0228aa22 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 25 Mar 2017 19:38:23 -0400 Subject: [PATCH 68/96] x86: don't wank with magical size in __copy_in_user() ... especially since copy_in_user() doesn't Signed-off-by: Al Viro --- arch/x86/include/asm/uaccess_64.h | 58 ++----------------------------- arch/x86/lib/usercopy_64.c | 9 ----- 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 242936b0cb4b..497ca1bb0440 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -185,62 +185,8 @@ int __copy_to_user(void __user *dst, const void *src, unsigned size) static __always_inline __must_check int __copy_in_user(void __user *dst, const void __user *src, unsigned size) { - int ret = 0; - - might_fault(); - if (!__builtin_constant_p(size)) - return copy_user_generic((__force void *)dst, - (__force void *)src, size); - switch (size) { - case 1: { - u8 tmp; - __uaccess_begin(); - __get_user_asm(tmp, (u8 __user *)src, - ret, "b", "b", "=q", 1); - if (likely(!ret)) - __put_user_asm(tmp, (u8 __user *)dst, - ret, "b", "b", "iq", 1); - __uaccess_end(); - return ret; - } - case 2: { - u16 tmp; - __uaccess_begin(); - __get_user_asm(tmp, (u16 __user *)src, - ret, "w", "w", "=r", 2); - if (likely(!ret)) - __put_user_asm(tmp, (u16 __user *)dst, - ret, "w", "w", "ir", 2); - __uaccess_end(); - return ret; - } - - case 4: { - u32 tmp; - __uaccess_begin(); - __get_user_asm(tmp, (u32 __user *)src, - ret, "l", "k", "=r", 4); - if (likely(!ret)) - __put_user_asm(tmp, (u32 __user *)dst, - ret, "l", "k", "ir", 4); - __uaccess_end(); - return ret; - } - case 8: { - u64 tmp; - __uaccess_begin(); - __get_user_asm(tmp, (u64 __user *)src, - ret, "q", "", "=r", 8); - if (likely(!ret)) - __put_user_asm(tmp, (u64 __user *)dst, - ret, "q", "", "er", 8); - __uaccess_end(); - return ret; - } - default: - return copy_user_generic((__force void *)dst, - (__force void *)src, size); - } + return copy_user_generic((__force void *)dst, + (__force void *)src, size); } static __must_check __always_inline int diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 6c8b6a6c1b38..3b7c40a2e3e1 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -54,15 +54,6 @@ unsigned long clear_user(void __user *to, unsigned long n) } EXPORT_SYMBOL(clear_user); -unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len) -{ - if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len)) { - return copy_user_generic((__force void *)to, (__force void *)from, len); - } - return len; -} -EXPORT_SYMBOL(copy_in_user); - /* * Try to copy last bytes and clear the rest if needed. * Since protection fault in copy_from/to_user is not a normal situation, From beba3a20bf90ce1b93e24592c3ebf0d0bb581bbe Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 25 Mar 2017 19:33:21 -0400 Subject: [PATCH 69/96] x86: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/x86/Kconfig | 1 + arch/x86/include/asm/uaccess.h | 53 ---------- arch/x86/include/asm/uaccess_32.h | 95 +++-------------- arch/x86/include/asm/uaccess_64.h | 45 +------- arch/x86/lib/usercopy.c | 55 +--------- arch/x86/lib/usercopy_32.c | 170 +----------------------------- 6 files changed, 24 insertions(+), 395 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index cc98d5a294ee..5f59fc388063 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -175,6 +175,7 @@ config X86 select USER_STACKTRACE_SUPPORT select VIRT_TO_BUS select X86_FEATURE_NAMES if PROC_FS + select ARCH_HAS_RAW_COPY_USER config INSTRUCTION_DECODER def_bool y diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 26410afcb8b0..68766b276d9e 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -682,59 +682,6 @@ extern struct movsl_mask { # include #endif -unsigned long __must_check _copy_from_user(void *to, const void __user *from, - unsigned n); -unsigned long __must_check _copy_to_user(void __user *to, const void *from, - unsigned n); - -extern void __compiletime_error("usercopy buffer size is too small") -__bad_copy_user(void); - -static inline void copy_user_overflow(int size, unsigned long count) -{ - WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); -} - -static __always_inline unsigned long __must_check -copy_from_user(void *to, const void __user *from, unsigned long n) -{ - int sz = __compiletime_object_size(to); - - might_fault(); - - kasan_check_write(to, n); - - if (likely(sz < 0 || sz >= n)) { - check_object_size(to, n, false); - n = _copy_from_user(to, from, n); - } else if (!__builtin_constant_p(n)) - copy_user_overflow(sz, n); - else - __bad_copy_user(); - - return n; -} - -static __always_inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long n) -{ - int sz = __compiletime_object_size(from); - - kasan_check_read(from, n); - - might_fault(); - - if (likely(sz < 0 || sz >= n)) { - check_object_size(from, n, true); - n = _copy_to_user(to, from, n); - } else if (!__builtin_constant_p(n)) - copy_user_overflow(sz, n); - else - __bad_copy_user(); - - return n; -} - /* * We rely on the nested NMI work to allow atomic faults from the NMI path; the * nested NMI paths are careful to preserve CR2. diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h index 19e6c050c438..aeda9bb8af50 100644 --- a/arch/x86/include/asm/uaccess_32.h +++ b/arch/x86/include/asm/uaccess_32.h @@ -8,113 +8,48 @@ #include #include -unsigned long __must_check __copy_to_user_ll - (void __user *to, const void *from, unsigned long n); -unsigned long __must_check __copy_from_user_ll - (void *to, const void __user *from, unsigned long n); -unsigned long __must_check __copy_from_user_ll_nozero - (void *to, const void __user *from, unsigned long n); +unsigned long __must_check __copy_user_ll + (void *to, const void *from, unsigned long n); unsigned long __must_check __copy_from_user_ll_nocache_nozero (void *to, const void __user *from, unsigned long n); -/** - * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. - * - * Copy data from kernel space to user space. Caller must check - * the specified block with access_ok() before calling this function. - * The caller should also make sure he pins the user space address - * so that we don't result in page fault and sleep. - */ static __always_inline unsigned long __must_check -__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n) +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - check_object_size(from, n, true); - return __copy_to_user_ll(to, from, n); -} - -/** - * __copy_to_user: - Copy a block of data into user space, with less checking. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -static __always_inline unsigned long __must_check -__copy_to_user(void __user *to, const void *from, unsigned long n) -{ - might_fault(); - return __copy_to_user_inatomic(to, from, n); + return __copy_user_ll((__force void *)to, from, n); } static __always_inline unsigned long -__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n) +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { - return __copy_from_user_ll_nozero(to, from, n); -} - -/** - * __copy_from_user: - Copy a block of data from user space, with less checking. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - * - * An alternate version - __copy_from_user_inatomic() - may be called from - * atomic context and will fail rather than sleep. In this case the - * uncopied bytes will *NOT* be padded with zeros. See fs/filemap.h - * for explanation of why this is needed. - */ -static __always_inline unsigned long -__copy_from_user(void *to, const void __user *from, unsigned long n) -{ - might_fault(); - check_object_size(to, n, false); if (__builtin_constant_p(n)) { unsigned long ret; switch (n) { case 1: + ret = 0; __uaccess_begin(); - __get_user_size(*(u8 *)to, from, 1, ret, 1); + __get_user_asm_nozero(*(u8 *)to, from, ret, + "b", "b", "=q", 1); __uaccess_end(); return ret; case 2: + ret = 0; __uaccess_begin(); - __get_user_size(*(u16 *)to, from, 2, ret, 2); + __get_user_asm_nozero(*(u16 *)to, from, ret, + "w", "w", "=r", 2); __uaccess_end(); return ret; case 4: + ret = 0; __uaccess_begin(); - __get_user_size(*(u32 *)to, from, 4, ret, 4); + __get_user_asm_nozero(*(u32 *)to, from, ret, + "l", "k", "=r", 4); __uaccess_end(); return ret; } } - return __copy_from_user_ll(to, from, n); + return __copy_user_ll(to, (__force const void *)from, n); } static __always_inline unsigned long diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 497ca1bb0440..c5504b9a472e 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -45,15 +45,11 @@ copy_user_generic(void *to, const void *from, unsigned len) return ret; } -__must_check unsigned long -copy_in_user(void __user *to, const void __user *from, unsigned len); - -static __always_inline __must_check -int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size) +static __always_inline __must_check unsigned long +raw_copy_from_user(void *dst, const void __user *src, unsigned long size) { int ret = 0; - check_object_size(dst, size, false); if (!__builtin_constant_p(size)) return copy_user_generic(dst, (__force void *)src, size); switch (size) { @@ -106,20 +102,11 @@ int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size) } } -static __always_inline __must_check -int __copy_from_user(void *dst, const void __user *src, unsigned size) -{ - might_fault(); - kasan_check_write(dst, size); - return __copy_from_user_nocheck(dst, src, size); -} - -static __always_inline __must_check -int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size) +static __always_inline __must_check unsigned long +raw_copy_to_user(void __user *dst, const void *src, unsigned long size) { int ret = 0; - check_object_size(src, size, true); if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst, src, size); switch (size) { @@ -175,34 +162,12 @@ int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size) } static __always_inline __must_check -int __copy_to_user(void __user *dst, const void *src, unsigned size) -{ - might_fault(); - kasan_check_read(src, size); - return __copy_to_user_nocheck(dst, src, size); -} - -static __always_inline __must_check -int __copy_in_user(void __user *dst, const void __user *src, unsigned size) +unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigned long size) { return copy_user_generic((__force void *)dst, (__force void *)src, size); } -static __must_check __always_inline int -__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size) -{ - kasan_check_write(dst, size); - return __copy_from_user_nocheck(dst, src, size); -} - -static __must_check __always_inline int -__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) -{ - kasan_check_read(src, size); - return __copy_to_user_nocheck(dst, src, size); -} - extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest); diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c index a851f3d199c2..c8c6ad0d58b8 100644 --- a/arch/x86/lib/usercopy.c +++ b/arch/x86/lib/usercopy.c @@ -4,12 +4,9 @@ * For licencing details see kernel-base/COPYING */ -#include +#include #include -#include -#include - /* * We rely on the nested NMI work to allow atomic faults from the NMI path; the * nested NMI paths are careful to preserve CR2. @@ -34,53 +31,3 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n) return ret; } EXPORT_SYMBOL_GPL(copy_from_user_nmi); - -/** - * copy_to_user: - Copy a block of data into user space. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -unsigned long _copy_to_user(void __user *to, const void *from, unsigned n) -{ - if (access_ok(VERIFY_WRITE, to, n)) - n = __copy_to_user(to, from, n); - return n; -} -EXPORT_SYMBOL(_copy_to_user); - -/** - * copy_from_user: - Copy a block of data from user space. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - */ -unsigned long _copy_from_user(void *to, const void __user *from, unsigned n) -{ - unsigned long res = n; - if (access_ok(VERIFY_READ, from, n)) - res = __copy_from_user_inatomic(to, from, n); - if (unlikely(res)) - memset(to + n - res, 0, res); - return res; -} -EXPORT_SYMBOL(_copy_from_user); diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index 02aa7aa8b9f3..bd057a4ffe6e 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -5,12 +5,7 @@ * Copyright 1997 Andi Kleen * Copyright 1997 Linus Torvalds */ -#include -#include -#include #include -#include -#include #include #include #include @@ -201,98 +196,6 @@ __copy_user_intel(void __user *to, const void *from, unsigned long size) return size; } -static unsigned long -__copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size) -{ - int d0, d1; - __asm__ __volatile__( - " .align 2,0x90\n" - "0: movl 32(%4), %%eax\n" - " cmpl $67, %0\n" - " jbe 2f\n" - "1: movl 64(%4), %%eax\n" - " .align 2,0x90\n" - "2: movl 0(%4), %%eax\n" - "21: movl 4(%4), %%edx\n" - " movl %%eax, 0(%3)\n" - " movl %%edx, 4(%3)\n" - "3: movl 8(%4), %%eax\n" - "31: movl 12(%4),%%edx\n" - " movl %%eax, 8(%3)\n" - " movl %%edx, 12(%3)\n" - "4: movl 16(%4), %%eax\n" - "41: movl 20(%4), %%edx\n" - " movl %%eax, 16(%3)\n" - " movl %%edx, 20(%3)\n" - "10: movl 24(%4), %%eax\n" - "51: movl 28(%4), %%edx\n" - " movl %%eax, 24(%3)\n" - " movl %%edx, 28(%3)\n" - "11: movl 32(%4), %%eax\n" - "61: movl 36(%4), %%edx\n" - " movl %%eax, 32(%3)\n" - " movl %%edx, 36(%3)\n" - "12: movl 40(%4), %%eax\n" - "71: movl 44(%4), %%edx\n" - " movl %%eax, 40(%3)\n" - " movl %%edx, 44(%3)\n" - "13: movl 48(%4), %%eax\n" - "81: movl 52(%4), %%edx\n" - " movl %%eax, 48(%3)\n" - " movl %%edx, 52(%3)\n" - "14: movl 56(%4), %%eax\n" - "91: movl 60(%4), %%edx\n" - " movl %%eax, 56(%3)\n" - " movl %%edx, 60(%3)\n" - " addl $-64, %0\n" - " addl $64, %4\n" - " addl $64, %3\n" - " cmpl $63, %0\n" - " ja 0b\n" - "5: movl %0, %%eax\n" - " shrl $2, %0\n" - " andl $3, %%eax\n" - " cld\n" - "6: rep; movsl\n" - " movl %%eax,%0\n" - "7: rep; movsb\n" - "8:\n" - ".section .fixup,\"ax\"\n" - "9: lea 0(%%eax,%0,4),%0\n" - "16: pushl %0\n" - " pushl %%eax\n" - " xorl %%eax,%%eax\n" - " rep; stosb\n" - " popl %%eax\n" - " popl %0\n" - " jmp 8b\n" - ".previous\n" - _ASM_EXTABLE(0b,16b) - _ASM_EXTABLE(1b,16b) - _ASM_EXTABLE(2b,16b) - _ASM_EXTABLE(21b,16b) - _ASM_EXTABLE(3b,16b) - _ASM_EXTABLE(31b,16b) - _ASM_EXTABLE(4b,16b) - _ASM_EXTABLE(41b,16b) - _ASM_EXTABLE(10b,16b) - _ASM_EXTABLE(51b,16b) - _ASM_EXTABLE(11b,16b) - _ASM_EXTABLE(61b,16b) - _ASM_EXTABLE(12b,16b) - _ASM_EXTABLE(71b,16b) - _ASM_EXTABLE(13b,16b) - _ASM_EXTABLE(81b,16b) - _ASM_EXTABLE(14b,16b) - _ASM_EXTABLE(91b,16b) - _ASM_EXTABLE(6b,9b) - _ASM_EXTABLE(7b,16b) - : "=&c"(size), "=&D" (d0), "=&S" (d1) - : "1"(to), "2"(from), "0"(size) - : "eax", "edx", "memory"); - return size; -} - static unsigned long __copy_user_intel_nocache(void *to, const void __user *from, unsigned long size) { @@ -387,8 +290,6 @@ static unsigned long __copy_user_intel_nocache(void *to, * Leave these declared but undefined. They should not be any references to * them */ -unsigned long __copy_user_zeroing_intel(void *to, const void __user *from, - unsigned long size); unsigned long __copy_user_intel(void __user *to, const void *from, unsigned long size); #endif /* CONFIG_X86_INTEL_USERCOPY */ @@ -427,47 +328,7 @@ do { \ : "memory"); \ } while (0) -#define __copy_user_zeroing(to, from, size) \ -do { \ - int __d0, __d1, __d2; \ - __asm__ __volatile__( \ - " cmp $7,%0\n" \ - " jbe 1f\n" \ - " movl %1,%0\n" \ - " negl %0\n" \ - " andl $7,%0\n" \ - " subl %0,%3\n" \ - "4: rep; movsb\n" \ - " movl %3,%0\n" \ - " shrl $2,%0\n" \ - " andl $3,%3\n" \ - " .align 2,0x90\n" \ - "0: rep; movsl\n" \ - " movl %3,%0\n" \ - "1: rep; movsb\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "5: addl %3,%0\n" \ - " jmp 6f\n" \ - "3: lea 0(%3,%0,4),%0\n" \ - "6: pushl %0\n" \ - " pushl %%eax\n" \ - " xorl %%eax,%%eax\n" \ - " rep; stosb\n" \ - " popl %%eax\n" \ - " popl %0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(4b,5b) \ - _ASM_EXTABLE(0b,3b) \ - _ASM_EXTABLE(1b,6b) \ - : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ - : "3"(size), "0"(size), "1"(to), "2"(from) \ - : "memory"); \ -} while (0) - -unsigned long __copy_to_user_ll(void __user *to, const void *from, - unsigned long n) +unsigned long __copy_user_ll(void *to, const void *from, unsigned long n) { stac(); if (movsl_is_ok(to, from, n)) @@ -477,34 +338,7 @@ unsigned long __copy_to_user_ll(void __user *to, const void *from, clac(); return n; } -EXPORT_SYMBOL(__copy_to_user_ll); - -unsigned long __copy_from_user_ll(void *to, const void __user *from, - unsigned long n) -{ - stac(); - if (movsl_is_ok(to, from, n)) - __copy_user_zeroing(to, from, n); - else - n = __copy_user_zeroing_intel(to, from, n); - clac(); - return n; -} -EXPORT_SYMBOL(__copy_from_user_ll); - -unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from, - unsigned long n) -{ - stac(); - if (movsl_is_ok(to, from, n)) - __copy_user(to, from, n); - else - n = __copy_user_intel((void __user *)to, - (const void *)from, n); - clac(); - return n; -} -EXPORT_SYMBOL(__copy_from_user_ll_nozero); +EXPORT_SYMBOL(__copy_user_ll); unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, unsigned long n) From e13909a4acc4b68b30527d8442d4267658963757 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Wed, 29 Mar 2017 11:53:33 -0700 Subject: [PATCH 70/96] ARC: uaccess: enable INLINE_COPY_{TO,FROM}_USER ... ... and switch to generic out of line version in lib/usercopy.c Signed-off-by: Vineet Gupta Signed-off-by: Al Viro --- arch/arc/include/asm/uaccess.h | 16 ++++++---------- arch/arc/mm/extable.c | 14 -------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index c4d26e8a21b3..f35974ee7264 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -168,7 +168,7 @@ static inline unsigned long -__arc_copy_from_user(void *to, const void __user *from, unsigned long n) +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { long res = 0; char val; @@ -395,7 +395,7 @@ __arc_copy_from_user(void *to, const void __user *from, unsigned long n) } static inline unsigned long -__arc_copy_to_user(void __user *to, const void *from, unsigned long n) +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { long res = 0; char val; @@ -721,24 +721,20 @@ static inline long __arc_strnlen_user(const char __user *s, long n) } #ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE -#define raw_copy_from_user __arc_copy_from_user -#define raw_copy_to_user __arc_copy_to_user + +#define INLINE_COPY_TO_USER +#define INLINE_COPY_FROM_USER + #define __clear_user(d, n) __arc_clear_user(d, n) #define __strncpy_from_user(d, s, n) __arc_strncpy_from_user(d, s, n) #define __strnlen_user(s, n) __arc_strnlen_user(s, n) #else -extern long arc_copy_from_user_noinline(void *to, const void __user * from, - unsigned long n); -extern long arc_copy_to_user_noinline(void __user *to, const void *from, - unsigned long n); extern unsigned long arc_clear_user_noinline(void __user *to, unsigned long n); extern long arc_strncpy_from_user_noinline (char *dst, const char __user *src, long count); extern long arc_strnlen_user_noinline(const char __user *src, long n); -#define raw_copy_from_user arc_copy_from_user_noinline -#define raw_copy_to_user arc_copy_to_user_noinline #define __clear_user(d, n) arc_clear_user_noinline(d, n) #define __strncpy_from_user(d, s, n) arc_strncpy_from_user_noinline(d, s, n) #define __strnlen_user(s, n) arc_strnlen_user_noinline(s, n) diff --git a/arch/arc/mm/extable.c b/arch/arc/mm/extable.c index c86906b41bfe..72125a34e780 100644 --- a/arch/arc/mm/extable.c +++ b/arch/arc/mm/extable.c @@ -28,20 +28,6 @@ int fixup_exception(struct pt_regs *regs) #ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE -long arc_copy_from_user_noinline(void *to, const void __user *from, - unsigned long n) -{ - return __arc_copy_from_user(to, from, n); -} -EXPORT_SYMBOL(arc_copy_from_user_noinline); - -long arc_copy_to_user_noinline(void __user *to, const void *from, - unsigned long n) -{ - return __arc_copy_to_user(to, from, n); -} -EXPORT_SYMBOL(arc_copy_to_user_noinline); - unsigned long arc_clear_user_noinline(void __user *to, unsigned long n) { From 37096003c8a7de5d24c6cf86234004c635fdd617 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 15:06:24 -0400 Subject: [PATCH 71/96] s390: get rid of zeroing, switch to RAW_COPY_USER [folded a fix from Martin] Signed-off-by: Al Viro --- arch/s390/Kconfig | 1 + arch/s390/include/asm/uaccess.h | 120 +++----------------------------- arch/s390/lib/uaccess.c | 68 ++++++------------ 3 files changed, 33 insertions(+), 156 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index a2dcef0aacc7..0724dbf2d8f8 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -178,6 +178,7 @@ config S390 select ARCH_HAS_SCALED_CPUTIME select VIRT_TO_BUS select HAVE_NMI + select ARCH_HAS_RAW_COPY_USER config SCHED_OMIT_FRAME_POINTER diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 29f5bf24e5fa..72d69b9585c2 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -60,47 +60,14 @@ static inline int __range_ok(unsigned long addr, unsigned long size) #define access_ok(type, addr, size) __access_ok(addr, size) -/** - * __copy_from_user: - Copy a block of data from user space, with less checking. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - */ -unsigned long __must_check __copy_from_user(void *to, const void __user *from, - unsigned long n); +unsigned long __must_check +raw_copy_from_user(void *to, const void __user *from, unsigned long n); -/** - * __copy_to_user: - Copy a block of data into user space, with less checking. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -unsigned long __must_check __copy_to_user(void __user *to, const void *from, - unsigned long n); +unsigned long __must_check +raw_copy_to_user(void __user *to, const void *from, unsigned long n); -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER #ifdef CONFIG_HAVE_MARCH_Z10_FEATURES @@ -189,13 +156,13 @@ static inline int __get_user_fn(void *x, const void __user *ptr, unsigned long s static inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) { - size = __copy_to_user(ptr, x, size); + size = raw_copy_to_user(ptr, x, size); return size ? -EFAULT : 0; } static inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size) { - size = __copy_from_user(x, ptr, size); + size = raw_copy_from_user(x, ptr, size); return size ? -EFAULT : 0; } @@ -285,77 +252,8 @@ int __get_user_bad(void) __attribute__((noreturn)); #define __put_user_unaligned __put_user #define __get_user_unaligned __get_user -extern void __compiletime_error("usercopy buffer size is too small") -__bad_copy_user(void); - -static inline void copy_user_overflow(int size, unsigned long count) -{ - WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); -} - -/** - * copy_to_user: - Copy a block of data into user space. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -static inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long n) -{ - might_fault(); - return __copy_to_user(to, from, n); -} - -/** - * copy_from_user: - Copy a block of data from user space. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - */ -static inline unsigned long __must_check -copy_from_user(void *to, const void __user *from, unsigned long n) -{ - unsigned int sz = __compiletime_object_size(to); - - might_fault(); - if (unlikely(sz != -1 && sz < n)) { - if (!__builtin_constant_p(n)) - copy_user_overflow(sz, n); - else - __bad_copy_user(); - return n; - } - return __copy_from_user(to, from, n); -} - unsigned long __must_check -__copy_in_user(void __user *to, const void __user *from, unsigned long n); - -static inline unsigned long __must_check -copy_in_user(void __user *to, const void __user *from, unsigned long n) -{ - might_fault(); - return __copy_in_user(to, from, n); -} +raw_copy_in_user(void __user *to, const void __user *from, unsigned long n); /* * Copy a null terminated string from userspace. diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index f481fcde067b..1e5bb2b86c42 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -26,7 +26,7 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr tmp1 = -4096UL; asm volatile( "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n" - "9: jz 7f\n" + "6: jz 4f\n" "1: algr %0,%3\n" " slgr %1,%3\n" " slgr %2,%3\n" @@ -35,23 +35,13 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ " slgr %4,%1\n" " clgr %0,%4\n" /* copy crosses next page boundary? */ - " jnh 4f\n" + " jnh 5f\n" "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n" - "10:slgr %0,%4\n" - " algr %2,%4\n" - "4: lghi %4,-1\n" - " algr %4,%0\n" /* copy remaining size, subtract 1 */ - " bras %3,6f\n" /* memset loop */ - " xc 0(1,%2),0(%2)\n" - "5: xc 0(256,%2),0(%2)\n" - " la %2,256(%2)\n" - "6: aghi %4,-256\n" - " jnm 5b\n" - " ex %4,0(%3)\n" - " j 8f\n" - "7: slgr %0,%0\n" - "8:\n" - EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b) + "7: slgr %0,%4\n" + " j 5f\n" + "4: slgr %0,%0\n" + "5:\n" + EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) : "d" (reg0) : "cc", "memory"); return size; @@ -67,49 +57,38 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, asm volatile( " sacf 0\n" "0: mvcp 0(%0,%2),0(%1),%3\n" - "10:jz 8f\n" + "7: jz 5f\n" "1: algr %0,%3\n" " la %1,256(%1)\n" " la %2,256(%2)\n" "2: mvcp 0(%0,%2),0(%1),%3\n" - "11:jnz 1b\n" - " j 8f\n" + "8: jnz 1b\n" + " j 5f\n" "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ " lghi %3,-4096\n" " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ " slgr %4,%1\n" " clgr %0,%4\n" /* copy crosses next page boundary? */ - " jnh 5f\n" + " jnh 6f\n" "4: mvcp 0(%4,%2),0(%1),%3\n" - "12:slgr %0,%4\n" - " algr %2,%4\n" - "5: lghi %4,-1\n" - " algr %4,%0\n" /* copy remaining size, subtract 1 */ - " bras %3,7f\n" /* memset loop */ - " xc 0(1,%2),0(%2)\n" - "6: xc 0(256,%2),0(%2)\n" - " la %2,256(%2)\n" - "7: aghi %4,-256\n" - " jnm 6b\n" - " ex %4,0(%3)\n" - " j 9f\n" - "8: slgr %0,%0\n" - "9: sacf 768\n" - EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,5b) - EX_TABLE(10b,3b) EX_TABLE(11b,3b) EX_TABLE(12b,5b) + "9: slgr %0,%4\n" + " j 6f\n" + "5: slgr %0,%0\n" + "6: sacf 768\n" + EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b) + EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) : : "cc", "memory"); return size; } -unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) +unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { - check_object_size(to, n, false); if (static_branch_likely(&have_mvcos)) return copy_from_user_mvcos(to, from, n); return copy_from_user_mvcp(to, from, n); } -EXPORT_SYMBOL(__copy_from_user); +EXPORT_SYMBOL(raw_copy_from_user); static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x, unsigned long size) @@ -176,14 +155,13 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, return size; } -unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) +unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - check_object_size(from, n, true); if (static_branch_likely(&have_mvcos)) return copy_to_user_mvcos(to, from, n); return copy_to_user_mvcs(to, from, n); } -EXPORT_SYMBOL(__copy_to_user); +EXPORT_SYMBOL(raw_copy_to_user); static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from, unsigned long size) @@ -240,13 +218,13 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user return size; } -unsigned long __copy_in_user(void __user *to, const void __user *from, unsigned long n) +unsigned long raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) { if (static_branch_likely(&have_mvcos)) return copy_in_user_mvcos(to, from, n); return copy_in_user_mvc(to, from, n); } -EXPORT_SYMBOL(__copy_in_user); +EXPORT_SYMBOL(raw_copy_in_user); static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size) { From f64fd180ec2c0bcca2a141314db3b5d3b6bfef2f Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 24 Mar 2017 23:11:26 -0400 Subject: [PATCH 72/96] parisc: switch to RAW_COPY_USER ... and remove dead declarations, while we are at it Signed-off-by: Al Viro --- arch/parisc/Kconfig | 1 + arch/parisc/include/asm/uaccess.h | 64 ++++--------------------------- arch/parisc/lib/memcpy.c | 14 +++---- 3 files changed, 16 insertions(+), 63 deletions(-) diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index ad294b3fb90b..15b7b279a169 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -41,6 +41,7 @@ config PARISC select GENERIC_CLOCKEVENTS select ARCH_NO_COHERENT_DMA_MMAP select CPU_NO_EFFICIENT_FFS + select ARCH_HAS_RAW_COPY_USER help The PA-RISC microprocessor is designed by Hewlett-Packard and used diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index cbb0aaca8d39..852322f2d1f9 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -187,9 +187,6 @@ struct exception_data { * Complex access routines -- external declarations */ -extern unsigned long lcopy_to_user(void __user *, const void *, unsigned long); -extern unsigned long lcopy_from_user(void *, const void __user *, unsigned long); -extern unsigned long lcopy_in_user(void __user *, const void __user *, unsigned long); extern long strncpy_from_user(char *, const char __user *, long); extern unsigned lclear_user(void __user *, unsigned long); extern long lstrnlen_user(const char __user *, long); @@ -203,59 +200,14 @@ extern long lstrnlen_user(const char __user *, long); #define clear_user lclear_user #define __clear_user lclear_user -unsigned long __must_check __copy_to_user(void __user *dst, const void *src, - unsigned long len); -unsigned long __must_check __copy_from_user(void *dst, const void __user *src, - unsigned long len); -unsigned long copy_in_user(void __user *dst, const void __user *src, - unsigned long len); -#define __copy_in_user copy_in_user -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - -extern void __compiletime_error("usercopy buffer size is too small") -__bad_copy_user(void); - -static inline void copy_user_overflow(int size, unsigned long count) -{ - WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); -} - -static __always_inline unsigned long __must_check -copy_from_user(void *to, const void __user *from, unsigned long n) -{ - int sz = __compiletime_object_size(to); - unsigned long ret = n; - - if (likely(sz < 0 || sz >= n)) { - check_object_size(to, n, false); - ret = __copy_from_user(to, from, n); - } else if (!__builtin_constant_p(n)) - copy_user_overflow(sz, n); - else - __bad_copy_user(); - - if (unlikely(ret)) - memset(to + (n - ret), 0, ret); - - return ret; -} - -static __always_inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long n) -{ - int sz = __compiletime_object_size(from); - - if (likely(sz < 0 || sz >= n)) { - check_object_size(from, n, true); - n = __copy_to_user(to, from, n); - } else if (!__builtin_constant_p(n)) - copy_user_overflow(sz, n); - else - __bad_copy_user(); - - return n; -} +unsigned long __must_check raw_copy_to_user(void __user *dst, const void *src, + unsigned long len); +unsigned long __must_check raw_copy_from_user(void *dst, const void __user *src, + unsigned long len); +unsigned long __must_check raw_copy_in_user(void __user *dst, const void __user *src, + unsigned long len); +#define INLINE_COPY_TO_USER +#define INLINE_COPY_FROM_USER struct pt_regs; int fixup_exception(struct pt_regs *regs); diff --git a/arch/parisc/lib/memcpy.c b/arch/parisc/lib/memcpy.c index e88e4a5572dc..99115cd9e790 100644 --- a/arch/parisc/lib/memcpy.c +++ b/arch/parisc/lib/memcpy.c @@ -36,25 +36,25 @@ DECLARE_PER_CPU(struct exception_data, exception_data); extern unsigned long pa_memcpy(void *dst, const void *src, unsigned long len); -unsigned long __copy_to_user(void __user *dst, const void *src, - unsigned long len) +unsigned long raw_copy_to_user(void __user *dst, const void *src, + unsigned long len) { mtsp(get_kernel_space(), 1); mtsp(get_user_space(), 2); return pa_memcpy((void __force *)dst, src, len); } -EXPORT_SYMBOL(__copy_to_user); +EXPORT_SYMBOL(raw_copy_to_user); -unsigned long __copy_from_user(void *dst, const void __user *src, +unsigned long raw_copy_from_user(void *dst, const void __user *src, unsigned long len) { mtsp(get_user_space(), 1); mtsp(get_kernel_space(), 2); return pa_memcpy(dst, (void __force *)src, len); } -EXPORT_SYMBOL(__copy_from_user); +EXPORT_SYMBOL(raw_copy_from_user); -unsigned long copy_in_user(void __user *dst, const void __user *src, unsigned long len) +unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigned long len) { mtsp(get_user_space(), 1); mtsp(get_user_space(), 2); @@ -70,7 +70,7 @@ void * memcpy(void * dst,const void *src, size_t count) return dst; } -EXPORT_SYMBOL(copy_in_user); +EXPORT_SYMBOL(raw_copy_in_user); EXPORT_SYMBOL(memcpy); long probe_kernel_read(void *dst, const void *src, size_t size) From 31af2f36d50e3b9b2fb7f17aa430c11c91f946c4 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 17:04:45 -0400 Subject: [PATCH 73/96] sparc: switch to RAW_COPY_USER ... and drop zeroing in sparc32. Signed-off-by: Al Viro --- arch/sparc/Kconfig | 1 + arch/sparc/include/asm/uaccess_32.h | 29 ++++--------------------- arch/sparc/include/asm/uaccess_64.h | 33 +++++------------------------ arch/sparc/lib/GENcopy_from_user.S | 2 +- arch/sparc/lib/GENcopy_to_user.S | 2 +- arch/sparc/lib/GENpatch.S | 4 ++-- arch/sparc/lib/NG2copy_from_user.S | 2 +- arch/sparc/lib/NG2copy_to_user.S | 2 +- arch/sparc/lib/NG2patch.S | 4 ++-- arch/sparc/lib/NG4copy_from_user.S | 2 +- arch/sparc/lib/NG4copy_to_user.S | 2 +- arch/sparc/lib/NG4patch.S | 4 ++-- arch/sparc/lib/NGcopy_from_user.S | 2 +- arch/sparc/lib/NGcopy_to_user.S | 2 +- arch/sparc/lib/NGpatch.S | 4 ++-- arch/sparc/lib/U1copy_from_user.S | 4 ++-- arch/sparc/lib/U1copy_to_user.S | 4 ++-- arch/sparc/lib/U3copy_to_user.S | 2 +- arch/sparc/lib/U3patch.S | 4 ++-- arch/sparc/lib/copy_in_user.S | 6 +++--- arch/sparc/lib/copy_user.S | 16 +------------- 21 files changed, 37 insertions(+), 94 deletions(-) diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 68ac5c7cd982..68353048f24e 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -45,6 +45,7 @@ config SPARC select HAVE_ARCH_HARDENED_USERCOPY select PROVE_LOCKING_SMALL if PROVE_LOCKING select ARCH_WANT_RELAX_ORDER + select ARCH_HAS_RAW_COPY_USER config SPARC32 def_bool !64BIT diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 93910845733a..12ebee2d97c7 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -235,39 +235,18 @@ int __get_user_bad(void); unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size); -static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) +static inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - if (n && __access_ok((unsigned long) to, n)) { - check_object_size(from, n, true); - return __copy_user(to, (__force void __user *) from, n); - } else - return n; -} - -static inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) -{ - check_object_size(from, n, true); return __copy_user(to, (__force void __user *) from, n); } -static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) -{ - if (n && __access_ok((unsigned long) from, n)) { - check_object_size(to, n, false); - return __copy_user((__force void __user *) to, from, n); - } else { - memset(to, 0, n); - return n; - } -} - -static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) +static inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { return __copy_user((__force void __user *) to, from, n); } -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER static inline unsigned long __clear_user(void __user *addr, unsigned long size) { diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 7afb4f64553f..6096d671aa63 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -176,39 +176,19 @@ __asm__ __volatile__( \ int __get_user_bad(void); -unsigned long __must_check ___copy_from_user(void *to, +unsigned long __must_check raw_copy_from_user(void *to, const void __user *from, unsigned long size); -static inline unsigned long __must_check -copy_from_user(void *to, const void __user *from, unsigned long size) -{ - check_object_size(to, size, false); - return ___copy_from_user(to, from, size); -} -#define __copy_from_user copy_from_user - -unsigned long __must_check ___copy_to_user(void __user *to, +unsigned long __must_check raw_copy_to_user(void __user *to, const void *from, unsigned long size); -static inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long size) -{ - check_object_size(from, size, true); +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER - return ___copy_to_user(to, from, size); -} -#define __copy_to_user copy_to_user - -unsigned long __must_check ___copy_in_user(void __user *to, +unsigned long __must_check raw_copy_in_user(void __user *to, const void __user *from, unsigned long size); -static inline unsigned long __must_check -copy_in_user(void __user *to, void __user *from, unsigned long size) -{ - return ___copy_in_user(to, from, size); -} -#define __copy_in_user copy_in_user unsigned long __must_check __clear_user(void __user *, unsigned long); @@ -217,9 +197,6 @@ unsigned long __must_check __clear_user(void __user *, unsigned long); __must_check long strlen_user(const char __user *str); __must_check long strnlen_user(const char __user *str, long n); -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - struct pt_regs; unsigned long compute_effective_address(struct pt_regs *, unsigned int insn, diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S index 69a439fa2fc1..8aa16ef113f2 100644 --- a/arch/sparc/lib/GENcopy_from_user.S +++ b/arch/sparc/lib/GENcopy_from_user.S @@ -23,7 +23,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S index 9947427ce354..311c8fa5e98e 100644 --- a/arch/sparc/lib/GENcopy_to_user.S +++ b/arch/sparc/lib/GENcopy_to_user.S @@ -27,7 +27,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/GENpatch.S b/arch/sparc/lib/GENpatch.S index fab9e89f16bd..95e2f1f9e477 100644 --- a/arch/sparc/lib/GENpatch.S +++ b/arch/sparc/lib/GENpatch.S @@ -26,8 +26,8 @@ .type generic_patch_copyops,#function generic_patch_copyops: GEN_DO_PATCH(memcpy, GENmemcpy) - GEN_DO_PATCH(___copy_from_user, GENcopy_from_user) - GEN_DO_PATCH(___copy_to_user, GENcopy_to_user) + GEN_DO_PATCH(raw_copy_from_user, GENcopy_from_user) + GEN_DO_PATCH(raw_copy_to_user, GENcopy_to_user) retl nop .size generic_patch_copyops,.-generic_patch_copyops diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S index b79a6998d87c..0d8a018118c2 100644 --- a/arch/sparc/lib/NG2copy_from_user.S +++ b/arch/sparc/lib/NG2copy_from_user.S @@ -36,7 +36,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S index dcec55f254ab..a7a0ea0d8a0b 100644 --- a/arch/sparc/lib/NG2copy_to_user.S +++ b/arch/sparc/lib/NG2copy_to_user.S @@ -45,7 +45,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG2patch.S b/arch/sparc/lib/NG2patch.S index 28c36f06a6d1..56ccc19adde8 100644 --- a/arch/sparc/lib/NG2patch.S +++ b/arch/sparc/lib/NG2patch.S @@ -26,8 +26,8 @@ .type niagara2_patch_copyops,#function niagara2_patch_copyops: NG_DO_PATCH(memcpy, NG2memcpy) - NG_DO_PATCH(___copy_from_user, NG2copy_from_user) - NG_DO_PATCH(___copy_to_user, NG2copy_to_user) + NG_DO_PATCH(raw_copy_from_user, NG2copy_from_user) + NG_DO_PATCH(raw_copy_to_user, NG2copy_to_user) retl nop .size niagara2_patch_copyops,.-niagara2_patch_copyops diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S index 16a286c1a528..5bb506bd61fa 100644 --- a/arch/sparc/lib/NG4copy_from_user.S +++ b/arch/sparc/lib/NG4copy_from_user.S @@ -31,7 +31,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S index 6b0276ffc858..a82d4d45fc1c 100644 --- a/arch/sparc/lib/NG4copy_to_user.S +++ b/arch/sparc/lib/NG4copy_to_user.S @@ -40,7 +40,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S index a114cbcf2a48..3cc0f8cc95df 100644 --- a/arch/sparc/lib/NG4patch.S +++ b/arch/sparc/lib/NG4patch.S @@ -26,8 +26,8 @@ .type niagara4_patch_copyops,#function niagara4_patch_copyops: NG_DO_PATCH(memcpy, NG4memcpy) - NG_DO_PATCH(___copy_from_user, NG4copy_from_user) - NG_DO_PATCH(___copy_to_user, NG4copy_to_user) + NG_DO_PATCH(raw_copy_from_user, NG4copy_from_user) + NG_DO_PATCH(raw_copy_to_user, NG4copy_to_user) retl nop .size niagara4_patch_copyops,.-niagara4_patch_copyops diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S index 9cd42fcbc781..2333b6f3e824 100644 --- a/arch/sparc/lib/NGcopy_from_user.S +++ b/arch/sparc/lib/NGcopy_from_user.S @@ -25,7 +25,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S index 5c358afd464e..07ba20bc4ea4 100644 --- a/arch/sparc/lib/NGcopy_to_user.S +++ b/arch/sparc/lib/NGcopy_to_user.S @@ -28,7 +28,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NGpatch.S b/arch/sparc/lib/NGpatch.S index 3b0674fc3366..62ccda7e7b38 100644 --- a/arch/sparc/lib/NGpatch.S +++ b/arch/sparc/lib/NGpatch.S @@ -26,8 +26,8 @@ .type niagara_patch_copyops,#function niagara_patch_copyops: NG_DO_PATCH(memcpy, NGmemcpy) - NG_DO_PATCH(___copy_from_user, NGcopy_from_user) - NG_DO_PATCH(___copy_to_user, NGcopy_to_user) + NG_DO_PATCH(raw_copy_from_user, NGcopy_from_user) + NG_DO_PATCH(raw_copy_to_user, NGcopy_to_user) retl nop .size niagara_patch_copyops,.-niagara_patch_copyops diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S index bb6ff73229e3..9a6e68a9bf4a 100644 --- a/arch/sparc/lib/U1copy_from_user.S +++ b/arch/sparc/lib/U1copy_from_user.S @@ -19,7 +19,7 @@ .text; \ .align 4; -#define FUNC_NAME ___copy_from_user +#define FUNC_NAME raw_copy_from_user #define LOAD(type,addr,dest) type##a [addr] %asi, dest #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest #define EX_RETVAL(x) 0 @@ -31,7 +31,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U1memcpy.S" diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S index ed92ce739558..d7b28491eddf 100644 --- a/arch/sparc/lib/U1copy_to_user.S +++ b/arch/sparc/lib/U1copy_to_user.S @@ -19,7 +19,7 @@ .text; \ .align 4; -#define FUNC_NAME ___copy_to_user +#define FUNC_NAME raw_copy_to_user #define STORE(type,src,addr) type##a src, [addr] ASI_AIUS #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS #define EX_RETVAL(x) 0 @@ -31,7 +31,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U1memcpy.S" diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S index c4ee858e352a..f48fb87fe9f2 100644 --- a/arch/sparc/lib/U3copy_to_user.S +++ b/arch/sparc/lib/U3copy_to_user.S @@ -31,7 +31,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U3memcpy.S" diff --git a/arch/sparc/lib/U3patch.S b/arch/sparc/lib/U3patch.S index ecc302619a6e..91cd6539b6e1 100644 --- a/arch/sparc/lib/U3patch.S +++ b/arch/sparc/lib/U3patch.S @@ -26,8 +26,8 @@ .type cheetah_patch_copyops,#function cheetah_patch_copyops: ULTRA3_DO_PATCH(memcpy, U3memcpy) - ULTRA3_DO_PATCH(___copy_from_user, U3copy_from_user) - ULTRA3_DO_PATCH(___copy_to_user, U3copy_to_user) + ULTRA3_DO_PATCH(raw_copy_from_user, U3copy_from_user) + ULTRA3_DO_PATCH(raw_copy_to_user, U3copy_to_user) retl nop .size cheetah_patch_copyops,.-cheetah_patch_copyops diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S index 0252b218de45..1b73bb80aeb0 100644 --- a/arch/sparc/lib/copy_in_user.S +++ b/arch/sparc/lib/copy_in_user.S @@ -44,7 +44,7 @@ __retl_o2_plus_1: * to copy register windows around during thread cloning. */ -ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ +ENTRY(raw_copy_in_user) /* %o0=dst, %o1=src, %o2=len */ cmp %o2, 0 be,pn %XCC, 85f or %o0, %o1, %o3 @@ -105,5 +105,5 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ add %o0, 1, %o0 retl clr %o0 -ENDPROC(___copy_in_user) -EXPORT_SYMBOL(___copy_in_user) +ENDPROC(raw_copy_in_user) +EXPORT_SYMBOL(raw_copy_in_user) diff --git a/arch/sparc/lib/copy_user.S b/arch/sparc/lib/copy_user.S index cea644dc67a6..bc243ee807cc 100644 --- a/arch/sparc/lib/copy_user.S +++ b/arch/sparc/lib/copy_user.S @@ -364,21 +364,7 @@ short_aligned_end: 97: mov %o2, %g3 fixupretl: - sethi %hi(PAGE_OFFSET), %g1 - cmp %o0, %g1 - blu 1f - cmp %o1, %g1 - bgeu 1f - ld [%g6 + TI_PREEMPT], %g1 - cmp %g1, 0 - bne 1f - nop - save %sp, -64, %sp - mov %i0, %o0 - call __bzero - mov %g3, %o1 - restore -1: retl + retl mov %g3, %o0 /* exception routine sets %g2 to (broken_insn - first_insn)>>2 */ From 7d4914db8fda6d38d92b1b8a740bafbd6c6d89a1 Mon Sep 17 00:00:00 2001 From: Max Filippov Date: Tue, 4 Apr 2017 13:26:29 -0700 Subject: [PATCH 74/96] xtensa: fix prefetch in the raw_copy_to_user 'from' is the input buffer, it should be prefetched with prefetch, not prefetchw. Tested-by: Max Filippov Signed-off-by: Max Filippov Signed-off-by: Al Viro --- arch/xtensa/include/asm/uaccess.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 8e93ed8ad1fe..2e7bac0d4b2c 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -245,7 +245,7 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n) static inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - prefetchw(from); + prefetch(from); return __xtensa_copy_user((__force void *)to, from, n); } #define INLINE_COPY_FROM_USER From 840db3f93849830c4a55c795ddd6057ad40d4f2d Mon Sep 17 00:00:00 2001 From: James Hogan Date: Tue, 4 Apr 2017 15:46:52 +0100 Subject: [PATCH 75/96] metag/usercopy: Switch to RAW_COPY_USER Switch to using raw user copy instead of providing metag specific [__]copy_{to,from}_user[_inatomic](). This simplifies the metag uaccess.h and allows us to take advantage of extra checking in the generic versions. Signed-off-by: James Hogan Cc: Al Viro Cc: linux-metag@vger.kernel.org Signed-off-by: Al Viro --- arch/metag/Kconfig | 1 + arch/metag/include/asm/uaccess.h | 31 ++----------------------------- arch/metag/lib/usercopy.c | 6 +++--- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/arch/metag/Kconfig b/arch/metag/Kconfig index 5b7a45d99cfb..ecce0c5ec8e8 100644 --- a/arch/metag/Kconfig +++ b/arch/metag/Kconfig @@ -1,5 +1,6 @@ config METAG def_bool y + select ARCH_HAS_RAW_COPY_USER select EMBEDDED select GENERIC_ATOMIC64 select GENERIC_CLOCKEVENTS diff --git a/arch/metag/include/asm/uaccess.h b/arch/metag/include/asm/uaccess.h index 61805d9f9375..5ebc2850690e 100644 --- a/arch/metag/include/asm/uaccess.h +++ b/arch/metag/include/asm/uaccess.h @@ -174,35 +174,8 @@ extern long __must_check strnlen_user(const char __user *src, long count); extern unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n); - -static inline unsigned long -copy_from_user(void *to, const void __user *from, unsigned long n) -{ - unsigned long res = n; - if (likely(access_ok(VERIFY_READ, from, n))) - res = raw_copy_from_user(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; -} - -#define __copy_from_user(to, from, n) raw_copy_from_user(to, from, n) -#define __copy_from_user_inatomic __copy_from_user - -extern unsigned long __must_check __copy_user(void __user *to, - const void *from, - unsigned long n); - -static inline unsigned long copy_to_user(void __user *to, const void *from, - unsigned long n) -{ - if (access_ok(VERIFY_WRITE, to, n)) - return __copy_user(to, from, n); - return n; -} - -#define __copy_to_user(to, from, n) __copy_user(to, from, n) -#define __copy_to_user_inatomic __copy_to_user +extern unsigned long raw_copy_to_user(void __user *to, const void *from, + unsigned long n); /* * Zero Userspace diff --git a/arch/metag/lib/usercopy.c b/arch/metag/lib/usercopy.c index 2792fc621088..e8a4ea83cabb 100644 --- a/arch/metag/lib/usercopy.c +++ b/arch/metag/lib/usercopy.c @@ -548,8 +548,8 @@ "SUB %1, %1, D0Ar2\n" \ "SUB %3, %3, D1Ar1\n") -unsigned long __copy_user(void __user *pdst, const void *psrc, - unsigned long n) +unsigned long raw_copy_to_user(void __user *pdst, const void *psrc, + unsigned long n) { register char __user *dst asm ("A0.2") = pdst; register const char *src asm ("A1.2") = psrc; @@ -654,7 +654,7 @@ unsigned long __copy_user(void __user *pdst, const void *psrc, */ return retn; } -EXPORT_SYMBOL(__copy_user); +EXPORT_SYMBOL(raw_copy_to_user); #define __asm_copy_from_user_1(to, from, ret) \ __asm_copy_user_cont(to, from, ret, \ From f0a955f4eeec0f16bdbdd0fb15d8ec0937d1de23 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 10:10:53 -0500 Subject: [PATCH 76/96] mips: sanitize __access_ok() for one thing, the last argument is always __access_mask and had been such since 2.4.0-test3pre8; for another, it can bloody well be a static inline - -O2 or -Os, __builtin_constant_p() propagates through static inline calls. Signed-off-by: Al Viro --- arch/mips/include/asm/uaccess.h | 21 ++++++--------------- arch/mips/kernel/mips-r2-to-r6-emul.c | 24 ++++++++++++------------ arch/mips/kernel/syscall.c | 2 +- arch/mips/oprofile/backtrace.c | 2 +- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index 70ca8eee166a..0c59911349ce 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -128,23 +128,14 @@ static inline bool eva_kernel_access(void) * this function, memory access functions may still return -EFAULT. */ -#define __access_mask get_fs().seg - -#define __access_ok(addr, size, mask) \ -({ \ - unsigned long __addr = (unsigned long) (addr); \ - unsigned long __size = size; \ - unsigned long __mask = mask; \ - unsigned long __ok; \ - \ - __chk_user_ptr(addr); \ - __ok = (signed long)(__mask & (__addr | (__addr + __size) | \ - __ua_size(__size))); \ - __ok == 0; \ -}) +static inline int __access_ok(const void __user *p, unsigned long size) +{ + unsigned long addr = (unsigned long)p; + return (get_fs().seg & (addr | (addr + size) | __ua_size(size))) == 0; +} #define access_ok(type, addr, size) \ - likely(__access_ok((addr), (size), __access_mask)) + likely(__access_ok((addr), (size))) /* * put_user: - Write a simple value into user space. diff --git a/arch/mips/kernel/mips-r2-to-r6-emul.c b/arch/mips/kernel/mips-r2-to-r6-emul.c index d8f1cf1ec370..550e7d03090a 100644 --- a/arch/mips/kernel/mips-r2-to-r6-emul.c +++ b/arch/mips/kernel/mips-r2-to-r6-emul.c @@ -1200,7 +1200,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) case lwl_op: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_READ, vaddr, 4)) { + if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1273,7 +1273,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) case lwr_op: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_READ, vaddr, 4)) { + if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1347,7 +1347,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) case swl_op: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_WRITE, vaddr, 4)) { + if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1417,7 +1417,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) case swr_op: rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_WRITE, vaddr, 4)) { + if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1492,7 +1492,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_READ, vaddr, 8)) { + if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1611,7 +1611,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_READ, vaddr, 8)) { + if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1730,7 +1730,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_WRITE, vaddr, 8)) { + if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1848,7 +1848,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) rt = regs->regs[MIPSInst_RT(inst)]; vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); - if (!access_ok(VERIFY_WRITE, vaddr, 8)) { + if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGSEGV; break; @@ -1965,7 +1965,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) err = SIGBUS; break; } - if (!access_ok(VERIFY_READ, vaddr, 4)) { + if (!access_ok(VERIFY_READ, (void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGBUS; break; @@ -2021,7 +2021,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) err = SIGBUS; break; } - if (!access_ok(VERIFY_WRITE, vaddr, 4)) { + if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 4)) { current->thread.cp0_baduaddr = vaddr; err = SIGBUS; break; @@ -2084,7 +2084,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) err = SIGBUS; break; } - if (!access_ok(VERIFY_READ, vaddr, 8)) { + if (!access_ok(VERIFY_READ, (void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGBUS; break; @@ -2145,7 +2145,7 @@ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) err = SIGBUS; break; } - if (!access_ok(VERIFY_WRITE, vaddr, 8)) { + if (!access_ok(VERIFY_WRITE, (void __user *)vaddr, 8)) { current->thread.cp0_baduaddr = vaddr; err = SIGBUS; break; diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c index f1d17ece4181..1dfa7f5796c7 100644 --- a/arch/mips/kernel/syscall.c +++ b/arch/mips/kernel/syscall.c @@ -98,7 +98,7 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new) if (unlikely(addr & 3)) return -EINVAL; - if (unlikely(!access_ok(VERIFY_WRITE, addr, 4))) + if (unlikely(!access_ok(VERIFY_WRITE, (const void __user *)addr, 4))) return -EINVAL; if (cpu_has_llsc && R10000_LLSC_WAR) { diff --git a/arch/mips/oprofile/backtrace.c b/arch/mips/oprofile/backtrace.c index 5e645c9a3162..16ace558cd9d 100644 --- a/arch/mips/oprofile/backtrace.c +++ b/arch/mips/oprofile/backtrace.c @@ -18,7 +18,7 @@ struct stackframe { static inline int get_mem(unsigned long addr, unsigned long *result) { unsigned long *address = (unsigned long *) addr; - if (!access_ok(VERIFY_READ, addr, sizeof(unsigned long))) + if (!access_ok(VERIFY_READ, address, sizeof(unsigned long))) return -1; if (__copy_from_user_inatomic(result, address, sizeof(unsigned long))) return -3; From c12a1d7ad916105fc5f7809f9273ea4fe7ce4b28 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 20 Mar 2017 11:40:15 -0400 Subject: [PATCH 77/96] mips: consolidate __invoke_... wrappers Signed-off-by: Al Viro --- arch/mips/include/asm/uaccess.h | 144 ++++++++++---------------------- 1 file changed, 44 insertions(+), 100 deletions(-) diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index 0c59911349ce..0cd0accdd0fa 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -797,8 +797,30 @@ extern void __put_user_unaligned_unknown(void); extern size_t __copy_user(void *__to, const void *__from, size_t __n); -#ifndef CONFIG_EVA -#define __invoke_copy_to_user(to, from, n) \ +#define __invoke_copy_from(func, to, from, n) \ +({ \ + register void *__cu_to_r __asm__("$4"); \ + register const void __user *__cu_from_r __asm__("$5"); \ + register long __cu_len_r __asm__("$6"); \ + \ + __cu_to_r = (to); \ + __cu_from_r = (from); \ + __cu_len_r = (n); \ + __asm__ __volatile__( \ + ".set\tnoreorder\n\t" \ + __MODULE_JAL(func) \ + ".set\tnoat\n\t" \ + __UA_ADDU "\t$1, %1, %2\n\t" \ + ".set\tat\n\t" \ + ".set\treorder" \ + : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ + : \ + : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ + DADDI_SCRATCH, "memory"); \ + __cu_len_r; \ +}) + +#define __invoke_copy_to(func, to, from, n) \ ({ \ register void __user *__cu_to_r __asm__("$4"); \ register const void *__cu_from_r __asm__("$5"); \ @@ -808,7 +830,7 @@ extern size_t __copy_user(void *__to, const void *__from, size_t __n); __cu_from_r = (from); \ __cu_len_r = (n); \ __asm__ __volatile__( \ - __MODULE_JAL(__copy_user) \ + __MODULE_JAL(func) \ : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ : \ : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ @@ -816,8 +838,12 @@ extern size_t __copy_user(void *__to, const void *__from, size_t __n); __cu_len_r; \ }) +#ifndef CONFIG_EVA +#define __invoke_copy_to_user(to, from, n) \ + __invoke_copy_to(__copy_user, to, from, n) + #define __invoke_copy_to_kernel(to, from, n) \ - __invoke_copy_to_user(to, from, n) + __invoke_copy_to(__copy_user, to, from, n) #endif @@ -948,64 +974,24 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n); #ifndef CONFIG_EVA #define __invoke_copy_from_user(to, from, n) \ -({ \ - register void *__cu_to_r __asm__("$4"); \ - register const void __user *__cu_from_r __asm__("$5"); \ - register long __cu_len_r __asm__("$6"); \ - \ - __cu_to_r = (to); \ - __cu_from_r = (from); \ - __cu_len_r = (n); \ - __asm__ __volatile__( \ - ".set\tnoreorder\n\t" \ - __MODULE_JAL(__copy_user) \ - ".set\tnoat\n\t" \ - __UA_ADDU "\t$1, %1, %2\n\t" \ - ".set\tat\n\t" \ - ".set\treorder" \ - : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ - : \ - : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ - DADDI_SCRATCH, "memory"); \ - __cu_len_r; \ -}) + __invoke_copy_from(__copy_user, to, from, n) #define __invoke_copy_from_kernel(to, from, n) \ - __invoke_copy_from_user(to, from, n) + __invoke_copy_from(__copy_user, to, from, n) /* For userland <-> userland operations */ #define ___invoke_copy_in_user(to, from, n) \ - __invoke_copy_from_user(to, from, n) + __invoke_copy_from(__copy_user, to, from, n) /* For kernel <-> kernel operations */ #define ___invoke_copy_in_kernel(to, from, n) \ - __invoke_copy_from_user(to, from, n) + __invoke_copy_from(__copy_user, to, from, n) #define __invoke_copy_from_user_inatomic(to, from, n) \ -({ \ - register void *__cu_to_r __asm__("$4"); \ - register const void __user *__cu_from_r __asm__("$5"); \ - register long __cu_len_r __asm__("$6"); \ - \ - __cu_to_r = (to); \ - __cu_from_r = (from); \ - __cu_len_r = (n); \ - __asm__ __volatile__( \ - ".set\tnoreorder\n\t" \ - __MODULE_JAL(__copy_user_inatomic) \ - ".set\tnoat\n\t" \ - __UA_ADDU "\t$1, %1, %2\n\t" \ - ".set\tat\n\t" \ - ".set\treorder" \ - : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ - : \ - : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ - DADDI_SCRATCH, "memory"); \ - __cu_len_r; \ -}) + __invoke_copy_from(__copy_user_inatomic, to, from, n) #define __invoke_copy_from_kernel_inatomic(to, from, n) \ - __invoke_copy_from_user_inatomic(to, from, n) \ + __invoke_copy_from(__copy_user_inatomic, to, from, n) #else @@ -1019,79 +1005,37 @@ extern size_t __copy_to_user_eva(void *__to, const void *__from, size_t __n); extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); -#define __invoke_copy_from_user_eva_generic(to, from, n, func_ptr) \ -({ \ - register void *__cu_to_r __asm__("$4"); \ - register const void __user *__cu_from_r __asm__("$5"); \ - register long __cu_len_r __asm__("$6"); \ - \ - __cu_to_r = (to); \ - __cu_from_r = (from); \ - __cu_len_r = (n); \ - __asm__ __volatile__( \ - ".set\tnoreorder\n\t" \ - __MODULE_JAL(func_ptr) \ - ".set\tnoat\n\t" \ - __UA_ADDU "\t$1, %1, %2\n\t" \ - ".set\tat\n\t" \ - ".set\treorder" \ - : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ - : \ - : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ - DADDI_SCRATCH, "memory"); \ - __cu_len_r; \ -}) - -#define __invoke_copy_to_user_eva_generic(to, from, n, func_ptr) \ -({ \ - register void *__cu_to_r __asm__("$4"); \ - register const void __user *__cu_from_r __asm__("$5"); \ - register long __cu_len_r __asm__("$6"); \ - \ - __cu_to_r = (to); \ - __cu_from_r = (from); \ - __cu_len_r = (n); \ - __asm__ __volatile__( \ - __MODULE_JAL(func_ptr) \ - : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \ - : \ - : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31", \ - DADDI_SCRATCH, "memory"); \ - __cu_len_r; \ -}) - /* * Source or destination address is in userland. We need to go through * the TLB */ #define __invoke_copy_from_user(to, from, n) \ - __invoke_copy_from_user_eva_generic(to, from, n, __copy_from_user_eva) + __invoke_copy_from(__copy_from_user_eva, to, from, n) #define __invoke_copy_from_user_inatomic(to, from, n) \ - __invoke_copy_from_user_eva_generic(to, from, n, \ - __copy_user_inatomic_eva) + __invoke_copy_from(__copy_user_inatomic_eva, to, from, n) #define __invoke_copy_to_user(to, from, n) \ - __invoke_copy_to_user_eva_generic(to, from, n, __copy_to_user_eva) + __invoke_copy_to(__copy_to_user_eva, to, from, n) #define ___invoke_copy_in_user(to, from, n) \ - __invoke_copy_from_user_eva_generic(to, from, n, __copy_in_user_eva) + __invoke_copy_from(__copy_in_user_eva, to, from, n) /* * Source or destination address in the kernel. We are not going through * the TLB */ #define __invoke_copy_from_kernel(to, from, n) \ - __invoke_copy_from_user_eva_generic(to, from, n, __copy_user) + __invoke_copy_from(__copy_user, to, from, n) #define __invoke_copy_from_kernel_inatomic(to, from, n) \ - __invoke_copy_from_user_eva_generic(to, from, n, __copy_user_inatomic) + __invoke_copy_from(__copy_user_inatomic, to, from, n) #define __invoke_copy_to_kernel(to, from, n) \ - __invoke_copy_to_user_eva_generic(to, from, n, __copy_user) + __invoke_copy_to(__copy_user, to, from, n) #define ___invoke_copy_in_kernel(to, from, n) \ - __invoke_copy_from_user_eva_generic(to, from, n, __copy_user) + __invoke_copy_from(__copy_user, to, from, n) #endif /* CONFIG_EVA */ From b0bb945c14e81b3b33075d59469d5bf5b736a4fb Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 20 Mar 2017 11:51:34 -0400 Subject: [PATCH 78/96] mips: clean and reorder the forest of macros... Signed-off-by: Al Viro --- arch/mips/include/asm/uaccess.h | 122 +++++++++++++------------------- 1 file changed, 50 insertions(+), 72 deletions(-) diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index 0cd0accdd0fa..9c05262a85f3 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -838,14 +838,60 @@ extern size_t __copy_user(void *__to, const void *__from, size_t __n); __cu_len_r; \ }) -#ifndef CONFIG_EVA -#define __invoke_copy_to_user(to, from, n) \ - __invoke_copy_to(__copy_user, to, from, n) +#define __invoke_copy_from_kernel(to, from, n) \ + __invoke_copy_from(__copy_user, to, from, n) + +#define __invoke_copy_from_kernel_inatomic(to, from, n) \ + __invoke_copy_from(__copy_user_inatomic, to, from, n) #define __invoke_copy_to_kernel(to, from, n) \ __invoke_copy_to(__copy_user, to, from, n) -#endif +#define ___invoke_copy_in_kernel(to, from, n) \ + __invoke_copy_from(__copy_user, to, from, n) + +#ifndef CONFIG_EVA +#define __invoke_copy_from_user(to, from, n) \ + __invoke_copy_from(__copy_user, to, from, n) + +#define __invoke_copy_from_user_inatomic(to, from, n) \ + __invoke_copy_from(__copy_user_inatomic, to, from, n) + +#define __invoke_copy_to_user(to, from, n) \ + __invoke_copy_to(__copy_user, to, from, n) + +#define ___invoke_copy_in_user(to, from, n) \ + __invoke_copy_from(__copy_user, to, from, n) + +#else + +/* EVA specific functions */ + +extern size_t __copy_user_inatomic_eva(void *__to, const void *__from, + size_t __n); +extern size_t __copy_from_user_eva(void *__to, const void *__from, + size_t __n); +extern size_t __copy_to_user_eva(void *__to, const void *__from, + size_t __n); +extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); + +/* + * Source or destination address is in userland. We need to go through + * the TLB + */ +#define __invoke_copy_from_user(to, from, n) \ + __invoke_copy_from(__copy_from_user_eva, to, from, n) + +#define __invoke_copy_from_user_inatomic(to, from, n) \ + __invoke_copy_from(__copy_user_inatomic_eva, to, from, n) + +#define __invoke_copy_to_user(to, from, n) \ + __invoke_copy_to(__copy_to_user_eva, to, from, n) + +#define ___invoke_copy_in_user(to, from, n) \ + __invoke_copy_from(__copy_in_user_eva, to, from, n) + +#endif /* CONFIG_EVA */ /* * __copy_to_user: - Copy a block of data into user space, with less checking. @@ -971,74 +1017,6 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n); __cu_len; \ }) -#ifndef CONFIG_EVA - -#define __invoke_copy_from_user(to, from, n) \ - __invoke_copy_from(__copy_user, to, from, n) - -#define __invoke_copy_from_kernel(to, from, n) \ - __invoke_copy_from(__copy_user, to, from, n) - -/* For userland <-> userland operations */ -#define ___invoke_copy_in_user(to, from, n) \ - __invoke_copy_from(__copy_user, to, from, n) - -/* For kernel <-> kernel operations */ -#define ___invoke_copy_in_kernel(to, from, n) \ - __invoke_copy_from(__copy_user, to, from, n) - -#define __invoke_copy_from_user_inatomic(to, from, n) \ - __invoke_copy_from(__copy_user_inatomic, to, from, n) - -#define __invoke_copy_from_kernel_inatomic(to, from, n) \ - __invoke_copy_from(__copy_user_inatomic, to, from, n) - -#else - -/* EVA specific functions */ - -extern size_t __copy_user_inatomic_eva(void *__to, const void *__from, - size_t __n); -extern size_t __copy_from_user_eva(void *__to, const void *__from, - size_t __n); -extern size_t __copy_to_user_eva(void *__to, const void *__from, - size_t __n); -extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); - -/* - * Source or destination address is in userland. We need to go through - * the TLB - */ -#define __invoke_copy_from_user(to, from, n) \ - __invoke_copy_from(__copy_from_user_eva, to, from, n) - -#define __invoke_copy_from_user_inatomic(to, from, n) \ - __invoke_copy_from(__copy_user_inatomic_eva, to, from, n) - -#define __invoke_copy_to_user(to, from, n) \ - __invoke_copy_to(__copy_to_user_eva, to, from, n) - -#define ___invoke_copy_in_user(to, from, n) \ - __invoke_copy_from(__copy_in_user_eva, to, from, n) - -/* - * Source or destination address in the kernel. We are not going through - * the TLB - */ -#define __invoke_copy_from_kernel(to, from, n) \ - __invoke_copy_from(__copy_user, to, from, n) - -#define __invoke_copy_from_kernel_inatomic(to, from, n) \ - __invoke_copy_from(__copy_user_inatomic, to, from, n) - -#define __invoke_copy_to_kernel(to, from, n) \ - __invoke_copy_to(__copy_user, to, from, n) - -#define ___invoke_copy_in_kernel(to, from, n) \ - __invoke_copy_from(__copy_user, to, from, n) - -#endif /* CONFIG_EVA */ - /* * __copy_from_user: - Copy a block of data from user space, with less checking. * @to: Destination address, in kernel space. From ab0aca27c4dfcfa72eb1339ae94edbfbdd751ae8 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 20 Mar 2017 11:54:21 -0400 Subject: [PATCH 79/96] mips: make copy_from_user() zero tail explicitly Signed-off-by: Al Viro --- arch/mips/include/asm/uaccess.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index 9c05262a85f3..712dc40625b6 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -1080,29 +1080,29 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n); ({ \ void *__cu_to; \ const void __user *__cu_from; \ - long __cu_len; \ + long __cu_len, __cu_res; \ \ __cu_to = (to); \ __cu_from = (from); \ - __cu_len = (n); \ + __cu_res = __cu_len = (n); \ \ check_object_size(__cu_to, __cu_len, false); \ \ if (eva_kernel_access()) { \ - __cu_len = __invoke_copy_from_kernel(__cu_to, \ + __cu_res = __invoke_copy_from_kernel(__cu_to, \ __cu_from, \ __cu_len); \ } else { \ if (access_ok(VERIFY_READ, __cu_from, __cu_len)) { \ might_fault(); \ - __cu_len = __invoke_copy_from_user(__cu_to, \ + __cu_res = __invoke_copy_from_user(__cu_to, \ __cu_from, \ __cu_len); \ - } else { \ - memset(__cu_to, 0, __cu_len); \ } \ } \ - __cu_len; \ + if (unlikely(__cu_res)) \ + memset(__cu_to + __cu_len - __cu_res, 0, __cu_res); \ + __cu_res; \ }) #define __copy_in_user(to, from, n) \ From 1a4fded6d32596053d4f0bc8c49028faf4114a17 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 20 Mar 2017 12:03:16 -0400 Subject: [PATCH 80/96] mips: get rid of tail-zeroing in primitives Signed-off-by: Al Viro --- arch/mips/cavium-octeon/octeon-memcpy.S | 31 +--------------- arch/mips/include/asm/uaccess.h | 19 +--------- arch/mips/lib/memcpy.S | 49 ------------------------- 3 files changed, 3 insertions(+), 96 deletions(-) diff --git a/arch/mips/cavium-octeon/octeon-memcpy.S b/arch/mips/cavium-octeon/octeon-memcpy.S index cfd97f6448bb..0a7c9834b81c 100644 --- a/arch/mips/cavium-octeon/octeon-memcpy.S +++ b/arch/mips/cavium-octeon/octeon-memcpy.S @@ -139,15 +139,6 @@ .set noreorder .set noat -/* - * t7 is used as a flag to note inatomic mode. - */ -LEAF(__copy_user_inatomic) -EXPORT_SYMBOL(__copy_user_inatomic) - b __copy_user_common - li t7, 1 - END(__copy_user_inatomic) - /* * A combined memcpy/__copy_user * __copy_user sets len to 0 for success; else to an upper bound of @@ -161,8 +152,6 @@ EXPORT_SYMBOL(memcpy) __memcpy: FEXPORT(__copy_user) EXPORT_SYMBOL(__copy_user) - li t7, 0 /* not inatomic */ -__copy_user_common: /* * Note: dst & src may be unaligned, len may be 0 * Temps @@ -414,25 +403,7 @@ l_exc: LOAD t0, TI_TASK($28) LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address SUB len, AT, t0 # len number of uncopied bytes - bnez t7, 2f /* Skip the zeroing out part if inatomic */ - /* - * Here's where we rely on src and dst being incremented in tandem, - * See (3) above. - * dst += (fault addr - src) to put dst at first byte to clear - */ - ADD dst, t0 # compute start address in a1 - SUB dst, src - /* - * Clear len bytes starting at dst. Can't call __bzero because it - * might modify len. An inefficient loop for these rare times... - */ - beqz len, done - SUB src, len, 1 -1: sb zero, 0(dst) - ADD dst, dst, 1 - bnez src, 1b - SUB src, src, 1 -2: jr ra + jr ra nop diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index 712dc40625b6..988cd3b64a59 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -841,9 +841,6 @@ extern size_t __copy_user(void *__to, const void *__from, size_t __n); #define __invoke_copy_from_kernel(to, from, n) \ __invoke_copy_from(__copy_user, to, from, n) -#define __invoke_copy_from_kernel_inatomic(to, from, n) \ - __invoke_copy_from(__copy_user_inatomic, to, from, n) - #define __invoke_copy_to_kernel(to, from, n) \ __invoke_copy_to(__copy_user, to, from, n) @@ -854,9 +851,6 @@ extern size_t __copy_user(void *__to, const void *__from, size_t __n); #define __invoke_copy_from_user(to, from, n) \ __invoke_copy_from(__copy_user, to, from, n) -#define __invoke_copy_from_user_inatomic(to, from, n) \ - __invoke_copy_from(__copy_user_inatomic, to, from, n) - #define __invoke_copy_to_user(to, from, n) \ __invoke_copy_to(__copy_user, to, from, n) @@ -867,8 +861,6 @@ extern size_t __copy_user(void *__to, const void *__from, size_t __n); /* EVA specific functions */ -extern size_t __copy_user_inatomic_eva(void *__to, const void *__from, - size_t __n); extern size_t __copy_from_user_eva(void *__to, const void *__from, size_t __n); extern size_t __copy_to_user_eva(void *__to, const void *__from, @@ -882,9 +874,6 @@ extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); #define __invoke_copy_from_user(to, from, n) \ __invoke_copy_from(__copy_from_user_eva, to, from, n) -#define __invoke_copy_from_user_inatomic(to, from, n) \ - __invoke_copy_from(__copy_user_inatomic_eva, to, from, n) - #define __invoke_copy_to_user(to, from, n) \ __invoke_copy_to(__copy_to_user_eva, to, from, n) @@ -930,8 +919,6 @@ extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); __cu_len; \ }) -extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n); - #define __copy_to_user_inatomic(to, from, n) \ ({ \ void __user *__cu_to; \ @@ -966,12 +953,10 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n); check_object_size(__cu_to, __cu_len, false); \ \ if (eva_kernel_access()) \ - __cu_len = __invoke_copy_from_kernel_inatomic(__cu_to, \ - __cu_from,\ + __cu_len = __invoke_copy_from_kernel(__cu_to, __cu_from,\ __cu_len);\ else \ - __cu_len = __invoke_copy_from_user_inatomic(__cu_to, \ - __cu_from, \ + __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ __cu_len); \ __cu_len; \ }) diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S index c3031f18c572..3114a2ed1f4e 100644 --- a/arch/mips/lib/memcpy.S +++ b/arch/mips/lib/memcpy.S @@ -562,39 +562,9 @@ LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address nop SUB len, AT, t0 # len number of uncopied bytes - bnez t6, .Ldone\@ /* Skip the zeroing part if inatomic */ - /* - * Here's where we rely on src and dst being incremented in tandem, - * See (3) above. - * dst += (fault addr - src) to put dst at first byte to clear - */ - ADD dst, t0 # compute start address in a1 - SUB dst, src - /* - * Clear len bytes starting at dst. Can't call __bzero because it - * might modify len. An inefficient loop for these rare times... - */ - .set reorder /* DADDI_WAR */ - SUB src, len, 1 - beqz len, .Ldone\@ - .set noreorder -1: sb zero, 0(dst) - ADD dst, dst, 1 -#ifndef CONFIG_CPU_DADDI_WORKAROUNDS - bnez src, 1b - SUB src, src, 1 -#else - .set push - .set noat - li v1, 1 - bnez src, 1b - SUB src, src, v1 - .set pop -#endif jr ra nop - #define SEXC(n) \ .set reorder; /* DADDI_WAR */ \ .Ls_exc_p ## n ## u\@: \ @@ -672,15 +642,6 @@ LEAF(__rmemcpy) /* a0=dst a1=src a2=len */ move a2, zero END(__rmemcpy) -/* - * t6 is used as a flag to note inatomic mode. - */ -LEAF(__copy_user_inatomic) -EXPORT_SYMBOL(__copy_user_inatomic) - b __copy_user_common - li t6, 1 - END(__copy_user_inatomic) - /* * A combined memcpy/__copy_user * __copy_user sets len to 0 for success; else to an upper bound of @@ -694,8 +655,6 @@ EXPORT_SYMBOL(memcpy) .L__memcpy: FEXPORT(__copy_user) EXPORT_SYMBOL(__copy_user) - li t6, 0 /* not inatomic */ -__copy_user_common: /* Legacy Mode, user <-> user */ __BUILD_COPY_USER LEGACY_MODE USEROP USEROP @@ -708,20 +667,12 @@ __copy_user_common: * space */ -LEAF(__copy_user_inatomic_eva) -EXPORT_SYMBOL(__copy_user_inatomic_eva) - b __copy_from_user_common - li t6, 1 - END(__copy_user_inatomic_eva) - /* * __copy_from_user (EVA) */ LEAF(__copy_from_user_eva) EXPORT_SYMBOL(__copy_from_user_eva) - li t6, 0 /* not inatomic */ -__copy_from_user_common: __BUILD_COPY_USER EVA_MODE USEROP KERNELOP END(__copy_from_user_eva) From 2260ea86c0e7623d4e99886638b2b83ef21efa60 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 15:59:26 -0400 Subject: [PATCH 81/96] mips: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/mips/Kconfig | 1 + arch/mips/include/asm/uaccess.h | 274 +++----------------------------- 2 files changed, 27 insertions(+), 248 deletions(-) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index a008a9f03072..aff5633bfe4d 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -69,6 +69,7 @@ config MIPS select HAVE_EXIT_THREAD select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_ARCH_HARDENED_USERCOPY + select ARCH_HAS_RAW_COPY_USER menu "Machine selection" diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index 988cd3b64a59..99e629a590a5 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -882,257 +882,35 @@ extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n); #endif /* CONFIG_EVA */ -/* - * __copy_to_user: - Copy a block of data into user space, with less checking. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -#define __copy_to_user(to, from, n) \ -({ \ - void __user *__cu_to; \ - const void *__cu_from; \ - long __cu_len; \ - \ - __cu_to = (to); \ - __cu_from = (from); \ - __cu_len = (n); \ - \ - check_object_size(__cu_from, __cu_len, true); \ - might_fault(); \ - \ - if (eva_kernel_access()) \ - __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \ - __cu_len); \ - else \ - __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \ - __cu_len); \ - __cu_len; \ -}) +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + if (eva_kernel_access()) + return __invoke_copy_to_kernel(to, from, n); + else + return __invoke_copy_to_user(to, from, n); +} -#define __copy_to_user_inatomic(to, from, n) \ -({ \ - void __user *__cu_to; \ - const void *__cu_from; \ - long __cu_len; \ - \ - __cu_to = (to); \ - __cu_from = (from); \ - __cu_len = (n); \ - \ - check_object_size(__cu_from, __cu_len, true); \ - \ - if (eva_kernel_access()) \ - __cu_len = __invoke_copy_to_kernel(__cu_to, __cu_from, \ - __cu_len); \ - else \ - __cu_len = __invoke_copy_to_user(__cu_to, __cu_from, \ - __cu_len); \ - __cu_len; \ -}) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + if (eva_kernel_access()) + return __invoke_copy_from_kernel(to, from, n); + else + return __invoke_copy_from_user(to, from, n); +} -#define __copy_from_user_inatomic(to, from, n) \ -({ \ - void *__cu_to; \ - const void __user *__cu_from; \ - long __cu_len; \ - \ - __cu_to = (to); \ - __cu_from = (from); \ - __cu_len = (n); \ - \ - check_object_size(__cu_to, __cu_len, false); \ - \ - if (eva_kernel_access()) \ - __cu_len = __invoke_copy_from_kernel(__cu_to, __cu_from,\ - __cu_len);\ - else \ - __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ - __cu_len); \ - __cu_len; \ -}) +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER -/* - * copy_to_user: - Copy a block of data into user space. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -#define copy_to_user(to, from, n) \ -({ \ - void __user *__cu_to; \ - const void *__cu_from; \ - long __cu_len; \ - \ - __cu_to = (to); \ - __cu_from = (from); \ - __cu_len = (n); \ - \ - check_object_size(__cu_from, __cu_len, true); \ - \ - if (eva_kernel_access()) { \ - __cu_len = __invoke_copy_to_kernel(__cu_to, \ - __cu_from, \ - __cu_len); \ - } else { \ - if (access_ok(VERIFY_WRITE, __cu_to, __cu_len)) { \ - might_fault(); \ - __cu_len = __invoke_copy_to_user(__cu_to, \ - __cu_from, \ - __cu_len); \ - } \ - } \ - __cu_len; \ -}) - -/* - * __copy_from_user: - Copy a block of data from user space, with less checking. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - */ -#define __copy_from_user(to, from, n) \ -({ \ - void *__cu_to; \ - const void __user *__cu_from; \ - long __cu_len; \ - \ - __cu_to = (to); \ - __cu_from = (from); \ - __cu_len = (n); \ - \ - check_object_size(__cu_to, __cu_len, false); \ - \ - if (eva_kernel_access()) { \ - __cu_len = __invoke_copy_from_kernel(__cu_to, \ - __cu_from, \ - __cu_len); \ - } else { \ - might_fault(); \ - __cu_len = __invoke_copy_from_user(__cu_to, __cu_from, \ - __cu_len); \ - } \ - __cu_len; \ -}) - -/* - * copy_from_user: - Copy a block of data from user space. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - */ -#define copy_from_user(to, from, n) \ -({ \ - void *__cu_to; \ - const void __user *__cu_from; \ - long __cu_len, __cu_res; \ - \ - __cu_to = (to); \ - __cu_from = (from); \ - __cu_res = __cu_len = (n); \ - \ - check_object_size(__cu_to, __cu_len, false); \ - \ - if (eva_kernel_access()) { \ - __cu_res = __invoke_copy_from_kernel(__cu_to, \ - __cu_from, \ - __cu_len); \ - } else { \ - if (access_ok(VERIFY_READ, __cu_from, __cu_len)) { \ - might_fault(); \ - __cu_res = __invoke_copy_from_user(__cu_to, \ - __cu_from, \ - __cu_len); \ - } \ - } \ - if (unlikely(__cu_res)) \ - memset(__cu_to + __cu_len - __cu_res, 0, __cu_res); \ - __cu_res; \ -}) - -#define __copy_in_user(to, from, n) \ -({ \ - void __user *__cu_to; \ - const void __user *__cu_from; \ - long __cu_len; \ - \ - __cu_to = (to); \ - __cu_from = (from); \ - __cu_len = (n); \ - if (eva_kernel_access()) { \ - __cu_len = ___invoke_copy_in_kernel(__cu_to, __cu_from, \ - __cu_len); \ - } else { \ - might_fault(); \ - __cu_len = ___invoke_copy_in_user(__cu_to, __cu_from, \ - __cu_len); \ - } \ - __cu_len; \ -}) - -#define copy_in_user(to, from, n) \ -({ \ - void __user *__cu_to; \ - const void __user *__cu_from; \ - long __cu_len; \ - \ - __cu_to = (to); \ - __cu_from = (from); \ - __cu_len = (n); \ - if (eva_kernel_access()) { \ - __cu_len = ___invoke_copy_in_kernel(__cu_to,__cu_from, \ - __cu_len); \ - } else { \ - if (likely(access_ok(VERIFY_READ, __cu_from, __cu_len) &&\ - access_ok(VERIFY_WRITE, __cu_to, __cu_len))) {\ - might_fault(); \ - __cu_len = ___invoke_copy_in_user(__cu_to, \ - __cu_from, \ - __cu_len); \ - } \ - } \ - __cu_len; \ -}) +static inline unsigned long +raw_copy_in_user(void __user*to, const void __user *from, unsigned long n) +{ + if (eva_kernel_access()) + return ___invoke_copy_in_kernel(to, from, n); + else + return ___invoke_copy_in_user(to, from, n); +} extern __kernel_size_t __bzero_kernel(void __user *addr, __kernel_size_t size); extern __kernel_size_t __bzero(void __user *addr, __kernel_size_t size); From e73a67f7cd471a14bb1351954609e0b0ef5d2187 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 18 Mar 2017 21:20:27 -0400 Subject: [PATCH 82/96] don't open-code kernel_setsockopt() Signed-off-by: Al Viro --- fs/ocfs2/cluster/tcp.c | 25 ++++--------------------- net/rds/tcp.c | 5 +---- net/rds/tcp_send.c | 8 +------- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/fs/ocfs2/cluster/tcp.c b/fs/ocfs2/cluster/tcp.c index 4348027384f5..58fb1bc2bb32 100644 --- a/fs/ocfs2/cluster/tcp.c +++ b/fs/ocfs2/cluster/tcp.c @@ -1460,27 +1460,10 @@ static void o2net_rx_until_empty(struct work_struct *work) static int o2net_set_nodelay(struct socket *sock) { - int ret, val = 1; - mm_segment_t oldfs; + int val = 1; - oldfs = get_fs(); - set_fs(KERNEL_DS); - - /* - * Dear unsuspecting programmer, - * - * Don't use sock_setsockopt() for SOL_TCP. It doesn't check its level - * argument and assumes SOL_SOCKET so, say, your TCP_NODELAY will - * silently turn into SO_DEBUG. - * - * Yours, - * Keeper of hilariously fragile interfaces. - */ - ret = sock->ops->setsockopt(sock, SOL_TCP, TCP_NODELAY, - (char __user *)&val, sizeof(val)); - - set_fs(oldfs); - return ret; + return kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, + (void *)&val, sizeof(val)); } static int o2net_set_usertimeout(struct socket *sock) @@ -1488,7 +1471,7 @@ static int o2net_set_usertimeout(struct socket *sock) int user_timeout = O2NET_TCP_USER_TIMEOUT; return kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, - (char *)&user_timeout, sizeof(user_timeout)); + (void *)&user_timeout, sizeof(user_timeout)); } static void o2net_initialize_handshake(void) diff --git a/net/rds/tcp.c b/net/rds/tcp.c index a973d3b4dff0..731282fbaf58 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -84,13 +84,10 @@ static struct ctl_table rds_tcp_sysctl_table[] = { /* doing it this way avoids calling tcp_sk() */ void rds_tcp_nonagle(struct socket *sock) { - mm_segment_t oldfs = get_fs(); int val = 1; - set_fs(KERNEL_DS); - sock->ops->setsockopt(sock, SOL_TCP, TCP_NODELAY, (char __user *)&val, + kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&val, sizeof(val)); - set_fs(oldfs); } u32 rds_tcp_snd_nxt(struct rds_tcp_connection *tc) diff --git a/net/rds/tcp_send.c b/net/rds/tcp_send.c index dcf4742083ea..52d11d7725c8 100644 --- a/net/rds/tcp_send.c +++ b/net/rds/tcp_send.c @@ -40,13 +40,7 @@ static void rds_tcp_cork(struct socket *sock, int val) { - mm_segment_t oldfs; - - oldfs = get_fs(); - set_fs(KERNEL_DS); - sock->ops->setsockopt(sock, SOL_TCP, TCP_CORK, (char __user *)&val, - sizeof(val)); - set_fs(oldfs); + kernel_setsockopt(sock, SOL_TCP, TCP_CORK, (void *)&val, sizeof(val)); } void rds_tcp_xmit_path_prepare(struct rds_conn_path *cp) From 1c87ea4566612ef37e31f6389cbb232bd49cdd45 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 25 Mar 2017 00:43:43 -0400 Subject: [PATCH 83/96] alpha: fix stack smashing in old_adjtimex(2) Signed-off-by: Al Viro --- arch/alpha/kernel/osf_sys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index 0b961093ca5c..6d76e528ab8f 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -1290,7 +1290,7 @@ SYSCALL_DEFINE1(old_adjtimex, struct timex32 __user *, txc_p) /* copy relevant bits of struct timex. */ if (copy_from_user(&txc, txc_p, offsetof(struct timex32, time)) || copy_from_user(&txc.tick, &txc_p->tick, sizeof(struct timex32) - - offsetof(struct timex32, time))) + offsetof(struct timex32, tick))) return -EFAULT; ret = do_adjtimex(&txc); From bf7af0cea8a7980269ee1762f336cccb7b7e813e Mon Sep 17 00:00:00 2001 From: Al Viro Date: Thu, 30 Mar 2017 12:27:21 -0400 Subject: [PATCH 84/96] esas2r: don't open-code memdup_user() Signed-off-by: Al Viro --- drivers/scsi/esas2r/esas2r_ioctl.c | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/drivers/scsi/esas2r/esas2r_ioctl.c b/drivers/scsi/esas2r/esas2r_ioctl.c index b35ed3829421..2d4b7f049a68 100644 --- a/drivers/scsi/esas2r/esas2r_ioctl.c +++ b/drivers/scsi/esas2r/esas2r_ioctl.c @@ -1289,32 +1289,13 @@ int esas2r_ioctl_handler(void *hostdata, int cmd, void __user *arg) || (cmd > EXPRESS_IOCTL_MAX)) return -ENOTSUPP; - if (!access_ok(VERIFY_WRITE, arg, sizeof(struct atto_express_ioctl))) { + ioctl = memdup_user(arg, sizeof(struct atto_express_ioctl)); + if (IS_ERR(ioctl)) { esas2r_log(ESAS2R_LOG_WARN, "ioctl_handler access_ok failed for cmd %d, " "address %p", cmd, arg); - return -EFAULT; - } - - /* allocate a kernel memory buffer for the IOCTL data */ - ioctl = kzalloc(sizeof(struct atto_express_ioctl), GFP_KERNEL); - if (ioctl == NULL) { - esas2r_log(ESAS2R_LOG_WARN, - "ioctl_handler kzalloc failed for %zu bytes", - sizeof(struct atto_express_ioctl)); - return -ENOMEM; - } - - err = __copy_from_user(ioctl, arg, sizeof(struct atto_express_ioctl)); - if (err != 0) { - esas2r_log(ESAS2R_LOG_WARN, - "copy_from_user didn't copy everything (err %d, cmd %d)", - err, - cmd); - kfree(ioctl); - - return -EFAULT; + return PTR_ERR(ioctl); } /* verify the signature */ From 3448890c32c32c482c3ec20baa8fdd2ab4f94cc0 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 16:35:08 -0400 Subject: [PATCH 85/96] powerpc: get rid of zeroing, switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/powerpc/Kconfig | 1 + arch/powerpc/include/asm/uaccess.h | 59 +++++------------------------- arch/powerpc/lib/Makefile | 2 +- arch/powerpc/lib/copy_32.S | 14 ------- arch/powerpc/lib/copyuser_64.S | 35 ++---------------- arch/powerpc/lib/usercopy_64.c | 41 --------------------- 6 files changed, 14 insertions(+), 138 deletions(-) delete mode 100644 arch/powerpc/lib/usercopy_64.c diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 97a8bc8a095c..839f08887269 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -87,6 +87,7 @@ config PPC select ARCH_HAS_DMA_SET_COHERENT_MASK select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_HAS_RAW_COPY_USER select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE select ARCH_HAS_SG_CHAIN select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index b19883f6273c..5c0d8a8cdae5 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -269,42 +269,19 @@ extern unsigned long __copy_tofrom_user(void __user *to, #ifndef __powerpc64__ -static inline unsigned long copy_from_user(void *to, - const void __user *from, unsigned long n) -{ - if (likely(access_ok(VERIFY_READ, from, n))) { - check_object_size(to, n, false); - return __copy_tofrom_user((__force void __user *)to, from, n); - } - memset(to, 0, n); - return n; -} - -static inline unsigned long copy_to_user(void __user *to, - const void *from, unsigned long n) -{ - if (access_ok(VERIFY_WRITE, to, n)) { - check_object_size(from, n, true); - return __copy_tofrom_user(to, (__force void __user *)from, n); - } - return n; -} +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER #else /* __powerpc64__ */ -#define __copy_in_user(to, from, size) \ - __copy_tofrom_user((to), (from), (size)) - -extern unsigned long copy_from_user(void *to, const void __user *from, - unsigned long n); -extern unsigned long copy_to_user(void __user *to, const void *from, - unsigned long n); -extern unsigned long copy_in_user(void __user *to, const void __user *from, - unsigned long n); - +static inline unsigned long +raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) +{ + return __copy_tofrom_user(to, from, n); +} #endif /* __powerpc64__ */ -static inline unsigned long __copy_from_user_inatomic(void *to, +static inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { if (__builtin_constant_p(n) && (n <= 8)) { @@ -328,12 +305,10 @@ static inline unsigned long __copy_from_user_inatomic(void *to, return 0; } - check_object_size(to, n, false); - return __copy_tofrom_user((__force void __user *)to, from, n); } -static inline unsigned long __copy_to_user_inatomic(void __user *to, +static inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { if (__builtin_constant_p(n) && (n <= 8)) { @@ -357,25 +332,9 @@ static inline unsigned long __copy_to_user_inatomic(void __user *to, return 0; } - check_object_size(from, n, true); - return __copy_tofrom_user(to, (__force const void __user *)from, n); } -static inline unsigned long __copy_from_user(void *to, - const void __user *from, unsigned long size) -{ - might_fault(); - return __copy_from_user_inatomic(to, from, size); -} - -static inline unsigned long __copy_to_user(void __user *to, - const void *from, unsigned long size) -{ - might_fault(); - return __copy_to_user_inatomic(to, from, size); -} - extern unsigned long __clear_user(void __user *addr, unsigned long size); static inline unsigned long clear_user(void __user *addr, unsigned long size) diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index 2b5e09020cfe..ed7dfce331e0 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -14,7 +14,7 @@ obj-y += string.o alloc.o crtsavres.o code-patching.o \ obj-$(CONFIG_PPC32) += div64.o copy_32.o -obj64-y += copypage_64.o copyuser_64.o usercopy_64.o mem_64.o hweight_64.o \ +obj64-y += copypage_64.o copyuser_64.o mem_64.o hweight_64.o \ copyuser_power7.o string_64.o copypage_power7.o memcpy_power7.o \ memcpy_64.o memcmp_64.o diff --git a/arch/powerpc/lib/copy_32.S b/arch/powerpc/lib/copy_32.S index ff0d894d7ff9..8aedbb5f4b86 100644 --- a/arch/powerpc/lib/copy_32.S +++ b/arch/powerpc/lib/copy_32.S @@ -477,18 +477,6 @@ _GLOBAL(__copy_tofrom_user) bdnz 130b /* then clear out the destination: r3 bytes starting at 4(r6) */ 132: mfctr r3 - srwi. r0,r3,2 - li r9,0 - mtctr r0 - beq 113f -112: stwu r9,4(r6) - bdnz 112b -113: andi. r0,r3,3 - mtctr r0 - beq 120f -114: stb r9,4(r6) - addi r6,r6,1 - bdnz 114b 120: blr EX_TABLE(30b,108b) @@ -497,7 +485,5 @@ _GLOBAL(__copy_tofrom_user) EX_TABLE(41b,111b) EX_TABLE(130b,132b) EX_TABLE(131b,120b) - EX_TABLE(112b,120b) - EX_TABLE(114b,120b) EXPORT_SYMBOL(__copy_tofrom_user) diff --git a/arch/powerpc/lib/copyuser_64.S b/arch/powerpc/lib/copyuser_64.S index aee6e24e81ab..08da06e1bd72 100644 --- a/arch/powerpc/lib/copyuser_64.S +++ b/arch/powerpc/lib/copyuser_64.S @@ -319,32 +319,9 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) blr /* - * here we have trapped again, need to clear ctr bytes starting at r3 + * here we have trapped again, amount remaining is in ctr. */ -143: mfctr r5 - li r0,0 - mr r4,r3 - mr r3,r5 /* return the number of bytes not copied */ -1: andi. r9,r4,7 - beq 3f -90: stb r0,0(r4) - addic. r5,r5,-1 - addi r4,r4,1 - bne 1b - blr -3: cmpldi cr1,r5,8 - srdi r9,r5,3 - andi. r5,r5,7 - blt cr1,93f - mtctr r9 -91: std r0,0(r4) - addi r4,r4,8 - bdnz 91b -93: beqlr - mtctr r5 -92: stb r0,0(r4) - addi r4,r4,1 - bdnz 92b +143: mfctr r3 blr /* @@ -389,10 +366,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) ld r5,-8(r1) add r6,r6,r5 subf r3,r3,r6 /* #bytes not copied */ -190: -191: -192: - blr /* #bytes not copied in r3 */ + blr EX_TABLE(20b,120b) EX_TABLE(220b,320b) @@ -451,9 +425,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) EX_TABLE(88b,188b) EX_TABLE(43b,143b) EX_TABLE(89b,189b) - EX_TABLE(90b,190b) - EX_TABLE(91b,191b) - EX_TABLE(92b,192b) /* * Routine to copy a whole page of data, optimized for POWER4. diff --git a/arch/powerpc/lib/usercopy_64.c b/arch/powerpc/lib/usercopy_64.c deleted file mode 100644 index 9bd3a3dad78d..000000000000 --- a/arch/powerpc/lib/usercopy_64.c +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Functions which are too large to be inlined. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ -#include -#include - -unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) -{ - if (likely(access_ok(VERIFY_READ, from, n))) - n = __copy_from_user(to, from, n); - else - memset(to, 0, n); - return n; -} - -unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) -{ - if (likely(access_ok(VERIFY_WRITE, to, n))) - n = __copy_to_user(to, from, n); - return n; -} - -unsigned long copy_in_user(void __user *to, const void __user *from, - unsigned long n) -{ - might_sleep(); - if (likely(access_ok(VERIFY_READ, from, n) && - access_ok(VERIFY_WRITE, to, n))) - n =__copy_tofrom_user(to, from, n); - return n; -} - -EXPORT_SYMBOL(copy_from_user); -EXPORT_SYMBOL(copy_to_user); -EXPORT_SYMBOL(copy_in_user); - From 8bec2717265601516bfe129acb4100a3070806ce Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sun, 25 Dec 2016 14:24:35 -0500 Subject: [PATCH 86/96] ia64: add extable.h Signed-off-by: Al Viro --- arch/ia64/include/asm/extable.h | 11 +++++++++++ arch/ia64/include/asm/uaccess.h | 8 +------- arch/ia64/mm/extable.c | 5 ++++- 3 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 arch/ia64/include/asm/extable.h diff --git a/arch/ia64/include/asm/extable.h b/arch/ia64/include/asm/extable.h new file mode 100644 index 000000000000..20376e71eab4 --- /dev/null +++ b/arch/ia64/include/asm/extable.h @@ -0,0 +1,11 @@ +#ifndef _ASM_IA64_EXTABLE_H +#define _ASM_IA64_EXTABLE_H + +#define ARCH_HAS_RELATIVE_EXTABLE + +struct exception_table_entry { + int insn; /* location-relative address of insn this fixup is for */ + int fixup; /* location-relative continuation addr.; if bit 2 is set, r9 is set to 0 */ +}; + +#endif diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index d471d1a1afd0..3f5ca0b8328a 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -39,6 +39,7 @@ #include #include #include +#include /* * For historical reasons, the following macros are grossly misnamed: @@ -341,13 +342,6 @@ extern unsigned long __strnlen_user (const char __user *, long); __su_ret; \ }) -#define ARCH_HAS_RELATIVE_EXTABLE - -struct exception_table_entry { - int insn; /* location-relative address of insn this fixup is for */ - int fixup; /* location-relative continuation addr.; if bit 2 is set, r9 is set to 0 */ -}; - #define ARCH_HAS_TRANSLATE_MEM_PTR 1 static __inline__ void * xlate_dev_mem_ptr(phys_addr_t p) diff --git a/arch/ia64/mm/extable.c b/arch/ia64/mm/extable.c index 4edb816aba9a..10dd4a66e167 100644 --- a/arch/ia64/mm/extable.c +++ b/arch/ia64/mm/extable.c @@ -5,7 +5,10 @@ * David Mosberger-Tang */ -#include +#include +#include +#include +#include void ia64_handle_exception (struct pt_regs *regs, const struct exception_table_entry *e) From 1bd5986bbccf1c44bc365f3cd5280350e8aa76bc Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 10:19:22 -0500 Subject: [PATCH 87/96] ia64: get rid of 'segment' argument of __{get,put}_user_check() always equal to get_fs() Signed-off-by: Al Viro --- arch/ia64/include/asm/uaccess.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index 3f5ca0b8328a..4132497d8ee6 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -76,8 +76,8 @@ * (a) re-use the arguments for side effects (sizeof/typeof is ok) * (b) require any knowledge of processes at this stage */ -#define put_user(x, ptr) __put_user_check((__typeof__(*(ptr))) (x), (ptr), sizeof(*(ptr)), get_fs()) -#define get_user(x, ptr) __get_user_check((x), (ptr), sizeof(*(ptr)), get_fs()) +#define put_user(x, ptr) __put_user_check((__typeof__(*(ptr))) (x), (ptr), sizeof(*(ptr))) +#define get_user(x, ptr) __get_user_check((x), (ptr), sizeof(*(ptr))) /* * The "__xxx" versions do not do address space checking, useful when @@ -199,7 +199,7 @@ extern void __get_user_unknown (void); }) #define __get_user_nocheck(x, ptr, size) __do_get_user(0, x, ptr, size, KERNEL_DS) -#define __get_user_check(x, ptr, size, segment) __do_get_user(1, x, ptr, size, segment) +#define __get_user_check(x, ptr, size) __do_get_user(1, x, ptr, size, get_fs()) extern void __put_user_unknown (void); @@ -226,7 +226,7 @@ extern void __put_user_unknown (void); }) #define __put_user_nocheck(x, ptr, size) __do_put_user(0, x, ptr, size, KERNEL_DS) -#define __put_user_check(x, ptr, size, segment) __do_put_user(1, x, ptr, size, segment) +#define __put_user_check(x, ptr, size) __do_put_user(1, x, ptr, size, get_fs()) /* * Complex access routines From 11836ecee96e8e1a62d5bd3ddd60eb9ecc01c375 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 10:21:09 -0500 Subject: [PATCH 88/96] ia64: get rid of 'segment' argument of __do_{get,put}_user() it's only evaluated if the first argument is not 0, and in those cases it's always equal to get_fs() Signed-off-by: Al Viro --- arch/ia64/include/asm/uaccess.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index 4132497d8ee6..24b39341f816 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -180,13 +180,13 @@ extern void __get_user_unknown (void); * could clobber r8 and r9 (among others). Thus, be careful not to evaluate it while * using r8/r9. */ -#define __do_get_user(check, x, ptr, size, segment) \ +#define __do_get_user(check, x, ptr, size) \ ({ \ const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \ __typeof__ (size) __gu_size = (size); \ long __gu_err = -EFAULT; \ unsigned long __gu_val = 0; \ - if (!check || __access_ok(__gu_ptr, size, segment)) \ + if (!check || __access_ok(__gu_ptr, size, get_fs())) \ switch (__gu_size) { \ case 1: __get_user_size(__gu_val, __gu_ptr, 1, __gu_err); break; \ case 2: __get_user_size(__gu_val, __gu_ptr, 2, __gu_err); break; \ @@ -198,8 +198,8 @@ extern void __get_user_unknown (void); __gu_err; \ }) -#define __get_user_nocheck(x, ptr, size) __do_get_user(0, x, ptr, size, KERNEL_DS) -#define __get_user_check(x, ptr, size) __do_get_user(1, x, ptr, size, get_fs()) +#define __get_user_nocheck(x, ptr, size) __do_get_user(0, x, ptr, size) +#define __get_user_check(x, ptr, size) __do_get_user(1, x, ptr, size) extern void __put_user_unknown (void); @@ -207,14 +207,14 @@ extern void __put_user_unknown (void); * Evaluating arguments X, PTR, SIZE, and SEGMENT may involve subroutine-calls, which * could clobber r8 (among others). Thus, be careful not to evaluate them while using r8. */ -#define __do_put_user(check, x, ptr, size, segment) \ +#define __do_put_user(check, x, ptr, size) \ ({ \ __typeof__ (x) __pu_x = (x); \ __typeof__ (*(ptr)) __user *__pu_ptr = (ptr); \ __typeof__ (size) __pu_size = (size); \ long __pu_err = -EFAULT; \ \ - if (!check || __access_ok(__pu_ptr, __pu_size, segment)) \ + if (!check || __access_ok(__pu_ptr, __pu_size, get_fs())) \ switch (__pu_size) { \ case 1: __put_user_size(__pu_x, __pu_ptr, 1, __pu_err); break; \ case 2: __put_user_size(__pu_x, __pu_ptr, 2, __pu_err); break; \ @@ -225,8 +225,8 @@ extern void __put_user_unknown (void); __pu_err; \ }) -#define __put_user_nocheck(x, ptr, size) __do_put_user(0, x, ptr, size, KERNEL_DS) -#define __put_user_check(x, ptr, size) __do_put_user(1, x, ptr, size, get_fs()) +#define __put_user_nocheck(x, ptr, size) __do_put_user(0, x, ptr, size) +#define __put_user_check(x, ptr, size) __do_put_user(1, x, ptr, size) /* * Complex access routines From 7bb8a503e8fad51e4fb5ed2453b9e23bedd82012 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Dec 2016 10:29:34 -0500 Subject: [PATCH 89/96] ia64: sanitize __access_ok() turn into static inline, kill the 'segment' argument. Signed-off-by: Al Viro --- arch/ia64/include/asm/uaccess.h | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index 24b39341f816..22c4e6d5d104 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -59,14 +59,14 @@ * address TASK_SIZE is never valid. We also need to make sure that the address doesn't * point inside the virtually mapped linear page table. */ -#define __access_ok(addr, size, segment) \ -({ \ - __chk_user_ptr(addr); \ - (likely((unsigned long) (addr) <= (segment).seg) \ - && ((segment).seg == KERNEL_DS.seg \ - || likely(REGION_OFFSET((unsigned long) (addr)) < RGN_MAP_LIMIT))); \ -}) -#define access_ok(type, addr, size) __access_ok((addr), (size), get_fs()) +static inline int __access_ok(const void __user *p, unsigned long size) +{ + unsigned long addr = (unsigned long)p; + unsigned long seg = get_fs().seg; + return likely(addr <= seg) && + (seg == KERNEL_DS.seg || likely(REGION_OFFSET(addr) < RGN_MAP_LIMIT)); +} +#define access_ok(type, addr, size) __access_ok((addr), (size)) /* * These are the main single-value transfer routines. They automatically @@ -186,7 +186,7 @@ extern void __get_user_unknown (void); __typeof__ (size) __gu_size = (size); \ long __gu_err = -EFAULT; \ unsigned long __gu_val = 0; \ - if (!check || __access_ok(__gu_ptr, size, get_fs())) \ + if (!check || __access_ok(__gu_ptr, size)) \ switch (__gu_size) { \ case 1: __get_user_size(__gu_val, __gu_ptr, 1, __gu_err); break; \ case 2: __get_user_size(__gu_val, __gu_ptr, 2, __gu_err); break; \ @@ -214,7 +214,7 @@ extern void __put_user_unknown (void); __typeof__ (size) __pu_size = (size); \ long __pu_err = -EFAULT; \ \ - if (!check || __access_ok(__pu_ptr, __pu_size, get_fs())) \ + if (!check || __access_ok(__pu_ptr, __pu_size)) \ switch (__pu_size) { \ case 1: __put_user_size(__pu_x, __pu_ptr, 1, __pu_err); break; \ case 2: __put_user_size(__pu_x, __pu_ptr, 2, __pu_err); break; \ @@ -258,7 +258,7 @@ __copy_from_user (void *to, const void __user *from, unsigned long count) const void *__cu_from = (from); \ long __cu_len = (n); \ \ - if (__access_ok(__cu_to, __cu_len, get_fs())) { \ + if (__access_ok(__cu_to, __cu_len)) { \ check_object_size(__cu_from, __cu_len, true); \ __cu_len = __copy_user(__cu_to, (__force void __user *) __cu_from, __cu_len); \ } \ @@ -269,7 +269,7 @@ static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) { check_object_size(to, n, false); - if (likely(__access_ok(from, n, get_fs()))) + if (likely(__access_ok(from, n))) n = __copy_user((__force void __user *) to, from, n); else memset(to, 0, n); @@ -293,7 +293,7 @@ extern unsigned long __do_clear_user (void __user *, unsigned long); #define clear_user(to, n) \ ({ \ unsigned long __cu_len = (n); \ - if (__access_ok(to, __cu_len, get_fs())) \ + if (__access_ok(to, __cu_len)) \ __cu_len = __do_clear_user(to, __cu_len); \ __cu_len; \ }) @@ -309,7 +309,7 @@ extern long __must_check __strncpy_from_user (char *to, const char __user *from, ({ \ const char __user * __sfu_from = (from); \ long __sfu_ret = -EFAULT; \ - if (__access_ok(__sfu_from, 0, get_fs())) \ + if (__access_ok(__sfu_from, 0)) \ __sfu_ret = __strncpy_from_user((to), __sfu_from, (n)); \ __sfu_ret; \ }) @@ -321,7 +321,7 @@ extern unsigned long __strlen_user (const char __user *); ({ \ const char __user *__su_str = (str); \ unsigned long __su_ret = 0; \ - if (__access_ok(__su_str, 0, get_fs())) \ + if (__access_ok(__su_str, 0)) \ __su_ret = __strlen_user(__su_str); \ __su_ret; \ }) @@ -337,7 +337,7 @@ extern unsigned long __strnlen_user (const char __user *, long); ({ \ const char __user *__su_str = (str); \ unsigned long __su_ret = 0; \ - if (__access_ok(__su_str, 0, get_fs())) \ + if (__access_ok(__su_str, 0)) \ __su_ret = __strnlen_user(__su_str, len); \ __su_ret; \ }) From 652c1aaca2a026d374d139058e8e7f3175e861d6 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 17:29:42 -0400 Subject: [PATCH 90/96] ia64: get rid of copy_in_user() it hadn't been biarch for years Signed-off-by: Al Viro --- arch/ia64/include/asm/uaccess.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index 22c4e6d5d104..bd159723dfde 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -276,16 +276,6 @@ copy_from_user(void *to, const void __user *from, unsigned long n) return n; } -#define __copy_in_user(to, from, size) __copy_user((to), (from), (size)) - -static inline unsigned long -copy_in_user (void __user *to, const void __user *from, unsigned long n) -{ - if (likely(access_ok(VERIFY_READ, from, n) && access_ok(VERIFY_WRITE, to, n))) - n = __copy_user(to, from, n); - return n; -} - extern unsigned long __do_clear_user (void __user *, unsigned long); #define __clear_user(to, n) __do_clear_user(to, n) From b3622d3217eb351717b157d64ce0895e662b7657 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 5 Apr 2017 18:56:55 -0400 Subject: [PATCH 91/96] get rid of padding, switch to RAW_COPY_USER Merced is fucked, so what else is new? Signed-off-by: Al Viro --- arch/ia64/Kconfig | 1 + arch/ia64/include/asm/uaccess.h | 35 ++++----------------------------- arch/ia64/lib/memcpy_mck.S | 13 +----------- 3 files changed, 6 insertions(+), 43 deletions(-) diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 18ca6a9ce566..0ed0e44856b2 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -53,6 +53,7 @@ config IA64 select ARCH_USE_CMPXCHG_LOCKREF select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_HARDENED_USERCOPY + select ARCH_HAS_RAW_COPY_USER default y help The Itanium Processor Family is Intel's 64-bit successor to diff --git a/arch/ia64/include/asm/uaccess.h b/arch/ia64/include/asm/uaccess.h index bd159723dfde..82a7646c4416 100644 --- a/arch/ia64/include/asm/uaccess.h +++ b/arch/ia64/include/asm/uaccess.h @@ -235,46 +235,19 @@ extern unsigned long __must_check __copy_user (void __user *to, const void __use unsigned long count); static inline unsigned long -__copy_to_user (void __user *to, const void *from, unsigned long count) +raw_copy_to_user(void __user *to, const void *from, unsigned long count) { - check_object_size(from, count, true); - return __copy_user(to, (__force void __user *) from, count); } static inline unsigned long -__copy_from_user (void *to, const void __user *from, unsigned long count) +raw_copy_from_user(void *to, const void __user *from, unsigned long count) { - check_object_size(to, count, false); - return __copy_user((__force void __user *) to, from, count); } -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user -#define copy_to_user(to, from, n) \ -({ \ - void __user *__cu_to = (to); \ - const void *__cu_from = (from); \ - long __cu_len = (n); \ - \ - if (__access_ok(__cu_to, __cu_len)) { \ - check_object_size(__cu_from, __cu_len, true); \ - __cu_len = __copy_user(__cu_to, (__force void __user *) __cu_from, __cu_len); \ - } \ - __cu_len; \ -}) - -static inline unsigned long -copy_from_user(void *to, const void __user *from, unsigned long n) -{ - check_object_size(to, n, false); - if (likely(__access_ok(from, n))) - n = __copy_user((__force void __user *) to, from, n); - else - memset(to, 0, n); - return n; -} +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER extern unsigned long __do_clear_user (void __user *, unsigned long); diff --git a/arch/ia64/lib/memcpy_mck.S b/arch/ia64/lib/memcpy_mck.S index b264b6a7967b..bbbadc478f5b 100644 --- a/arch/ia64/lib/memcpy_mck.S +++ b/arch/ia64/lib/memcpy_mck.S @@ -556,9 +556,6 @@ EK(.ex_handler, (p17) st8 [dst1]=r39,8); \ #define D r22 #define F r28 -#define memset_arg0 r32 -#define memset_arg2 r33 - #define saved_retval loc0 #define saved_rtlink loc1 #define saved_pfs_stack loc2 @@ -622,7 +619,7 @@ EK(.ex_handler, (p17) st8 [dst1]=r39,8); \ * (faulting_addr - orig_dst) -> len to faulting st address * B = (cur_dst - orig_dst) -> len copied so far * C = A - B -> len need to be copied - * D = orig_len - A -> len need to be zeroed + * D = orig_len - A -> len need to be left along */ (p6) sub A = F, saved_in0 (p7) sub A = F, saved_in1 @@ -638,9 +635,6 @@ EK(.ex_handler, (p17) st8 [dst1]=r39,8); \ sub D = saved_in2, A ;; cmp.gt p8,p0=C,r0 // more than 1 byte? - add memset_arg0=saved_in0, A -(p6) mov memset_arg2=0 // copy_to_user should not call memset -(p7) mov memset_arg2=D // copy_from_user need to have kbuf zeroed mov r8=0 mov saved_retval = D mov saved_rtlink = b0 @@ -652,11 +646,6 @@ EK(.ex_handler, (p17) st8 [dst1]=r39,8); \ ;; add saved_retval=saved_retval,r8 // above might return non-zero value - cmp.gt p8,p0=memset_arg2,r0 // more than 1 byte? - mov out0=memset_arg0 // *s - mov out1=r0 // c - mov out2=memset_arg2 // n -(p8) br.call.sptk.few b0=memset ;; mov retval=saved_retval From d491afb8654e84f972ed00cb0e087ad2fae5a1ac Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 12:10:12 -0400 Subject: [PATCH 92/96] microblaze: switch to RAW_COPY_USER [kudos to Piotr Sroka for spotting a braino in the previous variant] Signed-off-by: Al Viro --- arch/microblaze/Kconfig | 1 + arch/microblaze/include/asm/uaccess.h | 36 ++++++--------------------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 85885a501dce..1aff3658a104 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -34,6 +34,7 @@ config MICROBLAZE select TRACING_SUPPORT select VIRT_TO_BUS select CPU_NO_EFFICIENT_FFS + select ARCH_HAS_RAW_COPY_USER config SWAP def_bool n diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index 2c39375ac871..38f2c9ccef10 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -336,39 +336,19 @@ extern long __user_bad(void); __gu_err; \ }) - -/* copy_to_from_user */ -#define __copy_from_user(to, from, n) \ - __copy_tofrom_user((__force void __user *)(to), \ - (void __user *)(from), (n)) -#define __copy_from_user_inatomic(to, from, n) \ - __copy_from_user((to), (from), (n)) - -static inline long copy_from_user(void *to, - const void __user *from, unsigned long n) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { - unsigned long res = n; - might_fault(); - if (likely(access_ok(VERIFY_READ, from, n))) - res = __copy_from_user(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; + return __copy_tofrom_user((__force void __user *)to, from, n); } -#define __copy_to_user(to, from, n) \ - __copy_tofrom_user((void __user *)(to), \ - (__force const void __user *)(from), (n)) -#define __copy_to_user_inatomic(to, from, n) __copy_to_user((to), (from), (n)) - -static inline long copy_to_user(void __user *to, - const void *from, unsigned long n) +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { - might_fault(); - if (access_ok(VERIFY_WRITE, to, n)) - return __copy_to_user(to, from, n); - return n; + return __copy_tofrom_user(to, (__force const void __user *)from, n); } +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER /* * Copy a null terminated string from userspace. From ac4691fac8ad9bbb72268b6e4c73a168e670a196 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 28 Mar 2017 01:28:09 -0400 Subject: [PATCH 93/96] hexagon: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/hexagon/Kconfig | 1 + arch/hexagon/include/asm/uaccess.h | 23 ++++++++++------------- arch/hexagon/kernel/hexagon_ksyms.c | 4 ++-- arch/hexagon/mm/copy_from_user.S | 2 +- arch/hexagon/mm/copy_to_user.S | 2 +- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index 1941e4baaee6..0c536a83ea71 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -26,6 +26,7 @@ config HEXAGON select GENERIC_CLOCKEVENTS_BROADCAST select MODULES_USE_ELF_RELA select GENERIC_CPU_DEVICES + select ARCH_HAS_RAW_COPY_USER ---help--- Qualcomm Hexagon is a processor architecture designed for high performance and low power across a wide variety of applications. diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index 3a7f818e5ef7..458b69886b34 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -65,19 +65,12 @@ */ /* Assembly somewhat optimized copy routines */ -unsigned long __copy_from_user_hexagon(void *to, const void __user *from, +unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n); -unsigned long __copy_to_user_hexagon(void __user *to, const void *from, +unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n); - -#define __copy_from_user(to, from, n) __copy_from_user_hexagon(to, from, n) -#define __copy_to_user(to, from, n) __copy_to_user_hexagon(to, from, n) - -/* - * XXX todo: some additonal performance gain is possible by - * implementing __copy_to/from_user_inatomic, which is much - * like __copy_to/from_user, but performs slightly less checking. - */ +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER __kernel_size_t __clear_user_hexagon(void __user *dest, unsigned long count); #define __clear_user(a, s) __clear_user_hexagon((a), (s)) @@ -104,10 +97,14 @@ static inline long hexagon_strncpy_from_user(char *dst, const char __user *src, return -EFAULT; if (res > n) { - copy_from_user(dst, src, n); + long left = raw_copy_from_user(dst, src, n); + if (unlikely(left)) + memset(dst + (n - left), 0, left); return n; } else { - copy_from_user(dst, src, res); + long left = raw_copy_from_user(dst, src, res); + if (unlikely(left)) + memset(dst + (res - left), 0, left); return res-1; } } diff --git a/arch/hexagon/kernel/hexagon_ksyms.c b/arch/hexagon/kernel/hexagon_ksyms.c index af9dec4c28eb..00bcad9cbd8f 100644 --- a/arch/hexagon/kernel/hexagon_ksyms.c +++ b/arch/hexagon/kernel/hexagon_ksyms.c @@ -25,8 +25,8 @@ /* Additional functions */ EXPORT_SYMBOL(__clear_user_hexagon); -EXPORT_SYMBOL(__copy_from_user_hexagon); -EXPORT_SYMBOL(__copy_to_user_hexagon); +EXPORT_SYMBOL(raw_copy_from_user); +EXPORT_SYMBOL(raw_copy_to_user); EXPORT_SYMBOL(__iounmap); EXPORT_SYMBOL(__strnlen_user); EXPORT_SYMBOL(__vmgetie); diff --git a/arch/hexagon/mm/copy_from_user.S b/arch/hexagon/mm/copy_from_user.S index 7fc94f3e6642..7da066fbd16f 100644 --- a/arch/hexagon/mm/copy_from_user.S +++ b/arch/hexagon/mm/copy_from_user.S @@ -44,7 +44,7 @@ #define bytes r2 #define loopcount r5 -#define FUNCNAME __copy_from_user_hexagon +#define FUNCNAME raw_copy_from_user #include "copy_user_template.S" /* LOAD FAULTS from COPY_FROM_USER */ diff --git a/arch/hexagon/mm/copy_to_user.S b/arch/hexagon/mm/copy_to_user.S index 0cfbcc09d1d9..a7b7f8db21df 100644 --- a/arch/hexagon/mm/copy_to_user.S +++ b/arch/hexagon/mm/copy_to_user.S @@ -43,7 +43,7 @@ #define bytes r2 #define loopcount r5 -#define FUNCNAME __copy_to_user_hexagon +#define FUNCNAME raw_copy_to_user #include "copy_user_template.S" /* STORE FAULTS from COPY_TO_USER */ From 9a677341cd8de01a9eaf2c24a4b735939e507943 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 21 Mar 2017 12:02:25 -0400 Subject: [PATCH 94/96] m32r: switch to RAW_COPY_USER Signed-off-by: Al Viro --- arch/m32r/Kconfig | 1 + arch/m32r/include/asm/uaccess.h | 98 +++------------------------------ arch/m32r/kernel/m32r_ksyms.c | 2 - arch/m32r/lib/usercopy.c | 22 -------- 4 files changed, 8 insertions(+), 115 deletions(-) diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig index 95474460b367..b3e82bdd6db0 100644 --- a/arch/m32r/Kconfig +++ b/arch/m32r/Kconfig @@ -19,6 +19,7 @@ config M32R select HAVE_DEBUG_STACKOVERFLOW select CPU_NO_EFFICIENT_FFS select DMA_NOOP_OPS + select ARCH_HAS_RAW_COPY_USER config SBUS bool diff --git a/arch/m32r/include/asm/uaccess.h b/arch/m32r/include/asm/uaccess.h index d5c5e68fa2fb..07be349c00ad 100644 --- a/arch/m32r/include/asm/uaccess.h +++ b/arch/m32r/include/asm/uaccess.h @@ -13,6 +13,7 @@ */ #include #include +#include /* * The fs value determines whether argument validity checking should be @@ -463,107 +464,22 @@ do { \ /* We let the __ versions of copy_from/to_user inline, because they're often * used in fast paths and have only a small space overhead. */ -static inline unsigned long __generic_copy_from_user_nocheck(void *to, - const void __user *from, unsigned long n) +static inline unsigned long +raw_copy_from_user(void *to, const void __user *from, unsigned long n) { + prefetchw(to); __copy_user(to, from, n); return n; } -static inline unsigned long __generic_copy_to_user_nocheck(void __user *to, - const void *from, unsigned long n) +static inline unsigned long +raw_copy_to_user(void __user *to, const void *from, unsigned long n) { + prefetch(from); __copy_user(to, from, n); return n; } -unsigned long __generic_copy_to_user(void __user *, const void *, unsigned long); -unsigned long __generic_copy_from_user(void *, const void __user *, unsigned long); - -/** - * __copy_to_user: - Copy a block of data into user space, with less checking. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -#define __copy_to_user(to, from, n) \ - __generic_copy_to_user_nocheck((to), (from), (n)) - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user - -/** - * copy_to_user: - Copy a block of data into user space. - * @to: Destination address, in user space. - * @from: Source address, in kernel space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from kernel space to user space. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - */ -#define copy_to_user(to, from, n) \ -({ \ - might_fault(); \ - __generic_copy_to_user((to), (from), (n)); \ -}) - -/** - * __copy_from_user: - Copy a block of data from user space, with less checking. * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. Caller must check - * the specified block with access_ok() before calling this function. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - */ -#define __copy_from_user(to, from, n) \ - __generic_copy_from_user_nocheck((to), (from), (n)) - -/** - * copy_from_user: - Copy a block of data from user space. - * @to: Destination address, in kernel space. - * @from: Source address, in user space. - * @n: Number of bytes to copy. - * - * Context: User context only. This function may sleep if pagefaults are - * enabled. - * - * Copy data from user space to kernel space. - * - * Returns number of bytes that could not be copied. - * On success, this will be zero. - * - * If some data could not be copied, this function will pad the copied - * data to the requested size using zero bytes. - */ -#define copy_from_user(to, from, n) \ -({ \ - might_fault(); \ - __generic_copy_from_user((to), (from), (n)); \ -}) - long __must_check strncpy_from_user(char *dst, const char __user *src, long count); long __must_check __strncpy_from_user(char *dst, diff --git a/arch/m32r/kernel/m32r_ksyms.c b/arch/m32r/kernel/m32r_ksyms.c index d763f0bd2106..a4d43b5cc102 100644 --- a/arch/m32r/kernel/m32r_ksyms.c +++ b/arch/m32r/kernel/m32r_ksyms.c @@ -26,8 +26,6 @@ EXPORT_SYMBOL(strncpy_from_user); EXPORT_SYMBOL(__strncpy_from_user); EXPORT_SYMBOL(clear_user); EXPORT_SYMBOL(__clear_user); -EXPORT_SYMBOL(__generic_copy_from_user); -EXPORT_SYMBOL(__generic_copy_to_user); EXPORT_SYMBOL(strnlen_user); #ifdef CONFIG_SMP diff --git a/arch/m32r/lib/usercopy.c b/arch/m32r/lib/usercopy.c index 6aacf5ba0a58..b3ef2c899f96 100644 --- a/arch/m32r/lib/usercopy.c +++ b/arch/m32r/lib/usercopy.c @@ -11,28 +11,6 @@ #include #include -unsigned long -__generic_copy_to_user(void __user *to, const void *from, unsigned long n) -{ - prefetch(from); - if (access_ok(VERIFY_WRITE, to, n)) - __copy_user(to,from,n); - return n; -} - -unsigned long -__generic_copy_from_user(void *to, const void __user *from, unsigned long n) -{ - unsigned long ret = n; - prefetchw(to); - if (access_ok(VERIFY_READ, from, n)) - ret = __copy_user(to,from,n); - if (unlikely(ret)) - memset(to + n - ret, 0, ret); - return ret; -} - - /* * Copy a null terminated string from userspace. */ From 701cac61d0250912b89cbc28589969530179099a Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 5 Apr 2017 19:15:53 -0400 Subject: [PATCH 95/96] CONFIG_ARCH_HAS_RAW_COPY_USER is unconditional now all architectures converted Signed-off-by: Al Viro --- arch/Kconfig | 3 --- arch/alpha/Kconfig | 1 - arch/arc/Kconfig | 1 - arch/arm/Kconfig | 1 - arch/arm64/Kconfig | 1 - arch/avr32/Kconfig | 1 - arch/blackfin/Kconfig | 1 - arch/c6x/Kconfig | 1 - arch/cris/Kconfig | 1 - arch/frv/Kconfig | 1 - arch/h8300/Kconfig | 1 - arch/hexagon/Kconfig | 1 - arch/ia64/Kconfig | 1 - arch/m32r/Kconfig | 1 - arch/m68k/Kconfig | 1 - arch/metag/Kconfig | 1 - arch/microblaze/Kconfig | 1 - arch/mips/Kconfig | 1 - arch/mn10300/Kconfig | 1 - arch/nios2/Kconfig | 1 - arch/openrisc/Kconfig | 1 - arch/parisc/Kconfig | 1 - arch/powerpc/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/score/Kconfig | 1 - arch/sh/Kconfig | 1 - arch/sparc/Kconfig | 1 - arch/tile/Kconfig | 1 - arch/um/Kconfig.common | 1 - arch/unicore32/Kconfig | 1 - arch/x86/Kconfig | 1 - arch/xtensa/Kconfig | 1 - include/asm-generic/uaccess.h | 41 ----------------------------------- include/linux/uaccess.h | 7 ++---- lib/Makefile | 4 +--- 35 files changed, 3 insertions(+), 83 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 315d37626ddc..cd211a14a88f 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -847,7 +847,4 @@ config STRICT_MODULE_RWX config ARCH_WANT_RELAX_ORDER bool -config ARCH_HAS_RAW_COPY_USER - bool - source "kernel/gcov/Kconfig" diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 1be5f61dc630..0e49d39ea74a 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -26,7 +26,6 @@ config ALPHA select ODD_RT_SIGACTION select OLD_SIGSUSPEND select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67 - select ARCH_HAS_RAW_COPY_USER help The Alpha is a 64-bit general-purpose processor designed and marketed by the Digital Equipment Corporation of blessed memory, diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 7e213ff4f01f..c9f30f4763ab 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -44,7 +44,6 @@ config ARC select HAVE_GENERIC_DMA_COHERENT select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZMA - select ARCH_HAS_RAW_COPY_USER config MIGHT_HAVE_PCI bool diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 6fab7f34739c..0d4e71b42c77 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -96,7 +96,6 @@ config ARM select PERF_USE_VMALLOC select RTC_LIB select SYS_SUPPORTS_APM_EMULATION - select ARCH_HAS_RAW_COPY_USER # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. help diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3c833ff3303c..3741859765cf 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -115,7 +115,6 @@ config ARM64 select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK - select ARCH_HAS_RAW_COPY_USER help ARM 64-bit (AArch64) Linux support. diff --git a/arch/avr32/Kconfig b/arch/avr32/Kconfig index 8c349f2a9ebb..7e75d45e20cd 100644 --- a/arch/avr32/Kconfig +++ b/arch/avr32/Kconfig @@ -19,7 +19,6 @@ config AVR32 select HAVE_MOD_ARCH_SPECIFIC select MODULES_USE_ELF_RELA select HAVE_NMI - select ARCH_HAS_RAW_COPY_USER help AVR32 is a high-performance 32-bit RISC microprocessor core, designed for cost-sensitive embedded applications, with particular diff --git a/arch/blackfin/Kconfig b/arch/blackfin/Kconfig index 919dad1436f7..3c1bd640042a 100644 --- a/arch/blackfin/Kconfig +++ b/arch/blackfin/Kconfig @@ -41,7 +41,6 @@ config BLACKFIN select MODULES_USE_ELF_RELA select HAVE_DEBUG_STACKOVERFLOW select HAVE_NMI - select ARCH_HAS_RAW_COPY_USER config GENERIC_CSUM def_bool y diff --git a/arch/c6x/Kconfig b/arch/c6x/Kconfig index 3c7bd9a29f90..5aa8ea8bad2d 100644 --- a/arch/c6x/Kconfig +++ b/arch/c6x/Kconfig @@ -18,7 +18,6 @@ config C6X select GENERIC_CLOCKEVENTS select MODULES_USE_ELF_RELA select ARCH_NO_COHERENT_DMA_MMAP - select ARCH_HAS_RAW_COPY_USER config MMU def_bool n diff --git a/arch/cris/Kconfig b/arch/cris/Kconfig index 36f94c45e3f9..71b758dc3a96 100644 --- a/arch/cris/Kconfig +++ b/arch/cris/Kconfig @@ -71,7 +71,6 @@ config CRIS select GENERIC_SCHED_CLOCK if ETRAX_ARCH_V32 select HAVE_DEBUG_BUGVERBOSE if ETRAX_ARCH_V32 select HAVE_NMI - select ARCH_HAS_RAW_COPY_USER config HZ int diff --git a/arch/frv/Kconfig b/arch/frv/Kconfig index e489fef111dd..eefd9a4ed156 100644 --- a/arch/frv/Kconfig +++ b/arch/frv/Kconfig @@ -16,7 +16,6 @@ config FRV select OLD_SIGACTION select HAVE_DEBUG_STACKOVERFLOW select ARCH_NO_COHERENT_DMA_MMAP - select ARCH_HAS_RAW_COPY_USER config ZONE_DMA bool diff --git a/arch/h8300/Kconfig b/arch/h8300/Kconfig index 473883417004..3ae852507e57 100644 --- a/arch/h8300/Kconfig +++ b/arch/h8300/Kconfig @@ -22,7 +22,6 @@ config H8300 select HAVE_ARCH_KGDB select HAVE_ARCH_HASH select CPU_NO_EFFICIENT_FFS - select ARCH_HAS_RAW_COPY_USER config RWSEM_GENERIC_SPINLOCK def_bool y diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig index 0c536a83ea71..1941e4baaee6 100644 --- a/arch/hexagon/Kconfig +++ b/arch/hexagon/Kconfig @@ -26,7 +26,6 @@ config HEXAGON select GENERIC_CLOCKEVENTS_BROADCAST select MODULES_USE_ELF_RELA select GENERIC_CPU_DEVICES - select ARCH_HAS_RAW_COPY_USER ---help--- Qualcomm Hexagon is a processor architecture designed for high performance and low power across a wide variety of applications. diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 0ed0e44856b2..18ca6a9ce566 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -53,7 +53,6 @@ config IA64 select ARCH_USE_CMPXCHG_LOCKREF select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_HARDENED_USERCOPY - select ARCH_HAS_RAW_COPY_USER default y help The Itanium Processor Family is Intel's 64-bit successor to diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig index b3e82bdd6db0..95474460b367 100644 --- a/arch/m32r/Kconfig +++ b/arch/m32r/Kconfig @@ -19,7 +19,6 @@ config M32R select HAVE_DEBUG_STACKOVERFLOW select CPU_NO_EFFICIENT_FFS select DMA_NOOP_OPS - select ARCH_HAS_RAW_COPY_USER config SBUS bool diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 7d345758ea16..d140206d5d29 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -22,7 +22,6 @@ config M68K select MODULES_USE_ELF_RELA select OLD_SIGSUSPEND3 select OLD_SIGACTION - select ARCH_HAS_RAW_COPY_USER config RWSEM_GENERIC_SPINLOCK bool diff --git a/arch/metag/Kconfig b/arch/metag/Kconfig index ecce0c5ec8e8..5b7a45d99cfb 100644 --- a/arch/metag/Kconfig +++ b/arch/metag/Kconfig @@ -1,6 +1,5 @@ config METAG def_bool y - select ARCH_HAS_RAW_COPY_USER select EMBEDDED select GENERIC_ATOMIC64 select GENERIC_CLOCKEVENTS diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 1aff3658a104..85885a501dce 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -34,7 +34,6 @@ config MICROBLAZE select TRACING_SUPPORT select VIRT_TO_BUS select CPU_NO_EFFICIENT_FFS - select ARCH_HAS_RAW_COPY_USER config SWAP def_bool n diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index aff5633bfe4d..a008a9f03072 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -69,7 +69,6 @@ config MIPS select HAVE_EXIT_THREAD select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_ARCH_HARDENED_USERCOPY - select ARCH_HAS_RAW_COPY_USER menu "Machine selection" diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig index a96f3dcb0119..38e3494bfb63 100644 --- a/arch/mn10300/Kconfig +++ b/arch/mn10300/Kconfig @@ -16,7 +16,6 @@ config MN10300 select OLD_SIGACTION select HAVE_DEBUG_STACKOVERFLOW select ARCH_NO_COHERENT_DMA_MMAP - select ARCH_HAS_RAW_COPY_USER config AM33_2 def_bool n diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index 45b4727e3136..51a56c8b04b4 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -16,7 +16,6 @@ config NIOS2 select SPARSE_IRQ select USB_ARCH_HAS_HCD if USB_SUPPORT select CPU_NO_EFFICIENT_FFS - select ARCH_HAS_RAW_COPY_USER config GENERIC_CSUM def_bool y diff --git a/arch/openrisc/Kconfig b/arch/openrisc/Kconfig index 38954181fa96..1e95920b0737 100644 --- a/arch/openrisc/Kconfig +++ b/arch/openrisc/Kconfig @@ -28,7 +28,6 @@ config OPENRISC select OR1K_PIC select CPU_NO_EFFICIENT_FFS if !OPENRISC_HAVE_INST_FF1 select NO_BOOTMEM - select ARCH_HAS_RAW_COPY_USER config MMU def_bool y diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 15b7b279a169..ad294b3fb90b 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -41,7 +41,6 @@ config PARISC select GENERIC_CLOCKEVENTS select ARCH_NO_COHERENT_DMA_MMAP select CPU_NO_EFFICIENT_FFS - select ARCH_HAS_RAW_COPY_USER help The PA-RISC microprocessor is designed by Hewlett-Packard and used diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 839f08887269..97a8bc8a095c 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -87,7 +87,6 @@ config PPC select ARCH_HAS_DMA_SET_COHERENT_MASK select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_GCOV_PROFILE_ALL - select ARCH_HAS_RAW_COPY_USER select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE select ARCH_HAS_SG_CHAIN select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 0724dbf2d8f8..a2dcef0aacc7 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -178,7 +178,6 @@ config S390 select ARCH_HAS_SCALED_CPUTIME select VIRT_TO_BUS select HAVE_NMI - select ARCH_HAS_RAW_COPY_USER config SCHED_OMIT_FRAME_POINTER diff --git a/arch/score/Kconfig b/arch/score/Kconfig index 4d241cfedc3b..507d63181389 100644 --- a/arch/score/Kconfig +++ b/arch/score/Kconfig @@ -15,7 +15,6 @@ config SCORE select MODULES_USE_ELF_REL select CLONE_BACKWARDS select CPU_NO_EFFICIENT_FFS - select ARCH_HAS_RAW_COPY_USER choice prompt "System type" diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index c689645eb8b7..ee086958b2b2 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -48,7 +48,6 @@ config SUPERH select HAVE_ARCH_AUDITSYSCALL select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_NMI - select ARCH_HAS_RAW_COPY_USER help The SuperH is a RISC processor targeted for use in embedded systems and consumer electronics; it was also used in the Sega Dreamcast diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 68353048f24e..68ac5c7cd982 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -45,7 +45,6 @@ config SPARC select HAVE_ARCH_HARDENED_USERCOPY select PROVE_LOCKING_SMALL if PROVE_LOCKING select ARCH_WANT_RELAX_ORDER - select ARCH_HAS_RAW_COPY_USER config SPARC32 def_bool !64BIT diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig index 27808d86eb57..4583c0320059 100644 --- a/arch/tile/Kconfig +++ b/arch/tile/Kconfig @@ -33,7 +33,6 @@ config TILE select USER_STACKTRACE_SUPPORT select USE_PMC if PERF_EVENTS select VIRT_TO_BUS - select ARCH_HAS_RAW_COPY_USER config MMU def_bool y diff --git a/arch/um/Kconfig.common b/arch/um/Kconfig.common index e13f763849b1..fd443852103c 100644 --- a/arch/um/Kconfig.common +++ b/arch/um/Kconfig.common @@ -13,7 +13,6 @@ config UML select GENERIC_CLOCKEVENTS select HAVE_GCC_PLUGINS select TTY # Needed for line.c - select ARCH_HAS_RAW_COPY_USER config MMU bool diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig index 319519eaed24..0769066929c6 100644 --- a/arch/unicore32/Kconfig +++ b/arch/unicore32/Kconfig @@ -18,7 +18,6 @@ config UNICORE32 select ARCH_WANT_FRAME_POINTERS select GENERIC_IOMAP select MODULES_USE_ELF_REL - select ARCH_HAS_RAW_COPY_USER help UniCore-32 is 32-bit Instruction Set Architecture, including a series of low-power-consumption RISC chip diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 5f59fc388063..cc98d5a294ee 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -175,7 +175,6 @@ config X86 select USER_STACKTRACE_SUPPORT select VIRT_TO_BUS select X86_FEATURE_NAMES if PROC_FS - select ARCH_HAS_RAW_COPY_USER config INSTRUCTION_DECODER def_bool y diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 043d37d45919..f4126cf997a4 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -29,7 +29,6 @@ config XTENSA select NO_BOOTMEM select PERF_USE_VMALLOC select VIRT_TO_BUS - select ARCH_HAS_RAW_COPY_USER help Xtensa processors are 32-bit RISC machines designed by Tensilica primarily for embedded systems. These processors are both diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index d65c311eb128..bbe4bb438e39 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -86,11 +86,7 @@ static inline int __access_ok(unsigned long addr, unsigned long size) static inline int __put_user_fn(size_t size, void __user *ptr, void *x) { -#ifdef CONFIG_ARCH_HAS_RAW_COPY_USER return unlikely(raw_copy_to_user(ptr, x, size)) ? -EFAULT : 0; -#else - return unlikely(__copy_to_user(ptr, x, size)) ? -EFAULT : 0; -#endif } #define __put_user_fn(sz, u, k) __put_user_fn(sz, u, k) @@ -151,11 +147,7 @@ extern int __put_user_bad(void) __attribute__((noreturn)); #ifndef __get_user_fn static inline int __get_user_fn(size_t size, const void __user *ptr, void *x) { -#ifdef CONFIG_ARCH_HAS_RAW_COPY_USER return unlikely(raw_copy_from_user(x, ptr, size)) ? -EFAULT : 0; -#else - return unlikely(__copy_from_user(x, ptr, size)) ? -EFAULT : 0; -#endif } #define __get_user_fn(sz, u, k) __get_user_fn(sz, u, k) @@ -164,39 +156,6 @@ static inline int __get_user_fn(size_t size, const void __user *ptr, void *x) extern int __get_user_bad(void) __attribute__((noreturn)); -#ifndef CONFIG_ARCH_HAS_RAW_COPY_USER - -#ifndef __copy_from_user_inatomic -#define __copy_from_user_inatomic __copy_from_user -#endif - -#ifndef __copy_to_user_inatomic -#define __copy_to_user_inatomic __copy_to_user -#endif - -static inline long copy_from_user(void *to, - const void __user * from, unsigned long n) -{ - unsigned long res = n; - might_fault(); - if (likely(access_ok(VERIFY_READ, from, n))) - res = __copy_from_user(to, from, n); - if (unlikely(res)) - memset(to + (n - res), 0, res); - return res; -} - -static inline long copy_to_user(void __user *to, - const void *from, unsigned long n) -{ - might_fault(); - if (access_ok(VERIFY_WRITE, to, n)) - return __copy_to_user(to, from, n); - else - return n; -} -#endif - /* * Copy a null terminated string from userspace. */ diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 7fc2104b88bc..e0cbfb09e60f 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -12,12 +12,10 @@ #include -#ifdef CONFIG_ARCH_HAS_RAW_COPY_USER /* * Architectures should provide two primitives (raw_copy_{to,from}_user()) - * select ARCH_HAS_RAW_COPY_FROM_USER and get rid of their private instances - * of copy_{to,from}_user() and __copy_{to,from}_user{,_inatomic}(). Once - * all of them switch, this part of linux/uaccess.h will become unconditional. + * and get rid of their private instances of copy_{to,from}_user() and + * __copy_{to,from}_user{,_inatomic}(). * * raw_copy_{to,from}_user(to, from, size) should copy up to size bytes and * return the amount left to copy. They should assume that access_ok() has @@ -196,7 +194,6 @@ copy_in_user(void __user *to, const void *from, unsigned long n) return n; } #endif -#endif static __always_inline void pagefault_disabled_inc(void) { diff --git a/lib/Makefile b/lib/Makefile index 7d875c389172..b47cf97e1e68 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -41,7 +41,7 @@ obj-y += bcd.o div64.o sort.o parser.o debug_locks.o random32.o \ gcd.o lcm.o list_sort.o uuid.o flex_array.o iov_iter.o clz_ctz.o \ bsearch.o find_bit.o llist.o memweight.o kfifo.o \ percpu-refcount.o percpu_ida.o rhashtable.o reciprocal_div.o \ - once.o refcount.o + once.o refcount.o usercopy.o obj-y += string_helpers.o obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o obj-y += hexdump.o @@ -242,5 +242,3 @@ UBSAN_SANITIZE_ubsan.o := n obj-$(CONFIG_SBITMAP) += sbitmap.o obj-$(CONFIG_PARMAN) += parman.o - -obj-$(CONFIG_ARCH_HAS_RAW_COPY_USER) += usercopy.o From 2fefc97b2180518bac923fba3f79fdca1f41dc15 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 5 Apr 2017 19:17:18 -0400 Subject: [PATCH 96/96] HAVE_ARCH_HARDENED_USERCOPY is unconditional now Signed-off-by: Al Viro --- arch/arm/Kconfig | 1 - arch/arm64/Kconfig | 1 - arch/ia64/Kconfig | 1 - arch/mips/Kconfig | 1 - arch/parisc/Kconfig | 1 - arch/powerpc/Kconfig | 1 - arch/s390/Kconfig | 1 - arch/sparc/Kconfig | 1 - arch/x86/Kconfig | 1 - security/Kconfig | 9 --------- 10 files changed, 18 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 0d4e71b42c77..8a7ab5e73af9 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -41,7 +41,6 @@ config ARM select HARDIRQS_SW_RESEND select HAVE_ARCH_AUDITSYSCALL if (AEABI && !OABI_COMPAT) select HAVE_ARCH_BITREVERSE if (CPU_32v7M || CPU_32v7) && !CPU_32v6 - select HAVE_ARCH_HARDENED_USERCOPY select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU select HAVE_ARCH_MMAP_RND_BITS if MMU diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3741859765cf..b88550ab1a0c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -60,7 +60,6 @@ config ARM64 select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_BITREVERSE - select HAVE_ARCH_HARDENED_USERCOPY select HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KASAN if SPARSEMEM_VMEMMAP && !(ARM64_16K_PAGES && ARM64_VA_BITS_48) diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 18ca6a9ce566..6a15083cc366 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -52,7 +52,6 @@ config IA64 select MODULES_USE_ELF_RELA select ARCH_USE_CMPXCHG_LOCKREF select HAVE_ARCH_AUDITSYSCALL - select HAVE_ARCH_HARDENED_USERCOPY default y help The Itanium Processor Family is Intel's 64-bit successor to diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index a008a9f03072..81ce5b4b9251 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -68,7 +68,6 @@ config MIPS select HANDLE_DOMAIN_IRQ select HAVE_EXIT_THREAD select HAVE_REGS_AND_STACK_ACCESS_API - select HAVE_ARCH_HARDENED_USERCOPY menu "Machine selection" diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index ad294b3fb90b..531da9eb8f43 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -26,7 +26,6 @@ config PARISC select SYSCTL_ARCH_UNALIGN_ALLOW select SYSCTL_EXCEPTION_TRACE select HAVE_MOD_ARCH_SPECIFIC - select HAVE_ARCH_HARDENED_USERCOPY select VIRT_TO_BUS select MODULES_USE_ELF_RELA select CLONE_BACKWARDS diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 97a8bc8a095c..053382616533 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -117,7 +117,6 @@ config PPC select GENERIC_STRNLEN_USER select GENERIC_TIME_VSYSCALL_OLD select HAVE_ARCH_AUDITSYSCALL - select HAVE_ARCH_HARDENED_USERCOPY select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KGDB select HAVE_ARCH_SECCOMP_FILTER diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index a2dcef0aacc7..b8b143432381 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -124,7 +124,6 @@ config S390 select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_EARLY_PFN_TO_NID - select HAVE_ARCH_HARDENED_USERCOPY select HAVE_ARCH_JUMP_LABEL select CPU_NO_EFFICIENT_FFS if !HAVE_MARCH_Z9_109_FEATURES select HAVE_ARCH_SECCOMP_FILTER diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 68ac5c7cd982..b577dbff5efb 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -42,7 +42,6 @@ config SPARC select OLD_SIGSUSPEND select ARCH_HAS_SG_CHAIN select CPU_NO_EFFICIENT_FFS - select HAVE_ARCH_HARDENED_USERCOPY select PROVE_LOCKING_SMALL if PROVE_LOCKING select ARCH_WANT_RELAX_ORDER diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index cc98d5a294ee..1d50fdff77ee 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -98,7 +98,6 @@ config X86 select HAVE_ACPI_APEI_NMI if ACPI select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL - select HAVE_ARCH_HARDENED_USERCOPY select HAVE_ARCH_HUGE_VMAP if X86_64 || X86_PAE select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KASAN if X86_64 && SPARSEMEM_VMEMMAP diff --git a/security/Kconfig b/security/Kconfig index d900f47eaa68..213df4d4f2c1 100644 --- a/security/Kconfig +++ b/security/Kconfig @@ -125,17 +125,8 @@ config HAVE_HARDENED_USERCOPY_ALLOCATOR validating memory ranges against heap object sizes in support of CONFIG_HARDENED_USERCOPY. -config HAVE_ARCH_HARDENED_USERCOPY - bool - help - The architecture supports CONFIG_HARDENED_USERCOPY by - calling check_object_size() just before performing the - userspace copies in the low level implementation of - copy_to_user() and copy_from_user(). - config HARDENED_USERCOPY bool "Harden memory copies between kernel and userspace" - depends on HAVE_ARCH_HARDENED_USERCOPY depends on HAVE_HARDENED_USERCOPY_ALLOCATOR select BUG help