2008-10-23 13:26:29 +08:00
|
|
|
#ifndef _ASM_X86_CPUFEATURE_H
|
|
|
|
|
#define _ASM_X86_CPUFEATURE_H
|
2008-01-30 20:30:07 +08:00
|
|
|
|
2016-01-27 05:12:04 +08:00
|
|
|
#include <asm/processor.h>
|
2013-03-20 22:07:24 +08:00
|
|
|
|
2008-02-04 23:48:00 +08:00
|
|
|
#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
|
|
|
|
|
|
2010-05-12 08:47:07 +08:00
|
|
|
#include <asm/asm.h>
|
2008-02-04 23:48:00 +08:00
|
|
|
#include <linux/bitops.h>
|
|
|
|
|
|
2015-12-07 17:39:40 +08:00
|
|
|
enum cpuid_leafs
|
|
|
|
|
{
|
|
|
|
|
CPUID_1_EDX = 0,
|
|
|
|
|
CPUID_8000_0001_EDX,
|
|
|
|
|
CPUID_8086_0001_EDX,
|
|
|
|
|
CPUID_LNX_1,
|
|
|
|
|
CPUID_1_ECX,
|
|
|
|
|
CPUID_C000_0001_EDX,
|
|
|
|
|
CPUID_8000_0001_ECX,
|
|
|
|
|
CPUID_LNX_2,
|
|
|
|
|
CPUID_LNX_3,
|
|
|
|
|
CPUID_7_0_EBX,
|
|
|
|
|
CPUID_D_1_EAX,
|
|
|
|
|
CPUID_F_0_EDX,
|
|
|
|
|
CPUID_F_1_EDX,
|
|
|
|
|
CPUID_8000_0008_EBX,
|
|
|
|
|
CPUID_6_EAX,
|
|
|
|
|
CPUID_8000_000A_EDX,
|
x86/cpufeature, x86/mm/pkeys: Add protection keys related CPUID definitions
There are two CPUID bits for protection keys. One is for whether
the CPU contains the feature, and the other will appear set once
the OS enables protection keys. Specifically:
Bit 04: OSPKE. If 1, OS has set CR4.PKE to enable
Protection keys (and the RDPKRU/WRPKRU instructions)
This is because userspace can not see CR4 contents, but it can
see CPUID contents.
X86_FEATURE_PKU is referred to as "PKU" in the hardware documentation:
CPUID.(EAX=07H,ECX=0H):ECX.PKU [bit 3]
X86_FEATURE_OSPKE is "OSPKU":
CPUID.(EAX=07H,ECX=0H):ECX.OSPKE [bit 4]
These are the first CPU features which need to look at the
ECX word in CPUID leaf 0x7, so this patch also includes
fetching that word in to the cpuinfo->x86_capability[] array.
Add it to the disabled-features mask when its config option is
off. Even though we are not using it here, we also extend the
REQUIRED_MASK_BIT_SET() macro to keep it mirroring the
DISABLED_MASK_BIT_SET() version.
This means that in almost all code, you should use:
cpu_has(c, X86_FEATURE_PKU)
and *not* the CONFIG option.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20160212210201.7714C250@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-02-13 05:02:01 +08:00
|
|
|
CPUID_7_ECX,
|
2016-05-11 20:58:26 +08:00
|
|
|
CPUID_8000_0007_EBX,
|
2015-12-07 17:39:40 +08:00
|
|
|
};
|
|
|
|
|
|
2013-10-30 23:09:45 +08:00
|
|
|
#ifdef CONFIG_X86_FEATURE_NAMES
|
2008-02-04 23:48:00 +08:00
|
|
|
extern const char * const x86_cap_flags[NCAPINTS*32];
|
|
|
|
|
extern const char * const x86_power_flags[32];
|
2013-10-30 23:09:45 +08:00
|
|
|
#define X86_CAP_FMT "%s"
|
|
|
|
|
#define x86_cap_flag(flag) x86_cap_flags[flag]
|
|
|
|
|
#else
|
|
|
|
|
#define X86_CAP_FMT "%d:%d"
|
|
|
|
|
#define x86_cap_flag(flag) ((flag) >> 5), ((flag) & 31)
|
|
|
|
|
#endif
|
2008-02-04 23:48:00 +08:00
|
|
|
|
2014-06-24 19:25:03 +08:00
|
|
|
/*
|
|
|
|
|
* In order to save room, we index into this array by doing
|
|
|
|
|
* X86_BUG_<name> - NCAPINTS*32.
|
|
|
|
|
*/
|
|
|
|
|
extern const char * const x86_bug_flags[NBUGINTS*32];
|
|
|
|
|
|
2008-02-26 15:34:21 +08:00
|
|
|
#define test_cpu_cap(c, bit) \
|
|
|
|
|
test_bit(bit, (unsigned long *)((c)->x86_capability))
|
|
|
|
|
|
2011-03-12 19:50:10 +08:00
|
|
|
#define REQUIRED_MASK_BIT_SET(bit) \
|
x86/cpufeature, x86/mm/pkeys: Add protection keys related CPUID definitions
There are two CPUID bits for protection keys. One is for whether
the CPU contains the feature, and the other will appear set once
the OS enables protection keys. Specifically:
Bit 04: OSPKE. If 1, OS has set CR4.PKE to enable
Protection keys (and the RDPKRU/WRPKRU instructions)
This is because userspace can not see CR4 contents, but it can
see CPUID contents.
X86_FEATURE_PKU is referred to as "PKU" in the hardware documentation:
CPUID.(EAX=07H,ECX=0H):ECX.PKU [bit 3]
X86_FEATURE_OSPKE is "OSPKU":
CPUID.(EAX=07H,ECX=0H):ECX.OSPKE [bit 4]
These are the first CPU features which need to look at the
ECX word in CPUID leaf 0x7, so this patch also includes
fetching that word in to the cpuinfo->x86_capability[] array.
Add it to the disabled-features mask when its config option is
off. Even though we are not using it here, we also extend the
REQUIRED_MASK_BIT_SET() macro to keep it mirroring the
DISABLED_MASK_BIT_SET() version.
This means that in almost all code, you should use:
cpu_has(c, X86_FEATURE_PKU)
and *not* the CONFIG option.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20160212210201.7714C250@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-02-13 05:02:01 +08:00
|
|
|
( (((bit)>>5)==0 && (1UL<<((bit)&31) & REQUIRED_MASK0 )) || \
|
|
|
|
|
(((bit)>>5)==1 && (1UL<<((bit)&31) & REQUIRED_MASK1 )) || \
|
|
|
|
|
(((bit)>>5)==2 && (1UL<<((bit)&31) & REQUIRED_MASK2 )) || \
|
|
|
|
|
(((bit)>>5)==3 && (1UL<<((bit)&31) & REQUIRED_MASK3 )) || \
|
|
|
|
|
(((bit)>>5)==4 && (1UL<<((bit)&31) & REQUIRED_MASK4 )) || \
|
|
|
|
|
(((bit)>>5)==5 && (1UL<<((bit)&31) & REQUIRED_MASK5 )) || \
|
|
|
|
|
(((bit)>>5)==6 && (1UL<<((bit)&31) & REQUIRED_MASK6 )) || \
|
|
|
|
|
(((bit)>>5)==7 && (1UL<<((bit)&31) & REQUIRED_MASK7 )) || \
|
|
|
|
|
(((bit)>>5)==8 && (1UL<<((bit)&31) & REQUIRED_MASK8 )) || \
|
|
|
|
|
(((bit)>>5)==9 && (1UL<<((bit)&31) & REQUIRED_MASK9 )) || \
|
|
|
|
|
(((bit)>>5)==10 && (1UL<<((bit)&31) & REQUIRED_MASK10)) || \
|
|
|
|
|
(((bit)>>5)==11 && (1UL<<((bit)&31) & REQUIRED_MASK11)) || \
|
|
|
|
|
(((bit)>>5)==12 && (1UL<<((bit)&31) & REQUIRED_MASK12)) || \
|
|
|
|
|
(((bit)>>5)==13 && (1UL<<((bit)&31) & REQUIRED_MASK13)) || \
|
2016-05-14 06:13:28 +08:00
|
|
|
(((bit)>>5)==14 && (1UL<<((bit)&31) & REQUIRED_MASK14)) || \
|
|
|
|
|
(((bit)>>5)==15 && (1UL<<((bit)&31) & REQUIRED_MASK15)) || \
|
|
|
|
|
(((bit)>>5)==16 && (1UL<<((bit)&31) & REQUIRED_MASK16)) )
|
2011-03-12 19:50:10 +08:00
|
|
|
|
x86: Introduce disabled-features
I believe the REQUIRED_MASK aproach was taken so that it was
easier to consult in assembly (arch/x86/kernel/verify_cpu.S).
DISABLED_MASK does not have the same restriction, but I
implemented it the same way for consistency.
We have a REQUIRED_MASK... which does two things:
1. Keeps a list of cpuid bits to check in very early boot and
refuse to boot if those are not present.
2. Consulted during cpu_has() checks, which allows us to
optimize out things at compile-time. In other words, if we
*KNOW* we will not boot with the feature off, then we can
safely assume that it will be present forever.
But, we don't have a similar mechanism for CPU features which
may be present but that we know we will not use. We simply
use our existing mechanisms to repeatedly check the status of
the bit at runtime (well, the alternatives patching helps here
but it does not provide compile-time optimization).
Adding a feature to disabled-features.h allows the bit to be
checked via a new macro: cpu_feature_enabled(). Note that
for features in DISABLED_MASK, checks with this macro have
all of the benefits of an #ifdef. Before, we would have done
this in a header:
#ifdef CONFIG_X86_INTEL_MPX
#define cpu_has_mpx cpu_has(X86_FEATURE_MPX)
#else
#define cpu_has_mpx 0
#endif
and this in the code:
if (cpu_has_mpx)
do_some_mpx_thing();
Now, just add your feature to DISABLED_MASK and you can do this
everywhere, and get the same benefits you would have from
#ifdefs:
if (cpu_feature_enabled(X86_FEATURE_MPX))
do_some_mpx_thing();
We need a new function and *not* a modification to cpu_has()
because there are cases where we actually need to check the CPU
itself, despite what features the kernel supports. The best
example of this is a hypervisor which has no control over what
features its guests are using and where the guest does not depend
on the host for support.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: http://lkml.kernel.org/r/20140911211513.9E35E931@viggo.jf.intel.com
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2014-09-12 05:15:13 +08:00
|
|
|
#define DISABLED_MASK_BIT_SET(bit) \
|
x86/cpufeature, x86/mm/pkeys: Add protection keys related CPUID definitions
There are two CPUID bits for protection keys. One is for whether
the CPU contains the feature, and the other will appear set once
the OS enables protection keys. Specifically:
Bit 04: OSPKE. If 1, OS has set CR4.PKE to enable
Protection keys (and the RDPKRU/WRPKRU instructions)
This is because userspace can not see CR4 contents, but it can
see CPUID contents.
X86_FEATURE_PKU is referred to as "PKU" in the hardware documentation:
CPUID.(EAX=07H,ECX=0H):ECX.PKU [bit 3]
X86_FEATURE_OSPKE is "OSPKU":
CPUID.(EAX=07H,ECX=0H):ECX.OSPKE [bit 4]
These are the first CPU features which need to look at the
ECX word in CPUID leaf 0x7, so this patch also includes
fetching that word in to the cpuinfo->x86_capability[] array.
Add it to the disabled-features mask when its config option is
off. Even though we are not using it here, we also extend the
REQUIRED_MASK_BIT_SET() macro to keep it mirroring the
DISABLED_MASK_BIT_SET() version.
This means that in almost all code, you should use:
cpu_has(c, X86_FEATURE_PKU)
and *not* the CONFIG option.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20160212210201.7714C250@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-02-13 05:02:01 +08:00
|
|
|
( (((bit)>>5)==0 && (1UL<<((bit)&31) & DISABLED_MASK0 )) || \
|
|
|
|
|
(((bit)>>5)==1 && (1UL<<((bit)&31) & DISABLED_MASK1 )) || \
|
|
|
|
|
(((bit)>>5)==2 && (1UL<<((bit)&31) & DISABLED_MASK2 )) || \
|
|
|
|
|
(((bit)>>5)==3 && (1UL<<((bit)&31) & DISABLED_MASK3 )) || \
|
|
|
|
|
(((bit)>>5)==4 && (1UL<<((bit)&31) & DISABLED_MASK4 )) || \
|
|
|
|
|
(((bit)>>5)==5 && (1UL<<((bit)&31) & DISABLED_MASK5 )) || \
|
|
|
|
|
(((bit)>>5)==6 && (1UL<<((bit)&31) & DISABLED_MASK6 )) || \
|
|
|
|
|
(((bit)>>5)==7 && (1UL<<((bit)&31) & DISABLED_MASK7 )) || \
|
|
|
|
|
(((bit)>>5)==8 && (1UL<<((bit)&31) & DISABLED_MASK8 )) || \
|
|
|
|
|
(((bit)>>5)==9 && (1UL<<((bit)&31) & DISABLED_MASK9 )) || \
|
|
|
|
|
(((bit)>>5)==10 && (1UL<<((bit)&31) & DISABLED_MASK10)) || \
|
|
|
|
|
(((bit)>>5)==11 && (1UL<<((bit)&31) & DISABLED_MASK11)) || \
|
|
|
|
|
(((bit)>>5)==12 && (1UL<<((bit)&31) & DISABLED_MASK12)) || \
|
|
|
|
|
(((bit)>>5)==13 && (1UL<<((bit)&31) & DISABLED_MASK13)) || \
|
2016-05-14 06:13:28 +08:00
|
|
|
(((bit)>>5)==14 && (1UL<<((bit)&31) & DISABLED_MASK14)) || \
|
|
|
|
|
(((bit)>>5)==15 && (1UL<<((bit)&31) & DISABLED_MASK15)) || \
|
|
|
|
|
(((bit)>>5)==16 && (1UL<<((bit)&31) & DISABLED_MASK16)) )
|
x86: Introduce disabled-features
I believe the REQUIRED_MASK aproach was taken so that it was
easier to consult in assembly (arch/x86/kernel/verify_cpu.S).
DISABLED_MASK does not have the same restriction, but I
implemented it the same way for consistency.
We have a REQUIRED_MASK... which does two things:
1. Keeps a list of cpuid bits to check in very early boot and
refuse to boot if those are not present.
2. Consulted during cpu_has() checks, which allows us to
optimize out things at compile-time. In other words, if we
*KNOW* we will not boot with the feature off, then we can
safely assume that it will be present forever.
But, we don't have a similar mechanism for CPU features which
may be present but that we know we will not use. We simply
use our existing mechanisms to repeatedly check the status of
the bit at runtime (well, the alternatives patching helps here
but it does not provide compile-time optimization).
Adding a feature to disabled-features.h allows the bit to be
checked via a new macro: cpu_feature_enabled(). Note that
for features in DISABLED_MASK, checks with this macro have
all of the benefits of an #ifdef. Before, we would have done
this in a header:
#ifdef CONFIG_X86_INTEL_MPX
#define cpu_has_mpx cpu_has(X86_FEATURE_MPX)
#else
#define cpu_has_mpx 0
#endif
and this in the code:
if (cpu_has_mpx)
do_some_mpx_thing();
Now, just add your feature to DISABLED_MASK and you can do this
everywhere, and get the same benefits you would have from
#ifdefs:
if (cpu_feature_enabled(X86_FEATURE_MPX))
do_some_mpx_thing();
We need a new function and *not* a modification to cpu_has()
because there are cases where we actually need to check the CPU
itself, despite what features the kernel supports. The best
example of this is a hypervisor which has no control over what
features its guests are using and where the guest does not depend
on the host for support.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: http://lkml.kernel.org/r/20140911211513.9E35E931@viggo.jf.intel.com
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2014-09-12 05:15:13 +08:00
|
|
|
|
2011-03-12 19:50:10 +08:00
|
|
|
#define cpu_has(c, bit) \
|
|
|
|
|
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
|
2008-02-26 15:34:21 +08:00
|
|
|
test_cpu_cap(c, bit))
|
|
|
|
|
|
2011-03-12 19:50:10 +08:00
|
|
|
#define this_cpu_has(bit) \
|
|
|
|
|
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \
|
|
|
|
|
x86_this_cpu_test_bit(bit, (unsigned long *)&cpu_info.x86_capability))
|
|
|
|
|
|
x86: Introduce disabled-features
I believe the REQUIRED_MASK aproach was taken so that it was
easier to consult in assembly (arch/x86/kernel/verify_cpu.S).
DISABLED_MASK does not have the same restriction, but I
implemented it the same way for consistency.
We have a REQUIRED_MASK... which does two things:
1. Keeps a list of cpuid bits to check in very early boot and
refuse to boot if those are not present.
2. Consulted during cpu_has() checks, which allows us to
optimize out things at compile-time. In other words, if we
*KNOW* we will not boot with the feature off, then we can
safely assume that it will be present forever.
But, we don't have a similar mechanism for CPU features which
may be present but that we know we will not use. We simply
use our existing mechanisms to repeatedly check the status of
the bit at runtime (well, the alternatives patching helps here
but it does not provide compile-time optimization).
Adding a feature to disabled-features.h allows the bit to be
checked via a new macro: cpu_feature_enabled(). Note that
for features in DISABLED_MASK, checks with this macro have
all of the benefits of an #ifdef. Before, we would have done
this in a header:
#ifdef CONFIG_X86_INTEL_MPX
#define cpu_has_mpx cpu_has(X86_FEATURE_MPX)
#else
#define cpu_has_mpx 0
#endif
and this in the code:
if (cpu_has_mpx)
do_some_mpx_thing();
Now, just add your feature to DISABLED_MASK and you can do this
everywhere, and get the same benefits you would have from
#ifdefs:
if (cpu_feature_enabled(X86_FEATURE_MPX))
do_some_mpx_thing();
We need a new function and *not* a modification to cpu_has()
because there are cases where we actually need to check the CPU
itself, despite what features the kernel supports. The best
example of this is a hypervisor which has no control over what
features its guests are using and where the guest does not depend
on the host for support.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: http://lkml.kernel.org/r/20140911211513.9E35E931@viggo.jf.intel.com
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2014-09-12 05:15:13 +08:00
|
|
|
/*
|
|
|
|
|
* This macro is for detection of features which need kernel
|
|
|
|
|
* infrastructure to be used. It may *not* directly test the CPU
|
|
|
|
|
* itself. Use the cpu_has() family if you want true runtime
|
|
|
|
|
* testing of CPU features, like in hypervisor code where you are
|
|
|
|
|
* supporting a possible guest feature where host support for it
|
|
|
|
|
* is not relevant.
|
|
|
|
|
*/
|
|
|
|
|
#define cpu_feature_enabled(bit) \
|
2016-02-16 07:19:18 +08:00
|
|
|
(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit))
|
x86: Introduce disabled-features
I believe the REQUIRED_MASK aproach was taken so that it was
easier to consult in assembly (arch/x86/kernel/verify_cpu.S).
DISABLED_MASK does not have the same restriction, but I
implemented it the same way for consistency.
We have a REQUIRED_MASK... which does two things:
1. Keeps a list of cpuid bits to check in very early boot and
refuse to boot if those are not present.
2. Consulted during cpu_has() checks, which allows us to
optimize out things at compile-time. In other words, if we
*KNOW* we will not boot with the feature off, then we can
safely assume that it will be present forever.
But, we don't have a similar mechanism for CPU features which
may be present but that we know we will not use. We simply
use our existing mechanisms to repeatedly check the status of
the bit at runtime (well, the alternatives patching helps here
but it does not provide compile-time optimization).
Adding a feature to disabled-features.h allows the bit to be
checked via a new macro: cpu_feature_enabled(). Note that
for features in DISABLED_MASK, checks with this macro have
all of the benefits of an #ifdef. Before, we would have done
this in a header:
#ifdef CONFIG_X86_INTEL_MPX
#define cpu_has_mpx cpu_has(X86_FEATURE_MPX)
#else
#define cpu_has_mpx 0
#endif
and this in the code:
if (cpu_has_mpx)
do_some_mpx_thing();
Now, just add your feature to DISABLED_MASK and you can do this
everywhere, and get the same benefits you would have from
#ifdefs:
if (cpu_feature_enabled(X86_FEATURE_MPX))
do_some_mpx_thing();
We need a new function and *not* a modification to cpu_has()
because there are cases where we actually need to check the CPU
itself, despite what features the kernel supports. The best
example of this is a hypervisor which has no control over what
features its guests are using and where the guest does not depend
on the host for support.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: http://lkml.kernel.org/r/20140911211513.9E35E931@viggo.jf.intel.com
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2014-09-12 05:15:13 +08:00
|
|
|
|
2008-01-30 20:30:07 +08:00
|
|
|
#define boot_cpu_has(bit) cpu_has(&boot_cpu_data, bit)
|
|
|
|
|
|
2008-01-30 20:30:55 +08:00
|
|
|
#define set_cpu_cap(c, bit) set_bit(bit, (unsigned long *)((c)->x86_capability))
|
|
|
|
|
#define clear_cpu_cap(c, bit) clear_bit(bit, (unsigned long *)((c)->x86_capability))
|
2008-01-30 20:33:20 +08:00
|
|
|
#define setup_clear_cpu_cap(bit) do { \
|
|
|
|
|
clear_cpu_cap(&boot_cpu_data, bit); \
|
2009-05-10 14:47:42 +08:00
|
|
|
set_bit(bit, (unsigned long *)cpu_caps_cleared); \
|
2008-01-30 20:33:20 +08:00
|
|
|
} while (0)
|
2008-01-30 20:33:20 +08:00
|
|
|
#define setup_force_cpu_cap(bit) do { \
|
|
|
|
|
set_cpu_cap(&boot_cpu_data, bit); \
|
2009-05-10 14:47:42 +08:00
|
|
|
set_bit(bit, (unsigned long *)cpu_caps_set); \
|
2008-01-30 20:33:20 +08:00
|
|
|
} while (0)
|
2008-01-30 20:30:55 +08:00
|
|
|
|
2016-01-27 16:43:25 +08:00
|
|
|
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_X86_FAST_FEATURE_TESTS)
|
2010-05-12 08:47:07 +08:00
|
|
|
/*
|
|
|
|
|
* Static testing of CPU features. Used the same as boot_cpu_has().
|
2016-01-27 16:43:25 +08:00
|
|
|
* These will statically patch the target code for additional
|
|
|
|
|
* performance.
|
2010-05-12 08:47:07 +08:00
|
|
|
*/
|
2016-01-27 05:12:05 +08:00
|
|
|
static __always_inline __pure bool _static_cpu_has(u16 bit)
|
2013-06-09 18:07:33 +08:00
|
|
|
{
|
2016-01-27 16:45:25 +08:00
|
|
|
asm_volatile_goto("1: jmp 6f\n"
|
2013-06-09 18:07:33 +08:00
|
|
|
"2:\n"
|
2014-12-27 17:41:52 +08:00
|
|
|
".skip -(((5f-4f) - (2b-1b)) > 0) * "
|
|
|
|
|
"((5f-4f) - (2b-1b)),0x90\n"
|
|
|
|
|
"3:\n"
|
2013-06-09 18:07:33 +08:00
|
|
|
".section .altinstructions,\"a\"\n"
|
|
|
|
|
" .long 1b - .\n" /* src offset */
|
2014-12-27 17:41:52 +08:00
|
|
|
" .long 4f - .\n" /* repl offset */
|
2013-06-09 18:07:33 +08:00
|
|
|
" .word %P1\n" /* always replace */
|
2014-12-27 17:41:52 +08:00
|
|
|
" .byte 3b - 1b\n" /* src len */
|
|
|
|
|
" .byte 5f - 4f\n" /* repl len */
|
|
|
|
|
" .byte 3b - 2b\n" /* pad len */
|
2013-06-09 18:07:33 +08:00
|
|
|
".previous\n"
|
|
|
|
|
".section .altinstr_replacement,\"ax\"\n"
|
x86/alternatives: Make JMPs more robust
Up until now we had to pay attention to relative JMPs in alternatives
about how their relative offset gets computed so that the jump target
is still correct. Or, as it is the case for near CALLs (opcode e8), we
still have to go and readjust the offset at patching time.
What is more, the static_cpu_has_safe() facility had to forcefully
generate 5-byte JMPs since we couldn't rely on the compiler to generate
properly sized ones so we had to force the longest ones. Worse than
that, sometimes it would generate a replacement JMP which is longer than
the original one, thus overwriting the beginning of the next instruction
at patching time.
So, in order to alleviate all that and make using JMPs more
straight-forward we go and pad the original instruction in an
alternative block with NOPs at build time, should the replacement(s) be
longer. This way, alternatives users shouldn't pay special attention
so that original and replacement instruction sizes are fine but the
assembler would simply add padding where needed and not do anything
otherwise.
As a second aspect, we go and recompute JMPs at patching time so that we
can try to make 5-byte JMPs into two-byte ones if possible. If not, we
still have to recompute the offsets as the replacement JMP gets put far
away in the .altinstr_replacement section leading to a wrong offset if
copied verbatim.
For example, on a locally generated kernel image
old insn VA: 0xffffffff810014bd, CPU feat: X86_FEATURE_ALWAYS, size: 2
__switch_to:
ffffffff810014bd: eb 21 jmp ffffffff810014e0
repl insn: size: 5
ffffffff81d0b23c: e9 b1 62 2f ff jmpq ffffffff810014f2
gets corrected to a 2-byte JMP:
apply_alternatives: feat: 3*32+21, old: (ffffffff810014bd, len: 2), repl: (ffffffff81d0b23c, len: 5)
alt_insn: e9 b1 62 2f ff
recompute_jumps: next_rip: ffffffff81d0b241, tgt_rip: ffffffff810014f2, new_displ: 0x00000033, ret len: 2
converted to: eb 33 90 90 90
and a 5-byte JMP:
old insn VA: 0xffffffff81001516, CPU feat: X86_FEATURE_ALWAYS, size: 2
__switch_to:
ffffffff81001516: eb 30 jmp ffffffff81001548
repl insn: size: 5
ffffffff81d0b241: e9 10 63 2f ff jmpq ffffffff81001556
gets shortened into a two-byte one:
apply_alternatives: feat: 3*32+21, old: (ffffffff81001516, len: 2), repl: (ffffffff81d0b241, len: 5)
alt_insn: e9 10 63 2f ff
recompute_jumps: next_rip: ffffffff81d0b246, tgt_rip: ffffffff81001556, new_displ: 0x0000003e, ret len: 2
converted to: eb 3e 90 90 90
... and so on.
This leads to a net win of around
40ish replacements * 3 bytes savings =~ 120 bytes of I$
on an AMD guest which means some savings of precious instruction cache
bandwidth. The padding to the shorter 2-byte JMPs are single-byte NOPs
which on smart microarchitectures means discarding NOPs at decode time
and thus freeing up execution bandwidth.
Signed-off-by: Borislav Petkov <bp@suse.de>
2015-01-05 20:48:41 +08:00
|
|
|
"4: jmp %l[t_no]\n"
|
2014-12-27 17:41:52 +08:00
|
|
|
"5:\n"
|
2013-06-09 18:07:33 +08:00
|
|
|
".previous\n"
|
|
|
|
|
".section .altinstructions,\"a\"\n"
|
|
|
|
|
" .long 1b - .\n" /* src offset */
|
|
|
|
|
" .long 0\n" /* no replacement */
|
|
|
|
|
" .word %P0\n" /* feature bit */
|
2014-12-27 17:41:52 +08:00
|
|
|
" .byte 3b - 1b\n" /* src len */
|
2013-06-09 18:07:33 +08:00
|
|
|
" .byte 0\n" /* repl len */
|
2014-12-27 17:41:52 +08:00
|
|
|
" .byte 0\n" /* pad len */
|
2013-06-09 18:07:33 +08:00
|
|
|
".previous\n"
|
2016-01-27 16:45:25 +08:00
|
|
|
".section .altinstr_aux,\"ax\"\n"
|
|
|
|
|
"6:\n"
|
|
|
|
|
" testb %[bitnum],%[cap_byte]\n"
|
|
|
|
|
" jnz %l[t_yes]\n"
|
|
|
|
|
" jmp %l[t_no]\n"
|
|
|
|
|
".previous\n"
|
|
|
|
|
: : "i" (bit), "i" (X86_FEATURE_ALWAYS),
|
|
|
|
|
[bitnum] "i" (1 << (bit & 7)),
|
|
|
|
|
[cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3])
|
|
|
|
|
: : t_yes, t_no);
|
|
|
|
|
t_yes:
|
2013-06-09 18:07:33 +08:00
|
|
|
return true;
|
|
|
|
|
t_no:
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
2016-01-27 05:12:05 +08:00
|
|
|
#define static_cpu_has(bit) \
|
2013-06-09 18:07:33 +08:00
|
|
|
( \
|
|
|
|
|
__builtin_constant_p(boot_cpu_has(bit)) ? \
|
|
|
|
|
boot_cpu_has(bit) : \
|
2016-01-27 05:12:05 +08:00
|
|
|
_static_cpu_has(bit) \
|
2013-06-09 18:07:33 +08:00
|
|
|
)
|
2010-05-28 03:02:00 +08:00
|
|
|
#else
|
|
|
|
|
/*
|
2016-01-27 16:43:25 +08:00
|
|
|
* Fall back to dynamic for gcc versions which don't support asm goto. Should be
|
|
|
|
|
* a minority now anyway.
|
2010-05-28 03:02:00 +08:00
|
|
|
*/
|
2013-06-09 18:07:33 +08:00
|
|
|
#define static_cpu_has(bit) boot_cpu_has(bit)
|
2010-05-28 03:02:00 +08:00
|
|
|
#endif
|
2010-05-12 08:47:07 +08:00
|
|
|
|
2014-06-18 06:06:23 +08:00
|
|
|
#define cpu_has_bug(c, bit) cpu_has(c, (bit))
|
|
|
|
|
#define set_cpu_bug(c, bit) set_cpu_cap(c, (bit))
|
|
|
|
|
#define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit))
|
2013-03-20 22:07:23 +08:00
|
|
|
|
2014-06-18 06:06:23 +08:00
|
|
|
#define static_cpu_has_bug(bit) static_cpu_has((bit))
|
|
|
|
|
#define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit))
|
2013-03-20 22:07:23 +08:00
|
|
|
|
2014-06-18 06:06:23 +08:00
|
|
|
#define MAX_CPU_FEATURES (NCAPINTS * 32)
|
|
|
|
|
#define cpu_have_feature boot_cpu_has
|
2014-02-08 20:34:10 +08:00
|
|
|
|
2014-06-18 06:06:23 +08:00
|
|
|
#define CPU_FEATURE_TYPEFMT "x86,ven%04Xfam%04Xmod%04X"
|
|
|
|
|
#define CPU_FEATURE_TYPEVAL boot_cpu_data.x86_vendor, boot_cpu_data.x86, \
|
|
|
|
|
boot_cpu_data.x86_model
|
2014-02-08 20:34:10 +08:00
|
|
|
|
2008-02-04 23:48:00 +08:00
|
|
|
#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
|
2008-10-23 13:26:29 +08:00
|
|
|
#endif /* _ASM_X86_CPUFEATURE_H */
|