Merge branch 'speck-v20' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Merge speculative store buffer bypass fixes from Thomas Gleixner:
- rework of the SPEC_CTRL MSR management to accomodate the new fancy
SSBD (Speculative Store Bypass Disable) bit handling.
- the CPU bug and sysfs infrastructure for the exciting new Speculative
Store Bypass 'feature'.
- support for disabling SSB via LS_CFG MSR on AMD CPUs including
Hyperthread synchronization on ZEN.
- PRCTL support for dynamic runtime control of SSB
- SECCOMP integration to automatically disable SSB for sandboxed
processes with a filter flag for opt-out.
- KVM integration to allow guests fiddling with SSBD including the new
software MSR VIRT_SPEC_CTRL to handle the LS_CFG based oddities on
AMD.
- BPF protection against SSB
.. this is just the core and x86 side, other architecture support will
come separately.
* 'speck-v20' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (49 commits)
bpf: Prevent memory disambiguation attack
x86/bugs: Rename SSBD_NO to SSB_NO
KVM: SVM: Implement VIRT_SPEC_CTRL support for SSBD
x86/speculation, KVM: Implement support for VIRT_SPEC_CTRL/LS_CFG
x86/bugs: Rework spec_ctrl base and mask logic
x86/bugs: Remove x86_spec_ctrl_set()
x86/bugs: Expose x86_spec_ctrl_base directly
x86/bugs: Unify x86_spec_ctrl_{set_guest,restore_host}
x86/speculation: Rework speculative_store_bypass_update()
x86/speculation: Add virtualized speculative store bypass disable support
x86/bugs, KVM: Extend speculation control for VIRT_SPEC_CTRL
x86/speculation: Handle HT correctly on AMD
x86/cpufeatures: Add FEATURE_ZEN
x86/cpufeatures: Disentangle SSBD enumeration
x86/cpufeatures: Disentangle MSR_SPEC_CTRL enumeration from IBRS
x86/speculation: Use synthetic bits for IBRS/IBPB/STIBP
KVM: SVM: Move spec control call after restore of GS
x86/cpu: Make alternative_msr_write work for 32-bit code
x86/bugs: Fix the parameters alignment and missing void
x86/bugs: Make cpu_show_common() static
...
This commit is contained in:
Linus Torvalds
commit
3b78ce4a34
|
|
@ -478,6 +478,7 @@ What: /sys/devices/system/cpu/vulnerabilities
|
|||
/sys/devices/system/cpu/vulnerabilities/meltdown
|
||||
/sys/devices/system/cpu/vulnerabilities/spectre_v1
|
||||
/sys/devices/system/cpu/vulnerabilities/spectre_v2
|
||||
/sys/devices/system/cpu/vulnerabilities/spec_store_bypass
|
||||
Date: January 2018
|
||||
Contact: Linux kernel mailing list <linux-kernel@vger.kernel.org>
|
||||
Description: Information about CPU vulnerabilities
|
||||
|
|
|
|||
|
|
@ -2680,6 +2680,9 @@
|
|||
allow data leaks with this option, which is equivalent
|
||||
to spectre_v2=off.
|
||||
|
||||
nospec_store_bypass_disable
|
||||
[HW] Disable all mitigations for the Speculative Store Bypass vulnerability
|
||||
|
||||
noxsave [BUGS=X86] Disables x86 extended register state save
|
||||
and restore using xsave. The kernel will fallback to
|
||||
enabling legacy floating-point and sse state.
|
||||
|
|
@ -4025,6 +4028,48 @@
|
|||
Not specifying this option is equivalent to
|
||||
spectre_v2=auto.
|
||||
|
||||
spec_store_bypass_disable=
|
||||
[HW] Control Speculative Store Bypass (SSB) Disable mitigation
|
||||
(Speculative Store Bypass vulnerability)
|
||||
|
||||
Certain CPUs are vulnerable to an exploit against a
|
||||
a common industry wide performance optimization known
|
||||
as "Speculative Store Bypass" in which recent stores
|
||||
to the same memory location may not be observed by
|
||||
later loads during speculative execution. The idea
|
||||
is that such stores are unlikely and that they can
|
||||
be detected prior to instruction retirement at the
|
||||
end of a particular speculation execution window.
|
||||
|
||||
In vulnerable processors, the speculatively forwarded
|
||||
store can be used in a cache side channel attack, for
|
||||
example to read memory to which the attacker does not
|
||||
directly have access (e.g. inside sandboxed code).
|
||||
|
||||
This parameter controls whether the Speculative Store
|
||||
Bypass optimization is used.
|
||||
|
||||
on - Unconditionally disable Speculative Store Bypass
|
||||
off - Unconditionally enable Speculative Store Bypass
|
||||
auto - Kernel detects whether the CPU model contains an
|
||||
implementation of Speculative Store Bypass and
|
||||
picks the most appropriate mitigation. If the
|
||||
CPU is not vulnerable, "off" is selected. If the
|
||||
CPU is vulnerable the default mitigation is
|
||||
architecture and Kconfig dependent. See below.
|
||||
prctl - Control Speculative Store Bypass per thread
|
||||
via prctl. Speculative Store Bypass is enabled
|
||||
for a process by default. The state of the control
|
||||
is inherited on fork.
|
||||
seccomp - Same as "prctl" above, but all seccomp threads
|
||||
will disable SSB unless they explicitly opt out.
|
||||
|
||||
Not specifying this option is equivalent to
|
||||
spec_store_bypass_disable=auto.
|
||||
|
||||
Default mitigations:
|
||||
X86: If CONFIG_SECCOMP=y "seccomp", otherwise "prctl"
|
||||
|
||||
spia_io_base= [HW,MTD]
|
||||
spia_fio_base=
|
||||
spia_pedr=
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ place where this information is gathered.
|
|||
no_new_privs
|
||||
seccomp_filter
|
||||
unshare
|
||||
spec_ctrl
|
||||
|
||||
.. only:: subproject and html
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
===================
|
||||
Speculation Control
|
||||
===================
|
||||
|
||||
Quite some CPUs have speculation-related misfeatures which are in
|
||||
fact vulnerabilities causing data leaks in various forms even across
|
||||
privilege domains.
|
||||
|
||||
The kernel provides mitigation for such vulnerabilities in various
|
||||
forms. Some of these mitigations are compile-time configurable and some
|
||||
can be supplied on the kernel command line.
|
||||
|
||||
There is also a class of mitigations which are very expensive, but they can
|
||||
be restricted to a certain set of processes or tasks in controlled
|
||||
environments. The mechanism to control these mitigations is via
|
||||
:manpage:`prctl(2)`.
|
||||
|
||||
There are two prctl options which are related to this:
|
||||
|
||||
* PR_GET_SPECULATION_CTRL
|
||||
|
||||
* PR_SET_SPECULATION_CTRL
|
||||
|
||||
PR_GET_SPECULATION_CTRL
|
||||
-----------------------
|
||||
|
||||
PR_GET_SPECULATION_CTRL returns the state of the speculation misfeature
|
||||
which is selected with arg2 of prctl(2). The return value uses bits 0-3 with
|
||||
the following meaning:
|
||||
|
||||
==== ===================== ===================================================
|
||||
Bit Define Description
|
||||
==== ===================== ===================================================
|
||||
0 PR_SPEC_PRCTL Mitigation can be controlled per task by
|
||||
PR_SET_SPECULATION_CTRL.
|
||||
1 PR_SPEC_ENABLE The speculation feature is enabled, mitigation is
|
||||
disabled.
|
||||
2 PR_SPEC_DISABLE The speculation feature is disabled, mitigation is
|
||||
enabled.
|
||||
3 PR_SPEC_FORCE_DISABLE Same as PR_SPEC_DISABLE, but cannot be undone. A
|
||||
subsequent prctl(..., PR_SPEC_ENABLE) will fail.
|
||||
==== ===================== ===================================================
|
||||
|
||||
If all bits are 0 the CPU is not affected by the speculation misfeature.
|
||||
|
||||
If PR_SPEC_PRCTL is set, then the per-task control of the mitigation is
|
||||
available. If not set, prctl(PR_SET_SPECULATION_CTRL) for the speculation
|
||||
misfeature will fail.
|
||||
|
||||
PR_SET_SPECULATION_CTRL
|
||||
-----------------------
|
||||
|
||||
PR_SET_SPECULATION_CTRL allows to control the speculation misfeature, which
|
||||
is selected by arg2 of :manpage:`prctl(2)` per task. arg3 is used to hand
|
||||
in the control value, i.e. either PR_SPEC_ENABLE or PR_SPEC_DISABLE or
|
||||
PR_SPEC_FORCE_DISABLE.
|
||||
|
||||
Common error codes
|
||||
------------------
|
||||
======= =================================================================
|
||||
Value Meaning
|
||||
======= =================================================================
|
||||
EINVAL The prctl is not implemented by the architecture or unused
|
||||
prctl(2) arguments are not 0.
|
||||
|
||||
ENODEV arg2 is selecting a not supported speculation misfeature.
|
||||
======= =================================================================
|
||||
|
||||
PR_SET_SPECULATION_CTRL error codes
|
||||
-----------------------------------
|
||||
======= =================================================================
|
||||
Value Meaning
|
||||
======= =================================================================
|
||||
0 Success
|
||||
|
||||
ERANGE arg3 is incorrect, i.e. it's neither PR_SPEC_ENABLE nor
|
||||
PR_SPEC_DISABLE nor PR_SPEC_FORCE_DISABLE.
|
||||
|
||||
ENXIO Control of the selected speculation misfeature is not possible.
|
||||
See PR_GET_SPECULATION_CTRL.
|
||||
|
||||
EPERM Speculation was disabled with PR_SPEC_FORCE_DISABLE and caller
|
||||
tried to enable it again.
|
||||
======= =================================================================
|
||||
|
||||
Speculation misfeature controls
|
||||
-------------------------------
|
||||
- PR_SPEC_STORE_BYPASS: Speculative Store Bypass
|
||||
|
||||
Invocations:
|
||||
* prctl(PR_GET_SPECULATION_CTRL, PR_SPEC_STORE_BYPASS, 0, 0, 0);
|
||||
* prctl(PR_SET_SPECULATION_CTRL, PR_SPEC_STORE_BYPASS, PR_SPEC_ENABLE, 0, 0);
|
||||
* prctl(PR_SET_SPECULATION_CTRL, PR_SPEC_STORE_BYPASS, PR_SPEC_DISABLE, 0, 0);
|
||||
* prctl(PR_SET_SPECULATION_CTRL, PR_SPEC_STORE_BYPASS, PR_SPEC_FORCE_DISABLE, 0, 0);
|
||||
|
|
@ -198,7 +198,6 @@
|
|||
#define X86_FEATURE_CAT_L2 ( 7*32+ 5) /* Cache Allocation Technology L2 */
|
||||
#define X86_FEATURE_CDP_L3 ( 7*32+ 6) /* Code and Data Prioritization L3 */
|
||||
#define X86_FEATURE_INVPCID_SINGLE ( 7*32+ 7) /* Effectively INVPCID && CR4.PCIDE=1 */
|
||||
|
||||
#define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */
|
||||
#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */
|
||||
#define X86_FEATURE_SME ( 7*32+10) /* AMD Secure Memory Encryption */
|
||||
|
|
@ -207,13 +206,19 @@
|
|||
#define X86_FEATURE_RETPOLINE_AMD ( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */
|
||||
#define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */
|
||||
#define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */
|
||||
|
||||
#define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */
|
||||
#define X86_FEATURE_SSBD ( 7*32+17) /* Speculative Store Bypass Disable */
|
||||
#define X86_FEATURE_MBA ( 7*32+18) /* Memory Bandwidth Allocation */
|
||||
#define X86_FEATURE_RSB_CTXSW ( 7*32+19) /* "" Fill RSB on context switches */
|
||||
#define X86_FEATURE_SEV ( 7*32+20) /* AMD Secure Encrypted Virtualization */
|
||||
|
||||
#define X86_FEATURE_USE_IBPB ( 7*32+21) /* "" Indirect Branch Prediction Barrier enabled */
|
||||
#define X86_FEATURE_USE_IBRS_FW ( 7*32+22) /* "" Use IBRS during runtime firmware calls */
|
||||
#define X86_FEATURE_SPEC_STORE_BYPASS_DISABLE ( 7*32+23) /* "" Disable Speculative Store Bypass. */
|
||||
#define X86_FEATURE_LS_CFG_SSBD ( 7*32+24) /* "" AMD SSBD implementation via LS_CFG MSR */
|
||||
#define X86_FEATURE_IBRS ( 7*32+25) /* Indirect Branch Restricted Speculation */
|
||||
#define X86_FEATURE_IBPB ( 7*32+26) /* Indirect Branch Prediction Barrier */
|
||||
#define X86_FEATURE_STIBP ( 7*32+27) /* Single Thread Indirect Branch Predictors */
|
||||
#define X86_FEATURE_ZEN ( 7*32+28) /* "" CPU is AMD family 0x17 (Zen) */
|
||||
|
||||
/* Virtualization flags: Linux defined, word 8 */
|
||||
#define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
|
||||
|
|
@ -274,9 +279,10 @@
|
|||
#define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */
|
||||
#define X86_FEATURE_IRPERF (13*32+ 1) /* Instructions Retired Count */
|
||||
#define X86_FEATURE_XSAVEERPTR (13*32+ 2) /* Always save/restore FP error pointers */
|
||||
#define X86_FEATURE_IBPB (13*32+12) /* Indirect Branch Prediction Barrier */
|
||||
#define X86_FEATURE_IBRS (13*32+14) /* Indirect Branch Restricted Speculation */
|
||||
#define X86_FEATURE_STIBP (13*32+15) /* Single Thread Indirect Branch Predictors */
|
||||
#define X86_FEATURE_AMD_IBPB (13*32+12) /* "" Indirect Branch Prediction Barrier */
|
||||
#define X86_FEATURE_AMD_IBRS (13*32+14) /* "" Indirect Branch Restricted Speculation */
|
||||
#define X86_FEATURE_AMD_STIBP (13*32+15) /* "" Single Thread Indirect Branch Predictors */
|
||||
#define X86_FEATURE_VIRT_SSBD (13*32+25) /* Virtualized Speculative Store Bypass Disable */
|
||||
|
||||
/* Thermal and Power Management Leaf, CPUID level 0x00000006 (EAX), word 14 */
|
||||
#define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */
|
||||
|
|
@ -334,6 +340,7 @@
|
|||
#define X86_FEATURE_SPEC_CTRL (18*32+26) /* "" Speculation Control (IBRS + IBPB) */
|
||||
#define X86_FEATURE_INTEL_STIBP (18*32+27) /* "" Single Thread Indirect Branch Predictors */
|
||||
#define X86_FEATURE_ARCH_CAPABILITIES (18*32+29) /* IA32_ARCH_CAPABILITIES MSR (Intel) */
|
||||
#define X86_FEATURE_SPEC_CTRL_SSBD (18*32+31) /* "" Speculative Store Bypass Disable */
|
||||
|
||||
/*
|
||||
* BUG word(s)
|
||||
|
|
@ -363,5 +370,6 @@
|
|||
#define X86_BUG_CPU_MELTDOWN X86_BUG(14) /* CPU is affected by meltdown attack and needs kernel page table isolation */
|
||||
#define X86_BUG_SPECTRE_V1 X86_BUG(15) /* CPU is affected by Spectre variant 1 attack with conditional branches */
|
||||
#define X86_BUG_SPECTRE_V2 X86_BUG(16) /* CPU is affected by Spectre variant 2 attack with indirect branches */
|
||||
#define X86_BUG_SPEC_STORE_BYPASS X86_BUG(17) /* CPU is affected by speculative store bypass attack */
|
||||
|
||||
#endif /* _ASM_X86_CPUFEATURES_H */
|
||||
|
|
|
|||
|
|
@ -924,7 +924,7 @@ struct kvm_x86_ops {
|
|||
int (*hardware_setup)(void); /* __init */
|
||||
void (*hardware_unsetup)(void); /* __exit */
|
||||
bool (*cpu_has_accelerated_tpr)(void);
|
||||
bool (*cpu_has_high_real_mode_segbase)(void);
|
||||
bool (*has_emulated_msr)(int index);
|
||||
void (*cpuid_update)(struct kvm_vcpu *vcpu);
|
||||
|
||||
struct kvm *(*vm_alloc)(void);
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@
|
|||
#define MSR_IA32_SPEC_CTRL 0x00000048 /* Speculation Control */
|
||||
#define SPEC_CTRL_IBRS (1 << 0) /* Indirect Branch Restricted Speculation */
|
||||
#define SPEC_CTRL_STIBP (1 << 1) /* Single Thread Indirect Branch Predictors */
|
||||
#define SPEC_CTRL_SSBD_SHIFT 2 /* Speculative Store Bypass Disable bit */
|
||||
#define SPEC_CTRL_SSBD (1 << SPEC_CTRL_SSBD_SHIFT) /* Speculative Store Bypass Disable */
|
||||
|
||||
#define MSR_IA32_PRED_CMD 0x00000049 /* Prediction Command */
|
||||
#define PRED_CMD_IBPB (1 << 0) /* Indirect Branch Prediction Barrier */
|
||||
|
|
@ -68,6 +70,11 @@
|
|||
#define MSR_IA32_ARCH_CAPABILITIES 0x0000010a
|
||||
#define ARCH_CAP_RDCL_NO (1 << 0) /* Not susceptible to Meltdown */
|
||||
#define ARCH_CAP_IBRS_ALL (1 << 1) /* Enhanced IBRS support */
|
||||
#define ARCH_CAP_SSB_NO (1 << 4) /*
|
||||
* Not susceptible to Speculative Store Bypass
|
||||
* attack, so no Speculative Store Bypass
|
||||
* control required.
|
||||
*/
|
||||
|
||||
#define MSR_IA32_BBL_CR_CTL 0x00000119
|
||||
#define MSR_IA32_BBL_CR_CTL3 0x0000011e
|
||||
|
|
@ -340,6 +347,8 @@
|
|||
#define MSR_AMD64_SEV_ENABLED_BIT 0
|
||||
#define MSR_AMD64_SEV_ENABLED BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT)
|
||||
|
||||
#define MSR_AMD64_VIRT_SPEC_CTRL 0xc001011f
|
||||
|
||||
/* Fam 17h MSRs */
|
||||
#define MSR_F17H_IRPERF 0xc00000e9
|
||||
|
||||
|
|
|
|||
|
|
@ -217,6 +217,14 @@ enum spectre_v2_mitigation {
|
|||
SPECTRE_V2_IBRS,
|
||||
};
|
||||
|
||||
/* The Speculative Store Bypass disable variants */
|
||||
enum ssb_mitigation {
|
||||
SPEC_STORE_BYPASS_NONE,
|
||||
SPEC_STORE_BYPASS_DISABLE,
|
||||
SPEC_STORE_BYPASS_PRCTL,
|
||||
SPEC_STORE_BYPASS_SECCOMP,
|
||||
};
|
||||
|
||||
extern char __indirect_thunk_start[];
|
||||
extern char __indirect_thunk_end[];
|
||||
|
||||
|
|
@ -241,22 +249,27 @@ static inline void vmexit_fill_RSB(void)
|
|||
#endif
|
||||
}
|
||||
|
||||
#define alternative_msr_write(_msr, _val, _feature) \
|
||||
asm volatile(ALTERNATIVE("", \
|
||||
"movl %[msr], %%ecx\n\t" \
|
||||
"movl %[val], %%eax\n\t" \
|
||||
"movl $0, %%edx\n\t" \
|
||||
"wrmsr", \
|
||||
_feature) \
|
||||
: : [msr] "i" (_msr), [val] "i" (_val) \
|
||||
: "eax", "ecx", "edx", "memory")
|
||||
static __always_inline
|
||||
void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
|
||||
{
|
||||
asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
|
||||
: : "c" (msr),
|
||||
"a" ((u32)val),
|
||||
"d" ((u32)(val >> 32)),
|
||||
[feature] "i" (feature)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
static inline void indirect_branch_prediction_barrier(void)
|
||||
{
|
||||
alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB,
|
||||
X86_FEATURE_USE_IBPB);
|
||||
u64 val = PRED_CMD_IBPB;
|
||||
|
||||
alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB);
|
||||
}
|
||||
|
||||
/* The Intel SPEC CTRL MSR base value cache */
|
||||
extern u64 x86_spec_ctrl_base;
|
||||
|
||||
/*
|
||||
* With retpoline, we must use IBRS to restrict branch prediction
|
||||
* before calling into firmware.
|
||||
|
|
@ -265,14 +278,18 @@ static inline void indirect_branch_prediction_barrier(void)
|
|||
*/
|
||||
#define firmware_restrict_branch_speculation_start() \
|
||||
do { \
|
||||
u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \
|
||||
\
|
||||
preempt_disable(); \
|
||||
alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS, \
|
||||
alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \
|
||||
X86_FEATURE_USE_IBRS_FW); \
|
||||
} while (0)
|
||||
|
||||
#define firmware_restrict_branch_speculation_end() \
|
||||
do { \
|
||||
alternative_msr_write(MSR_IA32_SPEC_CTRL, 0, \
|
||||
u64 val = x86_spec_ctrl_base; \
|
||||
\
|
||||
alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \
|
||||
X86_FEATURE_USE_IBRS_FW); \
|
||||
preempt_enable(); \
|
||||
} while (0)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef _ASM_X86_SPECCTRL_H_
|
||||
#define _ASM_X86_SPECCTRL_H_
|
||||
|
||||
#include <linux/thread_info.h>
|
||||
#include <asm/nospec-branch.h>
|
||||
|
||||
/*
|
||||
* On VMENTER we must preserve whatever view of the SPEC_CTRL MSR
|
||||
* the guest has, while on VMEXIT we restore the host view. This
|
||||
* would be easier if SPEC_CTRL were architecturally maskable or
|
||||
* shadowable for guests but this is not (currently) the case.
|
||||
* Takes the guest view of SPEC_CTRL MSR as a parameter and also
|
||||
* the guest's version of VIRT_SPEC_CTRL, if emulated.
|
||||
*/
|
||||
extern void x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool guest);
|
||||
|
||||
/**
|
||||
* x86_spec_ctrl_set_guest - Set speculation control registers for the guest
|
||||
* @guest_spec_ctrl: The guest content of MSR_SPEC_CTRL
|
||||
* @guest_virt_spec_ctrl: The guest controlled bits of MSR_VIRT_SPEC_CTRL
|
||||
* (may get translated to MSR_AMD64_LS_CFG bits)
|
||||
*
|
||||
* Avoids writing to the MSR if the content/bits are the same
|
||||
*/
|
||||
static inline
|
||||
void x86_spec_ctrl_set_guest(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl)
|
||||
{
|
||||
x86_virt_spec_ctrl(guest_spec_ctrl, guest_virt_spec_ctrl, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* x86_spec_ctrl_restore_host - Restore host speculation control registers
|
||||
* @guest_spec_ctrl: The guest content of MSR_SPEC_CTRL
|
||||
* @guest_virt_spec_ctrl: The guest controlled bits of MSR_VIRT_SPEC_CTRL
|
||||
* (may get translated to MSR_AMD64_LS_CFG bits)
|
||||
*
|
||||
* Avoids writing to the MSR if the content/bits are the same
|
||||
*/
|
||||
static inline
|
||||
void x86_spec_ctrl_restore_host(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl)
|
||||
{
|
||||
x86_virt_spec_ctrl(guest_spec_ctrl, guest_virt_spec_ctrl, false);
|
||||
}
|
||||
|
||||
/* AMD specific Speculative Store Bypass MSR data */
|
||||
extern u64 x86_amd_ls_cfg_base;
|
||||
extern u64 x86_amd_ls_cfg_ssbd_mask;
|
||||
|
||||
static inline u64 ssbd_tif_to_spec_ctrl(u64 tifn)
|
||||
{
|
||||
BUILD_BUG_ON(TIF_SSBD < SPEC_CTRL_SSBD_SHIFT);
|
||||
return (tifn & _TIF_SSBD) >> (TIF_SSBD - SPEC_CTRL_SSBD_SHIFT);
|
||||
}
|
||||
|
||||
static inline unsigned long ssbd_spec_ctrl_to_tif(u64 spec_ctrl)
|
||||
{
|
||||
BUILD_BUG_ON(TIF_SSBD < SPEC_CTRL_SSBD_SHIFT);
|
||||
return (spec_ctrl & SPEC_CTRL_SSBD) << (TIF_SSBD - SPEC_CTRL_SSBD_SHIFT);
|
||||
}
|
||||
|
||||
static inline u64 ssbd_tif_to_amd_ls_cfg(u64 tifn)
|
||||
{
|
||||
return (tifn & _TIF_SSBD) ? x86_amd_ls_cfg_ssbd_mask : 0ULL;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
extern void speculative_store_bypass_ht_init(void);
|
||||
#else
|
||||
static inline void speculative_store_bypass_ht_init(void) { }
|
||||
#endif
|
||||
|
||||
extern void speculative_store_bypass_update(unsigned long tif);
|
||||
|
||||
static inline void speculative_store_bypass_update_current(void)
|
||||
{
|
||||
speculative_store_bypass_update(current_thread_info()->flags);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -79,6 +79,7 @@ struct thread_info {
|
|||
#define TIF_SIGPENDING 2 /* signal pending */
|
||||
#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
|
||||
#define TIF_SINGLESTEP 4 /* reenable singlestep on user return*/
|
||||
#define TIF_SSBD 5 /* Reduced data speculation */
|
||||
#define TIF_SYSCALL_EMU 6 /* syscall emulation active */
|
||||
#define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */
|
||||
#define TIF_SECCOMP 8 /* secure computing */
|
||||
|
|
@ -105,6 +106,7 @@ struct thread_info {
|
|||
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
|
||||
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
|
||||
#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
|
||||
#define _TIF_SSBD (1 << TIF_SSBD)
|
||||
#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
|
||||
#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
|
||||
#define _TIF_SECCOMP (1 << TIF_SECCOMP)
|
||||
|
|
@ -144,7 +146,7 @@ struct thread_info {
|
|||
|
||||
/* flags to check in __switch_to() */
|
||||
#define _TIF_WORK_CTXSW \
|
||||
(_TIF_IO_BITMAP|_TIF_NOCPUID|_TIF_NOTSC|_TIF_BLOCKSTEP)
|
||||
(_TIF_IO_BITMAP|_TIF_NOCPUID|_TIF_NOTSC|_TIF_BLOCKSTEP|_TIF_SSBD)
|
||||
|
||||
#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)
|
||||
#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include <asm/processor.h>
|
||||
#include <asm/apic.h>
|
||||
#include <asm/cpu.h>
|
||||
#include <asm/spec-ctrl.h>
|
||||
#include <asm/smp.h>
|
||||
#include <asm/pci-direct.h>
|
||||
#include <asm/delay.h>
|
||||
|
|
@ -554,6 +555,26 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
|
|||
rdmsrl(MSR_FAM10H_NODE_ID, value);
|
||||
nodes_per_socket = ((value >> 3) & 7) + 1;
|
||||
}
|
||||
|
||||
if (c->x86 >= 0x15 && c->x86 <= 0x17) {
|
||||
unsigned int bit;
|
||||
|
||||
switch (c->x86) {
|
||||
case 0x15: bit = 54; break;
|
||||
case 0x16: bit = 33; break;
|
||||
case 0x17: bit = 10; break;
|
||||
default: return;
|
||||
}
|
||||
/*
|
||||
* Try to cache the base value so further operations can
|
||||
* avoid RMW. If that faults, do not enable SSBD.
|
||||
*/
|
||||
if (!rdmsrl_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) {
|
||||
setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD);
|
||||
setup_force_cpu_cap(X86_FEATURE_SSBD);
|
||||
x86_amd_ls_cfg_ssbd_mask = 1ULL << bit;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
|
||||
|
|
@ -791,6 +812,7 @@ static void init_amd_bd(struct cpuinfo_x86 *c)
|
|||
|
||||
static void init_amd_zn(struct cpuinfo_x86 *c)
|
||||
{
|
||||
set_cpu_cap(c, X86_FEATURE_ZEN);
|
||||
/*
|
||||
* Fix erratum 1076: CPB feature bit not being set in CPUID. It affects
|
||||
* all up to and including B1.
|
||||
|
|
|
|||
|
|
@ -12,8 +12,10 @@
|
|||
#include <linux/utsname.h>
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/nospec.h>
|
||||
#include <linux/prctl.h>
|
||||
|
||||
#include <asm/nospec-branch.h>
|
||||
#include <asm/spec-ctrl.h>
|
||||
#include <asm/cmdline.h>
|
||||
#include <asm/bugs.h>
|
||||
#include <asm/processor.h>
|
||||
|
|
@ -27,6 +29,27 @@
|
|||
#include <asm/intel-family.h>
|
||||
|
||||
static void __init spectre_v2_select_mitigation(void);
|
||||
static void __init ssb_select_mitigation(void);
|
||||
|
||||
/*
|
||||
* Our boot-time value of the SPEC_CTRL MSR. We read it once so that any
|
||||
* writes to SPEC_CTRL contain whatever reserved bits have been set.
|
||||
*/
|
||||
u64 __ro_after_init x86_spec_ctrl_base;
|
||||
EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
|
||||
|
||||
/*
|
||||
* The vendor and possibly platform specific bits which can be modified in
|
||||
* x86_spec_ctrl_base.
|
||||
*/
|
||||
static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
|
||||
|
||||
/*
|
||||
* AMD specific MSR info for Speculative Store Bypass control.
|
||||
* x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
|
||||
*/
|
||||
u64 __ro_after_init x86_amd_ls_cfg_base;
|
||||
u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
|
||||
|
||||
void __init check_bugs(void)
|
||||
{
|
||||
|
|
@ -37,9 +60,27 @@ void __init check_bugs(void)
|
|||
print_cpu_info(&boot_cpu_data);
|
||||
}
|
||||
|
||||
/*
|
||||
* Read the SPEC_CTRL MSR to account for reserved bits which may
|
||||
* have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
|
||||
* init code as it is not enumerated and depends on the family.
|
||||
*/
|
||||
if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
|
||||
rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
|
||||
|
||||
/* Allow STIBP in MSR_SPEC_CTRL if supported */
|
||||
if (boot_cpu_has(X86_FEATURE_STIBP))
|
||||
x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
|
||||
|
||||
/* Select the proper spectre mitigation before patching alternatives */
|
||||
spectre_v2_select_mitigation();
|
||||
|
||||
/*
|
||||
* Select proper mitigation for any exposure to the Speculative Store
|
||||
* Bypass vulnerability.
|
||||
*/
|
||||
ssb_select_mitigation();
|
||||
|
||||
#ifdef CONFIG_X86_32
|
||||
/*
|
||||
* Check whether we are able to run this kernel safely on SMP.
|
||||
|
|
@ -93,7 +134,76 @@ static const char *spectre_v2_strings[] = {
|
|||
#undef pr_fmt
|
||||
#define pr_fmt(fmt) "Spectre V2 : " fmt
|
||||
|
||||
static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
|
||||
static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
|
||||
SPECTRE_V2_NONE;
|
||||
|
||||
void
|
||||
x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
|
||||
{
|
||||
u64 msrval, guestval, hostval = x86_spec_ctrl_base;
|
||||
struct thread_info *ti = current_thread_info();
|
||||
|
||||
/* Is MSR_SPEC_CTRL implemented ? */
|
||||
if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
|
||||
/*
|
||||
* Restrict guest_spec_ctrl to supported values. Clear the
|
||||
* modifiable bits in the host base value and or the
|
||||
* modifiable bits from the guest value.
|
||||
*/
|
||||
guestval = hostval & ~x86_spec_ctrl_mask;
|
||||
guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
|
||||
|
||||
/* SSBD controlled in MSR_SPEC_CTRL */
|
||||
if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
|
||||
hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
|
||||
|
||||
if (hostval != guestval) {
|
||||
msrval = setguest ? guestval : hostval;
|
||||
wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
|
||||
* MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
|
||||
*/
|
||||
if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
|
||||
!static_cpu_has(X86_FEATURE_VIRT_SSBD))
|
||||
return;
|
||||
|
||||
/*
|
||||
* If the host has SSBD mitigation enabled, force it in the host's
|
||||
* virtual MSR value. If its not permanently enabled, evaluate
|
||||
* current's TIF_SSBD thread flag.
|
||||
*/
|
||||
if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
|
||||
hostval = SPEC_CTRL_SSBD;
|
||||
else
|
||||
hostval = ssbd_tif_to_spec_ctrl(ti->flags);
|
||||
|
||||
/* Sanitize the guest value */
|
||||
guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
|
||||
|
||||
if (hostval != guestval) {
|
||||
unsigned long tif;
|
||||
|
||||
tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
|
||||
ssbd_spec_ctrl_to_tif(hostval);
|
||||
|
||||
speculative_store_bypass_update(tif);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
|
||||
|
||||
static void x86_amd_ssb_disable(void)
|
||||
{
|
||||
u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
|
||||
|
||||
if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
|
||||
wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
|
||||
else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
|
||||
wrmsrl(MSR_AMD64_LS_CFG, msrval);
|
||||
}
|
||||
|
||||
#ifdef RETPOLINE
|
||||
static bool spectre_v2_bad_module;
|
||||
|
|
@ -312,32 +422,289 @@ static void __init spectre_v2_select_mitigation(void)
|
|||
}
|
||||
|
||||
#undef pr_fmt
|
||||
#define pr_fmt(fmt) "Speculative Store Bypass: " fmt
|
||||
|
||||
static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
|
||||
|
||||
/* The kernel command line selection */
|
||||
enum ssb_mitigation_cmd {
|
||||
SPEC_STORE_BYPASS_CMD_NONE,
|
||||
SPEC_STORE_BYPASS_CMD_AUTO,
|
||||
SPEC_STORE_BYPASS_CMD_ON,
|
||||
SPEC_STORE_BYPASS_CMD_PRCTL,
|
||||
SPEC_STORE_BYPASS_CMD_SECCOMP,
|
||||
};
|
||||
|
||||
static const char *ssb_strings[] = {
|
||||
[SPEC_STORE_BYPASS_NONE] = "Vulnerable",
|
||||
[SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled",
|
||||
[SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl",
|
||||
[SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
|
||||
};
|
||||
|
||||
static const struct {
|
||||
const char *option;
|
||||
enum ssb_mitigation_cmd cmd;
|
||||
} ssb_mitigation_options[] = {
|
||||
{ "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */
|
||||
{ "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */
|
||||
{ "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */
|
||||
{ "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */
|
||||
{ "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
|
||||
};
|
||||
|
||||
static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
|
||||
{
|
||||
enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
|
||||
char arg[20];
|
||||
int ret, i;
|
||||
|
||||
if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
|
||||
return SPEC_STORE_BYPASS_CMD_NONE;
|
||||
} else {
|
||||
ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
|
||||
arg, sizeof(arg));
|
||||
if (ret < 0)
|
||||
return SPEC_STORE_BYPASS_CMD_AUTO;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
|
||||
if (!match_option(arg, ret, ssb_mitigation_options[i].option))
|
||||
continue;
|
||||
|
||||
cmd = ssb_mitigation_options[i].cmd;
|
||||
break;
|
||||
}
|
||||
|
||||
if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
|
||||
pr_err("unknown option (%s). Switching to AUTO select\n", arg);
|
||||
return SPEC_STORE_BYPASS_CMD_AUTO;
|
||||
}
|
||||
}
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
static enum ssb_mitigation __init __ssb_select_mitigation(void)
|
||||
{
|
||||
enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
|
||||
enum ssb_mitigation_cmd cmd;
|
||||
|
||||
if (!boot_cpu_has(X86_FEATURE_SSBD))
|
||||
return mode;
|
||||
|
||||
cmd = ssb_parse_cmdline();
|
||||
if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
|
||||
(cmd == SPEC_STORE_BYPASS_CMD_NONE ||
|
||||
cmd == SPEC_STORE_BYPASS_CMD_AUTO))
|
||||
return mode;
|
||||
|
||||
switch (cmd) {
|
||||
case SPEC_STORE_BYPASS_CMD_AUTO:
|
||||
case SPEC_STORE_BYPASS_CMD_SECCOMP:
|
||||
/*
|
||||
* Choose prctl+seccomp as the default mode if seccomp is
|
||||
* enabled.
|
||||
*/
|
||||
if (IS_ENABLED(CONFIG_SECCOMP))
|
||||
mode = SPEC_STORE_BYPASS_SECCOMP;
|
||||
else
|
||||
mode = SPEC_STORE_BYPASS_PRCTL;
|
||||
break;
|
||||
case SPEC_STORE_BYPASS_CMD_ON:
|
||||
mode = SPEC_STORE_BYPASS_DISABLE;
|
||||
break;
|
||||
case SPEC_STORE_BYPASS_CMD_PRCTL:
|
||||
mode = SPEC_STORE_BYPASS_PRCTL;
|
||||
break;
|
||||
case SPEC_STORE_BYPASS_CMD_NONE:
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* We have three CPU feature flags that are in play here:
|
||||
* - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
|
||||
* - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
|
||||
* - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
|
||||
*/
|
||||
if (mode == SPEC_STORE_BYPASS_DISABLE) {
|
||||
setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
|
||||
/*
|
||||
* Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD uses
|
||||
* a completely different MSR and bit dependent on family.
|
||||
*/
|
||||
switch (boot_cpu_data.x86_vendor) {
|
||||
case X86_VENDOR_INTEL:
|
||||
x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
|
||||
x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
|
||||
wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
|
||||
break;
|
||||
case X86_VENDOR_AMD:
|
||||
x86_amd_ssb_disable();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return mode;
|
||||
}
|
||||
|
||||
static void ssb_select_mitigation(void)
|
||||
{
|
||||
ssb_mode = __ssb_select_mitigation();
|
||||
|
||||
if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
|
||||
pr_info("%s\n", ssb_strings[ssb_mode]);
|
||||
}
|
||||
|
||||
#undef pr_fmt
|
||||
#define pr_fmt(fmt) "Speculation prctl: " fmt
|
||||
|
||||
static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
|
||||
{
|
||||
bool update;
|
||||
|
||||
if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
|
||||
ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
|
||||
return -ENXIO;
|
||||
|
||||
switch (ctrl) {
|
||||
case PR_SPEC_ENABLE:
|
||||
/* If speculation is force disabled, enable is not allowed */
|
||||
if (task_spec_ssb_force_disable(task))
|
||||
return -EPERM;
|
||||
task_clear_spec_ssb_disable(task);
|
||||
update = test_and_clear_tsk_thread_flag(task, TIF_SSBD);
|
||||
break;
|
||||
case PR_SPEC_DISABLE:
|
||||
task_set_spec_ssb_disable(task);
|
||||
update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
|
||||
break;
|
||||
case PR_SPEC_FORCE_DISABLE:
|
||||
task_set_spec_ssb_disable(task);
|
||||
task_set_spec_ssb_force_disable(task);
|
||||
update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
|
||||
break;
|
||||
default:
|
||||
return -ERANGE;
|
||||
}
|
||||
|
||||
/*
|
||||
* If being set on non-current task, delay setting the CPU
|
||||
* mitigation until it is next scheduled.
|
||||
*/
|
||||
if (task == current && update)
|
||||
speculative_store_bypass_update_current();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
|
||||
unsigned long ctrl)
|
||||
{
|
||||
switch (which) {
|
||||
case PR_SPEC_STORE_BYPASS:
|
||||
return ssb_prctl_set(task, ctrl);
|
||||
default:
|
||||
return -ENODEV;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SECCOMP
|
||||
void arch_seccomp_spec_mitigate(struct task_struct *task)
|
||||
{
|
||||
if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
|
||||
ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int ssb_prctl_get(struct task_struct *task)
|
||||
{
|
||||
switch (ssb_mode) {
|
||||
case SPEC_STORE_BYPASS_DISABLE:
|
||||
return PR_SPEC_DISABLE;
|
||||
case SPEC_STORE_BYPASS_SECCOMP:
|
||||
case SPEC_STORE_BYPASS_PRCTL:
|
||||
if (task_spec_ssb_force_disable(task))
|
||||
return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
|
||||
if (task_spec_ssb_disable(task))
|
||||
return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
|
||||
return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
|
||||
default:
|
||||
if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
|
||||
return PR_SPEC_ENABLE;
|
||||
return PR_SPEC_NOT_AFFECTED;
|
||||
}
|
||||
}
|
||||
|
||||
int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
|
||||
{
|
||||
switch (which) {
|
||||
case PR_SPEC_STORE_BYPASS:
|
||||
return ssb_prctl_get(task);
|
||||
default:
|
||||
return -ENODEV;
|
||||
}
|
||||
}
|
||||
|
||||
void x86_spec_ctrl_setup_ap(void)
|
||||
{
|
||||
if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
|
||||
wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
|
||||
|
||||
if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
|
||||
x86_amd_ssb_disable();
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SYSFS
|
||||
ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
|
||||
|
||||
static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
|
||||
char *buf, unsigned int bug)
|
||||
{
|
||||
if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
|
||||
if (!boot_cpu_has_bug(bug))
|
||||
return sprintf(buf, "Not affected\n");
|
||||
|
||||
switch (bug) {
|
||||
case X86_BUG_CPU_MELTDOWN:
|
||||
if (boot_cpu_has(X86_FEATURE_PTI))
|
||||
return sprintf(buf, "Mitigation: PTI\n");
|
||||
return sprintf(buf, "Vulnerable\n");
|
||||
}
|
||||
|
||||
ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
|
||||
{
|
||||
if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1))
|
||||
return sprintf(buf, "Not affected\n");
|
||||
break;
|
||||
|
||||
case X86_BUG_SPECTRE_V1:
|
||||
return sprintf(buf, "Mitigation: __user pointer sanitization\n");
|
||||
}
|
||||
|
||||
ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
|
||||
{
|
||||
if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
|
||||
return sprintf(buf, "Not affected\n");
|
||||
|
||||
case X86_BUG_SPECTRE_V2:
|
||||
return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
|
||||
boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
|
||||
boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
|
||||
spectre_v2_module_string());
|
||||
|
||||
case X86_BUG_SPEC_STORE_BYPASS:
|
||||
return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return sprintf(buf, "Vulnerable\n");
|
||||
}
|
||||
|
||||
ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
|
||||
{
|
||||
return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
|
||||
}
|
||||
|
||||
ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
|
||||
{
|
||||
return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
|
||||
}
|
||||
|
||||
ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
|
||||
{
|
||||
return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
|
||||
}
|
||||
|
||||
ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
|
||||
{
|
||||
return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -757,17 +757,32 @@ static void init_speculation_control(struct cpuinfo_x86 *c)
|
|||
* and they also have a different bit for STIBP support. Also,
|
||||
* a hypervisor might have set the individual AMD bits even on
|
||||
* Intel CPUs, for finer-grained selection of what's available.
|
||||
*
|
||||
* We use the AMD bits in 0x8000_0008 EBX as the generic hardware
|
||||
* features, which are visible in /proc/cpuinfo and used by the
|
||||
* kernel. So set those accordingly from the Intel bits.
|
||||
*/
|
||||
if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
|
||||
set_cpu_cap(c, X86_FEATURE_IBRS);
|
||||
set_cpu_cap(c, X86_FEATURE_IBPB);
|
||||
set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
|
||||
}
|
||||
|
||||
if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
|
||||
set_cpu_cap(c, X86_FEATURE_STIBP);
|
||||
|
||||
if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
|
||||
cpu_has(c, X86_FEATURE_VIRT_SSBD))
|
||||
set_cpu_cap(c, X86_FEATURE_SSBD);
|
||||
|
||||
if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
|
||||
set_cpu_cap(c, X86_FEATURE_IBRS);
|
||||
set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
|
||||
}
|
||||
|
||||
if (cpu_has(c, X86_FEATURE_AMD_IBPB))
|
||||
set_cpu_cap(c, X86_FEATURE_IBPB);
|
||||
|
||||
if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
|
||||
set_cpu_cap(c, X86_FEATURE_STIBP);
|
||||
set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
|
||||
}
|
||||
}
|
||||
|
||||
void get_cpu_cap(struct cpuinfo_x86 *c)
|
||||
|
|
@ -927,21 +942,55 @@ static const __initconst struct x86_cpu_id cpu_no_meltdown[] = {
|
|||
{}
|
||||
};
|
||||
|
||||
static bool __init cpu_vulnerable_to_meltdown(struct cpuinfo_x86 *c)
|
||||
static const __initconst struct x86_cpu_id cpu_no_spec_store_bypass[] = {
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_PINEVIEW },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_LINCROFT },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_PENWELL },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_CLOVERVIEW },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_CEDARVIEW },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_SILVERMONT1 },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_AIRMONT },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_SILVERMONT2 },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_MERRIFIELD },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_CORE_YONAH },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_XEON_PHI_KNL },
|
||||
{ X86_VENDOR_INTEL, 6, INTEL_FAM6_XEON_PHI_KNM },
|
||||
{ X86_VENDOR_CENTAUR, 5, },
|
||||
{ X86_VENDOR_INTEL, 5, },
|
||||
{ X86_VENDOR_NSC, 5, },
|
||||
{ X86_VENDOR_AMD, 0x12, },
|
||||
{ X86_VENDOR_AMD, 0x11, },
|
||||
{ X86_VENDOR_AMD, 0x10, },
|
||||
{ X86_VENDOR_AMD, 0xf, },
|
||||
{ X86_VENDOR_ANY, 4, },
|
||||
{}
|
||||
};
|
||||
|
||||
static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
|
||||
{
|
||||
u64 ia32_cap = 0;
|
||||
|
||||
if (x86_match_cpu(cpu_no_meltdown))
|
||||
return false;
|
||||
|
||||
if (cpu_has(c, X86_FEATURE_ARCH_CAPABILITIES))
|
||||
rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
|
||||
|
||||
if (!x86_match_cpu(cpu_no_spec_store_bypass) &&
|
||||
!(ia32_cap & ARCH_CAP_SSB_NO))
|
||||
setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
|
||||
|
||||
if (x86_match_cpu(cpu_no_speculation))
|
||||
return;
|
||||
|
||||
setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
|
||||
setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
|
||||
|
||||
if (x86_match_cpu(cpu_no_meltdown))
|
||||
return;
|
||||
|
||||
/* Rogue Data Cache Load? No! */
|
||||
if (ia32_cap & ARCH_CAP_RDCL_NO)
|
||||
return false;
|
||||
return;
|
||||
|
||||
return true;
|
||||
setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -992,12 +1041,7 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
|
|||
|
||||
setup_force_cpu_cap(X86_FEATURE_ALWAYS);
|
||||
|
||||
if (!x86_match_cpu(cpu_no_speculation)) {
|
||||
if (cpu_vulnerable_to_meltdown(c))
|
||||
setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
|
||||
setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
|
||||
setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
|
||||
}
|
||||
cpu_set_bug_bits(c);
|
||||
|
||||
fpu__init_system(c);
|
||||
|
||||
|
|
@ -1359,6 +1403,7 @@ void identify_secondary_cpu(struct cpuinfo_x86 *c)
|
|||
#endif
|
||||
mtrr_ap_init();
|
||||
validate_apic_and_package_id(c);
|
||||
x86_spec_ctrl_setup_ap();
|
||||
}
|
||||
|
||||
static __init int setup_noclflush(char *arg)
|
||||
|
|
|
|||
|
|
@ -50,4 +50,6 @@ extern void cpu_detect_cache_sizes(struct cpuinfo_x86 *c);
|
|||
|
||||
unsigned int aperfmperf_get_khz(int cpu);
|
||||
|
||||
extern void x86_spec_ctrl_setup_ap(void);
|
||||
|
||||
#endif /* ARCH_X86_CPU_H */
|
||||
|
|
|
|||
|
|
@ -188,7 +188,10 @@ static void early_init_intel(struct cpuinfo_x86 *c)
|
|||
setup_clear_cpu_cap(X86_FEATURE_IBPB);
|
||||
setup_clear_cpu_cap(X86_FEATURE_STIBP);
|
||||
setup_clear_cpu_cap(X86_FEATURE_SPEC_CTRL);
|
||||
setup_clear_cpu_cap(X86_FEATURE_MSR_SPEC_CTRL);
|
||||
setup_clear_cpu_cap(X86_FEATURE_INTEL_STIBP);
|
||||
setup_clear_cpu_cap(X86_FEATURE_SSBD);
|
||||
setup_clear_cpu_cap(X86_FEATURE_SPEC_CTRL_SSBD);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@
|
|||
#include <asm/switch_to.h>
|
||||
#include <asm/desc.h>
|
||||
#include <asm/prctl.h>
|
||||
#include <asm/spec-ctrl.h>
|
||||
|
||||
/*
|
||||
* per-CPU TSS segments. Threads are completely 'soft' on Linux,
|
||||
|
|
@ -278,6 +279,148 @@ static inline void switch_to_bitmap(struct tss_struct *tss,
|
|||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
|
||||
struct ssb_state {
|
||||
struct ssb_state *shared_state;
|
||||
raw_spinlock_t lock;
|
||||
unsigned int disable_state;
|
||||
unsigned long local_state;
|
||||
};
|
||||
|
||||
#define LSTATE_SSB 0
|
||||
|
||||
static DEFINE_PER_CPU(struct ssb_state, ssb_state);
|
||||
|
||||
void speculative_store_bypass_ht_init(void)
|
||||
{
|
||||
struct ssb_state *st = this_cpu_ptr(&ssb_state);
|
||||
unsigned int this_cpu = smp_processor_id();
|
||||
unsigned int cpu;
|
||||
|
||||
st->local_state = 0;
|
||||
|
||||
/*
|
||||
* Shared state setup happens once on the first bringup
|
||||
* of the CPU. It's not destroyed on CPU hotunplug.
|
||||
*/
|
||||
if (st->shared_state)
|
||||
return;
|
||||
|
||||
raw_spin_lock_init(&st->lock);
|
||||
|
||||
/*
|
||||
* Go over HT siblings and check whether one of them has set up the
|
||||
* shared state pointer already.
|
||||
*/
|
||||
for_each_cpu(cpu, topology_sibling_cpumask(this_cpu)) {
|
||||
if (cpu == this_cpu)
|
||||
continue;
|
||||
|
||||
if (!per_cpu(ssb_state, cpu).shared_state)
|
||||
continue;
|
||||
|
||||
/* Link it to the state of the sibling: */
|
||||
st->shared_state = per_cpu(ssb_state, cpu).shared_state;
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* First HT sibling to come up on the core. Link shared state of
|
||||
* the first HT sibling to itself. The siblings on the same core
|
||||
* which come up later will see the shared state pointer and link
|
||||
* themself to the state of this CPU.
|
||||
*/
|
||||
st->shared_state = st;
|
||||
}
|
||||
|
||||
/*
|
||||
* Logic is: First HT sibling enables SSBD for both siblings in the core
|
||||
* and last sibling to disable it, disables it for the whole core. This how
|
||||
* MSR_SPEC_CTRL works in "hardware":
|
||||
*
|
||||
* CORE_SPEC_CTRL = THREAD0_SPEC_CTRL | THREAD1_SPEC_CTRL
|
||||
*/
|
||||
static __always_inline void amd_set_core_ssb_state(unsigned long tifn)
|
||||
{
|
||||
struct ssb_state *st = this_cpu_ptr(&ssb_state);
|
||||
u64 msr = x86_amd_ls_cfg_base;
|
||||
|
||||
if (!static_cpu_has(X86_FEATURE_ZEN)) {
|
||||
msr |= ssbd_tif_to_amd_ls_cfg(tifn);
|
||||
wrmsrl(MSR_AMD64_LS_CFG, msr);
|
||||
return;
|
||||
}
|
||||
|
||||
if (tifn & _TIF_SSBD) {
|
||||
/*
|
||||
* Since this can race with prctl(), block reentry on the
|
||||
* same CPU.
|
||||
*/
|
||||
if (__test_and_set_bit(LSTATE_SSB, &st->local_state))
|
||||
return;
|
||||
|
||||
msr |= x86_amd_ls_cfg_ssbd_mask;
|
||||
|
||||
raw_spin_lock(&st->shared_state->lock);
|
||||
/* First sibling enables SSBD: */
|
||||
if (!st->shared_state->disable_state)
|
||||
wrmsrl(MSR_AMD64_LS_CFG, msr);
|
||||
st->shared_state->disable_state++;
|
||||
raw_spin_unlock(&st->shared_state->lock);
|
||||
} else {
|
||||
if (!__test_and_clear_bit(LSTATE_SSB, &st->local_state))
|
||||
return;
|
||||
|
||||
raw_spin_lock(&st->shared_state->lock);
|
||||
st->shared_state->disable_state--;
|
||||
if (!st->shared_state->disable_state)
|
||||
wrmsrl(MSR_AMD64_LS_CFG, msr);
|
||||
raw_spin_unlock(&st->shared_state->lock);
|
||||
}
|
||||
}
|
||||
#else
|
||||
static __always_inline void amd_set_core_ssb_state(unsigned long tifn)
|
||||
{
|
||||
u64 msr = x86_amd_ls_cfg_base | ssbd_tif_to_amd_ls_cfg(tifn);
|
||||
|
||||
wrmsrl(MSR_AMD64_LS_CFG, msr);
|
||||
}
|
||||
#endif
|
||||
|
||||
static __always_inline void amd_set_ssb_virt_state(unsigned long tifn)
|
||||
{
|
||||
/*
|
||||
* SSBD has the same definition in SPEC_CTRL and VIRT_SPEC_CTRL,
|
||||
* so ssbd_tif_to_spec_ctrl() just works.
|
||||
*/
|
||||
wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn));
|
||||
}
|
||||
|
||||
static __always_inline void intel_set_ssb_state(unsigned long tifn)
|
||||
{
|
||||
u64 msr = x86_spec_ctrl_base | ssbd_tif_to_spec_ctrl(tifn);
|
||||
|
||||
wrmsrl(MSR_IA32_SPEC_CTRL, msr);
|
||||
}
|
||||
|
||||
static __always_inline void __speculative_store_bypass_update(unsigned long tifn)
|
||||
{
|
||||
if (static_cpu_has(X86_FEATURE_VIRT_SSBD))
|
||||
amd_set_ssb_virt_state(tifn);
|
||||
else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD))
|
||||
amd_set_core_ssb_state(tifn);
|
||||
else
|
||||
intel_set_ssb_state(tifn);
|
||||
}
|
||||
|
||||
void speculative_store_bypass_update(unsigned long tif)
|
||||
{
|
||||
preempt_disable();
|
||||
__speculative_store_bypass_update(tif);
|
||||
preempt_enable();
|
||||
}
|
||||
|
||||
void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
|
||||
struct tss_struct *tss)
|
||||
{
|
||||
|
|
@ -309,6 +452,9 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
|
|||
|
||||
if ((tifp ^ tifn) & _TIF_NOCPUID)
|
||||
set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
|
||||
|
||||
if ((tifp ^ tifn) & _TIF_SSBD)
|
||||
__speculative_store_bypass_update(tifn);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@
|
|||
#include <asm/qspinlock.h>
|
||||
#include <asm/intel-family.h>
|
||||
#include <asm/cpu_device_id.h>
|
||||
#include <asm/spec-ctrl.h>
|
||||
|
||||
/* Number of siblings per CPU package */
|
||||
int smp_num_siblings = 1;
|
||||
|
|
@ -244,6 +245,8 @@ static void notrace start_secondary(void *unused)
|
|||
*/
|
||||
check_tsc_sync_target();
|
||||
|
||||
speculative_store_bypass_ht_init();
|
||||
|
||||
/*
|
||||
* Lock vector_lock, set CPU online and bring the vector
|
||||
* allocator online. Online must be set with vector_lock held
|
||||
|
|
@ -1292,6 +1295,8 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
|
|||
set_mtrr_aps_delayed_init();
|
||||
|
||||
smp_quirk_init_udelay();
|
||||
|
||||
speculative_store_bypass_ht_init();
|
||||
}
|
||||
|
||||
void arch_enable_nonboot_cpus_begin(void)
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
|
|||
|
||||
/* cpuid 0x80000008.ebx */
|
||||
const u32 kvm_cpuid_8000_0008_ebx_x86_features =
|
||||
F(IBPB) | F(IBRS);
|
||||
F(AMD_IBPB) | F(AMD_IBRS) | F(VIRT_SSBD);
|
||||
|
||||
/* cpuid 0xC0000001.edx */
|
||||
const u32 kvm_cpuid_C000_0001_edx_x86_features =
|
||||
|
|
@ -407,7 +407,7 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
|
|||
|
||||
/* cpuid 7.0.edx*/
|
||||
const u32 kvm_cpuid_7_0_edx_x86_features =
|
||||
F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) |
|
||||
F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) | F(SSBD) |
|
||||
F(ARCH_CAPABILITIES);
|
||||
|
||||
/* all calls to cpuid_count() should be made on the same cpu */
|
||||
|
|
@ -647,13 +647,20 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
|
|||
g_phys_as = phys_as;
|
||||
entry->eax = g_phys_as | (virt_as << 8);
|
||||
entry->edx = 0;
|
||||
/* IBRS and IBPB aren't necessarily present in hardware cpuid */
|
||||
if (boot_cpu_has(X86_FEATURE_IBPB))
|
||||
entry->ebx |= F(IBPB);
|
||||
if (boot_cpu_has(X86_FEATURE_IBRS))
|
||||
entry->ebx |= F(IBRS);
|
||||
/*
|
||||
* IBRS, IBPB and VIRT_SSBD aren't necessarily present in
|
||||
* hardware cpuid
|
||||
*/
|
||||
if (boot_cpu_has(X86_FEATURE_AMD_IBPB))
|
||||
entry->ebx |= F(AMD_IBPB);
|
||||
if (boot_cpu_has(X86_FEATURE_AMD_IBRS))
|
||||
entry->ebx |= F(AMD_IBRS);
|
||||
if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
|
||||
entry->ebx |= F(VIRT_SSBD);
|
||||
entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features;
|
||||
cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX);
|
||||
if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
|
||||
entry->ebx |= F(VIRT_SSBD);
|
||||
break;
|
||||
}
|
||||
case 0x80000019:
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@
|
|||
#include <asm/debugreg.h>
|
||||
#include <asm/kvm_para.h>
|
||||
#include <asm/irq_remapping.h>
|
||||
#include <asm/nospec-branch.h>
|
||||
#include <asm/spec-ctrl.h>
|
||||
|
||||
#include <asm/virtext.h>
|
||||
#include "trace.h"
|
||||
|
|
@ -213,6 +213,12 @@ struct vcpu_svm {
|
|||
} host;
|
||||
|
||||
u64 spec_ctrl;
|
||||
/*
|
||||
* Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be
|
||||
* translated into the appropriate L2_CFG bits on the host to
|
||||
* perform speculative control.
|
||||
*/
|
||||
u64 virt_spec_ctrl;
|
||||
|
||||
u32 *msrpm;
|
||||
|
||||
|
|
@ -2060,6 +2066,7 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
|
|||
|
||||
vcpu->arch.microcode_version = 0x01000065;
|
||||
svm->spec_ctrl = 0;
|
||||
svm->virt_spec_ctrl = 0;
|
||||
|
||||
if (!init_event) {
|
||||
svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
|
||||
|
|
@ -4108,11 +4115,18 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
|
|||
break;
|
||||
case MSR_IA32_SPEC_CTRL:
|
||||
if (!msr_info->host_initiated &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS))
|
||||
return 1;
|
||||
|
||||
msr_info->data = svm->spec_ctrl;
|
||||
break;
|
||||
case MSR_AMD64_VIRT_SPEC_CTRL:
|
||||
if (!msr_info->host_initiated &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
|
||||
return 1;
|
||||
|
||||
msr_info->data = svm->virt_spec_ctrl;
|
||||
break;
|
||||
case MSR_F15H_IC_CFG: {
|
||||
|
||||
int family, model;
|
||||
|
|
@ -4203,7 +4217,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
|
|||
break;
|
||||
case MSR_IA32_SPEC_CTRL:
|
||||
if (!msr->host_initiated &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS))
|
||||
return 1;
|
||||
|
||||
/* The STIBP bit doesn't fault even if it's not advertised */
|
||||
|
|
@ -4230,7 +4244,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
|
|||
break;
|
||||
case MSR_IA32_PRED_CMD:
|
||||
if (!msr->host_initiated &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_IBPB))
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
|
||||
return 1;
|
||||
|
||||
if (data & ~PRED_CMD_IBPB)
|
||||
|
|
@ -4244,6 +4258,16 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
|
|||
break;
|
||||
set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
|
||||
break;
|
||||
case MSR_AMD64_VIRT_SPEC_CTRL:
|
||||
if (!msr->host_initiated &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
|
||||
return 1;
|
||||
|
||||
if (data & ~SPEC_CTRL_SSBD)
|
||||
return 1;
|
||||
|
||||
svm->virt_spec_ctrl = data;
|
||||
break;
|
||||
case MSR_STAR:
|
||||
svm->vmcb->save.star = data;
|
||||
break;
|
||||
|
|
@ -5557,8 +5581,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
|
|||
* is no need to worry about the conditional branch over the wrmsr
|
||||
* being speculatively taken.
|
||||
*/
|
||||
if (svm->spec_ctrl)
|
||||
native_wrmsrl(MSR_IA32_SPEC_CTRL, svm->spec_ctrl);
|
||||
x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
|
||||
|
||||
asm volatile (
|
||||
"push %%" _ASM_BP "; \n\t"
|
||||
|
|
@ -5652,6 +5675,18 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
|
|||
#endif
|
||||
);
|
||||
|
||||
/* Eliminate branch target predictions from guest mode */
|
||||
vmexit_fill_RSB();
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
wrmsrl(MSR_GS_BASE, svm->host.gs_base);
|
||||
#else
|
||||
loadsegment(fs, svm->host.fs);
|
||||
#ifndef CONFIG_X86_32_LAZY_GS
|
||||
loadsegment(gs, svm->host.gs);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* We do not use IBRS in the kernel. If this vCPU has used the
|
||||
* SPEC_CTRL MSR it may have left it on; save the value and
|
||||
|
|
@ -5670,20 +5705,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
|
|||
if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
|
||||
svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
|
||||
|
||||
if (svm->spec_ctrl)
|
||||
native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
|
||||
|
||||
/* Eliminate branch target predictions from guest mode */
|
||||
vmexit_fill_RSB();
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
wrmsrl(MSR_GS_BASE, svm->host.gs_base);
|
||||
#else
|
||||
loadsegment(fs, svm->host.fs);
|
||||
#ifndef CONFIG_X86_32_LAZY_GS
|
||||
loadsegment(gs, svm->host.gs);
|
||||
#endif
|
||||
#endif
|
||||
x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
|
||||
|
||||
reload_tss(vcpu);
|
||||
|
||||
|
|
@ -5786,7 +5808,7 @@ static bool svm_cpu_has_accelerated_tpr(void)
|
|||
return false;
|
||||
}
|
||||
|
||||
static bool svm_has_high_real_mode_segbase(void)
|
||||
static bool svm_has_emulated_msr(int index)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
|
@ -7012,7 +7034,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
|
|||
.hardware_enable = svm_hardware_enable,
|
||||
.hardware_disable = svm_hardware_disable,
|
||||
.cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
|
||||
.cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
|
||||
.has_emulated_msr = svm_has_emulated_msr,
|
||||
|
||||
.vcpu_create = svm_create_vcpu,
|
||||
.vcpu_free = svm_free_vcpu,
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@
|
|||
#include <asm/apic.h>
|
||||
#include <asm/irq_remapping.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/nospec-branch.h>
|
||||
#include <asm/spec-ctrl.h>
|
||||
#include <asm/mshyperv.h>
|
||||
|
||||
#include "trace.h"
|
||||
|
|
@ -3529,7 +3529,6 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
|
|||
return kvm_get_msr_common(vcpu, msr_info);
|
||||
case MSR_IA32_SPEC_CTRL:
|
||||
if (!msr_info->host_initiated &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_IBRS) &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
|
||||
return 1;
|
||||
|
||||
|
|
@ -3648,12 +3647,11 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
|
|||
break;
|
||||
case MSR_IA32_SPEC_CTRL:
|
||||
if (!msr_info->host_initiated &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_IBRS) &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
|
||||
return 1;
|
||||
|
||||
/* The STIBP bit doesn't fault even if it's not advertised */
|
||||
if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP))
|
||||
if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD))
|
||||
return 1;
|
||||
|
||||
vmx->spec_ctrl = data;
|
||||
|
|
@ -3679,7 +3677,6 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
|
|||
break;
|
||||
case MSR_IA32_PRED_CMD:
|
||||
if (!msr_info->host_initiated &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_IBPB) &&
|
||||
!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
|
||||
return 1;
|
||||
|
||||
|
|
@ -9488,9 +9485,21 @@ static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
|
|||
}
|
||||
STACK_FRAME_NON_STANDARD(vmx_handle_external_intr);
|
||||
|
||||
static bool vmx_has_high_real_mode_segbase(void)
|
||||
static bool vmx_has_emulated_msr(int index)
|
||||
{
|
||||
switch (index) {
|
||||
case MSR_IA32_SMBASE:
|
||||
/*
|
||||
* We cannot do SMM unless we can run the guest in big
|
||||
* real mode.
|
||||
*/
|
||||
return enable_unrestricted_guest || emulate_invalid_guest_state;
|
||||
case MSR_AMD64_VIRT_SPEC_CTRL:
|
||||
/* This is AMD only. */
|
||||
return false;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
static bool vmx_mpx_supported(void)
|
||||
|
|
@ -9722,8 +9731,7 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
|
|||
* is no need to worry about the conditional branch over the wrmsr
|
||||
* being speculatively taken.
|
||||
*/
|
||||
if (vmx->spec_ctrl)
|
||||
native_wrmsrl(MSR_IA32_SPEC_CTRL, vmx->spec_ctrl);
|
||||
x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0);
|
||||
|
||||
vmx->__launched = vmx->loaded_vmcs->launched;
|
||||
|
||||
|
|
@ -9871,8 +9879,7 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
|
|||
if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
|
||||
vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
|
||||
|
||||
if (vmx->spec_ctrl)
|
||||
native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
|
||||
x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
|
||||
|
||||
/* Eliminate branch target predictions from guest mode */
|
||||
vmexit_fill_RSB();
|
||||
|
|
@ -12632,7 +12639,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
|
|||
.hardware_enable = hardware_enable,
|
||||
.hardware_disable = hardware_disable,
|
||||
.cpu_has_accelerated_tpr = report_flexpriority,
|
||||
.cpu_has_high_real_mode_segbase = vmx_has_high_real_mode_segbase,
|
||||
.has_emulated_msr = vmx_has_emulated_msr,
|
||||
|
||||
.vm_init = vmx_vm_init,
|
||||
.vm_alloc = vmx_vm_alloc,
|
||||
|
|
|
|||
|
|
@ -1061,6 +1061,7 @@ static u32 emulated_msrs[] = {
|
|||
MSR_SMI_COUNT,
|
||||
MSR_PLATFORM_INFO,
|
||||
MSR_MISC_FEATURES_ENABLES,
|
||||
MSR_AMD64_VIRT_SPEC_CTRL,
|
||||
};
|
||||
|
||||
static unsigned num_emulated_msrs;
|
||||
|
|
@ -2906,7 +2907,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
|
|||
* fringe case that is not enabled except via specific settings
|
||||
* of the module parameters.
|
||||
*/
|
||||
r = kvm_x86_ops->cpu_has_high_real_mode_segbase();
|
||||
r = kvm_x86_ops->has_emulated_msr(MSR_IA32_SMBASE);
|
||||
break;
|
||||
case KVM_CAP_VAPIC:
|
||||
r = !kvm_x86_ops->cpu_has_accelerated_tpr();
|
||||
|
|
@ -4606,14 +4607,8 @@ static void kvm_init_msr_list(void)
|
|||
num_msrs_to_save = j;
|
||||
|
||||
for (i = j = 0; i < ARRAY_SIZE(emulated_msrs); i++) {
|
||||
switch (emulated_msrs[i]) {
|
||||
case MSR_IA32_SMBASE:
|
||||
if (!kvm_x86_ops->cpu_has_high_real_mode_segbase())
|
||||
if (!kvm_x86_ops->has_emulated_msr(emulated_msrs[i]))
|
||||
continue;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (j < i)
|
||||
emulated_msrs[j] = emulated_msrs[i];
|
||||
|
|
|
|||
|
|
@ -534,14 +534,22 @@ ssize_t __weak cpu_show_spectre_v2(struct device *dev,
|
|||
return sprintf(buf, "Not affected\n");
|
||||
}
|
||||
|
||||
ssize_t __weak cpu_show_spec_store_bypass(struct device *dev,
|
||||
struct device_attribute *attr, char *buf)
|
||||
{
|
||||
return sprintf(buf, "Not affected\n");
|
||||
}
|
||||
|
||||
static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL);
|
||||
static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL);
|
||||
static DEVICE_ATTR(spectre_v2, 0444, cpu_show_spectre_v2, NULL);
|
||||
static DEVICE_ATTR(spec_store_bypass, 0444, cpu_show_spec_store_bypass, NULL);
|
||||
|
||||
static struct attribute *cpu_root_vulnerabilities_attrs[] = {
|
||||
&dev_attr_meltdown.attr,
|
||||
&dev_attr_spectre_v1.attr,
|
||||
&dev_attr_spectre_v2.attr,
|
||||
&dev_attr_spec_store_bypass.attr,
|
||||
NULL
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@
|
|||
#include <linux/delayacct.h>
|
||||
#include <linux/seq_file.h>
|
||||
#include <linux/pid_namespace.h>
|
||||
#include <linux/prctl.h>
|
||||
#include <linux/ptrace.h>
|
||||
#include <linux/tracehook.h>
|
||||
#include <linux/string_helpers.h>
|
||||
|
|
@ -335,6 +336,30 @@ static inline void task_seccomp(struct seq_file *m, struct task_struct *p)
|
|||
#ifdef CONFIG_SECCOMP
|
||||
seq_put_decimal_ull(m, "\nSeccomp:\t", p->seccomp.mode);
|
||||
#endif
|
||||
seq_printf(m, "\nSpeculation_Store_Bypass:\t");
|
||||
switch (arch_prctl_spec_ctrl_get(p, PR_SPEC_STORE_BYPASS)) {
|
||||
case -EINVAL:
|
||||
seq_printf(m, "unknown");
|
||||
break;
|
||||
case PR_SPEC_NOT_AFFECTED:
|
||||
seq_printf(m, "not vulnerable");
|
||||
break;
|
||||
case PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE:
|
||||
seq_printf(m, "thread force mitigated");
|
||||
break;
|
||||
case PR_SPEC_PRCTL | PR_SPEC_DISABLE:
|
||||
seq_printf(m, "thread mitigated");
|
||||
break;
|
||||
case PR_SPEC_PRCTL | PR_SPEC_ENABLE:
|
||||
seq_printf(m, "thread vulnerable");
|
||||
break;
|
||||
case PR_SPEC_DISABLE:
|
||||
seq_printf(m, "globally mitigated");
|
||||
break;
|
||||
default:
|
||||
seq_printf(m, "vulnerable");
|
||||
break;
|
||||
}
|
||||
seq_putc(m, '\n');
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ struct bpf_insn_aux_data {
|
|||
s32 call_imm; /* saved imm field of call insn */
|
||||
};
|
||||
int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
|
||||
int sanitize_stack_off; /* stack slot to be cleared */
|
||||
bool seen; /* this insn was processed by the verifier */
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,8 @@ extern ssize_t cpu_show_spectre_v1(struct device *dev,
|
|||
struct device_attribute *attr, char *buf);
|
||||
extern ssize_t cpu_show_spectre_v2(struct device *dev,
|
||||
struct device_attribute *attr, char *buf);
|
||||
extern ssize_t cpu_show_spec_store_bypass(struct device *dev,
|
||||
struct device_attribute *attr, char *buf);
|
||||
|
||||
extern __printf(4, 5)
|
||||
struct device *cpu_device_create(struct device *parent, void *drvdata,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
#define _LINUX_NOSPEC_H
|
||||
#include <asm/barrier.h>
|
||||
|
||||
struct task_struct;
|
||||
|
||||
/**
|
||||
* array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
|
||||
* @index: array element index
|
||||
|
|
@ -55,4 +57,12 @@ static inline unsigned long array_index_mask_nospec(unsigned long index,
|
|||
\
|
||||
(typeof(_i)) (_i & _mask); \
|
||||
})
|
||||
|
||||
/* Speculation control prctl */
|
||||
int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which);
|
||||
int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
|
||||
unsigned long ctrl);
|
||||
/* Speculation control for seccomp enforced mitigation */
|
||||
void arch_seccomp_spec_mitigate(struct task_struct *task);
|
||||
|
||||
#endif /* _LINUX_NOSPEC_H */
|
||||
|
|
|
|||
|
|
@ -1433,7 +1433,8 @@ static inline bool is_percpu_thread(void)
|
|||
#define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */
|
||||
#define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */
|
||||
#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */
|
||||
|
||||
#define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */
|
||||
#define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/
|
||||
|
||||
#define TASK_PFA_TEST(name, func) \
|
||||
static inline bool task_##func(struct task_struct *p) \
|
||||
|
|
@ -1458,6 +1459,13 @@ TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
|
|||
TASK_PFA_SET(SPREAD_SLAB, spread_slab)
|
||||
TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
|
||||
|
||||
TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
|
||||
TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
|
||||
TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
|
||||
|
||||
TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
|
||||
TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
|
||||
|
||||
static inline void
|
||||
current_restore_flags(unsigned long orig_flags, unsigned long flags)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@
|
|||
#include <uapi/linux/seccomp.h>
|
||||
|
||||
#define SECCOMP_FILTER_FLAG_MASK (SECCOMP_FILTER_FLAG_TSYNC | \
|
||||
SECCOMP_FILTER_FLAG_LOG)
|
||||
SECCOMP_FILTER_FLAG_LOG | \
|
||||
SECCOMP_FILTER_FLAG_SPEC_ALLOW)
|
||||
|
||||
#ifdef CONFIG_SECCOMP
|
||||
|
||||
|
|
|
|||
|
|
@ -207,4 +207,16 @@ struct prctl_mm_map {
|
|||
# define PR_SVE_VL_LEN_MASK 0xffff
|
||||
# define PR_SVE_VL_INHERIT (1 << 17) /* inherit across exec */
|
||||
|
||||
/* Per task speculation control */
|
||||
#define PR_GET_SPECULATION_CTRL 52
|
||||
#define PR_SET_SPECULATION_CTRL 53
|
||||
/* Speculation control variants */
|
||||
# define PR_SPEC_STORE_BYPASS 0
|
||||
/* Return and control values for PR_SET/GET_SPECULATION_CTRL */
|
||||
# define PR_SPEC_NOT_AFFECTED 0
|
||||
# define PR_SPEC_PRCTL (1UL << 0)
|
||||
# define PR_SPEC_ENABLE (1UL << 1)
|
||||
# define PR_SPEC_DISABLE (1UL << 2)
|
||||
# define PR_SPEC_FORCE_DISABLE (1UL << 3)
|
||||
|
||||
#endif /* _LINUX_PRCTL_H */
|
||||
|
|
|
|||
|
|
@ -17,8 +17,9 @@
|
|||
#define SECCOMP_GET_ACTION_AVAIL 2
|
||||
|
||||
/* Valid flags for SECCOMP_SET_MODE_FILTER */
|
||||
#define SECCOMP_FILTER_FLAG_TSYNC 1
|
||||
#define SECCOMP_FILTER_FLAG_LOG 2
|
||||
#define SECCOMP_FILTER_FLAG_TSYNC (1UL << 0)
|
||||
#define SECCOMP_FILTER_FLAG_LOG (1UL << 1)
|
||||
#define SECCOMP_FILTER_FLAG_SPEC_ALLOW (1UL << 2)
|
||||
|
||||
/*
|
||||
* All BPF programs must return a 32-bit value.
|
||||
|
|
|
|||
|
|
@ -978,7 +978,7 @@ static bool register_is_null(struct bpf_reg_state *reg)
|
|||
*/
|
||||
static int check_stack_write(struct bpf_verifier_env *env,
|
||||
struct bpf_func_state *state, /* func where register points to */
|
||||
int off, int size, int value_regno)
|
||||
int off, int size, int value_regno, int insn_idx)
|
||||
{
|
||||
struct bpf_func_state *cur; /* state of the current function */
|
||||
int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
|
||||
|
|
@ -1017,8 +1017,33 @@ static int check_stack_write(struct bpf_verifier_env *env,
|
|||
state->stack[spi].spilled_ptr = cur->regs[value_regno];
|
||||
state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
|
||||
|
||||
for (i = 0; i < BPF_REG_SIZE; i++)
|
||||
for (i = 0; i < BPF_REG_SIZE; i++) {
|
||||
if (state->stack[spi].slot_type[i] == STACK_MISC &&
|
||||
!env->allow_ptr_leaks) {
|
||||
int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off;
|
||||
int soff = (-spi - 1) * BPF_REG_SIZE;
|
||||
|
||||
/* detected reuse of integer stack slot with a pointer
|
||||
* which means either llvm is reusing stack slot or
|
||||
* an attacker is trying to exploit CVE-2018-3639
|
||||
* (speculative store bypass)
|
||||
* Have to sanitize that slot with preemptive
|
||||
* store of zero.
|
||||
*/
|
||||
if (*poff && *poff != soff) {
|
||||
/* disallow programs where single insn stores
|
||||
* into two different stack slots, since verifier
|
||||
* cannot sanitize them
|
||||
*/
|
||||
verbose(env,
|
||||
"insn %d cannot access two stack slots fp%d and fp%d",
|
||||
insn_idx, *poff, soff);
|
||||
return -EINVAL;
|
||||
}
|
||||
*poff = soff;
|
||||
}
|
||||
state->stack[spi].slot_type[i] = STACK_SPILL;
|
||||
}
|
||||
} else {
|
||||
u8 type = STACK_MISC;
|
||||
|
||||
|
|
@ -1694,7 +1719,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
|
|||
|
||||
if (t == BPF_WRITE)
|
||||
err = check_stack_write(env, state, off, size,
|
||||
value_regno);
|
||||
value_regno, insn_idx);
|
||||
else
|
||||
err = check_stack_read(env, state, off, size,
|
||||
value_regno);
|
||||
|
|
@ -5169,6 +5194,34 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
|
|||
else
|
||||
continue;
|
||||
|
||||
if (type == BPF_WRITE &&
|
||||
env->insn_aux_data[i + delta].sanitize_stack_off) {
|
||||
struct bpf_insn patch[] = {
|
||||
/* Sanitize suspicious stack slot with zero.
|
||||
* There are no memory dependencies for this store,
|
||||
* since it's only using frame pointer and immediate
|
||||
* constant of zero
|
||||
*/
|
||||
BPF_ST_MEM(BPF_DW, BPF_REG_FP,
|
||||
env->insn_aux_data[i + delta].sanitize_stack_off,
|
||||
0),
|
||||
/* the original STX instruction will immediately
|
||||
* overwrite the same stack slot with appropriate value
|
||||
*/
|
||||
*insn,
|
||||
};
|
||||
|
||||
cnt = ARRAY_SIZE(patch);
|
||||
new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
|
||||
if (!new_prog)
|
||||
return -ENOMEM;
|
||||
|
||||
delta += cnt - 1;
|
||||
env->prog = new_prog;
|
||||
insn = new_prog->insnsi + i + delta;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (env->insn_aux_data[i + delta].ptr_type != PTR_TO_CTX)
|
||||
continue;
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@
|
|||
#include <linux/compat.h>
|
||||
#include <linux/coredump.h>
|
||||
#include <linux/kmemleak.h>
|
||||
#include <linux/nospec.h>
|
||||
#include <linux/prctl.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/sched/task_stack.h>
|
||||
#include <linux/seccomp.h>
|
||||
|
|
@ -227,8 +229,11 @@ static inline bool seccomp_may_assign_mode(unsigned long seccomp_mode)
|
|||
return true;
|
||||
}
|
||||
|
||||
void __weak arch_seccomp_spec_mitigate(struct task_struct *task) { }
|
||||
|
||||
static inline void seccomp_assign_mode(struct task_struct *task,
|
||||
unsigned long seccomp_mode)
|
||||
unsigned long seccomp_mode,
|
||||
unsigned long flags)
|
||||
{
|
||||
assert_spin_locked(&task->sighand->siglock);
|
||||
|
||||
|
|
@ -238,6 +243,9 @@ static inline void seccomp_assign_mode(struct task_struct *task,
|
|||
* filter) is set.
|
||||
*/
|
||||
smp_mb__before_atomic();
|
||||
/* Assume default seccomp processes want spec flaw mitigation. */
|
||||
if ((flags & SECCOMP_FILTER_FLAG_SPEC_ALLOW) == 0)
|
||||
arch_seccomp_spec_mitigate(task);
|
||||
set_tsk_thread_flag(task, TIF_SECCOMP);
|
||||
}
|
||||
|
||||
|
|
@ -305,7 +313,7 @@ static inline pid_t seccomp_can_sync_threads(void)
|
|||
* without dropping the locks.
|
||||
*
|
||||
*/
|
||||
static inline void seccomp_sync_threads(void)
|
||||
static inline void seccomp_sync_threads(unsigned long flags)
|
||||
{
|
||||
struct task_struct *thread, *caller;
|
||||
|
||||
|
|
@ -346,7 +354,8 @@ static inline void seccomp_sync_threads(void)
|
|||
* allow one thread to transition the other.
|
||||
*/
|
||||
if (thread->seccomp.mode == SECCOMP_MODE_DISABLED)
|
||||
seccomp_assign_mode(thread, SECCOMP_MODE_FILTER);
|
||||
seccomp_assign_mode(thread, SECCOMP_MODE_FILTER,
|
||||
flags);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -469,7 +478,7 @@ static long seccomp_attach_filter(unsigned int flags,
|
|||
|
||||
/* Now that the new filter is in place, synchronize to all threads. */
|
||||
if (flags & SECCOMP_FILTER_FLAG_TSYNC)
|
||||
seccomp_sync_threads();
|
||||
seccomp_sync_threads(flags);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -818,7 +827,7 @@ static long seccomp_set_mode_strict(void)
|
|||
#ifdef TIF_NOTSC
|
||||
disable_TSC();
|
||||
#endif
|
||||
seccomp_assign_mode(current, seccomp_mode);
|
||||
seccomp_assign_mode(current, seccomp_mode, 0);
|
||||
ret = 0;
|
||||
|
||||
out:
|
||||
|
|
@ -876,7 +885,7 @@ static long seccomp_set_mode_filter(unsigned int flags,
|
|||
/* Do not free the successfully attached filter. */
|
||||
prepared = NULL;
|
||||
|
||||
seccomp_assign_mode(current, seccomp_mode);
|
||||
seccomp_assign_mode(current, seccomp_mode, flags);
|
||||
out:
|
||||
spin_unlock_irq(¤t->sighand->siglock);
|
||||
if (flags & SECCOMP_FILTER_FLAG_TSYNC)
|
||||
|
|
|
|||
23
kernel/sys.c
23
kernel/sys.c
|
|
@ -61,6 +61,8 @@
|
|||
#include <linux/uidgid.h>
|
||||
#include <linux/cred.h>
|
||||
|
||||
#include <linux/nospec.h>
|
||||
|
||||
#include <linux/kmsg_dump.h>
|
||||
/* Move somewhere else to avoid recompiling? */
|
||||
#include <generated/utsrelease.h>
|
||||
|
|
@ -2242,6 +2244,17 @@ static int propagate_has_child_subreaper(struct task_struct *p, void *data)
|
|||
return 1;
|
||||
}
|
||||
|
||||
int __weak arch_prctl_spec_ctrl_get(struct task_struct *t, unsigned long which)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
int __weak arch_prctl_spec_ctrl_set(struct task_struct *t, unsigned long which,
|
||||
unsigned long ctrl)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
|
||||
unsigned long, arg4, unsigned long, arg5)
|
||||
{
|
||||
|
|
@ -2450,6 +2463,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
|
|||
case PR_SVE_GET_VL:
|
||||
error = SVE_GET_VL();
|
||||
break;
|
||||
case PR_GET_SPECULATION_CTRL:
|
||||
if (arg3 || arg4 || arg5)
|
||||
return -EINVAL;
|
||||
error = arch_prctl_spec_ctrl_get(me, arg2);
|
||||
break;
|
||||
case PR_SET_SPECULATION_CTRL:
|
||||
if (arg4 || arg5)
|
||||
return -EINVAL;
|
||||
error = arch_prctl_spec_ctrl_set(me, arg2, arg3);
|
||||
break;
|
||||
default:
|
||||
error = -EINVAL;
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -134,11 +134,15 @@ struct seccomp_data {
|
|||
#endif
|
||||
|
||||
#ifndef SECCOMP_FILTER_FLAG_TSYNC
|
||||
#define SECCOMP_FILTER_FLAG_TSYNC 1
|
||||
#define SECCOMP_FILTER_FLAG_TSYNC (1UL << 0)
|
||||
#endif
|
||||
|
||||
#ifndef SECCOMP_FILTER_FLAG_LOG
|
||||
#define SECCOMP_FILTER_FLAG_LOG 2
|
||||
#define SECCOMP_FILTER_FLAG_LOG (1UL << 1)
|
||||
#endif
|
||||
|
||||
#ifndef SECCOMP_FILTER_FLAG_SPEC_ALLOW
|
||||
#define SECCOMP_FILTER_FLAG_SPEC_ALLOW (1UL << 2)
|
||||
#endif
|
||||
|
||||
#ifndef PTRACE_SECCOMP_GET_METADATA
|
||||
|
|
@ -2072,14 +2076,26 @@ TEST(seccomp_syscall_mode_lock)
|
|||
TEST(detect_seccomp_filter_flags)
|
||||
{
|
||||
unsigned int flags[] = { SECCOMP_FILTER_FLAG_TSYNC,
|
||||
SECCOMP_FILTER_FLAG_LOG };
|
||||
SECCOMP_FILTER_FLAG_LOG,
|
||||
SECCOMP_FILTER_FLAG_SPEC_ALLOW };
|
||||
unsigned int flag, all_flags;
|
||||
int i;
|
||||
long ret;
|
||||
|
||||
/* Test detection of known-good filter flags */
|
||||
for (i = 0, all_flags = 0; i < ARRAY_SIZE(flags); i++) {
|
||||
int bits = 0;
|
||||
|
||||
flag = flags[i];
|
||||
/* Make sure the flag is a single bit! */
|
||||
while (flag) {
|
||||
if (flag & 0x1)
|
||||
bits ++;
|
||||
flag >>= 1;
|
||||
}
|
||||
ASSERT_EQ(1, bits);
|
||||
flag = flags[i];
|
||||
|
||||
ret = seccomp(SECCOMP_SET_MODE_FILTER, flag, NULL);
|
||||
ASSERT_NE(ENOSYS, errno) {
|
||||
TH_LOG("Kernel does not support seccomp syscall!");
|
||||
|
|
|
|||
Loading…
Reference in New Issue