Current File : //lib/modules/3.10.0-1160.81.1.el7.centos.plus.x86_64/source/arch/x86/include/asm/spec_ctrl.h
#ifndef _ASM_X86_SPEC_CTRL_H
#define _ASM_X86_SPEC_CTRL_H
#define SPEC_CTRL_PCP_IBRS_ENTRY (1<<0)
#define SPEC_CTRL_PCP_IBRS_EXIT (1<<1)
#define SPEC_CTRL_PCP_IBRS (SPEC_CTRL_PCP_IBRS_ENTRY|SPEC_CTRL_PCP_IBRS_EXIT)
#define IBRS_ENABLED_PCP PER_CPU_VAR(spec_ctrl_pcp + \
KERNEL_IBRS_SPEC_CTRL_enabled)
#define IBRS_ENTRY_PCP PER_CPU_VAR(spec_ctrl_pcp + \
KERNEL_IBRS_SPEC_CTRL_entry)
#define IBRS_EXIT_PCP PER_CPU_VAR(spec_ctrl_pcp + \
KERNEL_IBRS_SPEC_CTRL_exit)
#define IBRS_HI32_PCP PER_CPU_VAR(spec_ctrl_pcp + \
KERNEL_IBRS_SPEC_CTRL_hi32)
#ifdef __ASSEMBLY__
#include <asm/msr-index.h>
#include <asm/jump_label.h>
.macro __IBRS_ENTRY
movl IBRS_HI32_PCP, %edx
movl IBRS_ENTRY_PCP, %eax
movl $MSR_IA32_SPEC_CTRL, %ecx
wrmsr
.endm
/*
* IBRS_ENTRY is used only in the syscall paths. We can use dynamic patching
* without problem. In interrupt/exception/nmi paths, dynamic patching
* can be problematic and so per-cpu variable check is still being done.
*/
.macro IBRS_ENTRY
STATIC_JUMP .Librs_\@, ibrs_entry_key
jmp .Lend_\@
.Librs_\@:
pushq %rax
pushq %rcx
pushq %rdx
__IBRS_ENTRY
popq %rdx
popq %rcx
popq %rax
.Lend_\@:
.endm
.macro IBRS_ENTRY_CLOBBER
testl $SPEC_CTRL_PCP_IBRS_ENTRY, IBRS_ENABLED_PCP
jz .Lskip_\@
__IBRS_ENTRY
jmp .Lend_\@
.Lskip_\@:
lfence
.Lend_\@:
.endm
#define NO_IBRS_RESTORE (-1) /* No restore on exit */
/*
* The save_reg is initialize to NO_IBRS_RESTORE just in case IBRS is
* enabled in the middle of an exception, this avoids the very remote risk
* of writing random save_reg content into the SPEC_CTRL MSR in such case.
*/
.macro IBRS_ENTRY_SAVE_AND_CLOBBER save_reg:req
movl $NO_IBRS_RESTORE, \save_reg
testl $SPEC_CTRL_PCP_IBRS_ENTRY, IBRS_ENABLED_PCP
jz .Lskip_\@
movl $MSR_IA32_SPEC_CTRL, %ecx
rdmsr
/*
* If the content of the MSR matches the kernel entry value,
* we should still rewrite the MSR anyway to enforce the
* barrier-like semantics in some IBRS implementations.
* Nowever, we can leave the save_reg as NO_IBRS_RESTORE
* so that we won't do a rewrite on exit,
*
* %edx is initialized by rdmsr above, and so it doesn't need
* to be touched.
*/
movl IBRS_ENTRY_PCP, %ecx
cmpl %eax, %ecx
je .Lwrmsr_\@
movl %eax, \save_reg
.Lwrmsr_\@:
movl $MSR_IA32_SPEC_CTRL, %ecx
wrmsr
jmp .Lend_\@
.Lskip_\@:
lfence
.Lend_\@:
.endm
.macro __IBRS_EXIT
movl IBRS_HI32_PCP, %edx
movl IBRS_EXIT_PCP, %eax
movl $MSR_IA32_SPEC_CTRL, %ecx
wrmsr
.endm
.macro IBRS_EXIT
STATIC_JUMP .Librs_\@, ibrs_exit_key
jmp .Lend_\@
.Librs_\@:
pushq %rax
pushq %rcx
pushq %rdx
__IBRS_EXIT
popq %rdx
popq %rcx
popq %rax
.Lend_\@:
.endm
.macro IBRS_EXIT_RESTORE_CLOBBER save_reg:req
testl $SPEC_CTRL_PCP_IBRS, IBRS_ENABLED_PCP
jz .Lskip_\@
cmpl $NO_IBRS_RESTORE, \save_reg
je .Lskip_\@
movl $MSR_IA32_SPEC_CTRL, %ecx
movl IBRS_HI32_PCP, %edx
movl \save_reg, %eax
wrmsr
.Lskip_\@:
.endm
.macro IBRS_EXIT_CLOBBER
testl $SPEC_CTRL_PCP_IBRS_EXIT, IBRS_ENABLED_PCP
jz .Lskip_\@
__IBRS_EXIT
.Lskip_\@:
.endm
.macro CLEAR_EXTRA_REGS
xorq %r15, %r15
xorq %r14, %r14
xorq %r13, %r13
xorq %r12, %r12
xorq %rbp, %rbp
xorq %rbx, %rbx
.endm
.macro CLEAR_R8_TO_R15
xorq %r15, %r15
xorq %r14, %r14
xorq %r13, %r13
xorq %r12, %r12
xorq %r11, %r11
xorq %r10, %r10
xorq %r9, %r9
xorq %r8, %r8
.endm
.macro CLEAR_R10_TO_R15
xorq %r15, %r15
xorq %r14, %r14
xorq %r13, %r13
xorq %r12, %r12
xorq %r11, %r11
xorq %r10, %r10
.endm
#else /* __ASSEMBLY__ */
#include <linux/ptrace.h>
#include <asm/microcode.h>
#include <asm/nospec-branch.h>
extern struct static_key retp_enabled_key;
extern struct static_key ibrs_present_key;
extern struct static_key ssbd_userset_key;
extern struct static_key ibpb_enabled_key;
/*
* Special SPEC_CTRL MSR value to write the content of the spec_ctrl_pcp.
*/
#define SPEC_CTRL_MSR_REFRESH ((unsigned)-1)
extern void spec_ctrl_rescan_cpuid(void);
extern void spec_ctrl_init(void);
extern void spec_ctrl_cpu_init(void);
extern void ssb_select_mitigation(void);
extern void ssb_print_mitigation(void);
extern void mds_print_mitigation(void);
bool spec_ctrl_force_enable_ibrs(void);
bool spec_ctrl_cond_enable_ibrs(bool full_retpoline);
bool spec_ctrl_enable_ibrs_always(void);
void spec_ctrl_enable_ibrs_enhanced(void);
bool spec_ctrl_force_enable_ibp_disabled(void);
bool spec_ctrl_cond_enable_ibp_disabled(void);
void spec_ctrl_enable_retpoline(void);
bool spec_ctrl_enable_retpoline_ibrs_user(void);
void spec_ctrl_set_ssbd(bool ssbd_on);
enum spectre_v2_mitigation spec_ctrl_get_mitigation(void);
bool unprotected_firmware_begin(void);
void unprotected_firmware_end(bool ibrs_on);
/*
* Percpu IBRS kernel entry/exit control structure
*/
struct kernel_ibrs_spec_ctrl {
unsigned int enabled; /* Entry and exit enabled control bits */
unsigned int exit; /* Lower 32-bit of SPEC_CTRL MSR for exit */
union {
struct {
/*
* The lower and upper 32-bit of SPEC_CTRL MSR
* when entering kernel.
*/
unsigned int entry;
unsigned int hi32;
};
u64 entry64; /* Full 64-bit SPEC_CTRL MSR */
};
};
DECLARE_PER_CPU_USER_MAPPED(struct kernel_ibrs_spec_ctrl, spec_ctrl_pcp);
extern void x86_amd_ssbd_enable(void);
/* The Intel SPEC CTRL MSR base value cache */
extern u64 x86_spec_ctrl_base;
extern u64 x86_spec_ctrl_mask;
static inline u64 ssbd_tif_to_spec_ctrl(u64 tifn)
{
BUILD_BUG_ON(TIF_SSBD < SPEC_CTRL_SSBD_SHIFT);
return (tifn & _TIF_SSBD) >> (TIF_SSBD - SPEC_CTRL_SSBD_SHIFT);
}
static inline unsigned long ssbd_spec_ctrl_to_tif(u64 spec_ctrl)
{
BUILD_BUG_ON(TIF_SSBD < SPEC_CTRL_SSBD_SHIFT);
return (spec_ctrl & SPEC_CTRL_SSBD) << (TIF_SSBD - SPEC_CTRL_SSBD_SHIFT);
}
static inline u64 ssbd_tif_to_amd_ls_cfg(u64 tifn)
{
return (tifn & _TIF_SSBD) ? x86_amd_ls_cfg_ssbd_mask : 0ULL;
}
#ifdef CONFIG_SMP
extern void speculative_store_bypass_ht_init(void);
#else
static inline void speculative_store_bypass_ht_init(void) { }
#endif
extern void speculation_ctrl_update(unsigned long tif);
extern void speculation_ctrl_update_current(void);
enum {
IBRS_DISABLED,
/* in host kernel, disabled in guest and userland */
IBRS_ENABLED,
/* in host kernel and host userland, disabled in guest */
IBRS_ENABLED_ALWAYS,
/* in host userland, disabled in kernel and guest */
IBRS_ENABLED_USER,
/* in both kernel and host userland (enhanced IBRS) */
IBRS_ENHANCED,
IBRS_MAX = IBRS_ENHANCED,
};
static __always_inline int cpu_has_spec_ctrl(void)
{
return static_key_false(&ibrs_present_key);
}
static __always_inline bool ibrs_enabled_kernel(void)
{
/*
* We don't need to do the cpu_has_spec_ctrl() check here as
* the IBRS bit won't be on if no such capability exists.
*/
unsigned int ibrs = __this_cpu_read(spec_ctrl_pcp.entry);
return ibrs & SPEC_CTRL_IBRS;
}
static inline bool retp_enabled(void)
{
return static_key_false(&retp_enabled_key);
}
static inline bool retp_enabled_full(void)
{
return retp_enabled() && retp_compiler();
}
static inline bool ibpb_enabled(void)
{
return static_key_false(&ibpb_enabled_key);
}
/*
* On VMENTER we must preserve whatever view of the SPEC_CTRL MSR
* the guest has, while on VMEXIT we restore the kernel view. This
* would be easier if SPEC_CTRL were architecturally maskable or
* shadowable for guests but this is not (currently) the case.
* Takes the guest view of SPEC_CTRL MSR as a parameter and also
* the guest's version of VIRT_SPEC_CTRL, if emulated.
*/
static __always_inline void
x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
{
/*
* The per-cpu spec_ctrl_pcp.entry64 will be the SPEC_CTRL MSR value
* to be used in host kernel. This is performance critical code.
* Preemption is disabled, so we cannot race with sysfs writes.
*/
u64 msr, guestval, hostval = this_cpu_read(spec_ctrl_pcp.entry64);
struct thread_info *ti = current_thread_info();
bool write_msr;
if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
/*
* Restrict guest_spec_ctrl to supported values. Clear the
* modifiable bits in the host base value and or the
* modifiable bits from the guest value.
*/
guestval = hostval & ~x86_spec_ctrl_mask;
guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
/*
* IBRS may have barrier semantics so it must be set
* during vmexit (!setguest) if SPEC_CTRL MSR write is
* enabled at kernel entry.
*/
write_msr = (!setguest &&
(this_cpu_read(spec_ctrl_pcp.enabled) &
SPEC_CTRL_PCP_IBRS_ENTRY)) ||
(hostval != guestval);
if (unlikely(write_msr)) {
msr = setguest ? guestval : hostval;
native_wrmsrl(MSR_IA32_SPEC_CTRL, msr);
}
}
/*
* If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
* MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
*/
if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
!static_cpu_has(X86_FEATURE_VIRT_SSBD))
goto ret;
/*
* If the SSBD mode is not user settable, grab the SSBD bit
* from x86_spec_ctrl_base. Otherwise, evaluate current's TIF_SSBD
* thread flag.
*/
if (!static_key_false(&ssbd_userset_key))
hostval = x86_spec_ctrl_base & SPEC_CTRL_SSBD;
else
hostval = ssbd_tif_to_spec_ctrl(ti->flags);
/* Sanitize the guest value */
guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
if (hostval != guestval) {
unsigned long tif;
tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
ssbd_spec_ctrl_to_tif(hostval);
speculation_ctrl_update(tif);
}
ret:
/* rmb not needed when entering guest */
if (!setguest) {
/*
* This is an unconditional jump, no wrong speculation
* is possible.
*/
if (retp_enabled_full())
return;
/* rmb to prevent wrong speculation for security */
rmb();
}
}
/**
* x86_spec_ctrl_set_guest - Set speculation control registers for the guest
* @guest_spec_ctrl: The guest content of MSR_SPEC_CTRL
* @guest_virt_spec_ctrl: The guest controlled bits of MSR_VIRT_SPEC_CTRL
* (may get translated to MSR_AMD64_LS_CFG bits)
*
* Avoids writing to the MSR if the content/bits are the same
*/
static __always_inline void x86_spec_ctrl_set_guest(u64 guest_spec_ctrl,
u64 guest_virt_spec_ctrl)
{
x86_virt_spec_ctrl(guest_spec_ctrl, guest_virt_spec_ctrl, true);
}
/**
* x86_spec_ctrl_restore_host - Restore host speculation control registers
* @guest_spec_ctrl: The guest content of MSR_SPEC_CTRL
* @guest_virt_spec_ctrl: The guest controlled bits of MSR_VIRT_SPEC_CTRL
* (may get translated to MSR_AMD64_LS_CFG bits)
*
* Avoids writing to the MSR if the content/bits are the same
*/
static __always_inline void x86_spec_ctrl_restore_host(u64 guest_spec_ctrl,
u64 guest_virt_spec_ctrl)
{
x86_virt_spec_ctrl(guest_spec_ctrl, guest_virt_spec_ctrl, false);
}
/*
* The spec_ctrl_ibrs_off() is called before a cpu enters idle state and
* spec_ctrl_ibrs_off() is called after exit from an idle state.
*
* There is no need to turn off and on IBRS when entering and exiting
* idle state if enhanced IBRS feature is present.
*/
static __always_inline void spec_ctrl_ibrs_on(void)
{
/*
* IBRS may have barrier semantics so it must be set even for ALWAYS
* mode.
*/
if (static_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
if (ibrs_enabled_kernel())
return;
/* Fall back to retpoline check */
} else if (ibrs_enabled_kernel()) {
u64 spec_ctrl = this_cpu_read(spec_ctrl_pcp.entry64);
native_wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
return;
}
/* This is an unconditional jump, no wrong speculation is possible. */
if (retp_enabled_full())
return;
/* rmb to prevent wrong speculation for security */
rmb();
}
static __always_inline void spec_ctrl_ibrs_off(void)
{
if (!static_cpu_has(X86_FEATURE_IBRS_ENHANCED) &&
ibrs_enabled_kernel()) {
u64 spec_ctrl = x86_spec_ctrl_base;
/* SSBD controlled in MSR_SPEC_CTRL */
if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
spec_ctrl |= ssbd_tif_to_spec_ctrl(
current_thread_info()->flags);
native_wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
}
/* rmb not needed when disabling IBRS */
}
/*
* These functions are called before calling into firmware. Firmware might
* have indirect branches, so if we're running with retpolines, we need to
* enable IBRS to protect the kernel from spectre v2.
*
* The 'ibrs_on' variable is used to prevent race conditions. Otherwise, if
* the admin disabled IBRS while a CPU was running in firmware, IBRS could get
* stuck on indefinitely.
*
* There are still other race conditions possible, but they're generally not a
* problem because they'll get corrected on the next kernel exit.
*/
static inline bool spec_ctrl_ibrs_on_firmware(void)
{
bool ibrs_on = false;
if (cpu_has_spec_ctrl() && retp_enabled() && !ibrs_enabled_kernel()) {
u64 spec_ctrl = this_cpu_read(spec_ctrl_pcp.entry64) |
SPEC_CTRL_IBRS;
native_wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
ibrs_on = true;
} else {
/* rmb to prevent wrong speculation for security */
rmb();
}
return ibrs_on;
}
static inline void spec_ctrl_ibrs_off_firmware(bool ibrs_on)
{
if (ibrs_on) {
u64 spec_ctrl = this_cpu_read(spec_ctrl_pcp.entry64);
native_wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl);
} else {
/* rmb to prevent wrong speculation for security */
rmb();
}
}
static inline void __spec_ctrl_ibpb(void)
{
native_wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
}
static inline void spec_ctrl_ibpb(void)
{
if (ibpb_enabled())
__spec_ctrl_ibpb();
}
static inline void spec_ctrl_ibpb_if_different_creds(struct task_struct *next)
{
if (ibpb_enabled() &&
(!next || __ptrace_may_access(next, PTRACE_MODE_IBPB))) {
__spec_ctrl_ibpb();
if (static_cpu_has(X86_FEATURE_SMEP))
fill_RSB();
}
}
extern enum ssb_mitigation ssb_mode;
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_SPEC_CTRL_H */
Mini Shell