2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Copyright (C) 1994 Linus Torvalds
|
|
|
|
*
|
|
|
|
* Pentium III FXSR, SSE support
|
|
|
|
* General FPU state handling cleanups
|
|
|
|
* Gareth Hughes <gareth@valinux.com>, May 2000
|
|
|
|
*/
|
2015-04-24 08:54:44 +08:00
|
|
|
#include <asm/fpu/internal.h>
|
2015-04-30 14:53:18 +08:00
|
|
|
#include <asm/fpu/regset.h>
|
2015-04-30 14:45:02 +08:00
|
|
|
#include <asm/fpu/signal.h>
|
2015-04-30 15:29:38 +08:00
|
|
|
#include <asm/traps.h>
|
2015-04-30 14:45:02 +08:00
|
|
|
|
2015-04-26 22:57:55 +08:00
|
|
|
#include <linux/hardirq.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2015-04-30 17:07:06 +08:00
|
|
|
/*
|
|
|
|
* Represents the initial FPU state. It's mostly (but not completely) zeroes,
|
|
|
|
* depending on the FPU hardware format:
|
|
|
|
*/
|
|
|
|
union thread_xstate init_fpstate __read_mostly;
|
|
|
|
|
2015-04-22 22:52:03 +08:00
|
|
|
/*
|
|
|
|
* Track whether the kernel is using the FPU state
|
|
|
|
* currently.
|
|
|
|
*
|
|
|
|
* This flag is used:
|
|
|
|
*
|
|
|
|
* - by IRQ context code to potentially use the FPU
|
|
|
|
* if it's unused.
|
|
|
|
*
|
|
|
|
* - to debug kernel_fpu_begin()/end() correctness
|
|
|
|
*/
|
2015-01-16 03:19:43 +08:00
|
|
|
static DEFINE_PER_CPU(bool, in_kernel_fpu);
|
|
|
|
|
2015-04-23 18:13:04 +08:00
|
|
|
/*
|
2015-04-23 18:18:28 +08:00
|
|
|
* Track which context is using the FPU on the CPU:
|
2015-04-23 18:13:04 +08:00
|
|
|
*/
|
2015-04-23 18:18:28 +08:00
|
|
|
DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
|
2015-04-23 18:13:04 +08:00
|
|
|
|
2015-04-22 22:33:08 +08:00
|
|
|
static void kernel_fpu_disable(void)
|
2015-01-16 03:20:28 +08:00
|
|
|
{
|
|
|
|
WARN_ON(this_cpu_read(in_kernel_fpu));
|
|
|
|
this_cpu_write(in_kernel_fpu, true);
|
|
|
|
}
|
|
|
|
|
2015-04-22 22:33:08 +08:00
|
|
|
static void kernel_fpu_enable(void)
|
2015-01-16 03:20:28 +08:00
|
|
|
{
|
2015-04-22 22:40:56 +08:00
|
|
|
WARN_ON_ONCE(!this_cpu_read(in_kernel_fpu));
|
2015-01-16 03:20:28 +08:00
|
|
|
this_cpu_write(in_kernel_fpu, false);
|
|
|
|
}
|
|
|
|
|
2015-04-22 22:52:03 +08:00
|
|
|
static bool kernel_fpu_disabled(void)
|
|
|
|
{
|
|
|
|
return this_cpu_read(in_kernel_fpu);
|
|
|
|
}
|
|
|
|
|
2012-02-22 02:25:45 +08:00
|
|
|
/*
|
|
|
|
* Were we in an interrupt that interrupted kernel mode?
|
|
|
|
*
|
2012-08-25 05:13:02 +08:00
|
|
|
* On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
|
2012-02-22 02:25:45 +08:00
|
|
|
* pair does nothing at all: the thread must not have fpu (so
|
|
|
|
* that we don't try to save the FPU state), and TS must
|
|
|
|
* be set (so that the clts/stts pair does nothing that is
|
|
|
|
* visible in the interrupted kernel thread).
|
2013-05-13 20:32:07 +08:00
|
|
|
*
|
2015-01-20 02:51:51 +08:00
|
|
|
* Except for the eagerfpu case when we return true; in the likely case
|
|
|
|
* the thread has FPU but we are not going to set/clear TS.
|
2012-02-22 02:25:45 +08:00
|
|
|
*/
|
2015-04-22 22:33:08 +08:00
|
|
|
static bool interrupted_kernel_fpu_idle(void)
|
2012-02-22 02:25:45 +08:00
|
|
|
{
|
2015-04-22 22:52:03 +08:00
|
|
|
if (kernel_fpu_disabled())
|
2015-01-16 03:19:43 +08:00
|
|
|
return false;
|
|
|
|
|
2012-09-07 05:58:52 +08:00
|
|
|
if (use_eager_fpu())
|
2015-01-20 02:51:51 +08:00
|
|
|
return true;
|
2012-08-25 05:13:02 +08:00
|
|
|
|
2015-04-24 20:19:26 +08:00
|
|
|
return !current->thread.fpu.fpregs_active && (read_cr0() & X86_CR0_TS);
|
2012-02-22 02:25:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Were we in user mode (or vm86 mode) when we were
|
|
|
|
* interrupted?
|
|
|
|
*
|
|
|
|
* Doing kernel_fpu_begin/end() is ok if we are running
|
|
|
|
* in an interrupt context from user mode - we'll just
|
|
|
|
* save the FPU state as required.
|
|
|
|
*/
|
2015-04-22 22:33:08 +08:00
|
|
|
static bool interrupted_user_mode(void)
|
2012-02-22 02:25:45 +08:00
|
|
|
{
|
|
|
|
struct pt_regs *regs = get_irq_regs();
|
2015-03-19 09:33:33 +08:00
|
|
|
return regs && user_mode(regs);
|
2012-02-22 02:25:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Can we use the FPU in kernel mode with the
|
|
|
|
* whole "kernel_fpu_begin/end()" sequence?
|
|
|
|
*
|
|
|
|
* It's always ok in process context (ie "not interrupt")
|
|
|
|
* but it is sometimes ok even from an irq.
|
|
|
|
*/
|
|
|
|
bool irq_fpu_usable(void)
|
|
|
|
{
|
|
|
|
return !in_interrupt() ||
|
|
|
|
interrupted_user_mode() ||
|
|
|
|
interrupted_kernel_fpu_idle();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(irq_fpu_usable);
|
|
|
|
|
2012-09-21 02:01:49 +08:00
|
|
|
void __kernel_fpu_begin(void)
|
2012-02-22 02:25:45 +08:00
|
|
|
{
|
2015-04-23 18:18:28 +08:00
|
|
|
struct fpu *fpu = ¤t->thread.fpu;
|
2012-02-22 02:25:45 +08:00
|
|
|
|
2015-04-22 22:40:56 +08:00
|
|
|
kernel_fpu_disable();
|
2015-01-16 03:19:43 +08:00
|
|
|
|
2015-04-24 20:19:26 +08:00
|
|
|
if (fpu->fpregs_active) {
|
x86/fpu: Rename fpu_save_init() to copy_fpregs_to_fpstate()
So fpu_save_init() is a historic name that got its name when the only
way the FPU state was FNSAVE, which cleared (well, destroyed) the FPU
state after saving it.
Nowadays the name is misleading, because ever since the introduction of
FXSAVE (and more modern FPU saving instructions) the 'we need to reload
the FPU state' part is only true if there's a pending FPU exception [*],
which is almost never the case.
So rename it to copy_fpregs_to_fpstate() to make it clear what's
happening. Also add a few comments about why we cannot keep registers
in certain cases.
Also clean up the control flow a bit, to make it more apparent when
we are dropping/keeping FP registers, and to optimize the common
case (of keeping fpregs) some more.
[*] Probably not true anymore, modern instructions always leave the FPU
state intact, even if exceptions are pending: because pending FP
exceptions are posted on the next FP instruction, not asynchronously.
They were truly asynchronous back in the IRQ13 case, and we had to
synchronize with them, but that code is not working anymore: we don't
have IRQ13 mapped in the IDT anymore.
But a cleanup patch is obviously not the place to change subtle behavior.
Reviewed-by: Borislav Petkov <bp@alien8.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-27 08:53:16 +08:00
|
|
|
copy_fpregs_to_fpstate(fpu);
|
2015-01-20 02:51:32 +08:00
|
|
|
} else {
|
2015-04-23 18:18:28 +08:00
|
|
|
this_cpu_write(fpu_fpregs_owner_ctx, NULL);
|
2015-04-27 14:58:45 +08:00
|
|
|
__fpregs_activate_hw();
|
2012-02-22 02:25:45 +08:00
|
|
|
}
|
|
|
|
}
|
2012-09-21 02:01:49 +08:00
|
|
|
EXPORT_SYMBOL(__kernel_fpu_begin);
|
2012-02-22 02:25:45 +08:00
|
|
|
|
2012-09-21 02:01:49 +08:00
|
|
|
void __kernel_fpu_end(void)
|
2012-02-22 02:25:45 +08:00
|
|
|
{
|
2015-04-23 23:34:20 +08:00
|
|
|
struct fpu *fpu = ¤t->thread.fpu;
|
2015-01-16 03:20:05 +08:00
|
|
|
|
2015-04-24 20:19:26 +08:00
|
|
|
if (fpu->fpregs_active) {
|
2015-04-30 02:10:43 +08:00
|
|
|
if (WARN_ON(copy_fpstate_to_fpregs(fpu)))
|
2015-04-30 13:12:46 +08:00
|
|
|
fpu__clear(fpu);
|
2015-04-27 14:58:45 +08:00
|
|
|
} else {
|
|
|
|
__fpregs_deactivate_hw();
|
x86, fpu: Check tsk_used_math() in kernel_fpu_end() for eager FPU
For non-eager fpu mode, thread's fpu state is allocated during the first
fpu usage (in the context of device not available exception). This
(math_state_restore()) can be a blocking call and hence we enable
interrupts (which were originally disabled when the exception happened),
allocate memory and disable interrupts etc.
But the eager-fpu mode, call's the same math_state_restore() from
kernel_fpu_end(). The assumption being that tsk_used_math() is always
set for the eager-fpu mode and thus avoid the code path of enabling
interrupts, allocating fpu state using blocking call and disable
interrupts etc.
But the below issue was noticed by Maarten Baert, Nate Eldredge and
few others:
If a user process dumps core on an ecrypt fs while aesni-intel is loaded,
we get a BUG() in __find_get_block() complaining that it was called with
interrupts disabled; then all further accesses to our ecrypt fs hang
and we have to reboot.
The aesni-intel code (encrypting the core file that we are writing) needs
the FPU and quite properly wraps its code in kernel_fpu_{begin,end}(),
the latter of which calls math_state_restore(). So after kernel_fpu_end(),
interrupts may be disabled, which nobody seems to expect, and they stay
that way until we eventually get to __find_get_block() which barfs.
For eager fpu, most the time, tsk_used_math() is true. At few instances
during thread exit, signal return handling etc, tsk_used_math() might
be false.
In kernel_fpu_end(), for eager-fpu, call math_state_restore()
only if tsk_used_math() is set. Otherwise, don't bother. Kernel code
path which cleared tsk_used_math() knows what needs to be done
with the fpu state.
Reported-by: Maarten Baert <maarten-baert@hotmail.com>
Reported-by: Nate Eldredge <nate@thatsmathematics.com>
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Suresh Siddha <sbsiddha@gmail.com>
Link: http://lkml.kernel.org/r/1391410583.3801.6.camel@europa
Cc: George Spelvin <linux@horizon.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2014-02-03 14:56:23 +08:00
|
|
|
}
|
2015-01-16 03:19:43 +08:00
|
|
|
|
2015-04-22 22:40:56 +08:00
|
|
|
kernel_fpu_enable();
|
2012-02-22 02:25:45 +08:00
|
|
|
}
|
2012-09-21 02:01:49 +08:00
|
|
|
EXPORT_SYMBOL(__kernel_fpu_end);
|
2012-02-22 02:25:45 +08:00
|
|
|
|
2015-04-26 18:07:18 +08:00
|
|
|
void kernel_fpu_begin(void)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
|
|
|
WARN_ON_ONCE(!irq_fpu_usable());
|
|
|
|
__kernel_fpu_begin();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(kernel_fpu_begin);
|
|
|
|
|
|
|
|
void kernel_fpu_end(void)
|
|
|
|
{
|
|
|
|
__kernel_fpu_end();
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(kernel_fpu_end);
|
|
|
|
|
2015-04-26 22:57:55 +08:00
|
|
|
/*
|
|
|
|
* CR0::TS save/restore functions:
|
|
|
|
*/
|
|
|
|
int irq_ts_save(void)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* If in process context and not atomic, we can take a spurious DNA fault.
|
|
|
|
* Otherwise, doing clts() in process context requires disabling preemption
|
|
|
|
* or some heavy lifting like kernel_fpu_begin()
|
|
|
|
*/
|
|
|
|
if (!in_atomic())
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (read_cr0() & X86_CR0_TS) {
|
|
|
|
clts();
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(irq_ts_save);
|
|
|
|
|
|
|
|
void irq_ts_restore(int TS_state)
|
|
|
|
{
|
|
|
|
if (TS_state)
|
|
|
|
stts();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(irq_ts_restore);
|
|
|
|
|
2015-04-03 17:01:36 +08:00
|
|
|
/*
|
2015-04-27 15:45:12 +08:00
|
|
|
* Save the FPU state (mark it for reload if necessary):
|
2015-04-03 17:06:43 +08:00
|
|
|
*
|
|
|
|
* This only ever gets called for the current task.
|
2015-04-03 17:01:36 +08:00
|
|
|
*/
|
2015-04-23 23:57:24 +08:00
|
|
|
void fpu__save(struct fpu *fpu)
|
2012-02-22 02:25:45 +08:00
|
|
|
{
|
2015-04-23 23:57:24 +08:00
|
|
|
WARN_ON(fpu != ¤t->thread.fpu);
|
2015-04-03 17:06:43 +08:00
|
|
|
|
2012-02-22 02:25:45 +08:00
|
|
|
preempt_disable();
|
2015-04-24 20:19:26 +08:00
|
|
|
if (fpu->fpregs_active) {
|
2015-04-27 15:45:12 +08:00
|
|
|
if (!copy_fpregs_to_fpstate(fpu))
|
2015-04-24 20:31:27 +08:00
|
|
|
fpregs_deactivate(fpu);
|
2015-02-07 04:01:58 +08:00
|
|
|
}
|
2012-02-22 02:25:45 +08:00
|
|
|
preempt_enable();
|
|
|
|
}
|
2015-04-03 17:01:36 +08:00
|
|
|
EXPORT_SYMBOL_GPL(fpu__save);
|
2012-02-22 02:25:45 +08:00
|
|
|
|
2015-04-30 16:08:36 +08:00
|
|
|
/*
|
|
|
|
* Legacy x87 fpstate state init:
|
|
|
|
*/
|
|
|
|
static inline void fpstate_init_fstate(struct i387_fsave_struct *fp)
|
|
|
|
{
|
|
|
|
fp->cwd = 0xffff037fu;
|
|
|
|
fp->swd = 0xffff0000u;
|
|
|
|
fp->twd = 0xffffffffu;
|
|
|
|
fp->fos = 0xffff0000u;
|
|
|
|
}
|
|
|
|
|
2015-04-30 16:23:42 +08:00
|
|
|
void fpstate_init(union thread_xstate *state)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2013-04-29 22:04:20 +08:00
|
|
|
if (!cpu_has_fpu) {
|
2015-04-30 16:23:42 +08:00
|
|
|
fpstate_init_soft(&state->soft);
|
2010-05-06 16:45:46 +08:00
|
|
|
return;
|
2008-05-24 07:26:37 +08:00
|
|
|
}
|
|
|
|
|
2015-04-30 16:23:42 +08:00
|
|
|
memset(state, 0, xstate_size);
|
2015-03-10 14:06:25 +08:00
|
|
|
|
2015-04-30 16:08:36 +08:00
|
|
|
if (cpu_has_fxsr)
|
2015-04-30 16:23:42 +08:00
|
|
|
fpstate_init_fxstate(&state->fxsave);
|
2015-04-30 16:08:36 +08:00
|
|
|
else
|
2015-04-30 16:23:42 +08:00
|
|
|
fpstate_init_fstate(&state->fsave);
|
2010-05-06 16:45:46 +08:00
|
|
|
}
|
2015-04-03 19:01:52 +08:00
|
|
|
EXPORT_SYMBOL_GPL(fpstate_init);
|
2010-05-06 16:45:46 +08:00
|
|
|
|
2015-04-23 14:55:34 +08:00
|
|
|
/*
|
|
|
|
* Copy the current task's FPU state to a new task's FPU context.
|
|
|
|
*
|
|
|
|
* In the 'eager' case we just save to the destination context.
|
|
|
|
*
|
|
|
|
* In the 'lazy' case we save to the source context, mark the FPU lazy
|
|
|
|
* via stts() and copy the source context into the destination context.
|
|
|
|
*/
|
2015-04-24 08:07:33 +08:00
|
|
|
static void fpu_copy(struct fpu *dst_fpu, struct fpu *src_fpu)
|
2015-04-23 02:09:29 +08:00
|
|
|
{
|
2015-04-24 08:07:33 +08:00
|
|
|
WARN_ON(src_fpu != ¤t->thread.fpu);
|
2015-04-23 14:55:34 +08:00
|
|
|
|
2015-04-27 16:08:39 +08:00
|
|
|
/*
|
|
|
|
* Don't let 'init optimized' areas of the XSAVE area
|
|
|
|
* leak into the child task:
|
|
|
|
*/
|
|
|
|
if (use_eager_fpu())
|
x86/fpu: Simplify FPU handling by embedding the fpstate in task_struct (again)
So 6 years ago we made the FPU fpstate dynamically allocated:
aa283f49276e ("x86, fpu: lazy allocation of FPU area - v5")
61c4628b5386 ("x86, fpu: split FPU state from task struct - v5")
In hindsight this was a mistake:
- it complicated context allocation failure handling, such as:
/* kthread execs. TODO: cleanup this horror. */
if (WARN_ON(fpstate_alloc_init(fpu)))
force_sig(SIGKILL, tsk);
- it caused us to enable irqs in fpu__restore():
local_irq_enable();
/*
* does a slab alloc which can sleep
*/
if (fpstate_alloc_init(fpu)) {
/*
* ran out of memory!
*/
do_group_exit(SIGKILL);
return;
}
local_irq_disable();
- it (slightly) slowed down task creation/destruction by adding
slab allocation/free pattens.
- it made access to context contents (slightly) slower by adding
one more pointer dereference.
The motivation for the dynamic allocation was two-fold:
- reduce memory consumption by non-FPU tasks
- allocate and handle only the necessary amount of context for
various XSAVE processors that have varying hardware frame
sizes.
These days, with glibc using SSE memcpy by default and GCC optimizing
for SSE/AVX by default, the scope of FPU using apps on an x86 system is
much larger than it was 6 years ago.
For example on a freshly installed Fedora 21 desktop system, with a
recent kernel, all non-kthread tasks have used the FPU shortly after
bootup.
Also, even modern embedded x86 CPUs try to support the latest vector
instruction set - so they'll too often use the larger xstate frame
sizes.
So remove the dynamic allocation complication by embedding the FPU
fpstate in task_struct again. This should make the FPU a lot more
accessible to all sorts of atomic contexts.
We could still optimize for the xstate frame size in the future,
by moving the state structure to the last element of task_struct,
and allocating only a part of that.
This change is kept minimal by still keeping the ctx_alloc()/free()
routines (that now do nothing substantial) - we'll remove them in
the following patches.
Reviewed-by: Borislav Petkov <bp@alien8.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-27 10:19:39 +08:00
|
|
|
memset(&dst_fpu->state.xsave, 0, xstate_size);
|
2015-04-27 16:08:39 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Save current FPU registers directly into the child
|
|
|
|
* FPU context, without any memory-to-memory copying.
|
|
|
|
*
|
|
|
|
* If the FPU context got destroyed in the process (FNSAVE
|
|
|
|
* done on old CPUs) then copy it back into the source
|
|
|
|
* context and mark the current task for lazy restore.
|
|
|
|
*
|
|
|
|
* We have to do all this with preemption disabled,
|
|
|
|
* mostly because of the FNSAVE case, because in that
|
|
|
|
* case we must not allow preemption in the window
|
|
|
|
* between the FNSAVE and us marking the context lazy.
|
|
|
|
*
|
|
|
|
* It shouldn't be an issue as even FNSAVE is plenty
|
|
|
|
* fast in terms of critical section length.
|
|
|
|
*/
|
|
|
|
preempt_disable();
|
|
|
|
if (!copy_fpregs_to_fpstate(dst_fpu)) {
|
|
|
|
memcpy(&src_fpu->state, &dst_fpu->state, xstate_size);
|
|
|
|
fpregs_deactivate(src_fpu);
|
2015-04-23 02:09:29 +08:00
|
|
|
}
|
2015-04-27 16:08:39 +08:00
|
|
|
preempt_enable();
|
2015-04-23 02:09:29 +08:00
|
|
|
}
|
|
|
|
|
2015-04-24 08:07:15 +08:00
|
|
|
int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu)
|
2015-04-22 21:47:05 +08:00
|
|
|
{
|
2015-04-24 08:07:15 +08:00
|
|
|
dst_fpu->counter = 0;
|
2015-04-24 20:19:26 +08:00
|
|
|
dst_fpu->fpregs_active = 0;
|
2015-04-24 08:07:15 +08:00
|
|
|
dst_fpu->last_cpu = -1;
|
2015-04-22 21:47:05 +08:00
|
|
|
|
2015-04-27 11:52:40 +08:00
|
|
|
if (src_fpu->fpstate_active)
|
2015-04-24 08:07:33 +08:00
|
|
|
fpu_copy(dst_fpu, src_fpu);
|
2015-04-27 11:52:40 +08:00
|
|
|
|
2015-04-22 21:47:05 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-04-03 18:02:02 +08:00
|
|
|
/*
|
2015-04-27 13:18:17 +08:00
|
|
|
* Activate the current task's in-memory FPU context,
|
|
|
|
* if it has not been used before:
|
2015-04-03 18:02:02 +08:00
|
|
|
*/
|
2015-04-27 13:18:17 +08:00
|
|
|
void fpu__activate_curr(struct fpu *fpu)
|
2015-04-03 18:02:02 +08:00
|
|
|
{
|
2015-04-27 12:46:52 +08:00
|
|
|
WARN_ON_ONCE(fpu != ¤t->thread.fpu);
|
2015-04-03 18:02:02 +08:00
|
|
|
|
2015-04-27 13:18:17 +08:00
|
|
|
if (!fpu->fpstate_active) {
|
2015-04-30 16:23:42 +08:00
|
|
|
fpstate_init(&fpu->state);
|
2015-04-03 18:02:02 +08:00
|
|
|
|
2015-04-27 13:18:17 +08:00
|
|
|
/* Safe to do for the current task: */
|
|
|
|
fpu->fpstate_active = 1;
|
|
|
|
}
|
2015-04-03 18:02:02 +08:00
|
|
|
}
|
2015-04-27 13:18:17 +08:00
|
|
|
EXPORT_SYMBOL_GPL(fpu__activate_curr);
|
2015-04-03 18:02:02 +08:00
|
|
|
|
2010-05-06 16:45:46 +08:00
|
|
|
/*
|
2015-04-27 13:22:58 +08:00
|
|
|
* This function must be called before we modify a stopped child's
|
|
|
|
* fpstate.
|
2015-04-23 20:06:05 +08:00
|
|
|
*
|
|
|
|
* If the child has not used the FPU before then initialize its
|
2015-04-27 13:22:58 +08:00
|
|
|
* fpstate.
|
2015-04-23 20:06:05 +08:00
|
|
|
*
|
|
|
|
* If the child has used the FPU before then unlazy it.
|
|
|
|
*
|
2015-04-27 13:22:58 +08:00
|
|
|
* [ After this function call, after registers in the fpstate are
|
|
|
|
* modified and the child task has woken up, the child task will
|
|
|
|
* restore the modified FPU state from the modified context. If we
|
2015-04-23 20:06:05 +08:00
|
|
|
* didn't clear its lazy status here then the lazy in-registers
|
2015-04-27 13:22:58 +08:00
|
|
|
* state pending on its former CPU could be restored, corrupting
|
2015-04-23 20:06:05 +08:00
|
|
|
* the modifications. ]
|
|
|
|
*
|
|
|
|
* This function is also called before we read a stopped child's
|
2015-04-27 13:22:58 +08:00
|
|
|
* FPU state - to make sure it's initialized if the child has
|
|
|
|
* no active FPU state.
|
2015-04-23 20:06:05 +08:00
|
|
|
*
|
|
|
|
* TODO: A future optimization would be to skip the unlazying in
|
|
|
|
* the read-only case, it's not strictly necessary for
|
|
|
|
* read-only access to the context.
|
2010-05-06 16:45:46 +08:00
|
|
|
*/
|
2015-04-30 18:59:30 +08:00
|
|
|
void fpu__activate_stopped(struct fpu *child_fpu)
|
2010-05-06 16:45:46 +08:00
|
|
|
{
|
2015-04-27 12:55:54 +08:00
|
|
|
WARN_ON_ONCE(child_fpu == ¤t->thread.fpu);
|
2015-04-03 18:21:03 +08:00
|
|
|
|
2015-04-23 18:49:20 +08:00
|
|
|
if (child_fpu->fpstate_active) {
|
2015-04-24 08:18:23 +08:00
|
|
|
child_fpu->last_cpu = -1;
|
2015-04-27 12:55:54 +08:00
|
|
|
} else {
|
2015-04-30 16:23:42 +08:00
|
|
|
fpstate_init(&child_fpu->state);
|
2015-04-03 18:29:47 +08:00
|
|
|
|
2015-04-27 12:55:54 +08:00
|
|
|
/* Safe to do for stopped child tasks: */
|
|
|
|
child_fpu->fpstate_active = 1;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2015-04-22 18:50:13 +08:00
|
|
|
/*
|
2015-04-28 18:53:45 +08:00
|
|
|
* 'fpu__restore()' is called to copy FPU registers from
|
|
|
|
* the FPU fpstate to the live hw registers and to activate
|
|
|
|
* access to the hardware registers, so that FPU instructions
|
|
|
|
* can be used afterwards.
|
2015-04-22 18:50:13 +08:00
|
|
|
*
|
2015-04-28 18:53:45 +08:00
|
|
|
* Must be called with kernel preemption disabled (for example
|
|
|
|
* with local interrupts disabled, as it is in the case of
|
|
|
|
* do_device_not_available()).
|
2015-04-22 18:50:13 +08:00
|
|
|
*/
|
2015-04-22 19:16:47 +08:00
|
|
|
void fpu__restore(void)
|
2015-04-22 18:50:13 +08:00
|
|
|
{
|
|
|
|
struct task_struct *tsk = current;
|
2015-04-23 18:31:17 +08:00
|
|
|
struct fpu *fpu = &tsk->thread.fpu;
|
2015-04-22 18:50:13 +08:00
|
|
|
|
2015-04-27 13:18:17 +08:00
|
|
|
fpu__activate_curr(fpu);
|
2015-04-22 18:50:13 +08:00
|
|
|
|
2015-04-24 20:30:38 +08:00
|
|
|
/* Avoid __kernel_fpu_begin() right after fpregs_activate() */
|
2015-04-22 18:50:13 +08:00
|
|
|
kernel_fpu_disable();
|
2015-04-24 20:30:38 +08:00
|
|
|
fpregs_activate(fpu);
|
2015-04-30 02:10:43 +08:00
|
|
|
if (unlikely(copy_fpstate_to_fpregs(fpu))) {
|
2015-04-30 13:12:46 +08:00
|
|
|
fpu__clear(fpu);
|
2015-04-22 18:50:13 +08:00
|
|
|
force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
|
|
|
|
} else {
|
|
|
|
tsk->thread.fpu.counter++;
|
|
|
|
}
|
|
|
|
kernel_fpu_enable();
|
|
|
|
}
|
2015-04-22 19:16:47 +08:00
|
|
|
EXPORT_SYMBOL_GPL(fpu__restore);
|
2015-04-22 18:50:13 +08:00
|
|
|
|
2015-04-30 02:24:14 +08:00
|
|
|
/*
|
|
|
|
* Drops current FPU state: deactivates the fpregs and
|
|
|
|
* the fpstate. NOTE: it still leaves previous contents
|
|
|
|
* in the fpregs in the eager-FPU case.
|
|
|
|
*
|
|
|
|
* This function can be used in cases where we know that
|
|
|
|
* a state-restore is coming: either an explicit one,
|
|
|
|
* or a reschedule.
|
|
|
|
*/
|
|
|
|
void fpu__drop(struct fpu *fpu)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
|
|
|
fpu->counter = 0;
|
|
|
|
|
|
|
|
if (fpu->fpregs_active) {
|
|
|
|
/* Ignore delayed exceptions from user space */
|
|
|
|
asm volatile("1: fwait\n"
|
|
|
|
"2:\n"
|
|
|
|
_ASM_EXTABLE(1b, 2b));
|
|
|
|
fpregs_deactivate(fpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
fpu->fpstate_active = 0;
|
|
|
|
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
2015-04-30 17:21:59 +08:00
|
|
|
/*
|
|
|
|
* Clear FPU registers by setting them up from
|
|
|
|
* the init fpstate:
|
|
|
|
*/
|
|
|
|
static inline void copy_init_fpstate_to_fpregs(void)
|
|
|
|
{
|
|
|
|
if (use_xsave())
|
2015-04-30 17:34:09 +08:00
|
|
|
copy_kernel_to_xregs(&init_fpstate.xsave, -1);
|
2015-04-30 17:21:59 +08:00
|
|
|
else
|
2015-04-30 17:34:09 +08:00
|
|
|
copy_kernel_to_fxregs(&init_fpstate.fxsave);
|
2015-04-30 17:21:59 +08:00
|
|
|
}
|
|
|
|
|
2015-04-30 02:24:14 +08:00
|
|
|
/*
|
2015-04-30 13:12:46 +08:00
|
|
|
* Clear the FPU state back to init state.
|
|
|
|
*
|
|
|
|
* Called by sys_execve(), by the signal handler code and by various
|
|
|
|
* error paths.
|
2015-04-29 14:46:26 +08:00
|
|
|
*/
|
2015-04-30 02:35:33 +08:00
|
|
|
void fpu__clear(struct fpu *fpu)
|
2015-04-22 17:52:13 +08:00
|
|
|
{
|
2015-04-30 02:35:33 +08:00
|
|
|
WARN_ON_ONCE(fpu != ¤t->thread.fpu); /* Almost certainly an anomaly */
|
2015-04-23 18:46:20 +08:00
|
|
|
|
2015-04-22 17:52:13 +08:00
|
|
|
if (!use_eager_fpu()) {
|
|
|
|
/* FPU state will be reallocated lazily at the first use. */
|
x86/fpu: Synchronize the naming of drop_fpu() and fpu_reset_state()
drop_fpu() and fpu_reset_state() are similar in functionality
and in scope, yet this is not apparent from their names.
drop_fpu() deactivates FPU contents (both the fpregs and the fpstate),
but leaves register contents intact in the eager-FPU case, mostly as an
optimization. It disables fpregs in the lazy FPU case. The drop_fpu()
method can be used to destroy FPU state in an optimized way, when we
know that a new state will be loaded before user-space might see
any remains of the old FPU state:
- such as in sys_exit()'s exit_thread() where we know this task
won't execute any user-space instructions anymore and the
next context switch cleans up the FPU. The old FPU state
might still be around in the eagerfpu case but won't be
saved.
- in __restore_xstate_sig(), where we use drop_fpu() before
copying a new state into the fpstate and activating that one.
No user-pace instructions can execute between those steps.
- in sys_execve()'s fpu__clear(): there we use drop_fpu() in
the !eagerfpu case, where it's equivalent to a full reinit.
fpu_reset_state() is a stronger version of drop_fpu(): both in
the eagerfpu and the lazy-FPU case it guarantees that fpregs
are reinitialized to init state. This method is used in cases
where we need a full reset:
- handle_signal() uses fpu_reset_state() to reset the FPU state
to init before executing a user-space signal handler. While we
have already saved the original FPU state at this point, and
always restore the original state, the signal handling code
still has to do this reinit, because signals may interrupt
any user-space instruction, and the FPU might be in various
intermediate states (such as an unbalanced x87 stack) that is
not immediately usable for general C signal handler code.
- __restore_xstate_sig() uses fpu_reset_state() when the signal
frame has no FP context. Since the signal handler may have
modified the FPU state, it gets reset back to init state.
- in another branch __restore_xstate_sig() uses fpu_reset_state()
to handle a restoration error: when restore_user_xstate() fails
to restore FPU state and we might have inconsistent FPU data,
fpu_reset_state() is used to reset it back to a known good
state.
- __kernel_fpu_end() uses fpu_reset_state() in an error branch.
This is in a 'must not trigger' error branch, so on bug-free
kernels this never triggers.
- fpu__restore() uses fpu_reset_state() in an error path
as well: if the fpstate was set up with invalid FPU state
(via ptrace or via a signal handler), then it's reset back
to init state.
- likewise, the scheduler's switch_fpu_finish() uses it in a
restoration error path too.
Move both drop_fpu() and fpu_reset_state() to the fpu__*() namespace
and harmonize their naming with their function:
fpu__drop()
fpu__reset()
This clearly shows that both methods operate on the full state of the
FPU, just like fpu__restore().
Also add comments to explain what each function does.
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-04-30 01:04:31 +08:00
|
|
|
fpu__drop(fpu);
|
2015-04-22 17:52:13 +08:00
|
|
|
} else {
|
2015-04-23 18:49:20 +08:00
|
|
|
if (!fpu->fpstate_active) {
|
2015-04-27 13:18:17 +08:00
|
|
|
fpu__activate_curr(fpu);
|
2015-04-22 17:52:13 +08:00
|
|
|
user_fpu_begin();
|
|
|
|
}
|
2015-04-30 17:21:59 +08:00
|
|
|
copy_init_fpstate_to_fpregs();
|
2015-04-22 17:52:13 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-04-30 15:29:38 +08:00
|
|
|
/*
|
|
|
|
* x87 math exception handling:
|
|
|
|
*/
|
|
|
|
|
|
|
|
static inline unsigned short get_fpu_cwd(struct fpu *fpu)
|
|
|
|
{
|
|
|
|
if (cpu_has_fxsr) {
|
|
|
|
return fpu->state.fxsave.cwd;
|
|
|
|
} else {
|
|
|
|
return (unsigned short)fpu->state.fsave.cwd;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned short get_fpu_swd(struct fpu *fpu)
|
|
|
|
{
|
|
|
|
if (cpu_has_fxsr) {
|
|
|
|
return fpu->state.fxsave.swd;
|
|
|
|
} else {
|
|
|
|
return (unsigned short)fpu->state.fsave.swd;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned short get_fpu_mxcsr(struct fpu *fpu)
|
|
|
|
{
|
|
|
|
if (cpu_has_xmm) {
|
|
|
|
return fpu->state.fxsave.mxcsr;
|
|
|
|
} else {
|
|
|
|
return MXCSR_DEFAULT;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int fpu__exception_code(struct fpu *fpu, int trap_nr)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (trap_nr == X86_TRAP_MF) {
|
|
|
|
unsigned short cwd, swd;
|
|
|
|
/*
|
|
|
|
* (~cwd & swd) will mask out exceptions that are not set to unmasked
|
|
|
|
* status. 0x3f is the exception bits in these regs, 0x200 is the
|
|
|
|
* C1 reg you need in case of a stack fault, 0x040 is the stack
|
|
|
|
* fault bit. We should only be taking one exception at a time,
|
|
|
|
* so if this combination doesn't produce any single exception,
|
|
|
|
* then we have a bad program that isn't synchronizing its FPU usage
|
|
|
|
* and it will suffer the consequences since we won't be able to
|
|
|
|
* fully reproduce the context of the exception
|
|
|
|
*/
|
|
|
|
cwd = get_fpu_cwd(fpu);
|
|
|
|
swd = get_fpu_swd(fpu);
|
|
|
|
|
|
|
|
err = swd & ~cwd;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The SIMD FPU exceptions are handled a little differently, as there
|
|
|
|
* is only a single status/control register. Thus, to determine which
|
|
|
|
* unmasked exception was caught we must mask the exception mask bits
|
|
|
|
* at 0x1f80, and then use these to mask the exception bits at 0x3f.
|
|
|
|
*/
|
|
|
|
unsigned short mxcsr = get_fpu_mxcsr(fpu);
|
|
|
|
err = ~(mxcsr >> 7) & mxcsr;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (err & 0x001) { /* Invalid op */
|
|
|
|
/*
|
|
|
|
* swd & 0x240 == 0x040: Stack Underflow
|
|
|
|
* swd & 0x240 == 0x240: Stack Overflow
|
|
|
|
* User must clear the SF bit (0x40) if set
|
|
|
|
*/
|
|
|
|
return FPE_FLTINV;
|
|
|
|
} else if (err & 0x004) { /* Divide by Zero */
|
|
|
|
return FPE_FLTDIV;
|
|
|
|
} else if (err & 0x008) { /* Overflow */
|
|
|
|
return FPE_FLTOVF;
|
|
|
|
} else if (err & 0x012) { /* Denormal, Underflow */
|
|
|
|
return FPE_FLTUND;
|
|
|
|
} else if (err & 0x020) { /* Precision */
|
|
|
|
return FPE_FLTRES;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we're using IRQ 13, or supposedly even some trap
|
|
|
|
* X86_TRAP_MF implementations, it's possible
|
|
|
|
* we get a spurious trap, which is not an error.
|
|
|
|
*/
|
|
|
|
return 0;
|
|
|
|
}
|