2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* linux/arch/x86_64/entry.S
|
|
|
|
*
|
|
|
|
* Copyright (C) 1991, 1992 Linus Torvalds
|
|
|
|
* Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
|
|
|
|
* Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* entry.S contains the system-call and fault low-level handling routines.
|
|
|
|
*
|
|
|
|
* NOTE: This code handles signal-recognition, which happens every time
|
|
|
|
* after an interrupt and after each system call.
|
|
|
|
*
|
|
|
|
* Normal syscalls and interrupts don't save a full stack frame, this is
|
|
|
|
* only done for syscall tracing, signals or fork/exec et.al.
|
|
|
|
*
|
|
|
|
* A note on terminology:
|
|
|
|
* - top of stack: Architecture defined interrupt frame from SS to RIP
|
|
|
|
* at the top of the kernel process stack.
|
|
|
|
* - partial stack frame: partially saved registers upto R11.
|
|
|
|
* - full stack frame: Like partial stack frame, but all register saved.
|
2006-09-26 16:52:29 +08:00
|
|
|
*
|
|
|
|
* Some macro usage:
|
|
|
|
* - CFI macros are used to generate dwarf2 unwind information for better
|
|
|
|
* backtraces. They don't change any code.
|
|
|
|
* - SAVE_ALL/RESTORE_ALL - Save/restore all registers
|
|
|
|
* - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
|
|
|
|
* There are unfortunately lots of special cases where some registers
|
|
|
|
* not touched. The macro is a big mess that should be cleaned up.
|
|
|
|
* - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
|
|
|
|
* Gives a full stack frame.
|
|
|
|
* - ENTRY/END Define functions in the symbol table.
|
|
|
|
* - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
|
|
|
|
* frame that is otherwise undefined after a SYSCALL
|
|
|
|
* - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
|
|
|
|
* - errorentry/paranoidentry/zeroentry - Define exception entry points.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/linkage.h>
|
|
|
|
#include <asm/segment.h>
|
|
|
|
#include <asm/cache.h>
|
|
|
|
#include <asm/errno.h>
|
|
|
|
#include <asm/dwarf2.h>
|
|
|
|
#include <asm/calling.h>
|
2005-09-10 03:28:48 +08:00
|
|
|
#include <asm/asm-offsets.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/msr.h>
|
|
|
|
#include <asm/unistd.h>
|
|
|
|
#include <asm/thread_info.h>
|
|
|
|
#include <asm/hw_irq.h>
|
2006-01-16 08:56:39 +08:00
|
|
|
#include <asm/page.h>
|
2006-07-03 15:24:45 +08:00
|
|
|
#include <asm/irqflags.h>
|
2008-01-30 20:32:08 +08:00
|
|
|
#include <asm/paravirt.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
.code64
|
|
|
|
|
2005-04-17 06:25:05 +08:00
|
|
|
#ifndef CONFIG_PREEMPT
|
2005-04-17 06:20:36 +08:00
|
|
|
#define retint_kernel retint_restore_args
|
|
|
|
#endif
|
2006-07-03 15:24:45 +08:00
|
|
|
|
2008-01-30 20:32:08 +08:00
|
|
|
#ifdef CONFIG_PARAVIRT
|
|
|
|
ENTRY(native_irq_enable_syscall_ret)
|
|
|
|
movq %gs:pda_oldrsp,%rsp
|
|
|
|
swapgs
|
|
|
|
sysretq
|
|
|
|
#endif /* CONFIG_PARAVIRT */
|
|
|
|
|
2006-07-03 15:24:45 +08:00
|
|
|
|
|
|
|
.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
|
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
|
|
|
|
jnc 1f
|
|
|
|
TRACE_IRQS_ON
|
|
|
|
1:
|
|
|
|
#endif
|
|
|
|
.endm
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* C code is not supposed to know about undefined top of stack. Every time
|
|
|
|
* a C function with an pt_regs argument is called from the SYSCALL based
|
|
|
|
* fast path FIXUP_TOP_OF_STACK is needed.
|
|
|
|
* RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
|
|
|
|
* manipulation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* %rsp:at FRAMEEND */
|
|
|
|
.macro FIXUP_TOP_OF_STACK tmp
|
|
|
|
movq %gs:pda_oldrsp,\tmp
|
|
|
|
movq \tmp,RSP(%rsp)
|
|
|
|
movq $__USER_DS,SS(%rsp)
|
|
|
|
movq $__USER_CS,CS(%rsp)
|
|
|
|
movq $-1,RCX(%rsp)
|
|
|
|
movq R11(%rsp),\tmp /* get eflags */
|
|
|
|
movq \tmp,EFLAGS(%rsp)
|
|
|
|
.endm
|
|
|
|
|
|
|
|
.macro RESTORE_TOP_OF_STACK tmp,offset=0
|
|
|
|
movq RSP-\offset(%rsp),\tmp
|
|
|
|
movq \tmp,%gs:pda_oldrsp
|
|
|
|
movq EFLAGS-\offset(%rsp),\tmp
|
|
|
|
movq \tmp,R11-\offset(%rsp)
|
|
|
|
.endm
|
|
|
|
|
|
|
|
.macro FAKE_STACK_FRAME child_rip
|
|
|
|
/* push in order ss, rsp, eflags, cs, rip */
|
2005-07-29 12:15:48 +08:00
|
|
|
xorl %eax, %eax
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq %rax /* ss */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-09-13 00:49:24 +08:00
|
|
|
/*CFI_REL_OFFSET ss,0*/
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq %rax /* rsp */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_REL_OFFSET rsp,0
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq $(1<<9) /* eflags - interrupts on */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-09-13 00:49:24 +08:00
|
|
|
/*CFI_REL_OFFSET rflags,0*/
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq $__KERNEL_CS /* cs */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-09-13 00:49:24 +08:00
|
|
|
/*CFI_REL_OFFSET cs,0*/
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq \child_rip /* rip */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_REL_OFFSET rip,0
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq %rax /* orig rax */
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
|
|
|
.endm
|
|
|
|
|
|
|
|
.macro UNFAKE_STACK_FRAME
|
|
|
|
addq $8*6, %rsp
|
|
|
|
CFI_ADJUST_CFA_OFFSET -(6*8)
|
|
|
|
.endm
|
|
|
|
|
2005-09-13 00:49:24 +08:00
|
|
|
.macro CFI_DEFAULT_STACK start=1
|
|
|
|
.if \start
|
|
|
|
CFI_STARTPROC simple
|
2006-09-26 16:52:41 +08:00
|
|
|
CFI_SIGNAL_FRAME
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_DEF_CFA rsp,SS+8
|
|
|
|
.else
|
|
|
|
CFI_DEF_CFA_OFFSET SS+8
|
|
|
|
.endif
|
|
|
|
CFI_REL_OFFSET r15,R15
|
|
|
|
CFI_REL_OFFSET r14,R14
|
|
|
|
CFI_REL_OFFSET r13,R13
|
|
|
|
CFI_REL_OFFSET r12,R12
|
|
|
|
CFI_REL_OFFSET rbp,RBP
|
|
|
|
CFI_REL_OFFSET rbx,RBX
|
|
|
|
CFI_REL_OFFSET r11,R11
|
|
|
|
CFI_REL_OFFSET r10,R10
|
|
|
|
CFI_REL_OFFSET r9,R9
|
|
|
|
CFI_REL_OFFSET r8,R8
|
|
|
|
CFI_REL_OFFSET rax,RAX
|
|
|
|
CFI_REL_OFFSET rcx,RCX
|
|
|
|
CFI_REL_OFFSET rdx,RDX
|
|
|
|
CFI_REL_OFFSET rsi,RSI
|
|
|
|
CFI_REL_OFFSET rdi,RDI
|
|
|
|
CFI_REL_OFFSET rip,RIP
|
|
|
|
/*CFI_REL_OFFSET cs,CS*/
|
|
|
|
/*CFI_REL_OFFSET rflags,EFLAGS*/
|
|
|
|
CFI_REL_OFFSET rsp,RSP
|
|
|
|
/*CFI_REL_OFFSET ss,SS*/
|
2005-04-17 06:20:36 +08:00
|
|
|
.endm
|
|
|
|
/*
|
|
|
|
* A newly forked process directly context switches into this.
|
|
|
|
*/
|
|
|
|
/* rdi: prev */
|
|
|
|
ENTRY(ret_from_fork)
|
|
|
|
CFI_DEFAULT_STACK
|
2006-09-26 16:52:41 +08:00
|
|
|
push kernel_eflags(%rip)
|
|
|
|
CFI_ADJUST_CFA_OFFSET 4
|
|
|
|
popf # reset kernel eflags
|
|
|
|
CFI_ADJUST_CFA_OFFSET -4
|
2005-04-17 06:20:36 +08:00
|
|
|
call schedule_tail
|
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
|
|
|
|
jnz rff_trace
|
|
|
|
rff_action:
|
|
|
|
RESTORE_REST
|
|
|
|
testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
|
|
|
|
je int_ret_from_sys_call
|
|
|
|
testl $_TIF_IA32,threadinfo_flags(%rcx)
|
|
|
|
jnz int_ret_from_sys_call
|
|
|
|
RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
|
|
|
|
jmp ret_from_sys_call
|
|
|
|
rff_trace:
|
|
|
|
movq %rsp,%rdi
|
|
|
|
call syscall_trace_leave
|
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
jmp rff_action
|
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
END(ret_from_fork)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* System call entry. Upto 6 arguments in registers are supported.
|
|
|
|
*
|
|
|
|
* SYSCALL does not save anything on the stack and does not change the
|
|
|
|
* stack pointer.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Register setup:
|
|
|
|
* rax system call number
|
|
|
|
* rdi arg0
|
|
|
|
* rcx return address for syscall/sysret, C arg3
|
|
|
|
* rsi arg1
|
|
|
|
* rdx arg2
|
|
|
|
* r10 arg3 (--> moved to rcx for C)
|
|
|
|
* r8 arg4
|
|
|
|
* r9 arg5
|
|
|
|
* r11 eflags for syscall/sysret, temporary for C
|
|
|
|
* r12-r15,rbp,rbx saved by C code, not touched.
|
|
|
|
*
|
|
|
|
* Interrupts are off on entry.
|
|
|
|
* Only called from user space.
|
|
|
|
*
|
|
|
|
* XXX if we had a free scratch register we could save the RSP into the stack frame
|
|
|
|
* and report it properly in ps. Unfortunately we haven't.
|
2006-04-08 01:50:00 +08:00
|
|
|
*
|
|
|
|
* When user can change the frames always force IRET. That is because
|
|
|
|
* it deals with uncanonical addresses better. SYSRET has trouble
|
|
|
|
* with them due to bugs in both AMD and Intel CPUs.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
|
|
|
|
ENTRY(system_call)
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_STARTPROC simple
|
2006-09-26 16:52:41 +08:00
|
|
|
CFI_SIGNAL_FRAME
|
2006-06-26 19:57:38 +08:00
|
|
|
CFI_DEF_CFA rsp,PDA_STACKOFFSET
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_REGISTER rip,rcx
|
|
|
|
/*CFI_REGISTER rflags,r11*/
|
2008-01-30 20:32:08 +08:00
|
|
|
SWAPGS_UNSAFE_STACK
|
|
|
|
/*
|
|
|
|
* A hypervisor implementation might want to use a label
|
|
|
|
* after the swapgs, so that it can do the swapgs
|
|
|
|
* for the guest and jump here on syscall.
|
|
|
|
*/
|
|
|
|
ENTRY(system_call_after_swapgs)
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
movq %rsp,%gs:pda_oldrsp
|
|
|
|
movq %gs:pda_kernelstack,%rsp
|
2006-07-03 15:24:45 +08:00
|
|
|
/*
|
|
|
|
* No need to follow this irqs off/on section - it's straight
|
|
|
|
* and short:
|
|
|
|
*/
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2005-04-17 06:20:36 +08:00
|
|
|
SAVE_ARGS 8,1
|
|
|
|
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
|
2005-09-13 00:49:24 +08:00
|
|
|
movq %rcx,RIP-ARGOFFSET(%rsp)
|
|
|
|
CFI_REL_OFFSET rip,RIP-ARGOFFSET
|
2005-04-17 06:20:36 +08:00
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
|
|
|
|
jnz tracesys
|
|
|
|
cmpq $__NR_syscall_max,%rax
|
|
|
|
ja badsys
|
|
|
|
movq %r10,%rcx
|
|
|
|
call *sys_call_table(,%rax,8) # XXX: rip relative
|
|
|
|
movq %rax,RAX-ARGOFFSET(%rsp)
|
|
|
|
/*
|
|
|
|
* Syscall return path ending with SYSRET (fast path)
|
|
|
|
* Has incomplete stack frame and undefined top of stack.
|
|
|
|
*/
|
|
|
|
ret_from_sys_call:
|
2005-04-17 06:25:02 +08:00
|
|
|
movl $_TIF_ALLWORK_MASK,%edi
|
2005-04-17 06:20:36 +08:00
|
|
|
/* edi: flagmask */
|
|
|
|
sysret_check:
|
2007-10-12 04:11:12 +08:00
|
|
|
LOCKDEP_SYS_EXIT
|
2005-04-17 06:20:36 +08:00
|
|
|
GET_THREAD_INFO(%rcx)
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-17 06:20:36 +08:00
|
|
|
movl threadinfo_flags(%rcx),%edx
|
|
|
|
andl %edi,%edx
|
|
|
|
jnz sysret_careful
|
2006-12-07 09:14:02 +08:00
|
|
|
CFI_REMEMBER_STATE
|
2006-07-03 15:24:45 +08:00
|
|
|
/*
|
|
|
|
* sysretq will re-enable interrupts:
|
|
|
|
*/
|
|
|
|
TRACE_IRQS_ON
|
2005-04-17 06:20:36 +08:00
|
|
|
movq RIP-ARGOFFSET(%rsp),%rcx
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_REGISTER rip,rcx
|
2005-04-17 06:20:36 +08:00
|
|
|
RESTORE_ARGS 0,-ARG_SKIP,1
|
2005-09-13 00:49:24 +08:00
|
|
|
/*CFI_REGISTER rflags,r11*/
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS_SYSCALL_RET
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-12-07 09:14:02 +08:00
|
|
|
CFI_RESTORE_STATE
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Handle reschedules */
|
|
|
|
/* edx: work, edi: workmask */
|
|
|
|
sysret_careful:
|
|
|
|
bt $TIF_NEED_RESCHED,%edx
|
|
|
|
jnc sysret_signal
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq %rdi
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-04-17 06:20:36 +08:00
|
|
|
call schedule
|
|
|
|
popq %rdi
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2005-04-17 06:20:36 +08:00
|
|
|
jmp sysret_check
|
|
|
|
|
|
|
|
/* Handle a signal */
|
|
|
|
sysret_signal:
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2008-01-26 04:08:29 +08:00
|
|
|
testl $_TIF_DO_NOTIFY_MASK,%edx
|
2005-05-17 12:53:19 +08:00
|
|
|
jz 1f
|
|
|
|
|
|
|
|
/* Really a signal */
|
|
|
|
/* edx: work flags (arg3) */
|
2005-04-17 06:20:36 +08:00
|
|
|
leaq do_notify_resume(%rip),%rax
|
|
|
|
leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
|
|
|
|
xorl %esi,%esi # oldset -> arg2
|
|
|
|
call ptregscall_common
|
2005-05-17 12:53:19 +08:00
|
|
|
1: movl $_TIF_NEED_RESCHED,%edi
|
2006-04-08 01:50:00 +08:00
|
|
|
/* Use IRET because user could have changed frame. This
|
|
|
|
works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2006-04-08 01:50:00 +08:00
|
|
|
jmp int_with_check
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-09-13 00:49:24 +08:00
|
|
|
badsys:
|
|
|
|
movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
|
|
|
|
jmp ret_from_sys_call
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Do syscall tracing */
|
|
|
|
tracesys:
|
|
|
|
SAVE_REST
|
2008-03-17 12:59:11 +08:00
|
|
|
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
|
2005-04-17 06:20:36 +08:00
|
|
|
FIXUP_TOP_OF_STACK %rdi
|
|
|
|
movq %rsp,%rdi
|
|
|
|
call syscall_trace_enter
|
|
|
|
LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
|
|
|
|
RESTORE_REST
|
|
|
|
cmpq $__NR_syscall_max,%rax
|
2008-03-17 12:59:11 +08:00
|
|
|
ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
|
2005-04-17 06:20:36 +08:00
|
|
|
movq %r10,%rcx /* fixup for C */
|
|
|
|
call *sys_call_table(,%rax,8)
|
2008-03-17 12:59:11 +08:00
|
|
|
movq %rax,RAX-ARGOFFSET(%rsp)
|
2006-04-08 01:50:00 +08:00
|
|
|
/* Use IRET because user could have changed frame */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Syscall return path ending with IRET.
|
|
|
|
* Has correct top of stack, but partial stack frame.
|
2006-12-07 09:14:02 +08:00
|
|
|
*/
|
|
|
|
.globl int_ret_from_sys_call
|
|
|
|
int_ret_from_sys_call:
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-17 06:20:36 +08:00
|
|
|
testl $3,CS-ARGOFFSET(%rsp)
|
|
|
|
je retint_restore_args
|
|
|
|
movl $_TIF_ALLWORK_MASK,%edi
|
|
|
|
/* edi: mask to check */
|
|
|
|
int_with_check:
|
2007-10-12 04:11:12 +08:00
|
|
|
LOCKDEP_SYS_EXIT_IRQ
|
2005-04-17 06:20:36 +08:00
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
movl threadinfo_flags(%rcx),%edx
|
|
|
|
andl %edi,%edx
|
|
|
|
jnz int_careful
|
2006-01-12 05:44:06 +08:00
|
|
|
andl $~TS_COMPAT,threadinfo_status(%rcx)
|
2005-04-17 06:20:36 +08:00
|
|
|
jmp retint_swapgs
|
|
|
|
|
|
|
|
/* Either reschedule or signal or syscall exit tracking needed. */
|
|
|
|
/* First do a reschedule test. */
|
|
|
|
/* edx: work, edi: workmask */
|
|
|
|
int_careful:
|
|
|
|
bt $TIF_NEED_RESCHED,%edx
|
|
|
|
jnc int_very_careful
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq %rdi
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-04-17 06:20:36 +08:00
|
|
|
call schedule
|
|
|
|
popq %rdi
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-17 06:20:36 +08:00
|
|
|
jmp int_with_check
|
|
|
|
|
|
|
|
/* handle signals and tracing -- both require a full stack frame */
|
|
|
|
int_very_careful:
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2005-04-17 06:20:36 +08:00
|
|
|
SAVE_REST
|
|
|
|
/* Check for syscall exit trace */
|
|
|
|
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
|
|
|
|
jz int_signal
|
|
|
|
pushq %rdi
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-04-17 06:20:36 +08:00
|
|
|
leaq 8(%rsp),%rdi # &ptregs -> arg1
|
|
|
|
call syscall_trace_leave
|
|
|
|
popq %rdi
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2005-04-17 06:25:01 +08:00
|
|
|
andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
|
2005-04-17 06:20:36 +08:00
|
|
|
jmp int_restore_rest
|
|
|
|
|
|
|
|
int_signal:
|
2008-01-26 04:08:29 +08:00
|
|
|
testl $_TIF_DO_NOTIFY_MASK,%edx
|
2005-04-17 06:20:36 +08:00
|
|
|
jz 1f
|
|
|
|
movq %rsp,%rdi # &ptregs -> arg1
|
|
|
|
xorl %esi,%esi # oldset -> arg2
|
|
|
|
call do_notify_resume
|
|
|
|
1: movl $_TIF_NEED_RESCHED,%edi
|
|
|
|
int_restore_rest:
|
|
|
|
RESTORE_REST
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-17 06:20:36 +08:00
|
|
|
jmp int_with_check
|
|
|
|
CFI_ENDPROC
|
2006-12-07 09:14:02 +08:00
|
|
|
END(system_call)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Certain special system calls that need to save a complete full stack frame.
|
|
|
|
*/
|
|
|
|
|
|
|
|
.macro PTREGSCALL label,func,arg
|
|
|
|
.globl \label
|
|
|
|
\label:
|
|
|
|
leaq \func(%rip),%rax
|
|
|
|
leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
|
|
|
|
jmp ptregscall_common
|
2006-06-26 19:56:55 +08:00
|
|
|
END(\label)
|
2005-04-17 06:20:36 +08:00
|
|
|
.endm
|
|
|
|
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_STARTPROC
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
PTREGSCALL stub_clone, sys_clone, %r8
|
|
|
|
PTREGSCALL stub_fork, sys_fork, %rdi
|
|
|
|
PTREGSCALL stub_vfork, sys_vfork, %rdi
|
|
|
|
PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
|
|
|
|
PTREGSCALL stub_iopl, sys_iopl, %rsi
|
|
|
|
|
|
|
|
ENTRY(ptregscall_common)
|
|
|
|
popq %r11
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
|
|
|
CFI_REGISTER rip, r11
|
2005-04-17 06:20:36 +08:00
|
|
|
SAVE_REST
|
|
|
|
movq %r11, %r15
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_REGISTER rip, r15
|
2005-04-17 06:20:36 +08:00
|
|
|
FIXUP_TOP_OF_STACK %r11
|
|
|
|
call *%rax
|
|
|
|
RESTORE_TOP_OF_STACK %r11
|
|
|
|
movq %r15, %r11
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_REGISTER rip, r11
|
2005-04-17 06:20:36 +08:00
|
|
|
RESTORE_REST
|
|
|
|
pushq %r11
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
|
|
|
CFI_REL_OFFSET rip, 0
|
2005-04-17 06:20:36 +08:00
|
|
|
ret
|
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
END(ptregscall_common)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(stub_execve)
|
|
|
|
CFI_STARTPROC
|
|
|
|
popq %r11
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
|
|
|
CFI_REGISTER rip, r11
|
2005-04-17 06:20:36 +08:00
|
|
|
SAVE_REST
|
|
|
|
FIXUP_TOP_OF_STACK %r11
|
2008-02-26 19:55:57 +08:00
|
|
|
movq %rsp, %rcx
|
2005-04-17 06:20:36 +08:00
|
|
|
call sys_execve
|
|
|
|
RESTORE_TOP_OF_STACK %r11
|
|
|
|
movq %rax,RAX(%rsp)
|
|
|
|
RESTORE_REST
|
|
|
|
jmp int_ret_from_sys_call
|
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
END(stub_execve)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* sigreturn is special because it needs to restore all registers on return.
|
|
|
|
* This cannot be done with SYSRET, so use the IRET return path instead.
|
|
|
|
*/
|
|
|
|
ENTRY(stub_rt_sigreturn)
|
|
|
|
CFI_STARTPROC
|
2005-09-13 00:49:24 +08:00
|
|
|
addq $8, %rsp
|
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2005-04-17 06:20:36 +08:00
|
|
|
SAVE_REST
|
|
|
|
movq %rsp,%rdi
|
|
|
|
FIXUP_TOP_OF_STACK %r11
|
|
|
|
call sys_rt_sigreturn
|
|
|
|
movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
|
|
|
|
RESTORE_REST
|
|
|
|
jmp int_ret_from_sys_call
|
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
END(stub_rt_sigreturn)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-09-13 00:49:24 +08:00
|
|
|
/*
|
|
|
|
* initial frame state for interrupts and exceptions
|
|
|
|
*/
|
|
|
|
.macro _frame ref
|
|
|
|
CFI_STARTPROC simple
|
2006-09-26 16:52:41 +08:00
|
|
|
CFI_SIGNAL_FRAME
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_DEF_CFA rsp,SS+8-\ref
|
|
|
|
/*CFI_REL_OFFSET ss,SS-\ref*/
|
|
|
|
CFI_REL_OFFSET rsp,RSP-\ref
|
|
|
|
/*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
|
|
|
|
/*CFI_REL_OFFSET cs,CS-\ref*/
|
|
|
|
CFI_REL_OFFSET rip,RIP-\ref
|
|
|
|
.endm
|
|
|
|
|
|
|
|
/* initial frame state for interrupts (and exceptions without error code) */
|
|
|
|
#define INTR_FRAME _frame RIP
|
|
|
|
/* initial frame state for exceptions with error code (and interrupts with
|
|
|
|
vector already pushed) */
|
|
|
|
#define XCPT_FRAME _frame ORIG_RAX
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Interrupt entry/exit.
|
|
|
|
*
|
|
|
|
* Interrupt entry points save only callee clobbered registers in fast path.
|
|
|
|
*
|
|
|
|
* Entry runs with interrupts off.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* 0(%rsp): interrupt number */
|
|
|
|
.macro interrupt func
|
|
|
|
cld
|
|
|
|
SAVE_ARGS
|
|
|
|
leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
|
2006-06-26 19:57:35 +08:00
|
|
|
pushq %rbp
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
|
|
|
CFI_REL_OFFSET rbp, 0
|
|
|
|
movq %rsp,%rbp
|
|
|
|
CFI_DEF_CFA_REGISTER rbp
|
2005-04-17 06:20:36 +08:00
|
|
|
testl $3,CS(%rdi)
|
|
|
|
je 1f
|
2008-01-30 20:32:08 +08:00
|
|
|
SWAPGS
|
2006-09-26 16:52:39 +08:00
|
|
|
/* irqcount is used to check if a CPU is already on an interrupt
|
|
|
|
stack or not. While this is essentially redundant with preempt_count
|
|
|
|
it is a little cheaper to use a separate counter in the PDA
|
|
|
|
(short of moving irq_enter into assembly, which would be too
|
|
|
|
much work) */
|
|
|
|
1: incl %gs:pda_irqcount
|
2006-06-26 19:57:35 +08:00
|
|
|
cmoveq %gs:pda_irqstackptr,%rsp
|
2006-08-03 04:37:28 +08:00
|
|
|
push %rbp # backlink for old unwinder
|
2006-07-03 15:24:45 +08:00
|
|
|
/*
|
|
|
|
* We entered an interrupt context - irqs are off:
|
|
|
|
*/
|
|
|
|
TRACE_IRQS_OFF
|
2005-04-17 06:20:36 +08:00
|
|
|
call \func
|
|
|
|
.endm
|
|
|
|
|
|
|
|
ENTRY(common_interrupt)
|
2005-09-13 00:49:24 +08:00
|
|
|
XCPT_FRAME
|
2005-04-17 06:20:36 +08:00
|
|
|
interrupt do_IRQ
|
|
|
|
/* 0(%rsp): oldrsp-ARGOFFSET */
|
2005-09-13 00:49:24 +08:00
|
|
|
ret_from_intr:
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2005-07-29 12:15:48 +08:00
|
|
|
decl %gs:pda_irqcount
|
2006-06-26 19:57:35 +08:00
|
|
|
leaveq
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_DEF_CFA_REGISTER rsp
|
2006-06-26 19:57:35 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2005-09-13 00:49:24 +08:00
|
|
|
exit_intr:
|
2005-04-17 06:20:36 +08:00
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
testl $3,CS-ARGOFFSET(%rsp)
|
|
|
|
je retint_kernel
|
|
|
|
|
|
|
|
/* Interrupt came from user space */
|
|
|
|
/*
|
|
|
|
* Has a correct top of stack, but a partial stack frame
|
|
|
|
* %rcx: thread info. Interrupts off.
|
|
|
|
*/
|
|
|
|
retint_with_reschedule:
|
|
|
|
movl $_TIF_WORK_MASK,%edi
|
2005-09-13 00:49:24 +08:00
|
|
|
retint_check:
|
2007-10-12 04:11:12 +08:00
|
|
|
LOCKDEP_SYS_EXIT_IRQ
|
2005-04-17 06:20:36 +08:00
|
|
|
movl threadinfo_flags(%rcx),%edx
|
|
|
|
andl %edi,%edx
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_REMEMBER_STATE
|
2005-04-17 06:20:36 +08:00
|
|
|
jnz retint_careful
|
2007-10-12 04:11:12 +08:00
|
|
|
|
|
|
|
retint_swapgs: /* return to user-space */
|
2006-07-03 15:24:45 +08:00
|
|
|
/*
|
|
|
|
* The iretq could re-enable interrupts:
|
|
|
|
*/
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_ANY)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_IRETQ
|
2008-01-30 20:32:08 +08:00
|
|
|
SWAPGS
|
2006-07-03 15:24:45 +08:00
|
|
|
jmp restore_args
|
|
|
|
|
2007-10-12 04:11:12 +08:00
|
|
|
retint_restore_args: /* return to kernel space */
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_ANY)
|
2006-07-03 15:24:45 +08:00
|
|
|
/*
|
|
|
|
* The iretq could re-enable interrupts:
|
|
|
|
*/
|
|
|
|
TRACE_IRQS_IRETQ
|
|
|
|
restore_args:
|
2008-02-10 06:24:08 +08:00
|
|
|
RESTORE_ARGS 0,8,0
|
|
|
|
|
2008-02-14 05:29:53 +08:00
|
|
|
irq_return:
|
2008-01-30 20:32:08 +08:00
|
|
|
INTERRUPT_RETURN
|
2008-02-10 06:24:08 +08:00
|
|
|
|
|
|
|
.section __ex_table, "a"
|
|
|
|
.quad irq_return, bad_iret
|
|
|
|
.previous
|
|
|
|
|
|
|
|
#ifdef CONFIG_PARAVIRT
|
2008-01-30 20:32:08 +08:00
|
|
|
ENTRY(native_iret)
|
2005-04-17 06:20:36 +08:00
|
|
|
iretq
|
|
|
|
|
|
|
|
.section __ex_table,"a"
|
2008-01-30 20:32:08 +08:00
|
|
|
.quad native_iret, bad_iret
|
2005-04-17 06:20:36 +08:00
|
|
|
.previous
|
2008-02-10 06:24:08 +08:00
|
|
|
#endif
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
.section .fixup,"ax"
|
|
|
|
bad_iret:
|
2008-02-07 05:39:43 +08:00
|
|
|
/*
|
|
|
|
* The iret traps when the %cs or %ss being restored is bogus.
|
|
|
|
* We've lost the original trap vector and error code.
|
|
|
|
* #GPF is the most likely one to get for an invalid selector.
|
|
|
|
* So pretend we completed the iret and took the #GPF in user mode.
|
|
|
|
*
|
|
|
|
* We are now running with the kernel GS after exception recovery.
|
|
|
|
* But error_entry expects us to have user GS to match the user %cs,
|
|
|
|
* so swap back.
|
|
|
|
*/
|
|
|
|
pushq $0
|
|
|
|
|
|
|
|
SWAPGS
|
|
|
|
jmp general_protection
|
|
|
|
|
2008-01-30 20:32:08 +08:00
|
|
|
.previous
|
|
|
|
|
2005-09-13 00:49:24 +08:00
|
|
|
/* edi: workmask, edx: work */
|
2005-04-17 06:20:36 +08:00
|
|
|
retint_careful:
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_RESTORE_STATE
|
2005-04-17 06:20:36 +08:00
|
|
|
bt $TIF_NEED_RESCHED,%edx
|
|
|
|
jnc retint_signal
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq %rdi
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-04-17 06:20:36 +08:00
|
|
|
call schedule
|
|
|
|
popq %rdi
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2005-04-17 06:20:36 +08:00
|
|
|
GET_THREAD_INFO(%rcx)
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-17 06:20:36 +08:00
|
|
|
jmp retint_check
|
|
|
|
|
|
|
|
retint_signal:
|
2008-01-26 04:08:29 +08:00
|
|
|
testl $_TIF_DO_NOTIFY_MASK,%edx
|
2005-05-17 12:53:19 +08:00
|
|
|
jz retint_swapgs
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_ON
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2005-04-17 06:20:36 +08:00
|
|
|
SAVE_REST
|
|
|
|
movq $-1,ORIG_RAX(%rsp)
|
2005-07-29 12:15:48 +08:00
|
|
|
xorl %esi,%esi # oldset
|
2005-04-17 06:20:36 +08:00
|
|
|
movq %rsp,%rdi # &pt_regs
|
|
|
|
call do_notify_resume
|
|
|
|
RESTORE_REST
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2005-05-17 12:53:19 +08:00
|
|
|
movl $_TIF_NEED_RESCHED,%edi
|
2005-05-01 23:58:51 +08:00
|
|
|
GET_THREAD_INFO(%rcx)
|
2005-04-17 06:20:36 +08:00
|
|
|
jmp retint_check
|
|
|
|
|
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
/* Returning to kernel space. Check if we need preemption */
|
|
|
|
/* rcx: threadinfo. interrupts off. */
|
2006-09-26 16:52:29 +08:00
|
|
|
ENTRY(retint_kernel)
|
2005-04-17 06:20:36 +08:00
|
|
|
cmpl $0,threadinfo_preempt_count(%rcx)
|
|
|
|
jnz retint_restore_args
|
|
|
|
bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
|
|
|
|
jnc retint_restore_args
|
|
|
|
bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
|
|
|
|
jnc retint_restore_args
|
|
|
|
call preempt_schedule_irq
|
|
|
|
jmp exit_intr
|
|
|
|
#endif
|
2006-06-26 19:56:55 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
END(common_interrupt)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* APIC interrupts.
|
|
|
|
*/
|
|
|
|
.macro apicinterrupt num,func
|
2005-09-13 00:49:24 +08:00
|
|
|
INTR_FRAME
|
2006-06-27 17:53:44 +08:00
|
|
|
pushq $~(\num)
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-04-17 06:20:36 +08:00
|
|
|
interrupt \func
|
|
|
|
jmp ret_from_intr
|
|
|
|
CFI_ENDPROC
|
|
|
|
.endm
|
|
|
|
|
|
|
|
ENTRY(thermal_interrupt)
|
|
|
|
apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
|
2006-06-26 19:56:55 +08:00
|
|
|
END(thermal_interrupt)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-06 00:25:53 +08:00
|
|
|
ENTRY(threshold_interrupt)
|
|
|
|
apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
|
2006-06-26 19:56:55 +08:00
|
|
|
END(threshold_interrupt)
|
2005-11-06 00:25:53 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
ENTRY(reschedule_interrupt)
|
|
|
|
apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
|
2006-06-26 19:56:55 +08:00
|
|
|
END(reschedule_interrupt)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-09-13 00:49:24 +08:00
|
|
|
.macro INVALIDATE_ENTRY num
|
|
|
|
ENTRY(invalidate_interrupt\num)
|
|
|
|
apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
|
2006-06-26 19:56:55 +08:00
|
|
|
END(invalidate_interrupt\num)
|
2005-09-13 00:49:24 +08:00
|
|
|
.endm
|
|
|
|
|
|
|
|
INVALIDATE_ENTRY 0
|
|
|
|
INVALIDATE_ENTRY 1
|
|
|
|
INVALIDATE_ENTRY 2
|
|
|
|
INVALIDATE_ENTRY 3
|
|
|
|
INVALIDATE_ENTRY 4
|
|
|
|
INVALIDATE_ENTRY 5
|
|
|
|
INVALIDATE_ENTRY 6
|
|
|
|
INVALIDATE_ENTRY 7
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(call_function_interrupt)
|
|
|
|
apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
|
2006-06-26 19:56:55 +08:00
|
|
|
END(call_function_interrupt)
|
2007-02-23 19:40:58 +08:00
|
|
|
ENTRY(irq_move_cleanup_interrupt)
|
|
|
|
apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
|
|
|
|
END(irq_move_cleanup_interrupt)
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
ENTRY(apic_timer_interrupt)
|
|
|
|
apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
|
2006-06-26 19:56:55 +08:00
|
|
|
END(apic_timer_interrupt)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-06-02 21:56:14 +08:00
|
|
|
ENTRY(uv_bau_message_intr1)
|
|
|
|
apicinterrupt 220,uv_bau_message_interrupt
|
|
|
|
END(uv_bau_message_intr1)
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
ENTRY(error_interrupt)
|
|
|
|
apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
|
2006-06-26 19:56:55 +08:00
|
|
|
END(error_interrupt)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(spurious_interrupt)
|
|
|
|
apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
|
2006-06-26 19:56:55 +08:00
|
|
|
END(spurious_interrupt)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Exception entry points.
|
|
|
|
*/
|
|
|
|
.macro zeroentry sym
|
2005-09-13 00:49:24 +08:00
|
|
|
INTR_FRAME
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq $0 /* push error code/oldrax */
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq %rax /* push real oldrax to the rdi slot */
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2007-05-03 01:27:05 +08:00
|
|
|
CFI_REL_OFFSET rax,0
|
2005-04-17 06:20:36 +08:00
|
|
|
leaq \sym(%rip),%rax
|
|
|
|
jmp error_entry
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ENDPROC
|
2005-04-17 06:20:36 +08:00
|
|
|
.endm
|
|
|
|
|
|
|
|
.macro errorentry sym
|
2005-09-13 00:49:24 +08:00
|
|
|
XCPT_FRAME
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq %rax
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2007-05-03 01:27:05 +08:00
|
|
|
CFI_REL_OFFSET rax,0
|
2005-04-17 06:20:36 +08:00
|
|
|
leaq \sym(%rip),%rax
|
|
|
|
jmp error_entry
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ENDPROC
|
2005-04-17 06:20:36 +08:00
|
|
|
.endm
|
|
|
|
|
|
|
|
/* error code is on the stack already */
|
|
|
|
/* handle NMI like exceptions that can happen everywhere */
|
2006-07-03 15:24:45 +08:00
|
|
|
.macro paranoidentry sym, ist=0, irqtrace=1
|
2005-04-17 06:20:36 +08:00
|
|
|
SAVE_ALL
|
|
|
|
cld
|
|
|
|
movl $1,%ebx
|
|
|
|
movl $MSR_GS_BASE,%ecx
|
|
|
|
rdmsr
|
|
|
|
testl %edx,%edx
|
|
|
|
js 1f
|
2008-01-30 20:32:08 +08:00
|
|
|
SWAPGS
|
2005-04-17 06:20:36 +08:00
|
|
|
xorl %ebx,%ebx
|
2006-01-12 05:43:00 +08:00
|
|
|
1:
|
|
|
|
.if \ist
|
|
|
|
movq %gs:pda_data_offset, %rbp
|
|
|
|
.endif
|
|
|
|
movq %rsp,%rdi
|
2005-04-17 06:20:36 +08:00
|
|
|
movq ORIG_RAX(%rsp),%rsi
|
|
|
|
movq $-1,ORIG_RAX(%rsp)
|
2006-01-12 05:43:00 +08:00
|
|
|
.if \ist
|
2006-01-16 08:56:39 +08:00
|
|
|
subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
|
2006-01-12 05:43:00 +08:00
|
|
|
.endif
|
2005-04-17 06:20:36 +08:00
|
|
|
call \sym
|
2006-01-12 05:43:00 +08:00
|
|
|
.if \ist
|
2006-01-16 08:56:39 +08:00
|
|
|
addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
|
2006-01-12 05:43:00 +08:00
|
|
|
.endif
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
.if \irqtrace
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
.endif
|
2005-04-17 06:20:36 +08:00
|
|
|
.endm
|
2006-07-03 15:24:45 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* "Paranoid" exit path from exception stack.
|
|
|
|
* Paranoid because this is used by NMIs and cannot take
|
|
|
|
* any kernel state for granted.
|
|
|
|
* We don't do kernel preemption checks here, because only
|
|
|
|
* NMI should be common and it does not enable IRQs and
|
|
|
|
* cannot get reschedule ticks.
|
|
|
|
*
|
|
|
|
* "trace" is 0 for the NMI handler only, because irq-tracing
|
|
|
|
* is fundamentally NMI-unsafe. (we cannot change the soft and
|
|
|
|
* hard flags at once, atomically)
|
|
|
|
*/
|
|
|
|
.macro paranoidexit trace=1
|
|
|
|
/* ebx: no swapgs flag */
|
|
|
|
paranoid_exit\trace:
|
|
|
|
testl %ebx,%ebx /* swapgs needed? */
|
|
|
|
jnz paranoid_restore\trace
|
|
|
|
testl $3,CS(%rsp)
|
|
|
|
jnz paranoid_userspace\trace
|
|
|
|
paranoid_swapgs\trace:
|
2006-09-26 16:52:37 +08:00
|
|
|
.if \trace
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_IRETQ 0
|
2006-09-26 16:52:37 +08:00
|
|
|
.endif
|
2008-01-30 20:32:08 +08:00
|
|
|
SWAPGS_UNSAFE_STACK
|
2006-07-03 15:24:45 +08:00
|
|
|
paranoid_restore\trace:
|
|
|
|
RESTORE_ALL 8
|
2008-02-10 06:24:08 +08:00
|
|
|
jmp irq_return
|
2006-07-03 15:24:45 +08:00
|
|
|
paranoid_userspace\trace:
|
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
movl threadinfo_flags(%rcx),%ebx
|
|
|
|
andl $_TIF_WORK_MASK,%ebx
|
|
|
|
jz paranoid_swapgs\trace
|
|
|
|
movq %rsp,%rdi /* &pt_regs */
|
|
|
|
call sync_regs
|
|
|
|
movq %rax,%rsp /* switch stack for scheduling */
|
|
|
|
testl $_TIF_NEED_RESCHED,%ebx
|
|
|
|
jnz paranoid_schedule\trace
|
|
|
|
movl %ebx,%edx /* arg3: thread flags */
|
|
|
|
.if \trace
|
|
|
|
TRACE_IRQS_ON
|
|
|
|
.endif
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
xorl %esi,%esi /* arg2: oldset */
|
|
|
|
movq %rsp,%rdi /* arg1: &pt_regs */
|
|
|
|
call do_notify_resume
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
.if \trace
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
.endif
|
|
|
|
jmp paranoid_userspace\trace
|
|
|
|
paranoid_schedule\trace:
|
|
|
|
.if \trace
|
|
|
|
TRACE_IRQS_ON
|
|
|
|
.endif
|
2008-01-30 20:32:08 +08:00
|
|
|
ENABLE_INTERRUPTS(CLBR_ANY)
|
2006-07-03 15:24:45 +08:00
|
|
|
call schedule
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_ANY)
|
2006-07-03 15:24:45 +08:00
|
|
|
.if \trace
|
|
|
|
TRACE_IRQS_OFF
|
|
|
|
.endif
|
|
|
|
jmp paranoid_userspace\trace
|
|
|
|
CFI_ENDPROC
|
|
|
|
.endm
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Exception entry point. This expects an error code/orig_rax on the stack
|
|
|
|
* and the exception handler in %rax.
|
|
|
|
*/
|
[PATCH] x86: error_code is not safe for kprobes
This patch moves the entry.S:error_entry to .kprobes.text section,
since code marked unsafe for kprobes jumps directly to entry.S::error_entry,
that must be marked unsafe as well.
This patch also moves all the ".previous.text" asm directives to ".previous"
for kprobes section.
AK: Following a similar i386 patch from Chuck Ebbert
AK: Also merged Jeremy's fix in.
+From: Jeremy Fitzhardinge <jeremy@goop.org>
KPROBE_ENTRY does a .section .kprobes.text, and expects its users to
do a .previous at the end of the function.
Unfortunately, if any code within the function switches sections, for
example .fixup, then the .previous ends up putting all subsequent code
into .fixup. Worse, any subsequent .fixup code gets intermingled with
the code its supposed to be fixing (which is also in .fixup). It's
surprising this didn't cause more havok.
The fix is to use .pushsection/.popsection, so this stuff nests
properly. A further cleanup would be to get rid of all
.section/.previous pairs, since they're inherently fragile.
+From: Chuck Ebbert <76306.1226@compuserve.com>
Because code marked unsafe for kprobes jumps directly to
entry.S::error_code, that must be marked unsafe as well.
The easiest way to do that is to move the page fault entry
point to just before error_code and let it inherit the same
section.
Also moved all the ".previous" asm directives for kprobes
sections to column 1 and removed ".text" from them.
Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com>
Signed-off-by: Andi Kleen <ak@suse.de>
2006-09-26 16:52:34 +08:00
|
|
|
KPROBE_ENTRY(error_entry)
|
2005-09-13 00:49:24 +08:00
|
|
|
_frame RDI
|
2007-05-03 01:27:05 +08:00
|
|
|
CFI_REL_OFFSET rax,0
|
2005-04-17 06:20:36 +08:00
|
|
|
/* rdi slot contains rax, oldrax contains error code */
|
|
|
|
cld
|
|
|
|
subq $14*8,%rsp
|
|
|
|
CFI_ADJUST_CFA_OFFSET (14*8)
|
|
|
|
movq %rsi,13*8(%rsp)
|
|
|
|
CFI_REL_OFFSET rsi,RSI
|
|
|
|
movq 14*8(%rsp),%rsi /* load rax from rdi slot */
|
2007-05-03 01:27:05 +08:00
|
|
|
CFI_REGISTER rax,rsi
|
2005-04-17 06:20:36 +08:00
|
|
|
movq %rdx,12*8(%rsp)
|
|
|
|
CFI_REL_OFFSET rdx,RDX
|
|
|
|
movq %rcx,11*8(%rsp)
|
|
|
|
CFI_REL_OFFSET rcx,RCX
|
|
|
|
movq %rsi,10*8(%rsp) /* store rax */
|
|
|
|
CFI_REL_OFFSET rax,RAX
|
|
|
|
movq %r8, 9*8(%rsp)
|
|
|
|
CFI_REL_OFFSET r8,R8
|
|
|
|
movq %r9, 8*8(%rsp)
|
|
|
|
CFI_REL_OFFSET r9,R9
|
|
|
|
movq %r10,7*8(%rsp)
|
|
|
|
CFI_REL_OFFSET r10,R10
|
|
|
|
movq %r11,6*8(%rsp)
|
|
|
|
CFI_REL_OFFSET r11,R11
|
|
|
|
movq %rbx,5*8(%rsp)
|
|
|
|
CFI_REL_OFFSET rbx,RBX
|
|
|
|
movq %rbp,4*8(%rsp)
|
|
|
|
CFI_REL_OFFSET rbp,RBP
|
|
|
|
movq %r12,3*8(%rsp)
|
|
|
|
CFI_REL_OFFSET r12,R12
|
|
|
|
movq %r13,2*8(%rsp)
|
|
|
|
CFI_REL_OFFSET r13,R13
|
|
|
|
movq %r14,1*8(%rsp)
|
|
|
|
CFI_REL_OFFSET r14,R14
|
|
|
|
movq %r15,(%rsp)
|
|
|
|
CFI_REL_OFFSET r15,R15
|
|
|
|
xorl %ebx,%ebx
|
|
|
|
testl $3,CS(%rsp)
|
|
|
|
je error_kernelspace
|
|
|
|
error_swapgs:
|
2008-01-30 20:32:08 +08:00
|
|
|
SWAPGS
|
2005-04-17 06:20:36 +08:00
|
|
|
error_sti:
|
|
|
|
movq %rdi,RDI(%rsp)
|
2007-05-03 01:27:05 +08:00
|
|
|
CFI_REL_OFFSET rdi,RDI
|
2005-04-17 06:20:36 +08:00
|
|
|
movq %rsp,%rdi
|
|
|
|
movq ORIG_RAX(%rsp),%rsi /* get error code */
|
|
|
|
movq $-1,ORIG_RAX(%rsp)
|
|
|
|
call *%rax
|
2007-10-12 04:11:12 +08:00
|
|
|
/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
|
|
|
|
error_exit:
|
|
|
|
movl %ebx,%eax
|
2005-04-17 06:20:36 +08:00
|
|
|
RESTORE_REST
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_NONE)
|
2006-07-03 15:24:45 +08:00
|
|
|
TRACE_IRQS_OFF
|
2005-04-17 06:20:36 +08:00
|
|
|
GET_THREAD_INFO(%rcx)
|
|
|
|
testl %eax,%eax
|
|
|
|
jne retint_kernel
|
2007-10-12 04:11:12 +08:00
|
|
|
LOCKDEP_SYS_EXIT_IRQ
|
2005-04-17 06:20:36 +08:00
|
|
|
movl threadinfo_flags(%rcx),%edx
|
|
|
|
movl $_TIF_WORK_MASK,%edi
|
|
|
|
andl %edi,%edx
|
|
|
|
jnz retint_careful
|
2007-10-12 04:11:12 +08:00
|
|
|
jmp retint_swapgs
|
2005-04-17 06:20:36 +08:00
|
|
|
CFI_ENDPROC
|
|
|
|
|
|
|
|
error_kernelspace:
|
|
|
|
incl %ebx
|
|
|
|
/* There are two places in the kernel that can potentially fault with
|
|
|
|
usergs. Handle them here. The exception handlers after
|
|
|
|
iret run with kernel gs again, so don't set the user space flag.
|
|
|
|
B stepping K8s sometimes report an truncated RIP for IRET
|
|
|
|
exceptions returning to compat mode. Check for these here too. */
|
2008-06-27 23:22:17 +08:00
|
|
|
leaq irq_return(%rip),%rcx
|
|
|
|
cmpq %rcx,RIP(%rsp)
|
2005-04-17 06:20:36 +08:00
|
|
|
je error_swapgs
|
2008-06-27 23:22:17 +08:00
|
|
|
movl %ecx,%ecx /* zero extend */
|
|
|
|
cmpq %rcx,RIP(%rsp)
|
2005-04-17 06:20:36 +08:00
|
|
|
je error_swapgs
|
|
|
|
cmpq $gs_change,RIP(%rsp)
|
|
|
|
je error_swapgs
|
|
|
|
jmp error_sti
|
[PATCH] x86: error_code is not safe for kprobes
This patch moves the entry.S:error_entry to .kprobes.text section,
since code marked unsafe for kprobes jumps directly to entry.S::error_entry,
that must be marked unsafe as well.
This patch also moves all the ".previous.text" asm directives to ".previous"
for kprobes section.
AK: Following a similar i386 patch from Chuck Ebbert
AK: Also merged Jeremy's fix in.
+From: Jeremy Fitzhardinge <jeremy@goop.org>
KPROBE_ENTRY does a .section .kprobes.text, and expects its users to
do a .previous at the end of the function.
Unfortunately, if any code within the function switches sections, for
example .fixup, then the .previous ends up putting all subsequent code
into .fixup. Worse, any subsequent .fixup code gets intermingled with
the code its supposed to be fixing (which is also in .fixup). It's
surprising this didn't cause more havok.
The fix is to use .pushsection/.popsection, so this stuff nests
properly. A further cleanup would be to get rid of all
.section/.previous pairs, since they're inherently fragile.
+From: Chuck Ebbert <76306.1226@compuserve.com>
Because code marked unsafe for kprobes jumps directly to
entry.S::error_code, that must be marked unsafe as well.
The easiest way to do that is to move the page fault entry
point to just before error_code and let it inherit the same
section.
Also moved all the ".previous" asm directives for kprobes
sections to column 1 and removed ".text" from them.
Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com>
Signed-off-by: Andi Kleen <ak@suse.de>
2006-09-26 16:52:34 +08:00
|
|
|
KPROBE_END(error_entry)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* Reload gs selector with exception handling */
|
|
|
|
/* edi: new selector */
|
|
|
|
ENTRY(load_gs_index)
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_STARTPROC
|
2005-04-17 06:20:36 +08:00
|
|
|
pushf
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2008-01-30 20:32:08 +08:00
|
|
|
DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
|
|
|
|
SWAPGS
|
2005-04-17 06:20:36 +08:00
|
|
|
gs_change:
|
|
|
|
movl %edi,%gs
|
|
|
|
2: mfence /* workaround */
|
2008-01-30 20:32:08 +08:00
|
|
|
SWAPGS
|
2005-04-17 06:20:36 +08:00
|
|
|
popf
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2005-04-17 06:20:36 +08:00
|
|
|
ret
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
ENDPROC(load_gs_index)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
.section __ex_table,"a"
|
|
|
|
.align 8
|
|
|
|
.quad gs_change,bad_gs
|
|
|
|
.previous
|
|
|
|
.section .fixup,"ax"
|
|
|
|
/* running with kernelgs */
|
|
|
|
bad_gs:
|
2008-01-30 20:32:08 +08:00
|
|
|
SWAPGS /* switch back to user gs */
|
2005-04-17 06:20:36 +08:00
|
|
|
xorl %eax,%eax
|
|
|
|
movl %eax,%gs
|
|
|
|
jmp 2b
|
|
|
|
.previous
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create a kernel thread.
|
|
|
|
*
|
|
|
|
* C extern interface:
|
|
|
|
* extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
|
|
|
|
*
|
|
|
|
* asm input arguments:
|
|
|
|
* rdi: fn, rsi: arg, rdx: flags
|
|
|
|
*/
|
|
|
|
ENTRY(kernel_thread)
|
|
|
|
CFI_STARTPROC
|
|
|
|
FAKE_STACK_FRAME $child_rip
|
|
|
|
SAVE_ALL
|
|
|
|
|
|
|
|
# rdi: flags, rsi: usp, rdx: will be &pt_regs
|
|
|
|
movq %rdx,%rdi
|
|
|
|
orq kernel_thread_flags(%rip),%rdi
|
|
|
|
movq $-1, %rsi
|
|
|
|
movq %rsp, %rdx
|
|
|
|
|
|
|
|
xorl %r8d,%r8d
|
|
|
|
xorl %r9d,%r9d
|
|
|
|
|
|
|
|
# clone now
|
|
|
|
call do_fork
|
|
|
|
movq %rax,RAX(%rsp)
|
|
|
|
xorl %edi,%edi
|
|
|
|
|
|
|
|
/*
|
|
|
|
* It isn't worth to check for reschedule here,
|
|
|
|
* so internally to the x86_64 port you can rely on kernel_thread()
|
|
|
|
* not to reschedule the child before returning, this avoids the need
|
|
|
|
* of hacks for example to fork off the per-CPU idle tasks.
|
|
|
|
* [Hopefully no generic code relies on the reschedule -AK]
|
|
|
|
*/
|
|
|
|
RESTORE_ALL
|
|
|
|
UNFAKE_STACK_FRAME
|
|
|
|
ret
|
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
ENDPROC(kernel_thread)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
child_rip:
|
2006-08-31 01:37:08 +08:00
|
|
|
pushq $0 # fake return address
|
|
|
|
CFI_STARTPROC
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Here we are in the child and the registers are set as they were
|
|
|
|
* at kernel_thread() invocation in the parent.
|
|
|
|
*/
|
|
|
|
movq %rdi, %rax
|
|
|
|
movq %rsi, %rdi
|
|
|
|
call *%rax
|
|
|
|
# exit
|
2007-10-18 00:04:33 +08:00
|
|
|
mov %eax, %edi
|
2005-04-17 06:20:36 +08:00
|
|
|
call do_exit
|
2006-08-31 01:37:08 +08:00
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
ENDPROC(child_rip)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
|
|
|
|
*
|
|
|
|
* C extern interface:
|
|
|
|
* extern long execve(char *name, char **argv, char **envp)
|
|
|
|
*
|
|
|
|
* asm input arguments:
|
|
|
|
* rdi: name, rsi: argv, rdx: envp
|
|
|
|
*
|
|
|
|
* We want to fallback into:
|
2008-02-26 19:55:57 +08:00
|
|
|
* extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
* do_sys_execve asm fallback arguments:
|
2008-02-26 19:55:57 +08:00
|
|
|
* rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2006-10-02 17:18:31 +08:00
|
|
|
ENTRY(kernel_execve)
|
2005-04-17 06:20:36 +08:00
|
|
|
CFI_STARTPROC
|
|
|
|
FAKE_STACK_FRAME $0
|
|
|
|
SAVE_ALL
|
2008-02-26 19:55:57 +08:00
|
|
|
movq %rsp,%rcx
|
2005-04-17 06:20:36 +08:00
|
|
|
call sys_execve
|
|
|
|
movq %rax, RAX(%rsp)
|
|
|
|
RESTORE_REST
|
|
|
|
testq %rax,%rax
|
|
|
|
je int_ret_from_sys_call
|
|
|
|
RESTORE_ARGS
|
|
|
|
UNFAKE_STACK_FRAME
|
|
|
|
ret
|
|
|
|
CFI_ENDPROC
|
2006-10-02 17:18:31 +08:00
|
|
|
ENDPROC(kernel_execve)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-09-07 06:19:28 +08:00
|
|
|
KPROBE_ENTRY(page_fault)
|
2005-04-17 06:20:36 +08:00
|
|
|
errorentry do_page_fault
|
[PATCH] x86: error_code is not safe for kprobes
This patch moves the entry.S:error_entry to .kprobes.text section,
since code marked unsafe for kprobes jumps directly to entry.S::error_entry,
that must be marked unsafe as well.
This patch also moves all the ".previous.text" asm directives to ".previous"
for kprobes section.
AK: Following a similar i386 patch from Chuck Ebbert
AK: Also merged Jeremy's fix in.
+From: Jeremy Fitzhardinge <jeremy@goop.org>
KPROBE_ENTRY does a .section .kprobes.text, and expects its users to
do a .previous at the end of the function.
Unfortunately, if any code within the function switches sections, for
example .fixup, then the .previous ends up putting all subsequent code
into .fixup. Worse, any subsequent .fixup code gets intermingled with
the code its supposed to be fixing (which is also in .fixup). It's
surprising this didn't cause more havok.
The fix is to use .pushsection/.popsection, so this stuff nests
properly. A further cleanup would be to get rid of all
.section/.previous pairs, since they're inherently fragile.
+From: Chuck Ebbert <76306.1226@compuserve.com>
Because code marked unsafe for kprobes jumps directly to
entry.S::error_code, that must be marked unsafe as well.
The easiest way to do that is to move the page fault entry
point to just before error_code and let it inherit the same
section.
Also moved all the ".previous" asm directives for kprobes
sections to column 1 and removed ".text" from them.
Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com>
Signed-off-by: Andi Kleen <ak@suse.de>
2006-09-26 16:52:34 +08:00
|
|
|
KPROBE_END(page_fault)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(coprocessor_error)
|
|
|
|
zeroentry do_coprocessor_error
|
2006-06-26 19:56:55 +08:00
|
|
|
END(coprocessor_error)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(simd_coprocessor_error)
|
|
|
|
zeroentry do_simd_coprocessor_error
|
2006-06-26 19:56:55 +08:00
|
|
|
END(simd_coprocessor_error)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(device_not_available)
|
|
|
|
zeroentry math_state_restore
|
2006-06-26 19:56:55 +08:00
|
|
|
END(device_not_available)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* runs on exception stack */
|
2005-09-07 06:19:28 +08:00
|
|
|
KPROBE_ENTRY(debug)
|
2005-09-13 00:49:24 +08:00
|
|
|
INTR_FRAME
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq $0
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2006-01-16 08:56:39 +08:00
|
|
|
paranoidentry do_debug, DEBUG_STACK
|
2006-07-03 15:24:45 +08:00
|
|
|
paranoidexit
|
[PATCH] x86: error_code is not safe for kprobes
This patch moves the entry.S:error_entry to .kprobes.text section,
since code marked unsafe for kprobes jumps directly to entry.S::error_entry,
that must be marked unsafe as well.
This patch also moves all the ".previous.text" asm directives to ".previous"
for kprobes section.
AK: Following a similar i386 patch from Chuck Ebbert
AK: Also merged Jeremy's fix in.
+From: Jeremy Fitzhardinge <jeremy@goop.org>
KPROBE_ENTRY does a .section .kprobes.text, and expects its users to
do a .previous at the end of the function.
Unfortunately, if any code within the function switches sections, for
example .fixup, then the .previous ends up putting all subsequent code
into .fixup. Worse, any subsequent .fixup code gets intermingled with
the code its supposed to be fixing (which is also in .fixup). It's
surprising this didn't cause more havok.
The fix is to use .pushsection/.popsection, so this stuff nests
properly. A further cleanup would be to get rid of all
.section/.previous pairs, since they're inherently fragile.
+From: Chuck Ebbert <76306.1226@compuserve.com>
Because code marked unsafe for kprobes jumps directly to
entry.S::error_code, that must be marked unsafe as well.
The easiest way to do that is to move the page fault entry
point to just before error_code and let it inherit the same
section.
Also moved all the ".previous" asm directives for kprobes
sections to column 1 and removed ".text" from them.
Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com>
Signed-off-by: Andi Kleen <ak@suse.de>
2006-09-26 16:52:34 +08:00
|
|
|
KPROBE_END(debug)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* runs on exception stack */
|
2006-02-04 04:50:41 +08:00
|
|
|
KPROBE_ENTRY(nmi)
|
2005-09-13 00:49:24 +08:00
|
|
|
INTR_FRAME
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq $-1
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2006-07-03 15:24:45 +08:00
|
|
|
paranoidentry do_nmi, 0, 0
|
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
paranoidexit 0
|
|
|
|
#else
|
|
|
|
jmp paranoid_exit1
|
|
|
|
CFI_ENDPROC
|
|
|
|
#endif
|
[PATCH] x86: error_code is not safe for kprobes
This patch moves the entry.S:error_entry to .kprobes.text section,
since code marked unsafe for kprobes jumps directly to entry.S::error_entry,
that must be marked unsafe as well.
This patch also moves all the ".previous.text" asm directives to ".previous"
for kprobes section.
AK: Following a similar i386 patch from Chuck Ebbert
AK: Also merged Jeremy's fix in.
+From: Jeremy Fitzhardinge <jeremy@goop.org>
KPROBE_ENTRY does a .section .kprobes.text, and expects its users to
do a .previous at the end of the function.
Unfortunately, if any code within the function switches sections, for
example .fixup, then the .previous ends up putting all subsequent code
into .fixup. Worse, any subsequent .fixup code gets intermingled with
the code its supposed to be fixing (which is also in .fixup). It's
surprising this didn't cause more havok.
The fix is to use .pushsection/.popsection, so this stuff nests
properly. A further cleanup would be to get rid of all
.section/.previous pairs, since they're inherently fragile.
+From: Chuck Ebbert <76306.1226@compuserve.com>
Because code marked unsafe for kprobes jumps directly to
entry.S::error_code, that must be marked unsafe as well.
The easiest way to do that is to move the page fault entry
point to just before error_code and let it inherit the same
section.
Also moved all the ".previous" asm directives for kprobes
sections to column 1 and removed ".text" from them.
Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com>
Signed-off-by: Andi Kleen <ak@suse.de>
2006-09-26 16:52:34 +08:00
|
|
|
KPROBE_END(nmi)
|
2005-04-17 06:25:03 +08:00
|
|
|
|
2005-09-07 06:19:28 +08:00
|
|
|
KPROBE_ENTRY(int3)
|
2006-01-12 05:43:00 +08:00
|
|
|
INTR_FRAME
|
|
|
|
pushq $0
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
2006-01-16 08:56:39 +08:00
|
|
|
paranoidentry do_int3, DEBUG_STACK
|
2006-07-03 15:24:45 +08:00
|
|
|
jmp paranoid_exit1
|
2006-01-12 05:43:00 +08:00
|
|
|
CFI_ENDPROC
|
[PATCH] x86: error_code is not safe for kprobes
This patch moves the entry.S:error_entry to .kprobes.text section,
since code marked unsafe for kprobes jumps directly to entry.S::error_entry,
that must be marked unsafe as well.
This patch also moves all the ".previous.text" asm directives to ".previous"
for kprobes section.
AK: Following a similar i386 patch from Chuck Ebbert
AK: Also merged Jeremy's fix in.
+From: Jeremy Fitzhardinge <jeremy@goop.org>
KPROBE_ENTRY does a .section .kprobes.text, and expects its users to
do a .previous at the end of the function.
Unfortunately, if any code within the function switches sections, for
example .fixup, then the .previous ends up putting all subsequent code
into .fixup. Worse, any subsequent .fixup code gets intermingled with
the code its supposed to be fixing (which is also in .fixup). It's
surprising this didn't cause more havok.
The fix is to use .pushsection/.popsection, so this stuff nests
properly. A further cleanup would be to get rid of all
.section/.previous pairs, since they're inherently fragile.
+From: Chuck Ebbert <76306.1226@compuserve.com>
Because code marked unsafe for kprobes jumps directly to
entry.S::error_code, that must be marked unsafe as well.
The easiest way to do that is to move the page fault entry
point to just before error_code and let it inherit the same
section.
Also moved all the ".previous" asm directives for kprobes
sections to column 1 and removed ".text" from them.
Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com>
Signed-off-by: Andi Kleen <ak@suse.de>
2006-09-26 16:52:34 +08:00
|
|
|
KPROBE_END(int3)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(overflow)
|
|
|
|
zeroentry do_overflow
|
2006-06-26 19:56:55 +08:00
|
|
|
END(overflow)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(bounds)
|
|
|
|
zeroentry do_bounds
|
2006-06-26 19:56:55 +08:00
|
|
|
END(bounds)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(invalid_op)
|
|
|
|
zeroentry do_invalid_op
|
2006-06-26 19:56:55 +08:00
|
|
|
END(invalid_op)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(coprocessor_segment_overrun)
|
|
|
|
zeroentry do_coprocessor_segment_overrun
|
2006-06-26 19:56:55 +08:00
|
|
|
END(coprocessor_segment_overrun)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* runs on exception stack */
|
|
|
|
ENTRY(double_fault)
|
2005-09-13 00:49:24 +08:00
|
|
|
XCPT_FRAME
|
2005-04-17 06:20:36 +08:00
|
|
|
paranoidentry do_double_fault
|
2006-07-03 15:24:45 +08:00
|
|
|
jmp paranoid_exit1
|
2005-04-17 06:20:36 +08:00
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
END(double_fault)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(invalid_TSS)
|
|
|
|
errorentry do_invalid_TSS
|
2006-06-26 19:56:55 +08:00
|
|
|
END(invalid_TSS)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(segment_not_present)
|
|
|
|
errorentry do_segment_not_present
|
2006-06-26 19:56:55 +08:00
|
|
|
END(segment_not_present)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* runs on exception stack */
|
|
|
|
ENTRY(stack_segment)
|
2005-09-13 00:49:24 +08:00
|
|
|
XCPT_FRAME
|
2005-04-17 06:20:36 +08:00
|
|
|
paranoidentry do_stack_segment
|
2006-07-03 15:24:45 +08:00
|
|
|
jmp paranoid_exit1
|
2005-04-17 06:20:36 +08:00
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
END(stack_segment)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-09-07 06:19:28 +08:00
|
|
|
KPROBE_ENTRY(general_protection)
|
2005-04-17 06:20:36 +08:00
|
|
|
errorentry do_general_protection
|
[PATCH] x86: error_code is not safe for kprobes
This patch moves the entry.S:error_entry to .kprobes.text section,
since code marked unsafe for kprobes jumps directly to entry.S::error_entry,
that must be marked unsafe as well.
This patch also moves all the ".previous.text" asm directives to ".previous"
for kprobes section.
AK: Following a similar i386 patch from Chuck Ebbert
AK: Also merged Jeremy's fix in.
+From: Jeremy Fitzhardinge <jeremy@goop.org>
KPROBE_ENTRY does a .section .kprobes.text, and expects its users to
do a .previous at the end of the function.
Unfortunately, if any code within the function switches sections, for
example .fixup, then the .previous ends up putting all subsequent code
into .fixup. Worse, any subsequent .fixup code gets intermingled with
the code its supposed to be fixing (which is also in .fixup). It's
surprising this didn't cause more havok.
The fix is to use .pushsection/.popsection, so this stuff nests
properly. A further cleanup would be to get rid of all
.section/.previous pairs, since they're inherently fragile.
+From: Chuck Ebbert <76306.1226@compuserve.com>
Because code marked unsafe for kprobes jumps directly to
entry.S::error_code, that must be marked unsafe as well.
The easiest way to do that is to move the page fault entry
point to just before error_code and let it inherit the same
section.
Also moved all the ".previous" asm directives for kprobes
sections to column 1 and removed ".text" from them.
Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com>
Signed-off-by: Andi Kleen <ak@suse.de>
2006-09-26 16:52:34 +08:00
|
|
|
KPROBE_END(general_protection)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(alignment_check)
|
|
|
|
errorentry do_alignment_check
|
2006-06-26 19:56:55 +08:00
|
|
|
END(alignment_check)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(divide_error)
|
|
|
|
zeroentry do_divide_error
|
2006-06-26 19:56:55 +08:00
|
|
|
END(divide_error)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
ENTRY(spurious_interrupt_bug)
|
|
|
|
zeroentry do_spurious_interrupt_bug
|
2006-06-26 19:56:55 +08:00
|
|
|
END(spurious_interrupt_bug)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_X86_MCE
|
|
|
|
/* runs on exception stack */
|
|
|
|
ENTRY(machine_check)
|
2005-09-13 00:49:24 +08:00
|
|
|
INTR_FRAME
|
2005-04-17 06:20:36 +08:00
|
|
|
pushq $0
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
|
|
|
paranoidentry do_machine_check
|
2006-07-03 15:24:45 +08:00
|
|
|
jmp paranoid_exit1
|
2005-04-17 06:20:36 +08:00
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
END(machine_check)
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
2006-08-03 04:37:28 +08:00
|
|
|
/* Call softirq on interrupt stack. Interrupts are off. */
|
2005-07-29 12:15:49 +08:00
|
|
|
ENTRY(call_softirq)
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_STARTPROC
|
2006-08-03 04:37:28 +08:00
|
|
|
push %rbp
|
|
|
|
CFI_ADJUST_CFA_OFFSET 8
|
|
|
|
CFI_REL_OFFSET rbp,0
|
|
|
|
mov %rsp,%rbp
|
|
|
|
CFI_DEF_CFA_REGISTER rbp
|
2005-07-29 12:15:49 +08:00
|
|
|
incl %gs:pda_irqcount
|
2006-08-03 04:37:28 +08:00
|
|
|
cmove %gs:pda_irqstackptr,%rsp
|
|
|
|
push %rbp # backlink for old unwinder
|
2005-07-29 12:15:49 +08:00
|
|
|
call __do_softirq
|
2006-08-03 04:37:28 +08:00
|
|
|
leaveq
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_DEF_CFA_REGISTER rsp
|
2006-08-03 04:37:28 +08:00
|
|
|
CFI_ADJUST_CFA_OFFSET -8
|
2005-07-29 12:15:49 +08:00
|
|
|
decl %gs:pda_irqcount
|
|
|
|
ret
|
2005-09-13 00:49:24 +08:00
|
|
|
CFI_ENDPROC
|
2006-06-26 19:56:55 +08:00
|
|
|
ENDPROC(call_softirq)
|
2007-06-23 08:29:25 +08:00
|
|
|
|
|
|
|
KPROBE_ENTRY(ignore_sysret)
|
|
|
|
CFI_STARTPROC
|
|
|
|
mov $-ENOSYS,%eax
|
|
|
|
sysret
|
|
|
|
CFI_ENDPROC
|
|
|
|
ENDPROC(ignore_sysret)
|