openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

x86/asm/entry: Do mass removal of 'ARGOFFSET' 

ARGOFFSET is zero now, removing it changes no code.

A few macros lost "offset" parameter, since it is always zero
now too.

No code changes - verified with objdump.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/8689f937622d9d2db0ab8be82331fa15e4ed4713.1424989793.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Denys Vlasenko 2015-02-26 14:40:30 -08:00 committed by Ingo Molnar
parent 0d55083698
commit f2db9382c1
3 changed files with 114 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
arch/x86/ia32/ia32entry.S
View File

@ -41,13 +41,13 @@
movl %edx,%edx /* zero extension */ movl %edx,%edx /* zero extension */
.endm .endm
/* clobbers %eax */ /* clobbers %rax */
.macro CLEAR_RREGS offset=0, _r9=rax .macro CLEAR_RREGS _r9=rax
xorl %eax,%eax xorl %eax,%eax
movq %rax,\offset+R11(%rsp) movq %rax,R11(%rsp)
movq %rax,\offset+R10(%rsp) movq %rax,R10(%rsp)
movq %\_r9,\offset+R9(%rsp) movq %\_r9,R9(%rsp)
movq %rax,\offset+R8(%rsp) movq %rax,R8(%rsp)
.endm .endm
/* /*
@ -60,14 +60,14 @@
* If it's -1 to make us punt the syscall, then (u32)-1 is still * If it's -1 to make us punt the syscall, then (u32)-1 is still
* an appropriately invalid value. * an appropriately invalid value.
*/ */
.macro LOAD_ARGS32 offset, _r9=0 .macro LOAD_ARGS32 _r9=0
.if \_r9 .if \_r9
movl \offset+R9(%rsp),%r9d movl R9(%rsp),%r9d
.endif .endif
movl \offset+RCX(%rsp),%ecx movl RCX(%rsp),%ecx
movl \offset+RDX(%rsp),%edx movl RDX(%rsp),%edx
movl \offset+RSI(%rsp),%esi movl RSI(%rsp),%esi
movl \offset+RDI(%rsp),%edi movl RDI(%rsp),%edi
movl %eax,%eax /* zero extension */ movl %eax,%eax /* zero extension */
.endm .endm
@ -158,12 +158,12 @@ ENTRY(ia32_sysenter_target)
* ourselves. To save a few cycles, we can check whether * ourselves. To save a few cycles, we can check whether
* NT was set instead of doing an unconditional popfq. * NT was set instead of doing an unconditional popfq.
*/ */
testl $X86_EFLAGS_NT,EFLAGS-ARGOFFSET(%rsp) testl $X86_EFLAGS_NT,EFLAGS(%rsp)
jnz sysenter_fix_flags jnz sysenter_fix_flags
sysenter_flags_fixed: sysenter_flags_fixed:
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
CFI_REMEMBER_STATE CFI_REMEMBER_STATE
jnz sysenter_tracesys jnz sysenter_tracesys
cmpq $(IA32_NR_syscalls-1),%rax cmpq $(IA32_NR_syscalls-1),%rax
@ -172,16 +172,16 @@ sysenter_do_call:
IA32_ARG_FIXUP IA32_ARG_FIXUP
sysenter_dispatch: sysenter_dispatch:
call *ia32_sys_call_table(,%rax,8) call *ia32_sys_call_table(,%rax,8)
movq %rax,RAX-ARGOFFSET(%rsp) movq %rax,RAX(%rsp)
DISABLE_INTERRUPTS(CLBR_NONE) DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF TRACE_IRQS_OFF
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
jnz sysexit_audit jnz sysexit_audit
sysexit_from_sys_call: sysexit_from_sys_call:
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
/* clear IF, that popfq doesn't enable interrupts early */ /* clear IF, that popfq doesn't enable interrupts early */
andl $~0x200,EFLAGS-ARGOFFSET(%rsp) andl $~0x200,EFLAGS(%rsp)
movl RIP-ARGOFFSET(%rsp),%edx /* User %eip */ movl RIP(%rsp),%edx /* User %eip */
CFI_REGISTER rip,rdx CFI_REGISTER rip,rdx
RESTORE_RSI_RDI RESTORE_RSI_RDI
REMOVE_PT_GPREGS_FROM_STACK 3*8 REMOVE_PT_GPREGS_FROM_STACK 3*8
@ -207,18 +207,18 @@ sysexit_from_sys_call:
movl %ebx,%esi /* 2nd arg: 1st syscall arg */ movl %ebx,%esi /* 2nd arg: 1st syscall arg */
movl %eax,%edi /* 1st arg: syscall number */ movl %eax,%edi /* 1st arg: syscall number */
call __audit_syscall_entry call __audit_syscall_entry
movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall number */ movl RAX(%rsp),%eax /* reload syscall number */
cmpq $(IA32_NR_syscalls-1),%rax cmpq $(IA32_NR_syscalls-1),%rax
ja ia32_badsys ja ia32_badsys
movl %ebx,%edi /* reload 1st syscall arg */ movl %ebx,%edi /* reload 1st syscall arg */
movl RCX-ARGOFFSET(%rsp),%esi /* reload 2nd syscall arg */ movl RCX(%rsp),%esi /* reload 2nd syscall arg */
movl RDX-ARGOFFSET(%rsp),%edx /* reload 3rd syscall arg */ movl RDX(%rsp),%edx /* reload 3rd syscall arg */
movl RSI-ARGOFFSET(%rsp),%ecx /* reload 4th syscall arg */ movl RSI(%rsp),%ecx /* reload 4th syscall arg */
movl RDI-ARGOFFSET(%rsp),%r8d /* reload 5th syscall arg */ movl RDI(%rsp),%r8d /* reload 5th syscall arg */
.endm .endm
.macro auditsys_exit exit .macro auditsys_exit exit
testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
jnz ia32_ret_from_sys_call jnz ia32_ret_from_sys_call
TRACE_IRQS_ON TRACE_IRQS_ON
ENABLE_INTERRUPTS(CLBR_NONE) ENABLE_INTERRUPTS(CLBR_NONE)
@ -229,13 +229,13 @@ sysexit_from_sys_call:
1: setbe %al /* 1 if error, 0 if not */ 1: setbe %al /* 1 if error, 0 if not */
movzbl %al,%edi /* zero-extend that into %edi */ movzbl %al,%edi /* zero-extend that into %edi */
call __audit_syscall_exit call __audit_syscall_exit
movq RAX-ARGOFFSET(%rsp),%rax /* reload syscall return value */ movq RAX(%rsp),%rax /* reload syscall return value */
movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
DISABLE_INTERRUPTS(CLBR_NONE) DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF TRACE_IRQS_OFF
testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl %edi,TI_flags+THREAD_INFO(%rsp,RIP)
jz \exit jz \exit
CLEAR_RREGS -ARGOFFSET CLEAR_RREGS
jmp int_with_check jmp int_with_check
.endm .endm
@ -255,7 +255,7 @@ sysenter_fix_flags:
sysenter_tracesys: sysenter_tracesys:
#ifdef CONFIG_AUDITSYSCALL #ifdef CONFIG_AUDITSYSCALL
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
jz sysenter_auditsys jz sysenter_auditsys
#endif #endif
SAVE_EXTRA_REGS SAVE_EXTRA_REGS
@ -263,7 +263,7 @@ sysenter_tracesys:
movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */ movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
movq %rsp,%rdi /* &pt_regs -> arg1 */ movq %rsp,%rdi /* &pt_regs -> arg1 */
call syscall_trace_enter call syscall_trace_enter
LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */ LOAD_ARGS32 /* reload args from stack in case ptrace changed it */
RESTORE_EXTRA_REGS RESTORE_EXTRA_REGS
cmpq $(IA32_NR_syscalls-1),%rax cmpq $(IA32_NR_syscalls-1),%rax
ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */ ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
@ -309,17 +309,17 @@ ENTRY(ia32_cstar_target)
ALLOC_PT_GPREGS_ON_STACK 8 ALLOC_PT_GPREGS_ON_STACK 8
SAVE_C_REGS_EXCEPT_RCX_R891011 SAVE_C_REGS_EXCEPT_RCX_R891011
movl %eax,%eax /* zero extension */ movl %eax,%eax /* zero extension */
movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rax,ORIG_RAX(%rsp)
movq %rcx,RIP-ARGOFFSET(%rsp) movq %rcx,RIP(%rsp)
CFI_REL_OFFSET rip,RIP-ARGOFFSET CFI_REL_OFFSET rip,RIP
movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */ movq %rbp,RCX(%rsp) /* this lies slightly to ptrace */
movl %ebp,%ecx movl %ebp,%ecx
movq $__USER32_CS,CS-ARGOFFSET(%rsp) movq $__USER32_CS,CS(%rsp)
movq $__USER32_DS,SS-ARGOFFSET(%rsp) movq $__USER32_DS,SS(%rsp)
movq %r11,EFLAGS-ARGOFFSET(%rsp) movq %r11,EFLAGS(%rsp)
/*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/ /*CFI_REL_OFFSET rflags,EFLAGS*/
movq %r8,RSP-ARGOFFSET(%rsp) movq %r8,RSP(%rsp)
CFI_REL_OFFSET rsp,RSP-ARGOFFSET CFI_REL_OFFSET rsp,RSP
/* no need to do an access_ok check here because r8 has been /* no need to do an access_ok check here because r8 has been
32bit zero extended */ 32bit zero extended */
/* hardware stack frame is complete now */ /* hardware stack frame is complete now */
@ -327,8 +327,8 @@ ENTRY(ia32_cstar_target)
1: movl (%r8),%r9d 1: movl (%r8),%r9d
_ASM_EXTABLE(1b,ia32_badarg) _ASM_EXTABLE(1b,ia32_badarg)
ASM_CLAC ASM_CLAC
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
CFI_REMEMBER_STATE CFI_REMEMBER_STATE
jnz cstar_tracesys jnz cstar_tracesys
cmpq $IA32_NR_syscalls-1,%rax cmpq $IA32_NR_syscalls-1,%rax
@ -337,32 +337,32 @@ cstar_do_call:
IA32_ARG_FIXUP 1 IA32_ARG_FIXUP 1
cstar_dispatch: cstar_dispatch:
call *ia32_sys_call_table(,%rax,8) call *ia32_sys_call_table(,%rax,8)
movq %rax,RAX-ARGOFFSET(%rsp) movq %rax,RAX(%rsp)
DISABLE_INTERRUPTS(CLBR_NONE) DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF TRACE_IRQS_OFF
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
jnz sysretl_audit jnz sysretl_audit
sysretl_from_sys_call: sysretl_from_sys_call:
andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
RESTORE_RSI_RDI_RDX RESTORE_RSI_RDI_RDX
movl RIP-ARGOFFSET(%rsp),%ecx movl RIP(%rsp),%ecx
CFI_REGISTER rip,rcx CFI_REGISTER rip,rcx
movl EFLAGS-ARGOFFSET(%rsp),%r11d movl EFLAGS(%rsp),%r11d
/*CFI_REGISTER rflags,r11*/ /*CFI_REGISTER rflags,r11*/
xorq %r10,%r10 xorq %r10,%r10
xorq %r9,%r9 xorq %r9,%r9
xorq %r8,%r8 xorq %r8,%r8
TRACE_IRQS_ON TRACE_IRQS_ON
movl RSP-ARGOFFSET(%rsp),%esp movl RSP(%rsp),%esp
CFI_RESTORE rsp CFI_RESTORE rsp
USERGS_SYSRET32 USERGS_SYSRET32
#ifdef CONFIG_AUDITSYSCALL #ifdef CONFIG_AUDITSYSCALL
cstar_auditsys: cstar_auditsys:
CFI_RESTORE_STATE CFI_RESTORE_STATE
movl %r9d,R9-ARGOFFSET(%rsp) /* register to be clobbered by call */ movl %r9d,R9(%rsp) /* register to be clobbered by call */
auditsys_entry_common auditsys_entry_common
movl R9-ARGOFFSET(%rsp),%r9d /* reload 6th syscall arg */ movl R9(%rsp),%r9d /* reload 6th syscall arg */
jmp cstar_dispatch jmp cstar_dispatch
sysretl_audit: sysretl_audit:
@ -371,16 +371,16 @@ sysretl_audit:
cstar_tracesys: cstar_tracesys:
#ifdef CONFIG_AUDITSYSCALL #ifdef CONFIG_AUDITSYSCALL
testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP)
jz cstar_auditsys jz cstar_auditsys
#endif #endif
xchgl %r9d,%ebp xchgl %r9d,%ebp
SAVE_EXTRA_REGS SAVE_EXTRA_REGS
CLEAR_RREGS 0, r9 CLEAR_RREGS r9
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */ movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
movq %rsp,%rdi /* &pt_regs -> arg1 */ movq %rsp,%rdi /* &pt_regs -> arg1 */
call syscall_trace_enter call syscall_trace_enter
LOAD_ARGS32 ARGOFFSET, 1 /* reload args from stack in case ptrace changed it */ LOAD_ARGS32 1 /* reload args from stack in case ptrace changed it */
RESTORE_EXTRA_REGS RESTORE_EXTRA_REGS
xchgl %ebp,%r9d xchgl %ebp,%r9d
cmpq $(IA32_NR_syscalls-1),%rax cmpq $(IA32_NR_syscalls-1),%rax
@ -438,8 +438,8 @@ ENTRY(ia32_syscall)
this could be a problem. */ this could be a problem. */
ALLOC_PT_GPREGS_ON_STACK ALLOC_PT_GPREGS_ON_STACK
SAVE_C_REGS_EXCEPT_R891011 SAVE_C_REGS_EXCEPT_R891011
orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP)
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
jnz ia32_tracesys jnz ia32_tracesys
cmpq $(IA32_NR_syscalls-1),%rax cmpq $(IA32_NR_syscalls-1),%rax
ja ia32_badsys ja ia32_badsys
@ -447,9 +447,9 @@ ia32_do_call:
IA32_ARG_FIXUP IA32_ARG_FIXUP
call *ia32_sys_call_table(,%rax,8) # xxx: rip relative call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
ia32_sysret: ia32_sysret:
movq %rax,RAX-ARGOFFSET(%rsp) movq %rax,RAX(%rsp)
ia32_ret_from_sys_call: ia32_ret_from_sys_call:
CLEAR_RREGS -ARGOFFSET CLEAR_RREGS
jmp int_ret_from_sys_call jmp int_ret_from_sys_call
ia32_tracesys: ia32_tracesys:
@ -458,7 +458,7 @@ ia32_tracesys:
movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */ movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
movq %rsp,%rdi /* &pt_regs -> arg1 */ movq %rsp,%rdi /* &pt_regs -> arg1 */
call syscall_trace_enter call syscall_trace_enter
LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */ LOAD_ARGS32 /* reload args from stack in case ptrace changed it */
RESTORE_EXTRA_REGS RESTORE_EXTRA_REGS
cmpq $(IA32_NR_syscalls-1),%rax cmpq $(IA32_NR_syscalls-1),%rax
ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */ ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */
@ -466,7 +466,7 @@ ia32_tracesys:
END(ia32_syscall) END(ia32_syscall)
ia32_badsys: ia32_badsys:
movq $0,ORIG_RAX-ARGOFFSET(%rsp) movq $0,ORIG_RAX(%rsp)
movq $-ENOSYS,%rax movq $-ENOSYS,%rax
jmp ia32_sysret jmp ia32_sysret
@ -499,17 +499,17 @@ ia32_ptregs_common:
CFI_ENDPROC CFI_ENDPROC
CFI_STARTPROC32 simple CFI_STARTPROC32 simple
CFI_SIGNAL_FRAME CFI_SIGNAL_FRAME
CFI_DEF_CFA rsp,SS+8-ARGOFFSET CFI_DEF_CFA rsp,SS+8
CFI_REL_OFFSET rax,RAX-ARGOFFSET CFI_REL_OFFSET rax,RAX
CFI_REL_OFFSET rcx,RCX-ARGOFFSET CFI_REL_OFFSET rcx,RCX
CFI_REL_OFFSET rdx,RDX-ARGOFFSET CFI_REL_OFFSET rdx,RDX
CFI_REL_OFFSET rsi,RSI-ARGOFFSET CFI_REL_OFFSET rsi,RSI
CFI_REL_OFFSET rdi,RDI-ARGOFFSET CFI_REL_OFFSET rdi,RDI
CFI_REL_OFFSET rip,RIP-ARGOFFSET CFI_REL_OFFSET rip,RIP
/* CFI_REL_OFFSET cs,CS-ARGOFFSET*/ /* CFI_REL_OFFSET cs,CS*/
/* CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/ /* CFI_REL_OFFSET rflags,EFLAGS*/
CFI_REL_OFFSET rsp,RSP-ARGOFFSET CFI_REL_OFFSET rsp,RSP
/* CFI_REL_OFFSET ss,SS-ARGOFFSET*/ /* CFI_REL_OFFSET ss,SS*/
SAVE_EXTRA_REGS 8 SAVE_EXTRA_REGS 8
call *%rax call *%rax
RESTORE_EXTRA_REGS 8 RESTORE_EXTRA_REGS 8

2
arch/x86/include/asm/calling.h
View File

@ -88,8 +88,6 @@ For 32-bit we have the following conventions - kernel is built with
#define RSP 19*8 #define RSP 19*8
#define SS 20*8 #define SS 20*8
#define ARGOFFSET 0
.macro ALLOC_PT_GPREGS_ON_STACK addskip=0 .macro ALLOC_PT_GPREGS_ON_STACK addskip=0
subq $15*8+\addskip, %rsp subq $15*8+\addskip, %rsp
CFI_ADJUST_CFA_OFFSET 15*8+\addskip CFI_ADJUST_CFA_OFFSET 15*8+\addskip

84
arch/x86/kernel/entry_64.S
View File

@ -73,9 +73,9 @@ ENDPROC(native_usergs_sysret64)
#endif /* CONFIG_PARAVIRT */ #endif /* CONFIG_PARAVIRT */
.macro TRACE_IRQS_IRETQ offset=ARGOFFSET .macro TRACE_IRQS_IRETQ
#ifdef CONFIG_TRACE_IRQFLAGS #ifdef CONFIG_TRACE_IRQFLAGS
bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */ bt $9,EFLAGS(%rsp) /* interrupts off? */
jnc 1f jnc 1f
TRACE_IRQS_ON TRACE_IRQS_ON
1: 1:
@ -107,8 +107,8 @@ ENDPROC(native_usergs_sysret64)
call debug_stack_reset call debug_stack_reset
.endm .endm
.macro TRACE_IRQS_IRETQ_DEBUG offset=ARGOFFSET .macro TRACE_IRQS_IRETQ_DEBUG
bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */ bt $9,EFLAGS(%rsp) /* interrupts off? */
jnc 1f jnc 1f
TRACE_IRQS_ON_DEBUG TRACE_IRQS_ON_DEBUG
1: 1:
@ -184,16 +184,16 @@ ENDPROC(native_usergs_sysret64)
* frame that enables passing a complete pt_regs to a C function. * frame that enables passing a complete pt_regs to a C function.
*/ */
.macro DEFAULT_FRAME start=1 offset=0 .macro DEFAULT_FRAME start=1 offset=0
XCPT_FRAME \start, ORIG_RAX+\offset-ARGOFFSET XCPT_FRAME \start, ORIG_RAX+\offset
CFI_REL_OFFSET rdi, RDI+\offset-ARGOFFSET CFI_REL_OFFSET rdi, RDI+\offset
CFI_REL_OFFSET rsi, RSI+\offset-ARGOFFSET CFI_REL_OFFSET rsi, RSI+\offset
CFI_REL_OFFSET rdx, RDX+\offset-ARGOFFSET CFI_REL_OFFSET rdx, RDX+\offset
CFI_REL_OFFSET rcx, RCX+\offset-ARGOFFSET CFI_REL_OFFSET rcx, RCX+\offset
CFI_REL_OFFSET rax, RAX+\offset-ARGOFFSET CFI_REL_OFFSET rax, RAX+\offset
CFI_REL_OFFSET r8, R8+\offset-ARGOFFSET CFI_REL_OFFSET r8, R8+\offset
CFI_REL_OFFSET r9, R9+\offset-ARGOFFSET CFI_REL_OFFSET r9, R9+\offset
CFI_REL_OFFSET r10, R10+\offset-ARGOFFSET CFI_REL_OFFSET r10, R10+\offset
CFI_REL_OFFSET r11, R11+\offset-ARGOFFSET CFI_REL_OFFSET r11, R11+\offset
CFI_REL_OFFSET rbx, RBX+\offset CFI_REL_OFFSET rbx, RBX+\offset
CFI_REL_OFFSET rbp, RBP+\offset CFI_REL_OFFSET rbp, RBP+\offset
CFI_REL_OFFSET r12, R12+\offset CFI_REL_OFFSET r12, R12+\offset
@ -237,13 +237,13 @@ ENTRY(ret_from_fork)
RESTORE_EXTRA_REGS RESTORE_EXTRA_REGS
testl $3, CS-ARGOFFSET(%rsp) # from kernel_thread? testl $3,CS(%rsp) # from kernel_thread?
jz 1f jz 1f
testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET
jnz int_ret_from_sys_call jnz int_ret_from_sys_call
RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET RESTORE_TOP_OF_STACK %rdi
jmp ret_from_sys_call # go to the SYSRET fastpath jmp ret_from_sys_call # go to the SYSRET fastpath
1: 1:
@ -310,11 +310,11 @@ GLOBAL(system_call_after_swapgs)
ENABLE_INTERRUPTS(CLBR_NONE) ENABLE_INTERRUPTS(CLBR_NONE)
ALLOC_PT_GPREGS_ON_STACK 8 ALLOC_PT_GPREGS_ON_STACK 8
SAVE_C_REGS_EXCEPT_RAX_RCX SAVE_C_REGS_EXCEPT_RAX_RCX
movq $-ENOSYS,RAX-ARGOFFSET(%rsp) movq $-ENOSYS,RAX(%rsp)
movq_cfi rax,(ORIG_RAX-ARGOFFSET) movq_cfi rax,ORIG_RAX
movq %rcx,RIP-ARGOFFSET(%rsp) movq %rcx,RIP(%rsp)
CFI_REL_OFFSET rip,RIP-ARGOFFSET CFI_REL_OFFSET rip,RIP
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP)
jnz tracesys jnz tracesys
system_call_fastpath: system_call_fastpath:
#if __SYSCALL_MASK == ~0 #if __SYSCALL_MASK == ~0
@ -326,13 +326,13 @@ system_call_fastpath:
ja ret_from_sys_call /* and return regs->ax */ ja ret_from_sys_call /* and return regs->ax */
movq %r10,%rcx movq %r10,%rcx
call *sys_call_table(,%rax,8) # XXX: rip relative call *sys_call_table(,%rax,8) # XXX: rip relative
movq %rax,RAX-ARGOFFSET(%rsp) movq %rax,RAX(%rsp)
/* /*
* Syscall return path ending with SYSRET (fast path) * Syscall return path ending with SYSRET (fast path)
* Has incomplete stack frame and undefined top of stack. * Has incomplete stack frame and undefined top of stack.
*/ */
ret_from_sys_call: ret_from_sys_call:
testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP)
jnz int_ret_from_sys_call_fixup /* Go the the slow path */ jnz int_ret_from_sys_call_fixup /* Go the the slow path */
LOCKDEP_SYS_EXIT LOCKDEP_SYS_EXIT
@ -344,7 +344,7 @@ ret_from_sys_call:
*/ */
TRACE_IRQS_ON TRACE_IRQS_ON
RESTORE_C_REGS_EXCEPT_RCX RESTORE_C_REGS_EXCEPT_RCX
movq RIP-ARGOFFSET(%rsp),%rcx movq RIP(%rsp),%rcx
CFI_REGISTER rip,rcx CFI_REGISTER rip,rcx
/*CFI_REGISTER rflags,r11*/ /*CFI_REGISTER rflags,r11*/
movq PER_CPU_VAR(old_rsp), %rsp movq PER_CPU_VAR(old_rsp), %rsp
@ -353,7 +353,7 @@ ret_from_sys_call:
CFI_RESTORE_STATE CFI_RESTORE_STATE
int_ret_from_sys_call_fixup: int_ret_from_sys_call_fixup:
FIXUP_TOP_OF_STACK %r11, -ARGOFFSET FIXUP_TOP_OF_STACK %r11
jmp int_ret_from_sys_call jmp int_ret_from_sys_call
/* Do syscall tracing */ /* Do syscall tracing */
@ -364,7 +364,7 @@ tracesys:
test %rax, %rax test %rax, %rax
jnz tracesys_phase2 /* if needed, run the slow path */ jnz tracesys_phase2 /* if needed, run the slow path */
RESTORE_C_REGS_EXCEPT_RAX /* else restore clobbered regs */ RESTORE_C_REGS_EXCEPT_RAX /* else restore clobbered regs */
movq ORIG_RAX-ARGOFFSET(%rsp), %rax movq ORIG_RAX(%rsp), %rax
jmp system_call_fastpath /* and return to the fast path */ jmp system_call_fastpath /* and return to the fast path */
tracesys_phase2: tracesys_phase2:
@ -391,7 +391,7 @@ tracesys_phase2:
ja int_ret_from_sys_call /* RAX(%rsp) is already set */ ja int_ret_from_sys_call /* RAX(%rsp) is already set */
movq %r10,%rcx /* fixup for C */ movq %r10,%rcx /* fixup for C */
call *sys_call_table(,%rax,8) call *sys_call_table(,%rax,8)
movq %rax,RAX-ARGOFFSET(%rsp) movq %rax,RAX(%rsp)
/* Use IRET because user could have changed frame */ /* Use IRET because user could have changed frame */
/* /*
@ -475,9 +475,9 @@ END(stub_\func)
ENTRY(\label) ENTRY(\label)
CFI_STARTPROC CFI_STARTPROC
DEFAULT_FRAME 0, 8 /* offset 8: return address */ DEFAULT_FRAME 0, 8 /* offset 8: return address */
FIXUP_TOP_OF_STACK %r11, 8-ARGOFFSET FIXUP_TOP_OF_STACK %r11, 8
call \func call \func
RESTORE_TOP_OF_STACK %r11, 8-ARGOFFSET RESTORE_TOP_OF_STACK %r11, 8
ret ret
CFI_ENDPROC CFI_ENDPROC
END(\label) END(\label)
@ -677,7 +677,7 @@ common_interrupt:
ASM_CLAC ASM_CLAC
addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */ addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
interrupt do_IRQ interrupt do_IRQ
/* 0(%rsp): old_rsp-ARGOFFSET */ /* 0(%rsp): old_rsp */
ret_from_intr: ret_from_intr:
DISABLE_INTERRUPTS(CLBR_NONE) DISABLE_INTERRUPTS(CLBR_NONE)
TRACE_IRQS_OFF TRACE_IRQS_OFF
@ -687,13 +687,13 @@ ret_from_intr:
popq %rsi popq %rsi
CFI_DEF_CFA rsi,SS+8-RBP /* reg/off reset after def_cfa_expr */ CFI_DEF_CFA rsi,SS+8-RBP /* reg/off reset after def_cfa_expr */
/* return code expects complete pt_regs - adjust rsp accordingly: */ /* return code expects complete pt_regs - adjust rsp accordingly: */
leaq ARGOFFSET-RBP(%rsi), %rsp leaq -RBP(%rsi),%rsp
CFI_DEF_CFA_REGISTER rsp CFI_DEF_CFA_REGISTER rsp
CFI_ADJUST_CFA_OFFSET RBP-ARGOFFSET CFI_ADJUST_CFA_OFFSET RBP
exit_intr: exit_intr:
GET_THREAD_INFO(%rcx) GET_THREAD_INFO(%rcx)
testl $3,CS-ARGOFFSET(%rsp) testl $3,CS(%rsp)
je retint_kernel je retint_kernel
/* Interrupt came from user space */ /* Interrupt came from user space */
@ -721,8 +721,8 @@ retint_swapgs: /* return to user-space */
* Try to use SYSRET instead of IRET if we're returning to * Try to use SYSRET instead of IRET if we're returning to
* a completely clean 64-bit userspace context. * a completely clean 64-bit userspace context.
*/ */
movq (RCX-ARGOFFSET)(%rsp), %rcx movq RCX(%rsp),%rcx
cmpq %rcx,(RIP-ARGOFFSET)(%rsp) /* RCX == RIP */ cmpq %rcx,RIP(%rsp) /* RCX == RIP */
jne opportunistic_sysret_failed jne opportunistic_sysret_failed
/* /*
@ -743,11 +743,11 @@ retint_swapgs: /* return to user-space */
shr $__VIRTUAL_MASK_SHIFT, %rcx shr $__VIRTUAL_MASK_SHIFT, %rcx
jnz opportunistic_sysret_failed jnz opportunistic_sysret_failed
cmpq $__USER_CS,(CS-ARGOFFSET)(%rsp) /* CS must match SYSRET */ cmpq $__USER_CS,CS(%rsp) /* CS must match SYSRET */
jne opportunistic_sysret_failed jne opportunistic_sysret_failed
movq (R11-ARGOFFSET)(%rsp), %r11 movq R11(%rsp),%r11
cmpq %r11,(EFLAGS-ARGOFFSET)(%rsp) /* R11 == RFLAGS */ cmpq %r11,EFLAGS(%rsp) /* R11 == RFLAGS */
jne opportunistic_sysret_failed jne opportunistic_sysret_failed
testq $X86_EFLAGS_RF,%r11 /* sysret can't restore RF */ testq $X86_EFLAGS_RF,%r11 /* sysret can't restore RF */
@ -755,7 +755,7 @@ retint_swapgs: /* return to user-space */
/* nothing to check for RSP */ /* nothing to check for RSP */
cmpq $__USER_DS,(SS-ARGOFFSET)(%rsp) /* SS must match SYSRET */ cmpq $__USER_DS,SS(%rsp) /* SS must match SYSRET */
jne opportunistic_sysret_failed jne opportunistic_sysret_failed
/* /*
@ -870,7 +870,7 @@ retint_signal:
ENTRY(retint_kernel) ENTRY(retint_kernel)
cmpl $0,PER_CPU_VAR(__preempt_count) cmpl $0,PER_CPU_VAR(__preempt_count)
jnz retint_restore_args jnz retint_restore_args
bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ bt $9,EFLAGS(%rsp) /* interrupts off? */
jnc retint_restore_args jnc retint_restore_args
call preempt_schedule_irq call preempt_schedule_irq
jmp exit_intr jmp exit_intr
@ -1286,11 +1286,11 @@ ENTRY(paranoid_exit)
TRACE_IRQS_OFF_DEBUG TRACE_IRQS_OFF_DEBUG
testl %ebx,%ebx /* swapgs needed? */ testl %ebx,%ebx /* swapgs needed? */
jnz paranoid_exit_no_swapgs jnz paranoid_exit_no_swapgs
TRACE_IRQS_IRETQ 0 TRACE_IRQS_IRETQ
SWAPGS_UNSAFE_STACK SWAPGS_UNSAFE_STACK
jmp paranoid_exit_restore jmp paranoid_exit_restore
paranoid_exit_no_swapgs: paranoid_exit_no_swapgs:
TRACE_IRQS_IRETQ_DEBUG 0 TRACE_IRQS_IRETQ_DEBUG
paranoid_exit_restore: paranoid_exit_restore:
RESTORE_EXTRA_REGS RESTORE_EXTRA_REGS
RESTORE_C_REGS RESTORE_C_REGS