x86/uaccess/64: Make the __copy_user_nocache() assembly code more readable
Add comments to __copy_user_nocache() to clarify its procedures
and alignment requirements.
Also change numeric branch target labels to named local labels.
No code changed:
arch/x86/lib/copy_user_64.o:
text data bss dec hex filename
1239 0 0 1239 4d7 copy_user_64.o.before
1239 0 0 1239 4d7 copy_user_64.o.after
md5:
58bed94c2db98c1ca9a2d46d0680aaae copy_user_64.o.before.asm
58bed94c2db98c1ca9a2d46d0680aaae copy_user_64.o.after.asm
Signed-off-by: Toshi Kani <toshi.kani@hpe.com>
Cc: <stable@vger.kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Luis R. Rodriguez <mcgrof@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Toshi Kani <toshi.kani@hp.com>
Cc: brian.boylston@hpe.com
Cc: dan.j.williams@intel.com
Cc: linux-nvdimm@lists.01.org
Cc: micah.parrish@hpe.com
Cc: ross.zwisler@linux.intel.com
Cc: vishal.l.verma@intel.com
Link: http://lkml.kernel.org/r/1455225857-12039-2-git-send-email-toshi.kani@hpe.com
[ Small readability edits and added object file comparison. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Toshi Kani
Ingo Molnar
parent
02a5f765c9
commit
ee9737c924
|
|
@ -232,17 +232,30 @@ ENDPROC(copy_user_enhanced_fast_string)
|
|||
|
||||
/*
|
||||
* copy_user_nocache - Uncached memory copy with exception handling
|
||||
* This will force destination/source out of cache for more performance.
|
||||
* This will force destination out of cache for more performance.
|
||||
*
|
||||
* Note: Cached memory copy is used when destination or size is not
|
||||
* naturally aligned. That is:
|
||||
* - Require 8-byte alignment when size is 8 bytes or larger.
|
||||
*/
|
||||
ENTRY(__copy_user_nocache)
|
||||
ASM_STAC
|
||||
|
||||
/* If size is less than 8 bytes, go to byte copy */
|
||||
cmpl $8,%edx
|
||||
jb 20f /* less then 8 bytes, go to byte copy loop */
|
||||
jb .L_1b_cache_copy_entry
|
||||
|
||||
/* If destination is not 8-byte aligned, "cache" copy to align it */
|
||||
ALIGN_DESTINATION
|
||||
|
||||
/* Set 4x8-byte copy count and remainder */
|
||||
movl %edx,%ecx
|
||||
andl $63,%edx
|
||||
shrl $6,%ecx
|
||||
jz 17f
|
||||
jz .L_8b_nocache_copy_entry /* jump if count is 0 */
|
||||
|
||||
/* Perform 4x8-byte nocache loop-copy */
|
||||
.L_4x8b_nocache_copy_loop:
|
||||
1: movq (%rsi),%r8
|
||||
2: movq 1*8(%rsi),%r9
|
||||
3: movq 2*8(%rsi),%r10
|
||||
|
|
@ -262,60 +275,79 @@ ENTRY(__copy_user_nocache)
|
|||
leaq 64(%rsi),%rsi
|
||||
leaq 64(%rdi),%rdi
|
||||
decl %ecx
|
||||
jnz 1b
|
||||
17: movl %edx,%ecx
|
||||
jnz .L_4x8b_nocache_copy_loop
|
||||
|
||||
/* Set 8-byte copy count and remainder */
|
||||
.L_8b_nocache_copy_entry:
|
||||
movl %edx,%ecx
|
||||
andl $7,%edx
|
||||
shrl $3,%ecx
|
||||
jz 20f
|
||||
18: movq (%rsi),%r8
|
||||
19: movnti %r8,(%rdi)
|
||||
jz .L_1b_cache_copy_entry /* jump if count is 0 */
|
||||
|
||||
/* Perform 8-byte nocache loop-copy */
|
||||
.L_8b_nocache_copy_loop:
|
||||
20: movq (%rsi),%r8
|
||||
21: movnti %r8,(%rdi)
|
||||
leaq 8(%rsi),%rsi
|
||||
leaq 8(%rdi),%rdi
|
||||
decl %ecx
|
||||
jnz 18b
|
||||
20: andl %edx,%edx
|
||||
jz 23f
|
||||
jnz .L_8b_nocache_copy_loop
|
||||
|
||||
/* If no byte left, we're done */
|
||||
.L_1b_cache_copy_entry:
|
||||
andl %edx,%edx
|
||||
jz .L_finish_copy
|
||||
|
||||
/* Perform byte "cache" loop-copy for the remainder */
|
||||
movl %edx,%ecx
|
||||
21: movb (%rsi),%al
|
||||
22: movb %al,(%rdi)
|
||||
.L_1b_cache_copy_loop:
|
||||
40: movb (%rsi),%al
|
||||
41: movb %al,(%rdi)
|
||||
incq %rsi
|
||||
incq %rdi
|
||||
decl %ecx
|
||||
jnz 21b
|
||||
23: xorl %eax,%eax
|
||||
jnz .L_1b_cache_copy_loop
|
||||
|
||||
/* Finished copying; fence the prior stores */
|
||||
.L_finish_copy:
|
||||
xorl %eax,%eax
|
||||
ASM_CLAC
|
||||
sfence
|
||||
ret
|
||||
|
||||
.section .fixup,"ax"
|
||||
30: shll $6,%ecx
|
||||
.L_fixup_4x8b_copy:
|
||||
shll $6,%ecx
|
||||
addl %ecx,%edx
|
||||
jmp 60f
|
||||
40: lea (%rdx,%rcx,8),%rdx
|
||||
jmp 60f
|
||||
50: movl %ecx,%edx
|
||||
60: sfence
|
||||
jmp .L_fixup_handle_tail
|
||||
.L_fixup_8b_copy:
|
||||
lea (%rdx,%rcx,8),%rdx
|
||||
jmp .L_fixup_handle_tail
|
||||
.L_fixup_1b_copy:
|
||||
movl %ecx,%edx
|
||||
.L_fixup_handle_tail:
|
||||
sfence
|
||||
jmp copy_user_handle_tail
|
||||
.previous
|
||||
|
||||
_ASM_EXTABLE(1b,30b)
|
||||
_ASM_EXTABLE(2b,30b)
|
||||
_ASM_EXTABLE(3b,30b)
|
||||
_ASM_EXTABLE(4b,30b)
|
||||
_ASM_EXTABLE(5b,30b)
|
||||
_ASM_EXTABLE(6b,30b)
|
||||
_ASM_EXTABLE(7b,30b)
|
||||
_ASM_EXTABLE(8b,30b)
|
||||
_ASM_EXTABLE(9b,30b)
|
||||
_ASM_EXTABLE(10b,30b)
|
||||
_ASM_EXTABLE(11b,30b)
|
||||
_ASM_EXTABLE(12b,30b)
|
||||
_ASM_EXTABLE(13b,30b)
|
||||
_ASM_EXTABLE(14b,30b)
|
||||
_ASM_EXTABLE(15b,30b)
|
||||
_ASM_EXTABLE(16b,30b)
|
||||
_ASM_EXTABLE(18b,40b)
|
||||
_ASM_EXTABLE(19b,40b)
|
||||
_ASM_EXTABLE(21b,50b)
|
||||
_ASM_EXTABLE(22b,50b)
|
||||
_ASM_EXTABLE(1b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(2b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(3b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(4b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(5b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(6b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(7b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(8b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(9b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(10b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(11b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(12b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(13b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(14b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(15b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(16b,.L_fixup_4x8b_copy)
|
||||
_ASM_EXTABLE(20b,.L_fixup_8b_copy)
|
||||
_ASM_EXTABLE(21b,.L_fixup_8b_copy)
|
||||
_ASM_EXTABLE(40b,.L_fixup_1b_copy)
|
||||
_ASM_EXTABLE(41b,.L_fixup_1b_copy)
|
||||
ENDPROC(__copy_user_nocache)
|
||||
|
|
|
|||
Loading…
Reference in New Issue