/* Copyright 2002 Andi Kleen, SuSE Labs. * Subject to the GNU Public License v2. * * Functions to copy from and to user space. */ #include <linux/linkage.h> #include <asm/dwarf2.h> #define FIX_ALIGNMENT 1 #include <asm/current.h> #include <asm/asm-offsets.h> #include <asm/thread_info.h> #include <asm/cpufeature.h> /* * copy_user_nocache - Uncached memory copy with exception handling * This will force destination/source out of cache for more performance. * * Input: * rdi destination * rsi source * rdx count * rcx zero flag when 1 zero on exception * * Output: * eax uncopied bytes or 0 if successful. */ ENTRY(__copy_user_nocache) CFI_STARTPROC pushq %rbx CFI_ADJUST_CFA_OFFSET 8 CFI_REL_OFFSET rbx, 0 pushq %rcx /* save zero flag */ CFI_ADJUST_CFA_OFFSET 8 CFI_REL_OFFSET rcx, 0 xorl %eax,%eax /* zero for the exception handler */ #ifdef FIX_ALIGNMENT /* check for bad alignment of destination */ movl %edi,%ecx andl $7,%ecx jnz .Lbad_alignment .Lafter_bad_alignment: #endif movq %rdx,%rcx movl $64,%ebx shrq $6,%rdx decq %rdx js .Lhandle_tail .p2align 4 .Lloop: .Ls1: movq (%rsi),%r11 .Ls2: movq 1*8(%rsi),%r8 .Ls3: movq 2*8(%rsi),%r9 .Ls4: movq 3*8(%rsi),%r10 .Ld1: movnti %r11,(%rdi) .Ld2: movnti %r8,1*8(%rdi) .Ld3: movnti %r9,2*8(%rdi) .Ld4: movnti %r10,3*8(%rdi) .Ls5: movq 4*8(%rsi),%r11 .Ls6: movq 5*8(%rsi),%r8 .Ls7: movq 6*8(%rsi),%r9 .Ls8: movq 7*8(%rsi),%r10 .Ld5: movnti %r11,4*8(%rdi) .Ld6: movnti %r8,5*8(%rdi) .Ld7: movnti %r9,6*8(%rdi) .Ld8: movnti %r10,7*8(%rdi) dec %rdx leaq 64(%rsi),%rsi leaq 64(%rdi),%rdi jns .Lloop .p2align 4 .Lhandle_tail: movl %ecx,%edx andl $63,%ecx shrl $3,%ecx jz .Lhandle_7 movl $8,%ebx .p2align 4 .Lloop_8: .Ls9: movq (%rsi),%r8 .Ld9: movnti %r8,(%rdi) decl %ecx leaq 8(%rdi),%rdi leaq 8(%rsi),%rsi jnz .Lloop_8 .Lhandle_7: movl %edx,%ecx andl $7,%ecx jz .Lende .p2align 4 .Lloop_1: .Ls10: movb (%rsi),%bl .Ld10: movb %bl,(%rdi) incq %rdi incq %rsi decl %ecx jnz .Lloop_1 CFI_REMEMBER_STATE .Lende: popq %rcx CFI_ADJUST_CFA_OFFSET -8 CFI_RESTORE %rcx popq %rbx CFI_ADJUST_CFA_OFFSET -8 CFI_RESTORE rbx sfence ret CFI_RESTORE_STATE #ifdef FIX_ALIGNMENT /* align destination */ .p2align 4 .Lbad_alignment: movl $8,%r9d subl %ecx,%r9d movl %r9d,%ecx cmpq %r9,%rdx jz .Lhandle_7 js .Lhandle_7 .Lalign_1: .Ls11: movb (%rsi),%bl .Ld11: movb %bl,(%rdi) incq %rsi incq %rdi decl %ecx jnz .Lalign_1 subq %r9,%rdx jmp .Lafter_bad_alignment #endif /* table sorted by exception address */ .section __ex_table,"a" .align 8 .quad .Ls1,.Ls1e /* .Ls[1-4] - 0 bytes copied */ .quad .Ls2,.Ls1e .quad .Ls3,.Ls1e .quad .Ls4,.Ls1e .quad .Ld1,.Ls1e /* .Ld[1-4] - 0..24 bytes coped */ .quad .Ld2,.Ls2e .quad .Ld3,.Ls3e .quad .Ld4,.Ls4e .quad .Ls5,.Ls5e /* .Ls[5-8] - 32 bytes copied */ .quad .Ls6,.Ls5e .quad .Ls7,.Ls5e .quad .Ls8,.Ls5e .quad .Ld5,.Ls5e /* .Ld[5-8] - 32..56 bytes copied */ .quad .Ld6,.Ls6e .quad .Ld7,.Ls7e .quad .Ld8,.Ls8e .quad .Ls9,.Le_quad .quad .Ld9,.Le_quad .quad .Ls10,.Le_byte .quad .Ld10,.Le_byte #ifdef FIX_ALIGNMENT .quad .Ls11,.Lzero_rest .quad .Ld11,.Lzero_rest #endif .quad .Le5,.Le_zero .previous /* eax: zero, ebx: 64 */ .Ls1e: addl $8,%eax /* eax: bytes left uncopied: Ls1e: 64 .. Ls8e: 8 */ .Ls2e: addl $8,%eax .Ls3e: addl $8,%eax .Ls4e: addl $8,%eax .Ls5e: addl $8,%eax .Ls6e: addl $8,%eax .Ls7e: addl $8,%eax .Ls8e: addl $8,%eax addq %rbx,%rdi /* +64 */ subq %rax,%rdi /* correct destination with computed offset */ shlq $6,%rdx /* loop counter * 64 (stride length) */ addq %rax,%rdx /* add offset to loopcnt */ andl $63,%ecx /* remaining bytes */ addq %rcx,%rdx /* add them */ jmp .Lzero_rest /* exception on quad word loop in tail handling */ /* ecx: loopcnt/8, %edx: length, rdi: correct */ .Le_quad: shll $3,%ecx andl $7,%edx addl %ecx,%edx /* edx: bytes to zero, rdi: dest, eax:zero */ .Lzero_rest: cmpl $0,(%rsp) /* zero flag set? */ jz .Le_zero movq %rdx,%rcx .Le_byte: xorl %eax,%eax .Le5: rep stosb /* when there is another exception while zeroing the rest just return */ .Le_zero: movq %rdx,%rax jmp .Lende CFI_ENDPROC ENDPROC(__copy_user_nocache)