514 lines
12 KiB
ArmAsm
514 lines
12 KiB
ArmAsm
/*
|
|
* Copyright (C) 2010 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
/*
|
|
* Contributed by: Intel Corporation
|
|
*/
|
|
|
|
#ifndef L
|
|
# define L(label) .L##label
|
|
#endif
|
|
|
|
#ifndef ALIGN
|
|
# define ALIGN(n) .p2align n
|
|
#endif
|
|
|
|
#ifndef cfi_startproc
|
|
# define cfi_startproc .cfi_startproc
|
|
#endif
|
|
|
|
#ifndef cfi_endproc
|
|
# define cfi_endproc .cfi_endproc
|
|
#endif
|
|
|
|
#ifndef cfi_rel_offset
|
|
# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
|
|
#endif
|
|
|
|
#ifndef cfi_restore
|
|
# define cfi_restore(reg) .cfi_restore reg
|
|
#endif
|
|
|
|
#ifndef cfi_adjust_cfa_offset
|
|
# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
|
|
#endif
|
|
|
|
#ifndef ENTRY
|
|
# define ENTRY(name) \
|
|
.type name, @function; \
|
|
.globl name; \
|
|
.p2align 4; \
|
|
name: \
|
|
cfi_startproc
|
|
#endif
|
|
|
|
#ifndef END
|
|
# define END(name) \
|
|
cfi_endproc; \
|
|
.size name, .-name
|
|
#endif
|
|
|
|
#define CFI_PUSH(REG) \
|
|
cfi_adjust_cfa_offset (4); \
|
|
cfi_rel_offset (REG, 0)
|
|
|
|
#define CFI_POP(REG) \
|
|
cfi_adjust_cfa_offset (-4); \
|
|
cfi_restore (REG)
|
|
|
|
#define PUSH(REG) pushl REG; CFI_PUSH (REG)
|
|
#define POP(REG) popl REG; CFI_POP (REG)
|
|
|
|
#ifdef USE_AS_BZERO32
|
|
# define DEST PARMS
|
|
# define LEN DEST+4
|
|
#else
|
|
# define DEST PARMS
|
|
# define DWDS DEST+4
|
|
# define LEN DWDS+4
|
|
#endif
|
|
|
|
#ifdef USE_AS_WMEMSET32
|
|
# define SETRTNVAL movl DEST(%esp), %eax
|
|
#else
|
|
# define SETRTNVAL
|
|
#endif
|
|
|
|
#ifdef SHARED
|
|
# define ENTRANCE PUSH (%ebx);
|
|
# define RETURN_END POP (%ebx); ret
|
|
# define RETURN RETURN_END; CFI_PUSH (%ebx)
|
|
# define PARMS 8 /* Preserve EBX. */
|
|
# define JMPTBL(I, B) I - B
|
|
|
|
/* Load an entry in a jump table into EBX and branch to it. TABLE is a
|
|
jump table with relative offsets. */
|
|
# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \
|
|
/* We first load PC into EBX. */ \
|
|
call __i686.get_pc_thunk.bx; \
|
|
/* Get the address of the jump table. */ \
|
|
add $(TABLE - .), %ebx; \
|
|
/* Get the entry and convert the relative offset to the \
|
|
absolute address. */ \
|
|
add (%ebx,%ecx,4), %ebx; \
|
|
/* We loaded the jump table and adjuested EDX. Go. */ \
|
|
jmp *%ebx
|
|
|
|
.section .gnu.linkonce.t.__i686.get_pc_thunk.bx,"ax",@progbits
|
|
.globl __i686.get_pc_thunk.bx
|
|
.hidden __i686.get_pc_thunk.bx
|
|
ALIGN (4)
|
|
.type __i686.get_pc_thunk.bx,@function
|
|
__i686.get_pc_thunk.bx:
|
|
movl (%esp), %ebx
|
|
ret
|
|
#else
|
|
# define ENTRANCE
|
|
# define RETURN_END ret
|
|
# define RETURN RETURN_END
|
|
# define PARMS 4
|
|
# define JMPTBL(I, B) I
|
|
|
|
/* Branch to an entry in a jump table. TABLE is a jump table with
|
|
absolute offsets. */
|
|
# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \
|
|
jmp *TABLE(,%ecx,4)
|
|
#endif
|
|
|
|
.section .text.sse2,"ax",@progbits
|
|
ALIGN (4)
|
|
ENTRY (sse2_memset32_atom)
|
|
ENTRANCE
|
|
|
|
movl LEN(%esp), %ecx
|
|
#ifdef USE_AS_ANDROID
|
|
shr $2, %ecx
|
|
#endif
|
|
#ifdef USE_AS_BZERO32
|
|
xor %eax, %eax
|
|
#else
|
|
mov DWDS(%esp), %eax
|
|
mov %eax, %edx
|
|
#endif
|
|
movl DEST(%esp), %edx
|
|
cmp $16, %ecx
|
|
jae L(16dbwordsormore)
|
|
|
|
L(write_less16dbwords):
|
|
lea (%edx, %ecx, 4), %edx
|
|
BRANCH_TO_JMPTBL_ENTRY (L(table_less16dbwords))
|
|
|
|
.pushsection .rodata.sse2,"a",@progbits
|
|
ALIGN (2)
|
|
L(table_less16dbwords):
|
|
.int JMPTBL (L(write_0dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_1dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_2dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_3dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_4dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_5dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_6dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_7dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_8dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_9dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_10dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_11dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_12dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_13dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_14dbwords), L(table_less16dbwords))
|
|
.int JMPTBL (L(write_15dbwords), L(table_less16dbwords))
|
|
.popsection
|
|
|
|
ALIGN (4)
|
|
L(write_15dbwords):
|
|
movl %eax, -60(%edx)
|
|
L(write_14dbwords):
|
|
movl %eax, -56(%edx)
|
|
L(write_13dbwords):
|
|
movl %eax, -52(%edx)
|
|
L(write_12dbwords):
|
|
movl %eax, -48(%edx)
|
|
L(write_11dbwords):
|
|
movl %eax, -44(%edx)
|
|
L(write_10dbwords):
|
|
movl %eax, -40(%edx)
|
|
L(write_9dbwords):
|
|
movl %eax, -36(%edx)
|
|
L(write_8dbwords):
|
|
movl %eax, -32(%edx)
|
|
L(write_7dbwords):
|
|
movl %eax, -28(%edx)
|
|
L(write_6dbwords):
|
|
movl %eax, -24(%edx)
|
|
L(write_5dbwords):
|
|
movl %eax, -20(%edx)
|
|
L(write_4dbwords):
|
|
movl %eax, -16(%edx)
|
|
L(write_3dbwords):
|
|
movl %eax, -12(%edx)
|
|
L(write_2dbwords):
|
|
movl %eax, -8(%edx)
|
|
L(write_1dbwords):
|
|
movl %eax, -4(%edx)
|
|
L(write_0dbwords):
|
|
SETRTNVAL
|
|
RETURN
|
|
|
|
ALIGN (4)
|
|
L(16dbwordsormore):
|
|
test $3, %edx
|
|
jz L(aligned4bytes)
|
|
mov %eax, (%edx)
|
|
mov %eax, -4(%edx, %ecx, 4)
|
|
sub $1, %ecx
|
|
rol $24, %eax
|
|
add $1, %edx
|
|
test $3, %edx
|
|
jz L(aligned4bytes)
|
|
ror $8, %eax
|
|
add $1, %edx
|
|
test $3, %edx
|
|
jz L(aligned4bytes)
|
|
ror $8, %eax
|
|
add $1, %edx
|
|
L(aligned4bytes):
|
|
shl $2, %ecx
|
|
|
|
#ifdef USE_AS_BZERO32
|
|
pxor %xmm0, %xmm0
|
|
#else
|
|
movd %eax, %xmm0
|
|
pshufd $0, %xmm0, %xmm0
|
|
#endif
|
|
testl $0xf, %edx
|
|
jz L(aligned_16)
|
|
/* ECX > 32 and EDX is not 16 byte aligned. */
|
|
L(not_aligned_16):
|
|
movdqu %xmm0, (%edx)
|
|
movl %edx, %eax
|
|
and $-16, %edx
|
|
add $16, %edx
|
|
sub %edx, %eax
|
|
add %eax, %ecx
|
|
movd %xmm0, %eax
|
|
ALIGN (4)
|
|
L(aligned_16):
|
|
cmp $128, %ecx
|
|
jae L(128bytesormore)
|
|
|
|
L(aligned_16_less128bytes):
|
|
add %ecx, %edx
|
|
shr $2, %ecx
|
|
BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
|
|
|
|
ALIGN (4)
|
|
L(128bytesormore):
|
|
#ifdef SHARED_CACHE_SIZE
|
|
PUSH (%ebx)
|
|
mov $SHARED_CACHE_SIZE, %ebx
|
|
#else
|
|
# ifdef SHARED
|
|
call __i686.get_pc_thunk.bx
|
|
add $_GLOBAL_OFFSET_TABLE_, %ebx
|
|
mov __x86_shared_cache_size@GOTOFF(%ebx), %ebx
|
|
# else
|
|
PUSH (%ebx)
|
|
mov __x86_shared_cache_size, %ebx
|
|
# endif
|
|
#endif
|
|
cmp %ebx, %ecx
|
|
jae L(128bytesormore_nt_start)
|
|
|
|
#ifdef DATA_CACHE_SIZE
|
|
POP (%ebx)
|
|
# define RESTORE_EBX_STATE CFI_PUSH (%ebx)
|
|
cmp $DATA_CACHE_SIZE, %ecx
|
|
#else
|
|
# ifdef SHARED
|
|
# define RESTORE_EBX_STATE
|
|
call __i686.get_pc_thunk.bx
|
|
add $_GLOBAL_OFFSET_TABLE_, %ebx
|
|
cmp __x86_data_cache_size@GOTOFF(%ebx), %ecx
|
|
# else
|
|
POP (%ebx)
|
|
# define RESTORE_EBX_STATE CFI_PUSH (%ebx)
|
|
cmp __x86_data_cache_size, %ecx
|
|
# endif
|
|
#endif
|
|
|
|
jae L(128bytes_L2_normal)
|
|
subl $128, %ecx
|
|
L(128bytesormore_normal):
|
|
sub $128, %ecx
|
|
movdqa %xmm0, (%edx)
|
|
movdqa %xmm0, 0x10(%edx)
|
|
movdqa %xmm0, 0x20(%edx)
|
|
movdqa %xmm0, 0x30(%edx)
|
|
movdqa %xmm0, 0x40(%edx)
|
|
movdqa %xmm0, 0x50(%edx)
|
|
movdqa %xmm0, 0x60(%edx)
|
|
movdqa %xmm0, 0x70(%edx)
|
|
lea 128(%edx), %edx
|
|
jb L(128bytesless_normal)
|
|
|
|
|
|
sub $128, %ecx
|
|
movdqa %xmm0, (%edx)
|
|
movdqa %xmm0, 0x10(%edx)
|
|
movdqa %xmm0, 0x20(%edx)
|
|
movdqa %xmm0, 0x30(%edx)
|
|
movdqa %xmm0, 0x40(%edx)
|
|
movdqa %xmm0, 0x50(%edx)
|
|
movdqa %xmm0, 0x60(%edx)
|
|
movdqa %xmm0, 0x70(%edx)
|
|
lea 128(%edx), %edx
|
|
jae L(128bytesormore_normal)
|
|
|
|
L(128bytesless_normal):
|
|
lea 128(%ecx), %ecx
|
|
add %ecx, %edx
|
|
shr $2, %ecx
|
|
BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
|
|
|
|
ALIGN (4)
|
|
L(128bytes_L2_normal):
|
|
prefetcht0 0x380(%edx)
|
|
prefetcht0 0x3c0(%edx)
|
|
sub $128, %ecx
|
|
movdqa %xmm0, (%edx)
|
|
movaps %xmm0, 0x10(%edx)
|
|
movaps %xmm0, 0x20(%edx)
|
|
movaps %xmm0, 0x30(%edx)
|
|
movaps %xmm0, 0x40(%edx)
|
|
movaps %xmm0, 0x50(%edx)
|
|
movaps %xmm0, 0x60(%edx)
|
|
movaps %xmm0, 0x70(%edx)
|
|
add $128, %edx
|
|
cmp $128, %ecx
|
|
jae L(128bytes_L2_normal)
|
|
|
|
L(128bytesless_L2_normal):
|
|
add %ecx, %edx
|
|
shr $2, %ecx
|
|
BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
|
|
|
|
RESTORE_EBX_STATE
|
|
L(128bytesormore_nt_start):
|
|
sub %ebx, %ecx
|
|
mov %ebx, %eax
|
|
and $0x7f, %eax
|
|
add %eax, %ecx
|
|
movd %xmm0, %eax
|
|
ALIGN (4)
|
|
L(128bytesormore_shared_cache_loop):
|
|
prefetcht0 0x3c0(%edx)
|
|
prefetcht0 0x380(%edx)
|
|
sub $0x80, %ebx
|
|
movdqa %xmm0, (%edx)
|
|
movdqa %xmm0, 0x10(%edx)
|
|
movdqa %xmm0, 0x20(%edx)
|
|
movdqa %xmm0, 0x30(%edx)
|
|
movdqa %xmm0, 0x40(%edx)
|
|
movdqa %xmm0, 0x50(%edx)
|
|
movdqa %xmm0, 0x60(%edx)
|
|
movdqa %xmm0, 0x70(%edx)
|
|
add $0x80, %edx
|
|
cmp $0x80, %ebx
|
|
jae L(128bytesormore_shared_cache_loop)
|
|
cmp $0x80, %ecx
|
|
jb L(shared_cache_loop_end)
|
|
|
|
ALIGN (4)
|
|
L(128bytesormore_nt):
|
|
sub $0x80, %ecx
|
|
movntdq %xmm0, (%edx)
|
|
movntdq %xmm0, 0x10(%edx)
|
|
movntdq %xmm0, 0x20(%edx)
|
|
movntdq %xmm0, 0x30(%edx)
|
|
movntdq %xmm0, 0x40(%edx)
|
|
movntdq %xmm0, 0x50(%edx)
|
|
movntdq %xmm0, 0x60(%edx)
|
|
movntdq %xmm0, 0x70(%edx)
|
|
add $0x80, %edx
|
|
cmp $0x80, %ecx
|
|
jae L(128bytesormore_nt)
|
|
sfence
|
|
L(shared_cache_loop_end):
|
|
#if defined DATA_CACHE_SIZE || !defined SHARED
|
|
POP (%ebx)
|
|
#endif
|
|
add %ecx, %edx
|
|
shr $2, %ecx
|
|
BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
|
|
|
|
.pushsection .rodata.sse2,"a",@progbits
|
|
ALIGN (2)
|
|
L(table_16_128bytes):
|
|
.int JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes))
|
|
.int JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes))
|
|
.popsection
|
|
|
|
ALIGN (4)
|
|
L(aligned_16_112bytes):
|
|
movdqa %xmm0, -112(%edx)
|
|
L(aligned_16_96bytes):
|
|
movdqa %xmm0, -96(%edx)
|
|
L(aligned_16_80bytes):
|
|
movdqa %xmm0, -80(%edx)
|
|
L(aligned_16_64bytes):
|
|
movdqa %xmm0, -64(%edx)
|
|
L(aligned_16_48bytes):
|
|
movdqa %xmm0, -48(%edx)
|
|
L(aligned_16_32bytes):
|
|
movdqa %xmm0, -32(%edx)
|
|
L(aligned_16_16bytes):
|
|
movdqa %xmm0, -16(%edx)
|
|
L(aligned_16_0bytes):
|
|
SETRTNVAL
|
|
RETURN
|
|
|
|
ALIGN (4)
|
|
L(aligned_16_116bytes):
|
|
movdqa %xmm0, -116(%edx)
|
|
L(aligned_16_100bytes):
|
|
movdqa %xmm0, -100(%edx)
|
|
L(aligned_16_84bytes):
|
|
movdqa %xmm0, -84(%edx)
|
|
L(aligned_16_68bytes):
|
|
movdqa %xmm0, -68(%edx)
|
|
L(aligned_16_52bytes):
|
|
movdqa %xmm0, -52(%edx)
|
|
L(aligned_16_36bytes):
|
|
movdqa %xmm0, -36(%edx)
|
|
L(aligned_16_20bytes):
|
|
movdqa %xmm0, -20(%edx)
|
|
L(aligned_16_4bytes):
|
|
movl %eax, -4(%edx)
|
|
SETRTNVAL
|
|
RETURN
|
|
|
|
ALIGN (4)
|
|
L(aligned_16_120bytes):
|
|
movdqa %xmm0, -120(%edx)
|
|
L(aligned_16_104bytes):
|
|
movdqa %xmm0, -104(%edx)
|
|
L(aligned_16_88bytes):
|
|
movdqa %xmm0, -88(%edx)
|
|
L(aligned_16_72bytes):
|
|
movdqa %xmm0, -72(%edx)
|
|
L(aligned_16_56bytes):
|
|
movdqa %xmm0, -56(%edx)
|
|
L(aligned_16_40bytes):
|
|
movdqa %xmm0, -40(%edx)
|
|
L(aligned_16_24bytes):
|
|
movdqa %xmm0, -24(%edx)
|
|
L(aligned_16_8bytes):
|
|
movq %xmm0, -8(%edx)
|
|
SETRTNVAL
|
|
RETURN
|
|
|
|
ALIGN (4)
|
|
L(aligned_16_124bytes):
|
|
movdqa %xmm0, -124(%edx)
|
|
L(aligned_16_108bytes):
|
|
movdqa %xmm0, -108(%edx)
|
|
L(aligned_16_92bytes):
|
|
movdqa %xmm0, -92(%edx)
|
|
L(aligned_16_76bytes):
|
|
movdqa %xmm0, -76(%edx)
|
|
L(aligned_16_60bytes):
|
|
movdqa %xmm0, -60(%edx)
|
|
L(aligned_16_44bytes):
|
|
movdqa %xmm0, -44(%edx)
|
|
L(aligned_16_28bytes):
|
|
movdqa %xmm0, -28(%edx)
|
|
L(aligned_16_12bytes):
|
|
movq %xmm0, -12(%edx)
|
|
movl %eax, -4(%edx)
|
|
SETRTNVAL
|
|
RETURN
|
|
|
|
END (sse2_memset32_atom)
|