From 458d1253f5eab0d00bb5c862194ce0663471257f Mon Sep 17 00:00:00 2001 From: Varvara Rainchik Date: Mon, 8 Sep 2014 16:27:01 +0400 Subject: [PATCH] Clean up memset[16,32] routines: unify files names, remove redundant wrapper for 32-bit, remove Atom caches sizes for 64-bit, fix license. Change-Id: Ieda6367d7b21cf25b2beda6dd8d77cf668d3f2af Signed-off-by: Varvara Rainchik --- libcutils/Android.mk | 4 +- libcutils/arch-x86/android_memset16.S | 710 ++++++++++++++++- libcutils/arch-x86/android_memset32.S | 501 +++++++++++- .../arch-x86/{cache_wrapper.S => cache.h} | 3 - libcutils/arch-x86/sse2-memset16-atom.S | 722 ------------------ libcutils/arch-x86/sse2-memset32-atom.S | 513 ------------- ...emset16_SSE2-atom.S => android_memset16.S} | 11 +- ...emset32_SSE2-atom.S => android_memset32.S} | 11 +- libcutils/arch-x86_64/cache.h | 9 - 9 files changed, 1209 insertions(+), 1275 deletions(-) mode change 100644 => 100755 libcutils/arch-x86/android_memset16.S mode change 100644 => 100755 libcutils/arch-x86/android_memset32.S rename libcutils/arch-x86/{cache_wrapper.S => cache.h} (96%) delete mode 100755 libcutils/arch-x86/sse2-memset16-atom.S delete mode 100755 libcutils/arch-x86/sse2-memset32-atom.S rename libcutils/arch-x86_64/{android_memset16_SSE2-atom.S => android_memset16.S} (99%) rename libcutils/arch-x86_64/{android_memset32_SSE2-atom.S => android_memset32.S} (98%) diff --git a/libcutils/Android.mk b/libcutils/Android.mk index 5a4be9309..469cdcfd9 100644 --- a/libcutils/Android.mk +++ b/libcutils/Android.mk @@ -140,8 +140,8 @@ LOCAL_SRC_FILES_x86 += \ arch-x86/android_memset32.S \ LOCAL_SRC_FILES_x86_64 += \ - arch-x86_64/android_memset16_SSE2-atom.S \ - arch-x86_64/android_memset32_SSE2-atom.S \ + arch-x86_64/android_memset16.S \ + arch-x86_64/android_memset32.S \ LOCAL_CFLAGS_arm += -DHAVE_MEMSET16 -DHAVE_MEMSET32 LOCAL_CFLAGS_arm64 += -DHAVE_MEMSET16 -DHAVE_MEMSET32 diff --git a/libcutils/arch-x86/android_memset16.S b/libcutils/arch-x86/android_memset16.S old mode 100644 new mode 100755 index f8b79bd23..cb2ff1412 --- a/libcutils/arch-x86/android_memset16.S +++ b/libcutils/arch-x86/android_memset16.S @@ -13,13 +13,707 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Contributed by: Intel Corporation - */ -# include "cache_wrapper.S" -# undef __i686 -# define USE_AS_ANDROID -# define sse2_memset16_atom android_memset16 -# include "sse2-memset16-atom.S" +#include "cache.h" +#ifndef MEMSET +# define MEMSET android_memset16 +#endif + +#ifndef L +# define L(label) .L##label +#endif + +#ifndef ALIGN +# define ALIGN(n) .p2align n +#endif + +#ifndef cfi_startproc +# define cfi_startproc .cfi_startproc +#endif + +#ifndef cfi_endproc +# define cfi_endproc .cfi_endproc +#endif + +#ifndef cfi_rel_offset +# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off +#endif + +#ifndef cfi_restore +# define cfi_restore(reg) .cfi_restore reg +#endif + +#ifndef cfi_adjust_cfa_offset +# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off +#endif + +#ifndef ENTRY +# define ENTRY(name) \ + .type name, @function; \ + .globl name; \ + .p2align 4; \ +name: \ + cfi_startproc +#endif + +#ifndef END +# define END(name) \ + cfi_endproc; \ + .size name, .-name +#endif + +#define CFI_PUSH(REG) \ + cfi_adjust_cfa_offset (4); \ + cfi_rel_offset (REG, 0) + +#define CFI_POP(REG) \ + cfi_adjust_cfa_offset (-4); \ + cfi_restore (REG) + +#define PUSH(REG) pushl REG; CFI_PUSH (REG) +#define POP(REG) popl REG; CFI_POP (REG) + +#ifdef USE_AS_BZERO16 +# define DEST PARMS +# define LEN DEST+4 +# define SETRTNVAL +#else +# define DEST PARMS +# define CHR DEST+4 +# define LEN CHR+4 +# define SETRTNVAL movl DEST(%esp), %eax +#endif + +#if (defined SHARED || defined __PIC__) +# define ENTRANCE PUSH (%ebx); +# define RETURN_END POP (%ebx); ret +# define RETURN RETURN_END; CFI_PUSH (%ebx) +# define PARMS 8 /* Preserve EBX. */ +# define JMPTBL(I, B) I - B + +/* Load an entry in a jump table into EBX and branch to it. TABLE is a + jump table with relative offsets. */ +# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \ + /* We first load PC into EBX. */ \ + call __x86.get_pc_thunk.bx; \ + /* Get the address of the jump table. */ \ + add $(TABLE - .), %ebx; \ + /* Get the entry and convert the relative offset to the \ + absolute address. */ \ + add (%ebx,%ecx,4), %ebx; \ + /* We loaded the jump table and adjuested EDX. Go. */ \ + jmp *%ebx + + .section .gnu.linkonce.t.__x86.get_pc_thunk.bx,"ax",@progbits + .globl __x86.get_pc_thunk.bx + .hidden __x86.get_pc_thunk.bx + ALIGN (4) + .type __x86.get_pc_thunk.bx,@function +__x86.get_pc_thunk.bx: + movl (%esp), %ebx + ret +#else +# define ENTRANCE +# define RETURN_END ret +# define RETURN RETURN_END +# define PARMS 4 +# define JMPTBL(I, B) I + +/* Branch to an entry in a jump table. TABLE is a jump table with + absolute offsets. */ +# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \ + jmp *TABLE(,%ecx,4) +#endif + + .section .text.sse2,"ax",@progbits + ALIGN (4) +ENTRY (MEMSET) + ENTRANCE + + movl LEN(%esp), %ecx + shr $1, %ecx +#ifdef USE_AS_BZERO16 + xor %eax, %eax +#else + movzwl CHR(%esp), %eax + mov %eax, %edx + shl $16, %eax + or %edx, %eax +#endif + movl DEST(%esp), %edx + cmp $32, %ecx + jae L(32wordsormore) + +L(write_less32words): + lea (%edx, %ecx, 2), %edx + BRANCH_TO_JMPTBL_ENTRY (L(table_less32words)) + + + .pushsection .rodata.sse2,"a",@progbits + ALIGN (2) +L(table_less32words): + .int JMPTBL (L(write_0words), L(table_less32words)) + .int JMPTBL (L(write_1words), L(table_less32words)) + .int JMPTBL (L(write_2words), L(table_less32words)) + .int JMPTBL (L(write_3words), L(table_less32words)) + .int JMPTBL (L(write_4words), L(table_less32words)) + .int JMPTBL (L(write_5words), L(table_less32words)) + .int JMPTBL (L(write_6words), L(table_less32words)) + .int JMPTBL (L(write_7words), L(table_less32words)) + .int JMPTBL (L(write_8words), L(table_less32words)) + .int JMPTBL (L(write_9words), L(table_less32words)) + .int JMPTBL (L(write_10words), L(table_less32words)) + .int JMPTBL (L(write_11words), L(table_less32words)) + .int JMPTBL (L(write_12words), L(table_less32words)) + .int JMPTBL (L(write_13words), L(table_less32words)) + .int JMPTBL (L(write_14words), L(table_less32words)) + .int JMPTBL (L(write_15words), L(table_less32words)) + .int JMPTBL (L(write_16words), L(table_less32words)) + .int JMPTBL (L(write_17words), L(table_less32words)) + .int JMPTBL (L(write_18words), L(table_less32words)) + .int JMPTBL (L(write_19words), L(table_less32words)) + .int JMPTBL (L(write_20words), L(table_less32words)) + .int JMPTBL (L(write_21words), L(table_less32words)) + .int JMPTBL (L(write_22words), L(table_less32words)) + .int JMPTBL (L(write_23words), L(table_less32words)) + .int JMPTBL (L(write_24words), L(table_less32words)) + .int JMPTBL (L(write_25words), L(table_less32words)) + .int JMPTBL (L(write_26words), L(table_less32words)) + .int JMPTBL (L(write_27words), L(table_less32words)) + .int JMPTBL (L(write_28words), L(table_less32words)) + .int JMPTBL (L(write_29words), L(table_less32words)) + .int JMPTBL (L(write_30words), L(table_less32words)) + .int JMPTBL (L(write_31words), L(table_less32words)) + .popsection + + ALIGN (4) +L(write_28words): + movl %eax, -56(%edx) + movl %eax, -52(%edx) +L(write_24words): + movl %eax, -48(%edx) + movl %eax, -44(%edx) +L(write_20words): + movl %eax, -40(%edx) + movl %eax, -36(%edx) +L(write_16words): + movl %eax, -32(%edx) + movl %eax, -28(%edx) +L(write_12words): + movl %eax, -24(%edx) + movl %eax, -20(%edx) +L(write_8words): + movl %eax, -16(%edx) + movl %eax, -12(%edx) +L(write_4words): + movl %eax, -8(%edx) + movl %eax, -4(%edx) +L(write_0words): + SETRTNVAL + RETURN + + ALIGN (4) +L(write_29words): + movl %eax, -58(%edx) + movl %eax, -54(%edx) +L(write_25words): + movl %eax, -50(%edx) + movl %eax, -46(%edx) +L(write_21words): + movl %eax, -42(%edx) + movl %eax, -38(%edx) +L(write_17words): + movl %eax, -34(%edx) + movl %eax, -30(%edx) +L(write_13words): + movl %eax, -26(%edx) + movl %eax, -22(%edx) +L(write_9words): + movl %eax, -18(%edx) + movl %eax, -14(%edx) +L(write_5words): + movl %eax, -10(%edx) + movl %eax, -6(%edx) +L(write_1words): + mov %ax, -2(%edx) + SETRTNVAL + RETURN + + ALIGN (4) +L(write_30words): + movl %eax, -60(%edx) + movl %eax, -56(%edx) +L(write_26words): + movl %eax, -52(%edx) + movl %eax, -48(%edx) +L(write_22words): + movl %eax, -44(%edx) + movl %eax, -40(%edx) +L(write_18words): + movl %eax, -36(%edx) + movl %eax, -32(%edx) +L(write_14words): + movl %eax, -28(%edx) + movl %eax, -24(%edx) +L(write_10words): + movl %eax, -20(%edx) + movl %eax, -16(%edx) +L(write_6words): + movl %eax, -12(%edx) + movl %eax, -8(%edx) +L(write_2words): + movl %eax, -4(%edx) + SETRTNVAL + RETURN + + ALIGN (4) +L(write_31words): + movl %eax, -62(%edx) + movl %eax, -58(%edx) +L(write_27words): + movl %eax, -54(%edx) + movl %eax, -50(%edx) +L(write_23words): + movl %eax, -46(%edx) + movl %eax, -42(%edx) +L(write_19words): + movl %eax, -38(%edx) + movl %eax, -34(%edx) +L(write_15words): + movl %eax, -30(%edx) + movl %eax, -26(%edx) +L(write_11words): + movl %eax, -22(%edx) + movl %eax, -18(%edx) +L(write_7words): + movl %eax, -14(%edx) + movl %eax, -10(%edx) +L(write_3words): + movl %eax, -6(%edx) + movw %ax, -2(%edx) + SETRTNVAL + RETURN + + ALIGN (4) + +L(32wordsormore): + shl $1, %ecx + test $0x01, %edx + jz L(aligned2bytes) + mov %eax, (%edx) + mov %eax, -4(%edx, %ecx) + sub $2, %ecx + add $1, %edx + rol $8, %eax +L(aligned2bytes): +#ifdef USE_AS_BZERO16 + pxor %xmm0, %xmm0 +#else + movd %eax, %xmm0 + pshufd $0, %xmm0, %xmm0 +#endif + testl $0xf, %edx + jz L(aligned_16) +/* ECX > 32 and EDX is not 16 byte aligned. */ +L(not_aligned_16): + movdqu %xmm0, (%edx) + movl %edx, %eax + and $-16, %edx + add $16, %edx + sub %edx, %eax + add %eax, %ecx + movd %xmm0, %eax + + ALIGN (4) +L(aligned_16): + cmp $128, %ecx + jae L(128bytesormore) + +L(aligned_16_less128bytes): + add %ecx, %edx + shr $1, %ecx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) + + ALIGN (4) +L(128bytesormore): +#ifdef SHARED_CACHE_SIZE + PUSH (%ebx) + mov $SHARED_CACHE_SIZE, %ebx +#else +# if (defined SHARED || defined __PIC__) + call __x86.get_pc_thunk.bx + add $_GLOBAL_OFFSET_TABLE_, %ebx + mov __x86_shared_cache_size@GOTOFF(%ebx), %ebx +# else + PUSH (%ebx) + mov __x86_shared_cache_size, %ebx +# endif +#endif + cmp %ebx, %ecx + jae L(128bytesormore_nt_start) + + +#ifdef DATA_CACHE_SIZE + POP (%ebx) +# define RESTORE_EBX_STATE CFI_PUSH (%ebx) + cmp $DATA_CACHE_SIZE, %ecx +#else +# if (defined SHARED || defined __PIC__) +# define RESTORE_EBX_STATE + call __x86.get_pc_thunk.bx + add $_GLOBAL_OFFSET_TABLE_, %ebx + cmp __x86_data_cache_size@GOTOFF(%ebx), %ecx +# else + POP (%ebx) +# define RESTORE_EBX_STATE CFI_PUSH (%ebx) + cmp __x86_data_cache_size, %ecx +# endif +#endif + + jae L(128bytes_L2_normal) + subl $128, %ecx +L(128bytesormore_normal): + sub $128, %ecx + movdqa %xmm0, (%edx) + movdqa %xmm0, 0x10(%edx) + movdqa %xmm0, 0x20(%edx) + movdqa %xmm0, 0x30(%edx) + movdqa %xmm0, 0x40(%edx) + movdqa %xmm0, 0x50(%edx) + movdqa %xmm0, 0x60(%edx) + movdqa %xmm0, 0x70(%edx) + lea 128(%edx), %edx + jb L(128bytesless_normal) + + + sub $128, %ecx + movdqa %xmm0, (%edx) + movdqa %xmm0, 0x10(%edx) + movdqa %xmm0, 0x20(%edx) + movdqa %xmm0, 0x30(%edx) + movdqa %xmm0, 0x40(%edx) + movdqa %xmm0, 0x50(%edx) + movdqa %xmm0, 0x60(%edx) + movdqa %xmm0, 0x70(%edx) + lea 128(%edx), %edx + jae L(128bytesormore_normal) + +L(128bytesless_normal): + lea 128(%ecx), %ecx + add %ecx, %edx + shr $1, %ecx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) + + ALIGN (4) +L(128bytes_L2_normal): + prefetcht0 0x380(%edx) + prefetcht0 0x3c0(%edx) + sub $128, %ecx + movdqa %xmm0, (%edx) + movaps %xmm0, 0x10(%edx) + movaps %xmm0, 0x20(%edx) + movaps %xmm0, 0x30(%edx) + movaps %xmm0, 0x40(%edx) + movaps %xmm0, 0x50(%edx) + movaps %xmm0, 0x60(%edx) + movaps %xmm0, 0x70(%edx) + add $128, %edx + cmp $128, %ecx + jae L(128bytes_L2_normal) + +L(128bytesless_L2_normal): + add %ecx, %edx + shr $1, %ecx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) + + RESTORE_EBX_STATE +L(128bytesormore_nt_start): + sub %ebx, %ecx + mov %ebx, %eax + and $0x7f, %eax + add %eax, %ecx + movd %xmm0, %eax + ALIGN (4) +L(128bytesormore_shared_cache_loop): + prefetcht0 0x3c0(%edx) + prefetcht0 0x380(%edx) + sub $0x80, %ebx + movdqa %xmm0, (%edx) + movdqa %xmm0, 0x10(%edx) + movdqa %xmm0, 0x20(%edx) + movdqa %xmm0, 0x30(%edx) + movdqa %xmm0, 0x40(%edx) + movdqa %xmm0, 0x50(%edx) + movdqa %xmm0, 0x60(%edx) + movdqa %xmm0, 0x70(%edx) + add $0x80, %edx + cmp $0x80, %ebx + jae L(128bytesormore_shared_cache_loop) + cmp $0x80, %ecx + jb L(shared_cache_loop_end) + ALIGN (4) +L(128bytesormore_nt): + sub $0x80, %ecx + movntdq %xmm0, (%edx) + movntdq %xmm0, 0x10(%edx) + movntdq %xmm0, 0x20(%edx) + movntdq %xmm0, 0x30(%edx) + movntdq %xmm0, 0x40(%edx) + movntdq %xmm0, 0x50(%edx) + movntdq %xmm0, 0x60(%edx) + movntdq %xmm0, 0x70(%edx) + add $0x80, %edx + cmp $0x80, %ecx + jae L(128bytesormore_nt) + sfence +L(shared_cache_loop_end): +#if defined DATA_CACHE_SIZE || !(defined SHARED || defined __PIC__) + POP (%ebx) +#endif + add %ecx, %edx + shr $1, %ecx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) + + + .pushsection .rodata.sse2,"a",@progbits + ALIGN (2) +L(table_16_128bytes): + .int JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_2bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_6bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_10bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_14bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_18bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_22bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_26bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_30bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_34bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_38bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_42bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_46bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_50bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_54bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_58bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_62bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_66bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_70bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_74bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_78bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_82bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_86bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_90bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_94bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_98bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_102bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_106bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_110bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_114bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_118bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_122bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_126bytes), L(table_16_128bytes)) + .popsection + + + ALIGN (4) +L(aligned_16_112bytes): + movdqa %xmm0, -112(%edx) +L(aligned_16_96bytes): + movdqa %xmm0, -96(%edx) +L(aligned_16_80bytes): + movdqa %xmm0, -80(%edx) +L(aligned_16_64bytes): + movdqa %xmm0, -64(%edx) +L(aligned_16_48bytes): + movdqa %xmm0, -48(%edx) +L(aligned_16_32bytes): + movdqa %xmm0, -32(%edx) +L(aligned_16_16bytes): + movdqa %xmm0, -16(%edx) +L(aligned_16_0bytes): + SETRTNVAL + RETURN + + + ALIGN (4) +L(aligned_16_114bytes): + movdqa %xmm0, -114(%edx) +L(aligned_16_98bytes): + movdqa %xmm0, -98(%edx) +L(aligned_16_82bytes): + movdqa %xmm0, -82(%edx) +L(aligned_16_66bytes): + movdqa %xmm0, -66(%edx) +L(aligned_16_50bytes): + movdqa %xmm0, -50(%edx) +L(aligned_16_34bytes): + movdqa %xmm0, -34(%edx) +L(aligned_16_18bytes): + movdqa %xmm0, -18(%edx) +L(aligned_16_2bytes): + movw %ax, -2(%edx) + SETRTNVAL + RETURN + + ALIGN (4) +L(aligned_16_116bytes): + movdqa %xmm0, -116(%edx) +L(aligned_16_100bytes): + movdqa %xmm0, -100(%edx) +L(aligned_16_84bytes): + movdqa %xmm0, -84(%edx) +L(aligned_16_68bytes): + movdqa %xmm0, -68(%edx) +L(aligned_16_52bytes): + movdqa %xmm0, -52(%edx) +L(aligned_16_36bytes): + movdqa %xmm0, -36(%edx) +L(aligned_16_20bytes): + movdqa %xmm0, -20(%edx) +L(aligned_16_4bytes): + movl %eax, -4(%edx) + SETRTNVAL + RETURN + + + ALIGN (4) +L(aligned_16_118bytes): + movdqa %xmm0, -118(%edx) +L(aligned_16_102bytes): + movdqa %xmm0, -102(%edx) +L(aligned_16_86bytes): + movdqa %xmm0, -86(%edx) +L(aligned_16_70bytes): + movdqa %xmm0, -70(%edx) +L(aligned_16_54bytes): + movdqa %xmm0, -54(%edx) +L(aligned_16_38bytes): + movdqa %xmm0, -38(%edx) +L(aligned_16_22bytes): + movdqa %xmm0, -22(%edx) +L(aligned_16_6bytes): + movl %eax, -6(%edx) + movw %ax, -2(%edx) + SETRTNVAL + RETURN + + + ALIGN (4) +L(aligned_16_120bytes): + movdqa %xmm0, -120(%edx) +L(aligned_16_104bytes): + movdqa %xmm0, -104(%edx) +L(aligned_16_88bytes): + movdqa %xmm0, -88(%edx) +L(aligned_16_72bytes): + movdqa %xmm0, -72(%edx) +L(aligned_16_56bytes): + movdqa %xmm0, -56(%edx) +L(aligned_16_40bytes): + movdqa %xmm0, -40(%edx) +L(aligned_16_24bytes): + movdqa %xmm0, -24(%edx) +L(aligned_16_8bytes): + movq %xmm0, -8(%edx) + SETRTNVAL + RETURN + + + ALIGN (4) +L(aligned_16_122bytes): + movdqa %xmm0, -122(%edx) +L(aligned_16_106bytes): + movdqa %xmm0, -106(%edx) +L(aligned_16_90bytes): + movdqa %xmm0, -90(%edx) +L(aligned_16_74bytes): + movdqa %xmm0, -74(%edx) +L(aligned_16_58bytes): + movdqa %xmm0, -58(%edx) +L(aligned_16_42bytes): + movdqa %xmm0, -42(%edx) +L(aligned_16_26bytes): + movdqa %xmm0, -26(%edx) +L(aligned_16_10bytes): + movq %xmm0, -10(%edx) + movw %ax, -2(%edx) + SETRTNVAL + RETURN + + + ALIGN (4) +L(aligned_16_124bytes): + movdqa %xmm0, -124(%edx) +L(aligned_16_108bytes): + movdqa %xmm0, -108(%edx) +L(aligned_16_92bytes): + movdqa %xmm0, -92(%edx) +L(aligned_16_76bytes): + movdqa %xmm0, -76(%edx) +L(aligned_16_60bytes): + movdqa %xmm0, -60(%edx) +L(aligned_16_44bytes): + movdqa %xmm0, -44(%edx) +L(aligned_16_28bytes): + movdqa %xmm0, -28(%edx) +L(aligned_16_12bytes): + movq %xmm0, -12(%edx) + movl %eax, -4(%edx) + SETRTNVAL + RETURN + + + ALIGN (4) +L(aligned_16_126bytes): + movdqa %xmm0, -126(%edx) +L(aligned_16_110bytes): + movdqa %xmm0, -110(%edx) +L(aligned_16_94bytes): + movdqa %xmm0, -94(%edx) +L(aligned_16_78bytes): + movdqa %xmm0, -78(%edx) +L(aligned_16_62bytes): + movdqa %xmm0, -62(%edx) +L(aligned_16_46bytes): + movdqa %xmm0, -46(%edx) +L(aligned_16_30bytes): + movdqa %xmm0, -30(%edx) +L(aligned_16_14bytes): + movq %xmm0, -14(%edx) + movl %eax, -6(%edx) + movw %ax, -2(%edx) + SETRTNVAL + RETURN + +END (MEMSET) diff --git a/libcutils/arch-x86/android_memset32.S b/libcutils/arch-x86/android_memset32.S old mode 100644 new mode 100755 index 6249fcece..f4326dc90 --- a/libcutils/arch-x86/android_memset32.S +++ b/libcutils/arch-x86/android_memset32.S @@ -13,13 +13,498 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Contributed by: Intel Corporation - */ -# include "cache_wrapper.S" -# undef __i686 -# define USE_AS_ANDROID -# define sse2_memset32_atom android_memset32 -# include "sse2-memset32-atom.S" +#include "cache.h" +#ifndef MEMSET +# define MEMSET android_memset32 +#endif + +#ifndef L +# define L(label) .L##label +#endif + +#ifndef ALIGN +# define ALIGN(n) .p2align n +#endif + +#ifndef cfi_startproc +# define cfi_startproc .cfi_startproc +#endif + +#ifndef cfi_endproc +# define cfi_endproc .cfi_endproc +#endif + +#ifndef cfi_rel_offset +# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off +#endif + +#ifndef cfi_restore +# define cfi_restore(reg) .cfi_restore reg +#endif + +#ifndef cfi_adjust_cfa_offset +# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off +#endif + +#ifndef ENTRY +# define ENTRY(name) \ + .type name, @function; \ + .globl name; \ + .p2align 4; \ +name: \ + cfi_startproc +#endif + +#ifndef END +# define END(name) \ + cfi_endproc; \ + .size name, .-name +#endif + +#define CFI_PUSH(REG) \ + cfi_adjust_cfa_offset (4); \ + cfi_rel_offset (REG, 0) + +#define CFI_POP(REG) \ + cfi_adjust_cfa_offset (-4); \ + cfi_restore (REG) + +#define PUSH(REG) pushl REG; CFI_PUSH (REG) +#define POP(REG) popl REG; CFI_POP (REG) + +#ifdef USE_AS_BZERO32 +# define DEST PARMS +# define LEN DEST+4 +# define SETRTNVAL +#else +# define DEST PARMS +# define DWDS DEST+4 +# define LEN DWDS+4 +# define SETRTNVAL movl DEST(%esp), %eax +#endif + +#if (defined SHARED || defined __PIC__) +# define ENTRANCE PUSH (%ebx); +# define RETURN_END POP (%ebx); ret +# define RETURN RETURN_END; CFI_PUSH (%ebx) +# define PARMS 8 /* Preserve EBX. */ +# define JMPTBL(I, B) I - B + +/* Load an entry in a jump table into EBX and branch to it. TABLE is a + jump table with relative offsets. */ +# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \ + /* We first load PC into EBX. */ \ + call __x86.get_pc_thunk.bx; \ + /* Get the address of the jump table. */ \ + add $(TABLE - .), %ebx; \ + /* Get the entry and convert the relative offset to the \ + absolute address. */ \ + add (%ebx,%ecx,4), %ebx; \ + /* We loaded the jump table and adjuested EDX. Go. */ \ + jmp *%ebx + + .section .gnu.linkonce.t.__x86.get_pc_thunk.bx,"ax",@progbits + .globl __x86.get_pc_thunk.bx + .hidden __x86.get_pc_thunk.bx + ALIGN (4) + .type __x86.get_pc_thunk.bx,@function +__x86.get_pc_thunk.bx: + movl (%esp), %ebx + ret +#else +# define ENTRANCE +# define RETURN_END ret +# define RETURN RETURN_END +# define PARMS 4 +# define JMPTBL(I, B) I + +/* Branch to an entry in a jump table. TABLE is a jump table with + absolute offsets. */ +# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \ + jmp *TABLE(,%ecx,4) +#endif + + .section .text.sse2,"ax",@progbits + ALIGN (4) +ENTRY (MEMSET) + ENTRANCE + + movl LEN(%esp), %ecx + shr $2, %ecx +#ifdef USE_AS_BZERO32 + xor %eax, %eax +#else + mov DWDS(%esp), %eax + mov %eax, %edx +#endif + movl DEST(%esp), %edx + cmp $16, %ecx + jae L(16dbwordsormore) + +L(write_less16dbwords): + lea (%edx, %ecx, 4), %edx + BRANCH_TO_JMPTBL_ENTRY (L(table_less16dbwords)) + + .pushsection .rodata.sse2,"a",@progbits + ALIGN (2) +L(table_less16dbwords): + .int JMPTBL (L(write_0dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_1dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_2dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_3dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_4dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_5dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_6dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_7dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_8dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_9dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_10dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_11dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_12dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_13dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_14dbwords), L(table_less16dbwords)) + .int JMPTBL (L(write_15dbwords), L(table_less16dbwords)) + .popsection + + ALIGN (4) +L(write_15dbwords): + movl %eax, -60(%edx) +L(write_14dbwords): + movl %eax, -56(%edx) +L(write_13dbwords): + movl %eax, -52(%edx) +L(write_12dbwords): + movl %eax, -48(%edx) +L(write_11dbwords): + movl %eax, -44(%edx) +L(write_10dbwords): + movl %eax, -40(%edx) +L(write_9dbwords): + movl %eax, -36(%edx) +L(write_8dbwords): + movl %eax, -32(%edx) +L(write_7dbwords): + movl %eax, -28(%edx) +L(write_6dbwords): + movl %eax, -24(%edx) +L(write_5dbwords): + movl %eax, -20(%edx) +L(write_4dbwords): + movl %eax, -16(%edx) +L(write_3dbwords): + movl %eax, -12(%edx) +L(write_2dbwords): + movl %eax, -8(%edx) +L(write_1dbwords): + movl %eax, -4(%edx) +L(write_0dbwords): + SETRTNVAL + RETURN + + ALIGN (4) +L(16dbwordsormore): + test $3, %edx + jz L(aligned4bytes) + mov %eax, (%edx) + mov %eax, -4(%edx, %ecx, 4) + sub $1, %ecx + rol $24, %eax + add $1, %edx + test $3, %edx + jz L(aligned4bytes) + ror $8, %eax + add $1, %edx + test $3, %edx + jz L(aligned4bytes) + ror $8, %eax + add $1, %edx +L(aligned4bytes): + shl $2, %ecx + +#ifdef USE_AS_BZERO32 + pxor %xmm0, %xmm0 +#else + movd %eax, %xmm0 + pshufd $0, %xmm0, %xmm0 +#endif + testl $0xf, %edx + jz L(aligned_16) +/* ECX > 32 and EDX is not 16 byte aligned. */ +L(not_aligned_16): + movdqu %xmm0, (%edx) + movl %edx, %eax + and $-16, %edx + add $16, %edx + sub %edx, %eax + add %eax, %ecx + movd %xmm0, %eax + ALIGN (4) +L(aligned_16): + cmp $128, %ecx + jae L(128bytesormore) + +L(aligned_16_less128bytes): + add %ecx, %edx + shr $2, %ecx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) + + ALIGN (4) +L(128bytesormore): +#ifdef SHARED_CACHE_SIZE + PUSH (%ebx) + mov $SHARED_CACHE_SIZE, %ebx +#else +# if (defined SHARED || defined __PIC__) + call __x86.get_pc_thunk.bx + add $_GLOBAL_OFFSET_TABLE_, %ebx + mov __x86_shared_cache_size@GOTOFF(%ebx), %ebx +# else + PUSH (%ebx) + mov __x86_shared_cache_size, %ebx +# endif +#endif + cmp %ebx, %ecx + jae L(128bytesormore_nt_start) + +#ifdef DATA_CACHE_SIZE + POP (%ebx) +# define RESTORE_EBX_STATE CFI_PUSH (%ebx) + cmp $DATA_CACHE_SIZE, %ecx +#else +# if (defined SHARED || defined __PIC__) +# define RESTORE_EBX_STATE + call __x86.get_pc_thunk.bx + add $_GLOBAL_OFFSET_TABLE_, %ebx + cmp __x86_data_cache_size@GOTOFF(%ebx), %ecx +# else + POP (%ebx) +# define RESTORE_EBX_STATE CFI_PUSH (%ebx) + cmp __x86_data_cache_size, %ecx +# endif +#endif + + jae L(128bytes_L2_normal) + subl $128, %ecx +L(128bytesormore_normal): + sub $128, %ecx + movdqa %xmm0, (%edx) + movdqa %xmm0, 0x10(%edx) + movdqa %xmm0, 0x20(%edx) + movdqa %xmm0, 0x30(%edx) + movdqa %xmm0, 0x40(%edx) + movdqa %xmm0, 0x50(%edx) + movdqa %xmm0, 0x60(%edx) + movdqa %xmm0, 0x70(%edx) + lea 128(%edx), %edx + jb L(128bytesless_normal) + + + sub $128, %ecx + movdqa %xmm0, (%edx) + movdqa %xmm0, 0x10(%edx) + movdqa %xmm0, 0x20(%edx) + movdqa %xmm0, 0x30(%edx) + movdqa %xmm0, 0x40(%edx) + movdqa %xmm0, 0x50(%edx) + movdqa %xmm0, 0x60(%edx) + movdqa %xmm0, 0x70(%edx) + lea 128(%edx), %edx + jae L(128bytesormore_normal) + +L(128bytesless_normal): + lea 128(%ecx), %ecx + add %ecx, %edx + shr $2, %ecx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) + + ALIGN (4) +L(128bytes_L2_normal): + prefetcht0 0x380(%edx) + prefetcht0 0x3c0(%edx) + sub $128, %ecx + movdqa %xmm0, (%edx) + movaps %xmm0, 0x10(%edx) + movaps %xmm0, 0x20(%edx) + movaps %xmm0, 0x30(%edx) + movaps %xmm0, 0x40(%edx) + movaps %xmm0, 0x50(%edx) + movaps %xmm0, 0x60(%edx) + movaps %xmm0, 0x70(%edx) + add $128, %edx + cmp $128, %ecx + jae L(128bytes_L2_normal) + +L(128bytesless_L2_normal): + add %ecx, %edx + shr $2, %ecx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) + + RESTORE_EBX_STATE +L(128bytesormore_nt_start): + sub %ebx, %ecx + mov %ebx, %eax + and $0x7f, %eax + add %eax, %ecx + movd %xmm0, %eax + ALIGN (4) +L(128bytesormore_shared_cache_loop): + prefetcht0 0x3c0(%edx) + prefetcht0 0x380(%edx) + sub $0x80, %ebx + movdqa %xmm0, (%edx) + movdqa %xmm0, 0x10(%edx) + movdqa %xmm0, 0x20(%edx) + movdqa %xmm0, 0x30(%edx) + movdqa %xmm0, 0x40(%edx) + movdqa %xmm0, 0x50(%edx) + movdqa %xmm0, 0x60(%edx) + movdqa %xmm0, 0x70(%edx) + add $0x80, %edx + cmp $0x80, %ebx + jae L(128bytesormore_shared_cache_loop) + cmp $0x80, %ecx + jb L(shared_cache_loop_end) + + ALIGN (4) +L(128bytesormore_nt): + sub $0x80, %ecx + movntdq %xmm0, (%edx) + movntdq %xmm0, 0x10(%edx) + movntdq %xmm0, 0x20(%edx) + movntdq %xmm0, 0x30(%edx) + movntdq %xmm0, 0x40(%edx) + movntdq %xmm0, 0x50(%edx) + movntdq %xmm0, 0x60(%edx) + movntdq %xmm0, 0x70(%edx) + add $0x80, %edx + cmp $0x80, %ecx + jae L(128bytesormore_nt) + sfence +L(shared_cache_loop_end): +#if defined DATA_CACHE_SIZE || !(defined SHARED || defined __PIC__) + POP (%ebx) +#endif + add %ecx, %edx + shr $2, %ecx + BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) + + .pushsection .rodata.sse2,"a",@progbits + ALIGN (2) +L(table_16_128bytes): + .int JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes)) + .int JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes)) + .popsection + + ALIGN (4) +L(aligned_16_112bytes): + movdqa %xmm0, -112(%edx) +L(aligned_16_96bytes): + movdqa %xmm0, -96(%edx) +L(aligned_16_80bytes): + movdqa %xmm0, -80(%edx) +L(aligned_16_64bytes): + movdqa %xmm0, -64(%edx) +L(aligned_16_48bytes): + movdqa %xmm0, -48(%edx) +L(aligned_16_32bytes): + movdqa %xmm0, -32(%edx) +L(aligned_16_16bytes): + movdqa %xmm0, -16(%edx) +L(aligned_16_0bytes): + SETRTNVAL + RETURN + + ALIGN (4) +L(aligned_16_116bytes): + movdqa %xmm0, -116(%edx) +L(aligned_16_100bytes): + movdqa %xmm0, -100(%edx) +L(aligned_16_84bytes): + movdqa %xmm0, -84(%edx) +L(aligned_16_68bytes): + movdqa %xmm0, -68(%edx) +L(aligned_16_52bytes): + movdqa %xmm0, -52(%edx) +L(aligned_16_36bytes): + movdqa %xmm0, -36(%edx) +L(aligned_16_20bytes): + movdqa %xmm0, -20(%edx) +L(aligned_16_4bytes): + movl %eax, -4(%edx) + SETRTNVAL + RETURN + + ALIGN (4) +L(aligned_16_120bytes): + movdqa %xmm0, -120(%edx) +L(aligned_16_104bytes): + movdqa %xmm0, -104(%edx) +L(aligned_16_88bytes): + movdqa %xmm0, -88(%edx) +L(aligned_16_72bytes): + movdqa %xmm0, -72(%edx) +L(aligned_16_56bytes): + movdqa %xmm0, -56(%edx) +L(aligned_16_40bytes): + movdqa %xmm0, -40(%edx) +L(aligned_16_24bytes): + movdqa %xmm0, -24(%edx) +L(aligned_16_8bytes): + movq %xmm0, -8(%edx) + SETRTNVAL + RETURN + + ALIGN (4) +L(aligned_16_124bytes): + movdqa %xmm0, -124(%edx) +L(aligned_16_108bytes): + movdqa %xmm0, -108(%edx) +L(aligned_16_92bytes): + movdqa %xmm0, -92(%edx) +L(aligned_16_76bytes): + movdqa %xmm0, -76(%edx) +L(aligned_16_60bytes): + movdqa %xmm0, -60(%edx) +L(aligned_16_44bytes): + movdqa %xmm0, -44(%edx) +L(aligned_16_28bytes): + movdqa %xmm0, -28(%edx) +L(aligned_16_12bytes): + movq %xmm0, -12(%edx) + movl %eax, -4(%edx) + SETRTNVAL + RETURN + +END (MEMSET) diff --git a/libcutils/arch-x86/cache_wrapper.S b/libcutils/arch-x86/cache.h similarity index 96% rename from libcutils/arch-x86/cache_wrapper.S rename to libcutils/arch-x86/cache.h index 9eee25c69..1c22feaa7 100644 --- a/libcutils/arch-x86/cache_wrapper.S +++ b/libcutils/arch-x86/cache.h @@ -13,9 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Contributed by: Intel Corporation - */ #if defined(__slm__) /* Values are optimized for Silvermont */ diff --git a/libcutils/arch-x86/sse2-memset16-atom.S b/libcutils/arch-x86/sse2-memset16-atom.S deleted file mode 100755 index c2a762ba6..000000000 --- a/libcutils/arch-x86/sse2-memset16-atom.S +++ /dev/null @@ -1,722 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - * Contributed by: Intel Corporation - */ - -#ifndef L -# define L(label) .L##label -#endif - -#ifndef ALIGN -# define ALIGN(n) .p2align n -#endif - -#ifndef cfi_startproc -# define cfi_startproc .cfi_startproc -#endif - -#ifndef cfi_endproc -# define cfi_endproc .cfi_endproc -#endif - -#ifndef cfi_rel_offset -# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off -#endif - -#ifndef cfi_restore -# define cfi_restore(reg) .cfi_restore reg -#endif - -#ifndef cfi_adjust_cfa_offset -# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off -#endif - -#ifndef ENTRY -# define ENTRY(name) \ - .type name, @function; \ - .globl name; \ - .p2align 4; \ -name: \ - cfi_startproc -#endif - -#ifndef END -# define END(name) \ - cfi_endproc; \ - .size name, .-name -#endif - -#define CFI_PUSH(REG) \ - cfi_adjust_cfa_offset (4); \ - cfi_rel_offset (REG, 0) - -#define CFI_POP(REG) \ - cfi_adjust_cfa_offset (-4); \ - cfi_restore (REG) - -#define PUSH(REG) pushl REG; CFI_PUSH (REG) -#define POP(REG) popl REG; CFI_POP (REG) - -#ifdef USE_AS_BZERO16 -# define DEST PARMS -# define LEN DEST+4 -#else -# define DEST PARMS -# define CHR DEST+4 -# define LEN CHR+4 -#endif - -#if 1 -# define SETRTNVAL -#else -# define SETRTNVAL movl DEST(%esp), %eax -#endif - -#if (defined SHARED || defined __PIC__) -# define ENTRANCE PUSH (%ebx); -# define RETURN_END POP (%ebx); ret -# define RETURN RETURN_END; CFI_PUSH (%ebx) -# define PARMS 8 /* Preserve EBX. */ -# define JMPTBL(I, B) I - B - -/* Load an entry in a jump table into EBX and branch to it. TABLE is a - jump table with relative offsets. */ -# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \ - /* We first load PC into EBX. */ \ - call __i686.get_pc_thunk.bx; \ - /* Get the address of the jump table. */ \ - add $(TABLE - .), %ebx; \ - /* Get the entry and convert the relative offset to the \ - absolute address. */ \ - add (%ebx,%ecx,4), %ebx; \ - /* We loaded the jump table and adjuested EDX. Go. */ \ - jmp *%ebx - - .section .gnu.linkonce.t.__i686.get_pc_thunk.bx,"ax",@progbits - .globl __i686.get_pc_thunk.bx - .hidden __i686.get_pc_thunk.bx - ALIGN (4) - .type __i686.get_pc_thunk.bx,@function -__i686.get_pc_thunk.bx: - movl (%esp), %ebx - ret -#else -# define ENTRANCE -# define RETURN_END ret -# define RETURN RETURN_END -# define PARMS 4 -# define JMPTBL(I, B) I - -/* Branch to an entry in a jump table. TABLE is a jump table with - absolute offsets. */ -# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \ - jmp *TABLE(,%ecx,4) -#endif - - .section .text.sse2,"ax",@progbits - ALIGN (4) -ENTRY (sse2_memset16_atom) - ENTRANCE - - movl LEN(%esp), %ecx -#ifdef USE_AS_ANDROID - shr $1, %ecx -#endif -#ifdef USE_AS_BZERO16 - xor %eax, %eax -#else - movzwl CHR(%esp), %eax - mov %eax, %edx - shl $16, %eax - or %edx, %eax -#endif - movl DEST(%esp), %edx - cmp $32, %ecx - jae L(32wordsormore) - -L(write_less32words): - lea (%edx, %ecx, 2), %edx - BRANCH_TO_JMPTBL_ENTRY (L(table_less32words)) - - - .pushsection .rodata.sse2,"a",@progbits - ALIGN (2) -L(table_less32words): - .int JMPTBL (L(write_0words), L(table_less32words)) - .int JMPTBL (L(write_1words), L(table_less32words)) - .int JMPTBL (L(write_2words), L(table_less32words)) - .int JMPTBL (L(write_3words), L(table_less32words)) - .int JMPTBL (L(write_4words), L(table_less32words)) - .int JMPTBL (L(write_5words), L(table_less32words)) - .int JMPTBL (L(write_6words), L(table_less32words)) - .int JMPTBL (L(write_7words), L(table_less32words)) - .int JMPTBL (L(write_8words), L(table_less32words)) - .int JMPTBL (L(write_9words), L(table_less32words)) - .int JMPTBL (L(write_10words), L(table_less32words)) - .int JMPTBL (L(write_11words), L(table_less32words)) - .int JMPTBL (L(write_12words), L(table_less32words)) - .int JMPTBL (L(write_13words), L(table_less32words)) - .int JMPTBL (L(write_14words), L(table_less32words)) - .int JMPTBL (L(write_15words), L(table_less32words)) - .int JMPTBL (L(write_16words), L(table_less32words)) - .int JMPTBL (L(write_17words), L(table_less32words)) - .int JMPTBL (L(write_18words), L(table_less32words)) - .int JMPTBL (L(write_19words), L(table_less32words)) - .int JMPTBL (L(write_20words), L(table_less32words)) - .int JMPTBL (L(write_21words), L(table_less32words)) - .int JMPTBL (L(write_22words), L(table_less32words)) - .int JMPTBL (L(write_23words), L(table_less32words)) - .int JMPTBL (L(write_24words), L(table_less32words)) - .int JMPTBL (L(write_25words), L(table_less32words)) - .int JMPTBL (L(write_26words), L(table_less32words)) - .int JMPTBL (L(write_27words), L(table_less32words)) - .int JMPTBL (L(write_28words), L(table_less32words)) - .int JMPTBL (L(write_29words), L(table_less32words)) - .int JMPTBL (L(write_30words), L(table_less32words)) - .int JMPTBL (L(write_31words), L(table_less32words)) - .popsection - - ALIGN (4) -L(write_28words): - movl %eax, -56(%edx) - movl %eax, -52(%edx) -L(write_24words): - movl %eax, -48(%edx) - movl %eax, -44(%edx) -L(write_20words): - movl %eax, -40(%edx) - movl %eax, -36(%edx) -L(write_16words): - movl %eax, -32(%edx) - movl %eax, -28(%edx) -L(write_12words): - movl %eax, -24(%edx) - movl %eax, -20(%edx) -L(write_8words): - movl %eax, -16(%edx) - movl %eax, -12(%edx) -L(write_4words): - movl %eax, -8(%edx) - movl %eax, -4(%edx) -L(write_0words): - SETRTNVAL - RETURN - - ALIGN (4) -L(write_29words): - movl %eax, -58(%edx) - movl %eax, -54(%edx) -L(write_25words): - movl %eax, -50(%edx) - movl %eax, -46(%edx) -L(write_21words): - movl %eax, -42(%edx) - movl %eax, -38(%edx) -L(write_17words): - movl %eax, -34(%edx) - movl %eax, -30(%edx) -L(write_13words): - movl %eax, -26(%edx) - movl %eax, -22(%edx) -L(write_9words): - movl %eax, -18(%edx) - movl %eax, -14(%edx) -L(write_5words): - movl %eax, -10(%edx) - movl %eax, -6(%edx) -L(write_1words): - mov %ax, -2(%edx) - SETRTNVAL - RETURN - - ALIGN (4) -L(write_30words): - movl %eax, -60(%edx) - movl %eax, -56(%edx) -L(write_26words): - movl %eax, -52(%edx) - movl %eax, -48(%edx) -L(write_22words): - movl %eax, -44(%edx) - movl %eax, -40(%edx) -L(write_18words): - movl %eax, -36(%edx) - movl %eax, -32(%edx) -L(write_14words): - movl %eax, -28(%edx) - movl %eax, -24(%edx) -L(write_10words): - movl %eax, -20(%edx) - movl %eax, -16(%edx) -L(write_6words): - movl %eax, -12(%edx) - movl %eax, -8(%edx) -L(write_2words): - movl %eax, -4(%edx) - SETRTNVAL - RETURN - - ALIGN (4) -L(write_31words): - movl %eax, -62(%edx) - movl %eax, -58(%edx) -L(write_27words): - movl %eax, -54(%edx) - movl %eax, -50(%edx) -L(write_23words): - movl %eax, -46(%edx) - movl %eax, -42(%edx) -L(write_19words): - movl %eax, -38(%edx) - movl %eax, -34(%edx) -L(write_15words): - movl %eax, -30(%edx) - movl %eax, -26(%edx) -L(write_11words): - movl %eax, -22(%edx) - movl %eax, -18(%edx) -L(write_7words): - movl %eax, -14(%edx) - movl %eax, -10(%edx) -L(write_3words): - movl %eax, -6(%edx) - movw %ax, -2(%edx) - SETRTNVAL - RETURN - - ALIGN (4) - -L(32wordsormore): - shl $1, %ecx - test $0x01, %edx - jz L(aligned2bytes) - mov %eax, (%edx) - mov %eax, -4(%edx, %ecx) - sub $2, %ecx - add $1, %edx - rol $8, %eax -L(aligned2bytes): -#ifdef USE_AS_BZERO16 - pxor %xmm0, %xmm0 -#else - movd %eax, %xmm0 - pshufd $0, %xmm0, %xmm0 -#endif - testl $0xf, %edx - jz L(aligned_16) -/* ECX > 32 and EDX is not 16 byte aligned. */ -L(not_aligned_16): - movdqu %xmm0, (%edx) - movl %edx, %eax - and $-16, %edx - add $16, %edx - sub %edx, %eax - add %eax, %ecx - movd %xmm0, %eax - - ALIGN (4) -L(aligned_16): - cmp $128, %ecx - jae L(128bytesormore) - -L(aligned_16_less128bytes): - add %ecx, %edx - shr $1, %ecx - BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) - - ALIGN (4) -L(128bytesormore): -#ifdef SHARED_CACHE_SIZE - PUSH (%ebx) - mov $SHARED_CACHE_SIZE, %ebx -#else -# if (defined SHARED || defined __PIC__) - call __i686.get_pc_thunk.bx - add $_GLOBAL_OFFSET_TABLE_, %ebx - mov __x86_shared_cache_size@GOTOFF(%ebx), %ebx -# else - PUSH (%ebx) - mov __x86_shared_cache_size, %ebx -# endif -#endif - cmp %ebx, %ecx - jae L(128bytesormore_nt_start) - - -#ifdef DATA_CACHE_SIZE - POP (%ebx) -# define RESTORE_EBX_STATE CFI_PUSH (%ebx) - cmp $DATA_CACHE_SIZE, %ecx -#else -# if (defined SHARED || defined __PIC__) -# define RESTORE_EBX_STATE - call __i686.get_pc_thunk.bx - add $_GLOBAL_OFFSET_TABLE_, %ebx - cmp __x86_data_cache_size@GOTOFF(%ebx), %ecx -# else - POP (%ebx) -# define RESTORE_EBX_STATE CFI_PUSH (%ebx) - cmp __x86_data_cache_size, %ecx -# endif -#endif - - jae L(128bytes_L2_normal) - subl $128, %ecx -L(128bytesormore_normal): - sub $128, %ecx - movdqa %xmm0, (%edx) - movdqa %xmm0, 0x10(%edx) - movdqa %xmm0, 0x20(%edx) - movdqa %xmm0, 0x30(%edx) - movdqa %xmm0, 0x40(%edx) - movdqa %xmm0, 0x50(%edx) - movdqa %xmm0, 0x60(%edx) - movdqa %xmm0, 0x70(%edx) - lea 128(%edx), %edx - jb L(128bytesless_normal) - - - sub $128, %ecx - movdqa %xmm0, (%edx) - movdqa %xmm0, 0x10(%edx) - movdqa %xmm0, 0x20(%edx) - movdqa %xmm0, 0x30(%edx) - movdqa %xmm0, 0x40(%edx) - movdqa %xmm0, 0x50(%edx) - movdqa %xmm0, 0x60(%edx) - movdqa %xmm0, 0x70(%edx) - lea 128(%edx), %edx - jae L(128bytesormore_normal) - -L(128bytesless_normal): - lea 128(%ecx), %ecx - add %ecx, %edx - shr $1, %ecx - BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) - - ALIGN (4) -L(128bytes_L2_normal): - prefetcht0 0x380(%edx) - prefetcht0 0x3c0(%edx) - sub $128, %ecx - movdqa %xmm0, (%edx) - movaps %xmm0, 0x10(%edx) - movaps %xmm0, 0x20(%edx) - movaps %xmm0, 0x30(%edx) - movaps %xmm0, 0x40(%edx) - movaps %xmm0, 0x50(%edx) - movaps %xmm0, 0x60(%edx) - movaps %xmm0, 0x70(%edx) - add $128, %edx - cmp $128, %ecx - jae L(128bytes_L2_normal) - -L(128bytesless_L2_normal): - add %ecx, %edx - shr $1, %ecx - BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) - - RESTORE_EBX_STATE -L(128bytesormore_nt_start): - sub %ebx, %ecx - mov %ebx, %eax - and $0x7f, %eax - add %eax, %ecx - movd %xmm0, %eax - ALIGN (4) -L(128bytesormore_shared_cache_loop): - prefetcht0 0x3c0(%edx) - prefetcht0 0x380(%edx) - sub $0x80, %ebx - movdqa %xmm0, (%edx) - movdqa %xmm0, 0x10(%edx) - movdqa %xmm0, 0x20(%edx) - movdqa %xmm0, 0x30(%edx) - movdqa %xmm0, 0x40(%edx) - movdqa %xmm0, 0x50(%edx) - movdqa %xmm0, 0x60(%edx) - movdqa %xmm0, 0x70(%edx) - add $0x80, %edx - cmp $0x80, %ebx - jae L(128bytesormore_shared_cache_loop) - cmp $0x80, %ecx - jb L(shared_cache_loop_end) - ALIGN (4) -L(128bytesormore_nt): - sub $0x80, %ecx - movntdq %xmm0, (%edx) - movntdq %xmm0, 0x10(%edx) - movntdq %xmm0, 0x20(%edx) - movntdq %xmm0, 0x30(%edx) - movntdq %xmm0, 0x40(%edx) - movntdq %xmm0, 0x50(%edx) - movntdq %xmm0, 0x60(%edx) - movntdq %xmm0, 0x70(%edx) - add $0x80, %edx - cmp $0x80, %ecx - jae L(128bytesormore_nt) - sfence -L(shared_cache_loop_end): -#if defined DATA_CACHE_SIZE || !(defined SHARED || defined __PIC__) - POP (%ebx) -#endif - add %ecx, %edx - shr $1, %ecx - BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) - - - .pushsection .rodata.sse2,"a",@progbits - ALIGN (2) -L(table_16_128bytes): - .int JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_2bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_6bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_10bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_14bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_18bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_22bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_26bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_30bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_34bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_38bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_42bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_46bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_50bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_54bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_58bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_62bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_66bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_70bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_74bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_78bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_82bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_86bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_90bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_94bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_98bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_102bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_106bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_110bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_114bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_118bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_122bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_126bytes), L(table_16_128bytes)) - .popsection - - - ALIGN (4) -L(aligned_16_112bytes): - movdqa %xmm0, -112(%edx) -L(aligned_16_96bytes): - movdqa %xmm0, -96(%edx) -L(aligned_16_80bytes): - movdqa %xmm0, -80(%edx) -L(aligned_16_64bytes): - movdqa %xmm0, -64(%edx) -L(aligned_16_48bytes): - movdqa %xmm0, -48(%edx) -L(aligned_16_32bytes): - movdqa %xmm0, -32(%edx) -L(aligned_16_16bytes): - movdqa %xmm0, -16(%edx) -L(aligned_16_0bytes): - SETRTNVAL - RETURN - - - ALIGN (4) -L(aligned_16_114bytes): - movdqa %xmm0, -114(%edx) -L(aligned_16_98bytes): - movdqa %xmm0, -98(%edx) -L(aligned_16_82bytes): - movdqa %xmm0, -82(%edx) -L(aligned_16_66bytes): - movdqa %xmm0, -66(%edx) -L(aligned_16_50bytes): - movdqa %xmm0, -50(%edx) -L(aligned_16_34bytes): - movdqa %xmm0, -34(%edx) -L(aligned_16_18bytes): - movdqa %xmm0, -18(%edx) -L(aligned_16_2bytes): - movw %ax, -2(%edx) - SETRTNVAL - RETURN - - ALIGN (4) -L(aligned_16_116bytes): - movdqa %xmm0, -116(%edx) -L(aligned_16_100bytes): - movdqa %xmm0, -100(%edx) -L(aligned_16_84bytes): - movdqa %xmm0, -84(%edx) -L(aligned_16_68bytes): - movdqa %xmm0, -68(%edx) -L(aligned_16_52bytes): - movdqa %xmm0, -52(%edx) -L(aligned_16_36bytes): - movdqa %xmm0, -36(%edx) -L(aligned_16_20bytes): - movdqa %xmm0, -20(%edx) -L(aligned_16_4bytes): - movl %eax, -4(%edx) - SETRTNVAL - RETURN - - - ALIGN (4) -L(aligned_16_118bytes): - movdqa %xmm0, -118(%edx) -L(aligned_16_102bytes): - movdqa %xmm0, -102(%edx) -L(aligned_16_86bytes): - movdqa %xmm0, -86(%edx) -L(aligned_16_70bytes): - movdqa %xmm0, -70(%edx) -L(aligned_16_54bytes): - movdqa %xmm0, -54(%edx) -L(aligned_16_38bytes): - movdqa %xmm0, -38(%edx) -L(aligned_16_22bytes): - movdqa %xmm0, -22(%edx) -L(aligned_16_6bytes): - movl %eax, -6(%edx) - movw %ax, -2(%edx) - SETRTNVAL - RETURN - - - ALIGN (4) -L(aligned_16_120bytes): - movdqa %xmm0, -120(%edx) -L(aligned_16_104bytes): - movdqa %xmm0, -104(%edx) -L(aligned_16_88bytes): - movdqa %xmm0, -88(%edx) -L(aligned_16_72bytes): - movdqa %xmm0, -72(%edx) -L(aligned_16_56bytes): - movdqa %xmm0, -56(%edx) -L(aligned_16_40bytes): - movdqa %xmm0, -40(%edx) -L(aligned_16_24bytes): - movdqa %xmm0, -24(%edx) -L(aligned_16_8bytes): - movq %xmm0, -8(%edx) - SETRTNVAL - RETURN - - - ALIGN (4) -L(aligned_16_122bytes): - movdqa %xmm0, -122(%edx) -L(aligned_16_106bytes): - movdqa %xmm0, -106(%edx) -L(aligned_16_90bytes): - movdqa %xmm0, -90(%edx) -L(aligned_16_74bytes): - movdqa %xmm0, -74(%edx) -L(aligned_16_58bytes): - movdqa %xmm0, -58(%edx) -L(aligned_16_42bytes): - movdqa %xmm0, -42(%edx) -L(aligned_16_26bytes): - movdqa %xmm0, -26(%edx) -L(aligned_16_10bytes): - movq %xmm0, -10(%edx) - movw %ax, -2(%edx) - SETRTNVAL - RETURN - - - ALIGN (4) -L(aligned_16_124bytes): - movdqa %xmm0, -124(%edx) -L(aligned_16_108bytes): - movdqa %xmm0, -108(%edx) -L(aligned_16_92bytes): - movdqa %xmm0, -92(%edx) -L(aligned_16_76bytes): - movdqa %xmm0, -76(%edx) -L(aligned_16_60bytes): - movdqa %xmm0, -60(%edx) -L(aligned_16_44bytes): - movdqa %xmm0, -44(%edx) -L(aligned_16_28bytes): - movdqa %xmm0, -28(%edx) -L(aligned_16_12bytes): - movq %xmm0, -12(%edx) - movl %eax, -4(%edx) - SETRTNVAL - RETURN - - - ALIGN (4) -L(aligned_16_126bytes): - movdqa %xmm0, -126(%edx) -L(aligned_16_110bytes): - movdqa %xmm0, -110(%edx) -L(aligned_16_94bytes): - movdqa %xmm0, -94(%edx) -L(aligned_16_78bytes): - movdqa %xmm0, -78(%edx) -L(aligned_16_62bytes): - movdqa %xmm0, -62(%edx) -L(aligned_16_46bytes): - movdqa %xmm0, -46(%edx) -L(aligned_16_30bytes): - movdqa %xmm0, -30(%edx) -L(aligned_16_14bytes): - movq %xmm0, -14(%edx) - movl %eax, -6(%edx) - movw %ax, -2(%edx) - SETRTNVAL - RETURN - -END (sse2_memset16_atom) diff --git a/libcutils/arch-x86/sse2-memset32-atom.S b/libcutils/arch-x86/sse2-memset32-atom.S deleted file mode 100755 index 05eb64f64..000000000 --- a/libcutils/arch-x86/sse2-memset32-atom.S +++ /dev/null @@ -1,513 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - * Contributed by: Intel Corporation - */ - -#ifndef L -# define L(label) .L##label -#endif - -#ifndef ALIGN -# define ALIGN(n) .p2align n -#endif - -#ifndef cfi_startproc -# define cfi_startproc .cfi_startproc -#endif - -#ifndef cfi_endproc -# define cfi_endproc .cfi_endproc -#endif - -#ifndef cfi_rel_offset -# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off -#endif - -#ifndef cfi_restore -# define cfi_restore(reg) .cfi_restore reg -#endif - -#ifndef cfi_adjust_cfa_offset -# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off -#endif - -#ifndef ENTRY -# define ENTRY(name) \ - .type name, @function; \ - .globl name; \ - .p2align 4; \ -name: \ - cfi_startproc -#endif - -#ifndef END -# define END(name) \ - cfi_endproc; \ - .size name, .-name -#endif - -#define CFI_PUSH(REG) \ - cfi_adjust_cfa_offset (4); \ - cfi_rel_offset (REG, 0) - -#define CFI_POP(REG) \ - cfi_adjust_cfa_offset (-4); \ - cfi_restore (REG) - -#define PUSH(REG) pushl REG; CFI_PUSH (REG) -#define POP(REG) popl REG; CFI_POP (REG) - -#ifdef USE_AS_BZERO32 -# define DEST PARMS -# define LEN DEST+4 -#else -# define DEST PARMS -# define DWDS DEST+4 -# define LEN DWDS+4 -#endif - -#ifdef USE_AS_WMEMSET32 -# define SETRTNVAL movl DEST(%esp), %eax -#else -# define SETRTNVAL -#endif - -#if (defined SHARED || defined __PIC__) -# define ENTRANCE PUSH (%ebx); -# define RETURN_END POP (%ebx); ret -# define RETURN RETURN_END; CFI_PUSH (%ebx) -# define PARMS 8 /* Preserve EBX. */ -# define JMPTBL(I, B) I - B - -/* Load an entry in a jump table into EBX and branch to it. TABLE is a - jump table with relative offsets. */ -# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \ - /* We first load PC into EBX. */ \ - call __i686.get_pc_thunk.bx; \ - /* Get the address of the jump table. */ \ - add $(TABLE - .), %ebx; \ - /* Get the entry and convert the relative offset to the \ - absolute address. */ \ - add (%ebx,%ecx,4), %ebx; \ - /* We loaded the jump table and adjuested EDX. Go. */ \ - jmp *%ebx - - .section .gnu.linkonce.t.__i686.get_pc_thunk.bx,"ax",@progbits - .globl __i686.get_pc_thunk.bx - .hidden __i686.get_pc_thunk.bx - ALIGN (4) - .type __i686.get_pc_thunk.bx,@function -__i686.get_pc_thunk.bx: - movl (%esp), %ebx - ret -#else -# define ENTRANCE -# define RETURN_END ret -# define RETURN RETURN_END -# define PARMS 4 -# define JMPTBL(I, B) I - -/* Branch to an entry in a jump table. TABLE is a jump table with - absolute offsets. */ -# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \ - jmp *TABLE(,%ecx,4) -#endif - - .section .text.sse2,"ax",@progbits - ALIGN (4) -ENTRY (sse2_memset32_atom) - ENTRANCE - - movl LEN(%esp), %ecx -#ifdef USE_AS_ANDROID - shr $2, %ecx -#endif -#ifdef USE_AS_BZERO32 - xor %eax, %eax -#else - mov DWDS(%esp), %eax - mov %eax, %edx -#endif - movl DEST(%esp), %edx - cmp $16, %ecx - jae L(16dbwordsormore) - -L(write_less16dbwords): - lea (%edx, %ecx, 4), %edx - BRANCH_TO_JMPTBL_ENTRY (L(table_less16dbwords)) - - .pushsection .rodata.sse2,"a",@progbits - ALIGN (2) -L(table_less16dbwords): - .int JMPTBL (L(write_0dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_1dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_2dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_3dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_4dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_5dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_6dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_7dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_8dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_9dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_10dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_11dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_12dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_13dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_14dbwords), L(table_less16dbwords)) - .int JMPTBL (L(write_15dbwords), L(table_less16dbwords)) - .popsection - - ALIGN (4) -L(write_15dbwords): - movl %eax, -60(%edx) -L(write_14dbwords): - movl %eax, -56(%edx) -L(write_13dbwords): - movl %eax, -52(%edx) -L(write_12dbwords): - movl %eax, -48(%edx) -L(write_11dbwords): - movl %eax, -44(%edx) -L(write_10dbwords): - movl %eax, -40(%edx) -L(write_9dbwords): - movl %eax, -36(%edx) -L(write_8dbwords): - movl %eax, -32(%edx) -L(write_7dbwords): - movl %eax, -28(%edx) -L(write_6dbwords): - movl %eax, -24(%edx) -L(write_5dbwords): - movl %eax, -20(%edx) -L(write_4dbwords): - movl %eax, -16(%edx) -L(write_3dbwords): - movl %eax, -12(%edx) -L(write_2dbwords): - movl %eax, -8(%edx) -L(write_1dbwords): - movl %eax, -4(%edx) -L(write_0dbwords): - SETRTNVAL - RETURN - - ALIGN (4) -L(16dbwordsormore): - test $3, %edx - jz L(aligned4bytes) - mov %eax, (%edx) - mov %eax, -4(%edx, %ecx, 4) - sub $1, %ecx - rol $24, %eax - add $1, %edx - test $3, %edx - jz L(aligned4bytes) - ror $8, %eax - add $1, %edx - test $3, %edx - jz L(aligned4bytes) - ror $8, %eax - add $1, %edx -L(aligned4bytes): - shl $2, %ecx - -#ifdef USE_AS_BZERO32 - pxor %xmm0, %xmm0 -#else - movd %eax, %xmm0 - pshufd $0, %xmm0, %xmm0 -#endif - testl $0xf, %edx - jz L(aligned_16) -/* ECX > 32 and EDX is not 16 byte aligned. */ -L(not_aligned_16): - movdqu %xmm0, (%edx) - movl %edx, %eax - and $-16, %edx - add $16, %edx - sub %edx, %eax - add %eax, %ecx - movd %xmm0, %eax - ALIGN (4) -L(aligned_16): - cmp $128, %ecx - jae L(128bytesormore) - -L(aligned_16_less128bytes): - add %ecx, %edx - shr $2, %ecx - BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) - - ALIGN (4) -L(128bytesormore): -#ifdef SHARED_CACHE_SIZE - PUSH (%ebx) - mov $SHARED_CACHE_SIZE, %ebx -#else -# if (defined SHARED || defined __PIC__) - call __i686.get_pc_thunk.bx - add $_GLOBAL_OFFSET_TABLE_, %ebx - mov __x86_shared_cache_size@GOTOFF(%ebx), %ebx -# else - PUSH (%ebx) - mov __x86_shared_cache_size, %ebx -# endif -#endif - cmp %ebx, %ecx - jae L(128bytesormore_nt_start) - -#ifdef DATA_CACHE_SIZE - POP (%ebx) -# define RESTORE_EBX_STATE CFI_PUSH (%ebx) - cmp $DATA_CACHE_SIZE, %ecx -#else -# if (defined SHARED || defined __PIC__) -# define RESTORE_EBX_STATE - call __i686.get_pc_thunk.bx - add $_GLOBAL_OFFSET_TABLE_, %ebx - cmp __x86_data_cache_size@GOTOFF(%ebx), %ecx -# else - POP (%ebx) -# define RESTORE_EBX_STATE CFI_PUSH (%ebx) - cmp __x86_data_cache_size, %ecx -# endif -#endif - - jae L(128bytes_L2_normal) - subl $128, %ecx -L(128bytesormore_normal): - sub $128, %ecx - movdqa %xmm0, (%edx) - movdqa %xmm0, 0x10(%edx) - movdqa %xmm0, 0x20(%edx) - movdqa %xmm0, 0x30(%edx) - movdqa %xmm0, 0x40(%edx) - movdqa %xmm0, 0x50(%edx) - movdqa %xmm0, 0x60(%edx) - movdqa %xmm0, 0x70(%edx) - lea 128(%edx), %edx - jb L(128bytesless_normal) - - - sub $128, %ecx - movdqa %xmm0, (%edx) - movdqa %xmm0, 0x10(%edx) - movdqa %xmm0, 0x20(%edx) - movdqa %xmm0, 0x30(%edx) - movdqa %xmm0, 0x40(%edx) - movdqa %xmm0, 0x50(%edx) - movdqa %xmm0, 0x60(%edx) - movdqa %xmm0, 0x70(%edx) - lea 128(%edx), %edx - jae L(128bytesormore_normal) - -L(128bytesless_normal): - lea 128(%ecx), %ecx - add %ecx, %edx - shr $2, %ecx - BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) - - ALIGN (4) -L(128bytes_L2_normal): - prefetcht0 0x380(%edx) - prefetcht0 0x3c0(%edx) - sub $128, %ecx - movdqa %xmm0, (%edx) - movaps %xmm0, 0x10(%edx) - movaps %xmm0, 0x20(%edx) - movaps %xmm0, 0x30(%edx) - movaps %xmm0, 0x40(%edx) - movaps %xmm0, 0x50(%edx) - movaps %xmm0, 0x60(%edx) - movaps %xmm0, 0x70(%edx) - add $128, %edx - cmp $128, %ecx - jae L(128bytes_L2_normal) - -L(128bytesless_L2_normal): - add %ecx, %edx - shr $2, %ecx - BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) - - RESTORE_EBX_STATE -L(128bytesormore_nt_start): - sub %ebx, %ecx - mov %ebx, %eax - and $0x7f, %eax - add %eax, %ecx - movd %xmm0, %eax - ALIGN (4) -L(128bytesormore_shared_cache_loop): - prefetcht0 0x3c0(%edx) - prefetcht0 0x380(%edx) - sub $0x80, %ebx - movdqa %xmm0, (%edx) - movdqa %xmm0, 0x10(%edx) - movdqa %xmm0, 0x20(%edx) - movdqa %xmm0, 0x30(%edx) - movdqa %xmm0, 0x40(%edx) - movdqa %xmm0, 0x50(%edx) - movdqa %xmm0, 0x60(%edx) - movdqa %xmm0, 0x70(%edx) - add $0x80, %edx - cmp $0x80, %ebx - jae L(128bytesormore_shared_cache_loop) - cmp $0x80, %ecx - jb L(shared_cache_loop_end) - - ALIGN (4) -L(128bytesormore_nt): - sub $0x80, %ecx - movntdq %xmm0, (%edx) - movntdq %xmm0, 0x10(%edx) - movntdq %xmm0, 0x20(%edx) - movntdq %xmm0, 0x30(%edx) - movntdq %xmm0, 0x40(%edx) - movntdq %xmm0, 0x50(%edx) - movntdq %xmm0, 0x60(%edx) - movntdq %xmm0, 0x70(%edx) - add $0x80, %edx - cmp $0x80, %ecx - jae L(128bytesormore_nt) - sfence -L(shared_cache_loop_end): -#if defined DATA_CACHE_SIZE || !(defined SHARED || defined __PIC__) - POP (%ebx) -#endif - add %ecx, %edx - shr $2, %ecx - BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes)) - - .pushsection .rodata.sse2,"a",@progbits - ALIGN (2) -L(table_16_128bytes): - .int JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes)) - .int JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes)) - .popsection - - ALIGN (4) -L(aligned_16_112bytes): - movdqa %xmm0, -112(%edx) -L(aligned_16_96bytes): - movdqa %xmm0, -96(%edx) -L(aligned_16_80bytes): - movdqa %xmm0, -80(%edx) -L(aligned_16_64bytes): - movdqa %xmm0, -64(%edx) -L(aligned_16_48bytes): - movdqa %xmm0, -48(%edx) -L(aligned_16_32bytes): - movdqa %xmm0, -32(%edx) -L(aligned_16_16bytes): - movdqa %xmm0, -16(%edx) -L(aligned_16_0bytes): - SETRTNVAL - RETURN - - ALIGN (4) -L(aligned_16_116bytes): - movdqa %xmm0, -116(%edx) -L(aligned_16_100bytes): - movdqa %xmm0, -100(%edx) -L(aligned_16_84bytes): - movdqa %xmm0, -84(%edx) -L(aligned_16_68bytes): - movdqa %xmm0, -68(%edx) -L(aligned_16_52bytes): - movdqa %xmm0, -52(%edx) -L(aligned_16_36bytes): - movdqa %xmm0, -36(%edx) -L(aligned_16_20bytes): - movdqa %xmm0, -20(%edx) -L(aligned_16_4bytes): - movl %eax, -4(%edx) - SETRTNVAL - RETURN - - ALIGN (4) -L(aligned_16_120bytes): - movdqa %xmm0, -120(%edx) -L(aligned_16_104bytes): - movdqa %xmm0, -104(%edx) -L(aligned_16_88bytes): - movdqa %xmm0, -88(%edx) -L(aligned_16_72bytes): - movdqa %xmm0, -72(%edx) -L(aligned_16_56bytes): - movdqa %xmm0, -56(%edx) -L(aligned_16_40bytes): - movdqa %xmm0, -40(%edx) -L(aligned_16_24bytes): - movdqa %xmm0, -24(%edx) -L(aligned_16_8bytes): - movq %xmm0, -8(%edx) - SETRTNVAL - RETURN - - ALIGN (4) -L(aligned_16_124bytes): - movdqa %xmm0, -124(%edx) -L(aligned_16_108bytes): - movdqa %xmm0, -108(%edx) -L(aligned_16_92bytes): - movdqa %xmm0, -92(%edx) -L(aligned_16_76bytes): - movdqa %xmm0, -76(%edx) -L(aligned_16_60bytes): - movdqa %xmm0, -60(%edx) -L(aligned_16_44bytes): - movdqa %xmm0, -44(%edx) -L(aligned_16_28bytes): - movdqa %xmm0, -28(%edx) -L(aligned_16_12bytes): - movq %xmm0, -12(%edx) - movl %eax, -4(%edx) - SETRTNVAL - RETURN - -END (sse2_memset32_atom) diff --git a/libcutils/arch-x86_64/android_memset16_SSE2-atom.S b/libcutils/arch-x86_64/android_memset16.S similarity index 99% rename from libcutils/arch-x86_64/android_memset16_SSE2-atom.S rename to libcutils/arch-x86_64/android_memset16.S index 48a10ed66..cb6d4a320 100644 --- a/libcutils/arch-x86_64/android_memset16_SSE2-atom.S +++ b/libcutils/arch-x86_64/android_memset16.S @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Contributed by: Intel Corporation - */ #include "cache.h" +#ifndef MEMSET +# define MEMSET android_memset16 +#endif + #ifndef L # define L(label) .L##label #endif @@ -63,7 +64,7 @@ name: \ .section .text.sse2,"ax",@progbits ALIGN (4) -ENTRY (android_memset16) // Address in rdi +ENTRY (MEMSET) // Address in rdi shr $1, %rdx // Count in rdx movzwl %si, %ecx /* Fill the whole ECX with pattern. */ @@ -561,4 +562,4 @@ L(aligned_16_14bytes): movw %cx, -2(%rdi) ret -END (android_memset16) +END (MEMSET) diff --git a/libcutils/arch-x86_64/android_memset32_SSE2-atom.S b/libcutils/arch-x86_64/android_memset32.S similarity index 98% rename from libcutils/arch-x86_64/android_memset32_SSE2-atom.S rename to libcutils/arch-x86_64/android_memset32.S index 4bdea8e0a..1514aa2c8 100644 --- a/libcutils/arch-x86_64/android_memset32_SSE2-atom.S +++ b/libcutils/arch-x86_64/android_memset32.S @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Contributed by: Intel Corporation - */ #include "cache.h" +#ifndef MEMSET +# define MEMSET android_memset32 +#endif + #ifndef L # define L(label) .L##label #endif @@ -63,7 +64,7 @@ name: \ .section .text.sse2,"ax",@progbits ALIGN (4) -ENTRY (android_memset32) // Address in rdi +ENTRY (MEMSET) // Address in rdi shr $2, %rdx // Count in rdx movl %esi, %ecx // Pattern in ecx @@ -369,4 +370,4 @@ L(aligned_16_12bytes): movl %ecx, -4(%rdi) ret -END (android_memset32) +END (MEMSET) diff --git a/libcutils/arch-x86_64/cache.h b/libcutils/arch-x86_64/cache.h index ab5dd2fdc..f14430939 100644 --- a/libcutils/arch-x86_64/cache.h +++ b/libcutils/arch-x86_64/cache.h @@ -13,19 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Contributed by: Intel Corporation - */ -#if defined(__slm__) /* Values are optimized for Silvermont */ #define SHARED_CACHE_SIZE (1024*1024) /* Silvermont L2 Cache */ #define DATA_CACHE_SIZE (24*1024) /* Silvermont L1 Data Cache */ -#else -/* Values are optimized for Atom */ -#define SHARED_CACHE_SIZE (512*1024) /* Atom L2 Cache */ -#define DATA_CACHE_SIZE (24*1024) /* Atom L1 Data Cache */ -#endif #define SHARED_CACHE_SIZE_HALF (SHARED_CACHE_SIZE / 2) #define DATA_CACHE_SIZE_HALF (DATA_CACHE_SIZE / 2)