openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'x86-atomic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86/atomic changes from Ingo Molnar.

* 'x86-atomic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86: atomic64 assembly improvements
  x86: Adjust asm constraints in atomic64 wrappers
This commit is contained in:
Linus Torvalds 2012-03-22 09:23:57 -07:00
parent e17fdf5c67 cb8095bba6
commit 1b674bf106
5 changed files with 102 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
arch/x86/include/asm/alternative.h
View File

@ -145,6 +145,12 @@ static inline int alternatives_text_reserved(void *start, void *end)
*/ */
#define ASM_OUTPUT2(a...) a #define ASM_OUTPUT2(a...) a
/*
* use this macro if you need clobbers but no inputs in
* alternative_{input,io,call}()
*/
#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
struct paravirt_patch_site; struct paravirt_patch_site;
#ifdef CONFIG_PARAVIRT #ifdef CONFIG_PARAVIRT
void apply_paravirt(struct paravirt_patch_site *start, void apply_paravirt(struct paravirt_patch_site *start,

148
arch/x86/include/asm/atomic64_32.h
View File

@ -14,13 +14,52 @@ typedef struct {
#define ATOMIC64_INIT(val) { (val) } #define ATOMIC64_INIT(val) { (val) }
#ifdef CONFIG_X86_CMPXCHG64 #define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...)
#define ATOMIC64_ALTERNATIVE_(f, g) "call atomic64_" #g "_cx8" #ifndef ATOMIC64_EXPORT
#define ATOMIC64_DECL_ONE __ATOMIC64_DECL
#else #else
#define ATOMIC64_ALTERNATIVE_(f, g) ALTERNATIVE("call atomic64_" #f "_386", "call atomic64_" #g "_cx8", X86_FEATURE_CX8) #define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \
ATOMIC64_EXPORT(atomic64_##sym)
#endif #endif
#define ATOMIC64_ALTERNATIVE(f) ATOMIC64_ALTERNATIVE_(f, f) #ifdef CONFIG_X86_CMPXCHG64
#define __alternative_atomic64(f, g, out, in...) \
asm volatile("call %P[func]" \
: out : [func] "i" (atomic64_##g##_cx8), ## in)
#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8)
#else
#define __alternative_atomic64(f, g, out, in...) \
alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \
X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in)
#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \
ATOMIC64_DECL_ONE(sym##_386)
ATOMIC64_DECL_ONE(add_386);
ATOMIC64_DECL_ONE(sub_386);
ATOMIC64_DECL_ONE(inc_386);
ATOMIC64_DECL_ONE(dec_386);
#endif
#define alternative_atomic64(f, out, in...) \
__alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in)
ATOMIC64_DECL(read);
ATOMIC64_DECL(set);
ATOMIC64_DECL(xchg);
ATOMIC64_DECL(add_return);
ATOMIC64_DECL(sub_return);
ATOMIC64_DECL(inc_return);
ATOMIC64_DECL(dec_return);
ATOMIC64_DECL(dec_if_positive);
ATOMIC64_DECL(inc_not_zero);
ATOMIC64_DECL(add_unless);
#undef ATOMIC64_DECL
#undef ATOMIC64_DECL_ONE
#undef __ATOMIC64_DECL
#undef ATOMIC64_EXPORT
/** /**
* atomic64_cmpxchg - cmpxchg atomic64 variable * atomic64_cmpxchg - cmpxchg atomic64 variable
@ -50,11 +89,9 @@ static inline long long atomic64_xchg(atomic64_t *v, long long n)
long long o; long long o;
unsigned high = (unsigned)(n >> 32); unsigned high = (unsigned)(n >> 32);
unsigned low = (unsigned)n; unsigned low = (unsigned)n;
asm volatile(ATOMIC64_ALTERNATIVE(xchg) alternative_atomic64(xchg, "=&A" (o),
: "=A" (o), "+b" (low), "+c" (high) "S" (v), "b" (low), "c" (high)
: "S" (v) : "memory");
: "memory"
);
return o; return o;
} }
@ -69,11 +106,9 @@ static inline void atomic64_set(atomic64_t *v, long long i)
{ {
unsigned high = (unsigned)(i >> 32); unsigned high = (unsigned)(i >> 32);
unsigned low = (unsigned)i; unsigned low = (unsigned)i;
asm volatile(ATOMIC64_ALTERNATIVE(set) alternative_atomic64(set, /* no output */,
: "+b" (low), "+c" (high) "S" (v), "b" (low), "c" (high)
: "S" (v) : "eax", "edx", "memory");
: "eax", "edx", "memory"
);
} }
/** /**
@ -85,10 +120,7 @@ static inline void atomic64_set(atomic64_t *v, long long i)
static inline long long atomic64_read(const atomic64_t *v) static inline long long atomic64_read(const atomic64_t *v)
{ {
long long r; long long r;
asm volatile(ATOMIC64_ALTERNATIVE(read) alternative_atomic64(read, "=&A" (r), "c" (v) : "memory");
: "=A" (r), "+c" (v)
: : "memory"
);
return r; return r;
} }
@ -101,10 +133,9 @@ static inline long long atomic64_read(const atomic64_t *v)
*/ */
static inline long long atomic64_add_return(long long i, atomic64_t *v) static inline long long atomic64_add_return(long long i, atomic64_t *v)
{ {
asm volatile(ATOMIC64_ALTERNATIVE(add_return) alternative_atomic64(add_return,
: "+A" (i), "+c" (v) ASM_OUTPUT2("+A" (i), "+c" (v)),
: : "memory" ASM_NO_INPUT_CLOBBER("memory"));
);
return i; return i;
} }
@ -113,32 +144,25 @@ static inline long long atomic64_add_return(long long i, atomic64_t *v)
*/ */
static inline long long atomic64_sub_return(long long i, atomic64_t *v) static inline long long atomic64_sub_return(long long i, atomic64_t *v)
{ {
asm volatile(ATOMIC64_ALTERNATIVE(sub_return) alternative_atomic64(sub_return,
: "+A" (i), "+c" (v) ASM_OUTPUT2("+A" (i), "+c" (v)),
: : "memory" ASM_NO_INPUT_CLOBBER("memory"));
);
return i; return i;
} }
static inline long long atomic64_inc_return(atomic64_t *v) static inline long long atomic64_inc_return(atomic64_t *v)
{ {
long long a; long long a;
asm volatile(ATOMIC64_ALTERNATIVE(inc_return) alternative_atomic64(inc_return, "=&A" (a),
: "=A" (a) "S" (v) : "memory", "ecx");
: "S" (v)
: "memory", "ecx"
);
return a; return a;
} }
static inline long long atomic64_dec_return(atomic64_t *v) static inline long long atomic64_dec_return(atomic64_t *v)
{ {
long long a; long long a;
asm volatile(ATOMIC64_ALTERNATIVE(dec_return) alternative_atomic64(dec_return, "=&A" (a),
: "=A" (a) "S" (v) : "memory", "ecx");
: "S" (v)
: "memory", "ecx"
);
return a; return a;
} }
@ -151,10 +175,9 @@ static inline long long atomic64_dec_return(atomic64_t *v)
*/ */
static inline long long atomic64_add(long long i, atomic64_t *v) static inline long long atomic64_add(long long i, atomic64_t *v)
{ {
asm volatile(ATOMIC64_ALTERNATIVE_(add, add_return) __alternative_atomic64(add, add_return,
: "+A" (i), "+c" (v) ASM_OUTPUT2("+A" (i), "+c" (v)),
: : "memory" ASM_NO_INPUT_CLOBBER("memory"));
);
return i; return i;
} }
@ -167,10 +190,9 @@ static inline long long atomic64_add(long long i, atomic64_t *v)
*/ */
static inline long long atomic64_sub(long long i, atomic64_t *v) static inline long long atomic64_sub(long long i, atomic64_t *v)
{ {
asm volatile(ATOMIC64_ALTERNATIVE_(sub, sub_return) __alternative_atomic64(sub, sub_return,
: "+A" (i), "+c" (v) ASM_OUTPUT2("+A" (i), "+c" (v)),
: : "memory" ASM_NO_INPUT_CLOBBER("memory"));
);
return i; return i;
} }
@ -196,10 +218,8 @@ static inline int atomic64_sub_and_test(long long i, atomic64_t *v)
*/ */
static inline void atomic64_inc(atomic64_t *v) static inline void atomic64_inc(atomic64_t *v)
{ {
asm volatile(ATOMIC64_ALTERNATIVE_(inc, inc_return) __alternative_atomic64(inc, inc_return, /* no output */,
: : "S" (v) "S" (v) : "memory", "eax", "ecx", "edx");
: "memory", "eax", "ecx", "edx"
);
} }
/** /**
@ -210,10 +230,8 @@ static inline void atomic64_inc(atomic64_t *v)
*/ */
static inline void atomic64_dec(atomic64_t *v) static inline void atomic64_dec(atomic64_t *v)
{ {
asm volatile(ATOMIC64_ALTERNATIVE_(dec, dec_return) __alternative_atomic64(dec, dec_return, /* no output */,
: : "S" (v) "S" (v) : "memory", "eax", "ecx", "edx");
: "memory", "eax", "ecx", "edx"
);
} }
/** /**
@ -263,15 +281,15 @@ static inline int atomic64_add_negative(long long i, atomic64_t *v)
* @u: ...unless v is equal to u. * @u: ...unless v is equal to u.
* *
* Atomically adds @a to @v, so long as it was not @u. * Atomically adds @a to @v, so long as it was not @u.
* Returns the old value of @v. * Returns non-zero if the add was done, zero otherwise.
*/ */
static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
{ {
unsigned low = (unsigned)u; unsigned low = (unsigned)u;
unsigned high = (unsigned)(u >> 32); unsigned high = (unsigned)(u >> 32);
asm volatile(ATOMIC64_ALTERNATIVE(add_unless) "\n\t" alternative_atomic64(add_unless,
: "+A" (a), "+c" (v), "+S" (low), "+D" (high) ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)),
: : "memory"); "S" (v) : "memory");
return (int)a; return (int)a;
} }
@ -279,26 +297,20 @@ static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
static inline int atomic64_inc_not_zero(atomic64_t *v) static inline int atomic64_inc_not_zero(atomic64_t *v)
{ {
int r; int r;
asm volatile(ATOMIC64_ALTERNATIVE(inc_not_zero) alternative_atomic64(inc_not_zero, "=&a" (r),
: "=a" (r) "S" (v) : "ecx", "edx", "memory");
: "S" (v)
: "ecx", "edx", "memory"
);
return r; return r;
} }
static inline long long atomic64_dec_if_positive(atomic64_t *v) static inline long long atomic64_dec_if_positive(atomic64_t *v)
{ {
long long r; long long r;
asm volatile(ATOMIC64_ALTERNATIVE(dec_if_positive) alternative_atomic64(dec_if_positive, "=&A" (r),
: "=A" (r) "S" (v) : "ecx", "memory");
: "S" (v)
: "ecx", "memory"
);
return r; return r;
} }
#undef ATOMIC64_ALTERNATIVE #undef alternative_atomic64
#undef ATOMIC64_ALTERNATIVE_ #undef __alternative_atomic64
#endif /* _ASM_X86_ATOMIC64_32_H */ #endif /* _ASM_X86_ATOMIC64_32_H */

59
arch/x86/lib/atomic64_32.c
View File

@ -1,59 +1,4 @@
#include <linux/compiler.h> #define ATOMIC64_EXPORT EXPORT_SYMBOL
#include <linux/module.h>
#include <linux/types.h>
#include <asm/processor.h> #include <linux/export.h>
#include <asm/cmpxchg.h>
#include <linux/atomic.h> #include <linux/atomic.h>
long long atomic64_read_cx8(long long, const atomic64_t *v);
EXPORT_SYMBOL(atomic64_read_cx8);
long long atomic64_set_cx8(long long, const atomic64_t *v);
EXPORT_SYMBOL(atomic64_set_cx8);
long long atomic64_xchg_cx8(long long, unsigned high);
EXPORT_SYMBOL(atomic64_xchg_cx8);
long long atomic64_add_return_cx8(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_add_return_cx8);
long long atomic64_sub_return_cx8(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_sub_return_cx8);
long long atomic64_inc_return_cx8(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_inc_return_cx8);
long long atomic64_dec_return_cx8(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_dec_return_cx8);
long long atomic64_dec_if_positive_cx8(atomic64_t *v);
EXPORT_SYMBOL(atomic64_dec_if_positive_cx8);
int atomic64_inc_not_zero_cx8(atomic64_t *v);
EXPORT_SYMBOL(atomic64_inc_not_zero_cx8);
int atomic64_add_unless_cx8(atomic64_t *v, long long a, long long u);
EXPORT_SYMBOL(atomic64_add_unless_cx8);
#ifndef CONFIG_X86_CMPXCHG64
long long atomic64_read_386(long long, const atomic64_t *v);
EXPORT_SYMBOL(atomic64_read_386);
long long atomic64_set_386(long long, const atomic64_t *v);
EXPORT_SYMBOL(atomic64_set_386);
long long atomic64_xchg_386(long long, unsigned high);
EXPORT_SYMBOL(atomic64_xchg_386);
long long atomic64_add_return_386(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_add_return_386);
long long atomic64_sub_return_386(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_sub_return_386);
long long atomic64_inc_return_386(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_inc_return_386);
long long atomic64_dec_return_386(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_dec_return_386);
long long atomic64_add_386(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_add_386);
long long atomic64_sub_386(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_sub_386);
long long atomic64_inc_386(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_inc_386);
long long atomic64_dec_386(long long a, atomic64_t *v);
EXPORT_SYMBOL(atomic64_dec_386);
long long atomic64_dec_if_positive_386(atomic64_t *v);
EXPORT_SYMBOL(atomic64_dec_if_positive_386);
int atomic64_inc_not_zero_386(atomic64_t *v);
EXPORT_SYMBOL(atomic64_inc_not_zero_386);
int atomic64_add_unless_386(atomic64_t *v, long long a, long long u);
EXPORT_SYMBOL(atomic64_add_unless_386);
#endif

6
arch/x86/lib/atomic64_386_32.S
View File

@ -137,13 +137,13 @@ BEGIN(dec_return)
RET_ENDP RET_ENDP
#undef v #undef v
#define v %ecx #define v %esi
BEGIN(add_unless) BEGIN(add_unless)
addl %eax, %esi addl %eax, %ecx
adcl %edx, %edi adcl %edx, %edi
addl (v), %eax addl (v), %eax
adcl 4(v), %edx adcl 4(v), %edx
cmpl %eax, %esi cmpl %eax, %ecx
je 3f je 3f
1: 1:
movl %eax, (v) movl %eax, (v)

29
arch/x86/lib/atomic64_cx8_32.S
View File

@ -55,8 +55,6 @@ ENDPROC(atomic64_set_cx8)
ENTRY(atomic64_xchg_cx8) ENTRY(atomic64_xchg_cx8)
CFI_STARTPROC CFI_STARTPROC
movl %ebx, %eax
movl %ecx, %edx
1: 1:
LOCK_PREFIX LOCK_PREFIX
cmpxchg8b (%esi) cmpxchg8b (%esi)
@ -78,7 +76,7 @@ ENTRY(atomic64_\func\()_return_cx8)
movl %edx, %edi movl %edx, %edi
movl %ecx, %ebp movl %ecx, %ebp
read64 %ebp read64 %ecx
1: 1:
movl %eax, %ebx movl %eax, %ebx
movl %edx, %ecx movl %edx, %ecx
@ -159,23 +157,22 @@ ENTRY(atomic64_add_unless_cx8)
SAVE ebx SAVE ebx
/* these just push these two parameters on the stack */ /* these just push these two parameters on the stack */
SAVE edi SAVE edi
SAVE esi SAVE ecx
movl %ecx, %ebp movl %eax, %ebp
movl %eax, %esi
movl %edx, %edi movl %edx, %edi
read64 %ebp read64 %esi
1: 1:
cmpl %eax, 0(%esp) cmpl %eax, 0(%esp)
je 4f je 4f
2: 2:
movl %eax, %ebx movl %eax, %ebx
movl %edx, %ecx movl %edx, %ecx
addl %esi, %ebx addl %ebp, %ebx
adcl %edi, %ecx adcl %edi, %ecx
LOCK_PREFIX LOCK_PREFIX
cmpxchg8b (%ebp) cmpxchg8b (%esi)
jne 1b jne 1b
movl $1, %eax movl $1, %eax
@ -199,13 +196,13 @@ ENTRY(atomic64_inc_not_zero_cx8)
read64 %esi read64 %esi
1: 1:
testl %eax, %eax movl %eax, %ecx
je 4f orl %edx, %ecx
2: jz 3f
movl %eax, %ebx movl %eax, %ebx
movl %edx, %ecx xorl %ecx, %ecx
addl $1, %ebx addl $1, %ebx
adcl $0, %ecx adcl %edx, %ecx
LOCK_PREFIX LOCK_PREFIX
cmpxchg8b (%esi) cmpxchg8b (%esi)
jne 1b jne 1b
@ -214,9 +211,5 @@ ENTRY(atomic64_inc_not_zero_cx8)
3: 3:
RESTORE ebx RESTORE ebx
ret ret
4:
testl %edx, %edx
jne 2b
jmp 3b
CFI_ENDPROC CFI_ENDPROC
ENDPROC(atomic64_inc_not_zero_cx8) ENDPROC(atomic64_inc_not_zero_cx8)