2012-03-05 19:49:34 +08:00
|
|
|
/*
|
|
|
|
|
* Based on arch/arm/include/asm/cmpxchg.h
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef __ASM_CMPXCHG_H
|
|
|
|
|
#define __ASM_CMPXCHG_H
|
|
|
|
|
|
|
|
|
|
#include <linux/bug.h>
|
2014-10-24 20:22:20 +08:00
|
|
|
#include <linux/mmdebug.h>
|
2012-03-05 19:49:34 +08:00
|
|
|
|
2015-04-24 03:08:49 +08:00
|
|
|
#include <asm/atomic.h>
|
2012-03-05 19:49:34 +08:00
|
|
|
#include <asm/barrier.h>
|
2015-03-31 21:11:24 +08:00
|
|
|
#include <asm/lse.h>
|
2012-03-05 19:49:34 +08:00
|
|
|
|
|
|
|
|
static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
|
|
|
|
|
{
|
|
|
|
|
unsigned long ret, tmp;
|
|
|
|
|
|
|
|
|
|
switch (size) {
|
|
|
|
|
case 1:
|
2015-03-31 21:11:24 +08:00
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
2015-05-29 20:31:10 +08:00
|
|
|
" prfm pstl1strm, %2\n"
|
arm64: atomics: fix use of acquire + release for full barrier semantics
Linux requires a number of atomic operations to provide full barrier
semantics, that is no memory accesses after the operation can be
observed before any accesses up to and including the operation in
program order.
On arm64, these operations have been incorrectly implemented as follows:
// A, B, C are independent memory locations
<Access [A]>
// atomic_op (B)
1: ldaxr x0, [B] // Exclusive load with acquire
<op(B)>
stlxr w1, x0, [B] // Exclusive store with release
cbnz w1, 1b
<Access [C]>
The assumption here being that two half barriers are equivalent to a
full barrier, so the only permitted ordering would be A -> B -> C
(where B is the atomic operation involving both a load and a store).
Unfortunately, this is not the case by the letter of the architecture
and, in fact, the accesses to A and C are permitted to pass their
nearest half barrier resulting in orderings such as Bl -> A -> C -> Bs
or Bl -> C -> A -> Bs (where Bl is the load-acquire on B and Bs is the
store-release on B). This is a clear violation of the full barrier
requirement.
The simple way to fix this is to implement the same algorithm as ARMv7
using explicit barriers:
<Access [A]>
// atomic_op (B)
dmb ish // Full barrier
1: ldxr x0, [B] // Exclusive load
<op(B)>
stxr w1, x0, [B] // Exclusive store
cbnz w1, 1b
dmb ish // Full barrier
<Access [C]>
but this has the undesirable effect of introducing *two* full barrier
instructions. A better approach is actually the following, non-intuitive
sequence:
<Access [A]>
// atomic_op (B)
1: ldxr x0, [B] // Exclusive load
<op(B)>
stlxr w1, x0, [B] // Exclusive store with release
cbnz w1, 1b
dmb ish // Full barrier
<Access [C]>
The simple observations here are:
- The dmb ensures that no subsequent accesses (e.g. the access to C)
can enter or pass the atomic sequence.
- The dmb also ensures that no prior accesses (e.g. the access to A)
can pass the atomic sequence.
- Therefore, no prior access can pass a subsequent access, or
vice-versa (i.e. A is strictly ordered before C).
- The stlxr ensures that no prior access can pass the store component
of the atomic operation.
The only tricky part remaining is the ordering between the ldxr and the
access to A, since the absence of the first dmb means that we're now
permitting re-ordering between the ldxr and any prior accesses.
From an (arbitrary) observer's point of view, there are two scenarios:
1. We have observed the ldxr. This means that if we perform a store to
[B], the ldxr will still return older data. If we can observe the
ldxr, then we can potentially observe the permitted re-ordering
with the access to A, which is clearly an issue when compared to
the dmb variant of the code. Thankfully, the exclusive monitor will
save us here since it will be cleared as a result of the store and
the ldxr will retry. Notice that any use of a later memory
observation to imply observation of the ldxr will also imply
observation of the access to A, since the stlxr/dmb ensure strict
ordering.
2. We have not observed the ldxr. This means we can perform a store
and influence the later ldxr. However, that doesn't actually tell
us anything about the access to [A], so we've not lost anything
here either when compared to the dmb variant.
This patch implements this solution for our barriered atomic operations,
ensuring that we satisfy the full barrier requirements where they are
needed.
Cc: <stable@vger.kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-02-04 20:29:12 +08:00
|
|
|
"1: ldxrb %w0, %2\n"
|
2013-02-04 20:12:33 +08:00
|
|
|
" stlxrb %w1, %w3, %2\n"
|
2012-03-05 19:49:34 +08:00
|
|
|
" cbnz %w1, 1b\n"
|
2015-03-31 21:11:24 +08:00
|
|
|
" dmb ish",
|
|
|
|
|
/* LSE atomics */
|
|
|
|
|
" nop\n"
|
2015-05-29 20:31:10 +08:00
|
|
|
" nop\n"
|
2015-03-31 21:11:24 +08:00
|
|
|
" swpalb %w3, %w0, %2\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" nop")
|
2013-02-04 20:12:33 +08:00
|
|
|
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
|
|
|
|
|
: "r" (x)
|
2014-02-04 20:29:13 +08:00
|
|
|
: "memory");
|
2012-03-05 19:49:34 +08:00
|
|
|
break;
|
|
|
|
|
case 2:
|
2015-03-31 21:11:24 +08:00
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
2015-05-29 20:31:10 +08:00
|
|
|
" prfm pstl1strm, %2\n"
|
arm64: atomics: fix use of acquire + release for full barrier semantics
Linux requires a number of atomic operations to provide full barrier
semantics, that is no memory accesses after the operation can be
observed before any accesses up to and including the operation in
program order.
On arm64, these operations have been incorrectly implemented as follows:
// A, B, C are independent memory locations
<Access [A]>
// atomic_op (B)
1: ldaxr x0, [B] // Exclusive load with acquire
<op(B)>
stlxr w1, x0, [B] // Exclusive store with release
cbnz w1, 1b
<Access [C]>
The assumption here being that two half barriers are equivalent to a
full barrier, so the only permitted ordering would be A -> B -> C
(where B is the atomic operation involving both a load and a store).
Unfortunately, this is not the case by the letter of the architecture
and, in fact, the accesses to A and C are permitted to pass their
nearest half barrier resulting in orderings such as Bl -> A -> C -> Bs
or Bl -> C -> A -> Bs (where Bl is the load-acquire on B and Bs is the
store-release on B). This is a clear violation of the full barrier
requirement.
The simple way to fix this is to implement the same algorithm as ARMv7
using explicit barriers:
<Access [A]>
// atomic_op (B)
dmb ish // Full barrier
1: ldxr x0, [B] // Exclusive load
<op(B)>
stxr w1, x0, [B] // Exclusive store
cbnz w1, 1b
dmb ish // Full barrier
<Access [C]>
but this has the undesirable effect of introducing *two* full barrier
instructions. A better approach is actually the following, non-intuitive
sequence:
<Access [A]>
// atomic_op (B)
1: ldxr x0, [B] // Exclusive load
<op(B)>
stlxr w1, x0, [B] // Exclusive store with release
cbnz w1, 1b
dmb ish // Full barrier
<Access [C]>
The simple observations here are:
- The dmb ensures that no subsequent accesses (e.g. the access to C)
can enter or pass the atomic sequence.
- The dmb also ensures that no prior accesses (e.g. the access to A)
can pass the atomic sequence.
- Therefore, no prior access can pass a subsequent access, or
vice-versa (i.e. A is strictly ordered before C).
- The stlxr ensures that no prior access can pass the store component
of the atomic operation.
The only tricky part remaining is the ordering between the ldxr and the
access to A, since the absence of the first dmb means that we're now
permitting re-ordering between the ldxr and any prior accesses.
From an (arbitrary) observer's point of view, there are two scenarios:
1. We have observed the ldxr. This means that if we perform a store to
[B], the ldxr will still return older data. If we can observe the
ldxr, then we can potentially observe the permitted re-ordering
with the access to A, which is clearly an issue when compared to
the dmb variant of the code. Thankfully, the exclusive monitor will
save us here since it will be cleared as a result of the store and
the ldxr will retry. Notice that any use of a later memory
observation to imply observation of the ldxr will also imply
observation of the access to A, since the stlxr/dmb ensure strict
ordering.
2. We have not observed the ldxr. This means we can perform a store
and influence the later ldxr. However, that doesn't actually tell
us anything about the access to [A], so we've not lost anything
here either when compared to the dmb variant.
This patch implements this solution for our barriered atomic operations,
ensuring that we satisfy the full barrier requirements where they are
needed.
Cc: <stable@vger.kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-02-04 20:29:12 +08:00
|
|
|
"1: ldxrh %w0, %2\n"
|
2013-02-04 20:12:33 +08:00
|
|
|
" stlxrh %w1, %w3, %2\n"
|
2012-03-05 19:49:34 +08:00
|
|
|
" cbnz %w1, 1b\n"
|
2015-03-31 21:11:24 +08:00
|
|
|
" dmb ish",
|
|
|
|
|
/* LSE atomics */
|
|
|
|
|
" nop\n"
|
2015-05-29 20:31:10 +08:00
|
|
|
" nop\n"
|
2015-03-31 21:11:24 +08:00
|
|
|
" swpalh %w3, %w0, %2\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" nop")
|
2013-02-04 20:12:33 +08:00
|
|
|
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
|
|
|
|
|
: "r" (x)
|
2014-02-04 20:29:13 +08:00
|
|
|
: "memory");
|
2012-03-05 19:49:34 +08:00
|
|
|
break;
|
|
|
|
|
case 4:
|
2015-03-31 21:11:24 +08:00
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
2015-05-29 20:31:10 +08:00
|
|
|
" prfm pstl1strm, %2\n"
|
arm64: atomics: fix use of acquire + release for full barrier semantics
Linux requires a number of atomic operations to provide full barrier
semantics, that is no memory accesses after the operation can be
observed before any accesses up to and including the operation in
program order.
On arm64, these operations have been incorrectly implemented as follows:
// A, B, C are independent memory locations
<Access [A]>
// atomic_op (B)
1: ldaxr x0, [B] // Exclusive load with acquire
<op(B)>
stlxr w1, x0, [B] // Exclusive store with release
cbnz w1, 1b
<Access [C]>
The assumption here being that two half barriers are equivalent to a
full barrier, so the only permitted ordering would be A -> B -> C
(where B is the atomic operation involving both a load and a store).
Unfortunately, this is not the case by the letter of the architecture
and, in fact, the accesses to A and C are permitted to pass their
nearest half barrier resulting in orderings such as Bl -> A -> C -> Bs
or Bl -> C -> A -> Bs (where Bl is the load-acquire on B and Bs is the
store-release on B). This is a clear violation of the full barrier
requirement.
The simple way to fix this is to implement the same algorithm as ARMv7
using explicit barriers:
<Access [A]>
// atomic_op (B)
dmb ish // Full barrier
1: ldxr x0, [B] // Exclusive load
<op(B)>
stxr w1, x0, [B] // Exclusive store
cbnz w1, 1b
dmb ish // Full barrier
<Access [C]>
but this has the undesirable effect of introducing *two* full barrier
instructions. A better approach is actually the following, non-intuitive
sequence:
<Access [A]>
// atomic_op (B)
1: ldxr x0, [B] // Exclusive load
<op(B)>
stlxr w1, x0, [B] // Exclusive store with release
cbnz w1, 1b
dmb ish // Full barrier
<Access [C]>
The simple observations here are:
- The dmb ensures that no subsequent accesses (e.g. the access to C)
can enter or pass the atomic sequence.
- The dmb also ensures that no prior accesses (e.g. the access to A)
can pass the atomic sequence.
- Therefore, no prior access can pass a subsequent access, or
vice-versa (i.e. A is strictly ordered before C).
- The stlxr ensures that no prior access can pass the store component
of the atomic operation.
The only tricky part remaining is the ordering between the ldxr and the
access to A, since the absence of the first dmb means that we're now
permitting re-ordering between the ldxr and any prior accesses.
From an (arbitrary) observer's point of view, there are two scenarios:
1. We have observed the ldxr. This means that if we perform a store to
[B], the ldxr will still return older data. If we can observe the
ldxr, then we can potentially observe the permitted re-ordering
with the access to A, which is clearly an issue when compared to
the dmb variant of the code. Thankfully, the exclusive monitor will
save us here since it will be cleared as a result of the store and
the ldxr will retry. Notice that any use of a later memory
observation to imply observation of the ldxr will also imply
observation of the access to A, since the stlxr/dmb ensure strict
ordering.
2. We have not observed the ldxr. This means we can perform a store
and influence the later ldxr. However, that doesn't actually tell
us anything about the access to [A], so we've not lost anything
here either when compared to the dmb variant.
This patch implements this solution for our barriered atomic operations,
ensuring that we satisfy the full barrier requirements where they are
needed.
Cc: <stable@vger.kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-02-04 20:29:12 +08:00
|
|
|
"1: ldxr %w0, %2\n"
|
2013-02-04 20:12:33 +08:00
|
|
|
" stlxr %w1, %w3, %2\n"
|
2012-03-05 19:49:34 +08:00
|
|
|
" cbnz %w1, 1b\n"
|
2015-03-31 21:11:24 +08:00
|
|
|
" dmb ish",
|
|
|
|
|
/* LSE atomics */
|
|
|
|
|
" nop\n"
|
2015-05-29 20:31:10 +08:00
|
|
|
" nop\n"
|
2015-03-31 21:11:24 +08:00
|
|
|
" swpal %w3, %w0, %2\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" nop")
|
2013-02-04 20:12:33 +08:00
|
|
|
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
|
|
|
|
|
: "r" (x)
|
2014-02-04 20:29:13 +08:00
|
|
|
: "memory");
|
2012-03-05 19:49:34 +08:00
|
|
|
break;
|
|
|
|
|
case 8:
|
2015-03-31 21:11:24 +08:00
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
/* LL/SC */
|
2015-05-29 20:31:10 +08:00
|
|
|
" prfm pstl1strm, %2\n"
|
arm64: atomics: fix use of acquire + release for full barrier semantics
Linux requires a number of atomic operations to provide full barrier
semantics, that is no memory accesses after the operation can be
observed before any accesses up to and including the operation in
program order.
On arm64, these operations have been incorrectly implemented as follows:
// A, B, C are independent memory locations
<Access [A]>
// atomic_op (B)
1: ldaxr x0, [B] // Exclusive load with acquire
<op(B)>
stlxr w1, x0, [B] // Exclusive store with release
cbnz w1, 1b
<Access [C]>
The assumption here being that two half barriers are equivalent to a
full barrier, so the only permitted ordering would be A -> B -> C
(where B is the atomic operation involving both a load and a store).
Unfortunately, this is not the case by the letter of the architecture
and, in fact, the accesses to A and C are permitted to pass their
nearest half barrier resulting in orderings such as Bl -> A -> C -> Bs
or Bl -> C -> A -> Bs (where Bl is the load-acquire on B and Bs is the
store-release on B). This is a clear violation of the full barrier
requirement.
The simple way to fix this is to implement the same algorithm as ARMv7
using explicit barriers:
<Access [A]>
// atomic_op (B)
dmb ish // Full barrier
1: ldxr x0, [B] // Exclusive load
<op(B)>
stxr w1, x0, [B] // Exclusive store
cbnz w1, 1b
dmb ish // Full barrier
<Access [C]>
but this has the undesirable effect of introducing *two* full barrier
instructions. A better approach is actually the following, non-intuitive
sequence:
<Access [A]>
// atomic_op (B)
1: ldxr x0, [B] // Exclusive load
<op(B)>
stlxr w1, x0, [B] // Exclusive store with release
cbnz w1, 1b
dmb ish // Full barrier
<Access [C]>
The simple observations here are:
- The dmb ensures that no subsequent accesses (e.g. the access to C)
can enter or pass the atomic sequence.
- The dmb also ensures that no prior accesses (e.g. the access to A)
can pass the atomic sequence.
- Therefore, no prior access can pass a subsequent access, or
vice-versa (i.e. A is strictly ordered before C).
- The stlxr ensures that no prior access can pass the store component
of the atomic operation.
The only tricky part remaining is the ordering between the ldxr and the
access to A, since the absence of the first dmb means that we're now
permitting re-ordering between the ldxr and any prior accesses.
From an (arbitrary) observer's point of view, there are two scenarios:
1. We have observed the ldxr. This means that if we perform a store to
[B], the ldxr will still return older data. If we can observe the
ldxr, then we can potentially observe the permitted re-ordering
with the access to A, which is clearly an issue when compared to
the dmb variant of the code. Thankfully, the exclusive monitor will
save us here since it will be cleared as a result of the store and
the ldxr will retry. Notice that any use of a later memory
observation to imply observation of the ldxr will also imply
observation of the access to A, since the stlxr/dmb ensure strict
ordering.
2. We have not observed the ldxr. This means we can perform a store
and influence the later ldxr. However, that doesn't actually tell
us anything about the access to [A], so we've not lost anything
here either when compared to the dmb variant.
This patch implements this solution for our barriered atomic operations,
ensuring that we satisfy the full barrier requirements where they are
needed.
Cc: <stable@vger.kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-02-04 20:29:12 +08:00
|
|
|
"1: ldxr %0, %2\n"
|
2013-02-04 20:12:33 +08:00
|
|
|
" stlxr %w1, %3, %2\n"
|
2012-03-05 19:49:34 +08:00
|
|
|
" cbnz %w1, 1b\n"
|
2015-03-31 21:11:24 +08:00
|
|
|
" dmb ish",
|
|
|
|
|
/* LSE atomics */
|
|
|
|
|
" nop\n"
|
2015-05-29 20:31:10 +08:00
|
|
|
" nop\n"
|
2015-03-31 21:11:24 +08:00
|
|
|
" swpal %3, %0, %2\n"
|
|
|
|
|
" nop\n"
|
|
|
|
|
" nop")
|
2013-02-04 20:12:33 +08:00
|
|
|
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
|
|
|
|
|
: "r" (x)
|
2014-02-04 20:29:13 +08:00
|
|
|
: "memory");
|
2012-03-05 19:49:34 +08:00
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
BUILD_BUG();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define xchg(ptr,x) \
|
arm64: xchg: prevent warning if return value is unused
Some users of xchg() don't bother using the return value, which results
in a compiler warning like the following (from kgdb):
In file included from linux/arch/arm64/include/asm/atomic.h:27:0,
from include/linux/atomic.h:4,
from include/linux/spinlock.h:402,
from include/linux/seqlock.h:35,
from include/linux/time.h:5,
from include/uapi/linux/timex.h:56,
from include/linux/timex.h:56,
from include/linux/sched.h:19,
from include/linux/pid_namespace.h:4,
from kernel/debug/debug_core.c:30:
kernel/debug/debug_core.c: In function ‘kgdb_cpu_enter’:
linux/arch/arm64/include/asm/cmpxchg.h:75:3: warning: value computed is not used [-Wunused-value]
((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
^
linux/arch/arm64/include/asm/atomic.h:132:30: note: in expansion of macro ‘xchg’
#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
kernel/debug/debug_core.c:504:4: note: in expansion of macro ‘atomic_xchg’
atomic_xchg(&kgdb_active, cpu);
^
This patch makes use of the same trick as we do for cmpxchg, by assigning
the return value to a dummy variable in the xchg() macro itself.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2014-04-30 23:23:06 +08:00
|
|
|
({ \
|
|
|
|
|
__typeof__(*(ptr)) __ret; \
|
|
|
|
|
__ret = (__typeof__(*(ptr))) \
|
|
|
|
|
__xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
|
|
|
|
|
__ret; \
|
|
|
|
|
})
|
2012-03-05 19:49:34 +08:00
|
|
|
|
|
|
|
|
static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
|
|
|
|
|
unsigned long new, int size)
|
|
|
|
|
{
|
|
|
|
|
switch (size) {
|
|
|
|
|
case 1:
|
2015-07-31 02:19:43 +08:00
|
|
|
return __cmpxchg_case_1(ptr, (u8)old, new);
|
2012-03-05 19:49:34 +08:00
|
|
|
case 2:
|
2015-07-31 02:19:43 +08:00
|
|
|
return __cmpxchg_case_2(ptr, (u16)old, new);
|
2012-03-05 19:49:34 +08:00
|
|
|
case 4:
|
2015-04-24 03:08:49 +08:00
|
|
|
return __cmpxchg_case_4(ptr, old, new);
|
2012-03-05 19:49:34 +08:00
|
|
|
case 8:
|
2015-04-24 03:08:49 +08:00
|
|
|
return __cmpxchg_case_8(ptr, old, new);
|
2012-03-05 19:49:34 +08:00
|
|
|
default:
|
|
|
|
|
BUILD_BUG();
|
|
|
|
|
}
|
|
|
|
|
|
2015-04-24 03:08:49 +08:00
|
|
|
unreachable();
|
2012-03-05 19:49:34 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
|
|
|
|
|
unsigned long new, int size)
|
|
|
|
|
{
|
2015-04-24 03:08:49 +08:00
|
|
|
switch (size) {
|
|
|
|
|
case 1:
|
2015-07-31 02:19:43 +08:00
|
|
|
return __cmpxchg_case_mb_1(ptr, (u8)old, new);
|
2015-04-24 03:08:49 +08:00
|
|
|
case 2:
|
2015-07-31 02:19:43 +08:00
|
|
|
return __cmpxchg_case_mb_2(ptr, (u16)old, new);
|
2015-04-24 03:08:49 +08:00
|
|
|
case 4:
|
|
|
|
|
return __cmpxchg_case_mb_4(ptr, old, new);
|
|
|
|
|
case 8:
|
|
|
|
|
return __cmpxchg_case_mb_8(ptr, old, new);
|
|
|
|
|
default:
|
|
|
|
|
BUILD_BUG();
|
|
|
|
|
}
|
2012-03-05 19:49:34 +08:00
|
|
|
|
2015-04-24 03:08:49 +08:00
|
|
|
unreachable();
|
2012-03-05 19:49:34 +08:00
|
|
|
}
|
|
|
|
|
|
2013-12-04 03:19:12 +08:00
|
|
|
#define cmpxchg(ptr, o, n) \
|
|
|
|
|
({ \
|
|
|
|
|
__typeof__(*(ptr)) __ret; \
|
|
|
|
|
__ret = (__typeof__(*(ptr))) \
|
|
|
|
|
__cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
|
|
|
|
|
sizeof(*(ptr))); \
|
|
|
|
|
__ret; \
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#define cmpxchg_local(ptr, o, n) \
|
|
|
|
|
({ \
|
|
|
|
|
__typeof__(*(ptr)) __ret; \
|
|
|
|
|
__ret = (__typeof__(*(ptr))) \
|
|
|
|
|
__cmpxchg((ptr), (unsigned long)(o), \
|
|
|
|
|
(unsigned long)(n), sizeof(*(ptr))); \
|
|
|
|
|
__ret; \
|
|
|
|
|
})
|
2012-03-05 19:49:34 +08:00
|
|
|
|
2015-05-15 01:05:50 +08:00
|
|
|
#define system_has_cmpxchg_double() 1
|
|
|
|
|
|
|
|
|
|
#define __cmpxchg_double_check(ptr1, ptr2) \
|
|
|
|
|
({ \
|
|
|
|
|
if (sizeof(*(ptr1)) != 8) \
|
|
|
|
|
BUILD_BUG(); \
|
|
|
|
|
VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \
|
|
|
|
|
})
|
|
|
|
|
|
2014-10-24 20:22:20 +08:00
|
|
|
#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
|
|
|
|
|
({\
|
|
|
|
|
int __ret;\
|
2015-05-15 01:05:50 +08:00
|
|
|
__cmpxchg_double_check(ptr1, ptr2); \
|
|
|
|
|
__ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
|
|
|
|
|
(unsigned long)(n1), (unsigned long)(n2), \
|
|
|
|
|
ptr1); \
|
2014-10-24 20:22:20 +08:00
|
|
|
__ret; \
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
|
|
|
|
|
({\
|
|
|
|
|
int __ret;\
|
2015-05-15 01:05:50 +08:00
|
|
|
__cmpxchg_double_check(ptr1, ptr2); \
|
|
|
|
|
__ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
|
|
|
|
|
(unsigned long)(n1), (unsigned long)(n2), \
|
|
|
|
|
ptr1); \
|
2014-10-24 20:22:20 +08:00
|
|
|
__ret; \
|
|
|
|
|
})
|
|
|
|
|
|
2015-03-22 22:51:51 +08:00
|
|
|
#define _protect_cmpxchg_local(pcp, o, n) \
|
|
|
|
|
({ \
|
|
|
|
|
typeof(*raw_cpu_ptr(&(pcp))) __ret; \
|
|
|
|
|
preempt_disable(); \
|
|
|
|
|
__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
|
|
|
|
|
preempt_enable(); \
|
|
|
|
|
__ret; \
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
|
|
|
|
|
#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
|
|
|
|
|
#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
|
|
|
|
|
#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
|
|
|
|
|
|
|
|
|
|
#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
|
|
|
|
|
({ \
|
|
|
|
|
int __ret; \
|
|
|
|
|
preempt_disable(); \
|
|
|
|
|
__ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
|
|
|
|
|
raw_cpu_ptr(&(ptr2)), \
|
|
|
|
|
o1, o2, n1, n2); \
|
|
|
|
|
preempt_enable(); \
|
|
|
|
|
__ret; \
|
|
|
|
|
})
|
2014-10-24 20:22:20 +08:00
|
|
|
|
2013-04-22 13:08:41 +08:00
|
|
|
#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
|
|
|
|
|
#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
|
|
|
|
|
|
2013-10-09 22:54:28 +08:00
|
|
|
#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n))
|
|
|
|
|
|
2012-03-05 19:49:34 +08:00
|
|
|
#endif /* __ASM_CMPXCHG_H */
|