2018-01-30 01:26:07 +08:00
|
|
|
/*
|
|
|
|
|
* This file provides wrappers with KASAN instrumentation for atomic operations.
|
|
|
|
|
* To use this functionality an arch's atomic.h file needs to define all
|
|
|
|
|
* atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
|
|
|
|
|
* this file at the end. This file provides atomic_read() that forwards to
|
|
|
|
|
* arch_atomic_read() for actual atomic operation.
|
|
|
|
|
* Note: if an arch atomic operation is implemented by means of other atomic
|
|
|
|
|
* operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
|
|
|
|
|
* arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
|
|
|
|
|
* double instrumentation.
|
|
|
|
|
*/
|
|
|
|
|
|
2018-01-30 01:26:04 +08:00
|
|
|
#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
|
|
|
|
|
#define _LINUX_ATOMIC_INSTRUMENTED_H
|
|
|
|
|
|
|
|
|
|
#include <linux/build_bug.h>
|
2018-01-30 01:26:06 +08:00
|
|
|
#include <linux/kasan-checks.h>
|
2018-01-30 01:26:04 +08:00
|
|
|
|
|
|
|
|
static __always_inline int atomic_read(const atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_read(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_read(v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_read(const atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_read(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_read(v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic_set(atomic_t *v, int i)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic_set(v, i);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic64_set(atomic64_t *v, s64 i)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic64_set(v, i);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline int atomic_xchg(atomic_t *v, int i)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_xchg(v, i);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_xchg(v, i);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_cmpxchg(v, old, new);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_cmpxchg(v, old, new);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifdef arch_atomic_try_cmpxchg
|
|
|
|
|
#define atomic_try_cmpxchg atomic_try_cmpxchg
|
|
|
|
|
static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
|
|
|
|
kasan_check_read(old, sizeof(*old));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_try_cmpxchg(v, old, new);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef arch_atomic64_try_cmpxchg
|
|
|
|
|
#define atomic64_try_cmpxchg atomic64_try_cmpxchg
|
|
|
|
|
static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
|
|
|
|
kasan_check_read(old, sizeof(*old));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_try_cmpxchg(v, old, new);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2018-06-21 20:13:09 +08:00
|
|
|
#ifdef arch_atomic_fetch_add_unless
|
|
|
|
|
#define atomic_fetch_add_unless atomic_fetch_add_unless
|
atomics/treewide: Rename __atomic_add_unless() => atomic_fetch_add_unless()
While __atomic_add_unless() was originally intended as a building-block
for atomic_add_unless(), it's now used in a number of places around the
kernel. It's the only common atomic operation named __atomic*(), rather
than atomic_*(), and for consistency it would be better named
atomic_fetch_add_unless().
This lack of consistency is slightly confusing, and gets in the way of
scripting atomics. Given that, let's clean things up and promote it to
an official part of the atomics API, in the form of
atomic_fetch_add_unless().
This patch converts definitions and invocations over to the new name,
including the instrumented version, using the following script:
----
git grep -w __atomic_add_unless | while read line; do
sed -i '{s/\<__atomic_add_unless\>/atomic_fetch_add_unless/}' "${line%%:*}";
done
git grep -w __arch_atomic_add_unless | while read line; do
sed -i '{s/\<__arch_atomic_add_unless\>/arch_atomic_fetch_add_unless/}' "${line%%:*}";
done
----
Note that we do not have atomic{64,_long}_fetch_add_unless(), which will
be introduced by later patches.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-2-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:04 +08:00
|
|
|
static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)
|
2018-01-30 01:26:04 +08:00
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
atomics/treewide: Rename __atomic_add_unless() => atomic_fetch_add_unless()
While __atomic_add_unless() was originally intended as a building-block
for atomic_add_unless(), it's now used in a number of places around the
kernel. It's the only common atomic operation named __atomic*(), rather
than atomic_*(), and for consistency it would be better named
atomic_fetch_add_unless().
This lack of consistency is slightly confusing, and gets in the way of
scripting atomics. Given that, let's clean things up and promote it to
an official part of the atomics API, in the form of
atomic_fetch_add_unless().
This patch converts definitions and invocations over to the new name,
including the instrumented version, using the following script:
----
git grep -w __atomic_add_unless | while read line; do
sed -i '{s/\<__atomic_add_unless\>/atomic_fetch_add_unless/}' "${line%%:*}";
done
git grep -w __arch_atomic_add_unless | while read line; do
sed -i '{s/\<__arch_atomic_add_unless\>/arch_atomic_fetch_add_unless/}' "${line%%:*}";
done
----
Note that we do not have atomic{64,_long}_fetch_add_unless(), which will
be introduced by later patches.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-2-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:04 +08:00
|
|
|
return arch_atomic_fetch_add_unless(v, a, u);
|
2018-01-30 01:26:04 +08:00
|
|
|
}
|
2018-06-21 20:13:09 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:10 +08:00
|
|
|
#ifdef arch_atomic64_fetch_add_unless
|
|
|
|
|
#define atomic64_fetch_add_unless atomic64_fetch_add_unless
|
|
|
|
|
static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
|
|
|
|
|
{
|
|
|
|
|
kasan_check_write(v, sizeof(*v));
|
|
|
|
|
return arch_atomic64_fetch_add_unless(v, a, u);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:19 +08:00
|
|
|
#ifdef arch_atomic_inc
|
|
|
|
|
#define atomic_inc atomic_inc
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline void atomic_inc(atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic_inc(v);
|
|
|
|
|
}
|
2018-06-21 20:13:19 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:19 +08:00
|
|
|
#ifdef arch_atomic64_inc
|
|
|
|
|
#define atomic64_inc atomic64_inc
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline void atomic64_inc(atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic64_inc(v);
|
|
|
|
|
}
|
2018-06-21 20:13:19 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:19 +08:00
|
|
|
#ifdef arch_atomic_dec
|
|
|
|
|
#define atomic_dec atomic_dec
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline void atomic_dec(atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic_dec(v);
|
|
|
|
|
}
|
2018-06-21 20:13:19 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:19 +08:00
|
|
|
#ifdef atch_atomic64_dec
|
|
|
|
|
#define atomic64_dec
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline void atomic64_dec(atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic64_dec(v);
|
|
|
|
|
}
|
2018-06-21 20:13:19 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
|
|
|
|
static __always_inline void atomic_add(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic_add(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic64_add(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic64_add(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic_sub(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic_sub(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic64_sub(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic64_sub(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic_and(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic_and(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic64_and(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic64_and(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic_or(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic_or(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic64_or(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic64_or(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic_xor(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic_xor(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline void atomic64_xor(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
arch_atomic64_xor(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
2018-06-21 20:13:19 +08:00
|
|
|
#ifdef arch_atomic_inc_return
|
|
|
|
|
#define atomic_inc_return atomic_inc_return
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline int atomic_inc_return(atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_inc_return(v);
|
|
|
|
|
}
|
2018-06-21 20:13:19 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:19 +08:00
|
|
|
#ifdef arch_atomic64_in_return
|
|
|
|
|
#define atomic64_inc_return atomic64_inc_return
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline s64 atomic64_inc_return(atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_inc_return(v);
|
|
|
|
|
}
|
2018-06-21 20:13:19 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:19 +08:00
|
|
|
#ifdef arch_atomic_dec_return
|
|
|
|
|
#define atomic_dec_return atomic_dec_return
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline int atomic_dec_return(atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_dec_return(v);
|
|
|
|
|
}
|
2018-06-21 20:13:19 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:19 +08:00
|
|
|
#ifdef arch_atomic64_dec_return
|
|
|
|
|
#define atomic64_dec_return atomic64_dec_return
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline s64 atomic64_dec_return(atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_dec_return(v);
|
|
|
|
|
}
|
2018-06-21 20:13:19 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:08 +08:00
|
|
|
#ifdef arch_atomic64_inc_not_zero
|
|
|
|
|
#define atomic64_inc_not_zero atomic64_inc_not_zero
|
2018-06-21 20:13:07 +08:00
|
|
|
static __always_inline bool atomic64_inc_not_zero(atomic64_t *v)
|
2018-01-30 01:26:04 +08:00
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_inc_not_zero(v);
|
|
|
|
|
}
|
2018-06-21 20:13:08 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-06-21 20:13:20 +08:00
|
|
|
#ifdef arch_atomic64_dec_if_positive
|
|
|
|
|
#define atomic64_dec_if_positive atomic64_dec_if_positive
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_dec_if_positive(v);
|
|
|
|
|
}
|
2018-06-21 20:13:20 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#ifdef arch_atomic_dec_and_test
|
|
|
|
|
#define atomic_dec_and_test atomic_dec_and_test
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline bool atomic_dec_and_test(atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_dec_and_test(v);
|
|
|
|
|
}
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#ifdef arch_atomic64_dec_and_test
|
|
|
|
|
#define atomic64_dec_and_test atomic64_dec_and_test
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline bool atomic64_dec_and_test(atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_dec_and_test(v);
|
|
|
|
|
}
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#ifdef arch_atomic_inc_and_test
|
|
|
|
|
#define atomic_inc_and_test atomic_inc_and_test
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline bool atomic_inc_and_test(atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_inc_and_test(v);
|
|
|
|
|
}
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#ifdef arch_atomic64_inc_and_test
|
|
|
|
|
#define atomic64_inc_and_test atomic64_inc_and_test
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline bool atomic64_inc_and_test(atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_inc_and_test(v);
|
|
|
|
|
}
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
|
|
|
|
static __always_inline int atomic_add_return(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_add_return(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_add_return(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline int atomic_sub_return(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_sub_return(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_sub_return(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline int atomic_fetch_add(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_fetch_add(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_fetch_add(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline int atomic_fetch_sub(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_fetch_sub(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_fetch_sub(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline int atomic_fetch_and(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_fetch_and(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_fetch_and(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline int atomic_fetch_or(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_fetch_or(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_fetch_or(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline int atomic_fetch_xor(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_fetch_xor(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_fetch_xor(i, v);
|
|
|
|
|
}
|
|
|
|
|
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#ifdef arch_atomic_sub_and_test
|
|
|
|
|
#define atomic_sub_and_test atomic_sub_and_test
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline bool atomic_sub_and_test(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_sub_and_test(i, v);
|
|
|
|
|
}
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#ifdef arch_atomic64_sub_and_test
|
|
|
|
|
#define atomic64_sub_and_test atomic64_sub_and_test
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_sub_and_test(i, v);
|
|
|
|
|
}
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#ifdef arch_atomic_add_negative
|
|
|
|
|
#define atomic_add_negative atomic_add_negative
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline bool atomic_add_negative(int i, atomic_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic_add_negative(i, v);
|
|
|
|
|
}
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#ifdef arch_atomic64_add_negative
|
|
|
|
|
#define atomic64_add_negative atomic64_add_negative
|
2018-01-30 01:26:04 +08:00
|
|
|
static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v)
|
|
|
|
|
{
|
2018-01-30 01:26:06 +08:00
|
|
|
kasan_check_write(v, sizeof(*v));
|
2018-01-30 01:26:04 +08:00
|
|
|
return arch_atomic64_add_negative(i, v);
|
|
|
|
|
}
|
atomics/treewide: Make test ops optional
Some of the atomics return the result of a test applied after the atomic
operation, and almost all architectures implement these as trivial
wrappers around the underlying atomic. Specifically:
* <atomic>_inc_and_test(v) is (<atomic>_inc_return(v) == 0)
* <atomic>_dec_and_test(v) is (<atomic>_dec_return(v) == 0)
* <atomic>_sub_and_test(i, v) is (<atomic>_sub_return(i, v) == 0)
* <atomic>_add_negative(i, v) is (<atomic>_add_return(i, v) < 0)
Rather than have these definitions duplicated in all architectures, with
minor inconsistencies in formatting and documentation, let's make these
operations optional, with default fallbacks as above. Implementations
must now provide a preprocessor symbol.
The instrumented atomics are updated accordingly.
Both x86 and m68k have custom implementations, which are left as-is,
given preprocessor symbols to avoid being overridden.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Acked-by: Geert Uytterhoeven <geert@linux-m68k.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Palmer Dabbelt <palmer@sifive.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: https://lore.kernel.org/lkml/20180621121321.4761-16-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-06-21 20:13:18 +08:00
|
|
|
#endif
|
2018-01-30 01:26:04 +08:00
|
|
|
|
2018-07-16 19:30:09 +08:00
|
|
|
#define xchg(ptr, new) \
|
|
|
|
|
({ \
|
|
|
|
|
typeof(ptr) __ai_ptr = (ptr); \
|
|
|
|
|
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
|
|
|
|
|
arch_xchg(__ai_ptr, (new)); \
|
|
|
|
|
})
|
|
|
|
|
|
2018-01-30 01:26:04 +08:00
|
|
|
#define cmpxchg(ptr, old, new) \
|
|
|
|
|
({ \
|
locking/atomics: Simplify cmpxchg() instrumentation
Currently we define some fairly verbose wrappers for the cmpxchg()
family so that we can pass a pointer and size into kasan_check_write().
The wrappers duplicate the size-switching logic necessary in arch code,
and only work for scalar types. On some architectures, (cmp)xchg are
used on non-scalar types, and thus the instrumented wrappers need to be
able to handle this.
We could take the type-punning logic from {READ,WRITE}_ONCE(), but this
makes the wrappers even more verbose, and requires several local
variables in the macros.
Instead, let's simplify the wrappers into simple macros which:
* snapshot the pointer into a single local variable, called __ai_ptr to
avoid conflicts with variables in the scope of the caller.
* call kasan_check_write() on __ai_ptr.
* invoke the relevant arch_*() function, passing the original arguments,
bar __ai_ptr being substituted for ptr.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: andy.shevchenko@gmail.com
Cc: arnd@arndb.de
Cc: aryabinin@virtuozzo.com
Cc: catalin.marinas@arm.com
Cc: glider@google.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: parri.andrea@gmail.com
Cc: peter@hurleysoftware.com
Link: http://lkml.kernel.org/r/20180716113017.3909-4-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-07-16 19:30:08 +08:00
|
|
|
typeof(ptr) __ai_ptr = (ptr); \
|
|
|
|
|
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
|
|
|
|
|
arch_cmpxchg(__ai_ptr, (old), (new)); \
|
2018-01-30 01:26:04 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#define sync_cmpxchg(ptr, old, new) \
|
|
|
|
|
({ \
|
locking/atomics: Simplify cmpxchg() instrumentation
Currently we define some fairly verbose wrappers for the cmpxchg()
family so that we can pass a pointer and size into kasan_check_write().
The wrappers duplicate the size-switching logic necessary in arch code,
and only work for scalar types. On some architectures, (cmp)xchg are
used on non-scalar types, and thus the instrumented wrappers need to be
able to handle this.
We could take the type-punning logic from {READ,WRITE}_ONCE(), but this
makes the wrappers even more verbose, and requires several local
variables in the macros.
Instead, let's simplify the wrappers into simple macros which:
* snapshot the pointer into a single local variable, called __ai_ptr to
avoid conflicts with variables in the scope of the caller.
* call kasan_check_write() on __ai_ptr.
* invoke the relevant arch_*() function, passing the original arguments,
bar __ai_ptr being substituted for ptr.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: andy.shevchenko@gmail.com
Cc: arnd@arndb.de
Cc: aryabinin@virtuozzo.com
Cc: catalin.marinas@arm.com
Cc: glider@google.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: parri.andrea@gmail.com
Cc: peter@hurleysoftware.com
Link: http://lkml.kernel.org/r/20180716113017.3909-4-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-07-16 19:30:08 +08:00
|
|
|
typeof(ptr) __ai_ptr = (ptr); \
|
|
|
|
|
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
|
|
|
|
|
arch_sync_cmpxchg(__ai_ptr, (old), (new)); \
|
2018-01-30 01:26:04 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#define cmpxchg_local(ptr, old, new) \
|
|
|
|
|
({ \
|
locking/atomics: Simplify cmpxchg() instrumentation
Currently we define some fairly verbose wrappers for the cmpxchg()
family so that we can pass a pointer and size into kasan_check_write().
The wrappers duplicate the size-switching logic necessary in arch code,
and only work for scalar types. On some architectures, (cmp)xchg are
used on non-scalar types, and thus the instrumented wrappers need to be
able to handle this.
We could take the type-punning logic from {READ,WRITE}_ONCE(), but this
makes the wrappers even more verbose, and requires several local
variables in the macros.
Instead, let's simplify the wrappers into simple macros which:
* snapshot the pointer into a single local variable, called __ai_ptr to
avoid conflicts with variables in the scope of the caller.
* call kasan_check_write() on __ai_ptr.
* invoke the relevant arch_*() function, passing the original arguments,
bar __ai_ptr being substituted for ptr.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: andy.shevchenko@gmail.com
Cc: arnd@arndb.de
Cc: aryabinin@virtuozzo.com
Cc: catalin.marinas@arm.com
Cc: glider@google.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: parri.andrea@gmail.com
Cc: peter@hurleysoftware.com
Link: http://lkml.kernel.org/r/20180716113017.3909-4-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-07-16 19:30:08 +08:00
|
|
|
typeof(ptr) __ai_ptr = (ptr); \
|
|
|
|
|
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
|
|
|
|
|
arch_cmpxchg_local(__ai_ptr, (old), (new)); \
|
2018-01-30 01:26:04 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#define cmpxchg64(ptr, old, new) \
|
|
|
|
|
({ \
|
locking/atomics: Simplify cmpxchg() instrumentation
Currently we define some fairly verbose wrappers for the cmpxchg()
family so that we can pass a pointer and size into kasan_check_write().
The wrappers duplicate the size-switching logic necessary in arch code,
and only work for scalar types. On some architectures, (cmp)xchg are
used on non-scalar types, and thus the instrumented wrappers need to be
able to handle this.
We could take the type-punning logic from {READ,WRITE}_ONCE(), but this
makes the wrappers even more verbose, and requires several local
variables in the macros.
Instead, let's simplify the wrappers into simple macros which:
* snapshot the pointer into a single local variable, called __ai_ptr to
avoid conflicts with variables in the scope of the caller.
* call kasan_check_write() on __ai_ptr.
* invoke the relevant arch_*() function, passing the original arguments,
bar __ai_ptr being substituted for ptr.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: andy.shevchenko@gmail.com
Cc: arnd@arndb.de
Cc: aryabinin@virtuozzo.com
Cc: catalin.marinas@arm.com
Cc: glider@google.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: parri.andrea@gmail.com
Cc: peter@hurleysoftware.com
Link: http://lkml.kernel.org/r/20180716113017.3909-4-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-07-16 19:30:08 +08:00
|
|
|
typeof(ptr) __ai_ptr = (ptr); \
|
|
|
|
|
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
|
|
|
|
|
arch_cmpxchg64(__ai_ptr, (old), (new)); \
|
2018-01-30 01:26:04 +08:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#define cmpxchg64_local(ptr, old, new) \
|
|
|
|
|
({ \
|
locking/atomics: Simplify cmpxchg() instrumentation
Currently we define some fairly verbose wrappers for the cmpxchg()
family so that we can pass a pointer and size into kasan_check_write().
The wrappers duplicate the size-switching logic necessary in arch code,
and only work for scalar types. On some architectures, (cmp)xchg are
used on non-scalar types, and thus the instrumented wrappers need to be
able to handle this.
We could take the type-punning logic from {READ,WRITE}_ONCE(), but this
makes the wrappers even more verbose, and requires several local
variables in the macros.
Instead, let's simplify the wrappers into simple macros which:
* snapshot the pointer into a single local variable, called __ai_ptr to
avoid conflicts with variables in the scope of the caller.
* call kasan_check_write() on __ai_ptr.
* invoke the relevant arch_*() function, passing the original arguments,
bar __ai_ptr being substituted for ptr.
There should be no functional change as a result of this patch.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Will Deacon <will.deacon@arm.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: andy.shevchenko@gmail.com
Cc: arnd@arndb.de
Cc: aryabinin@virtuozzo.com
Cc: catalin.marinas@arm.com
Cc: glider@google.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: parri.andrea@gmail.com
Cc: peter@hurleysoftware.com
Link: http://lkml.kernel.org/r/20180716113017.3909-4-mark.rutland@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-07-16 19:30:08 +08:00
|
|
|
typeof(ptr) __ai_ptr = (ptr); \
|
|
|
|
|
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
|
|
|
|
|
arch_cmpxchg64_local(__ai_ptr, (old), (new)); \
|
2018-01-30 01:26:04 +08:00
|
|
|
})
|
|
|
|
|
|
2018-01-30 01:26:07 +08:00
|
|
|
/*
|
|
|
|
|
* Originally we had the following code here:
|
|
|
|
|
* __typeof__(p1) ____p1 = (p1);
|
|
|
|
|
* kasan_check_write(____p1, 2 * sizeof(*____p1));
|
|
|
|
|
* arch_cmpxchg_double(____p1, (p2), (o1), (o2), (n1), (n2));
|
|
|
|
|
* But it leads to compilation failures (see gcc issue 72873).
|
|
|
|
|
* So for now it's left non-instrumented.
|
|
|
|
|
* There are few callers of cmpxchg_double(), so it's not critical.
|
|
|
|
|
*/
|
2018-01-30 01:26:04 +08:00
|
|
|
#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
|
|
|
|
|
({ \
|
|
|
|
|
arch_cmpxchg_double((p1), (p2), (o1), (o2), (n1), (n2)); \
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
|
|
|
|
|
({ \
|
|
|
|
|
arch_cmpxchg_double_local((p1), (p2), (o1), (o2), (n1), (n2)); \
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
|