2012-03-29 01:30:01 +08:00
|
|
|
#ifndef __ASM_BARRIER_H
|
|
|
|
|
#define __ASM_BARRIER_H
|
|
|
|
|
|
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
|
|
|
|
#define nop() __asm__ __volatile__("mov\tr0,r0\t@ nop\n\t");
|
|
|
|
|
|
|
|
|
|
#if __LINUX_ARM_ARCH__ >= 7 || \
|
|
|
|
|
(__LINUX_ARM_ARCH__ == 6 && defined(CONFIG_CPU_32v6K))
|
|
|
|
|
#define sev() __asm__ __volatile__ ("sev" : : : "memory")
|
|
|
|
|
#define wfe() __asm__ __volatile__ ("wfe" : : : "memory")
|
|
|
|
|
#define wfi() __asm__ __volatile__ ("wfi" : : : "memory")
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if __LINUX_ARM_ARCH__ >= 7
|
2013-05-11 01:07:19 +08:00
|
|
|
#define isb(option) __asm__ __volatile__ ("isb " #option : : : "memory")
|
|
|
|
|
#define dsb(option) __asm__ __volatile__ ("dsb " #option : : : "memory")
|
|
|
|
|
#define dmb(option) __asm__ __volatile__ ("dmb " #option : : : "memory")
|
2012-03-29 01:30:01 +08:00
|
|
|
#elif defined(CONFIG_CPU_XSC3) || __LINUX_ARM_ARCH__ == 6
|
2013-05-11 01:07:19 +08:00
|
|
|
#define isb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \
|
2012-03-29 01:30:01 +08:00
|
|
|
: : "r" (0) : "memory")
|
2013-05-11 01:07:19 +08:00
|
|
|
#define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \
|
2012-03-29 01:30:01 +08:00
|
|
|
: : "r" (0) : "memory")
|
2013-05-11 01:07:19 +08:00
|
|
|
#define dmb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 5" \
|
2012-03-29 01:30:01 +08:00
|
|
|
: : "r" (0) : "memory")
|
|
|
|
|
#elif defined(CONFIG_CPU_FA526)
|
2013-05-11 01:07:19 +08:00
|
|
|
#define isb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \
|
2012-03-29 01:30:01 +08:00
|
|
|
: : "r" (0) : "memory")
|
2013-05-11 01:07:19 +08:00
|
|
|
#define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \
|
2012-03-29 01:30:01 +08:00
|
|
|
: : "r" (0) : "memory")
|
2013-05-11 01:07:19 +08:00
|
|
|
#define dmb(x) __asm__ __volatile__ ("" : : : "memory")
|
2012-03-29 01:30:01 +08:00
|
|
|
#else
|
2013-05-11 01:07:19 +08:00
|
|
|
#define isb(x) __asm__ __volatile__ ("" : : : "memory")
|
|
|
|
|
#define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \
|
2012-03-29 01:30:01 +08:00
|
|
|
: : "r" (0) : "memory")
|
2013-05-11 01:07:19 +08:00
|
|
|
#define dmb(x) __asm__ __volatile__ ("" : : : "memory")
|
2012-03-29 01:30:01 +08:00
|
|
|
#endif
|
|
|
|
|
|
ARM: move heavy barrier support out of line
The existing memory barrier macro causes a significant amount of code
to be inserted inline at every call site. For example, in
gpio_set_irq_type(), we have this for mb():
c0344c08: f57ff04e dsb st
c0344c0c: e59f8190 ldr r8, [pc, #400] ; c0344da4 <gpio_set_irq_type+0x230>
c0344c10: e3590004 cmp r9, #4
c0344c14: e5983014 ldr r3, [r8, #20]
c0344c18: 0a000054 beq c0344d70 <gpio_set_irq_type+0x1fc>
c0344c1c: e3530000 cmp r3, #0
c0344c20: 0a000004 beq c0344c38 <gpio_set_irq_type+0xc4>
c0344c24: e50b2030 str r2, [fp, #-48] ; 0xffffffd0
c0344c28: e50bc034 str ip, [fp, #-52] ; 0xffffffcc
c0344c2c: e12fff33 blx r3
c0344c30: e51bc034 ldr ip, [fp, #-52] ; 0xffffffcc
c0344c34: e51b2030 ldr r2, [fp, #-48] ; 0xffffffd0
c0344c38: e5963004 ldr r3, [r6, #4]
Moving the outer_cache_sync() call out of line reduces the impact of
the barrier:
c0344968: f57ff04e dsb st
c034496c: e35a0004 cmp sl, #4
c0344970: e50b2030 str r2, [fp, #-48] ; 0xffffffd0
c0344974: 0a000044 beq c0344a8c <gpio_set_irq_type+0x1b8>
c0344978: ebf363dd bl c001d8f4 <arm_heavy_mb>
c034497c: e5953004 ldr r3, [r5, #4]
This should reduce the cache footprint of this code. Overall, this
results in a reduction of around 20K in the kernel size:
text data bss dec hex filename
10773970 667392 10369656 21811018 14ccf4a ../build/imx6/vmlinux-old
10754219 667392 10369656 21791267 14c8223 ../build/imx6/vmlinux-new
Another advantage to this approach is that we can finally resolve the
issue of SoCs which have their own memory barrier requirements within
multiplatform kernels (such as OMAP.) Here, the bus interconnects
need additional handling to ensure that writes become visible in the
correct order (eg, between dma_map() operations, writes to DMA
coherent memory, and MMIO accesses.)
Acked-by: Tony Lindgren <tony@atomide.com>
Acked-by: Richard Woodruff <r-woodruff2@ti.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2015-06-02 06:44:46 +08:00
|
|
|
#ifdef CONFIG_ARM_HEAVY_MB
|
2015-06-03 20:10:16 +08:00
|
|
|
extern void (*soc_mb)(void);
|
ARM: move heavy barrier support out of line
The existing memory barrier macro causes a significant amount of code
to be inserted inline at every call site. For example, in
gpio_set_irq_type(), we have this for mb():
c0344c08: f57ff04e dsb st
c0344c0c: e59f8190 ldr r8, [pc, #400] ; c0344da4 <gpio_set_irq_type+0x230>
c0344c10: e3590004 cmp r9, #4
c0344c14: e5983014 ldr r3, [r8, #20]
c0344c18: 0a000054 beq c0344d70 <gpio_set_irq_type+0x1fc>
c0344c1c: e3530000 cmp r3, #0
c0344c20: 0a000004 beq c0344c38 <gpio_set_irq_type+0xc4>
c0344c24: e50b2030 str r2, [fp, #-48] ; 0xffffffd0
c0344c28: e50bc034 str ip, [fp, #-52] ; 0xffffffcc
c0344c2c: e12fff33 blx r3
c0344c30: e51bc034 ldr ip, [fp, #-52] ; 0xffffffcc
c0344c34: e51b2030 ldr r2, [fp, #-48] ; 0xffffffd0
c0344c38: e5963004 ldr r3, [r6, #4]
Moving the outer_cache_sync() call out of line reduces the impact of
the barrier:
c0344968: f57ff04e dsb st
c034496c: e35a0004 cmp sl, #4
c0344970: e50b2030 str r2, [fp, #-48] ; 0xffffffd0
c0344974: 0a000044 beq c0344a8c <gpio_set_irq_type+0x1b8>
c0344978: ebf363dd bl c001d8f4 <arm_heavy_mb>
c034497c: e5953004 ldr r3, [r5, #4]
This should reduce the cache footprint of this code. Overall, this
results in a reduction of around 20K in the kernel size:
text data bss dec hex filename
10773970 667392 10369656 21811018 14ccf4a ../build/imx6/vmlinux-old
10754219 667392 10369656 21791267 14c8223 ../build/imx6/vmlinux-new
Another advantage to this approach is that we can finally resolve the
issue of SoCs which have their own memory barrier requirements within
multiplatform kernels (such as OMAP.) Here, the bus interconnects
need additional handling to ensure that writes become visible in the
correct order (eg, between dma_map() operations, writes to DMA
coherent memory, and MMIO accesses.)
Acked-by: Tony Lindgren <tony@atomide.com>
Acked-by: Richard Woodruff <r-woodruff2@ti.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2015-06-02 06:44:46 +08:00
|
|
|
extern void arm_heavy_mb(void);
|
|
|
|
|
#define __arm_heavy_mb(x...) do { dsb(x); arm_heavy_mb(); } while (0)
|
|
|
|
|
#else
|
|
|
|
|
#define __arm_heavy_mb(x...) dsb(x)
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-03-29 01:30:01 +08:00
|
|
|
#ifdef CONFIG_ARCH_HAS_BARRIERS
|
|
|
|
|
#include <mach/barriers.h>
|
|
|
|
|
#elif defined(CONFIG_ARM_DMA_MEM_BUFFERABLE) || defined(CONFIG_SMP)
|
ARM: move heavy barrier support out of line
The existing memory barrier macro causes a significant amount of code
to be inserted inline at every call site. For example, in
gpio_set_irq_type(), we have this for mb():
c0344c08: f57ff04e dsb st
c0344c0c: e59f8190 ldr r8, [pc, #400] ; c0344da4 <gpio_set_irq_type+0x230>
c0344c10: e3590004 cmp r9, #4
c0344c14: e5983014 ldr r3, [r8, #20]
c0344c18: 0a000054 beq c0344d70 <gpio_set_irq_type+0x1fc>
c0344c1c: e3530000 cmp r3, #0
c0344c20: 0a000004 beq c0344c38 <gpio_set_irq_type+0xc4>
c0344c24: e50b2030 str r2, [fp, #-48] ; 0xffffffd0
c0344c28: e50bc034 str ip, [fp, #-52] ; 0xffffffcc
c0344c2c: e12fff33 blx r3
c0344c30: e51bc034 ldr ip, [fp, #-52] ; 0xffffffcc
c0344c34: e51b2030 ldr r2, [fp, #-48] ; 0xffffffd0
c0344c38: e5963004 ldr r3, [r6, #4]
Moving the outer_cache_sync() call out of line reduces the impact of
the barrier:
c0344968: f57ff04e dsb st
c034496c: e35a0004 cmp sl, #4
c0344970: e50b2030 str r2, [fp, #-48] ; 0xffffffd0
c0344974: 0a000044 beq c0344a8c <gpio_set_irq_type+0x1b8>
c0344978: ebf363dd bl c001d8f4 <arm_heavy_mb>
c034497c: e5953004 ldr r3, [r5, #4]
This should reduce the cache footprint of this code. Overall, this
results in a reduction of around 20K in the kernel size:
text data bss dec hex filename
10773970 667392 10369656 21811018 14ccf4a ../build/imx6/vmlinux-old
10754219 667392 10369656 21791267 14c8223 ../build/imx6/vmlinux-new
Another advantage to this approach is that we can finally resolve the
issue of SoCs which have their own memory barrier requirements within
multiplatform kernels (such as OMAP.) Here, the bus interconnects
need additional handling to ensure that writes become visible in the
correct order (eg, between dma_map() operations, writes to DMA
coherent memory, and MMIO accesses.)
Acked-by: Tony Lindgren <tony@atomide.com>
Acked-by: Richard Woodruff <r-woodruff2@ti.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2015-06-02 06:44:46 +08:00
|
|
|
#define mb() __arm_heavy_mb()
|
2012-03-29 01:30:01 +08:00
|
|
|
#define rmb() dsb()
|
ARM: move heavy barrier support out of line
The existing memory barrier macro causes a significant amount of code
to be inserted inline at every call site. For example, in
gpio_set_irq_type(), we have this for mb():
c0344c08: f57ff04e dsb st
c0344c0c: e59f8190 ldr r8, [pc, #400] ; c0344da4 <gpio_set_irq_type+0x230>
c0344c10: e3590004 cmp r9, #4
c0344c14: e5983014 ldr r3, [r8, #20]
c0344c18: 0a000054 beq c0344d70 <gpio_set_irq_type+0x1fc>
c0344c1c: e3530000 cmp r3, #0
c0344c20: 0a000004 beq c0344c38 <gpio_set_irq_type+0xc4>
c0344c24: e50b2030 str r2, [fp, #-48] ; 0xffffffd0
c0344c28: e50bc034 str ip, [fp, #-52] ; 0xffffffcc
c0344c2c: e12fff33 blx r3
c0344c30: e51bc034 ldr ip, [fp, #-52] ; 0xffffffcc
c0344c34: e51b2030 ldr r2, [fp, #-48] ; 0xffffffd0
c0344c38: e5963004 ldr r3, [r6, #4]
Moving the outer_cache_sync() call out of line reduces the impact of
the barrier:
c0344968: f57ff04e dsb st
c034496c: e35a0004 cmp sl, #4
c0344970: e50b2030 str r2, [fp, #-48] ; 0xffffffd0
c0344974: 0a000044 beq c0344a8c <gpio_set_irq_type+0x1b8>
c0344978: ebf363dd bl c001d8f4 <arm_heavy_mb>
c034497c: e5953004 ldr r3, [r5, #4]
This should reduce the cache footprint of this code. Overall, this
results in a reduction of around 20K in the kernel size:
text data bss dec hex filename
10773970 667392 10369656 21811018 14ccf4a ../build/imx6/vmlinux-old
10754219 667392 10369656 21791267 14c8223 ../build/imx6/vmlinux-new
Another advantage to this approach is that we can finally resolve the
issue of SoCs which have their own memory barrier requirements within
multiplatform kernels (such as OMAP.) Here, the bus interconnects
need additional handling to ensure that writes become visible in the
correct order (eg, between dma_map() operations, writes to DMA
coherent memory, and MMIO accesses.)
Acked-by: Tony Lindgren <tony@atomide.com>
Acked-by: Richard Woodruff <r-woodruff2@ti.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
2015-06-02 06:44:46 +08:00
|
|
|
#define wmb() __arm_heavy_mb(st)
|
2014-12-12 07:02:06 +08:00
|
|
|
#define dma_rmb() dmb(osh)
|
|
|
|
|
#define dma_wmb() dmb(oshst)
|
2012-03-29 01:30:01 +08:00
|
|
|
#else
|
2012-08-21 18:26:24 +08:00
|
|
|
#define mb() barrier()
|
|
|
|
|
#define rmb() barrier()
|
|
|
|
|
#define wmb() barrier()
|
2014-12-12 07:02:06 +08:00
|
|
|
#define dma_rmb() barrier()
|
|
|
|
|
#define dma_wmb() barrier()
|
2012-03-29 01:30:01 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
|
#define smp_mb() barrier()
|
|
|
|
|
#define smp_rmb() barrier()
|
|
|
|
|
#define smp_wmb() barrier()
|
|
|
|
|
#else
|
2013-05-11 01:07:19 +08:00
|
|
|
#define smp_mb() dmb(ish)
|
|
|
|
|
#define smp_rmb() smp_mb()
|
|
|
|
|
#define smp_wmb() dmb(ishst)
|
2012-03-29 01:30:01 +08:00
|
|
|
#endif
|
|
|
|
|
|
2013-11-06 21:57:36 +08:00
|
|
|
#define smp_store_release(p, v) \
|
|
|
|
|
do { \
|
|
|
|
|
compiletime_assert_atomic_type(*p); \
|
|
|
|
|
smp_mb(); \
|
locking, arch: use WRITE_ONCE()/READ_ONCE() in smp_store_release()/smp_load_acquire()
Replace ACCESS_ONCE() macro in smp_store_release() and smp_load_acquire()
with WRITE_ONCE() and READ_ONCE() on x86, arm, arm64, ia64, metag, mips,
powerpc, s390, sparc and asm-generic since ACCESS_ONCE() does not work
reliably on non-scalar types.
WRITE_ONCE() and READ_ONCE() were introduced in the following commits:
230fa253df63 ("kernel: Provide READ_ONCE and ASSIGN_ONCE")
43239cbe79fc ("kernel: Change ASSIGN_ONCE(val, x) to WRITE_ONCE(x, val)")
Signed-off-by: Andrey Konovalov <andreyknvl@google.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Davidlohr Bueso <dbueso@suse.de>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc)
Acked-by: Ralf Baechle <ralf@linux-mips.org>
Cc: Alexander Duyck <alexander.h.duyck@redhat.com>
Cc: Andre Przywara <andre.przywara@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Borislav Petkov <bp@suse.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: James Hogan <james.hogan@imgtec.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: linux-arch@vger.kernel.org
Link: http://lkml.kernel.org/r/1438528264-714-1-git-send-email-andreyknvl@google.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-08-02 23:11:04 +08:00
|
|
|
WRITE_ONCE(*p, v); \
|
2013-11-06 21:57:36 +08:00
|
|
|
} while (0)
|
|
|
|
|
|
|
|
|
|
#define smp_load_acquire(p) \
|
|
|
|
|
({ \
|
locking, arch: use WRITE_ONCE()/READ_ONCE() in smp_store_release()/smp_load_acquire()
Replace ACCESS_ONCE() macro in smp_store_release() and smp_load_acquire()
with WRITE_ONCE() and READ_ONCE() on x86, arm, arm64, ia64, metag, mips,
powerpc, s390, sparc and asm-generic since ACCESS_ONCE() does not work
reliably on non-scalar types.
WRITE_ONCE() and READ_ONCE() were introduced in the following commits:
230fa253df63 ("kernel: Provide READ_ONCE and ASSIGN_ONCE")
43239cbe79fc ("kernel: Change ASSIGN_ONCE(val, x) to WRITE_ONCE(x, val)")
Signed-off-by: Andrey Konovalov <andreyknvl@google.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Davidlohr Bueso <dbueso@suse.de>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc)
Acked-by: Ralf Baechle <ralf@linux-mips.org>
Cc: Alexander Duyck <alexander.h.duyck@redhat.com>
Cc: Andre Przywara <andre.przywara@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Borislav Petkov <bp@suse.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: James Hogan <james.hogan@imgtec.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: linux-arch@vger.kernel.org
Link: http://lkml.kernel.org/r/1438528264-714-1-git-send-email-andreyknvl@google.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-08-02 23:11:04 +08:00
|
|
|
typeof(*p) ___p1 = READ_ONCE(*p); \
|
2013-11-06 21:57:36 +08:00
|
|
|
compiletime_assert_atomic_type(*p); \
|
|
|
|
|
smp_mb(); \
|
|
|
|
|
___p1; \
|
|
|
|
|
})
|
|
|
|
|
|
2012-03-29 01:30:01 +08:00
|
|
|
#define read_barrier_depends() do { } while(0)
|
|
|
|
|
#define smp_read_barrier_depends() do { } while(0)
|
|
|
|
|
|
2015-05-12 16:51:55 +08:00
|
|
|
#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
|
2012-03-29 01:30:01 +08:00
|
|
|
|
2014-03-13 00:11:00 +08:00
|
|
|
#define smp_mb__before_atomic() smp_mb()
|
|
|
|
|
#define smp_mb__after_atomic() smp_mb()
|
|
|
|
|
|
2012-03-29 01:30:01 +08:00
|
|
|
#endif /* !__ASSEMBLY__ */
|
|
|
|
|
#endif /* __ASM_BARRIER_H */
|