arm64: locks: introduce ticket-based spinlock implementation

This patch introduces a ticket lock implementation for arm64, along the
same lines as the implementation for arch/arm/.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

arch/arm64/include/asm/spinlock.h

@@ -22,17 +22,10 @@
 /*
  * Spinlock implementation.
  *
- * The old value is read exclusively and the new one, if unlocked, is written
- * exclusively. In case of failure, the loop is restarted.
- *
  * The memory barriers are implicit with the load-acquire and store-release
  * instructions.
- *
- * Unlocked value: 0
- * Locked value: 1
  */
 
-#define arch_spin_is_locked(x)		((x)->lock != 0)
 #define arch_spin_unlock_wait(lock) \
 	do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0)
 
@@ -41,32 +34,51 @@
 static inline void arch_spin_lock(arch_spinlock_t *lock)
 {
 	unsigned int tmp;
+	arch_spinlock_t lockval, newval;
 
 	asm volatile(
+	/* Atomically increment the next ticket. */
+"	prfm	pstl1strm, %3\n"
+"1:	ldaxr	%w0, %3\n"
+"	add	%w1, %w0, %w5\n"
+"	stxr	%w2, %w1, %3\n"
+"	cbnz	%w2, 1b\n"
+	/* Did we get the lock? */
+"	eor	%w1, %w0, %w0, ror #16\n"
+"	cbz	%w1, 3f\n"
+	/*
+	 * No: spin on the owner. Send a local event to avoid missing an
+	 * unlock before the exclusive load.
+	 */
 "	sevl\n"
-	"1:	wfe\n"
+"2:	wfe\n"
-	"2:	ldaxr	%w0, %1\n"
+"	ldaxrh	%w2, %4\n"
-	"	cbnz	%w0, 1b\n"
+"	eor	%w1, %w2, %w0, lsr #16\n"
-	"	stxr	%w0, %w2, %1\n"
+"	cbnz	%w1, 2b\n"
-	"	cbnz	%w0, 2b\n"
+	/* We got the lock. Critical section starts here. */
-	: "=&r" (tmp), "+Q" (lock->lock)
+"3:"
-	: "r" (1)
+	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
-	: "cc", "memory");
+	: "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
+	: "memory");
 }
 
 static inline int arch_spin_trylock(arch_spinlock_t *lock)
 {
 	unsigned int tmp;
+	arch_spinlock_t lockval;
 
 	asm volatile(
-	"2:	ldaxr	%w0, %1\n"
+"	prfm	pstl1strm, %2\n"
-	"	cbnz	%w0, 1f\n"
+"1:	ldaxr	%w0, %2\n"
-	"	stxr	%w0, %w2, %1\n"
+"	eor	%w1, %w0, %w0, ror #16\n"
-	"	cbnz	%w0, 2b\n"
+"	cbnz	%w1, 2f\n"
-	"1:\n"
+"	add	%w0, %w0, %3\n"
-	: "=&r" (tmp), "+Q" (lock->lock)
+"	stxr	%w1, %w0, %2\n"
-	: "r" (1)
+"	cbnz	%w1, 1b\n"
-	: "cc", "memory");
+"2:"
+	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
+	: "I" (1 << TICKET_SHIFT)
+	: "memory");
 
 	return !tmp;
 }
@@ -74,10 +86,25 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock)
 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 {
 	asm volatile(
-	"	stlr	%w1, %0\n"
+"	stlrh	%w1, %0\n"
-	: "=Q" (lock->lock) : "r" (0) : "memory");
+	: "=Q" (lock->owner)
+	: "r" (lock->owner + 1)
+	: "memory");
 }
 
+static inline int arch_spin_is_locked(arch_spinlock_t *lock)
+{
+	arch_spinlock_t lockval = ACCESS_ONCE(*lock);
+	return lockval.owner != lockval.next;
+}
+
+static inline int arch_spin_is_contended(arch_spinlock_t *lock)
+{
+	arch_spinlock_t lockval = ACCESS_ONCE(*lock);
+	return (lockval.next - lockval.owner) > 1;
+}
+#define arch_spin_is_contended	arch_spin_is_contended
+
 /*
  * Write lock implementation.
  *

arch/arm64/include/asm/spinlock_types.h

@@ -20,14 +20,14 @@
 # error "please don't include this file directly"
 #endif
 
-/* We only require natural alignment for exclusive accesses. */
+#define TICKET_SHIFT	16
-#define __lock_aligned
 
 typedef struct {
-	volatile unsigned int lock;
+	u16 owner;
-} arch_spinlock_t;
+	u16 next;
+} __aligned(4) arch_spinlock_t;
 
-#define __ARCH_SPIN_LOCK_UNLOCKED	{ 0 }
+#define __ARCH_SPIN_LOCK_UNLOCKED	{ 0 , 0 }
 
 typedef struct {
 	volatile unsigned int lock;