powerpc/qspinlock: convert atomic operations to assembly (b3a73b7d) · Commits · EulixOS / Software / Kernel

arch/powerpc/include/asm/qspinlock.h

+19 −5

Original line number	Original line	Diff line number	Diff line
	@@ -2,29 +2,43 @@
	#ifndef _ASM_POWERPC_QSPINLOCK_H		#ifndef _ASM_POWERPC_QSPINLOCK_H
	#define _ASM_POWERPC_QSPINLOCK_H		#define _ASM_POWERPC_QSPINLOCK_H

	#include <linux/atomic.h>
	#include <linux/compiler.h>		#include <linux/compiler.h>
	#include <asm/qspinlock_types.h>		#include <asm/qspinlock_types.h>
	#include <asm/paravirt.h>		#include <asm/paravirt.h>

	static __always_inline int queued_spin_is_locked(struct qspinlock *lock)		static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
	{		{
	return atomic_read(&lock->val);		return READ_ONCE(lock->val);
	}		}

	static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)		static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)
	{		{
	return !atomic_read(&lock.val);		return !lock.val;
	}		}

	static __always_inline int queued_spin_is_contended(struct qspinlock *lock)		static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
	{		{
	return !!(atomic_read(&lock->val) & _Q_TAIL_CPU_MASK);		return !!(READ_ONCE(lock->val) & _Q_TAIL_CPU_MASK);
	}		}

	static __always_inline int queued_spin_trylock(struct qspinlock *lock)		static __always_inline int queued_spin_trylock(struct qspinlock *lock)
	{		{
	return atomic_cmpxchg_acquire(&lock->val, 0, _Q_LOCKED_VAL) == 0;		u32 prev;

			asm volatile(
			"1: lwarx %0,0,%1,%3 # queued_spin_trylock \n"
			" cmpwi 0,%0,0 \n"
			" bne- 2f \n"
			" stwcx. %2,0,%1 \n"
			" bne- 1b \n"
			"\t" PPC_ACQUIRE_BARRIER " \n"
			"2: \n"
			: "=&r" (prev)
			: "r" (&lock->val), "r" (_Q_LOCKED_VAL),
			"i" (IS_ENABLED(CONFIG_PPC64))
			: "cr0", "memory");

			return likely(prev == 0);
	}		}

	void queued_spin_lock_slowpath(struct qspinlock *lock);		void queued_spin_lock_slowpath(struct qspinlock *lock);

arch/powerpc/include/asm/qspinlock_types.h

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -7,7 +7,7 @@

	typedef struct qspinlock {		typedef struct qspinlock {
	union {		union {
	atomic_t val;		u32 val;

	#ifdef __LITTLE_ENDIAN		#ifdef __LITTLE_ENDIAN
	struct {		struct {
	@@ -23,7 +23,7 @@ typedef struct qspinlock {
	};		};
	} arch_spinlock_t;		} arch_spinlock_t;

	#define __ARCH_SPIN_LOCK_UNLOCKED { { .val = ATOMIC_INIT(0) } }		#define __ARCH_SPIN_LOCK_UNLOCKED { { .val = 0 } }

	/*		/*
	* Bitfields in the lock word:		* Bitfields in the lock word:

arch/powerpc/lib/qspinlock.c

+47 −35

Original line number	Original line	Diff line number	Diff line
	// SPDX-License-Identifier: GPL-2.0-or-later		// SPDX-License-Identifier: GPL-2.0-or-later
	#include <linux/atomic.h>
	#include <linux/bug.h>		#include <linux/bug.h>
	#include <linux/compiler.h>		#include <linux/compiler.h>
	#include <linux/export.h>		#include <linux/export.h>
	@@ -22,12 +21,12 @@ struct qnodes {

	static DEFINE_PER_CPU_ALIGNED(struct qnodes, qnodes);		static DEFINE_PER_CPU_ALIGNED(struct qnodes, qnodes);

	static inline int encode_tail_cpu(int cpu)		static inline u32 encode_tail_cpu(int cpu)
	{		{
	return (cpu + 1) << _Q_TAIL_CPU_OFFSET;		return (cpu + 1) << _Q_TAIL_CPU_OFFSET;
	}		}

	static inline int decode_tail_cpu(int val)		static inline int decode_tail_cpu(u32 val)
	{		{
	return (val >> _Q_TAIL_CPU_OFFSET) - 1;		return (val >> _Q_TAIL_CPU_OFFSET) - 1;
	}		}
	@@ -39,26 +38,34 @@ static inline int decode_tail_cpu(int val)
	* This is used by the head of the queue to acquire the lock and clean up		* This is used by the head of the queue to acquire the lock and clean up
	* its tail if it was the last one queued.		* its tail if it was the last one queued.
	*/		*/
	static __always_inline int set_locked_clean_tail(struct qspinlock *lock, int tail)		static __always_inline u32 set_locked_clean_tail(struct qspinlock *lock, u32 tail)
	{		{
	int val = atomic_read(&lock->val);		u32 newval = _Q_LOCKED_VAL;
			u32 prev, tmp;
	BUG_ON(val & _Q_LOCKED_VAL);
			asm volatile(
	/* If we're the last queued, must clean up the tail. */		"1: lwarx %0,0,%2,%6 # set_locked_clean_tail \n"
	if ((val & _Q_TAIL_CPU_MASK) == tail) {		/* Test whether the lock tail == tail */
	if (atomic_cmpxchg_acquire(&lock->val, val, _Q_LOCKED_VAL) == val)		" and %1,%0,%5 \n"
	return val;		" cmpw 0,%1,%3 \n"
	/* Another waiter must have enqueued */		/* Merge the new locked value */
	val = atomic_read(&lock->val);		" or %1,%1,%4 \n"
	BUG_ON(val & _Q_LOCKED_VAL);		" bne 2f \n"
	}		/* If the lock tail matched, then clear it, otherwise leave it. */
			" andc %1,%1,%5 \n"
	/* We must be the owner, just set the lock bit and acquire */		"2: stwcx. %1,0,%2 \n"
	atomic_or(_Q_LOCKED_VAL, &lock->val);		" bne- 1b \n"
	__atomic_acquire_fence();		"\t" PPC_ACQUIRE_BARRIER " \n"
			"3: \n"
	return val;		: "=&r" (prev), "=&r" (tmp)
			: "r" (&lock->val), "r"(tail), "r" (newval),
			"r" (_Q_TAIL_CPU_MASK),
			"i" (IS_ENABLED(CONFIG_PPC64))
			: "cr0", "memory");

			BUG_ON(prev & _Q_LOCKED_VAL);

			return prev;
	}		}

	/*		/*
	@@ -68,20 +75,25 @@ static __always_inline int set_locked_clean_tail(struct qspinlock *lock, int tai
	* acquire barrier in get_tail_qnode() when the next CPU finds this tail		* acquire barrier in get_tail_qnode() when the next CPU finds this tail
	* value.		* value.
	*/		*/
	static __always_inline int publish_tail_cpu(struct qspinlock *lock, int tail)		static __always_inline u32 publish_tail_cpu(struct qspinlock *lock, u32 tail)
	{		{
	for (;;) {		u32 prev, tmp;
	int val = atomic_read(&lock->val);
	int newval = (val & ~_Q_TAIL_CPU_MASK) \| tail;		asm volatile(
	int old;		"\t" PPC_RELEASE_BARRIER " \n"
			"1: lwarx %0,0,%2 # publish_tail_cpu \n"
	old = atomic_cmpxchg_release(&lock->val, val, newval);		" andc %1,%0,%4 \n"
	if (old == val)		" or %1,%1,%3 \n"
	return old;		" stwcx. %1,0,%2 \n"
	}		" bne- 1b \n"
			: "=&r" (prev), "=&r"(tmp)
			: "r" (&lock->val), "r" (tail), "r"(_Q_TAIL_CPU_MASK)
			: "cr0", "memory");

			return prev;
	}		}

	static struct qnode get_tail_qnode(struct qspinlock lock, int val)		static struct qnode get_tail_qnode(struct qspinlock lock, u32 val)
	{		{
	int cpu = decode_tail_cpu(val);		int cpu = decode_tail_cpu(val);
	struct qnodes *qnodesp = per_cpu_ptr(&qnodes, cpu);		struct qnodes *qnodesp = per_cpu_ptr(&qnodes, cpu);
	@@ -109,7 +121,7 @@ static inline void queued_spin_lock_mcs_queue(struct qspinlock *lock)
	{		{
	struct qnodes *qnodesp;		struct qnodes *qnodesp;
	struct qnode next, node;		struct qnode next, node;
	int val, old, tail;		u32 val, old, tail;
	int idx;		int idx;

	BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));		BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
	@@ -156,7 +168,7 @@ static inline void queued_spin_lock_mcs_queue(struct qspinlock *lock)

	/* We're at the head of the waitqueue, wait for the lock. */		/* We're at the head of the waitqueue, wait for the lock. */
	for (;;) {		for (;;) {
	val = atomic_read(&lock->val);		val = READ_ONCE(lock->val);
	if (!(val & _Q_LOCKED_VAL))		if (!(val & _Q_LOCKED_VAL))
	break;		break;