Commit d4a72e7f authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Dave Hansen
Browse files

x86/mm/pae: Get rid of set_64bit() 



Recognise that set_64bit() is a special case of our previously
introduced pxx_xchg64(), so use that and get rid of set_64bit().

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20221022114425.233481884%40infradead.org
parent 9ee850ac

arch/x86/include/asm/cmpxchg_32.h

+0 −28
Original line number Diff line number Diff line
@@ -7,34 +7,6 @@ 
 *       you need to test for the feature in boot_cpu_data.

 */



/*

 * CMPXCHG8B only writes to the target if we had the previous

 * value in registers, otherwise it acts as a read and gives us the

 * "new previous" value.  That is why there is a loop.  Preloading

 * EDX:EAX is a performance optimization: in the common case it means

 * we need only one locked operation.

 *

 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very

 * least an FPU save and/or %cr0.ts manipulation.

 *

 * cmpxchg8b must be used with the lock prefix here to allow the

 * instruction to be executed atomically.  We need to have the reader

 * side to see the coherent 64bit value.

 */

static inline void set_64bit(volatile u64 *ptr, u64 value)

{

	u32 low  = value;

	u32 high = value >> 32;

	u64 prev = *ptr;



	asm volatile("\n1:\t"

		     LOCK_PREFIX "cmpxchg8b %0\n\t"

		     "jnz 1b"

		     : "=m" (*ptr), "+A" (prev)

		     : "b" (low), "c" (high)

		     : "memory");

}



#ifdef CONFIG_X86_CMPXCHG64

#define arch_cmpxchg64(ptr, o, n)					\

	((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \

arch/x86/include/asm/pgtable-3level.h

+12 −11
Original line number Diff line number Diff line
@@ -19,7 +19,15 @@ 
	pr_err("%s:%d: bad pgd %p(%016Lx)\n",				\

	       __FILE__, __LINE__, &(e), pgd_val(e))



/* Rules for using set_pte: the pte being assigned *must* be

#define pxx_xchg64(_pxx, _ptr, _val) ({					\

	_pxx##val_t *_p = (_pxx##val_t *)_ptr;				\

	_pxx##val_t _o = *_p;						\

	do { } while (!try_cmpxchg64(_p, &_o, (_val)));			\

	native_make_##_pxx(_o);						\

})



/*

 * Rules for using set_pte: the pte being assigned *must* be

 * either not present or in a state where the hardware will

 * not attempt to update the pte.  In places where this is

 * not possible, use pte_get_and_clear to obtain the old pte
@@ -34,12 +42,12 @@ static inline void native_set_pte(pte_t *ptep, pte_t pte) 


static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)

{

	set_64bit((unsigned long long *)(ptep), native_pte_val(pte));

	pxx_xchg64(pte, ptep, native_pte_val(pte));

}



static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)

{

	set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd));

	pxx_xchg64(pmd, pmdp, native_pmd_val(pmd));

}



static inline void native_set_pud(pud_t *pudp, pud_t pud)
@@ -47,7 +55,7 @@ static inline void native_set_pud(pud_t *pudp, pud_t pud) 
#ifdef CONFIG_PAGE_TABLE_ISOLATION

	pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd);

#endif

	set_64bit((unsigned long long *)(pudp), native_pud_val(pud));

	pxx_xchg64(pud, pudp, native_pud_val(pud));

}



/*
@@ -91,13 +99,6 @@ static inline void pud_clear(pud_t *pudp) 
}





#define pxx_xchg64(_pxx, _ptr, _val) ({					\

	_pxx##val_t *_p = (_pxx##val_t *)_ptr;				\

	_pxx##val_t _o = *_p;						\

	do { } while (!try_cmpxchg64(_p, &_o, (_val)));			\

	native_make_##_pxx(_o);						\

})



#ifdef CONFIG_SMP

static inline pte_t native_ptep_get_and_clear(pte_t *ptep)

{