Commit fbfdec99 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Dave Hansen
Browse files

x86/mm/pae: Make pmd_t similar to pte_t 



Instead of mucking about with at least 2 different ways of fudging
it, do the same thing we do for pte_t.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20221022114424.580310787%40infradead.org
parent 93b3037a

arch/x86/include/asm/pgtable-3level.h

+14 −28
Original line number Diff line number Diff line
@@ -87,7 +87,7 @@ static inline pmd_t pmd_read_atomic(pmd_t *pmdp) 
		ret |= ((pmdval_t)*(tmp + 1)) << 32;

	}



	return (pmd_t) { ret };

	return (pmd_t) { .pmd = ret };

}



static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
@@ -121,12 +121,11 @@ static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, 
	ptep->pte_high = 0;

}



static inline void native_pmd_clear(pmd_t *pmd)

static inline void native_pmd_clear(pmd_t *pmdp)

{

	u32 *tmp = (u32 *)pmd;

	*tmp = 0;

	pmdp->pmd_low = 0;

	smp_wmb();

	*(tmp + 1) = 0;

	pmdp->pmd_high = 0;

}



static inline void native_pud_clear(pud_t *pudp)
@@ -162,25 +161,17 @@ static inline pte_t native_ptep_get_and_clear(pte_t *ptep) 
#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)

#endif



union split_pmd {

	struct {

		u32 pmd_low;

		u32 pmd_high;

	};

	pmd_t pmd;

};



#ifdef CONFIG_SMP

static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)

{

	union split_pmd res, *orig = (union split_pmd *)pmdp;

	pmd_t res;



	/* xchg acts as a barrier before setting of the high bits */

	res.pmd_low = xchg(&orig->pmd_low, 0);

	res.pmd_high = orig->pmd_high;

	orig->pmd_high = 0;

	res.pmd_low = xchg(&pmdp->pmd_low, 0);

	res.pmd_high = READ_ONCE(pmdp->pmd_high);

	WRITE_ONCE(pmdp->pmd_high, 0);



	return res.pmd;

	return res;

}

#else

#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
@@ -199,17 +190,12 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, 
	 * anybody.

	 */

	if (!(pmd_val(pmd) & _PAGE_PRESENT)) {

		union split_pmd old, new, *ptr;



		ptr = (union split_pmd *)pmdp;



		new.pmd = pmd;



		/* xchg acts as a barrier before setting of the high bits */

		old.pmd_low = xchg(&ptr->pmd_low, new.pmd_low);

		old.pmd_high = ptr->pmd_high;

		ptr->pmd_high = new.pmd_high;

		return old.pmd;

		old.pmd_low = xchg(&pmdp->pmd_low, pmd.pmd_low);

		old.pmd_high = READ_ONCE(pmdp->pmd_high);

		WRITE_ONCE(pmdp->pmd_high, pmd.pmd_high);



		return old;

	}



	do {

arch/x86/include/asm/pgtable-3level_types.h

+7 −0
Original line number Diff line number Diff line
@@ -18,6 +18,13 @@ typedef union { 
	};

	pteval_t pte;

} pte_t;



typedef union {

	struct {

		unsigned long pmd_low, pmd_high;

	};

	pmdval_t pmd;

} pmd_t;

#endif	/* !__ASSEMBLY__ */



#define SHARED_KERNEL_PMD	(!static_cpu_has(X86_FEATURE_PTI))

arch/x86/include/asm/pgtable_64_types.h

+1 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ typedef unsigned long pgdval_t; 
typedef unsigned long	pgprotval_t;



typedef struct { pteval_t pte; } pte_t;

typedef struct { pmdval_t pmd; } pmd_t;



#ifdef CONFIG_X86_5LEVEL

extern unsigned int __pgtable_l5_enabled;

arch/x86/include/asm/pgtable_types.h

+1 −3
Original line number Diff line number Diff line
@@ -361,11 +361,9 @@ static inline pudval_t native_pud_val(pud_t pud) 
#endif



#if CONFIG_PGTABLE_LEVELS > 2

typedef struct { pmdval_t pmd; } pmd_t;



static inline pmd_t native_make_pmd(pmdval_t val)

{

	return (pmd_t) { val };

	return (pmd_t) { .pmd = val };

}



static inline pmdval_t native_pmd_val(pmd_t pmd)