Commit e9f63768 authored by Mike Rapoport's avatar Mike Rapoport Committed by Linus Torvalds
Browse files

arm64: add support for folded p4d page tables 

Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate, replace 5level-fixup.h with pgtable-nop4d.h and
remove __ARCH_USE_5LEVEL_HACK.

[arnd@arndb.de: fix gcc-10 shift warning]
  Link: http://lkml.kernel.org/r/20200429185657.4085975-1-arnd@arndb.de


Signed-off-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Brian Cain <bcain@codeaurora.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christophe Leroy <christophe.leroy@c-s.fr>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Geert Uytterhoeven <geert+renesas@glider.be>
Cc: Guan Xuetao <gxt@pku.edu.cn>
Cc: James Morse <james.morse@arm.com>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Julien Thierry <julien.thierry.kdev@gmail.com>
Cc: Ley Foon Tan <ley.foon.tan@intel.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Rich Felker <dalias@libc.org>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Stafford Horne <shorne@gmail.com>
Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Link: http://lkml.kernel.org/r/20200414153455.21744-4-rppt@kernel.org


Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 84e6ffb2

arch/arm64/include/asm/kvm_mmu.h

+8 −2
Original line number Diff line number Diff line
@@ -172,8 +172,8 @@ void kvm_clear_hyp_idmap(void); 
	__pmd(__phys_to_pmd_val(__pa(ptep)) | PMD_TYPE_TABLE)

#define kvm_mk_pud(pmdp)					\

	__pud(__phys_to_pud_val(__pa(pmdp)) | PMD_TYPE_TABLE)

#define kvm_mk_pgd(pudp)					\

	__pgd(__phys_to_pgd_val(__pa(pudp)) | PUD_TYPE_TABLE)

#define kvm_mk_p4d(pmdp)					\

	__p4d(__phys_to_p4d_val(__pa(pmdp)) | PUD_TYPE_TABLE)



#define kvm_set_pud(pudp, pud)		set_pud(pudp, pud)
@@ -299,6 +299,12 @@ static inline bool kvm_s2pud_young(pud_t pud) 
#define hyp_pud_table_empty(pudp) kvm_page_empty(pudp)

#endif



#ifdef __PAGETABLE_P4D_FOLDED

#define hyp_p4d_table_empty(p4dp) (0)

#else

#define hyp_p4d_table_empty(p4dp) kvm_page_empty(p4dp)

#endif



struct kvm;



#define kvm_flush_dcache_to_poc(a,l)	__flush_dcache_area((a), (l))

arch/arm64/include/asm/pgalloc.h

+5 −5
Original line number Diff line number Diff line
@@ -73,17 +73,17 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pudp) 
	free_page((unsigned long)pudp);

}



static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot)

static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)

{

	set_pgd(pgdp, __pgd(__phys_to_pgd_val(pudp) | prot));

	set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot));

}



static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, pud_t *pudp)

static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp)

{

	__pgd_populate(pgdp, __pa(pudp), PUD_TYPE_TABLE);

	__p4d_populate(p4dp, __pa(pudp), PUD_TYPE_TABLE);

}

#else

static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot)

static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)

{

	BUILD_BUG();

}

arch/arm64/include/asm/pgtable-types.h

+2 −3
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 
typedef u64 pteval_t;

typedef u64 pmdval_t;

typedef u64 pudval_t;

typedef u64 p4dval_t;

typedef u64 pgdval_t;



/*
@@ -44,13 +45,11 @@ typedef struct { pteval_t pgprot; } pgprot_t; 
#define __pgprot(x)	((pgprot_t) { (x) } )



#if CONFIG_PGTABLE_LEVELS == 2

#define __ARCH_USE_5LEVEL_HACK

#include <asm-generic/pgtable-nopmd.h>

#elif CONFIG_PGTABLE_LEVELS == 3

#define __ARCH_USE_5LEVEL_HACK

#include <asm-generic/pgtable-nopud.h>

#elif CONFIG_PGTABLE_LEVELS == 4

#include <asm-generic/5level-fixup.h>

#include <asm-generic/pgtable-nop4d.h>

#endif



#endif	/* __ASM_PGTABLE_TYPES_H */

arch/arm64/include/asm/pgtable.h

+23 −14
Original line number Diff line number Diff line
@@ -298,6 +298,11 @@ static inline pte_t pgd_pte(pgd_t pgd) 
	return __pte(pgd_val(pgd));

}



static inline pte_t p4d_pte(p4d_t p4d)

{

	return __pte(p4d_val(p4d));

}



static inline pte_t pud_pte(pud_t pud)

{

	return __pte(pud_val(pud));
@@ -401,6 +406,9 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd) 


#define set_pmd_at(mm, addr, pmdp, pmd)	set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))



#define __p4d_to_phys(p4d)	__pte_to_phys(p4d_pte(p4d))

#define __phys_to_p4d_val(phys)	__phys_to_pte_val(phys)



#define __pgd_to_phys(pgd)	__pte_to_phys(pgd_pte(pgd))

#define __phys_to_pgd_val(phys)	__phys_to_pte_val(phys)
@@ -592,49 +600,50 @@ static inline phys_addr_t pud_page_paddr(pud_t pud) 


#define pud_ERROR(pud)		__pud_error(__FILE__, __LINE__, pud_val(pud))



#define pgd_none(pgd)		(!pgd_val(pgd))

#define pgd_bad(pgd)		(!(pgd_val(pgd) & 2))

#define pgd_present(pgd)	(pgd_val(pgd))

#define p4d_none(p4d)		(!p4d_val(p4d))

#define p4d_bad(p4d)		(!(p4d_val(p4d) & 2))

#define p4d_present(p4d)	(p4d_val(p4d))



static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)

static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)

{

	if (in_swapper_pgdir(pgdp)) {

		set_swapper_pgd(pgdp, pgd);

	if (in_swapper_pgdir(p4dp)) {

		set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));

		return;

	}



	WRITE_ONCE(*pgdp, pgd);

	WRITE_ONCE(*p4dp, p4d);

	dsb(ishst);

	isb();

}



static inline void pgd_clear(pgd_t *pgdp)

static inline void p4d_clear(p4d_t *p4dp)

{

	set_pgd(pgdp, __pgd(0));

	set_p4d(p4dp, __p4d(0));

}



static inline phys_addr_t pgd_page_paddr(pgd_t pgd)

static inline phys_addr_t p4d_page_paddr(p4d_t p4d)

{

	return __pgd_to_phys(pgd);

	return __p4d_to_phys(p4d);

}



/* Find an entry in the frst-level page table. */

#define pud_index(addr)		(((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))



#define pud_offset_phys(dir, addr)	(pgd_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))

#define pud_offset_phys(dir, addr)	(p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))

#define pud_offset(dir, addr)		((pud_t *)__va(pud_offset_phys((dir), (addr))))



#define pud_set_fixmap(addr)		((pud_t *)set_fixmap_offset(FIX_PUD, addr))

#define pud_set_fixmap_offset(pgd, addr)	pud_set_fixmap(pud_offset_phys(pgd, addr))

#define pud_set_fixmap_offset(p4d, addr)	pud_set_fixmap(pud_offset_phys(p4d, addr))

#define pud_clear_fixmap()		clear_fixmap(FIX_PUD)



#define pgd_page(pgd)			phys_to_page(__pgd_to_phys(pgd))

#define p4d_page(p4d)		pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))



/* use ONLY for statically allocated translation tables */

#define pud_offset_kimg(dir,addr)	((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))



#else



#define p4d_page_paddr(p4d)	({ BUILD_BUG(); 0;})

#define pgd_page_paddr(pgd)	({ BUILD_BUG(); 0;})



/* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */

arch/arm64/include/asm/stage2_pgtable.h

+37 −11
Original line number Diff line number Diff line
@@ -68,41 +68,67 @@ static inline bool kvm_stage2_has_pud(struct kvm *kvm) 
#define S2_PUD_SIZE			(1UL << S2_PUD_SHIFT)

#define S2_PUD_MASK			(~(S2_PUD_SIZE - 1))



static inline bool stage2_pgd_none(struct kvm *kvm, pgd_t pgd)

#define stage2_pgd_none(kvm, pgd)		pgd_none(pgd)

#define stage2_pgd_clear(kvm, pgd)		pgd_clear(pgd)

#define stage2_pgd_present(kvm, pgd)		pgd_present(pgd)

#define stage2_pgd_populate(kvm, pgd, p4d)	pgd_populate(NULL, pgd, p4d)



static inline p4d_t *stage2_p4d_offset(struct kvm *kvm,

				       pgd_t *pgd, unsigned long address)

{

	return p4d_offset(pgd, address);

}



static inline void stage2_p4d_free(struct kvm *kvm, p4d_t *p4d)

{

}



static inline bool stage2_p4d_table_empty(struct kvm *kvm, p4d_t *p4dp)

{

	return false;

}



static inline phys_addr_t stage2_p4d_addr_end(struct kvm *kvm,

					      phys_addr_t addr, phys_addr_t end)

{

	return end;

}



static inline bool stage2_p4d_none(struct kvm *kvm, p4d_t p4d)

{

	if (kvm_stage2_has_pud(kvm))

		return pgd_none(pgd);

		return p4d_none(p4d);

	else

		return 0;

}



static inline void stage2_pgd_clear(struct kvm *kvm, pgd_t *pgdp)

static inline void stage2_p4d_clear(struct kvm *kvm, p4d_t *p4dp)

{

	if (kvm_stage2_has_pud(kvm))

		pgd_clear(pgdp);

		p4d_clear(p4dp);

}



static inline bool stage2_pgd_present(struct kvm *kvm, pgd_t pgd)

static inline bool stage2_p4d_present(struct kvm *kvm, p4d_t p4d)

{

	if (kvm_stage2_has_pud(kvm))

		return pgd_present(pgd);

		return p4d_present(p4d);

	else

		return 1;

}



static inline void stage2_pgd_populate(struct kvm *kvm, pgd_t *pgd, pud_t *pud)

static inline void stage2_p4d_populate(struct kvm *kvm, p4d_t *p4d, pud_t *pud)

{

	if (kvm_stage2_has_pud(kvm))

		pgd_populate(NULL, pgd, pud);

		p4d_populate(NULL, p4d, pud);

}



static inline pud_t *stage2_pud_offset(struct kvm *kvm,

				       pgd_t *pgd, unsigned long address)

				       p4d_t *p4d, unsigned long address)

{

	if (kvm_stage2_has_pud(kvm))

		return pud_offset(pgd, address);

		return pud_offset(p4d, address);

	else

		return (pud_t *)pgd;

		return (pud_t *)p4d;

}



static inline void stage2_pud_free(struct kvm *kvm, pud_t *pud)