Unverified Commit 677b9eb8 authored by Qinglin Pan's avatar Qinglin Pan Committed by Palmer Dabbelt
Browse files

riscv: mm: Prepare pt_ops helper functions for sv57 



This patch prepare some pt_ops helper functions which will be used in
creating sv57 mappings during boot time.

Signed-off-by: default avatarQinglin Pan <panqinglin2020@iscas.ac.cn>
Signed-off-by: default avatarPalmer Dabbelt <palmer@rivosinc.com>
parent d10efa21

arch/riscv/include/asm/fixmap.h

+1 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ enum fixed_addresses { 
	FIX_PTE,

	FIX_PMD,

	FIX_PUD,

	FIX_P4D,

	FIX_TEXT_POKE1,

	FIX_TEXT_POKE0,

	FIX_EARLYCON_MEM_BASE,

arch/riscv/include/asm/pgtable.h

+2 −0
Original line number Diff line number Diff line
@@ -133,6 +133,8 @@ struct pt_alloc_ops { 
	phys_addr_t (*alloc_pmd)(uintptr_t va);

	pud_t *(*get_pud_virt)(phys_addr_t pa);

	phys_addr_t (*alloc_pud)(uintptr_t va);

	p4d_t *(*get_p4d_virt)(phys_addr_t pa);

	phys_addr_t (*alloc_p4d)(uintptr_t va);

#endif

};

arch/riscv/mm/init.c

+116 −15
Original line number Diff line number Diff line
@@ -229,6 +229,7 @@ pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; 
static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss;



pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);

static p4d_t __maybe_unused early_dtb_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);

static pud_t __maybe_unused early_dtb_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE);

static pmd_t __maybe_unused early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
@@ -319,6 +320,16 @@ static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); 
#define early_pmd      ((pmd_t *)XIP_FIXUP(early_pmd))

#endif /* CONFIG_XIP_KERNEL */



static p4d_t trampoline_p4d[PTRS_PER_P4D] __page_aligned_bss;

static p4d_t fixmap_p4d[PTRS_PER_P4D] __page_aligned_bss;

static p4d_t early_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);



#ifdef CONFIG_XIP_KERNEL

#define trampoline_p4d ((p4d_t *)XIP_FIXUP(trampoline_p4d))

#define fixmap_p4d     ((p4d_t *)XIP_FIXUP(fixmap_p4d))

#define early_p4d      ((p4d_t *)XIP_FIXUP(early_p4d))

#endif /* CONFIG_XIP_KERNEL */



static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss;

static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss;

static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE);
@@ -433,6 +444,44 @@ static phys_addr_t alloc_pud_late(uintptr_t va) 
	return __pa(vaddr);

}



static p4d_t *__init get_p4d_virt_early(phys_addr_t pa)

{

	return (p4d_t *)((uintptr_t)pa);

}



static p4d_t *__init get_p4d_virt_fixmap(phys_addr_t pa)

{

	clear_fixmap(FIX_P4D);

	return (p4d_t *)set_fixmap_offset(FIX_P4D, pa);

}



static p4d_t *__init get_p4d_virt_late(phys_addr_t pa)

{

	return (p4d_t *)__va(pa);

}



static phys_addr_t __init alloc_p4d_early(uintptr_t va)

{

	/* Only one P4D is available for early mapping */

	BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT);



	return (uintptr_t)early_p4d;

}



static phys_addr_t __init alloc_p4d_fixmap(uintptr_t va)

{

	return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);

}



static phys_addr_t alloc_p4d_late(uintptr_t va)

{

	unsigned long vaddr;



	vaddr = __get_free_page(GFP_KERNEL);

	BUG_ON(!vaddr);

	return __pa(vaddr);

}



static void __init create_pud_mapping(pud_t *pudp,

				      uintptr_t va, phys_addr_t pa,

				      phys_addr_t sz, pgprot_t prot)
@@ -460,21 +509,55 @@ static void __init create_pud_mapping(pud_t *pudp, 
	create_pmd_mapping(nextp, va, pa, sz, prot);

}



#define pgd_next_t		pud_t

#define alloc_pgd_next(__va)	(pgtable_l4_enabled ?			\

		pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))

#define get_pgd_next_virt(__pa)	(pgtable_l4_enabled ?			\

		pt_ops.get_pud_virt(__pa) : (pgd_next_t *)pt_ops.get_pmd_virt(__pa))

static void __init create_p4d_mapping(p4d_t *p4dp,

				      uintptr_t va, phys_addr_t pa,

				      phys_addr_t sz, pgprot_t prot)

{

	pud_t *nextp;

	phys_addr_t next_phys;

	uintptr_t p4d_index = p4d_index(va);



	if (sz == P4D_SIZE) {

		if (p4d_val(p4dp[p4d_index]) == 0)

			p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot);

		return;

	}



	if (p4d_val(p4dp[p4d_index]) == 0) {

		next_phys = pt_ops.alloc_pud(va);

		p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE);

		nextp = pt_ops.get_pud_virt(next_phys);

		memset(nextp, 0, PAGE_SIZE);

	} else {

		next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index]));

		nextp = pt_ops.get_pud_virt(next_phys);

	}



	create_pud_mapping(nextp, va, pa, sz, prot);

}



#define pgd_next_t		p4d_t

#define alloc_pgd_next(__va)	(pgtable_l5_enabled ?			\

		pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ?		\

		pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va)))

#define get_pgd_next_virt(__pa)	(pgtable_l5_enabled ?			\

		pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ?	\

		pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa)))

#define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot)	\

				(pgtable_l5_enabled ?			\

		create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \

				(pgtable_l4_enabled ?			\

		create_pud_mapping(__nextp, __va, __pa, __sz, __prot) :	\

		create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))

#define fixmap_pgd_next		(pgtable_l4_enabled ?			\

		(uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)

#define trampoline_pgd_next	(pgtable_l4_enabled ?			\

		(uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)

#define early_dtb_pgd_next	(pgtable_l4_enabled ?			\

		(uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd)

		create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) :	\

		create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot)))

#define fixmap_pgd_next		(pgtable_l5_enabled ?			\

		(uintptr_t)fixmap_p4d : (pgtable_l4_enabled ?		\

		(uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd))

#define trampoline_pgd_next	(pgtable_l5_enabled ?			\

		(uintptr_t)trampoline_p4d : (pgtable_l4_enabled ?	\

		(uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd))

#define early_dtb_pgd_next	(pgtable_l5_enabled ?			\

		(uintptr_t)early_dtb_p4d : (pgtable_l4_enabled ?	\

		(uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd))

#else

#define pgd_next_t		pte_t

#define alloc_pgd_next(__va)	pt_ops.alloc_pte(__va)
@@ -483,6 +566,7 @@ static void __init create_pud_mapping(pud_t *pudp, 
	create_pte_mapping(__nextp, __va, __pa, __sz, __prot)

#define fixmap_pgd_next		((uintptr_t)fixmap_pte)

#define early_dtb_pgd_next	((uintptr_t)early_dtb_pmd)

#define create_p4d_mapping(__pmdp, __va, __pa, __sz, __prot)

#define create_pud_mapping(__pmdp, __va, __pa, __sz, __prot)

#define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot)

#endif /* __PAGETABLE_PMD_FOLDED */
@@ -693,10 +777,13 @@ static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa) 
			   PGDIR_SIZE,

			   IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL);



	if (pgtable_l4_enabled) {

	if (pgtable_l5_enabled)

		create_p4d_mapping(early_dtb_p4d, DTB_EARLY_BASE_VA,

				   (uintptr_t)early_dtb_pud, P4D_SIZE, PAGE_TABLE);



	if (pgtable_l4_enabled)

		create_pud_mapping(early_dtb_pud, DTB_EARLY_BASE_VA,

				   (uintptr_t)early_dtb_pmd, PUD_SIZE, PAGE_TABLE);

	}



	if (IS_ENABLED(CONFIG_64BIT)) {

		create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA,
@@ -732,6 +819,8 @@ void __init pt_ops_set_early(void) 
	pt_ops.get_pmd_virt = get_pmd_virt_early;

	pt_ops.alloc_pud = alloc_pud_early;

	pt_ops.get_pud_virt = get_pud_virt_early;

	pt_ops.alloc_p4d = alloc_p4d_early;

	pt_ops.get_p4d_virt = get_p4d_virt_early;

#endif

}
@@ -752,6 +841,8 @@ void __init pt_ops_set_fixmap(void) 
	pt_ops.get_pmd_virt = kernel_mapping_pa_to_va((uintptr_t)get_pmd_virt_fixmap);

	pt_ops.alloc_pud = kernel_mapping_pa_to_va((uintptr_t)alloc_pud_fixmap);

	pt_ops.get_pud_virt = kernel_mapping_pa_to_va((uintptr_t)get_pud_virt_fixmap);

	pt_ops.alloc_p4d = kernel_mapping_pa_to_va((uintptr_t)alloc_p4d_fixmap);

	pt_ops.get_p4d_virt = kernel_mapping_pa_to_va((uintptr_t)get_p4d_virt_fixmap);

#endif

}
@@ -768,6 +859,8 @@ void __init pt_ops_set_late(void) 
	pt_ops.get_pmd_virt = get_pmd_virt_late;

	pt_ops.alloc_pud = alloc_pud_late;

	pt_ops.get_pud_virt = get_pud_virt_late;

	pt_ops.alloc_p4d = alloc_p4d_late;

	pt_ops.get_p4d_virt = get_p4d_virt_late;

#endif

}
@@ -828,6 +921,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) 
			   fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE);



#ifndef __PAGETABLE_PMD_FOLDED

	/* Setup fixmap P4D and PUD */

	if (pgtable_l5_enabled)

		create_p4d_mapping(fixmap_p4d, FIXADDR_START,

				   (uintptr_t)fixmap_pud, P4D_SIZE, PAGE_TABLE);

	/* Setup fixmap PUD and PMD */

	if (pgtable_l4_enabled)

		create_pud_mapping(fixmap_pud, FIXADDR_START,
@@ -837,6 +934,9 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) 
	/* Setup trampoline PGD and PMD */

	create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr,

			   trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE);

	if (pgtable_l5_enabled)

		create_p4d_mapping(trampoline_p4d, kernel_map.virt_addr,

				   (uintptr_t)trampoline_pud, P4D_SIZE, PAGE_TABLE);

	if (pgtable_l4_enabled)

		create_pud_mapping(trampoline_pud, kernel_map.virt_addr,

				   (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE);
@@ -938,6 +1038,7 @@ static void __init setup_vm_final(void) 
	clear_fixmap(FIX_PTE);

	clear_fixmap(FIX_PMD);

	clear_fixmap(FIX_PUD);

	clear_fixmap(FIX_P4D);



	/* Move to swapper page table */

	csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | satp_mode);