Unverified Commit 2bfc6cd8 authored by Alexandre Ghiti's avatar Alexandre Ghiti Committed by Palmer Dabbelt
Browse files

riscv: Move kernel mapping outside of linear mapping 



This is a preparatory patch for relocatable kernel and sv48 support.

The kernel used to be linked at PAGE_OFFSET address therefore we could use
the linear mapping for the kernel mapping. But the relocated kernel base
address will be different from PAGE_OFFSET and since in the linear mapping,
two different virtual addresses cannot point to the same physical address,
the kernel mapping needs to lie outside the linear mapping so that we don't
have to copy it at the same physical offset.

The kernel mapping is moved to the last 2GB of the address space, BPF
is now always after the kernel and modules use the 2GB memory range right
before the kernel, so BPF and modules regions do not overlap. KASLR
implementation will simply have to move the kernel in the last 2GB range
and just take care of leaving enough space for BPF.

In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.

Suggested-by: default avatarArnd Bergmann <arnd@arndb.de>
Signed-off-by: default avatarAlexandre Ghiti <alex@ghiti.fr>
[Palmer: Squash the STRICT_RWX fix, and a !MMU fix]
Signed-off-by: default avatarPalmer Dabbelt <palmerdabbelt@google.com>
parent 8a07ac39

arch/riscv/boot/loader.lds.S

+2 −1
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */



#include <asm/page.h>

#include <asm/pgtable.h>



OUTPUT_ARCH(riscv)

ENTRY(_start)



SECTIONS

{

	. = PAGE_OFFSET;

	. = KERNEL_LINK_ADDR;



	.payload : {

		*(.payload)

arch/riscv/include/asm/page.h

+24 −2
Original line number Diff line number Diff line
@@ -90,15 +90,37 @@ typedef struct page *pgtable_t; 


#ifdef CONFIG_MMU

extern unsigned long va_pa_offset;

#ifdef CONFIG_64BIT

extern unsigned long va_kernel_pa_offset;

#endif

extern unsigned long pfn_base;

#define ARCH_PFN_OFFSET		(pfn_base)

#else

#define va_pa_offset		0

#ifdef CONFIG_64BIT

#define va_kernel_pa_offset	0

#endif

#define ARCH_PFN_OFFSET		(PAGE_OFFSET >> PAGE_SHIFT)

#endif /* CONFIG_MMU */



#ifdef CONFIG_64BIT

extern unsigned long kernel_virt_addr;



#define linear_mapping_pa_to_va(x)	((void *)((unsigned long)(x) + va_pa_offset))

#define kernel_mapping_pa_to_va(x)	((void *)((unsigned long)(x) + va_kernel_pa_offset))

#define __pa_to_va_nodebug(x)		linear_mapping_pa_to_va(x)



#define linear_mapping_va_to_pa(x)	((unsigned long)(x) - va_pa_offset)

#define kernel_mapping_va_to_pa(x)	((unsigned long)(x) - va_kernel_pa_offset)

#define __va_to_pa_nodebug(x)	({						\

	unsigned long _x = x;							\

	(_x < kernel_virt_addr) ?						\

		linear_mapping_va_to_pa(_x) : kernel_mapping_va_to_pa(_x);	\

	})

#else

#define __pa_to_va_nodebug(x)  ((void *)((unsigned long) (x) + va_pa_offset))

#define __va_to_pa_nodebug(x)  ((unsigned long)(x) - va_pa_offset)

#endif



#ifdef CONFIG_DEBUG_VIRTUAL

extern phys_addr_t __virt_to_phys(unsigned long x);

arch/riscv/include/asm/pgtable.h

+31 −8
Original line number Diff line number Diff line
@@ -11,23 +11,38 @@ 


#include <asm/pgtable-bits.h>



#ifndef __ASSEMBLY__

#ifndef CONFIG_MMU

#define KERNEL_LINK_ADDR	PAGE_OFFSET

#else



/* Page Upper Directory not used in RISC-V */

#include <asm-generic/pgtable-nopud.h>

#include <asm/page.h>

#include <asm/tlbflush.h>

#include <linux/mm_types.h>

#define ADDRESS_SPACE_END	(UL(-1))



#ifdef CONFIG_MMU

#ifdef CONFIG_64BIT

/* Leave 2GB for kernel and BPF at the end of the address space */

#define KERNEL_LINK_ADDR	(ADDRESS_SPACE_END - SZ_2G + 1)

#else

#define KERNEL_LINK_ADDR	PAGE_OFFSET

#endif



#define VMALLOC_SIZE     (KERN_VIRT_SIZE >> 1)

#define VMALLOC_END      (PAGE_OFFSET - 1)

#define VMALLOC_START    (PAGE_OFFSET - VMALLOC_SIZE)



#define BPF_JIT_REGION_SIZE	(SZ_128M)

#ifdef CONFIG_64BIT

/* KASLR should leave at least 128MB for BPF after the kernel */

#define BPF_JIT_REGION_START	PFN_ALIGN((unsigned long)&_end)

#define BPF_JIT_REGION_END	(BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)

#else

#define BPF_JIT_REGION_START	(PAGE_OFFSET - BPF_JIT_REGION_SIZE)

#define BPF_JIT_REGION_END	(VMALLOC_END)

#endif



/* Modules always live before the kernel */

#ifdef CONFIG_64BIT

#define MODULES_VADDR	(PFN_ALIGN((unsigned long)&_end) - SZ_2G)

#define MODULES_END	(PFN_ALIGN((unsigned long)&_start))

#endif



/*

 * Roughly size the vmemmap space to be large enough to fit enough
@@ -57,9 +72,16 @@ 
#define FIXADDR_SIZE     PGDIR_SIZE

#endif

#define FIXADDR_START    (FIXADDR_TOP - FIXADDR_SIZE)



#endif



#ifndef __ASSEMBLY__



/* Page Upper Directory not used in RISC-V */

#include <asm-generic/pgtable-nopud.h>

#include <asm/page.h>

#include <asm/tlbflush.h>

#include <linux/mm_types.h>



#ifdef CONFIG_64BIT

#include <asm/pgtable-64.h>

#else
@@ -484,6 +506,7 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, 


#define kern_addr_valid(addr)   (1) /* FIXME */



extern char _start[];

extern void *dtb_early_va;

extern uintptr_t dtb_early_pa;

void setup_bootmem(void);

arch/riscv/include/asm/set_memory.h

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ int set_memory_x(unsigned long addr, int numpages); 
int set_memory_nx(unsigned long addr, int numpages);

int set_memory_rw_nx(unsigned long addr, int numpages);

void protect_kernel_text_data(void);

void protect_kernel_linear_mapping_text_rodata(void);

#else

static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; }

static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; }

arch/riscv/kernel/head.S

+2 −1
Original line number Diff line number Diff line
@@ -69,7 +69,8 @@ pe_head_start: 
#ifdef CONFIG_MMU

relocate:

	/* Relocate return address */

	li a1, PAGE_OFFSET

	la a1, kernel_virt_addr

	REG_L a1, 0(a1)

	la a2, _start

	sub a1, a1, a2

	add ra, ra, a1