Commit 8184a8bc authored by Will Deacon's avatar Will Deacon
Browse files

Merge branch 'for-next/kpti' into for-next/core 

* for-next/kpti:
  arm64: correct the effect of mitigations off on kpti
  arm64: entry: simplify trampoline data page
  arm64: mm: install KPTI nG mappings with MMU enabled
  arm64: kpti-ng: simplify page table traversal logic
parents b7c47fd7 e92b2573

Documentation/admin-guide/kernel-parameters.txt

+1 −1
Original line number Diff line number Diff line
@@ -3161,7 +3161,7 @@ 
				improves system performance, but it may also

				expose users to several CPU vulnerabilities.

				Equivalent to: nopti [X86,PPC]

					       kpti=0 [ARM64]

					       if nokaslr then kpti=0 [ARM64]

					       nospectre_v1 [X86,PPC]

					       nobp=0 [S390]

					       nospectre_v2 [X86,PPC,S390,ARM64]

arch/arm64/include/asm/fixmap.h

+3 −1
Original line number Diff line number Diff line
@@ -62,10 +62,12 @@ enum fixed_addresses { 
#endif /* CONFIG_ACPI_APEI_GHES */



#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

#ifdef CONFIG_RELOCATABLE

	FIX_ENTRY_TRAMP_TEXT4,	/* one extra slot for the data page */

#endif

	FIX_ENTRY_TRAMP_TEXT3,

	FIX_ENTRY_TRAMP_TEXT2,

	FIX_ENTRY_TRAMP_TEXT1,

	FIX_ENTRY_TRAMP_DATA,

#define TRAMP_VALIAS		(__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))

#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */

	__end_of_permanent_fixed_addresses,

arch/arm64/kernel/cpufeature.c

+51 −3
Original line number Diff line number Diff line
@@ -1646,14 +1646,34 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry, 
}



#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

#define KPTI_NG_TEMP_VA		(-(1UL << PMD_SHIFT))



extern

void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,

			     phys_addr_t size, pgprot_t prot,

			     phys_addr_t (*pgtable_alloc)(int), int flags);



static phys_addr_t kpti_ng_temp_alloc;



static phys_addr_t kpti_ng_pgd_alloc(int shift)

{

	kpti_ng_temp_alloc -= PAGE_SIZE;

	return kpti_ng_temp_alloc;

}



static void __nocfi

kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)

{

	typedef void (kpti_remap_fn)(int, int, phys_addr_t);

	typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);

	extern kpti_remap_fn idmap_kpti_install_ng_mappings;

	kpti_remap_fn *remap_fn;



	int cpu = smp_processor_id();

	int levels = CONFIG_PGTABLE_LEVELS;

	int order = order_base_2(levels);

	u64 kpti_ng_temp_pgd_pa = 0;

	pgd_t *kpti_ng_temp_pgd;

	u64 alloc = 0;



	if (__this_cpu_read(this_cpu_vector) == vectors) {

		const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
@@ -1671,13 +1691,41 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) 


	remap_fn = (void *)__pa_symbol(function_nocfi(idmap_kpti_install_ng_mappings));



	if (!cpu) {

		alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);

		kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE);

		kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd);



		//

		// Create a minimal page table hierarchy that permits us to map

		// the swapper page tables temporarily as we traverse them.

		//

		// The physical pages are laid out as follows:

		//

		// +--------+-/-------+-/------ +-\\--------+

		// :  PTE[] : | PMD[] : | PUD[] : || PGD[]  :

		// +--------+-\-------+-\------ +-//--------+

		//      ^

		// The first page is mapped into this hierarchy at a PMD_SHIFT

		// aligned virtual address, so that we can manipulate the PTE

		// level entries while the mapping is active. The first entry

		// covers the PTE[] page itself, the remaining entries are free

		// to be used as a ad-hoc fixmap.

		//

		create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc),

					KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,

					kpti_ng_pgd_alloc, 0);

	}



	cpu_install_idmap();

	remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));

	remap_fn(cpu, num_online_cpus(), kpti_ng_temp_pgd_pa, KPTI_NG_TEMP_VA);

	cpu_uninstall_idmap();



	if (!cpu)

	if (!cpu) {

		free_pages(alloc, order);

		arm64_use_ng_mappings = true;

	}

}

#else

static void

kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)

arch/arm64/kernel/entry.S

+22 −31
Original line number Diff line number Diff line
@@ -636,18 +636,28 @@ alternative_else_nop_endif 
	 */

	.endm



	.macro tramp_data_page	dst

	adr_l	\dst, .entry.tramp.text

	sub	\dst, \dst, PAGE_SIZE

	.endm



	.macro		tramp_data_read_var	dst, var

#ifdef CONFIG_RANDOMIZE_BASE

	tramp_data_page		\dst

	add	\dst, \dst, #:lo12:__entry_tramp_data_\var

	ldr	\dst, [\dst]

#ifdef CONFIG_RELOCATABLE

	ldr		\dst, .L__tramp_data_\var

	.ifndef		.L__tramp_data_\var

	.pushsection	".entry.tramp.rodata", "a", %progbits

	.align		3

.L__tramp_data_\var:

	.quad		\var

	.popsection

	.endif

#else

	ldr	\dst, =\var

	/*

	 * As !RELOCATABLE implies !RANDOMIZE_BASE the address is always a

	 * compile time constant (and hence not secret and not worth hiding).

	 *

	 * As statically allocated kernel code and data always live in the top

	 * 47 bits of the address space we can sign-extend bit 47 and avoid an

	 * instruction to load the upper 16 bits (which must be 0xFFFF).

	 */

	movz		\dst, :abs_g2_s:\var

	movk		\dst, :abs_g1_nc:\var

	movk		\dst, :abs_g0_nc:\var

#endif

	.endm
@@ -695,7 +705,7 @@ alternative_else_nop_endif 
	msr	vbar_el1, x30

	isb

	.else

	ldr	x30, =vectors

	adr_l	x30, vectors

	.endif // \kpti == 1



	.if	\bhb == BHB_MITIGATION_FW
@@ -764,24 +774,7 @@ SYM_CODE_END(tramp_exit_native) 
SYM_CODE_START(tramp_exit_compat)

	tramp_exit	32

SYM_CODE_END(tramp_exit_compat)



	.ltorg

	.popsection				// .entry.tramp.text

#ifdef CONFIG_RANDOMIZE_BASE

	.pushsection ".rodata", "a"

	.align PAGE_SHIFT

SYM_DATA_START(__entry_tramp_data_start)

__entry_tramp_data_vectors:

	.quad	vectors

#ifdef CONFIG_ARM_SDE_INTERFACE

__entry_tramp_data___sdei_asm_handler:

	.quad	__sdei_asm_handler

#endif /* CONFIG_ARM_SDE_INTERFACE */

__entry_tramp_data_this_cpu_vector:

	.quad	this_cpu_vector

SYM_DATA_END(__entry_tramp_data_start)

	.popsection				// .rodata

#endif /* CONFIG_RANDOMIZE_BASE */

#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */



/*
@@ -932,7 +925,6 @@ NOKPROBE(call_on_irq_stack) 
 * This clobbers x4, __sdei_handler() will restore this from firmware's

 * copy.

 */

.ltorg

.pushsection ".entry.tramp.text", "ax"

SYM_CODE_START(__sdei_asm_entry_trampoline)

	mrs	x4, ttbr1_el1
@@ -967,7 +959,6 @@ SYM_CODE_START(__sdei_asm_exit_trampoline) 
1:	sdei_handler_exit exit_mode=x2

SYM_CODE_END(__sdei_asm_exit_trampoline)

NOKPROBE(__sdei_asm_exit_trampoline)

	.ltorg

.popsection		// .entry.tramp.text

#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */

arch/arm64/kernel/vmlinux.lds.S

+2 −1
Original line number Diff line number Diff line
@@ -115,7 +115,8 @@ jiffies = jiffies_64; 
	__entry_tramp_text_start = .;			\

	*(.entry.tramp.text)				\

	. = ALIGN(PAGE_SIZE);				\

	__entry_tramp_text_end = .;

	__entry_tramp_text_end = .;			\

	*(.entry.tramp.rodata)

#else

#define TRAMP_TEXT

#endif