Commit 6ec6259d authored by David Brazdil's avatar David Brazdil Committed by Marc Zyngier
Browse files

KVM: arm64: Apply hyp relocations at runtime 



KVM nVHE code runs under a different VA mapping than the kernel, hence
so far it avoided using absolute addressing because the VA in a constant
pool is relocated by the linker to a kernel VA (see hyp_symbol_addr).

Now the kernel has access to a list of positions that contain a kimg VA
but will be accessed only in hyp execution context. These are generated
by the gen-hyprel build-time tool and stored in .hyp.reloc.

Add early boot pass over the entries and convert the kimg VAs to hyp VAs.
Note that this requires for .hyp* ELF sections to be mapped read-write
at that point.

Signed-off-by: default avatarDavid Brazdil <dbrazdil@google.com>
Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20210105180541.65031-6-dbrazdil@google.com
parent 8c49b5d4

arch/arm64/include/asm/kvm_mmu.h

+1 −0
Original line number Diff line number Diff line
@@ -129,6 +129,7 @@ alternative_cb_end 
void kvm_update_va_mask(struct alt_instr *alt,

			__le32 *origptr, __le32 *updptr, int nr_inst);

void kvm_compute_layout(void);

void kvm_apply_hyp_relocations(void);



static __always_inline unsigned long __kern_hyp_va(unsigned long v)

{

arch/arm64/include/asm/sections.h

+1 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[]; 
extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];

extern char __hyp_text_start[], __hyp_text_end[];

extern char __hyp_rodata_start[], __hyp_rodata_end[];

extern char __hyp_reloc_begin[], __hyp_reloc_end[];

extern char __idmap_text_start[], __idmap_text_end[];

extern char __initdata_begin[], __initdata_end[];

extern char __inittext_begin[], __inittext_end[];

arch/arm64/kernel/smp.c

+3 −1
Original line number Diff line number Diff line
@@ -434,8 +434,10 @@ static void __init hyp_mode_check(void) 
			   "CPU: CPUs started in inconsistent modes");

	else

		pr_info("CPU: All CPU(s) started at EL1\n");

	if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode())

	if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode()) {

		kvm_compute_layout();

		kvm_apply_hyp_relocations();

	}

}



void __init smp_cpus_done(unsigned int max_cpus)

arch/arm64/kvm/va_layout.c

+28 −0
Original line number Diff line number Diff line
@@ -81,6 +81,34 @@ __init void kvm_compute_layout(void) 
	init_hyp_physvirt_offset();

}



/*

 * The .hyp.reloc ELF section contains a list of kimg positions that

 * contains kimg VAs but will be accessed only in hyp execution context.

 * Convert them to hyp VAs. See gen-hyprel.c for more details.

 */

__init void kvm_apply_hyp_relocations(void)

{

	int32_t *rel;

	int32_t *begin = (int32_t *)__hyp_reloc_begin;

	int32_t *end = (int32_t *)__hyp_reloc_end;



	for (rel = begin; rel < end; ++rel) {

		uintptr_t *ptr, kimg_va;



		/*

		 * Each entry contains a 32-bit relative offset from itself

		 * to a kimg VA position.

		 */

		ptr = (uintptr_t *)lm_alias((char *)rel + *rel);



		/* Read the kimg VA value at the relocation address. */

		kimg_va = *ptr;



		/* Convert to hyp VA and store back to the relocation address. */

		*ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));

	}

}



static u32 compute_instruction(int n, u32 rd, u32 rn)

{

	u32 insn = AARCH64_BREAK_FAULT;