Commit 63fec243 authored by David Brazdil's avatar David Brazdil Committed by Marc Zyngier
Browse files

KVM: arm64: Move hyp-init params to a per-CPU struct 



Once we start initializing KVM on newly booted cores before the rest of
the kernel, parameters to __do_hyp_init will need to be provided by EL2
rather than EL1. At that point it will not be possible to pass its three
arguments directly because PSCI_CPU_ON only supports one context
argument.

Refactor __do_hyp_init to accept its parameters in a struct. This
prepares the code for KVM booting cores as well as removes any limits on
the number of __do_hyp_init arguments.

Signed-off-by: default avatarDavid Brazdil <dbrazdil@google.com>
Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20201202184122.26046-11-dbrazdil@google.com
parent 5be1d622

arch/arm64/include/asm/kvm_asm.h

+6 −0
Original line number Diff line number Diff line
@@ -150,6 +150,12 @@ extern void *__vhe_undefined_symbol; 


#endif



struct kvm_nvhe_init_params {

	unsigned long tpidr_el2;

	unsigned long stack_hyp_va;

	phys_addr_t pgd_pa;

};



/* Translate a kernel address @ptr into its equivalent linear mapping */

#define kvm_ksym_ref(ptr)						\

	({								\

arch/arm64/include/asm/kvm_hyp.h

+1 −1
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 


DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);

DECLARE_PER_CPU(unsigned long, kvm_hyp_vector);

DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);



#define read_sysreg_elx(r,nvh,vh)					\

	({								\
@@ -98,4 +99,3 @@ void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par); 
#endif



#endif /* __ARM64_KVM_HYP_H__ */

arch/arm64/kernel/asm-offsets.c

+3 −0
Original line number Diff line number Diff line
@@ -109,6 +109,9 @@ int main(void) 
  DEFINE(CPU_APGAKEYLO_EL1,	offsetof(struct kvm_cpu_context, sys_regs[APGAKEYLO_EL1]));

  DEFINE(HOST_CONTEXT_VCPU,	offsetof(struct kvm_cpu_context, __hyp_running_vcpu));

  DEFINE(HOST_DATA_CONTEXT,	offsetof(struct kvm_host_data, host_ctxt));

  DEFINE(NVHE_INIT_TPIDR_EL2,	offsetof(struct kvm_nvhe_init_params, tpidr_el2));

  DEFINE(NVHE_INIT_STACK_HYP_VA,	offsetof(struct kvm_nvhe_init_params, stack_hyp_va));

  DEFINE(NVHE_INIT_PGD_PA,	offsetof(struct kvm_nvhe_init_params, pgd_pa));

#endif

#ifdef CONFIG_CPU_PM

  DEFINE(CPU_CTX_SP,		offsetof(struct cpu_suspend_ctx, sp));

arch/arm64/kvm/arm.c

+13 −10
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector); 


static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);

unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];

DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);



/* The VMID used in the VTTBR */

static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
@@ -1333,9 +1334,7 @@ static int kvm_map_vectors(void) 


static void cpu_init_hyp_mode(void)

{

	phys_addr_t pgd_ptr;

	unsigned long hyp_stack_ptr;

	unsigned long tpidr_el2;

	struct kvm_nvhe_init_params *params = this_cpu_ptr_nvhe_sym(kvm_init_params);

	struct arm_smccc_res res;



	/* Switch from the HYP stub to our own HYP init vector */
@@ -1346,12 +1345,17 @@ static void cpu_init_hyp_mode(void) 
	 * kernel's mapping to the linear mapping, and store it in tpidr_el2

	 * so that we can use adr_l to access per-cpu variables in EL2.

	 */

	tpidr_el2 = (unsigned long)this_cpu_ptr_nvhe_sym(__per_cpu_start) -

	params->tpidr_el2 = (unsigned long)this_cpu_ptr_nvhe_sym(__per_cpu_start) -

			    (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));



	pgd_ptr = kvm_mmu_get_httbr();

	hyp_stack_ptr = __this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE;

	hyp_stack_ptr = kern_hyp_va(hyp_stack_ptr);

	params->stack_hyp_va = kern_hyp_va(__this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE);

	params->pgd_pa = kvm_mmu_get_httbr();



	/*

	 * Flush the init params from the data cache because the struct will

	 * be read while the MMU is off.

	 */

	kvm_flush_dcache_to_poc(params, sizeof(*params));



	/*

	 * Call initialization code, and switch to the full blown HYP code.
@@ -1360,8 +1364,7 @@ static void cpu_init_hyp_mode(void) 
	 * cpus_have_const_cap() wrapper.

	 */

	BUG_ON(!system_capabilities_finalized());

	arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init),

			  pgd_ptr, tpidr_el2, hyp_stack_ptr, &res);

	arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);

	WARN_ON(res.a0 != SMCCC_RET_SUCCESS);



	/*

arch/arm64/kvm/hyp/nvhe/hyp-init.S

+7 −9
Original line number Diff line number Diff line
@@ -47,9 +47,7 @@ __invalid: 


	/*

	 * x0: SMCCC function ID

	 * x1: HYP pgd

	 * x2: per-CPU offset

	 * x3: HYP stack

	 * x1: struct kvm_nvhe_init_params PA

	 */

__do_hyp_init:

	/* Check for a stub HVC call */
@@ -70,10 +68,13 @@ __do_hyp_init: 
	mov	x0, #SMCCC_RET_NOT_SUPPORTED

	eret



1:

	/* Set tpidr_el2 for use by HYP to free a register */

	msr	tpidr_el2, x2

1:	ldr	x0, [x1, #NVHE_INIT_TPIDR_EL2]

	msr	tpidr_el2, x0



	ldr	x0, [x1, #NVHE_INIT_STACK_HYP_VA]

	mov	sp, x0



	ldr	x1, [x1, #NVHE_INIT_PGD_PA]

	phys_to_ttbr x0, x1

alternative_if ARM64_HAS_CNP

	orr	x0, x0, #TTBR_CNP_BIT
@@ -133,9 +134,6 @@ alternative_else_nop_endif 
	msr	sctlr_el2, x0

	isb



	/* Set the stack */

	mov	sp, x3



	/* Set the host vector */

	ldr	x0, =__kvm_hyp_host_vector

	kimg_hyp_va x0, x1