Commit d58335d1 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch kvm-arm64/tlbi-range into kvmarm-master/next 



* kvm-arm64/tlbi-range:
  : .
  : FEAT_TLBIRANGE support, courtesy of Raghavendra Rao Ananta.
  : From the cover letter:
  :
  : "In certain code paths, KVM/ARM currently invalidates the entire VM's
  : page-tables instead of just invalidating a necessary range. For example,
  : when collapsing a table PTE to a block PTE, instead of iterating over
  : each PTE and flushing them, KVM uses 'vmalls12e1is' TLBI operation to
  : flush all the entries. This is inefficient since the guest would have
  : to refill the TLBs again, even for the addresses that aren't covered
  : by the table entry. The performance impact would scale poorly if many
  : addresses in the VM is going through this remapping.
  :
  : For architectures that implement FEAT_TLBIRANGE, KVM can replace such
  : inefficient paths by performing the invalidations only on the range of
  : addresses that are in scope. This series tries to achieve the same in
  : the areas of stage-2 map, unmap and write-protecting the pages."
  : .
  KVM: arm64: Use TLBI range-based instructions for unmap
  KVM: arm64: Invalidate the table entries upon a range
  KVM: arm64: Flush only the memslot after write-protect
  KVM: arm64: Implement kvm_arch_flush_remote_tlbs_range()
  KVM: arm64: Define kvm_tlb_flush_vmid_range()
  KVM: arm64: Implement __kvm_tlb_flush_vmid_range()
  arm64: tlb: Implement __flush_s2_tlb_range_op()
  arm64: tlb: Refactor the core flush algorithm of __flush_tlb_range
  KVM: Move kvm_arch_flush_remote_tlbs_memslot() to common code
  KVM: Allow range-based TLB invalidation from common code
  KVM: Remove CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
  KVM: arm64: Use kvm_arch_flush_remote_tlbs()
  KVM: Declare kvm_arch_flush_remote_tlbs() globally
  KVM: Rename kvm_arch_flush_remote_tlb() to kvm_arch_flush_remote_tlbs()

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents c1907626 7657ea92

arch/arm64/include/asm/kvm_asm.h

+3 −0
Original line number Diff line number Diff line
@@ -70,6 +70,7 @@ enum __kvm_host_smccc_func { 
	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa,

	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_ipa_nsh,

	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid,

	__KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range,

	__KVM_HOST_SMCCC_FUNC___kvm_flush_cpu_context,

	__KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff,

	__KVM_HOST_SMCCC_FUNC___vgic_v3_read_vmcr,
@@ -229,6 +230,8 @@ extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa, 
extern void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu,

					 phys_addr_t ipa,

					 int level);

extern void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu,

					phys_addr_t start, unsigned long pages);

extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);



extern void __kvm_timer_set_cntvoff(u64 cntvoff);

arch/arm64/include/asm/kvm_host.h

+4 −0
Original line number Diff line number Diff line
@@ -1120,6 +1120,10 @@ int __init kvm_set_ipa_limit(void); 
#define __KVM_HAVE_ARCH_VM_ALLOC

struct kvm *kvm_arch_alloc_vm(void);



#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS



#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE



static inline bool kvm_vm_is_protected(struct kvm *kvm)

{

	return false;

arch/arm64/include/asm/kvm_pgtable.h

+10 −0
Original line number Diff line number Diff line
@@ -746,4 +746,14 @@ enum kvm_pgtable_prot kvm_pgtable_stage2_pte_prot(kvm_pte_t pte); 
 *	   kvm_pgtable_prot format.

 */

enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte);



/**

 * kvm_tlb_flush_vmid_range() - Invalidate/flush a range of TLB entries

 *

 * @mmu:	Stage-2 KVM MMU struct

 * @addr:	The base Intermediate physical address from which to invalidate

 * @size:	Size of the range from the base to invalidate

 */

void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu,

				phys_addr_t addr, size_t size);

#endif	/* __ARM64_KVM_PGTABLE_H__ */

arch/arm64/include/asm/tlbflush.h

+71 −53
Original line number Diff line number Diff line
@@ -278,14 +278,77 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, 
 */

#define MAX_TLBI_OPS	PTRS_PER_PTE



/*

 * __flush_tlb_range_op - Perform TLBI operation upon a range

 *

 * @op:	TLBI instruction that operates on a range (has 'r' prefix)

 * @start:	The start address of the range

 * @pages:	Range as the number of pages from 'start'

 * @stride:	Flush granularity

 * @asid:	The ASID of the task (0 for IPA instructions)

 * @tlb_level:	Translation Table level hint, if known

 * @tlbi_user:	If 'true', call an additional __tlbi_user()

 *              (typically for user ASIDs). 'flase' for IPA instructions

 *

 * When the CPU does not support TLB range operations, flush the TLB

 * entries one by one at the granularity of 'stride'. If the TLB

 * range ops are supported, then:

 *

 * 1. If 'pages' is odd, flush the first page through non-range

 *    operations;

 *

 * 2. For remaining pages: the minimum range granularity is decided

 *    by 'scale', so multiple range TLBI operations may be required.

 *    Start from scale = 0, flush the corresponding number of pages

 *    ((num+1)*2^(5*scale+1) starting from 'addr'), then increase it

 *    until no pages left.

 *

 * Note that certain ranges can be represented by either num = 31 and

 * scale or num = 0 and scale + 1. The loop below favours the latter

 * since num is limited to 30 by the __TLBI_RANGE_NUM() macro.

 */

#define __flush_tlb_range_op(op, start, pages, stride,			\

				asid, tlb_level, tlbi_user)		\

do {									\

	int num = 0;							\

	int scale = 0;							\

	unsigned long addr;						\

									\

	while (pages > 0) {						\

		if (!system_supports_tlb_range() ||			\

		    pages % 2 == 1) {					\

			addr = __TLBI_VADDR(start, asid);		\

			__tlbi_level(op, addr, tlb_level);		\

			if (tlbi_user)					\

				__tlbi_user_level(op, addr, tlb_level);	\

			start += stride;				\

			pages -= stride >> PAGE_SHIFT;			\

			continue;					\

		}							\

									\

		num = __TLBI_RANGE_NUM(pages, scale);			\

		if (num >= 0) {						\

			addr = __TLBI_VADDR_RANGE(start, asid, scale,	\

						  num, tlb_level);	\

			__tlbi(r##op, addr);				\

			if (tlbi_user)					\

				__tlbi_user(r##op, addr);		\

			start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \

			pages -= __TLBI_RANGE_PAGES(num, scale);	\

		}							\

		scale++;						\

	}								\

} while (0)



#define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \

	__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false)



static inline void __flush_tlb_range(struct vm_area_struct *vma,

				     unsigned long start, unsigned long end,

				     unsigned long stride, bool last_level,

				     int tlb_level)

{

	int num = 0;

	int scale = 0;

	unsigned long asid, addr, pages;

	unsigned long asid, pages;



	start = round_down(start, stride);

	end = round_up(end, stride);
@@ -307,56 +370,11 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma, 
	dsb(ishst);

	asid = ASID(vma->vm_mm);



	/*

	 * When the CPU does not support TLB range operations, flush the TLB

	 * entries one by one at the granularity of 'stride'. If the TLB

	 * range ops are supported, then:

	 *

	 * 1. If 'pages' is odd, flush the first page through non-range

	 *    operations;

	 *

	 * 2. For remaining pages: the minimum range granularity is decided

	 *    by 'scale', so multiple range TLBI operations may be required.

	 *    Start from scale = 0, flush the corresponding number of pages

	 *    ((num+1)*2^(5*scale+1) starting from 'addr'), then increase it

	 *    until no pages left.

	 *

	 * Note that certain ranges can be represented by either num = 31 and

	 * scale or num = 0 and scale + 1. The loop below favours the latter

	 * since num is limited to 30 by the __TLBI_RANGE_NUM() macro.

	 */

	while (pages > 0) {

		if (!system_supports_tlb_range() ||

		    pages % 2 == 1) {

			addr = __TLBI_VADDR(start, asid);

			if (last_level) {

				__tlbi_level(vale1is, addr, tlb_level);

				__tlbi_user_level(vale1is, addr, tlb_level);

			} else {

				__tlbi_level(vae1is, addr, tlb_level);

				__tlbi_user_level(vae1is, addr, tlb_level);

			}

			start += stride;

			pages -= stride >> PAGE_SHIFT;

			continue;

		}

	if (last_level)

		__flush_tlb_range_op(vale1is, start, pages, stride, asid, tlb_level, true);

	else

		__flush_tlb_range_op(vae1is, start, pages, stride, asid, tlb_level, true);



		num = __TLBI_RANGE_NUM(pages, scale);

		if (num >= 0) {

			addr = __TLBI_VADDR_RANGE(start, asid, scale,

						  num, tlb_level);

			if (last_level) {

				__tlbi(rvale1is, addr);

				__tlbi_user(rvale1is, addr);

			} else {

				__tlbi(rvae1is, addr);

				__tlbi_user(rvae1is, addr);

			}

			start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT;

			pages -= __TLBI_RANGE_PAGES(num, scale);

		}

		scale++;

	}

	dsb(ish);

}

arch/arm64/kvm/Kconfig

+0 −1
Original line number Diff line number Diff line
@@ -25,7 +25,6 @@ menuconfig KVM 
	select MMU_NOTIFIER

	select PREEMPT_NOTIFIERS

	select HAVE_KVM_CPU_RELAX_INTERCEPT

	select HAVE_KVM_ARCH_TLB_FLUSH_ALL

	select KVM_MMIO

	select KVM_GENERIC_DIRTYLOG_READ_PROTECT

	select KVM_XFER_TO_GUEST_WORK