Unverified Commit 3f1e7829 authored by Guo Ren's avatar Guo Ren Committed by Palmer Dabbelt
Browse files

riscv: add ASID-based tlbflushing methods 



Implement optimized version of the tlb flushing routines for systems
using ASIDs. These are behind the use_asid_allocator static branch to
not affect existing systems not using ASIDs.

Signed-off-by: default avatarGuo Ren <guoren@linux.alibaba.com>
[hch: rebased on top of previous cleanups, use the same algorithm as
      the non-ASID based code for local vs global flushes, keep functions
      as local as possible]
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Tested-by: default avatarGuo Ren <guoren@kernel.org>
Signed-off-by: default avatarPalmer Dabbelt <palmerdabbelt@google.com>
parent 70c7605c

arch/riscv/include/asm/mmu_context.h

+2 −0
Original line number Diff line number Diff line
@@ -33,6 +33,8 @@ static inline int init_new_context(struct task_struct *tsk, 
	return 0;

}



DECLARE_STATIC_KEY_FALSE(use_asid_allocator);



#include <asm-generic/mmu_context.h>



#endif /* _ASM_RISCV_MMU_CONTEXT_H */

arch/riscv/mm/context.c

+1 −1
Original line number Diff line number Diff line
@@ -18,7 +18,7 @@ 


#ifdef CONFIG_MMU



static DEFINE_STATIC_KEY_FALSE(use_asid_allocator);

DEFINE_STATIC_KEY_FALSE(use_asid_allocator);



static unsigned long asid_bits;

static unsigned long num_asids;

arch/riscv/mm/tlbflush.c

+40 −7
Original line number Diff line number Diff line
@@ -4,6 +4,24 @@ 
#include <linux/smp.h>

#include <linux/sched.h>

#include <asm/sbi.h>

#include <asm/mmu_context.h>



static inline void local_flush_tlb_all_asid(unsigned long asid)

{

	__asm__ __volatile__ ("sfence.vma x0, %0"

			:

			: "r" (asid)

			: "memory");

}



static inline void local_flush_tlb_page_asid(unsigned long addr,

		unsigned long asid)

{

	__asm__ __volatile__ ("sfence.vma %0, %1"

			:

			: "r" (addr), "r" (asid)

			: "memory");

}



void flush_tlb_all(void)

{
@@ -16,21 +34,36 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start, 
	struct cpumask *cmask = mm_cpumask(mm);

	struct cpumask hmask;

	unsigned int cpuid;

	bool broadcast;



	if (cpumask_empty(cmask))

		return;



	cpuid = get_cpu();

	/* check if the tlbflush needs to be sent to other CPUs */

	broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;

	if (static_branch_unlikely(&use_asid_allocator)) {

		unsigned long asid = atomic_long_read(&mm->context.id);



	if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {

		/* local cpu is the only cpu present in cpumask */

		if (size <= stride)

			local_flush_tlb_page(start);

		else

			local_flush_tlb_all();

		if (broadcast) {

			riscv_cpuid_to_hartid_mask(cmask, &hmask);

			sbi_remote_sfence_vma_asid(cpumask_bits(&hmask),

						   start, size, asid);

		} else if (size <= stride) {

			local_flush_tlb_page_asid(start, asid);

		} else {

			local_flush_tlb_all_asid(asid);

		}

	} else {

		if (broadcast) {

			riscv_cpuid_to_hartid_mask(cmask, &hmask);

		sbi_remote_sfence_vma(cpumask_bits(&hmask), start, size);

			sbi_remote_sfence_vma(cpumask_bits(&hmask),

					      start, size);

		} else if (size <= stride) {

			local_flush_tlb_page(start);

		} else {

			local_flush_tlb_all();

		}

	}



	put_cpu();