Commit f1581bf1 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Benjamin Herrenschmidt
Browse files

powerpc/mm/thp: Remove code duplication 



Rename invalidate_old_hpte to flush_hash_hugepage and use that in
other places.

Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent c4f3eb5f

arch/powerpc/include/asm/tlbflush.h

+2 −1
Original line number Diff line number Diff line
@@ -127,7 +127,8 @@ static inline void arch_leave_lazy_mmu_mode(void) 
extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize,

			    int ssize, int local);

extern void flush_hash_range(unsigned long number, int local);



extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr,

				pmd_t *pmdp, unsigned int psize, int ssize);



static inline void local_flush_tlb_mm(struct mm_struct *mm)

{

arch/powerpc/mm/hash_utils_64.c

+52 −0
Original line number Diff line number Diff line
@@ -1315,6 +1315,58 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize, 
#endif

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

void flush_hash_hugepage(unsigned long vsid, unsigned long addr,

			 pmd_t *pmdp, unsigned int psize, int ssize)

{

	int i, max_hpte_count, valid;

	unsigned long s_addr;

	unsigned char *hpte_slot_array;

	unsigned long hidx, shift, vpn, hash, slot;



	s_addr = addr & HPAGE_PMD_MASK;

	hpte_slot_array = get_hpte_slot_array(pmdp);

	/*

	 * IF we try to do a HUGE PTE update after a withdraw is done.

	 * we will find the below NULL. This happens when we do

	 * split_huge_page_pmd

	 */

	if (!hpte_slot_array)

		return;



	if (ppc_md.hugepage_invalidate)

		return ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,

						  psize, ssize);

	/*

	 * No bluk hpte removal support, invalidate each entry

	 */

	shift = mmu_psize_defs[psize].shift;

	max_hpte_count = HPAGE_PMD_SIZE >> shift;

	for (i = 0; i < max_hpte_count; i++) {

		/*

		 * 8 bits per each hpte entries

		 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]

		 */

		valid = hpte_valid(hpte_slot_array, i);

		if (!valid)

			continue;

		hidx =  hpte_hash_index(hpte_slot_array, i);



		/* get the vpn */

		addr = s_addr + (i * (1ul << shift));

		vpn = hpt_vpn(addr, vsid, ssize);

		hash = hpt_hash(vpn, shift, ssize);

		if (hidx & _PTEIDX_SECONDARY)

			hash = ~hash;



		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;

		slot += hidx & _PTEIDX_GROUP_IX;

		ppc_md.hpte_invalidate(slot, vpn, psize,

				       MMU_PAGE_16M, ssize, 0);

	}

}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



void flush_hash_range(unsigned long number, int local)

{

	if (ppc_md.flush_hash_range)

arch/powerpc/mm/hugepage-hash64.c

+2 −52
Original line number Diff line number Diff line
@@ -18,57 +18,6 @@ 
#include <linux/mm.h>

#include <asm/machdep.h>



static void invalidate_old_hpte(unsigned long vsid, unsigned long addr,

				pmd_t *pmdp, unsigned int psize, int ssize)

{

	int i, max_hpte_count, valid;

	unsigned long s_addr;

	unsigned char *hpte_slot_array;

	unsigned long hidx, shift, vpn, hash, slot;



	s_addr = addr & HPAGE_PMD_MASK;

	hpte_slot_array = get_hpte_slot_array(pmdp);

	/*

	 * IF we try to do a HUGE PTE update after a withdraw is done.

	 * we will find the below NULL. This happens when we do

	 * split_huge_page_pmd

	 */

	if (!hpte_slot_array)

		return;



	if (ppc_md.hugepage_invalidate)

		return ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,

						  psize, ssize);

	/*

	 * No bluk hpte removal support, invalidate each entry

	 */

	shift = mmu_psize_defs[psize].shift;

	max_hpte_count = HPAGE_PMD_SIZE >> shift;

	for (i = 0; i < max_hpte_count; i++) {

		/*

		 * 8 bits per each hpte entries

		 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]

		 */

		valid = hpte_valid(hpte_slot_array, i);

		if (!valid)

			continue;

		hidx =  hpte_hash_index(hpte_slot_array, i);



		/* get the vpn */

		addr = s_addr + (i * (1ul << shift));

		vpn = hpt_vpn(addr, vsid, ssize);

		hash = hpt_hash(vpn, shift, ssize);

		if (hidx & _PTEIDX_SECONDARY)

			hash = ~hash;



		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;

		slot += hidx & _PTEIDX_GROUP_IX;

		ppc_md.hpte_invalidate(slot, vpn, psize,

				       MMU_PAGE_16M, ssize, 0);

	}

}





int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid,

		    pmd_t *pmdp, unsigned long trap, int local, int ssize,

		    unsigned int psize)
@@ -145,7 +94,8 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, 
		 * hash page table entries.

		 */

		if ((old_pmd & _PAGE_HASHPTE) && !(old_pmd & _PAGE_COMBO))

			invalidate_old_hpte(vsid, ea, pmdp, MMU_PAGE_64K, ssize);

			flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K,

					    ssize);

	}



	valid = hpte_valid(hpte_slot_array, index);

arch/powerpc/mm/pgtable_64.c

+9 −55
Original line number Diff line number Diff line
@@ -739,29 +739,13 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 
void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,

			    pmd_t *pmdp, unsigned long old_pmd)

{

	int ssize, i;

	unsigned long s_addr;

	int max_hpte_count;

	unsigned int psize, valid;

	unsigned char *hpte_slot_array;

	unsigned long hidx, vpn, vsid, hash, shift, slot;



	/*

	 * Flush all the hptes mapping this hugepage

	 */

	s_addr = addr & HPAGE_PMD_MASK;

	hpte_slot_array = get_hpte_slot_array(pmdp);

	/*

	 * IF we try to do a HUGE PTE update after a withdraw is done.

	 * we will find the below NULL. This happens when we do

	 * split_huge_page_pmd

	 */

	if (!hpte_slot_array)

		return;

	int ssize;

	unsigned int psize;

	unsigned long vsid;



	/* get the base page size,vsid and segment size */

#ifdef CONFIG_DEBUG_VM

	psize = get_slice_psize(mm, s_addr);

	psize = get_slice_psize(mm, addr);

	BUG_ON(psize == MMU_PAGE_16M);

#endif

	if (old_pmd & _PAGE_COMBO)
@@ -769,46 +753,16 @@ void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, 
	else

		psize = MMU_PAGE_64K;



	if (!is_kernel_addr(s_addr)) {

		ssize = user_segment_size(s_addr);

		vsid = get_vsid(mm->context.id, s_addr, ssize);

	if (!is_kernel_addr(addr)) {

		ssize = user_segment_size(addr);

		vsid = get_vsid(mm->context.id, addr, ssize);

		WARN_ON(vsid == 0);

	} else {

		vsid = get_kernel_vsid(s_addr, mmu_kernel_ssize);

		vsid = get_kernel_vsid(addr, mmu_kernel_ssize);

		ssize = mmu_kernel_ssize;

	}



	if (ppc_md.hugepage_invalidate)

		return ppc_md.hugepage_invalidate(vsid, s_addr,

						  hpte_slot_array,

						  psize, ssize);

	/*

	 * No bluk hpte removal support, invalidate each entry

	 */

	shift = mmu_psize_defs[psize].shift;

	max_hpte_count = HPAGE_PMD_SIZE >> shift;

	for (i = 0; i < max_hpte_count; i++) {

		/*

		 * 8 bits per each hpte entries

		 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]

		 */

		valid = hpte_valid(hpte_slot_array, i);

		if (!valid)

			continue;

		hidx =  hpte_hash_index(hpte_slot_array, i);



		/* get the vpn */

		addr = s_addr + (i * (1ul << shift));

		vpn = hpt_vpn(addr, vsid, ssize);

		hash = hpt_hash(vpn, shift, ssize);

		if (hidx & _PTEIDX_SECONDARY)

			hash = ~hash;



		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;

		slot += hidx & _PTEIDX_GROUP_IX;

		ppc_md.hpte_invalidate(slot, vpn, psize,

				       MMU_PAGE_16M, ssize, 0);

	}

	return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize);

}



static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)