Commit 80217147 authored by Martin Schwidefsky's avatar Martin Schwidefsky Committed by Martin Schwidefsky
Browse files

[S390] lockless get_user_pages_fast() 



Implement get_user_pages_fast without locking in the fastpath on s390.

Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent 87799eba

arch/s390/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -101,6 +101,7 @@ config S390 
	select HAVE_KERNEL_BZIP2

	select HAVE_KERNEL_LZMA

	select HAVE_KERNEL_LZO

	select HAVE_GET_USER_PAGES_FAST

	select ARCH_INLINE_SPIN_TRYLOCK

	select ARCH_INLINE_SPIN_TRYLOCK_BH

	select ARCH_INLINE_SPIN_LOCK

arch/s390/include/asm/pgalloc.h

+4 −0
Original line number Diff line number Diff line
@@ -21,9 +21,11 @@ 


unsigned long *crst_table_alloc(struct mm_struct *, int);

void crst_table_free(struct mm_struct *, unsigned long *);

void crst_table_free_rcu(struct mm_struct *, unsigned long *);



unsigned long *page_table_alloc(struct mm_struct *);

void page_table_free(struct mm_struct *, unsigned long *);

void page_table_free_rcu(struct mm_struct *, unsigned long *);

void disable_noexec(struct mm_struct *, struct task_struct *);



static inline void clear_table(unsigned long *s, unsigned long val, size_t n)
@@ -176,4 +178,6 @@ static inline void pmd_populate(struct mm_struct *mm, 
#define pte_free_kernel(mm, pte) page_table_free(mm, (unsigned long *) pte)

#define pte_free(mm, pte) page_table_free(mm, (unsigned long *) pte)



extern void rcu_table_freelist_finish(void);



#endif /* _S390_PGALLOC_H */

arch/s390/include/asm/pgtable.h

+1 −0
Original line number Diff line number Diff line
@@ -316,6 +316,7 @@ extern unsigned long VMALLOC_START; 


/* Bits in the segment table entry */

#define _SEGMENT_ENTRY_ORIGIN	0x7fffffc0UL	/* page table origin	    */

#define _SEGMENT_ENTRY_RO	0x200	/* page protection bit		    */

#define _SEGMENT_ENTRY_INV	0x20	/* invalid segment table entry	    */

#define _SEGMENT_ENTRY_COMMON	0x10	/* common segment bit		    */

#define _SEGMENT_ENTRY_PTL	0x0f	/* page table length		    */

arch/s390/include/asm/tlb.h

+7 −6
Original line number Diff line number Diff line
@@ -64,10 +64,9 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb, 
	if (!tlb->fullmm && (tlb->nr_ptes > 0 || tlb->nr_pxds < TLB_NR_PTRS))

		__tlb_flush_mm(tlb->mm);

	while (tlb->nr_ptes > 0)

		pte_free(tlb->mm, tlb->array[--tlb->nr_ptes]);

		page_table_free_rcu(tlb->mm, tlb->array[--tlb->nr_ptes]);

	while (tlb->nr_pxds < TLB_NR_PTRS)

		/* pgd_free frees the pointer as region or segment table */

		pgd_free(tlb->mm, tlb->array[tlb->nr_pxds++]);

		crst_table_free_rcu(tlb->mm, tlb->array[tlb->nr_pxds++]);

}



static inline void tlb_finish_mmu(struct mmu_gather *tlb,
@@ -75,6 +74,8 @@ static inline void tlb_finish_mmu(struct mmu_gather *tlb, 
{

	tlb_flush_mmu(tlb, start, end);



	rcu_table_freelist_finish();



	/* keep the page table cache within bounds */

	check_pgt_cache();
@@ -103,7 +104,7 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 
		if (tlb->nr_ptes >= tlb->nr_pxds)

			tlb_flush_mmu(tlb, 0, 0);

	} else

		pte_free(tlb->mm, pte);

		page_table_free(tlb->mm, (unsigned long *) pte);

}



/*
@@ -124,7 +125,7 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 
		if (tlb->nr_ptes >= tlb->nr_pxds)

			tlb_flush_mmu(tlb, 0, 0);

	} else

		pmd_free(tlb->mm, pmd);

		crst_table_free(tlb->mm, (unsigned long *) pmd);

#endif

}
@@ -146,7 +147,7 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 
		if (tlb->nr_ptes >= tlb->nr_pxds)

			tlb_flush_mmu(tlb, 0, 0);

	} else

		pud_free(tlb->mm, pud);

		crst_table_free(tlb->mm, (unsigned long *) pud);

#endif

}

arch/s390/mm/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -3,6 +3,6 @@ 
#



obj-y	 := init.o fault.o extmem.o mmap.o vmem.o pgtable.o maccess.o \

	    page-states.o

	    page-states.o gup.o

obj-$(CONFIG_CMM) += cmm.o

obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o