Commit 0214feb8 authored by ZhangPeng's avatar ZhangPeng Committed by Peng Zhang
Browse files

mm/userswap: introduce UFFDIO_COPY_MODE_DIRECT_MAP 

hulk inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8KESX


CVE: NA

--------------------------------

Add a new UFFDIO_COPY mode UFFDIO_COPY_MODE_DIRECT_MAP to map physical
pages without copy_from_user(). We use uswap_unmap_anon_page() to unmap
an anonymous page and uswap_map_anon_page() to map page to src addr. We
introduce mfill_atomic_pte_nocopy() to achieve zero copy by unmapping
src_addr to the physical page and establishing the mapping from dst_addr
to the physical page.

Signed-off-by: default avatarZhangPeng <zhangpeng362@huawei.com>
parent b54b55c3

fs/userfaultfd.c

+7 −1
Original line number Diff line number Diff line
@@ -1797,10 +1797,16 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, 
		goto out;



	ret = -EINVAL;

	if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))

	if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE |

				 UFFDIO_COPY_MODE_WP |

				 IS_ENABLED(CONFIG_USERSWAP) ?

				 UFFDIO_COPY_MODE_DIRECT_MAP : 0))

		goto out;

	if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP)

		flags |= MFILL_ATOMIC_WP;

	if (IS_ENABLED(CONFIG_USERSWAP) &&

	    (uffdio_copy.mode & UFFDIO_COPY_MODE_DIRECT_MAP))

		flags |= MFILL_ATOMIC_DIRECT_MAP;

	if (mmget_not_zero(ctx->mm)) {

		ret = mfill_atomic_copy(ctx->mm, uffdio_copy.dst, uffdio_copy.src,

					uffdio_copy.len, &ctx->mmap_changing,

include/linux/userfaultfd_k.h

+5 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ enum mfill_atomic_mode { 
	MFILL_ATOMIC_ZEROPAGE,

	MFILL_ATOMIC_CONTINUE,

	MFILL_ATOMIC_POISON,

	MFILL_ATOMIC_DIRECT_MAP,

	NR_MFILL_ATOMIC_MODES,

};
@@ -62,6 +63,10 @@ static inline bool uffd_flags_mode_is(uffd_flags_t flags, enum mfill_atomic_mode 


static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_atomic_mode mode)

{

	if (IS_ENABLED(CONFIG_USERSWAP) && (flags & MFILL_ATOMIC_DIRECT_MAP) &&

	    uffd_flags_mode_is(mode, MFILL_ATOMIC_COPY))

		mode = MFILL_ATOMIC_DIRECT_MAP;



	flags &= ~MFILL_ATOMIC_MODE_MASK;

	return flags | ((__force uffd_flags_t) mode);

}

include/linux/userswap.h

+13 −0
Original line number Diff line number Diff line
@@ -29,6 +29,10 @@ bool uswap_adjust_uffd_range(struct uffdio_register *uffdio_register, 
vm_fault_t do_uswap_page(swp_entry_t entry, struct vm_fault *vmf,

			 struct vm_area_struct *vma);



int mfill_atomic_pte_nocopy(struct mm_struct *dst_mm, pmd_t *dst_pmd,

			    struct vm_area_struct *dst_vma,

			    unsigned long dst_addr, unsigned long src_addr);



static inline void uswap_must_wait(unsigned long reason, pte_t pte, bool *ret)

{

	if (!static_branch_unlikely(&userswap_enabled))
@@ -37,5 +41,14 @@ static inline void uswap_must_wait(unsigned long reason, pte_t pte, bool *ret) 
		*ret = true;

}



static inline bool uswap_check_copy(struct vm_area_struct *vma,

				    uffd_flags_t flags)

{

	if (!!uffd_flags_mode_is(flags, MFILL_ATOMIC_DIRECT_MAP) ^

	    !!(vma->vm_flags & VM_USWAP))

		return false;

	return true;

}



#endif /* CONFIG_USERSWAP */

#endif /* _LINUX_USERSWAP_H */

include/uapi/linux/userfaultfd.h

+1 −0
Original line number Diff line number Diff line
@@ -270,6 +270,7 @@ struct uffdio_copy { 
	 * according to the uffdio_register.ioctls.

	 */

#define UFFDIO_COPY_MODE_WP			((__u64)1<<1)

#define UFFDIO_COPY_MODE_DIRECT_MAP		((__u64)1<<10)

	__u64 mode;



	/*

mm/userfaultfd.c

+12 −0
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ 
#include <linux/mmu_notifier.h>

#include <linux/hugetlb.h>

#include <linux/shmem_fs.h>

#include <linux/userswap.h>

#include <asm/tlbflush.h>

#include <asm/tlb.h>

#include "internal.h"
@@ -603,6 +604,10 @@ static __always_inline ssize_t mfill_atomic(struct mm_struct *dst_mm, 
		goto out_unlock;



	err = -EINVAL;

#ifdef CONFIG_USERSWAP

	if (!uswap_check_copy(dst_vma, flags))

		goto out_unlock;

#endif

	/*

	 * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but

	 * it will overwrite vm_ops, so vma_is_anonymous must return false.
@@ -675,6 +680,13 @@ static __always_inline ssize_t mfill_atomic(struct mm_struct *dst_mm, 
		BUG_ON(pmd_none(*dst_pmd));

		BUG_ON(pmd_trans_huge(*dst_pmd));



#ifdef CONFIG_USERSWAP

		if (static_branch_unlikely(&userswap_enabled) &&

		    uffd_flags_mode_is(flags, MFILL_ATOMIC_DIRECT_MAP))

			err = mfill_atomic_pte_nocopy(dst_mm, dst_pmd, dst_vma,

						      dst_addr, src_addr);

		else

#endif

		err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr,

				       src_addr, flags, &folio);

		cond_resched();