Commit c2436190 authored by Shiyang Ruan's avatar Shiyang Ruan Committed by Darrick J. Wong
Browse files

fsdax: factor out a dax_fault_actor() helper 



The core logic in the two dax page fault functions is similar. So, move
the logic into a common helper function. Also, to facilitate the
addition of new features, such as CoW, switch-case is no longer used to
handle different iomap types.

Signed-off-by: default avatarShiyang Ruan <ruansy.fnst@fujitsu.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarRitesh Harjani <riteshh@linux.ibm.com>
Reviewed-by: default avatarDarrick J. Wong <djwong@kernel.org>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarDarrick J. Wong <djwong@kernel.org>
parent 55f81639

fs/dax.c

+149 −148
Original line number Diff line number Diff line
@@ -1066,6 +1066,66 @@ static vm_fault_t dax_load_hole(struct xa_state *xas, 
	return ret;

}



#ifdef CONFIG_FS_DAX_PMD

static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,

		struct iomap *iomap, void **entry)

{

	struct address_space *mapping = vmf->vma->vm_file->f_mapping;

	unsigned long pmd_addr = vmf->address & PMD_MASK;

	struct vm_area_struct *vma = vmf->vma;

	struct inode *inode = mapping->host;

	pgtable_t pgtable = NULL;

	struct page *zero_page;

	spinlock_t *ptl;

	pmd_t pmd_entry;

	pfn_t pfn;



	zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);



	if (unlikely(!zero_page))

		goto fallback;



	pfn = page_to_pfn_t(zero_page);

	*entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,

			DAX_PMD | DAX_ZERO_PAGE, false);



	if (arch_needs_pgtable_deposit()) {

		pgtable = pte_alloc_one(vma->vm_mm);

		if (!pgtable)

			return VM_FAULT_OOM;

	}



	ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);

	if (!pmd_none(*(vmf->pmd))) {

		spin_unlock(ptl);

		goto fallback;

	}



	if (pgtable) {

		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);

		mm_inc_nr_ptes(vma->vm_mm);

	}

	pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);

	pmd_entry = pmd_mkhuge(pmd_entry);

	set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);

	spin_unlock(ptl);

	trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry);

	return VM_FAULT_NOPAGE;



fallback:

	if (pgtable)

		pte_free(vma->vm_mm, pgtable);

	trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry);

	return VM_FAULT_FALLBACK;

}

#else

static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,

		struct iomap *iomap, void **entry)

{

	return VM_FAULT_FALLBACK;

}

#endif /* CONFIG_FS_DAX_PMD */



s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap)

{

	sector_t sector = iomap_sector(iomap, pos & PAGE_MASK);
@@ -1302,6 +1362,63 @@ static vm_fault_t dax_fault_cow_page(struct vm_fault *vmf, struct iomap *iomap, 
	return ret;

}



/**

 * dax_fault_actor - Common actor to handle pfn insertion in PTE/PMD fault.

 * @vmf:	vm fault instance

 * @pfnp:	pfn to be returned

 * @xas:	the dax mapping tree of a file

 * @entry:	an unlocked dax entry to be inserted

 * @pmd:	distinguish whether it is a pmd fault

 * @flags:	iomap flags

 * @iomap:	from iomap_begin()

 * @srcmap:	from iomap_begin(), not equal to iomap if it is a CoW

 */

static vm_fault_t dax_fault_actor(struct vm_fault *vmf, pfn_t *pfnp,

		struct xa_state *xas, void **entry, bool pmd,

		unsigned int flags, struct iomap *iomap, struct iomap *srcmap)

{

	struct address_space *mapping = vmf->vma->vm_file->f_mapping;

	size_t size = pmd ? PMD_SIZE : PAGE_SIZE;

	loff_t pos = (loff_t)xas->xa_index << PAGE_SHIFT;

	bool write = vmf->flags & FAULT_FLAG_WRITE;

	bool sync = dax_fault_is_synchronous(flags, vmf->vma, iomap);

	unsigned long entry_flags = pmd ? DAX_PMD : 0;

	int err = 0;

	pfn_t pfn;



	/* if we are reading UNWRITTEN and HOLE, return a hole. */

	if (!write &&

	    (iomap->type == IOMAP_UNWRITTEN || iomap->type == IOMAP_HOLE)) {

		if (!pmd)

			return dax_load_hole(xas, mapping, entry, vmf);

		return dax_pmd_load_hole(xas, vmf, iomap, entry);

	}



	if (iomap->type != IOMAP_MAPPED) {

		WARN_ON_ONCE(1);

		return pmd ? VM_FAULT_FALLBACK : VM_FAULT_SIGBUS;

	}



	err = dax_iomap_pfn(iomap, pos, size, &pfn);

	if (err)

		return pmd ? VM_FAULT_FALLBACK : dax_fault_return(err);



	*entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn, entry_flags,

				  write && !sync);



	if (sync)

		return dax_fault_synchronous_pfnp(pfnp, pfn);



	/* insert PMD pfn */

	if (pmd)

		return vmf_insert_pfn_pmd(vmf, pfn, write);



	/* insert PTE pfn */

	if (write)

		return vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);

	return vmf_insert_mixed(vmf->vma, vmf->address, pfn);

}



static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,

			       int *iomap_errp, const struct iomap_ops *ops)

{
@@ -1309,17 +1426,14 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, 
	struct address_space *mapping = vma->vm_file->f_mapping;

	XA_STATE(xas, &mapping->i_pages, vmf->pgoff);

	struct inode *inode = mapping->host;

	unsigned long vaddr = vmf->address;

	loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;

	struct iomap iomap = { .type = IOMAP_HOLE };

	struct iomap srcmap = { .type = IOMAP_HOLE };

	unsigned flags = IOMAP_FAULT;

	int error, major = 0;

	int error;

	bool write = vmf->flags & FAULT_FLAG_WRITE;

	bool sync;

	vm_fault_t ret = 0;

	vm_fault_t ret = 0, major = 0;

	void *entry;

	pfn_t pfn;



	trace_dax_pte_fault(inode, vmf, ret);

	/*
@@ -1365,8 +1479,8 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, 
		goto unlock_entry;

	}

	if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {

		error = -EIO;	/* fs corruption? */

		goto error_finish_iomap;

		ret = VM_FAULT_SIGBUS;	/* fs corruption? */

		goto finish_iomap;

	}



	if (vmf->cow_page) {
@@ -1374,48 +1488,18 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, 
		goto finish_iomap;

	}



	sync = dax_fault_is_synchronous(flags, vma, &iomap);

	ret = dax_fault_actor(vmf, pfnp, &xas, &entry, false, flags,

			      &iomap, &srcmap);

	if (ret == VM_FAULT_SIGBUS)

		goto finish_iomap;



	switch (iomap.type) {

	case IOMAP_MAPPED:

	/* read/write MAPPED, CoW UNWRITTEN */

	if (iomap.flags & IOMAP_F_NEW) {

		count_vm_event(PGMAJFAULT);

		count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);

		major = VM_FAULT_MAJOR;

	}

		error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);

		if (error < 0)

			goto error_finish_iomap;



		entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,

						 0, write && !sync);



		if (sync) {

			ret = dax_fault_synchronous_pfnp(pfnp, pfn);

			goto finish_iomap;

		}

		trace_dax_insert_mapping(inode, vmf, entry);

		if (write)

			ret = vmf_insert_mixed_mkwrite(vma, vaddr, pfn);

		else

			ret = vmf_insert_mixed(vma, vaddr, pfn);



		goto finish_iomap;

	case IOMAP_UNWRITTEN:

	case IOMAP_HOLE:

		if (!write) {

			ret = dax_load_hole(&xas, mapping, &entry, vmf);

			goto finish_iomap;

		}

		fallthrough;

	default:

		WARN_ON_ONCE(1);

		error = -EIO;

		break;

	}



 error_finish_iomap:

	ret = dax_fault_return(error);

finish_iomap:

	if (ops->iomap_end) {

		int copied = PAGE_SIZE;
@@ -1438,58 +1522,6 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp, 
}



#ifdef CONFIG_FS_DAX_PMD

static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,

		struct iomap *iomap, void **entry)

{

	struct address_space *mapping = vmf->vma->vm_file->f_mapping;

	unsigned long pmd_addr = vmf->address & PMD_MASK;

	struct vm_area_struct *vma = vmf->vma;

	struct inode *inode = mapping->host;

	pgtable_t pgtable = NULL;

	struct page *zero_page;

	spinlock_t *ptl;

	pmd_t pmd_entry;

	pfn_t pfn;



	zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);



	if (unlikely(!zero_page))

		goto fallback;



	pfn = page_to_pfn_t(zero_page);

	*entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,

			DAX_PMD | DAX_ZERO_PAGE, false);



	if (arch_needs_pgtable_deposit()) {

		pgtable = pte_alloc_one(vma->vm_mm);

		if (!pgtable)

			return VM_FAULT_OOM;

	}



	ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);

	if (!pmd_none(*(vmf->pmd))) {

		spin_unlock(ptl);

		goto fallback;

	}



	if (pgtable) {

		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);

		mm_inc_nr_ptes(vma->vm_mm);

	}

	pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);

	pmd_entry = pmd_mkhuge(pmd_entry);

	set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);

	spin_unlock(ptl);

	trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry);

	return VM_FAULT_NOPAGE;



fallback:

	if (pgtable)

		pte_free(vma->vm_mm, pgtable);

	trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry);

	return VM_FAULT_FALLBACK;

}



static bool dax_fault_check_fallback(struct vm_fault *vmf, struct xa_state *xas,

		pgoff_t max_pgoff)

{
@@ -1530,17 +1562,15 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 
	struct address_space *mapping = vma->vm_file->f_mapping;

	XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);

	bool write = vmf->flags & FAULT_FLAG_WRITE;

	bool sync;

	unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;

	unsigned int flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;

	struct inode *inode = mapping->host;

	vm_fault_t result = VM_FAULT_FALLBACK;

	vm_fault_t ret = VM_FAULT_FALLBACK;

	struct iomap iomap = { .type = IOMAP_HOLE };

	struct iomap srcmap = { .type = IOMAP_HOLE };

	pgoff_t max_pgoff;

	void *entry;

	loff_t pos;

	int error;

	pfn_t pfn;



	/*

	 * Check whether offset isn't beyond end of file now. Caller is
@@ -1552,7 +1582,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 
	trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);



	if (xas.xa_index >= max_pgoff) {

		result = VM_FAULT_SIGBUS;

		ret = VM_FAULT_SIGBUS;

		goto out;

	}
@@ -1567,7 +1597,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 
	 */

	entry = grab_mapping_entry(&xas, mapping, PMD_ORDER);

	if (xa_is_internal(entry)) {

		result = xa_to_internal(entry);

		ret = xa_to_internal(entry);

		goto fallback;

	}
@@ -1579,7 +1609,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 
	 */

	if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&

			!pmd_devmap(*vmf->pmd)) {

		result = 0;

		ret = 0;

		goto unlock_entry;

	}
@@ -1589,49 +1619,21 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 
	 * to look up our filesystem block.

	 */

	pos = (loff_t)xas.xa_index << PAGE_SHIFT;

	error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap,

			&srcmap);

	error = ops->iomap_begin(inode, pos, PMD_SIZE, flags, &iomap, &srcmap);

	if (error)

		goto unlock_entry;



	if (iomap.offset + iomap.length < pos + PMD_SIZE)

		goto finish_iomap;



	sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);



	switch (iomap.type) {

	case IOMAP_MAPPED:

		error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);

		if (error < 0)

			goto finish_iomap;



		entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,

						DAX_PMD, write && !sync);



		if (sync) {

			result = dax_fault_synchronous_pfnp(pfnp, pfn);

			goto finish_iomap;

		}



		trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);

		result = vmf_insert_pfn_pmd(vmf, pfn, write);

		break;

	case IOMAP_UNWRITTEN:

	case IOMAP_HOLE:

		if (WARN_ON_ONCE(write))

			break;

		result = dax_pmd_load_hole(&xas, vmf, &iomap, &entry);

		break;

	default:

		WARN_ON_ONCE(1);

		break;

	}

	ret = dax_fault_actor(vmf, pfnp, &xas, &entry, true, flags,

			      &iomap, &srcmap);



finish_iomap:

	if (ops->iomap_end) {

		int copied = PMD_SIZE;



		if (result == VM_FAULT_FALLBACK)

		if (ret == VM_FAULT_FALLBACK)

			copied = 0;

		/*

		 * The fault is done by now and there's no way back (other
@@ -1639,19 +1641,18 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, 
		 * Just ignore error from ->iomap_end since we cannot do much

		 * with it.

		 */

		ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,

				&iomap);

		ops->iomap_end(inode, pos, PMD_SIZE, copied, flags, &iomap);

	}

unlock_entry:

	dax_unlock_entry(&xas, entry);

fallback:

	if (result == VM_FAULT_FALLBACK) {

	if (ret == VM_FAULT_FALLBACK) {

		split_huge_pmd(vma, vmf->pmd, vmf->address);

		count_vm_event(THP_FAULT_FALLBACK);

	}

out:

	trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);

	return result;

	trace_dax_pmd_fault_done(inode, vmf, max_pgoff, ret);

	return ret;

}

#else

static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,