Commit 4f661701 authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle)
Browse files

filemap: Allow __filemap_get_folio to allocate large folios 



Allow callers of __filemap_get_folio() to specify a preferred folio
order in the FGP flags.  This is only honoured in the FGP_CREATE path;
if there is already a folio in the page cache that covers the index,
we will return it, no matter what its order is.  No create-around is
attempted; we will only create folios which start at the specified index.
Unmodified callers will continue to allocate order 0 folios.

Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarDarrick J. Wong <djwong@kernel.org>
parent ffc143db

include/linux/pagemap.h

+34 −0
Original line number Diff line number Diff line
@@ -470,6 +470,19 @@ static inline void *detach_page_private(struct page *page) 
	return folio_detach_private(page_folio(page));

}



/*

 * There are some parts of the kernel which assume that PMD entries

 * are exactly HPAGE_PMD_ORDER.  Those should be fixed, but until then,

 * limit the maximum allocation order to PMD size.  I'm not aware of any

 * assumptions about maximum order if THP are disabled, but 8 seems like

 * a good order (that's 1MB if you're using 4kB pages)

 */

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define MAX_PAGECACHE_ORDER	HPAGE_PMD_ORDER

#else

#define MAX_PAGECACHE_ORDER	8

#endif



#ifdef CONFIG_NUMA

struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order);

#else
@@ -535,9 +548,30 @@ typedef unsigned int __bitwise fgf_t; 
#define FGP_NOWAIT		((__force fgf_t)0x00000020)

#define FGP_FOR_MMAP		((__force fgf_t)0x00000040)

#define FGP_STABLE		((__force fgf_t)0x00000080)

#define FGF_GET_ORDER(fgf)	(((__force unsigned)fgf) >> 26)	/* top 6 bits */



#define FGP_WRITEBEGIN		(FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE)



/**

 * fgf_set_order - Encode a length in the fgf_t flags.

 * @size: The suggested size of the folio to create.

 *

 * The caller of __filemap_get_folio() can use this to suggest a preferred

 * size for the folio that is created.  If there is already a folio at

 * the index, it will be returned, no matter what its size.  If a folio

 * is freshly created, it may be of a different size than requested

 * due to alignment constraints, memory pressure, or the presence of

 * other folios at nearby indices.

 */

static inline fgf_t fgf_set_order(size_t size)

{

	unsigned int shift = ilog2(size);



	if (shift <= PAGE_SHIFT)

		return 0;

	return (__force fgf_t)((shift - PAGE_SHIFT) << 26);

}



void *filemap_get_entry(struct address_space *mapping, pgoff_t index);

struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,

		fgf_t fgp_flags, gfp_t gfp);

mm/filemap.c

+33 −13
Original line number Diff line number Diff line
@@ -1905,7 +1905,9 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, 
		folio_wait_stable(folio);

no_page:

	if (!folio && (fgp_flags & FGP_CREAT)) {

		unsigned order = FGF_GET_ORDER(fgp_flags);

		int err;



		if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))

			gfp |= __GFP_WRITE;

		if (fgp_flags & FGP_NOFS)
@@ -1914,26 +1916,44 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, 
			gfp &= ~GFP_KERNEL;

			gfp |= GFP_NOWAIT | __GFP_NOWARN;

		}



		folio = filemap_alloc_folio(gfp, 0);

		if (!folio)

			return ERR_PTR(-ENOMEM);



		if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP))))

			fgp_flags |= FGP_LOCK;



		if (!mapping_large_folio_support(mapping))

			order = 0;

		if (order > MAX_PAGECACHE_ORDER)

			order = MAX_PAGECACHE_ORDER;

		/* If we're not aligned, allocate a smaller folio */

		if (index & ((1UL << order) - 1))

			order = __ffs(index);



		do {

			gfp_t alloc_gfp = gfp;



			err = -ENOMEM;

			if (order == 1)

				order = 0;

			if (order > 0)

				alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN;

			folio = filemap_alloc_folio(alloc_gfp, order);

			if (!folio)

				continue;



			/* Init accessed so avoid atomic mark_page_accessed later */

			if (fgp_flags & FGP_ACCESSED)

				__folio_set_referenced(folio);



			err = filemap_add_folio(mapping, folio, index, gfp);

		if (unlikely(err)) {

			if (!err)

				break;

			folio_put(folio);

			folio = NULL;

		} while (order-- > 0);



		if (err == -EEXIST)

			goto repeat;

		}



		if (err)

			return ERR_PTR(err);

		/*

		 * filemap_add_folio locks the page, and for mmap

		 * we expect an unlocked page.

mm/readahead.c

+0 −13
Original line number Diff line number Diff line
@@ -461,19 +461,6 @@ static int try_context_readahead(struct address_space *mapping, 
	return 1;

}



/*

 * There are some parts of the kernel which assume that PMD entries

 * are exactly HPAGE_PMD_ORDER.  Those should be fixed, but until then,

 * limit the maximum allocation order to PMD size.  I'm not aware of any

 * assumptions about maximum order if THP are disabled, but 8 seems like

 * a good order (that's 1MB if you're using 4kB pages)

 */

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#define MAX_PAGECACHE_ORDER	HPAGE_PMD_ORDER

#else

#define MAX_PAGECACHE_ORDER	8

#endif



static inline int ra_alloc_folio(struct readahead_control *ractl, pgoff_t index,

		pgoff_t mark, unsigned int order, gfp_t gfp)

{