Commit f62e52d1 authored by David Howells's avatar David Howells Committed by Steve French
Browse files

iov_iter: Define flags to qualify page extraction. 



Define flags to qualify page extraction to pass into iov_iter_*_pages*()
rather than passing in FOLL_* flags.

For now only a flag to allow peer-to-peer DMA is supported.

Signed-off-by: default avatarDavid Howells <dhowells@redhat.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarJohn Hubbard <jhubbard@nvidia.com>
Reviewed-by: default avatarJens Axboe <axboe@kernel.dk>
cc: Al Viro <viro@zeniv.linux.org.uk>
cc: Logan Gunthorpe <logang@deltatee.com>
cc: linux-fsdevel@vger.kernel.org
cc: linux-block@vger.kernel.org
Signed-off-by: default avatarSteve French <stfrench@microsoft.com>
parent 33b3b041

block/bio.c

+3 −3
Original line number Diff line number Diff line
@@ -1245,11 +1245,11 @@ static int bio_iov_add_zone_append_page(struct bio *bio, struct page *page, 
 */

static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)

{

	iov_iter_extraction_t extraction_flags = 0;

	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;

	unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;

	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;

	struct page **pages = (struct page **)bv;

	unsigned int gup_flags = 0;

	ssize_t size, left;

	unsigned len, i = 0;

	size_t offset, trim;
@@ -1264,7 +1264,7 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) 
	pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);



	if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue))

		gup_flags |= FOLL_PCI_P2PDMA;

		extraction_flags |= ITER_ALLOW_P2PDMA;



	/*

	 * Each segment in the iov is required to be a block size multiple.
@@ -1275,7 +1275,7 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) 
	 */

	size = iov_iter_get_pages(iter, pages,

				  UINT_MAX - bio->bi_iter.bi_size,

				  nr_pages, &offset, gup_flags);

				  nr_pages, &offset, extraction_flags);

	if (unlikely(size <= 0))

		return size ? size : -EFAULT;

block/blk-map.c

+4 −4
Original line number Diff line number Diff line
@@ -265,9 +265,9 @@ static struct bio *blk_rq_map_bio_alloc(struct request *rq, 
static int bio_map_user_iov(struct request *rq, struct iov_iter *iter,

		gfp_t gfp_mask)

{

	iov_iter_extraction_t extraction_flags = 0;

	unsigned int max_sectors = queue_max_hw_sectors(rq->q);

	unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS);

	unsigned int gup_flags = 0;

	struct bio *bio;

	int ret;

	int j;
@@ -280,7 +280,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, 
		return -ENOMEM;



	if (blk_queue_pci_p2pdma(rq->q))

		gup_flags |= FOLL_PCI_P2PDMA;

		extraction_flags |= ITER_ALLOW_P2PDMA;



	while (iov_iter_count(iter)) {

		struct page **pages, *stack_pages[UIO_FASTIOV];
@@ -291,10 +291,10 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, 
		if (nr_vecs <= ARRAY_SIZE(stack_pages)) {

			pages = stack_pages;

			bytes = iov_iter_get_pages(iter, pages, LONG_MAX,

						   nr_vecs, &offs, gup_flags);

						   nr_vecs, &offs, extraction_flags);

		} else {

			bytes = iov_iter_get_pages_alloc(iter, &pages,

						LONG_MAX, &offs, gup_flags);

						LONG_MAX, &offs, extraction_flags);

		}

		if (unlikely(bytes <= 0)) {

			ret = bytes ? bytes : -EFAULT;

include/linux/uio.h

+8 −2
Original line number Diff line number Diff line
@@ -13,6 +13,8 @@ 
struct page;

struct pipe_inode_info;



typedef unsigned int __bitwise iov_iter_extraction_t;



struct kvec {

	void *iov_base; /* and that should *never* hold a userland pointer */

	size_t iov_len;
@@ -252,12 +254,12 @@ void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray * 
		     loff_t start, size_t count);

ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages,

		size_t maxsize, unsigned maxpages, size_t *start,

		unsigned gup_flags);

		iov_iter_extraction_t extraction_flags);

ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,

			size_t maxsize, unsigned maxpages, size_t *start);

ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,

		struct page ***pages, size_t maxsize, size_t *start,

		unsigned gup_flags);

		iov_iter_extraction_t extraction_flags);

ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, struct page ***pages,

			size_t maxsize, size_t *start);

int iov_iter_npages(const struct iov_iter *i, int maxpages);
@@ -360,4 +362,8 @@ static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction, 
	};

}



/* Flags for iov_iter_get/extract_pages*() */

/* Allow P2PDMA on the extracted pages */

#define ITER_ALLOW_P2PDMA	((__force iov_iter_extraction_t)0x01)



#endif

lib/iov_iter.c

+8 −6
Original line number Diff line number Diff line
@@ -1426,9 +1426,9 @@ static struct page *first_bvec_segment(const struct iov_iter *i, 
static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,

		   struct page ***pages, size_t maxsize,

		   unsigned int maxpages, size_t *start,

		   unsigned int gup_flags)

		   iov_iter_extraction_t extraction_flags)

{

	unsigned int n;

	unsigned int n, gup_flags = 0;



	if (maxsize > i->count)

		maxsize = i->count;
@@ -1436,6 +1436,8 @@ static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i, 
		return 0;

	if (maxsize > MAX_RW_COUNT)

		maxsize = MAX_RW_COUNT;

	if (extraction_flags & ITER_ALLOW_P2PDMA)

		gup_flags |= FOLL_PCI_P2PDMA;



	if (likely(user_backed_iter(i))) {

		unsigned long addr;
@@ -1489,14 +1491,14 @@ static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i, 


ssize_t iov_iter_get_pages(struct iov_iter *i,

		   struct page **pages, size_t maxsize, unsigned maxpages,

		   size_t *start, unsigned gup_flags)

		   size_t *start, iov_iter_extraction_t extraction_flags)

{

	if (!maxpages)

		return 0;

	BUG_ON(!pages);



	return __iov_iter_get_pages_alloc(i, &pages, maxsize, maxpages,

					  start, gup_flags);

					  start, extraction_flags);

}

EXPORT_SYMBOL_GPL(iov_iter_get_pages);
@@ -1509,14 +1511,14 @@ EXPORT_SYMBOL(iov_iter_get_pages2); 


ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,

		   struct page ***pages, size_t maxsize,

		   size_t *start, unsigned gup_flags)

		   size_t *start, iov_iter_extraction_t extraction_flags)

{

	ssize_t len;



	*pages = NULL;



	len = __iov_iter_get_pages_alloc(i, pages, maxsize, ~0U, start,

					 gup_flags);

					 extraction_flags);

	if (len <= 0) {

		kvfree(*pages);

		*pages = NULL;