Commit 130879f1 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

block: move bio_map_* to blk-map.c 



The bio_map_* helpers are just the low-level helpers for the
blk_rq_map_* APIs.  Move them together for better logical grouping,
as no there isn't much overlap with other code in bio.c.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent f01b411f

block/bio.c

+1 −509
Original line number Diff line number Diff line
@@ -780,7 +780,7 @@ static bool bio_try_merge_pc_page(struct request_queue *q, struct bio *bio, 
 *

 *	This should only be used by passthrough bios.

 */

static int __bio_add_pc_page(struct request_queue *q, struct bio *bio,

int __bio_add_pc_page(struct request_queue *q, struct bio *bio,

		struct page *page, unsigned int len, unsigned int offset,

		bool *same_page)

{
@@ -1194,90 +1194,6 @@ void bio_list_copy_data(struct bio *dst, struct bio *src) 
}

EXPORT_SYMBOL(bio_list_copy_data);



struct bio_map_data {

	int is_our_pages;

	struct iov_iter iter;

	struct iovec iov[];

};



static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,

					       gfp_t gfp_mask)

{

	struct bio_map_data *bmd;

	if (data->nr_segs > UIO_MAXIOV)

		return NULL;



	bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask);

	if (!bmd)

		return NULL;

	memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs);

	bmd->iter = *data;

	bmd->iter.iov = bmd->iov;

	return bmd;

}



/**

 * bio_copy_from_iter - copy all pages from iov_iter to bio

 * @bio: The &struct bio which describes the I/O as destination

 * @iter: iov_iter as source

 *

 * Copy all pages from iov_iter to bio.

 * Returns 0 on success, or error on failure.

 */

static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)

{

	struct bio_vec *bvec;

	struct bvec_iter_all iter_all;



	bio_for_each_segment_all(bvec, bio, iter_all) {

		ssize_t ret;



		ret = copy_page_from_iter(bvec->bv_page,

					  bvec->bv_offset,

					  bvec->bv_len,

					  iter);



		if (!iov_iter_count(iter))

			break;



		if (ret < bvec->bv_len)

			return -EFAULT;

	}



	return 0;

}



/**

 * bio_copy_to_iter - copy all pages from bio to iov_iter

 * @bio: The &struct bio which describes the I/O as source

 * @iter: iov_iter as destination

 *

 * Copy all pages from bio to iov_iter.

 * Returns 0 on success, or error on failure.

 */

static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)

{

	struct bio_vec *bvec;

	struct bvec_iter_all iter_all;



	bio_for_each_segment_all(bvec, bio, iter_all) {

		ssize_t ret;



		ret = copy_page_to_iter(bvec->bv_page,

					bvec->bv_offset,

					bvec->bv_len,

					&iter);



		if (!iov_iter_count(&iter))

			break;



		if (ret < bvec->bv_len)

			return -EFAULT;

	}



	return 0;

}



void bio_free_pages(struct bio *bio)

{

	struct bio_vec *bvec;
@@ -1288,430 +1204,6 @@ void bio_free_pages(struct bio *bio) 
}

EXPORT_SYMBOL(bio_free_pages);



/**

 *	bio_uncopy_user	-	finish previously mapped bio

 *	@bio: bio being terminated

 *

 *	Free pages allocated from bio_copy_user_iov() and write back data

 *	to user space in case of a read.

 */

int bio_uncopy_user(struct bio *bio)

{

	struct bio_map_data *bmd = bio->bi_private;

	int ret = 0;



	if (!bio_flagged(bio, BIO_NULL_MAPPED)) {

		/*

		 * if we're in a workqueue, the request is orphaned, so

		 * don't copy into a random user address space, just free

		 * and return -EINTR so user space doesn't expect any data.

		 */

		if (!current->mm)

			ret = -EINTR;

		else if (bio_data_dir(bio) == READ)

			ret = bio_copy_to_iter(bio, bmd->iter);

		if (bmd->is_our_pages)

			bio_free_pages(bio);

	}

	kfree(bmd);

	bio_put(bio);

	return ret;

}



/**

 *	bio_copy_user_iov	-	copy user data to bio

 *	@q:		destination block queue

 *	@map_data:	pointer to the rq_map_data holding pages (if necessary)

 *	@iter:		iovec iterator

 *	@gfp_mask:	memory allocation flags

 *

 *	Prepares and returns a bio for indirect user io, bouncing data

 *	to/from kernel pages as necessary. Must be paired with

 *	call bio_uncopy_user() on io completion.

 */

struct bio *bio_copy_user_iov(struct request_queue *q,

			      struct rq_map_data *map_data,

			      struct iov_iter *iter,

			      gfp_t gfp_mask)

{

	struct bio_map_data *bmd;

	struct page *page;

	struct bio *bio;

	int i = 0, ret;

	int nr_pages;

	unsigned int len = iter->count;

	unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;



	bmd = bio_alloc_map_data(iter, gfp_mask);

	if (!bmd)

		return ERR_PTR(-ENOMEM);



	/*

	 * We need to do a deep copy of the iov_iter including the iovecs.

	 * The caller provided iov might point to an on-stack or otherwise

	 * shortlived one.

	 */

	bmd->is_our_pages = map_data ? 0 : 1;



	nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);

	if (nr_pages > BIO_MAX_PAGES)

		nr_pages = BIO_MAX_PAGES;



	ret = -ENOMEM;

	bio = bio_kmalloc(gfp_mask, nr_pages);

	if (!bio)

		goto out_bmd;



	ret = 0;



	if (map_data) {

		nr_pages = 1 << map_data->page_order;

		i = map_data->offset / PAGE_SIZE;

	}

	while (len) {

		unsigned int bytes = PAGE_SIZE;



		bytes -= offset;



		if (bytes > len)

			bytes = len;



		if (map_data) {

			if (i == map_data->nr_entries * nr_pages) {

				ret = -ENOMEM;

				break;

			}



			page = map_data->pages[i / nr_pages];

			page += (i % nr_pages);



			i++;

		} else {

			page = alloc_page(q->bounce_gfp | gfp_mask);

			if (!page) {

				ret = -ENOMEM;

				break;

			}

		}



		if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) {

			if (!map_data)

				__free_page(page);

			break;

		}



		len -= bytes;

		offset = 0;

	}



	if (ret)

		goto cleanup;



	if (map_data)

		map_data->offset += bio->bi_iter.bi_size;



	/*

	 * success

	 */

	if ((iov_iter_rw(iter) == WRITE && (!map_data || !map_data->null_mapped)) ||

	    (map_data && map_data->from_user)) {

		ret = bio_copy_from_iter(bio, iter);

		if (ret)

			goto cleanup;

	} else {

		if (bmd->is_our_pages)

			zero_fill_bio(bio);

		iov_iter_advance(iter, bio->bi_iter.bi_size);

	}



	bio->bi_private = bmd;

	if (map_data && map_data->null_mapped)

		bio_set_flag(bio, BIO_NULL_MAPPED);

	return bio;

cleanup:

	if (!map_data)

		bio_free_pages(bio);

	bio_put(bio);

out_bmd:

	kfree(bmd);

	return ERR_PTR(ret);

}



/**

 *	bio_map_user_iov - map user iovec into bio

 *	@q:		the struct request_queue for the bio

 *	@iter:		iovec iterator

 *	@gfp_mask:	memory allocation flags

 *

 *	Map the user space address into a bio suitable for io to a block

 *	device. Returns an error pointer in case of error.

 */

struct bio *bio_map_user_iov(struct request_queue *q,

			     struct iov_iter *iter,

			     gfp_t gfp_mask)

{

	int j;

	struct bio *bio;

	int ret;



	if (!iov_iter_count(iter))

		return ERR_PTR(-EINVAL);



	bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES));

	if (!bio)

		return ERR_PTR(-ENOMEM);



	while (iov_iter_count(iter)) {

		struct page **pages;

		ssize_t bytes;

		size_t offs, added = 0;

		int npages;



		bytes = iov_iter_get_pages_alloc(iter, &pages, LONG_MAX, &offs);

		if (unlikely(bytes <= 0)) {

			ret = bytes ? bytes : -EFAULT;

			goto out_unmap;

		}



		npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE);



		if (unlikely(offs & queue_dma_alignment(q))) {

			ret = -EINVAL;

			j = 0;

		} else {

			for (j = 0; j < npages; j++) {

				struct page *page = pages[j];

				unsigned int n = PAGE_SIZE - offs;

				bool same_page = false;



				if (n > bytes)

					n = bytes;



				if (!__bio_add_pc_page(q, bio, page, n, offs,

						&same_page)) {

					if (same_page)

						put_page(page);

					break;

				}



				added += n;

				bytes -= n;

				offs = 0;

			}

			iov_iter_advance(iter, added);

		}

		/*

		 * release the pages we didn't map into the bio, if any

		 */

		while (j < npages)

			put_page(pages[j++]);

		kvfree(pages);

		/* couldn't stuff something into bio? */

		if (bytes)

			break;

	}



	bio_set_flag(bio, BIO_USER_MAPPED);



	/*

	 * subtle -- if bio_map_user_iov() ended up bouncing a bio,

	 * it would normally disappear when its bi_end_io is run.

	 * however, we need it for the unmap, so grab an extra

	 * reference to it

	 */

	bio_get(bio);

	return bio;



 out_unmap:

	bio_release_pages(bio, false);

	bio_put(bio);

	return ERR_PTR(ret);

}



/**

 *	bio_unmap_user	-	unmap a bio

 *	@bio:		the bio being unmapped

 *

 *	Unmap a bio previously mapped by bio_map_user_iov(). Must be called from

 *	process context.

 *

 *	bio_unmap_user() may sleep.

 */

void bio_unmap_user(struct bio *bio)

{

	bio_release_pages(bio, bio_data_dir(bio) == READ);

	bio_put(bio);

	bio_put(bio);

}



static void bio_invalidate_vmalloc_pages(struct bio *bio)

{

#ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE

	if (bio->bi_private && !op_is_write(bio_op(bio))) {

		unsigned long i, len = 0;



		for (i = 0; i < bio->bi_vcnt; i++)

			len += bio->bi_io_vec[i].bv_len;

		invalidate_kernel_vmap_range(bio->bi_private, len);

	}

#endif

}



static void bio_map_kern_endio(struct bio *bio)

{

	bio_invalidate_vmalloc_pages(bio);

	bio_put(bio);

}



/**

 *	bio_map_kern	-	map kernel address into bio

 *	@q: the struct request_queue for the bio

 *	@data: pointer to buffer to map

 *	@len: length in bytes

 *	@gfp_mask: allocation flags for bio allocation

 *

 *	Map the kernel address into a bio suitable for io to a block

 *	device. Returns an error pointer in case of error.

 */

struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,

			 gfp_t gfp_mask)

{

	unsigned long kaddr = (unsigned long)data;

	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;

	unsigned long start = kaddr >> PAGE_SHIFT;

	const int nr_pages = end - start;

	bool is_vmalloc = is_vmalloc_addr(data);

	struct page *page;

	int offset, i;

	struct bio *bio;



	bio = bio_kmalloc(gfp_mask, nr_pages);

	if (!bio)

		return ERR_PTR(-ENOMEM);



	if (is_vmalloc) {

		flush_kernel_vmap_range(data, len);

		bio->bi_private = data;

	}



	offset = offset_in_page(kaddr);

	for (i = 0; i < nr_pages; i++) {

		unsigned int bytes = PAGE_SIZE - offset;



		if (len <= 0)

			break;



		if (bytes > len)

			bytes = len;



		if (!is_vmalloc)

			page = virt_to_page(data);

		else

			page = vmalloc_to_page(data);

		if (bio_add_pc_page(q, bio, page, bytes,

				    offset) < bytes) {

			/* we don't support partial mappings */

			bio_put(bio);

			return ERR_PTR(-EINVAL);

		}



		data += bytes;

		len -= bytes;

		offset = 0;

	}



	bio->bi_end_io = bio_map_kern_endio;

	return bio;

}



static void bio_copy_kern_endio(struct bio *bio)

{

	bio_free_pages(bio);

	bio_put(bio);

}



static void bio_copy_kern_endio_read(struct bio *bio)

{

	char *p = bio->bi_private;

	struct bio_vec *bvec;

	struct bvec_iter_all iter_all;



	bio_for_each_segment_all(bvec, bio, iter_all) {

		memcpy(p, page_address(bvec->bv_page), bvec->bv_len);

		p += bvec->bv_len;

	}



	bio_copy_kern_endio(bio);

}



/**

 *	bio_copy_kern	-	copy kernel address into bio

 *	@q: the struct request_queue for the bio

 *	@data: pointer to buffer to copy

 *	@len: length in bytes

 *	@gfp_mask: allocation flags for bio and page allocation

 *	@reading: data direction is READ

 *

 *	copy the kernel address into a bio suitable for io to a block

 *	device. Returns an error pointer in case of error.

 */

struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,

			  gfp_t gfp_mask, int reading)

{

	unsigned long kaddr = (unsigned long)data;

	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;

	unsigned long start = kaddr >> PAGE_SHIFT;

	struct bio *bio;

	void *p = data;

	int nr_pages = 0;



	/*

	 * Overflow, abort

	 */

	if (end < start)

		return ERR_PTR(-EINVAL);



	nr_pages = end - start;

	bio = bio_kmalloc(gfp_mask, nr_pages);

	if (!bio)

		return ERR_PTR(-ENOMEM);



	while (len) {

		struct page *page;

		unsigned int bytes = PAGE_SIZE;



		if (bytes > len)

			bytes = len;



		page = alloc_page(q->bounce_gfp | gfp_mask);

		if (!page)

			goto cleanup;



		if (!reading)

			memcpy(page_address(page), p, bytes);



		if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)

			break;



		len -= bytes;

		p += bytes;

	}



	if (reading) {

		bio->bi_end_io = bio_copy_kern_endio_read;

		bio->bi_private = data;

	} else {

		bio->bi_end_io = bio_copy_kern_endio;

	}



	return bio;



cleanup:

	bio_free_pages(bio);

	bio_put(bio);

	return ERR_PTR(-ENOMEM);

}



/*

 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions

 * for performing direct-IO in BIOs.

block/blk-map.c

+508 −0
Original line number Diff line number Diff line
@@ -11,6 +11,514 @@ 


#include "blk.h"



struct bio_map_data {

	int is_our_pages;

	struct iov_iter iter;

	struct iovec iov[];

};



static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,

					       gfp_t gfp_mask)

{

	struct bio_map_data *bmd;



	if (data->nr_segs > UIO_MAXIOV)

		return NULL;



	bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask);

	if (!bmd)

		return NULL;

	memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs);

	bmd->iter = *data;

	bmd->iter.iov = bmd->iov;

	return bmd;

}



/**

 * bio_copy_from_iter - copy all pages from iov_iter to bio

 * @bio: The &struct bio which describes the I/O as destination

 * @iter: iov_iter as source

 *

 * Copy all pages from iov_iter to bio.

 * Returns 0 on success, or error on failure.

 */

static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)

{

	struct bio_vec *bvec;

	struct bvec_iter_all iter_all;



	bio_for_each_segment_all(bvec, bio, iter_all) {

		ssize_t ret;



		ret = copy_page_from_iter(bvec->bv_page,

					  bvec->bv_offset,

					  bvec->bv_len,

					  iter);



		if (!iov_iter_count(iter))

			break;



		if (ret < bvec->bv_len)

			return -EFAULT;

	}



	return 0;

}



/**

 * bio_copy_to_iter - copy all pages from bio to iov_iter

 * @bio: The &struct bio which describes the I/O as source

 * @iter: iov_iter as destination

 *

 * Copy all pages from bio to iov_iter.

 * Returns 0 on success, or error on failure.

 */

static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)

{

	struct bio_vec *bvec;

	struct bvec_iter_all iter_all;



	bio_for_each_segment_all(bvec, bio, iter_all) {

		ssize_t ret;



		ret = copy_page_to_iter(bvec->bv_page,

					bvec->bv_offset,

					bvec->bv_len,

					&iter);



		if (!iov_iter_count(&iter))

			break;



		if (ret < bvec->bv_len)

			return -EFAULT;

	}



	return 0;

}



/**

 *	bio_uncopy_user	-	finish previously mapped bio

 *	@bio: bio being terminated

 *

 *	Free pages allocated from bio_copy_user_iov() and write back data

 *	to user space in case of a read.

 */

static int bio_uncopy_user(struct bio *bio)

{

	struct bio_map_data *bmd = bio->bi_private;

	int ret = 0;



	if (!bio_flagged(bio, BIO_NULL_MAPPED)) {

		/*

		 * if we're in a workqueue, the request is orphaned, so

		 * don't copy into a random user address space, just free

		 * and return -EINTR so user space doesn't expect any data.

		 */

		if (!current->mm)

			ret = -EINTR;

		else if (bio_data_dir(bio) == READ)

			ret = bio_copy_to_iter(bio, bmd->iter);

		if (bmd->is_our_pages)

			bio_free_pages(bio);

	}

	kfree(bmd);

	bio_put(bio);

	return ret;

}



/**

 *	bio_copy_user_iov	-	copy user data to bio

 *	@q:		destination block queue

 *	@map_data:	pointer to the rq_map_data holding pages (if necessary)

 *	@iter:		iovec iterator

 *	@gfp_mask:	memory allocation flags

 *

 *	Prepares and returns a bio for indirect user io, bouncing data

 *	to/from kernel pages as necessary. Must be paired with

 *	call bio_uncopy_user() on io completion.

 */

static struct bio *bio_copy_user_iov(struct request_queue *q,

		struct rq_map_data *map_data, struct iov_iter *iter,

		gfp_t gfp_mask)

{

	struct bio_map_data *bmd;

	struct page *page;

	struct bio *bio;

	int i = 0, ret;

	int nr_pages;

	unsigned int len = iter->count;

	unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;



	bmd = bio_alloc_map_data(iter, gfp_mask);

	if (!bmd)

		return ERR_PTR(-ENOMEM);



	/*

	 * We need to do a deep copy of the iov_iter including the iovecs.

	 * The caller provided iov might point to an on-stack or otherwise

	 * shortlived one.

	 */

	bmd->is_our_pages = map_data ? 0 : 1;



	nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);

	if (nr_pages > BIO_MAX_PAGES)

		nr_pages = BIO_MAX_PAGES;



	ret = -ENOMEM;

	bio = bio_kmalloc(gfp_mask, nr_pages);

	if (!bio)

		goto out_bmd;



	ret = 0;



	if (map_data) {

		nr_pages = 1 << map_data->page_order;

		i = map_data->offset / PAGE_SIZE;

	}

	while (len) {

		unsigned int bytes = PAGE_SIZE;



		bytes -= offset;



		if (bytes > len)

			bytes = len;



		if (map_data) {

			if (i == map_data->nr_entries * nr_pages) {

				ret = -ENOMEM;

				break;

			}



			page = map_data->pages[i / nr_pages];

			page += (i % nr_pages);



			i++;

		} else {

			page = alloc_page(q->bounce_gfp | gfp_mask);

			if (!page) {

				ret = -ENOMEM;

				break;

			}

		}



		if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) {

			if (!map_data)

				__free_page(page);

			break;

		}



		len -= bytes;

		offset = 0;

	}



	if (ret)

		goto cleanup;



	if (map_data)

		map_data->offset += bio->bi_iter.bi_size;



	/*

	 * success

	 */

	if ((iov_iter_rw(iter) == WRITE &&

	     (!map_data || !map_data->null_mapped)) ||

	    (map_data && map_data->from_user)) {

		ret = bio_copy_from_iter(bio, iter);

		if (ret)

			goto cleanup;

	} else {

		if (bmd->is_our_pages)

			zero_fill_bio(bio);

		iov_iter_advance(iter, bio->bi_iter.bi_size);

	}



	bio->bi_private = bmd;

	if (map_data && map_data->null_mapped)

		bio_set_flag(bio, BIO_NULL_MAPPED);

	return bio;

cleanup:

	if (!map_data)

		bio_free_pages(bio);

	bio_put(bio);

out_bmd:

	kfree(bmd);

	return ERR_PTR(ret);

}



/**

 *	bio_map_user_iov - map user iovec into bio

 *	@q:		the struct request_queue for the bio

 *	@iter:		iovec iterator

 *	@gfp_mask:	memory allocation flags

 *

 *	Map the user space address into a bio suitable for io to a block

 *	device. Returns an error pointer in case of error.

 */

static struct bio *bio_map_user_iov(struct request_queue *q,

		struct iov_iter *iter, gfp_t gfp_mask)

{

	int j;

	struct bio *bio;

	int ret;



	if (!iov_iter_count(iter))

		return ERR_PTR(-EINVAL);



	bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES));

	if (!bio)

		return ERR_PTR(-ENOMEM);



	while (iov_iter_count(iter)) {

		struct page **pages;

		ssize_t bytes;

		size_t offs, added = 0;

		int npages;



		bytes = iov_iter_get_pages_alloc(iter, &pages, LONG_MAX, &offs);

		if (unlikely(bytes <= 0)) {

			ret = bytes ? bytes : -EFAULT;

			goto out_unmap;

		}



		npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE);



		if (unlikely(offs & queue_dma_alignment(q))) {

			ret = -EINVAL;

			j = 0;

		} else {

			for (j = 0; j < npages; j++) {

				struct page *page = pages[j];

				unsigned int n = PAGE_SIZE - offs;

				bool same_page = false;



				if (n > bytes)

					n = bytes;



				if (!__bio_add_pc_page(q, bio, page, n, offs,

						&same_page)) {

					if (same_page)

						put_page(page);

					break;

				}



				added += n;

				bytes -= n;

				offs = 0;

			}

			iov_iter_advance(iter, added);

		}

		/*

		 * release the pages we didn't map into the bio, if any

		 */

		while (j < npages)

			put_page(pages[j++]);

		kvfree(pages);

		/* couldn't stuff something into bio? */

		if (bytes)

			break;

	}



	bio_set_flag(bio, BIO_USER_MAPPED);



	/*

	 * subtle -- if bio_map_user_iov() ended up bouncing a bio,

	 * it would normally disappear when its bi_end_io is run.

	 * however, we need it for the unmap, so grab an extra

	 * reference to it

	 */

	bio_get(bio);

	return bio;



 out_unmap:

	bio_release_pages(bio, false);

	bio_put(bio);

	return ERR_PTR(ret);

}



/**

 *	bio_unmap_user	-	unmap a bio

 *	@bio:		the bio being unmapped

 *

 *	Unmap a bio previously mapped by bio_map_user_iov(). Must be called from

 *	process context.

 *

 *	bio_unmap_user() may sleep.

 */

static void bio_unmap_user(struct bio *bio)

{

	bio_release_pages(bio, bio_data_dir(bio) == READ);

	bio_put(bio);

	bio_put(bio);

}



static void bio_invalidate_vmalloc_pages(struct bio *bio)

{

#ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE

	if (bio->bi_private && !op_is_write(bio_op(bio))) {

		unsigned long i, len = 0;



		for (i = 0; i < bio->bi_vcnt; i++)

			len += bio->bi_io_vec[i].bv_len;

		invalidate_kernel_vmap_range(bio->bi_private, len);

	}

#endif

}



static void bio_map_kern_endio(struct bio *bio)

{

	bio_invalidate_vmalloc_pages(bio);

	bio_put(bio);

}



/**

 *	bio_map_kern	-	map kernel address into bio

 *	@q: the struct request_queue for the bio

 *	@data: pointer to buffer to map

 *	@len: length in bytes

 *	@gfp_mask: allocation flags for bio allocation

 *

 *	Map the kernel address into a bio suitable for io to a block

 *	device. Returns an error pointer in case of error.

 */

static struct bio *bio_map_kern(struct request_queue *q, void *data,

		unsigned int len, gfp_t gfp_mask)

{

	unsigned long kaddr = (unsigned long)data;

	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;

	unsigned long start = kaddr >> PAGE_SHIFT;

	const int nr_pages = end - start;

	bool is_vmalloc = is_vmalloc_addr(data);

	struct page *page;

	int offset, i;

	struct bio *bio;



	bio = bio_kmalloc(gfp_mask, nr_pages);

	if (!bio)

		return ERR_PTR(-ENOMEM);



	if (is_vmalloc) {

		flush_kernel_vmap_range(data, len);

		bio->bi_private = data;

	}



	offset = offset_in_page(kaddr);

	for (i = 0; i < nr_pages; i++) {

		unsigned int bytes = PAGE_SIZE - offset;



		if (len <= 0)

			break;



		if (bytes > len)

			bytes = len;



		if (!is_vmalloc)

			page = virt_to_page(data);

		else

			page = vmalloc_to_page(data);

		if (bio_add_pc_page(q, bio, page, bytes,

				    offset) < bytes) {

			/* we don't support partial mappings */

			bio_put(bio);

			return ERR_PTR(-EINVAL);

		}



		data += bytes;

		len -= bytes;

		offset = 0;

	}



	bio->bi_end_io = bio_map_kern_endio;

	return bio;

}



static void bio_copy_kern_endio(struct bio *bio)

{

	bio_free_pages(bio);

	bio_put(bio);

}



static void bio_copy_kern_endio_read(struct bio *bio)

{

	char *p = bio->bi_private;

	struct bio_vec *bvec;

	struct bvec_iter_all iter_all;



	bio_for_each_segment_all(bvec, bio, iter_all) {

		memcpy(p, page_address(bvec->bv_page), bvec->bv_len);

		p += bvec->bv_len;

	}



	bio_copy_kern_endio(bio);

}



/**

 *	bio_copy_kern	-	copy kernel address into bio

 *	@q: the struct request_queue for the bio

 *	@data: pointer to buffer to copy

 *	@len: length in bytes

 *	@gfp_mask: allocation flags for bio and page allocation

 *	@reading: data direction is READ

 *

 *	copy the kernel address into a bio suitable for io to a block

 *	device. Returns an error pointer in case of error.

 */

static struct bio *bio_copy_kern(struct request_queue *q, void *data,

		unsigned int len, gfp_t gfp_mask, int reading)

{

	unsigned long kaddr = (unsigned long)data;

	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;

	unsigned long start = kaddr >> PAGE_SHIFT;

	struct bio *bio;

	void *p = data;

	int nr_pages = 0;



	/*

	 * Overflow, abort

	 */

	if (end < start)

		return ERR_PTR(-EINVAL);



	nr_pages = end - start;

	bio = bio_kmalloc(gfp_mask, nr_pages);

	if (!bio)

		return ERR_PTR(-ENOMEM);



	while (len) {

		struct page *page;

		unsigned int bytes = PAGE_SIZE;



		if (bytes > len)

			bytes = len;



		page = alloc_page(q->bounce_gfp | gfp_mask);

		if (!page)

			goto cleanup;



		if (!reading)

			memcpy(page_address(page), p, bytes);



		if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)

			break;



		len -= bytes;

		p += bytes;

	}



	if (reading) {

		bio->bi_end_io = bio_copy_kern_endio_read;

		bio->bi_private = data;

	} else {

		bio->bi_end_io = bio_copy_kern_endio;

	}



	return bio;



cleanup:

	bio_free_pages(bio);

	bio_put(bio);

	return ERR_PTR(-ENOMEM);

}



/*

 * Append a bio to a passthrough request.  Only works if the bio can be merged

 * into the request based on the driver constraints.

block/blk.h

+4 −0
Original line number Diff line number Diff line
@@ -484,4 +484,8 @@ static inline void part_nr_sects_write(struct hd_struct *part, sector_t size) 


struct request_queue *__blk_alloc_queue(int node_id);



int __bio_add_pc_page(struct request_queue *q, struct bio *bio,

		struct page *page, unsigned int len, unsigned int offset,

		bool *same_page);



#endif /* BLK_INTERNAL_H */

include/linux/bio.h

+0 −14
Original line number Diff line number Diff line
@@ -441,14 +441,6 @@ void __bio_add_page(struct bio *bio, struct page *page, 
		unsigned int len, unsigned int off);

int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);

void bio_release_pages(struct bio *bio, bool mark_dirty);

struct rq_map_data;

extern struct bio *bio_map_user_iov(struct request_queue *,

				    struct iov_iter *, gfp_t);

extern void bio_unmap_user(struct bio *);

extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,

				gfp_t);

extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,

				 gfp_t, int);

extern void bio_set_pages_dirty(struct bio *bio);

extern void bio_check_pages_dirty(struct bio *bio);
@@ -463,12 +455,6 @@ extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, 
extern void bio_copy_data(struct bio *dst, struct bio *src);

extern void bio_list_copy_data(struct bio *dst, struct bio *src);

extern void bio_free_pages(struct bio *bio);



extern struct bio *bio_copy_user_iov(struct request_queue *,

				     struct rq_map_data *,

				     struct iov_iter *,

				     gfp_t);

extern int bio_uncopy_user(struct bio *);

void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);

void bio_truncate(struct bio *bio, unsigned new_size);

void guard_bio_eod(struct bio *bio);