Commit d8c2237d authored by Jens Axboe's avatar Jens Axboe
Browse files

io_uring: add io_pin_pages() helper 



Abstract this out from io_sqe_buffer_register() so we can use it
elsewhere too without duplicating this code.

No intended functional changes in this patch.

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 3d200242

fs/io_uring.c

+50 −27
Original line number Diff line number Diff line
@@ -9845,30 +9845,18 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages, 
	return ret;

}



static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,

				  struct io_mapped_ubuf **pimu,

				  struct page **last_hpage)

static struct page **io_pin_pages(unsigned long ubuf, unsigned long len,

				  int *npages)

{

	struct io_mapped_ubuf *imu = NULL;

	unsigned long start, end, nr_pages;

	struct vm_area_struct **vmas = NULL;

	struct page **pages = NULL;

	unsigned long off, start, end, ubuf;

	size_t size;

	int ret, pret, nr_pages, i;

	int i, pret, ret = -ENOMEM;



	if (!iov->iov_base) {

		*pimu = ctx->dummy_ubuf;

		return 0;

	}



	ubuf = (unsigned long) iov->iov_base;

	end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;

	end = (ubuf + len + PAGE_SIZE - 1) >> PAGE_SHIFT;

	start = ubuf >> PAGE_SHIFT;

	nr_pages = end - start;



	*pimu = NULL;

	ret = -ENOMEM;



	pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);

	if (!pages)

		goto done;
@@ -9878,10 +9866,6 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, 
	if (!vmas)

		goto done;



	imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);

	if (!imu)

		goto done;



	ret = 0;

	mmap_read_lock(current->mm);

	pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
@@ -9899,6 +9883,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, 
				break;

			}

		}

		*npages = nr_pages;

	} else {

		ret = pret < 0 ? pret : -EFAULT;

	}
@@ -9912,14 +9897,53 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, 
			unpin_user_pages(pages, pret);

		goto done;

	}

	ret = 0;

done:

	kvfree(vmas);

	if (ret < 0) {

		kvfree(pages);

		pages = ERR_PTR(ret);

	}

	return pages;

}



	ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);

static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,

				  struct io_mapped_ubuf **pimu,

				  struct page **last_hpage)

{

	struct io_mapped_ubuf *imu = NULL;

	struct page **pages = NULL;

	unsigned long off;

	size_t size;

	int ret, nr_pages, i;



	if (!iov->iov_base) {

		*pimu = ctx->dummy_ubuf;

		return 0;

	}



	*pimu = NULL;

	ret = -ENOMEM;



	pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,

				&nr_pages);

	if (IS_ERR(pages)) {

		ret = PTR_ERR(pages);

		pages = NULL;

		goto done;

	}



	imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);

	if (!imu)

		goto done;



	ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);

	if (ret) {

		unpin_user_pages(pages, pret);

		unpin_user_pages(pages, nr_pages);

		goto done;

	}



	off = ubuf & ~PAGE_MASK;

	off = (unsigned long) iov->iov_base & ~PAGE_MASK;

	size = iov->iov_len;

	for (i = 0; i < nr_pages; i++) {

		size_t vec_len;
@@ -9932,8 +9956,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, 
		size -= vec_len;

	}

	/* store original address for later verification */

	imu->ubuf = ubuf;

	imu->ubuf_end = ubuf + iov->iov_len;

	imu->ubuf = (unsigned long) iov->iov_base;

	imu->ubuf_end = imu->ubuf + iov->iov_len;

	imu->nr_bvecs = nr_pages;

	*pimu = imu;

	ret = 0;
@@ -9941,7 +9965,6 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, 
	if (ret)

		kvfree(imu);

	kvfree(pages);

	kvfree(vmas);

	return ret;

}