Commit fb041b59 authored by David Laight's avatar David Laight Committed by Al Viro
Browse files

iov_iter: move rw_copy_check_uvector() into lib/iov_iter.c 



This lets the compiler inline it into import_iovec() generating
much better code.

Signed-off-by: default avatarDavid Laight <david.laight@aculab.com>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent 576d0703

fs/read_write.c

+0 −179
Original line number Diff line number Diff line
@@ -752,185 +752,6 @@ static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter, 
	return ret;

}



/**

 * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace

 *     into the kernel and check that it is valid.

 *

 * @type: One of %CHECK_IOVEC_ONLY, %READ, or %WRITE.

 * @uvector: Pointer to the userspace array.

 * @nr_segs: Number of elements in userspace array.

 * @fast_segs: Number of elements in @fast_pointer.

 * @fast_pointer: Pointer to (usually small on-stack) kernel array.

 * @ret_pointer: (output parameter) Pointer to a variable that will point to

 *     either @fast_pointer, a newly allocated kernel array, or NULL,

 *     depending on which array was used.

 *

 * This function copies an array of &struct iovec of @nr_segs from

 * userspace into the kernel and checks that each element is valid (e.g.

 * it does not point to a kernel address or cause overflow by being too

 * large, etc.).

 *

 * As an optimization, the caller may provide a pointer to a small

 * on-stack array in @fast_pointer, typically %UIO_FASTIOV elements long

 * (the size of this array, or 0 if unused, should be given in @fast_segs).

 *

 * @ret_pointer will always point to the array that was used, so the

 * caller must take care not to call kfree() on it e.g. in case the

 * @fast_pointer array was used and it was allocated on the stack.

 *

 * Return: The total number of bytes covered by the iovec array on success

 *   or a negative error code on error.

 */

ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,

			      unsigned long nr_segs, unsigned long fast_segs,

			      struct iovec *fast_pointer,

			      struct iovec **ret_pointer)

{

	unsigned long seg;

	ssize_t ret;

	struct iovec *iov = fast_pointer;



	/*

	 * SuS says "The readv() function *may* fail if the iovcnt argument

	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has

	 * traditionally returned zero for zero segments, so...

	 */

	if (nr_segs == 0) {

		ret = 0;

		goto out;

	}



	/*

	 * First get the "struct iovec" from user memory and

	 * verify all the pointers

	 */

	if (nr_segs > UIO_MAXIOV) {

		ret = -EINVAL;

		goto out;

	}

	if (nr_segs > fast_segs) {

		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);

		if (iov == NULL) {

			ret = -ENOMEM;

			goto out;

		}

	}

	if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {

		ret = -EFAULT;

		goto out;

	}



	/*

	 * According to the Single Unix Specification we should return EINVAL

	 * if an element length is < 0 when cast to ssize_t or if the

	 * total length would overflow the ssize_t return value of the

	 * system call.

	 *

	 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the

	 * overflow case.

	 */

	ret = 0;

	for (seg = 0; seg < nr_segs; seg++) {

		void __user *buf = iov[seg].iov_base;

		ssize_t len = (ssize_t)iov[seg].iov_len;



		/* see if we we're about to use an invalid len or if

		 * it's about to overflow ssize_t */

		if (len < 0) {

			ret = -EINVAL;

			goto out;

		}

		if (type >= 0

		    && unlikely(!access_ok(buf, len))) {

			ret = -EFAULT;

			goto out;

		}

		if (len > MAX_RW_COUNT - ret) {

			len = MAX_RW_COUNT - ret;

			iov[seg].iov_len = len;

		}

		ret += len;

	}

out:

	*ret_pointer = iov;

	return ret;

}



#ifdef CONFIG_COMPAT

ssize_t compat_rw_copy_check_uvector(int type,

		const struct compat_iovec __user *uvector, unsigned long nr_segs,

		unsigned long fast_segs, struct iovec *fast_pointer,

		struct iovec **ret_pointer)

{

	compat_ssize_t tot_len;

	struct iovec *iov = *ret_pointer = fast_pointer;

	ssize_t ret = 0;

	int seg;



	/*

	 * SuS says "The readv() function *may* fail if the iovcnt argument

	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has

	 * traditionally returned zero for zero segments, so...

	 */

	if (nr_segs == 0)

		goto out;



	ret = -EINVAL;

	if (nr_segs > UIO_MAXIOV)

		goto out;

	if (nr_segs > fast_segs) {

		ret = -ENOMEM;

		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);

		if (iov == NULL)

			goto out;

	}

	*ret_pointer = iov;



	ret = -EFAULT;

	if (!access_ok(uvector, nr_segs*sizeof(*uvector)))

		goto out;



	/*

	 * Single unix specification:

	 * We should -EINVAL if an element length is not >= 0 and fitting an

	 * ssize_t.

	 *

	 * In Linux, the total length is limited to MAX_RW_COUNT, there is

	 * no overflow possibility.

	 */

	tot_len = 0;

	ret = -EINVAL;

	for (seg = 0; seg < nr_segs; seg++) {

		compat_uptr_t buf;

		compat_ssize_t len;



		if (__get_user(len, &uvector->iov_len) ||

		   __get_user(buf, &uvector->iov_base)) {

			ret = -EFAULT;

			goto out;

		}

		if (len < 0)	/* size_t not fitting in compat_ssize_t .. */

			goto out;

		if (type >= 0 &&

		    !access_ok(compat_ptr(buf), len)) {

			ret = -EFAULT;

			goto out;

		}

		if (len > MAX_RW_COUNT - tot_len)

			len = MAX_RW_COUNT - tot_len;

		tot_len += len;

		iov->iov_base = compat_ptr(buf);

		iov->iov_len = (compat_size_t) len;

		uvector++;

		iov++;

	}

	ret = tot_len;



out:

	return ret;

}

#endif



static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,

		loff_t *pos, rwf_t flags)

{

lib/iov_iter.c

+176 −0
Original line number Diff line number Diff line
@@ -1650,6 +1650,109 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) 
}

EXPORT_SYMBOL(dup_iter);



/**

 * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace

 *     into the kernel and check that it is valid.

 *

 * @type: One of %CHECK_IOVEC_ONLY, %READ, or %WRITE.

 * @uvector: Pointer to the userspace array.

 * @nr_segs: Number of elements in userspace array.

 * @fast_segs: Number of elements in @fast_pointer.

 * @fast_pointer: Pointer to (usually small on-stack) kernel array.

 * @ret_pointer: (output parameter) Pointer to a variable that will point to

 *     either @fast_pointer, a newly allocated kernel array, or NULL,

 *     depending on which array was used.

 *

 * This function copies an array of &struct iovec of @nr_segs from

 * userspace into the kernel and checks that each element is valid (e.g.

 * it does not point to a kernel address or cause overflow by being too

 * large, etc.).

 *

 * As an optimization, the caller may provide a pointer to a small

 * on-stack array in @fast_pointer, typically %UIO_FASTIOV elements long

 * (the size of this array, or 0 if unused, should be given in @fast_segs).

 *

 * @ret_pointer will always point to the array that was used, so the

 * caller must take care not to call kfree() on it e.g. in case the

 * @fast_pointer array was used and it was allocated on the stack.

 *

 * Return: The total number of bytes covered by the iovec array on success

 *   or a negative error code on error.

 */

ssize_t rw_copy_check_uvector(int type, const struct iovec __user *uvector,

		unsigned long nr_segs, unsigned long fast_segs,

		struct iovec *fast_pointer, struct iovec **ret_pointer)

{

	unsigned long seg;

	ssize_t ret;

	struct iovec *iov = fast_pointer;



	/*

	 * SuS says "The readv() function *may* fail if the iovcnt argument

	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has

	 * traditionally returned zero for zero segments, so...

	 */

	if (nr_segs == 0) {

		ret = 0;

		goto out;

	}



	/*

	 * First get the "struct iovec" from user memory and

	 * verify all the pointers

	 */

	if (nr_segs > UIO_MAXIOV) {

		ret = -EINVAL;

		goto out;

	}

	if (nr_segs > fast_segs) {

		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);

		if (iov == NULL) {

			ret = -ENOMEM;

			goto out;

		}

	}

	if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {

		ret = -EFAULT;

		goto out;

	}



	/*

	 * According to the Single Unix Specification we should return EINVAL

	 * if an element length is < 0 when cast to ssize_t or if the

	 * total length would overflow the ssize_t return value of the

	 * system call.

	 *

	 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the

	 * overflow case.

	 */

	ret = 0;

	for (seg = 0; seg < nr_segs; seg++) {

		void __user *buf = iov[seg].iov_base;

		ssize_t len = (ssize_t)iov[seg].iov_len;



		/* see if we we're about to use an invalid len or if

		 * it's about to overflow ssize_t */

		if (len < 0) {

			ret = -EINVAL;

			goto out;

		}

		if (type >= 0

		    && unlikely(!access_ok(buf, len))) {

			ret = -EFAULT;

			goto out;

		}

		if (len > MAX_RW_COUNT - ret) {

			len = MAX_RW_COUNT - ret;

			iov[seg].iov_len = len;

		}

		ret += len;

	}

out:

	*ret_pointer = iov;

	return ret;

}



/**

 * import_iovec() - Copy an array of &struct iovec from userspace

 *     into the kernel, check that it is valid, and initialize a new
@@ -1695,6 +1798,79 @@ EXPORT_SYMBOL(import_iovec); 
#ifdef CONFIG_COMPAT

#include <linux/compat.h>



ssize_t compat_rw_copy_check_uvector(int type,

		const struct compat_iovec __user *uvector,

		unsigned long nr_segs, unsigned long fast_segs,

		struct iovec *fast_pointer, struct iovec **ret_pointer)

{

	compat_ssize_t tot_len;

	struct iovec *iov = *ret_pointer = fast_pointer;

	ssize_t ret = 0;

	int seg;



	/*

	 * SuS says "The readv() function *may* fail if the iovcnt argument

	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has

	 * traditionally returned zero for zero segments, so...

	 */

	if (nr_segs == 0)

		goto out;



	ret = -EINVAL;

	if (nr_segs > UIO_MAXIOV)

		goto out;

	if (nr_segs > fast_segs) {

		ret = -ENOMEM;

		iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);

		if (iov == NULL)

			goto out;

	}

	*ret_pointer = iov;



	ret = -EFAULT;

	if (!access_ok(uvector, nr_segs*sizeof(*uvector)))

		goto out;



	/*

	 * Single unix specification:

	 * We should -EINVAL if an element length is not >= 0 and fitting an

	 * ssize_t.

	 *

	 * In Linux, the total length is limited to MAX_RW_COUNT, there is

	 * no overflow possibility.

	 */

	tot_len = 0;

	ret = -EINVAL;

	for (seg = 0; seg < nr_segs; seg++) {

		compat_uptr_t buf;

		compat_ssize_t len;



		if (__get_user(len, &uvector->iov_len) ||

		   __get_user(buf, &uvector->iov_base)) {

			ret = -EFAULT;

			goto out;

		}

		if (len < 0)	/* size_t not fitting in compat_ssize_t .. */

			goto out;

		if (type >= 0 &&

		    !access_ok(compat_ptr(buf), len)) {

			ret = -EFAULT;

			goto out;

		}

		if (len > MAX_RW_COUNT - tot_len)

			len = MAX_RW_COUNT - tot_len;

		tot_len += len;

		iov->iov_base = compat_ptr(buf);

		iov->iov_len = (compat_size_t) len;

		uvector++;

		iov++;

	}

	ret = tot_len;



out:

	return ret;

}



ssize_t compat_import_iovec(int type,

		const struct compat_iovec __user * uvector,

		unsigned nr_segs, unsigned fast_segs,