Commit 28f38db7 authored by Al Viro's avatar Al Viro
Browse files

iov_iter: reorder handling of flavours in primitives 



iovec is the most common one; test it first and test explicitly,
rather than "not anything else".  Replace all flavour checks with
use of iov_iter_is_...() helpers.

Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent 4b6c132b

lib/iov_iter.c

+45 −46
Original line number Diff line number Diff line
@@ -117,22 +117,21 @@ 
#define iterate_all_kinds(i, n, v, I, B, K, X) {		\

	if (likely(n)) {					\

		size_t skip = i->iov_offset;			\

		if (unlikely(i->type & ITER_BVEC)) {		\

		if (likely(iter_is_iovec(i))) {			\

			const struct iovec *iov;		\

			struct iovec v;				\

			iterate_iovec(i, n, v, iov, skip, (I))	\

		} else if (iov_iter_is_bvec(i)) {		\

			struct bio_vec v;			\

			struct bvec_iter __bi;			\

			iterate_bvec(i, n, v, __bi, skip, (B))	\

		} else if (unlikely(i->type & ITER_KVEC)) {	\

		} else if (iov_iter_is_kvec(i)) {		\

			const struct kvec *kvec;		\

			struct kvec v;				\

			iterate_kvec(i, n, v, kvec, skip, (K))	\

		} else if (unlikely(i->type & ITER_DISCARD)) {	\

		} else if (unlikely(i->type & ITER_XARRAY)) {	\

		} else if (iov_iter_is_xarray(i)) {		\

			struct bio_vec v;			\

			iterate_xarray(i, n, v, skip, (X));	\

		} else {					\

			const struct iovec *iov;		\

			struct iovec v;				\

			iterate_iovec(i, n, v, iov, skip, (I))	\

		}						\

	}							\

}
@@ -142,7 +141,17 @@ 
		n = i->count;					\

	if (i->count) {						\

		size_t skip = i->iov_offset;			\

		if (unlikely(i->type & ITER_BVEC)) {		\

		if (likely(iter_is_iovec(i))) {			\

			const struct iovec *iov;		\

			struct iovec v;				\

			iterate_iovec(i, n, v, iov, skip, (I))	\

			if (skip == iov->iov_len) {		\

				iov++;				\

				skip = 0;			\

			}					\

			i->nr_segs -= iov - i->iov;		\

			i->iov = iov;				\

		} else if (iov_iter_is_bvec(i)) {		\

			const struct bio_vec *bvec = i->bvec;	\

			struct bio_vec v;			\

			struct bvec_iter __bi;			\
@@ -150,7 +159,7 @@ 
			i->bvec = __bvec_iter_bvec(i->bvec, __bi);	\

			i->nr_segs -= i->bvec - bvec;		\

			skip = __bi.bi_bvec_done;		\

		} else if (unlikely(i->type & ITER_KVEC)) {	\

		} else if (iov_iter_is_kvec(i)) {		\

			const struct kvec *kvec;		\

			struct kvec v;				\

			iterate_kvec(i, n, v, kvec, skip, (K))	\
@@ -160,21 +169,11 @@ 
			}					\

			i->nr_segs -= kvec - i->kvec;		\

			i->kvec = kvec;				\

		} else if (unlikely(i->type & ITER_DISCARD)) {	\

			skip += n;				\

		} else if (unlikely(i->type & ITER_XARRAY)) {	\

		} else if (iov_iter_is_xarray(i)) {		\

			struct bio_vec v;			\

			iterate_xarray(i, n, v, skip, (X))	\

		} else {					\

			const struct iovec *iov;		\

			struct iovec v;				\

			iterate_iovec(i, n, v, iov, skip, (I))	\

			if (skip == iov->iov_len) {		\

				iov++;				\

				skip = 0;			\

			}					\

			i->nr_segs -= iov - i->iov;		\

			i->iov = iov;				\

		} else if (iov_iter_is_discard(i)) {		\

			skip += n;				\

		}						\

		i->count -= n;					\

		i->iov_offset = skip;				\
@@ -905,20 +904,24 @@ static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) 
static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes,

			 struct iov_iter *i)

{

	if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) {

	if (likely(iter_is_iovec(i)))

		return copy_page_to_iter_iovec(page, offset, bytes, i);

	if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) {

		void *kaddr = kmap_atomic(page);

		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);

		kunmap_atomic(kaddr);

		return wanted;

	} else if (unlikely(iov_iter_is_discard(i))) {

	}

	if (iov_iter_is_pipe(i))

		return copy_page_to_iter_pipe(page, offset, bytes, i);

	if (unlikely(iov_iter_is_discard(i))) {

		if (unlikely(i->count < bytes))

			bytes = i->count;

		i->count -= bytes;

		return bytes;

	} else if (likely(!iov_iter_is_pipe(i)))

		return copy_page_to_iter_iovec(page, offset, bytes, i);

	else

		return copy_page_to_iter_pipe(page, offset, bytes, i);

	}

	WARN_ON(1);

	return 0;

}



size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
@@ -951,17 +954,16 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 
{

	if (unlikely(!page_copy_sane(page, offset, bytes)))

		return 0;

	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {

		WARN_ON(1);

		return 0;

	}

	if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) {

	if (likely(iter_is_iovec(i)))

		return copy_page_from_iter_iovec(page, offset, bytes, i);

	if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) {

		void *kaddr = kmap_atomic(page);

		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);

		kunmap_atomic(kaddr);

		return wanted;

	} else

		return copy_page_from_iter_iovec(page, offset, bytes, i);

	}

	WARN_ON(1);

	return 0;

}

EXPORT_SYMBOL(copy_page_from_iter);
@@ -1203,16 +1205,13 @@ EXPORT_SYMBOL(iov_iter_revert); 
 */

size_t iov_iter_single_seg_count(const struct iov_iter *i)

{

	if (unlikely(iov_iter_is_pipe(i)))

		return i->count;	// it is a silly place, anyway

	if (i->nr_segs == 1)

		return i->count;

	if (unlikely(iov_iter_is_discard(i) || iov_iter_is_xarray(i)))

		return i->count;

	if (i->nr_segs > 1) {

		if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))

			return min(i->count, i->iov->iov_len - i->iov_offset);

		if (iov_iter_is_bvec(i))

			return min(i->count, i->bvec->bv_len - i->iov_offset);

	else

		return min(i->count, i->iov->iov_len - i->iov_offset);

	}

	return i->count;

}

EXPORT_SYMBOL(iov_iter_single_seg_count);