iov_iter: reorder handling of flavours in primitives

iovec is the most common one; test it first and test explicitly,
rather than "not anything else".  Replace all flavour checks with
use of iov_iter_is_...() helpers.

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

lib/iov_iter.c

@@ -117,22 +117,21 @@
 #define iterate_all_kinds(i, n, v, I, B, K, X) {		\
 	if (likely(n)) {					\
 		size_t skip = i->iov_offset;			\
-		if (unlikely(i->type & ITER_BVEC)) {		\
+		if (likely(iter_is_iovec(i))) {			\
-			struct bio_vec v;			\
-			struct bvec_iter __bi;			\
-			iterate_bvec(i, n, v, __bi, skip, (B))	\
-		} else if (unlikely(i->type & ITER_KVEC)) {	\
-			const struct kvec *kvec;		\
-			struct kvec v;				\
-			iterate_kvec(i, n, v, kvec, skip, (K))	\
-		} else if (unlikely(i->type & ITER_DISCARD)) {	\
-		} else if (unlikely(i->type & ITER_XARRAY)) {	\
-			struct bio_vec v;			\
-			iterate_xarray(i, n, v, skip, (X));	\
-		} else {					\
 			const struct iovec *iov;		\
 			struct iovec v;				\
 			iterate_iovec(i, n, v, iov, skip, (I))	\
+		} else if (iov_iter_is_bvec(i)) {		\
+			struct bio_vec v;			\
+			struct bvec_iter __bi;			\
+			iterate_bvec(i, n, v, __bi, skip, (B))	\
+		} else if (iov_iter_is_kvec(i)) {		\
+			const struct kvec *kvec;		\
+			struct kvec v;				\
+			iterate_kvec(i, n, v, kvec, skip, (K))	\
+		} else if (iov_iter_is_xarray(i)) {		\
+			struct bio_vec v;			\
+			iterate_xarray(i, n, v, skip, (X));	\
 		}						\
 	}							\
 }
@@ -142,30 +141,7 @@
 		n = i->count;					\
 	if (i->count) {						\
 		size_t skip = i->iov_offset;			\
-		if (unlikely(i->type & ITER_BVEC)) {		\
+		if (likely(iter_is_iovec(i))) {			\
-			const struct bio_vec *bvec = i->bvec;	\
-			struct bio_vec v;			\
-			struct bvec_iter __bi;			\
-			iterate_bvec(i, n, v, __bi, skip, (B))	\
-			i->bvec = __bvec_iter_bvec(i->bvec, __bi);	\
-			i->nr_segs -= i->bvec - bvec;		\
-			skip = __bi.bi_bvec_done;		\
-		} else if (unlikely(i->type & ITER_KVEC)) {	\
-			const struct kvec *kvec;		\
-			struct kvec v;				\
-			iterate_kvec(i, n, v, kvec, skip, (K))	\
-			if (skip == kvec->iov_len) {		\
-				kvec++;				\
-				skip = 0;			\
-			}					\
-			i->nr_segs -= kvec - i->kvec;		\
-			i->kvec = kvec;				\
-		} else if (unlikely(i->type & ITER_DISCARD)) {	\
-			skip += n;				\
-		} else if (unlikely(i->type & ITER_XARRAY)) {	\
-			struct bio_vec v;			\
-			iterate_xarray(i, n, v, skip, (X))	\
-		} else {					\
 			const struct iovec *iov;		\
 			struct iovec v;				\
 			iterate_iovec(i, n, v, iov, skip, (I))	\
@@ -175,6 +151,29 @@
 			}					\
 			i->nr_segs -= iov - i->iov;		\
 			i->iov = iov;				\
+		} else if (iov_iter_is_bvec(i)) {		\
+			const struct bio_vec *bvec = i->bvec;	\
+			struct bio_vec v;			\
+			struct bvec_iter __bi;			\
+			iterate_bvec(i, n, v, __bi, skip, (B))	\
+			i->bvec = __bvec_iter_bvec(i->bvec, __bi);	\
+			i->nr_segs -= i->bvec - bvec;		\
+			skip = __bi.bi_bvec_done;		\
+		} else if (iov_iter_is_kvec(i)) {		\
+			const struct kvec *kvec;		\
+			struct kvec v;				\
+			iterate_kvec(i, n, v, kvec, skip, (K))	\
+			if (skip == kvec->iov_len) {		\
+				kvec++;				\
+				skip = 0;			\
+			}					\
+			i->nr_segs -= kvec - i->kvec;		\
+			i->kvec = kvec;				\
+		} else if (iov_iter_is_xarray(i)) {		\
+			struct bio_vec v;			\
+			iterate_xarray(i, n, v, skip, (X))	\
+		} else if (iov_iter_is_discard(i)) {		\
+			skip += n;				\
 		}						\
 		i->count -= n;					\
 		i->iov_offset = skip;				\
@@ -905,20 +904,24 @@ static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
 static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
 			 struct iov_iter *i)
 {
-	if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) {
+	if (likely(iter_is_iovec(i)))
+		return copy_page_to_iter_iovec(page, offset, bytes, i);
+	if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) {
 		void *kaddr = kmap_atomic(page);
 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
 		kunmap_atomic(kaddr);
 		return wanted;
-	} else if (unlikely(iov_iter_is_discard(i))) {
+	}
+	if (iov_iter_is_pipe(i))
+		return copy_page_to_iter_pipe(page, offset, bytes, i);
+	if (unlikely(iov_iter_is_discard(i))) {
 		if (unlikely(i->count < bytes))
 			bytes = i->count;
 		i->count -= bytes;
 		return bytes;
-	} else if (likely(!iov_iter_is_pipe(i)))
+	}
-		return copy_page_to_iter_iovec(page, offset, bytes, i);
+	WARN_ON(1);
-	else
+	return 0;
-		return copy_page_to_iter_pipe(page, offset, bytes, i);
 }
 
 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
@@ -951,17 +954,16 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
 {
 	if (unlikely(!page_copy_sane(page, offset, bytes)))
 		return 0;
-	if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
+	if (likely(iter_is_iovec(i)))
-		WARN_ON(1);
+		return copy_page_from_iter_iovec(page, offset, bytes, i);
-		return 0;
+	if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) {
-	}
-	if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) {
 		void *kaddr = kmap_atomic(page);
 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
 		kunmap_atomic(kaddr);
 		return wanted;
-	} else
+	}
-		return copy_page_from_iter_iovec(page, offset, bytes, i);
+	WARN_ON(1);
+	return 0;
 }
 EXPORT_SYMBOL(copy_page_from_iter);
 
@@ -1203,16 +1205,13 @@ EXPORT_SYMBOL(iov_iter_revert);
  */
 size_t iov_iter_single_seg_count(const struct iov_iter *i)
 {
-	if (unlikely(iov_iter_is_pipe(i)))
+	if (i->nr_segs > 1) {
-		return i->count;	// it is a silly place, anyway
+		if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
-	if (i->nr_segs == 1)
+			return min(i->count, i->iov->iov_len - i->iov_offset);
-		return i->count;
+		if (iov_iter_is_bvec(i))
-	if (unlikely(iov_iter_is_discard(i) || iov_iter_is_xarray(i)))
+			return min(i->count, i->bvec->bv_len - i->iov_offset);
-		return i->count;
+	}
-	if (iov_iter_is_bvec(i))
+	return i->count;
-		return min(i->count, i->bvec->bv_len - i->iov_offset);
-	else
-		return min(i->count, i->iov->iov_len - i->iov_offset);
 }
 EXPORT_SYMBOL(iov_iter_single_seg_count);