block: optimise for_each_bvec() advance (6b6667aa) · Commits · 戴 / test

Because of how for_each_bvec() works it never advances across multiple entries at a time, so bvec_iter_advance() is an overkill. Add specialised bvec_iter_advance_single() that is faster. It also handles zero-len bvecs, so can kill bvec_iter_skip_zero_bvec(). text data bss dec hex filename before: 23977 805 0 24782 60ce lib/iov_iter.o before, bvec_iter_advance() w/o WARN_ONCE() 22886 600 0 23486 5bbe ./lib/iov_iter.o after: 21862 600 0 22462 57be lib/iov_iter.o Signed-off-by:

Pavel Begunkov <asml.silence@gmail.com> Reviewed-by:

Ming Lei <ming.lei@redhat.com> Reviewed-by:

Christoph Hellwig <hch@lst.de> Signed-off-by:

Jens Axboe <axboe@kernel.dk>

include/linux/bvec.h

+15 −5

Original line number	Diff line number	Diff line
		@@ -121,18 +121,28 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,
		return true;
		}

		static inline void bvec_iter_skip_zero_bvec(struct bvec_iter *iter)
		/*
		* A simpler version of bvec_iter_advance(), @bytes should not span
		* across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len
		*/
		static inline void bvec_iter_advance_single(const struct bio_vec *bv,
		struct bvec_iter *iter, unsigned int bytes)
		{
		iter->bi_bvec_done = 0;
		unsigned int done = iter->bi_bvec_done + bytes;

		if (done == bv[iter->bi_idx].bv_len) {
		done = 0;
		iter->bi_idx++;
		}
		iter->bi_bvec_done = done;
		iter->bi_size -= bytes;
		}

		#define for_each_bvec(bvl, bio_vec, iter, start) \
		for (iter = (start); \
		(iter).bi_size && \
		((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \
		(bvl).bv_len ? (void)bvec_iter_advance((bio_vec), &(iter), \
		(bvl).bv_len) : bvec_iter_skip_zero_bvec(&(iter)))
		bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))

		/* for iterating one bio from start to end */
		#define BVEC_ITER_ALL_INIT (struct bvec_iter) \

Admin message