Commit c4cf5261 authored by Jens Axboe's avatar Jens Axboe
Browse files

bio: skip atomic inc/dec of ->bi_remaining for non-chains 



Struct bio has an atomic ref count for chained bio's, and we use this
to know when to end IO on the bio. However, most bio's are not chained,
so we don't need to always introduce this atomic operation as part of
ending IO.

Add a helper to elevate the bi_remaining count, and flag the bio as
now actually needing the decrement at end_io time. Rename the field
to __bi_remaining to catch any current users of this doing the
incrementing manually.

For high IOPS workloads, this reduces the overhead of bio_endio()
substantially.

Tested-by: default avatarRobert Elliott <elliott@hp.com>
Acked-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
Reviewed-by: default avatarJan Kara <jack@suse.cz>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent d9cee5d4

block/bio.c

+29 −9
Original line number Diff line number Diff line
@@ -270,7 +270,7 @@ void bio_init(struct bio *bio) 
{

	memset(bio, 0, sizeof(*bio));

	bio->bi_flags = 1 << BIO_UPTODATE;

	atomic_set(&bio->bi_remaining, 1);

	atomic_set(&bio->__bi_remaining, 1);

	atomic_set(&bio->bi_cnt, 1);

}

EXPORT_SYMBOL(bio_init);
@@ -293,7 +293,7 @@ void bio_reset(struct bio *bio) 


	memset(bio, 0, BIO_RESET_BYTES);

	bio->bi_flags = flags | (1 << BIO_UPTODATE);

	atomic_set(&bio->bi_remaining, 1);

	atomic_set(&bio->__bi_remaining, 1);

}

EXPORT_SYMBOL(bio_reset);
@@ -320,7 +320,7 @@ void bio_chain(struct bio *bio, struct bio *parent) 


	bio->bi_private = parent;

	bio->bi_end_io	= bio_chain_endio;

	atomic_inc(&parent->bi_remaining);

	bio_inc_remaining(parent);

}

EXPORT_SYMBOL(bio_chain);
@@ -1741,6 +1741,23 @@ void bio_flush_dcache_pages(struct bio *bi) 
EXPORT_SYMBOL(bio_flush_dcache_pages);

#endif



static inline bool bio_remaining_done(struct bio *bio)

{

	/*

	 * If we're not chaining, then ->__bi_remaining is always 1 and

	 * we always end io on the first invocation.

	 */

	if (!bio_flagged(bio, BIO_CHAIN))

		return true;



	BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);



	if (atomic_dec_and_test(&bio->__bi_remaining))

		return true;



	return false;

}



/**

 * bio_endio - end I/O on a bio

 * @bio:	bio
@@ -1758,15 +1775,13 @@ EXPORT_SYMBOL(bio_flush_dcache_pages); 
void bio_endio(struct bio *bio, int error)

{

	while (bio) {

		BUG_ON(atomic_read(&bio->bi_remaining) <= 0);



		if (error)

			clear_bit(BIO_UPTODATE, &bio->bi_flags);

		else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))

			error = -EIO;



		if (!atomic_dec_and_test(&bio->bi_remaining))

			return;

		if (unlikely(!bio_remaining_done(bio)))

			break;



		/*

		 * Need to have a real endio function for chained bios,
@@ -1799,7 +1814,12 @@ EXPORT_SYMBOL(bio_endio); 
 **/

void bio_endio_nodec(struct bio *bio, int error)

{

	atomic_inc(&bio->bi_remaining);

	/*

	 * If it's not flagged as a chain, we are not going to dec the count

	 */

	if (bio_flagged(bio, BIO_CHAIN))

		bio_inc_remaining(bio);



	bio_endio(bio, error);

}

EXPORT_SYMBOL(bio_endio_nodec);

drivers/md/dm-cache-target.c

+1 −1
Original line number Diff line number Diff line
@@ -91,7 +91,7 @@ static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio) 
	 * Must bump bi_remaining to allow bio to complete with

	 * restored bi_end_io.

	 */

	atomic_inc(&bio->bi_remaining);

	bio_inc_remaining(bio);

}



/*----------------------------------------------------------------*/

drivers/md/dm-raid1.c

+1 −1
Original line number Diff line number Diff line
@@ -1254,7 +1254,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error) 
			dm_bio_restore(bd, bio);

			bio_record->details.bi_bdev = NULL;



			atomic_inc(&bio->bi_remaining);

			bio_inc_remaining(bio);



			queue_bio(ms, bio, rw);

			return DM_ENDIO_INCOMPLETE;

drivers/md/dm-snap.c

+1 −1
Original line number Diff line number Diff line
@@ -1478,7 +1478,7 @@ out: 
	if (full_bio) {

		full_bio->bi_end_io = pe->full_bio_end_io;

		full_bio->bi_private = pe->full_bio_private;

		atomic_inc(&full_bio->bi_remaining);

		bio_inc_remaining(full_bio);

	}

	increment_pending_exceptions_done_count();

drivers/md/dm-thin.c

+2 −2
Original line number Diff line number Diff line
@@ -795,7 +795,7 @@ static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m) 
{

	if (m->bio) {

		m->bio->bi_end_io = m->saved_bi_end_io;

		atomic_inc(&m->bio->bi_remaining);

		bio_inc_remaining(m->bio);

	}

	cell_error(m->tc->pool, m->cell);

	list_del(&m->list);
@@ -812,7 +812,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) 
	bio = m->bio;

	if (bio) {

		bio->bi_end_io = m->saved_bi_end_io;

		atomic_inc(&bio->bi_remaining);

		bio_inc_remaining(bio);

	}



	if (m->err) {

CRA Git | Maintained and supported by SUSTech CRA and CCSE