Commit cb8432d6 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

block: allocate struct hd_struct as part of struct bdev_inode 



Allocate hd_struct together with struct block_device to pre-load
the lifetime rule changes in preparation of merging the two structures.

Note that part0 was previously embedded into struct gendisk, but is
a separate allocation now, and already points to the block_device instead
of the hd_struct.  The lifetime of struct gendisk is still controlled by
the struct device embedded in the part0 hd_struct.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarJan Kara <jack@suse.cz>
Reviewed-by: default avatarHannes Reinecke <hare@suse.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 83950d35

block/blk-core.c

+7 −9
Original line number Diff line number Diff line
@@ -714,7 +714,8 @@ static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part) 


static noinline int should_fail_bio(struct bio *bio)

{

	if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))

	if (should_fail_request(bio->bi_disk->part0->bd_part,

			bio->bi_iter.bi_size))

		return -EIO;

	return 0;

}
@@ -831,7 +832,7 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio) 
		if (unlikely(blk_partition_remap(bio)))

			goto end_io;

	} else {

		if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))

		if (unlikely(bio_check_ro(bio, bio->bi_disk->part0->bd_part)))

			goto end_io;

		if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))

			goto end_io;
@@ -1203,7 +1204,7 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request * 
		return ret;



	if (rq->rq_disk &&

	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))

	    should_fail_request(rq->rq_disk->part0->bd_part, blk_rq_bytes(rq)))

		return BLK_STS_IOERR;



	if (blk_crypto_insert_cloned_request(rq))
@@ -1272,7 +1273,7 @@ again: 
			__part_stat_add(part, io_ticks, end ? now - stamp : 1);

	}

	if (part->partno) {

		part = &part_to_disk(part)->part0;

		part = part_to_disk(part)->part0->bd_part;

		goto again;

	}

}
@@ -1309,8 +1310,6 @@ void blk_account_io_done(struct request *req, u64 now) 
		part_stat_inc(part, ios[sgrp]);

		part_stat_add(part, nsecs[sgrp], now - req->start_time_ns);

		part_stat_unlock();



		hd_struct_put(part);

	}

}
@@ -1354,7 +1353,7 @@ EXPORT_SYMBOL_GPL(part_start_io_acct); 
unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,

				 unsigned int op)

{

	return __part_start_io_acct(&disk->part0, sectors, op);

	return __part_start_io_acct(disk->part0->bd_part, sectors, op);

}

EXPORT_SYMBOL(disk_start_io_acct);
@@ -1376,14 +1375,13 @@ void part_end_io_acct(struct hd_struct *part, struct bio *bio, 
		      unsigned long start_time)

{

	__part_end_io_acct(part, bio_op(bio), start_time);

	hd_struct_put(part);

}

EXPORT_SYMBOL_GPL(part_end_io_acct);



void disk_end_io_acct(struct gendisk *disk, unsigned int op,

		      unsigned long start_time)

{

	__part_end_io_acct(&disk->part0, op, start_time);

	__part_end_io_acct(disk->part0->bd_part, op, start_time);

}

EXPORT_SYMBOL(disk_end_io_acct);

block/blk-flush.c

+1 −1
Original line number Diff line number Diff line
@@ -139,7 +139,7 @@ static void blk_flush_queue_rq(struct request *rq, bool add_front) 


static void blk_account_io_flush(struct request *rq)

{

	struct hd_struct *part = &rq->rq_disk->part0;

	struct hd_struct *part = rq->rq_disk->part0->bd_part;



	part_stat_lock();

	part_stat_inc(part, ios[STAT_FLUSH]);

block/blk-merge.c

+0 −2
Original line number Diff line number Diff line
@@ -683,8 +683,6 @@ static void blk_account_io_merge_request(struct request *req) 
		part_stat_lock();

		part_stat_inc(req->part, merges[op_stat_group(req_op(req))]);

		part_stat_unlock();



		hd_struct_put(req->part);

	}

}

block/blk.h

+0 −21
Original line number Diff line number Diff line
@@ -363,27 +363,6 @@ int bdev_del_partition(struct block_device *bdev, int partno); 
int bdev_resize_partition(struct block_device *bdev, int partno,

		sector_t start, sector_t length);

int disk_expand_part_tbl(struct gendisk *disk, int target);

int hd_ref_init(struct hd_struct *part);



/* no need to get/put refcount of part0 */

static inline int hd_struct_try_get(struct hd_struct *part)

{

	if (part->partno)

		return percpu_ref_tryget_live(&part->ref);

	return 1;

}



static inline void hd_struct_put(struct hd_struct *part)

{

	if (part->partno)

		percpu_ref_put(&part->ref);

}



static inline void hd_free_part(struct hd_struct *part)

{

	bdput(part->bdev);

	percpu_ref_exit(&part->ref);

}



int bio_add_hw_page(struct request_queue *q, struct bio *bio,

		struct page *page, unsigned int len, unsigned int offset,

block/genhd.c

+19 −31
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ static void disk_release_events(struct gendisk *disk); 


void set_capacity(struct gendisk *disk, sector_t sectors)

{

	struct block_device *bdev = disk->part0.bdev;

	struct block_device *bdev = disk->part0;



	spin_lock(&bdev->bd_size_lock);

	i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT);
@@ -318,9 +318,7 @@ static inline int sector_in_part(struct hd_struct *part, sector_t sector) 
 * primarily used for stats accounting.

 *

 * CONTEXT:

 * RCU read locked.  The returned partition pointer is always valid

 * because its refcount is grabbed except for part0, which lifetime

 * is same with the disk.

 * RCU read locked.

 *

 * RETURNS:

 * Found partition on success, part0 is returned if no partition matches
@@ -336,26 +334,19 @@ struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector) 
	ptbl = rcu_dereference(disk->part_tbl);



	part = rcu_dereference(ptbl->last_lookup);

	if (part && sector_in_part(part, sector) && hd_struct_try_get(part))

	if (part && sector_in_part(part, sector))

		goto out_unlock;



	for (i = 1; i < ptbl->len; i++) {

		part = rcu_dereference(ptbl->part[i]);



		if (part && sector_in_part(part, sector)) {

			/*

			 * only live partition can be cached for lookup,

			 * so use-after-free on cached & deleting partition

			 * can be avoided

			 */

			if (!hd_struct_try_get(part))

				break;

			rcu_assign_pointer(ptbl->last_lookup, part);

			goto out_unlock;

		}

	}



	part = &disk->part0;

	part = disk->part0->bd_part;

out_unlock:

	rcu_read_unlock();

	return part;
@@ -681,7 +672,7 @@ static void register_disk(struct device *parent, struct gendisk *disk, 
	 */

	pm_runtime_set_memalloc_noio(ddev, true);



	disk->part0.bdev->bd_holder_dir =

	disk->part0->bd_holder_dir =

		kobject_create_and_add("holders", &ddev->kobj);

	disk->slave_dir = kobject_create_and_add("slaves", &ddev->kobj);
@@ -748,7 +739,7 @@ static void __device_add_disk(struct device *parent, struct gendisk *disk, 


	disk->flags |= GENHD_FL_UP;



	retval = blk_alloc_devt(&disk->part0, &devt);

	retval = blk_alloc_devt(disk->part0->bd_part, &devt);

	if (retval) {

		WARN_ON(1);

		return;
@@ -775,7 +766,7 @@ static void __device_add_disk(struct device *parent, struct gendisk *disk, 
		ret = bdi_register(bdi, "%u:%u", MAJOR(devt), MINOR(devt));

		WARN_ON(ret);

		bdi_set_owner(bdi, dev);

		bdev_add(disk->part0.bdev, devt);

		bdev_add(disk->part0, devt);

	}

	register_disk(parent, disk, groups);

	if (register_queue)
@@ -888,11 +879,11 @@ void del_gendisk(struct gendisk *disk) 


	blk_unregister_queue(disk);



	kobject_put(disk->part0.bdev->bd_holder_dir);

	kobject_put(disk->part0->bd_holder_dir);

	kobject_put(disk->slave_dir);



	part_stat_set_all(&disk->part0, 0);

	disk->part0.bdev->bd_stamp = 0;

	part_stat_set_all(disk->part0->bd_part, 0);

	disk->part0->bd_stamp = 0;

	if (!sysfs_deprecated)

		sysfs_remove_link(block_depr, dev_name(disk_to_dev(disk)));

	pm_runtime_set_memalloc_noio(disk_to_dev(disk), false);
@@ -1005,7 +996,7 @@ void __init printk_all_partitions(void) 
		 */

		disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);

		while ((part = disk_part_iter_next(&piter))) {

			bool is_part0 = part == &disk->part0;

			bool is_part0 = part == disk->part0->bd_part;



			printk("%s%s %10llu %s %s", is_part0 ? "" : "  ",

			       bdevt_str(part_devt(part), devt_buf),
@@ -1460,7 +1451,7 @@ static void disk_release(struct device *dev) 
	disk_release_events(disk);

	kfree(disk->random);

	disk_replace_part_tbl(disk, NULL);

	hd_free_part(&disk->part0);

	bdput(disk->part0);

	if (disk->queue)

		blk_put_queue(disk->queue);

	kfree(disk);
@@ -1626,8 +1617,8 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) 
	if (!disk)

		return NULL;



	disk->part0.bdev = bdev_alloc(disk, 0);

	if (!disk->part0.bdev)

	disk->part0 = bdev_alloc(disk, 0);

	if (!disk->part0)

		goto out_free_disk;



	disk->node_id = node_id;
@@ -1635,10 +1626,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) 
		goto out_bdput;



	ptbl = rcu_dereference_protected(disk->part_tbl, 1);

	rcu_assign_pointer(ptbl->part[0], &disk->part0);



	if (hd_ref_init(&disk->part0))

		goto out_bdput;

	rcu_assign_pointer(ptbl->part[0], disk->part0->bd_part);



	disk->minors = minors;

	rand_initialize_disk(disk);
@@ -1648,7 +1636,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) 
	return disk;



out_bdput:

	bdput(disk->part0.bdev);

	bdput(disk->part0);

out_free_disk:

	kfree(disk);

	return NULL;
@@ -1687,9 +1675,9 @@ void set_disk_ro(struct gendisk *disk, int flag) 
	struct disk_part_iter piter;

	struct hd_struct *part;



	if (disk->part0.bdev->bd_read_only != flag) {

	if (disk->part0->bd_read_only != flag) {

		set_disk_ro_uevent(disk, flag);

		disk->part0.bdev->bd_read_only = flag;

		disk->part0->bd_read_only = flag;

	}



	disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY);

CRA Git | Maintained and supported by SUSTech CRA and CCSE