Commit 7e5a8794 authored by Tejun Heo's avatar Tejun Heo Committed by Jens Axboe
Browse files

block, cfq: move io_cq exit/release to blk-ioc.c 



With kmem_cache managed by blk-ioc, io_cq exit/release can be moved to
blk-ioc too.  The odd ->io_cq->exit/release() callbacks are replaced
with elevator_ops->elevator_exit_icq_fn() with unlinking from both ioc
and q, and freeing automatically handled by blk-ioc.  The elevator
operation only need to perform exit operation specific to the elevator
- in cfq's case, exiting the cfqq's.

Also, clearing of io_cq's on q detach is moved to block core and
automatically performed on elevator switch and q release.

Because the q io_cq points to might be freed before RCU callback for
the io_cq runs, blk-ioc code should remember to which cache the io_cq
needs to be freed when the io_cq is released.  New field
io_cq->__rcu_icq_cache is added for this purpose.  As both the new
field and rcu_head are used only after io_cq is released and the
q/ioc_node fields aren't, they are put into unions.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 3d3c2379

block/blk-ioc.c

+68 −8
Original line number Diff line number Diff line
@@ -44,6 +44,51 @@ EXPORT_SYMBOL(get_io_context); 
#define ioc_release_depth_dec(q)	do { } while (0)

#endif



static void icq_free_icq_rcu(struct rcu_head *head)

{

	struct io_cq *icq = container_of(head, struct io_cq, __rcu_head);



	kmem_cache_free(icq->__rcu_icq_cache, icq);

}



/*

 * Exit and free an icq.  Called with both ioc and q locked.

 */

static void ioc_exit_icq(struct io_cq *icq)

{

	struct io_context *ioc = icq->ioc;

	struct request_queue *q = icq->q;

	struct elevator_type *et = q->elevator->type;



	lockdep_assert_held(&ioc->lock);

	lockdep_assert_held(q->queue_lock);



	radix_tree_delete(&ioc->icq_tree, icq->q->id);

	hlist_del_init(&icq->ioc_node);

	list_del_init(&icq->q_node);



	/*

	 * Both setting lookup hint to and clearing it from @icq are done

	 * under queue_lock.  If it's not pointing to @icq now, it never

	 * will.  Hint assignment itself can race safely.

	 */

	if (rcu_dereference_raw(ioc->icq_hint) == icq)

		rcu_assign_pointer(ioc->icq_hint, NULL);



	if (et->ops.elevator_exit_icq_fn) {

		ioc_release_depth_inc(q);

		et->ops.elevator_exit_icq_fn(icq);

		ioc_release_depth_dec(q);

	}



	/*

	 * @icq->q might have gone away by the time RCU callback runs

	 * making it impossible to determine icq_cache.  Record it in @icq.

	 */

	icq->__rcu_icq_cache = et->icq_cache;

	call_rcu(&icq->__rcu_head, icq_free_icq_rcu);

}



/*

 * Slow path for ioc release in put_io_context().  Performs double-lock

 * dancing to unlink all icq's and then frees ioc.
@@ -87,10 +132,7 @@ static void ioc_release_fn(struct work_struct *work) 
			spin_lock(&ioc->lock);

			continue;

		}

		ioc_release_depth_inc(this_q);

		icq->exit(icq);

		icq->release(icq);

		ioc_release_depth_dec(this_q);

		ioc_exit_icq(icq);

	}



	if (last_q) {
@@ -167,10 +209,7 @@ void put_io_context(struct io_context *ioc, struct request_queue *locked_q) 
			last_q = this_q;

			continue;

		}

		ioc_release_depth_inc(this_q);

		icq->exit(icq);

		icq->release(icq);

		ioc_release_depth_dec(this_q);

		ioc_exit_icq(icq);

	}



	if (last_q && last_q != locked_q)
@@ -203,6 +242,27 @@ void exit_io_context(struct task_struct *task) 
	put_io_context(ioc, NULL);

}



/**

 * ioc_clear_queue - break any ioc association with the specified queue

 * @q: request_queue being cleared

 *

 * Walk @q->icq_list and exit all io_cq's.  Must be called with @q locked.

 */

void ioc_clear_queue(struct request_queue *q)

{

	lockdep_assert_held(q->queue_lock);



	while (!list_empty(&q->icq_list)) {

		struct io_cq *icq = list_entry(q->icq_list.next,

					       struct io_cq, q_node);

		struct io_context *ioc = icq->ioc;



		spin_lock(&ioc->lock);

		ioc_exit_icq(icq);

		spin_unlock(&ioc->lock);

	}

}



void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags,

				int node)

{

block/blk-sysfs.c

+5 −1
Original line number Diff line number Diff line
@@ -479,8 +479,12 @@ static void blk_release_queue(struct kobject *kobj) 


	blk_sync_queue(q);



	if (q->elevator)

	if (q->elevator) {

		spin_lock_irq(q->queue_lock);

		ioc_clear_queue(q);

		spin_unlock_irq(q->queue_lock);

		elevator_exit(q->elevator);

	}



	blk_throtl_exit(q);

block/blk.h

+1 −0
Original line number Diff line number Diff line
@@ -200,6 +200,7 @@ static inline int blk_do_io_stat(struct request *rq) 
 */

void get_io_context(struct io_context *ioc);

struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q);

void ioc_clear_queue(struct request_queue *q);



void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_mask,

				int node);

block/cfq-iosched.c

+2 −45
Original line number Diff line number Diff line
@@ -2674,26 +2674,6 @@ static void cfq_put_queue(struct cfq_queue *cfqq) 
	cfq_put_cfqg(cfqg);

}



static void cfq_icq_free_rcu(struct rcu_head *head)

{

	kmem_cache_free(cfq_icq_pool,

			icq_to_cic(container_of(head, struct io_cq, rcu_head)));

}



static void cfq_icq_free(struct io_cq *icq)

{

	call_rcu(&icq->rcu_head, cfq_icq_free_rcu);

}



static void cfq_release_icq(struct io_cq *icq)

{

	struct io_context *ioc = icq->ioc;



	radix_tree_delete(&ioc->icq_tree, icq->q->id);

	hlist_del(&icq->ioc_node);

	cfq_icq_free(icq);

}



static void cfq_put_cooperator(struct cfq_queue *cfqq)

{

	struct cfq_queue *__cfqq, *next;
@@ -2731,17 +2711,6 @@ static void cfq_exit_icq(struct io_cq *icq) 
{

	struct cfq_io_cq *cic = icq_to_cic(icq);

	struct cfq_data *cfqd = cic_to_cfqd(cic);

	struct io_context *ioc = icq->ioc;



	list_del_init(&icq->q_node);



	/*

	 * Both setting lookup hint to and clearing it from @icq are done

	 * under queue_lock.  If it's not pointing to @icq now, it never

	 * will.  Hint assignment itself can race safely.

	 */

	if (rcu_dereference_raw(ioc->icq_hint) == icq)

		rcu_assign_pointer(ioc->icq_hint, NULL);



	if (cic->cfqq[BLK_RW_ASYNC]) {

		cfq_exit_cfqq(cfqd, cic->cfqq[BLK_RW_ASYNC]);
@@ -2764,8 +2733,6 @@ static struct cfq_io_cq *cfq_alloc_cic(struct cfq_data *cfqd, gfp_t gfp_mask) 
		cic->ttime.last_end_request = jiffies;

		INIT_LIST_HEAD(&cic->icq.q_node);

		INIT_HLIST_NODE(&cic->icq.ioc_node);

		cic->icq.exit = cfq_exit_icq;

		cic->icq.release = cfq_release_icq;

	}



	return cic;
@@ -3034,7 +3001,7 @@ out: 
	if (ret)

		printk(KERN_ERR "cfq: icq link failed!\n");

	if (icq)

		cfq_icq_free(icq);

		kmem_cache_free(cfq_icq_pool, icq);

	return ret;

}
@@ -3774,17 +3741,6 @@ static void cfq_exit_queue(struct elevator_queue *e) 
	if (cfqd->active_queue)

		__cfq_slice_expired(cfqd, cfqd->active_queue, 0);



	while (!list_empty(&q->icq_list)) {

		struct io_cq *icq = list_entry(q->icq_list.next,

					       struct io_cq, q_node);

		struct io_context *ioc = icq->ioc;



		spin_lock(&ioc->lock);

		cfq_exit_icq(icq);

		cfq_release_icq(icq);

		spin_unlock(&ioc->lock);

	}



	cfq_put_async_queues(cfqd);

	cfq_release_cfq_groups(cfqd);
@@ -4019,6 +3975,7 @@ static struct elevator_type iosched_cfq = { 
		.elevator_completed_req_fn =	cfq_completed_request,

		.elevator_former_req_fn =	elv_rb_former_request,

		.elevator_latter_req_fn =	elv_rb_latter_request,

		.elevator_exit_icq_fn =		cfq_exit_icq,

		.elevator_set_req_fn =		cfq_set_request,

		.elevator_put_req_fn =		cfq_put_request,

		.elevator_may_queue_fn =	cfq_may_queue,

block/elevator.c

+2 −1
Original line number Diff line number Diff line
@@ -979,8 +979,9 @@ static int elevator_switch(struct request_queue *q, struct elevator_type *new_e) 
			goto fail_register;

	}



	/* done, replace the old one with new one and turn off BYPASS */

	/* done, clear io_cq's, switch elevators and turn off BYPASS */

	spin_lock_irq(q->queue_lock);

	ioc_clear_queue(q);

	old_elevator = q->elevator;

	q->elevator = e;

	spin_unlock_irq(q->queue_lock);

CRA Git | Maintained and supported by SUSTech CRA and CCSE