blkcg: make sure blkg_lookup() returns %NULL if @q is bypassing (80fd9979) · Commits · 戴 / test

block/blk-cgroup.c

+33 −17

Original line number	Diff line number	Diff line
		@@ -137,6 +137,38 @@ static struct blkio_group blkg_alloc(struct blkio_cgroup blkcg,
		return blkg;
		}

		static struct blkio_group __blkg_lookup(struct blkio_cgroup blkcg,
		struct request_queue *q)
		{
		struct blkio_group *blkg;
		struct hlist_node *n;

		hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node)
		if (blkg->q == q)
		return blkg;
		return NULL;
		}

		/**
		* blkg_lookup - lookup blkg for the specified blkcg - q pair
		* @blkcg: blkcg of interest
		* @q: request_queue of interest
		*
		* Lookup blkg for the @blkcg - @q pair. This function should be called
		* under RCU read lock and is guaranteed to return %NULL if @q is bypassing
		* - see blk_queue_bypass_start() for details.
		*/
		struct blkio_group blkg_lookup(struct blkio_cgroup blkcg,
		struct request_queue *q)
		{
		WARN_ON_ONCE(!rcu_read_lock_held());

		if (unlikely(blk_queue_bypass(q)))
		return NULL;
		return __blkg_lookup(blkcg, q);
		}
		EXPORT_SYMBOL_GPL(blkg_lookup);

		struct blkio_group blkg_lookup_create(struct blkio_cgroup blkcg,
		struct request_queue *q,
		bool for_root)
		@@ -150,13 +182,11 @@ struct blkio_group blkg_lookup_create(struct blkio_cgroup blkcg,
		/*
		* This could be the first entry point of blkcg implementation and
		* we shouldn't allow anything to go through for a bypassing queue.
		* The following can be removed if blkg lookup is guaranteed to
		* fail on a bypassing queue.
		*/
		if (unlikely(blk_queue_bypass(q)) && !for_root)
		return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);

		blkg = blkg_lookup(blkcg, q);
		blkg = __blkg_lookup(blkcg, q);
		if (blkg)
		return blkg;

		@@ -185,20 +215,6 @@ out:
		}
		EXPORT_SYMBOL_GPL(blkg_lookup_create);

		/* called under rcu_read_lock(). */
		struct blkio_group blkg_lookup(struct blkio_cgroup blkcg,
		struct request_queue *q)
		{
		struct blkio_group *blkg;
		struct hlist_node *n;

		hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node)
		if (blkg->q == q)
		return blkg;
		return NULL;
		}
		EXPORT_SYMBOL_GPL(blkg_lookup);

		static void blkg_destroy(struct blkio_group *blkg)
		{
		struct request_queue *q = blkg->q;

block/blk-core.c

+13 −2

Original line number	Diff line number	Diff line
		@@ -416,7 +416,8 @@ void blk_drain_queue(struct request_queue *q, bool drain_all)
		* In bypass mode, only the dispatch FIFO queue of @q is used. This
		* function makes @q enter bypass mode and drains all requests which were
		* throttled or issued before. On return, it's guaranteed that no request
		* is being throttled or has ELVPRIV set.
		* is being throttled or has ELVPRIV set and blk_queue_bypass() %true
		* inside queue or RCU read lock.
		*/
		void blk_queue_bypass_start(struct request_queue *q)
		{
		@@ -426,6 +427,8 @@ void blk_queue_bypass_start(struct request_queue *q)
		spin_unlock_irq(q->queue_lock);

		blk_drain_queue(q, false);
		/* ensure blk_queue_bypass() is %true inside RCU read lock */
		synchronize_rcu();
		}
		EXPORT_SYMBOL_GPL(blk_queue_bypass_start);

		@@ -462,7 +465,15 @@ void blk_cleanup_queue(struct request_queue *q)

		spin_lock_irq(lock);

		/* dead queue is permanently in bypass mode till released */
		/*
		* Dead queue is permanently in bypass mode till released. Note
		* that, unlike blk_queue_bypass_start(), we aren't performing
		* synchronize_rcu() after entering bypass mode to avoid the delay
		* as some drivers create and destroy a lot of queues while
		* probing. This is still safe because blk_release_queue() will be
		* called only after the queue refcnt drops to zero and nothing,
		* RCU or not, would be traversing the queue by then.
		*/
		q->bypass_depth++;
		queue_flag_set(QUEUE_FLAG_BYPASS, q);

Admin message