blk-mq: Remove generation seqeunce

	rq->internal_tag = -1;

	rq->start_time_ns = ktime_get_ns();

	rq->part = NULL;

	seqcount_init(&rq->gstate_seq);

	u64_stats_init(&rq->aborted_gstate_sync);

	/*

	 * See comment of blk_mq_init_request

	 */

	WRITE_ONCE(rq->gstate, MQ_RQ_GEN_INC);

}

EXPORT_SYMBOL(blk_rq_init);

	RQF_NAME(STATS),

	RQF_NAME(SPECIAL_PAYLOAD),

	RQF_NAME(ZONE_WRITE_LOCKED),

	RQF_NAME(MQ_TIMEOUT_EXPIRED),

	RQF_NAME(MQ_POLL_SLEPT),

};

#undef RQF_NAME

#endif

	data->ctx->rq_dispatched[op_is_sync(op)]++;

	refcount_set(&rq->ref, 1);

	return rq;

}

}

EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);

static void __blk_mq_free_request(struct request *rq)

{

	struct request_queue *q = rq->q;

	struct blk_mq_ctx *ctx = rq->mq_ctx;

	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);

	const int sched_tag = rq->internal_tag;

	if (rq->tag != -1)

		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);

	if (sched_tag != -1)

		blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);

	blk_mq_sched_restart(hctx);

	blk_queue_exit(q);

}

void blk_mq_free_request(struct request *rq)

{

	struct request_queue *q = rq->q;

	struct elevator_queue *e = q->elevator;

	struct blk_mq_ctx *ctx = rq->mq_ctx;

	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);

	const int sched_tag = rq->internal_tag;

	if (rq->rq_flags & RQF_ELVPRIV) {

		if (e && e->type->ops.mq.finish_request)

	if (blk_rq_rl(rq))

		blk_put_rl(blk_rq_rl(rq));

	blk_mq_rq_update_state(rq, MQ_RQ_IDLE);

	if (rq->tag != -1)

		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);

	if (sched_tag != -1)

		blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);

	blk_mq_sched_restart(hctx);

	blk_queue_exit(q);

	WRITE_ONCE(rq->state, MQ_RQ_IDLE);

	if (refcount_dec_and_test(&rq->ref))

		__blk_mq_free_request(rq);

}

EXPORT_SYMBOL_GPL(blk_mq_free_request);

	bool shared = false;

	int cpu;

	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT);

	blk_mq_rq_update_state(rq, MQ_RQ_COMPLETE);

	if (cmpxchg(&rq->state, MQ_RQ_IN_FLIGHT, MQ_RQ_COMPLETE) !=

			MQ_RQ_IN_FLIGHT)

		return;

	if (rq->internal_tag != -1)

		blk_mq_sched_completed_request(rq);

		*srcu_idx = srcu_read_lock(hctx->srcu);

}

static void blk_mq_rq_update_aborted_gstate(struct request *rq, u64 gstate)

{

	unsigned long flags;

	/*

	 * blk_mq_rq_aborted_gstate() is used from the completion path and

	 * can thus be called from irq context.  u64_stats_fetch in the

	 * middle of update on the same CPU leads to lockup.  Disable irq

	 * while updating.

	 */

	local_irq_save(flags);

	u64_stats_update_begin(&rq->aborted_gstate_sync);

	rq->aborted_gstate = gstate;

	u64_stats_update_end(&rq->aborted_gstate_sync);

	local_irq_restore(flags);

}

static u64 blk_mq_rq_aborted_gstate(struct request *rq)

{

	unsigned int start;

	u64 aborted_gstate;

	do {

		start = u64_stats_fetch_begin(&rq->aborted_gstate_sync);

		aborted_gstate = rq->aborted_gstate;

	} while (u64_stats_fetch_retry(&rq->aborted_gstate_sync, start));

	return aborted_gstate;

}

/**

 * blk_mq_complete_request - end I/O on a request

 * @rq:		the request being processed

 **/

void blk_mq_complete_request(struct request *rq)

{

	struct request_queue *q = rq->q;

	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);

	int srcu_idx;

	if (unlikely(blk_should_fake_timeout(q)))

	if (unlikely(blk_should_fake_timeout(rq->q)))

		return;

	/*

	 * If @rq->aborted_gstate equals the current instance, timeout is

	 * claiming @rq and we lost.  This is synchronized through

	 * hctx_lock().  See blk_mq_timeout_work() for details.

	 *

	 * Completion path never blocks and we can directly use RCU here

	 * instead of hctx_lock() which can be either RCU or SRCU.

	 * However, that would complicate paths which want to synchronize

	 * against us.  Let stay in sync with the issue path so that

	 * hctx_lock() covers both issue and completion paths.

	 */

	hctx_lock(hctx, &srcu_idx);

	if (blk_mq_rq_aborted_gstate(rq) != rq->gstate)

	__blk_mq_complete_request(rq);

	hctx_unlock(hctx, srcu_idx);

}

EXPORT_SYMBOL(blk_mq_complete_request);

	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);

	/*

	 * Mark @rq in-flight which also advances the generation number,

	 * and register for timeout.  Protect with a seqcount to allow the

	 * timeout path to read both @rq->gstate and @rq->deadline

	 * coherently.

	 *

	 * This is the only place where a request is marked in-flight.  If

	 * the timeout path reads an in-flight @rq->gstate, the

	 * @rq->deadline it reads together under @rq->gstate_seq is

	 * guaranteed to be the matching one.

	 */

	preempt_disable();

	write_seqcount_begin(&rq->gstate_seq);

	blk_add_timer(rq);

	blk_mq_rq_update_state(rq, MQ_RQ_IN_FLIGHT);

	write_seqcount_end(&rq->gstate_seq);

	preempt_enable();

	WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);

	if (q->dma_drain_size && blk_rq_bytes(rq)) {

		/*

}

EXPORT_SYMBOL(blk_mq_start_request);

/*

 * When we reach here because queue is busy, it's safe to change the state

 * to IDLE without checking @rq->aborted_gstate because we should still be

 * holding the RCU read lock and thus protected against timeout.

 */

static void __blk_mq_requeue_request(struct request *rq)

{

	struct request_queue *q = rq->q;

	trace_block_rq_requeue(q, rq);

	wbt_requeue(q->rq_wb, rq);

	if (blk_mq_rq_state(rq) != MQ_RQ_IDLE) {

		blk_mq_rq_update_state(rq, MQ_RQ_IDLE);

	if (blk_mq_request_started(rq)) {

		WRITE_ONCE(rq->state, MQ_RQ_IDLE);

		if (q->dma_drain_size && blk_rq_bytes(rq))

			rq->nr_phys_segments--;

	}

}

EXPORT_SYMBOL(blk_mq_tag_to_rq);

struct blk_mq_timeout_data {

	unsigned long next;

	unsigned int next_set;

	unsigned int nr_expired;

};

static void blk_mq_rq_timed_out(struct request *req, bool reserved)

{

	const struct blk_mq_ops *ops = req->q->mq_ops;

	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;

	req->rq_flags |= RQF_MQ_TIMEOUT_EXPIRED;

	if (ops->timeout)

		ret = ops->timeout(req, reserved);

	switch (ret) {

	case BLK_EH_HANDLED:

		if (blk_mq_rq_state(req) == MQ_RQ_IN_FLIGHT)

			__blk_mq_complete_request(req);

		break;

	case BLK_EH_RESET_TIMER:

		/*

		 * As nothing prevents from completion happening while

		 * ->aborted_gstate is set, this may lead to ignored

		 * completions and further spurious timeouts.

		 */

		blk_mq_rq_update_aborted_gstate(req, 0);

		blk_add_timer(req);

		break;

	case BLK_EH_NOT_HANDLED:

	}

}

static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,

		struct request *rq, void *priv, bool reserved)

static bool blk_mq_req_expired(struct request *rq, unsigned long *next)

{

	struct blk_mq_timeout_data *data = priv;

	unsigned long gstate, deadline;

	int start;

	might_sleep();

	unsigned long deadline;

	if (rq->rq_flags & RQF_MQ_TIMEOUT_EXPIRED)

		return;

	if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)

		return false;

	/* read coherent snapshots of @rq->state_gen and @rq->deadline */

	while (true) {

		start = read_seqcount_begin(&rq->gstate_seq);

		gstate = READ_ONCE(rq->gstate);

	deadline = blk_rq_deadline(rq);

		if (!read_seqcount_retry(&rq->gstate_seq, start))

			break;

		cond_resched();

	}

	if (time_after_eq(jiffies, deadline))

		return true;

	/* if in-flight && overdue, mark for abortion */

	if ((gstate & MQ_RQ_STATE_MASK) == MQ_RQ_IN_FLIGHT &&

	    time_after_eq(jiffies, deadline)) {

		blk_mq_rq_update_aborted_gstate(rq, gstate);

		data->nr_expired++;

		hctx->nr_expired++;

	} else if (!data->next_set || time_after(data->next, deadline)) {

		data->next = deadline;

		data->next_set = 1;

	}

	if (*next == 0)

		*next = deadline;

	else if (time_after(*next, deadline))

		*next = deadline;

	return false;

}

static void blk_mq_terminate_expired(struct blk_mq_hw_ctx *hctx,

static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,

		struct request *rq, void *priv, bool reserved)

{

	unsigned long *next = priv;

	/*

	 * Just do a quick check if it is expired before locking the request in

	 * so we're not unnecessarilly synchronizing across CPUs.

	 */

	if (!blk_mq_req_expired(rq, next))

		return;

	/*

	 * We marked @rq->aborted_gstate and waited for RCU.  If there were

	 * completions that we lost to, they would have finished and

	 * updated @rq->gstate by now; otherwise, the completion path is

	 * now guaranteed to see @rq->aborted_gstate and yield.  If

	 * @rq->aborted_gstate still matches @rq->gstate, @rq is ours.

	 * We have reason to believe the request may be expired. Take a

	 * reference on the request to lock this request lifetime into its

	 * currently allocated context to prevent it from being reallocated in

	 * the event the completion by-passes this timeout handler.

	 *

	 * If the reference was already released, then the driver beat the

	 * timeout handler to posting a natural completion.

	 */

	if (!refcount_inc_not_zero(&rq->ref))

		return;

	/*

	 * The request is now locked and cannot be reallocated underneath the

	 * timeout handler's processing. Re-verify this exact request is truly

	 * expired; if it is not expired, then the request was completed and

	 * reallocated as a new request.

	 */

	if (!(rq->rq_flags & RQF_MQ_TIMEOUT_EXPIRED) &&

	    READ_ONCE(rq->gstate) == rq->aborted_gstate)

	if (blk_mq_req_expired(rq, next))

		blk_mq_rq_timed_out(rq, reserved);

	if (refcount_dec_and_test(&rq->ref))

		__blk_mq_free_request(rq);

}

static void blk_mq_timeout_work(struct work_struct *work)

{

	struct request_queue *q =

		container_of(work, struct request_queue, timeout_work);

	struct blk_mq_timeout_data data = {

		.next		= 0,

		.next_set	= 0,

		.nr_expired	= 0,

	};

	unsigned long next = 0;

	struct blk_mq_hw_ctx *hctx;

	int i;

	if (!percpu_ref_tryget(&q->q_usage_counter))

		return;

	/* scan for the expired ones and set their ->aborted_gstate */

	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);

	if (data.nr_expired) {

		bool has_rcu = false;

		/*

		 * Wait till everyone sees ->aborted_gstate.  The

		 * sequential waits for SRCUs aren't ideal.  If this ever

		 * becomes a problem, we can add per-hw_ctx rcu_head and

		 * wait in parallel.

		 */

		queue_for_each_hw_ctx(q, hctx, i) {

			if (!hctx->nr_expired)

				continue;

			if (!(hctx->flags & BLK_MQ_F_BLOCKING))

				has_rcu = true;

			else

				synchronize_srcu(hctx->srcu);

			hctx->nr_expired = 0;

		}

		if (has_rcu)

			synchronize_rcu();

		/* terminate the ones we won */

		blk_mq_queue_tag_busy_iter(q, blk_mq_terminate_expired, NULL);

	}

	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);

	if (data.next_set) {

		data.next = blk_rq_timeout(round_jiffies_up(data.next));

		mod_timer(&q->timeout, data.next);

	if (next != 0) {

		mod_timer(&q->timeout, next);

	} else {

		/*

		 * Request timeouts are handled as a forward rolling timer. If

			return ret;

	}

	seqcount_init(&rq->gstate_seq);

	u64_stats_init(&rq->aborted_gstate_sync);

	/*

	 * start gstate with gen 1 instead of 0, otherwise it will be equal

	 * to aborted_gstate, and be identified timed out by

	 * blk_mq_terminate_expired.

	 */

	WRITE_ONCE(rq->gstate, MQ_RQ_GEN_INC);

	WRITE_ONCE(rq->state, MQ_RQ_IDLE);

	return 0;

}

	struct kobject		kobj;

} ____cacheline_aligned_in_smp;

/*

 * Bits for request->gstate.  The lower two bits carry MQ_RQ_* state value

 * and the upper bits the generation number.

 */

enum mq_rq_state {

	MQ_RQ_IDLE		= 0,

	MQ_RQ_IN_FLIGHT		= 1,

	MQ_RQ_COMPLETE		= 2,

	MQ_RQ_STATE_BITS	= 2,

	MQ_RQ_STATE_MASK	= (1 << MQ_RQ_STATE_BITS) - 1,

	MQ_RQ_GEN_INC		= 1 << MQ_RQ_STATE_BITS,

};

void blk_mq_freeze_queue(struct request_queue *q);

void blk_mq_free_queue(struct request_queue *q);

int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);

 * blk_mq_rq_state() - read the current MQ_RQ_* state of a request

 * @rq: target request.

 */

static inline int blk_mq_rq_state(struct request *rq)

static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)

{

	return READ_ONCE(rq->gstate) & MQ_RQ_STATE_MASK;

}

/**

 * blk_mq_rq_update_state() - set the current MQ_RQ_* state of a request

 * @rq: target request.

 * @state: new state to set.

 *

 * Set @rq's state to @state.  The caller is responsible for ensuring that

 * there are no other updaters.  A request can transition into IN_FLIGHT

 * only from IDLE and doing so increments the generation number.

 */

static inline void blk_mq_rq_update_state(struct request *rq,

					  enum mq_rq_state state)

{

	u64 old_val = READ_ONCE(rq->gstate);

	u64 new_val = (old_val & ~MQ_RQ_STATE_MASK) | state;

	if (state == MQ_RQ_IN_FLIGHT) {

		WARN_ON_ONCE((old_val & MQ_RQ_STATE_MASK) != MQ_RQ_IDLE);

		new_val += MQ_RQ_GEN_INC;

	}

	/* avoid exposing interim values */

	WRITE_ONCE(rq->gstate, new_val);

	return READ_ONCE(rq->state);

}

static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,

		req->timeout = q->rq_timeout;

	blk_rq_set_deadline(req, jiffies + req->timeout);

	req->rq_flags &= ~RQF_MQ_TIMEOUT_EXPIRED;

	/*

	 * Only the non-mq case needs to add the request to a protected list.