Commit 7ae7a8de authored by Jens Axboe's avatar Jens Axboe
Browse files

Merge tag 'nvme-5.10-2020-11-05' of git://git.infradead.org/nvme into block-5.10 

Pull NVMe fixes from Christoph:

"nvme fixes for 5.10:

 - revert a nvme_queue size optimization (Keith Bush)
 - fabrics timeout races fixes (Chao Leng and Sagi Grimberg)"

* tag 'nvme-5.10-2020-11-05' of git://git.infradead.org/nvme:
  nvme-tcp: avoid repeated request completion
  nvme-rdma: avoid repeated request completion
  nvme-tcp: avoid race between time out and tear down
  nvme-rdma: avoid race between time out and tear down
  nvme: introduce nvme_sync_io_queues
  Revert "nvme-pci: remove last_sq_tail"
parents 3cea11cd 0a8a2c85

drivers/nvme/host/core.c

+6 −2
Original line number Diff line number Diff line
@@ -4582,8 +4582,7 @@ void nvme_start_queues(struct nvme_ctrl *ctrl) 
}

EXPORT_SYMBOL_GPL(nvme_start_queues);





void nvme_sync_queues(struct nvme_ctrl *ctrl)

void nvme_sync_io_queues(struct nvme_ctrl *ctrl)

{

	struct nvme_ns *ns;
@@ -4591,7 +4590,12 @@ void nvme_sync_queues(struct nvme_ctrl *ctrl) 
	list_for_each_entry(ns, &ctrl->namespaces, list)

		blk_sync_queue(ns->queue);

	up_read(&ctrl->namespaces_rwsem);

}

EXPORT_SYMBOL_GPL(nvme_sync_io_queues);



void nvme_sync_queues(struct nvme_ctrl *ctrl)

{

	nvme_sync_io_queues(ctrl);

	if (ctrl->admin_q)

		blk_sync_queue(ctrl->admin_q);

}

drivers/nvme/host/nvme.h

+1 −0
Original line number Diff line number Diff line
@@ -602,6 +602,7 @@ void nvme_stop_queues(struct nvme_ctrl *ctrl); 
void nvme_start_queues(struct nvme_ctrl *ctrl);

void nvme_kill_queues(struct nvme_ctrl *ctrl);

void nvme_sync_queues(struct nvme_ctrl *ctrl);

void nvme_sync_io_queues(struct nvme_ctrl *ctrl);

void nvme_unfreeze(struct nvme_ctrl *ctrl);

void nvme_wait_freeze(struct nvme_ctrl *ctrl);

int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);

drivers/nvme/host/pci.c

+19 −4
Original line number Diff line number Diff line
@@ -198,6 +198,7 @@ struct nvme_queue { 
	u32 q_depth;

	u16 cq_vector;

	u16 sq_tail;

	u16 last_sq_tail;

	u16 cq_head;

	u16 qid;

	u8 cq_phase;
@@ -455,11 +456,24 @@ static int nvme_pci_map_queues(struct blk_mq_tag_set *set) 
	return 0;

}



static inline void nvme_write_sq_db(struct nvme_queue *nvmeq)

/*

 * Write sq tail if we are asked to, or if the next command would wrap.

 */

static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq)

{

	if (!write_sq) {

		u16 next_tail = nvmeq->sq_tail + 1;



		if (next_tail == nvmeq->q_depth)

			next_tail = 0;

		if (next_tail != nvmeq->last_sq_tail)

			return;

	}



	if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail,

			nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei))

		writel(nvmeq->sq_tail, nvmeq->q_db);

	nvmeq->last_sq_tail = nvmeq->sq_tail;

}



/**
@@ -476,8 +490,7 @@ static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd, 
	       cmd, sizeof(*cmd));

	if (++nvmeq->sq_tail == nvmeq->q_depth)

		nvmeq->sq_tail = 0;

	if (write_sq)

		nvme_write_sq_db(nvmeq);

	nvme_write_sq_db(nvmeq, write_sq);

	spin_unlock(&nvmeq->sq_lock);

}
@@ -486,7 +499,8 @@ static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx) 
	struct nvme_queue *nvmeq = hctx->driver_data;



	spin_lock(&nvmeq->sq_lock);

	nvme_write_sq_db(nvmeq);

	if (nvmeq->sq_tail != nvmeq->last_sq_tail)

		nvme_write_sq_db(nvmeq, true);

	spin_unlock(&nvmeq->sq_lock);

}
@@ -1496,6 +1510,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 
	struct nvme_dev *dev = nvmeq->dev;



	nvmeq->sq_tail = 0;

	nvmeq->last_sq_tail = 0;

	nvmeq->cq_head = 0;

	nvmeq->cq_phase = 1;

	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];

drivers/nvme/host/rdma.c

+3 −11
Original line number Diff line number Diff line
@@ -122,7 +122,6 @@ struct nvme_rdma_ctrl { 
	struct sockaddr_storage src_addr;



	struct nvme_ctrl	ctrl;

	struct mutex		teardown_lock;

	bool			use_inline_data;

	u32			io_queues[HCTX_MAX_TYPES];

};
@@ -1010,8 +1009,8 @@ out_free_io_queues: 
static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl,

		bool remove)

{

	mutex_lock(&ctrl->teardown_lock);

	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);

	blk_sync_queue(ctrl->ctrl.admin_q);

	nvme_rdma_stop_queue(&ctrl->queues[0]);

	if (ctrl->ctrl.admin_tagset) {

		blk_mq_tagset_busy_iter(ctrl->ctrl.admin_tagset,
@@ -1021,16 +1020,15 @@ static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl, 
	if (remove)

		blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);

	nvme_rdma_destroy_admin_queue(ctrl, remove);

	mutex_unlock(&ctrl->teardown_lock);

}



static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl,

		bool remove)

{

	mutex_lock(&ctrl->teardown_lock);

	if (ctrl->ctrl.queue_count > 1) {

		nvme_start_freeze(&ctrl->ctrl);

		nvme_stop_queues(&ctrl->ctrl);

		nvme_sync_io_queues(&ctrl->ctrl);

		nvme_rdma_stop_io_queues(ctrl);

		if (ctrl->ctrl.tagset) {

			blk_mq_tagset_busy_iter(ctrl->ctrl.tagset,
@@ -1041,7 +1039,6 @@ static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl, 
			nvme_start_queues(&ctrl->ctrl);

		nvme_rdma_destroy_io_queues(ctrl, remove);

	}

	mutex_unlock(&ctrl->teardown_lock);

}



static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
@@ -1976,16 +1973,12 @@ static void nvme_rdma_complete_timed_out(struct request *rq) 
{

	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);

	struct nvme_rdma_queue *queue = req->queue;

	struct nvme_rdma_ctrl *ctrl = queue->ctrl;



	/* fence other contexts that may complete the command */

	mutex_lock(&ctrl->teardown_lock);

	nvme_rdma_stop_queue(queue);

	if (!blk_mq_request_completed(rq)) {

	if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) {

		nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;

		blk_mq_complete_request(rq);

	}

	mutex_unlock(&ctrl->teardown_lock);

}



static enum blk_eh_timer_return
@@ -2320,7 +2313,6 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, 
		return ERR_PTR(-ENOMEM);

	ctrl->ctrl.opts = opts;

	INIT_LIST_HEAD(&ctrl->list);

	mutex_init(&ctrl->teardown_lock);



	if (!(opts->mask & NVMF_OPT_TRSVCID)) {

		opts->trsvcid =

drivers/nvme/host/tcp.c

+4 −12
Original line number Diff line number Diff line
@@ -124,7 +124,6 @@ struct nvme_tcp_ctrl { 
	struct sockaddr_storage src_addr;

	struct nvme_ctrl	ctrl;



	struct mutex		teardown_lock;

	struct work_struct	err_work;

	struct delayed_work	connect_work;

	struct nvme_tcp_request async_req;
@@ -1886,8 +1885,8 @@ out_free_queue: 
static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,

		bool remove)

{

	mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);

	blk_mq_quiesce_queue(ctrl->admin_q);

	blk_sync_queue(ctrl->admin_q);

	nvme_tcp_stop_queue(ctrl, 0);

	if (ctrl->admin_tagset) {

		blk_mq_tagset_busy_iter(ctrl->admin_tagset,
@@ -1897,18 +1896,17 @@ static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 
	if (remove)

		blk_mq_unquiesce_queue(ctrl->admin_q);

	nvme_tcp_destroy_admin_queue(ctrl, remove);

	mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);

}



static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,

		bool remove)

{

	mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);

	if (ctrl->queue_count <= 1)

		goto out;

		return;

	blk_mq_quiesce_queue(ctrl->admin_q);

	nvme_start_freeze(ctrl);

	nvme_stop_queues(ctrl);

	nvme_sync_io_queues(ctrl);

	nvme_tcp_stop_io_queues(ctrl);

	if (ctrl->tagset) {

		blk_mq_tagset_busy_iter(ctrl->tagset,
@@ -1918,8 +1916,6 @@ static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 
	if (remove)

		nvme_start_queues(ctrl);

	nvme_tcp_destroy_io_queues(ctrl, remove);

out:

	mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);

}



static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
@@ -2171,14 +2167,11 @@ static void nvme_tcp_complete_timed_out(struct request *rq) 
	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);

	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;



	/* fence other contexts that may complete the command */

	mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);

	nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));

	if (!blk_mq_request_completed(rq)) {

	if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) {

		nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;

		blk_mq_complete_request(rq);

	}

	mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);

}



static enum blk_eh_timer_return
@@ -2455,7 +2448,6 @@ static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 
			nvme_tcp_reconnect_ctrl_work);

	INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);

	INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);

	mutex_init(&ctrl->teardown_lock);



	if (!(opts->mask & NVMF_OPT_TRSVCID)) {

		opts->trsvcid =

CRA Git | Maintained and supported by SUSTech CRA and CCSE