Commit ff029451 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

nvme: use blk_mq_complete_request_remote to avoid an indirect function call 



Use the new blk_mq_complete_request_remote helper to avoid an indirect
function call in the completion fast path.

Reviewed-by: default avatarDaniel Wagner <dwagner@suse.de>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 8446546c

drivers/nvme/host/fc.c

+3 −1
Original line number Diff line number Diff line
@@ -227,6 +227,7 @@ static DECLARE_COMPLETION(nvme_fc_unload_proceed); 
 */

static struct device *fc_udev_device;



static void nvme_fc_complete_rq(struct request *rq);



/* *********************** FC-NVME Port Management ************************ */
@@ -2033,7 +2034,8 @@ done: 
	}



	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);

	nvme_end_request(rq, status, result);

	if (!nvme_end_request(rq, status, result))

		nvme_fc_complete_rq(rq);



check_error:

	if (terminate_assoc)

drivers/nvme/host/nvme.h

+4 −3
Original line number Diff line number Diff line
@@ -472,7 +472,7 @@ static inline u32 nvme_bytes_to_numd(size_t len) 
	return (len >> 2) - 1;

}



static inline void nvme_end_request(struct request *req, __le16 status,

static inline bool nvme_end_request(struct request *req, __le16 status,

		union nvme_result result)

{

	struct nvme_request *rq = nvme_req(req);
@@ -481,8 +481,9 @@ static inline void nvme_end_request(struct request *req, __le16 status, 
	rq->result = result;

	/* inject error when permitted by fault injection framework */

	nvme_should_fail(req);

	if (likely(!blk_should_fake_timeout(req->q)))

		blk_mq_complete_request(req);

	if (unlikely(blk_should_fake_timeout(req->q)))

		return true;

	return blk_mq_complete_request_remote(req);

}



static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)

drivers/nvme/host/pci.c

+2 −1
Original line number Diff line number Diff line
@@ -963,7 +963,8 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) 


	req = blk_mq_tag_to_rq(nvme_queue_tagset(nvmeq), cqe->command_id);

	trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail);

	nvme_end_request(req, cqe->status, cqe->result);

	if (!nvme_end_request(req, cqe->status, cqe->result))

		nvme_pci_complete_rq(req);

}



static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)

drivers/nvme/host/rdma.c

+3 −1
Original line number Diff line number Diff line
@@ -149,6 +149,7 @@ MODULE_PARM_DESC(register_always, 
static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,

		struct rdma_cm_event *event);

static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);

static void nvme_rdma_complete_rq(struct request *rq);



static const struct blk_mq_ops nvme_rdma_mq_ops;

static const struct blk_mq_ops nvme_rdma_admin_mq_ops;
@@ -1155,7 +1156,8 @@ static void nvme_rdma_end_request(struct nvme_rdma_request *req) 


	if (!refcount_dec_and_test(&req->ref))

		return;

	nvme_end_request(rq, req->status, req->result);

	if (!nvme_end_request(rq, req->status, req->result))

		nvme_rdma_complete_rq(rq);

}



static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,

drivers/nvme/host/tcp.c

+4 −2
Original line number Diff line number Diff line
@@ -464,7 +464,8 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 
		return -EINVAL;

	}



	nvme_end_request(rq, cqe->status, cqe->result);

	if (!nvme_end_request(rq, cqe->status, cqe->result))

		nvme_complete_rq(rq);

	queue->nr_cqe++;



	return 0;
@@ -654,7 +655,8 @@ static inline void nvme_tcp_end_request(struct request *rq, u16 status) 
{

	union nvme_result res = {};



	nvme_end_request(rq, cpu_to_le16(status << 1), res);

	if (!nvme_end_request(rq, cpu_to_le16(status << 1), res))

		nvme_complete_rq(rq);

}



static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,

CRA Git | Maintained and supported by SUSTech CRA and CCSE