Commit 920d13a8 authored by Sagi Grimberg's avatar Sagi Grimberg Committed by Jens Axboe
Browse files

nvme-pci: factor out the cqe reading mechanics from __nvme_process_cq



Also, maintain a consumed counter to rely on for doorbell and
cqe_seen update instead of directly relying on the cq head and phase.

Signed-off-by: default avatarSagi Grimberg <sagi@grimberg.me>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarKeith Busch <keith.busch@intel.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 83a12fb7
Loading
Loading
Loading
Loading
+26 −22
Original line number Diff line number Diff line
@@ -770,37 +770,41 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
	nvme_end_request(req, cqe->status, cqe->result);
}

static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
static inline bool nvme_read_cqe(struct nvme_queue *nvmeq,
		struct nvme_completion *cqe)
{
	u16 head, phase;

	head = nvmeq->cq_head;
	phase = nvmeq->cq_phase;

	while (nvme_cqe_valid(nvmeq, head, phase)) {
		struct nvme_completion cqe = nvmeq->cqes[head];
	if (nvme_cqe_valid(nvmeq, nvmeq->cq_head, nvmeq->cq_phase)) {
		*cqe = nvmeq->cqes[nvmeq->cq_head];

		if (++head == nvmeq->q_depth) {
			head = 0;
			phase = !phase;
		if (++nvmeq->cq_head == nvmeq->q_depth) {
			nvmeq->cq_head = 0;
			nvmeq->cq_phase = !nvmeq->cq_phase;
		}
		return true;
	}
	return false;
}

		if (tag && *tag == cqe.command_id)
			*tag = -1;
static void __nvme_process_cq(struct nvme_queue *nvmeq, int *tag)
{
	struct nvme_completion cqe;
	int consumed = 0;

	while (nvme_read_cqe(nvmeq, &cqe)) {
		nvme_handle_cqe(nvmeq, &cqe);
	}
		consumed++;

	if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
		return;

	nvmeq->cq_head = head;
	nvmeq->cq_phase = phase;
		if (tag && *tag == cqe.command_id) {
			*tag = -1;
			break;
		}
	}

	if (consumed) {
		nvme_ring_cq_doorbell(nvmeq);

		nvmeq->cqe_seen = 1;
	}
}

static void nvme_process_cq(struct nvme_queue *nvmeq)
{