IB/hfi1: Move rvt_cq_wc struct into uapi directory (239b0e52) · Commits · 戴 / test

drivers/infiniband/hw/hfi1/qp.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -702,8 +702,8 @@ void qp_iter_print(struct seq_file s, struct rvt_qp_iter iter)
		sde ? sde->this_idx : 0,
		send_context,
		send_context ? send_context->sw_index : 0,
		ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->head,
		ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->tail,
		ib_cq_head(qp->ibqp.send_cq),
		ib_cq_tail(qp->ibqp.send_cq),
		qp->pid,
		qp->s_state,
		qp->s_ack_state,

drivers/infiniband/sw/rdmavt/cq.c

+123 −69

Original line number	Diff line number	Diff line
		@@ -63,19 +63,33 @@ static struct workqueue_struct *comp_vector_wq;
		*/
		void rvt_cq_enter(struct rvt_cq cq, struct ib_wc entry, bool solicited)
		{
		struct rvt_cq_wc *wc;
		struct ib_uverbs_wc *uqueue = NULL;
		struct ib_wc *kqueue = NULL;
		struct rvt_cq_wc *u_wc = NULL;
		struct rvt_k_cq_wc *k_wc = NULL;
		unsigned long flags;
		u32 head;
		u32 next;
		u32 tail;

		spin_lock_irqsave(&cq->lock, flags);

		if (cq->ip) {
		u_wc = cq->queue;
		uqueue = &u_wc->uqueue[0];
		head = RDMA_READ_UAPI_ATOMIC(u_wc->head);
		tail = RDMA_READ_UAPI_ATOMIC(u_wc->tail);
		} else {
		k_wc = cq->kqueue;
		kqueue = &k_wc->kqueue[0];
		head = k_wc->head;
		tail = k_wc->tail;
		}

		/*
		* Note that the head pointer might be writable by user processes.
		* Take care to verify it is a sane value.
		* Note that the head pointer might be writable by
		* user processes.Take care to verify it is a sane value.
		*/
		wc = cq->queue;
		head = wc->head;
		if (head >= (unsigned)cq->ibcq.cqe) {
		head = cq->ibcq.cqe;
		next = 0;
		@@ -83,7 +97,7 @@ void rvt_cq_enter(struct rvt_cq cq, struct ib_wc entry, bool solicited)
		next = head + 1;
		}

		if (unlikely(next == wc->tail)) {
		if (unlikely(next == tail)) {
		spin_unlock_irqrestore(&cq->lock, flags);
		if (cq->ibcq.event_handler) {
		struct ib_event ev;
		@@ -96,27 +110,27 @@ void rvt_cq_enter(struct rvt_cq cq, struct ib_wc entry, bool solicited)
		return;
		}
		trace_rvt_cq_enter(cq, entry, head);
		if (cq->ip) {
		wc->uqueue[head].wr_id = entry->wr_id;
		wc->uqueue[head].status = entry->status;
		wc->uqueue[head].opcode = entry->opcode;
		wc->uqueue[head].vendor_err = entry->vendor_err;
		wc->uqueue[head].byte_len = entry->byte_len;
		wc->uqueue[head].ex.imm_data = entry->ex.imm_data;
		wc->uqueue[head].qp_num = entry->qp->qp_num;
		wc->uqueue[head].src_qp = entry->src_qp;
		wc->uqueue[head].wc_flags = entry->wc_flags;
		wc->uqueue[head].pkey_index = entry->pkey_index;
		wc->uqueue[head].slid = ib_lid_cpu16(entry->slid);
		wc->uqueue[head].sl = entry->sl;
		wc->uqueue[head].dlid_path_bits = entry->dlid_path_bits;
		wc->uqueue[head].port_num = entry->port_num;
		if (uqueue) {
		uqueue[head].wr_id = entry->wr_id;
		uqueue[head].status = entry->status;
		uqueue[head].opcode = entry->opcode;
		uqueue[head].vendor_err = entry->vendor_err;
		uqueue[head].byte_len = entry->byte_len;
		uqueue[head].ex.imm_data = entry->ex.imm_data;
		uqueue[head].qp_num = entry->qp->qp_num;
		uqueue[head].src_qp = entry->src_qp;
		uqueue[head].wc_flags = entry->wc_flags;
		uqueue[head].pkey_index = entry->pkey_index;
		uqueue[head].slid = ib_lid_cpu16(entry->slid);
		uqueue[head].sl = entry->sl;
		uqueue[head].dlid_path_bits = entry->dlid_path_bits;
		uqueue[head].port_num = entry->port_num;
		/* Make sure entry is written before the head index. */
		smp_wmb();
		RDMA_WRITE_UAPI_ATOMIC(u_wc->head, next);
		} else {
		wc->kqueue[head] = *entry;
		kqueue[head] = *entry;
		k_wc->head = next;
		}
		wc->head = next;

		if (cq->notify == IB_CQ_NEXT_COMP \|\|
		(cq->notify == IB_CQ_SOLICITED &&
		@@ -179,8 +193,9 @@ int rvt_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
		{
		struct ib_device *ibdev = ibcq->device;
		struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
		struct rvt_cq *cq = container_of(ibcq, struct rvt_cq, ibcq);
		struct rvt_cq_wc *wc;
		struct rvt_cq *cq = ibcq_to_rvtcq(ibcq);
		struct rvt_cq_wc *u_wc = NULL;
		struct rvt_k_cq_wc *k_wc = NULL;
		u32 sz;
		unsigned int entries = attr->cqe;
		int comp_vector = attr->comp_vector;
		@@ -204,22 +219,28 @@ int rvt_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
		* We need to use vmalloc() in order to support mmap and large
		* numbers of entries.
		*/
		sz = sizeof(*wc);
		if (udata && udata->outlen >= sizeof(__u64))
		sz += sizeof(struct ib_uverbs_wc) * (entries + 1);
		else
		sz += sizeof(struct ib_wc) * (entries + 1);
		wc = udata ?
		vmalloc_user(sz) :
		vzalloc_node(sz, rdi->dparms.node);
		if (!wc)
		if (udata && udata->outlen >= sizeof(__u64)) {
		sz = sizeof(struct ib_uverbs_wc) * (entries + 1);
		sz += sizeof(*u_wc);
		u_wc = vmalloc_user(sz);
		if (!u_wc)
		return -ENOMEM;
		} else {
		sz = sizeof(struct ib_wc) * (entries + 1);
		sz += sizeof(*k_wc);
		k_wc = vzalloc_node(sz, rdi->dparms.node);
		if (!k_wc)
		return -ENOMEM;
		}

		/*
		* Return the address of the WC as the offset to mmap.
		* See rvt_mmap() for details.
		*/
		if (udata && udata->outlen >= sizeof(__u64)) {
		cq->ip = rvt_create_mmap_info(rdi, sz, udata, wc);
		int err;

		cq->ip = rvt_create_mmap_info(rdi, sz, udata, u_wc);
		if (!cq->ip) {
		err = -ENOMEM;
		goto bail_wc;
		@@ -264,7 +285,10 @@ int rvt_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
		cq->notify = RVT_CQ_NONE;
		spin_lock_init(&cq->lock);
		INIT_WORK(&cq->comptask, send_complete);
		cq->queue = wc;
		if (u_wc)
		cq->queue = u_wc;
		else
		cq->kqueue = k_wc;

		trace_rvt_create_cq(cq, attr);
		return 0;
		@@ -272,7 +296,8 @@ int rvt_create_cq(struct ib_cq ibcq, const struct ib_cq_init_attr attr,
		bail_ip:
		kfree(cq->ip);
		bail_wc:
		vfree(wc);
		vfree(u_wc);
		vfree(k_wc);
		return err;
		}

		@@ -322,9 +347,16 @@ int rvt_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags)
		if (cq->notify != IB_CQ_NEXT_COMP)
		cq->notify = notify_flags & IB_CQ_SOLICITED_MASK;

		if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
		cq->queue->head != cq->queue->tail)
		if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
		if (cq->queue) {
		if (RDMA_READ_UAPI_ATOMIC(cq->queue->head) !=
		RDMA_READ_UAPI_ATOMIC(cq->queue->tail))
		ret = 1;
		} else {
		if (cq->kqueue->head != cq->kqueue->tail)
		ret = 1;
		}
		}

		spin_unlock_irqrestore(&cq->lock, flags);

		@@ -340,12 +372,14 @@ int rvt_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags)
		int rvt_resize_cq(struct ib_cq ibcq, int cqe, struct ib_udata udata)
		{
		struct rvt_cq *cq = ibcq_to_rvtcq(ibcq);
		struct rvt_cq_wc *old_wc;
		struct rvt_cq_wc *wc;
		u32 head, tail, n;
		int ret;
		u32 sz;
		struct rvt_dev_info *rdi = cq->rdi;
		struct rvt_cq_wc *u_wc = NULL;
		struct rvt_cq_wc *old_u_wc = NULL;
		struct rvt_k_cq_wc *k_wc = NULL;
		struct rvt_k_cq_wc *old_k_wc = NULL;

		if (cqe < 1 \|\| cqe > rdi->dparms.props.max_cqe)
		return -EINVAL;
		@@ -353,17 +387,19 @@ int rvt_resize_cq(struct ib_cq ibcq, int cqe, struct ib_udata udata)
		/*
		* Need to use vmalloc() if we want to support large #s of entries.
		*/
		sz = sizeof(*wc);
		if (udata && udata->outlen >= sizeof(__u64))
		sz += sizeof(struct ib_uverbs_wc) * (cqe + 1);
		else
		sz += sizeof(struct ib_wc) * (cqe + 1);
		wc = udata ?
		vmalloc_user(sz) :
		vzalloc_node(sz, rdi->dparms.node);
		if (!wc)
		if (udata && udata->outlen >= sizeof(__u64)) {
		sz = sizeof(struct ib_uverbs_wc) * (cqe + 1);
		sz += sizeof(*u_wc);
		u_wc = vmalloc_user(sz);
		if (!u_wc)
		return -ENOMEM;

		} else {
		sz = sizeof(struct ib_wc) * (cqe + 1);
		sz += sizeof(*k_wc);
		k_wc = vzalloc_node(sz, rdi->dparms.node);
		if (!k_wc)
		return -ENOMEM;
		}
		/* Check that we can write the offset to mmap. */
		if (udata && udata->outlen >= sizeof(__u64)) {
		__u64 offset = 0;
		@@ -378,11 +414,18 @@ int rvt_resize_cq(struct ib_cq ibcq, int cqe, struct ib_udata udata)
		* Make sure head and tail are sane since they
		* might be user writable.
		*/
		old_wc = cq->queue;
		head = old_wc->head;
		if (u_wc) {
		old_u_wc = cq->queue;
		head = RDMA_READ_UAPI_ATOMIC(old_u_wc->head);
		tail = RDMA_READ_UAPI_ATOMIC(old_u_wc->tail);
		} else {
		old_k_wc = cq->kqueue;
		head = old_k_wc->head;
		tail = old_k_wc->tail;
		}

		if (head > (u32)cq->ibcq.cqe)
		head = (u32)cq->ibcq.cqe;
		tail = old_wc->tail;
		if (tail > (u32)cq->ibcq.cqe)
		tail = (u32)cq->ibcq.cqe;
		if (head < tail)
		@@ -394,27 +437,36 @@ int rvt_resize_cq(struct ib_cq ibcq, int cqe, struct ib_udata udata)
		goto bail_unlock;
		}
		for (n = 0; tail != head; n++) {
		if (cq->ip)
		wc->uqueue[n] = old_wc->uqueue[tail];
		if (u_wc)
		u_wc->uqueue[n] = old_u_wc->uqueue[tail];
		else
		wc->kqueue[n] = old_wc->kqueue[tail];
		k_wc->kqueue[n] = old_k_wc->kqueue[tail];
		if (tail == (u32)cq->ibcq.cqe)
		tail = 0;
		else
		tail++;
		}
		cq->ibcq.cqe = cqe;
		wc->head = n;
		wc->tail = 0;
		cq->queue = wc;
		if (u_wc) {
		RDMA_WRITE_UAPI_ATOMIC(u_wc->head, n);
		RDMA_WRITE_UAPI_ATOMIC(u_wc->tail, 0);
		cq->queue = u_wc;
		} else {
		k_wc->head = n;
		k_wc->tail = 0;
		cq->kqueue = k_wc;
		}
		spin_unlock_irq(&cq->lock);

		vfree(old_wc);
		if (u_wc)
		vfree(old_u_wc);
		else
		vfree(old_k_wc);

		if (cq->ip) {
		struct rvt_mmap_info *ip = cq->ip;

		rvt_update_mmap_info(rdi, ip, sz, wc);
		rvt_update_mmap_info(rdi, ip, sz, u_wc);

		/*
		* Return the offset to mmap.
		@@ -438,7 +490,9 @@ int rvt_resize_cq(struct ib_cq ibcq, int cqe, struct ib_udata udata)
		bail_unlock:
		spin_unlock_irq(&cq->lock);
		bail_free:
		vfree(wc);
		vfree(u_wc);
		vfree(k_wc);

		return ret;
		}

		@@ -456,7 +510,7 @@ bail_free:
		int rvt_poll_cq(struct ib_cq ibcq, int num_entries, struct ib_wc entry)
		{
		struct rvt_cq *cq = ibcq_to_rvtcq(ibcq);
		struct rvt_cq_wc *wc;
		struct rvt_k_cq_wc *wc;
		unsigned long flags;
		int npolled;
		u32 tail;
		@@ -467,7 +521,7 @@ int rvt_poll_cq(struct ib_cq ibcq, int num_entries, struct ib_wc entry)

		spin_lock_irqsave(&cq->lock, flags);

		wc = cq->queue;
		wc = cq->kqueue;
		tail = wc->tail;
		if (tail > (u32)cq->ibcq.cqe)
		tail = (u32)cq->ibcq.cqe;

include/rdma/rdmavt_cq.h

+16 −6

Original line number	Diff line number	Diff line
		@@ -60,19 +60,28 @@
		*/
		#define RVT_CQ_NONE (IB_CQ_NEXT_COMP + 1)

		/*
		* Define read macro that apply smp_load_acquire memory barrier
		* when reading indice of circular buffer that mmaped to user space.
		*/
		#define RDMA_READ_UAPI_ATOMIC(member) smp_load_acquire(&(member).val)

		/*
		* Define write macro that uses smp_store_release memory barrier
		* when writing indice of circular buffer that mmaped to user space.
		*/
		#define RDMA_WRITE_UAPI_ATOMIC(member, x) smp_store_release(&(member).val, x)
		#include <rdma/rvt-abi.h>

		/*
		* This structure is used to contain the head pointer, tail pointer,
		* and completion queue entries as a single memory allocation so
		* it can be mmap'ed into user space.
		*/
		struct rvt_cq_wc {
		struct rvt_k_cq_wc {
		u32 head; /* index of next entry to fill */
		u32 tail; /* index of next ib_poll_cq() entry */
		union {
		/* these are actually size ibcq.cqe + 1 */
		struct ib_uverbs_wc uqueue[0];
		struct ib_wc kqueue[0];
		};
		struct ib_wc kqueue[];
		};

		/*
		@@ -88,6 +97,7 @@ struct rvt_cq {
		struct rvt_dev_info *rdi;
		struct rvt_cq_wc *queue;
		struct rvt_mmap_info *ip;
		struct rvt_k_cq_wc *kqueue;
		};

		static inline struct rvt_cq ibcq_to_rvtcq(struct ib_cq ibcq)

include/rdma/rdmavt_qp.h

+32 −0

Original line number	Diff line number	Diff line
		@@ -820,6 +820,38 @@ struct rvt_qp_iter {
		int n;
		};

		/**
		* ib_cq_tail - Return tail index of cq buffer
		* @send_cq - The cq for send
		*
		* This is called in qp_iter_print to get tail
		* of cq buffer.
		*/
		static inline u32 ib_cq_tail(struct ib_cq *send_cq)
		{
		struct rvt_cq *cq = ibcq_to_rvtcq(send_cq);

		return ibcq_to_rvtcq(send_cq)->ip ?
		RDMA_READ_UAPI_ATOMIC(cq->queue->tail) :
		ibcq_to_rvtcq(send_cq)->kqueue->tail;
		}

		/**
		* ib_cq_head - Return head index of cq buffer
		* @send_cq - The cq for send
		*
		* This is called in qp_iter_print to get head
		* of cq buffer.
		*/
		static inline u32 ib_cq_head(struct ib_cq *send_cq)
		{
		struct rvt_cq *cq = ibcq_to_rvtcq(send_cq);

		return ibcq_to_rvtcq(send_cq)->ip ?
		RDMA_READ_UAPI_ATOMIC(cq->queue->head) :
		ibcq_to_rvtcq(send_cq)->kqueue->head;
		}

		struct rvt_qp_iter rvt_qp_iter_init(struct rvt_dev_info rdi,
		u64 v,
		void (cb)(struct rvt_qp qp, u64 v));

include/uapi/rdma/rvt-abi.h

0 → 100644

+32 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */

		/*
		* This file contains defines, structures, etc. that are used
		* to communicate between kernel and user code.
		*/

		#ifndef RVT_ABI_USER_H
		#define RVT_ABI_USER_H

		#include <linux/types.h>
		#include <rdma/ib_user_verbs.h>
		#ifndef RDMA_ATOMIC_UAPI
		#define RDMA_ATOMIC_UAPI(_type, _name) struct{ _type val; } _name
		#endif

		/*
		* This structure is used to contain the head pointer, tail pointer,
		* and completion queue entries as a single memory allocation so
		* it can be mmap'ed into user space.
		*/
		struct rvt_cq_wc {
		/* index of next entry to fill */
		RDMA_ATOMIC_UAPI(__u32, head);
		/* index of next ib_poll_cq() entry */
		RDMA_ATOMIC_UAPI(__u32, tail);

		/* these are actually size ibcq.cqe + 1 */
		struct ib_uverbs_wc uqueue[];
		};

		#endif /* RVT_ABI_USER_H */

Admin message