IB/hfi1: Move receive work queue struct into uapi directory

	}

}

/**

 * rvt_alloc_rq - allocate memory for user or kernel buffer

 * @rq: receive queue data structure

 * @size: number of request queue entries

 * @node: The NUMA node

 * @udata: True if user data is available or not false

 *

 * Return: If memory allocation failed, return -ENONEM

 * This function is used by both shared receive

 * queues and non-shared receive queues to allocate

 * memory.

 */

int rvt_alloc_rq(struct rvt_rq *rq, u32 size, int node,

		 struct ib_udata *udata)

{

	if (udata) {

		rq->wq = vmalloc_user(sizeof(struct rvt_rwq) + size);

		if (!rq->wq)

			goto bail;

		/* need kwq with no buffers */

		rq->kwq = kzalloc_node(sizeof(*rq->kwq), GFP_KERNEL, node);

		if (!rq->kwq)

			goto bail;

		rq->kwq->curr_wq = rq->wq->wq;

	} else {

		/* need kwq with buffers */

		rq->kwq =

			vzalloc_node(sizeof(struct rvt_krwq) + size, node);

		if (!rq->kwq)

			goto bail;

		rq->kwq->curr_wq = rq->kwq->wq;

	}

	spin_lock_init(&rq->lock);

	return 0;

bail:

	rvt_free_rq(rq);

	return -ENOMEM;

}

/**

 * rvt_init_qp - initialize the QP state to the reset state

 * @qp: the QP to init or reinit

	qp->s_tail_ack_queue = 0;

	qp->s_acked_ack_queue = 0;

	qp->s_num_rd_atomic = 0;

	if (qp->r_rq.wq) {

		qp->r_rq.wq->head = 0;

		qp->r_rq.wq->tail = 0;

	}

	qp->r_sge.num_sge = 0;

	atomic_set(&qp->s_reserved_used, 0);

}

			qp->r_rq.max_sge = init_attr->cap.max_recv_sge;

			sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) +

				sizeof(struct rvt_rwqe);

			if (udata)

				qp->r_rq.wq = vmalloc_user(

						sizeof(struct rvt_rwq) +

						qp->r_rq.size * sz);

			else

				qp->r_rq.wq = vzalloc_node(

						sizeof(struct rvt_rwq) +

						qp->r_rq.size * sz,

						rdi->dparms.node);

			if (!qp->r_rq.wq)

			err = rvt_alloc_rq(&qp->r_rq, qp->r_rq.size * sz,

					   rdi->dparms.node, udata);

			if (err) {

				ret = ERR_PTR(err);

				goto bail_driver_priv;

			}

		}

		/*

		 * ib_create_qp() will initialize qp->ibqp

	rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num);

bail_rq_wq:

	if (!qp->ip)

		vfree(qp->r_rq.wq);

	rvt_free_rq(&qp->r_rq);

bail_driver_priv:

	rdi->driver_f.qp_priv_free(rdi, qp);

	}

	wc.status = IB_WC_WR_FLUSH_ERR;

	if (qp->r_rq.wq) {

		struct rvt_rwq *wq;

	if (qp->r_rq.kwq) {

		u32 head;

		u32 tail;

		struct rvt_rwq *wq = NULL;

		struct rvt_krwq *kwq = NULL;

		spin_lock(&qp->r_rq.lock);

		/* sanity check pointers before trusting them */

		/* qp->ip used to validate if there is a  user buffer mmaped */

		if (qp->ip) {

			wq = qp->r_rq.wq;

		head = wq->head;

			head = RDMA_READ_UAPI_ATOMIC(wq->head);

			tail = RDMA_READ_UAPI_ATOMIC(wq->tail);

		} else {

			kwq = qp->r_rq.kwq;

			head = kwq->head;

			tail = kwq->tail;

		}

		/* sanity check pointers before trusting them */

		if (head >= qp->r_rq.size)

			head = 0;

		tail = wq->tail;

		if (tail >= qp->r_rq.size)

			tail = 0;

		while (tail != head) {

				tail = 0;

			rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1);

		}

		wq->tail = tail;

		if (qp->ip)

			RDMA_WRITE_UAPI_ATOMIC(wq->tail, tail);

		else

			kwq->tail = tail;

		spin_unlock(&qp->r_rq.lock);

	} else if (qp->ibqp.event_handler) {

		ret = 1;

	if (qp->ip)

		kref_put(&qp->ip->ref, rvt_release_mmap_info);

	else

		vfree(qp->r_rq.wq);

	kvfree(qp->r_rq.kwq);

	rdi->driver_f.qp_priv_free(rdi, qp);

	kfree(qp->s_ack_queue);

	rdma_destroy_ah_attr(&qp->remote_ah_attr);

		  const struct ib_recv_wr **bad_wr)

{

	struct rvt_qp *qp = ibqp_to_rvtqp(ibqp);

	struct rvt_rwq *wq = qp->r_rq.wq;

	struct rvt_krwq *wq = qp->r_rq.kwq;

	unsigned long flags;

	int qp_err_flush = (ib_rvt_state_ops[qp->state] & RVT_FLUSH_RECV) &&

				!qp->ibqp.srq;

		next = wq->head + 1;

		if (next >= qp->r_rq.size)

			next = 0;

		if (next == wq->tail) {

		if (next == READ_ONCE(wq->tail)) {

			spin_unlock_irqrestore(&qp->r_rq.lock, flags);

			*bad_wr = wr;

			return -ENOMEM;

			 * Make sure queue entry is written

			 * before the head index.

			 */

			smp_wmb();

			wq->head = next;

			smp_store_release(&wq->head, next);

		}

		spin_unlock_irqrestore(&qp->r_rq.lock, flags);

	}

		      const struct ib_recv_wr **bad_wr)

{

	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);

	struct rvt_rwq *wq;

	struct rvt_krwq *wq;

	unsigned long flags;

	for (; wr; wr = wr->next) {

		}

		spin_lock_irqsave(&srq->rq.lock, flags);

		wq = srq->rq.wq;

		wq = srq->rq.kwq;

		next = wq->head + 1;

		if (next >= srq->rq.size)

			next = 0;

		if (next == wq->tail) {

		if (next == READ_ONCE(wq->tail)) {

			spin_unlock_irqrestore(&srq->rq.lock, flags);

			*bad_wr = wr;

			return -ENOMEM;

		for (i = 0; i < wr->num_sge; i++)

			wqe->sg_list[i] = wr->sg_list[i];

		/* Make sure queue entry is written before the head index. */

		smp_wmb();

		wq->head = next;

		smp_store_release(&wq->head, next);

		spin_unlock_irqrestore(&srq->rq.lock, flags);

	}

	return 0;

	return 0;

}

/**

 * get_rvt_head - get head indices of the circular buffer

 * @rq: data structure for request queue entry

 * @ip: the QP

 *

 * Return - head index value

 */

static inline u32 get_rvt_head(struct rvt_rq *rq, void *ip)

{

	u32 head;

	if (ip)

		head = RDMA_READ_UAPI_ATOMIC(rq->wq->head);

	else

		head = rq->kwq->head;

	return head;

}

/**

 * rvt_get_rwqe - copy the next RWQE into the QP's RWQE

 * @qp: the QP

{

	unsigned long flags;

	struct rvt_rq *rq;

	struct rvt_krwq *kwq;

	struct rvt_rwq *wq;

	struct rvt_srq *srq;

	struct rvt_rwqe *wqe;

	void (*handler)(struct ib_event *, void *);

	u32 tail;

	u32 head;

	int ret;

	void *ip = NULL;

	if (qp->ibqp.srq) {

		srq = ibsrq_to_rvtsrq(qp->ibqp.srq);

		handler = srq->ibsrq.event_handler;

		rq = &srq->rq;

		ip = srq->ip;

	} else {

		srq = NULL;

		handler = NULL;

		rq = &qp->r_rq;

		ip = qp->ip;

	}

	spin_lock_irqsave(&rq->lock, flags);

		ret = 0;

		goto unlock;

	}

	if (ip) {

		wq = rq->wq;

	tail = wq->tail;

		tail = RDMA_READ_UAPI_ATOMIC(wq->tail);

	} else {

		kwq = rq->kwq;

		tail = kwq->tail;

	}

	/* Validate tail before using it since it is user writable. */

	if (tail >= rq->size)

		tail = 0;

	if (unlikely(tail == wq->head)) {

	head = get_rvt_head(rq, ip);

	if (unlikely(tail == head)) {

		ret = 0;

		goto unlock;

	}

	/* Make sure entry is read after head index is read. */

	/* Make sure entry is read after the count is read. */

	smp_rmb();

	wqe = rvt_get_rwqe_ptr(rq, tail);

	/*

	 */

	if (++tail >= rq->size)

		tail = 0;

	wq->tail = tail;

	if (ip)

		RDMA_WRITE_UAPI_ATOMIC(wq->tail, tail);

	else

		kwq->tail = tail;

	if (!wr_id_only && !init_sge(qp, wqe)) {

		ret = -1;

		goto unlock;

		 * Validate head pointer value and compute

		 * the number of remaining WQEs.

		 */

		n = wq->head;

		n = get_rvt_head(rq, ip);

		if (n >= rq->size)

			n = 0;

		if (n < tail)

		      const struct ib_recv_wr **bad_wr);

int rvt_wss_init(struct rvt_dev_info *rdi);

void rvt_wss_exit(struct rvt_dev_info *rdi);

int rvt_alloc_rq(struct rvt_rq *rq, u32 size, int node,

		 struct ib_udata *udata);

#endif          /* DEF_RVTQP_H */

	} else {

		u32 min, max, x;

		u32 credits;

		struct rvt_rwq *wq = qp->r_rq.wq;

		u32 head;

		u32 tail;

		/* sanity check pointers before trusting them */

		head = wq->head;

		if (qp->ip) {

			head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);

			tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);

		} else {

			head = READ_ONCE(qp->r_rq.kwq->head);

			tail = READ_ONCE(qp->r_rq.kwq->tail);

		}

		if (head >= qp->r_rq.size)

			head = 0;

		tail = wq->tail;

		if (tail >= qp->r_rq.size)

			tail = 0;

		/*

#include "srq.h"

#include "vt.h"

#include "qp.h"

/**

 * rvt_driver_srq_init - init srq resources on a per driver basis

 * @rdi: rvt dev structure

	srq->rq.max_sge = srq_init_attr->attr.max_sge;

	sz = sizeof(struct ib_sge) * srq->rq.max_sge +

		sizeof(struct rvt_rwqe);

	srq->rq.wq = udata ?

		vmalloc_user(sizeof(struct rvt_rwq) + srq->rq.size * sz) :

		vzalloc_node(sizeof(struct rvt_rwq) + srq->rq.size * sz,

			     dev->dparms.node);

	if (!srq->rq.wq) {

	if (rvt_alloc_rq(&srq->rq, srq->rq.size * sz,

			 dev->dparms.node, udata)) {

		ret = -ENOMEM;

		goto bail_srq;

	}

bail_ip:

	kfree(srq->ip);

bail_wq:

	vfree(srq->rq.wq);

	rvt_free_rq(&srq->rq);

bail_srq:

	return ret;

}

{

	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);

	struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);

	struct rvt_rwq *wq;

	struct rvt_rq tmp_rq = {};

	int ret = 0;

	if (attr_mask & IB_SRQ_MAX_WR) {

		struct rvt_rwq *owq;

		struct rvt_krwq *okwq = NULL;

		struct rvt_rwq *owq = NULL;

		struct rvt_rwqe *p;

		u32 sz, size, n, head, tail;

		    ((attr_mask & IB_SRQ_LIMIT) ?

		     attr->srq_limit : srq->limit) > attr->max_wr)

			return -EINVAL;

		sz = sizeof(struct rvt_rwqe) +

			srq->rq.max_sge * sizeof(struct ib_sge);

		size = attr->max_wr + 1;

		wq = udata ?

			vmalloc_user(sizeof(struct rvt_rwq) + size * sz) :

			vzalloc_node(sizeof(struct rvt_rwq) + size * sz,

				     dev->dparms.node);

		if (!wq)

		if (rvt_alloc_rq(&tmp_rq, size * sz, dev->dparms.node,

				 udata))

			return -ENOMEM;

		/* Check that we can write the offset to mmap. */

		if (udata && udata->inlen >= sizeof(__u64)) {

			__u64 offset_addr;

		 * validate head and tail pointer values and compute

		 * the number of remaining WQEs.

		 */

		if (udata) {

			owq = srq->rq.wq;

		head = owq->head;

		tail = owq->tail;

			head = RDMA_READ_UAPI_ATOMIC(owq->head);

			tail = RDMA_READ_UAPI_ATOMIC(owq->tail);

		} else {

			okwq = srq->rq.kwq;

			head = okwq->head;

			tail = okwq->tail;

		}

		if (head >= srq->rq.size || tail >= srq->rq.size) {

			ret = -EINVAL;

			goto bail_unlock;

			goto bail_unlock;

		}

		n = 0;

		p = wq->wq;

		p = tmp_rq.kwq->curr_wq;

		while (tail != head) {

			struct rvt_rwqe *wqe;

			int i;

			if (++tail >= srq->rq.size)

				tail = 0;

		}

		srq->rq.wq = wq;

		srq->rq.kwq = tmp_rq.kwq;

		if (udata) {

			srq->rq.wq = tmp_rq.wq;

			RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->head, n);

			RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->tail, 0);

		} else {

			tmp_rq.kwq->head = n;

			tmp_rq.kwq->tail = 0;

		}

		srq->rq.size = size;

		wq->head = n;

		wq->tail = 0;

		if (attr_mask & IB_SRQ_LIMIT)

			srq->limit = attr->srq_limit;

		spin_unlock_irq(&srq->rq.lock);

		vfree(owq);

		kvfree(okwq);

		if (srq->ip) {

			struct rvt_mmap_info *ip = srq->ip;

			struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device);

			u32 s = sizeof(struct rvt_rwq) + size * sz;

			rvt_update_mmap_info(dev, ip, s, wq);

			rvt_update_mmap_info(dev, ip, s, tmp_rq.wq);

			/*

			 * Return the offset to mmap.

bail_unlock:

	spin_unlock_irq(&srq->rq.lock);

bail_free:

	vfree(wq);

	rvt_free_rq(&tmp_rq);

	return ret;

}

	spin_unlock(&dev->n_srqs_lock);

	if (srq->ip)

		kref_put(&srq->ip->ref, rvt_release_mmap_info);

	else

		vfree(srq->rq.wq);

	kvfree(srq->rq.kwq);

}

#include <rdma/ib_pack.h>

#include <rdma/ib_verbs.h>

#include <rdma/rdmavt_cq.h>

#include <rdma/rvt-abi.h>

/*

 * Atomic bit definitions for r_aflags.

 */

	struct rvt_sge sg_list[0];

};

/*

 * Receive work request queue entry.

 * The size of the sg_list is determined when the QP (or SRQ) is created

 * and stored in qp->r_rq.max_sge (or srq->rq.max_sge).

/**

 * struct rvt_krwq - kernel struct receive work request

 * @head: index of next entry to fill

 * @tail: index of next entry to pull

 * @count: count is aproximate of total receive enteries posted

 * @rvt_rwqe: struct of receive work request queue entry

 *

 * This structure is used to contain the head pointer,

 * tail pointer and receive work queue entries for kernel

 * mode user.

 */

struct rvt_rwqe {

	u64 wr_id;

	u8 num_sge;

	struct ib_sge sg_list[0];

};

/*

 * This structure is used to contain the head pointer, tail pointer,

 * and receive work queue entries as a single memory allocation so

 * it can be mmap'ed into user space.

 * Note that the wq array elements are variable size so you can't

 * just index into the array to get the N'th element;

 * use get_rwqe_ptr() instead.

 */

struct rvt_rwq {

struct rvt_krwq {

	u32 head;               /* new work requests posted to the head */

	u32 tail;               /* receives pull requests from here. */

	struct rvt_rwqe wq[0];

	struct rvt_rwqe *curr_wq;

	struct rvt_rwqe wq[];

};

struct rvt_rq {

	struct rvt_rwq *wq;

	struct rvt_krwq *kwq;

	u32 size;               /* size of RWQE array */

	u8 max_sge;

	/* protect changes in this struct */

static inline struct rvt_rwqe *rvt_get_rwqe_ptr(struct rvt_rq *rq, unsigned n)

{

	return (struct rvt_rwqe *)

		((char *)rq->wq->wq +

		((char *)rq->kwq->curr_wq +

		 (sizeof(struct rvt_rwqe) +

		  rq->max_sge * sizeof(struct ib_sge)) * n);

}

	       ibcq_to_rvtcq(send_cq)->kqueue->head;

}

/**

 * rvt_free_rq - free memory allocated for rvt_rq struct

 * @rvt_rq: request queue data structure

 *

 * This function should only be called if the rvt_mmap_info()

 * has not succeeded.

 */

static inline void rvt_free_rq(struct rvt_rq *rq)

{

	kvfree(rq->kwq);

	rq->kwq = NULL;

	vfree(rq->wq);

	rq->wq = NULL;

}

struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi,

				     u64 v,

				     void (*cb)(struct rvt_qp *qp, u64 v));