Commit e7e55829 authored by Steve Wise's avatar Steve Wise Committed by Roland Dreier
Browse files

RDMA/cxgb3: MEM_MGT_EXTENSIONS support 



- set IB_DEVICE_MEM_MGT_EXTENSIONS capability bit if fw supports it.
- set max_fast_reg_page_list_len device attribute.
- add iwch_alloc_fast_reg_mr function.
- add iwch_alloc_fastreg_pbl
- add iwch_free_fastreg_pbl
- adjust the WQ depth for kernel mode work queues to account for
  fastreg possibly taking 2 WR slots.
- add fastreg_mr work request support.
- add local_inv work request support.
- add send_with_inv and send_with_se_inv work request support.
- removed useless duplicate enums/defines for TPT/MW/MR stuff.

Signed-off-by: default avatarSteve Wise <swise@opengridcomputing.com>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent 00f7ec36

drivers/infiniband/hw/cxgb3/cxio_hal.c

+12 −3
Original line number Diff line number Diff line
@@ -145,7 +145,9 @@ static int cxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid) 
	}

	wqe = (struct t3_modify_qp_wr *) skb_put(skb, sizeof(*wqe));

	memset(wqe, 0, sizeof(*wqe));

	build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0, qpid, 7);

	build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD,

		       T3_COMPLETION_FLAG | T3_NOTIFY_FLAG, 0, qpid, 7,

		       T3_SOPEOP);

	wqe->flags = cpu_to_be32(MODQP_WRITE_EC);

	sge_cmd = qpid << 8 | 3;

	wqe->sge_cmd = cpu_to_be64(sge_cmd);
@@ -558,7 +560,7 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p) 
	wqe = (struct t3_modify_qp_wr *) skb_put(skb, sizeof(*wqe));

	memset(wqe, 0, sizeof(*wqe));

	build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 0, 0,

		       T3_CTL_QP_TID, 7);

		       T3_CTL_QP_TID, 7, T3_SOPEOP);

	wqe->flags = cpu_to_be32(MODQP_WRITE_EC);

	sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3;

	wqe->sge_cmd = cpu_to_be64(sge_cmd);
@@ -674,7 +676,7 @@ static int cxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr, 
		build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag,

			       Q_GENBIT(rdev_p->ctrl_qp.wptr,

					T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID,

			       wr_len);

			       wr_len, T3_SOPEOP);

		if (flag == T3_COMPLETION_FLAG)

			ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID);

		len -= 96;
@@ -816,6 +818,13 @@ int cxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag) 
			     0, 0);

}



int cxio_allocate_stag(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr)

{

	*stag = T3_STAG_UNSET;

	return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_NON_SHARED_MR,

			     0, 0, 0ULL, 0, 0, pbl_size, pbl_addr);

}



int cxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr)

{

	struct t3_rdma_init_wr *wqe;

drivers/infiniband/hw/cxgb3/cxio_hal.h

+1 −0
Original line number Diff line number Diff line
@@ -165,6 +165,7 @@ int cxio_reregister_phys_mem(struct cxio_rdev *rdev, u32 * stag, u32 pdid, 
int cxio_dereg_mem(struct cxio_rdev *rdev, u32 stag, u32 pbl_size,

		   u32 pbl_addr);

int cxio_allocate_window(struct cxio_rdev *rdev, u32 * stag, u32 pdid);

int cxio_allocate_stag(struct cxio_rdev *rdev, u32 *stag, u32 pdid, u32 pbl_size, u32 pbl_addr);

int cxio_deallocate_window(struct cxio_rdev *rdev, u32 stag);

int cxio_rdma_init(struct cxio_rdev *rdev, struct t3_rdma_init_attr *attr);

void cxio_register_ev_cb(cxio_hal_ev_callback_func_t ev_cb);

drivers/infiniband/hw/cxgb3/cxio_wr.h

+70 −20
Original line number Diff line number Diff line
@@ -72,7 +72,8 @@ enum t3_wr_opcode { 
	T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,

	T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,

	T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,

	T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP

	T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP,

	T3_WR_FASTREG = FW_WROPCODE_RI_FASTREGISTER_MR

} __attribute__ ((packed));



enum t3_rdma_opcode {
@@ -89,7 +90,8 @@ enum t3_rdma_opcode { 
	T3_FAST_REGISTER,

	T3_LOCAL_INV,

	T3_QP_MOD,

	T3_BYPASS

	T3_BYPASS,

	T3_RDMA_READ_REQ_WITH_INV,

} __attribute__ ((packed));



static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
@@ -103,6 +105,7 @@ static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop) 
		case T3_WR_BIND: return T3_BIND_MW;

		case T3_WR_INIT: return T3_RDMA_INIT;

		case T3_WR_QP_MOD: return T3_QP_MOD;

		case T3_WR_FASTREG: return T3_FAST_REGISTER;

		default: break;

	}

	return -1;
@@ -170,11 +173,54 @@ struct t3_send_wr { 
	struct t3_sge sgl[T3_MAX_SGE];	/* 4+ */

};



#define T3_MAX_FASTREG_DEPTH 24

#define T3_MAX_FASTREG_FRAG 10



struct t3_fastreg_wr {

	struct fw_riwrh wrh;	/* 0 */

	union t3_wrid wrid;	/* 1 */

	__be32 stag;		/* 2 */

	__be32 len;

	__be32 va_base_hi;	/* 3 */

	__be32 va_base_lo_fbo;

	__be32 page_type_perms; /* 4 */

	__be32 reserved1;

	__be64 pbl_addrs[0];	/* 5+ */

};



/*

 * If a fastreg wr spans multiple wqes, then the 2nd fragment look like this.

 */

struct t3_pbl_frag {

	struct fw_riwrh wrh;	/* 0 */

	__be64 pbl_addrs[14];	/* 1..14 */

};



#define S_FR_PAGE_COUNT		24

#define M_FR_PAGE_COUNT		0xff

#define V_FR_PAGE_COUNT(x)	((x) << S_FR_PAGE_COUNT)

#define G_FR_PAGE_COUNT(x)	((((x) >> S_FR_PAGE_COUNT)) & M_FR_PAGE_COUNT)



#define S_FR_PAGE_SIZE		16

#define M_FR_PAGE_SIZE		0x1f

#define V_FR_PAGE_SIZE(x)	((x) << S_FR_PAGE_SIZE)

#define G_FR_PAGE_SIZE(x)	((((x) >> S_FR_PAGE_SIZE)) & M_FR_PAGE_SIZE)



#define S_FR_TYPE		8

#define M_FR_TYPE		0x1

#define V_FR_TYPE(x)		((x) << S_FR_TYPE)

#define G_FR_TYPE(x)		((((x) >> S_FR_TYPE)) & M_FR_TYPE)



#define S_FR_PERMS		0

#define M_FR_PERMS		0xff

#define V_FR_PERMS(x)		((x) << S_FR_PERMS)

#define G_FR_PERMS(x)		((((x) >> S_FR_PERMS)) & M_FR_PERMS)



struct t3_local_inv_wr {

	struct fw_riwrh wrh;	/* 0 */

	union t3_wrid wrid;	/* 1 */

	__be32 stag;		/* 2 */

	__be32 reserved3;

	__be32 reserved;

};



struct t3_rdma_write_wr {
@@ -193,7 +239,8 @@ struct t3_rdma_read_wr { 
	struct fw_riwrh wrh;	/* 0 */

	union t3_wrid wrid;	/* 1 */

	u8 rdmaop;		/* 2 */

	u8 reserved[3];

	u8 local_inv;

	u8 reserved[2];

	__be32 rem_stag;

	__be64 rem_to;		/* 3 */

	__be32 local_stag;	/* 4 */
@@ -201,18 +248,6 @@ struct t3_rdma_read_wr { 
	__be64 local_to;	/* 5 */

};



enum t3_addr_type {

	T3_VA_BASED_TO = 0x0,

	T3_ZERO_BASED_TO = 0x1

} __attribute__ ((packed));



enum t3_mem_perms {

	T3_MEM_ACCESS_LOCAL_READ = 0x1,

	T3_MEM_ACCESS_LOCAL_WRITE = 0x2,

	T3_MEM_ACCESS_REM_READ = 0x4,

	T3_MEM_ACCESS_REM_WRITE = 0x8

} __attribute__ ((packed));



struct t3_bind_mw_wr {

	struct fw_riwrh wrh;	/* 0 */

	union t3_wrid wrid;	/* 1 */
@@ -336,6 +371,11 @@ struct t3_genbit { 
	__be64 genbit;

};



struct t3_wq_in_err {

	u64 flit[13];

	u64 err;

};



enum rdma_init_wr_flags {

	MPA_INITIATOR = (1<<0),

	PRIV_QP = (1<<1),
@@ -346,13 +386,16 @@ union t3_wr { 
	struct t3_rdma_write_wr write;

	struct t3_rdma_read_wr read;

	struct t3_receive_wr recv;

	struct t3_fastreg_wr fastreg;

	struct t3_pbl_frag pbl_frag;

	struct t3_local_inv_wr local_inv;

	struct t3_bind_mw_wr bind;

	struct t3_bypass_wr bypass;

	struct t3_rdma_init_wr init;

	struct t3_modify_qp_wr qp_mod;

	struct t3_genbit genbit;

	u64 flit[16];

	struct t3_wq_in_err wq_in_err;

	__be64 flit[16];

};



#define T3_SQ_CQE_FLIT	  13
@@ -366,12 +409,18 @@ static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe) 
	return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));

}



enum t3_wr_hdr_bits {

	T3_EOP = 1,

	T3_SOP = 2,

	T3_SOPEOP = T3_EOP|T3_SOP,

};



static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,

				  enum t3_wr_flags flags, u8 genbit, u32 tid,

				  u8 len)

				  u8 len, u8 sopeop)

{

	wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |

					 V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |

					 V_FW_RIWR_SOPEOP(sopeop) |

					 V_FW_RIWR_FLAGS(flags));

	wmb();

	wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
@@ -404,6 +453,7 @@ enum tpt_addr_type { 
};



enum tpt_mem_perm {

	TPT_MW_BIND = 0x10,

	TPT_LOCAL_READ = 0x8,

	TPT_LOCAL_WRITE = 0x4,

	TPT_REMOTE_READ = 0x2,
@@ -659,7 +709,7 @@ struct t3_cq { 


static inline void cxio_set_wq_in_error(struct t3_wq *wq)

{

	wq->queue->flit[13] = 1;

	wq->queue->wq_in_err.err = 1;

}



static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)

drivers/infiniband/hw/cxgb3/iwch_cq.c

+12 −3
Original line number Diff line number Diff line
@@ -81,6 +81,7 @@ static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp, 
	wc->wr_id = cookie;

	wc->qp = &qhp->ibqp;

	wc->vendor_err = CQE_STATUS(cqe);

	wc->wc_flags = 0;



	PDBG("%s qpid 0x%x type %d opcode %d status 0x%x wrid hi 0x%x "

	     "lo 0x%x cookie 0x%llx\n", __func__,
@@ -94,6 +95,11 @@ static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp, 
		else

			wc->byte_len = 0;

		wc->opcode = IB_WC_RECV;

		if (CQE_OPCODE(cqe) == T3_SEND_WITH_INV ||

		    CQE_OPCODE(cqe) == T3_SEND_WITH_SE_INV) {

			wc->ex.invalidate_rkey = CQE_WRID_STAG(cqe);

			wc->wc_flags |= IB_WC_WITH_INVALIDATE;

		}

	} else {

		switch (CQE_OPCODE(cqe)) {

		case T3_RDMA_WRITE:
@@ -105,17 +111,20 @@ static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp, 
			break;

		case T3_SEND:

		case T3_SEND_WITH_SE:

		case T3_SEND_WITH_INV:

		case T3_SEND_WITH_SE_INV:

			wc->opcode = IB_WC_SEND;

			break;

		case T3_BIND_MW:

			wc->opcode = IB_WC_BIND_MW;

			break;



		/* these aren't supported yet */

		case T3_SEND_WITH_INV:

		case T3_SEND_WITH_SE_INV:

		case T3_LOCAL_INV:

			wc->opcode = IB_WC_LOCAL_INV;

			break;

		case T3_FAST_REGISTER:

			wc->opcode = IB_WC_FAST_REG_MR;

			break;

		default:

			printk(KERN_ERR MOD "Unexpected opcode %d "

			       "in the CQE received for QPID=0x%0x\n",

drivers/infiniband/hw/cxgb3/iwch_provider.c

+101 −3
Original line number Diff line number Diff line
@@ -768,6 +768,68 @@ static int iwch_dealloc_mw(struct ib_mw *mw) 
	return 0;

}



static struct ib_mr *iwch_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth)

{

	struct iwch_dev *rhp;

	struct iwch_pd *php;

	struct iwch_mr *mhp;

	u32 mmid;

	u32 stag = 0;

	int ret;



	php = to_iwch_pd(pd);

	rhp = php->rhp;

	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);

	if (!mhp)

		return ERR_PTR(-ENOMEM);



	mhp->rhp = rhp;

	ret = iwch_alloc_pbl(mhp, pbl_depth);

	if (ret) {

		kfree(mhp);

		return ERR_PTR(ret);

	}

	mhp->attr.pbl_size = pbl_depth;

	ret = cxio_allocate_stag(&rhp->rdev, &stag, php->pdid,

				 mhp->attr.pbl_size, mhp->attr.pbl_addr);

	if (ret) {

		iwch_free_pbl(mhp);

		kfree(mhp);

		return ERR_PTR(ret);

	}

	mhp->attr.pdid = php->pdid;

	mhp->attr.type = TPT_NON_SHARED_MR;

	mhp->attr.stag = stag;

	mhp->attr.state = 1;

	mmid = (stag) >> 8;

	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;

	insert_handle(rhp, &rhp->mmidr, mhp, mmid);

	PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);

	return &(mhp->ibmr);

}



static struct ib_fast_reg_page_list *iwch_alloc_fastreg_pbl(

					struct ib_device *device,

					int page_list_len)

{

	struct ib_fast_reg_page_list *page_list;



	page_list = kmalloc(sizeof *page_list + page_list_len * sizeof(u64),

			    GFP_KERNEL);

	if (!page_list)

		return ERR_PTR(-ENOMEM);



	page_list->page_list = (u64 *)(page_list + 1);

	page_list->max_page_list_len = page_list_len;



	return page_list;

}



static void iwch_free_fastreg_pbl(struct ib_fast_reg_page_list *page_list)

{

	kfree(page_list);

}



static int iwch_destroy_qp(struct ib_qp *ib_qp)

{

	struct iwch_dev *rhp;
@@ -843,6 +905,15 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, 
	 */

	sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);

	wqsize = roundup_pow_of_two(rqsize + sqsize);



	/*

	 * Kernel users need more wq space for fastreg WRs which can take

	 * 2 WR fragments.

	 */

	ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;

	if (!ucontext && wqsize < (rqsize + (2 * sqsize)))

		wqsize = roundup_pow_of_two(rqsize +

				roundup_pow_of_two(attrs->cap.max_send_wr * 2));

	PDBG("%s wqsize %d sqsize %d rqsize %d\n", __func__,

	     wqsize, sqsize, rqsize);

	qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
@@ -851,7 +922,6 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, 
	qhp->wq.size_log2 = ilog2(wqsize);

	qhp->wq.rq_size_log2 = ilog2(rqsize);

	qhp->wq.sq_size_log2 = ilog2(sqsize);

	ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;

	if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,

			   ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {

		kfree(qhp);
@@ -1048,6 +1118,7 @@ static int iwch_query_device(struct ib_device *ibdev, 
	props->max_mr = dev->attr.max_mem_regs;

	props->max_pd = dev->attr.max_pds;

	props->local_ca_ack_delay = 0;

	props->max_fast_reg_page_list_len = T3_MAX_FASTREG_DEPTH;



	return 0;

}
@@ -1088,6 +1159,28 @@ static ssize_t show_rev(struct device *dev, struct device_attribute *attr, 
	return sprintf(buf, "%d\n", iwch_dev->rdev.t3cdev_p->type);

}



static int fw_supports_fastreg(struct iwch_dev *iwch_dev)

{

	struct ethtool_drvinfo info;

	struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;

	char *cp, *next;

	unsigned fw_maj, fw_min;



	rtnl_lock();

	lldev->ethtool_ops->get_drvinfo(lldev, &info);

	rtnl_unlock();



	next = info.fw_version+1;

	cp = strsep(&next, ".");

	sscanf(cp, "%i", &fw_maj);

	cp = strsep(&next, ".");

	sscanf(cp, "%i", &fw_min);



	PDBG("%s maj %u min %u\n", __func__, fw_maj, fw_min);



	return fw_maj > 6 || (fw_maj == 6 && fw_min > 0);

}



static ssize_t show_fw_ver(struct device *dev, struct device_attribute *attr, char *buf)

{

	struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
@@ -1149,8 +1242,10 @@ int iwch_register_device(struct iwch_dev *dev) 
	memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));

	memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);

	dev->ibdev.owner = THIS_MODULE;

	dev->device_cap_flags =

	    (IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW);

	dev->device_cap_flags = IB_DEVICE_ZERO_STAG |

				IB_DEVICE_MEM_WINDOW;

	if (fw_supports_fastreg(dev))

		dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;



	dev->ibdev.uverbs_cmd_mask =

	    (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
@@ -1202,6 +1297,9 @@ int iwch_register_device(struct iwch_dev *dev) 
	dev->ibdev.alloc_mw = iwch_alloc_mw;

	dev->ibdev.bind_mw = iwch_bind_mw;

	dev->ibdev.dealloc_mw = iwch_dealloc_mw;

	dev->ibdev.alloc_fast_reg_mr = iwch_alloc_fast_reg_mr;

	dev->ibdev.alloc_fast_reg_page_list = iwch_alloc_fastreg_pbl;

	dev->ibdev.free_fast_reg_page_list = iwch_free_fastreg_pbl;



	dev->ibdev.attach_mcast = iwch_multicast_attach;

	dev->ibdev.detach_mcast = iwch_multicast_detach;

CRA Git | Maintained and supported by SUSTech CRA and CCSE