Commit a4e6a1dd authored by Gal Pressman's avatar Gal Pressman Committed by Jason Gunthorpe
Browse files

RDMA/efa: Introduce SRD RNR retry 

This patch introduces the ability to configure SRD QPs with the RNR retry
parameter when issuing a modify QP command.

In addition, a capability bit was added to report support to the userspace
library.

Link: https://lore.kernel.org/r/20200731060420.17053-5-galpress@amazon.com


Reviewed-by: default avatarFiras JahJah <firasj@amazon.com>
Reviewed-by: default avatarYossi Leybovich <sleybo@amazon.com>
Signed-off-by: default avatarGal Pressman <galpress@amazon.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 22c50e06

drivers/infiniband/hw/efa/efa_admin_cmds_defs.h

+11 −6
Original line number Diff line number Diff line
@@ -197,7 +197,8 @@ struct efa_admin_modify_qp_cmd { 
	 * 2 : qkey

	 * 3 : sq_psn

	 * 4 : sq_drained_async_notify

	 * 31:5 : reserved

	 * 5 : rnr_retry

	 * 31:6 : reserved

	 */

	u32 modify_mask;
@@ -219,8 +220,8 @@ struct efa_admin_modify_qp_cmd { 
	/* Enable async notification when SQ is drained */

	u8 sq_drained_async_notify;



	/* MBZ */

	u8 reserved1;

	/* Number of RNR retries (valid only for SRD QPs) */

	u8 rnr_retry;



	/* MBZ */

	u16 reserved2;
@@ -255,8 +256,8 @@ struct efa_admin_query_qp_resp { 
	/* Indicates that draining is in progress */

	u8 sq_draining;



	/* MBZ */

	u8 reserved1;

	/* Number of RNR retries (valid only for SRD QPs) */

	u8 rnr_retry;



	/* MBZ */

	u16 reserved2;
@@ -573,7 +574,9 @@ struct efa_admin_feature_device_attr_desc { 
	/*

	 * 0 : rdma_read - If set, RDMA Read is supported on

	 *    TX queues

	 * 31:1 : reserved - MBZ

	 * 1 : rnr_retry - If set, RNR retry is supported on

	 *    modify QP command

	 * 31:2 : reserved - MBZ

	 */

	u32 device_caps;
@@ -865,6 +868,7 @@ struct efa_admin_host_info { 
#define EFA_ADMIN_MODIFY_QP_CMD_QKEY_MASK                   BIT(2)

#define EFA_ADMIN_MODIFY_QP_CMD_SQ_PSN_MASK                 BIT(3)

#define EFA_ADMIN_MODIFY_QP_CMD_SQ_DRAINED_ASYNC_NOTIFY_MASK BIT(4)

#define EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY_MASK              BIT(5)



/* reg_mr_cmd */

#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK      GENMASK(4, 0)
@@ -882,6 +886,7 @@ struct efa_admin_host_info { 


/* feature_device_attr_desc */

#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_READ_MASK   BIT(0)

#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RNR_RETRY_MASK   BIT(1)



/* host_info */

#define EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE_MASK         GENMASK(7, 0)

drivers/infiniband/hw/efa/efa_com_cmd.c

+2 −0
Original line number Diff line number Diff line
@@ -76,6 +76,7 @@ int efa_com_modify_qp(struct efa_com_dev *edev, 
	cmd.qkey = params->qkey;

	cmd.sq_psn = params->sq_psn;

	cmd.sq_drained_async_notify = params->sq_drained_async_notify;

	cmd.rnr_retry = params->rnr_retry;



	err = efa_com_cmd_exec(aq,

			       (struct efa_admin_aq_entry *)&cmd,
@@ -121,6 +122,7 @@ int efa_com_query_qp(struct efa_com_dev *edev, 
	result->qkey = resp.qkey;

	result->sq_draining = resp.sq_draining;

	result->sq_psn = resp.sq_psn;

	result->rnr_retry = resp.rnr_retry;



	return 0;

}

drivers/infiniband/hw/efa/efa_com_cmd.h

+2 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ struct efa_com_modify_qp_params { 
	u32 qkey;

	u32 sq_psn;

	u8 sq_drained_async_notify;

	u8 rnr_retry;

};



struct efa_com_query_qp_params {
@@ -58,6 +59,7 @@ struct efa_com_query_qp_result { 
	u32 qkey;

	u32 sq_draining;

	u32 sq_psn;

	u8 rnr_retry;

};



struct efa_com_destroy_qp_params {

drivers/infiniband/hw/efa/efa_verbs.c

+16 −3
Original line number Diff line number Diff line
@@ -223,6 +223,9 @@ int efa_query_device(struct ib_device *ibdev, 
		if (EFA_DEV_CAP(dev, RDMA_READ))

			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_READ;



		if (EFA_DEV_CAP(dev, RNR_RETRY))

			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RNR_RETRY;



		err = ib_copy_to_udata(udata, &resp,

				       min(sizeof(resp), udata->outlen));

		if (err) {
@@ -268,7 +271,7 @@ int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 


#define EFA_QUERY_QP_SUPP_MASK \

	(IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | \

	 IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP)

	 IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP | IB_QP_RNR_RETRY)



	if (qp_attr_mask & ~EFA_QUERY_QP_SUPP_MASK) {

		ibdev_dbg(&dev->ibdev,
@@ -290,6 +293,7 @@ int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 
	qp_attr->sq_psn = result.sq_psn;

	qp_attr->sq_draining = result.sq_draining;

	qp_attr->port_num = 1;

	qp_attr->rnr_retry = result.rnr_retry;



	qp_attr->cap.max_send_wr = qp->max_send_wr;

	qp_attr->cap.max_recv_wr = qp->max_recv_wr;
@@ -776,7 +780,9 @@ static const struct { 
			.valid = 1,

			.req_param = IB_QP_SQ_PSN,

			.opt_param = IB_QP_CUR_STATE |

				     IB_QP_QKEY,

				     IB_QP_QKEY |

				     IB_QP_RNR_RETRY,



		}

	},

	[IB_QPS_RTS] = {
@@ -856,7 +862,8 @@ static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp, 


#define EFA_MODIFY_QP_SUPP_MASK \

	(IB_QP_STATE | IB_QP_CUR_STATE | IB_QP_EN_SQD_ASYNC_NOTIFY | \

	 IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN)

	 IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN | \

	 IB_QP_RNR_RETRY)



	if (qp_attr_mask & ~EFA_MODIFY_QP_SUPP_MASK) {

		ibdev_dbg(&dev->ibdev,
@@ -943,6 +950,12 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 
		params.sq_psn = qp_attr->sq_psn;

	}



	if (qp_attr_mask & IB_QP_RNR_RETRY) {

		EFA_SET(&params.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY,

			1);

		params.rnr_retry = qp_attr->rnr_retry;

	}



	err = efa_com_modify_qp(&dev->edev, &params);

	if (err)

		return err;

include/uapi/rdma/efa-abi.h

+1 −0
Original line number Diff line number Diff line
@@ -105,6 +105,7 @@ struct efa_ibv_create_ah_resp { 


enum {

	EFA_QUERY_DEVICE_CAPS_RDMA_READ = 1 << 0,

	EFA_QUERY_DEVICE_CAPS_RNR_RETRY = 1 << 1,

};



struct efa_ibv_ex_query_device_resp {

CRA Git | Maintained and supported by SUSTech CRA and CCSE