Commit 051f2630 authored by Leon Romanovsky's avatar Leon Romanovsky Committed by Doug Ledford
Browse files

IB/mlx5: Add driver cross-channel support 



Add support of cross-channel functionality to mlx5
driver. This includes ability to ignore overrun for CQ
which intended for cross-channel, export device capability and
configure the QP to be sync master/slave queues.

The cross-channel enabled QP supports combination of
three possible properties:
* WQE processing on the receive queue of this QP
* WQE processing on the send queue of this QP
* WQE are supported on the send queue

Reviewed-by: default avatarSagi Grimberg <sagig@mellanox.com>
Signed-off-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 8a06ce59

drivers/infiniband/hw/mlx5/cq.c

+6 −1
Original line number Diff line number Diff line
@@ -778,7 +778,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, 
	int eqn;

	int err;



	if (attr->flags)

	if (check_cq_create_flags(attr->flags))

		return ERR_PTR(-EINVAL);



	if (entries < 0)
@@ -800,6 +800,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, 
	spin_lock_init(&cq->lock);

	cq->resize_buf = NULL;

	cq->resize_umem = NULL;

	cq->create_flags = attr->flags;



	if (context) {

		err = create_cq_user(dev, udata, context, cq, entries,
@@ -817,6 +818,10 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, 


	cq->cqe_size = cqe_size;

	cqb->ctx.cqe_sz_flags = cqe_sz_to_mlx_sz(cqe_size) << 5;



	if (cq->create_flags & IB_CQ_FLAGS_IGNORE_OVERRUN)

		cqb->ctx.cqe_sz_flags |= (1 << 1);



	cqb->ctx.log_sz_usr_page = cpu_to_be32((ilog2(entries) << 24) | index);

	err = mlx5_vector2eqn(dev->mdev, vector, &eqn, &irqn);

	if (err)

drivers/infiniband/hw/mlx5/main.c

+3 −0
Original line number Diff line number Diff line
@@ -512,6 +512,9 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, 
	props->odp_caps = dev->odp_caps;

#endif



	if (MLX5_CAP_GEN(mdev, cd))

		props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;



	return 0;

}

drivers/infiniband/hw/mlx5/mlx5_ib.h

+16 −0
Original line number Diff line number Diff line
@@ -90,6 +90,10 @@ enum mlx5_ib_mad_ifc_flags { 
	MLX5_MAD_IFC_NET_VIEW		= 4,

};



enum {

	MLX5_CROSS_CHANNEL_UUAR         = 0,

};



struct mlx5_ib_ucontext {

	struct ib_ucontext	ibucontext;

	struct list_head	db_page_list;
@@ -247,6 +251,9 @@ struct mlx5_ib_cq_buf { 
enum mlx5_ib_qp_flags {

	MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK     = 1 << 0,

	MLX5_IB_QP_SIGNATURE_HANDLING           = 1 << 1,

	MLX5_IB_QP_CROSS_CHANNEL		= 1 << 2,

	MLX5_IB_QP_MANAGED_SEND			= 1 << 3,

	MLX5_IB_QP_MANAGED_RECV			= 1 << 4,

};



struct mlx5_umr_wr {
@@ -289,6 +296,7 @@ struct mlx5_ib_cq { 
	struct mlx5_ib_cq_buf  *resize_buf;

	struct ib_umem	       *resize_umem;

	int			cqe_size;

	u32			create_flags;

};



struct mlx5_ib_srq {
@@ -678,4 +686,12 @@ static inline int is_qp1(enum ib_qp_type qp_type) 
#define MLX5_MAX_UMR_SHIFT 16

#define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)



static inline u32 check_cq_create_flags(u32 flags)

{

	/*

	 * It returns non-zero value for unsupported CQ

	 * create flags, otherwise it returns zero.

	 */

	return (flags & ~IB_CQ_FLAGS_IGNORE_OVERRUN);

}

#endif /* MLX5_IB_H */

drivers/infiniband/hw/mlx5/qp.c

+44 −10
Original line number Diff line number Diff line
@@ -616,6 +616,10 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, 
	/*

	 * TBD: should come from the verbs when we have the API

	 */

	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)

		/* In CROSS_CHANNEL CQ and QP must use the same UAR */

		uuarn = MLX5_CROSS_CHANNEL_UUAR;

	else {

		uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH);

		if (uuarn < 0) {

			mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n");
@@ -631,6 +635,7 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, 
				}

			}

		}

	}



	uar_index = uuarn_to_uar_index(&context->uuari, uuarn);

	mlx5_ib_dbg(dev, "uuarn 0x%x, uar_index 0x%x\n", uuarn, uar_index);
@@ -881,6 +886,21 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd, 
		}

	}



	if (init_attr->create_flags &

			(IB_QP_CREATE_CROSS_CHANNEL |

			 IB_QP_CREATE_MANAGED_SEND |

			 IB_QP_CREATE_MANAGED_RECV)) {

		if (!MLX5_CAP_GEN(mdev, cd)) {

			mlx5_ib_dbg(dev, "cross-channel isn't supported\n");

			return -EINVAL;

		}

		if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL)

			qp->flags |= MLX5_IB_QP_CROSS_CHANNEL;

		if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND)

			qp->flags |= MLX5_IB_QP_MANAGED_SEND;

		if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV)

			qp->flags |= MLX5_IB_QP_MANAGED_RECV;

	}

	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)

		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
@@ -955,6 +975,13 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd, 
	if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)

		in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_BLOCK_MCAST);



	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)

		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_MASTER);

	if (qp->flags & MLX5_IB_QP_MANAGED_SEND)

		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_SLAVE_SEND);

	if (qp->flags & MLX5_IB_QP_MANAGED_RECV)

		in->ctx.params2 |= cpu_to_be32(MLX5_QP_BIT_CC_SLAVE_RECV);



	if (qp->scat_cqe && is_connected(init_attr->qp_type)) {

		int rcqe_sz;

		int scqe_sz;
@@ -3130,6 +3157,13 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr 
	if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)

		qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;



	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)

		qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL;

	if (qp->flags & MLX5_IB_QP_MANAGED_SEND)

		qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND;

	if (qp->flags & MLX5_IB_QP_MANAGED_RECV)

		qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV;



	qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?

		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;

include/linux/mlx5/qp.h

+3 −0
Original line number Diff line number Diff line
@@ -130,6 +130,9 @@ enum { 
	MLX5_QP_BIT_RWE				= 1 << 14,

	MLX5_QP_BIT_RAE				= 1 << 13,

	MLX5_QP_BIT_RIC				= 1 <<	4,

	MLX5_QP_BIT_CC_SLAVE_RECV		= 1 <<  2,

	MLX5_QP_BIT_CC_SLAVE_SEND		= 1 <<  1,

	MLX5_QP_BIT_CC_MASTER			= 1 <<  0

};



enum {

CRA Git | Maintained and supported by SUSTech CRA and CCSE