IB/mlx5: Use blue flame register allocator in mlx5_ib (5fe9dec0) · Commits · 戴 / test

drivers/infiniband/hw/mlx5/cq.c

+3 −5

Original line number	Original line	Diff line number	Diff line
	@@ -689,7 +689,7 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
	{		{
	struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;		struct mlx5_core_dev *mdev = to_mdev(ibcq->device)->mdev;
	struct mlx5_ib_cq *cq = to_mcq(ibcq);		struct mlx5_ib_cq *cq = to_mcq(ibcq);
	void __iomem *uar_page = mdev->priv.bfregi.uars[0].map;		void __iomem *uar_page = mdev->priv.uar->map;
	unsigned long irq_flags;		unsigned long irq_flags;
	int ret = 0;		int ret = 0;

	@@ -704,9 +704,7 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
	mlx5_cq_arm(&cq->mcq,		mlx5_cq_arm(&cq->mcq,
	(flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?		(flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
	MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,		MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT,
	uar_page,		uar_page, to_mcq(ibcq)->mcq.cons_index);
	MLX5_GET_DOORBELL_LOCK(&mdev->priv.cq_uar_lock),
	to_mcq(ibcq)->mcq.cons_index);

	return ret;		return ret;
	}		}
	@@ -886,7 +884,7 @@ static int create_cq_kernel(struct mlx5_ib_dev dev, struct mlx5_ib_cq cq,
	MLX5_SET(cqc, cqc, log_page_size,		MLX5_SET(cqc, cqc, log_page_size,
	cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);		cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);

	*index = dev->mdev->priv.bfregi.uars[0].index;		*index = dev->mdev->priv.uar->index;

	return 0;		return 0;

drivers/infiniband/hw/mlx5/main.c

+25 −3

Original line number	Original line	Diff line number	Diff line
	@@ -3074,8 +3074,6 @@ static void mlx5_ib_add(struct mlx5_core_dev mdev)
	if (mlx5_use_mad_ifc(dev))		if (mlx5_use_mad_ifc(dev))
	get_ext_port_caps(dev);		get_ext_port_caps(dev);

	MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);

	if (!mlx5_lag_is_active(mdev))		if (!mlx5_lag_is_active(mdev))
	name = "mlx5_%d";		name = "mlx5_%d";
	else		else
	@@ -3251,9 +3249,21 @@ static void mlx5_ib_add(struct mlx5_core_dev mdev)
	if (err)		if (err)
	goto err_odp;		goto err_odp;

			dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
			if (!dev->mdev->priv.uar)
			goto err_q_cnt;

			err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
			if (err)
			goto err_uar_page;

			err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
			if (err)
			goto err_bfreg;

	err = ib_register_device(&dev->ib_dev, NULL);		err = ib_register_device(&dev->ib_dev, NULL);
	if (err)		if (err)
	goto err_q_cnt;		goto err_fp_bfreg;

	err = create_umr_res(dev);		err = create_umr_res(dev);
	if (err)		if (err)
	@@ -3276,6 +3286,15 @@ err_umrc:
	err_dev:		err_dev:
	ib_unregister_device(&dev->ib_dev);		ib_unregister_device(&dev->ib_dev);

			err_fp_bfreg:
			mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);

			err_bfreg:
			mlx5_free_bfreg(dev->mdev, &dev->bfreg);

			err_uar_page:
			mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);

	err_q_cnt:		err_q_cnt:
	mlx5_ib_dealloc_q_counters(dev);		mlx5_ib_dealloc_q_counters(dev);

	@@ -3307,6 +3326,9 @@ static void mlx5_ib_remove(struct mlx5_core_dev mdev, void context)

	mlx5_remove_netdev_notifier(dev);		mlx5_remove_netdev_notifier(dev);
	ib_unregister_device(&dev->ib_dev);		ib_unregister_device(&dev->ib_dev);
			mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
			mlx5_free_bfreg(dev->mdev, &dev->bfreg);
			mlx5_put_uars_page(dev->mdev, mdev->priv.uar);
	mlx5_ib_dealloc_q_counters(dev);		mlx5_ib_dealloc_q_counters(dev);
	destroy_umrc_res(dev);		destroy_umrc_res(dev);
	mlx5_ib_odp_remove_one(dev);		mlx5_ib_odp_remove_one(dev);

drivers/infiniband/hw/mlx5/mlx5_ib.h

+9 −2

Original line number	Original line	Diff line number	Diff line
	@@ -324,6 +324,12 @@ struct mlx5_ib_raw_packet_qp {
	struct mlx5_ib_rq rq;		struct mlx5_ib_rq rq;
	};		};

			struct mlx5_bf {
			int buf_size;
			unsigned long offset;
			struct mlx5_sq_bfreg *bfreg;
			};

	struct mlx5_ib_qp {		struct mlx5_ib_qp {
	struct ib_qp ibqp;		struct ib_qp ibqp;
	union {		union {
	@@ -349,7 +355,7 @@ struct mlx5_ib_qp {
	int wq_sig;		int wq_sig;
	int scat_cqe;		int scat_cqe;
	int max_inline_data;		int max_inline_data;
	struct mlx5_bf *bf;		struct mlx5_bf bf;
	int has_rq;		int has_rq;

	/* only for user space QPs. For kernel		/* only for user space QPs. For kernel
	@@ -591,7 +597,6 @@ struct mlx5_ib_dev {
	struct ib_device ib_dev;		struct ib_device ib_dev;
	struct mlx5_core_dev *mdev;		struct mlx5_core_dev *mdev;
	struct mlx5_roce roce;		struct mlx5_roce roce;
	MLX5_DECLARE_DOORBELL_LOCK(uar_lock);
	int num_ports;		int num_ports;
	/* serialize update of capability mask		/* serialize update of capability mask
	*/		*/
	@@ -621,6 +626,8 @@ struct mlx5_ib_dev {
	struct list_head qp_list;		struct list_head qp_list;
	/* Array with num_ports elements */		/* Array with num_ports elements */
	struct mlx5_ib_port *port;		struct mlx5_ib_port *port;
			struct mlx5_sq_bfreg bfreg;
			struct mlx5_sq_bfreg fp_bfreg;
	};		};

	static inline struct mlx5_ib_cq to_mibcq(struct mlx5_core_cq mcq)		static inline struct mlx5_ib_cq to_mibcq(struct mlx5_core_cq mcq)

drivers/infiniband/hw/mlx5/qp.c

+14 −59

Original line number	Original line	Diff line number	Diff line
	@@ -909,14 +909,10 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
	u32 *in, int inlen,		u32 *in, int inlen,
	struct mlx5_ib_qp_base *base)		struct mlx5_ib_qp_base *base)
	{		{
	enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW;
	struct mlx5_bfreg_info *bfregi;
	int uar_index;		int uar_index;
	void *qpc;		void *qpc;
	int bfregn;
	int err;		int err;

	bfregi = &dev->mdev->priv.bfregi;
	if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN \|		if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN \|
	IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK \|		IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK \|
	IB_QP_CREATE_IPOIB_UD_LSO \|		IB_QP_CREATE_IPOIB_UD_LSO \|
	@@ -924,21 +920,17 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
	return -EINVAL;		return -EINVAL;

	if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)		if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
	lc = MLX5_IB_LATENCY_CLASS_FAST_PATH;		qp->bf.bfreg = &dev->fp_bfreg;
			else
	bfregn = alloc_bfreg(bfregi, lc);		qp->bf.bfreg = &dev->bfreg;
	if (bfregn < 0) {
	mlx5_ib_dbg(dev, "\n");
	return -ENOMEM;
	}

	qp->bf = &bfregi->bfs[bfregn];		qp->bf.buf_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
	uar_index = qp->bf->uar->index;		uar_index = qp->bf.bfreg->index;

	err = calc_sq_size(dev, init_attr, qp);		err = calc_sq_size(dev, init_attr, qp);
	if (err < 0) {		if (err < 0) {
	mlx5_ib_dbg(dev, "err %d\n", err);		mlx5_ib_dbg(dev, "err %d\n", err);
	goto err_bfreg;		return err;
	}		}

	qp->rq.offset = 0;		qp->rq.offset = 0;
	@@ -948,7 +940,7 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
	err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);		err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);
	if (err) {		if (err) {
	mlx5_ib_dbg(dev, "err %d\n", err);		mlx5_ib_dbg(dev, "err %d\n", err);
	goto err_bfreg;		return err;
	}		}

	qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);		qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);
	@@ -1010,9 +1002,6 @@ err_free:

	err_buf:		err_buf:
	mlx5_buf_free(dev->mdev, &qp->buf);		mlx5_buf_free(dev->mdev, &qp->buf);

	err_bfreg:
	free_bfreg(&dev->mdev->priv.bfregi, bfregn);
	return err;		return err;
	}		}

	@@ -1025,7 +1014,6 @@ static void destroy_qp_kernel(struct mlx5_ib_dev dev, struct mlx5_ib_qp qp)
	kfree(qp->rq.wrid);		kfree(qp->rq.wrid);
	mlx5_db_free(dev->mdev, &qp->db);		mlx5_db_free(dev->mdev, &qp->db);
	mlx5_buf_free(dev->mdev, &qp->buf);		mlx5_buf_free(dev->mdev, &qp->buf);
	free_bfreg(&dev->mdev->priv.bfregi, qp->bf->bfregn);
	}		}

	static u32 get_rx_type(struct mlx5_ib_qp qp, struct ib_qp_init_attr attr)		static u32 get_rx_type(struct mlx5_ib_qp qp, struct ib_qp_init_attr attr)
	@@ -3744,24 +3732,6 @@ static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
	}		}
	}		}

	static void mlx5_bf_copy(u64 __iomem dst, u64 src,
	unsigned bytecnt, struct mlx5_ib_qp *qp)
	{
	while (bytecnt > 0) {
	__iowrite64_copy(dst++, src++, 8);
	__iowrite64_copy(dst++, src++, 8);
	__iowrite64_copy(dst++, src++, 8);
	__iowrite64_copy(dst++, src++, 8);
	__iowrite64_copy(dst++, src++, 8);
	__iowrite64_copy(dst++, src++, 8);
	__iowrite64_copy(dst++, src++, 8);
	__iowrite64_copy(dst++, src++, 8);
	bytecnt -= 64;
	if (unlikely(src == qp->sq.qend))
	src = mlx5_get_send_wqe(qp, 0);
	}
	}

	static u8 get_fence(u8 fence, struct ib_send_wr *wr)		static u8 get_fence(u8 fence, struct ib_send_wr *wr)
	{		{
	if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&		if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&
	@@ -3857,7 +3827,7 @@ int mlx5_ib_post_send(struct ib_qp ibqp, struct ib_send_wr wr,
	return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);		return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);

	qp = to_mqp(ibqp);		qp = to_mqp(ibqp);
	bf = qp->bf;		bf = &qp->bf;
	qend = qp->sq.qend;		qend = qp->sq.qend;

	spin_lock_irqsave(&qp->sq.lock, flags);		spin_lock_irqsave(&qp->sq.lock, flags);
	@@ -4130,28 +4100,13 @@ out:
	* we hit doorbell */		* we hit doorbell */
	wmb();		wmb();

	if (bf->need_lock)		/* currently we support only regular doorbells */
	spin_lock(&bf->lock);		mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset, NULL);
	else
	__acquire(&bf->lock);

	/* TBD enable WC */
	if (0 && nreq == 1 && bf->bfregn && inl && size > 1 && size <= bf->buf_size / 16) {
	mlx5_bf_copy(bf->reg + bf->offset, (u64 )ctrl, ALIGN(size 16, 64), qp);
	/* wc_wmb(); */
	} else {
	mlx5_write64((__be32 *)ctrl, bf->regreg + bf->offset,
	MLX5_GET_DOORBELL_LOCK(&bf->lock32));
	/* Make sure doorbells don't leak out of SQ spinlock		/* Make sure doorbells don't leak out of SQ spinlock
	* and reach the HCA out of order.		* and reach the HCA out of order.
	*/		*/
	mmiowb();		mmiowb();
	}
	bf->offset ^= bf->buf_size;		bf->offset ^= bf->buf_size;
	if (bf->need_lock)
	spin_unlock(&bf->lock);
	else
	__release(&bf->lock);
	}		}

	spin_unlock_irqrestore(&qp->sq.lock, flags);		spin_unlock_irqrestore(&qp->sq.lock, flags);

drivers/net/ethernet/mellanox/mlx5/core/en.h

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -832,7 +832,7 @@ static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
	struct mlx5_core_cq *mcq;		struct mlx5_core_cq *mcq;

	mcq = &cq->mcq;		mcq = &cq->mcq;
	mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, NULL, cq->wq.cc);		mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
	}		}

	static inline u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix)		static inline u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix)

Admin message