Commit b5883008 authored by Li Dongyang's avatar Li Dongyang Committed by Doug Ledford
Browse files

IB/mlx5: use kvmalloc_array for mlx5_ib_wq 



We observed multiple times on our Lustre OSS servers that when
the system memory is fragmented, kmalloc() in create_kernel_qp()
could fail order 4/5 allocations while we still have many free pages.

Switch to kvmalloc_array() to allow the operation to contine.

Signed-off-by: default avatarLi Dongyang <dongyang.li@anu.edu.au>
Acked-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 7be05753

drivers/infiniband/hw/mlx5/qp.c

+20 −15
Original line number Diff line number Diff line
@@ -965,11 +965,16 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, 
		goto err_free;

	}



	qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wrid), GFP_KERNEL);

	qp->sq.wr_data = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_data), GFP_KERNEL);

	qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof(*qp->rq.wrid), GFP_KERNEL);

	qp->sq.w_list = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.w_list), GFP_KERNEL);

	qp->sq.wqe_head = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wqe_head), GFP_KERNEL);

	qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,

				     sizeof(*qp->sq.wrid), GFP_KERNEL);

	qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,

					sizeof(*qp->sq.wr_data), GFP_KERNEL);

	qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,

				     sizeof(*qp->rq.wrid), GFP_KERNEL);

	qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,

				       sizeof(*qp->sq.w_list), GFP_KERNEL);

	qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,

					 sizeof(*qp->sq.wqe_head), GFP_KERNEL);



	if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||

	    !qp->sq.w_list || !qp->sq.wqe_head) {
@@ -981,11 +986,11 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, 
	return 0;



err_wrid:

	kfree(qp->sq.wqe_head);

	kfree(qp->sq.w_list);

	kfree(qp->sq.wrid);

	kfree(qp->sq.wr_data);

	kfree(qp->rq.wrid);

	kvfree(qp->sq.wqe_head);

	kvfree(qp->sq.w_list);

	kvfree(qp->sq.wrid);

	kvfree(qp->sq.wr_data);

	kvfree(qp->rq.wrid);

	mlx5_db_free(dev->mdev, &qp->db);



err_free:
@@ -998,11 +1003,11 @@ err_buf: 


static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)

{

	kfree(qp->sq.wqe_head);

	kfree(qp->sq.w_list);

	kfree(qp->sq.wrid);

	kfree(qp->sq.wr_data);

	kfree(qp->rq.wrid);

	kvfree(qp->sq.wqe_head);

	kvfree(qp->sq.w_list);

	kvfree(qp->sq.wrid);

	kvfree(qp->sq.wr_data);

	kvfree(qp->rq.wrid);

	mlx5_db_free(dev->mdev, &qp->db);

	mlx5_buf_free(dev->mdev, &qp->buf);

}

drivers/infiniband/hw/mlx5/srq.c

+2 −2
Original line number Diff line number Diff line
@@ -196,7 +196,7 @@ static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, 
	}

	mlx5_fill_page_array(&srq->buf, in->pas);



	srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL);

	srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL);

	if (!srq->wrid) {

		err = -ENOMEM;

		goto err_in;
@@ -230,7 +230,7 @@ static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq) 


static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)

{

	kfree(srq->wrid);

	kvfree(srq->wrid);

	mlx5_buf_free(dev->mdev, &srq->buf);

	mlx5_db_free(dev->mdev, &srq->db);

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE