Commit f76a5c75 authored by Doug Ledford's avatar Doug Ledford
Browse files

Merge tag 'mlx5-updates-2018-02-21' of... 

Merge tag 'mlx5-updates-2018-02-21' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux

 into k.o/wip/dl-for-next

mlx5-updates-2018-02-21

This series includes shared code updates for mlx5 core driver for both
netdev and rdma subsystems.

By Saeed,
First six patches of the series are meant to address a performance issue
and should provide a performance boost for multi core IRQ interrupt hungry
workloads.  The issue is fixed in the first patch, all other patches are
meant to refactor the code in light of this fix.

The problem it comes to fix, is a shared spinlock accessed across all HCA
IRQs which protects the CQ database.  To solve this we simply move the CQ
database and its spinlock to be per EQ (IRQ), thus per core.

By Yonatan,
Fragmented completion queue (CQ) for RDMA,
core driver implementation to create fragmented CQ buffers rather than
one large contiguous memory buffer, the implementation scheme already
exist and used by the netdev CQs, the patch shares that code with the
rdma CQ creation flow and makes use of the new API in mlx5_ib driver.

Thanks,
Saeed.

Merged into rdma-next tree as well as net-next tree to prevent conflicts
in future patches between the two trees.

Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parents 3a148896 388ca8be

drivers/infiniband/hw/mlx5/cq.c

+39 −25
Original line number Diff line number Diff line
@@ -64,14 +64,9 @@ static void mlx5_ib_cq_event(struct mlx5_core_cq *mcq, enum mlx5_event type) 
	}

}



static void *get_cqe_from_buf(struct mlx5_ib_cq_buf *buf, int n, int size)

{

	return mlx5_buf_offset(&buf->buf, n * size);

}



static void *get_cqe(struct mlx5_ib_cq *cq, int n)

{

	return get_cqe_from_buf(&cq->buf, n, cq->mcq.cqe_sz);

	return mlx5_frag_buf_get_wqe(&cq->buf.fbc, n);

}



static u8 sw_ownership_bit(int n, int nent)
@@ -403,7 +398,7 @@ static void handle_atomics(struct mlx5_ib_qp *qp, struct mlx5_cqe64 *cqe64, 


static void free_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf)

{

	mlx5_buf_free(dev->mdev, &buf->buf);

	mlx5_frag_buf_free(dev->mdev, &buf->fbc.frag_buf);

}



static void get_sig_err_item(struct mlx5_sig_err_cqe *cqe,
@@ -724,12 +719,25 @@ int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 
	return ret;

}



static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,

			int nent, int cqe_size)

static int alloc_cq_frag_buf(struct mlx5_ib_dev *dev,

			     struct mlx5_ib_cq_buf *buf,

			     int nent,

			     int cqe_size)

{

	struct mlx5_frag_buf_ctrl *c = &buf->fbc;

	struct mlx5_frag_buf *frag_buf = &c->frag_buf;

	u32 cqc_buff[MLX5_ST_SZ_DW(cqc)] = {0};

	int err;



	err = mlx5_buf_alloc(dev->mdev, nent * cqe_size, &buf->buf);

	MLX5_SET(cqc, cqc_buff, log_cq_size, ilog2(cqe_size));

	MLX5_SET(cqc, cqc_buff, cqe_sz, (cqe_size == 128) ? 1 : 0);



	mlx5_core_init_cq_frag_buf(&buf->fbc, cqc_buff);



	err = mlx5_frag_buf_alloc_node(dev->mdev,

				       nent * cqe_size,

				       frag_buf,

				       dev->mdev->priv.numa_node);

	if (err)

		return err;
@@ -862,14 +870,15 @@ static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context) 
	ib_umem_release(cq->buf.umem);

}



static void init_cq_buf(struct mlx5_ib_cq *cq, struct mlx5_ib_cq_buf *buf)

static void init_cq_frag_buf(struct mlx5_ib_cq *cq,

			     struct mlx5_ib_cq_buf *buf)

{

	int i;

	void *cqe;

	struct mlx5_cqe64 *cqe64;



	for (i = 0; i < buf->nent; i++) {

		cqe = get_cqe_from_buf(buf, i, buf->cqe_size);

		cqe = get_cqe(cq, i);

		cqe64 = buf->cqe_size == 64 ? cqe : cqe + 64;

		cqe64->op_own = MLX5_CQE_INVALID << 4;

	}
@@ -891,14 +900,15 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, 
	cq->mcq.arm_db     = cq->db.db + 1;

	cq->mcq.cqe_sz = cqe_size;



	err = alloc_cq_buf(dev, &cq->buf, entries, cqe_size);

	err = alloc_cq_frag_buf(dev, &cq->buf, entries, cqe_size);

	if (err)

		goto err_db;



	init_cq_buf(cq, &cq->buf);

	init_cq_frag_buf(cq, &cq->buf);



	*inlen = MLX5_ST_SZ_BYTES(create_cq_in) +

		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * cq->buf.buf.npages;

		 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) *

		 cq->buf.fbc.frag_buf.npages;

	*cqb = kvzalloc(*inlen, GFP_KERNEL);

	if (!*cqb) {

		err = -ENOMEM;
@@ -906,11 +916,12 @@ static int create_cq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, 
	}



	pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, *cqb, pas);

	mlx5_fill_page_array(&cq->buf.buf, pas);

	mlx5_fill_page_frag_array(&cq->buf.fbc.frag_buf, pas);



	cqc = MLX5_ADDR_OF(create_cq_in, *cqb, cq_context);

	MLX5_SET(cqc, cqc, log_page_size,

		 cq->buf.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);

		 cq->buf.fbc.frag_buf.page_shift -

		 MLX5_ADAPTER_PAGE_SHIFT);



	*index = dev->mdev->priv.uar->index;
@@ -1207,11 +1218,11 @@ static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, 
	if (!cq->resize_buf)

		return -ENOMEM;



	err = alloc_cq_buf(dev, cq->resize_buf, entries, cqe_size);

	err = alloc_cq_frag_buf(dev, cq->resize_buf, entries, cqe_size);

	if (err)

		goto ex;



	init_cq_buf(cq, cq->resize_buf);

	init_cq_frag_buf(cq, cq->resize_buf);



	return 0;
@@ -1256,9 +1267,8 @@ static int copy_resize_cqes(struct mlx5_ib_cq *cq) 
	}



	while ((scqe64->op_own >> 4) != MLX5_CQE_RESIZE_CQ) {

		dcqe = get_cqe_from_buf(cq->resize_buf,

					(i + 1) & (cq->resize_buf->nent),

					dsize);

		dcqe = mlx5_frag_buf_get_wqe(&cq->resize_buf->fbc,

					     (i + 1) & cq->resize_buf->nent);

		dcqe64 = dsize == 64 ? dcqe : dcqe + 64;

		sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent);

		memcpy(dcqe, scqe, dsize);
@@ -1324,8 +1334,11 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) 
		cqe_size = 64;

		err = resize_kernel(dev, cq, entries, cqe_size);

		if (!err) {

			npas = cq->resize_buf->buf.npages;

			page_shift = cq->resize_buf->buf.page_shift;

			struct mlx5_frag_buf_ctrl *c;



			c = &cq->resize_buf->fbc;

			npas = c->frag_buf.npages;

			page_shift = c->frag_buf.page_shift;

		}

	}
@@ -1346,7 +1359,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) 
		mlx5_ib_populate_pas(dev, cq->resize_umem, page_shift,

				     pas, 0);

	else

		mlx5_fill_page_array(&cq->resize_buf->buf, pas);

		mlx5_fill_page_frag_array(&cq->resize_buf->fbc.frag_buf,

					  pas);



	MLX5_SET(modify_cq_in, in,

		 modify_field_select_resize_field_select.resize_field_select.resize_field_select,

drivers/infiniband/hw/mlx5/mlx5_ib.h

+3 −3
Original line number Diff line number Diff line
@@ -371,7 +371,7 @@ struct mlx5_ib_qp { 
		struct mlx5_ib_rss_qp rss_qp;

		struct mlx5_ib_dct dct;

	};

	struct mlx5_buf		buf;

	struct mlx5_frag_buf	buf;



	struct mlx5_db		db;

	struct mlx5_ib_wq	rq;
@@ -413,7 +413,7 @@ struct mlx5_ib_qp { 
};



struct mlx5_ib_cq_buf {

	struct mlx5_buf		buf;

	struct mlx5_frag_buf_ctrl fbc;

	struct ib_umem		*umem;

	int			cqe_size;

	int			nent;
@@ -495,7 +495,7 @@ struct mlx5_ib_wc { 
struct mlx5_ib_srq {

	struct ib_srq		ibsrq;

	struct mlx5_core_srq	msrq;

	struct mlx5_buf		buf;

	struct mlx5_frag_buf	buf;

	struct mlx5_db		db;

	u64		       *wrid;

	/* protect SRQ hanlding

drivers/net/ethernet/mellanox/mlx5/core/alloc.c

+25 −12
Original line number Diff line number Diff line
@@ -71,19 +71,24 @@ static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev, 
}



int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,

			struct mlx5_buf *buf, int node)

			struct mlx5_frag_buf *buf, int node)

{

	dma_addr_t t;



	buf->size = size;

	buf->npages       = 1;

	buf->page_shift   = (u8)get_order(size) + PAGE_SHIFT;

	buf->direct.buf   = mlx5_dma_zalloc_coherent_node(dev, size,

							  &t, node);

	if (!buf->direct.buf)



	buf->frags = kzalloc(sizeof(*buf->frags), GFP_KERNEL);

	if (!buf->frags)

		return -ENOMEM;



	buf->direct.map = t;

	buf->frags->buf   = mlx5_dma_zalloc_coherent_node(dev, size,

							  &t, node);

	if (!buf->frags->buf)

		goto err_out;



	buf->frags->map = t;



	while (t & ((1 << buf->page_shift) - 1)) {

		--buf->page_shift;
@@ -91,18 +96,24 @@ int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, 
	}



	return 0;

err_out:

	kfree(buf->frags);

	return -ENOMEM;

}



int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf)

int mlx5_buf_alloc(struct mlx5_core_dev *dev,

		   int size, struct mlx5_frag_buf *buf)

{

	return mlx5_buf_alloc_node(dev, size, buf, dev->priv.numa_node);

}

EXPORT_SYMBOL_GPL(mlx5_buf_alloc);

EXPORT_SYMBOL(mlx5_buf_alloc);



void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf)

void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf)

{

	dma_free_coherent(&dev->pdev->dev, buf->size, buf->direct.buf,

			  buf->direct.map);

	dma_free_coherent(&dev->pdev->dev, buf->size, buf->frags->buf,

			  buf->frags->map);



	kfree(buf->frags);

}

EXPORT_SYMBOL_GPL(mlx5_buf_free);
@@ -147,6 +158,7 @@ err_free_buf: 
err_out:

	return -ENOMEM;

}

EXPORT_SYMBOL_GPL(mlx5_frag_buf_alloc_node);



void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf)

{
@@ -162,6 +174,7 @@ void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) 
	}

	kfree(buf->frags);

}

EXPORT_SYMBOL_GPL(mlx5_frag_buf_free);



static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev,

						 int node)
@@ -275,13 +288,13 @@ void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db) 
}

EXPORT_SYMBOL_GPL(mlx5_db_free);



void mlx5_fill_page_array(struct mlx5_buf *buf, __be64 *pas)

void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas)

{

	u64 addr;

	int i;



	for (i = 0; i < buf->npages; i++) {

		addr = buf->direct.map + (i << buf->page_shift);

		addr = buf->frags->map + (i << buf->page_shift);



		pas[i] = cpu_to_be64(addr);

	}

drivers/net/ethernet/mellanox/mlx5/core/cq.c

+24 −92
Original line number Diff line number Diff line
@@ -58,8 +58,7 @@ void mlx5_cq_tasklet_cb(unsigned long data) 
				 tasklet_ctx.list) {

		list_del_init(&mcq->tasklet_ctx.list);

		mcq->tasklet_ctx.comp(mcq);

		if (refcount_dec_and_test(&mcq->refcount))

			complete(&mcq->free);

		mlx5_cq_put(mcq);

		if (time_after(jiffies, end))

			break;

	}
@@ -80,69 +79,19 @@ static void mlx5_add_cq_to_tasklet(struct mlx5_core_cq *cq) 
	 * still arrive.

	 */

	if (list_empty_careful(&cq->tasklet_ctx.list)) {

		refcount_inc(&cq->refcount);

		mlx5_cq_hold(cq);

		list_add_tail(&cq->tasklet_ctx.list, &tasklet_ctx->list);

	}

	spin_unlock_irqrestore(&tasklet_ctx->lock, flags);

}



void mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn)

{

	struct mlx5_core_cq *cq;

	struct mlx5_cq_table *table = &dev->priv.cq_table;



	spin_lock(&table->lock);

	cq = radix_tree_lookup(&table->tree, cqn);

	if (likely(cq))

		refcount_inc(&cq->refcount);

	spin_unlock(&table->lock);



	if (!cq) {

		mlx5_core_warn(dev, "Completion event for bogus CQ 0x%x\n", cqn);

		return;

	}



	++cq->arm_sn;



	cq->comp(cq);



	if (refcount_dec_and_test(&cq->refcount))

		complete(&cq->free);

}



void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type)

{

	struct mlx5_cq_table *table = &dev->priv.cq_table;

	struct mlx5_core_cq *cq;



	spin_lock(&table->lock);



	cq = radix_tree_lookup(&table->tree, cqn);

	if (cq)

		refcount_inc(&cq->refcount);



	spin_unlock(&table->lock);



	if (!cq) {

		mlx5_core_warn(dev, "Async event for bogus CQ 0x%x\n", cqn);

		return;

	}



	cq->event(cq, event_type);



	if (refcount_dec_and_test(&cq->refcount))

		complete(&cq->free);

}



int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,

			u32 *in, int inlen)

{

	struct mlx5_cq_table *table = &dev->priv.cq_table;

	int eqn = MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context), c_eqn);

	u32 dout[MLX5_ST_SZ_DW(destroy_cq_out)];

	u32 out[MLX5_ST_SZ_DW(create_cq_out)];

	u32 din[MLX5_ST_SZ_DW(destroy_cq_in)];

	u32 dout[MLX5_ST_SZ_DW(destroy_cq_out)];

	int eqn = MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context),

			   c_eqn);

	struct mlx5_eq *eq;

	int err;
@@ -159,7 +108,9 @@ int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 
	cq->cqn = MLX5_GET(create_cq_out, out, cqn);

	cq->cons_index = 0;

	cq->arm_sn     = 0;

	refcount_set(&cq->refcount, 1);

	cq->eq         = eq;

	refcount_set(&cq->refcount, 0);

	mlx5_cq_hold(cq);

	init_completion(&cq->free);

	if (!cq->comp)

		cq->comp = mlx5_add_cq_to_tasklet;
@@ -167,12 +118,16 @@ int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 
	cq->tasklet_ctx.priv = &eq->tasklet_ctx;

	INIT_LIST_HEAD(&cq->tasklet_ctx.list);



	spin_lock_irq(&table->lock);

	err = radix_tree_insert(&table->tree, cq->cqn, cq);

	spin_unlock_irq(&table->lock);

	/* Add to comp EQ CQ tree to recv comp events */

	err = mlx5_eq_add_cq(eq, cq);

	if (err)

		goto err_cmd;



	/* Add to async EQ CQ tree to recv async events */

	err = mlx5_eq_add_cq(&dev->priv.eq_table.async_eq, cq);

	if (err)

		goto err_cq_add;



	cq->pid = current->pid;

	err = mlx5_debug_cq_add(dev, cq);

	if (err)
@@ -183,6 +138,8 @@ int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 


	return 0;



err_cq_add:

	mlx5_eq_del_cq(eq, cq);

err_cmd:

	memset(din, 0, sizeof(din));

	memset(dout, 0, sizeof(dout));
@@ -195,23 +152,17 @@ EXPORT_SYMBOL(mlx5_core_create_cq); 


int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq)

{

	struct mlx5_cq_table *table = &dev->priv.cq_table;

	u32 out[MLX5_ST_SZ_DW(destroy_cq_out)] = {0};

	u32 in[MLX5_ST_SZ_DW(destroy_cq_in)] = {0};

	struct mlx5_core_cq *tmp;

	int err;



	spin_lock_irq(&table->lock);

	tmp = radix_tree_delete(&table->tree, cq->cqn);

	spin_unlock_irq(&table->lock);

	if (!tmp) {

		mlx5_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn);

		return -EINVAL;

	}

	if (tmp != cq) {

		mlx5_core_warn(dev, "corruption on srqn 0x%x\n", cq->cqn);

		return -EINVAL;

	}

	err = mlx5_eq_del_cq(&dev->priv.eq_table.async_eq, cq);

	if (err)

		return err;



	err = mlx5_eq_del_cq(cq->eq, cq);

	if (err)

		return err;



	MLX5_SET(destroy_cq_in, in, opcode, MLX5_CMD_OP_DESTROY_CQ);

	MLX5_SET(destroy_cq_in, in, cqn, cq->cqn);
@@ -222,8 +173,7 @@ int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq) 
	synchronize_irq(cq->irqn);



	mlx5_debug_cq_remove(dev, cq);

	if (refcount_dec_and_test(&cq->refcount))

		complete(&cq->free);

	mlx5_cq_put(cq);

	wait_for_completion(&cq->free);



	return 0;
@@ -270,21 +220,3 @@ int mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev, 
	return mlx5_core_modify_cq(dev, cq, in, sizeof(in));

}

EXPORT_SYMBOL(mlx5_core_modify_cq_moderation);
 


int mlx5_init_cq_table(struct mlx5_core_dev *dev)

{

	struct mlx5_cq_table *table = &dev->priv.cq_table;

	int err;



	memset(table, 0, sizeof(*table));

	spin_lock_init(&table->lock);

	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);

	err = mlx5_cq_debugfs_init(dev);



	return err;

}



void mlx5_cleanup_cq_table(struct mlx5_core_dev *dev)

{

	mlx5_cq_debugfs_cleanup(dev);

}

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

+6 −5
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config) 
static inline void mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cqcc,

				       void *data)

{

	u32 ci = cqcc & cq->wq.sz_m1;

	u32 ci = cqcc & cq->wq.fbc.sz_m1;



	memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, ci), sizeof(struct mlx5_cqe64));

}
@@ -74,9 +74,10 @@ static inline void mlx5e_read_mini_arr_slot(struct mlx5e_cq *cq, u32 cqcc) 


static inline void mlx5e_cqes_update_owner(struct mlx5e_cq *cq, u32 cqcc, int n)

{

	u8 op_own = (cqcc >> cq->wq.log_sz) & 1;

	u32 wq_sz = 1 << cq->wq.log_sz;

	u32 ci = cqcc & cq->wq.sz_m1;

	struct mlx5_frag_buf_ctrl *fbc = &cq->wq.fbc;

	u8 op_own = (cqcc >> fbc->log_sz) & 1;

	u32 wq_sz = 1 << fbc->log_sz;

	u32 ci = cqcc & fbc->sz_m1;

	u32 ci_top = min_t(u32, wq_sz, ci + n);



	for (; ci < ci_top; ci++, n--) {
@@ -101,7 +102,7 @@ static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq, 
	cq->title.byte_cnt     = cq->mini_arr[cq->mini_arr_idx].byte_cnt;

	cq->title.check_sum    = cq->mini_arr[cq->mini_arr_idx].checksum;

	cq->title.op_own      &= 0xf0;

	cq->title.op_own      |= 0x01 & (cqcc >> cq->wq.log_sz);

	cq->title.op_own      |= 0x01 & (cqcc >> cq->wq.fbc.log_sz);

	cq->title.wqe_counter  = cpu_to_be16(cq->decmprs_wqe_counter);



	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)

CRA Git | Maintained and supported by SUSTech CRA and CCSE