Commit fadd59fc authored by Aviv Heller's avatar Aviv Heller Committed by Saeed Mahameed
Browse files

net/mlx5: Introduce inter-device communication mechanism 



This introduces devcom, a generic mechanism for performing operations
on both physical functions of the same Connect-X card.

The first user of this API is merged eswitch, which will be introduced
in subsequent patches.

Signed-off-by: default avatarAviv Heller <avivh@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
parent 64e4cf0d

drivers/net/ethernet/mellanox/mlx5/core/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ 
		health.o mcg.o cq.o alloc.o qp.o port.o mr.o pd.o \

		mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \

		fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \

		diag/fs_tracepoint.o diag/fw_tracer.o

		lib/devcom.o diag/fs_tracepoint.o diag/fw_tracer.o



#

# Netdev basic

drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c

0 → 100644
+255 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB

/* Copyright (c) 2018 Mellanox Technologies */



#include <linux/mlx5/vport.h>

#include "lib/devcom.h"



static LIST_HEAD(devcom_list);



#define devcom_for_each_component(priv, comp, iter) \

	for (iter = 0; \

	     comp = &(priv)->components[iter], iter < MLX5_DEVCOM_NUM_COMPONENTS; \

	     iter++)



struct mlx5_devcom_component {

	struct {

		void *data;

	} device[MLX5_MAX_PORTS];



	mlx5_devcom_event_handler_t handler;

	struct rw_semaphore sem;

	bool paired;

};



struct mlx5_devcom_list {

	struct list_head list;



	struct mlx5_devcom_component components[MLX5_DEVCOM_NUM_COMPONENTS];

	struct mlx5_core_dev *devs[MLX5_MAX_PORTS];

};



struct mlx5_devcom {

	struct mlx5_devcom_list *priv;

	int idx;

};



static struct mlx5_devcom_list *mlx5_devcom_list_alloc(void)

{

	struct mlx5_devcom_component *comp;

	struct mlx5_devcom_list *priv;

	int i;



	priv = kzalloc(sizeof(*priv), GFP_KERNEL);

	if (!priv)

		return NULL;



	devcom_for_each_component(priv, comp, i)

		init_rwsem(&comp->sem);



	return priv;

}



static struct mlx5_devcom *mlx5_devcom_alloc(struct mlx5_devcom_list *priv,

					     u8 idx)

{

	struct mlx5_devcom *devcom;



	devcom = kzalloc(sizeof(*devcom), GFP_KERNEL);

	if (!devcom)

		return NULL;



	devcom->priv = priv;

	devcom->idx = idx;

	return devcom;

}



/* Must be called with intf_mutex held */

struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev)

{

	struct mlx5_devcom_list *priv = NULL, *iter;

	struct mlx5_devcom *devcom = NULL;

	bool new_priv = false;

	u64 sguid0, sguid1;

	int idx, i;



	if (!mlx5_core_is_pf(dev))

		return NULL;



	sguid0 = mlx5_query_nic_system_image_guid(dev);

	list_for_each_entry(iter, &devcom_list, list) {

		struct mlx5_core_dev *tmp_dev = NULL;



		idx = -1;

		for (i = 0; i < MLX5_MAX_PORTS; i++) {

			if (iter->devs[i])

				tmp_dev = iter->devs[i];

			else

				idx = i;

		}



		if (idx == -1)

			continue;



		sguid1 = mlx5_query_nic_system_image_guid(tmp_dev);

		if (sguid0 != sguid1)

			continue;



		priv = iter;

		break;

	}



	if (!priv) {

		priv = mlx5_devcom_list_alloc();

		if (!priv)

			return ERR_PTR(-ENOMEM);



		idx = 0;

		new_priv = true;

	}



	priv->devs[idx] = dev;

	devcom = mlx5_devcom_alloc(priv, idx);

	if (!devcom) {

		kfree(priv);

		return ERR_PTR(-ENOMEM);

	}



	if (new_priv)

		list_add(&priv->list, &devcom_list);



	return devcom;

}



/* Must be called with intf_mutex held */

void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom)

{

	struct mlx5_devcom_list *priv;

	int i;



	if (IS_ERR_OR_NULL(devcom))

		return;



	priv = devcom->priv;

	priv->devs[devcom->idx] = NULL;



	kfree(devcom);



	for (i = 0; i < MLX5_MAX_PORTS; i++)

		if (priv->devs[i])

			break;



	if (i != MLX5_MAX_PORTS)

		return;



	list_del(&priv->list);

	kfree(priv);

}



void mlx5_devcom_register_component(struct mlx5_devcom *devcom,

				    enum mlx5_devcom_components id,

				    mlx5_devcom_event_handler_t handler,

				    void *data)

{

	struct mlx5_devcom_component *comp;



	if (IS_ERR_OR_NULL(devcom))

		return;



	WARN_ON(!data);



	comp = &devcom->priv->components[id];

	down_write(&comp->sem);

	comp->handler = handler;

	comp->device[devcom->idx].data = data;

	up_write(&comp->sem);

}



void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom,

				      enum mlx5_devcom_components id)

{

	struct mlx5_devcom_component *comp;



	if (IS_ERR_OR_NULL(devcom))

		return;



	comp = &devcom->priv->components[id];

	down_write(&comp->sem);

	comp->device[devcom->idx].data = NULL;

	up_write(&comp->sem);

}



int mlx5_devcom_send_event(struct mlx5_devcom *devcom,

			   enum mlx5_devcom_components id,

			   int event,

			   void *event_data)

{

	struct mlx5_devcom_component *comp;

	int err = -ENODEV, i;



	if (IS_ERR_OR_NULL(devcom))

		return err;



	comp = &devcom->priv->components[id];

	down_write(&comp->sem);

	for (i = 0; i < MLX5_MAX_PORTS; i++)

		if (i != devcom->idx && comp->device[i].data) {

			err = comp->handler(event, comp->device[i].data,

					    event_data);

			break;

		}



	up_write(&comp->sem);

	return err;

}



void mlx5_devcom_set_paired(struct mlx5_devcom *devcom,

			    enum mlx5_devcom_components id,

			    bool paired)

{

	struct mlx5_devcom_component *comp;



	comp = &devcom->priv->components[id];

	WARN_ON(!rwsem_is_locked(&comp->sem));



	comp->paired = paired;

}



bool mlx5_devcom_is_paired(struct mlx5_devcom *devcom,

			   enum mlx5_devcom_components id)

{

	if (IS_ERR_OR_NULL(devcom))

		return false;



	return devcom->priv->components[id].paired;

}



void *mlx5_devcom_get_peer_data(struct mlx5_devcom *devcom,

				enum mlx5_devcom_components id)

{

	struct mlx5_devcom_component *comp;

	int i;



	if (IS_ERR_OR_NULL(devcom))

		return NULL;



	comp = &devcom->priv->components[id];

	down_read(&comp->sem);

	if (!comp->paired) {

		up_read(&comp->sem);

		return NULL;

	}



	for (i = 0; i < MLX5_MAX_PORTS; i++)

		if (i != devcom->idx)

			break;



	return comp->device[i].data;

}



void mlx5_devcom_release_peer_data(struct mlx5_devcom *devcom,

				   enum mlx5_devcom_components id)

{

	struct mlx5_devcom_component *comp = &devcom->priv->components[id];



	up_read(&comp->sem);

}

drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h

0 → 100644
+44 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

/* Copyright (c) 2018 Mellanox Technologies */



#ifndef __LIB_MLX5_DEVCOM_H__

#define __LIB_MLX5_DEVCOM_H__



#include <linux/mlx5/driver.h>



enum mlx5_devcom_components {

	MLX5_DEVCOM_NUM_COMPONENTS,

};



typedef int (*mlx5_devcom_event_handler_t)(int event,

					   void *my_data,

					   void *event_data);



struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev);

void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom);



void mlx5_devcom_register_component(struct mlx5_devcom *devcom,

				    enum mlx5_devcom_components id,

				    mlx5_devcom_event_handler_t handler,

				    void *data);

void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom,

				      enum mlx5_devcom_components id);



int mlx5_devcom_send_event(struct mlx5_devcom *devcom,

			   enum mlx5_devcom_components id,

			   int event,

			   void *event_data);



void mlx5_devcom_set_paired(struct mlx5_devcom *devcom,

			    enum mlx5_devcom_components id,

			    bool paired);

bool mlx5_devcom_is_paired(struct mlx5_devcom *devcom,

			   enum mlx5_devcom_components id);



void *mlx5_devcom_get_peer_data(struct mlx5_devcom *devcom,

				enum mlx5_devcom_components id);

void mlx5_devcom_release_peer_data(struct mlx5_devcom *devcom,

				   enum mlx5_devcom_components id);



#endif

drivers/net/ethernet/mellanox/mlx5/core/main.c

+11 −3
Original line number Diff line number Diff line
@@ -63,6 +63,7 @@ 
#include "accel/tls.h"

#include "lib/clock.h"

#include "lib/vxlan.h"

#include "lib/devcom.h"

#include "diag/fw_tracer.h"



MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
@@ -722,16 +723,21 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 
	struct pci_dev *pdev = dev->pdev;

	int err;



	priv->devcom = mlx5_devcom_register_device(dev);

	if (IS_ERR(priv->devcom))

		dev_err(&pdev->dev, "failed to register with devcom (0x%p)\n",

			priv->devcom);



	err = mlx5_query_board_id(dev);

	if (err) {

		dev_err(&pdev->dev, "query board id failed\n");

		goto out;

		goto err_devcom;

	}



	err = mlx5_eq_table_init(dev);

	if (err) {

		dev_err(&pdev->dev, "failed to initialize eq\n");

		goto out;

		goto err_devcom;

	}



	err = mlx5_events_init(dev);
@@ -807,8 +813,9 @@ err_events_cleanup: 
	mlx5_events_cleanup(dev);

err_eq_cleanup:

	mlx5_eq_table_cleanup(dev);

err_devcom:

	mlx5_devcom_unregister_device(dev->priv.devcom);



out:

	return err;

}
@@ -828,6 +835,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) 
	mlx5_cq_debugfs_cleanup(dev);

	mlx5_events_cleanup(dev);

	mlx5_eq_table_cleanup(dev);

	mlx5_devcom_unregister_device(dev->priv.devcom);

}



static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,

include/linux/mlx5/driver.h

+2 −0
Original line number Diff line number Diff line
@@ -486,6 +486,7 @@ struct mlx5_events; 
struct mlx5_mpfs;

struct mlx5_eswitch;

struct mlx5_lag;

struct mlx5_devcom;

struct mlx5_eq_table;



struct mlx5_rate_limit {
@@ -560,6 +561,7 @@ struct mlx5_priv { 
	struct mlx5_eswitch     *eswitch;

	struct mlx5_core_sriov	sriov;

	struct mlx5_lag		*lag;

	struct mlx5_devcom	*devcom;

	unsigned long		pci_dev_data;

	struct mlx5_fc_stats		fc_stats;

	struct mlx5_rl_table            rl_table;

CRA Git | Maintained and supported by SUSTech CRA and CCSE