Commit c87e65cf authored by Leon Romanovsky's avatar Leon Romanovsky Committed by Jason Gunthorpe
Browse files

RDMA/cm: Move debug counters to be under relevant IB device 



The sysfs layout is created by CM incorrectly presented RDMA devices with
InfiniBand link layer. Layout of such devices represents device tree of
connections. By moving CM statistics to be under relevant port of IB
device, we will fix the following issues:

 * Symlink name - It used device name instead of specific identifier.
 * Target location - It was supposed to point to PCI-ID/infiniband_cm/
   instead of PCI-ID/infiniband/
 * Target name - It created extra device file under already existing
   device folder, e.g. mlx5_0/mlx5_0
 * Crash during boot with RDMA persistent naming patches.

 sysfs: cannot create duplicate filename '/class/infiniband_cm/mlx5_0'
 CPU: 29 PID: 433 Comm: modprobe Not tainted 5.0.0-rc5+ #178
 Call Trace:
  dump_stack+0xcc/0x180
  sysfs_warn_dup.cold.3+0x17/0x2d
  sysfs_do_create_link_sd.isra.2+0xd0/0xf0
  device_add+0x7cb/0x1450
  device_create_groups_vargs+0x1ae/0x220
  device_create+0x93/0xc0
  cm_add_one+0x38f/0xf60 [ib_cm]
  add_client_context+0x167/0x210 [ib_core]
  enable_device_and_get+0x230/0x3f0 [ib_core]
  ib_register_device+0x823/0xbf0 [ib_core]
  __mlx5_ib_add+0x45/0x150 [mlx5_ib]
  mlx5_ib_add+0x1b3/0x5e0 [mlx5_ib]
  mlx5_add_device+0x130/0x3a0 [mlx5_core]
  mlx5_register_interface+0x1a9/0x270 [mlx5_core]
  do_one_initcall+0x14f/0x5de
  do_init_module+0x247/0x7c0
  load_module+0x4c2f/0x60d0
  entry_SYSCALL_64_after_hwframe+0x49/0xbe

After this change:
[leonro@server ~]$ ls -al /sys/class/infiniband/ibp0s12f0/ports/1/
drwxr-xr-x  2 root root    0 Mar 11 11:17 cm_rx_duplicates
drwxr-xr-x  2 root root    0 Mar 11 11:17 cm_rx_msgs
drwxr-xr-x  2 root root    0 Mar 11 11:17 cm_tx_msgs
drwxr-xr-x  2 root root    0 Mar 11 11:17 cm_tx_retries

Fixes: 110cf374 ("infiniband: make cm_device use a struct device and not a kobject.")
Signed-off-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent 4d2e11d4

drivers/infiniband/core/cm.c

+8 −38
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ 
#include <rdma/ib_cache.h>

#include <rdma/ib_cm.h>

#include "cm_msgs.h"

#include "core_priv.h"



MODULE_AUTHOR("Sean Hefty");

MODULE_DESCRIPTION("InfiniBand CM");
@@ -220,7 +221,6 @@ struct cm_port { 
struct cm_device {

	struct list_head list;

	struct ib_device *ib_device;

	struct device *device;

	u8 ack_delay;

	int going_down;

	struct cm_port *port[0];
@@ -4272,18 +4272,6 @@ static struct kobj_type cm_counter_obj_type = { 
	.default_attrs = cm_counter_default_attrs

};



static void cm_release_port_obj(struct kobject *obj)

{

	struct cm_port *cm_port;



	cm_port = container_of(obj, struct cm_port, port_obj);

	kfree(cm_port);

}



static struct kobj_type cm_port_obj_type = {

	.release = cm_release_port_obj

};



static char *cm_devnode(struct device *dev, umode_t *mode)

{

	if (mode)
@@ -4302,19 +4290,12 @@ static int cm_create_port_fs(struct cm_port *port) 
{

	int i, ret;



	ret = kobject_init_and_add(&port->port_obj, &cm_port_obj_type,

				   &port->cm_dev->device->kobj,

				   "%d", port->port_num);

	if (ret) {

		kfree(port);

		return ret;

	}



	for (i = 0; i < CM_COUNTER_GROUPS; i++) {

		ret = kobject_init_and_add(&port->counter_group[i].obj,

		ret = ib_port_register_module_stat(port->cm_dev->ib_device,

						   port->port_num,

						   &port->counter_group[i].obj,

						   &cm_counter_obj_type,

					   &port->port_obj,

					   "%s", counter_group_names[i]);

						   counter_group_names[i]);

		if (ret)

			goto error;

	}
@@ -4323,8 +4304,7 @@ static int cm_create_port_fs(struct cm_port *port) 


error:

	while (i--)

		kobject_put(&port->counter_group[i].obj);

	kobject_put(&port->port_obj);

		ib_port_unregister_module_stat(&port->counter_group[i].obj);

	return ret;



}
@@ -4334,9 +4314,8 @@ static void cm_remove_port_fs(struct cm_port *port) 
	int i;



	for (i = 0; i < CM_COUNTER_GROUPS; i++)

		kobject_put(&port->counter_group[i].obj);

		ib_port_unregister_module_stat(&port->counter_group[i].obj);



	kobject_put(&port->port_obj);

}



static void cm_add_one(struct ib_device *ib_device)
@@ -4363,13 +4342,6 @@ static void cm_add_one(struct ib_device *ib_device) 
	cm_dev->ib_device = ib_device;

	cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;

	cm_dev->going_down = 0;

	cm_dev->device = device_create(&cm_class, &ib_device->dev,

				       MKDEV(0, 0), NULL,

				       "%s", dev_name(&ib_device->dev));

	if (IS_ERR(cm_dev->device)) {

		kfree(cm_dev);

		return;

	}



	set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);

	for (i = 1; i <= ib_device->phys_port_cnt; i++) {
@@ -4436,7 +4408,6 @@ error1: 
		cm_remove_port_fs(port);

	}

free:

	device_unregister(cm_dev->device);

	kfree(cm_dev);

}
@@ -4490,7 +4461,6 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data) 
		cm_remove_port_fs(port);

	}



	device_unregister(cm_dev->device);

	kfree(cm_dev);

}

drivers/infiniband/core/core_priv.h

+6 −0
Original line number Diff line number Diff line
@@ -344,4 +344,10 @@ int ib_setup_port_attrs(struct ib_core_device *coredev, 
			bool alloc_hw_stats);



int rdma_compatdev_set(u8 enable);



int ib_port_register_module_stat(struct ib_device *device, u8 port_num,

				 struct kobject *kobj, struct kobj_type *ktype,

				 const char *name);

void ib_port_unregister_module_stat(struct kobject *kobj);



#endif /* _CORE_PRIV_H */

drivers/infiniband/core/sysfs.c

+43 −0
Original line number Diff line number Diff line
@@ -1354,3 +1354,46 @@ void ib_device_unregister_sysfs(struct ib_device *device) 


	ib_free_port_attrs(&device->coredev);

}



/**

 * ib_port_register_module_stat - add module counters under relevant port

 *  of IB device.

 *

 * @device: IB device to add counters

 * @port_num: valid port number

 * @kobj: pointer to the kobject to initialize

 * @ktype: pointer to the ktype for this kobject.

 * @name: the name of the kobject

 */

int ib_port_register_module_stat(struct ib_device *device, u8 port_num,

				 struct kobject *kobj, struct kobj_type *ktype,

				 const char *name)

{

	struct kobject *p, *t;

	int ret;



	list_for_each_entry_safe(p, t, &device->coredev.port_list, entry) {

		struct ib_port *port = container_of(p, struct ib_port, kobj);



		if (port->port_num != port_num)

			continue;



		ret = kobject_init_and_add(kobj, ktype, &port->kobj, "%s",

					   name);

		if (ret)

			return ret;

	}



	return 0;

}

EXPORT_SYMBOL(ib_port_register_module_stat);



/**

 * ib_port_unregister_module_stat - release module counters

 * @kobj: pointer to the kobject to release

 */

void ib_port_unregister_module_stat(struct kobject *kobj)

{

	kobject_put(kobj);

}

EXPORT_SYMBOL(ib_port_unregister_module_stat);

CRA Git | Maintained and supported by SUSTech CRA and CCSE