Commit cebe556b authored by Parav Pandit's avatar Parav Pandit Committed by Jason Gunthorpe
Browse files

RDMA/core: Introduce ib_core_device to hold device 



In order to support sysfs entries in multiple net namespaces for a rdma
device, introduce a ib_core_device whose scope is limited to hold core
device and per port sysfs related entries.

This is preparation patch so that multiple ib_core_devices in each net
namespace can be created in subsequent patch who all can share ib_device.

(a) Move sysfs specific fields to ib_core_device.
(b) Make sysfs and device life cycle related routines to work on
    ib_core_device.
(c) Introduce and use rdma_init_coredev() helper to initialize
    coredev fields.

Signed-off-by: default avatarParav Pandit <parav@mellanox.com>
Signed-off-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent 629e6f9d

drivers/infiniband/core/device.c

+20 −4
Original line number Original line Diff line number Diff line
@@ -363,6 +363,25 @@ static struct class ib_class = { 
	.dev_uevent = ib_device_uevent,

	.dev_uevent = ib_device_uevent,

};

};





static void rdma_init_coredev(struct ib_core_device *coredev,

			      struct ib_device *dev)

{

	/* This BUILD_BUG_ON is intended to catch layout change

	 * of union of ib_core_device and device.

	 * dev must be the first element as ib_core and providers

	 * driver uses it. Adding anything in ib_core_device before

	 * device will break this assumption.

	 */

	BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) !=

		     offsetof(struct ib_device, dev));



	coredev->dev.class = &ib_class;

	coredev->dev.groups = dev->groups;

	device_initialize(&coredev->dev);

	coredev->owner = dev;

	INIT_LIST_HEAD(&coredev->port_list);

}



/**

/**

 * _ib_alloc_device - allocate an IB device struct

 * _ib_alloc_device - allocate an IB device struct

 * @size:size of structure to allocate

 * @size:size of structure to allocate
@@ -389,10 +408,8 @@ struct ib_device *_ib_alloc_device(size_t size) 
		return NULL;

		return NULL;

	}

	}





	device->dev.class = &ib_class;

	device->groups[0] = &ib_dev_attr_group;

	device->groups[0] = &ib_dev_attr_group;

	device->dev.groups = device->groups;

	rdma_init_coredev(&device->coredev, device);

	device_initialize(&device->dev);





	INIT_LIST_HEAD(&device->event_handler_list);

	INIT_LIST_HEAD(&device->event_handler_list);

	spin_lock_init(&device->event_handler_lock);

	spin_lock_init(&device->event_handler_lock);
@@ -403,7 +420,6 @@ struct ib_device *_ib_alloc_device(size_t size) 
	 */

	 */

	xa_init_flags(&device->client_data, XA_FLAGS_ALLOC);

	xa_init_flags(&device->client_data, XA_FLAGS_ALLOC);

	init_rwsem(&device->client_data_rwsem);

	init_rwsem(&device->client_data_rwsem);

	INIT_LIST_HEAD(&device->port_list);

	init_completion(&device->unreg_completion);

	init_completion(&device->unreg_completion);

	INIT_WORK(&device->unregistration_work, ib_unregister_work);

	INIT_WORK(&device->unregistration_work, ib_unregister_work);

drivers/infiniband/core/sysfs.c

+16 −13
Original line number Original line Diff line number Diff line
@@ -1015,8 +1015,9 @@ err_free_stats: 
	return;

	return;

}

}





static int add_port(struct ib_device *device, int port_num)

static int add_port(struct ib_core_device *coredev, int port_num)

{

{

	struct ib_device *device = rdma_device_to_ibdev(&coredev->dev);

	struct ib_port *p;

	struct ib_port *p;

	struct ib_port_attr attr;

	struct ib_port_attr attr;

	int i;

	int i;
@@ -1034,7 +1035,7 @@ static int add_port(struct ib_device *device, int port_num) 
	p->port_num   = port_num;

	p->port_num   = port_num;





	ret = kobject_init_and_add(&p->kobj, &port_type,

	ret = kobject_init_and_add(&p->kobj, &port_type,

				   device->ports_kobj,

				   coredev->ports_kobj,

				   "%d", port_num);

				   "%d", port_num);

	if (ret) {

	if (ret) {

		kfree(p);

		kfree(p);
@@ -1125,7 +1126,7 @@ static int add_port(struct ib_device *device, int port_num) 
	if (device->ops.alloc_hw_stats && port_num)

	if (device->ops.alloc_hw_stats && port_num)

		setup_hw_stats(device, p, port_num);

		setup_hw_stats(device, p, port_num);





	list_add_tail(&p->kobj.entry, &device->port_list);

	list_add_tail(&p->kobj.entry, &coredev->port_list);





	kobject_uevent(&p->kobj, KOBJ_ADD);

	kobject_uevent(&p->kobj, KOBJ_ADD);

	return 0;

	return 0;
@@ -1279,11 +1280,11 @@ const struct attribute_group ib_dev_attr_group = { 
	.attrs = ib_dev_attrs,

	.attrs = ib_dev_attrs,

};

};





static void ib_free_port_attrs(struct ib_device *device)

static void ib_free_port_attrs(struct ib_core_device *coredev)

{

{

	struct kobject *p, *t;

	struct kobject *p, *t;





	list_for_each_entry_safe(p, t, &device->port_list, entry) {

	list_for_each_entry_safe(p, t, &coredev->port_list, entry) {

		struct ib_port *port = container_of(p, struct ib_port, kobj);

		struct ib_port *port = container_of(p, struct ib_port, kobj);





		list_del(&p->entry);

		list_del(&p->entry);
@@ -1303,20 +1304,22 @@ static void ib_free_port_attrs(struct ib_device *device) 
		kobject_put(p);

		kobject_put(p);

	}

	}





	kobject_put(device->ports_kobj);

	kobject_put(coredev->ports_kobj);

}

}





static int ib_setup_port_attrs(struct ib_device *device)

static int ib_setup_port_attrs(struct ib_core_device *coredev)

{

{

	struct ib_device *device = rdma_device_to_ibdev(&coredev->dev);

	unsigned int port;

	unsigned int port;

	int ret;

	int ret;





	device->ports_kobj = kobject_create_and_add("ports", &device->dev.kobj);

	coredev->ports_kobj = kobject_create_and_add("ports",

	if (!device->ports_kobj)

						     &coredev->dev.kobj);

	if (!coredev->ports_kobj)

		return -ENOMEM;

		return -ENOMEM;





	rdma_for_each_port (device, port) {

	rdma_for_each_port (device, port) {

		ret = add_port(device, port);

		ret = add_port(coredev, port);

		if (ret)

		if (ret)

			goto err_put;

			goto err_put;

	}

	}
@@ -1324,7 +1327,7 @@ static int ib_setup_port_attrs(struct ib_device *device) 
	return 0;

	return 0;





err_put:

err_put:

	ib_free_port_attrs(device);

	ib_free_port_attrs(coredev);

	return ret;

	return ret;

}

}
@@ -1332,7 +1335,7 @@ int ib_device_register_sysfs(struct ib_device *device) 
{

{

	int ret;

	int ret;





	ret = ib_setup_port_attrs(device);

	ret = ib_setup_port_attrs(&device->coredev);

	if (ret)

	if (ret)

		return ret;

		return ret;
@@ -1348,5 +1351,5 @@ void ib_device_unregister_sysfs(struct ib_device *device) 
		free_hsag(&device->dev.kobj, device->hw_stats_ag);

		free_hsag(&device->dev.kobj, device->hw_stats_ag);

	kfree(device->hw_stats);

	kfree(device->hw_stats);





	ib_free_port_attrs(device);

	ib_free_port_attrs(&device->coredev);

}
 
}

include/rdma/ib_verbs.h

+19 −6
Original line number Original line Diff line number Diff line
@@ -2554,8 +2554,17 @@ struct ib_device_ops { 
	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);

	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);

};

};





struct rdma_restrack_root;

struct ib_core_device {

	/* device must be the first element in structure until,

	 * union of ib_core_device and device exists in ib_device.

	 */

	struct device dev;

	struct kobject *ports_kobj;

	struct list_head port_list;

	struct ib_device *owner; /* reach back to owner ib_device */

};





struct rdma_restrack_root;

struct ib_device {

struct ib_device {

	/* Do not access @dma_device directly from ULP nor from HW drivers. */

	/* Do not access @dma_device directly from ULP nor from HW drivers. */

	struct device                *dma_device;

	struct device                *dma_device;
@@ -2581,16 +2590,17 @@ struct ib_device { 
	struct iw_cm_verbs	     *iwcm;

	struct iw_cm_verbs	     *iwcm;





	struct module               *owner;

	struct module               *owner;

	union {

		struct device		dev;

		struct device		dev;

		struct ib_core_device	coredev;

	};



	/* First group for device attributes,

	/* First group for device attributes,

	 * Second group for driver provided attributes (optional).

	 * Second group for driver provided attributes (optional).

	 * It is NULL terminated array.

	 * It is NULL terminated array.

	 */

	 */

	const struct attribute_group	*groups[3];

	const struct attribute_group	*groups[3];





	struct kobject			*ports_kobj;

	struct list_head             port_list;



	int			     uverbs_abi_ver;

	int			     uverbs_abi_ver;

	u64			     uverbs_cmd_mask;

	u64			     uverbs_cmd_mask;

	u64			     uverbs_ex_cmd_mask;

	u64			     uverbs_ex_cmd_mask;
@@ -4349,7 +4359,10 @@ rdma_set_device_sysfs_group(struct ib_device *dev, 
 */

 */

static inline struct ib_device *rdma_device_to_ibdev(struct device *device)

static inline struct ib_device *rdma_device_to_ibdev(struct device *device)

{

{

	return container_of(device, struct ib_device, dev);

	struct ib_core_device *coredev =

		container_of(device, struct ib_core_device, dev);



	return coredev->owner;

}

}





/**

/**

CRA Git | Maintained and supported by SUSTech CRA and CCSE