Commit 8cf12d77 authored by Huy Nguyen's avatar Huy Nguyen Committed by Jason Gunthorpe
Browse files

IB/core: Increase number of char device minors 



There is a need to increase number of possible char devices to support
large number of SR-IOV instances. The current limit is in the range of
64-128 devices/ports. Increase it to support up to 1024.

The patch performs the following steps to refactor the code:
1. Removes the split bitmap for fixed and overflow dev numbers.
2. Pre-allocates the non-legacy major number range during driver
   initialization, choosen for simplicity.
3. Add new define (RDMA_MAX_PORTS) that is shared between all drivers.
   This is the maximum total number of ports on all struct ib_devices.
4. Set RDMA_MAX_PORTS to 1024.

Signed-off-by: default avatarHuy Nguyen <huyn@mellanox.com>
Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent 008b656f

drivers/infiniband/core/core_priv.h

+3 −0
Original line number Diff line number Diff line
@@ -42,6 +42,9 @@ 
#include <rdma/ib_mad.h>

#include "mad_priv.h"



/* Total number of ports combined across all struct ib_devices's */

#define RDMA_MAX_PORTS 1024



struct pkey_index_qp_list {

	struct list_head    pkey_index_list;

	u16                 pkey_index;

drivers/infiniband/core/ucm.c

+28 −45
Original line number Diff line number Diff line
@@ -53,6 +53,8 @@ 
#include <rdma/ib_user_cm.h>

#include <rdma/ib_marshall.h>



#include "core_priv.h"



MODULE_AUTHOR("Libor Michalek");

MODULE_DESCRIPTION("InfiniBand userspace Connection Manager access");

MODULE_LICENSE("Dual BSD/GPL");
@@ -104,10 +106,13 @@ struct ib_ucm_event { 
enum {

	IB_UCM_MAJOR = 231,

	IB_UCM_BASE_MINOR = 224,

	IB_UCM_MAX_DEVICES = 32

	IB_UCM_MAX_DEVICES = RDMA_MAX_PORTS,

	IB_UCM_NUM_FIXED_MINOR = 32,

	IB_UCM_NUM_DYNAMIC_MINOR = IB_UCM_MAX_DEVICES - IB_UCM_NUM_FIXED_MINOR,

};



#define IB_UCM_BASE_DEV MKDEV(IB_UCM_MAJOR, IB_UCM_BASE_MINOR)

static dev_t dynamic_ucm_dev;



static void ib_ucm_add_one(struct ib_device *device);

static void ib_ucm_remove_one(struct ib_device *device, void *client_data);
@@ -1199,7 +1204,6 @@ static int ib_ucm_close(struct inode *inode, struct file *filp) 
	return 0;

}



static DECLARE_BITMAP(overflow_map, IB_UCM_MAX_DEVICES);

static void ib_ucm_release_dev(struct device *dev)

{

	struct ib_ucm_device *ucm_dev;
@@ -1210,10 +1214,7 @@ static void ib_ucm_release_dev(struct device *dev) 


static void ib_ucm_free_dev(struct ib_ucm_device *ucm_dev)

{

	if (ucm_dev->devnum < IB_UCM_MAX_DEVICES)

	clear_bit(ucm_dev->devnum, dev_map);

	else

		clear_bit(ucm_dev->devnum - IB_UCM_MAX_DEVICES, overflow_map);

}



static const struct file_operations ucm_fops = {
@@ -1235,27 +1236,6 @@ static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr, 
}

static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);



static dev_t overflow_maj;

static int find_overflow_devnum(void)

{

	int ret;



	if (!overflow_maj) {

		ret = alloc_chrdev_region(&overflow_maj, 0, IB_UCM_MAX_DEVICES,

					  "infiniband_cm");

		if (ret) {

			pr_err("ucm: couldn't register dynamic device number\n");

			return ret;

		}

	}



	ret = find_first_zero_bit(overflow_map, IB_UCM_MAX_DEVICES);

	if (ret >= IB_UCM_MAX_DEVICES)

		return -1;



	return ret;

}



static void ib_ucm_add_one(struct ib_device *device)

{

	int devnum;
@@ -1274,19 +1254,14 @@ static void ib_ucm_add_one(struct ib_device *device) 
	ucm_dev->dev.release = ib_ucm_release_dev;



	devnum = find_first_zero_bit(dev_map, IB_UCM_MAX_DEVICES);

	if (devnum >= IB_UCM_MAX_DEVICES) {

		devnum = find_overflow_devnum();

		if (devnum < 0)

	if (devnum >= IB_UCM_MAX_DEVICES)

		goto err;



		ucm_dev->devnum = devnum + IB_UCM_MAX_DEVICES;

		base = devnum + overflow_maj;

		set_bit(devnum, overflow_map);

	} else {

	ucm_dev->devnum = devnum;

		base = devnum + IB_UCM_BASE_DEV;

	set_bit(devnum, dev_map);

	}

	if (devnum >= IB_UCM_NUM_FIXED_MINOR)

		base = dynamic_ucm_dev + devnum - IB_UCM_NUM_FIXED_MINOR;

	else

		base = IB_UCM_BASE_DEV + devnum;



	cdev_init(&ucm_dev->cdev, &ucm_fops);

	ucm_dev->cdev.owner = THIS_MODULE;
@@ -1334,13 +1309,20 @@ static int __init ib_ucm_init(void) 
{

	int ret;



	ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES,

	ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR,

				     "infiniband_cm");

	if (ret) {

		pr_err("ucm: couldn't register device number\n");

		goto error1;

	}



	ret = alloc_chrdev_region(&dynamic_ucm_dev, 0, IB_UCM_NUM_DYNAMIC_MINOR,

				  "infiniband_cm");

	if (ret) {

		pr_err("ucm: couldn't register dynamic device number\n");

		goto err_alloc;

	}



	ret = class_create_file(&cm_class, &class_attr_abi_version.attr);

	if (ret) {

		pr_err("ucm: couldn't create abi_version attribute\n");
@@ -1357,7 +1339,9 @@ static int __init ib_ucm_init(void) 
error3:

	class_remove_file(&cm_class, &class_attr_abi_version.attr);

error2:

	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES);

	unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);

err_alloc:

	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);

error1:

	return ret;

}
@@ -1366,9 +1350,8 @@ static void __exit ib_ucm_cleanup(void) 
{

	ib_unregister_client(&ucm_client);

	class_remove_file(&cm_class, &class_attr_abi_version.attr);

	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES);

	if (overflow_maj)

		unregister_chrdev_region(overflow_maj, IB_UCM_MAX_DEVICES);

	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);

	unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);

	idr_destroy(&ctx_id_table);

}

drivers/infiniband/core/user_mad.c

+48 −55
Original line number Diff line number Diff line
@@ -55,16 +55,21 @@ 
#include <rdma/ib_mad.h>

#include <rdma/ib_user_mad.h>



#include "core_priv.h"



MODULE_AUTHOR("Roland Dreier");

MODULE_DESCRIPTION("InfiniBand userspace MAD packet access");

MODULE_LICENSE("Dual BSD/GPL");



enum {

	IB_UMAD_MAX_PORTS  = 64,

	IB_UMAD_MAX_PORTS  = RDMA_MAX_PORTS,

	IB_UMAD_MAX_AGENTS = 32,



	IB_UMAD_MAJOR      = 231,

	IB_UMAD_MINOR_BASE = 0

	IB_UMAD_MINOR_BASE = 0,

	IB_UMAD_NUM_FIXED_MINOR = 64,

	IB_UMAD_NUM_DYNAMIC_MINOR = IB_UMAD_MAX_PORTS - IB_UMAD_NUM_FIXED_MINOR,

	IB_ISSM_MINOR_BASE        = IB_UMAD_NUM_FIXED_MINOR,

};



/*
@@ -127,7 +132,11 @@ struct ib_umad_packet { 


static struct class *umad_class;



static const dev_t base_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE);

static const dev_t base_umad_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE);

static const dev_t base_issm_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE) +

				   IB_UMAD_NUM_FIXED_MINOR;

static dev_t dynamic_umad_dev;

static dev_t dynamic_issm_dev;



static DECLARE_BITMAP(dev_map, IB_UMAD_MAX_PORTS);
@@ -1141,49 +1150,25 @@ static DEVICE_ATTR(port, S_IRUGO, show_port, NULL); 
static CLASS_ATTR_STRING(abi_version, S_IRUGO,

			 __stringify(IB_USER_MAD_ABI_VERSION));



static dev_t overflow_maj;

static DECLARE_BITMAP(overflow_map, IB_UMAD_MAX_PORTS);

static int find_overflow_devnum(struct ib_device *device)

{

	int ret;



	if (!overflow_maj) {

		ret = alloc_chrdev_region(&overflow_maj, 0, IB_UMAD_MAX_PORTS * 2,

					  "infiniband_mad");

		if (ret) {

			dev_err(&device->dev,

				"couldn't register dynamic device number\n");

			return ret;

		}

	}



	ret = find_first_zero_bit(overflow_map, IB_UMAD_MAX_PORTS);

	if (ret >= IB_UMAD_MAX_PORTS)

		return -1;



	return ret;

}



static int ib_umad_init_port(struct ib_device *device, int port_num,

			     struct ib_umad_device *umad_dev,

			     struct ib_umad_port *port)

{

	int devnum;

	dev_t base;

	dev_t base_umad;

	dev_t base_issm;



	devnum = find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);

	if (devnum >= IB_UMAD_MAX_PORTS) {

		devnum = find_overflow_devnum(device);

		if (devnum < 0)

	if (devnum >= IB_UMAD_MAX_PORTS)

		return -1;



		port->dev_num = devnum + IB_UMAD_MAX_PORTS;

		base = devnum + overflow_maj;

		set_bit(devnum, overflow_map);

	} else {

	port->dev_num = devnum;

		base = devnum + base_dev;

	set_bit(devnum, dev_map);

	if (devnum >= IB_UMAD_NUM_FIXED_MINOR) {

		base_umad = dynamic_umad_dev + devnum - IB_UMAD_NUM_FIXED_MINOR;

		base_issm = dynamic_issm_dev + devnum - IB_UMAD_NUM_FIXED_MINOR;

	} else {

		base_umad = devnum + base_umad_dev;

		base_issm = devnum + base_issm_dev;

	}



	port->ib_dev   = device;
@@ -1196,7 +1181,7 @@ static int ib_umad_init_port(struct ib_device *device, int port_num, 
	port->cdev.owner = THIS_MODULE;

	cdev_set_parent(&port->cdev, &umad_dev->kobj);

	kobject_set_name(&port->cdev.kobj, "umad%d", port->dev_num);

	if (cdev_add(&port->cdev, base, 1))

	if (cdev_add(&port->cdev, base_umad, 1))

		goto err_cdev;



	port->dev = device_create(umad_class, device->dev.parent,
@@ -1210,12 +1195,11 @@ static int ib_umad_init_port(struct ib_device *device, int port_num, 
	if (device_create_file(port->dev, &dev_attr_port))

		goto err_dev;



	base += IB_UMAD_MAX_PORTS;

	cdev_init(&port->sm_cdev, &umad_sm_fops);

	port->sm_cdev.owner = THIS_MODULE;

	cdev_set_parent(&port->sm_cdev, &umad_dev->kobj);

	kobject_set_name(&port->sm_cdev.kobj, "issm%d", port->dev_num);

	if (cdev_add(&port->sm_cdev, base, 1))

	if (cdev_add(&port->sm_cdev, base_issm, 1))

		goto err_sm_cdev;



	port->sm_dev = device_create(umad_class, device->dev.parent,
@@ -1242,10 +1226,7 @@ err_dev: 


err_cdev:

	cdev_del(&port->cdev);

	if (port->dev_num < IB_UMAD_MAX_PORTS)

	clear_bit(devnum, dev_map);

	else

		clear_bit(devnum, overflow_map);



	return -1;

}
@@ -1279,11 +1260,7 @@ static void ib_umad_kill_port(struct ib_umad_port *port) 
	}



	mutex_unlock(&port->file_mutex);



	if (port->dev_num < IB_UMAD_MAX_PORTS)

	clear_bit(port->dev_num, dev_map);

	else

		clear_bit(port->dev_num - IB_UMAD_MAX_PORTS, overflow_map);

}



static void ib_umad_add_one(struct ib_device *device)
@@ -1359,13 +1336,23 @@ static int __init ib_umad_init(void) 
{

	int ret;



	ret = register_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2,

	ret = register_chrdev_region(base_umad_dev,

				     IB_UMAD_NUM_FIXED_MINOR * 2,

				     "infiniband_mad");

	if (ret) {

		pr_err("couldn't register device number\n");

		goto out;

	}



	ret = alloc_chrdev_region(&dynamic_umad_dev, 0,

				  IB_UMAD_NUM_DYNAMIC_MINOR * 2,

				  "infiniband_mad");

	if (ret) {

		pr_err("couldn't register dynamic device number\n");

		goto out_alloc;

	}

	dynamic_issm_dev = dynamic_umad_dev + IB_UMAD_NUM_DYNAMIC_MINOR;



	umad_class = class_create(THIS_MODULE, "infiniband_mad");

	if (IS_ERR(umad_class)) {

		ret = PTR_ERR(umad_class);
@@ -1393,7 +1380,12 @@ out_class: 
	class_destroy(umad_class);



out_chrdev:

	unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2);

	unregister_chrdev_region(dynamic_umad_dev,

				 IB_UMAD_NUM_DYNAMIC_MINOR * 2);



out_alloc:

	unregister_chrdev_region(base_umad_dev,

				 IB_UMAD_NUM_FIXED_MINOR * 2);



out:

	return ret;
@@ -1403,9 +1395,10 @@ static void __exit ib_umad_cleanup(void) 
{

	ib_unregister_client(&umad_client);

	class_destroy(umad_class);

	unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2);

	if (overflow_maj)

		unregister_chrdev_region(overflow_maj, IB_UMAD_MAX_PORTS * 2);

	unregister_chrdev_region(base_umad_dev,

				 IB_UMAD_NUM_FIXED_MINOR * 2);

	unregister_chrdev_region(dynamic_umad_dev,

				 IB_UMAD_NUM_DYNAMIC_MINOR * 2);

}



module_init(ib_umad_init);

drivers/infiniband/core/uverbs_main.c

+34 −56
Original line number Diff line number Diff line
@@ -62,11 +62,14 @@ MODULE_LICENSE("Dual BSD/GPL"); 
enum {

	IB_UVERBS_MAJOR       = 231,

	IB_UVERBS_BASE_MINOR  = 192,

	IB_UVERBS_MAX_DEVICES = 32

	IB_UVERBS_MAX_DEVICES = RDMA_MAX_PORTS,

	IB_UVERBS_NUM_FIXED_MINOR = 32,

	IB_UVERBS_NUM_DYNAMIC_MINOR = IB_UVERBS_MAX_DEVICES - IB_UVERBS_NUM_FIXED_MINOR,

};



#define IB_UVERBS_BASE_DEV	MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)



static dev_t dynamic_uverbs_dev;

static struct class *uverbs_class;



static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES);
@@ -1004,34 +1007,6 @@ static DEVICE_ATTR(abi_version, S_IRUGO, show_dev_abi_version, NULL); 
static CLASS_ATTR_STRING(abi_version, S_IRUGO,

			 __stringify(IB_USER_VERBS_ABI_VERSION));



static dev_t overflow_maj;

static DECLARE_BITMAP(overflow_map, IB_UVERBS_MAX_DEVICES);



/*

 * If we have more than IB_UVERBS_MAX_DEVICES, dynamically overflow by

 * requesting a new major number and doubling the number of max devices we

 * support. It's stupid, but simple.

 */

static int find_overflow_devnum(void)

{

	int ret;



	if (!overflow_maj) {

		ret = alloc_chrdev_region(&overflow_maj, 0, IB_UVERBS_MAX_DEVICES,

					  "infiniband_verbs");

		if (ret) {

			pr_err("user_verbs: couldn't register dynamic device number\n");

			return ret;

		}

	}



	ret = find_first_zero_bit(overflow_map, IB_UVERBS_MAX_DEVICES);

	if (ret >= IB_UVERBS_MAX_DEVICES)

		return -1;



	return ret;

}



static void ib_uverbs_add_one(struct ib_device *device)

{

	int devnum;
@@ -1062,19 +1037,14 @@ static void ib_uverbs_add_one(struct ib_device *device) 
	INIT_LIST_HEAD(&uverbs_dev->uverbs_events_file_list);



	devnum = find_first_zero_bit(dev_map, IB_UVERBS_MAX_DEVICES);

	if (devnum >= IB_UVERBS_MAX_DEVICES) {

		devnum = find_overflow_devnum();

		if (devnum < 0)

	if (devnum >= IB_UVERBS_MAX_DEVICES)

		goto err;



		uverbs_dev->devnum = devnum + IB_UVERBS_MAX_DEVICES;

		base = devnum + overflow_maj;

		set_bit(devnum, overflow_map);

	} else {

	uverbs_dev->devnum = devnum;

		base = devnum + IB_UVERBS_BASE_DEV;

	set_bit(devnum, dev_map);

	}

	if (devnum >= IB_UVERBS_NUM_FIXED_MINOR)

		base = dynamic_uverbs_dev + devnum - IB_UVERBS_NUM_FIXED_MINOR;

	else

		base = IB_UVERBS_BASE_DEV + devnum;



	rcu_assign_pointer(uverbs_dev->ib_dev, device);

	uverbs_dev->num_comp_vectors = device->num_comp_vectors;
@@ -1119,10 +1089,7 @@ err_class: 


err_cdev:

	cdev_del(&uverbs_dev->cdev);

	if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)

	clear_bit(devnum, dev_map);

	else

		clear_bit(devnum, overflow_map);



err:

	if (atomic_dec_and_test(&uverbs_dev->refcount))
@@ -1214,11 +1181,7 @@ static void ib_uverbs_remove_one(struct ib_device *device, void *client_data) 
	dev_set_drvdata(uverbs_dev->dev, NULL);

	device_destroy(uverbs_class, uverbs_dev->cdev.dev);

	cdev_del(&uverbs_dev->cdev);



	if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)

	clear_bit(uverbs_dev->devnum, dev_map);

	else

		clear_bit(uverbs_dev->devnum - IB_UVERBS_MAX_DEVICES, overflow_map);



	if (device->disassociate_ucontext) {

		/* We disassociate HW resources and immediately return.
@@ -1260,13 +1223,22 @@ static int __init ib_uverbs_init(void) 
{

	int ret;



	ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES,

	ret = register_chrdev_region(IB_UVERBS_BASE_DEV,

				     IB_UVERBS_NUM_FIXED_MINOR,

				     "infiniband_verbs");

	if (ret) {

		pr_err("user_verbs: couldn't register device number\n");

		goto out;

	}



	ret = alloc_chrdev_region(&dynamic_uverbs_dev, 0,

				  IB_UVERBS_NUM_DYNAMIC_MINOR,

				  "infiniband_verbs");

	if (ret) {

		pr_err("couldn't register dynamic device number\n");

		goto out_alloc;

	}



	uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");

	if (IS_ERR(uverbs_class)) {

		ret = PTR_ERR(uverbs_class);
@@ -1294,7 +1266,12 @@ out_class: 
	class_destroy(uverbs_class);



out_chrdev:

	unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);

	unregister_chrdev_region(dynamic_uverbs_dev,

				 IB_UVERBS_NUM_DYNAMIC_MINOR);



out_alloc:

	unregister_chrdev_region(IB_UVERBS_BASE_DEV,

				 IB_UVERBS_NUM_FIXED_MINOR);



out:

	return ret;
@@ -1304,9 +1281,10 @@ static void __exit ib_uverbs_cleanup(void) 
{

	ib_unregister_client(&uverbs_client);

	class_destroy(uverbs_class);

	unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);

	if (overflow_maj)

		unregister_chrdev_region(overflow_maj, IB_UVERBS_MAX_DEVICES);

	unregister_chrdev_region(IB_UVERBS_BASE_DEV,

				 IB_UVERBS_NUM_FIXED_MINOR);

	unregister_chrdev_region(dynamic_uverbs_dev,

				 IB_UVERBS_NUM_DYNAMIC_MINOR);

}



module_init(ib_uverbs_init);

CRA Git | Maintained and supported by SUSTech CRA and CCSE