Commit 753f618a authored by Nir Muchtar's avatar Nir Muchtar Committed by Roland Dreier
Browse files

RDMA/cma: Add support for netlink statistics export 



Add callbacks and data types for statistics export of all current
devices/ids.  The schema for RDMA CM is a series of netlink messages.
Each one contains an rdma_cm_stat struct.  Additionally, two netlink
attributes are created for the addresses for each message (if
applicable).

Their types used are:
RDMA_NL_RDMA_CM_ATTR_SRC_ADDR (The source address for this ID)
RDMA_NL_RDMA_CM_ATTR_DST_ADDR (The destination address for this ID)
sockaddr_* structs are encapsulated within these attributes.

In other words, every transaction contains a series of messages like:

-------message 1-------
struct rdma_cm_id_stats {
       __u32 qp_num;
       __u32 bound_dev_if;
       __u32 port_space;
       __s32 pid;
       __u8 cm_state;
       __u8 node_type;
       __u8 port_num;
       __u8 reserved;
}
RDMA_NL_RDMA_CM_ATTR_SRC_ADDR attribute - contains the source address
RDMA_NL_RDMA_CM_ATTR_DST_ADDR attribute - contains the destination address
-------end 1-------
-------message 2-------
struct rdma_cm_id_stats
RDMA_NL_RDMA_CM_ATTR_SRC_ADDR attribute
RDMA_NL_RDMA_CM_ATTR_DST_ADDR attribute
-------end 2-------

Signed-off-by: default avatarNir Muchtar <nirm@voltaire.com>
Signed-off-by: default avatarRoland Dreier <roland@purestorage.com>
parent b26f9b99

drivers/infiniband/core/cma.c

+99 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ 


#include <rdma/rdma_cm.h>

#include <rdma/rdma_cm_ib.h>

#include <rdma/rdma_netlink.h>

#include <rdma/ib_cache.h>

#include <rdma/ib_cm.h>

#include <rdma/ib_sa.h>
@@ -3315,6 +3316,99 @@ static void cma_remove_one(struct ib_device *device) 
	kfree(cma_dev);

}



static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)

{

	struct nlmsghdr *nlh;

	struct rdma_cm_id_stats *id_stats;

	struct rdma_id_private *id_priv;

	struct rdma_cm_id *id = NULL;

	struct cma_device *cma_dev;

	int i_dev = 0, i_id = 0;



	/*

	 * We export all of the IDs as a sequence of messages.  Each

	 * ID gets its own netlink message.

	 */

	mutex_lock(&lock);



	list_for_each_entry(cma_dev, &dev_list, list) {

		if (i_dev < cb->args[0]) {

			i_dev++;

			continue;

		}



		i_id = 0;

		list_for_each_entry(id_priv, &cma_dev->id_list, list) {

			if (i_id < cb->args[1]) {

				i_id++;

				continue;

			}



			id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,

						sizeof *id_stats, RDMA_NL_RDMA_CM,

						RDMA_NL_RDMA_CM_ID_STATS);

			if (!id_stats)

				goto out;



			memset(id_stats, 0, sizeof *id_stats);

			id = &id_priv->id;

			id_stats->node_type = id->route.addr.dev_addr.dev_type;

			id_stats->port_num = id->port_num;

			id_stats->bound_dev_if =

				id->route.addr.dev_addr.bound_dev_if;



			if (id->route.addr.src_addr.ss_family == AF_INET) {

				if (ibnl_put_attr(skb, nlh,

						  sizeof(struct sockaddr_in),

						  &id->route.addr.src_addr,

						  RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {

					goto out;

				}

				if (ibnl_put_attr(skb, nlh,

						  sizeof(struct sockaddr_in),

						  &id->route.addr.dst_addr,

						  RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {

					goto out;

				}

			} else if (id->route.addr.src_addr.ss_family == AF_INET6) {

				if (ibnl_put_attr(skb, nlh,

						  sizeof(struct sockaddr_in6),

						  &id->route.addr.src_addr,

						  RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {

					goto out;

				}

				if (ibnl_put_attr(skb, nlh,

						  sizeof(struct sockaddr_in6),

						  &id->route.addr.dst_addr,

						  RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {

					goto out;

				}

			}



			id_stats->port_space	= id->ps;

			id_stats->cm_state	= id_priv->state;

			id_stats->qp_num	= id_priv->qp_num;

			id_stats->qp_type	= id->qp_type;



			i_id++;

		}



		cb->args[1] = 0;

		i_dev++;

	}



out:

	mutex_unlock(&lock);

	cb->args[0] = i_dev;

	cb->args[1] = i_id;



	return skb->len;

}



static const struct ibnl_client_cbs cma_cb_table[] = {

	[RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats },

};



static int __init cma_init(void)

{

	int ret;
@@ -3330,6 +3424,10 @@ static int __init cma_init(void) 
	ret = ib_register_client(&cma_client);

	if (ret)

		goto err;



	if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))

		printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");



	return 0;



err:
@@ -3342,6 +3440,7 @@ err: 


static void __exit cma_cleanup(void)

{

	ibnl_remove_client(RDMA_NL_RDMA_CM);

	ib_unregister_client(&cma_client);

	unregister_netdevice_notifier(&cma_nb);

	rdma_addr_unregister_client(&addr_client);

include/rdma/rdma_netlink.h

+28 −0
Original line number Diff line number Diff line 
#ifndef _RDMA_NETLINK_H

#define _RDMA_NETLINK_H



#include <linux/types.h>



enum {

	RDMA_NL_RDMA_CM = 1

};



#define RDMA_NL_GET_CLIENT(type) ((type & (((1 << 6) - 1) << 10)) >> 10)

#define RDMA_NL_GET_OP(type) (type & ((1 << 10) - 1))

#define RDMA_NL_GET_TYPE(client, op) ((client << 10) + op)



enum {

	RDMA_NL_RDMA_CM_ID_STATS = 0,

	RDMA_NL_RDMA_CM_NUM_OPS

};



enum {

	RDMA_NL_RDMA_CM_ATTR_SRC_ADDR = 1,

	RDMA_NL_RDMA_CM_ATTR_DST_ADDR,

	RDMA_NL_RDMA_CM_NUM_ATTR,

};



struct rdma_cm_id_stats {

	__u32	qp_num;

	__u32	bound_dev_if;

	__u32	port_space;

	__s32	pid;

	__u8	cm_state;

	__u8	node_type;

	__u8	port_num;

	__u8	qp_type;

};



#ifdef __KERNEL__



#include <linux/netlink.h>

CRA Git | Maintained and supported by SUSTech CRA and CCSE