net/smc: rework pnet table (890a2cb4) · Commits · 戴 / test

net/smc/smc_ib.c

+0 −1

Original line number	Diff line number	Diff line
		@@ -564,7 +564,6 @@ static void smc_ib_remove_dev(struct ib_device ibdev, void client_data)
		spin_lock(&smc_ib_devices.lock);
		list_del_init(&smcibdev->list); /* remove from smc_ib_devices */
		spin_unlock(&smc_ib_devices.lock);
		smc_pnet_remove_by_ibdev(smcibdev);
		smc_ib_cleanup_per_ibdev(smcibdev);
		ib_unregister_event_handler(&smcibdev->event_handler);
		kfree(smcibdev);

net/smc/smc_ib.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -42,6 +42,8 @@ struct smc_ib_device { /* ib-device infos for smc */
		/* mac address per port*/
		u8 pnetid[SMC_MAX_PORTS][SMC_MAX_PNETID_LEN];
		/* pnetid per port */
		bool pnetid_by_user[SMC_MAX_PORTS];
		/* pnetid defined by user? */
		u8 initialized : 1; /* ib dev CQ, evthdl done */
		struct work_struct port_event_work;
		unsigned long port_event_mask;

net/smc/smc_pnet.c

+247 −192

Original line number	Diff line number	Diff line
		@@ -24,6 +24,10 @@
		#include "smc_ib.h"
		#include "smc_ism.h"

		#define SMC_ASCII_BLANK 32

		static struct net_device pnet_find_base_ndev(struct net_device ndev);

		static struct nla_policy smc_pnet_policy[SMC_PNETID_MAX + 1] = {
		[SMC_PNETID_NAME] = {
		.type = NLA_NUL_STRING,
		@@ -56,13 +60,14 @@ static struct smc_pnettable {
		};

		/**
		* struct smc_pnetentry - pnet identifier name entry
		* struct smc_user_pnetentry - pnet identifier name entry for/from user
		* @list: List node.
		* @pnet_name: Pnet identifier name
		* @ndev: pointer to network device.
		* @smcibdev: Pointer to IB device.
		* @ib_port: Port of IB device.
		*/
		struct smc_pnetentry {
		struct smc_user_pnetentry {
		struct list_head list;
		char pnet_name[SMC_MAX_PNETID_LEN + 1];
		struct net_device *ndev;
		@@ -70,33 +75,26 @@ struct smc_pnetentry {
		u8 ib_port;
		};

		/* Check if two RDMA device entries are identical. Use device name and port
		* number for comparison.
		*/
		static bool smc_pnet_same_ibname(struct smc_pnetentry pnetelem, char ibname,
		u8 ibport)
		{
		return pnetelem->ib_port == ibport &&
		!strncmp(pnetelem->smcibdev->ibdev->name, ibname,
		sizeof(pnetelem->smcibdev->ibdev->name));
		}
		/* pnet entry stored in pnet table */
		struct smc_pnetentry {
		struct list_head list;
		char pnet_name[SMC_MAX_PNETID_LEN + 1];
		struct net_device *ndev;
		};

		/* Find a pnetid in the pnet table.
		*/
		static struct smc_pnetentry smc_pnet_find_pnetid(char pnet_name)
		/* Check if two given pnetids match */
		static bool smc_pnet_match(u8 pnetid1, u8 pnetid2)
		{
		struct smc_pnetentry pnetelem, found_pnetelem = NULL;
		int i;

		read_lock(&smc_pnettable.lock);
		list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) {
		if (!strncmp(pnetelem->pnet_name, pnet_name,
		sizeof(pnetelem->pnet_name))) {
		found_pnetelem = pnetelem;
		for (i = 0; i < SMC_MAX_PNETID_LEN; i++) {
		if ((pnetid1[i] == 0 \|\| pnetid1[i] == SMC_ASCII_BLANK) &&
		(pnetid2[i] == 0 \|\| pnetid2[i] == SMC_ASCII_BLANK))
		break;
		if (pnetid1[i] != pnetid2[i])
		return false;
		}
		}
		read_unlock(&smc_pnettable.lock);
		return found_pnetelem;
		return true;
		}

		/* Remove a pnetid from the pnet table.
		@@ -104,21 +102,39 @@ static struct smc_pnetentry smc_pnet_find_pnetid(char pnet_name)
		static int smc_pnet_remove_by_pnetid(char *pnet_name)
		{
		struct smc_pnetentry pnetelem, tmp_pe;
		struct smc_ib_device *ibdev;
		int rc = -ENOENT;
		int ibport;

		/* remove netdevices */
		write_lock(&smc_pnettable.lock);
		list_for_each_entry_safe(pnetelem, tmp_pe, &smc_pnettable.pnetlist,
		list) {
		if (!strncmp(pnetelem->pnet_name, pnet_name,
		sizeof(pnetelem->pnet_name))) {
		if (!pnet_name \|\|
		smc_pnet_match(pnetelem->pnet_name, pnet_name)) {
		list_del(&pnetelem->list);
		dev_put(pnetelem->ndev);
		kfree(pnetelem);
		rc = 0;
		break;
		}
		}
		write_unlock(&smc_pnettable.lock);
		/* remove ib devices */
		spin_lock(&smc_ib_devices.lock);
		list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
		for (ibport = 0; ibport < SMC_MAX_PORTS; ibport++) {
		if (ibdev->pnetid_by_user[ibport] &&
		(!pnet_name \|\|
		smc_pnet_match(pnet_name,
		ibdev->pnetid[ibport]))) {
		memset(ibdev->pnetid[ibport], 0,
		SMC_MAX_PNETID_LEN);
		ibdev->pnetid_by_user[ibport] = false;
		rc = 0;
		}
		}
		}
		spin_unlock(&smc_ib_devices.lock);
		return rc;
		}

		@@ -144,53 +160,66 @@ static int smc_pnet_remove_by_ndev(struct net_device *ndev)
		return rc;
		}

		/* Remove a pnet entry mentioning a given ib device from the pnet table.
		/* Append a pnetid to the end of the pnet table if not already on this list.
		*/
		int smc_pnet_remove_by_ibdev(struct smc_ib_device *ibdev)
		static int smc_pnet_enter(struct smc_user_pnetentry *new_pnetelem)
		{
		struct smc_pnetentry pnetelem, tmp_pe;
		int rc = -ENOENT;
		u8 pnet_null[SMC_MAX_PNETID_LEN] = {0};
		u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
		struct smc_pnetentry *tmp_pnetelem;
		struct smc_pnetentry *pnetelem;
		struct net_device *ndev;
		bool new_netdev = true;
		bool new_ibdev = false;

		write_lock(&smc_pnettable.lock);
		list_for_each_entry_safe(pnetelem, tmp_pe, &smc_pnettable.pnetlist,
		list) {
		if (pnetelem->smcibdev == ibdev) {
		list_del(&pnetelem->list);
		dev_put(pnetelem->ndev);
		kfree(pnetelem);
		rc = 0;
		break;
		}
		if (new_pnetelem->smcibdev) {
		struct smc_ib_device *ib_dev = new_pnetelem->smcibdev;
		int ib_port = new_pnetelem->ib_port;

		spin_lock(&smc_ib_devices.lock);
		if (smc_pnet_match(ib_dev->pnetid[ib_port - 1], pnet_null)) {
		memcpy(ib_dev->pnetid[ib_port - 1],
		new_pnetelem->pnet_name, SMC_MAX_PNETID_LEN);
		ib_dev->pnetid_by_user[ib_port - 1] = true;
		new_ibdev = true;
		}
		write_unlock(&smc_pnettable.lock);
		return rc;
		spin_unlock(&smc_ib_devices.lock);
		}

		/* Append a pnetid to the end of the pnet table if not already on this list.
		if (!new_pnetelem->ndev)
		return new_ibdev ? 0 : -EEXIST;

		/* check if (base) netdev already has a pnetid. If there is one, we do
		* not want to add a pnet table entry
		*/
		static int smc_pnet_enter(struct smc_pnetentry *new_pnetelem)
		{
		struct smc_pnetentry *pnetelem;
		int rc = -EEXIST;
		ndev = pnet_find_base_ndev(new_pnetelem->ndev);
		if (!smc_pnetid_by_dev_port(ndev->dev.parent, ndev->dev_port,
		ndev_pnetid))
		return new_ibdev ? 0 : -EEXIST;

		/* add a new netdev entry to the pnet table if there isn't one */
		tmp_pnetelem = kzalloc(sizeof(*pnetelem), GFP_KERNEL);
		if (!tmp_pnetelem)
		return -ENOMEM;
		memcpy(tmp_pnetelem->pnet_name, new_pnetelem->pnet_name,
		SMC_MAX_PNETID_LEN);
		tmp_pnetelem->ndev = new_pnetelem->ndev;

		write_lock(&smc_pnettable.lock);
		list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) {
		if (!strncmp(pnetelem->pnet_name, new_pnetelem->pnet_name,
		sizeof(new_pnetelem->pnet_name)) \|\|
		!strncmp(pnetelem->ndev->name, new_pnetelem->ndev->name,
		sizeof(new_pnetelem->ndev->name)) \|\|
		smc_pnet_same_ibname(pnetelem,
		new_pnetelem->smcibdev->ibdev->name,
		new_pnetelem->ib_port)) {
		dev_put(pnetelem->ndev);
		goto found;
		if (pnetelem->ndev == new_pnetelem->ndev)
		new_netdev = false;
		}
		}
		list_add_tail(&new_pnetelem->list, &smc_pnettable.pnetlist);
		rc = 0;
		found:
		if (new_netdev) {
		dev_hold(tmp_pnetelem->ndev);
		list_add_tail(&tmp_pnetelem->list, &smc_pnettable.pnetlist);
		write_unlock(&smc_pnettable.lock);
		return rc;
		} else {
		write_unlock(&smc_pnettable.lock);
		kfree(tmp_pnetelem);
		}

		return (new_netdev \|\| new_ibdev) ? 0 : -EEXIST;
		}

		/* The limit for pnetid is 16 characters.
		@@ -241,7 +270,8 @@ out:
		/* Parse the supplied netlink attributes and fill a pnetentry structure.
		* For ethernet and infiniband device names verify that the devices exist.
		*/
		static int smc_pnet_fill_entry(struct net net, struct smc_pnetentry pnetelem,
		static int smc_pnet_fill_entry(struct net *net,
		struct smc_user_pnetentry *pnetelem,
		struct nlattr *tb[])
		{
		char string, ibname;
		@@ -258,30 +288,29 @@ static int smc_pnet_fill_entry(struct net net, struct smc_pnetentry pnetelem,
		goto error;

		rc = -EINVAL;
		if (!tb[SMC_PNETID_ETHNAME])
		goto error;
		rc = -ENOENT;
		if (tb[SMC_PNETID_ETHNAME]) {
		string = (char *)nla_data(tb[SMC_PNETID_ETHNAME]);
		pnetelem->ndev = dev_get_by_name(net, string);
		if (!pnetelem->ndev)
		goto error;
		}

		rc = -EINVAL;
		if (!tb[SMC_PNETID_IBNAME])
		goto error;
		rc = -ENOENT;
		if (tb[SMC_PNETID_IBNAME]) {
		ibname = (char *)nla_data(tb[SMC_PNETID_IBNAME]);
		ibname = strim(ibname);
		pnetelem->smcibdev = smc_pnet_find_ib(ibname);
		if (!pnetelem->smcibdev)
		goto error;

		rc = -EINVAL;
		if (pnetelem->smcibdev) {
		if (!tb[SMC_PNETID_IBPORT])
		goto error;
		pnetelem->ib_port = nla_get_u8(tb[SMC_PNETID_IBPORT]);
		if (pnetelem->ib_port < 1 \|\| pnetelem->ib_port > SMC_MAX_PORTS)
		if (pnetelem->ib_port < 1 \|\|
		pnetelem->ib_port > SMC_MAX_PORTS)
		goto error;
		}
		}

		return 0;

		@@ -292,71 +321,44 @@ error:
		}

		/* Convert an smc_pnetentry to a netlink attribute sequence */
		static int smc_pnet_set_nla(struct sk_buff msg, struct smc_pnetentry pnetelem)
		static int smc_pnet_set_nla(struct sk_buff *msg,
		struct smc_user_pnetentry *pnetelem)
		{
		if (nla_put_string(msg, SMC_PNETID_NAME, pnetelem->pnet_name) \|\|
		nla_put_string(msg, SMC_PNETID_ETHNAME, pnetelem->ndev->name) \|\|
		nla_put_string(msg, SMC_PNETID_IBNAME,
		if (nla_put_string(msg, SMC_PNETID_NAME, pnetelem->pnet_name))
		return -1;
		if (pnetelem->ndev) {
		if (nla_put_string(msg, SMC_PNETID_ETHNAME,
		pnetelem->ndev->name))
		return -1;
		} else {
		if (nla_put_string(msg, SMC_PNETID_ETHNAME, "n/a"))
		return -1;
		}
		if (pnetelem->smcibdev) {
		if (nla_put_string(msg, SMC_PNETID_IBNAME,
		pnetelem->smcibdev->ibdev->name) \|\|
		nla_put_u8(msg, SMC_PNETID_IBPORT, pnetelem->ib_port))
		return -1;
		return 0;
		}

		/* Retrieve one PNETID entry */
		static int smc_pnet_get(struct sk_buff skb, struct genl_info info)
		{
		struct smc_pnetentry *pnetelem;
		struct sk_buff *msg;
		void *hdr;
		int rc;

		if (!info->attrs[SMC_PNETID_NAME])
		return -EINVAL;
		pnetelem = smc_pnet_find_pnetid(
		(char *)nla_data(info->attrs[SMC_PNETID_NAME]));
		if (!pnetelem)
		return -ENOENT;
		msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
		if (!msg)
		return -ENOMEM;

		hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,
		&smc_pnet_nl_family, 0, SMC_PNETID_GET);
		if (!hdr) {
		rc = -EMSGSIZE;
		goto err_out;
		}

		if (smc_pnet_set_nla(msg, pnetelem)) {
		rc = -ENOBUFS;
		goto err_out;
		} else {
		if (nla_put_string(msg, SMC_PNETID_IBNAME, "n/a") \|\|
		nla_put_u8(msg, SMC_PNETID_IBPORT, 0xff))
		return -1;
		}

		genlmsg_end(msg, hdr);
		return genlmsg_reply(msg, info);

		err_out:
		nlmsg_free(msg);
		return rc;
		return 0;
		}

		static int smc_pnet_add(struct sk_buff skb, struct genl_info info)
		{
		struct net *net = genl_info_net(info);
		struct smc_pnetentry *pnetelem;
		struct smc_user_pnetentry pnetelem;
		int rc;

		pnetelem = kzalloc(sizeof(*pnetelem), GFP_KERNEL);
		if (!pnetelem)
		return -ENOMEM;
		rc = smc_pnet_fill_entry(net, pnetelem, info->attrs);
		rc = smc_pnet_fill_entry(net, &pnetelem, info->attrs);
		if (!rc)
		rc = smc_pnet_enter(pnetelem);
		if (rc) {
		kfree(pnetelem);
		return rc;
		}
		rc = smc_pnet_enter(&pnetelem);
		if (pnetelem.ndev)
		dev_put(pnetelem.ndev);
		return rc;
		}

		@@ -376,7 +378,7 @@ static int smc_pnet_dump_start(struct netlink_callback *cb)

		static int smc_pnet_dumpinfo(struct sk_buff *skb,
		u32 portid, u32 seq, u32 flags,
		struct smc_pnetentry *pnetelem)
		struct smc_user_pnetentry *pnetelem)
		{
		void *hdr;

		@@ -392,42 +394,107 @@ static int smc_pnet_dumpinfo(struct sk_buff *skb,
		return 0;
		}

		static int smc_pnet_dump(struct sk_buff skb, struct netlink_callback cb)
		static int _smc_pnet_dump(struct sk_buff skb, u32 portid, u32 seq, u8 pnetid,
		int start_idx)
		{
		struct smc_user_pnetentry tmp_entry;
		struct smc_pnetentry *pnetelem;
		struct smc_ib_device *ibdev;
		int idx = 0;
		int ibport;

		/* dump netdevices */
		read_lock(&smc_pnettable.lock);
		list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) {
		if (idx++ < cb->args[0])
		if (pnetid && !smc_pnet_match(pnetelem->pnet_name, pnetid))
		continue;
		if (idx++ < start_idx)
		continue;
		if (smc_pnet_dumpinfo(skb, NETLINK_CB(cb->skb).portid,
		cb->nlh->nlmsg_seq, NLM_F_MULTI,
		pnetelem)) {
		memset(&tmp_entry, 0, sizeof(tmp_entry));
		memcpy(&tmp_entry.pnet_name, pnetelem->pnet_name,
		SMC_MAX_PNETID_LEN);
		tmp_entry.ndev = pnetelem->ndev;
		if (smc_pnet_dumpinfo(skb, portid, seq, NLM_F_MULTI,
		&tmp_entry)) {
		--idx;
		break;
		}
		}
		cb->args[0] = idx;
		read_unlock(&smc_pnettable.lock);

		/* dump ib devices */
		spin_lock(&smc_ib_devices.lock);
		list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
		for (ibport = 0; ibport < SMC_MAX_PORTS; ibport++) {
		if (ibdev->pnetid_by_user[ibport]) {
		if (pnetid &&
		!smc_pnet_match(ibdev->pnetid[ibport],
		pnetid))
		continue;
		if (idx++ < start_idx)
		continue;
		memset(&tmp_entry, 0, sizeof(tmp_entry));
		memcpy(&tmp_entry.pnet_name,
		ibdev->pnetid[ibport],
		SMC_MAX_PNETID_LEN);
		tmp_entry.smcibdev = ibdev;
		tmp_entry.ib_port = ibport + 1;
		if (smc_pnet_dumpinfo(skb, portid, seq,
		NLM_F_MULTI,
		&tmp_entry)) {
		--idx;
		break;
		}
		}
		}
		}
		spin_unlock(&smc_ib_devices.lock);

		return idx;
		}

		static int smc_pnet_dump(struct sk_buff skb, struct netlink_callback cb)
		{
		int idx;

		idx = _smc_pnet_dump(skb, NETLINK_CB(cb->skb).portid,
		cb->nlh->nlmsg_seq, NULL, cb->args[0]);

		cb->args[0] = idx;
		return skb->len;
		}

		/* Retrieve one PNETID entry */
		static int smc_pnet_get(struct sk_buff skb, struct genl_info info)
		{
		struct sk_buff *msg;
		void *hdr;

		if (!info->attrs[SMC_PNETID_NAME])
		return -EINVAL;

		msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
		if (!msg)
		return -ENOMEM;

		_smc_pnet_dump(msg, info->snd_portid, info->snd_seq,
		nla_data(info->attrs[SMC_PNETID_NAME]), 0);

		/* finish multi part message and send it */
		hdr = nlmsg_put(msg, info->snd_portid, info->snd_seq, NLMSG_DONE, 0,
		NLM_F_MULTI);
		if (!hdr) {
		nlmsg_free(msg);
		return -EMSGSIZE;
		}
		return genlmsg_reply(msg, info);
		}

		/* Remove and delete all pnetids from pnet table.
		*/
		static int smc_pnet_flush(struct sk_buff skb, struct genl_info info)
		{
		struct smc_pnetentry pnetelem, tmp_pe;

		write_lock(&smc_pnettable.lock);
		list_for_each_entry_safe(pnetelem, tmp_pe, &smc_pnettable.pnetlist,
		list) {
		list_del(&pnetelem->list);
		dev_put(pnetelem->ndev);
		kfree(pnetelem);
		}
		write_unlock(&smc_pnettable.lock);
		return 0;
		return smc_pnet_remove_by_pnetid(NULL);
		}

		/* SMC_PNETID generic netlink operation definition */
		@@ -534,6 +601,25 @@ static struct net_device pnet_find_base_ndev(struct net_device ndev)
		return ndev;
		}

		static int smc_pnet_find_ndev_pnetid_by_table(struct net_device *netdev,
		u8 *pnetid)
		{
		struct smc_pnetentry *pnetelem;
		int rc = -ENOENT;

		read_lock(&smc_pnettable.lock);
		list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) {
		if (netdev == pnetelem->ndev) {
		/* get pnetid of netdev device */
		memcpy(pnetid, pnetelem->pnet_name, SMC_MAX_PNETID_LEN);
		rc = 0;
		break;
		}
		}
		read_unlock(&smc_pnettable.lock);
		return rc;
		}

		/* Determine the corresponding IB device port based on the hardware PNETID.
		* Searching stops at the first matching active IB device port with vlan_id
		* configured.
		@@ -549,7 +635,8 @@ static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev,

		ndev = pnet_find_base_ndev(ndev);
		if (smc_pnetid_by_dev_port(ndev->dev.parent, ndev->dev_port,
		ndev_pnetid))
		ndev_pnetid) &&
		smc_pnet_find_ndev_pnetid_by_table(ndev, ndev_pnetid))
		return; /* pnetid could not be determined */

		spin_lock(&smc_ib_devices.lock);
		@@ -557,8 +644,7 @@ static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev,
		for (i = 1; i <= SMC_MAX_PORTS; i++) {
		if (!rdma_is_port_valid(ibdev->ibdev, i))
		continue;
		if (!memcmp(ibdev->pnetid[i - 1], ndev_pnetid,
		SMC_MAX_PNETID_LEN) &&
		if (smc_pnet_match(ibdev->pnetid[i - 1], ndev_pnetid) &&
		smc_ib_port_active(ibdev, i) &&
		!smc_ib_determine_gid(ibdev, i, vlan_id, gid,
		NULL)) {
		@@ -593,31 +679,6 @@ static void smc_pnet_find_ism_by_pnetid(struct net_device *ndev,
		spin_unlock(&smcd_dev_list.lock);
		}

		/* Lookup of coupled ib_device via SMC pnet table */
		static void smc_pnet_find_roce_by_table(struct net_device *netdev,
		struct smc_ib_device **smcibdev,
		u8 *ibport, unsigned short vlan_id,
		u8 gid[])
		{
		struct smc_pnetentry *pnetelem;

		read_lock(&smc_pnettable.lock);
		list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) {
		if (netdev == pnetelem->ndev) {
		if (smc_ib_port_active(pnetelem->smcibdev,
		pnetelem->ib_port) &&
		!smc_ib_determine_gid(pnetelem->smcibdev,
		pnetelem->ib_port, vlan_id,
		gid, NULL)) {
		*smcibdev = pnetelem->smcibdev;
		*ibport = pnetelem->ib_port;
		}
		break;
		}
		}
		read_unlock(&smc_pnettable.lock);
		}

		/* PNET table analysis for a given sock:
		* determine ib_device and port belonging to used internal TCP socket
		* ethernet interface.
		@@ -636,13 +697,7 @@ void smc_pnet_find_roce_resource(struct sock *sk,
		if (!dst->dev)
		goto out_rel;

		/* if possible, lookup via hardware-defined pnetid */
		smc_pnet_find_roce_by_pnetid(dst->dev, smcibdev, ibport, vlan_id, gid);
		if (*smcibdev)
		goto out_rel;

		/* lookup via SMC PNET table */
		smc_pnet_find_roce_by_table(dst->dev, smcibdev, ibport, vlan_id, gid);

		out_rel:
		dst_release(dst);

net/smc/smc_pnet.h

+0 −1

Original line number	Diff line number	Diff line
		@@ -31,7 +31,6 @@ static inline int smc_pnetid_by_dev_port(struct device *dev,

		int smc_pnet_init(void) __init;
		void smc_pnet_exit(void);
		int smc_pnet_remove_by_ibdev(struct smc_ib_device *ibdev);
		void smc_pnet_find_roce_resource(struct sock *sk,
		struct smc_ib_device *smcibdev, u8 ibport,
		unsigned short vlan_id, u8 gid[]);

Admin message