Commit 99ec41d0 authored by Jack Morgenstein's avatar Jack Morgenstein Committed by Roland Dreier
Browse files

mlx4: Add infrastructure for selecting VFs to enable QP0 via MLX proxy QPs 



This commit adds the infrastructure for enabling selected VFs to
operate SMI (QP0) MADs without restriction.

Additionally, for these enabled VFs, their QP0 proxy and tunnel QPs
are MLX QPs.  As such, they operate over VL15.  Therefore, they are
not affected by "credit" problems or changes in the VLArb table (which
may shut down VL0).

Non-enabled VFs may only create UD proxy QP0 qps (which are forced by
the hypervisor to send packets using the q-key it assigns and places
in the qp-context).  Thus, non-enabled VFs will not pose a security
risk.  The hypervisor discards any privileged MADs it receives from
these non-enabled VFs.

By default, all VFs are NOT enabled, and must explicitly be enabled
by the administrator.

The sysfs interface which operates the VF enablement infrastructure
is provided in the next commit.

Signed-off-by: default avatarJack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: default avatarOr Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: default avatarRoland Dreier <roland@purestorage.com>
parent 97982f5a

drivers/infiniband/hw/mlx4/qp.c

+40 −12
Original line number Diff line number Diff line
@@ -608,6 +608,16 @@ static int qp_has_rq(struct ib_qp_init_attr *attr) 
	return !attr->srq;

}



static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)

{

	int i;

	for (i = 0; i < dev->caps.num_ports; i++) {

		if (qpn == dev->caps.qp0_proxy[i])

			return !!dev->caps.qp0_qkey[i];

	}

	return 0;

}



static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,

			    struct ib_qp_init_attr *init_attr,

			    struct ib_udata *udata, int sqpn, struct mlx4_ib_qp **caller_qp)
@@ -625,11 +635,14 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, 
		     !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) {

			if (init_attr->qp_type == IB_QPT_GSI)

				qp_type = MLX4_IB_QPT_PROXY_GSI;

			else if (mlx4_is_master(dev->dev))

			else {

				if (mlx4_is_master(dev->dev) ||

				    qp0_enabled_vf(dev->dev, sqpn))

					qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER;

				else

					qp_type = MLX4_IB_QPT_PROXY_SMI;

			}

		}

		qpn = sqpn;

		/* add extra sg entry for tunneling */

		init_attr->cap.max_recv_sge++;
@@ -643,7 +656,9 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, 
			return -EINVAL;

		if (tnl_init->proxy_qp_type == IB_QPT_GSI)

			qp_type = MLX4_IB_QPT_TUN_GSI;

		else if (tnl_init->slave == mlx4_master_func_num(dev->dev))

		else if (tnl_init->slave == mlx4_master_func_num(dev->dev) ||

			 mlx4_vf_smi_enabled(dev->dev, tnl_init->slave,

					     tnl_init->port))

			qp_type = MLX4_IB_QPT_TUN_SMI_OWNER;

		else

			qp_type = MLX4_IB_QPT_TUN_SMI;
@@ -1930,6 +1945,19 @@ out: 
	return err;

}



static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)

{

	int i;

	for (i = 0; i < dev->caps.num_ports; i++) {

		if (qpn == dev->caps.qp0_proxy[i] ||

		    qpn == dev->caps.qp0_tunnel[i]) {

			*qkey = dev->caps.qp0_qkey[i];

			return 0;

		}

	}

	return -EINVAL;

}



static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,

				  struct ib_send_wr *wr,

				  void *wqe, unsigned *mlx_seg_len)
@@ -1987,8 +2015,13 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp, 
			cpu_to_be32(mdev->dev->caps.qp0_tunnel[sqp->qp.port - 1]);



	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));

	if (mlx4_is_master(mdev->dev)) {

		if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))

			return -EINVAL;

	} else {

		if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))

			return -EINVAL;

	}

	sqp->ud_header.deth.qkey = cpu_to_be32(qkey);

	sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn);
@@ -2682,11 +2715,6 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
			break;



		case MLX4_IB_QPT_PROXY_SMI_OWNER:

			if (unlikely(!mlx4_is_master(to_mdev(ibqp->device)->dev))) {

				err = -ENOSYS;

				*bad_wr = wr;

				goto out;

			}

			err = build_sriov_qp0_header(to_msqp(qp), wr, ctrl, &seglen);

			if (unlikely(err)) {

				*bad_wr = wr;

drivers/net/ethernet/mellanox/mlx4/cmd.c

+12 −1
Original line number Diff line number Diff line
@@ -1666,6 +1666,8 @@ static int mlx4_master_activate_admin_state(struct mlx4_priv *priv, int slave) 
	for (port = min_port; port <= max_port; port++) {

		if (!test_bit(port - 1, actv_ports.ports))

			continue;

		priv->mfunc.master.vf_oper[slave].smi_enabled[port] =

			priv->mfunc.master.vf_admin[slave].enable_smi[port];

		vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];

		vp_admin = &priv->mfunc.master.vf_admin[slave].vport[port];

		vp_oper->state = *vp_admin;
@@ -1717,6 +1719,8 @@ static void mlx4_master_deactivate_admin_state(struct mlx4_priv *priv, int slave 
	for (port = min_port; port <= max_port; port++) {

		if (!test_bit(port - 1, actv_ports.ports))

			continue;

		priv->mfunc.master.vf_oper[slave].smi_enabled[port] =

			MLX4_VF_SMI_DISABLED;

		vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];

		if (NO_INDX != vp_oper->vlan_idx) {

			__mlx4_unregister_vlan(&priv->dev,
@@ -2553,6 +2557,13 @@ EXPORT_SYMBOL_GPL(mlx4_set_vf_link_state); 


int mlx4_vf_smi_enabled(struct mlx4_dev *dev, int slave, int port)

{

	struct mlx4_priv *priv = mlx4_priv(dev);



	if (slave < 1 || slave >= dev->num_slaves ||

	    port < 1 || port > MLX4_MAX_PORTS)

		return 0;



	return priv->mfunc.master.vf_oper[slave].smi_enabled[port] ==

		MLX4_VF_SMI_ENABLED;

}

EXPORT_SYMBOL_GPL(mlx4_vf_smi_enabled);

drivers/net/ethernet/mellanox/mlx4/fw.c

+30 −14
Original line number Diff line number Diff line
@@ -178,8 +178,8 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, 
				struct mlx4_cmd_info *cmd)

{

	struct mlx4_priv *priv = mlx4_priv(dev);

	u8	field;

	u32	size;

	u8	field, port;

	u32	size, proxy_qp, qkey;

	int	err = 0;



#define QUERY_FUNC_CAP_FLAGS_OFFSET		0x0
@@ -209,6 +209,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, 


/* when opcode modifier = 1 */

#define QUERY_FUNC_CAP_PHYS_PORT_OFFSET		0x3

#define QUERY_FUNC_CAP_PRIV_VF_QKEY_OFFSET	0x4

#define QUERY_FUNC_CAP_FLAGS0_OFFSET		0x8

#define QUERY_FUNC_CAP_FLAGS1_OFFSET		0xc
@@ -221,6 +222,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, 
#define QUERY_FUNC_CAP_FLAGS1_FORCE_MAC		0x40

#define QUERY_FUNC_CAP_FLAGS1_FORCE_VLAN	0x80

#define QUERY_FUNC_CAP_FLAGS1_NIC_INFO			0x10

#define QUERY_FUNC_CAP_VF_ENABLE_QP0		0x08



#define QUERY_FUNC_CAP_FLAGS0_FORCE_PHY_WQE_GID 0x80
@@ -234,28 +236,35 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, 
			return -EINVAL;



		vhcr->in_modifier = converted_port;

		/* Set nic_info bit to mark new fields support */

		field  = QUERY_FUNC_CAP_FLAGS1_NIC_INFO;

		MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS1_OFFSET);



		/* phys-port = logical-port */

		field = vhcr->in_modifier -

			find_first_bit(actv_ports.ports, dev->caps.num_ports);

		MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_PHYS_PORT_OFFSET);



		field = vhcr->in_modifier;

		port = vhcr->in_modifier;

		proxy_qp = dev->phys_caps.base_proxy_sqpn + 8 * slave + port - 1;



		/* Set nic_info bit to mark new fields support */

		field  = QUERY_FUNC_CAP_FLAGS1_NIC_INFO;



		if (mlx4_vf_smi_enabled(dev, slave, port) &&

		    !mlx4_get_parav_qkey(dev, proxy_qp, &qkey)) {

			field |= QUERY_FUNC_CAP_VF_ENABLE_QP0;

			MLX4_PUT(outbox->buf, qkey,

				 QUERY_FUNC_CAP_PRIV_VF_QKEY_OFFSET);

		}

		MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS1_OFFSET);



		/* size is now the QP number */

		size = dev->phys_caps.base_tunnel_sqpn + 8 * slave + field - 1;

		size = dev->phys_caps.base_tunnel_sqpn + 8 * slave + port - 1;

		MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP0_TUNNEL);



		size += 2;

		MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP1_TUNNEL);



		size = dev->phys_caps.base_proxy_sqpn + 8 * slave + field - 1;

		MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP0_PROXY);



		size += 2;

		MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP1_PROXY);

		MLX4_PUT(outbox->buf, proxy_qp, QUERY_FUNC_CAP_QP0_PROXY);

		proxy_qp += 2;

		MLX4_PUT(outbox->buf, proxy_qp, QUERY_FUNC_CAP_QP1_PROXY);



		MLX4_PUT(outbox->buf, dev->caps.phys_port_id[vhcr->in_modifier],

			 QUERY_FUNC_CAP_PHYS_PORT_ID);
@@ -326,7 +335,7 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port, 
	struct mlx4_cmd_mailbox *mailbox;

	u32			*outbox;

	u8			field, op_modifier;

	u32			size;

	u32			size, qkey;

	int			err = 0, quotas = 0;



	op_modifier = !!gen_or_port; /* 0 = general, 1 = logical port */
@@ -442,6 +451,13 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port, 
		goto out;

	}



	if (func_cap->flags1 & QUERY_FUNC_CAP_VF_ENABLE_QP0) {

		MLX4_GET(qkey, outbox, QUERY_FUNC_CAP_PRIV_VF_QKEY_OFFSET);

		func_cap->qp0_qkey = qkey;

	} else {

		func_cap->qp0_qkey = 0;

	}



	MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_TUNNEL);

	func_cap->qp0_tunnel_qpn = size & 0xFFFFFF;

drivers/net/ethernet/mellanox/mlx4/fw.h

+1 −0
Original line number Diff line number Diff line
@@ -134,6 +134,7 @@ struct mlx4_func_cap { 
	int	max_eq;

	int	reserved_eq;

	int	mcg_quota;

	u32	qp0_qkey;

	u32	qp0_tunnel_qpn;

	u32	qp0_proxy_qpn;

	u32	qp1_tunnel_qpn;

drivers/net/ethernet/mellanox/mlx4/main.c

+13 −3
Original line number Diff line number Diff line
@@ -666,13 +666,15 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) 
		return -ENODEV;

	}



	dev->caps.qp0_qkey = kcalloc(dev->caps.num_ports, sizeof(u32), GFP_KERNEL);

	dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);

	dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);

	dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);

	dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);



	if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy ||

	    !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) {

	    !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy ||

	    !dev->caps.qp0_qkey) {

		err = -ENOMEM;

		goto err_mem;

	}
@@ -684,6 +686,7 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) 
				 " port %d, aborting (%d).\n", i, err);

			goto err_mem;

		}

		dev->caps.qp0_qkey[i - 1] = func_cap.qp0_qkey;

		dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn;

		dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn;

		dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn;
@@ -729,12 +732,16 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) 
	return 0;



err_mem:

	kfree(dev->caps.qp0_qkey);

	kfree(dev->caps.qp0_tunnel);

	kfree(dev->caps.qp0_proxy);

	kfree(dev->caps.qp1_tunnel);

	kfree(dev->caps.qp1_proxy);

	dev->caps.qp0_tunnel = dev->caps.qp0_proxy =

		dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL;

	dev->caps.qp0_qkey = NULL;

	dev->caps.qp0_tunnel = NULL;

	dev->caps.qp0_proxy = NULL;

	dev->caps.qp1_tunnel = NULL;

	dev->caps.qp1_proxy = NULL;



	return err;

}
@@ -1697,6 +1704,7 @@ unmap_bf: 
	unmap_bf_area(dev);



	if (mlx4_is_slave(dev)) {

		kfree(dev->caps.qp0_qkey);

		kfree(dev->caps.qp0_tunnel);

		kfree(dev->caps.qp0_proxy);

		kfree(dev->caps.qp1_tunnel);
@@ -2573,6 +2581,7 @@ err_master_mfunc: 
		mlx4_multi_func_cleanup(dev);



	if (mlx4_is_slave(dev)) {

		kfree(dev->caps.qp0_qkey);

		kfree(dev->caps.qp0_tunnel);

		kfree(dev->caps.qp0_proxy);

		kfree(dev->caps.qp1_tunnel);
@@ -2702,6 +2711,7 @@ static void __mlx4_remove_one(struct pci_dev *pdev) 
	if (!mlx4_is_slave(dev))

		mlx4_free_ownership(dev);



	kfree(dev->caps.qp0_qkey);

	kfree(dev->caps.qp0_tunnel);

	kfree(dev->caps.qp0_proxy);

	kfree(dev->caps.qp1_tunnel);

CRA Git | Maintained and supported by SUSTech CRA and CCSE