Commit b9a9ad78 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'smc-next' 



Ursula Braun says:

====================
net/smc: patches 2018-07-25

here are 4 more patches for SMC: The first one is just a small
code cleanup in preparation for patch 2. Patch 2 switches to the
use of the vlan-gid for VLAN traffic. Patch 3 improves diagnosis
when creating SMC connections. Patch 4 improves synchronization
between local and remote link groups.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents f5375305 0d18a0cb

include/uapi/linux/smc_diag.h

+6 −0
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ enum { 
	SMC_DIAG_LGRINFO,

	SMC_DIAG_SHUTDOWN,

	SMC_DIAG_DMBINFO,

	SMC_DIAG_FALLBACK,

	__SMC_DIAG_MAX,

};
@@ -92,6 +93,11 @@ struct smc_diag_lgrinfo { 
	__u8				role;

};



struct smc_diag_fallback {

	__u32 reason;

	__u32 peer_diagnosis;

};



struct smcd_diag_dmbinfo {		/* SMC-D Socket internals */

	__u32 linkid;			/* Link identifier */

	__u64 peer_gid;			/* Peer GID */

net/smc/af_smc.c

+45 −41
Original line number Diff line number Diff line
@@ -344,20 +344,17 @@ static int smc_clnt_conf_first_link(struct smc_sock *smc) 


	rc = smc_ib_modify_qp_rts(link);

	if (rc)

		return SMC_CLC_DECL_INTERR;

		return SMC_CLC_DECL_ERR_RDYLNK;



	smc_wr_remember_qp_attr(link);



	if (smc_reg_rmb(link, smc->conn.rmb_desc, false))

		return SMC_CLC_DECL_INTERR;

		return SMC_CLC_DECL_ERR_REGRMB;



	/* send CONFIRM LINK response over RoCE fabric */

	rc = smc_llc_send_confirm_link(link,

				       link->smcibdev->mac[link->ibport - 1],

				       &link->smcibdev->gid[link->ibport - 1],

				       SMC_LLC_RESP);

	rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);

	if (rc < 0)

		return SMC_CLC_DECL_TCL;

		return SMC_CLC_DECL_TIMEOUT_CL;



	/* receive ADD LINK request from server over RoCE fabric */

	rest = wait_for_completion_interruptible_timeout(&link->llc_add,
@@ -373,10 +370,9 @@ static int smc_clnt_conf_first_link(struct smc_sock *smc) 
	/* send add link reject message, only one link supported for now */

	rc = smc_llc_send_add_link(link,

				   link->smcibdev->mac[link->ibport - 1],

				   &link->smcibdev->gid[link->ibport - 1],

				   SMC_LLC_RESP);

				   link->gid, SMC_LLC_RESP);

	if (rc < 0)

		return SMC_CLC_DECL_TCL;

		return SMC_CLC_DECL_TIMEOUT_AL;



	smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
@@ -428,9 +424,10 @@ static void smc_link_save_peer_info(struct smc_link *link, 
}



/* fall back during connect */

static int smc_connect_fallback(struct smc_sock *smc)

static int smc_connect_fallback(struct smc_sock *smc, int reason_code)

{

	smc->use_fallback = true;

	smc->fallback_rsn = reason_code;

	smc_copy_sock_settings_to_clc(smc);

	if (smc->sk.sk_state == SMC_INIT)

		smc->sk.sk_state = SMC_ACTIVE;
@@ -447,7 +444,7 @@ static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code) 
			sock_put(&smc->sk); /* passive closing */

		return reason_code;

	}

	if (reason_code != SMC_CLC_DECL_REPLY) {

	if (reason_code != SMC_CLC_DECL_PEERDECL) {

		rc = smc_clc_send_decline(smc, reason_code);

		if (rc < 0) {

			if (smc->sk.sk_state == SMC_INIT)
@@ -455,7 +452,7 @@ static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code) 
			return rc;

		}

	}

	return smc_connect_fallback(smc);

	return smc_connect_fallback(smc, reason_code);

}



/* abort connecting */
@@ -472,7 +469,7 @@ static int smc_connect_abort(struct smc_sock *smc, int reason_code, 
/* check if there is a rdma device available for this connection. */

/* called for connect and listen */

static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,

			  u8 *ibport)

			  u8 *ibport, unsigned short vlan_id, u8 gid[])

{

	int reason_code = 0;
@@ -480,7 +477,8 @@ static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev, 
	 * within same PNETID that also contains the ethernet device

	 * used for the internal TCP socket

	 */

	smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport);

	smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport, vlan_id,

				    gid);

	if (!(*ibdev))

		reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
@@ -526,12 +524,12 @@ static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd, 
static int smc_connect_clc(struct smc_sock *smc, int smc_type,

			   struct smc_clc_msg_accept_confirm *aclc,

			   struct smc_ib_device *ibdev, u8 ibport,

			   struct smcd_dev *ismdev)

			   u8 gid[], struct smcd_dev *ismdev)

{

	int rc = 0;



	/* do inband token exchange */

	rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, ismdev);

	rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, gid, ismdev);

	if (rc)

		return rc;

	/* receive SMC Accept CLC message */
@@ -571,7 +569,7 @@ static int smc_connect_rdma(struct smc_sock *smc, 
		smc_link_save_peer_info(link, aclc);



	if (smc_rmb_rtoken_handling(&smc->conn, aclc))

		return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,

		return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,

					 local_contact);



	smc_close_init(smc);
@@ -579,12 +577,12 @@ static int smc_connect_rdma(struct smc_sock *smc, 


	if (local_contact == SMC_FIRST_CONTACT) {

		if (smc_ib_ready_link(link))

			return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,

			return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,

						 local_contact);

	} else {

		if (!smc->conn.rmb_desc->reused &&

		    smc_reg_rmb(link, smc->conn.rmb_desc, true))

			return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,

			return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,

						 local_contact);

	}

	smc_rmb_sync_sg_for_device(&smc->conn);
@@ -653,6 +651,7 @@ static int __smc_connect(struct smc_sock *smc) 
	struct smc_clc_msg_accept_confirm aclc;

	struct smc_ib_device *ibdev;

	struct smcd_dev *ismdev;

	u8 gid[SMC_GID_SIZE];

	unsigned short vlan;

	int smc_type;

	int rc = 0;
@@ -661,11 +660,11 @@ static int __smc_connect(struct smc_sock *smc) 
	sock_hold(&smc->sk); /* sock put in passive closing */



	if (smc->use_fallback)

		return smc_connect_fallback(smc);

		return smc_connect_fallback(smc, smc->fallback_rsn);



	/* if peer has not signalled SMC-capability, fall back */

	if (!tcp_sk(smc->clcsock->sk)->syn_smc)

		return smc_connect_fallback(smc);

		return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);



	/* IPSec connections opt out of SMC-R optimizations */

	if (using_ipsec(smc))
@@ -684,7 +683,7 @@ static int __smc_connect(struct smc_sock *smc) 
	}



	/* check if there is a rdma device available */

	if (!smc_check_rdma(smc, &ibdev, &ibport)) {

	if (!smc_check_rdma(smc, &ibdev, &ibport, vlan, gid)) {

		/* RDMA is supported for this connection */

		rdma_supported = true;

		if (ism_supported)
@@ -695,10 +694,10 @@ static int __smc_connect(struct smc_sock *smc) 


	/* if neither ISM nor RDMA are supported, fallback */

	if (!rdma_supported && !ism_supported)

		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);

		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);



	/* perform CLC handshake */

	rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, ismdev);

	rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, gid, ismdev);

	if (rc) {

		smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);

		return smc_connect_decline_fallback(smc, rc);
@@ -710,7 +709,7 @@ static int __smc_connect(struct smc_sock *smc) 
	else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)

		rc = smc_connect_ism(smc, &aclc, ismdev);

	else

		rc = SMC_CLC_DECL_CNFERR;

		rc = SMC_CLC_DECL_MODEUNSUPP;

	if (rc) {

		smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);

		return smc_connect_decline_fallback(smc, rc);
@@ -948,15 +947,12 @@ static int smc_serv_conf_first_link(struct smc_sock *smc) 
	link = &lgr->lnk[SMC_SINGLE_LINK];



	if (smc_reg_rmb(link, smc->conn.rmb_desc, false))

		return SMC_CLC_DECL_INTERR;

		return SMC_CLC_DECL_ERR_REGRMB;



	/* send CONFIRM LINK request to client over the RoCE fabric */

	rc = smc_llc_send_confirm_link(link,

				       link->smcibdev->mac[link->ibport - 1],

				       &link->smcibdev->gid[link->ibport - 1],

				       SMC_LLC_REQ);

	rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);

	if (rc < 0)

		return SMC_CLC_DECL_TCL;

		return SMC_CLC_DECL_TIMEOUT_CL;



	/* receive CONFIRM LINK response from client over the RoCE fabric */

	rest = wait_for_completion_interruptible_timeout(
@@ -976,10 +972,9 @@ static int smc_serv_conf_first_link(struct smc_sock *smc) 
	/* send ADD LINK request to client over the RoCE fabric */

	rc = smc_llc_send_add_link(link,

				   link->smcibdev->mac[link->ibport - 1],

				   &link->smcibdev->gid[link->ibport - 1],

				   SMC_LLC_REQ);

				   link->gid, SMC_LLC_REQ);

	if (rc < 0)

		return SMC_CLC_DECL_TCL;

		return SMC_CLC_DECL_TIMEOUT_AL;



	/* receive ADD LINK response from client over the RoCE fabric */

	rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
@@ -1054,7 +1049,8 @@ static void smc_listen_decline(struct smc_sock *new_smc, int reason_code, 
	}

	smc_conn_free(&new_smc->conn);

	new_smc->use_fallback = true;

	if (reason_code && reason_code != SMC_CLC_DECL_REPLY) {

	new_smc->fallback_rsn = reason_code;

	if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {

		if (smc_clc_send_decline(new_smc, reason_code) < 0) {

			smc_listen_out_err(new_smc);

			return;
@@ -1145,7 +1141,7 @@ static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact) 
	if (local_contact != SMC_FIRST_CONTACT) {

		if (!new_smc->conn.rmb_desc->reused) {

			if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))

				return SMC_CLC_DECL_INTERR;

				return SMC_CLC_DECL_ERR_REGRMB;

		}

	}

	smc_rmb_sync_sg_for_device(&new_smc->conn);
@@ -1165,13 +1161,13 @@ static void smc_listen_rdma_finish(struct smc_sock *new_smc, 
		smc_link_save_peer_info(link, cclc);



	if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {

		reason_code = SMC_CLC_DECL_INTERR;

		reason_code = SMC_CLC_DECL_ERR_RTOK;

		goto decline;

	}



	if (local_contact == SMC_FIRST_CONTACT) {

		if (smc_ib_ready_link(link)) {

			reason_code = SMC_CLC_DECL_INTERR;

			reason_code = SMC_CLC_DECL_ERR_RDYLNK;

			goto decline;

		}

		/* QP confirmation over RoCE fabric */
@@ -1199,6 +1195,7 @@ static void smc_listen_work(struct work_struct *work) 
	struct smcd_dev *ismdev;

	u8 buf[SMC_CLC_MAX_LEN];

	int local_contact = 0;

	unsigned short vlan;

	int reason_code = 0;

	int rc = 0;

	u8 ibport;
@@ -1211,6 +1208,7 @@ static void smc_listen_work(struct work_struct *work) 
	/* check if peer is smc capable */

	if (!tcp_sk(newclcsock->sk)->syn_smc) {

		new_smc->use_fallback = true;

		new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;

		smc_listen_out_connected(new_smc);

		return;

	}
@@ -1247,14 +1245,16 @@ static void smc_listen_work(struct work_struct *work) 
	/* check if RDMA is available */

	if (!ism_supported &&

	    ((pclc->hdr.path != SMC_TYPE_R && pclc->hdr.path != SMC_TYPE_B) ||

	     smc_check_rdma(new_smc, &ibdev, &ibport) ||

	     smc_vlan_by_tcpsk(new_smc->clcsock, &vlan) ||

	     smc_check_rdma(new_smc, &ibdev, &ibport, vlan, NULL) ||

	     smc_listen_rdma_check(new_smc, pclc) ||

	     smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,

				  &local_contact) ||

	     smc_listen_rdma_reg(new_smc, local_contact))) {

		/* SMC not supported, decline */

		mutex_unlock(&smc_create_lgr_pending);

		smc_listen_decline(new_smc, SMC_CLC_DECL_CNFERR, local_contact);

		smc_listen_decline(new_smc, SMC_CLC_DECL_MODEUNSUPP,

				   local_contact);

		return;

	}
@@ -1301,6 +1301,7 @@ static void smc_tcp_listen_work(struct work_struct *work) 


		new_smc->listen_smc = lsmc;

		new_smc->use_fallback = lsmc->use_fallback;

		new_smc->fallback_rsn = lsmc->fallback_rsn;

		sock_hold(lsk); /* sock_put in smc_listen_work */

		INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);

		smc_copy_sock_settings_to_smc(new_smc);
@@ -1455,6 +1456,7 @@ static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) 
	if (msg->msg_flags & MSG_FASTOPEN) {

		if (sk->sk_state == SMC_INIT) {

			smc->use_fallback = true;

			smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;

		} else {

			rc = -EINVAL;

			goto out;
@@ -1652,6 +1654,7 @@ static int smc_setsockopt(struct socket *sock, int level, int optname, 
		/* option not supported by SMC */

		if (sk->sk_state == SMC_INIT) {

			smc->use_fallback = true;

			smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;

		} else {

			if (!smc->use_fallback)

				rc = -EINVAL;
@@ -1889,6 +1892,7 @@ static int smc_create(struct net *net, struct socket *sock, int protocol, 
	/* create internal TCP socket for CLC handshake and fallback */

	smc = smc_sk(sk);

	smc->use_fallback = false; /* assume rdma capability first */

	smc->fallback_rsn = 0;

	rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,

			      &smc->clcsock);

	if (rc) {

net/smc/smc.h

+2 −0
Original line number Diff line number Diff line
@@ -208,6 +208,8 @@ struct smc_sock { /* smc sock container */ 
	struct list_head	accept_q;	/* sockets to be accepted */

	spinlock_t		accept_q_lock;	/* protects accept_q */

	bool			use_fallback;	/* fallback to tcp */

	int			fallback_rsn;	/* reason for fallback */

	u32			peer_diagnosis; /* decline reason from peer */

	int			sockopt_defer_accept;

						/* sockopt TCP_DEFER_ACCEPT

						 * value

net/smc/smc_clc.c

+9 −7
Original line number Diff line number Diff line
@@ -334,7 +334,11 @@ int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen, 
		goto out;

	}

	if (clcm->type == SMC_CLC_DECLINE) {

		reason_code = SMC_CLC_DECL_REPLY;

		struct smc_clc_msg_decline *dclc;



		dclc = (struct smc_clc_msg_decline *)clcm;

		reason_code = SMC_CLC_DECL_PEERDECL;

		smc->peer_diagnosis = ntohl(dclc->peer_diagnosis);

		if (((struct smc_clc_msg_decline *)buf)->hdr.flag) {

			smc->conn.lgr->sync_err = 1;

			smc_lgr_terminate(smc->conn.lgr);
@@ -378,7 +382,7 @@ int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info) 


/* send CLC PROPOSAL message across internal TCP socket */

int smc_clc_send_proposal(struct smc_sock *smc, int smc_type,

			  struct smc_ib_device *ibdev, u8 ibport,

			  struct smc_ib_device *ibdev, u8 ibport, u8 gid[],

			  struct smcd_dev *ismdev)

{

	struct smc_clc_ipv6_prefix ipv6_prfx[SMC_CLC_MAX_V6_PREFIX];
@@ -409,7 +413,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, int smc_type, 
		/* add SMC-R specifics */

		memcpy(pclc.lcl.id_for_peer, local_systemid,

		       sizeof(local_systemid));

		memcpy(&pclc.lcl.gid, &ibdev->gid[ibport - 1], SMC_GID_SIZE);

		memcpy(&pclc.lcl.gid, gid, SMC_GID_SIZE);

		memcpy(&pclc.lcl.mac, &ibdev->mac[ibport - 1], ETH_ALEN);

		pclc.iparea_offset = htons(0);

	}
@@ -492,8 +496,7 @@ int smc_clc_send_confirm(struct smc_sock *smc) 
		cclc.hdr.length = htons(SMCR_CLC_ACCEPT_CONFIRM_LEN);

		memcpy(cclc.lcl.id_for_peer, local_systemid,

		       sizeof(local_systemid));

		memcpy(&cclc.lcl.gid, &link->smcibdev->gid[link->ibport - 1],

		       SMC_GID_SIZE);

		memcpy(&cclc.lcl.gid, link->gid, SMC_GID_SIZE);

		memcpy(&cclc.lcl.mac, &link->smcibdev->mac[link->ibport - 1],

		       ETH_ALEN);

		hton24(cclc.qpn, link->roce_qp->qp_num);
@@ -566,8 +569,7 @@ int smc_clc_send_accept(struct smc_sock *new_smc, int srv_first_contact) 
		link = &conn->lgr->lnk[SMC_SINGLE_LINK];

		memcpy(aclc.lcl.id_for_peer, local_systemid,

		       sizeof(local_systemid));

		memcpy(&aclc.lcl.gid, &link->smcibdev->gid[link->ibport - 1],

		       SMC_GID_SIZE);

		memcpy(&aclc.lcl.gid, link->gid, SMC_GID_SIZE);

		memcpy(&aclc.lcl.mac, link->smcibdev->mac[link->ibport - 1],

		       ETH_ALEN);

		hton24(aclc.qpn, link->roce_qp->qp_num);

net/smc/smc_clc.h

+13 −7
Original line number Diff line number Diff line
@@ -28,15 +28,21 @@ 
#define SMC_TYPE_B		3		/* SMC-R and SMC-D	      */

#define CLC_WAIT_TIME		(6 * HZ)	/* max. wait time on clcsock  */

#define SMC_CLC_DECL_MEM	0x01010000  /* insufficient memory resources  */

#define SMC_CLC_DECL_TIMEOUT	0x02000000  /* timeout                        */

#define SMC_CLC_DECL_TIMEOUT_CL	0x02010000  /* timeout w4 QP confirm link     */

#define SMC_CLC_DECL_TIMEOUT_AL	0x02020000  /* timeout w4 QP add link	      */

#define SMC_CLC_DECL_CNFERR	0x03000000  /* configuration error            */

#define SMC_CLC_DECL_IPSEC	0x03030000  /* IPsec usage                    */

#define SMC_CLC_DECL_PEERNOSMC	0x03010000  /* peer did not indicate SMC      */

#define SMC_CLC_DECL_IPSEC	0x03020000  /* IPsec usage		      */

#define SMC_CLC_DECL_NOSMCDEV	0x03030000  /* no SMC device found	      */

#define SMC_CLC_DECL_MODEUNSUPP	0x03040000  /* smc modes do not match (R or D)*/

#define SMC_CLC_DECL_RMBE_EC	0x03050000  /* peer has eyecatcher in RMBE    */

#define SMC_CLC_DECL_OPTUNSUPP	0x03060000  /* fastopen sockopt not supported */

#define SMC_CLC_DECL_SYNCERR	0x04000000  /* synchronization error          */

#define SMC_CLC_DECL_REPLY	0x06000000  /* reply to a received decline    */

#define SMC_CLC_DECL_PEERDECL	0x05000000  /* peer declined during handshake */

#define SMC_CLC_DECL_INTERR	0x99990000  /* internal error                 */

#define SMC_CLC_DECL_TCL	0x02040000  /* timeout w4 QP confirm          */

#define SMC_CLC_DECL_SEND	0x07000000  /* sending problem                */

#define SMC_CLC_DECL_RMBE_EC	0x08000000  /* peer has eyecatcher in RMBE    */

#define SMC_CLC_DECL_ERR_RTOK	0x99990001  /*	 rtoken handling failed       */

#define SMC_CLC_DECL_ERR_RDYLNK	0x99990002  /*	 ib ready link failed	      */

#define SMC_CLC_DECL_ERR_REGRMB	0x99990003  /*	 reg rmb failed		      */



struct smc_clc_msg_hdr {	/* header1 of clc messages */

	u8 eyecatcher[4];	/* eye catcher */
@@ -179,7 +185,7 @@ int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen, 
		     u8 expected_type);

int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info);

int smc_clc_send_proposal(struct smc_sock *smc, int smc_type,

			  struct smc_ib_device *smcibdev, u8 ibport,

			  struct smc_ib_device *smcibdev, u8 ibport, u8 gid[],

			  struct smcd_dev *ismdev);

int smc_clc_send_confirm(struct smc_sock *smc);

int smc_clc_send_accept(struct smc_sock *smc, int srv_first_contact);

CRA Git | Maintained and supported by SUSTech CRA and CCSE