Commit 47ebcc0b authored by Yossi Kuperman's avatar Yossi Kuperman Committed by Steffen Klassert
Browse files

xfrm: Add support for network devices capable of removing the ESP trailer 

In conjunction with crypto offload [1], removing the ESP trailer by
hardware can potentially improve the performance by avoiding (1) a
cache miss incurred by reading the nexthdr field and (2) the necessity
to calculate the csum value of the trailer in order to keep skb->csum
valid.

This patch introduces the changes to the xfrm stack and merely serves
as an infrastructure. Subsequent patch to mlx5 driver will put this to
a good use.

[1] https://www.mail-archive.com/netdev@vger.kernel.org/msg175733.html



Signed-off-by: default avatarYossi Kuperman <yossiku@mellanox.com>
Signed-off-by: default avatarSteffen Klassert <steffen.klassert@secunet.com>
parent acfb98b9

include/net/xfrm.h

+1 −0
Original line number Diff line number Diff line
@@ -1019,6 +1019,7 @@ struct xfrm_offload { 
#define	CRYPTO_FALLBACK		8

#define	XFRM_GSO_SEGMENT	16

#define	XFRM_GRO		32

#define	XFRM_ESP_NO_TRAILER	64



	__u32			status;

#define CRYPTO_SUCCESS				1

net/ipv4/esp4.c

+47 −23
Original line number Diff line number Diff line
@@ -499,35 +499,69 @@ static int esp_output(struct xfrm_state *x, struct sk_buff *skb) 
	return esp_output_tail(x, skb, &esp);

}



int esp_input_done2(struct sk_buff *skb, int err)

static inline int esp_remove_trailer(struct sk_buff *skb)

{

	const struct iphdr *iph;

	struct xfrm_state *x = xfrm_input_state(skb);

	struct xfrm_offload *xo = xfrm_offload(skb);

	struct crypto_aead *aead = x->data;

	int alen = crypto_aead_authsize(aead);

	int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);

	int elen = skb->len - hlen;

	int ihl;

	u8 nexthdr[2];

	int alen, hlen, elen;

	int padlen, trimlen;

	__wsum csumdiff;

	u8 nexthdr[2];

	int ret;



	if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))

		kfree(ESP_SKB_CB(skb)->tmp);

	alen = crypto_aead_authsize(aead);

	hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);

	elen = skb->len - hlen;



	if (unlikely(err))

	if (xo && (xo->flags & XFRM_ESP_NO_TRAILER)) {

		ret = xo->proto;

		goto out;

	}



	if (skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2))

		BUG();



	err = -EINVAL;

	ret = -EINVAL;

	padlen = nexthdr[0];

	if (padlen + 2 + alen >= elen)

	if (padlen + 2 + alen >= elen) {

		net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",

				    padlen + 2, elen - alen);

		goto out;

	}



	/* ... check padding bits here. Silly. :-) */

	trimlen = alen + padlen + 2;

	if (skb->ip_summed == CHECKSUM_COMPLETE) {

		csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);

		skb->csum = csum_block_sub(skb->csum, csumdiff,

					   skb->len - trimlen);

	}

	pskb_trim(skb, skb->len - trimlen);



	ret = nexthdr[1];



out:

	return ret;

}



int esp_input_done2(struct sk_buff *skb, int err)

{

	const struct iphdr *iph;

	struct xfrm_state *x = xfrm_input_state(skb);

	struct xfrm_offload *xo = xfrm_offload(skb);

	struct crypto_aead *aead = x->data;

	int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);

	int ihl;



	if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))

		kfree(ESP_SKB_CB(skb)->tmp);



	if (unlikely(err))

		goto out;



	err = esp_remove_trailer(skb);

	if (unlikely(err < 0))

		goto out;



	iph = ip_hdr(skb);

	ihl = iph->ihl * 4;
@@ -569,22 +603,12 @@ int esp_input_done2(struct sk_buff *skb, int err) 
			skb->ip_summed = CHECKSUM_UNNECESSARY;

	}



	trimlen = alen + padlen + 2;

	if (skb->ip_summed == CHECKSUM_COMPLETE) {

		csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);

		skb->csum = csum_block_sub(skb->csum, csumdiff,

					   skb->len - trimlen);

	}

	pskb_trim(skb, skb->len - trimlen);



	skb_pull_rcsum(skb, hlen);

	if (x->props.mode == XFRM_MODE_TUNNEL)

		skb_reset_transport_header(skb);

	else

		skb_set_transport_header(skb, -ihl);



	err = nexthdr[1];



	/* RFC4303: Drop dummy packets without any error */

	if (err == IPPROTO_NONE)

		err = -EINVAL;

net/ipv6/esp6.c

+36 −15
Original line number Diff line number Diff line
@@ -461,29 +461,30 @@ static int esp6_output(struct xfrm_state *x, struct sk_buff *skb) 
	return esp6_output_tail(x, skb, &esp);

}



int esp6_input_done2(struct sk_buff *skb, int err)

static inline int esp_remove_trailer(struct sk_buff *skb)

{

	struct xfrm_state *x = xfrm_input_state(skb);

	struct xfrm_offload *xo = xfrm_offload(skb);

	struct crypto_aead *aead = x->data;

	int alen = crypto_aead_authsize(aead);

	int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);

	int elen = skb->len - hlen;

	int hdr_len = skb_network_header_len(skb);

	int alen, hlen, elen;

	int padlen, trimlen;

	__wsum csumdiff;

	u8 nexthdr[2];

	int ret;



	if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))

		kfree(ESP_SKB_CB(skb)->tmp);

	alen = crypto_aead_authsize(aead);

	hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);

	elen = skb->len - hlen;



	if (unlikely(err))

	if (xo && (xo->flags & XFRM_ESP_NO_TRAILER)) {

		ret = xo->proto;

		goto out;

	}



	if (skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2))

		BUG();



	err = -EINVAL;

	ret = -EINVAL;

	padlen = nexthdr[0];

	if (padlen + 2 + alen >= elen) {

		net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
@@ -491,26 +492,46 @@ int esp6_input_done2(struct sk_buff *skb, int err) 
		goto out;

	}



	/* ... check padding bits here. Silly. :-) */



	trimlen = alen + padlen + 2;

	if (skb->ip_summed == CHECKSUM_COMPLETE) {

		skb_postpull_rcsum(skb, skb_network_header(skb),

				   skb_network_header_len(skb));

		csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);

		skb->csum = csum_block_sub(skb->csum, csumdiff,

					   skb->len - trimlen);

	}

	pskb_trim(skb, skb->len - trimlen);



	ret = nexthdr[1];



out:

	return ret;

}



int esp6_input_done2(struct sk_buff *skb, int err)

{

	struct xfrm_state *x = xfrm_input_state(skb);

	struct xfrm_offload *xo = xfrm_offload(skb);

	struct crypto_aead *aead = x->data;

	int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);

	int hdr_len = skb_network_header_len(skb);



	if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))

		kfree(ESP_SKB_CB(skb)->tmp);



	if (unlikely(err))

		goto out;



	err = esp_remove_trailer(skb);

	if (unlikely(err < 0))

		goto out;



	skb_postpull_rcsum(skb, skb_network_header(skb),

			   skb_network_header_len(skb));

	skb_pull_rcsum(skb, hlen);

	if (x->props.mode == XFRM_MODE_TUNNEL)

		skb_reset_transport_header(skb);

	else

		skb_set_transport_header(skb, -hdr_len);



	err = nexthdr[1];



	/* RFC4303: Drop dummy packets without any error */

	if (err == IPPROTO_NONE)

		err = -EINVAL;

net/xfrm/xfrm_input.c

+5 −0
Original line number Diff line number Diff line
@@ -247,6 +247,11 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 
					goto drop;

				}



				if (xo->status & CRYPTO_INVALID_PROTOCOL) {

					XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);

					goto drop;

				}



				XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);

				goto drop;

			}

CRA Git | Maintained and supported by SUSTech CRA and CCSE