Commit 0927f71d authored by Doron Roberts-Kedes's avatar Doron Roberts-Kedes Committed by David S. Miller
Browse files

net/tls: Calculate nsg for zerocopy path without skb_cow_data. 



decrypt_skb fails if the number of sg elements required to map it
is greater than MAX_SKB_FRAGS. nsg must always be calculated, but
skb_cow_data adds unnecessary memcpy's for the zerocopy case.

The new function skb_nsg calculates the number of scatterlist elements
required to map the skb without the extra overhead of skb_cow_data.
This patch reduces memcpy by 50% on my encrypted NBD benchmarks.

Reported-by: default avatarVakul Garg <Vakul.garg@nxp.com>
Reviewed-by: default avatarVakul Garg <Vakul.garg@nxp.com>
Tested-by: default avatarVakul Garg <Vakul.garg@nxp.com>
Signed-off-by: default avatarDoron Roberts-Kedes <doronrk@fb.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7e8d5755

net/tls/tls_sw.c

+79 −1
Original line number Diff line number Diff line
@@ -43,6 +43,82 @@ 


#define MAX_IV_SIZE	TLS_CIPHER_AES_GCM_128_IV_SIZE



static int __skb_nsg(struct sk_buff *skb, int offset, int len,

                     unsigned int recursion_level)

{

        int start = skb_headlen(skb);

        int i, chunk = start - offset;

        struct sk_buff *frag_iter;

        int elt = 0;



        if (unlikely(recursion_level >= 24))

                return -EMSGSIZE;



        if (chunk > 0) {

                if (chunk > len)

                        chunk = len;

                elt++;

                len -= chunk;

                if (len == 0)

                        return elt;

                offset += chunk;

        }



        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {

                int end;



                WARN_ON(start > offset + len);



                end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);

                chunk = end - offset;

                if (chunk > 0) {

                        if (chunk > len)

                                chunk = len;

                        elt++;

                        len -= chunk;

                        if (len == 0)

                                return elt;

                        offset += chunk;

                }

                start = end;

        }



        if (unlikely(skb_has_frag_list(skb))) {

                skb_walk_frags(skb, frag_iter) {

                        int end, ret;



                        WARN_ON(start > offset + len);



                        end = start + frag_iter->len;

                        chunk = end - offset;

                        if (chunk > 0) {

                                if (chunk > len)

                                        chunk = len;

                                ret = __skb_nsg(frag_iter, offset - start, chunk,

                                                recursion_level + 1);

                                if (unlikely(ret < 0))

                                        return ret;

                                elt += ret;

                                len -= chunk;

                                if (len == 0)

                                        return elt;

                                offset += chunk;

                        }

                        start = end;

                }

        }

        BUG_ON(len);

        return elt;

}



/* Return the number of scatterlist elements required to completely map the

 * skb, or -EMSGSIZE if the recursion depth is exceeded.

 */

static int skb_nsg(struct sk_buff *skb, int offset, int len)

{

        return __skb_nsg(skb, offset, len, 0);

}



static int tls_do_decryption(struct sock *sk,

			     struct scatterlist *sgin,

			     struct scatterlist *sgout,
@@ -678,12 +754,14 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb, 
			n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1;

		else

			n_sgout = sg_nents(out_sg);

		n_sgin = skb_nsg(skb, rxm->offset + tls_ctx->rx.prepend_size,

				 rxm->full_len - tls_ctx->rx.prepend_size);

	} else {

		n_sgout = 0;

		*zc = false;

		n_sgin = skb_cow_data(skb, 0, &unused);

	}



	n_sgin = skb_cow_data(skb, 0, &unused);

	if (n_sgin < 1)

		return -EBADMSG;

CRA Git | Maintained and supported by SUSTech CRA and CCSE