Commit f53c7239 authored by Steffen Klassert's avatar Steffen Klassert
Browse files

net: Add asynchronous callbacks for xfrm on layer 2. 



This patch implements asynchronous crypto callbacks
and a backlog handler that can be used when IPsec
is done at layer 2 in the TX path. It also extends
the skb validate functions so that we can update
the driver transmit return codes based on async
crypto operation or to indicate that we queued the
packet in a backlog queue.

Joint work with: Aviv Heller <avivh@mellanox.com>

Signed-off-by: default avatarSteffen Klassert <steffen.klassert@secunet.com>
parent 3dca3f38

include/linux/netdevice.h

+4 −2
Original line number Diff line number Diff line
@@ -2793,7 +2793,9 @@ struct softnet_data { 
	struct Qdisc		*output_queue;

	struct Qdisc		**output_queue_tailp;

	struct sk_buff		*completion_queue;



#ifdef CONFIG_XFRM_OFFLOAD

	struct sk_buff_head	xfrm_backlog;

#endif

#ifdef CONFIG_RPS

	/* input_queue_head should be written by cpu owning this struct,

	 * and only read by other cpus. Worth using a cache line.
@@ -3325,7 +3327,7 @@ int dev_get_phys_port_id(struct net_device *dev, 
int dev_get_phys_port_name(struct net_device *dev,

			   char *name, size_t len);

int dev_change_proto_down(struct net_device *dev, bool proto_down);

struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev);

struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);

struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,

				    struct netdev_queue *txq, int *ret);

include/net/xfrm.h

+19 −3
Original line number Diff line number Diff line
@@ -1051,6 +1051,7 @@ struct xfrm_offload { 
#define	XFRM_GSO_SEGMENT	16

#define	XFRM_GRO		32

#define	XFRM_ESP_NO_TRAILER	64

#define	XFRM_DEV_RESUME		128



	__u32			status;

#define CRYPTO_SUCCESS				1
@@ -1874,21 +1875,28 @@ static inline struct xfrm_state *xfrm_input_state(struct sk_buff *skb) 
{

	return skb->sp->xvec[skb->sp->len - 1];

}

#endif



static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb)

{

#ifdef CONFIG_XFRM

	struct sec_path *sp = skb->sp;



	if (!sp || !sp->olen || sp->len != sp->olen)

		return NULL;



	return &sp->ovec[sp->olen - 1];

}

#else

	return NULL;

#endif

}



void __net_init xfrm_dev_init(void);



#ifdef CONFIG_XFRM_OFFLOAD

struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features);

void xfrm_dev_resume(struct sk_buff *skb);

void xfrm_dev_backlog(struct softnet_data *sd);

struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again);

int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,

		       struct xfrm_user_offload *xuo);

bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
@@ -1929,7 +1937,15 @@ static inline void xfrm_dev_state_free(struct xfrm_state *x) 
	}

}

#else

static inline struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features)

static inline void xfrm_dev_resume(struct sk_buff *skb)

{

}



static inline void xfrm_dev_backlog(struct softnet_data *sd)

{

}



static inline struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)

{

	return skb;

}

net/core/dev.c

+11 −5
Original line number Diff line number Diff line
@@ -3059,7 +3059,7 @@ int skb_csum_hwoffload_help(struct sk_buff *skb, 
}

EXPORT_SYMBOL(skb_csum_hwoffload_help);



static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)

static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)

{

	netdev_features_t features;
@@ -3099,7 +3099,7 @@ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device 
		}

	}



	skb = validate_xmit_xfrm(skb, features);

	skb = validate_xmit_xfrm(skb, features, again);



	return skb;
@@ -3110,7 +3110,7 @@ out_null: 
	return NULL;

}



struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)

struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)

{

	struct sk_buff *next, *head = NULL, *tail;
@@ -3121,7 +3121,7 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d 
		/* in case skb wont be segmented, point to itself */

		skb->prev = skb;



		skb = validate_xmit_skb(skb, dev);

		skb = validate_xmit_skb(skb, dev, again);

		if (!skb)

			continue;
@@ -3448,6 +3448,7 @@ static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) 
	struct netdev_queue *txq;

	struct Qdisc *q;

	int rc = -ENOMEM;

	bool again = false;



	skb_reset_mac_header(skb);
@@ -3509,7 +3510,7 @@ static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) 
				     XMIT_RECURSION_LIMIT))

				goto recursion_alert;



			skb = validate_xmit_skb(skb, dev);

			skb = validate_xmit_skb(skb, dev, &again);

			if (!skb)

				goto out;
@@ -4193,6 +4194,8 @@ static __latent_entropy void net_tx_action(struct softirq_action *h) 
				spin_unlock(root_lock);

		}

	}



	xfrm_dev_backlog(sd);

}



#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
@@ -8874,6 +8877,9 @@ static int __init net_dev_init(void) 


		skb_queue_head_init(&sd->input_pkt_queue);

		skb_queue_head_init(&sd->process_queue);

#ifdef CONFIG_XFRM_OFFLOAD

		skb_queue_head_init(&sd->xfrm_backlog);

#endif

		INIT_LIST_HEAD(&sd->poll_list);

		sd->output_queue_tailp = &sd->output_queue;

#ifdef CONFIG_RPS

net/ipv4/esp4.c

+21 −3
Original line number Diff line number Diff line
@@ -121,15 +121,33 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp) 
static void esp_output_done(struct crypto_async_request *base, int err)

{

	struct sk_buff *skb = base->data;

	struct xfrm_offload *xo = xfrm_offload(skb);

	void *tmp;

	struct dst_entry *dst = skb_dst(skb);

	struct xfrm_state *x = dst->xfrm;

	struct xfrm_state *x;



	if (xo && (xo->flags & XFRM_DEV_RESUME))

		x = skb->sp->xvec[skb->sp->len - 1];

	else

		x = skb_dst(skb)->xfrm;



	tmp = ESP_SKB_CB(skb)->tmp;

	esp_ssg_unref(x, tmp);

	kfree(tmp);



	if (xo && (xo->flags & XFRM_DEV_RESUME)) {

		if (err) {

			XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);

			kfree_skb(skb);

			return;

		}



		skb_push(skb, skb->data - skb_mac_header(skb));

		secpath_reset(skb);

		xfrm_dev_resume(skb);

	} else {

		xfrm_output_resume(skb, err);

	}

}



/* Move ESP header back into place. */

static void esp_restore_header(struct sk_buff *skb, unsigned int offset)

net/ipv6/esp6.c

+21 −3
Original line number Diff line number Diff line
@@ -141,15 +141,33 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp) 
static void esp_output_done(struct crypto_async_request *base, int err)

{

	struct sk_buff *skb = base->data;

	struct xfrm_offload *xo = xfrm_offload(skb);

	void *tmp;

	struct dst_entry *dst = skb_dst(skb);

	struct xfrm_state *x = dst->xfrm;

	struct xfrm_state *x;



	if (xo && (xo->flags & XFRM_DEV_RESUME))

		x = skb->sp->xvec[skb->sp->len - 1];

	else

		x = skb_dst(skb)->xfrm;



	tmp = ESP_SKB_CB(skb)->tmp;

	esp_ssg_unref(x, tmp);

	kfree(tmp);



	if (xo && (xo->flags & XFRM_DEV_RESUME)) {

		if (err) {

			XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);

			kfree_skb(skb);

			return;

		}



		skb_push(skb, skb->data - skb_mac_header(skb));

		secpath_reset(skb);

		xfrm_dev_resume(skb);

	} else {

		xfrm_output_resume(skb, err);

	}

}



/* Move ESP header back into place. */

static void esp_restore_header(struct sk_buff *skb, unsigned int offset)

CRA Git | Maintained and supported by SUSTech CRA and CCSE