Commit 8685255e authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'sch_act_lockless' 



Eric Dumazet says:

====================
net_sched: act: lockless operation

As mentioned by Alexei last week in Budapest, it is a bit weird
to take a spinlock in order to drop a packet in a tc filter...

Lets add percpu infra for tc actions and use it for gact & mirred.

Before changes, my host with 8 RX queues was handling 5 Mpps with gact,
and more than 11 Mpps after.

Mirred change is not yet visible if ifb+qdisc is used, as ifb is
not yet multi queue enabled, but is a step forward.
====================

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 0a6d4245 2ee22a90

include/net/act_api.h

+14 −1
Original line number Diff line number Diff line
@@ -21,6 +21,8 @@ struct tcf_common { 
	struct gnet_stats_rate_est64	tcfc_rate_est;

	spinlock_t			tcfc_lock;

	struct rcu_head			tcfc_rcu;

	struct gnet_stats_basic_cpu __percpu *cpu_bstats;

	struct gnet_stats_queue __percpu *cpu_qstats;

};

#define tcf_head	common.tcfc_head

#define tcf_index	common.tcfc_index
@@ -68,6 +70,17 @@ static inline void tcf_hashinfo_destroy(struct tcf_hashinfo *hf) 
	kfree(hf->htab);

}



/* Update lastuse only if needed, to avoid dirtying a cache line.

 * We use a temp variable to avoid fetching jiffies twice.

 */

static inline void tcf_lastuse_update(struct tcf_t *tm)

{

	unsigned long now = jiffies;



	if (tm->lastuse != now)

		tm->lastuse = now;

}



#ifdef CONFIG_NET_CLS_ACT



#define ACT_P_CREATED 1
@@ -103,7 +116,7 @@ int tcf_hash_release(struct tc_action *a, int bind); 
u32 tcf_hash_new_index(struct tcf_hashinfo *hinfo);

int tcf_hash_check(u32 index, struct tc_action *a, int bind);

int tcf_hash_create(u32 index, struct nlattr *est, struct tc_action *a,

		    int size, int bind);

		    int size, int bind, bool cpustats);

void tcf_hash_cleanup(struct tc_action *a, struct nlattr *est);

void tcf_hash_insert(struct tc_action *a);

include/net/sch_generic.h

+21 −10
Original line number Diff line number Diff line
@@ -513,17 +513,20 @@ static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 
	bstats->packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;

}



static inline void qdisc_bstats_update_cpu(struct Qdisc *sch,

static inline void bstats_cpu_update(struct gnet_stats_basic_cpu *bstats,

				     const struct sk_buff *skb)

{

	struct gnet_stats_basic_cpu *bstats =

				this_cpu_ptr(sch->cpu_bstats);



	u64_stats_update_begin(&bstats->syncp);

	bstats_update(&bstats->bstats, skb);

	u64_stats_update_end(&bstats->syncp);

}



static inline void qdisc_bstats_cpu_update(struct Qdisc *sch,

					   const struct sk_buff *skb)

{

	bstats_cpu_update(this_cpu_ptr(sch->cpu_bstats), skb);

}



static inline void qdisc_bstats_update(struct Qdisc *sch,

				       const struct sk_buff *skb)

{
@@ -547,16 +550,24 @@ static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) 
	sch->qstats.drops += count;

}



static inline void qdisc_qstats_drop(struct Qdisc *sch)

static inline void qstats_drop_inc(struct gnet_stats_queue *qstats)

{

	qstats->drops++;

}



static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats)

{

	sch->qstats.drops++;

	qstats->overlimits++;

}



static inline void qdisc_qstats_drop_cpu(struct Qdisc *sch)

static inline void qdisc_qstats_drop(struct Qdisc *sch)

{

	struct gnet_stats_queue *qstats = this_cpu_ptr(sch->cpu_qstats);

	qstats_drop_inc(&sch->qstats);

}



	qstats->drops++;

static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch)

{

	qstats_drop_inc(this_cpu_ptr(sch->cpu_qstats));

}



static inline void qdisc_qstats_overlimit(struct Qdisc *sch)

include/net/tc_act/tc_gact.h

+4 −3
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ struct tcf_gact { 
	u16			tcfg_ptype;

	u16			tcfg_pval;

	int			tcfg_paction;

	atomic_t		packets;

#endif

};

#define to_gact(a) \

include/net/tc_act/tc_mirred.h

+1 −1
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ struct tcf_mirred { 
	int			tcfm_eaction;

	int			tcfm_ifindex;

	int			tcfm_ok_push;

	struct net_device	*tcfm_dev;

	struct net_device __rcu	*tcfm_dev;

	struct list_head	tcfm_list;

};

#define to_mirred(a) \

net/core/dev.c

+2 −2
Original line number Diff line number Diff line
@@ -3646,7 +3646,7 @@ static inline struct sk_buff *handle_ing(struct sk_buff *skb, 


	qdisc_skb_cb(skb)->pkt_len = skb->len;

	skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);

	qdisc_bstats_update_cpu(cl->q, skb);

	qdisc_bstats_cpu_update(cl->q, skb);



	switch (tc_classify(skb, cl, &cl_res)) {

	case TC_ACT_OK:
@@ -3654,7 +3654,7 @@ static inline struct sk_buff *handle_ing(struct sk_buff *skb, 
		skb->tc_index = TC_H_MIN(cl_res.classid);

		break;

	case TC_ACT_SHOT:

		qdisc_qstats_drop_cpu(cl->q);

		qdisc_qstats_cpu_drop(cl->q);

	case TC_ACT_STOLEN:

	case TC_ACT_QUEUED:

		kfree_skb(skb);

CRA Git | Maintained and supported by SUSTech CRA and CCSE