Commit e83e5bb1 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'net-sched-bulk-dequeue' 



Eric Dumazet says:

====================
net_sched: bulk dequeue and deferred drops

First patch adds an additional parameter to ->enqueue() qdisc method
so that drops can be done outside of critical section
(after locks are released).

Then fq_codel can have a small optimization to reduce number of cache
lines misses during a drop event
(possibly accumulating hundreds of packets to be freed).

A small htb change exports the backlog in class dumps.

Final patch adds bulk dequeue to qdiscs that were lacking this feature.

This series brings a nice qdisc performance increase (more than 80 %
in some cases).
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 36195d86 4d202a0d

include/net/codel_qdisc.h

+1 −0
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ 
/* Qdiscs using codel plugin must use codel_skb_cb in their own cb[] */

struct codel_skb_cb {

	codel_time_t enqueue_time;

	unsigned int mem_usage;

};



static struct codel_skb_cb *get_codel_cb(const struct sk_buff *skb)

include/net/sch_generic.h

+34 −14
Original line number Diff line number Diff line
@@ -37,8 +37,10 @@ struct qdisc_size_table { 
};



struct Qdisc {

	int 			(*enqueue)(struct sk_buff *skb, struct Qdisc *dev);

	struct sk_buff *	(*dequeue)(struct Qdisc *dev);

	int 			(*enqueue)(struct sk_buff *skb,

					   struct Qdisc *sch,

					   struct sk_buff **to_free);

	struct sk_buff *	(*dequeue)(struct Qdisc *sch);

	unsigned int		flags;

#define TCQ_F_BUILTIN		1

#define TCQ_F_INGRESS		2
@@ -73,13 +75,14 @@ struct Qdisc { 
	/*

	 * For performance sake on SMP, we put highly modified fields at the end

	 */

	struct Qdisc		*next_sched ____cacheline_aligned_in_smp;

	struct sk_buff		*gso_skb;

	unsigned long		state;

	struct sk_buff		*gso_skb ____cacheline_aligned_in_smp;

	struct sk_buff_head	q;

	struct gnet_stats_basic_packed bstats;

	seqcount_t		running;

	struct gnet_stats_queue	qstats;

	unsigned long		state;

	struct Qdisc            *next_sched;

	struct sk_buff		*skb_bad_txq;

	struct rcu_head		rcu_head;

	int			padded;

	atomic_t		refcnt;
@@ -160,7 +163,9 @@ struct Qdisc_ops { 
	char			id[IFNAMSIZ];

	int			priv_size;



	int 			(*enqueue)(struct sk_buff *, struct Qdisc *);

	int 			(*enqueue)(struct sk_buff *skb,

					   struct Qdisc *sch,

					   struct sk_buff **to_free);

	struct sk_buff *	(*dequeue)(struct Qdisc *);

	struct sk_buff *	(*peek)(struct Qdisc *);
@@ -498,10 +503,11 @@ static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, 
#endif

}



static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)

static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,

				struct sk_buff **to_free)

{

	qdisc_calculate_pkt_len(skb, sch);

	return sch->enqueue(skb, sch);

	return sch->enqueue(skb, sch, to_free);

}



static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
@@ -626,24 +632,36 @@ static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 
	return __qdisc_dequeue_head(sch, &sch->q);

}



/* Instead of calling kfree_skb() while root qdisc lock is held,

 * queue the skb for future freeing at end of __dev_xmit_skb()

 */

static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free)

{

	skb->next = *to_free;

	*to_free = skb;

}



static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,

					      struct sk_buff_head *list)

						   struct sk_buff_head *list,

						   struct sk_buff **to_free)

{

	struct sk_buff *skb = __skb_dequeue(list);



	if (likely(skb != NULL)) {

		unsigned int len = qdisc_pkt_len(skb);



		qdisc_qstats_backlog_dec(sch, skb);

		kfree_skb(skb);

		__qdisc_drop(skb, to_free);

		return len;

	}



	return 0;

}



static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch)

static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch,

						 struct sk_buff **to_free)

{

	return __qdisc_queue_drop_head(sch, &sch->q);

	return __qdisc_queue_drop_head(sch, &sch->q, to_free);

}



static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)
@@ -724,9 +742,11 @@ static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 
	qdisc_qstats_drop(sch);

}



static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)



static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch,

			     struct sk_buff **to_free)

{

	kfree_skb(skb);

	__qdisc_drop(skb, to_free);

	qdisc_qstats_drop(sch);



	return NET_XMIT_DROP;

net/core/dev.c

+5 −2
Original line number Diff line number Diff line
@@ -3070,6 +3070,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 
				 struct netdev_queue *txq)

{

	spinlock_t *root_lock = qdisc_lock(q);

	struct sk_buff *to_free = NULL;

	bool contended;

	int rc;
@@ -3086,7 +3087,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 


	spin_lock(root_lock);

	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {

		kfree_skb(skb);

		__qdisc_drop(skb, &to_free);

		rc = NET_XMIT_DROP;

	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&

		   qdisc_run_begin(q)) {
@@ -3109,7 +3110,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 


		rc = NET_XMIT_SUCCESS;

	} else {

		rc = q->enqueue(skb, q) & NET_XMIT_MASK;

		rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;

		if (qdisc_run_begin(q)) {

			if (unlikely(contended)) {

				spin_unlock(&q->busylock);
@@ -3119,6 +3120,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 
		}

	}

	spin_unlock(root_lock);

	if (unlikely(to_free))

		kfree_skb_list(to_free);

	if (unlikely(contended))

		spin_unlock(&q->busylock);

	return rc;

net/sched/sch_atm.c

+5 −4
Original line number Diff line number Diff line
@@ -357,7 +357,8 @@ static struct tcf_proto __rcu **atm_tc_find_tcf(struct Qdisc *sch, 


/* --------------------------- Qdisc operations ---------------------------- */



static int atm_tc_enqueue(struct sk_buff *skb, struct Qdisc *sch)

static int atm_tc_enqueue(struct sk_buff *skb, struct Qdisc *sch,

			  struct sk_buff **to_free)

{

	struct atm_qdisc_data *p = qdisc_priv(sch);

	struct atm_flow_data *flow;
@@ -398,10 +399,10 @@ done: 
		switch (result) {

		case TC_ACT_QUEUED:

		case TC_ACT_STOLEN:

			kfree_skb(skb);

			__qdisc_drop(skb, to_free);

			return NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;

		case TC_ACT_SHOT:

			kfree_skb(skb);

			__qdisc_drop(skb, to_free);

			goto drop;

		case TC_ACT_RECLASSIFY:

			if (flow->excess)
@@ -413,7 +414,7 @@ done: 
#endif

	}



	ret = qdisc_enqueue(skb, flow->q);

	ret = qdisc_enqueue(skb, flow->q, to_free);

	if (ret != NET_XMIT_SUCCESS) {

drop: __maybe_unused

		if (net_xmit_drop_count(ret)) {

net/sched/sch_blackhole.c

+3 −2
Original line number Diff line number Diff line
@@ -17,9 +17,10 @@ 
#include <linux/skbuff.h>

#include <net/pkt_sched.h>



static int blackhole_enqueue(struct sk_buff *skb, struct Qdisc *sch)

static int blackhole_enqueue(struct sk_buff *skb, struct Qdisc *sch,

			     struct sk_buff **to_free)

{

	qdisc_drop(skb, sch);

	qdisc_drop(skb, sch, to_free);

	return NET_XMIT_SUCCESS;

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE