Commit 37437bb2 authored by David S. Miller's avatar David S. Miller
Browse files

pkt_sched: Schedule qdiscs instead of netdev_queue. 



When we have shared qdiscs, packets come out of the qdiscs
for multiple transmit queues.

Therefore it doesn't make any sense to schedule the transmit
queue when logically we cannot know ahead of time the TX
queue of the SKB that the qdisc->dequeue() will give us.

Just for sanity I added a BUG check to make sure we never
get into a state where the noop_qdisc is scheduled.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7698b4fc

include/linux/netdevice.h

+5 −7
Original line number Diff line number Diff line
@@ -275,7 +275,6 @@ enum netdev_state_t 
{

	__LINK_STATE_START,

	__LINK_STATE_PRESENT,

	__LINK_STATE_SCHED,

	__LINK_STATE_NOCARRIER,

	__LINK_STATE_LINKWATCH_PENDING,

	__LINK_STATE_DORMANT,
@@ -452,7 +451,6 @@ struct netdev_queue { 
	int			xmit_lock_owner;

	struct Qdisc		*qdisc_sleeping;

	struct list_head	qdisc_list;

	struct netdev_queue	*next_sched;

} ____cacheline_aligned_in_smp;



/*
@@ -969,7 +967,7 @@ static inline int unregister_gifconf(unsigned int family) 
 */

struct softnet_data

{

	struct netdev_queue	*output_queue;

	struct Qdisc		*output_queue;

	struct sk_buff_head	input_pkt_queue;

	struct list_head	poll_list;

	struct sk_buff		*completion_queue;
@@ -984,12 +982,12 @@ DECLARE_PER_CPU(struct softnet_data,softnet_data); 


#define HAVE_NETIF_QUEUE



extern void __netif_schedule(struct netdev_queue *txq);

extern void __netif_schedule(struct Qdisc *q);



static inline void netif_schedule_queue(struct netdev_queue *txq)

{

	if (!test_bit(__QUEUE_STATE_XOFF, &txq->state))

		__netif_schedule(txq);

		__netif_schedule(txq->qdisc);

}



static inline void netif_tx_schedule_all(struct net_device *dev)
@@ -1042,7 +1040,7 @@ static inline void netif_tx_wake_queue(struct netdev_queue *dev_queue) 
	}

#endif

	if (test_and_clear_bit(__QUEUE_STATE_XOFF, &dev_queue->state))

		__netif_schedule(dev_queue);

		__netif_schedule(dev_queue->qdisc);

}



static inline void netif_wake_queue(struct net_device *dev)
@@ -1186,7 +1184,7 @@ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index) 
		return;

#endif

	if (test_and_clear_bit(__QUEUE_STATE_XOFF, &txq->state))

		__netif_schedule(txq);

		__netif_schedule(txq->qdisc);

}



/**

include/net/pkt_sched.h

+4 −7
Original line number Diff line number Diff line
@@ -84,15 +84,12 @@ extern struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, 
		struct nlattr *tab);

extern void qdisc_put_rtab(struct qdisc_rate_table *tab);



extern void __qdisc_run(struct netdev_queue *txq);

extern void __qdisc_run(struct Qdisc *q);



static inline void qdisc_run(struct netdev_queue *txq)

static inline void qdisc_run(struct Qdisc *q)

{

	struct Qdisc *q = txq->qdisc;



	if (!netif_tx_queue_stopped(txq) &&

	    !test_and_set_bit(__QDISC_STATE_RUNNING, &q->state))

		__qdisc_run(txq);

	if (!test_and_set_bit(__QDISC_STATE_RUNNING, &q->state))

		__qdisc_run(q);

}



extern int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp,

include/net/sch_generic.h

+2 −0
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ struct qdisc_rate_table 
enum qdisc_state_t

{

	__QDISC_STATE_RUNNING,

	__QDISC_STATE_SCHED,

};



struct Qdisc
@@ -45,6 +46,7 @@ struct Qdisc 
	struct sk_buff		*gso_skb;

	struct sk_buff_head	q;

	struct netdev_queue	*dev_queue;

	struct Qdisc		*next_sched;

	struct list_head	list;



	struct gnet_stats_basic	bstats;

net/core/dev.c

+28 −40
Original line number Diff line number Diff line
@@ -1323,18 +1323,18 @@ static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 
}





void __netif_schedule(struct netdev_queue *txq)

void __netif_schedule(struct Qdisc *q)

{

	struct net_device *dev = txq->dev;

	BUG_ON(q == &noop_qdisc);



	if (!test_and_set_bit(__LINK_STATE_SCHED, &dev->state)) {

	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) {

		struct softnet_data *sd;

		unsigned long flags;



		local_irq_save(flags);

		sd = &__get_cpu_var(softnet_data);

		txq->next_sched = sd->output_queue;

		sd->output_queue = txq;

		q->next_sched = sd->output_queue;

		sd->output_queue = q;

		raise_softirq_irqoff(NET_TX_SOFTIRQ);

		local_irq_restore(flags);

	}
@@ -1771,38 +1771,24 @@ gso: 
	rcu_read_lock_bh();



	txq = dev_pick_tx(dev, skb);

	spin_lock_prefetch(&txq->lock);



	/* Updates of qdisc are serialized by queue->lock.

	 * The struct Qdisc which is pointed to by qdisc is now a

	 * rcu structure - it may be accessed without acquiring

	 * a lock (but the structure may be stale.) The freeing of the

	 * qdisc will be deferred until it's known that there are no

	 * more references to it.

	 *

	 * If the qdisc has an enqueue function, we still need to

	 * hold the queue->lock before calling it, since queue->lock

	 * also serializes access to the device queue.

	 */



	q = rcu_dereference(txq->qdisc);



#ifdef CONFIG_NET_CLS_ACT

	skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS);

#endif

	if (q->enqueue) {

		/* Grab device queue */

		spin_lock(&txq->lock);

		q = txq->qdisc;

		if (q->enqueue) {

		spinlock_t *root_lock = qdisc_root_lock(q);



		spin_lock(root_lock);



		rc = q->enqueue(skb, q);

			qdisc_run(txq);

			spin_unlock(&txq->lock);

		qdisc_run(q);



		spin_unlock(root_lock);



		rc = rc == NET_XMIT_BYPASS ? NET_XMIT_SUCCESS : rc;

		goto out;

	}

		spin_unlock(&txq->lock);

	}



	/* The device has no queue. Common case for software devices:

	   loopback, all the sorts of tunnels...
@@ -1974,7 +1960,7 @@ static void net_tx_action(struct softirq_action *h) 
	}



	if (sd->output_queue) {

		struct netdev_queue *head;

		struct Qdisc *head;



		local_irq_disable();

		head = sd->output_queue;
@@ -1982,18 +1968,20 @@ static void net_tx_action(struct softirq_action *h) 
		local_irq_enable();



		while (head) {

			struct netdev_queue *txq = head;

			struct net_device *dev = txq->dev;

			struct Qdisc *q = head;

			spinlock_t *root_lock;



			head = head->next_sched;



			smp_mb__before_clear_bit();

			clear_bit(__LINK_STATE_SCHED, &dev->state);

			clear_bit(__QDISC_STATE_SCHED, &q->state);



			if (spin_trylock(&txq->lock)) {

				qdisc_run(txq);

				spin_unlock(&txq->lock);

			root_lock = qdisc_root_lock(q);

			if (spin_trylock(root_lock)) {

				qdisc_run(q);

				spin_unlock(root_lock);

			} else {

				netif_schedule_queue(txq);

				__netif_schedule(q);

			}

		}

	}
@@ -4459,7 +4447,7 @@ static int dev_cpu_callback(struct notifier_block *nfb, 
			    void *ocpu)

{

	struct sk_buff **list_skb;

	struct netdev_queue **list_net;

	struct Qdisc **list_net;

	struct sk_buff *skb;

	unsigned int cpu, oldcpu = (unsigned long)ocpu;

	struct softnet_data *sd, *oldsd;

net/sched/sch_api.c

+1 −2
Original line number Diff line number Diff line
@@ -294,11 +294,10 @@ static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer) 
{

	struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,

						 timer);

	struct netdev_queue *txq = wd->qdisc->dev_queue;



	wd->qdisc->flags &= ~TCQ_F_THROTTLED;

	smp_wmb();

	netif_schedule_queue(txq);

	__netif_schedule(wd->qdisc);



	return HRTIMER_NORESTART;

}

CRA Git | Maintained and supported by SUSTech CRA and CCSE