tcp/dccp: get rid of central timewait timer (789f558c) · Commits · 戴 / test

include/net/inet_timewait_sock.h

+9 −98

Original line number	Diff line number	Diff line
		@@ -31,67 +31,14 @@

		struct inet_hashinfo;

		#define INET_TWDR_RECYCLE_SLOTS_LOG 5
		#define INET_TWDR_RECYCLE_SLOTS (1 << INET_TWDR_RECYCLE_SLOTS_LOG)

		/*
		* If time > 4sec, it is "slow" path, no recycling is required,
		* so that we select tick to get range about 4 seconds.
		*/
		#if HZ <= 16 \|\| HZ > 4096
		# error Unsupported: HZ <= 16 or HZ > 4096
		#elif HZ <= 32
		# define INET_TWDR_RECYCLE_TICK (5 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
		#elif HZ <= 64
		# define INET_TWDR_RECYCLE_TICK (6 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
		#elif HZ <= 128
		# define INET_TWDR_RECYCLE_TICK (7 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
		#elif HZ <= 256
		# define INET_TWDR_RECYCLE_TICK (8 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
		#elif HZ <= 512
		# define INET_TWDR_RECYCLE_TICK (9 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
		#elif HZ <= 1024
		# define INET_TWDR_RECYCLE_TICK (10 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
		#elif HZ <= 2048
		# define INET_TWDR_RECYCLE_TICK (11 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
		#else
		# define INET_TWDR_RECYCLE_TICK (12 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
		#endif

		static inline u32 inet_tw_time_stamp(void)
		{
		return jiffies;
		}

		/* TIME_WAIT reaping mechanism. */
		#define INET_TWDR_TWKILL_SLOTS 8 /* Please keep this a power of 2. */

		#define INET_TWDR_TWKILL_QUOTA 100

		struct inet_timewait_death_row {
		/* Short-time timewait calendar */
		int twcal_hand;
		unsigned long twcal_jiffie;
		struct timer_list twcal_timer;
		struct hlist_head twcal_row[INET_TWDR_RECYCLE_SLOTS];

		spinlock_t death_lock;
		int tw_count;
		int period;
		u32 thread_slots;
		struct work_struct twkill_work;
		struct timer_list tw_timer;
		int slot;
		struct hlist_head cells[INET_TWDR_TWKILL_SLOTS];
		struct inet_hashinfo *hashinfo;
		atomic_t tw_count;

		struct inet_hashinfo *hashinfo ____cacheline_aligned_in_smp;
		int sysctl_tw_recycle;
		int sysctl_max_tw_buckets;
		};

		void inet_twdr_hangman(unsigned long data);
		void inet_twdr_twkill_work(struct work_struct *work);
		void inet_twdr_twcal_tick(unsigned long data);

		struct inet_bind_bucket;

		/*
		@@ -133,52 +80,18 @@ struct inet_timewait_sock {
		__be16 tw_sport;
		kmemcheck_bitfield_begin(flags);
		/* And these are ours. */
		unsigned int tw_pad0 : 1, /* 1 bit hole */
		unsigned int tw_kill : 1,
		tw_transparent : 1,
		tw_flowlabel : 20,
		tw_pad : 2, /* 2 bits hole */
		tw_tos : 8;
		kmemcheck_bitfield_end(flags);
		u32 tw_ttd;
		struct timer_list tw_timer;
		struct inet_bind_bucket *tw_tb;
		struct hlist_node tw_death_node;
		struct inet_timewait_death_row *tw_dr;
		};
		#define tw_tclass tw_tos

		static inline int inet_twsk_dead_hashed(const struct inet_timewait_sock *tw)
		{
		return !hlist_unhashed(&tw->tw_death_node);
		}

		static inline void inet_twsk_dead_node_init(struct inet_timewait_sock *tw)
		{
		tw->tw_death_node.pprev = NULL;
		}

		static inline void __inet_twsk_del_dead_node(struct inet_timewait_sock *tw)
		{
		__hlist_del(&tw->tw_death_node);
		inet_twsk_dead_node_init(tw);
		}

		static inline int inet_twsk_del_dead_node(struct inet_timewait_sock *tw)
		{
		if (inet_twsk_dead_hashed(tw)) {
		__inet_twsk_del_dead_node(tw);
		return 1;
		}
		return 0;
		}

		#define inet_twsk_for_each(tw, node, head) \
		hlist_nulls_for_each_entry(tw, node, head, tw_node)

		#define inet_twsk_for_each_inmate(tw, jail) \
		hlist_for_each_entry(tw, jail, tw_death_node)

		#define inet_twsk_for_each_inmate_safe(tw, safe, jail) \
		hlist_for_each_entry_safe(tw, safe, jail, tw_death_node)

		static inline struct inet_timewait_sock inet_twsk(const struct sock sk)
		{
		return (struct inet_timewait_sock *)sk;
		@@ -193,16 +106,14 @@ int inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
		struct inet_hashinfo *hashinfo);

		struct inet_timewait_sock inet_twsk_alloc(const struct sock sk,
		struct inet_timewait_death_row *dr,
		const int state);

		void __inet_twsk_hashdance(struct inet_timewait_sock tw, struct sock sk,
		struct inet_hashinfo *hashinfo);

		void inet_twsk_schedule(struct inet_timewait_sock *tw,
		struct inet_timewait_death_row *twdr,
		const int timeo, const int timewait_len);
		void inet_twsk_deschedule(struct inet_timewait_sock *tw,
		struct inet_timewait_death_row *twdr);
		void inet_twsk_schedule(struct inet_timewait_sock *tw, const int timeo);
		void inet_twsk_deschedule(struct inet_timewait_sock *tw);

		void inet_twsk_purge(struct inet_hashinfo *hashinfo,
		struct inet_timewait_death_row *twdr, int family);

net/dccp/minisocks.c

+3 −16

Original line number	Diff line number	Diff line
		@@ -27,28 +27,16 @@

		struct inet_timewait_death_row dccp_death_row = {
		.sysctl_max_tw_buckets = NR_FILE * 2,
		.period = DCCP_TIMEWAIT_LEN / INET_TWDR_TWKILL_SLOTS,
		.death_lock = __SPIN_LOCK_UNLOCKED(dccp_death_row.death_lock),
		.hashinfo = &dccp_hashinfo,
		.tw_timer = TIMER_INITIALIZER(inet_twdr_hangman, 0,
		(unsigned long)&dccp_death_row),
		.twkill_work = __WORK_INITIALIZER(dccp_death_row.twkill_work,
		inet_twdr_twkill_work),
		/* Short-time timewait calendar */

		.twcal_hand = -1,
		.twcal_timer = TIMER_INITIALIZER(inet_twdr_twcal_tick, 0,
		(unsigned long)&dccp_death_row),
		};

		EXPORT_SYMBOL_GPL(dccp_death_row);

		void dccp_time_wait(struct sock *sk, int state, int timeo)
		{
		struct inet_timewait_sock *tw = NULL;
		struct inet_timewait_sock *tw;

		if (dccp_death_row.tw_count < dccp_death_row.sysctl_max_tw_buckets)
		tw = inet_twsk_alloc(sk, state);
		tw = inet_twsk_alloc(sk, &dccp_death_row, state);

		if (tw != NULL) {
		const struct inet_connection_sock *icsk = inet_csk(sk);
		@@ -71,8 +59,7 @@ void dccp_time_wait(struct sock *sk, int state, int timeo)
		if (state == DCCP_TIME_WAIT)
		timeo = DCCP_TIMEWAIT_LEN;

		inet_twsk_schedule(tw, &dccp_death_row, timeo,
		DCCP_TIMEWAIT_LEN);
		inet_twsk_schedule(tw, timeo);
		inet_twsk_put(tw);
		} else {
		/* Sorry, if we're out of memory, just CLOSE this

net/ipv4/inet_diag.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -248,7 +248,7 @@ static int inet_twsk_diag_fill(struct sock *sk,
		struct inet_timewait_sock *tw = inet_twsk(sk);
		struct inet_diag_msg *r;
		struct nlmsghdr *nlh;
		s32 tmo;
		long tmo;

		nlh = nlmsg_put(skb, portid, seq, unlh->nlmsg_type, sizeof(*r),
		nlmsg_flags);
		@@ -258,7 +258,7 @@ static int inet_twsk_diag_fill(struct sock *sk,
		r = nlmsg_data(nlh);
		BUG_ON(tw->tw_state != TCP_TIME_WAIT);

		tmo = tw->tw_ttd - inet_tw_time_stamp();
		tmo = tw->tw_timer.expires - jiffies;
		if (tmo < 0)
		tmo = 0;

net/ipv4/inet_hashtables.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -388,7 +388,7 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row,
		*twp = tw;
		} else if (tw) {
		/* Silly. Should hash-dance instead... */
		inet_twsk_deschedule(tw, death_row);
		inet_twsk_deschedule(tw);

		inet_twsk_put(tw);
		}
		@@ -565,7 +565,7 @@ ok:
		spin_unlock(&head->lock);

		if (tw) {
		inet_twsk_deschedule(tw, death_row);
		inet_twsk_deschedule(tw);
		while (twrefcnt) {
		twrefcnt--;
		inet_twsk_put(tw);

net/ipv4/inet_timewait_sock.c

+36 −234

Original line number	Diff line number	Diff line
		@@ -67,9 +67,9 @@ int inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
		}

		/* Must be called with locally disabled BHs. */
		static void __inet_twsk_kill(struct inet_timewait_sock *tw,
		struct inet_hashinfo *hashinfo)
		static void inet_twsk_kill(struct inet_timewait_sock *tw)
		{
		struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
		struct inet_bind_hashbucket *bhead;
		int refcnt;
		/* Unlink from established hashes. */
		@@ -89,6 +89,8 @@ static void __inet_twsk_kill(struct inet_timewait_sock *tw,

		BUG_ON(refcnt >= atomic_read(&tw->tw_refcnt));
		atomic_sub(refcnt, &tw->tw_refcnt);
		atomic_dec(&tw->tw_dr->tw_count);
		inet_twsk_put(tw);
		}

		void inet_twsk_free(struct inet_timewait_sock *tw)
		@@ -168,16 +170,34 @@ void __inet_twsk_hashdance(struct inet_timewait_sock tw, struct sock sk,
		}
		EXPORT_SYMBOL_GPL(__inet_twsk_hashdance);

		struct inet_timewait_sock inet_twsk_alloc(const struct sock sk, const int state)
		void tw_timer_handler(unsigned long data)
		{
		struct inet_timewait_sock tw = (struct inet_timewait_sock )data;

		if (tw->tw_kill)
		NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITKILLED);
		else
		NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITED);
		inet_twsk_kill(tw);
		}

		struct inet_timewait_sock inet_twsk_alloc(const struct sock sk,
		struct inet_timewait_death_row *dr,
		const int state)
		{
		struct inet_timewait_sock *tw =
		kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
		struct inet_timewait_sock *tw;

		if (atomic_read(&dr->tw_count) >= dr->sysctl_max_tw_buckets)
		return NULL;

		tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
		GFP_ATOMIC);
		if (tw) {
		const struct inet_sock *inet = inet_sk(sk);

		kmemcheck_annotate_bitfield(tw, flags);

		tw->tw_dr = dr;
		/* Give us an identity. */
		tw->tw_daddr = inet->inet_daddr;
		tw->tw_rcv_saddr = inet->inet_rcv_saddr;
		@@ -196,13 +216,14 @@ struct inet_timewait_sock inet_twsk_alloc(const struct sock sk, const int stat
		tw->tw_prot = sk->sk_prot_creator;
		atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
		twsk_net_set(tw, sock_net(sk));
		setup_timer(&tw->tw_timer, tw_timer_handler, (unsigned long)tw);
		/*
		* Because we use RCU lookups, we should not set tw_refcnt
		* to a non null value before everything is setup for this
		* timewait socket.
		*/
		atomic_set(&tw->tw_refcnt, 0);
		inet_twsk_dead_node_init(tw);

		__module_get(tw->tw_prot->owner);
		}

		@@ -210,139 +231,20 @@ struct inet_timewait_sock inet_twsk_alloc(const struct sock sk, const int stat
		}
		EXPORT_SYMBOL_GPL(inet_twsk_alloc);

		/* Returns non-zero if quota exceeded. */
		static int inet_twdr_do_twkill_work(struct inet_timewait_death_row *twdr,
		const int slot)
		{
		struct inet_timewait_sock *tw;
		unsigned int killed;
		int ret;

		/* NOTE: compare this to previous version where lock
		* was released after detaching chain. It was racy,
		* because tw buckets are scheduled in not serialized context
		* in 2.3 (with netfilter), and with softnet it is common, because
		* soft irqs are not sequenced.
		*/
		killed = 0;
		ret = 0;
		rescan:
		inet_twsk_for_each_inmate(tw, &twdr->cells[slot]) {
		__inet_twsk_del_dead_node(tw);
		spin_unlock(&twdr->death_lock);
		__inet_twsk_kill(tw, twdr->hashinfo);
		#ifdef CONFIG_NET_NS
		NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITED);
		#endif
		inet_twsk_put(tw);
		killed++;
		spin_lock(&twdr->death_lock);
		if (killed > INET_TWDR_TWKILL_QUOTA) {
		ret = 1;
		break;
		}

		/* While we dropped twdr->death_lock, another cpu may have
		* killed off the next TW bucket in the list, therefore
		* do a fresh re-read of the hlist head node with the
		* lock reacquired. We still use the hlist traversal
		* macro in order to get the prefetches.
		*/
		goto rescan;
		}

		twdr->tw_count -= killed;
		#ifndef CONFIG_NET_NS
		NET_ADD_STATS_BH(&init_net, LINUX_MIB_TIMEWAITED, killed);
		#endif
		return ret;
		}

		void inet_twdr_hangman(unsigned long data)
		{
		struct inet_timewait_death_row *twdr;
		unsigned int need_timer;

		twdr = (struct inet_timewait_death_row *)data;
		spin_lock(&twdr->death_lock);

		if (twdr->tw_count == 0)
		goto out;

		need_timer = 0;
		if (inet_twdr_do_twkill_work(twdr, twdr->slot)) {
		twdr->thread_slots \|= (1 << twdr->slot);
		schedule_work(&twdr->twkill_work);
		need_timer = 1;
		} else {
		/* We purged the entire slot, anything left? */
		if (twdr->tw_count)
		need_timer = 1;
		twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1));
		}
		if (need_timer)
		mod_timer(&twdr->tw_timer, jiffies + twdr->period);
		out:
		spin_unlock(&twdr->death_lock);
		}
		EXPORT_SYMBOL_GPL(inet_twdr_hangman);

		void inet_twdr_twkill_work(struct work_struct *work)
		{
		struct inet_timewait_death_row *twdr =
		container_of(work, struct inet_timewait_death_row, twkill_work);
		int i;

		BUILD_BUG_ON((INET_TWDR_TWKILL_SLOTS - 1) >
		(sizeof(twdr->thread_slots) * 8));

		while (twdr->thread_slots) {
		spin_lock_bh(&twdr->death_lock);
		for (i = 0; i < INET_TWDR_TWKILL_SLOTS; i++) {
		if (!(twdr->thread_slots & (1 << i)))
		continue;

		while (inet_twdr_do_twkill_work(twdr, i) != 0) {
		if (need_resched()) {
		spin_unlock_bh(&twdr->death_lock);
		schedule();
		spin_lock_bh(&twdr->death_lock);
		}
		}

		twdr->thread_slots &= ~(1 << i);
		}
		spin_unlock_bh(&twdr->death_lock);
		}
		}
		EXPORT_SYMBOL_GPL(inet_twdr_twkill_work);

		/* These are always called from BH context. See callers in
		* tcp_input.c to verify this.
		*/

		/* This is for handling early-kills of TIME_WAIT sockets. */
		void inet_twsk_deschedule(struct inet_timewait_sock *tw,
		struct inet_timewait_death_row *twdr)
		void inet_twsk_deschedule(struct inet_timewait_sock *tw)
		{
		spin_lock(&twdr->death_lock);
		if (inet_twsk_del_dead_node(tw)) {
		inet_twsk_put(tw);
		if (--twdr->tw_count == 0)
		del_timer(&twdr->tw_timer);
		}
		spin_unlock(&twdr->death_lock);
		__inet_twsk_kill(tw, twdr->hashinfo);
		if (del_timer_sync(&tw->tw_timer))
		inet_twsk_kill(tw);
		}
		EXPORT_SYMBOL(inet_twsk_deschedule);

		void inet_twsk_schedule(struct inet_timewait_sock *tw,
		struct inet_timewait_death_row *twdr,
		const int timeo, const int timewait_len)
		void inet_twsk_schedule(struct inet_timewait_sock *tw, const int timeo)
		{
		struct hlist_head *list;
		int slot;

		/* timeout := RTO * 3.5
		*
		* 3.5 = 1+2+0.5 to wait for two retransmits.
		@@ -367,115 +269,15 @@ void inet_twsk_schedule(struct inet_timewait_sock *tw,
		* is greater than TS tick!) and detect old duplicates with help
		* of PAWS.
		*/
		slot = (timeo + (1 << INET_TWDR_RECYCLE_TICK) - 1) >> INET_TWDR_RECYCLE_TICK;

		spin_lock(&twdr->death_lock);

		/* Unlink it, if it was scheduled */
		if (inet_twsk_del_dead_node(tw))
		twdr->tw_count--;
		else
		tw->tw_kill = timeo <= 4*HZ;
		if (!mod_timer_pinned(&tw->tw_timer, jiffies + timeo)) {
		atomic_inc(&tw->tw_refcnt);

		if (slot >= INET_TWDR_RECYCLE_SLOTS) {
		/* Schedule to slow timer */
		if (timeo >= timewait_len) {
		slot = INET_TWDR_TWKILL_SLOTS - 1;
		} else {
		slot = DIV_ROUND_UP(timeo, twdr->period);
		if (slot >= INET_TWDR_TWKILL_SLOTS)
		slot = INET_TWDR_TWKILL_SLOTS - 1;
		atomic_inc(&tw->tw_dr->tw_count);
		}
		tw->tw_ttd = inet_tw_time_stamp() + timeo;
		slot = (twdr->slot + slot) & (INET_TWDR_TWKILL_SLOTS - 1);
		list = &twdr->cells[slot];
		} else {
		tw->tw_ttd = inet_tw_time_stamp() + (slot << INET_TWDR_RECYCLE_TICK);

		if (twdr->twcal_hand < 0) {
		twdr->twcal_hand = 0;
		twdr->twcal_jiffie = jiffies;
		twdr->twcal_timer.expires = twdr->twcal_jiffie +
		(slot << INET_TWDR_RECYCLE_TICK);
		add_timer(&twdr->twcal_timer);
		} else {
		if (time_after(twdr->twcal_timer.expires,
		jiffies + (slot << INET_TWDR_RECYCLE_TICK)))
		mod_timer(&twdr->twcal_timer,
		jiffies + (slot << INET_TWDR_RECYCLE_TICK));
		slot = (twdr->twcal_hand + slot) & (INET_TWDR_RECYCLE_SLOTS - 1);
		}
		list = &twdr->twcal_row[slot];
		}

		hlist_add_head(&tw->tw_death_node, list);

		if (twdr->tw_count++ == 0)
		mod_timer(&twdr->tw_timer, jiffies + twdr->period);
		spin_unlock(&twdr->death_lock);
		}
		EXPORT_SYMBOL_GPL(inet_twsk_schedule);

		void inet_twdr_twcal_tick(unsigned long data)
		{
		struct inet_timewait_death_row *twdr;
		int n, slot;
		unsigned long j;
		unsigned long now = jiffies;
		int killed = 0;
		int adv = 0;

		twdr = (struct inet_timewait_death_row *)data;

		spin_lock(&twdr->death_lock);
		if (twdr->twcal_hand < 0)
		goto out;

		slot = twdr->twcal_hand;
		j = twdr->twcal_jiffie;

		for (n = 0; n < INET_TWDR_RECYCLE_SLOTS; n++) {
		if (time_before_eq(j, now)) {
		struct hlist_node *safe;
		struct inet_timewait_sock *tw;

		inet_twsk_for_each_inmate_safe(tw, safe,
		&twdr->twcal_row[slot]) {
		__inet_twsk_del_dead_node(tw);
		__inet_twsk_kill(tw, twdr->hashinfo);
		#ifdef CONFIG_NET_NS
		NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITKILLED);
		#endif
		inet_twsk_put(tw);
		killed++;
		}
		} else {
		if (!adv) {
		adv = 1;
		twdr->twcal_jiffie = j;
		twdr->twcal_hand = slot;
		}

		if (!hlist_empty(&twdr->twcal_row[slot])) {
		mod_timer(&twdr->twcal_timer, j);
		goto out;
		}
		}
		j += 1 << INET_TWDR_RECYCLE_TICK;
		slot = (slot + 1) & (INET_TWDR_RECYCLE_SLOTS - 1);
		}
		twdr->twcal_hand = -1;

		out:
		if ((twdr->tw_count -= killed) == 0)
		del_timer(&twdr->tw_timer);
		#ifndef CONFIG_NET_NS
		NET_ADD_STATS_BH(&init_net, LINUX_MIB_TIMEWAITKILLED, killed);
		#endif
		spin_unlock(&twdr->death_lock);
		}
		EXPORT_SYMBOL_GPL(inet_twdr_twcal_tick);

		void inet_twsk_purge(struct inet_hashinfo *hashinfo,
		struct inet_timewait_death_row *twdr, int family)
		{
		@@ -509,7 +311,7 @@ restart:

		rcu_read_unlock();
		local_bh_disable();
		inet_twsk_deschedule(tw, twdr);
		inet_twsk_deschedule(tw);
		local_bh_enable();
		inet_twsk_put(tw);
		goto restart_rcu;

Admin message