Commit 960632ec authored by Aaron Conole's avatar Aaron Conole Committed by Pablo Neira Ayuso
Browse files

netfilter: convert hook list to an array 



This converts the storage and layout of netfilter hook entries from a
linked list to an array.  After this commit, hook entries will be
stored adjacent in memory.  The next pointer is no longer required.

The ops pointers are stored at the end of the array as they are only
used in the register/unregister path and in the legacy br_netfilter code.

nf_unregister_net_hooks() is slower than needed as it just calls
nf_unregister_net_hook in a loop (i.e. at least n synchronize_net()
calls), this will be addressed in followup patch.

Test setup:
 - ixgbe 10gbit
 - netperf UDP_STREAM, 64 byte packets
 - 5 hooks: (raw + mangle prerouting, mangle+filter input, inet filter):
empty mangle and raw prerouting, mangle and filter input hooks:
353.9
this patch:
364.2

Signed-off-by: default avatarAaron Conole <aconole@bytheb.org>
Signed-off-by: default avatarFlorian Westphal <fw@strlen.de>
Signed-off-by: default avatarPablo Neira Ayuso <pablo@netfilter.org>
parent 5fd02ebe

include/linux/netdevice.h

+1 −1
Original line number Diff line number Diff line
@@ -1811,7 +1811,7 @@ struct net_device { 
#endif

	struct netdev_queue __rcu *ingress_queue;

#ifdef CONFIG_NETFILTER_INGRESS

	struct nf_hook_entry __rcu *nf_hooks_ingress;

	struct nf_hook_entries __rcu *nf_hooks_ingress;

#endif



	unsigned char		broadcast[MAX_ADDR_LEN];

include/linux/netfilter.h

+23 −22
Original line number Diff line number Diff line
@@ -72,25 +72,32 @@ struct nf_hook_ops { 
};



struct nf_hook_entry {

	struct nf_hook_entry __rcu	*next;

	nf_hookfn			*hook;

	void				*priv;

	const struct nf_hook_ops	*orig_ops;

};



static inline void

nf_hook_entry_init(struct nf_hook_entry *entry,	const struct nf_hook_ops *ops)

{

	entry->next = NULL;

	entry->hook = ops->hook;

	entry->priv = ops->priv;

	entry->orig_ops = ops;

}

struct nf_hook_entries {

	u16				num_hook_entries;

	/* padding */

	struct nf_hook_entry		hooks[];



static inline int

nf_hook_entry_priority(const struct nf_hook_entry *entry)

	/* trailer: pointers to original orig_ops of each hook.

	 *

	 * This is not part of struct nf_hook_entry since its only

	 * needed in slow path (hook register/unregister).

	 *

	 * const struct nf_hook_ops     *orig_ops[]

	 */

};



static inline struct nf_hook_ops **nf_hook_entries_get_hook_ops(const struct nf_hook_entries *e)

{

	return entry->orig_ops->priority;

	unsigned int n = e->num_hook_entries;

	const void *hook_end;



	hook_end = &e->hooks[n]; /* this is *past* ->hooks[]! */



	return (struct nf_hook_ops **)hook_end;

}



static inline int
@@ -100,12 +107,6 @@ nf_hook_entry_hookfn(const struct nf_hook_entry *entry, struct sk_buff *skb, 
	return entry->hook(entry->priv, skb, state);

}



static inline const struct nf_hook_ops *

nf_hook_entry_ops(const struct nf_hook_entry *entry)

{

	return entry->orig_ops;

}



static inline void nf_hook_state_init(struct nf_hook_state *p,

				      unsigned int hook,

				      u_int8_t pf,
@@ -168,7 +169,7 @@ extern struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; 
#endif



int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state,

		 struct nf_hook_entry *entry);

		 const struct nf_hook_entries *e, unsigned int i);



/**

 *	nf_hook - call a netfilter hook
@@ -182,7 +183,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net, 
			  struct net_device *indev, struct net_device *outdev,

			  int (*okfn)(struct net *, struct sock *, struct sk_buff *))

{

	struct nf_hook_entry *hook_head;

	struct nf_hook_entries *hook_head;

	int ret = 1;



#ifdef HAVE_JUMP_LABEL
@@ -200,7 +201,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net, 
		nf_hook_state_init(&state, hook, pf, indev, outdev,

				   sk, net, okfn);



		ret = nf_hook_slow(skb, &state, hook_head);

		ret = nf_hook_slow(skb, &state, hook_head, 0);

	}

	rcu_read_unlock();

include/linux/netfilter_ingress.h

+2 −2
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ static inline bool nf_hook_ingress_active(const struct sk_buff *skb) 
/* caller must hold rcu_read_lock */

static inline int nf_hook_ingress(struct sk_buff *skb)

{

	struct nf_hook_entry *e = rcu_dereference(skb->dev->nf_hooks_ingress);

	struct nf_hook_entries *e = rcu_dereference(skb->dev->nf_hooks_ingress);

	struct nf_hook_state state;

	int ret;
@@ -30,7 +30,7 @@ static inline int nf_hook_ingress(struct sk_buff *skb) 
	nf_hook_state_init(&state, NF_NETDEV_INGRESS,

			   NFPROTO_NETDEV, skb->dev, NULL, NULL,

			   dev_net(skb->dev), NULL);

	ret = nf_hook_slow(skb, &state, e);

	ret = nf_hook_slow(skb, &state, e, 0);

	if (ret == 0)

		return -1;

include/net/netfilter/nf_queue.h

+1 −1
Original line number Diff line number Diff line
@@ -10,9 +10,9 @@ struct nf_queue_entry { 
	struct list_head	list;

	struct sk_buff		*skb;

	unsigned int		id;

	unsigned int		hook_index;	/* index in hook_entries->hook[] */



	struct nf_hook_state	state;

	struct nf_hook_entry	*hook;

	u16			size; /* sizeof(entry) + saved route keys */



	/* extra space to store route keys */

include/net/netns/netfilter.h

+1 −1
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ struct netns_nf { 
#ifdef CONFIG_SYSCTL

	struct ctl_table_header *nf_log_dir_header;

#endif

	struct nf_hook_entry __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];

	struct nf_hook_entries __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];

#if IS_ENABLED(CONFIG_NF_DEFRAG_IPV4)

	bool			defrag_ipv4;

#endif

CRA Git | Maintained and supported by SUSTech CRA and CCSE