Commit e54740e6 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/jesse/openvswitch 



Jesse Gross says:

====================
A set of OVS changes for net-next/3.16.

The major change here is a switch from per-CPU to per-NUMA flow
statistics. This improves scalability by reducing kernel overhead
in flow setup and maintenance.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents ad2ebb3d 944df8ae

net/openvswitch/actions.c

+2 −2
Original line number Diff line number Diff line
@@ -134,8 +134,8 @@ static int set_eth_addr(struct sk_buff *skb, 


	skb_postpull_rcsum(skb, eth_hdr(skb), ETH_ALEN * 2);



	memcpy(eth_hdr(skb)->h_source, eth_key->eth_src, ETH_ALEN);

	memcpy(eth_hdr(skb)->h_dest, eth_key->eth_dst, ETH_ALEN);

	ether_addr_copy(eth_hdr(skb)->h_source, eth_key->eth_src);

	ether_addr_copy(eth_hdr(skb)->h_dest, eth_key->eth_dst);



	ovs_skb_postpush_rcsum(skb, eth_hdr(skb), ETH_ALEN * 2);

net/openvswitch/datapath.c

+5 −6
Original line number Diff line number Diff line
@@ -524,7 +524,7 @@ static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info) 
		packet->protocol = htons(ETH_P_802_2);



	/* Build an sw_flow for sending this packet. */

	flow = ovs_flow_alloc(false);

	flow = ovs_flow_alloc();

	err = PTR_ERR(flow);

	if (IS_ERR(flow))

		goto err_kfree_skb;
@@ -782,7 +782,6 @@ static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info) 
	struct datapath *dp;

	struct sw_flow_actions *acts = NULL;

	struct sw_flow_match match;

	bool exact_5tuple;

	int error;



	/* Extract key. */
@@ -791,7 +790,7 @@ static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info) 
		goto error;



	ovs_match_init(&match, &key, &mask);

	error = ovs_nla_get_match(&match, &exact_5tuple,

	error = ovs_nla_get_match(&match,

				  a[OVS_FLOW_ATTR_KEY], a[OVS_FLOW_ATTR_MASK]);

	if (error)

		goto error;
@@ -830,7 +829,7 @@ static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info) 
			goto err_unlock_ovs;



		/* Allocate flow. */

		flow = ovs_flow_alloc(!exact_5tuple);

		flow = ovs_flow_alloc();

		if (IS_ERR(flow)) {

			error = PTR_ERR(flow);

			goto err_unlock_ovs;
@@ -914,7 +913,7 @@ static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info) 
	}



	ovs_match_init(&match, &key, NULL);

	err = ovs_nla_get_match(&match, NULL, a[OVS_FLOW_ATTR_KEY], NULL);

	err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);

	if (err)

		return err;
@@ -968,7 +967,7 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info) 
	}



	ovs_match_init(&match, &key, NULL);

	err = ovs_nla_get_match(&match, NULL, a[OVS_FLOW_ATTR_KEY], NULL);

	err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);

	if (err)

		goto unlock;

net/openvswitch/datapath.h

+5 −3
Original line number Diff line number Diff line
@@ -195,6 +195,8 @@ int ovs_execute_actions(struct datapath *dp, struct sk_buff *skb); 
void ovs_dp_notify_wq(struct work_struct *work);



#define OVS_NLERR(fmt, ...)					\

	pr_info_once("netlink: " fmt, ##__VA_ARGS__)



do {								\

	if (net_ratelimit())					\

		pr_info("netlink: " fmt, ##__VA_ARGS__);	\

} while (0)

#endif /* datapath.h */

net/openvswitch/flow.c

+87 −62
Original line number Diff line number Diff line
@@ -65,87 +65,112 @@ void ovs_flow_stats_update(struct sw_flow *flow, struct sk_buff *skb) 
{

	struct flow_stats *stats;

	__be16 tcp_flags = 0;

	int node = numa_node_id();



	if (!flow->stats.is_percpu)

		stats = flow->stats.stat;

	else

		stats = this_cpu_ptr(flow->stats.cpu_stats);

	stats = rcu_dereference(flow->stats[node]);



	if ((flow->key.eth.type == htons(ETH_P_IP) ||

	     flow->key.eth.type == htons(ETH_P_IPV6)) &&

	    flow->key.ip.frag != OVS_FRAG_TYPE_LATER &&

	    flow->key.ip.proto == IPPROTO_TCP &&

	    likely(skb->len >= skb_transport_offset(skb) + sizeof(struct tcphdr))) {

		tcp_flags = TCP_FLAGS_BE16(tcp_hdr(skb));

	if (likely(flow->key.ip.proto == IPPROTO_TCP)) {

		if (likely(flow->key.eth.type == htons(ETH_P_IP)))

			tcp_flags = flow->key.ipv4.tp.flags;

		else if (likely(flow->key.eth.type == htons(ETH_P_IPV6)))

			tcp_flags = flow->key.ipv6.tp.flags;

	}



	/* Check if already have node-specific stats. */

	if (likely(stats)) {

		spin_lock(&stats->lock);

		/* Mark if we write on the pre-allocated stats. */

		if (node == 0 && unlikely(flow->stats_last_writer != node))

			flow->stats_last_writer = node;

	} else {

		stats = rcu_dereference(flow->stats[0]); /* Pre-allocated. */

		spin_lock(&stats->lock);



		/* If the current NUMA-node is the only writer on the

		 * pre-allocated stats keep using them.

		 */

		if (unlikely(flow->stats_last_writer != node)) {

			/* A previous locker may have already allocated the

			 * stats, so we need to check again.  If node-specific

			 * stats were already allocated, we update the pre-

			 * allocated stats as we have already locked them.

			 */

			if (likely(flow->stats_last_writer != NUMA_NO_NODE)

			    && likely(!rcu_dereference(flow->stats[node]))) {

				/* Try to allocate node-specific stats. */

				struct flow_stats *new_stats;



				new_stats =

					kmem_cache_alloc_node(flow_stats_cache,

							      GFP_THISNODE |

							      __GFP_NOMEMALLOC,

							      node);

				if (likely(new_stats)) {

					new_stats->used = jiffies;

					new_stats->packet_count = 1;

					new_stats->byte_count = skb->len;

					new_stats->tcp_flags = tcp_flags;

					spin_lock_init(&new_stats->lock);



					rcu_assign_pointer(flow->stats[node],

							   new_stats);

					goto unlock;

				}

			}

			flow->stats_last_writer = node;

		}

	}



	stats->used = jiffies;

	stats->packet_count++;

	stats->byte_count += skb->len;

	stats->tcp_flags |= tcp_flags;

	spin_unlock(&stats->lock);

}



static void stats_read(struct flow_stats *stats,

		       struct ovs_flow_stats *ovs_stats,

		       unsigned long *used, __be16 *tcp_flags)

{

	spin_lock(&stats->lock);

	if (!*used || time_after(stats->used, *used))

		*used = stats->used;

	*tcp_flags |= stats->tcp_flags;

	ovs_stats->n_packets += stats->packet_count;

	ovs_stats->n_bytes += stats->byte_count;

unlock:

	spin_unlock(&stats->lock);

}



void ovs_flow_stats_get(struct sw_flow *flow, struct ovs_flow_stats *ovs_stats,

			unsigned long *used, __be16 *tcp_flags)

{

	int cpu;

	int node;



	*used = 0;

	*tcp_flags = 0;

	memset(ovs_stats, 0, sizeof(*ovs_stats));



	local_bh_disable();

	if (!flow->stats.is_percpu) {

		stats_read(flow->stats.stat, ovs_stats, used, tcp_flags);

	} else {

		for_each_possible_cpu(cpu) {

			struct flow_stats *stats;

	for_each_node(node) {

		struct flow_stats *stats = rcu_dereference(flow->stats[node]);



			stats = per_cpu_ptr(flow->stats.cpu_stats, cpu);

			stats_read(stats, ovs_stats, used, tcp_flags);

		if (stats) {

			/* Local CPU may write on non-local stats, so we must

			 * block bottom-halves here.

			 */

			spin_lock_bh(&stats->lock);

			if (!*used || time_after(stats->used, *used))

				*used = stats->used;

			*tcp_flags |= stats->tcp_flags;

			ovs_stats->n_packets += stats->packet_count;

			ovs_stats->n_bytes += stats->byte_count;

			spin_unlock_bh(&stats->lock);

		}

	}

	local_bh_enable();

}



static void stats_reset(struct flow_stats *stats)

void ovs_flow_stats_clear(struct sw_flow *flow)

{

	spin_lock(&stats->lock);

	int node;



	for_each_node(node) {

		struct flow_stats *stats = rcu_dereference(flow->stats[node]);



		if (stats) {

			spin_lock_bh(&stats->lock);

			stats->used = 0;

			stats->packet_count = 0;

			stats->byte_count = 0;

			stats->tcp_flags = 0;

	spin_unlock(&stats->lock);

}



void ovs_flow_stats_clear(struct sw_flow *flow)

{

	int cpu;



	local_bh_disable();

	if (!flow->stats.is_percpu) {

		stats_reset(flow->stats.stat);

	} else {

		for_each_possible_cpu(cpu) {

			stats_reset(per_cpu_ptr(flow->stats.cpu_stats, cpu));

			spin_unlock_bh(&stats->lock);

		}

	}

	local_bh_enable();

}



static int check_header(struct sk_buff *skb, int len)
@@ -372,14 +397,14 @@ static int parse_icmpv6(struct sk_buff *skb, struct sw_flow_key *key, 
			    && opt_len == 8) {

				if (unlikely(!is_zero_ether_addr(key->ipv6.nd.sll)))

					goto invalid;

				memcpy(key->ipv6.nd.sll,

				    &nd->opt[offset+sizeof(*nd_opt)], ETH_ALEN);

				ether_addr_copy(key->ipv6.nd.sll,

						&nd->opt[offset+sizeof(*nd_opt)]);

			} else if (nd_opt->nd_opt_type == ND_OPT_TARGET_LL_ADDR

				   && opt_len == 8) {

				if (unlikely(!is_zero_ether_addr(key->ipv6.nd.tll)))

					goto invalid;

				memcpy(key->ipv6.nd.tll,

				    &nd->opt[offset+sizeof(*nd_opt)], ETH_ALEN);

				ether_addr_copy(key->ipv6.nd.tll,

						&nd->opt[offset+sizeof(*nd_opt)]);

			}



			icmp_len -= opt_len;
@@ -439,8 +464,8 @@ int ovs_flow_extract(struct sk_buff *skb, u16 in_port, struct sw_flow_key *key) 
	 * header in the linear data area.

	 */

	eth = eth_hdr(skb);

	memcpy(key->eth.src, eth->h_source, ETH_ALEN);

	memcpy(key->eth.dst, eth->h_dest, ETH_ALEN);

	ether_addr_copy(key->eth.src, eth->h_source);

	ether_addr_copy(key->eth.dst, eth->h_dest);



	__skb_pull(skb, 2 * ETH_ALEN);

	/* We are going to push all headers that we pull, so no need to
@@ -538,8 +563,8 @@ int ovs_flow_extract(struct sk_buff *skb, u16 in_port, struct sw_flow_key *key) 
				key->ip.proto = ntohs(arp->ar_op);

			memcpy(&key->ipv4.addr.src, arp->ar_sip, sizeof(key->ipv4.addr.src));

			memcpy(&key->ipv4.addr.dst, arp->ar_tip, sizeof(key->ipv4.addr.dst));

			memcpy(key->ipv4.arp.sha, arp->ar_sha, ETH_ALEN);

			memcpy(key->ipv4.arp.tha, arp->ar_tha, ETH_ALEN);

			ether_addr_copy(key->ipv4.arp.sha, arp->ar_sha);

			ether_addr_copy(key->ipv4.arp.tha, arp->ar_tha);

		}

	} else if (key->eth.type == htons(ETH_P_IPV6)) {

		int nh_len;             /* IPv6 Header + Extensions */

net/openvswitch/flow.h

+8 −10
Original line number Diff line number Diff line
@@ -155,24 +155,22 @@ struct flow_stats { 
	__be16 tcp_flags;		/* Union of seen TCP flags. */

};



struct sw_flow_stats {

	bool is_percpu;

	union {

		struct flow_stats *stat;

		struct flow_stats __percpu *cpu_stats;

	};

};



struct sw_flow {

	struct rcu_head rcu;

	struct hlist_node hash_node[2];

	u32 hash;



	int stats_last_writer;		/* NUMA-node id of the last writer on

					 * 'stats[0]'.

					 */

	struct sw_flow_key key;

	struct sw_flow_key unmasked_key;

	struct sw_flow_mask *mask;

	struct sw_flow_actions __rcu *sf_acts;

	struct sw_flow_stats stats;

	struct flow_stats __rcu *stats[]; /* One for each NUMA node.  First one

					   * is allocated at flow creation time,

					   * the rest are allocated on demand

					   * while holding the 'stats[0].lock'.

					   */

};



struct arp_eth_header {

CRA Git | Maintained and supported by SUSTech CRA and CCSE