Commit c6baf7ee authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'skbuff-micro-optimize-flow-dissection' 

Alexander Lobakin says:

====================
skbuff: micro-optimize flow dissection

This little number makes all of the flow dissection functions take
raw input data pointer as const (1-5) and shuffles the branches in
__skb_header_pointer() according to their hit probability.

The result is +20 Mbps per flow/core with one Flow Dissector pass
per packet. This affects RPS (with software hashing), drivers that
use eth_get_headlen() on their Rx path and so on.

From v2 [1]:
 - reword some commit messages as a potential fix for NIPA;
 - no functional changes.

From v1 [0]:
 - rebase on top of the latest net-next. This was super-weird, but
   I double-checked that the series applies with no conflicts, and
   then on Patchwork it didn't;
 - no other changes.

[0] https://lore.kernel.org/netdev/20210312194538.337504-1-alobakin@pm.me
[1] https://lore.kernel.org/netdev/20210313113645.5949-1-alobakin@pm.me


====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 3f79eb3c d206121f

include/linux/etherdevice.h

+2 −2
Original line number Diff line number Diff line
@@ -29,7 +29,7 @@ struct device; 
int eth_platform_get_mac_address(struct device *dev, u8 *mac_addr);

unsigned char *arch_get_platform_mac_address(void);

int nvmem_get_mac_address(struct device *dev, void *addrbuf);

u32 eth_get_headlen(const struct net_device *dev, void *data, unsigned int len);

u32 eth_get_headlen(const struct net_device *dev, const void *data, u32 len);

__be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev);

extern const struct header_ops eth_header_ops;

include/linux/skbuff.h

+12 −14
Original line number Diff line number Diff line
@@ -1292,10 +1292,10 @@ __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4) 
void __skb_get_hash(struct sk_buff *skb);

u32 __skb_get_hash_symmetric(const struct sk_buff *skb);

u32 skb_get_poff(const struct sk_buff *skb);

u32 __skb_get_poff(const struct sk_buff *skb, void *data,

u32 __skb_get_poff(const struct sk_buff *skb, const void *data,

		   const struct flow_keys_basic *keys, int hlen);

__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,

			    void *data, int hlen_proto);

			    const void *data, int hlen_proto);



static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,

					int thoff, u8 ip_proto)
@@ -1314,9 +1314,8 @@ bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, 
bool __skb_flow_dissect(const struct net *net,

			const struct sk_buff *skb,

			struct flow_dissector *flow_dissector,

			void *target_container,

			void *data, __be16 proto, int nhoff, int hlen,

			unsigned int flags);

			void *target_container, const void *data,

			__be16 proto, int nhoff, int hlen, unsigned int flags);



static inline bool skb_flow_dissect(const struct sk_buff *skb,

				    struct flow_dissector *flow_dissector,
@@ -1338,9 +1337,9 @@ static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb, 
static inline bool

skb_flow_dissect_flow_keys_basic(const struct net *net,

				 const struct sk_buff *skb,

				 struct flow_keys_basic *flow, void *data,

				 __be16 proto, int nhoff, int hlen,

				 unsigned int flags)

				 struct flow_keys_basic *flow,

				 const void *data, __be16 proto,

				 int nhoff, int hlen, unsigned int flags)

{

	memset(flow, 0, sizeof(*flow));

	return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow,
@@ -3678,14 +3677,13 @@ __wsum skb_checksum(const struct sk_buff *skb, int offset, int len, 
		    __wsum csum);



static inline void * __must_check

__skb_header_pointer(const struct sk_buff *skb, int offset,

		     int len, void *data, int hlen, void *buffer)

__skb_header_pointer(const struct sk_buff *skb, int offset, int len,

		     const void *data, int hlen, void *buffer)

{

	if (hlen - offset >= len)

		return data + offset;

	if (likely(hlen - offset >= len))

		return (void *)data + offset;



	if (!skb ||

	    skb_copy_bits(skb, offset, buffer, len) < 0)

	if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0))

		return NULL;



	return buffer;

include/net/flow_dissector.h

+3 −3
Original line number Diff line number Diff line
@@ -350,7 +350,7 @@ static inline bool flow_keys_have_l4(const struct flow_keys *keys) 
u32 flow_hash_from_keys(struct flow_keys *keys);

void skb_flow_get_icmp_tci(const struct sk_buff *skb,

			   struct flow_dissector_key_icmp *key_icmp,

			   void *data, int thoff, int hlen);

			   const void *data, int thoff, int hlen);



static inline bool dissector_uses_key(const struct flow_dissector *flow_dissector,

				      enum flow_dissector_key_id key_id)
@@ -368,8 +368,8 @@ static inline void *skb_flow_dissector_target(struct flow_dissector *flow_dissec 
struct bpf_flow_dissector {

	struct bpf_flow_keys	*flow_keys;

	const struct sk_buff	*skb;

	void			*data;

	void			*data_end;

	const void		*data;

	const void		*data_end;

};



static inline void

net/core/flow_dissector.c

+22 −19
Original line number Diff line number Diff line
@@ -114,7 +114,7 @@ int flow_dissector_bpf_prog_attach_check(struct net *net, 
 * is the protocol port offset returned from proto_ports_offset

 */

__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,

			    void *data, int hlen)

			    const void *data, int hlen)

{

	int poff = proto_ports_offset(ip_proto);
@@ -161,7 +161,7 @@ static bool icmp_has_id(u8 type) 
 */

void skb_flow_get_icmp_tci(const struct sk_buff *skb,

			   struct flow_dissector_key_icmp *key_icmp,

			   void *data, int thoff, int hlen)

			   const void *data, int thoff, int hlen)

{

	struct icmphdr *ih, _ih;
@@ -187,8 +187,8 @@ EXPORT_SYMBOL(skb_flow_get_icmp_tci); 
 */

static void __skb_flow_dissect_icmp(const struct sk_buff *skb,

				    struct flow_dissector *flow_dissector,

				    void *target_container,

				    void *data, int thoff, int hlen)

				    void *target_container, const void *data,

				    int thoff, int hlen)

{

	struct flow_dissector_key_icmp *key_icmp;
@@ -409,8 +409,8 @@ EXPORT_SYMBOL(skb_flow_dissect_hash); 
static enum flow_dissect_ret

__skb_flow_dissect_mpls(const struct sk_buff *skb,

			struct flow_dissector *flow_dissector,

			void *target_container, void *data, int nhoff, int hlen,

			int lse_index, bool *entropy_label)

			void *target_container, const void *data, int nhoff,

			int hlen, int lse_index, bool *entropy_label)

{

	struct mpls_label *hdr, _hdr;

	u32 entry, label, bos;
@@ -467,7 +467,8 @@ __skb_flow_dissect_mpls(const struct sk_buff *skb, 
static enum flow_dissect_ret

__skb_flow_dissect_arp(const struct sk_buff *skb,

		       struct flow_dissector *flow_dissector,

		       void *target_container, void *data, int nhoff, int hlen)

		       void *target_container, const void *data,

		       int nhoff, int hlen)

{

	struct flow_dissector_key_arp *key_arp;

	struct {
@@ -523,7 +524,7 @@ static enum flow_dissect_ret 
__skb_flow_dissect_gre(const struct sk_buff *skb,

		       struct flow_dissector_key_control *key_control,

		       struct flow_dissector *flow_dissector,

		       void *target_container, void *data,

		       void *target_container, const void *data,

		       __be16 *p_proto, int *p_nhoff, int *p_hlen,

		       unsigned int flags)

{
@@ -663,8 +664,8 @@ __skb_flow_dissect_gre(const struct sk_buff *skb, 
static enum flow_dissect_ret

__skb_flow_dissect_batadv(const struct sk_buff *skb,

			  struct flow_dissector_key_control *key_control,

			  void *data, __be16 *p_proto, int *p_nhoff, int hlen,

			  unsigned int flags)

			  const void *data, __be16 *p_proto, int *p_nhoff,

			  int hlen, unsigned int flags)

{

	struct {

		struct batadv_unicast_packet batadv_unicast;
@@ -695,7 +696,8 @@ __skb_flow_dissect_batadv(const struct sk_buff *skb, 
static void

__skb_flow_dissect_tcp(const struct sk_buff *skb,

		       struct flow_dissector *flow_dissector,

		       void *target_container, void *data, int thoff, int hlen)

		       void *target_container, const void *data,

		       int thoff, int hlen)

{

	struct flow_dissector_key_tcp *key_tcp;

	struct tcphdr *th, _th;
@@ -719,8 +721,8 @@ __skb_flow_dissect_tcp(const struct sk_buff *skb, 
static void

__skb_flow_dissect_ports(const struct sk_buff *skb,

			 struct flow_dissector *flow_dissector,

			 void *target_container, void *data, int nhoff,

			 u8 ip_proto, int hlen)

			 void *target_container, const void *data,

			 int nhoff, u8 ip_proto, int hlen)

{

	enum flow_dissector_key_id dissector_ports = FLOW_DISSECTOR_KEY_MAX;

	struct flow_dissector_key_ports *key_ports;
@@ -744,7 +746,8 @@ __skb_flow_dissect_ports(const struct sk_buff *skb, 
static void

__skb_flow_dissect_ipv4(const struct sk_buff *skb,

			struct flow_dissector *flow_dissector,

			void *target_container, void *data, const struct iphdr *iph)

			void *target_container, const void *data,

			const struct iphdr *iph)

{

	struct flow_dissector_key_ip *key_ip;
@@ -761,7 +764,8 @@ __skb_flow_dissect_ipv4(const struct sk_buff *skb, 
static void

__skb_flow_dissect_ipv6(const struct sk_buff *skb,

			struct flow_dissector *flow_dissector,

			void *target_container, void *data, const struct ipv6hdr *iph)

			void *target_container, const void *data,

			const struct ipv6hdr *iph)

{

	struct flow_dissector_key_ip *key_ip;
@@ -908,9 +912,8 @@ bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, 
bool __skb_flow_dissect(const struct net *net,

			const struct sk_buff *skb,

			struct flow_dissector *flow_dissector,

			void *target_container,

			void *data, __be16 proto, int nhoff, int hlen,

			unsigned int flags)

			void *target_container, const void *data,

			__be16 proto, int nhoff, int hlen, unsigned int flags)

{

	struct flow_dissector_key_control *key_control;

	struct flow_dissector_key_basic *key_basic;
@@ -1642,7 +1645,7 @@ __u32 skb_get_hash_perturb(const struct sk_buff *skb, 
}

EXPORT_SYMBOL(skb_get_hash_perturb);



u32 __skb_get_poff(const struct sk_buff *skb, void *data,

u32 __skb_get_poff(const struct sk_buff *skb, const void *data,

		   const struct flow_keys_basic *keys, int hlen)

{

	u32 poff = keys->control.thoff;

net/ethernet/eth.c

+1 −1
Original line number Diff line number Diff line
@@ -122,7 +122,7 @@ EXPORT_SYMBOL(eth_header); 
 * Make a best effort attempt to pull the length for all of the headers for

 * a given frame in a linear buffer.

 */

u32 eth_get_headlen(const struct net_device *dev, void *data, unsigned int len)

u32 eth_get_headlen(const struct net_device *dev, const void *data, u32 len)

{

	const unsigned int flags = FLOW_DISSECTOR_F_PARSE_1ST_FRAG;

	const struct ethhdr *eth = (const struct ethhdr *)data;