Commit e0bc8927 authored by Alexei Starovoitov's avatar Alexei Starovoitov
Browse files

Merge branch 'Generic XDP improvements' 

Kumar Kartikeya says:

====================

This small series makes some improvements to generic XDP mode and brings it
closer to native XDP. Patch 1 splits out generic XDP processing into reusable
parts, patch 2 adds pointer friendly wrappers for bitops (not have to cast back
and forth the address of local pointer to unsigned long *), patch 3 implements
generic cpumap support (details in commit) and patch 4 allows devmap bpf prog
execution before generic_xdp_tx is called.

Patch 5 just updates a couple of selftests to adapt to changes in behavior (in
that specifying devmap/cpumap prog fd in generic mode is now allowed).

Changelog:
----------
v5 -> v6
v5: https://lore.kernel.org/bpf/20210701002759.381983-1-memxor@gmail.com
 * Put rcpu->prog check before RCU-bh section to avoid do_softirq (Jesper)

v4 -> v5
v4: https://lore.kernel.org/bpf/20210628114746.129669-1-memxor@gmail.com
 * Add comments and examples for new bitops macros (Alexei)

v3 -> v4
v3: https://lore.kernel.org/bpf/20210622202835.1151230-1-memxor@gmail.com
 * Add detach now that attach of XDP program succeeds (Toke)
 * Clean up the test to use new ASSERT macros

v2 -> v3
v2: https://lore.kernel.org/bpf/20210622195527.1110497-1-memxor@gmail.com
 * list_for_each_entry -> list_for_each_entry_safe (due to deletion of skb)

v1 -> v2
v1: https://lore.kernel.org/bpf/20210620233200.855534-1-memxor@gmail.com


 * Move __ptr_{set,clear,test}_bit to bitops.h (Toke)
   Also changed argument order to match the bit op they wrap.
 * Remove map value size checking functions for cpumap/devmap (Toke)
 * Rework prog run for skb in cpu_map_kthread_run (Toke)
 * Set skb->dev to dst->dev after devmap prog has run
 * Don't set xdp rxq that will be overwritten in cpumap prog run
====================

Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parents a080cdcc 36246d5a

include/linux/bitops.h

+50 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 


#include <asm/types.h>

#include <linux/bits.h>

#include <linux/typecheck.h>



#include <uapi/linux/kernel.h>
@@ -253,6 +254,55 @@ static __always_inline void __assign_bit(long nr, volatile unsigned long *addr, 
		__clear_bit(nr, addr);

}



/**

 * __ptr_set_bit - Set bit in a pointer's value

 * @nr: the bit to set

 * @addr: the address of the pointer variable

 *

 * Example:

 *	void *p = foo();

 *	__ptr_set_bit(bit, &p);

 */

#define __ptr_set_bit(nr, addr)                         \

	({                                              \

		typecheck_pointer(*(addr));             \

		__set_bit(nr, (unsigned long *)(addr)); \

	})



/**

 * __ptr_clear_bit - Clear bit in a pointer's value

 * @nr: the bit to clear

 * @addr: the address of the pointer variable

 *

 * Example:

 *	void *p = foo();

 *	__ptr_clear_bit(bit, &p);

 */

#define __ptr_clear_bit(nr, addr)                         \

	({                                                \

		typecheck_pointer(*(addr));               \

		__clear_bit(nr, (unsigned long *)(addr)); \

	})



/**

 * __ptr_test_bit - Test bit in a pointer's value

 * @nr: the bit to test

 * @addr: the address of the pointer variable

 *

 * Example:

 *	void *p = foo();

 *	if (__ptr_test_bit(bit, &p)) {

 *	        ...

 *	} else {

 *		...

 *	}

 */

#define __ptr_test_bit(nr, addr)                       \

	({                                             \

		typecheck_pointer(*(addr));            \

		test_bit(nr, (unsigned long *)(addr)); \

	})



#ifdef __KERNEL__



#ifndef set_mask_bits

include/linux/bpf.h

+8 −2
Original line number Diff line number Diff line
@@ -1508,12 +1508,12 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 
int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,

			   struct bpf_prog *xdp_prog, struct bpf_map *map,

			   bool exclude_ingress);

bool dev_map_can_have_prog(struct bpf_map *map);



void __cpu_map_flush(void);

int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,

		    struct net_device *dev_rx);

bool cpu_map_prog_allowed(struct bpf_map *map);

int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,

			     struct sk_buff *skb);



/* Return map's numa specified by userspace */

static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
@@ -1710,6 +1710,12 @@ static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 
	return 0;

}



static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,

					   struct sk_buff *skb)

{

	return -EOPNOTSUPP;

}



static inline bool cpu_map_prog_allowed(struct bpf_map *map)

{

	return false;

include/linux/netdevice.h

+2 −0
Original line number Diff line number Diff line
@@ -3984,6 +3984,8 @@ static inline void dev_consume_skb_any(struct sk_buff *skb) 
	__dev_kfree_skb_any(skb, SKB_REASON_CONSUMED);

}



u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,

			     struct bpf_prog *xdp_prog);

void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);

int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);

int netif_rx(struct sk_buff *skb);

include/linux/skbuff.h

+2 −8
Original line number Diff line number Diff line
@@ -863,8 +863,8 @@ struct sk_buff { 
	__u8			tc_skip_classify:1;

	__u8			tc_at_ingress:1;

#endif

#ifdef CONFIG_NET_REDIRECT

	__u8			redirected:1;

#ifdef CONFIG_NET_REDIRECT

	__u8			from_ingress:1;

#endif

#ifdef CONFIG_TLS_DEVICE
@@ -4664,17 +4664,13 @@ static inline __wsum lco_csum(struct sk_buff *skb) 


static inline bool skb_is_redirected(const struct sk_buff *skb)

{

#ifdef CONFIG_NET_REDIRECT

	return skb->redirected;

#else

	return false;

#endif

}



static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress)

{

#ifdef CONFIG_NET_REDIRECT

	skb->redirected = 1;

#ifdef CONFIG_NET_REDIRECT

	skb->from_ingress = from_ingress;

	if (skb->from_ingress)

		skb->tstamp = 0;
@@ -4683,9 +4679,7 @@ static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress) 


static inline void skb_reset_redirect(struct sk_buff *skb)

{

#ifdef CONFIG_NET_REDIRECT

	skb->redirected = 0;

#endif

}



static inline bool skb_csum_is_sctp(struct sk_buff *skb)

include/linux/typecheck.h

+9 −0
Original line number Diff line number Diff line
@@ -22,4 +22,13 @@ 
	(void)__tmp; \

})



/*

 * Check at compile time that something is a pointer type.

 */

#define typecheck_pointer(x) \

({	typeof(x) __dummy; \

	(void)sizeof(*__dummy); \

	1; \

})



#endif		/* TYPECHECK_H_INCLUDED */