Commit 3dca3f38 authored by Steffen Klassert's avatar Steffen Klassert
Browse files

xfrm: Separate ESP handling from segmentation for GRO packets. 



We change the ESP GSO handlers to only segment the packets.
The ESP handling and encryption is defered to validate_xmit_xfrm()
where this is done for non GRO packets too. This makes the code
more robust and prepares for asynchronous crypto handling.

Signed-off-by: default avatarSteffen Klassert <steffen.klassert@secunet.com>
parent f39a5c01

include/net/xfrm.h

+3 −3
Original line number Diff line number Diff line
@@ -1888,7 +1888,7 @@ static inline struct xfrm_offload *xfrm_offload(struct sk_buff *skb) 
void __net_init xfrm_dev_init(void);



#ifdef CONFIG_XFRM_OFFLOAD

int validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features);

struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features);

int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,

		       struct xfrm_user_offload *xuo);

bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
@@ -1929,9 +1929,9 @@ static inline void xfrm_dev_state_free(struct xfrm_state *x) 
	}

}

#else

static inline int validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features)

static inline struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features)

{

	return 0;

	return skb;

}



static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo)

net/core/dev.c

+2 −3
Original line number Diff line number Diff line
@@ -3083,9 +3083,6 @@ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device 
		    __skb_linearize(skb))

			goto out_kfree_skb;



		if (validate_xmit_xfrm(skb, features))

			goto out_kfree_skb;



		/* If packet is not checksummed and device does not

		 * support checksumming for this protocol, complete

		 * checksumming here.
@@ -3102,6 +3099,8 @@ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device 
		}

	}



	skb = validate_xmit_xfrm(skb, features);



	return skb;



out_kfree_skb:

net/ipv4/esp4_offload.c

+21 −52
Original line number Diff line number Diff line
@@ -108,75 +108,36 @@ static void esp4_gso_encap(struct xfrm_state *x, struct sk_buff *skb) 
static struct sk_buff *esp4_gso_segment(struct sk_buff *skb,

				        netdev_features_t features)

{

	__u32 seq;

	int err = 0;

	struct sk_buff *skb2;

	struct xfrm_state *x;

	struct ip_esp_hdr *esph;

	struct crypto_aead *aead;

	struct sk_buff *segs = ERR_PTR(-EINVAL);

	netdev_features_t esp_features = features;

	struct xfrm_offload *xo = xfrm_offload(skb);



	if (!xo)

		goto out;



	seq = xo->seq.low;

		return ERR_PTR(-EINVAL);



	x = skb->sp->xvec[skb->sp->len - 1];

	aead = x->data;

	esph = ip_esp_hdr(skb);



	if (esph->spi != x->id.spi)

		goto out;

		return ERR_PTR(-EINVAL);



	if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)))

		goto out;

		return ERR_PTR(-EINVAL);



	__skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead));



	skb->encap_hdr_csum = 1;



	if (!(features & NETIF_F_HW_ESP))

	if (!(features & NETIF_F_HW_ESP) || !x->xso.offload_handle ||

	    (x->xso.dev != skb->dev))

		esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);



	segs = x->outer_mode->gso_segment(x, skb, esp_features);

	if (IS_ERR_OR_NULL(segs))

		goto out;



	__skb_pull(skb, skb->data - skb_mac_header(skb));



	skb2 = segs;

	do {

		struct sk_buff *nskb = skb2->next;



		xo = xfrm_offload(skb2);

	xo->flags |= XFRM_GSO_SEGMENT;

		xo->seq.low = seq;

		xo->seq.hi = xfrm_replay_seqhi(x, seq);



		if(!(features & NETIF_F_HW_ESP))

			xo->flags |= CRYPTO_FALLBACK;



		x->outer_mode->xmit(x, skb2);



		err = x->type_offload->xmit(x, skb2, esp_features);

		if (err) {

			kfree_skb_list(segs);

			return ERR_PTR(err);

		}



		if (!skb_is_gso(skb2))

			seq++;

		else

			seq += skb_shinfo(skb2)->gso_segs;



		skb_push(skb2, skb2->mac_len);

		skb2 = nskb;

	} while (skb2);



out:

	return segs;

	return x->outer_mode->gso_segment(x, skb, esp_features);

}



static int esp_input_tail(struct xfrm_state *x, struct sk_buff *skb)
@@ -203,6 +164,7 @@ static int esp_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features_ 
	struct crypto_aead *aead;

	struct esp_info esp;

	bool hw_offload = true;

	__u32 seq;



	esp.inplace = true;
@@ -241,23 +203,30 @@ static int esp_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features_ 
			return esp.nfrags;

	}



	seq = xo->seq.low;



	esph = esp.esph;

	esph->spi = x->id.spi;



	skb_push(skb, -skb_network_offset(skb));



	if (xo->flags & XFRM_GSO_SEGMENT) {

		esph->seq_no = htonl(xo->seq.low);

	} else {

		esph->seq_no = htonl(seq);



		if (!skb_is_gso(skb))

			xo->seq.low++;

		else

			xo->seq.low += skb_shinfo(skb)->gso_segs;

	}



	esp.seqno = cpu_to_be64(seq + ((u64)xo->seq.hi << 32));



	ip_hdr(skb)->tot_len = htons(skb->len);

	ip_send_check(ip_hdr(skb));

	}



	if (hw_offload)

		return 0;



	esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));



	err = esp_output_tail(x, skb, &esp);

	if (err)

		return err;

net/ipv4/xfrm4_mode_tunnel.c

+1 −4
Original line number Diff line number Diff line
@@ -105,18 +105,15 @@ static struct sk_buff *xfrm4_mode_tunnel_gso_segment(struct xfrm_state *x, 
{

	__skb_push(skb, skb->mac_len);

	return skb_mac_gso_segment(skb, features);



}



static void xfrm4_mode_tunnel_xmit(struct xfrm_state *x, struct sk_buff *skb)

{

	struct xfrm_offload *xo = xfrm_offload(skb);



	if (xo->flags & XFRM_GSO_SEGMENT) {

		skb->network_header = skb->network_header - x->props.header_len;

	if (xo->flags & XFRM_GSO_SEGMENT)

		skb->transport_header = skb->network_header +

					sizeof(struct iphdr);

	}



	skb_reset_mac_len(skb);

	pskb_pull(skb, skb->mac_len + x->props.header_len);

net/ipv6/esp6_offload.c

+24 −56
Original line number Diff line number Diff line
@@ -135,75 +135,36 @@ static void esp6_gso_encap(struct xfrm_state *x, struct sk_buff *skb) 
static struct sk_buff *esp6_gso_segment(struct sk_buff *skb,

				        netdev_features_t features)

{

	__u32 seq;

	int err = 0;

	struct sk_buff *skb2;

	struct xfrm_state *x;

	struct ip_esp_hdr *esph;

	struct crypto_aead *aead;

	struct sk_buff *segs = ERR_PTR(-EINVAL);

	netdev_features_t esp_features = features;

	struct xfrm_offload *xo = xfrm_offload(skb);



	if (!xo)

		goto out;



	seq = xo->seq.low;

		return ERR_PTR(-EINVAL);



	x = skb->sp->xvec[skb->sp->len - 1];

	aead = x->data;

	esph = ip_esp_hdr(skb);



	if (esph->spi != x->id.spi)

		goto out;

		return ERR_PTR(-EINVAL);



	if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)))

		goto out;

		return ERR_PTR(-EINVAL);



	__skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead));



	skb->encap_hdr_csum = 1;



	if (!(features & NETIF_F_HW_ESP))

	if (!(features & NETIF_F_HW_ESP) || !x->xso.offload_handle ||

	    (x->xso.dev != skb->dev))

		esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);



	segs = x->outer_mode->gso_segment(x, skb, esp_features);

	if (IS_ERR_OR_NULL(segs))

		goto out;



	__skb_pull(skb, skb->data - skb_mac_header(skb));



	skb2 = segs;

	do {

		struct sk_buff *nskb = skb2->next;



		xo = xfrm_offload(skb2);

	xo->flags |= XFRM_GSO_SEGMENT;

		xo->seq.low = seq;

		xo->seq.hi = xfrm_replay_seqhi(x, seq);



		if(!(features & NETIF_F_HW_ESP))

			xo->flags |= CRYPTO_FALLBACK;



		x->outer_mode->xmit(x, skb2);



		err = x->type_offload->xmit(x, skb2, esp_features);

		if (err) {

			kfree_skb_list(segs);

			return ERR_PTR(err);

		}



		if (!skb_is_gso(skb2))

			seq++;

		else

			seq += skb_shinfo(skb2)->gso_segs;



		skb_push(skb2, skb2->mac_len);

		skb2 = nskb;

	} while (skb2);



out:

	return segs;

	return x->outer_mode->gso_segment(x, skb, esp_features);

}



static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb)
@@ -222,6 +183,7 @@ static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb) 


static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb,  netdev_features_t features)

{

	int len;

	int err;

	int alen;

	int blksize;
@@ -230,6 +192,7 @@ static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features 
	struct crypto_aead *aead;

	struct esp_info esp;

	bool hw_offload = true;

	__u32 seq;



	esp.inplace = true;
@@ -265,28 +228,33 @@ static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features 
			return esp.nfrags;

	}



	seq = xo->seq.low;



	esph = ip_esp_hdr(skb);

	esph->spi = x->id.spi;



	skb_push(skb, -skb_network_offset(skb));



	if (xo->flags & XFRM_GSO_SEGMENT) {

		esph->seq_no = htonl(xo->seq.low);

	} else {

		int len;

		esph->seq_no = htonl(seq);



		if (!skb_is_gso(skb))

			xo->seq.low++;

		else

			xo->seq.low += skb_shinfo(skb)->gso_segs;

	}



	esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));



	len = skb->len - sizeof(struct ipv6hdr);

	if (len > IPV6_MAXPLEN)

		len = 0;



	ipv6_hdr(skb)->payload_len = htons(len);

	}



	if (hw_offload)

		return 0;



	esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));



	err = esp6_output_tail(x, skb, &esp);

	if (err)

		return err;