Commit 78cbac64 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'ip-faster-in-order-IP-fragments' 



Peter Oskolkov says:

====================
ip: faster in-order IP fragments

Added "Signed-off-by" in v2.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 6a92ef08 a4fd284a

include/net/inet_frag.h

+6 −0
Original line number Diff line number Diff line
@@ -57,7 +57,9 @@ struct frag_v6_compare_key { 
 * @lock: spinlock protecting this frag

 * @refcnt: reference count of the queue

 * @fragments: received fragments head

 * @rb_fragments: received fragments rb-tree root

 * @fragments_tail: received fragments tail

 * @last_run_head: the head of the last "run". see ip_fragment.c

 * @stamp: timestamp of the last received fragment

 * @len: total length of the original datagram

 * @meat: length of received fragments so far
@@ -78,6 +80,7 @@ struct inet_frag_queue { 
	struct sk_buff		*fragments;  /* Used in IPv6. */

	struct rb_root		rb_fragments; /* Used in IPv4. */

	struct sk_buff		*fragments_tail;

	struct sk_buff		*last_run_head;

	ktime_t			stamp;

	int			len;

	int			meat;
@@ -113,6 +116,9 @@ void inet_frag_kill(struct inet_frag_queue *q); 
void inet_frag_destroy(struct inet_frag_queue *q);

struct inet_frag_queue *inet_frag_find(struct netns_frags *nf, void *key);



/* Free all skbs in the queue; return the sum of their truesizes. */

unsigned int inet_frag_rbtree_purge(struct rb_root *root);



static inline void inet_frag_put(struct inet_frag_queue *q)

{

	if (refcount_dec_and_test(&q->refcnt))

net/ipv4/inet_fragment.c

+1 −1
Original line number Diff line number Diff line
@@ -146,7 +146,7 @@ void inet_frag_destroy(struct inet_frag_queue *q) 
			fp = xp;

		} while (fp);

	} else {

		sum_truesize = skb_rbtree_purge(&q->rb_fragments);

		sum_truesize = inet_frag_rbtree_purge(&q->rb_fragments);

	}

	sum = sum_truesize + f->qsize;

net/ipv4/ip_fragment.c

+142 −41
Original line number Diff line number Diff line
@@ -57,6 +57,57 @@ 
 */

static const char ip_frag_cache_name[] = "ip4-frags";



/* Use skb->cb to track consecutive/adjacent fragments coming at

 * the end of the queue. Nodes in the rb-tree queue will

 * contain "runs" of one or more adjacent fragments.

 *

 * Invariants:

 * - next_frag is NULL at the tail of a "run";

 * - the head of a "run" has the sum of all fragment lengths in frag_run_len.

 */

struct ipfrag_skb_cb {

	struct inet_skb_parm	h;

	struct sk_buff		*next_frag;

	int			frag_run_len;

};



#define FRAG_CB(skb)		((struct ipfrag_skb_cb *)((skb)->cb))



static void ip4_frag_init_run(struct sk_buff *skb)

{

	BUILD_BUG_ON(sizeof(struct ipfrag_skb_cb) > sizeof(skb->cb));



	FRAG_CB(skb)->next_frag = NULL;

	FRAG_CB(skb)->frag_run_len = skb->len;

}



/* Append skb to the last "run". */

static void ip4_frag_append_to_last_run(struct inet_frag_queue *q,

					struct sk_buff *skb)

{

	RB_CLEAR_NODE(&skb->rbnode);

	FRAG_CB(skb)->next_frag = NULL;



	FRAG_CB(q->last_run_head)->frag_run_len += skb->len;

	FRAG_CB(q->fragments_tail)->next_frag = skb;

	q->fragments_tail = skb;

}



/* Create a new "run" with the skb. */

static void ip4_frag_create_run(struct inet_frag_queue *q, struct sk_buff *skb)

{

	if (q->last_run_head)

		rb_link_node(&skb->rbnode, &q->last_run_head->rbnode,

			     &q->last_run_head->rbnode.rb_right);

	else

		rb_link_node(&skb->rbnode, NULL, &q->rb_fragments.rb_node);

	rb_insert_color(&skb->rbnode, &q->rb_fragments);



	ip4_frag_init_run(skb);

	q->fragments_tail = skb;

	q->last_run_head = skb;

}



/* Describe an entry in the "incomplete datagrams" queue. */

struct ipq {

	struct inet_frag_queue q;
@@ -75,8 +126,8 @@ static u8 ip4_frag_ecn(u8 tos) 


static struct inet_frags ip4_frags;



static int ip_frag_reasm(struct ipq *qp, struct sk_buff *prev,

			 struct net_device *dev);

static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,

			 struct sk_buff *prev_tail, struct net_device *dev);





static void ip4_frag_init(struct inet_frag_queue *q, const void *a)
@@ -168,6 +219,11 @@ static void ip_expire(struct timer_list *t) 
		head = skb_rb_first(&qp->q.rb_fragments);

		if (!head)

			goto out;

		if (FRAG_CB(head)->next_frag)

			rb_replace_node(&head->rbnode,

					&FRAG_CB(head)->next_frag->rbnode,

					&qp->q.rb_fragments);

		else

			rb_erase(&head->rbnode, &qp->q.rb_fragments);

		memset(&head->rbnode, 0, sizeof(head->rbnode));

		barrier();
@@ -269,7 +325,7 @@ static int ip_frag_reinit(struct ipq *qp) 
		return -ETIMEDOUT;

	}



	sum_truesize = skb_rbtree_purge(&qp->q.rb_fragments);

	sum_truesize = inet_frag_rbtree_purge(&qp->q.rb_fragments);

	sub_frag_mem_limit(qp->q.net, sum_truesize);



	qp->q.flags = 0;
@@ -278,6 +334,7 @@ static int ip_frag_reinit(struct ipq *qp) 
	qp->q.fragments = NULL;

	qp->q.rb_fragments = RB_ROOT;

	qp->q.fragments_tail = NULL;

	qp->q.last_run_head = NULL;

	qp->iif = 0;

	qp->ecn = 0;
@@ -289,7 +346,7 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) 
{

	struct net *net = container_of(qp->q.net, struct net, ipv4.frags);

	struct rb_node **rbn, *parent;

	struct sk_buff *skb1;

	struct sk_buff *skb1, *prev_tail;

	struct net_device *dev;

	unsigned int fragsize;

	int flags, offset;
@@ -367,38 +424,41 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) 
	 */



	/* Find out where to put this fragment.  */

	skb1 = qp->q.fragments_tail;

	if (!skb1) {

		/* This is the first fragment we've received. */

		rb_link_node(&skb->rbnode, NULL, &qp->q.rb_fragments.rb_node);

		qp->q.fragments_tail = skb;

	} else if ((skb1->ip_defrag_offset + skb1->len) < end) {

		/* This is the common/special case: skb goes to the end. */

	prev_tail = qp->q.fragments_tail;

	if (!prev_tail)

		ip4_frag_create_run(&qp->q, skb);  /* First fragment. */

	else if (prev_tail->ip_defrag_offset + prev_tail->len < end) {

		/* This is the common case: skb goes to the end. */

		/* Detect and discard overlaps. */

		if (offset < (skb1->ip_defrag_offset + skb1->len))

		if (offset < prev_tail->ip_defrag_offset + prev_tail->len)

			goto discard_qp;

		/* Insert after skb1. */

		rb_link_node(&skb->rbnode, &skb1->rbnode, &skb1->rbnode.rb_right);

		qp->q.fragments_tail = skb;

		if (offset == prev_tail->ip_defrag_offset + prev_tail->len)

			ip4_frag_append_to_last_run(&qp->q, skb);

		else

			ip4_frag_create_run(&qp->q, skb);

	} else {

		/* Binary search. Note that skb can become the first fragment, but

		 * not the last (covered above). */

		/* Binary search. Note that skb can become the first fragment,

		 * but not the last (covered above).

		 */

		rbn = &qp->q.rb_fragments.rb_node;

		do {

			parent = *rbn;

			skb1 = rb_to_skb(parent);

			if (end <= skb1->ip_defrag_offset)

				rbn = &parent->rb_left;

			else if (offset >= skb1->ip_defrag_offset + skb1->len)

			else if (offset >= skb1->ip_defrag_offset +

						FRAG_CB(skb1)->frag_run_len)

				rbn = &parent->rb_right;

			else /* Found an overlap with skb1. */

				goto discard_qp;

		} while (*rbn);

		/* Here we have parent properly set, and rbn pointing to

		 * one of its NULL left/right children. Insert skb. */

		 * one of its NULL left/right children. Insert skb.

		 */

		ip4_frag_init_run(skb);

		rb_link_node(&skb->rbnode, parent, rbn);

	}

		rb_insert_color(&skb->rbnode, &qp->q.rb_fragments);

	}



	if (dev)

		qp->iif = dev->ifindex;
@@ -425,7 +485,7 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) 
		unsigned long orefdst = skb->_skb_refdst;



		skb->_skb_refdst = 0UL;

		err = ip_frag_reasm(qp, skb, dev);

		err = ip_frag_reasm(qp, skb, prev_tail, dev);

		skb->_skb_refdst = orefdst;

		return err;

	}
@@ -444,7 +504,7 @@ static int ip_frag_queue(struct ipq *qp, struct sk_buff *skb) 


/* Build a new IP datagram from all its fragments. */

static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb,

			 struct net_device *dev)

			 struct sk_buff *prev_tail, struct net_device *dev)

{

	struct net *net = container_of(qp->q.net, struct net, ipv4.frags);

	struct iphdr *iph;
@@ -468,10 +528,16 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, 
		fp = skb_clone(skb, GFP_ATOMIC);

		if (!fp)

			goto out_nomem;

		rb_replace_node(&skb->rbnode, &fp->rbnode, &qp->q.rb_fragments);

		FRAG_CB(fp)->next_frag = FRAG_CB(skb)->next_frag;

		if (RB_EMPTY_NODE(&skb->rbnode))

			FRAG_CB(prev_tail)->next_frag = fp;

		else

			rb_replace_node(&skb->rbnode, &fp->rbnode,

					&qp->q.rb_fragments);

		if (qp->q.fragments_tail == skb)

			qp->q.fragments_tail = fp;

		skb_morph(skb, head);

		FRAG_CB(skb)->next_frag = FRAG_CB(head)->next_frag;

		rb_replace_node(&head->rbnode, &skb->rbnode,

				&qp->q.rb_fragments);

		consume_skb(head);
@@ -507,7 +573,7 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, 
		for (i = 0; i < skb_shinfo(head)->nr_frags; i++)

			plen += skb_frag_size(&skb_shinfo(head)->frags[i]);

		clone->len = clone->data_len = head->data_len - plen;

		skb->truesize += clone->truesize;

		head->truesize += clone->truesize;

		clone->csum = 0;

		clone->ip_summed = head->ip_summed;

		add_frag_mem_limit(qp->q.net, clone->truesize);
@@ -520,13 +586,15 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, 
	skb_push(head, head->data - skb_network_header(head));



	/* Traverse the tree in order, to build frag_list. */

	fp = FRAG_CB(head)->next_frag;

	rbn = rb_next(&head->rbnode);

	rb_erase(&head->rbnode, &qp->q.rb_fragments);

	while (rbn) {

		struct rb_node *rbnext = rb_next(rbn);

		fp = rb_to_skb(rbn);

		rb_erase(rbn, &qp->q.rb_fragments);

		rbn = rbnext;

	while (rbn || fp) {

		/* fp points to the next sk_buff in the current run;

		 * rbn points to the next run.

		 */

		/* Go through the current run. */

		while (fp) {

			*nextp = fp;

			nextp = &fp->next;

			fp->prev = NULL;
@@ -538,6 +606,16 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, 
			else if (head->ip_summed == CHECKSUM_COMPLETE)

				head->csum = csum_add(head->csum, fp->csum);

			head->truesize += fp->truesize;

			fp = FRAG_CB(fp)->next_frag;

		}

		/* Move to the next run. */

		if (rbn) {

			struct rb_node *rbnext = rb_next(rbn);



			fp = rb_to_skb(rbn);

			rb_erase(rbn, &qp->q.rb_fragments);

			rbn = rbnext;

		}

	}

	sub_frag_mem_limit(qp->q.net, head->truesize);
@@ -573,6 +651,7 @@ static int ip_frag_reasm(struct ipq *qp, struct sk_buff *skb, 
	qp->q.fragments = NULL;

	qp->q.rb_fragments = RB_ROOT;

	qp->q.fragments_tail = NULL;

	qp->q.last_run_head = NULL;

	return 0;



out_nomem:
@@ -654,6 +733,28 @@ struct sk_buff *ip_check_defrag(struct net *net, struct sk_buff *skb, u32 user) 
}

EXPORT_SYMBOL(ip_check_defrag);



unsigned int inet_frag_rbtree_purge(struct rb_root *root)

{

	struct rb_node *p = rb_first(root);

	unsigned int sum = 0;



	while (p) {

		struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);



		p = rb_next(p);

		rb_erase(&skb->rbnode, root);

		while (skb) {

			struct sk_buff *next = FRAG_CB(skb)->next_frag;



			sum += skb->truesize;

			kfree_skb(skb);

			skb = next;

		}

	}

	return sum;

}

EXPORT_SYMBOL(inet_frag_rbtree_purge);



#ifdef CONFIG_SYSCTL

static int dist_min;