Commit bf36c1a0 authored by Alexander Duyck's avatar Alexander Duyck Committed by Jeff Garzik
Browse files

igb: add page recycling support 



This patch adds support for page recycling by splitting the page into two
usable portions and tracking the reference count.

Signed-off-by: default avatarAlexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
Signed-off-by: default avatarJeff Garzik <jgarzik@redhat.com>
parent 7dfc16fa

drivers/net/igb/igb.h

+1 −3
Original line number Diff line number Diff line
@@ -125,6 +125,7 @@ struct igb_buffer { 
		struct {

			struct page *page;

			u64 page_dma;

			unsigned int page_offset;

		};

	};

};
@@ -163,9 +164,6 @@ struct igb_ring { 
		};

		/* RX */

		struct {

			/* arrays of page information for packet split */

			struct sk_buff *pending_skb;

			int pending_skb_page;

			int no_itr_adjust;

			struct igb_queue_stats rx_stats;

			struct napi_struct napi;

drivers/net/igb/igb_main.c

+62 −76
Original line number Diff line number Diff line
@@ -1725,7 +1725,6 @@ int igb_setup_rx_resources(struct igb_adapter *adapter, 


	rx_ring->next_to_clean = 0;

	rx_ring->next_to_use = 0;

	rx_ring->pending_skb = NULL;



	rx_ring->adapter = adapter;
@@ -1817,15 +1816,6 @@ static void igb_setup_rctl(struct igb_adapter *adapter) 
			rctl |= E1000_RCTL_SZ_2048;

			rctl &= ~E1000_RCTL_BSEX;

			break;

		case IGB_RXBUFFER_4096:

			rctl |= E1000_RCTL_SZ_4096;

			break;

		case IGB_RXBUFFER_8192:

			rctl |= E1000_RCTL_SZ_8192;

			break;

		case IGB_RXBUFFER_16384:

			rctl |= E1000_RCTL_SZ_16384;

			break;

		}

	} else {

		rctl &= ~E1000_RCTL_BSEX;
@@ -1843,10 +1833,8 @@ static void igb_setup_rctl(struct igb_adapter *adapter) 
	 * so only enable packet split for jumbo frames */

	if (rctl & E1000_RCTL_LPE) {

		adapter->rx_ps_hdr_size = IGB_RXBUFFER_128;

		srrctl = adapter->rx_ps_hdr_size <<

		srrctl |= adapter->rx_ps_hdr_size <<

			 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;

		/* buffer size is ALWAYS one page */

		srrctl |= PAGE_SIZE >> E1000_SRRCTL_BSIZEPKT_SHIFT;

		srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;

	} else {

		adapter->rx_ps_hdr_size = 0;
@@ -2151,20 +2139,17 @@ static void igb_clean_rx_ring(struct igb_ring *rx_ring) 
			buffer_info->skb = NULL;

		}

		if (buffer_info->page) {

			if (buffer_info->page_dma)

				pci_unmap_page(pdev, buffer_info->page_dma,

				       PAGE_SIZE, PCI_DMA_FROMDEVICE);

					       PAGE_SIZE / 2,

					       PCI_DMA_FROMDEVICE);

			put_page(buffer_info->page);

			buffer_info->page = NULL;

			buffer_info->page_dma = 0;

			buffer_info->page_offset = 0;

		}

	}



	/* there also may be some cached data from a chained receive */

	if (rx_ring->pending_skb) {

		dev_kfree_skb(rx_ring->pending_skb);

		rx_ring->pending_skb = NULL;

	}



	size = sizeof(struct igb_buffer) * rx_ring->count;

	memset(rx_ring->buffer_info, 0, size);
@@ -3091,7 +3076,11 @@ static int igb_change_mtu(struct net_device *netdev, int new_mtu) 
	else if (max_frame <= IGB_RXBUFFER_2048)

		adapter->rx_buffer_len = IGB_RXBUFFER_2048;

	else

		adapter->rx_buffer_len = IGB_RXBUFFER_4096;

#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384

		adapter->rx_buffer_len = IGB_RXBUFFER_16384;

#else

		adapter->rx_buffer_len = PAGE_SIZE / 2;

#endif

	/* adjust allocation if LPE protects us, and we aren't using SBP */

	if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||

	     (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE))
@@ -3796,7 +3785,7 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring, 
	union e1000_adv_rx_desc *rx_desc , *next_rxd;

	struct igb_buffer *buffer_info , *next_buffer;

	struct sk_buff *skb;

	unsigned int i, j;

	unsigned int i;

	u32 length, hlen, staterr;

	bool cleaned = false;

	int cleaned_count = 0;
@@ -3826,21 +3815,10 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring, 
		cleaned = true;

		cleaned_count++;



		if (rx_ring->pending_skb != NULL) {

			skb = rx_ring->pending_skb;

			rx_ring->pending_skb = NULL;

			j = rx_ring->pending_skb_page;

		} else {

		skb = buffer_info->skb;

		prefetch(skb->data - NET_IP_ALIGN);

		buffer_info->skb = NULL;

			if (hlen) {

				pci_unmap_single(pdev, buffer_info->dma,

						 adapter->rx_ps_hdr_size +

						   NET_IP_ALIGN,

						 PCI_DMA_FROMDEVICE);

				skb_put(skb, hlen);

			} else {

		if (!adapter->rx_ps_hdr_size) {

			pci_unmap_single(pdev, buffer_info->dma,

					 adapter->rx_buffer_len +

					   NET_IP_ALIGN,
@@ -3848,39 +3826,35 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring, 
			skb_put(skb, length);

			goto send_up;

		}

			j = 0;



		if (!skb_shinfo(skb)->nr_frags) {

			pci_unmap_single(pdev, buffer_info->dma,

					 adapter->rx_ps_hdr_size +

					   NET_IP_ALIGN,

					 PCI_DMA_FROMDEVICE);

			skb_put(skb, hlen);

		}



		while (length) {

		if (length) {

			pci_unmap_page(pdev, buffer_info->page_dma,

				PAGE_SIZE, PCI_DMA_FROMDEVICE);

				       PAGE_SIZE / 2, PCI_DMA_FROMDEVICE);

			buffer_info->page_dma = 0;

			skb_fill_page_desc(skb, j, buffer_info->page,

						0, length);



			skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags++,

						buffer_info->page,

						buffer_info->page_offset,

						length);



			if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||

			    (page_count(buffer_info->page) != 1))

				buffer_info->page = NULL;

			else

				get_page(buffer_info->page);



			skb->len += length;

			skb->data_len += length;

			skb->truesize += length;

			rx_desc->wb.upper.status_error = 0;

			if (staterr & E1000_RXD_STAT_EOP)

				break;



			j++;

			cleaned_count++;

			i++;

			if (i == rx_ring->count)

				i = 0;



			buffer_info = &rx_ring->buffer_info[i];

			rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);

			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);

			length = le16_to_cpu(rx_desc->wb.upper.length);

			if (!(staterr & E1000_RXD_STAT_DD)) {

				rx_ring->pending_skb = skb;

				rx_ring->pending_skb_page = j;

				goto out;

			}

			skb->truesize += length;

		}

send_up:

		i++;
@@ -3890,6 +3864,12 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring, 
		prefetch(next_rxd);

		next_buffer = &rx_ring->buffer_info[i];



		if (!(staterr & E1000_RXD_STAT_EOP)) {

			buffer_info->skb = xchg(&next_buffer->skb, skb);

			buffer_info->dma = xchg(&next_buffer->dma, 0);

			goto next_desc;

		}



		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {

			dev_kfree_skb_irq(skb);

			goto next_desc;
@@ -3922,7 +3902,7 @@ static bool igb_clean_rx_irq_adv(struct igb_ring *rx_ring, 


		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);

	}

out:



	rx_ring->next_to_clean = i;

	cleaned_count = IGB_DESC_UNUSED(rx_ring);
@@ -3960,16 +3940,22 @@ static void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, 
	while (cleaned_count--) {

		rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);



		if (adapter->rx_ps_hdr_size && !buffer_info->page) {

		if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {

			if (!buffer_info->page) {

				buffer_info->page = alloc_page(GFP_ATOMIC);

				if (!buffer_info->page) {

					adapter->alloc_rx_buff_failed++;

					goto no_buffers;

				}

				buffer_info->page_offset = 0;

			} else {

				buffer_info->page_offset ^= PAGE_SIZE / 2;

			}

			buffer_info->page_dma =

				pci_map_page(pdev,

					     buffer_info->page,

					     0, PAGE_SIZE,

					     buffer_info->page_offset,

					     PAGE_SIZE / 2,

					     PCI_DMA_FROMDEVICE);

		}