octeon_ep: add Tx/Rx processing and interrupt support (37d79d05) · Commits · EulixOS / Software / Kernel

drivers/net/ethernet/marvell/octeon_ep/octep_main.c

+587 −0

Original line number	Diff line number	Diff line
		@@ -32,6 +32,431 @@ MODULE_DESCRIPTION(OCTEP_DRV_STRING);
		MODULE_LICENSE("GPL");
		MODULE_VERSION(OCTEP_DRV_VERSION_STR);

		/**
		* octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
		*
		* @oct: Octeon device private data structure.
		*
		* Allocate resources to hold per Tx/Rx queue interrupt info.
		* This is the information passed to interrupt handler, from which napi poll
		* is scheduled and includes quick access to private data of Tx/Rx queue
		* corresponding to the interrupt being handled.
		*
		* Return: 0, on successful allocation of resources for all queue interrupts.
		* -1, if failed to allocate any resource.
		*/
		static int octep_alloc_ioq_vectors(struct octep_device *oct)
		{
		int i;
		struct octep_ioq_vector *ioq_vector;

		for (i = 0; i < oct->num_oqs; i++) {
		oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
		if (!oct->ioq_vector[i])
		goto free_ioq_vector;

		ioq_vector = oct->ioq_vector[i];
		ioq_vector->iq = oct->iq[i];
		ioq_vector->oq = oct->oq[i];
		ioq_vector->octep_dev = oct;
		}

		dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
		return 0;

		free_ioq_vector:
		while (i) {
		i--;
		vfree(oct->ioq_vector[i]);
		oct->ioq_vector[i] = NULL;
		}
		return -1;
		}

		/**
		* octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
		*
		* @oct: Octeon device private data structure.
		*/
		static void octep_free_ioq_vectors(struct octep_device *oct)
		{
		int i;

		for (i = 0; i < oct->num_oqs; i++) {
		if (oct->ioq_vector[i]) {
		vfree(oct->ioq_vector[i]);
		oct->ioq_vector[i] = NULL;
		}
		}
		netdev_info(oct->netdev, "Freed IOQ Vectors\n");
		}

		/**
		* octep_enable_msix_range() - enable MSI-x interrupts.
		*
		* @oct: Octeon device private data structure.
		*
		* Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
		* for the Octeon device.
		*
		* Return: 0, on successfully enabling all MSI-x interrupts.
		* -1, if failed to enable any MSI-x interrupt.
		*/
		static int octep_enable_msix_range(struct octep_device *oct)
		{
		int num_msix, msix_allocated;
		int i;

		/* Generic interrupts apart from input/output queues */
		num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
		oct->msix_entries = kcalloc(num_msix,
		sizeof(struct msix_entry), GFP_KERNEL);
		if (!oct->msix_entries)
		goto msix_alloc_err;

		for (i = 0; i < num_msix; i++)
		oct->msix_entries[i].entry = i;

		msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
		num_msix, num_msix);
		if (msix_allocated != num_msix) {
		dev_err(&oct->pdev->dev,
		"Failed to enable %d msix irqs; got only %d\n",
		num_msix, msix_allocated);
		goto enable_msix_err;
		}
		oct->num_irqs = msix_allocated;
		dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");

		return 0;

		enable_msix_err:
		if (msix_allocated > 0)
		pci_disable_msix(oct->pdev);
		kfree(oct->msix_entries);
		oct->msix_entries = NULL;
		msix_alloc_err:
		return -1;
		}

		/**
		* octep_disable_msix() - disable MSI-x interrupts.
		*
		* @oct: Octeon device private data structure.
		*
		* Disable MSI-x on the Octeon device.
		*/
		static void octep_disable_msix(struct octep_device *oct)
		{
		pci_disable_msix(oct->pdev);
		kfree(oct->msix_entries);
		oct->msix_entries = NULL;
		dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
		}

		/**
		* octep_non_ioq_intr_handler() - common handler for all generic interrupts.
		*
		* @irq: Interrupt number.
		* @data: interrupt data.
		*
		* this is common handler for all non-queue (generic) interrupts.
		*/
		static irqreturn_t octep_non_ioq_intr_handler(int irq, void *data)
		{
		struct octep_device *oct = data;

		return oct->hw_ops.non_ioq_intr_handler(oct);
		}

		/**
		* octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
		*
		* @irq: Interrupt number.
		* @data: interrupt data contains pointers to Tx/Rx queue private data
		* and correspong NAPI context.
		*
		* this is common handler for all non-queue (generic) interrupts.
		*/
		static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
		{
		struct octep_ioq_vector *ioq_vector = data;
		struct octep_device *oct = ioq_vector->octep_dev;

		return oct->hw_ops.ioq_intr_handler(ioq_vector);
		}

		/**
		* octep_request_irqs() - Register interrupt handlers.
		*
		* @oct: Octeon device private data structure.
		*
		* Register handlers for all queue and non-queue interrupts.
		*
		* Return: 0, on successful registration of all interrupt handlers.
		* -1, on any error.
		*/
		static int octep_request_irqs(struct octep_device *oct)
		{
		struct net_device *netdev = oct->netdev;
		struct octep_ioq_vector *ioq_vector;
		struct msix_entry *msix_entry;
		char **non_ioq_msix_names;
		int num_non_ioq_msix;
		int ret, i;

		num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
		non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);

		oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix,
		OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
		if (!oct->non_ioq_irq_names)
		goto alloc_err;

		/* First few MSI-X interrupts are non-queue interrupts */
		for (i = 0; i < num_non_ioq_msix; i++) {
		char *irq_name;

		irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
		msix_entry = &oct->msix_entries[i];

		snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
		"%s-%s", netdev->name, non_ioq_msix_names[i]);
		ret = request_irq(msix_entry->vector,
		octep_non_ioq_intr_handler, 0,
		irq_name, oct);
		if (ret) {
		netdev_err(netdev,
		"request_irq failed for %s; err=%d",
		irq_name, ret);
		goto non_ioq_irq_err;
		}
		}

		/* Request IRQs for Tx/Rx queues */
		for (i = 0; i < oct->num_oqs; i++) {
		ioq_vector = oct->ioq_vector[i];
		msix_entry = &oct->msix_entries[i + num_non_ioq_msix];

		snprintf(ioq_vector->name, sizeof(ioq_vector->name),
		"%s-q%d", netdev->name, i);
		ret = request_irq(msix_entry->vector,
		octep_ioq_intr_handler, 0,
		ioq_vector->name, ioq_vector);
		if (ret) {
		netdev_err(netdev,
		"request_irq failed for Q-%d; err=%d",
		i, ret);
		goto ioq_irq_err;
		}

		cpumask_set_cpu(i % num_online_cpus(),
		&ioq_vector->affinity_mask);
		irq_set_affinity_hint(msix_entry->vector,
		&ioq_vector->affinity_mask);
		}

		return 0;
		ioq_irq_err:
		while (i > num_non_ioq_msix) {
		--i;
		irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
		free_irq(oct->msix_entries[i].vector, oct->ioq_vector[i]);
		}
		non_ioq_irq_err:
		while (i) {
		--i;
		free_irq(oct->msix_entries[i].vector, oct);
		}
		alloc_err:
		return -1;
		}

		/**
		* octep_free_irqs() - free all registered interrupts.
		*
		* @oct: Octeon device private data structure.
		*
		* Free all queue and non-queue interrupts of the Octeon device.
		*/
		static void octep_free_irqs(struct octep_device *oct)
		{
		int i;

		/* First few MSI-X interrupts are non queue interrupts; free them */
		for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
		free_irq(oct->msix_entries[i].vector, oct);
		kfree(oct->non_ioq_irq_names);

		/* Free IRQs for Input/Output (Tx/Rx) queues */
		for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
		irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
		free_irq(oct->msix_entries[i].vector,
		oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
		}
		netdev_info(oct->netdev, "IRQs freed\n");
		}

		/**
		* octep_setup_irqs() - setup interrupts for the Octeon device.
		*
		* @oct: Octeon device private data structure.
		*
		* Allocate data structures to hold per interrupt information, allocate/enable
		* MSI-x interrupt and register interrupt handlers.
		*
		* Return: 0, on successful allocation and registration of all interrupts.
		* -1, on any error.
		*/
		static int octep_setup_irqs(struct octep_device *oct)
		{
		if (octep_alloc_ioq_vectors(oct))
		goto ioq_vector_err;

		if (octep_enable_msix_range(oct))
		goto enable_msix_err;

		if (octep_request_irqs(oct))
		goto request_irq_err;

		return 0;

		request_irq_err:
		octep_disable_msix(oct);
		enable_msix_err:
		octep_free_ioq_vectors(oct);
		ioq_vector_err:
		return -1;
		}

		/**
		* octep_clean_irqs() - free all interrupts and its resources.
		*
		* @oct: Octeon device private data structure.
		*/
		static void octep_clean_irqs(struct octep_device *oct)
		{
		octep_free_irqs(oct);
		octep_disable_msix(oct);
		octep_free_ioq_vectors(oct);
		}

		/**
		* octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
		*
		* @iq: Octeon Tx queue data structure.
		* @oq: Octeon Rx queue data structure.
		*/
		static void octep_enable_ioq_irq(struct octep_iq iq, struct octep_oq oq)
		{
		u32 pkts_pend = oq->pkts_pending;

		netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
		if (iq->pkts_processed) {
		writel(iq->pkts_processed, iq->inst_cnt_reg);
		iq->pkt_in_done -= iq->pkts_processed;
		iq->pkts_processed = 0;
		}
		if (oq->last_pkt_count - pkts_pend) {
		writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg);
		oq->last_pkt_count = pkts_pend;
		}

		/* Flush the previous wrties before writing to RESEND bit */
		wmb();
		writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
		writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
		}

		/**
		* octep_napi_poll() - NAPI poll function for Tx/Rx.
		*
		* @napi: pointer to napi context.
		* @budget: max number of packets to be processed in single invocation.
		*/
		static int octep_napi_poll(struct napi_struct *napi, int budget)
		{
		struct octep_ioq_vector *ioq_vector =
		container_of(napi, struct octep_ioq_vector, napi);
		u32 tx_pending, rx_done;

		tx_pending = octep_iq_process_completions(ioq_vector->iq, budget);
		rx_done = octep_oq_process_rx(ioq_vector->oq, budget);

		/* need more polling if tx completion processing is still pending or
		* processed at least 'budget' number of rx packets.
		*/
		if (tx_pending \|\| rx_done >= budget)
		return budget;

		napi_complete(napi);
		octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
		return rx_done;
		}

		/**
		* octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
		*
		* @oct: Octeon device private data structure.
		*/
		static void octep_napi_add(struct octep_device *oct)
		{
		int i;

		for (i = 0; i < oct->num_oqs; i++) {
		netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
		netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
		octep_napi_poll, 64);
		oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
		}
		}

		/**
		* octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
		*
		* @oct: Octeon device private data structure.
		*/
		static void octep_napi_delete(struct octep_device *oct)
		{
		int i;

		for (i = 0; i < oct->num_oqs; i++) {
		netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
		netif_napi_del(&oct->ioq_vector[i]->napi);
		oct->oq[i]->napi = NULL;
		}
		}

		/**
		* octep_napi_enable() - enable NAPI for all Tx/Rx queues.
		*
		* @oct: Octeon device private data structure.
		*/
		static void octep_napi_enable(struct octep_device *oct)
		{
		int i;

		for (i = 0; i < oct->num_oqs; i++) {
		netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
		napi_enable(&oct->ioq_vector[i]->napi);
		}
		}

		/**
		* octep_napi_disable() - disable NAPI for all Tx/Rx queues.
		*
		* @oct: Octeon device private data structure.
		*/
		static void octep_napi_disable(struct octep_device *oct)
		{
		int i;

		for (i = 0; i < oct->num_oqs; i++) {
		netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
		napi_disable(&oct->ioq_vector[i]->napi);
		}
		}

		static void octep_link_up(struct net_device *netdev)
		{
		netif_carrier_on(netdev);
		@@ -63,6 +488,8 @@ static int octep_open(struct net_device *netdev)
		goto setup_iq_err;
		if (octep_setup_oqs(oct))
		goto setup_oq_err;
		if (octep_setup_irqs(oct))
		goto setup_irq_err;

		err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
		if (err)
		@@ -71,6 +498,9 @@ static int octep_open(struct net_device *netdev)
		if (err)
		goto set_queues_err;

		octep_napi_add(oct);
		octep_napi_enable(oct);

		oct->link_info.admin_up = 1;
		octep_set_rx_state(oct, true);

		@@ -93,6 +523,10 @@ static int octep_open(struct net_device *netdev)
		return 0;

		set_queues_err:
		octep_napi_disable(oct);
		octep_napi_delete(oct);
		octep_clean_irqs(oct);
		setup_irq_err:
		octep_free_oqs(oct);
		setup_oq_err:
		octep_free_iqs(oct);
		@@ -126,7 +560,10 @@ static int octep_stop(struct net_device *netdev)
		oct->link_info.oper_up = 0;

		oct->hw_ops.disable_interrupts(oct);
		octep_napi_disable(oct);
		octep_napi_delete(oct);

		octep_clean_irqs(oct);
		octep_clean_iqs(oct);

		oct->hw_ops.disable_io_queues(oct);
		@@ -137,6 +574,36 @@ static int octep_stop(struct net_device *netdev)
		return 0;
		}

		/**
		* octep_iq_full_check() - check if a Tx queue is full.
		*
		* @iq: Octeon Tx queue data structure.
		*
		* Return: 0, if the Tx queue is not full.
		* 1, if the Tx queue is full.
		*/
		static inline int octep_iq_full_check(struct octep_iq *iq)
		{
		if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >=
		OCTEP_WAKE_QUEUE_THRESHOLD))
		return 0;

		/* Stop the queue if unable to send */
		netif_stop_subqueue(iq->netdev, iq->q_no);

		/* check again and restart the queue, in case NAPI has just freed
		* enough Tx ring entries.
		*/
		if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >=
		OCTEP_WAKE_QUEUE_THRESHOLD)) {
		netif_start_subqueue(iq->netdev, iq->q_no);
		iq->stats.restart_cnt++;
		return 0;
		}

		return 1;
		}

		/**
		* octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
		*
		@@ -149,6 +616,126 @@ static int octep_stop(struct net_device *netdev)
		static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
		struct net_device *netdev)
		{
		struct octep_device *oct = netdev_priv(netdev);
		struct octep_tx_sglist_desc *sglist;
		struct octep_tx_buffer *tx_buffer;
		struct octep_tx_desc_hw *hw_desc;
		struct skb_shared_info *shinfo;
		struct octep_instr_hdr *ih;
		struct octep_iq *iq;
		skb_frag_t *frag;
		u16 nr_frags, si;
		u16 q_no, wi;

		q_no = skb_get_queue_mapping(skb);
		if (q_no >= oct->num_iqs) {
		netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
		q_no = q_no % oct->num_iqs;
		}

		iq = oct->iq[q_no];
		if (octep_iq_full_check(iq)) {
		iq->stats.tx_busy++;
		return NETDEV_TX_BUSY;
		}

		shinfo = skb_shinfo(skb);
		nr_frags = shinfo->nr_frags;

		wi = iq->host_write_index;
		hw_desc = &iq->desc_ring[wi];
		hw_desc->ih64 = 0;

		tx_buffer = iq->buff_info + wi;
		tx_buffer->skb = skb;

		ih = &hw_desc->ih;
		ih->tlen = skb->len;
		ih->pkind = oct->pkind;

		if (!nr_frags) {
		tx_buffer->gather = 0;
		tx_buffer->dma = dma_map_single(iq->dev, skb->data,
		skb->len, DMA_TO_DEVICE);
		if (dma_mapping_error(iq->dev, tx_buffer->dma))
		goto dma_map_err;
		hw_desc->dptr = tx_buffer->dma;
		} else {
		/* Scatter/Gather */
		dma_addr_t dma;
		u16 len;

		sglist = tx_buffer->sglist;

		ih->gsz = nr_frags + 1;
		ih->gather = 1;
		tx_buffer->gather = 1;

		len = skb_headlen(skb);
		dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
		if (dma_mapping_error(iq->dev, dma))
		goto dma_map_err;

		dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma,
		OCTEP_SGLIST_SIZE_PER_PKT,
		DMA_TO_DEVICE);
		memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
		sglist[0].len[3] = len;
		sglist[0].dma_ptr[0] = dma;

		si = 1; /* entry 0 is main skb, mapped above */
		frag = &shinfo->frags[0];
		while (nr_frags--) {
		len = skb_frag_size(frag);
		dma = skb_frag_dma_map(iq->dev, frag, 0,
		len, DMA_TO_DEVICE);
		if (dma_mapping_error(iq->dev, dma))
		goto dma_map_sg_err;

		sglist[si >> 2].len[3 - (si & 3)] = len;
		sglist[si >> 2].dma_ptr[si & 3] = dma;

		frag++;
		si++;
		}
		dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma,
		OCTEP_SGLIST_SIZE_PER_PKT,
		DMA_TO_DEVICE);

		hw_desc->dptr = tx_buffer->sglist_dma;
		}

		/* Flush the hw descriptor before writing to doorbell */
		wmb();

		/* Ring Doorbell to notify the NIC there is a new packet */
		writel(1, iq->doorbell_reg);
		atomic_inc(&iq->instr_pending);
		wi++;
		if (wi == iq->max_count)
		wi = 0;
		iq->host_write_index = wi;

		netdev_tx_sent_queue(iq->netdev_q, skb->len);
		iq->stats.instr_posted++;
		skb_tx_timestamp(skb);
		return NETDEV_TX_OK;

		dma_map_sg_err:
		if (si > 0) {
		dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
		sglist[0].len[0], DMA_TO_DEVICE);
		sglist[0].len[0] = 0;
		}
		while (si > 1) {
		dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
		sglist[si >> 2].len[si & 3], DMA_TO_DEVICE);
		sglist[si >> 2].len[si & 3] = 0;
		si--;
		}
		tx_buffer->gather = 0;
		dma_map_err:
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
		}

drivers/net/ethernet/marvell/octeon_ep/octep_rx.c

+244 −0

Original line number	Diff line number	Diff line
		@@ -68,6 +68,50 @@ static int octep_oq_fill_ring_buffers(struct octep_oq *oq)
		return -1;
		}

		/**
		* octep_oq_refill() - refill buffers for used Rx ring descriptors.
		*
		* @oct: Octeon device private data structure.
		* @oq: Octeon Rx queue data structure.
		*
		* Return: number of descriptors successfully refilled with receive buffers.
		*/
		static int octep_oq_refill(struct octep_device oct, struct octep_oq oq)
		{
		struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
		struct page *page;
		u32 refill_idx, i;

		refill_idx = oq->host_refill_idx;
		for (i = 0; i < oq->refill_count; i++) {
		page = dev_alloc_page();
		if (unlikely(!page)) {
		dev_err(oq->dev, "refill: rx buffer alloc failed\n");
		oq->stats.alloc_failures++;
		break;
		}

		desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
		PAGE_SIZE, DMA_FROM_DEVICE);
		if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
		dev_err(oq->dev,
		"OQ-%d buffer refill: DMA mapping error!\n",
		oq->q_no);
		put_page(page);
		oq->stats.alloc_failures++;
		break;
		}
		oq->buff_info[refill_idx].page = page;
		refill_idx++;
		if (refill_idx == oq->max_count)
		refill_idx = 0;
		}
		oq->host_refill_idx = refill_idx;
		oq->refill_count -= i;

		return i;
		}

		/**
		* octep_setup_oq() - Setup a Rx queue.
		*
		@@ -262,3 +306,203 @@ void octep_free_oqs(struct octep_device *oct)
		"Successfully freed OQ(RxQ)-%d.\n", i);
		}
		}

		/**
		* octep_oq_check_hw_for_pkts() - Check for new Rx packets.
		*
		* @oct: Octeon device private data structure.
		* @oq: Octeon Rx queue data structure.
		*
		* Return: packets received after previous check.
		*/
		static int octep_oq_check_hw_for_pkts(struct octep_device *oct,
		struct octep_oq *oq)
		{
		u32 pkt_count, new_pkts;

		pkt_count = readl(oq->pkts_sent_reg);
		new_pkts = pkt_count - oq->last_pkt_count;

		/* Clear the hardware packets counter register if the rx queue is
		* being processed continuously with-in a single interrupt and
		* reached half its max value.
		* this counter is not cleared every time read, to save write cycles.
		*/
		if (unlikely(pkt_count > 0xF0000000U)) {
		writel(pkt_count, oq->pkts_sent_reg);
		pkt_count = readl(oq->pkts_sent_reg);
		new_pkts += pkt_count;
		}
		oq->last_pkt_count = pkt_count;
		oq->pkts_pending += new_pkts;
		return new_pkts;
		}

		/**
		* __octep_oq_process_rx() - Process hardware Rx queue and push to stack.
		*
		* @oct: Octeon device private data structure.
		* @oq: Octeon Rx queue data structure.
		* @pkts_to_process: number of packets to be processed.
		*
		* Process the new packets in Rx queue.
		* Packets larger than single Rx buffer arrive in consecutive descriptors.
		* But, count returned by the API only accounts full packets, not fragments.
		*
		* Return: number of packets processed and pushed to stack.
		*/
		static int __octep_oq_process_rx(struct octep_device *oct,
		struct octep_oq *oq, u16 pkts_to_process)
		{
		struct octep_oq_resp_hw_ext *resp_hw_ext = NULL;
		struct octep_rx_buffer *buff_info;
		struct octep_oq_resp_hw *resp_hw;
		u32 pkt, rx_bytes, desc_used;
		struct sk_buff *skb;
		u16 data_offset;
		u32 read_idx;

		read_idx = oq->host_read_idx;
		rx_bytes = 0;
		desc_used = 0;
		for (pkt = 0; pkt < pkts_to_process; pkt++) {
		buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];
		dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
		PAGE_SIZE, DMA_FROM_DEVICE);
		resp_hw = page_address(buff_info->page);
		buff_info->page = NULL;

		/* Swap the length field that is in Big-Endian to CPU */
		buff_info->len = be64_to_cpu(resp_hw->length);
		if (oct->caps_enabled & OCTEP_CAP_RX_CHECKSUM) {
		/* Extended response header is immediately after
		* response header (resp_hw)
		*/
		resp_hw_ext = (struct octep_oq_resp_hw_ext *)
		(resp_hw + 1);
		buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE;
		/* Packet Data is immediately after
		* extended response header.
		*/
		data_offset = OCTEP_OQ_RESP_HW_SIZE +
		OCTEP_OQ_RESP_HW_EXT_SIZE;
		} else {
		/* Data is immediately after
		* Hardware Rx response header.
		*/
		data_offset = OCTEP_OQ_RESP_HW_SIZE;
		}
		rx_bytes += buff_info->len;

		if (buff_info->len <= oq->max_single_buffer_size) {
		skb = build_skb((void *)resp_hw, PAGE_SIZE);
		skb_reserve(skb, data_offset);
		skb_put(skb, buff_info->len);
		read_idx++;
		desc_used++;
		if (read_idx == oq->max_count)
		read_idx = 0;
		} else {
		struct skb_shared_info *shinfo;
		u16 data_len;

		skb = build_skb((void *)resp_hw, PAGE_SIZE);
		skb_reserve(skb, data_offset);
		/* Head fragment includes response header(s);
		* subsequent fragments contains only data.
		*/
		skb_put(skb, oq->max_single_buffer_size);
		read_idx++;
		desc_used++;
		if (read_idx == oq->max_count)
		read_idx = 0;

		shinfo = skb_shinfo(skb);
		data_len = buff_info->len - oq->max_single_buffer_size;
		while (data_len) {
		dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
		PAGE_SIZE, DMA_FROM_DEVICE);
		buff_info = (struct octep_rx_buffer *)
		&oq->buff_info[read_idx];
		if (data_len < oq->buffer_size) {
		buff_info->len = data_len;
		data_len = 0;
		} else {
		buff_info->len = oq->buffer_size;
		data_len -= oq->buffer_size;
		}

		skb_add_rx_frag(skb, shinfo->nr_frags,
		buff_info->page, 0,
		buff_info->len,
		buff_info->len);
		buff_info->page = NULL;
		read_idx++;
		desc_used++;
		if (read_idx == oq->max_count)
		read_idx = 0;
		}
		}

		skb->dev = oq->netdev;
		skb->protocol = eth_type_trans(skb, skb->dev);
		if (resp_hw_ext &&
		resp_hw_ext->csum_verified == OCTEP_CSUM_VERIFIED)
		skb->ip_summed = CHECKSUM_UNNECESSARY;
		else
		skb->ip_summed = CHECKSUM_NONE;
		napi_gro_receive(oq->napi, skb);
		}

		oq->host_read_idx = read_idx;
		oq->refill_count += desc_used;
		oq->stats.packets += pkt;
		oq->stats.bytes += rx_bytes;

		return pkt;
		}

		/**
		* octep_oq_process_rx() - Process Rx queue.
		*
		* @oq: Octeon Rx queue data structure.
		* @budget: max number of packets can be processed in one invocation.
		*
		* Check for newly received packets and process them.
		* Keeps checking for new packets until budget is used or no new packets seen.
		*
		* Return: number of packets processed.
		*/
		int octep_oq_process_rx(struct octep_oq *oq, int budget)
		{
		u32 pkts_available, pkts_processed, total_pkts_processed;
		struct octep_device *oct = oq->octep_dev;

		pkts_available = 0;
		pkts_processed = 0;
		total_pkts_processed = 0;
		while (total_pkts_processed < budget) {
		/* update pending count only when current one exhausted */
		if (oq->pkts_pending == 0)
		octep_oq_check_hw_for_pkts(oct, oq);
		pkts_available = min(budget - total_pkts_processed,
		oq->pkts_pending);
		if (!pkts_available)
		break;

		pkts_processed = __octep_oq_process_rx(oct, oq,
		pkts_available);
		oq->pkts_pending -= pkts_processed;
		total_pkts_processed += pkts_processed;
		}

		if (oq->refill_count >= oq->refill_threshold) {
		u32 desc_refilled = octep_oq_refill(oct, oq);

		/* flush pending writes before updating credits */
		wmb();
		writel(desc_refilled, oq->pkts_credit_reg);
		}

		return total_pkts_processed;
		}

drivers/net/ethernet/marvell/octeon_ep/octep_tx.c

+73 −0

File changed.

Preview size limit exceeded, changes collapsed.