Commit 21d6aac4 authored by Phil Elwell's avatar Phil Elwell Committed by popcornmix
Browse files

bcm2835-dma: Add proper 40-bit DMA support 



The 40-bit additions are not fully tested, but it should be
capable of supporting both 40-bit memcpy on BCM2711 and regular
Lite channels on BCM2835.

Signed-off-by: default avatarPhil Elwell <phil@raspberrypi.org>
parent e86d6046

arch/arm/boot/dts/bcm2838.dtsi

+21 −12
Original line number Diff line number Diff line
@@ -372,6 +372,23 @@ 
			};

		};



		dma40: dma@7e007b00 {

			compatible = "brcm,bcm2838-dma";

			reg = <0x0 0x7e007b00 0x400>;

			interrupts =

				<GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>, /* dma4 11 */

				<GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>, /* dma4 12 */

				<GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>, /* dma4 13 */

				<GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>; /* dma4 14 */

			interrupt-names = "dma11",

				"dma12",

				"dma13",

				"dma14";

			#dma-cells = <1>;

			brcm,dma-channel-mask = <0x7000>;

		};

		/* DMA4 - 40 bit DMA engines */



		xhci: xhci@7e9c0000 {

			compatible = "generic-xhci";

			status = "disabled";
@@ -689,6 +706,7 @@ 
};



&dma {

	reg = <0x7e007000 0xb00>;

	interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>,

		<GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>,

		<GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>,
@@ -699,12 +717,7 @@ 
		<GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>, /* dmalite  7 */

		<GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>, /* dmalite  8 */

		<GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>, /* dmalite  9 */

		<GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>, /* dmalite 10 */

		/* DMA4 - 40 bit DMA engines */

		<GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>, /* dma4    11 */

		<GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>, /* dma4    12 */

		<GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>, /* dma4    13 */

		<GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>; /* dma4    14 */

		<GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>; /* dmalite 10 */

	interrupt-names = "dma0",

			  "dma1",

			  "dma2",
@@ -715,10 +728,6 @@ 
			  "dma7",

			  "dma8",

			  "dma9",

			  "dma10",

			  "dma11",

			  "dma12",

			  "dma13",

			  "dma14";

	brcm,dma-channel-mask = <0x7ef5>;

			  "dma10";

	brcm,dma-channel-mask = <0x01f5>;

};

drivers/dma/bcm2835-dma.c

+326 −100
Original line number Diff line number Diff line
@@ -50,12 +50,18 @@ 
#define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14

#define BCM2835_DMA_CHAN_NAME_SIZE 8

#define BCM2835_DMA_BULK_MASK  BIT(0)

#define BCM2838_DMA_MEMCPY_CHAN 14



struct bcm2835_dma_cfg_data {

	u32	chan_40bit_mask;

};



struct bcm2835_dmadev {

	struct dma_device ddev;

	spinlock_t lock;

	void __iomem *base;

	struct device_dma_parameters dma_parms;

	const struct bcm2835_dma_cfg_data *cfg_data;

};



struct bcm2835_dma_cb {
@@ -100,6 +106,7 @@ struct bcm2835_chan { 
	unsigned int irq_flags;



	bool is_lite_channel;

	bool is_40bit_channel;

};



struct bcm2835_desc {
@@ -189,7 +196,8 @@ struct bcm2835_desc { 
#define BCM2835_DMA_DATA_TYPE_S128	16



/* Valid only for channels 0 - 14, 15 has its own base address */

#define BCM2835_DMA_CHAN(n)	((n) << 8) /* Base address */

#define BCM2835_DMA_CHAN_SIZE	0x100

#define BCM2835_DMA_CHAN(n)	((n) * BCM2835_DMA_CHAN_SIZE) /* Base address */

#define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n))



/* the max dma length for different channels */
@@ -200,7 +208,7 @@ struct bcm2835_desc { 
#define BCM2838_DMA40_CS	0x00

#define BCM2838_DMA40_CB	0x04

#define BCM2838_DMA40_DEBUG	0x0c

#define BCM2858_DMA40_TI	0x10

#define BCM2838_DMA40_TI	0x10

#define BCM2838_DMA40_SRC	0x14

#define BCM2838_DMA40_SRCI	0x18

#define BCM2838_DMA40_DEST	0x1c
@@ -209,32 +217,97 @@ struct bcm2835_desc { 
#define BCM2838_DMA40_NEXT_CB	0x28

#define BCM2838_DMA40_DEBUG2	0x2c



#define BCM2838_DMA40_CS_ACTIVE	BIT(0)

#define BCM2838_DMA40_CS_END	BIT(1)

#define BCM2838_DMA40_ACTIVE		BIT(0)

#define BCM2838_DMA40_END		BIT(1)

#define BCM2838_DMA40_INT		BIT(2)

#define BCM2838_DMA40_DREQ		BIT(3)  /* DREQ state */

#define BCM2838_DMA40_RD_PAUSED		BIT(4)  /* Reading is paused */

#define BCM2838_DMA40_WR_PAUSED		BIT(5)  /* Writing is paused */

#define BCM2838_DMA40_DREQ_PAUSED	BIT(6)  /* Is paused by DREQ flow control */

#define BCM2838_DMA40_WAITING_FOR_WRITES BIT(7)  /* Waiting for last write */

#define BCM2838_DMA40_ERR		BIT(10)

#define BCM2838_DMA40_QOS(x)		(((x) & 0x1f) << 16)

#define BCM2838_DMA40_PANIC_QOS(x)	(((x) & 0x1f) << 20)

#define BCM2838_DMA40_WAIT_FOR_WRITES	BIT(28)

#define BCM2838_DMA40_DISDEBUG		BIT(29)

#define BCM2838_DMA40_ABORT		BIT(30)

#define BCM2838_DMA40_HALT		BIT(31)

#define BCM2838_DMA40_CS_FLAGS(x) (x & (BCM2838_DMA40_QOS(15) | \

					BCM2838_DMA40_PANIC_QOS(15) | \

					BCM2838_DMA40_WAIT_FOR_WRITES |	\

					BCM2838_DMA40_DISDEBUG))



/* Transfer information bits */

#define BCM2838_DMA40_INTEN		BIT(0)

#define BCM2838_DMA40_TDMODE		BIT(1) /* 2D-Mode */

#define BCM2838_DMA40_WAIT_RESP		BIT(2) /* wait for AXI write to be acked */

#define BCM2838_DMA40_WAIT_RD_RESP	BIT(3) /* wait for AXI read to complete */

#define BCM2838_DMA40_PER_MAP(x)	((x & 31) << 9) /* REQ source */

#define BCM2838_DMA40_S_DREQ		BIT(14) /* enable SREQ for source */

#define BCM2838_DMA40_D_DREQ		BIT(15) /* enable DREQ for destination */

#define BCM2838_DMA40_S_WAIT(x)		((x & 0xff) << 16) /* add DMA read-wait cycles */

#define BCM2838_DMA40_D_WAIT(x)		((x & 0xff) << 24) /* add DMA write-wait cycles */



#define BCM2838_DMA40_CS_QOS(x)	(((x) & 0x1f) << 16)

#define BCM2838_DMA40_CS_PANIC_QOS(x)	(((x) & 0x1f) << 20)

#define BCM2838_DMA40_CS_WRITE_WAIT	BIT(28)

/* debug register bits */

#define BCM2838_DMA40_DEBUG_WRITE_ERR		BIT(0)

#define BCM2838_DMA40_DEBUG_FIFO_ERR		BIT(1)

#define BCM2838_DMA40_DEBUG_READ_ERR		BIT(2)

#define BCM2838_DMA40_DEBUG_READ_CB_ERR		BIT(3)

#define BCM2838_DMA40_DEBUG_IN_ON_ERR		BIT(8)

#define BCM2838_DMA40_DEBUG_ABORT_ON_ERR	BIT(9)

#define BCM2838_DMA40_DEBUG_HALT_ON_ERR		BIT(10)

#define BCM2838_DMA40_DEBUG_DISABLE_CLK_GATE	BIT(11)

#define BCM2838_DMA40_DEBUG_RSTATE_SHIFT	14

#define BCM2838_DMA40_DEBUG_RSTATE_BITS		4

#define BCM2838_DMA40_DEBUG_WSTATE_SHIFT	18

#define BCM2838_DMA40_DEBUG_WSTATE_BITS		4

#define BCM2838_DMA40_DEBUG_RESET		BIT(23)

#define BCM2838_DMA40_DEBUG_ID_SHIFT		24

#define BCM2838_DMA40_DEBUG_ID_BITS		4

#define BCM2838_DMA40_DEBUG_VERSION_SHIFT	28

#define BCM2838_DMA40_DEBUG_VERSION_BITS	4



/* Valid only for channels 0 - 3 (11 - 14) */

#define BCM2838_DMA40_CHAN(n)	(((n) + 11) << 8) /* Base address */

#define BCM2838_DMA40_CHANIO(base, n) ((base) + BCM2838_DMA_CHAN(n))



/* the max dma length for different channels */

#define MAX_DMA40_LEN SZ_1G



#define BCM2838_DMA40_BURST_LEN(x)	((((x) - 1) & 0xf) << 8)

#define BCM2838_DMA40_BURST_LEN(x)	((min(x,16) - 1) << 8)

#define BCM2838_DMA40_INC		BIT(12)

#define BCM2838_DMA40_SIZE_32		(0 << 13)

#define BCM2838_DMA40_SIZE_64		(1 << 13)

#define BCM2838_DMA40_SIZE_128		(2 << 13)

#define BCM2838_DMA40_SIZE_256		(3 << 13)

#define BCM2838_DMA40_IGNORE		BIT(15)

#define BCM2838_DMA40_STRIDE(x)		((x) << 16) /* For 2D mode */



#define BCM2838_DMA40_MEMCPY_QOS \

	(BCM2838_DMA40_CS_QOS(0x0) | \

	 BCM2838_DMA40_CS_PANIC_QOS(0x0) | \

	 BCM2838_DMA40_CS_WRITE_WAIT)

#define BCM2838_DMA40_MEMCPY_FLAGS \

	(BCM2838_DMA40_QOS(0) | \

	 BCM2838_DMA40_PANIC_QOS(0) | \

	 BCM2838_DMA40_WAIT_FOR_WRITES | \

	 BCM2838_DMA40_DISDEBUG)



#define BCM2838_DMA40_MEMCPY_XFER_INFO \

	(BCM2838_DMA40_SIZE_128 | \

	 BCM2838_DMA40_INC | \

	 BCM2838_DMA40_BURST_LEN(16))



struct bcm2835_dmadev *memcpy_parent;

static void __iomem *memcpy_chan;

static struct bcm2838_dma40_scb *memcpy_scb;

static dma_addr_t memcpy_scb_dma;

DEFINE_SPINLOCK(memcpy_lock);



static const struct bcm2835_dma_cfg_data bcm2835_dma_cfg = {

	.chan_40bit_mask = 0,

};



static const struct bcm2835_dma_cfg_data bcm2838_dma_cfg = {

	.chan_40bit_mask = BIT(11) | BIT(12) | BIT(13) | BIT(14),

};



static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c)

{

	/* lite and normal channels have different max frame length */
@@ -264,6 +337,32 @@ static inline struct bcm2835_desc *to_bcm2835_dma_desc( 
	return container_of(t, struct bcm2835_desc, vd.tx);

}



static inline uint32_t to_bcm2838_ti(uint32_t info)

{

	return ((info & BCM2835_DMA_INT_EN) ? BCM2838_DMA40_INTEN : 0) |

		((info & BCM2835_DMA_WAIT_RESP) ? BCM2838_DMA40_WAIT_RESP : 0) |

		((info & BCM2835_DMA_S_DREQ) ?

		 (BCM2838_DMA40_S_DREQ | BCM2838_DMA40_WAIT_RD_RESP) : 0) |

		((info & BCM2835_DMA_D_DREQ) ? BCM2838_DMA40_D_DREQ : 0) |

		BCM2838_DMA40_PER_MAP((info >> 16) & 0x1f);

}



static inline uint32_t to_bcm2838_srci(uint32_t info)

{

	return ((info & BCM2835_DMA_S_INC) ? BCM2838_DMA40_INC : 0);

}



static inline uint32_t to_bcm2838_dsti(uint32_t info)

{

	return ((info & BCM2835_DMA_D_INC) ? BCM2838_DMA40_INC : 0);

}



static inline uint32_t to_bcm2838_cbaddr(dma_addr_t addr)

{

	BUG_ON(addr & 0x1f);

	return (addr >> 5);

}



static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc)

{

	size_t i;
@@ -282,22 +381,20 @@ static void bcm2835_dma_desc_free(struct virt_dma_desc *vd) 
}



static void bcm2835_dma_create_cb_set_length(

	struct bcm2835_chan *chan,

	struct bcm2835_chan *c,

	struct bcm2835_dma_cb *control_block,

	size_t len,

	size_t period_len,

	size_t *total_len,

	u32 finalextrainfo)

{

	size_t max_len = bcm2835_dma_max_frame_length(chan);

	size_t max_len = bcm2835_dma_max_frame_length(c);

	uint32_t cb_len;



	/* set the length taking lite-channel limitations into account */

	control_block->length = min_t(u32, len, max_len);



	/* finished if we have no period_length */

	if (!period_len)

		return;

	cb_len = min_t(u32, len, max_len);



	if (period_len) {

		/*

		 * period_len means: that we need to generate

		 * transfers that are terminating at every
@@ -307,21 +404,31 @@ static void bcm2835_dma_create_cb_set_length( 
		 */



		/* have we filled in period_length yet? */

	if (*total_len + control_block->length < period_len) {

		if (*total_len + cb_len < period_len) {

			/* update number of bytes in this period so far */

		*total_len += control_block->length;

		return;

	}



	/* calculate the length that remains to reach period_length */

	control_block->length = period_len - *total_len;

			*total_len += cb_len;

		} else {

			/* calculate the length that remains to reach period_len */

			cb_len = period_len - *total_len;



			/* reset total_length for next period */

			*total_len = 0;

		}

	}



	if (c->is_40bit_channel) {

		struct bcm2838_dma40_scb *scb =

			(struct bcm2838_dma40_scb *)control_block;



	/* add extrainfo bits in info */

		scb->len = cb_len;

		/* add extrainfo bits to ti */

		scb->ti |= to_bcm2838_ti(finalextrainfo);

	} else {

		control_block->length = cb_len;

		/* add extrainfo bits to info */

		control_block->info |= finalextrainfo;

	}

}



static inline size_t bcm2835_dma_count_frames_for_sg(

	struct bcm2835_chan *c,
@@ -343,7 +450,7 @@ static inline size_t bcm2835_dma_count_frames_for_sg( 
/**

 * bcm2835_dma_create_cb_chain - create a control block and fills data in

 *

 * @chan:           the @dma_chan for which we run this

 * @c:              the @bcm2835_chan for which we run this

 * @direction:      the direction in which we transfer

 * @cyclic:         it is a cyclic transfer

 * @info:           the default info bits to apply per controlblock
@@ -361,12 +468,11 @@ static inline size_t bcm2835_dma_count_frames_for_sg( 
 * @gfp:            the GFP flag to use for allocation

 */

static struct bcm2835_desc *bcm2835_dma_create_cb_chain(

	struct dma_chan *chan, enum dma_transfer_direction direction,

	struct bcm2835_chan *c, enum dma_transfer_direction direction,

	bool cyclic, u32 info, u32 finalextrainfo, size_t frames,

	dma_addr_t src, dma_addr_t dst, size_t buf_len,

	size_t period_len, gfp_t gfp)

{

	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);

	size_t len = buf_len, total_len;

	size_t frame;

	struct bcm2835_desc *d;
@@ -399,11 +505,23 @@ static struct bcm2835_desc *bcm2835_dma_create_cb_chain( 


		/* fill in the control block */

		control_block = cb_entry->cb;

		if (c->is_40bit_channel) {

			struct bcm2838_dma40_scb *scb =

				(struct bcm2838_dma40_scb *)control_block;

			scb->ti = to_bcm2838_ti(info);

			scb->src = lower_32_bits(src);

			scb->srci= upper_32_bits(src) | to_bcm2838_srci(info);

			scb->dst = lower_32_bits(dst);

			scb->dsti = upper_32_bits(dst) | to_bcm2838_dsti(info);

			scb->next_cb = 0;

		} else {

			control_block->info = info;

			control_block->src = src;

			control_block->dst = dst;

			control_block->stride = 0;

			control_block->next = 0;

		}



		/* set up length in control_block if requested */

		if (buf_len) {

			/* calculate length honoring period_length */
@@ -417,7 +535,10 @@ static struct bcm2835_desc *bcm2835_dma_create_cb_chain( 
		}



		/* link this the last controlblock */

		if (frame)

		if (frame && c->is_40bit_channel)

			d->cb_list[frame - 1].cb->next =

				to_bcm2838_cbaddr(cb_entry->paddr);

		if (frame && !c->is_40bit_channel)

			d->cb_list[frame - 1].cb->next = cb_entry->paddr;



		/* update src and dst and length */
@@ -431,7 +552,14 @@ static struct bcm2835_desc *bcm2835_dma_create_cb_chain( 
	}



	/* the last frame requires extra flags */

	if (c->is_40bit_channel) {

		struct bcm2838_dma40_scb *scb =

			(struct bcm2838_dma40_scb *)d->cb_list[d->frames-1].cb;



		scb->ti |= to_bcm2838_ti(finalextrainfo);

	} else {

		d->cb_list[d->frames - 1].cb->info |= finalextrainfo;

	}



	/* detect a size missmatch */

	if (buf_len && (d->size != buf_len))
@@ -445,28 +573,51 @@ error_cb: 
}



static void bcm2835_dma_fill_cb_chain_with_sg(

	struct dma_chan *chan,

	struct bcm2835_chan *c,

	enum dma_transfer_direction direction,

	struct bcm2835_cb_entry *cb,

	struct scatterlist *sgl,

	unsigned int sg_len)

{

	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);

	size_t len, max_len;

	unsigned int i;

	dma_addr_t addr;

	struct scatterlist *sgent;



	pr_err("dma_fill_chain_with_sg(ch %d, dir %d):\n", c->ch, direction);



	max_len = bcm2835_dma_max_frame_length(c);

	for_each_sg(sgl, sgent, sg_len, i) {

		for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent);

		if (c->is_40bit_channel) {

			struct bcm2838_dma40_scb *scb =

				(struct bcm2838_dma40_scb *)cb->cb;

			for (addr = sg_dma_address(sgent),

				     len = sg_dma_len(sgent);

			     len > 0;

			     addr += scb->len, len -= scb->len, scb++) {

				if (direction == DMA_DEV_TO_MEM) {

					scb->dst = lower_32_bits(addr);

					scb->dsti = upper_32_bits(addr) | BCM2838_DMA40_INC;

				} else {

					scb->src = lower_32_bits(addr);

					scb->srci = upper_32_bits(addr) | BCM2838_DMA40_INC;

				}

				scb->len = min(len, max_len);

				pr_err("  %llx, %x\n", (u64)addr, scb->len);

			}

		} else {

			for (addr = sg_dma_address(sgent),

				     len = sg_dma_len(sgent);

			     len > 0;

		     addr += cb->cb->length, len -= cb->cb->length, cb++) {

			     addr += cb->cb->length, len -= cb->cb->length,

			     cb++) {

				if (direction == DMA_DEV_TO_MEM)

					cb->cb->dst = addr;

				else

					cb->cb->src = addr;

				cb->cb->length = min(len, max_len);

				pr_err("  %llx, %x\n", (u64)addr, cb->cb->length);

			}

		}

	}

}
@@ -475,6 +626,10 @@ static int bcm2835_dma_abort(struct bcm2835_chan *c) 
{

	void __iomem *chan_base = c->chan_base;

	long int timeout = 10000;

	u32 wait_mask = BCM2835_DMA_WAITING_FOR_WRITES;



	if (c->is_40bit_channel)

		wait_mask = BCM2838_DMA40_WAITING_FOR_WRITES;



	/*

	 * A zero control block address means the channel is idle.
@@ -487,8 +642,7 @@ static int bcm2835_dma_abort(struct bcm2835_chan *c) 
	writel(0, chan_base + BCM2835_DMA_CS);



	/* Wait for any current AXI transfer to complete */

	while ((readl(chan_base + BCM2835_DMA_CS) &

		BCM2835_DMA_WAITING_FOR_WRITES) && --timeout)

	while ((readl(chan_base + BCM2835_DMA_CS) & wait_mask) && --timeout)

		cpu_relax();



	/* Peripheral might be stuck and fail to signal AXI write responses */
@@ -505,6 +659,7 @@ static void bcm2835_dma_start_desc(struct bcm2835_chan *c) 
	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);

	struct bcm2835_desc *d;



	pr_err("dma_start_desc(%px)\n", vd);

	if (!vd) {

		c->desc = NULL;

		return;
@@ -514,10 +669,17 @@ static void bcm2835_dma_start_desc(struct bcm2835_chan *c) 


	c->desc = d = to_bcm2835_dma_desc(&vd->tx);



	if (c->is_40bit_channel) {

		writel(to_bcm2838_cbaddr(d->cb_list[0].paddr),

		       c->chan_base + BCM2838_DMA40_CB);

		writel(BCM2838_DMA40_ACTIVE | BCM2838_DMA40_CS_FLAGS(c->dreq),

		       c->chan_base + BCM2838_DMA40_CS);

	} else {

		writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR);

		writel(BCM2835_DMA_ACTIVE | BCM2835_DMA_CS_FLAGS(c->dreq),

		       c->chan_base + BCM2835_DMA_CS);

	}

}



static irqreturn_t bcm2835_dma_callback(int irq, void *data)

{
@@ -544,7 +706,8 @@ static irqreturn_t bcm2835_dma_callback(int irq, void *data) 
	 * will remain idle despite the ACTIVE flag being set.

	 */

	writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE |

	       BCM2835_DMA_CS_FLAGS(c->dreq),

	       (c->is_40bit_channel ? BCM2838_DMA40_CS_FLAGS(c->dreq) :

		BCM2835_DMA_CS_FLAGS(c->dreq)),

	       c->chan_base + BCM2835_DMA_CS);



	d = c->desc;
@@ -643,9 +806,17 @@ static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan, 
		struct bcm2835_desc *d = c->desc;

		dma_addr_t pos;



		if (d->dir == DMA_MEM_TO_DEV)

		if (d->dir == DMA_MEM_TO_DEV && c->is_40bit_channel)

			pos = readl(c->chan_base + BCM2838_DMA40_SRC) +

				((readl(c->chan_base + BCM2838_DMA40_SRCI) &

				  0xff) << 8);

		else if (d->dir == DMA_MEM_TO_DEV && !c->is_40bit_channel)

			pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD);

		else if (d->dir == DMA_DEV_TO_MEM)

		else if (d->dir == DMA_DEV_TO_MEM && c->is_40bit_channel)

			pos = readl(c->chan_base + BCM2838_DMA40_DEST) +

				((readl(c->chan_base + BCM2838_DMA40_DESTI) &

				  0xff) << 8);

		else if (d->dir == DMA_DEV_TO_MEM && !c->is_40bit_channel)

			pos = readl(c->chan_base + BCM2835_DMA_DEST_AD);

		else

			pos = 0;
@@ -691,7 +862,7 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy( 
	frames = bcm2835_dma_frames_for_length(len, max_len);



	/* allocate the CB chain - this also fills in the pointers */

	d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false,

	d = bcm2835_dma_create_cb_chain(c, DMA_MEM_TO_MEM, false,

					info, extra, frames,

					src, dst, len, 0, GFP_KERNEL);

	if (!d)
@@ -726,11 +897,21 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg( 
		if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)

			return NULL;

		src = c->cfg.src_addr;

		/*

		 * One would think it ought to be possible to get the physical

		 * to dma address mapping information from the dma-ranges DT

		 * property, but I've not found a way yet that doesn't involve

		 * open-coding the whole thing.

		 */

		if (c->is_40bit_channel)

		    src |= 0x400000000ull;

		info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC;

	} else {

		if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)

			return NULL;

		dst = c->cfg.dst_addr;

		if (c->is_40bit_channel)

		    dst |= 0x400000000ull;

		info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC;

	}
@@ -738,7 +919,7 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg( 
	frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len);



	/* allocate the CB chain */

	d = bcm2835_dma_create_cb_chain(chan, direction, false,

	d = bcm2835_dma_create_cb_chain(c, direction, false,

					info, extra,

					frames, src, dst, 0, 0,

					GFP_KERNEL);
@@ -746,7 +927,7 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg( 
		return NULL;



	/* fill in frames with scatterlist pointers */

	bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list,

	bcm2835_dma_fill_cb_chain_with_sg(c, direction, d->cb_list,

					  sgl, sg_len);



	return vchan_tx_prep(&c->vc, &d->vd, flags);
@@ -815,7 +996,7 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic( 
	 * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine

	 * implementation calls prep_dma_cyclic with interrupts disabled.

	 */

	d = bcm2835_dma_create_cb_chain(chan, direction, true,

	d = bcm2835_dma_create_cb_chain(c, direction, true,

					info, extra,

					frames, src, dst, buf_len,

					period_len, GFP_NOWAIT);
@@ -823,7 +1004,8 @@ static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic( 
		return NULL;



	/* wrap around into a loop */

	d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr;

	d->cb_list[d->frames - 1].cb->next = c->is_40bit_channel ?

		to_bcm2838_cbaddr(d->cb_list[0].paddr) : d->cb_list[0].paddr;



	return vchan_tx_prep(&c->vc, &d->vd, flags);

}
@@ -899,8 +1081,10 @@ static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, 
	c->irq_number = irq;

	c->irq_flags = irq_flags;



	/* check in DEBUG register if this is a LITE channel */

	if (readl(c->chan_base + BCM2835_DMA_DEBUG) &

	/* check for 40bit and lite channels */

	if (d->cfg_data->chan_40bit_mask & BIT(chan_id))

		c->is_40bit_channel = true;

	else if (readl(c->chan_base + BCM2835_DMA_DEBUG) &

		 BCM2835_DMA_DEBUG_LITE)

		c->is_lite_channel = true;
@@ -918,18 +1102,16 @@ static void bcm2835_dma_free(struct bcm2835_dmadev *od) 
	}

}



int bcm2838_dma40_memcpy_init(struct device *dev)

int bcm2838_dma40_memcpy_init(void)

{

	if (memcpy_scb)

		return 0;

	if (!memcpy_parent)

		return -EPROBE_DEFER;



	memcpy_scb = dma_alloc_coherent(dev, sizeof(*memcpy_scb),

					&memcpy_scb_dma, GFP_KERNEL);

	if (!memcpy_chan)

		return -EINVAL;



	if (!memcpy_scb) {

		pr_err("bcm2838_dma40_memcpy_init failed!\n");

	if (!memcpy_scb)

		return -ENOMEM;

	}



	return 0;

}
@@ -956,20 +1138,22 @@ void bcm2838_dma40_memcpy(dma_addr_t dst, dma_addr_t src, size_t size) 
	scb->next_cb = 0;



	writel((u32)(memcpy_scb_dma >> 5), memcpy_chan + BCM2838_DMA40_CB);

	writel(BCM2838_DMA40_MEMCPY_QOS + BCM2838_DMA40_CS_ACTIVE,

	writel(BCM2838_DMA40_MEMCPY_FLAGS + BCM2838_DMA40_ACTIVE,

	       memcpy_chan + BCM2838_DMA40_CS);



	/* Poll for completion */

	while (!(readl(memcpy_chan + BCM2838_DMA40_CS) & BCM2838_DMA40_CS_END))

	while (!(readl(memcpy_chan + BCM2838_DMA40_CS) & BCM2838_DMA40_END))

		cpu_relax();



	writel(BCM2838_DMA40_CS_END, memcpy_chan + BCM2838_DMA40_CS);

	writel(BCM2838_DMA40_END, memcpy_chan + BCM2838_DMA40_CS);



	spin_unlock_irqrestore(&memcpy_lock, flags);

}

EXPORT_SYMBOL(bcm2838_dma40_memcpy);



static const struct of_device_id bcm2835_dma_of_match[] = {

	{ .compatible = "brcm,bcm2835-dma", },

	{ .compatible = "brcm,bcm2835-dma", .data = &bcm2835_dma_cfg },

	{ .compatible = "brcm,bcm2838-dma", .data = &bcm2838_dma_cfg },

	{},

};

MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match);
@@ -1001,6 +1185,8 @@ static int bcm2835_dma_probe(struct platform_device *pdev) 
	int irq_flags;

	uint32_t chans_available;

	char chan_name[BCM2835_DMA_CHAN_NAME_SIZE];

	const struct of_device_id *of_id;

	int chan_count, chan_start, chan_end;



	if (!pdev->dev.dma_mask)

		pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
@@ -1020,9 +1206,13 @@ static int bcm2835_dma_probe(struct platform_device *pdev) 
	base = devm_ioremap_resource(&pdev->dev, res);

	if (IS_ERR(base))

		return PTR_ERR(base);

	rc = bcm_dmaman_probe(pdev, base, BCM2835_DMA_BULK_MASK);

	if (rc)

		dev_err(&pdev->dev, "Failed to initialize the legacy API\n");



	/* The set of channels can be split across multiple instances. */

	chan_start = ((u32)base / BCM2835_DMA_CHAN_SIZE) & 0xf;

	base -= BCM2835_DMA_CHAN(chan_start);

	chan_count = resource_size(res) / BCM2835_DMA_CHAN_SIZE;

	chan_end = min(chan_start + chan_count,

			 BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1);



	od->base = base;
@@ -1052,6 +1242,14 @@ static int bcm2835_dma_probe(struct platform_device *pdev) 


	platform_set_drvdata(pdev, od);



	of_id = of_match_node(bcm2835_dma_of_match, pdev->dev.of_node);

	if (!of_id) {

		dev_err(&pdev->dev, "Failed to match compatible string\n");

		return -EINVAL;

	}



	od->cfg_data = of_id->data;



	/* Request DMA channel mask from device tree */

	if (of_property_read_u32(pdev->dev.of_node,

			"brcm,dma-channel-mask",
@@ -1061,18 +1259,34 @@ static int bcm2835_dma_probe(struct platform_device *pdev) 
		goto err_no_dma;

	}



	/* Channel 0 is used by the legacy API */

	/* One channel is reserved for the legacy API */

	if (chans_available & BCM2835_DMA_BULK_MASK) {

		rc = bcm_dmaman_probe(pdev, base,

				      chans_available & BCM2835_DMA_BULK_MASK);

		if (rc)

			dev_err(&pdev->dev,

				"Failed to initialize the legacy API\n");



		chans_available &= ~BCM2835_DMA_BULK_MASK;

	}



	/* We can't use channels 11-13 yet */

	chans_available &= ~(BIT(11) | BIT(12) | BIT(13));

	/* And possibly one for the 40-bit DMA memcpy API */

	if (chans_available & od->cfg_data->chan_40bit_mask &

	    BIT(BCM2838_DMA_MEMCPY_CHAN)) {

		memcpy_parent = od;

		memcpy_chan = BCM2835_DMA_CHANIO(base, BCM2838_DMA_MEMCPY_CHAN);

		memcpy_scb = dma_alloc_coherent(memcpy_parent->ddev.dev,

						sizeof(*memcpy_scb),

						&memcpy_scb_dma, GFP_KERNEL);

		if (!memcpy_scb)

			dev_warn(&pdev->dev,

				 "Failed to allocated memcpy scb\n");



	/* Grab channel 14 for the 40-bit DMA memcpy */

	chans_available &= ~BIT(14);

	memcpy_chan = BCM2835_DMA_CHANIO(base, 14);

		chans_available &= ~BIT(BCM2838_DMA_MEMCPY_CHAN);

	}



	/* get irqs for each channel that we support */

	for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {

	for (i = chan_start; i < chan_end; i++) {

		/* skip masked out channels */

		if (!(chans_available & (1 << i))) {

			irq[i] = -1;
@@ -1095,13 +1309,17 @@ static int bcm2835_dma_probe(struct platform_device *pdev) 
		irq[i] = platform_get_irq(pdev, i < 11 ? i : 11);

	}



	chan_count = 0;



	/* get irqs for each channel */

	for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) {

	for (i = chan_start; i < chan_end; i++) {

		/* skip channels without irq */

		if (irq[i] < 0)

			continue;



		/* check if there are other channels that also use this irq */

		/* FIXME: This will fail if interrupts are shared across

		   instances */

		irq_flags = 0;

		for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++)

			if ((i != j) && (irq[j] == irq[i])) {
@@ -1113,9 +1331,10 @@ static int bcm2835_dma_probe(struct platform_device *pdev) 
		rc = bcm2835_dma_chan_init(od, i, irq[i], irq_flags);

		if (rc)

			goto err_no_dma;

		chan_count++;

	}



	dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i);

	dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", chan_count);



	/* Device-tree DMA controller registration */

	rc = of_dma_controller_register(pdev->dev.of_node,
@@ -1147,6 +1366,13 @@ static int bcm2835_dma_remove(struct platform_device *pdev) 


	bcm_dmaman_remove(pdev);

	dma_async_device_unregister(&od->ddev);

	if (memcpy_parent == od) {

		dma_free_coherent(&pdev->dev, sizeof(*memcpy_scb), memcpy_scb,

				  memcpy_scb_dma);

		memcpy_parent = NULL;

		memcpy_scb = NULL;

		memcpy_chan = NULL;

	}

	bcm2835_dma_free(od);



	return 0;

drivers/pci/controller/pcie-brcmstb-bounce.c

+17 −13

File changed.

Preview size limit exceeded, changes collapsed.

drivers/pci/controller/pcie-brcmstb-bounce.h

+13 −8
Original line number Diff line number Diff line
@@ -8,21 +8,26 @@ 


#ifdef CONFIG_ARM



int brcm_pcie_bounce_register_dev(struct device *dev, unsigned long buffer_size,

int brcm_pcie_bounce_init(struct device *dev, unsigned long buffer_size,

			  dma_addr_t threshold);



int brcm_pcie_bounce_unregister_dev(struct device *dev);

int brcm_pcie_bounce_uninit(struct device *dev);

int brcm_pcie_bounce_register_dev(struct device *dev);



#else



static inline int brcm_pcie_bounce_register_dev(struct device *dev,

static inline int brcm_pcie_bounce_init(struct device *dev,

					unsigned long buffer_size,

					dma_addr_t threshold)

{

	return 0;

}



static inline int brcm_pcie_bounce_unregister_dev(struct device *dev)

static inline int brcm_pcie_bounce_uninit(struct device *dev)

{

	return 0;

}



static inline int brcm_pcie_bounce_register_dev(struct device *dev)

{

	return 0;

}

drivers/pci/controller/pcie-brcmstb.c

+18 −5

File changed.

Preview size limit exceeded, changes collapsed.