dmaengine: dw-edma: Add support for native HDMA (e74c3957) · Commits · EulixOS / Software / Kernel

drivers/dma/dw-edma/Makefile

+3 −2

Original line number	Diff line number	Diff line
		@@ -3,5 +3,6 @@
		obj-$(CONFIG_DW_EDMA) += dw-edma.o
		dw-edma-$(CONFIG_DEBUG_FS) := dw-edma-v0-debugfs.o
		dw-edma-objs := dw-edma-core.o \
		dw-edma-v0-core.o $(dw-edma-y)
		dw-edma-v0-core.o \
		dw-hdma-v0-core.o $(dw-edma-y)
		obj-$(CONFIG_DW_EDMA_PCIE) += dw-edma-pcie.o

drivers/dma/dw-edma/dw-edma-core.c

+5 −1

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@

		#include "dw-edma-core.h"
		#include "dw-edma-v0-core.h"
		#include "dw-hdma-v0-core.h"
		#include "../dmaengine.h"
		#include "../virt-dma.h"

		@@ -922,6 +923,9 @@ int dw_edma_probe(struct dw_edma_chip *chip)

		dw->chip = chip;

		if (dw->chip->mf == EDMA_MF_HDMA_NATIVE)
		dw_hdma_v0_core_register(dw);
		else
		dw_edma_v0_core_register(dw);

		raw_spin_lock_init(&dw->lock);

drivers/dma/dw-edma/dw-hdma-v0-core.c

0 → 100644

+294 −0

Original line number	Diff line number	Diff line
		// SPDX-License-Identifier: GPL-2.0
		/*
		* Copyright (c) 2023 Cai Huoqing
		* Synopsys DesignWare HDMA v0 core
		*/

		#include <linux/bitfield.h>
		#include <linux/irqreturn.h>
		#include <linux/io-64-nonatomic-lo-hi.h>

		#include "dw-edma-core.h"
		#include "dw-hdma-v0-core.h"
		#include "dw-hdma-v0-regs.h"

		enum dw_hdma_control {
		DW_HDMA_V0_CB = BIT(0),
		DW_HDMA_V0_TCB = BIT(1),
		DW_HDMA_V0_LLP = BIT(2),
		DW_HDMA_V0_LIE = BIT(3),
		DW_HDMA_V0_RIE = BIT(4),
		DW_HDMA_V0_CCS = BIT(8),
		DW_HDMA_V0_LLE = BIT(9),
		};

		static inline struct dw_hdma_v0_regs __iomem __dw_regs(struct dw_edma dw)
		{
		return dw->chip->reg_base;
		}

		static inline struct dw_hdma_v0_ch_regs __iomem *
		__dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
		{
		if (dir == EDMA_DIR_WRITE)
		return &(__dw_regs(dw)->ch[ch].wr);
		else
		return &(__dw_regs(dw)->ch[ch].rd);
		}

		#define SET_CH_32(dw, dir, ch, name, value) \
		writel(value, &(__dw_ch_regs(dw, dir, ch)->name))

		#define GET_CH_32(dw, dir, ch, name) \
		readl(&(__dw_ch_regs(dw, dir, ch)->name))

		#define SET_BOTH_CH_32(dw, ch, name, value) \
		do { \
		writel(value, &(__dw_ch_regs(dw, EDMA_DIR_WRITE, ch)->name)); \
		writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name)); \
		} while (0)

		/* HDMA management callbacks */
		static void dw_hdma_v0_core_off(struct dw_edma *dw)
		{
		int id;

		for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) {
		SET_BOTH_CH_32(dw, id, int_setup,
		HDMA_V0_STOP_INT_MASK \| HDMA_V0_ABORT_INT_MASK);
		SET_BOTH_CH_32(dw, id, int_clear,
		HDMA_V0_STOP_INT_MASK \| HDMA_V0_ABORT_INT_MASK);
		SET_BOTH_CH_32(dw, id, ch_en, 0);
		}
		}

		static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
		{
		u32 num_ch = 0;
		int id;

		for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) {
		if (GET_CH_32(dw, id, dir, ch_en) & BIT(0))
		num_ch++;
		}

		if (num_ch > HDMA_V0_MAX_NR_CH)
		num_ch = HDMA_V0_MAX_NR_CH;

		return (u16)num_ch;
		}

		static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan)
		{
		struct dw_edma *dw = chan->dw;
		u32 tmp;

		tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK,
		GET_CH_32(dw, chan->id, chan->dir, ch_stat));

		if (tmp == 1)
		return DMA_IN_PROGRESS;
		else if (tmp == 3)
		return DMA_COMPLETE;
		else
		return DMA_ERROR;
		}

		static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
		{
		struct dw_edma *dw = chan->dw;

		SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);
		}

		static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
		{
		struct dw_edma *dw = chan->dw;

		SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_ABORT_INT_MASK);
		}

		static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
		{
		struct dw_edma *dw = chan->dw;

		return GET_CH_32(dw, chan->dir, chan->id, int_stat);
		}

		static irqreturn_t
		dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
		dw_edma_handler_t done, dw_edma_handler_t abort)
		{
		struct dw_edma *dw = dw_irq->dw;
		unsigned long total, pos, val;
		irqreturn_t ret = IRQ_NONE;
		struct dw_edma_chan *chan;
		unsigned long off, mask;

		if (dir == EDMA_DIR_WRITE) {
		total = dw->wr_ch_cnt;
		off = 0;
		mask = dw_irq->wr_mask;
		} else {
		total = dw->rd_ch_cnt;
		off = dw->wr_ch_cnt;
		mask = dw_irq->rd_mask;
		}

		for_each_set_bit(pos, &mask, total) {
		chan = &dw->chan[pos + off];

		val = dw_hdma_v0_core_status_int(chan);
		if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
		dw_hdma_v0_core_clear_done_int(chan);
		done(chan);

		ret = IRQ_HANDLED;
		}

		if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
		dw_hdma_v0_core_clear_abort_int(chan);
		abort(chan);

		ret = IRQ_HANDLED;
		}
		}

		return ret;
		}

		static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
		u32 control, u32 size, u64 sar, u64 dar)
		{
		ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);

		if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
		struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;

		lli->control = control;
		lli->transfer_size = size;
		lli->sar.reg = sar;
		lli->dar.reg = dar;
		} else {
		struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;

		writel(control, &lli->control);
		writel(size, &lli->transfer_size);
		writeq(sar, &lli->sar.reg);
		writeq(dar, &lli->dar.reg);
		}
		}

		static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk,
		int i, u32 control, u64 pointer)
		{
		ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);

		if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
		struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;

		llp->control = control;
		llp->llp.reg = pointer;
		} else {
		struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;

		writel(control, &llp->control);
		writeq(pointer, &llp->llp.reg);
		}
		}

		static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
		{
		struct dw_edma_burst *child;
		struct dw_edma_chan *chan = chunk->chan;
		u32 control = 0, i = 0;
		int j;

		if (chunk->cb)
		control = DW_HDMA_V0_CB;

		j = chunk->bursts_alloc;
		list_for_each_entry(child, &chunk->burst->list, list) {
		j--;
		if (!j) {
		control \|= DW_HDMA_V0_LIE;
		if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
		control \|= DW_HDMA_V0_RIE;
		}

		dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz,
		child->sar, child->dar);
		}

		control = DW_HDMA_V0_LLP \| DW_HDMA_V0_TCB;
		if (!chunk->cb)
		control \|= DW_HDMA_V0_CB;

		dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
		}

		static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
		{
		struct dw_edma_chan *chan = chunk->chan;
		struct dw_edma *dw = chan->dw;
		u32 tmp;

		dw_hdma_v0_core_write_chunk(chunk);

		if (first) {
		/* Enable engine */
		SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
		/* Interrupt enable&unmask - done, abort */
		tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup) \|
		HDMA_V0_STOP_INT_MASK \| HDMA_V0_ABORT_INT_MASK \|
		HDMA_V0_LOCAL_STOP_INT_EN \| HDMA_V0_LOCAL_STOP_INT_EN;
		SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
		/* Channel control */
		SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);
		/* Linked list */
		/* llp is not aligned on 64bit -> keep 32bit accesses */
		SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
		lower_32_bits(chunk->ll_region.paddr));
		SET_CH_32(dw, chan->dir, chan->id, llp.msb,
		upper_32_bits(chunk->ll_region.paddr));
		}
		/* Set consumer cycle */
		SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
		HDMA_V0_CONSUMER_CYCLE_STAT \| HDMA_V0_CONSUMER_CYCLE_BIT);
		/* Doorbell */
		SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
		}

		static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
		{
		struct dw_edma *dw = chan->dw;

		/* MSI done addr - low, high */
		SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);
		SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);
		/* MSI abort addr - low, high */
		SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);
		SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
		/* config MSI data */
		SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data);
		}

		/* HDMA debugfs callbacks */
		static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw)
		{
		}

		static const struct dw_edma_core_ops dw_hdma_v0_core = {
		.off = dw_hdma_v0_core_off,
		.ch_count = dw_hdma_v0_core_ch_count,
		.ch_status = dw_hdma_v0_core_ch_status,
		.handle_int = dw_hdma_v0_core_handle_int,
		.start = dw_hdma_v0_core_start,
		.ch_config = dw_hdma_v0_core_ch_config,
		.debugfs_on = dw_hdma_v0_core_debugfs_on,
		};

		void dw_hdma_v0_core_register(struct dw_edma *dw)
		{
		dw->core = &dw_hdma_v0_core;
		}

drivers/dma/dw-edma/dw-hdma-v0-core.h

0 → 100644

+17 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: GPL-2.0 */
		/*
		* Copyright (c) 2023 Cai Huoqing
		* Synopsys DesignWare HDMA v0 core
		*
		* Author: Cai Huoqing <cai.huoqing@linux.dev>
		*/

		#ifndef _DW_HDMA_V0_CORE_H
		#define _DW_HDMA_V0_CORE_H

		#include <linux/dma/edma.h>

		/* HDMA core register */
		void dw_hdma_v0_core_register(struct dw_edma *dw);

		#endif /* _DW_HDMA_V0_CORE_H */

drivers/dma/dw-edma/dw-hdma-v0-regs.h

0 → 100644

+129 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: GPL-2.0 */
		/*
		* Copyright (c) 2023 Cai Huoqing
		* Synopsys DesignWare HDMA v0 reg
		*
		* Author: Cai Huoqing <cai.huoqing@linux.dev>
		*/

		#ifndef _DW_HDMA_V0_REGS_H
		#define _DW_HDMA_V0_REGS_H

		#include <linux/dmaengine.h>

		#define HDMA_V0_MAX_NR_CH 8
		#define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6)
		#define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5)
		#define HDMA_V0_LOCAL_STOP_INT_EN BIT(4)
		#define HDMA_V0_REMOTEL_STOP_INT_EN BIT(3)
		#define HDMA_V0_ABORT_INT_MASK BIT(2)
		#define HDMA_V0_STOP_INT_MASK BIT(0)
		#define HDMA_V0_LINKLIST_EN BIT(0)
		#define HDMA_V0_CONSUMER_CYCLE_STAT BIT(1)
		#define HDMA_V0_CONSUMER_CYCLE_BIT BIT(0)
		#define HDMA_V0_DOORBELL_START BIT(0)
		#define HDMA_V0_CH_STATUS_MASK GENMASK(1, 0)

		struct dw_hdma_v0_ch_regs {
		u32 ch_en; /* 0x0000 */
		u32 doorbell; /* 0x0004 */
		u32 prefetch; /* 0x0008 */
		u32 handshake; /* 0x000c */
		union {
		u64 reg; /* 0x0010..0x0014 */
		struct {
		u32 lsb; /* 0x0010 */
		u32 msb; /* 0x0014 */
		};
		} llp;
		u32 cycle_sync; /* 0x0018 */
		u32 transfer_size; /* 0x001c */
		union {
		u64 reg; /* 0x0020..0x0024 */
		struct {
		u32 lsb; /* 0x0020 */
		u32 msb; /* 0x0024 */
		};
		} sar;
		union {
		u64 reg; /* 0x0028..0x002c */
		struct {
		u32 lsb; /* 0x0028 */
		u32 msb; /* 0x002c */
		};
		} dar;
		u32 watermark_en; /* 0x0030 */
		u32 control1; /* 0x0034 */
		u32 func_num; /* 0x0038 */
		u32 qos; /* 0x003c */
		u32 padding_1[16]; /* 0x0040..0x007c */
		u32 ch_stat; /* 0x0080 */
		u32 int_stat; /* 0x0084 */
		u32 int_setup; /* 0x0088 */
		u32 int_clear; /* 0x008c */
		union {
		u64 reg; /* 0x0090..0x0094 */
		struct {
		u32 lsb; /* 0x0090 */
		u32 msb; /* 0x0094 */
		};
		} msi_stop;
		union {
		u64 reg; /* 0x0098..0x009c */
		struct {
		u32 lsb; /* 0x0098 */
		u32 msb; /* 0x009c */
		};
		} msi_watermark;
		union {
		u64 reg; /* 0x00a0..0x00a4 */
		struct {
		u32 lsb; /* 0x00a0 */
		u32 msb; /* 0x00a4 */
		};
		} msi_abort;
		u32 msi_msgdata; /* 0x00a8 */
		u32 padding_2[21]; /* 0x00ac..0x00fc */
		} __packed;

		struct dw_hdma_v0_ch {
		struct dw_hdma_v0_ch_regs wr; /* 0x0000 */
		struct dw_hdma_v0_ch_regs rd; /* 0x0100 */
		} __packed;

		struct dw_hdma_v0_regs {
		struct dw_hdma_v0_ch ch[HDMA_V0_MAX_NR_CH]; /* 0x0000..0x0fa8 */
		} __packed;

		struct dw_hdma_v0_lli {
		u32 control;
		u32 transfer_size;
		union {
		u64 reg;
		struct {
		u32 lsb;
		u32 msb;
		};
		} sar;
		union {
		u64 reg;
		struct {
		u32 lsb;
		u32 msb;
		};
		} dar;
		} __packed;

		struct dw_hdma_v0_llp {
		u32 control;
		u32 reserved;
		union {
		u64 reg;
		struct {
		u32 lsb;
		u32 msb;
		};
		} llp;
		} __packed;

		#endif /* _DW_HDMA_V0_REGS_H */