iommu/amd: Convert AMD iommu driver to the dma-iommu api

	select PCI_PASID

	select IOMMU_API

	select IOMMU_IOVA

	select IOMMU_DMA

	depends on X86_64 && PCI && ACPI

	---help---

	  With this option you can enable support for AMD IOMMU hardware in

#include <linux/scatterlist.h>

#include <linux/dma-mapping.h>

#include <linux/dma-direct.h>

#include <linux/dma-iommu.h>

#include <linux/iommu-helper.h>

#include <linux/iommu.h>

#include <linux/delay.h>

static ATOMIC_NOTIFIER_HEAD(ppr_notifier);

int amd_iommu_max_glx_val = -1;

static const struct dma_map_ops amd_iommu_dma_ops;

/*

 * general struct to manage commands send to an IOMMU

 */

static void update_domain(struct protection_domain *domain);

static int protection_domain_init(struct protection_domain *domain);

static void detach_device(struct device *dev);

static void iova_domain_flush_tlb(struct iova_domain *iovad);

/*

 * Data container for a dma_ops specific protection domain

 */

struct dma_ops_domain {

	/* generic protection domain information */

	struct protection_domain domain;

	/* IOVA RB-Tree */

	struct iova_domain iovad;

};

static struct iova_domain reserved_iova_ranges;

static struct lock_class_key reserved_rbtree_key;

/****************************************************************************

 *

	return container_of(dom, struct protection_domain, domain);

}

static struct dma_ops_domain* to_dma_ops_domain(struct protection_domain *domain)

{

	BUG_ON(domain->flags != PD_DMA_OPS_MASK);

	return container_of(domain, struct dma_ops_domain, domain);

}

static struct iommu_dev_data *alloc_dev_data(u16 devid)

{

	struct iommu_dev_data *dev_data;

	__domain_flush_pages(domain, address, size, 0);

}

/* Flush the whole IO/TLB for a given protection domain */

static void domain_flush_tlb(struct protection_domain *domain)

{

	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);

}

/* Flush the whole IO/TLB for a given protection domain - including PDE */

static void domain_flush_tlb_pde(struct protection_domain *domain)

{

	return unmapped;

}

/****************************************************************************

 *

 * The next functions belong to the address allocator for the dma_ops

 * interface functions.

 *

 ****************************************************************************/

static unsigned long dma_ops_alloc_iova(struct device *dev,

					struct dma_ops_domain *dma_dom,

					unsigned int pages, u64 dma_mask)

{

	unsigned long pfn = 0;

	pages = __roundup_pow_of_two(pages);

	if (dma_mask > DMA_BIT_MASK(32))

		pfn = alloc_iova_fast(&dma_dom->iovad, pages,

				      IOVA_PFN(DMA_BIT_MASK(32)), false);

	if (!pfn)

		pfn = alloc_iova_fast(&dma_dom->iovad, pages,

				      IOVA_PFN(dma_mask), true);

	return (pfn << PAGE_SHIFT);

}

static void dma_ops_free_iova(struct dma_ops_domain *dma_dom,

			      unsigned long address,

			      unsigned int pages)

{

	pages = __roundup_pow_of_two(pages);

	address >>= PAGE_SHIFT;

	free_iova_fast(&dma_dom->iovad, address, pages);

}

/****************************************************************************

 *

 * The next functions belong to the domain allocation. A domain is

	free_page((unsigned long)domain->gcr3_tbl);

}

static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)

{

	unsigned long flags;

	spin_lock_irqsave(&dom->domain.lock, flags);

	domain_flush_tlb(&dom->domain);

	domain_flush_complete(&dom->domain);

	spin_unlock_irqrestore(&dom->domain.lock, flags);

}

static void iova_domain_flush_tlb(struct iova_domain *iovad)

{

	struct dma_ops_domain *dom;

	dom = container_of(iovad, struct dma_ops_domain, iovad);

	dma_ops_domain_flush_tlb(dom);

}

/*

 * Free a domain, only used if something went wrong in the

 * allocation path and we need to free an already allocated page table

 */

static void dma_ops_domain_free(struct dma_ops_domain *dom)

static void dma_ops_domain_free(struct protection_domain *domain)

{

	if (!dom)

	if (!domain)

		return;

	put_iova_domain(&dom->iovad);

	iommu_put_dma_cookie(&domain->domain);

	free_pagetable(&dom->domain);

	free_pagetable(domain);

	if (dom->domain.id)

		domain_id_free(dom->domain.id);

	if (domain->id)

		domain_id_free(domain->id);

	kfree(dom);

	kfree(domain);

}

/*

 * It also initializes the page table and the address allocator data

 * structures required for the dma_ops interface

 */

static struct dma_ops_domain *dma_ops_domain_alloc(void)

static struct protection_domain *dma_ops_domain_alloc(void)

{

	struct dma_ops_domain *dma_dom;

	struct protection_domain *domain;

	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);

	if (!dma_dom)

	domain = kzalloc(sizeof(struct protection_domain), GFP_KERNEL);

	if (!domain)

		return NULL;

	if (protection_domain_init(&dma_dom->domain))

		goto free_dma_dom;

	dma_dom->domain.mode = PAGE_MODE_3_LEVEL;

	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);

	dma_dom->domain.flags = PD_DMA_OPS_MASK;

	if (!dma_dom->domain.pt_root)

		goto free_dma_dom;

	init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_START_PFN);

	if (protection_domain_init(domain))

		goto free_domain;

	if (init_iova_flush_queue(&dma_dom->iovad, iova_domain_flush_tlb, NULL))

		goto free_dma_dom;

	domain->mode = PAGE_MODE_3_LEVEL;

	domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);

	domain->flags = PD_DMA_OPS_MASK;

	if (!domain->pt_root)

		goto free_domain;

	/* Initialize reserved ranges */

	copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);

	if (iommu_get_dma_cookie(&domain->domain) == -ENOMEM)

		goto free_domain;

	return dma_dom;

	return domain;

free_dma_dom:

	dma_ops_domain_free(dma_dom);

free_domain:

	dma_ops_domain_free(domain);

	return NULL;

}

	domain = iommu_get_domain_for_dev(dev);

	if (domain->type == IOMMU_DOMAIN_IDENTITY)

		dev_data->passthrough = true;

	else

		dev->dma_ops = &amd_iommu_dma_ops;

	else if (domain->type == IOMMU_DOMAIN_DMA)

		iommu_setup_dma_ops(dev, IOVA_START_PFN << PAGE_SHIFT, 0);

out:

	iommu_completion_wait(iommu);

	return acpihid_device_group(dev);

}

static int amd_iommu_domain_get_attr(struct iommu_domain *domain,

		enum iommu_attr attr, void *data)

{

	switch (domain->type) {

	case IOMMU_DOMAIN_UNMANAGED:

		return -ENODEV;

	case IOMMU_DOMAIN_DMA:

		switch (attr) {

		case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:

			*(int *)data = !amd_iommu_unmap_flush;

			return 0;

		default:

			return -ENODEV;

		}

		break;

	default:

		return -EINVAL;

	}

}

/*****************************************************************************

 *

 * The next functions belong to the dma_ops mapping/unmapping code.

 *

 *****************************************************************************/

/*

 * In the dma_ops path we only have the struct device. This function

 * finds the corresponding IOMMU, the protection domain and the

 * requestor id for a given device.

 * If the device is not yet associated with a domain this is also done

 * in this function.

 */

static struct protection_domain *get_domain(struct device *dev)

{

	struct protection_domain *domain;

	struct iommu_domain *io_domain;

	if (!check_device(dev))

		return ERR_PTR(-EINVAL);

	domain = get_dev_data(dev)->domain;

	if (domain == NULL && get_dev_data(dev)->defer_attach) {

		get_dev_data(dev)->defer_attach = false;

		io_domain = iommu_get_domain_for_dev(dev);

		domain = to_pdomain(io_domain);

		attach_device(dev, domain);

	}

	if (domain == NULL)

		return ERR_PTR(-EBUSY);

	if (!dma_ops_domain(domain))

		return ERR_PTR(-EBUSY);

	return domain;

}

static void update_device_table(struct protection_domain *domain)

{

	struct iommu_dev_data *dev_data;

	domain_flush_tlb_pde(domain);

}

static int dir2prot(enum dma_data_direction direction)

{

	if (direction == DMA_TO_DEVICE)

		return IOMMU_PROT_IR;

	else if (direction == DMA_FROM_DEVICE)

		return IOMMU_PROT_IW;

	else if (direction == DMA_BIDIRECTIONAL)

		return IOMMU_PROT_IW | IOMMU_PROT_IR;

	else

		return 0;

}

/*

 * This function contains common code for mapping of a physically

 * contiguous memory region into DMA address space. It is used by all

 * mapping functions provided with this IOMMU driver.

 * Must be called with the domain lock held.

 */

static dma_addr_t __map_single(struct device *dev,

			       struct dma_ops_domain *dma_dom,

			       phys_addr_t paddr,

			       size_t size,

			       enum dma_data_direction direction,

			       u64 dma_mask)

{

	dma_addr_t offset = paddr & ~PAGE_MASK;

	dma_addr_t address, start, ret;

	unsigned long flags;

	unsigned int pages;

	int prot = 0;

	int i;

	pages = iommu_num_pages(paddr, size, PAGE_SIZE);

	paddr &= PAGE_MASK;

	address = dma_ops_alloc_iova(dev, dma_dom, pages, dma_mask);

	if (!address)

		goto out;

	prot = dir2prot(direction);

	start = address;

	for (i = 0; i < pages; ++i) {

		ret = iommu_map_page(&dma_dom->domain, start, paddr,

				     PAGE_SIZE, prot, GFP_ATOMIC);

		if (ret)

			goto out_unmap;

		paddr += PAGE_SIZE;

		start += PAGE_SIZE;

	}

	address += offset;

	domain_flush_np_cache(&dma_dom->domain, address, size);

out:

	return address;

out_unmap:

	for (--i; i >= 0; --i) {

		start -= PAGE_SIZE;

		iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE);

	}

	spin_lock_irqsave(&dma_dom->domain.lock, flags);

	domain_flush_tlb(&dma_dom->domain);

	domain_flush_complete(&dma_dom->domain);

	spin_unlock_irqrestore(&dma_dom->domain.lock, flags);

	dma_ops_free_iova(dma_dom, address, pages);

	return DMA_MAPPING_ERROR;

}

/*

 * Does the reverse of the __map_single function. Must be called with

 * the domain lock held too

 */

static void __unmap_single(struct dma_ops_domain *dma_dom,

			   dma_addr_t dma_addr,

			   size_t size,

			   int dir)

{

	dma_addr_t i, start;

	unsigned int pages;

	pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);

	dma_addr &= PAGE_MASK;

	start = dma_addr;

	for (i = 0; i < pages; ++i) {

		iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE);

		start += PAGE_SIZE;

	}

	if (amd_iommu_unmap_flush) {

		unsigned long flags;

		spin_lock_irqsave(&dma_dom->domain.lock, flags);

		domain_flush_tlb(&dma_dom->domain);

		domain_flush_complete(&dma_dom->domain);

		spin_unlock_irqrestore(&dma_dom->domain.lock, flags);

		dma_ops_free_iova(dma_dom, dma_addr, pages);

	} else {

		pages = __roundup_pow_of_two(pages);

		queue_iova(&dma_dom->iovad, dma_addr >> PAGE_SHIFT, pages, 0);

	}

}

/*

 * The exported map_single function for dma_ops.

 */

static dma_addr_t map_page(struct device *dev, struct page *page,

			   unsigned long offset, size_t size,

			   enum dma_data_direction dir,

			   unsigned long attrs)

{

	phys_addr_t paddr = page_to_phys(page) + offset;

	struct protection_domain *domain;

	struct dma_ops_domain *dma_dom;

	u64 dma_mask;

	domain = get_domain(dev);

	if (PTR_ERR(domain) == -EINVAL)

		return (dma_addr_t)paddr;

	else if (IS_ERR(domain))

		return DMA_MAPPING_ERROR;

	dma_mask = *dev->dma_mask;

	dma_dom = to_dma_ops_domain(domain);

	return __map_single(dev, dma_dom, paddr, size, dir, dma_mask);

}

/*

 * The exported unmap_single function for dma_ops.

 */

static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,

		       enum dma_data_direction dir, unsigned long attrs)

{

	struct protection_domain *domain;

	struct dma_ops_domain *dma_dom;

	domain = get_domain(dev);

	if (IS_ERR(domain))

		return;

	dma_dom = to_dma_ops_domain(domain);

	__unmap_single(dma_dom, dma_addr, size, dir);

}

static int sg_num_pages(struct device *dev,

			struct scatterlist *sglist,

			int nelems)

{

	unsigned long mask, boundary_size;

	struct scatterlist *s;

	int i, npages = 0;

	mask          = dma_get_seg_boundary(dev);

	boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :

				   1UL << (BITS_PER_LONG - PAGE_SHIFT);

	for_each_sg(sglist, s, nelems, i) {

		int p, n;

		s->dma_address = npages << PAGE_SHIFT;

		p = npages % boundary_size;

		n = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);

		if (p + n > boundary_size)

			npages += boundary_size - p;

		npages += n;

	}

	return npages;

}

/*

 * The exported map_sg function for dma_ops (handles scatter-gather

 * lists).

 */

static int map_sg(struct device *dev, struct scatterlist *sglist,

		  int nelems, enum dma_data_direction direction,

		  unsigned long attrs)

{

	int mapped_pages = 0, npages = 0, prot = 0, i;

	struct protection_domain *domain;

	struct dma_ops_domain *dma_dom;

	struct scatterlist *s;

	unsigned long address;

	u64 dma_mask;

	int ret;

	domain = get_domain(dev);

	if (IS_ERR(domain))

		return 0;

	dma_dom  = to_dma_ops_domain(domain);

	dma_mask = *dev->dma_mask;

	npages = sg_num_pages(dev, sglist, nelems);

	address = dma_ops_alloc_iova(dev, dma_dom, npages, dma_mask);

	if (!address)

		goto out_err;

	prot = dir2prot(direction);

	/* Map all sg entries */

	for_each_sg(sglist, s, nelems, i) {

		int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);

		for (j = 0; j < pages; ++j) {

			unsigned long bus_addr, phys_addr;

			bus_addr  = address + s->dma_address + (j << PAGE_SHIFT);

			phys_addr = (sg_phys(s) & PAGE_MASK) + (j << PAGE_SHIFT);

			ret = iommu_map_page(domain, bus_addr, phys_addr,

					     PAGE_SIZE, prot,

					     GFP_ATOMIC | __GFP_NOWARN);

			if (ret)

				goto out_unmap;

			mapped_pages += 1;

		}

	}

	/* Everything is mapped - write the right values into s->dma_address */

	for_each_sg(sglist, s, nelems, i) {

		/*

		 * Add in the remaining piece of the scatter-gather offset that

		 * was masked out when we were determining the physical address

		 * via (sg_phys(s) & PAGE_MASK) earlier.

		 */

		s->dma_address += address + (s->offset & ~PAGE_MASK);

		s->dma_length   = s->length;

	}

	if (s)

		domain_flush_np_cache(domain, s->dma_address, s->dma_length);

	return nelems;

out_unmap:

	dev_err(dev, "IOMMU mapping error in map_sg (io-pages: %d reason: %d)\n",

		npages, ret);

	for_each_sg(sglist, s, nelems, i) {

		int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);

		for (j = 0; j < pages; ++j) {

			unsigned long bus_addr;

			bus_addr  = address + s->dma_address + (j << PAGE_SHIFT);

			iommu_unmap_page(domain, bus_addr, PAGE_SIZE);

			if (--mapped_pages == 0)

				goto out_free_iova;

		}

	}

out_free_iova:

	free_iova_fast(&dma_dom->iovad, address >> PAGE_SHIFT, npages);

out_err:

	return 0;

}

/*

 * The exported map_sg function for dma_ops (handles scatter-gather

 * lists).

 */

static void unmap_sg(struct device *dev, struct scatterlist *sglist,

		     int nelems, enum dma_data_direction dir,

		     unsigned long attrs)

{

	struct protection_domain *domain;

	struct dma_ops_domain *dma_dom;

	unsigned long startaddr;

	int npages;

	domain = get_domain(dev);

	if (IS_ERR(domain))

		return;

	startaddr = sg_dma_address(sglist) & PAGE_MASK;

	dma_dom   = to_dma_ops_domain(domain);

	npages    = sg_num_pages(dev, sglist, nelems);

	__unmap_single(dma_dom, startaddr, npages << PAGE_SHIFT, dir);

}

/*

 * The exported alloc_coherent function for dma_ops.

 */

static void *alloc_coherent(struct device *dev, size_t size,

			    dma_addr_t *dma_addr, gfp_t flag,

			    unsigned long attrs)

{

	u64 dma_mask = dev->coherent_dma_mask;

	struct protection_domain *domain;

	struct dma_ops_domain *dma_dom;

	struct page *page;

	domain = get_domain(dev);

	if (PTR_ERR(domain) == -EINVAL) {

		page = alloc_pages(flag, get_order(size));

		*dma_addr = page_to_phys(page);

		return page_address(page);

	} else if (IS_ERR(domain))

		return NULL;

	dma_dom   = to_dma_ops_domain(domain);

	size	  = PAGE_ALIGN(size);

	dma_mask  = dev->coherent_dma_mask;

	flag     &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);

	flag     |= __GFP_ZERO;

	page = alloc_pages(flag | __GFP_NOWARN,  get_order(size));

	if (!page) {

		if (!gfpflags_allow_blocking(flag))

			return NULL;

		page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,

					get_order(size), flag & __GFP_NOWARN);

		if (!page)

			return NULL;

	}

	if (!dma_mask)

		dma_mask = *dev->dma_mask;

	*dma_addr = __map_single(dev, dma_dom, page_to_phys(page),

				 size, DMA_BIDIRECTIONAL, dma_mask);

	if (*dma_addr == DMA_MAPPING_ERROR)

		goto out_free;

	return page_address(page);

out_free:

	if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))

		__free_pages(page, get_order(size));

	return NULL;

}

/*

 * The exported free_coherent function for dma_ops.

 */

static void free_coherent(struct device *dev, size_t size,

			  void *virt_addr, dma_addr_t dma_addr,

			  unsigned long attrs)

{

	struct protection_domain *domain;

	struct dma_ops_domain *dma_dom;

	struct page *page;

	page = virt_to_page(virt_addr);

	size = PAGE_ALIGN(size);

	domain = get_domain(dev);

	if (IS_ERR(domain))

		goto free_mem;

	dma_dom = to_dma_ops_domain(domain);

	__unmap_single(dma_dom, dma_addr, size, DMA_BIDIRECTIONAL);

free_mem:

	if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))

		__free_pages(page, get_order(size));

}

/*

 * This function is called by the DMA layer to find out if we can handle a

 * particular device. It is part of the dma_ops.

 */

static int amd_iommu_dma_supported(struct device *dev, u64 mask)

{

	if (!dma_direct_supported(dev, mask))