Commit c12f048f authored by David S. Miller's avatar David S. Miller
Browse files

sparc: Revert generic IOMMU allocator. 



I applied the wrong version of this patch series, V4 instead
of V10, due to a patchwork bundling snafu.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 04b7fe6a

arch/sparc/include/asm/iommu_64.h

+4 −3
Original line number Diff line number Diff line
@@ -16,7 +16,6 @@ 
#define IOPTE_WRITE   0x0000000000000002UL



#define IOMMU_NUM_CTXS	4096

#include <linux/iommu-common.h>



struct iommu_arena {

	unsigned long	*map;
@@ -25,10 +24,11 @@ struct iommu_arena { 
};



struct iommu {

	struct iommu_table	tbl;

	spinlock_t		lock;

	u32			dma_addr_mask;

	struct iommu_arena	arena;

	void			(*flush_all)(struct iommu *);

	iopte_t			*page_table;

	u32			page_table_map_base;

	unsigned long		iommu_control;

	unsigned long		iommu_tsbbase;

	unsigned long		iommu_flush;
@@ -40,6 +40,7 @@ struct iommu { 
	unsigned long		dummy_page_pa;

	unsigned long		ctx_lowest_free;

	DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS);

	u32			dma_addr_mask;

};



struct strbuf {

arch/sparc/kernel/iommu.c

+125 −63
Original line number Diff line number Diff line
@@ -13,15 +13,11 @@ 
#include <linux/errno.h>

#include <linux/iommu-helper.h>

#include <linux/bitmap.h>

#include <linux/hash.h>

#include <linux/iommu-common.h>



#ifdef CONFIG_PCI

#include <linux/pci.h>

#endif



static	DEFINE_PER_CPU(unsigned int, iommu_pool_hash);



#include <asm/iommu.h>



#include "iommu_common.h"
@@ -49,9 +45,8 @@ static DEFINE_PER_CPU(unsigned int, iommu_pool_hash); 
			       "i" (ASI_PHYS_BYPASS_EC_E))



/* Must be invoked under the IOMMU lock. */

static void iommu_flushall(struct iommu_table *iommu_table)

static void iommu_flushall(struct iommu *iommu)

{

	struct iommu *iommu = container_of(iommu_table, struct iommu, tbl);

	if (iommu->iommu_flushinv) {

		iommu_write(iommu->iommu_flushinv, ~(u64)0);

	} else {
@@ -92,22 +87,93 @@ static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte) 
	iopte_val(*iopte) = val;

}



static struct iommu_tbl_ops iommu_sparc_ops = {

	.reset	= iommu_flushall

};



static void setup_iommu_pool_hash(void)

/* Based almost entirely upon the ppc64 iommu allocator.  If you use the 'handle'

 * facility it must all be done in one pass while under the iommu lock.

 *

 * On sun4u platforms, we only flush the IOMMU once every time we've passed

 * over the entire page table doing allocations.  Therefore we only ever advance

 * the hint and cannot backtrack it.

 */

unsigned long iommu_range_alloc(struct device *dev,

				struct iommu *iommu,

				unsigned long npages,

				unsigned long *handle)

{

	unsigned int i;

	static bool do_once;

	unsigned long n, end, start, limit, boundary_size;

	struct iommu_arena *arena = &iommu->arena;

	int pass = 0;



	if (do_once)

		return;

	do_once = true;

	for_each_possible_cpu(i)

		per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);

	/* This allocator was derived from x86_64's bit string search */



	/* Sanity check */

	if (unlikely(npages == 0)) {

		if (printk_ratelimit())

			WARN_ON(1);

		return DMA_ERROR_CODE;

	}



	if (handle && *handle)

		start = *handle;

	else

		start = arena->hint;



	limit = arena->limit;



	/* The case below can happen if we have a small segment appended

	 * to a large, or when the previous alloc was at the very end of

	 * the available space. If so, go back to the beginning and flush.

	 */

	if (start >= limit) {

		start = 0;

		if (iommu->flush_all)

			iommu->flush_all(iommu);

	}



 again:



	if (dev)

		boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,

				      1 << IO_PAGE_SHIFT);

	else

		boundary_size = ALIGN(1UL << 32, 1 << IO_PAGE_SHIFT);



	n = iommu_area_alloc(arena->map, limit, start, npages,

			     iommu->page_table_map_base >> IO_PAGE_SHIFT,

			     boundary_size >> IO_PAGE_SHIFT, 0);

	if (n == -1) {

		if (likely(pass < 1)) {

			/* First failure, rescan from the beginning.  */

			start = 0;

			if (iommu->flush_all)

				iommu->flush_all(iommu);

			pass++;

			goto again;

		} else {

			/* Second failure, give up */

			return DMA_ERROR_CODE;

		}

	}



	end = n + npages;



	arena->hint = end;



	/* Update handle for SG allocations */

	if (handle)

		*handle = end;



	return n;

}



void iommu_range_free(struct iommu *iommu, dma_addr_t dma_addr, unsigned long npages)

{

	struct iommu_arena *arena = &iommu->arena;

	unsigned long entry;



	entry = (dma_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT;



	bitmap_clear(arena->map, entry, npages);

}



int iommu_table_init(struct iommu *iommu, int tsbsize,

		     u32 dma_offset, u32 dma_addr_mask,
@@ -121,22 +187,22 @@ int iommu_table_init(struct iommu *iommu, int tsbsize, 
	/* Setup initial software IOMMU state. */

	spin_lock_init(&iommu->lock);

	iommu->ctx_lowest_free = 1;

	iommu->tbl.page_table_map_base = dma_offset;

	iommu->page_table_map_base = dma_offset;

	iommu->dma_addr_mask = dma_addr_mask;



	/* Allocate and initialize the free area map.  */

	sz = num_tsb_entries / 8;

	sz = (sz + 7UL) & ~7UL;

	iommu->tbl.map = kmalloc_node(sz, GFP_KERNEL, numa_node);

	if (!iommu->tbl.map)

	iommu->arena.map = kmalloc_node(sz, GFP_KERNEL, numa_node);

	if (!iommu->arena.map) {

		printk(KERN_ERR "IOMMU: Error, kmalloc(arena.map) failed.\n");

		return -ENOMEM;

	memset(iommu->tbl.map, 0, sz);

	if (tlb_type != hypervisor)

		iommu_sparc_ops.reset = NULL; /* not needed on on sun4v */

	}

	memset(iommu->arena.map, 0, sz);

	iommu->arena.limit = num_tsb_entries;



	setup_iommu_pool_hash();

	iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT,

			    &iommu_sparc_ops, false, 1);

	if (tlb_type != hypervisor)

		iommu->flush_all = iommu_flushall;



	/* Allocate and initialize the dummy page which we

	 * set inactive IO PTEs to point to.
@@ -169,20 +235,18 @@ int iommu_table_init(struct iommu *iommu, int tsbsize, 
	iommu->dummy_page = 0UL;



out_free_map:

	kfree(iommu->tbl.map);

	iommu->tbl.map = NULL;

	kfree(iommu->arena.map);

	iommu->arena.map = NULL;



	return -ENOMEM;

}



static inline iopte_t *alloc_npages(struct device *dev,

				    struct iommu *iommu,

static inline iopte_t *alloc_npages(struct device *dev, struct iommu *iommu,

				    unsigned long npages)

{

	unsigned long entry;



	entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,

				      __this_cpu_read(iommu_pool_hash));

	entry = iommu_range_alloc(dev, iommu, npages, NULL);

	if (unlikely(entry == DMA_ERROR_CODE))

		return NULL;
@@ -220,7 +284,7 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, 
				   dma_addr_t *dma_addrp, gfp_t gfp,

				   struct dma_attrs *attrs)

{

	unsigned long order, first_page;

	unsigned long flags, order, first_page;

	struct iommu *iommu;

	struct page *page;

	int npages, nid;
@@ -242,14 +306,16 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, 


	iommu = dev->archdata.iommu;



	spin_lock_irqsave(&iommu->lock, flags);

	iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT);

	spin_unlock_irqrestore(&iommu->lock, flags);



	if (unlikely(iopte == NULL)) {

		free_pages(first_page, order);

		return NULL;

	}



	*dma_addrp = (iommu->tbl.page_table_map_base +

	*dma_addrp = (iommu->page_table_map_base +

		      ((iopte - iommu->page_table) << IO_PAGE_SHIFT));

	ret = (void *) first_page;

	npages = size >> IO_PAGE_SHIFT;
@@ -270,12 +336,16 @@ static void dma_4u_free_coherent(struct device *dev, size_t size, 
				 struct dma_attrs *attrs)

{

	struct iommu *iommu;

	unsigned long order, npages;

	unsigned long flags, order, npages;



	npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;

	iommu = dev->archdata.iommu;



	iommu_tbl_range_free(&iommu->tbl, dvma, npages, false, NULL);

	spin_lock_irqsave(&iommu->lock, flags);



	iommu_range_free(iommu, dvma, npages);



	spin_unlock_irqrestore(&iommu->lock, flags);



	order = get_order(size);

	if (order < 10)
@@ -305,8 +375,8 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, 
	npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);

	npages >>= IO_PAGE_SHIFT;



	base = alloc_npages(dev, iommu, npages);

	spin_lock_irqsave(&iommu->lock, flags);

	base = alloc_npages(dev, iommu, npages);

	ctx = 0;

	if (iommu->iommu_ctxflush)

		ctx = iommu_alloc_ctx(iommu);
@@ -315,7 +385,7 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, 
	if (unlikely(!base))

		goto bad;



	bus_addr = (iommu->tbl.page_table_map_base +

	bus_addr = (iommu->page_table_map_base +

		    ((base - iommu->page_table) << IO_PAGE_SHIFT));

	ret = bus_addr | (oaddr & ~IO_PAGE_MASK);

	base_paddr = __pa(oaddr & IO_PAGE_MASK);
@@ -426,7 +496,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, 
	npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);

	npages >>= IO_PAGE_SHIFT;

	base = iommu->page_table +

		((bus_addr - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT);

		((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);

	bus_addr &= IO_PAGE_MASK;



	spin_lock_irqsave(&iommu->lock, flags);
@@ -445,11 +515,11 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, 
	for (i = 0; i < npages; i++)

		iopte_make_dummy(iommu, base + i);



	iommu_range_free(iommu, bus_addr, npages);



	iommu_free_ctx(iommu, ctx);

	spin_unlock_irqrestore(&iommu->lock, flags);



	iommu_tbl_range_free(&iommu->tbl, bus_addr, npages,

			     false, NULL);

	spin_unlock_irqrestore(&iommu->lock, flags);

}



static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -497,7 +567,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, 
	max_seg_size = dma_get_max_seg_size(dev);

	seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,

				  IO_PAGE_SIZE) >> IO_PAGE_SHIFT;

	base_shift = iommu->tbl.page_table_map_base >> IO_PAGE_SHIFT;

	base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT;

	for_each_sg(sglist, s, nelems, i) {

		unsigned long paddr, npages, entry, out_entry = 0, slen;

		iopte_t *base;
@@ -511,8 +581,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, 
		/* Allocate iommu entries for that segment */

		paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s);

		npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE);

		entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, &handle,

					      __this_cpu_read(iommu_pool_hash));

		entry = iommu_range_alloc(dev, iommu, npages, &handle);



		/* Handle failure */

		if (unlikely(entry == DMA_ERROR_CODE)) {
@@ -525,7 +594,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, 
		base = iommu->page_table + entry;



		/* Convert entry to a dma_addr_t */

		dma_addr = iommu->tbl.page_table_map_base +

		dma_addr = iommu->page_table_map_base +

			(entry << IO_PAGE_SHIFT);

		dma_addr |= (s->offset & ~IO_PAGE_MASK);
@@ -585,17 +654,15 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, 
			vaddr = s->dma_address & IO_PAGE_MASK;

			npages = iommu_num_pages(s->dma_address, s->dma_length,

						 IO_PAGE_SIZE);

			iommu_range_free(iommu, vaddr, npages);



			entry = (vaddr - iommu->tbl.page_table_map_base)

			entry = (vaddr - iommu->page_table_map_base)

				>> IO_PAGE_SHIFT;

			base = iommu->page_table + entry;



			for (j = 0; j < npages; j++)

				iopte_make_dummy(iommu, base + j);



			iommu_tbl_range_free(&iommu->tbl, vaddr, npages,

					     false, NULL);



			s->dma_address = DMA_ERROR_CODE;

			s->dma_length = 0;

		}
@@ -610,19 +677,17 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, 
/* If contexts are being used, they are the same in all of the mappings

 * we make for a particular SG.

 */

static unsigned long fetch_sg_ctx(struct iommu *iommu,

				  struct scatterlist *sg)

static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg)

{

	unsigned long ctx = 0;



	if (iommu->iommu_ctxflush) {

		iopte_t *base;

		u32 bus_addr;

		struct iommu_table *tbl = &iommu->tbl;



		bus_addr = sg->dma_address & IO_PAGE_MASK;

		base = iommu->page_table +

		       ((bus_addr - tbl->page_table_map_base) >> IO_PAGE_SHIFT);

			((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);



		ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;

	}
@@ -658,8 +723,9 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, 
		if (!len)

			break;

		npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE);

		iommu_range_free(iommu, dma_handle, npages);



		entry = ((dma_handle - iommu->tbl.page_table_map_base)

		entry = ((dma_handle - iommu->page_table_map_base)

			 >> IO_PAGE_SHIFT);

		base = iommu->page_table + entry;
@@ -671,8 +737,6 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, 
		for (i = 0; i < npages; i++)

			iopte_make_dummy(iommu, base + i);



		iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, false,

				     NULL);

		sg = sg_next(sg);

	}
@@ -706,10 +770,9 @@ static void dma_4u_sync_single_for_cpu(struct device *dev, 
	if (iommu->iommu_ctxflush &&

	    strbuf->strbuf_ctxflush) {

		iopte_t *iopte;

		struct iommu_table *tbl = &iommu->tbl;



		iopte = iommu->page_table +

			((bus_addr - tbl->page_table_map_base)>>IO_PAGE_SHIFT);

			((bus_addr - iommu->page_table_map_base)>>IO_PAGE_SHIFT);

		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;

	}
@@ -742,10 +805,9 @@ static void dma_4u_sync_sg_for_cpu(struct device *dev, 
	if (iommu->iommu_ctxflush &&

	    strbuf->strbuf_ctxflush) {

		iopte_t *iopte;

		struct iommu_table *tbl = &iommu->tbl;



		iopte = iommu->page_table + ((sglist[0].dma_address -

			tbl->page_table_map_base) >> IO_PAGE_SHIFT);

		iopte = iommu->page_table +

			((sglist[0].dma_address - iommu->page_table_map_base) >> IO_PAGE_SHIFT);

		ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;

	}

arch/sparc/kernel/iommu_common.h

+8 −0
Original line number Diff line number Diff line
@@ -48,4 +48,12 @@ static inline int is_span_boundary(unsigned long entry, 
	return iommu_is_span_boundary(entry, nr, shift, boundary_size);

}



unsigned long iommu_range_alloc(struct device *dev,

				struct iommu *iommu,

				unsigned long npages,

				unsigned long *handle);

void iommu_range_free(struct iommu *iommu,

		      dma_addr_t dma_addr,

		      unsigned long npages);



#endif /* _IOMMU_COMMON_H */

arch/sparc/kernel/ldc.c

+88 −97
Original line number Diff line number Diff line
@@ -15,8 +15,6 @@ 
#include <linux/list.h>

#include <linux/init.h>

#include <linux/bitmap.h>

#include <linux/hash.h>

#include <linux/iommu-common.h>



#include <asm/hypervisor.h>

#include <asm/iommu.h>
@@ -29,11 +27,6 @@ 
#define DRV_MODULE_VERSION	"1.1"

#define DRV_MODULE_RELDATE	"July 22, 2008"



#define COOKIE_PGSZ_CODE	0xf000000000000000ULL

#define COOKIE_PGSZ_CODE_SHIFT	60ULL



static DEFINE_PER_CPU(unsigned int, ldc_pool_hash);



static char version[] =

	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";

#define LDC_PACKET_SIZE		64
@@ -105,10 +98,10 @@ static const struct ldc_mode_ops stream_ops; 
int ldom_domaining_enabled;



struct ldc_iommu {

	/* Protects ldc_unmap.  */

	/* Protects arena alloc/free.  */

	spinlock_t			lock;

	struct iommu_arena		arena;

	struct ldc_mtable_entry		*page_table;

	struct iommu_table		iommu_table;

};



struct ldc_channel {
@@ -1005,85 +998,31 @@ static void free_queue(unsigned long num_entries, struct ldc_packet *q) 
	free_pages((unsigned long)q, order);

}



static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)

{

	u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;

	/* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */



	cookie &= ~COOKIE_PGSZ_CODE;



	return (cookie >> (13ULL + (szcode * 3ULL)));

}



struct ldc_demap_arg {

	struct ldc_iommu *ldc_iommu;

	u64 cookie;

	unsigned long id;

};



static void ldc_demap(void *arg, unsigned long entry, unsigned long npages)

{

	struct ldc_demap_arg *ldc_demap_arg = arg;

	struct ldc_iommu *iommu = ldc_demap_arg->ldc_iommu;

	unsigned long id = ldc_demap_arg->id;

	u64 cookie = ldc_demap_arg->cookie;

	struct ldc_mtable_entry *base;

	unsigned long i, shift;



	shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;

	base = iommu->page_table + entry;

	for (i = 0; i < npages; i++) {

		if (base->cookie)

			sun4v_ldc_revoke(id, cookie + (i << shift),

					 base->cookie);

		base->mte = 0;

	}

}



/* XXX Make this configurable... XXX */

#define LDC_IOTABLE_SIZE	(8 * 1024)



struct iommu_tbl_ops ldc_iommu_ops = {

	.cookie_to_index = ldc_cookie_to_index,

	.demap = ldc_demap,

};



static void setup_ldc_pool_hash(void)

{

	unsigned int i;

	static bool do_once;



	if (do_once)

		return;

	do_once = true;

	for_each_possible_cpu(i)

		per_cpu(ldc_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);

}





static int ldc_iommu_init(const char *name, struct ldc_channel *lp)

static int ldc_iommu_init(struct ldc_channel *lp)

{

	unsigned long sz, num_tsb_entries, tsbsize, order;

	struct ldc_iommu *ldc_iommu = &lp->iommu;

	struct iommu_table *iommu = &ldc_iommu->iommu_table;

	struct ldc_iommu *iommu = &lp->iommu;

	struct ldc_mtable_entry *table;

	unsigned long hv_err;

	int err;



	num_tsb_entries = LDC_IOTABLE_SIZE;

	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);

	setup_ldc_pool_hash();

	spin_lock_init(&ldc_iommu->lock);



	spin_lock_init(&iommu->lock);



	sz = num_tsb_entries / 8;

	sz = (sz + 7UL) & ~7UL;

	iommu->map = kzalloc(sz, GFP_KERNEL);

	if (!iommu->map) {

	iommu->arena.map = kzalloc(sz, GFP_KERNEL);

	if (!iommu->arena.map) {

		printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);

		return -ENOMEM;

	}

	iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,

			    &ldc_iommu_ops, false, 1);



	iommu->arena.limit = num_tsb_entries;



	order = get_order(tsbsize);
@@ -1098,7 +1037,7 @@ static int ldc_iommu_init(const char *name, struct ldc_channel *lp) 


	memset(table, 0, PAGE_SIZE << order);



	ldc_iommu->page_table = table;

	iommu->page_table = table;



	hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),

					 num_tsb_entries);
@@ -1110,32 +1049,31 @@ static int ldc_iommu_init(const char *name, struct ldc_channel *lp) 


out_free_table:

	free_pages((unsigned long) table, order);

	ldc_iommu->page_table = NULL;

	iommu->page_table = NULL;



out_free_map:

	kfree(iommu->map);

	iommu->map = NULL;

	kfree(iommu->arena.map);

	iommu->arena.map = NULL;



	return err;

}



static void ldc_iommu_release(struct ldc_channel *lp)

{

	struct ldc_iommu *ldc_iommu = &lp->iommu;

	struct iommu_table *iommu = &ldc_iommu->iommu_table;

	struct ldc_iommu *iommu = &lp->iommu;

	unsigned long num_tsb_entries, tsbsize, order;



	(void) sun4v_ldc_set_map_table(lp->id, 0, 0);



	num_tsb_entries = iommu->poolsize * iommu->nr_pools;

	num_tsb_entries = iommu->arena.limit;

	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);

	order = get_order(tsbsize);



	free_pages((unsigned long) ldc_iommu->page_table, order);

	ldc_iommu->page_table = NULL;

	free_pages((unsigned long) iommu->page_table, order);

	iommu->page_table = NULL;



	kfree(iommu->map);

	iommu->map = NULL;

	kfree(iommu->arena.map);

	iommu->arena.map = NULL;

}



struct ldc_channel *ldc_alloc(unsigned long id,
@@ -1202,7 +1140,7 @@ struct ldc_channel *ldc_alloc(unsigned long id, 


	lp->id = id;



	err = ldc_iommu_init(name, lp);

	err = ldc_iommu_init(lp);

	if (err)

		goto out_free_ldc;
@@ -1947,6 +1885,40 @@ int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) 
}

EXPORT_SYMBOL(ldc_read);



static long arena_alloc(struct ldc_iommu *iommu, unsigned long npages)

{

	struct iommu_arena *arena = &iommu->arena;

	unsigned long n, start, end, limit;

	int pass;



	limit = arena->limit;

	start = arena->hint;

	pass = 0;



again:

	n = bitmap_find_next_zero_area(arena->map, limit, start, npages, 0);

	end = n + npages;

	if (unlikely(end >= limit)) {

		if (likely(pass < 1)) {

			limit = start;

			start = 0;

			pass++;

			goto again;

		} else {

			/* Scanned the whole thing, give up. */

			return -1;

		}

	}

	bitmap_set(arena->map, n, npages);



	arena->hint = end;



	return n;

}



#define COOKIE_PGSZ_CODE	0xf000000000000000ULL

#define COOKIE_PGSZ_CODE_SHIFT	60ULL



static u64 pagesize_code(void)

{

	switch (PAGE_SIZE) {
@@ -1973,14 +1945,23 @@ static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) 
		page_offset);

}



static u64 cookie_to_index(u64 cookie, unsigned long *shift)

{

	u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;



	cookie &= ~COOKIE_PGSZ_CODE;



	*shift = szcode * 3;



	return (cookie >> (13ULL + (szcode * 3ULL)));

}



static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,

					     unsigned long npages)

{

	long entry;



	entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_table, npages,

				     NULL, __this_cpu_read(ldc_pool_hash));

	entry = arena_alloc(iommu, npages);

	if (unlikely(entry < 0))

		return NULL;
@@ -2109,7 +2090,7 @@ int ldc_map_sg(struct ldc_channel *lp, 
	       struct ldc_trans_cookie *cookies, int ncookies,

	       unsigned int map_perm)

{

	unsigned long i, npages;

	unsigned long i, npages, flags;

	struct ldc_mtable_entry *base;

	struct cookie_state state;

	struct ldc_iommu *iommu;
@@ -2128,7 +2109,9 @@ int ldc_map_sg(struct ldc_channel *lp, 


	iommu = &lp->iommu;



	spin_lock_irqsave(&iommu->lock, flags);

	base = alloc_npages(iommu, npages);

	spin_unlock_irqrestore(&iommu->lock, flags);



	if (!base)

		return -ENOMEM;
@@ -2153,7 +2136,7 @@ int ldc_map_single(struct ldc_channel *lp, 
		   struct ldc_trans_cookie *cookies, int ncookies,

		   unsigned int map_perm)

{

	unsigned long npages, pa;

	unsigned long npages, pa, flags;

	struct ldc_mtable_entry *base;

	struct cookie_state state;

	struct ldc_iommu *iommu;
@@ -2169,7 +2152,9 @@ int ldc_map_single(struct ldc_channel *lp, 


	iommu = &lp->iommu;



	spin_lock_irqsave(&iommu->lock, flags);

	base = alloc_npages(iommu, npages);

	spin_unlock_irqrestore(&iommu->lock, flags);



	if (!base)

		return -ENOMEM;
@@ -2187,29 +2172,35 @@ int ldc_map_single(struct ldc_channel *lp, 
}

EXPORT_SYMBOL(ldc_map_single);





static void free_npages(unsigned long id, struct ldc_iommu *iommu,

			u64 cookie, u64 size)

{

	unsigned long npages;

	struct ldc_demap_arg demap_arg;



	demap_arg.ldc_iommu = iommu;

	demap_arg.cookie = cookie;

	demap_arg.id = id;

	struct iommu_arena *arena = &iommu->arena;

	unsigned long i, shift, index, npages;

	struct ldc_mtable_entry *base;



	npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;

	iommu_tbl_range_free(&iommu->iommu_table, cookie, npages, true,

			     &demap_arg);

	index = cookie_to_index(cookie, &shift);

	base = iommu->page_table + index;



	BUG_ON(index > arena->limit ||

	       (index + npages) > arena->limit);



	for (i = 0; i < npages; i++) {

		if (base->cookie)

			sun4v_ldc_revoke(id, cookie + (i << shift),

					 base->cookie);

		base->mte = 0;

		__clear_bit(index + i, arena->map);

	}

}



void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,

	       int ncookies)

{

	struct ldc_iommu *iommu = &lp->iommu;

	int i;

	unsigned long flags;

	int i;



	spin_lock_irqsave(&iommu->lock, flags);

	for (i = 0; i < ncookies; i++) {

arch/sparc/kernel/pci_sun4v.c

+99 −94

File changed.

Preview size limit exceeded, changes collapsed.