Commit b219372d authored by Patrik Jakobsson's avatar Patrik Jakobsson
Browse files

drm/gma500: Make SGX MMU driver actually do something 



Old MMU code never wrote PDs or PTEs to any registers. Now we do, and
that's a good start.

Signed-off-by: default avatarPatrik Jakobsson <patrik.r.jakobsson@gmail.com>
parent 786a7828

drivers/gpu/drm/gma500/mmu.c

+132 −126
Original line number Diff line number Diff line
@@ -59,15 +59,14 @@ struct psb_mmu_driver { 
	spinlock_t lock;



	atomic_t needs_tlbflush;



	uint8_t __iomem *register_map;

	atomic_t *msvdx_mmu_invaldc;

	struct psb_mmu_pd *default_pd;

	/*uint32_t bif_ctrl;*/

	uint32_t bif_ctrl;

	int has_clflush;

	int clflush_add;

	unsigned long clflush_mask;



	struct drm_psb_private *dev_priv;

	struct drm_device *dev;

};



struct psb_mmu_pd;
@@ -102,13 +101,13 @@ static inline uint32_t psb_mmu_pd_index(uint32_t offset) 
	return offset >> PSB_PDE_SHIFT;

}



#if defined(CONFIG_X86)

static inline void psb_clflush(void *addr)

{

	__asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");

}



static inline void psb_mmu_clflush(struct psb_mmu_driver *driver,

				   void *addr)

static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)

{

	if (!driver->has_clflush)

		return;
@@ -117,62 +116,77 @@ static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, 
	psb_clflush(addr);

	mb();

}

#else



static void psb_page_clflush(struct psb_mmu_driver *driver, struct page* page)

{

	uint32_t clflush_add = driver->clflush_add >> PAGE_SHIFT;

	uint32_t clflush_count = PAGE_SIZE / clflush_add;

	int i;

	uint8_t *clf;



	clf = kmap_atomic(page);

	mb();

	for (i = 0; i < clflush_count; ++i) {

		psb_clflush(clf);

		clf += clflush_add;

	}

	mb();

	kunmap_atomic(clf);

static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)

{;

}



static void psb_pages_clflush(struct psb_mmu_driver *driver,

				struct page *page[], unsigned long num_pages)

#endif



static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)

{

	int i;

	struct drm_device *dev = driver->dev;

	struct drm_psb_private *dev_priv = dev->dev_private;



	if (!driver->has_clflush)

		return ;

	if (atomic_read(&driver->needs_tlbflush) || force) {

		uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);

		PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);



	for (i = 0; i < num_pages; i++)

		psb_page_clflush(driver, *page++);

		/* Make sure data cache is turned off before enabling it */

		wmb();

		PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);

		(void)PSB_RSGX32(PSB_CR_BIF_CTRL);

		if (driver->msvdx_mmu_invaldc)

			atomic_set(driver->msvdx_mmu_invaldc, 1);

	}



static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver,

				    int force)

{

	atomic_set(&driver->needs_tlbflush, 0);

}



#if 0

static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)

{

	down_write(&driver->sem);

	psb_mmu_flush_pd_locked(driver, force);

	up_write(&driver->sem);

}

#endif



void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot)

void psb_mmu_flush(struct psb_mmu_driver *driver)

{

	if (rc_prot)

	struct drm_device *dev = driver->dev;

	struct drm_psb_private *dev_priv = dev->dev_private;

	uint32_t val;



	down_write(&driver->sem);

	if (rc_prot)

	val = PSB_RSGX32(PSB_CR_BIF_CTRL);

	if (atomic_read(&driver->needs_tlbflush))

		PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);

	else

		PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);



	/* Make sure data cache is turned off and MMU is flushed before

	   restoring bank interface control register */

	wmb();

	PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),

		   PSB_CR_BIF_CTRL);

	(void)PSB_RSGX32(PSB_CR_BIF_CTRL);



	atomic_set(&driver->needs_tlbflush, 0);

	if (driver->msvdx_mmu_invaldc)

		atomic_set(driver->msvdx_mmu_invaldc, 1);

	up_write(&driver->sem);

}



void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)

{

	/*ttm_tt_cache_flush(&pd->p, 1);*/

	psb_pages_clflush(pd->driver, &pd->p, 1);

	struct drm_device *dev = pd->driver->dev;

	struct drm_psb_private *dev_priv = dev->dev_private;

	uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :

			  PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;



	down_write(&pd->driver->sem);

	PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);

	wmb();

	psb_mmu_flush_pd_locked(pd->driver, 1);

	pd->hw_context = hw_context;
@@ -183,7 +197,6 @@ void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context) 
static inline unsigned long psb_pd_addr_end(unsigned long addr,

					    unsigned long end)

{



	addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;

	return (addr < end) ? addr : end;

}
@@ -223,11 +236,9 @@ struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 
		goto out_err3;



	if (!trap_pagefaults) {

		pd->invalid_pde =

		    psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),

		pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),

						   invalid_type);

		pd->invalid_pte =

		    psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),

		pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),

						   invalid_type);

	} else {

		pd->invalid_pde = 0;
@@ -279,12 +290,16 @@ static void psb_mmu_free_pt(struct psb_mmu_pt *pt) 
void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)

{

	struct psb_mmu_driver *driver = pd->driver;

	struct drm_device *dev = driver->dev;

	struct drm_psb_private *dev_priv = dev->dev_private;

	struct psb_mmu_pt *pt;

	int i;



	down_write(&driver->sem);

	if (pd->hw_context != -1)

	if (pd->hw_context != -1) {

		PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);

		psb_mmu_flush_pd_locked(driver, 1);

	}



	/* Should take the spinlock here, but we don't need to do that

	   since we have the semaphore in write mode. */
@@ -331,7 +346,7 @@ static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 
	for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)

		*ptes++ = pd->invalid_pte;





#if defined(CONFIG_X86)

	if (pd->driver->has_clflush && pd->hw_context != -1) {

		mb();

		for (i = 0; i < clflush_count; ++i) {
@@ -340,7 +355,7 @@ static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 
		}

		mb();

	}



#endif

	kunmap_atomic(v);

	spin_unlock(lock);
@@ -351,7 +366,7 @@ static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd) 
	return pt;

}



static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,

struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,

					     unsigned long addr)

{

	uint32_t index = psb_mmu_pd_index(addr);
@@ -420,8 +435,7 @@ static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 
		pd->tables[pt->index] = NULL;



		if (pd->hw_context != -1) {

			psb_mmu_clflush(pd->driver,

					(void *) &v[pt->index]);

			psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);

			atomic_set(&pd->driver->needs_tlbflush, 1);

		}

		kunmap_atomic(pt->v);
@@ -432,8 +446,8 @@ static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt) 
	spin_unlock(&pd->driver->lock);

}



static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt,

				   unsigned long addr, uint32_t pte)

static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,

				   uint32_t pte)

{

	pt->v[psb_mmu_pt_index(addr)] = pte;

}
@@ -444,69 +458,50 @@ static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt, 
	pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;

}





void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd,

			uint32_t mmu_offset, uint32_t gtt_start,

			uint32_t gtt_pages)

struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)

{

	uint32_t *v;

	uint32_t start = psb_mmu_pd_index(mmu_offset);

	struct psb_mmu_driver *driver = pd->driver;

	int num_pages = gtt_pages;

	struct psb_mmu_pd *pd;



	down_read(&driver->sem);

	spin_lock(&driver->lock);



	v = kmap_atomic(pd->p);

	v += start;



	while (gtt_pages--) {

		*v++ = gtt_start | pd->pd_mask;

		gtt_start += PAGE_SIZE;

	}



	/*ttm_tt_cache_flush(&pd->p, num_pages);*/

	psb_pages_clflush(pd->driver, &pd->p, num_pages);

	kunmap_atomic(v);

	spin_unlock(&driver->lock);



	if (pd->hw_context != -1)

		atomic_set(&pd->driver->needs_tlbflush, 1);

	pd = driver->default_pd;

	up_read(&driver->sem);



	up_read(&pd->driver->sem);

	psb_mmu_flush_pd(pd->driver, 0);

	return pd;

}



struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)

/* Returns the physical address of the PD shared by sgx/msvdx */

uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)

{

	struct psb_mmu_pd *pd;



	/* down_read(&driver->sem); */

	pd = driver->default_pd;

	/* up_read(&driver->sem); */



	return pd;

	pd = psb_mmu_get_default_pd(driver);

	return page_to_pfn(pd->p) << PAGE_SHIFT;

}



void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)

{

	struct drm_device *dev = driver->dev;

	struct drm_psb_private *dev_priv = dev->dev_private;



	PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);

	psb_mmu_free_pagedir(driver->default_pd);

	kfree(driver);

}



struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,

struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,

					   int trap_pagefaults,

					   int invalid_type,

					struct drm_psb_private *dev_priv)

					   atomic_t *msvdx_mmu_invaldc)

{

	struct psb_mmu_driver *driver;

	struct drm_psb_private *dev_priv = dev->dev_private;



	driver = kmalloc(sizeof(*driver), GFP_KERNEL);



	if (!driver)

		return NULL;

	driver->dev_priv = dev_priv;



	driver->dev = dev;

	driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,

					      invalid_type);

	if (!driver->default_pd)
@@ -515,17 +510,24 @@ struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers, 
	spin_lock_init(&driver->lock);

	init_rwsem(&driver->sem);

	down_write(&driver->sem);

	driver->register_map = registers;

	atomic_set(&driver->needs_tlbflush, 1);

	driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;



	driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);

	PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,

		   PSB_CR_BIF_CTRL);

	PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,

		   PSB_CR_BIF_CTRL);



	driver->has_clflush = 0;



#if defined(CONFIG_X86)

	if (boot_cpu_has(X86_FEATURE_CLFLSH)) {

		uint32_t tfms, misc, cap0, cap4, clflush_size;



		/*

		 * clflush size is determined at kernel setup for x86_64

		 *  but not for i386. We have to do it here.

		 * clflush size is determined at kernel setup for x86_64 but not

		 * for i386. We have to do it here.

		 */



		cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
@@ -536,6 +538,7 @@ struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers, 
		driver->clflush_mask = driver->clflush_add - 1;

		driver->clflush_mask = ~driver->clflush_mask;

	}

#endif



	up_write(&driver->sem);

	return driver;
@@ -545,9 +548,9 @@ struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers, 
	return NULL;

}



static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd,

			       unsigned long address, uint32_t num_pages,

			       uint32_t desired_tile_stride,

#if defined(CONFIG_X86)

static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,

			       uint32_t num_pages, uint32_t desired_tile_stride,

			       uint32_t hw_tile_stride)

{

	struct psb_mmu_pt *pt;
@@ -561,11 +564,8 @@ static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, 
	unsigned long clflush_add = pd->driver->clflush_add;

	unsigned long clflush_mask = pd->driver->clflush_mask;



	if (!pd->driver->has_clflush) {

		/*ttm_tt_cache_flush(&pd->p, num_pages);*/

		psb_pages_clflush(pd->driver, &pd->p, num_pages);

	if (!pd->driver->has_clflush)

		return;

	}



	if (hw_tile_stride)

		rows = num_pages / desired_tile_stride;
@@ -586,10 +586,8 @@ static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, 
			if (!pt)

				continue;

			do {

				psb_clflush(&pt->v

					    [psb_mmu_pt_index(addr)]);

			} while (addr +=

				 clflush_add,

				psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);

			} while (addr += clflush_add,

				 (addr & clflush_mask) < next);



			psb_mmu_pt_unmap_unlock(pt);
@@ -598,6 +596,14 @@ static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, 
	}

	mb();

}

#else

static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,

			       uint32_t num_pages, uint32_t desired_tile_stride,

			       uint32_t hw_tile_stride)

{

	drm_ttm_cache_flush();

}

#endif



void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,

				 unsigned long address, uint32_t num_pages)
@@ -633,7 +639,7 @@ void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd, 
	up_read(&pd->driver->sem);



	if (pd->hw_context != -1)

		psb_mmu_flush(pd->driver, 0);

		psb_mmu_flush(pd->driver);



	return;

}
@@ -660,7 +666,7 @@ void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 
	add = desired_tile_stride << PAGE_SHIFT;

	row_add = hw_tile_stride << PAGE_SHIFT;



	/* down_read(&pd->driver->sem); */

	down_read(&pd->driver->sem);



	/* Make sure we only need to flush this processor's cache */
@@ -688,10 +694,10 @@ void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address, 
		psb_mmu_flush_ptes(pd, f_address, num_pages,

				   desired_tile_stride, hw_tile_stride);



	/* up_read(&pd->driver->sem); */

	up_read(&pd->driver->sem);



	if (pd->hw_context != -1)

		psb_mmu_flush(pd->driver, 0);

		psb_mmu_flush(pd->driver);

}



int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
@@ -704,7 +710,7 @@ int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 
	unsigned long end;

	unsigned long next;

	unsigned long f_address = address;

	int ret = 0;

	int ret = -ENOMEM;



	down_read(&pd->driver->sem);
@@ -726,6 +732,7 @@ int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 
		psb_mmu_pt_unmap_unlock(pt);



	} while (addr = next, next != end);

	ret = 0;



out:

	if (pd->hw_context != -1)
@@ -734,15 +741,15 @@ int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn, 
	up_read(&pd->driver->sem);



	if (pd->hw_context != -1)

		psb_mmu_flush(pd->driver, 1);

		psb_mmu_flush(pd->driver);



	return ret;

	return 0;

}



int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,

			 unsigned long address, uint32_t num_pages,

			 uint32_t desired_tile_stride,

			 uint32_t hw_tile_stride, int type)

			 uint32_t desired_tile_stride, uint32_t hw_tile_stride,

			 int type)

{

	struct psb_mmu_pt *pt;

	uint32_t rows = 1;
@@ -754,7 +761,7 @@ int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 
	unsigned long add;

	unsigned long row_add;

	unsigned long f_address = address;

	int ret = 0;

	int ret = -ENOMEM;



	if (hw_tile_stride) {

		if (num_pages % desired_tile_stride != 0)
@@ -777,13 +784,10 @@ int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 
		do {

			next = psb_pd_addr_end(addr, end);

			pt = psb_mmu_pt_alloc_map_lock(pd, addr);

			if (!pt) {

				ret = -ENOMEM;

			if (!pt)

				goto out;

			}

			do {

				pte =

				    psb_mmu_mask_pte(page_to_pfn(*pages++),

				pte = psb_mmu_mask_pte(page_to_pfn(*pages++),

						       type);

				psb_mmu_set_pte(pt, addr, pte);

				pt->count++;
@@ -794,6 +798,8 @@ int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 


		address += row_add;

	}



	ret = 0;

out:

	if (pd->hw_context != -1)

		psb_mmu_flush_ptes(pd, f_address, num_pages,
@@ -802,7 +808,7 @@ int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages, 
	up_read(&pd->driver->sem);



	if (pd->hw_context != -1)

		psb_mmu_flush(pd->driver, 1);

		psb_mmu_flush(pd->driver);



	return ret;

}

drivers/gpu/drm/gma500/psb_drv.c

+1 −3
Original line number Diff line number Diff line
@@ -347,9 +347,7 @@ static int psb_driver_load(struct drm_device *dev, unsigned long chipset) 
	if (ret)

		goto out_err;



	dev_priv->mmu = psb_mmu_driver_init((void *)0,

					drm_psb_trap_pagefaults, 0,

					dev_priv);

	dev_priv->mmu = psb_mmu_driver_init(dev, drm_psb_trap_pagefaults, 0, 0);

	if (!dev_priv->mmu)

		goto out_err;

drivers/gpu/drm/gma500/psb_drv.h

+5 −5
Original line number Diff line number Diff line
@@ -727,10 +727,10 @@ static inline struct drm_psb_private *psb_priv(struct drm_device *dev) 
 * MMU stuff.

 */



extern struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,

extern struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,

						  int trap_pagefaults,

						  int invalid_type,

					struct drm_psb_private *dev_priv);

						  atomic_t *msvdx_mmu_invaldc);

extern void psb_mmu_driver_takedown(struct psb_mmu_driver *driver);

extern struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver

						 *driver);
@@ -740,7 +740,7 @@ extern struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver, 
					   int trap_pagefaults,

					   int invalid_type);

extern void psb_mmu_free_pagedir(struct psb_mmu_pd *pd);

extern void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot);

extern void psb_mmu_flush(struct psb_mmu_driver *driver);

extern void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,

					unsigned long address,

					uint32_t num_pages);