Commit f4710445 authored by Mark Rutland's avatar Mark Rutland Committed by Catalin Marinas
Browse files

arm64: mm: use fixmap when creating page tables 



As a preparatory step to allow us to allocate early page tables from
unmapped memory using memblock_alloc, modify the __create_mapping
callees to map and unmap the tables they modify using fixmap entries.

All but the top-level pgd initialisation is performed via the fixmap.
Subsequent patches will inject the pgd physical address, and migrate to
using the FIX_PGD slot.

Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Tested-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: default avatarJeremy Linton <jeremy.linton@arm.com>
Cc: Laura Abbott <labbott@fedoraproject.org>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 961faac1

arch/arm64/mm/mmu.c

+41 −20
Original line number Original line Diff line number Diff line
@@ -63,19 +63,30 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 
}

}

EXPORT_SYMBOL(phys_mem_access_prot);

EXPORT_SYMBOL(phys_mem_access_prot);





static void __init *early_pgtable_alloc(void)

static phys_addr_t __init early_pgtable_alloc(void)

{

{

	phys_addr_t phys;

	phys_addr_t phys;

	void *ptr;

	void *ptr;





	phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE);

	phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE);

	BUG_ON(!phys);

	BUG_ON(!phys);

	ptr = __va(phys);



	/*

	 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE

	 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise

	 * any level of table.

	 */

	ptr = pte_set_fixmap(phys);



	memset(ptr, 0, PAGE_SIZE);

	memset(ptr, 0, PAGE_SIZE);





	/* Ensure the zeroed page is visible to the page table walker */

	/*

	dsb(ishst);

	 * Implicit barriers also ensure the zeroed page is visible to the page

	return ptr;

	 * table walker

	 */

	pte_clear_fixmap();



	return phys;

}

}





/*

/*
@@ -99,24 +110,28 @@ static void split_pmd(pmd_t *pmd, pte_t *pte) 
static void alloc_init_pte(pmd_t *pmd, unsigned long addr,

static void alloc_init_pte(pmd_t *pmd, unsigned long addr,

				  unsigned long end, unsigned long pfn,

				  unsigned long end, unsigned long pfn,

				  pgprot_t prot,

				  pgprot_t prot,

				  void *(*pgtable_alloc)(void))

				  phys_addr_t (*pgtable_alloc)(void))

{

{

	pte_t *pte;

	pte_t *pte;





	if (pmd_none(*pmd) || pmd_sect(*pmd)) {

	if (pmd_none(*pmd) || pmd_sect(*pmd)) {

		pte = pgtable_alloc();

		phys_addr_t pte_phys = pgtable_alloc();

		pte = pte_set_fixmap(pte_phys);

		if (pmd_sect(*pmd))

		if (pmd_sect(*pmd))

			split_pmd(pmd, pte);

			split_pmd(pmd, pte);

		__pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);

		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);

		flush_tlb_all();

		flush_tlb_all();

		pte_clear_fixmap();

	}

	}

	BUG_ON(pmd_bad(*pmd));

	BUG_ON(pmd_bad(*pmd));





	pte = pte_offset_kernel(pmd, addr);

	pte = pte_set_fixmap_offset(pmd, addr);

	do {

	do {

		set_pte(pte, pfn_pte(pfn, prot));

		set_pte(pte, pfn_pte(pfn, prot));

		pfn++;

		pfn++;

	} while (pte++, addr += PAGE_SIZE, addr != end);

	} while (pte++, addr += PAGE_SIZE, addr != end);



	pte_clear_fixmap();

}

}





static void split_pud(pud_t *old_pud, pmd_t *pmd)

static void split_pud(pud_t *old_pud, pmd_t *pmd)
@@ -134,7 +149,7 @@ static void split_pud(pud_t *old_pud, pmd_t *pmd) 
static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,

static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,

				  unsigned long addr, unsigned long end,

				  unsigned long addr, unsigned long end,

				  phys_addr_t phys, pgprot_t prot,

				  phys_addr_t phys, pgprot_t prot,

				  void *(*pgtable_alloc)(void))

				  phys_addr_t (*pgtable_alloc)(void))

{

{

	pmd_t *pmd;

	pmd_t *pmd;

	unsigned long next;

	unsigned long next;
@@ -143,7 +158,8 @@ static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, 
	 * Check for initial section mappings in the pgd/pud and remove them.

	 * Check for initial section mappings in the pgd/pud and remove them.

	 */

	 */

	if (pud_none(*pud) || pud_sect(*pud)) {

	if (pud_none(*pud) || pud_sect(*pud)) {

		pmd = pgtable_alloc();

		phys_addr_t pmd_phys = pgtable_alloc();

		pmd = pmd_set_fixmap(pmd_phys);

		if (pud_sect(*pud)) {

		if (pud_sect(*pud)) {

			/*

			/*

			 * need to have the 1G of mappings continue to be

			 * need to have the 1G of mappings continue to be
@@ -151,12 +167,13 @@ static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, 
			 */

			 */

			split_pud(pud, pmd);

			split_pud(pud, pmd);

		}

		}

		pud_populate(mm, pud, pmd);

		__pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);

		flush_tlb_all();

		flush_tlb_all();

		pmd_clear_fixmap();

	}

	}

	BUG_ON(pud_bad(*pud));

	BUG_ON(pud_bad(*pud));





	pmd = pmd_offset(pud, addr);

	pmd = pmd_set_fixmap_offset(pud, addr);

	do {

	do {

		next = pmd_addr_end(addr, end);

		next = pmd_addr_end(addr, end);

		/* try section mapping first */

		/* try section mapping first */
@@ -182,6 +199,8 @@ static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, 
		}

		}

		phys += next - addr;

		phys += next - addr;

	} while (pmd++, addr = next, addr != end);

	} while (pmd++, addr = next, addr != end);



	pmd_clear_fixmap();

}

}





static inline bool use_1G_block(unsigned long addr, unsigned long next,

static inline bool use_1G_block(unsigned long addr, unsigned long next,
@@ -199,18 +218,18 @@ static inline bool use_1G_block(unsigned long addr, unsigned long next, 
static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,

static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,

				  unsigned long addr, unsigned long end,

				  unsigned long addr, unsigned long end,

				  phys_addr_t phys, pgprot_t prot,

				  phys_addr_t phys, pgprot_t prot,

				  void *(*pgtable_alloc)(void))

				  phys_addr_t (*pgtable_alloc)(void))

{

{

	pud_t *pud;

	pud_t *pud;

	unsigned long next;

	unsigned long next;





	if (pgd_none(*pgd)) {

	if (pgd_none(*pgd)) {

		pud = pgtable_alloc();

		phys_addr_t pud_phys = pgtable_alloc();

		pgd_populate(mm, pgd, pud);

		__pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE);

	}

	}

	BUG_ON(pgd_bad(*pgd));

	BUG_ON(pgd_bad(*pgd));





	pud = pud_offset(pgd, addr);

	pud = pud_set_fixmap_offset(pgd, addr);

	do {

	do {

		next = pud_addr_end(addr, end);

		next = pud_addr_end(addr, end);
@@ -243,6 +262,8 @@ static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, 
		}

		}

		phys += next - addr;

		phys += next - addr;

	} while (pud++, addr = next, addr != end);

	} while (pud++, addr = next, addr != end);



	pud_clear_fixmap();

}

}





/*

/*
@@ -252,7 +273,7 @@ static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, 
static void  __create_mapping(struct mm_struct *mm, pgd_t *pgd,

static void  __create_mapping(struct mm_struct *mm, pgd_t *pgd,

				    phys_addr_t phys, unsigned long virt,

				    phys_addr_t phys, unsigned long virt,

				    phys_addr_t size, pgprot_t prot,

				    phys_addr_t size, pgprot_t prot,

				    void *(*pgtable_alloc)(void))

				    phys_addr_t (*pgtable_alloc)(void))

{

{

	unsigned long addr, length, end, next;

	unsigned long addr, length, end, next;
@@ -275,14 +296,14 @@ static void __create_mapping(struct mm_struct *mm, pgd_t *pgd, 
	} while (pgd++, addr = next, addr != end);

	} while (pgd++, addr = next, addr != end);

}

}





static void *late_pgtable_alloc(void)

static phys_addr_t late_pgtable_alloc(void)

{

{

	void *ptr = (void *)__get_free_page(PGALLOC_GFP);

	void *ptr = (void *)__get_free_page(PGALLOC_GFP);

	BUG_ON(!ptr);

	BUG_ON(!ptr);





	/* Ensure the zeroed page is visible to the page table walker */

	/* Ensure the zeroed page is visible to the page table walker */

	dsb(ishst);

	dsb(ishst);

	return ptr;

	return __pa(ptr);

}

}





static void __init create_mapping(phys_addr_t phys, unsigned long virt,

static void __init create_mapping(phys_addr_t phys, unsigned long virt,