Commit 5615f69b authored Oct 25, 2020 by Linus Walleij Committed by Russell King Oct 27, 2020

ARM: 9016/2: Initialize the mapping of KASan shadow memory



This patch initializes KASan shadow region's page table and memory.
There are two stage for KASan initializing:

1. At early boot stage the whole shadow region is mapped to just
   one physical page (kasan_zero_page). It is finished by the function
   kasan_early_init which is called by __mmap_switched(arch/arm/kernel/
   head-common.S)

2. After the calling of paging_init, we use kasan_zero_page as zero
   shadow for some memory that KASan does not need to track, and we
   allocate a new shadow space for the other memory that KASan need to
   track. These issues are finished by the function kasan_init which is
   call by setup_arch.

When using KASan we also need to increase the THREAD_SIZE_ORDER
from 1 to 2 as the extra calls for shadow memory uses quite a bit
of stack.

As we need to make a temporary copy of the PGD when setting up
shadow memory we create a helpful PGD_SIZE definition for both
LPAE and non-LPAE setups.

The KASan core code unconditionally calls pud_populate() so this
needs to be changed from BUG() to do {} while (0) when building
with KASan enabled.

After the initial development by Andre Ryabinin several modifications
have been made to this code:

Abbott Liu <liuwenliang@huawei.com>
- Add support ARM LPAE: If LPAE is enabled, KASan shadow region's
  mapping table need be copied in the pgd_alloc() function.
- Change kasan_pte_populate,kasan_pmd_populate,kasan_pud_populate,
  kasan_pgd_populate from .meminit.text section to .init.text section.
  Reported by Florian Fainelli <f.fainelli@gmail.com>

Linus Walleij <linus.walleij@linaro.org>:
- Drop the custom mainpulation of TTBR0 and just use
  cpu_switch_mm() to switch the pgd table.
- Adopt to handle 4th level page tabel folding.
- Rewrite the entire page directory and page entry initialization
  sequence to be recursive based on ARM64:s kasan_init.c.

Ard Biesheuvel <ardb@kernel.org>:
- Necessary underlying fixes.
- Crucial bug fixes to the memory set-up code.

Co-developed-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
Co-developed-by: Abbott Liu <liuwenliang@huawei.com>
Co-developed-by: Ard Biesheuvel <ardb@kernel.org>

Cc: Alexander Potapenko <glider@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: kasan-dev@googlegroups.com
Cc: Mike Rapoport <rppt@linux.ibm.com>
Acked-by: Mike Rapoport <rppt@linux.ibm.com>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Tested-by: Ard Biesheuvel <ardb@kernel.org> # QEMU/KVM/mach-virt/LPAE/8G
Tested-by: Florian Fainelli <f.fainelli@gmail.com> # Brahma SoCs
Tested-by: Ahmad Fatoum <a.fatoum@pengutronix.de> # i.MX6Q
Reported-by: Russell King - ARM Linux <rmk+kernel@armlinux.org.uk>
Reported-by: Florian Fainelli <f.fainelli@gmail.com>
Signed-off-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
Signed-off-by: Abbott Liu <liuwenliang@huawei.com>
Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Linus Walleij <linus.walleij@linaro.org>
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>

parent c12366ba

arch/arm/include/asm/kasan.h

0 → 100644

+33 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: GPL-2.0 */
		/*
		* arch/arm/include/asm/kasan.h
		*
		* Copyright (c) 2015 Samsung Electronics Co., Ltd.
		* Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
		*
		*/

		#ifndef __ASM_KASAN_H
		#define __ASM_KASAN_H

		#ifdef CONFIG_KASAN

		#include <asm/kasan_def.h>

		#define KASAN_SHADOW_SCALE_SHIFT 3

		/*
		* The compiler uses a shadow offset assuming that addresses start
		* from 0. Kernel addresses don't start from 0, so shadow
		* for kernel really starts from 'compiler's shadow offset' +
		* ('kernel address space start' >> KASAN_SHADOW_SCALE_SHIFT)
		*/

		asmlinkage void kasan_early_init(void);
		extern void kasan_init(void);

		#else
		static inline void kasan_init(void) { }
		#endif

		#endif

arch/arm/include/asm/pgalloc.h

+7 −1

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@
		#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE \| PMD_BIT4 \| PMD_DOMAIN(DOMAIN_KERNEL))

		#ifdef CONFIG_ARM_LPAE
		#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))

		static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)
		{
		@@ -28,14 +29,19 @@ static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)
		}

		#else /* !CONFIG_ARM_LPAE */
		#define PGD_SIZE (PAGE_SIZE << 2)

		/*
		* Since we have only two-level page tables, these are trivial
		*/
		#define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); })
		#define pmd_free(mm, pmd) do { } while (0)
		#ifdef CONFIG_KASAN
		/* The KASan core unconditionally calls pud_populate() on all architectures */
		#define pud_populate(mm,pmd,pte) do { } while (0)
		#else
		#define pud_populate(mm,pmd,pte) BUG()

		#endif
		#endif /* CONFIG_ARM_LPAE */

		extern pgd_t pgd_alloc(struct mm_struct mm);

arch/arm/include/asm/thread_info.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -13,7 +13,15 @@
		#include <asm/fpstate.h>
		#include <asm/page.h>

		#ifdef CONFIG_KASAN
		/*
		* KASan uses a lot of extra stack space so the thread size order needs to
		* be increased.
		*/
		#define THREAD_SIZE_ORDER 2
		#else
		#define THREAD_SIZE_ORDER 1
		#endif
		#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
		#define THREAD_START_SP (THREAD_SIZE - 8)

arch/arm/kernel/head-common.S

+3 −0

Original line number	Diff line number	Diff line
		@@ -111,6 +111,9 @@ __mmap_switched:
		str r8, [r2] @ Save atags pointer
		cmp r3, #0
		strne r10, [r3] @ Save control register values
		#ifdef CONFIG_KASAN
		bl kasan_early_init
		#endif
		mov lr, #0
		b start_kernel
		ENDPROC(__mmap_switched)

arch/arm/kernel/setup.c

+2 −0

Original line number	Diff line number	Diff line
		@@ -59,6 +59,7 @@
		#include <asm/unwind.h>
		#include <asm/memblock.h>
		#include <asm/virt.h>
		#include <asm/kasan.h>

		#include "atags.h"

		@@ -1145,6 +1146,7 @@ void __init setup_arch(char **cmdline_p)
		early_ioremap_reset();

		paging_init(mdesc);
		kasan_init();
		request_standard_resources(mdesc);

		if (mdesc->restart)