Commit b073d7f8 authored by Alexander Potapenko's avatar Alexander Potapenko Committed by Andrew Morton
Browse files

mm: kmsan: maintain KMSAN metadata for page operations 

Insert KMSAN hooks that make the necessary bookkeeping changes:
 - poison page shadow and origins in alloc_pages()/free_page();
 - clear page shadow and origins in clear_page(), copy_user_highpage();
 - copy page metadata in copy_highpage(), wp_page_copy();
 - handle vmap()/vunmap()/iounmap();

Link: https://lkml.kernel.org/r/20220915150417.722975-15-glider@google.com


Signed-off-by: default avatarAlexander Potapenko <glider@google.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrey Konovalov <andreyknvl@gmail.com>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Eric Biggers <ebiggers@google.com>
Cc: Eric Biggers <ebiggers@kernel.org>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Ilya Leoshkevich <iii@linux.ibm.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Marco Elver <elver@google.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Michael S. Tsirkin <mst@redhat.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Petr Mladek <pmladek@suse.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vegard Nossum <vegard.nossum@oracle.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent d596b04f

arch/x86/include/asm/page_64.h

+7 −0
Original line number Diff line number Diff line
@@ -8,6 +8,8 @@ 
#include <asm/cpufeatures.h>

#include <asm/alternative.h>



#include <linux/kmsan-checks.h>



/* duplicated to the one in bootmem.h */

extern unsigned long max_pfn;

extern unsigned long phys_base;
@@ -47,6 +49,11 @@ void clear_page_erms(void *page); 


static inline void clear_page(void *page)

{

	/*

	 * Clean up KMSAN metadata for the page being cleared. The assembly call

	 * below clobbers @page, so we perform unpoisoning before it.

	 */

	kmsan_unpoison_memory(page, PAGE_SIZE);

	alternative_call_2(clear_page_orig,

			   clear_page_rep, X86_FEATURE_REP_GOOD,

			   clear_page_erms, X86_FEATURE_ERMS,

arch/x86/mm/ioremap.c

+3 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#include <linux/cc_platform.h>

#include <linux/efi.h>

#include <linux/pgtable.h>

#include <linux/kmsan.h>



#include <asm/set_memory.h>

#include <asm/e820/api.h>
@@ -479,6 +480,8 @@ void iounmap(volatile void __iomem *addr) 
		return;

	}



	kmsan_iounmap_page_range((unsigned long)addr,

		(unsigned long)addr + get_vm_area_size(p));

	memtype_free(p->phys_addr, p->phys_addr + get_vm_area_size(p));



	/* Finally remove it */

include/linux/highmem.h

+3 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
#include <linux/kernel.h>

#include <linux/bug.h>

#include <linux/cacheflush.h>

#include <linux/kmsan.h>

#include <linux/mm.h>

#include <linux/uaccess.h>

#include <linux/hardirq.h>
@@ -311,6 +312,7 @@ static inline void copy_user_highpage(struct page *to, struct page *from, 
	vfrom = kmap_local_page(from);

	vto = kmap_local_page(to);

	copy_user_page(vto, vfrom, vaddr, to);

	kmsan_unpoison_memory(page_address(to), PAGE_SIZE);

	kunmap_local(vto);

	kunmap_local(vfrom);

}
@@ -326,6 +328,7 @@ static inline void copy_highpage(struct page *to, struct page *from) 
	vfrom = kmap_local_page(from);

	vto = kmap_local_page(to);

	copy_page(vto, vfrom);

	kmsan_copy_page_meta(to, from);

	kunmap_local(vto);

	kunmap_local(vfrom);

}

include/linux/kmsan.h

0 → 100644
+145 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * KMSAN API for subsystems.

 *

 * Copyright (C) 2017-2022 Google LLC

 * Author: Alexander Potapenko <glider@google.com>

 *

 */

#ifndef _LINUX_KMSAN_H

#define _LINUX_KMSAN_H



#include <linux/gfp.h>

#include <linux/kmsan-checks.h>

#include <linux/types.h>



struct page;



#ifdef CONFIG_KMSAN



/**

 * kmsan_alloc_page() - Notify KMSAN about an alloc_pages() call.

 * @page:  struct page pointer returned by alloc_pages().

 * @order: order of allocated struct page.

 * @flags: GFP flags used by alloc_pages()

 *

 * KMSAN marks 1<<@order pages starting at @page as uninitialized, unless

 * @flags contain __GFP_ZERO.

 */

void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags);



/**

 * kmsan_free_page() - Notify KMSAN about a free_pages() call.

 * @page:  struct page pointer passed to free_pages().

 * @order: order of deallocated struct page.

 *

 * KMSAN marks freed memory as uninitialized.

 */

void kmsan_free_page(struct page *page, unsigned int order);



/**

 * kmsan_copy_page_meta() - Copy KMSAN metadata between two pages.

 * @dst: destination page.

 * @src: source page.

 *

 * KMSAN copies the contents of metadata pages for @src into the metadata pages

 * for @dst. If @dst has no associated metadata pages, nothing happens.

 * If @src has no associated metadata pages, @dst metadata pages are unpoisoned.

 */

void kmsan_copy_page_meta(struct page *dst, struct page *src);



/**

 * kmsan_map_kernel_range_noflush() - Notify KMSAN about a vmap.

 * @start:	start of vmapped range.

 * @end:	end of vmapped range.

 * @prot:	page protection flags used for vmap.

 * @pages:	array of pages.

 * @page_shift:	page_shift passed to vmap_range_noflush().

 *

 * KMSAN maps shadow and origin pages of @pages into contiguous ranges in

 * vmalloc metadata address range.

 */

void kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end,

				    pgprot_t prot, struct page **pages,

				    unsigned int page_shift);



/**

 * kmsan_vunmap_kernel_range_noflush() - Notify KMSAN about a vunmap.

 * @start: start of vunmapped range.

 * @end:   end of vunmapped range.

 *

 * KMSAN unmaps the contiguous metadata ranges created by

 * kmsan_map_kernel_range_noflush().

 */

void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end);



/**

 * kmsan_ioremap_page_range() - Notify KMSAN about a ioremap_page_range() call.

 * @addr:	range start.

 * @end:	range end.

 * @phys_addr:	physical range start.

 * @prot:	page protection flags used for ioremap_page_range().

 * @page_shift:	page_shift argument passed to vmap_range_noflush().

 *

 * KMSAN creates new metadata pages for the physical pages mapped into the

 * virtual memory.

 */

void kmsan_ioremap_page_range(unsigned long addr, unsigned long end,

			      phys_addr_t phys_addr, pgprot_t prot,

			      unsigned int page_shift);



/**

 * kmsan_iounmap_page_range() - Notify KMSAN about a iounmap_page_range() call.

 * @start: range start.

 * @end:   range end.

 *

 * KMSAN unmaps the metadata pages for the given range and, unlike for

 * vunmap_page_range(), also deallocates them.

 */

void kmsan_iounmap_page_range(unsigned long start, unsigned long end);



#else



static inline int kmsan_alloc_page(struct page *page, unsigned int order,

				   gfp_t flags)

{

	return 0;

}



static inline void kmsan_free_page(struct page *page, unsigned int order)

{

}



static inline void kmsan_copy_page_meta(struct page *dst, struct page *src)

{

}



static inline void kmsan_vmap_pages_range_noflush(unsigned long start,

						  unsigned long end,

						  pgprot_t prot,

						  struct page **pages,

						  unsigned int page_shift)

{

}



static inline void kmsan_vunmap_range_noflush(unsigned long start,

					      unsigned long end)

{

}



static inline void kmsan_ioremap_page_range(unsigned long start,

					    unsigned long end,

					    phys_addr_t phys_addr,

					    pgprot_t prot,

					    unsigned int page_shift)

{

}



static inline void kmsan_iounmap_page_range(unsigned long start,

					    unsigned long end)

{

}



#endif



#endif /* _LINUX_KMSAN_H */

mm/internal.h

+6 −0
Original line number Diff line number Diff line
@@ -818,8 +818,14 @@ int vmap_pages_range_noflush(unsigned long addr, unsigned long end, 
}

#endif



int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,

			       pgprot_t prot, struct page **pages,

			       unsigned int page_shift);



void vunmap_range_noflush(unsigned long start, unsigned long end);



void __vunmap_range_noflush(unsigned long start, unsigned long end);



int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,

		      unsigned long addr, int page_nid, int *flags);