init: kmsan: call KMSAN initialization routines

 */

void kmsan_task_exit(struct task_struct *task);

/**

 * kmsan_init_shadow() - Initialize KMSAN shadow at boot time.

 *

 * Allocate and initialize KMSAN metadata for early allocations.

 */

void __init kmsan_init_shadow(void);

/**

 * kmsan_init_runtime() - Initialize KMSAN state and enable KMSAN.

 */

void __init kmsan_init_runtime(void);

/**

 * kmsan_memblock_free_pages() - handle freeing of memblock pages.

 * @page:	struct page to free.

 * @order:	order of @page.

 *

 * Freed pages are either returned to buddy allocator or held back to be used

 * as metadata pages.

 */

bool __init kmsan_memblock_free_pages(struct page *page, unsigned int order);

/**

 * kmsan_alloc_page() - Notify KMSAN about an alloc_pages() call.

 * @page:  struct page pointer returned by alloc_pages().

#else

static inline void kmsan_init_shadow(void)

{

}

static inline void kmsan_init_runtime(void)

{

}

static inline bool kmsan_memblock_free_pages(struct page *page,

					     unsigned int order)

{

	return true;

}

static inline void kmsan_task_create(struct task_struct *task)

{

}

#include <linux/percpu.h>

#include <linux/kmod.h>

#include <linux/kprobes.h>

#include <linux/kmsan.h>

#include <linux/vmalloc.h>

#include <linux/kernel_stat.h>

#include <linux/start_kernel.h>

	init_mem_debugging_and_hardening();

	kfence_alloc_pool();

	report_meminit();

	kmsan_init_shadow();

	stack_depot_early_init();

	mem_init();

	mem_init_print_info();

	init_espfix_bsp();

	/* Should be run after espfix64 is set up. */

	pti_init();

	kmsan_init_runtime();

}

#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET

# Makefile for KernelMemorySanitizer (KMSAN).

#

#

obj-y := core.o instrumentation.o hooks.o report.o shadow.o

obj-y := core.o instrumentation.o init.o hooks.o report.o shadow.o

KMSAN_SANITIZE := n

KCOV_INSTRUMENT := n

CFLAGS_core.o := $(CC_FLAGS_KMSAN_RUNTIME)

CFLAGS_hooks.o := $(CC_FLAGS_KMSAN_RUNTIME)

CFLAGS_init.o := $(CC_FLAGS_KMSAN_RUNTIME)

CFLAGS_instrumentation.o := $(CC_FLAGS_KMSAN_RUNTIME)

CFLAGS_report.o := $(CC_FLAGS_KMSAN_RUNTIME)

CFLAGS_shadow.o := $(CC_FLAGS_KMSAN_RUNTIME)

// SPDX-License-Identifier: GPL-2.0

/*

 * KMSAN initialization routines.

 *

 * Copyright (C) 2017-2021 Google LLC

 * Author: Alexander Potapenko <glider@google.com>

 *

 */

#include "kmsan.h"

#include <asm/sections.h>

#include <linux/mm.h>

#include <linux/memblock.h>

#include "../internal.h"

#define NUM_FUTURE_RANGES 128

struct start_end_pair {

	u64 start, end;

};

static struct start_end_pair start_end_pairs[NUM_FUTURE_RANGES] __initdata;

static int future_index __initdata;

/*

 * Record a range of memory for which the metadata pages will be created once

 * the page allocator becomes available.

 */

static void __init kmsan_record_future_shadow_range(void *start, void *end)

{

	u64 nstart = (u64)start, nend = (u64)end, cstart, cend;

	bool merged = false;

	KMSAN_WARN_ON(future_index == NUM_FUTURE_RANGES);

	KMSAN_WARN_ON((nstart >= nend) || !nstart || !nend);

	nstart = ALIGN_DOWN(nstart, PAGE_SIZE);

	nend = ALIGN(nend, PAGE_SIZE);

	/*

	 * Scan the existing ranges to see if any of them overlaps with

	 * [start, end). In that case, merge the two ranges instead of

	 * creating a new one.

	 * The number of ranges is less than 20, so there is no need to organize

	 * them into a more intelligent data structure.

	 */

	for (int i = 0; i < future_index; i++) {

		cstart = start_end_pairs[i].start;

		cend = start_end_pairs[i].end;

		if ((cstart < nstart && cend < nstart) ||

		    (cstart > nend && cend > nend))

			/* ranges are disjoint - do not merge */

			continue;

		start_end_pairs[i].start = min(nstart, cstart);

		start_end_pairs[i].end = max(nend, cend);

		merged = true;

		break;

	}

	if (merged)

		return;

	start_end_pairs[future_index].start = nstart;

	start_end_pairs[future_index].end = nend;

	future_index++;

}

/*

 * Initialize the shadow for existing mappings during kernel initialization.

 * These include kernel text/data sections, NODE_DATA and future ranges

 * registered while creating other data (e.g. percpu).

 *

 * Allocations via memblock can be only done before slab is initialized.

 */

void __init kmsan_init_shadow(void)

{

	const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);

	phys_addr_t p_start, p_end;

	u64 loop;

	int nid;

	for_each_reserved_mem_range(loop, &p_start, &p_end)

		kmsan_record_future_shadow_range(phys_to_virt(p_start),

						 phys_to_virt(p_end));

	/* Allocate shadow for .data */

	kmsan_record_future_shadow_range(_sdata, _edata);

	for_each_online_node(nid)

		kmsan_record_future_shadow_range(

			NODE_DATA(nid), (char *)NODE_DATA(nid) + nd_size);

	for (int i = 0; i < future_index; i++)

		kmsan_init_alloc_meta_for_range(

			(void *)start_end_pairs[i].start,

			(void *)start_end_pairs[i].end);

}

struct metadata_page_pair {

	struct page *shadow, *origin;

};

static struct metadata_page_pair held_back[MAX_ORDER] __initdata;

/*

 * Eager metadata allocation. When the memblock allocator is freeing pages to

 * pagealloc, we use 2/3 of them as metadata for the remaining 1/3.

 * We store the pointers to the returned blocks of pages in held_back[] grouped

 * by their order: when kmsan_memblock_free_pages() is called for the first

 * time with a certain order, it is reserved as a shadow block, for the second

 * time - as an origin block. On the third time the incoming block receives its

 * shadow and origin ranges from the previously saved shadow and origin blocks,

 * after which held_back[order] can be used again.

 *

 * At the very end there may be leftover blocks in held_back[]. They are

 * collected later by kmsan_memblock_discard().

 */

bool kmsan_memblock_free_pages(struct page *page, unsigned int order)

{

	struct page *shadow, *origin;

	if (!held_back[order].shadow) {

		held_back[order].shadow = page;

		return false;

	}

	if (!held_back[order].origin) {

		held_back[order].origin = page;

		return false;

	}

	shadow = held_back[order].shadow;

	origin = held_back[order].origin;

	kmsan_setup_meta(page, shadow, origin, order);

	held_back[order].shadow = NULL;

	held_back[order].origin = NULL;

	return true;

}

#define MAX_BLOCKS 8

struct smallstack {

	struct page *items[MAX_BLOCKS];

	int index;

	int order;

};

static struct smallstack collect = {

	.index = 0,

	.order = MAX_ORDER,

};

static void smallstack_push(struct smallstack *stack, struct page *pages)

{

	KMSAN_WARN_ON(stack->index == MAX_BLOCKS);

	stack->items[stack->index] = pages;

	stack->index++;

}

#undef MAX_BLOCKS

static struct page *smallstack_pop(struct smallstack *stack)

{

	struct page *ret;

	KMSAN_WARN_ON(stack->index == 0);

	stack->index--;

	ret = stack->items[stack->index];

	stack->items[stack->index] = NULL;

	return ret;

}

static void do_collection(void)

{

	struct page *page, *shadow, *origin;

	while (collect.index >= 3) {

		page = smallstack_pop(&collect);

		shadow = smallstack_pop(&collect);

		origin = smallstack_pop(&collect);

		kmsan_setup_meta(page, shadow, origin, collect.order);

		__free_pages_core(page, collect.order);

	}

}

static void collect_split(void)

{

	struct smallstack tmp = {

		.order = collect.order - 1,

		.index = 0,

	};

	struct page *page;

	if (!collect.order)

		return;

	while (collect.index) {

		page = smallstack_pop(&collect);

		smallstack_push(&tmp, &page[0]);

		smallstack_push(&tmp, &page[1 << tmp.order]);

	}

	__memcpy(&collect, &tmp, sizeof(tmp));

}

/*

 * Memblock is about to go away. Split the page blocks left over in held_back[]

 * and return 1/3 of that memory to the system.

 */

static void kmsan_memblock_discard(void)

{

	/*

	 * For each order=N:

	 *  - push held_back[N].shadow and .origin to @collect;

	 *  - while there are >= 3 elements in @collect, do garbage collection:

	 *    - pop 3 ranges from @collect;

	 *    - use two of them as shadow and origin for the third one;

	 *    - repeat;

	 *  - split each remaining element from @collect into 2 ranges of

	 *    order=N-1,

	 *  - repeat.

	 */

	collect.order = MAX_ORDER - 1;

	for (int i = MAX_ORDER - 1; i >= 0; i--) {

		if (held_back[i].shadow)

			smallstack_push(&collect, held_back[i].shadow);

		if (held_back[i].origin)

			smallstack_push(&collect, held_back[i].origin);

		held_back[i].shadow = NULL;

		held_back[i].origin = NULL;

		do_collection();

		collect_split();

	}

}

void __init kmsan_init_runtime(void)

{

	/* Assuming current is init_task */

	kmsan_internal_task_create(current);

	kmsan_memblock_discard();

	pr_info("Starting KernelMemorySanitizer\n");

	pr_info("ATTENTION: KMSAN is a debugging tool! Do not use it on production machines!\n");

	kmsan_enabled = true;

}

struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *addr, u64 size,

						     bool store);

void *kmsan_get_metadata(void *addr, bool is_origin);

void __init kmsan_init_alloc_meta_for_range(void *start, void *end);

enum kmsan_bug_reason {

	REASON_ANY,

				 int reason);

struct page *kmsan_vmalloc_to_page_or_null(void *vaddr);

void kmsan_setup_meta(struct page *page, struct page *shadow,

		      struct page *origin, int order);

/*

 * kmsan_internal_is_module_addr() and kmsan_internal_is_vmalloc_addr() are