Commit 617d1d34 authored by Tianchen Ding's avatar Tianchen Ding Committed by Zheng Zengkai
Browse files

kfence: alloc kfence_pool after system startup 

mainline inclusion
from mainline-v5.18-rc1
commit b33f778b
category: feature
bugzilla: 187071, https://gitee.com/openeuler/kernel/issues/I5DLA7
CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=b33f778bba5ef3f76fe6708c611346c1ea03acd4

--------------------------------

Allow enabling KFENCE after system startup by allocating its pool via the
page allocator. This provides the flexibility to enable KFENCE even if it
wasn't enabled at boot time.

Link: https://lkml.kernel.org/r/20220307074516.6920-3-dtcccc@linux.alibaba.com


Signed-off-by: default avatarTianchen Ding <dtcccc@linux.alibaba.com>
Reviewed-by: default avatarMarco Elver <elver@google.com>
Tested-by: default avatarPeng Liu <liupeng256@huawei.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Conflicts:
	mm/kfence/core.c
Signed-off-by: default avatarLiu Shixin <liushixin2@huawei.com>
Reviewed-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Signed-off-by: default avatarZheng Zengkai <zhengzengkai@huawei.com>
parent fbd7f3f8

mm/kfence/core.c

+90 −21
Original line number Diff line number Diff line
@@ -109,7 +109,7 @@ static unsigned long kfence_skip_covered_thresh __read_mostly = 75; 
module_param_named(skip_covered_thresh, kfence_skip_covered_thresh, ulong, 0644);



/* The pool of pages used for guard pages and objects. */

char *__kfence_pool __ro_after_init;

char *__kfence_pool __read_mostly;

EXPORT_SYMBOL(__kfence_pool); /* Export for test modules. */



#ifdef CONFIG_KFENCE_DYNAMIC_OBJECTS
@@ -633,17 +633,19 @@ static void rcu_guarded_free(struct rcu_head *h) 
	kfence_guarded_free((void *)meta->addr, meta, false);

}



static bool __init kfence_init_pool(void)

/*

 * Initialization of the KFENCE pool after its allocation.

 * Returns 0 on success; otherwise returns the address up to

 * which partial initialization succeeded.

 */

static unsigned long kfence_init_pool(void)

{

	unsigned long addr = (unsigned long)__kfence_pool;

	struct page *pages;

	int i;



	if (!__kfence_pool)

		return false;



	if (!arch_kfence_init_pool())

		goto err;

		return addr;



	pages = virt_to_page(addr);
@@ -661,7 +663,7 @@ static bool __init kfence_init_pool(void) 


		/* Verify we do not have a compound head page. */

		if (WARN_ON(compound_head(&pages[i]) != &pages[i]))

			goto err;

			return addr;



		__SetPageSlab(&pages[i]);

	}
@@ -674,7 +676,7 @@ static bool __init kfence_init_pool(void) 
	 */

	for (i = 0; i < 2; i++) {

		if (unlikely(!kfence_protect(addr)))

			goto err;

			return addr;



		addr += PAGE_SIZE;

	}
@@ -691,7 +693,7 @@ static bool __init kfence_init_pool(void) 


		/* Protect the right redzone. */

		if (unlikely(!kfence_protect(addr + PAGE_SIZE)))

			goto err;

			return addr;



		addr += 2 * PAGE_SIZE;

	}
@@ -704,9 +706,21 @@ static bool __init kfence_init_pool(void) 
	 */

	kmemleak_free(__kfence_pool);



	return 0;

}



static bool __init kfence_init_pool_early(void)

{

	unsigned long addr;



	if (!__kfence_pool)

		return false;



	addr = kfence_init_pool();



	if (!addr)

		return true;



err:

	/*

	 * Only release unprotected pages, and do not try to go back and change

	 * page attributes due to risk of failing to do so as well. If changing
@@ -719,6 +733,26 @@ static bool __init kfence_init_pool(void) 
	return false;

}



static bool kfence_init_pool_late(void)

{

	unsigned long addr, free_size;



	addr = kfence_init_pool();



	if (!addr)

		return true;



	/* Same as above. */

	free_size = KFENCE_POOL_SIZE - (addr - (unsigned long)__kfence_pool);

#ifdef CONFIG_CONTIG_ALLOC

	free_contig_range(page_to_pfn(virt_to_page(addr)), free_size / PAGE_SIZE);

#else

	free_pages_exact((void *)addr, free_size);

#endif

	__kfence_pool = NULL;

	return false;

}



/* === DebugFS Interface ==================================================== */



static int stats_show(struct seq_file *seq, void *v)
@@ -927,31 +961,66 @@ void __init kfence_alloc_pool(void) 
	}

}



static void kfence_init_enable(void)

{

	if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS))

		static_branch_enable(&kfence_allocation_key);

	WRITE_ONCE(kfence_enabled, true);

	queue_delayed_work(system_unbound_wq, &kfence_timer, 0);

	pr_info("initialized - using %lu bytes for %lu objects at 0x%p-0x%p\n", KFENCE_POOL_SIZE,

		(unsigned long)KFENCE_NR_OBJECTS, (void *)__kfence_pool,

		(void *)(__kfence_pool + KFENCE_POOL_SIZE));

}



void __init kfence_init(void)

{

	stack_hash_seed = (u32)random_get_entropy();



	/* Setting kfence_sample_interval to 0 on boot disables KFENCE. */

	if (!kfence_sample_interval)

		return;



	stack_hash_seed = (u32)random_get_entropy();

	if (!kfence_init_pool()) {

	if (!kfence_init_pool_early()) {

		pr_err("%s failed\n", __func__);

		return;

	}



	if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS))

		static_branch_enable(&kfence_allocation_key);

	WRITE_ONCE(kfence_enabled, true);

	queue_delayed_work(system_unbound_wq, &kfence_timer, 0);

	pr_info("initialized - using %lu bytes for %lu objects at 0x%p-0x%p\n", KFENCE_POOL_SIZE,

		(unsigned long)KFENCE_NR_OBJECTS, (void *)__kfence_pool,

		(void *)(__kfence_pool + KFENCE_POOL_SIZE));

	kfence_init_enable();

}



static int kfence_init_late(void)

{

	const unsigned long nr_pages = KFENCE_POOL_SIZE / PAGE_SIZE;

#ifdef CONFIG_CONTIG_ALLOC

	struct page *pages;



	pages = alloc_contig_pages(nr_pages, GFP_KERNEL, first_online_node, NULL);

	if (!pages)

		return -ENOMEM;

	__kfence_pool = page_to_virt(pages);

#else

	if (nr_pages > MAX_ORDER_NR_PAGES) {

		pr_warn("KFENCE_NUM_OBJECTS too large for buddy allocator\n");

		return -EINVAL;

	}

	__kfence_pool = alloc_pages_exact(KFENCE_POOL_SIZE, GFP_KERNEL);

	if (!__kfence_pool)

		return -ENOMEM;

#endif



	if (!kfence_init_pool_late()) {

		pr_err("%s failed\n", __func__);

		return -EBUSY;

	}



	kfence_init_enable();

	return 0;

}



static int kfence_enable_late(void)

{

	if (!__kfence_pool)

		return -EINVAL;

		return kfence_init_late();



	WRITE_ONCE(kfence_enabled, true);

	queue_delayed_work(system_unbound_wq, &kfence_timer, 0);