Commit 559089e0 authored by Song Liu's avatar Song Liu Committed by Linus Torvalds
Browse files

vmalloc: replace VM_NO_HUGE_VMAP with VM_ALLOW_HUGE_VMAP 

Huge page backed vmalloc memory could benefit performance in many cases.
However, some users of vmalloc may not be ready to handle huge pages for
various reasons: hardware constraints, potential pages split, etc.
VM_NO_HUGE_VMAP was introduced to allow vmalloc users to opt-out huge
pages.  However, it is not easy to track down all the users that require
the opt-out, as the allocation are passed different stacks and may cause
issues in different layers.

To address this issue, replace VM_NO_HUGE_VMAP with an opt-in flag,
VM_ALLOW_HUGE_VMAP, so that users that benefit from huge pages could ask
specificially.

Also, remove vmalloc_no_huge() and add opt-in helper vmalloc_huge().

Fixes: fac54e2b ("x86/Kconfig: Select HAVE_ARCH_HUGE_VMALLOC with HAVE_ARCH_HUGE_VMAP")
Link: https://lore.kernel.org/netdev/14444103-d51b-0fb3-ee63-c3f182f0b546@molgen.mpg.de/

"
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarSong Liu <song@kernel.org>
Reviewed-by: default avatarRik van Riel <riel@surriel.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b7f73403

arch/Kconfig

+2 −4
Original line number Diff line number Diff line
@@ -854,10 +854,8 @@ config HAVE_ARCH_HUGE_VMAP 


#

#  Archs that select this would be capable of PMD-sized vmaps (i.e.,

#  arch_vmap_pmd_supported() returns true), and they must make no assumptions

#  that vmalloc memory is mapped with PAGE_SIZE ptes. The VM_NO_HUGE_VMAP flag

#  can be used to prohibit arch-specific allocations from using hugepages to

#  help with this (e.g., modules may require it).

#  arch_vmap_pmd_supported() returns true). The VM_ALLOW_HUGE_VMAP flag

#  must be used to enable allocations to use hugepages.

#

config HAVE_ARCH_HUGE_VMALLOC

	depends on HAVE_ARCH_HUGE_VMAP

arch/powerpc/kernel/module.c

+1 −1
Original line number Diff line number Diff line
@@ -101,7 +101,7 @@ __module_alloc(unsigned long size, unsigned long start, unsigned long end, bool 
	 * too.

	 */

	return __vmalloc_node_range(size, 1, start, end, gfp, prot,

				    VM_FLUSH_RESET_PERMS | VM_NO_HUGE_VMAP,

				    VM_FLUSH_RESET_PERMS,

				    NUMA_NO_NODE, __builtin_return_address(0));

}

arch/s390/kvm/pv.c

+1 −6
Original line number Diff line number Diff line
@@ -137,12 +137,7 @@ static int kvm_s390_pv_alloc_vm(struct kvm *kvm) 
	/* Allocate variable storage */

	vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE);

	vlen += uv_info.guest_virt_base_stor_len;

	/*

	 * The Create Secure Configuration Ultravisor Call does not support

	 * using large pages for the virtual memory area.

	 * This is a hardware limitation.

	 */

	kvm->arch.pv.stor_var = vmalloc_no_huge(vlen);

	kvm->arch.pv.stor_var = vzalloc(vlen);

	if (!kvm->arch.pv.stor_var)

		goto out_err;

	return 0;

include/linux/vmalloc.h

+2 −2
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ struct notifier_block; /* in notifier.h */ 
#define VM_KASAN		0x00000080      /* has allocated kasan shadow memory */

#define VM_FLUSH_RESET_PERMS	0x00000100	/* reset direct map and flush TLB on unmap, can't be freed in atomic context */

#define VM_MAP_PUT_PAGES	0x00000200	/* put pages and free array in vfree */

#define VM_NO_HUGE_VMAP		0x00000400	/* force PAGE_SIZE pte mapping */

#define VM_ALLOW_HUGE_VMAP	0x00000400      /* Allow for huge pages on archs with HAVE_ARCH_HUGE_VMALLOC */



#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \

	!defined(CONFIG_KASAN_VMALLOC)
@@ -153,7 +153,7 @@ extern void *__vmalloc_node_range(unsigned long size, unsigned long align, 
			const void *caller) __alloc_size(1);

void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask,

		int node, const void *caller) __alloc_size(1);

void *vmalloc_no_huge(unsigned long size) __alloc_size(1);

void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) __alloc_size(1);



extern void *__vmalloc_array(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2);

extern void *vmalloc_array(size_t n, size_t size) __alloc_size(1, 2);

mm/vmalloc.c

+10 −7
Original line number Diff line number Diff line
@@ -3095,7 +3095,7 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, 
		return NULL;

	}



	if (vmap_allow_huge && !(vm_flags & VM_NO_HUGE_VMAP)) {

	if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) {

		unsigned long size_per_node;



		/*
@@ -3262,21 +3262,24 @@ void *vmalloc(unsigned long size) 
EXPORT_SYMBOL(vmalloc);



/**

 * vmalloc_no_huge - allocate virtually contiguous memory using small pages

 * vmalloc_huge - allocate virtually contiguous memory, allow huge pages

 * @size:      allocation size

 * @gfp_mask:  flags for the page level allocator

 *

 * Allocate enough non-huge pages to cover @size from the page level

 * Allocate enough pages to cover @size from the page level

 * allocator and map them into contiguous kernel virtual space.

 * If @size is greater than or equal to PMD_SIZE, allow using

 * huge pages for the memory

 *

 * Return: pointer to the allocated memory or %NULL on error

 */

void *vmalloc_no_huge(unsigned long size)

void *vmalloc_huge(unsigned long size, gfp_t gfp_mask)

{

	return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,

				    GFP_KERNEL, PAGE_KERNEL, VM_NO_HUGE_VMAP,

				    gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,

				    NUMA_NO_NODE, __builtin_return_address(0));

}

EXPORT_SYMBOL(vmalloc_no_huge);

EXPORT_SYMBOL_GPL(vmalloc_huge);



/**

 * vzalloc - allocate virtually contiguous memory with zero fill