Commit b1405135 authored by Hyeonggon Yoo's avatar Hyeonggon Yoo Committed by Vlastimil Babka
Browse files

mm/sl[au]b: generalize kmalloc subsystem 



Now everything in kmalloc subsystem can be generalized.
Let's do it!

Generalize __do_kmalloc_node(), __kmalloc_node_track_caller(),
kfree(), __ksize(), __kmalloc(), __kmalloc_node() and move them
to slab_common.c.

In the meantime, rename kmalloc_large_node_notrace()
to __kmalloc_large_node() and make it static as it's now only called in
slab_common.c.

[ feng.tang@intel.com: adjust kfence skip list to include
  __kmem_cache_free so that kfence kunit tests do not fail ]

Signed-off-by: default avatarHyeonggon Yoo <42.hyeyoo@gmail.com>
Reviewed-by: default avatarVlastimil Babka <vbabka@suse.cz>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent ed4cd17e

mm/kfence/report.c

+1 −0
Original line number Diff line number Diff line
@@ -86,6 +86,7 @@ static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries 
		/* Also the *_bulk() variants by only checking prefixes. */

		if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||

		    str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||

		    str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||

		    str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||

		    str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))

			goto found;

mm/slab.c

+0 −108
Original line number Diff line number Diff line
@@ -3587,44 +3587,6 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, 
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);

#endif



static __always_inline void *

__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)

{

	struct kmem_cache *cachep;

	void *ret;



	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {

		ret = kmalloc_large_node_notrace(size, flags, node);



		trace_kmalloc_node(caller, ret, NULL, size,

				   PAGE_SIZE << get_order(size),

				   flags, node);

		return ret;

	}



	cachep = kmalloc_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(cachep)))

		return cachep;



	ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);

	ret = kasan_kmalloc(cachep, ret, size, flags);



	return ret;

}



void *__kmalloc_node(size_t size, gfp_t flags, int node)

{

	return __do_kmalloc_node(size, flags, node, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc_node);



void *__kmalloc_node_track_caller(size_t size, gfp_t flags,

		int node, unsigned long caller)

{

	return __do_kmalloc_node(size, flags, node, caller);

}

EXPORT_SYMBOL(__kmalloc_node_track_caller);



#ifdef CONFIG_PRINTK

void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)

{
@@ -3647,12 +3609,6 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 
}

#endif



void *__kmalloc(size_t size, gfp_t flags)

{

	return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc);



static __always_inline

void __do_kmem_cache_free(struct kmem_cache *cachep, void *objp,

			  unsigned long caller)
@@ -3730,43 +3686,6 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) 
}

EXPORT_SYMBOL(kmem_cache_free_bulk);



/**

 * kfree - free previously allocated memory

 * @objp: pointer returned by kmalloc.

 *

 * If @objp is NULL, no operation is performed.

 *

 * Don't free memory not originally allocated by kmalloc()

 * or you will run into trouble.

 */

void kfree(const void *objp)

{

	struct kmem_cache *c;

	unsigned long flags;

	struct folio *folio;



	trace_kfree(_RET_IP_, objp);



	if (unlikely(ZERO_OR_NULL_PTR(objp)))

		return;



	folio = virt_to_folio(objp);

	if (!folio_test_slab(folio)) {

		free_large_kmalloc(folio, (void *)objp);

		return;

	}



	c = folio_slab(folio)->slab_cache;



	local_irq_save(flags);

	kfree_debugcheck(objp);

	debug_check_no_locks_freed(objp, c->object_size);

	debug_check_no_obj_freed(objp, c->object_size);

	__cache_free(c, (void *)objp, _RET_IP_);

	local_irq_restore(flags);

}

EXPORT_SYMBOL(kfree);



/*

 * This initializes kmem_cache_node or resizes various caches for all nodes.

 */
@@ -4169,30 +4088,3 @@ void __check_heap_object(const void *ptr, unsigned long n, 
	usercopy_abort("SLAB object", cachep->name, to_user, offset, n);

}

#endif /* CONFIG_HARDENED_USERCOPY */
 


/**

 * __ksize -- Uninstrumented ksize.

 * @objp: pointer to the object

 *

 * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same

 * safety checks as ksize() with KASAN instrumentation enabled.

 *

 * Return: size of the actual memory used by @objp in bytes

 */

size_t __ksize(const void *objp)

{

	struct kmem_cache *c;

	struct folio *folio;



	BUG_ON(!objp);

	if (unlikely(objp == ZERO_SIZE_PTR))

		return 0;



	folio = virt_to_folio(objp);

	if (!folio_test_slab(folio))

		return folio_size(folio);



	c = folio_slab(folio)->slab_cache;

	return c->object_size;

}

EXPORT_SYMBOL(__ksize);

mm/slab.h

+0 −2
Original line number Diff line number Diff line
@@ -280,8 +280,6 @@ void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, 
void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller);

#endif



void *kmalloc_large_node_notrace(size_t size, gfp_t flags, int node);



gfp_t kmalloc_fix_flags(gfp_t flags);



/* Functions provided by the slab allocators */

mm/slab_common.c

+106 −3
Original line number Diff line number Diff line
@@ -897,6 +897,109 @@ void free_large_kmalloc(struct folio *folio, void *object) 
			      -(PAGE_SIZE << order));

	__free_pages(folio_page(folio, 0), order);

}



static void *__kmalloc_large_node(size_t size, gfp_t flags, int node);

static __always_inline

void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)

{

	struct kmem_cache *s;

	void *ret;



	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {

		ret = __kmalloc_large_node(size, flags, node);

		trace_kmalloc_node(caller, ret, NULL,

				   size, PAGE_SIZE << get_order(size),

				   flags, node);

		return ret;

	}



	s = kmalloc_slab(size, flags);



	if (unlikely(ZERO_OR_NULL_PTR(s)))

		return s;



	ret = __kmem_cache_alloc_node(s, flags, node, size, caller);

	ret = kasan_kmalloc(s, ret, size, flags);

	trace_kmalloc_node(caller, ret, s, size,

			   s->size, flags, node);

	return ret;

}



void *__kmalloc_node(size_t size, gfp_t flags, int node)

{

	return __do_kmalloc_node(size, flags, node, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc_node);



void *__kmalloc(size_t size, gfp_t flags)

{

	return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc);



void *__kmalloc_node_track_caller(size_t size, gfp_t flags,

				  int node, unsigned long caller)

{

	return __do_kmalloc_node(size, flags, node, caller);

}

EXPORT_SYMBOL(__kmalloc_node_track_caller);



/**

 * kfree - free previously allocated memory

 * @object: pointer returned by kmalloc.

 *

 * If @object is NULL, no operation is performed.

 *

 * Don't free memory not originally allocated by kmalloc()

 * or you will run into trouble.

 */

void kfree(const void *object)

{

	struct folio *folio;

	struct slab *slab;

	struct kmem_cache *s;



	trace_kfree(_RET_IP_, object);



	if (unlikely(ZERO_OR_NULL_PTR(object)))

		return;



	folio = virt_to_folio(object);

	if (unlikely(!folio_test_slab(folio))) {

		free_large_kmalloc(folio, (void *)object);

		return;

	}



	slab = folio_slab(folio);

	s = slab->slab_cache;

	__kmem_cache_free(s, (void *)object, _RET_IP_);

}

EXPORT_SYMBOL(kfree);



/**

 * __ksize -- Uninstrumented ksize.

 * @object: pointer to the object

 *

 * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same

 * safety checks as ksize() with KASAN instrumentation enabled.

 *

 * Return: size of the actual memory used by @object in bytes

 */

size_t __ksize(const void *object)

{

	struct folio *folio;



	if (unlikely(object == ZERO_SIZE_PTR))

		return 0;



	folio = virt_to_folio(object);



	if (unlikely(!folio_test_slab(folio)))

		return folio_size(folio);



	return slab_ksize(folio_slab(folio)->slab_cache);

}

EXPORT_SYMBOL(__ksize);

#endif /* !CONFIG_SLOB */



gfp_t kmalloc_fix_flags(gfp_t flags)
@@ -917,7 +1020,7 @@ gfp_t kmalloc_fix_flags(gfp_t flags) 
 * know the allocation order to free the pages properly in kfree.

 */



void *kmalloc_large_node_notrace(size_t size, gfp_t flags, int node)

static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)

{

	struct page *page;

	void *ptr = NULL;
@@ -943,7 +1046,7 @@ void *kmalloc_large_node_notrace(size_t size, gfp_t flags, int node) 


void *kmalloc_large(size_t size, gfp_t flags)

{

	void *ret = kmalloc_large_node_notrace(size, flags, NUMA_NO_NODE);

	void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);



	trace_kmalloc(_RET_IP_, ret, NULL, size,

		      PAGE_SIZE << get_order(size), flags);
@@ -953,7 +1056,7 @@ EXPORT_SYMBOL(kmalloc_large); 


void *kmalloc_large_node(size_t size, gfp_t flags, int node)

{

	void *ret = kmalloc_large_node_notrace(size, flags, node);

	void *ret = __kmalloc_large_node(size, flags, node);



	trace_kmalloc_node(_RET_IP_, ret, NULL, size,

			   PAGE_SIZE << get_order(size), flags, node);

mm/slub.c

+0 −87
Original line number Diff line number Diff line
@@ -4388,49 +4388,6 @@ static int __init setup_slub_min_objects(char *str) 


__setup("slub_min_objects=", setup_slub_min_objects);



static __always_inline

void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)

{

	struct kmem_cache *s;

	void *ret;



	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {

		ret = kmalloc_large_node_notrace(size, flags, node);



		trace_kmalloc_node(caller, ret, NULL,

				   size, PAGE_SIZE << get_order(size),

				   flags, node);



		return ret;

	}



	s = kmalloc_slab(size, flags);



	if (unlikely(ZERO_OR_NULL_PTR(s)))

		return s;



	ret = slab_alloc_node(s, NULL, flags, node, caller, size);



	trace_kmalloc_node(caller, ret, s, size, s->size, flags, node);



	ret = kasan_kmalloc(s, ret, size, flags);



	return ret;

}



void *__kmalloc_node(size_t size, gfp_t flags, int node)

{

	return __do_kmalloc_node(size, flags, node, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc_node);



void *__kmalloc(size_t size, gfp_t flags)

{

	return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc);





#ifdef CONFIG_HARDENED_USERCOPY

/*

 * Rejects incorrectly sized objects and objects that are to be copied
@@ -4481,43 +4438,6 @@ void __check_heap_object(const void *ptr, unsigned long n, 
}

#endif /* CONFIG_HARDENED_USERCOPY */



size_t __ksize(const void *object)

{

	struct folio *folio;



	if (unlikely(object == ZERO_SIZE_PTR))

		return 0;



	folio = virt_to_folio(object);



	if (unlikely(!folio_test_slab(folio)))

		return folio_size(folio);



	return slab_ksize(folio_slab(folio)->slab_cache);

}

EXPORT_SYMBOL(__ksize);



void kfree(const void *x)

{

	struct folio *folio;

	struct slab *slab;

	void *object = (void *)x;



	trace_kfree(_RET_IP_, x);



	if (unlikely(ZERO_OR_NULL_PTR(x)))

		return;



	folio = virt_to_folio(x);

	if (unlikely(!folio_test_slab(folio))) {

		free_large_kmalloc(folio, object);

		return;

	}

	slab = folio_slab(folio);

	slab_free(slab->slab_cache, slab, object, NULL, &object, 1, _RET_IP_);

}

EXPORT_SYMBOL(kfree);



#define SHRINK_PROMOTE_MAX 32



/*
@@ -4863,13 +4783,6 @@ int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags) 
	return 0;

}



void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,

				  int node, unsigned long caller)

{

	return __do_kmalloc_node(size, gfpflags, node, caller);

}

EXPORT_SYMBOL(__kmalloc_node_track_caller);



#ifdef CONFIG_SYSFS

static int count_inuse(struct slab *slab)

{