Commit 3662c13e authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

Merge branch 'slab/for-6.1/common_kmalloc' into slab/for-next 

The "common kmalloc v4" series [1] by Hyeonggon Yoo.

- Improves the mm/slab_common.c wrappers to allow deleting duplicated
  code between SLAB and SLUB.
- Large kmalloc() allocations in SLAB are passed to page allocator like
  in SLUB, reducing number of kmalloc caches.
- Removes the {kmem_cache_alloc,kmalloc}_node variants of tracepoints,
  node id parameter added to non-_node variants.
- 8 files changed, 341 insertions(+), 651 deletions(-)

[1] https://lore.kernel.org/all/20220817101826.236819-1-42.hyeyoo@gmail.com/

--
Merge resolves trivial conflict in mm/slub.c with commit 5373b8a0
("kasan: call kasan_malloc() from __kmalloc_*track_caller()")
parents 0467ca38 d5eff736

include/linux/slab.h

+50 −94
Original line number Diff line number Diff line
@@ -187,7 +187,6 @@ int kmem_cache_shrink(struct kmem_cache *s); 
void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __alloc_size(2);

void kfree(const void *objp);

void kfree_sensitive(const void *objp);

size_t __ksize(const void *objp);

size_t ksize(const void *objp);

#ifdef CONFIG_PRINTK

bool kmem_valid_obj(void *object);
@@ -243,27 +242,17 @@ static inline unsigned int arch_slab_minalign(void) 


#ifdef CONFIG_SLAB

/*

 * The largest kmalloc size supported by the SLAB allocators is

 * 32 megabyte (2^25) or the maximum allocatable page order if that is

 * less than 32 MB.

 *

 * WARNING: Its not easy to increase this value since the allocators have

 * to do various tricks to work around compiler limitations in order to

 * ensure proper constant folding.

 * SLAB and SLUB directly allocates requests fitting in to an order-1 page

 * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.

 */

#define KMALLOC_SHIFT_HIGH	((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \

				(MAX_ORDER + PAGE_SHIFT - 1) : 25)

#define KMALLOC_SHIFT_MAX	KMALLOC_SHIFT_HIGH

#define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)

#define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT - 1)

#ifndef KMALLOC_SHIFT_LOW

#define KMALLOC_SHIFT_LOW	5

#endif

#endif



#ifdef CONFIG_SLUB

/*

 * SLUB directly allocates requests fitting in to an order-1 page

 * (PAGE_SIZE*2).  Larger requests are passed to the page allocator.

 */

#define KMALLOC_SHIFT_HIGH	(PAGE_SHIFT + 1)

#define KMALLOC_SHIFT_MAX	(MAX_ORDER + PAGE_SHIFT - 1)

#ifndef KMALLOC_SHIFT_LOW
@@ -415,10 +404,6 @@ static __always_inline unsigned int __kmalloc_index(size_t size, 
	if (size <= 512 * 1024) return 19;

	if (size <= 1024 * 1024) return 20;

	if (size <=  2 * 1024 * 1024) return 21;

	if (size <=  4 * 1024 * 1024) return 22;

	if (size <=  8 * 1024 * 1024) return 23;

	if (size <=  16 * 1024 * 1024) return 24;

	if (size <=  32 * 1024 * 1024) return 25;



	if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)

		BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
@@ -428,6 +413,7 @@ static __always_inline unsigned int __kmalloc_index(size_t size, 
	/* Will never be reached. Needed because the compiler may complain */

	return -1;

}

static_assert(PAGE_SHIFT <= 20);

#define kmalloc_index(s) __kmalloc_index(s, true)

#endif /* !CONFIG_SLOB */
@@ -456,42 +442,22 @@ static __always_inline void kfree_bulk(size_t size, void **p) 
	kmem_cache_free_bulk(NULL, size, p);

}



#ifdef CONFIG_NUMA

void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment

							 __alloc_size(1);

void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment

									 __malloc;

#else

static __always_inline __alloc_size(1) void *__kmalloc_node(size_t size, gfp_t flags, int node)

{

	return __kmalloc(size, flags);

}



static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node)

{

	return kmem_cache_alloc(s, flags);

}

#endif



#ifdef CONFIG_TRACING

extern void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)

				   __assume_slab_alignment __alloc_size(3);

void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)

		    __assume_kmalloc_alignment __alloc_size(3);



#ifdef CONFIG_NUMA

extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,

					 int node, size_t size) __assume_slab_alignment

void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,

			 int node, size_t size) __assume_kmalloc_alignment

						__alloc_size(4);

#else

static __always_inline __alloc_size(4) void *kmem_cache_alloc_node_trace(struct kmem_cache *s,

						 gfp_t gfpflags, int node, size_t size)

{

	return kmem_cache_alloc_trace(s, gfpflags, size);

}

#endif /* CONFIG_NUMA */



#else /* CONFIG_TRACING */

static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_cache *s,

								    gfp_t flags, size_t size)

/* Save a function call when CONFIG_TRACING=n */

static __always_inline __alloc_size(3)

void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)

{

	void *ret = kmem_cache_alloc(s, flags);
@@ -499,7 +465,8 @@ static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_ 
	return ret;

}



static __always_inline void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,

static __always_inline __alloc_size(4)

void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,

			 int node, size_t size)

{

	void *ret = kmem_cache_alloc_node(s, gfpflags, node);
@@ -509,25 +476,11 @@ static __always_inline void *kmem_cache_alloc_node_trace(struct kmem_cache *s, g 
}

#endif /* CONFIG_TRACING */



extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment

void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment

					      __alloc_size(1);



#ifdef CONFIG_TRACING

extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)

				__assume_page_alignment __alloc_size(1);

#else

static __always_inline __alloc_size(1) void *kmalloc_order_trace(size_t size, gfp_t flags,

								 unsigned int order)

{

	return kmalloc_order(size, flags, order);

}

#endif



static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t flags)

{

	unsigned int order = get_order(size);

	return kmalloc_order_trace(size, flags, order);

}

void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment

							     __alloc_size(1);



/**

 * kmalloc - allocate memory
@@ -597,7 +550,7 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) 
		if (!index)

			return ZERO_SIZE_PTR;



		return kmem_cache_alloc_trace(

		return kmalloc_trace(

				kmalloc_caches[kmalloc_type(flags)][index],

				flags, size);

#endif
@@ -605,23 +558,35 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) 
	return __kmalloc(size, flags);

}



#ifndef CONFIG_SLOB

static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)

{

#ifndef CONFIG_SLOB

	if (__builtin_constant_p(size) &&

		size <= KMALLOC_MAX_CACHE_SIZE) {

		unsigned int i = kmalloc_index(size);

	if (__builtin_constant_p(size)) {

		unsigned int index;



		if (size > KMALLOC_MAX_CACHE_SIZE)

			return kmalloc_large_node(size, flags, node);



		index = kmalloc_index(size);



		if (!i)

		if (!index)

			return ZERO_SIZE_PTR;



		return kmem_cache_alloc_node_trace(

				kmalloc_caches[kmalloc_type(flags)][i],

		return kmalloc_node_trace(

				kmalloc_caches[kmalloc_type(flags)][index],

				flags, node, size);

	}

#endif

	return __kmalloc_node(size, flags, node);

}

#else

static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)

{

	if (__builtin_constant_p(size) && size > KMALLOC_MAX_CACHE_SIZE)

		return kmalloc_large_node(size, flags, node);



	return __kmalloc_node(size, flags, node);

}

#endif



/**

 * kmalloc_array - allocate memory for an array.
@@ -671,6 +636,12 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag 
	return kmalloc_array(n, size, flags | __GFP_ZERO);

}



void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,

				  unsigned long caller) __alloc_size(1);

#define kmalloc_node_track_caller(size, flags, node) \

	__kmalloc_node_track_caller(size, flags, node, \

				    _RET_IP_)



/*

 * kmalloc_track_caller is a special version of kmalloc that records the

 * calling function of the routine calling it for slab leak tracking instead
@@ -679,9 +650,9 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag 
 * allocator where we care about the real place the memory allocation

 * request comes from.

 */

extern void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller);

#define kmalloc_track_caller(size, flags) \

	__kmalloc_track_caller(size, flags, _RET_IP_)

	__kmalloc_node_track_caller(size, flags, \

				    NUMA_NO_NODE, _RET_IP_)



static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags,

							  int node)
@@ -700,21 +671,6 @@ static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t 
	return kmalloc_array_node(n, size, flags | __GFP_ZERO, node);

}





#ifdef CONFIG_NUMA

extern void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,

					 unsigned long caller) __alloc_size(1);

#define kmalloc_node_track_caller(size, flags, node) \

	__kmalloc_node_track_caller(size, flags, node, \

			_RET_IP_)



#else /* CONFIG_NUMA */



#define kmalloc_node_track_caller(size, flags, node) \

	kmalloc_track_caller(size, flags)



#endif /* CONFIG_NUMA */



/*

 * Shortcuts

 */

include/trace/events/kmem.h

+19 −55
Original line number Diff line number Diff line
@@ -9,16 +9,15 @@ 
#include <linux/tracepoint.h>

#include <trace/events/mmflags.h>



DECLARE_EVENT_CLASS(kmem_alloc,

TRACE_EVENT(kmem_cache_alloc,



	TP_PROTO(unsigned long call_site,

		 const void *ptr,

		 struct kmem_cache *s,

		 size_t bytes_req,

		 size_t bytes_alloc,

		 gfp_t gfp_flags),

		 gfp_t gfp_flags,

		 int node),



	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags),

	TP_ARGS(call_site, ptr, s, gfp_flags, node),



	TP_STRUCT__entry(

		__field(	unsigned long,	call_site	)
@@ -26,56 +25,42 @@ DECLARE_EVENT_CLASS(kmem_alloc, 
		__field(	size_t,		bytes_req	)

		__field(	size_t,		bytes_alloc	)

		__field(	unsigned long,	gfp_flags	)

		__field(	int,		node		)

		__field(	bool,		accounted	)

	),



	TP_fast_assign(

		__entry->call_site	= call_site;

		__entry->ptr		= ptr;

		__entry->bytes_req	= bytes_req;

		__entry->bytes_alloc	= bytes_alloc;

		__entry->bytes_req	= s->object_size;

		__entry->bytes_alloc	= s->size;

		__entry->gfp_flags	= (__force unsigned long)gfp_flags;

		__entry->node		= node;

		__entry->accounted	= IS_ENABLED(CONFIG_MEMCG_KMEM) ?

					  ((gfp_flags & __GFP_ACCOUNT) ||

					  (s && s->flags & SLAB_ACCOUNT)) : false;

					  (s->flags & SLAB_ACCOUNT)) : false;

	),



	TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s accounted=%s",

	TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d accounted=%s",

		(void *)__entry->call_site,

		__entry->ptr,

		__entry->bytes_req,

		__entry->bytes_alloc,

		show_gfp_flags(__entry->gfp_flags),

		__entry->node,

		__entry->accounted ? "true" : "false")

);



DEFINE_EVENT(kmem_alloc, kmalloc,



	TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s,

		 size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags),



	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags)

);



DEFINE_EVENT(kmem_alloc, kmem_cache_alloc,



	TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s,

		 size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags),



	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags)

);



DECLARE_EVENT_CLASS(kmem_alloc_node,

TRACE_EVENT(kmalloc,



	TP_PROTO(unsigned long call_site,

		 const void *ptr,

		 struct kmem_cache *s,

		 size_t bytes_req,

		 size_t bytes_alloc,

		 gfp_t gfp_flags,

		 int node),



	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node),

	TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node),



	TP_STRUCT__entry(

		__field(	unsigned long,	call_site	)
@@ -84,7 +69,6 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, 
		__field(	size_t,		bytes_alloc	)

		__field(	unsigned long,	gfp_flags	)

		__field(	int,		node		)

		__field(	bool,		accounted	)

	),



	TP_fast_assign(
@@ -94,9 +78,6 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, 
		__entry->bytes_alloc	= bytes_alloc;

		__entry->gfp_flags	= (__force unsigned long)gfp_flags;

		__entry->node		= node;

		__entry->accounted	= IS_ENABLED(CONFIG_MEMCG_KMEM) ?

					  ((gfp_flags & __GFP_ACCOUNT) ||

					  (s && s->flags & SLAB_ACCOUNT)) : false;

	),



	TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d accounted=%s",
@@ -106,25 +87,8 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, 
		__entry->bytes_alloc,

		show_gfp_flags(__entry->gfp_flags),

		__entry->node,

		__entry->accounted ? "true" : "false")

);



DEFINE_EVENT(kmem_alloc_node, kmalloc_node,



	TP_PROTO(unsigned long call_site, const void *ptr,

		 struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc,

		 gfp_t gfp_flags, int node),



	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node)

);



DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node,



	TP_PROTO(unsigned long call_site, const void *ptr,

		 struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc,

		 gfp_t gfp_flags, int node),



	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node)

		(IS_ENABLED(CONFIG_MEMCG_KMEM) &&

		 (__entry->gfp_flags & (__force unsigned long)__GFP_ACCOUNT)) ? "true" : "false")

);



TRACE_EVENT(kfree,
@@ -149,20 +113,20 @@ TRACE_EVENT(kfree, 


TRACE_EVENT(kmem_cache_free,



	TP_PROTO(unsigned long call_site, const void *ptr, const char *name),

	TP_PROTO(unsigned long call_site, const void *ptr, const struct kmem_cache *s),



	TP_ARGS(call_site, ptr, name),

	TP_ARGS(call_site, ptr, s),



	TP_STRUCT__entry(

		__field(	unsigned long,	call_site	)

		__field(	const void *,	ptr		)

		__string(	name,	name	)

		__string(	name,		s->name		)

	),



	TP_fast_assign(

		__entry->call_site	= call_site;

		__entry->ptr		= ptr;

		__assign_str(name, name);

		__assign_str(name, s->name);

	),



	TP_printk("call_site=%pS ptr=%p name=%s",

mm/kfence/report.c

+1 −0
Original line number Diff line number Diff line
@@ -86,6 +86,7 @@ static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries 
		/* Also the *_bulk() variants by only checking prefixes. */

		if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||

		    str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||

		    str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||

		    str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||

		    str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))

			goto found;

mm/slab.c

+70 −235
Original line number Diff line number Diff line
@@ -3181,84 +3181,46 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, 
}



static __always_inline void *

slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,

		   unsigned long caller)

__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags, int nodeid)

{

	unsigned long save_flags;

	void *ptr;

	void *objp = NULL;

	int slab_node = numa_mem_id();

	struct obj_cgroup *objcg = NULL;

	bool init = false;



	flags &= gfp_allowed_mask;

	cachep = slab_pre_alloc_hook(cachep, NULL, &objcg, 1, flags);

	if (unlikely(!cachep))

		return NULL;



	ptr = kfence_alloc(cachep, orig_size, flags);

	if (unlikely(ptr))

		goto out_hooks;



	local_irq_save(save_flags);



	if (nodeid == NUMA_NO_NODE)

		nodeid = slab_node;



	if (unlikely(!get_node(cachep, nodeid))) {

		/* Node not bootstrapped yet */

		ptr = fallback_alloc(cachep, flags);

	if (nodeid == NUMA_NO_NODE) {

		if (current->mempolicy || cpuset_do_slab_mem_spread()) {

			objp = alternate_node_alloc(cachep, flags);

			if (objp)

				goto out;

		}



	if (nodeid == slab_node) {

		/*

		 * Use the locally cached objects if possible.

		 * However ____cache_alloc does not allow fallback

		 * to other nodes. It may fail while we still have

		 * objects on other nodes available.

		 */

		ptr = ____cache_alloc(cachep, flags);

		if (ptr)

			goto out;

	}

	/* ___cache_alloc_node can fall back to other nodes */

	ptr = ____cache_alloc_node(cachep, flags, nodeid);

out:

	local_irq_restore(save_flags);

	ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);

	init = slab_want_init_on_alloc(flags, cachep);



out_hooks:

	slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr, init);

	return ptr;

}



static __always_inline void *

__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)

{

	void *objp;



	if (current->mempolicy || cpuset_do_slab_mem_spread()) {

		objp = alternate_node_alloc(cache, flags);

		if (objp)

		objp = ____cache_alloc(cachep, flags);

		nodeid = slab_node;

	} else if (nodeid == slab_node) {

		objp = ____cache_alloc(cachep, flags);

	} else if (!get_node(cachep, nodeid)) {

		/* Node not bootstrapped yet */

		objp = fallback_alloc(cachep, flags);

		goto out;

	}

	objp = ____cache_alloc(cache, flags);



	/*

	 * We may just have run out of memory on the local node.

	 * ____cache_alloc_node() knows how to locate memory on other nodes

	 */

	if (!objp)

		objp = ____cache_alloc_node(cache, flags, numa_mem_id());



		objp = ____cache_alloc_node(cachep, flags, nodeid);

out:

	return objp;

}

#else



static __always_inline void *

__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)

__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags, int nodeid __maybe_unused)

{

	return ____cache_alloc(cachep, flags);

}
@@ -3266,8 +3228,8 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 
#endif /* CONFIG_NUMA */



static __always_inline void *

slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags,

	   size_t orig_size, unsigned long caller)

slab_alloc_node(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags,

		int nodeid, size_t orig_size, unsigned long caller)

{

	unsigned long save_flags;

	void *objp;
@@ -3284,7 +3246,7 @@ slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags, 
		goto out;



	local_irq_save(save_flags);

	objp = __do_cache_alloc(cachep, flags);

	objp = __do_cache_alloc(cachep, flags, nodeid);

	local_irq_restore(save_flags);

	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);

	prefetchw(objp);
@@ -3295,6 +3257,14 @@ slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags, 
	return objp;

}



static __always_inline void *

slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags,

	   size_t orig_size, unsigned long caller)

{

	return slab_alloc_node(cachep, lru, flags, NUMA_NO_NODE, orig_size,

			       caller);

}



/*

 * Caller needs to acquire correct kmem_cache_node's list_lock

 * @list: List of detached free slabs should be freed by caller
@@ -3470,8 +3440,7 @@ void *__kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, 
{

	void *ret = slab_alloc(cachep, lru, flags, cachep->object_size, _RET_IP_);



	trace_kmem_cache_alloc(_RET_IP_, ret, cachep,

			       cachep->object_size, cachep->size, flags);

	trace_kmem_cache_alloc(_RET_IP_, ret, cachep, flags, NUMA_NO_NODE);



	return ret;

}
@@ -3521,7 +3490,8 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 


	local_irq_disable();

	for (i = 0; i < size; i++) {

		void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags);

		void *objp = kfence_alloc(s, s->object_size, flags) ?:

			     __do_cache_alloc(s, flags, NUMA_NO_NODE);



		if (unlikely(!objp))

			goto error;
@@ -3548,23 +3518,6 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 
}

EXPORT_SYMBOL(kmem_cache_alloc_bulk);



#ifdef CONFIG_TRACING

void *

kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)

{

	void *ret;



	ret = slab_alloc(cachep, NULL, flags, size, _RET_IP_);



	ret = kasan_kmalloc(cachep, ret, size, flags);

	trace_kmalloc(_RET_IP_, ret, cachep,

		      size, cachep->size, flags);

	return ret;

}

EXPORT_SYMBOL(kmem_cache_alloc_trace);

#endif



#ifdef CONFIG_NUMA

/**

 * kmem_cache_alloc_node - Allocate an object on the specified node

 * @cachep: The cache to allocate from.
@@ -3580,65 +3533,21 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace); 
 */

void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)

{

	void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_);

	void *ret = slab_alloc_node(cachep, NULL, flags, nodeid, cachep->object_size, _RET_IP_);



	trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep,

				    cachep->object_size, cachep->size,

				    flags, nodeid);

	trace_kmem_cache_alloc(_RET_IP_, ret, cachep, flags, nodeid);



	return ret;

}

EXPORT_SYMBOL(kmem_cache_alloc_node);



#ifdef CONFIG_TRACING

void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,

				  gfp_t flags,

				  int nodeid,

				  size_t size)

{

	void *ret;



	ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_);



	ret = kasan_kmalloc(cachep, ret, size, flags);

	trace_kmalloc_node(_RET_IP_, ret, cachep,

			   size, cachep->size,

			   flags, nodeid);

	return ret;

}

EXPORT_SYMBOL(kmem_cache_alloc_node_trace);

#endif



static __always_inline void *

__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)

{

	struct kmem_cache *cachep;

	void *ret;



	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))

		return NULL;

	cachep = kmalloc_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(cachep)))

		return cachep;

	ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);

	ret = kasan_kmalloc(cachep, ret, size, flags);



	return ret;

}



void *__kmalloc_node(size_t size, gfp_t flags, int node)

{

	return __do_kmalloc_node(size, flags, node, _RET_IP_);

}

EXPORT_SYMBOL(__kmalloc_node);



void *__kmalloc_node_track_caller(size_t size, gfp_t flags,

		int node, unsigned long caller)

void *__kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,

			     int nodeid, size_t orig_size,

			     unsigned long caller)

{

	return __do_kmalloc_node(size, flags, node, caller);

	return slab_alloc_node(cachep, NULL, flags, nodeid,

			       orig_size, caller);

}

EXPORT_SYMBOL(__kmalloc_node_track_caller);

#endif /* CONFIG_NUMA */



#ifdef CONFIG_PRINTK

void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
@@ -3662,45 +3571,25 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 
}

#endif



/**

 * __do_kmalloc - allocate memory

 * @size: how many bytes of memory are required.

 * @flags: the type of memory to allocate (see kmalloc).

 * @caller: function caller for debug tracking of the caller

 *

 * Return: pointer to the allocated memory or %NULL in case of error

 */

static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,

static __always_inline

void __do_kmem_cache_free(struct kmem_cache *cachep, void *objp,

			  unsigned long caller)

{

	struct kmem_cache *cachep;

	void *ret;



	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))

		return NULL;

	cachep = kmalloc_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(cachep)))

		return cachep;

	ret = slab_alloc(cachep, NULL, flags, size, caller);



	ret = kasan_kmalloc(cachep, ret, size, flags);

	trace_kmalloc(caller, ret, cachep,

		      size, cachep->size, flags);



	return ret;

}

	unsigned long flags;



void *__kmalloc(size_t size, gfp_t flags)

{

	return __do_kmalloc(size, flags, _RET_IP_);

	local_irq_save(flags);

	debug_check_no_locks_freed(objp, cachep->object_size);

	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))

		debug_check_no_obj_freed(objp, cachep->object_size);

	__cache_free(cachep, objp, caller);

	local_irq_restore(flags);

}

EXPORT_SYMBOL(__kmalloc);



void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)

void __kmem_cache_free(struct kmem_cache *cachep, void *objp,

		       unsigned long caller)

{

	return __do_kmalloc(size, flags, caller);

	__do_kmem_cache_free(cachep, objp, caller);

}

EXPORT_SYMBOL(__kmalloc_track_caller);



/**

 * kmem_cache_free - Deallocate an object
@@ -3712,34 +3601,38 @@ EXPORT_SYMBOL(__kmalloc_track_caller); 
 */

void kmem_cache_free(struct kmem_cache *cachep, void *objp)

{

	unsigned long flags;

	cachep = cache_from_obj(cachep, objp);

	if (!cachep)

		return;



	trace_kmem_cache_free(_RET_IP_, objp, cachep->name);

	local_irq_save(flags);

	debug_check_no_locks_freed(objp, cachep->object_size);

	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))

		debug_check_no_obj_freed(objp, cachep->object_size);

	__cache_free(cachep, objp, _RET_IP_);

	local_irq_restore(flags);

	trace_kmem_cache_free(_RET_IP_, objp, cachep);

	__do_kmem_cache_free(cachep, objp, _RET_IP_);

}

EXPORT_SYMBOL(kmem_cache_free);



void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)

{

	struct kmem_cache *s;

	size_t i;



	local_irq_disable();

	for (i = 0; i < size; i++) {

	for (int i = 0; i < size; i++) {

		void *objp = p[i];

		struct kmem_cache *s;



		if (!orig_s) /* called via kfree_bulk */

			s = virt_to_cache(objp);

		else

		if (!orig_s) {

			struct folio *folio = virt_to_folio(objp);



			/* called via kfree_bulk */

			if (!folio_test_slab(folio)) {

				local_irq_enable();

				free_large_kmalloc(folio, objp);

				local_irq_disable();

				continue;

			}

			s = folio_slab(folio)->slab_cache;

		} else {

			s = cache_from_obj(orig_s, objp);

		}



		if (!s)

			continue;
@@ -3755,39 +3648,6 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) 
}

EXPORT_SYMBOL(kmem_cache_free_bulk);



/**

 * kfree - free previously allocated memory

 * @objp: pointer returned by kmalloc.

 *

 * If @objp is NULL, no operation is performed.

 *

 * Don't free memory not originally allocated by kmalloc()

 * or you will run into trouble.

 */

void kfree(const void *objp)

{

	struct kmem_cache *c;

	unsigned long flags;



	trace_kfree(_RET_IP_, objp);



	if (unlikely(ZERO_OR_NULL_PTR(objp)))

		return;

	local_irq_save(flags);

	kfree_debugcheck(objp);

	c = virt_to_cache(objp);

	if (!c) {

		local_irq_restore(flags);

		return;

	}

	debug_check_no_locks_freed(objp, c->object_size);



	debug_check_no_obj_freed(objp, c->object_size);

	__cache_free(c, (void *)objp, _RET_IP_);

	local_irq_restore(flags);

}

EXPORT_SYMBOL(kfree);



/*

 * This initializes kmem_cache_node or resizes various caches for all nodes.

 */
@@ -4190,28 +4050,3 @@ void __check_heap_object(const void *ptr, unsigned long n, 
	usercopy_abort("SLAB object", cachep->name, to_user, offset, n);

}

#endif /* CONFIG_HARDENED_USERCOPY */
 


/**

 * __ksize -- Uninstrumented ksize.

 * @objp: pointer to the object

 *

 * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same

 * safety checks as ksize() with KASAN instrumentation enabled.

 *

 * Return: size of the actual memory used by @objp in bytes

 */

size_t __ksize(const void *objp)

{

	struct kmem_cache *c;

	size_t size;



	BUG_ON(!objp);

	if (unlikely(objp == ZERO_SIZE_PTR))

		return 0;



	c = virt_to_cache(objp);

	size = c ? c->object_size : 0;



	return size;

}

EXPORT_SYMBOL(__ksize);

mm/slab.h

+10 −0
Original line number Diff line number Diff line
@@ -273,6 +273,11 @@ void create_kmalloc_caches(slab_flags_t); 


/* Find the kmalloc slab corresponding for a certain size */

struct kmem_cache *kmalloc_slab(size_t, gfp_t);



void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags,

			      int node, size_t orig_size,

			      unsigned long caller);

void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller);

#endif



gfp_t kmalloc_fix_flags(gfp_t flags);
@@ -658,8 +663,13 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) 
		print_tracking(cachep, x);

	return cachep;

}



void free_large_kmalloc(struct folio *folio, void *object);



#endif /* CONFIG_SLOB */



size_t __ksize(const void *objp);



static inline size_t slab_ksize(const struct kmem_cache *s)

{

#ifndef CONFIG_SLUB