Commit 0dd1cabe authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'slab-for-5.20_or_6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab 

Pull slab updates from Vlastimil Babka:

 - An addition of 'accounted' flag to slab allocation tracepoints to
   indicate memcg_kmem accounting, by Vasily

 - An optimization of memcg handling in freeing paths, by Muchun

 - Various smaller fixes and cleanups

* tag 'slab-for-5.20_or_6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab:
  mm/slab_common: move generic bulk alloc/free functions to SLOB
  mm/sl[au]b: use own bulk free function when bulk alloc failed
  mm: slab: optimize memcg_slab_free_hook()
  mm/tracing: add 'accounted' entry into output of allocation tracepoints
  tools/vm/slabinfo: Handle files in debugfs
  mm/slub: Simplify __kmem_cache_alias()
  mm, slab: fix bad alignments
parents 0cec3f24 3041808b

include/trace/events/kmem.h

+26 −14
Original line number Diff line number Diff line
@@ -13,11 +13,12 @@ DECLARE_EVENT_CLASS(kmem_alloc, 


	TP_PROTO(unsigned long call_site,

		 const void *ptr,

		 struct kmem_cache *s,

		 size_t bytes_req,

		 size_t bytes_alloc,

		 gfp_t gfp_flags),



	TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags),

	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags),



	TP_STRUCT__entry(

		__field(	unsigned long,	call_site	)
@@ -25,6 +26,7 @@ DECLARE_EVENT_CLASS(kmem_alloc, 
		__field(	size_t,		bytes_req	)

		__field(	size_t,		bytes_alloc	)

		__field(	unsigned long,	gfp_flags	)

		__field(	bool,		accounted	)

	),



	TP_fast_assign(
@@ -33,42 +35,47 @@ DECLARE_EVENT_CLASS(kmem_alloc, 
		__entry->bytes_req	= bytes_req;

		__entry->bytes_alloc	= bytes_alloc;

		__entry->gfp_flags	= (__force unsigned long)gfp_flags;

		__entry->accounted	= IS_ENABLED(CONFIG_MEMCG_KMEM) ?

					  ((gfp_flags & __GFP_ACCOUNT) ||

					  (s && s->flags & SLAB_ACCOUNT)) : false;

	),



	TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s",

	TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s accounted=%s",

		(void *)__entry->call_site,

		__entry->ptr,

		__entry->bytes_req,

		__entry->bytes_alloc,

		show_gfp_flags(__entry->gfp_flags))

		show_gfp_flags(__entry->gfp_flags),

		__entry->accounted ? "true" : "false")

);



DEFINE_EVENT(kmem_alloc, kmalloc,



	TP_PROTO(unsigned long call_site, const void *ptr,

	TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s,

		 size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags),



	TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags)

	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags)

);



DEFINE_EVENT(kmem_alloc, kmem_cache_alloc,



	TP_PROTO(unsigned long call_site, const void *ptr,

	TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s,

		 size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags),



	TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags)

	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags)

);



DECLARE_EVENT_CLASS(kmem_alloc_node,



	TP_PROTO(unsigned long call_site,

		 const void *ptr,

		 struct kmem_cache *s,

		 size_t bytes_req,

		 size_t bytes_alloc,

		 gfp_t gfp_flags,

		 int node),



	TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node),

	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node),



	TP_STRUCT__entry(

		__field(	unsigned long,	call_site	)
@@ -77,6 +84,7 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, 
		__field(	size_t,		bytes_alloc	)

		__field(	unsigned long,	gfp_flags	)

		__field(	int,		node		)

		__field(	bool,		accounted	)

	),



	TP_fast_assign(
@@ -86,33 +94,37 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, 
		__entry->bytes_alloc	= bytes_alloc;

		__entry->gfp_flags	= (__force unsigned long)gfp_flags;

		__entry->node		= node;

		__entry->accounted	= IS_ENABLED(CONFIG_MEMCG_KMEM) ?

					  ((gfp_flags & __GFP_ACCOUNT) ||

					  (s && s->flags & SLAB_ACCOUNT)) : false;

	),



	TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d",

	TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d accounted=%s",

		(void *)__entry->call_site,

		__entry->ptr,

		__entry->bytes_req,

		__entry->bytes_alloc,

		show_gfp_flags(__entry->gfp_flags),

		__entry->node)

		__entry->node,

		__entry->accounted ? "true" : "false")

);



DEFINE_EVENT(kmem_alloc_node, kmalloc_node,



	TP_PROTO(unsigned long call_site, const void *ptr,

		 size_t bytes_req, size_t bytes_alloc,

		 struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc,

		 gfp_t gfp_flags, int node),



	TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node)

	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node)

);



DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node,



	TP_PROTO(unsigned long call_site, const void *ptr,

		 size_t bytes_req, size_t bytes_alloc,

		 struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc,

		 gfp_t gfp_flags, int node),



	TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node)

	TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node)

);



TRACE_EVENT(kfree,

mm/slab.c

+10 −10
Original line number Diff line number Diff line
@@ -3406,9 +3406,10 @@ static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp, 
{

	bool init;



	memcg_slab_free_hook(cachep, virt_to_slab(objp), &objp, 1);



	if (is_kfence_address(objp)) {

		kmemleak_free_recursive(objp, cachep->flags);

		memcg_slab_free_hook(cachep, &objp, 1);

		__kfence_free(objp);

		return;

	}
@@ -3441,7 +3442,6 @@ void ___cache_free(struct kmem_cache *cachep, void *objp, 
	check_irq_off();

	kmemleak_free_recursive(objp, cachep->flags);

	objp = cache_free_debugcheck(cachep, objp, caller);

	memcg_slab_free_hook(cachep, &objp, 1);



	/*

	 * Skip calling cache_free_alien() when the platform is not numa.
@@ -3478,7 +3478,7 @@ void *__kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, 
{

	void *ret = slab_alloc(cachep, lru, flags, cachep->object_size, _RET_IP_);



	trace_kmem_cache_alloc(_RET_IP_, ret,

	trace_kmem_cache_alloc(_RET_IP_, ret, cachep,

			       cachep->object_size, cachep->size, flags);



	return ret;
@@ -3553,7 +3553,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 
	local_irq_enable();

	cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);

	slab_post_alloc_hook(s, objcg, flags, i, p, false);

	__kmem_cache_free_bulk(s, i, p);

	kmem_cache_free_bulk(s, i, p);

	return 0;

}

EXPORT_SYMBOL(kmem_cache_alloc_bulk);
@@ -3567,7 +3567,7 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) 
	ret = slab_alloc(cachep, NULL, flags, size, _RET_IP_);



	ret = kasan_kmalloc(cachep, ret, size, flags);

	trace_kmalloc(_RET_IP_, ret,

	trace_kmalloc(_RET_IP_, ret, cachep,

		      size, cachep->size, flags);

	return ret;

}
@@ -3592,7 +3592,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) 
{

	void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_);



	trace_kmem_cache_alloc_node(_RET_IP_, ret,

	trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep,

				    cachep->object_size, cachep->size,

				    flags, nodeid);
@@ -3611,7 +3611,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, 
	ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_);



	ret = kasan_kmalloc(cachep, ret, size, flags);

	trace_kmalloc_node(_RET_IP_, ret,

	trace_kmalloc_node(_RET_IP_, ret, cachep,

			   size, cachep->size,

			   flags, nodeid);

	return ret;
@@ -3694,7 +3694,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, 
	ret = slab_alloc(cachep, NULL, flags, size, caller);



	ret = kasan_kmalloc(cachep, ret, size, flags);

	trace_kmalloc(caller, ret,

	trace_kmalloc(caller, ret, cachep,

		      size, cachep->size, flags);



	return ret;

mm/slab.h

+8 −31
Original line number Diff line number Diff line
@@ -380,15 +380,6 @@ void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s); 
ssize_t slabinfo_write(struct file *file, const char __user *buffer,

		       size_t count, loff_t *ppos);



/*

 * Generic implementation of bulk operations

 * These are useful for situations in which the allocator cannot

 * perform optimizations. In that case segments of the object listed

 * may be allocated or freed using these operations.

 */

void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);

int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);



static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)

{

	return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
@@ -547,36 +538,22 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, 
	obj_cgroup_put(objcg);

}



static inline void memcg_slab_free_hook(struct kmem_cache *s_orig,

static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,

					void **p, int objects)

{

	struct kmem_cache *s;

	struct obj_cgroup **objcgs;

	struct obj_cgroup *objcg;

	struct slab *slab;

	unsigned int off;

	int i;



	if (!memcg_kmem_enabled())

		return;



	for (i = 0; i < objects; i++) {

		if (unlikely(!p[i]))

			continue;



		slab = virt_to_slab(p[i]);

		/* we could be given a kmalloc_large() object, skip those */

		if (!slab)

			continue;



	objcgs = slab_objcgs(slab);

	if (!objcgs)

			continue;

		return;



		if (!s_orig)

			s = slab->slab_cache;

		else

			s = s_orig;

	for (i = 0; i < objects; i++) {

		struct obj_cgroup *objcg;

		unsigned int off;



		off = obj_to_index(s, slab, p[i]);

		objcg = objcgs[off];
@@ -628,7 +605,7 @@ static inline void memcg_slab_post_alloc_hook(struct kmem_cache *s, 
{

}



static inline void memcg_slab_free_hook(struct kmem_cache *s,

static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,

					void **p, int objects)

{

}

mm/slab_common.c

+4 −32
Original line number Diff line number Diff line
@@ -26,13 +26,12 @@ 
#include <linux/memcontrol.h>

#include <linux/stackdepot.h>



#define CREATE_TRACE_POINTS

#include <trace/events/kmem.h>



#include "internal.h"



#include "slab.h"



#define CREATE_TRACE_POINTS

#include <trace/events/kmem.h>



enum slab_state slab_state;

LIST_HEAD(slab_caches);

DEFINE_MUTEX(slab_mutex);
@@ -105,33 +104,6 @@ static inline int kmem_cache_sanity_check(const char *name, unsigned int size) 
}

#endif



void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p)

{

	size_t i;



	for (i = 0; i < nr; i++) {

		if (s)

			kmem_cache_free(s, p[i]);

		else

			kfree(p[i]);

	}

}



int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,

								void **p)

{

	size_t i;



	for (i = 0; i < nr; i++) {

		void *x = p[i] = kmem_cache_alloc(s, flags);

		if (!x) {

			__kmem_cache_free_bulk(s, i, p);

			return 0;

		}

	}

	return i;

}



/*

 * Figure out what the alignment of the objects will be given a set of

 * flags, a user specified alignment and the size of the objects.
@@ -959,7 +931,7 @@ EXPORT_SYMBOL(kmalloc_order); 
void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)

{

	void *ret = kmalloc_order(size, flags, order);

	trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);

	trace_kmalloc(_RET_IP_, ret, NULL, size, PAGE_SIZE << order, flags);

	return ret;

}

EXPORT_SYMBOL(kmalloc_order_trace);

mm/slob.c

+25 −8
Original line number Diff line number Diff line
@@ -507,7 +507,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) 
		*m = size;

		ret = (void *)m + minalign;



		trace_kmalloc_node(caller, ret,

		trace_kmalloc_node(caller, ret, NULL,

				   size, size + minalign, gfp, node);

	} else {

		unsigned int order = get_order(size);
@@ -516,7 +516,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) 
			gfp |= __GFP_COMP;

		ret = slob_new_pages(gfp, order, node);



		trace_kmalloc_node(caller, ret,

		trace_kmalloc_node(caller, ret, NULL,

				   size, PAGE_SIZE << order, gfp, node);

	}
@@ -616,12 +616,12 @@ static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node) 


	if (c->size < PAGE_SIZE) {

		b = slob_alloc(c->size, flags, c->align, node, 0);

		trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,

		trace_kmem_cache_alloc_node(_RET_IP_, b, NULL, c->object_size,

					    SLOB_UNITS(c->size) * SLOB_UNIT,

					    flags, node);

	} else {

		b = slob_new_pages(flags, get_order(c->size), node);

		trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,

		trace_kmem_cache_alloc_node(_RET_IP_, b, NULL, c->object_size,

					    PAGE_SIZE << get_order(c->size),

					    flags, node);

	}
@@ -692,16 +692,33 @@ void kmem_cache_free(struct kmem_cache *c, void *b) 
}

EXPORT_SYMBOL(kmem_cache_free);



void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)

void kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p)

{

	__kmem_cache_free_bulk(s, size, p);

	size_t i;



	for (i = 0; i < nr; i++) {

		if (s)

			kmem_cache_free(s, p[i]);

		else

			kfree(p[i]);

	}

}

EXPORT_SYMBOL(kmem_cache_free_bulk);



int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,

int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,

								void **p)

{

	return __kmem_cache_alloc_bulk(s, flags, size, p);

	size_t i;



	for (i = 0; i < nr; i++) {

		void *x = p[i] = kmem_cache_alloc(s, flags);



		if (!x) {

			kmem_cache_free_bulk(s, i, p);

			return 0;

		}

	}

	return i;

}

EXPORT_SYMBOL(kmem_cache_alloc_bulk);