Commit 40f3bf0c authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

mm: Convert struct page to struct slab in functions used by other subsystems 



KASAN, KFENCE and memcg interact with SLAB or SLUB internals through
functions nearest_obj(), obj_to_index() and objs_per_slab() that use
struct page as parameter. This patch converts it to struct slab
including all callers, through a coccinelle semantic patch.

// Options: --include-headers --no-includes --smpl-spacing include/linux/slab_def.h include/linux/slub_def.h mm/slab.h mm/kasan/*.c mm/kfence/kfence_test.c mm/memcontrol.c mm/slab.c mm/slub.c
// Note: needs coccinelle 1.1.1 to avoid breaking whitespace

@@
@@

-objs_per_slab_page(
+objs_per_slab(
 ...
 )
 { ... }

@@
@@

-objs_per_slab_page(
+objs_per_slab(
 ...
 )

@@
identifier fn =~ "obj_to_index|objs_per_slab";
@@

 fn(...,
-   const struct page *page
+   const struct slab *slab
    ,...)
 {
<...
(
- page_address(page)
+ slab_address(slab)
|
- page
+ slab
)
...>
 }

@@
identifier fn =~ "nearest_obj";
@@

 fn(...,
-   struct page *page
+   const struct slab *slab
    ,...)
 {
<...
(
- page_address(page)
+ slab_address(slab)
|
- page
+ slab
)
...>
 }

@@
identifier fn =~ "nearest_obj|obj_to_index|objs_per_slab";
expression E;
@@

 fn(...,
(
- slab_page(E)
+ E
|
- virt_to_page(E)
+ virt_to_slab(E)
|
- virt_to_head_page(E)
+ virt_to_slab(E)
|
- page
+ page_slab(page)
)
  ,...)

Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Reviewed-by: default avatarAndrey Konovalov <andreyknvl@gmail.com>
Reviewed-by: default avatarRoman Gushchin <guro@fb.com>
Acked-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Cc: Julia Lawall <julia.lawall@inria.fr>
Cc: Luis Chamberlain <mcgrof@kernel.org>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Marco Elver <elver@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: <kasan-dev@googlegroups.com>
Cc: <cgroups@vger.kernel.org>
parent dd35f71a

include/linux/slab_def.h

+8 −8
Original line number Diff line number Diff line
@@ -87,11 +87,11 @@ struct kmem_cache { 
	struct kmem_cache_node *node[MAX_NUMNODES];

};



static inline void *nearest_obj(struct kmem_cache *cache, struct page *page,

static inline void *nearest_obj(struct kmem_cache *cache, const struct slab *slab,

				void *x)

{

	void *object = x - (x - page->s_mem) % cache->size;

	void *last_object = page->s_mem + (cache->num - 1) * cache->size;

	void *object = x - (x - slab->s_mem) % cache->size;

	void *last_object = slab->s_mem + (cache->num - 1) * cache->size;



	if (unlikely(object > last_object))

		return last_object;
@@ -106,16 +106,16 @@ static inline void *nearest_obj(struct kmem_cache *cache, struct page *page, 
 *   reciprocal_divide(offset, cache->reciprocal_buffer_size)

 */

static inline unsigned int obj_to_index(const struct kmem_cache *cache,

					const struct page *page, void *obj)

					const struct slab *slab, void *obj)

{

	u32 offset = (obj - page->s_mem);

	u32 offset = (obj - slab->s_mem);

	return reciprocal_divide(offset, cache->reciprocal_buffer_size);

}



static inline int objs_per_slab_page(const struct kmem_cache *cache,

				     const struct page *page)

static inline int objs_per_slab(const struct kmem_cache *cache,

				     const struct slab *slab)

{

	if (is_kfence_address(page_address(page)))

	if (is_kfence_address(slab_address(slab)))

		return 1;

	return cache->num;

}

include/linux/slub_def.h

+9 −9
Original line number Diff line number Diff line
@@ -158,11 +158,11 @@ static inline void sysfs_slab_release(struct kmem_cache *s) 


void *fixup_red_left(struct kmem_cache *s, void *p);



static inline void *nearest_obj(struct kmem_cache *cache, struct page *page,

static inline void *nearest_obj(struct kmem_cache *cache, const struct slab *slab,

				void *x) {

	void *object = x - (x - page_address(page)) % cache->size;

	void *last_object = page_address(page) +

		(page->objects - 1) * cache->size;

	void *object = x - (x - slab_address(slab)) % cache->size;

	void *last_object = slab_address(slab) +

		(slab->objects - 1) * cache->size;

	void *result = (unlikely(object > last_object)) ? last_object : object;



	result = fixup_red_left(cache, result);
@@ -178,16 +178,16 @@ static inline unsigned int __obj_to_index(const struct kmem_cache *cache, 
}



static inline unsigned int obj_to_index(const struct kmem_cache *cache,

					const struct page *page, void *obj)

					const struct slab *slab, void *obj)

{

	if (is_kfence_address(obj))

		return 0;

	return __obj_to_index(cache, page_address(page), obj);

	return __obj_to_index(cache, slab_address(slab), obj);

}



static inline int objs_per_slab_page(const struct kmem_cache *cache,

				     const struct page *page)

static inline int objs_per_slab(const struct kmem_cache *cache,

				     const struct slab *slab)

{

	return page->objects;

	return slab->objects;

}

#endif /* _LINUX_SLUB_DEF_H */

mm/kasan/common.c

+2 −2
Original line number Diff line number Diff line
@@ -298,7 +298,7 @@ static inline u8 assign_tag(struct kmem_cache *cache, 
	/* For caches that either have a constructor or SLAB_TYPESAFE_BY_RCU: */

#ifdef CONFIG_SLAB

	/* For SLAB assign tags based on the object index in the freelist. */

	return (u8)obj_to_index(cache, virt_to_head_page(object), (void *)object);

	return (u8)obj_to_index(cache, virt_to_slab(object), (void *)object);

#else

	/*

	 * For SLUB assign a random tag during slab creation, otherwise reuse
@@ -341,7 +341,7 @@ static inline bool ____kasan_slab_free(struct kmem_cache *cache, void *object, 
	if (is_kfence_address(object))

		return false;



	if (unlikely(nearest_obj(cache, virt_to_head_page(object), object) !=

	if (unlikely(nearest_obj(cache, virt_to_slab(object), object) !=

	    object)) {

		kasan_report_invalid_free(tagged_object, ip);

		return true;

mm/kasan/generic.c

+1 −1
Original line number Diff line number Diff line
@@ -339,7 +339,7 @@ static void __kasan_record_aux_stack(void *addr, bool can_alloc) 
		return;



	cache = page->slab_cache;

	object = nearest_obj(cache, page, addr);

	object = nearest_obj(cache, page_slab(page), addr);

	alloc_meta = kasan_get_alloc_meta(cache, object);

	if (!alloc_meta)

		return;

mm/kasan/report.c

+1 −1
Original line number Diff line number Diff line
@@ -249,7 +249,7 @@ static void print_address_description(void *addr, u8 tag) 


	if (page && PageSlab(page)) {

		struct kmem_cache *cache = page->slab_cache;

		void *object = nearest_obj(cache, page,	addr);

		void *object = nearest_obj(cache, page_slab(page),	addr);



		describe_object(cache, object, addr, tag);

	}