Commit 44f6a42d authored by Jann Horn's avatar Jann Horn Committed by Vlastimil Babka
Browse files

mm/slub: refactor freelist to use custom type 



Currently the SLUB code represents encoded freelist entries as "void*".
That's misleading, those things are encoded under
CONFIG_SLAB_FREELIST_HARDENED so that they're not actually dereferencable.

Give them their own type, and split freelist_ptr() into one function per
direction (one for encoding, one for decoding).

Signed-off-by: default avatarJann Horn <jannh@google.com>
Co-developed-by: default avatarMatteo Rizzo <matteorizzo@google.com>
Signed-off-by: default avatarMatteo Rizzo <matteorizzo@google.com>
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent 06c2afb8

mm/slub.c

+32 −11
Original line number Diff line number Diff line
@@ -360,13 +360,19 @@ static struct workqueue_struct *flushwq; 
 * 			Core slab cache functions

 *******************************************************************/



/*

 * freeptr_t represents a SLUB freelist pointer, which might be encoded

 * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.

 */

typedef struct { unsigned long v; } freeptr_t;



/*

 * Returns freelist pointer (ptr). With hardening, this is obfuscated

 * with an XOR of the address where the pointer is held and a per-cache

 * random number.

 */

static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,

				 unsigned long ptr_addr)

static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,

					    void *ptr, unsigned long ptr_addr)

{

#ifdef CONFIG_SLAB_FREELIST_HARDENED

	/*
@@ -379,25 +385,40 @@ static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr, 
	 * calls get_freepointer() with an untagged pointer, which causes the

	 * freepointer to be restored incorrectly.

	 */

	return (void *)((unsigned long)ptr ^ s->random ^

	return (freeptr_t){.v = (unsigned long)ptr ^ s->random ^

			swab((unsigned long)kasan_reset_tag((void *)ptr_addr))};

#else

	return (freeptr_t){.v = (unsigned long)ptr};

#endif

}



static inline void *freelist_ptr_decode(const struct kmem_cache *s,

					freeptr_t ptr, unsigned long ptr_addr)

{

	void *decoded;



#ifdef CONFIG_SLAB_FREELIST_HARDENED

	/* See the comment in freelist_ptr_encode */

	decoded = (void *)(ptr.v ^ s->random ^

		swab((unsigned long)kasan_reset_tag((void *)ptr_addr)));

#else

	return ptr;

	decoded = (void *)ptr.v;

#endif

	return decoded;

}



/* Returns the freelist pointer recorded at location ptr_addr. */

static inline void *freelist_dereference(const struct kmem_cache *s,

					 void *ptr_addr)

{

	return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),

	return freelist_ptr_decode(s, *(freeptr_t *)(ptr_addr),

			    (unsigned long)ptr_addr);

}



static inline void *get_freepointer(struct kmem_cache *s, void *object)

{

	object = kasan_reset_tag(object);

	return freelist_dereference(s, object + s->offset);

	return freelist_dereference(s, (freeptr_t *)(object + s->offset));

}



#ifndef CONFIG_SLUB_TINY
@@ -421,15 +442,15 @@ __no_kmsan_checks 
static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)

{

	unsigned long freepointer_addr;

	void *p;

	freeptr_t p;



	if (!debug_pagealloc_enabled_static())

		return get_freepointer(s, object);



	object = kasan_reset_tag(object);

	freepointer_addr = (unsigned long)object + s->offset;

	copy_from_kernel_nofault(&p, (void **)freepointer_addr, sizeof(p));

	return freelist_ptr(s, p, freepointer_addr);

	copy_from_kernel_nofault(&p, (freeptr_t *)freepointer_addr, sizeof(p));

	return freelist_ptr_decode(s, p, freepointer_addr);

}



static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
@@ -441,7 +462,7 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) 
#endif



	freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);

	*(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);

	*(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr);

}



/* Loop over all objects in a slab */