Commit d835eef4 authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle) Committed by Vlastimil Babka
Browse files

mm/slub: Convert kfree() to use a struct slab 



Convert kfree(), kmem_cache_free() and ___cache_free() to resolve object
addresses to struct slab, using folio as intermediate step where needed.
Keep passing the result as struct page for now in preparation for mass
conversion of internal functions.

[ vbabka@suse.cz: Use folio as intermediate step when checking for
  large kmalloc pages, and when freeing them - rename
  free_nonslab_page() to free_large_kmalloc() that takes struct folio ]

Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Reviewed-by: default avatarRoman Gushchin <guro@fb.com>
parent cc465c3b

mm/slub.c

+16 −13
Original line number Diff line number Diff line
@@ -3517,7 +3517,7 @@ static __always_inline void slab_free(struct kmem_cache *s, struct page *page, 
#ifdef CONFIG_KASAN_GENERIC

void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)

{

	do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);

	do_slab_free(cache, slab_page(virt_to_slab(x)), x, NULL, 1, addr);

}

#endif
@@ -3527,7 +3527,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x) 
	if (!s)

		return;

	trace_kmem_cache_free(_RET_IP_, x, s->name);

	slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);

	slab_free(s, slab_page(virt_to_slab(x)), x, NULL, 1, _RET_IP_);

}

EXPORT_SYMBOL(kmem_cache_free);
@@ -3539,16 +3539,17 @@ struct detached_freelist { 
	struct kmem_cache *s;

};



static inline void free_nonslab_page(struct page *page, void *object)

static inline void free_large_kmalloc(struct folio *folio, void *object)

{

	unsigned int order = compound_order(page);

	unsigned int order = folio_order(folio);



	if (WARN_ON_ONCE(!PageCompound(page)))

	if (WARN_ON_ONCE(order == 0))

		pr_warn_once("object pointer: 0x%p\n", object);



	kfree_hook(object);

	mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, -(PAGE_SIZE << order));

	__free_pages(page, order);

	mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,

			      -(PAGE_SIZE << order));

	__free_pages(folio_page(folio, 0), order);

}



/*
@@ -3588,7 +3589,7 @@ int build_detached_freelist(struct kmem_cache *s, size_t size, 
	if (!s) {

		/* Handle kalloc'ed objects */

		if (unlikely(!folio_test_slab(folio))) {

			free_nonslab_page(folio_page(folio, 0), object);

			free_large_kmalloc(folio, object);

			p[size] = NULL; /* mark object processed */

			return size;

		}
@@ -4547,7 +4548,8 @@ EXPORT_SYMBOL(__ksize); 


void kfree(const void *x)

{

	struct page *page;

	struct folio *folio;

	struct slab *slab;

	void *object = (void *)x;



	trace_kfree(_RET_IP_, x);
@@ -4555,12 +4557,13 @@ void kfree(const void *x) 
	if (unlikely(ZERO_OR_NULL_PTR(x)))

		return;



	page = virt_to_head_page(x);

	if (unlikely(!PageSlab(page))) {

		free_nonslab_page(page, object);

	folio = virt_to_folio(x);

	if (unlikely(!folio_test_slab(folio))) {

		free_large_kmalloc(folio, object);

		return;

	}

	slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);

	slab = folio_slab(folio);

	slab_free(slab->slab_cache, slab_page(slab), object, NULL, 1, _RET_IP_);

}

EXPORT_SYMBOL(kfree);