Commit 75c8ff28 authored by Vlastimil Babka's avatar Vlastimil Babka
Browse files

mm, slub: return slab page from get_partial() and set c->page afterwards 



The function get_partial() finds a suitable page on a partial list, acquires
and returns its freelist and assigns the page pointer to kmem_cache_cpu.
In later patch we will need more control over the kmem_cache_cpu.page
assignment, so instead of passing a kmem_cache_cpu pointer, pass a pointer to a
pointer to a page that get_partial() can fill and the caller can assign the
kmem_cache_cpu.page pointer. No functional change as all of this still happens
with disabled IRQs.

Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent 53a0de06

mm/slub.c

+11 −10
Original line number Diff line number Diff line
@@ -2017,7 +2017,7 @@ static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags); 
 * Try to allocate a partial slab from a specific node.

 */

static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,

				struct kmem_cache_cpu *c, gfp_t flags)

			      struct page **ret_page, gfp_t flags)

{

	struct page *page, *page2;

	void *object = NULL;
@@ -2046,7 +2046,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, 


		available += objects;

		if (!object) {

			c->page = page;

			*ret_page = page;

			stat(s, ALLOC_FROM_PARTIAL);

			object = t;

		} else {
@@ -2066,7 +2066,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, 
 * Get a page from somewhere. Search in increasing NUMA distances.

 */

static void *get_any_partial(struct kmem_cache *s, gfp_t flags,

		struct kmem_cache_cpu *c)

			     struct page **ret_page)

{

#ifdef CONFIG_NUMA

	struct zonelist *zonelist;
@@ -2108,7 +2108,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, 


			if (n && cpuset_zone_allowed(zone, flags) &&

					n->nr_partial > s->min_partial) {

				object = get_partial_node(s, n, c, flags);

				object = get_partial_node(s, n, ret_page, flags);

				if (object) {

					/*

					 * Don't check read_mems_allowed_retry()
@@ -2130,7 +2130,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, 
 * Get a partial page, lock it and return it.

 */

static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,

		struct kmem_cache_cpu *c)

			 struct page **ret_page)

{

	void *object;

	int searchnode = node;
@@ -2138,11 +2138,11 @@ static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, 
	if (node == NUMA_NO_NODE)

		searchnode = numa_mem_id();



	object = get_partial_node(s, get_node(s, searchnode), c, flags);

	object = get_partial_node(s, get_node(s, searchnode), ret_page, flags);

	if (object || node != NUMA_NO_NODE)

		return object;



	return get_any_partial(s, flags, c);

	return get_any_partial(s, flags, ret_page);

}



#ifdef CONFIG_PREEMPTION
@@ -2754,9 +2754,11 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 
		goto redo;

	}



	freelist = get_partial(s, gfpflags, node, c);

	if (freelist)

	freelist = get_partial(s, gfpflags, node, &page);

	if (freelist) {

		c->page = page;

		goto check_new_page;

	}



	page = new_slab(s, gfpflags, node);
@@ -2780,7 +2782,6 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 
	c->page = page;



check_new_page:

	page = c->page;

	if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))

		goto load_freelist;