mm/slub: Finish struct page to struct slab conversion

#ifdef CONFIG_SLUB_CPU_PARTIAL

	/* Number of per cpu partial objects to keep around */

	unsigned int cpu_partial;

	/* Number of per cpu partial pages to keep around */

	/* Number of per cpu partial slabs to keep around */

	unsigned int cpu_partial_slabs;

#endif

	struct kmem_cache_order_objects oo;

 *   1. slab_mutex (Global Mutex)

 *   2. node->list_lock (Spinlock)

 *   3. kmem_cache->cpu_slab->lock (Local lock)

 *   4. slab_lock(page) (Only on some arches or for debugging)

 *   4. slab_lock(slab) (Only on some arches or for debugging)

 *   5. object_map_lock (Only for debugging)

 *

 *   slab_mutex

 *

 *   The slab_lock is only used for debugging and on arches that do not

 *   have the ability to do a cmpxchg_double. It only protects:

 *	A. page->freelist	-> List of object free in a page

 *	B. page->inuse		-> Number of objects in use

 *	C. page->objects	-> Number of objects in page

 *	D. page->frozen		-> frozen state

 *	A. slab->freelist	-> List of free objects in a slab

 *	B. slab->inuse		-> Number of objects in use

 *	C. slab->objects	-> Number of objects in slab

 *	D. slab->frozen		-> frozen state

 *

 *   Frozen slabs

 *

 *   If a slab is frozen then it is exempt from list management. It is not

 *   on any list except per cpu partial list. The processor that froze the

 *   slab is the one who can perform list operations on the page. Other

 *   slab is the one who can perform list operations on the slab. Other

 *   processors may put objects onto the freelist but the processor that

 *   froze the slab is the only one that can retrieve the objects from the

 *   page's freelist.

 *   slab's freelist.

 *

 *   list_lock

 *

 * minimal so we rely on the page allocators per cpu caches for

 * fast frees and allocs.

 *

 * page->frozen		The slab is frozen and exempt from list processing.

 * slab->frozen		The slab is frozen and exempt from list processing.

 * 			This means that the slab is dedicated to a purpose

 * 			such as satisfying allocations for a specific

 * 			processor. Objects may be freed in the slab while

#define OO_SHIFT	16

#define OO_MASK		((1 << OO_SHIFT) - 1)

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

#define MAX_OBJS_PER_PAGE	32767 /* since slab.objects is u15 */

/* Internal SLUB flags */

/* Poison object */

	/*

	 * We take the number of objects but actually limit the number of

	 * pages on the per cpu partial list, in order to limit excessive

	 * growth of the list. For simplicity we assume that the pages will

	 * slabs on the per cpu partial list, in order to limit excessive

	 * growth of the list. For simplicity we assume that the slabs will

	 * be half-full.

	 */

	nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));

#endif

/*

 * Determine a map of object in use on a page.

 * Determine a map of objects in use in a slab.

 *

 * Node listlock must be held to guarantee that the page does

 * Node listlock must be held to guarantee that the slab does

 * not vanish from under us.

 */

static unsigned long *get_map(struct kmem_cache *s, struct slab *slab)

}

/*

 * Determine if a certain object on a page is on the freelist. Must hold the

 * Determine if a certain object in a slab is on the freelist. Must hold the

 * slab lock to guarantee that the chains are in a consistent state.

 */

static int on_freelist(struct kmem_cache *s, struct slab *slab, void *search)

}

/*

 * Get a page from somewhere. Search in increasing NUMA distances.

 * Get a slab from somewhere. Search in increasing NUMA distances.

 */

static void *get_any_partial(struct kmem_cache *s, gfp_t flags,

			     struct slab **ret_slab)

}

/*

 * Get a partial page, lock it and return it.

 * Get a partial slab, lock it and return it.

 */

static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,

			 struct slab **ret_slab)

}

/*

 * Finishes removing the cpu slab. Merges cpu's freelist with page's freelist,

 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,

 * unfreezes the slabs and puts it on the proper list.

 * Assumes the slab has been already safely taken away from kmem_cache_cpu

 * by the caller.

	}

	/*

	 * Stage two: Unfreeze the page while splicing the per-cpu

	 * freelist to the head of page's freelist.

	 * Stage two: Unfreeze the slab while splicing the per-cpu

	 * freelist to the head of slab's freelist.

	 *

	 * Ensure that the page is unfrozen while the list presence

	 * Ensure that the slab is unfrozen while the list presence

	 * reflects the actual number of objects during unfreeze.

	 *

	 * We setup the list membership and then perform a cmpxchg

	 * with the count. If there is a mismatch then the page

	 * is not unfrozen but the page is on the wrong list.

	 * with the count. If there is a mismatch then the slab

	 * is not unfrozen but the slab is on the wrong list.

	 *

	 * Then we restart the process which may have to remove

	 * the page from the list that we just put it on again

	 * the slab from the list that we just put it on again

	 * because the number of objects in the slab may have

	 * changed.

	 */

		if (!lock) {

			lock = 1;

			/*

			 * Taking the spinlock removes the possibility

			 * that acquire_slab() will see a slab page that

			 * is frozen

			 * Taking the spinlock removes the possibility that

			 * acquire_slab() will see a slab that is frozen

			 */

			spin_lock_irqsave(&n->list_lock, flags);

		}

}

/*

 * Put a page that was just frozen (in __slab_free|get_partial_node) into a

 * partial page slot if available.

 * Put a slab that was just frozen (in __slab_free|get_partial_node) into a

 * partial slab slot if available.

 *

 * If we did not find a slot then simply move all the partials to the

 * per node partial list.

}

/*

 * Check the page->freelist of a page and either transfer the freelist to the

 * per cpu freelist or deactivate the page.

 * Check the slab->freelist and either transfer the freelist to the

 * per cpu freelist or deactivate the slab.

 *

 * The page is still frozen if the return value is not NULL.

 * The slab is still frozen if the return value is not NULL.

 *

 * If this function returns NULL then the page has been unfrozen.

 * If this function returns NULL then the slab has been unfrozen.

 */

static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)

{

	stat(s, ALLOC_SLOWPATH);

reread_page:

reread_slab:

	slab = READ_ONCE(c->slab);

	if (!slab) {

	if (unlikely(!pfmemalloc_match(slab, gfpflags)))

		goto deactivate_slab;

	/* must check again c->page in case we got preempted and it changed */

	/* must check again c->slab in case we got preempted and it changed */

	local_lock_irqsave(&s->cpu_slab->lock, flags);

	if (unlikely(slab != c->slab)) {

		local_unlock_irqrestore(&s->cpu_slab->lock, flags);

		goto reread_page;

		goto reread_slab;

	}

	freelist = c->freelist;

	if (freelist)

	/*

	 * freelist is pointing to the list of objects to be used.

	 * page is pointing to the page from which the objects are obtained.

	 * That page must be frozen for per cpu allocations to work.

	 * slab is pointing to the slab from which the objects are obtained.

	 * That slab must be frozen for per cpu allocations to work.

	 */

	VM_BUG_ON(!c->slab->frozen);

	c->freelist = get_freepointer(s, freelist);

	local_lock_irqsave(&s->cpu_slab->lock, flags);

	if (slab != c->slab) {

		local_unlock_irqrestore(&s->cpu_slab->lock, flags);

		goto reread_page;

		goto reread_slab;

	}

	freelist = c->freelist;

	c->slab = NULL;

		local_lock_irqsave(&s->cpu_slab->lock, flags);

		if (unlikely(c->slab)) {

			local_unlock_irqrestore(&s->cpu_slab->lock, flags);

			goto reread_page;

			goto reread_slab;

		}

		if (unlikely(!slub_percpu_partial(c))) {

			local_unlock_irqrestore(&s->cpu_slab->lock, flags);

	freelist = get_partial(s, gfpflags, node, &slab);

	if (freelist)

		goto check_new_page;

		goto check_new_slab;

	slub_put_cpu_ptr(s->cpu_slab);

	slab = new_slab(s, gfpflags, node);

	}

	/*

	 * No other reference to the page yet so we can

	 * No other reference to the slab yet so we can

	 * muck around with it freely without cmpxchg

	 */

	freelist = slab->freelist;

	stat(s, ALLOC_SLAB);

check_new_page:

check_new_slab:

	if (kmem_cache_debug(s)) {

		if (!alloc_debug_processing(s, slab, freelist, addr)) {

		 */

		goto return_single;

retry_load_page:

retry_load_slab:

	local_lock_irqsave(&s->cpu_slab->lock, flags);

	if (unlikely(c->slab)) {

		stat(s, CPUSLAB_FLUSH);

		goto retry_load_page;

		goto retry_load_slab;

	}

	c->slab = slab;

	/*

	 * Irqless object alloc/free algorithm used here depends on sequence

	 * of fetching cpu_slab's data. tid should be fetched before anything

	 * on c to guarantee that object and page associated with previous tid

	 * on c to guarantee that object and slab associated with previous tid

	 * won't be used with current tid. If we fetch tid first, object and

	 * page could be one associated with next tid and our alloc/free

	 * slab could be one associated with next tid and our alloc/free

	 * request will be failed. In this case, we will retry. So, no problem.

	 */

	barrier();

 * have a longer lifetime than the cpu slabs in most processing loads.

 *

 * So we still attempt to reduce cache line usage. Just take the slab

 * lock and free the item. If there is no additional partial page

 * lock and free the item. If there is no additional partial slab

 * handling required then we can return immediately.

 */

static void __slab_free(struct kmem_cache *s, struct slab *slab,

			stat(s, FREE_FROZEN);

		} else if (new.frozen) {

			/*

			 * If we just froze the page then put it onto the

			 * If we just froze the slab then put it onto the

			 * per cpu partial list.

			 */

			put_cpu_partial(s, slab, 1);

 * with all sorts of special processing.

 *

 * Bulk free of a freelist with several objects (all pointing to the

 * same page) possible by specifying head and tail ptr, plus objects

 * same slab) possible by specifying head and tail ptr, plus objects

 * count (cnt). Bulk free indicated by tail pointer being set.

 */

static __always_inline void do_slab_free(struct kmem_cache *s,

#endif

	/*

	 * The larger the object size is, the more pages we want on the partial

	 * The larger the object size is, the more slabs we want on the partial

	 * list to avoid pounding the page allocator excessively.

	 */

	set_min_partial(s, ilog2(s->size) / 2);

		 * Build lists of slabs to discard or promote.

		 *

		 * Note that concurrent frees may occur while we hold the

		 * list_lock. page->inuse here is the upper limit.

		 * list_lock. slab->inuse here is the upper limit.

		 */

		list_for_each_entry_safe(slab, t, &n->partial, slab_list) {

			int free = slab->objects - slab->inuse;

			/* Do not reread page->inuse */

			/* Do not reread slab->inuse */

			barrier();

			/* We do not keep full slabs on the list */

			slabs += slab->slabs;

	}

	/* Approximate half-full pages , see slub_set_cpu_partial() */

	/* Approximate half-full slabs, see slub_set_cpu_partial() */

	objects = (slabs * oo_objects(s->oo)) / 2;

	len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);