Merge branch 'slab/for-6.1/slub_validation_locking' into slab/for-next

 *   1. slab_mutex (Global Mutex)

 *   2. node->list_lock (Spinlock)

 *   3. kmem_cache->cpu_slab->lock (Local lock)

 *   4. slab_lock(slab) (Only on some arches or for debugging)

 *   4. slab_lock(slab) (Only on some arches)

 *   5. object_map_lock (Only for debugging)

 *

 *   slab_mutex

 *   The slab_lock is a wrapper around the page lock, thus it is a bit

 *   spinlock.

 *

 *   The slab_lock is only used for debugging and on arches that do not

 *   have the ability to do a cmpxchg_double. It only protects:

 *   The slab_lock is only used on arches that do not have the ability

 *   to do a cmpxchg_double. It only protects:

 *

 *	A. slab->freelist	-> List of free objects in a slab

 *	B. slab->inuse		-> Number of objects in use

 *	C. slab->objects	-> Number of objects in slab

 *   allocating a long series of objects that fill up slabs does not require

 *   the list lock.

 *

 *   For debug caches, all allocations are forced to go through a list_lock

 *   protected region to serialize against concurrent validation.

 *

 *   cpu_slab->lock local lock

 *

 *   This locks protect slowpath manipulation of all kmem_cache_cpu fields

 *   except the stat counters. This is a percpu structure manipulated only by

 *   the local cpu, so the lock protects against being preempted or interrupted

 *   by an irq. Fast path operations rely on lockless operations instead.

 *   On PREEMPT_RT, the local lock does not actually disable irqs (and thus

 *   prevent the lockless operations), so fastpath operations also need to take

 *   the lock and are no longer lockless.

 *

 *   On PREEMPT_RT, the local lock neither disables interrupts nor preemption

 *   which means the lockless fastpath cannot be used as it might interfere with

 *   an in-progress slow path operations. In this case the local lock is always

 *   taken but it still utilizes the freelist for the common operations.

 *

 *   lockless fastpaths

 *

#ifndef CONFIG_PREEMPT_RT

#define slub_get_cpu_ptr(var)		get_cpu_ptr(var)

#define slub_put_cpu_ptr(var)		put_cpu_ptr(var)

#define USE_LOCKLESS_FAST_PATH()	(true)

#else

#define slub_get_cpu_ptr(var)		\

({					\

	(void)(var);			\

	migrate_enable();		\

} while (0)

#define USE_LOCKLESS_FAST_PATH()	(false)

#endif

#ifdef CONFIG_SLUB_DEBUG

/*

 * Per slab locking using the pagelock

 */

static __always_inline void __slab_lock(struct slab *slab)

static __always_inline void slab_lock(struct slab *slab)

{

	struct page *page = slab_page(slab);

	bit_spin_lock(PG_locked, &page->flags);

}

static __always_inline void __slab_unlock(struct slab *slab)

static __always_inline void slab_unlock(struct slab *slab)

{

	struct page *page = slab_page(slab);

	__bit_spin_unlock(PG_locked, &page->flags);

}

static __always_inline void slab_lock(struct slab *slab, unsigned long *flags)

{

	if (IS_ENABLED(CONFIG_PREEMPT_RT))

		local_irq_save(*flags);

	__slab_lock(slab);

}

static __always_inline void slab_unlock(struct slab *slab, unsigned long *flags)

{

	__slab_unlock(slab);

	if (IS_ENABLED(CONFIG_PREEMPT_RT))

		local_irq_restore(*flags);

}

/*

 * Interrupts must be disabled (for the fallback code to work right), typically

 * by an _irqsave() lock variant. Except on PREEMPT_RT where locks are different

 * so we disable interrupts as part of slab_[un]lock().

 * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is

 * part of bit_spin_lock(), is sufficient because the policy is not to allow any

 * allocation/ free operation in hardirq context. Therefore nothing can

 * interrupt the operation.

 */

static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab,

		void *freelist_old, unsigned long counters_old,

		void *freelist_new, unsigned long counters_new,

		const char *n)

{

	if (!IS_ENABLED(CONFIG_PREEMPT_RT))

	if (USE_LOCKLESS_FAST_PATH())

		lockdep_assert_irqs_disabled();

#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \

    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)

	} else

#endif

	{

		/* init to 0 to prevent spurious warnings */

		unsigned long flags = 0;

		slab_lock(slab, &flags);

		slab_lock(slab);

		if (slab->freelist == freelist_old &&

					slab->counters == counters_old) {

			slab->freelist = freelist_new;

			slab->counters = counters_new;

			slab_unlock(slab, &flags);

			slab_unlock(slab);

			return true;

		}

		slab_unlock(slab, &flags);

		slab_unlock(slab);

	}

	cpu_relax();

		unsigned long flags;

		local_irq_save(flags);

		__slab_lock(slab);

		slab_lock(slab);

		if (slab->freelist == freelist_old &&

					slab->counters == counters_old) {

			slab->freelist = freelist_new;

			slab->counters = counters_new;

			__slab_unlock(slab);

			slab_unlock(slab);

			local_irq_restore(flags);

			return true;

		}

		__slab_unlock(slab);

		slab_unlock(slab);

		local_irq_restore(flags);

	}

#ifdef CONFIG_SLUB_DEBUG

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_RAW_SPINLOCK(object_map_lock);

static DEFINE_SPINLOCK(object_map_lock);

static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,

		       struct slab *slab)

static inline bool slab_add_kunit_errors(void) { return false; }

#endif

/*

 * Determine a map of objects in use in a slab.

 *

 * Node listlock must be held to guarantee that the slab does

 * not vanish from under us.

 */

static unsigned long *get_map(struct kmem_cache *s, struct slab *slab)

	__acquires(&object_map_lock)

{

	VM_BUG_ON(!irqs_disabled());

	raw_spin_lock(&object_map_lock);

	__fill_map(object_map, s, slab);

	return object_map;

}

static void put_map(unsigned long *map) __releases(&object_map_lock)

{

	VM_BUG_ON(map != object_map);

	raw_spin_unlock(&object_map_lock);

}

static inline unsigned int size_from_object(struct kmem_cache *s)

{

	if (s->flags & SLAB_RED_ZONE)

}

static noinline int alloc_debug_processing(struct kmem_cache *s,

					struct slab *slab,

					void *object, unsigned long addr)

					struct slab *slab, void *object)

{

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {

		if (!alloc_consistency_checks(s, slab, object))

			goto bad;

	}

	/* Success perform special debug activities for allocs */

	if (s->flags & SLAB_STORE_USER)

		set_track(s, object, TRACK_ALLOC, addr);

	/* Success. Perform special debug activities for allocs */

	trace(s, slab, object, 1);

	init_object(s, object, SLUB_RED_ACTIVE);

	return 1;

	return 1;

}

/* Supports checking bulk free of a constructed freelist */

static noinline int free_debug_processing(

	struct kmem_cache *s, struct slab *slab,

	void *head, void *tail, int bulk_cnt,

	unsigned long addr)

{

	struct kmem_cache_node *n = get_node(s, slab_nid(slab));

	void *object = head;

	int cnt = 0;

	unsigned long flags, flags2;

	int ret = 0;

	depot_stack_handle_t handle = 0;

	if (s->flags & SLAB_STORE_USER)

		handle = set_track_prepare();

	spin_lock_irqsave(&n->list_lock, flags);

	slab_lock(slab, &flags2);

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {

		if (!check_slab(s, slab))

			goto out;

	}

next_object:

	cnt++;

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {

		if (!free_consistency_checks(s, slab, object, addr))

			goto out;

	}

	if (s->flags & SLAB_STORE_USER)

		set_track_update(s, object, TRACK_FREE, addr, handle);

	trace(s, slab, object, 0);

	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */

	init_object(s, object, SLUB_RED_INACTIVE);

	/* Reached end of constructed freelist yet? */

	if (object != tail) {

		object = get_freepointer(s, object);

		goto next_object;

	}

	ret = 1;

out:

	if (cnt != bulk_cnt)

		slab_err(s, slab, "Bulk freelist count(%d) invalid(%d)\n",

			 bulk_cnt, cnt);

	slab_unlock(slab, &flags2);

	spin_unlock_irqrestore(&n->list_lock, flags);

	if (!ret)

		slab_fix(s, "Object at 0x%p not freed", object);

	return ret;

}

/*

 * Parse a block of slub_debug options. Blocks are delimited by ';'

 *

void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}

static inline int alloc_debug_processing(struct kmem_cache *s,

	struct slab *slab, void *object, unsigned long addr) { return 0; }

	struct slab *slab, void *object) { return 0; }

static inline int free_debug_processing(

static inline void free_debug_processing(

	struct kmem_cache *s, struct slab *slab,

	void *head, void *tail, int bulk_cnt,

	unsigned long addr) { return 0; }

	unsigned long addr) {}

static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}

static inline int check_object(struct kmem_cache *s, struct slab *slab,

			void *object, u8 val) { return 1; }

static inline void set_track(struct kmem_cache *s, void *object,

			     enum track_item alloc, unsigned long addr) {}

static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,

					struct slab *slab) {}

static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,

		 */

		slab = alloc_slab_page(alloc_gfp, node, oo);

		if (unlikely(!slab))

			goto out;

			return NULL;

		stat(s, ORDER_FALLBACK);

	}

	slab->objects = oo_objects(oo);

	slab->inuse = 0;

	slab->frozen = 0;

	account_slab(slab, oo_order(oo), s, flags);

		set_freepointer(s, p, NULL);

	}

	slab->inuse = slab->objects;

	slab->frozen = 1;

out:

	if (!slab)

		return NULL;

	inc_slabs_node(s, slab_nid(slab), slab->objects);

	return slab;

}

	n->nr_partial--;

}

/*

 * Called only for kmem_cache_debug() caches instead of acquire_slab(), with a

 * slab from the n->partial list. Remove only a single object from the slab, do

 * the alloc_debug_processing() checks and leave the slab on the list, or move

 * it to full list if it was the last free object.

 */

static void *alloc_single_from_partial(struct kmem_cache *s,

		struct kmem_cache_node *n, struct slab *slab)

{

	void *object;

	lockdep_assert_held(&n->list_lock);

	object = slab->freelist;

	slab->freelist = get_freepointer(s, object);

	slab->inuse++;

	if (!alloc_debug_processing(s, slab, object)) {

		remove_partial(n, slab);

		return NULL;

	}

	if (slab->inuse == slab->objects) {

		remove_partial(n, slab);

		add_full(s, n, slab);

	}

	return object;

}

/*

 * Called only for kmem_cache_debug() caches to allocate from a freshly

 * allocated slab. Allocate a single object instead of whole freelist

 * and put the slab to the partial (or full) list.

 */

static void *alloc_single_from_new_slab(struct kmem_cache *s,

					struct slab *slab)

{

	int nid = slab_nid(slab);

	struct kmem_cache_node *n = get_node(s, nid);

	unsigned long flags;

	void *object;

	object = slab->freelist;

	slab->freelist = get_freepointer(s, object);

	slab->inuse = 1;

	if (!alloc_debug_processing(s, slab, object))

		/*

		 * It's not really expected that this would fail on a

		 * freshly allocated slab, but a concurrent memory

		 * corruption in theory could cause that.

		 */

		return NULL;

	spin_lock_irqsave(&n->list_lock, flags);

	if (slab->inuse == slab->objects)

		add_full(s, n, slab);

	else

		add_partial(n, slab, DEACTIVATE_TO_HEAD);

	inc_slabs_node(s, nid, slab->objects);

	spin_unlock_irqrestore(&n->list_lock, flags);

	return object;

}

/*

 * Remove slab from the partial list, freeze it and

 * return the pointer to the freelist.

		if (!pfmemalloc_match(slab, gfpflags))

			continue;

		if (kmem_cache_debug(s)) {

			object = alloc_single_from_partial(s, n, slab);

			if (object)

				break;

			continue;

		}

		t = acquire_slab(s, n, slab, object == NULL);

		if (!t)

			break;

{

	return atomic_long_read(&n->total_objects);

}

/* Supports checking bulk free of a constructed freelist */

static noinline void free_debug_processing(

	struct kmem_cache *s, struct slab *slab,

	void *head, void *tail, int bulk_cnt,

	unsigned long addr)

{

	struct kmem_cache_node *n = get_node(s, slab_nid(slab));

	struct slab *slab_free = NULL;

	void *object = head;

	int cnt = 0;

	unsigned long flags;

	bool checks_ok = false;

	depot_stack_handle_t handle = 0;

	if (s->flags & SLAB_STORE_USER)

		handle = set_track_prepare();

	spin_lock_irqsave(&n->list_lock, flags);

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {

		if (!check_slab(s, slab))

			goto out;

	}

	if (slab->inuse < bulk_cnt) {

		slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",

			 slab->inuse, bulk_cnt);

		goto out;

	}

next_object:

	if (++cnt > bulk_cnt)

		goto out_cnt;

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {

		if (!free_consistency_checks(s, slab, object, addr))

			goto out;

	}

	if (s->flags & SLAB_STORE_USER)

		set_track_update(s, object, TRACK_FREE, addr, handle);

	trace(s, slab, object, 0);

	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */

	init_object(s, object, SLUB_RED_INACTIVE);

	/* Reached end of constructed freelist yet? */

	if (object != tail) {

		object = get_freepointer(s, object);

		goto next_object;

	}

	checks_ok = true;

out_cnt:

	if (cnt != bulk_cnt)

		slab_err(s, slab, "Bulk free expected %d objects but found %d\n",

			 bulk_cnt, cnt);

out:

	if (checks_ok) {

		void *prior = slab->freelist;

		/* Perform the actual freeing while we still hold the locks */

		slab->inuse -= cnt;

		set_freepointer(s, tail, prior);

		slab->freelist = head;

		/* Do we need to remove the slab from full or partial list? */

		if (!prior) {

			remove_full(s, n, slab);

		} else if (slab->inuse == 0 &&

			   n->nr_partial >= s->min_partial) {

			remove_partial(n, slab);

			stat(s, FREE_REMOVE_PARTIAL);

		}

		/* Do we need to discard the slab or add to partial list? */

		if (slab->inuse == 0 && n->nr_partial >= s->min_partial) {

			slab_free = slab;

		} else if (!prior) {

			add_partial(n, slab, DEACTIVATE_TO_TAIL);

			stat(s, FREE_ADD_PARTIAL);

		}

	}

	if (slab_free) {

		/*

		 * Update the counters while still holding n->list_lock to

		 * prevent spurious validation warnings

		 */

		dec_slabs_node(s, slab_nid(slab_free), slab_free->objects);

	}

	spin_unlock_irqrestore(&n->list_lock, flags);

	if (!checks_ok)

		slab_fix(s, "Object at 0x%p not freed", object);

	if (slab_free) {

		stat(s, FREE_SLAB);

		free_slab(s, slab_free);

	}

}

#endif /* CONFIG_SLUB_DEBUG */

#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)

		return NULL;

	}

	stat(s, ALLOC_SLAB);

	if (kmem_cache_debug(s)) {

		freelist = alloc_single_from_new_slab(s, slab);

		if (unlikely(!freelist))

			goto new_objects;

		if (s->flags & SLAB_STORE_USER)

			set_track(s, freelist, TRACK_ALLOC, addr);

		return freelist;

	}

	/*

	 * No other reference to the slab yet so we can

	 * muck around with it freely without cmpxchg

	 */

	freelist = slab->freelist;

	slab->freelist = NULL;

	slab->inuse = slab->objects;

	slab->frozen = 1;

	stat(s, ALLOC_SLAB);

	inc_slabs_node(s, slab_nid(slab), slab->objects);

check_new_slab:

	if (kmem_cache_debug(s)) {

		if (!alloc_debug_processing(s, slab, freelist, addr)) {

			/* Slab failed checks. Next slab needed */

			goto new_slab;

		} else {

		/*

			 * For debug case, we don't load freelist so that all

			 * allocations go through alloc_debug_processing()

		 * For debug caches here we had to go through

		 * alloc_single_from_partial() so just store the tracking info

		 * and return the object

		 */

			goto return_single;

		}

		if (s->flags & SLAB_STORE_USER)

			set_track(s, freelist, TRACK_ALLOC, addr);

		return freelist;

	}

	if (unlikely(!pfmemalloc_match(slab, gfpflags)))

	if (unlikely(!pfmemalloc_match(slab, gfpflags))) {

		/*

		 * For !pfmemalloc_match() case we don't load freelist so that

		 * we don't make further mismatched allocations easier.

		 */

		goto return_single;

		deactivate_slab(s, slab, get_freepointer(s, freelist));

		return freelist;

	}

retry_load_slab:

	c->slab = slab;

	goto load_freelist;

return_single:

	deactivate_slab(s, slab, get_freepointer(s, freelist));

	return freelist;

}

/*

	object = c->freelist;

	slab = c->slab;

	/*

	 * We cannot use the lockless fastpath on PREEMPT_RT because if a

	 * slowpath has taken the local_lock_irqsave(), it is not protected

	 * against a fast path operation in an irq handler. So we need to take

	 * the slow path which uses local_lock. It is still relatively fast if

	 * there is a suitable cpu freelist.

	 */

	if (IS_ENABLED(CONFIG_PREEMPT_RT) ||

	if (!USE_LOCKLESS_FAST_PATH() ||

	    unlikely(!object || !slab || !node_match(slab, node))) {

		object = __slab_alloc(s, gfpflags, node, addr, c);

	} else {

	if (kfence_free(head))

		return;

	if (kmem_cache_debug(s) &&

	    !free_debug_processing(s, slab, head, tail, cnt, addr))

	if (kmem_cache_debug(s)) {

		free_debug_processing(s, slab, head, tail, cnt, addr);

		return;

	}

	do {

		if (unlikely(n)) {

	void *tail_obj = tail ? : head;

	struct kmem_cache_cpu *c;

	unsigned long tid;

	void **freelist;

redo:

	/*

	/* Same with comment on barrier() in slab_alloc_node() */

	barrier();

	if (likely(slab == c->slab)) {

#ifndef CONFIG_PREEMPT_RT

		void **freelist = READ_ONCE(c->freelist);

	if (unlikely(slab != c->slab)) {

		__slab_free(s, slab, head, tail_obj, cnt, addr);

		return;

	}

	if (USE_LOCKLESS_FAST_PATH()) {

		freelist = READ_ONCE(c->freelist);

		set_freepointer(s, tail_obj, freelist);

			note_cmpxchg_failure("slab_free", s, tid);

			goto redo;

		}

#else /* CONFIG_PREEMPT_RT */

		/*

		 * We cannot use the lockless fastpath on PREEMPT_RT because if

		 * a slowpath has taken the local_lock_irqsave(), it is not

		 * protected against a fast path operation in an irq handler. So

		 * we need to take the local_lock. We shouldn't simply defer to

		 * __slab_free() as that wouldn't use the cpu freelist at all.

		 */

		void **freelist;

	} else {

		/* Update the free list under the local lock */

		local_lock(&s->cpu_slab->lock);