Commit 69381c36 authored by ZhangPeng's avatar ZhangPeng Committed by Peng Zhang
Browse files

percpu_counter: introduce atomic mode for percpu_counter 

maillist inclusion
category: performance
bugzilla: https://gitee.com/openeuler/kernel/issues/I9IA1I
CVE: NA

Reference: https://lore.kernel.org/all/20240418142008.2775308-1-zhangpeng362@huawei.com/



--------------------------------

Depending on whether counters is NULL, we can support two modes:
atomic mode and perpcu mode. We implement both modes by grouping
the s64 count and atomic64_t count_atomic in a union. At the same time,
we create the interface for adding and reading in atomic mode and for
switching atomic mode to percpu mode.

Suggested-by: default avatarJan Kara <jack@suse.cz>
Signed-off-by: default avatarZhangPeng <zhangpeng362@huawei.com>
Signed-off-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
parent 0c8d7c88

include/linux/percpu_counter.h

+45 −3
Original line number Diff line number Diff line
@@ -21,7 +21,18 @@ 


struct percpu_counter {

	raw_spinlock_t lock;

	/*

	 * Depending on whether counters is NULL, we can support two modes,

	 * atomic mode using count_atomic and perpcu mode using count.

	 * The single-thread processes should use atomic mode to reduce the

	 * memory consumption and performance regression.

	 * The multiple-thread processes should use percpu mode to reduce the

	 * error margin.

	 */

	union {

		s64 count;

		atomic64_t count_atomic;

	};

#ifdef CONFIG_HOTPLUG_CPU

	struct list_head list;	/* All percpu_counters are on a list */

#endif
@@ -32,14 +43,14 @@ extern int percpu_counter_batch; 


int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,

			       gfp_t gfp, u32 nr_counters,

			       struct lock_class_key *key);

			       struct lock_class_key *key, bool switch_mode);



#define percpu_counter_init_many(fbc, value, gfp, nr_counters)		\

	({								\

		static struct lock_class_key __key;			\

									\

		__percpu_counter_init_many(fbc, value, gfp, nr_counters,\

					   &__key);			\

					   &__key, false);		\

	})
@@ -121,6 +132,20 @@ static inline bool percpu_counter_initialized(struct percpu_counter *fbc) 
	return (fbc->counters != NULL);

}



static inline s64 percpu_counter_atomic_read(struct percpu_counter *fbc)

{

	return atomic64_read(&fbc->count_atomic);

}



static inline void percpu_counter_atomic_add(struct percpu_counter *fbc,

					     s64 amount)

{

	atomic64_add(amount, &fbc->count_atomic);

}



int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc,

				       u32 nr_counters);



#else /* !CONFIG_SMP */



struct percpu_counter {
@@ -230,6 +255,23 @@ static inline bool percpu_counter_initialized(struct percpu_counter *fbc) 
static inline void percpu_counter_sync(struct percpu_counter *fbc)

{

}



static inline s64 percpu_counter_atomic_read(struct percpu_counter *fbc)

{

	return fbc->count;

}



static inline void percpu_counter_atomic_add(struct percpu_counter *fbc,

					     s64 amount)

{

	percpu_counter_add(fbc, amount);

}



static inline int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc,

						     u32 nr_counters)

{

	return 0;

}

#endif	/* CONFIG_SMP */



static inline void percpu_counter_inc(struct percpu_counter *fbc)

lib/percpu_counter.c

+33 −2
Original line number Diff line number Diff line
@@ -153,7 +153,7 @@ EXPORT_SYMBOL(__percpu_counter_sum); 


int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,

			       gfp_t gfp, u32 nr_counters,

			       struct lock_class_key *key)

			       struct lock_class_key *key, bool switch_mode)

{

	unsigned long flags __maybe_unused;

	size_t counter_size;
@@ -174,6 +174,7 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount, 
#ifdef CONFIG_HOTPLUG_CPU

		INIT_LIST_HEAD(&fbc[i].list);

#endif

		if (likely(!switch_mode))

			fbc[i].count = amount;

		fbc[i].counters = (void *)counters + (i * counter_size);
@@ -278,6 +279,36 @@ int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch) 
}

EXPORT_SYMBOL(__percpu_counter_compare);



/*

 * percpu_counter_switch_to_pcpu_many: Converts struct percpu_counters from

 * atomic mode to percpu mode.

 *

 * Return: 0 if percpu_counter is already in atomic mode or successfully

 * switched to atomic mode; -ENOMEM if perpcu memory allocation fails,

 * perpcu_counter is still in atomic mode.

 */

int percpu_counter_switch_to_pcpu_many(struct percpu_counter *fbc,

				       u32 nr_counters)

{

	static struct lock_class_key __key;

	unsigned long flags;

	bool ret = 0;



	if (percpu_counter_initialized(fbc))

		return 0;



	preempt_disable();

	local_irq_save(flags);

	if (likely(!percpu_counter_initialized(fbc)))

		ret = __percpu_counter_init_many(fbc, 0,

					GFP_ATOMIC|__GFP_NOWARN|__GFP_ZERO,

					nr_counters, &__key, true);

	local_irq_restore(flags);

	preempt_enable();



	return ret;

}



static int __init percpu_counter_startup(void)

{

	int ret;