Commit 44b0c295 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files

u64_stats: Streamline the implementation 



The u64 stats code handles 3 different cases:

  - 32bit UP
  - 32bit SMP
  - 64bit

with an unreadable #ifdef maze, which was recently expanded with PREEMPT_RT
conditionals.

Reduce it to two cases (32bit and 64bit) and drop the optimization for
32bit UP as suggested by Linus.

Use the new preempt_disable/enable_nested() helpers to get rid of the
CONFIG_PREEMPT_RT conditionals.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarSebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220825164131.402717-9-bigeasy@linutronix.de
parent 9458e0a7

include/linux/u64_stats_sync.h

+64 −81
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ 
 *

 * Key points :

 *

 * -  Use a seqcount on 32-bit SMP, only disable preemption for 32-bit UP.

 * -  Use a seqcount on 32-bit

 * -  The whole thing is a no-op on 64-bit architectures.

 *

 * Usage constraints:
@@ -20,7 +20,8 @@ 
 *    writer and also spin forever.

 *

 * 3) Write side must use the _irqsave() variant if other writers, or a reader,

 *    can be invoked from an IRQ context.

 *    can be invoked from an IRQ context. On 64bit systems this variant does not

 *    disable interrupts.

 *

 * 4) If reader fetches several counters, there is no guarantee the whole values

 *    are consistent w.r.t. each other (remember point #2: seqcounts are not
@@ -29,11 +30,6 @@ 
 * 5) Readers are allowed to sleep or be preempted/interrupted: they perform

 *    pure reads.

 *

 * 6) Readers must use both u64_stats_fetch_{begin,retry}_irq() if the stats

 *    might be updated from a hardirq or softirq context (remember point #1:

 *    seqcounts are not used for UP kernels). 32-bit UP stat readers could read

 *    corrupted 64-bit values otherwise.

 *

 * Usage :

 *

 * Stats producer (writer) should use following template granted it already got
@@ -66,7 +62,7 @@ 
#include <linux/seqlock.h>



struct u64_stats_sync {

#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))

#if BITS_PER_LONG == 32

	seqcount_t	seq;

#endif

};
@@ -98,7 +94,22 @@ static inline void u64_stats_inc(u64_stats_t *p) 
	local64_inc(&p->v);

}



#else

static inline void u64_stats_init(struct u64_stats_sync *syncp) { }

static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp) { }

static inline void __u64_stats_update_end(struct u64_stats_sync *syncp) { }

static inline unsigned long __u64_stats_irqsave(void) { return 0; }

static inline void __u64_stats_irqrestore(unsigned long flags) { }

static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)

{

	return 0;

}

static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,

					   unsigned int start)

{

	return false;

}



#else /* 64 bit */



typedef struct {

	u64		v;
@@ -123,123 +134,95 @@ static inline void u64_stats_inc(u64_stats_t *p) 
{

	p->v++;

}

#endif



#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))

#define u64_stats_init(syncp)	seqcount_init(&(syncp)->seq)

#else

static inline void u64_stats_init(struct u64_stats_sync *syncp)

{

	seqcount_init(&syncp->seq);

}

#endif



static inline void u64_stats_update_begin(struct u64_stats_sync *syncp)

static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp)

{

#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))

	if (IS_ENABLED(CONFIG_PREEMPT_RT))

		preempt_disable();

	preempt_disable_nested();

	write_seqcount_begin(&syncp->seq);

#endif

}



static inline void u64_stats_update_end(struct u64_stats_sync *syncp)

static inline void __u64_stats_update_end(struct u64_stats_sync *syncp)

{

#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))

	write_seqcount_end(&syncp->seq);

	if (IS_ENABLED(CONFIG_PREEMPT_RT))

		preempt_enable();

#endif

	preempt_enable_nested();

}



static inline unsigned long

u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp)

static inline unsigned long __u64_stats_irqsave(void)

{

	unsigned long flags = 0;

	unsigned long flags;



#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))

	if (IS_ENABLED(CONFIG_PREEMPT_RT))

		preempt_disable();

	else

	local_irq_save(flags);

	write_seqcount_begin(&syncp->seq);

#endif

	return flags;

}



static inline void

u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,

				unsigned long flags)

static inline void __u64_stats_irqrestore(unsigned long flags)

{

#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))

	write_seqcount_end(&syncp->seq);

	if (IS_ENABLED(CONFIG_PREEMPT_RT))

		preempt_enable();

	else

	local_irq_restore(flags);

#endif

}



static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp)

{

#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))

	return read_seqcount_begin(&syncp->seq);

#else

	return 0;

#endif

}



static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)

{

#if BITS_PER_LONG == 32 && (!defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT))

	preempt_disable();

#endif

	return __u64_stats_fetch_begin(syncp);

}



static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp,

					   unsigned int start)

{

#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT))

	return read_seqcount_retry(&syncp->seq, start);

#else

	return false;

#endif

}

#endif /* !64 bit */



static inline void u64_stats_update_begin(struct u64_stats_sync *syncp)

{

	__u64_stats_update_begin(syncp);

}



static inline void u64_stats_update_end(struct u64_stats_sync *syncp)

{

	__u64_stats_update_end(syncp);

}



static inline unsigned long u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp)

{

	unsigned long flags = __u64_stats_irqsave();



	__u64_stats_update_begin(syncp);

	return flags;

}



static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp,

						   unsigned long flags)

{

	__u64_stats_update_end(syncp);

	__u64_stats_irqrestore(flags);

}



static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp)

{

	return __u64_stats_fetch_begin(syncp);

}



static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp,

					 unsigned int start)

{

#if BITS_PER_LONG == 32 && (!defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT))

	preempt_enable();

#endif

	return __u64_stats_fetch_retry(syncp, start);

}



/*

 * In case irq handlers can update u64 counters, readers can use following helpers

 * - SMP 32bit arches use seqcount protection, irq safe.

 * - UP 32bit must disable irqs.

 * - 64bit have no problem atomically reading u64 values, irq safe.

 */

/* Obsolete interfaces */

static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp)

{

#if BITS_PER_LONG == 32 && defined(CONFIG_PREEMPT_RT)

	preempt_disable();

#elif BITS_PER_LONG == 32 && !defined(CONFIG_SMP)

	local_irq_disable();

#endif

	return __u64_stats_fetch_begin(syncp);

	return u64_stats_fetch_begin(syncp);

}



static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp,

					     unsigned int start)

{

#if BITS_PER_LONG == 32 && defined(CONFIG_PREEMPT_RT)

	preempt_enable();

#elif BITS_PER_LONG == 32 && !defined(CONFIG_SMP)

	local_irq_enable();

#endif

	return __u64_stats_fetch_retry(syncp, start);

	return u64_stats_fetch_retry(syncp, start);

}



#endif /* _LINUX_U64_STATS_SYNC_H */