Commit f1a79412 authored by Shakeel Butt's avatar Shakeel Butt Committed by Andrew Morton
Browse files

mm: convert mm's rss stats into percpu_counter 

Currently mm_struct maintains rss_stats which are updated on page fault
and the unmapping codepaths.  For page fault codepath the updates are
cached per thread with the batch of TASK_RSS_EVENTS_THRESH which is 64. 
The reason for caching is performance for multithreaded applications
otherwise the rss_stats updates may become hotspot for such applications.

However this optimization comes with the cost of error margin in the rss
stats.  The rss_stats for applications with large number of threads can be
very skewed.  At worst the error margin is (nr_threads * 64) and we have a
lot of applications with 100s of threads, so the error margin can be very
high.  Internally we had to reduce TASK_RSS_EVENTS_THRESH to 32.

Recently we started seeing the unbounded errors for rss_stats for specific
applications which use TCP rx0cp.  It seems like vm_insert_pages()
codepath does not sync rss_stats at all.

This patch converts the rss_stats into percpu_counter to convert the error
margin from (nr_threads * 64) to approximately (nr_cpus ^ 2).  However
this conversion enable us to get the accurate stats for situations where
accuracy is more important than the cpu cost.

This patch does not make such tradeoffs - we can just use
percpu_counter_add_local() for the updates and percpu_counter_sum() (or
percpu_counter_sync() + percpu_counter_read) for the readers.  At the
moment the readers are either procfs interface, oom_killer and memory
reclaim which I think are not performance critical and should be ok with
slow read.  However I think we can make that change in a separate patch.

Link: https://lkml.kernel.org/r/20221024052841.3291983-1-shakeelb@google.com


Signed-off-by: default avatarShakeel Butt <shakeelb@google.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 9cd6ffa6

include/linux/mm.h

+8 −18
Original line number Diff line number Diff line
@@ -2052,40 +2052,30 @@ static inline bool get_user_page_fast_only(unsigned long addr, 
 */

static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)

{

	long val = atomic_long_read(&mm->rss_stat.count[member]);



#ifdef SPLIT_RSS_COUNTING

	/*

	 * counter is updated in asynchronous manner and may go to minus.

	 * But it's never be expected number for users.

	 */

	if (val < 0)

		val = 0;

#endif

	return (unsigned long)val;

	return percpu_counter_read_positive(&mm->rss_stat[member]);

}



void mm_trace_rss_stat(struct mm_struct *mm, int member, long count);

void mm_trace_rss_stat(struct mm_struct *mm, int member);



static inline void add_mm_counter(struct mm_struct *mm, int member, long value)

{

	long count = atomic_long_add_return(value, &mm->rss_stat.count[member]);

	percpu_counter_add(&mm->rss_stat[member], value);



	mm_trace_rss_stat(mm, member, count);

	mm_trace_rss_stat(mm, member);

}



static inline void inc_mm_counter(struct mm_struct *mm, int member)

{

	long count = atomic_long_inc_return(&mm->rss_stat.count[member]);

	percpu_counter_inc(&mm->rss_stat[member]);



	mm_trace_rss_stat(mm, member, count);

	mm_trace_rss_stat(mm, member);

}



static inline void dec_mm_counter(struct mm_struct *mm, int member)

{

	long count = atomic_long_dec_return(&mm->rss_stat.count[member]);

	percpu_counter_dec(&mm->rss_stat[member]);



	mm_trace_rss_stat(mm, member, count);

	mm_trace_rss_stat(mm, member);

}



/* Optimized variant when page is already known not to be PageAnon */

include/linux/mm_types.h

+2 −5
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include <linux/page-flags-layout.h>

#include <linux/workqueue.h>

#include <linux/seqlock.h>

#include <linux/percpu_counter.h>



#include <asm/mmu.h>
@@ -626,11 +627,7 @@ struct mm_struct { 


		unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */



		/*

		 * Special counters, in some configurations protected by the

		 * page_table_lock, in other configurations by being atomic.

		 */

		struct mm_rss_stat rss_stat;

		struct percpu_counter rss_stat[NR_MM_COUNTERS];



		struct linux_binfmt *binfmt;

include/linux/mm_types_task.h

+0 −13
Original line number Diff line number Diff line
@@ -36,19 +36,6 @@ enum { 
	NR_MM_COUNTERS

};



#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU)

#define SPLIT_RSS_COUNTING

/* per-thread cached information, */

struct task_rss_stat {

	int events;	/* for synchronization threshold */

	int count[NR_MM_COUNTERS];

};

#endif /* USE_SPLIT_PTE_PTLOCKS */



struct mm_rss_stat {

	atomic_long_t count[NR_MM_COUNTERS];

};



struct page_frag {

	struct page *page;

#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)

include/linux/percpu_counter.h

+0 −1
Original line number Diff line number Diff line
@@ -13,7 +13,6 @@ 
#include <linux/threads.h>

#include <linux/percpu.h>

#include <linux/types.h>

#include <linux/gfp.h>



/* percpu_counter batch for local add or sub */

#define PERCPU_COUNTER_LOCAL_BATCH	INT_MAX

include/linux/sched.h

+0 −3
Original line number Diff line number Diff line
@@ -870,9 +870,6 @@ struct task_struct { 
	struct mm_struct		*mm;

	struct mm_struct		*active_mm;



#ifdef SPLIT_RSS_COUNTING

	struct task_rss_stat		rss_stat;

#endif

	int				exit_state;

	int				exit_code;

	int				exit_signal;