Commit 475733dd authored by Kirill Tkhai's avatar Kirill Tkhai Committed by Andrew Morton
Browse files

mm: vmscan: add shrinker_srcu_generation 

After we make slab shrink lockless with SRCU, the longest sleep
unregister_shrinker() will be a sleep waiting for all do_shrink_slab()
calls.

To avoid long unbreakable action in the unregister_shrinker(), add
shrinker_srcu_generation to restore a check similar to the
rwsem_is_contendent() check that we had before.

And for memcg slab shrink, we unlock SRCU and continue iterations from the
next shrinker id.

Link: https://lkml.kernel.org/r/20230313112819.38938-5-zhengqi.arch@bytedance.com


Signed-off-by: default avatarKirill Tkhai <tkhai@ya.ru>
Signed-off-by: default avatarQi Zheng <zhengqi.arch@bytedance.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarRoman Gushchin <roman.gushchin@linux.dev>
Cc: Christian König <christian.koenig@amd.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Paul E. McKenney <paulmck@kernel.org>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Sultan Alsawaf <sultan@kerneltoast.com>
Cc: Tetsuo Handa <penguin-kernel@i-love.sakura.ne.jp>
Cc: Yang Shi <shy828301@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent caa05325

mm/vmscan.c

+20 −4
Original line number Diff line number Diff line
@@ -204,6 +204,7 @@ static void set_task_reclaim_state(struct task_struct *task, 
LIST_HEAD(shrinker_list);

DECLARE_RWSEM(shrinker_rwsem);

DEFINE_SRCU(shrinker_srcu);

static atomic_t shrinker_srcu_generation = ATOMIC_INIT(0);



#ifdef CONFIG_MEMCG

static int shrinker_nr_max;
@@ -776,6 +777,7 @@ void unregister_shrinker(struct shrinker *shrinker) 
	debugfs_entry = shrinker_debugfs_remove(shrinker);

	up_write(&shrinker_rwsem);



	atomic_inc(&shrinker_srcu_generation);

	synchronize_srcu(&shrinker_srcu);



	debugfs_remove_recursive(debugfs_entry);
@@ -797,6 +799,7 @@ void synchronize_shrinkers(void) 
{

	down_write(&shrinker_rwsem);

	up_write(&shrinker_rwsem);

	atomic_inc(&shrinker_srcu_generation);

	synchronize_srcu(&shrinker_srcu);

}

EXPORT_SYMBOL(synchronize_shrinkers);
@@ -906,18 +909,20 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, 
{

	struct shrinker_info *info;

	unsigned long ret, freed = 0;

	int srcu_idx;

	int i;

	int srcu_idx, generation;

	int i = 0;



	if (!mem_cgroup_online(memcg))

		return 0;



again:

	srcu_idx = srcu_read_lock(&shrinker_srcu);

	info = shrinker_info_srcu(memcg, nid);

	if (unlikely(!info))

		goto unlock;



	for_each_set_bit(i, info->map, info->map_nr_max) {

	generation = atomic_read(&shrinker_srcu_generation);

	for_each_set_bit_from(i, info->map, info->map_nr_max) {

		struct shrink_control sc = {

			.gfp_mask = gfp_mask,

			.nid = nid,
@@ -963,6 +968,11 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, 
				set_shrinker_bit(memcg, nid, i);

		}

		freed += ret;

		if (atomic_read(&shrinker_srcu_generation) != generation) {

			srcu_read_unlock(&shrinker_srcu, srcu_idx);

			i++;

			goto again;

		}

	}

unlock:

	srcu_read_unlock(&shrinker_srcu, srcu_idx);
@@ -1002,7 +1012,7 @@ static unsigned long shrink_slab(gfp_t gfp_mask, int nid, 
{

	unsigned long ret, freed = 0;

	struct shrinker *shrinker;

	int srcu_idx;

	int srcu_idx, generation;



	/*

	 * The root memcg might be allocated even though memcg is disabled
@@ -1016,6 +1026,7 @@ static unsigned long shrink_slab(gfp_t gfp_mask, int nid, 


	srcu_idx = srcu_read_lock(&shrinker_srcu);



	generation = atomic_read(&shrinker_srcu_generation);

	list_for_each_entry_srcu(shrinker, &shrinker_list, list,

				 srcu_read_lock_held(&shrinker_srcu)) {

		struct shrink_control sc = {
@@ -1028,6 +1039,11 @@ static unsigned long shrink_slab(gfp_t gfp_mask, int nid, 
		if (ret == SHRINK_EMPTY)

			ret = 0;

		freed += ret;



		if (atomic_read(&shrinker_srcu_generation) != generation) {

			freed = freed ? : 1;

			break;

		}

	}



	srcu_read_unlock(&shrinker_srcu, srcu_idx);