Commit 2bfd3637 authored by Yang Shi's avatar Yang Shi Committed by Linus Torvalds
Browse files

mm: vmscan: consolidate shrinker_maps handling code 

The shrinker map management is not purely memcg specific, it is at the
intersection between memory cgroup and shrinkers.  It's allocation and
assignment of a structure, and the only memcg bit is the map is being
stored in a memcg structure.  So move the shrinker_maps handling code
into vmscan.c for tighter integration with shrinker code, and remove the
"memcg_" prefix.  There is no functional change.

Link: https://lkml.kernel.org/r/20210311190845.9708-3-shy828301@gmail.com


Signed-off-by: default avatarYang Shi <shy828301@gmail.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarKirill Tkhai <ktkhai@virtuozzo.com>
Acked-by: default avatarRoman Gushchin <guro@fb.com>
Reviewed-by: default avatarShakeel Butt <shakeelb@google.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 8efb4b59

include/linux/memcontrol.h

+5 −6
Original line number Diff line number Diff line
@@ -1610,10 +1610,9 @@ static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg) 
	return false;

}



extern int memcg_expand_shrinker_maps(int new_id);



extern void memcg_set_shrinker_bit(struct mem_cgroup *memcg,

				   int nid, int shrinker_id);

int alloc_shrinker_maps(struct mem_cgroup *memcg);

void free_shrinker_maps(struct mem_cgroup *memcg);

void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id);

#else

#define mem_cgroup_sockets_enabled 0

static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
@@ -1623,7 +1622,7 @@ static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg) 
	return false;

}



static inline void memcg_set_shrinker_bit(struct mem_cgroup *memcg,

static inline void set_shrinker_bit(struct mem_cgroup *memcg,

				    int nid, int shrinker_id)

{

}

mm/huge_memory.c

+2 −2
Original line number Diff line number Diff line
@@ -2830,7 +2830,7 @@ void deferred_split_huge_page(struct page *page) 
		ds_queue->split_queue_len++;

#ifdef CONFIG_MEMCG

		if (memcg)

			memcg_set_shrinker_bit(memcg, page_to_nid(page),

			set_shrinker_bit(memcg, page_to_nid(page),

					 deferred_split_shrinker.id);

#endif

	}

mm/list_lru.c

+3 −3
Original line number Diff line number Diff line
@@ -125,7 +125,7 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item) 
		list_add_tail(item, &l->list);

		/* Set shrinker bit if the first element was added */

		if (!l->nr_items++)

			memcg_set_shrinker_bit(memcg, nid,

			set_shrinker_bit(memcg, nid,

					 lru_shrinker_id(lru));

		nlru->nr_items++;

		spin_unlock(&nlru->lock);
@@ -540,7 +540,7 @@ static void memcg_drain_list_lru_node(struct list_lru *lru, int nid, 


	if (src->nr_items) {

		dst->nr_items += src->nr_items;

		memcg_set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru));

		set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru));

		src->nr_items = 0;

	}

mm/memcontrol.c

+3 −127
Original line number Diff line number Diff line
@@ -400,130 +400,6 @@ DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key); 
EXPORT_SYMBOL(memcg_kmem_enabled_key);

#endif



static int memcg_shrinker_map_size;

static DEFINE_MUTEX(memcg_shrinker_map_mutex);



static void memcg_free_shrinker_map_rcu(struct rcu_head *head)

{

	kvfree(container_of(head, struct memcg_shrinker_map, rcu));

}



static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,

					 int size, int old_size)

{

	struct memcg_shrinker_map *new, *old;

	struct mem_cgroup_per_node *pn;

	int nid;



	lockdep_assert_held(&memcg_shrinker_map_mutex);



	for_each_node(nid) {

		pn = memcg->nodeinfo[nid];

		old = rcu_dereference_protected(pn->shrinker_map, true);

		/* Not yet online memcg */

		if (!old)

			return 0;



		new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);

		if (!new)

			return -ENOMEM;



		/* Set all old bits, clear all new bits */

		memset(new->map, (int)0xff, old_size);

		memset((void *)new->map + old_size, 0, size - old_size);



		rcu_assign_pointer(pn->shrinker_map, new);

		call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);

	}



	return 0;

}



static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)

{

	struct mem_cgroup_per_node *pn;

	struct memcg_shrinker_map *map;

	int nid;



	if (mem_cgroup_is_root(memcg))

		return;



	for_each_node(nid) {

		pn = memcg->nodeinfo[nid];

		map = rcu_dereference_protected(pn->shrinker_map, true);

		kvfree(map);

		rcu_assign_pointer(pn->shrinker_map, NULL);

	}

}



static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)

{

	struct memcg_shrinker_map *map;

	int nid, size, ret = 0;



	if (mem_cgroup_is_root(memcg))

		return 0;



	mutex_lock(&memcg_shrinker_map_mutex);

	size = memcg_shrinker_map_size;

	for_each_node(nid) {

		map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid);

		if (!map) {

			memcg_free_shrinker_maps(memcg);

			ret = -ENOMEM;

			break;

		}

		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);

	}

	mutex_unlock(&memcg_shrinker_map_mutex);



	return ret;

}



int memcg_expand_shrinker_maps(int new_id)

{

	int size, old_size, ret = 0;

	struct mem_cgroup *memcg;



	size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);

	old_size = memcg_shrinker_map_size;

	if (size <= old_size)

		return 0;



	mutex_lock(&memcg_shrinker_map_mutex);

	if (!root_mem_cgroup)

		goto unlock;



	for_each_mem_cgroup(memcg) {

		if (mem_cgroup_is_root(memcg))

			continue;

		ret = memcg_expand_one_shrinker_map(memcg, size, old_size);

		if (ret) {

			mem_cgroup_iter_break(NULL, memcg);

			goto unlock;

		}

	}

unlock:

	if (!ret)

		memcg_shrinker_map_size = size;

	mutex_unlock(&memcg_shrinker_map_mutex);

	return ret;

}



void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)

{

	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {

		struct memcg_shrinker_map *map;



		rcu_read_lock();

		map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);

		/* Pairs with smp mb in shrink_slab() */

		smp_mb__before_atomic();

		set_bit(shrinker_id, map->map);

		rcu_read_unlock();

	}

}



/**

 * mem_cgroup_css_from_page - css of the memcg associated with a page

 * @page: page of interest
@@ -5242,11 +5118,11 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) 
	struct mem_cgroup *memcg = mem_cgroup_from_css(css);



	/*

	 * A memcg must be visible for memcg_expand_shrinker_maps()

	 * A memcg must be visible for expand_shrinker_maps()

	 * by the time the maps are allocated. So, we allocate maps

	 * here, when for_each_mem_cgroup() can't skip it.

	 */

	if (memcg_alloc_shrinker_maps(memcg)) {

	if (alloc_shrinker_maps(memcg)) {

		mem_cgroup_id_remove(memcg);

		return -ENOMEM;

	}
@@ -5310,7 +5186,7 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css) 
	vmpressure_cleanup(&memcg->vmpressure);

	cancel_work_sync(&memcg->high_work);

	mem_cgroup_remove_from_trees(memcg);

	memcg_free_shrinker_maps(memcg);

	free_shrinker_maps(memcg);

	memcg_free_kmem(memcg);

	mem_cgroup_free(memcg);

}

mm/vmscan.c

+129 −3
Original line number Diff line number Diff line
@@ -185,6 +185,132 @@ static LIST_HEAD(shrinker_list); 
static DECLARE_RWSEM(shrinker_rwsem);



#ifdef CONFIG_MEMCG



static int memcg_shrinker_map_size;

static DEFINE_MUTEX(memcg_shrinker_map_mutex);



static void free_shrinker_map_rcu(struct rcu_head *head)

{

	kvfree(container_of(head, struct memcg_shrinker_map, rcu));

}



static int expand_one_shrinker_map(struct mem_cgroup *memcg,

				   int size, int old_size)

{

	struct memcg_shrinker_map *new, *old;

	struct mem_cgroup_per_node *pn;

	int nid;



	lockdep_assert_held(&memcg_shrinker_map_mutex);



	for_each_node(nid) {

		pn = memcg->nodeinfo[nid];

		old = rcu_dereference_protected(pn->shrinker_map, true);

		/* Not yet online memcg */

		if (!old)

			return 0;



		new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);

		if (!new)

			return -ENOMEM;



		/* Set all old bits, clear all new bits */

		memset(new->map, (int)0xff, old_size);

		memset((void *)new->map + old_size, 0, size - old_size);



		rcu_assign_pointer(pn->shrinker_map, new);

		call_rcu(&old->rcu, free_shrinker_map_rcu);

	}



	return 0;

}



void free_shrinker_maps(struct mem_cgroup *memcg)

{

	struct mem_cgroup_per_node *pn;

	struct memcg_shrinker_map *map;

	int nid;



	if (mem_cgroup_is_root(memcg))

		return;



	for_each_node(nid) {

		pn = memcg->nodeinfo[nid];

		map = rcu_dereference_protected(pn->shrinker_map, true);

		kvfree(map);

		rcu_assign_pointer(pn->shrinker_map, NULL);

	}

}



int alloc_shrinker_maps(struct mem_cgroup *memcg)

{

	struct memcg_shrinker_map *map;

	int nid, size, ret = 0;



	if (mem_cgroup_is_root(memcg))

		return 0;



	mutex_lock(&memcg_shrinker_map_mutex);

	size = memcg_shrinker_map_size;

	for_each_node(nid) {

		map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid);

		if (!map) {

			free_shrinker_maps(memcg);

			ret = -ENOMEM;

			break;

		}

		rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);

	}

	mutex_unlock(&memcg_shrinker_map_mutex);



	return ret;

}



static int expand_shrinker_maps(int new_id)

{

	int size, old_size, ret = 0;

	struct mem_cgroup *memcg;



	size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);

	old_size = memcg_shrinker_map_size;

	if (size <= old_size)

		return 0;



	mutex_lock(&memcg_shrinker_map_mutex);

	if (!root_mem_cgroup)

		goto unlock;



	memcg = mem_cgroup_iter(NULL, NULL, NULL);

	do {

		if (mem_cgroup_is_root(memcg))

			continue;

		ret = expand_one_shrinker_map(memcg, size, old_size);

		if (ret) {

			mem_cgroup_iter_break(NULL, memcg);

			goto unlock;

		}

	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);

unlock:

	if (!ret)

		memcg_shrinker_map_size = size;

	mutex_unlock(&memcg_shrinker_map_mutex);

	return ret;

}



void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)

{

	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {

		struct memcg_shrinker_map *map;



		rcu_read_lock();

		map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);

		/* Pairs with smp mb in shrink_slab() */

		smp_mb__before_atomic();

		set_bit(shrinker_id, map->map);

		rcu_read_unlock();

	}

}



/*

 * We allow subsystems to populate their shrinker-related

 * LRU lists before register_shrinker_prepared() is called
@@ -212,7 +338,7 @@ static int prealloc_memcg_shrinker(struct shrinker *shrinker) 
		goto unlock;



	if (id >= shrinker_nr_max) {

		if (memcg_expand_shrinker_maps(id)) {

		if (expand_shrinker_maps(id)) {

			idr_remove(&shrinker_idr, id);

			goto unlock;

		}
@@ -590,7 +716,7 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, 
			 * case, we invoke the shrinker one more time and reset

			 * the bit if it reports that it is not empty anymore.

			 * The memory barrier here pairs with the barrier in

			 * memcg_set_shrinker_bit():

			 * set_shrinker_bit():

			 *

			 * list_lru_add()     shrink_slab_memcg()

			 *   list_add_tail()    clear_bit()
@@ -602,7 +728,7 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid, 
			if (ret == SHRINK_EMPTY)

				ret = 0;

			else

				memcg_set_shrinker_bit(memcg, nid, i);

				set_shrinker_bit(memcg, nid, i);

		}

		freed += ret;