Commit e267992f authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'for-5.14' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu 

Pull percpu updates from Dennis Zhou:

 - percpu chunk depopulation - depopulate backing pages for chunks with
   empty pages when we exceed a global threshold without those pages.
   This lets us reclaim a portion of memory that would previously be
   lost until the full chunk would be freed (possibly never).

 - memcg accounting cleanup - previously separate chunks were managed
   for normal allocations and __GFP_ACCOUNT allocations. These are now
   consolidated which cleans up the code quite a bit.

 - a few misc clean ups for clang warnings

* 'for-5.14' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu:
  percpu: optimize locking in pcpu_balance_workfn()
  percpu: initialize best_upa variable
  percpu: rework memcg accounting
  mm, memcg: introduce mem_cgroup_kmem_disabled()
  mm, memcg: mark cgroup_memory_nosocket, nokmem and noswap as __ro_after_init
  percpu: make symbol 'pcpu_free_slot' static
  percpu: implement partial chunk depopulation
  percpu: use pcpu_free_slot instead of pcpu_nr_slots - 1
  percpu: factor out pcpu_check_block_hint()
  percpu: split __pcpu_balance_workfn()
  percpu: fix a comment about the chunks ordering
parents 19b43859 e4d77700

include/linux/memcontrol.h

+5 −0
Original line number Diff line number Diff line
@@ -1619,6 +1619,7 @@ static inline void set_shrinker_bit(struct mem_cgroup *memcg, 
#endif



#ifdef CONFIG_MEMCG_KMEM

bool mem_cgroup_kmem_disabled(void);

int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order);

void __memcg_kmem_uncharge_page(struct page *page, int order);
@@ -1672,6 +1673,10 @@ static inline int memcg_cache_id(struct mem_cgroup *memcg) 
struct mem_cgroup *mem_cgroup_from_obj(void *p);



#else

static inline bool mem_cgroup_kmem_disabled(void)

{

	return true;

}



static inline int memcg_kmem_charge_page(struct page *page, gfp_t gfp,

					 int order)

mm/memcontrol.c

+8 −3
Original line number Diff line number Diff line
@@ -81,14 +81,14 @@ DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg); 
EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg);



/* Socket memory accounting disabled? */

static bool cgroup_memory_nosocket;

static bool cgroup_memory_nosocket __ro_after_init;



/* Kernel memory accounting disabled? */

bool cgroup_memory_nokmem;

bool cgroup_memory_nokmem __ro_after_init;



/* Whether the swap controller is active */

#ifdef CONFIG_MEMCG_SWAP

bool cgroup_memory_noswap __read_mostly;

bool cgroup_memory_noswap __ro_after_init;

#else

#define cgroup_memory_noswap		1

#endif
@@ -256,6 +256,11 @@ struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr) 
#ifdef CONFIG_MEMCG_KMEM

extern spinlock_t css_set_lock;



bool mem_cgroup_kmem_disabled(void)

{

	return cgroup_memory_nokmem;

}



static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,

				      unsigned int nr_pages);

mm/percpu-internal.h

+5 −51
Original line number Diff line number Diff line
@@ -5,25 +5,6 @@ 
#include <linux/types.h>

#include <linux/percpu.h>



/*

 * There are two chunk types: root and memcg-aware.

 * Chunks of each type have separate slots list.

 *

 * Memcg-aware chunks have an attached vector of obj_cgroup pointers, which is

 * used to store memcg membership data of a percpu object.  Obj_cgroups are

 * ref-counted pointers to a memory cgroup with an ability to switch dynamically

 * to the parent memory cgroup.  This allows to reclaim a deleted memory cgroup

 * without reclaiming of all outstanding objects, which hold a reference at it.

 */

enum pcpu_chunk_type {

	PCPU_CHUNK_ROOT,

#ifdef CONFIG_MEMCG_KMEM

	PCPU_CHUNK_MEMCG,

#endif

	PCPU_NR_CHUNK_TYPES,

	PCPU_FAIL_ALLOC = PCPU_NR_CHUNK_TYPES

};



/*

 * pcpu_block_md is the metadata block struct.

 * Each chunk's bitmap is split into a number of full blocks.
@@ -67,6 +48,8 @@ struct pcpu_chunk { 


	void			*data;		/* chunk data */

	bool			immutable;	/* no [de]population allowed */

	bool			isolated;	/* isolated from active chunk

						   slots */

	int			start_offset;	/* the overlap with the previous

						   region to have a page aligned

						   base_addr */
@@ -87,7 +70,9 @@ extern spinlock_t pcpu_lock; 


extern struct list_head *pcpu_chunk_lists;

extern int pcpu_nr_slots;

extern int pcpu_nr_empty_pop_pages[];

extern int pcpu_sidelined_slot;

extern int pcpu_to_depopulate_slot;

extern int pcpu_nr_empty_pop_pages;



extern struct pcpu_chunk *pcpu_first_chunk;

extern struct pcpu_chunk *pcpu_reserved_chunk;
@@ -128,37 +113,6 @@ static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 
	return pcpu_nr_pages_to_map_bits(chunk->nr_pages);

}



#ifdef CONFIG_MEMCG_KMEM

static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)

{

	if (chunk->obj_cgroups)

		return PCPU_CHUNK_MEMCG;

	return PCPU_CHUNK_ROOT;

}



static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)

{

	return chunk_type == PCPU_CHUNK_MEMCG;

}



#else

static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)

{

	return PCPU_CHUNK_ROOT;

}



static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)

{

	return false;

}

#endif



static inline struct list_head *pcpu_chunk_list(enum pcpu_chunk_type chunk_type)

{

	return &pcpu_chunk_lists[pcpu_nr_slots *

				 pcpu_is_memcg_chunk(chunk_type)];

}



#ifdef CONFIG_PERCPU_STATS



#include <linux/spinlock.h>

mm/percpu-km.c

+7 −3
Original line number Diff line number Diff line
@@ -44,8 +44,7 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, 
	/* nada */

}



static struct pcpu_chunk *pcpu_create_chunk(enum pcpu_chunk_type type,

					    gfp_t gfp)

static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)

{

	const int nr_pages = pcpu_group_sizes[0] >> PAGE_SHIFT;

	struct pcpu_chunk *chunk;
@@ -53,7 +52,7 @@ static struct pcpu_chunk *pcpu_create_chunk(enum pcpu_chunk_type type, 
	unsigned long flags;

	int i;



	chunk = pcpu_alloc_chunk(type, gfp);

	chunk = pcpu_alloc_chunk(gfp);

	if (!chunk)

		return NULL;
@@ -118,3 +117,8 @@ static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai) 


	return 0;

}



static bool pcpu_should_reclaim_chunk(struct pcpu_chunk *chunk)

{

	return false;

}

mm/percpu-stats.c

+15 −29
Original line number Diff line number Diff line
@@ -34,15 +34,11 @@ static int find_max_nr_alloc(void) 
{

	struct pcpu_chunk *chunk;

	int slot, max_nr_alloc;

	enum pcpu_chunk_type type;



	max_nr_alloc = 0;

	for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++)

	for (slot = 0; slot < pcpu_nr_slots; slot++)

			list_for_each_entry(chunk, &pcpu_chunk_list(type)[slot],

					    list)

				max_nr_alloc = max(max_nr_alloc,

						   chunk->nr_alloc);

		list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list)

			max_nr_alloc = max(max_nr_alloc, chunk->nr_alloc);



	return max_nr_alloc;

}
@@ -133,9 +129,6 @@ static void chunk_map_stats(struct seq_file *m, struct pcpu_chunk *chunk, 
	P("cur_min_alloc", cur_min_alloc);

	P("cur_med_alloc", cur_med_alloc);

	P("cur_max_alloc", cur_max_alloc);

#ifdef CONFIG_MEMCG_KMEM

	P("memcg_aware", pcpu_is_memcg_chunk(pcpu_chunk_type(chunk)));

#endif

	seq_putc(m, '\n');

}
@@ -144,8 +137,6 @@ static int percpu_stats_show(struct seq_file *m, void *v) 
	struct pcpu_chunk *chunk;

	int slot, max_nr_alloc;

	int *buffer;

	enum pcpu_chunk_type type;

	int nr_empty_pop_pages;



alloc_buffer:

	spin_lock_irq(&pcpu_lock);
@@ -166,10 +157,6 @@ static int percpu_stats_show(struct seq_file *m, void *v) 
		goto alloc_buffer;

	}



	nr_empty_pop_pages = 0;

	for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++)

		nr_empty_pop_pages += pcpu_nr_empty_pop_pages[type];



#define PL(X)								\

	seq_printf(m, "  %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X)
@@ -201,7 +188,7 @@ static int percpu_stats_show(struct seq_file *m, void *v) 
	PU(nr_max_chunks);

	PU(min_alloc_size);

	PU(max_alloc_size);

	P("empty_pop_pages", nr_empty_pop_pages);

	P("empty_pop_pages", pcpu_nr_empty_pop_pages);

	seq_putc(m, '\n');



#undef PU
@@ -215,20 +202,19 @@ static int percpu_stats_show(struct seq_file *m, void *v) 
		chunk_map_stats(m, pcpu_reserved_chunk, buffer);

	}



	for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++) {

	for (slot = 0; slot < pcpu_nr_slots; slot++) {

			list_for_each_entry(chunk, &pcpu_chunk_list(type)[slot],

					    list) {

				if (chunk == pcpu_first_chunk) {

		list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list) {

			if (chunk == pcpu_first_chunk)

				seq_puts(m, "Chunk: <- First Chunk\n");

					chunk_map_stats(m, chunk, buffer);

				} else {

			else if (slot == pcpu_to_depopulate_slot)

				seq_puts(m, "Chunk (to_depopulate)\n");

			else if (slot == pcpu_sidelined_slot)

				seq_puts(m, "Chunk (sidelined):\n");

			else

				seq_puts(m, "Chunk:\n");

			chunk_map_stats(m, chunk, buffer);

		}

	}

		}

	}



	spin_unlock_irq(&pcpu_lock);