Commit 0760fa3d authored by Roman Gushchin's avatar Roman Gushchin Committed by Dennis Zhou
Browse files

percpu: make pcpu_nr_empty_pop_pages per chunk type 

nr_empty_pop_pages is used to guarantee that there are some free
populated pages to satisfy atomic allocations. Accounted and
non-accounted allocations are using separate sets of chunks,
so both need to have a surplus of empty pages.

This commit makes pcpu_nr_empty_pop_pages and the corresponding logic
per chunk type.

[Dennis]
This issue came up as I was reviewing [1] and realized I missed this.
Simultaneously, it was reported btrfs was seeing failed atomic
allocations in fsstress tests [2] and [3].

[1] https://lore.kernel.org/linux-mm/20210324190626.564297-1-guro@fb.com/
[2] https://lore.kernel.org/linux-mm/20210401185158.3275.409509F4@e16-tech.com/
[3] https://lore.kernel.org/linux-mm/CAL3q7H5RNBjCi708GH7jnczAOe0BLnacT9C+OBgA-Dx9jhB6SQ@mail.gmail.com/



Fixes: 3c7be18a ("mm: memcg/percpu: account percpu memory to memory cgroups")
Cc: stable@vger.kernel.org # 5.9+
Signed-off-by: default avatarRoman Gushchin <guro@fb.com>
Tested-by: default avatarFilipe Manana <fdmanana@suse.com>
Signed-off-by: default avatarDennis Zhou <dennis@kernel.org>
parent e49d033b

mm/percpu-internal.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -87,7 +87,7 @@ extern spinlock_t pcpu_lock; 




extern struct list_head *pcpu_chunk_lists;

extern struct list_head *pcpu_chunk_lists;

extern int pcpu_nr_slots;

extern int pcpu_nr_slots;

extern int pcpu_nr_empty_pop_pages;

extern int pcpu_nr_empty_pop_pages[];





extern struct pcpu_chunk *pcpu_first_chunk;

extern struct pcpu_chunk *pcpu_first_chunk;

extern struct pcpu_chunk *pcpu_reserved_chunk;

extern struct pcpu_chunk *pcpu_reserved_chunk;

mm/percpu-stats.c

+7 −2
Original line number Original line Diff line number Diff line
@@ -145,6 +145,7 @@ static int percpu_stats_show(struct seq_file *m, void *v) 
	int slot, max_nr_alloc;

	int slot, max_nr_alloc;

	int *buffer;

	int *buffer;

	enum pcpu_chunk_type type;

	enum pcpu_chunk_type type;

	int nr_empty_pop_pages;





alloc_buffer:

alloc_buffer:

	spin_lock_irq(&pcpu_lock);

	spin_lock_irq(&pcpu_lock);
@@ -165,6 +166,10 @@ static int percpu_stats_show(struct seq_file *m, void *v) 
		goto alloc_buffer;

		goto alloc_buffer;

	}

	}





	nr_empty_pop_pages = 0;

	for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++)

		nr_empty_pop_pages += pcpu_nr_empty_pop_pages[type];



#define PL(X)								\

#define PL(X)								\

	seq_printf(m, "  %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X)

	seq_printf(m, "  %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X)
@@ -196,7 +201,7 @@ static int percpu_stats_show(struct seq_file *m, void *v) 
	PU(nr_max_chunks);

	PU(nr_max_chunks);

	PU(min_alloc_size);

	PU(min_alloc_size);

	PU(max_alloc_size);

	PU(max_alloc_size);

	P("empty_pop_pages", pcpu_nr_empty_pop_pages);

	P("empty_pop_pages", nr_empty_pop_pages);

	seq_putc(m, '\n');

	seq_putc(m, '\n');





#undef PU

#undef PU

mm/percpu.c

+7 −7
Original line number Original line Diff line number Diff line
@@ -173,10 +173,10 @@ struct list_head *pcpu_chunk_lists __ro_after_init; /* chunk list slots */ 
static LIST_HEAD(pcpu_map_extend_chunks);

static LIST_HEAD(pcpu_map_extend_chunks);





/*

/*

 * The number of empty populated pages, protected by pcpu_lock.  The

 * The number of empty populated pages by chunk type, protected by pcpu_lock.

 * reserved chunk doesn't contribute to the count.

 * The reserved chunk doesn't contribute to the count.

 */

 */

int pcpu_nr_empty_pop_pages;

int pcpu_nr_empty_pop_pages[PCPU_NR_CHUNK_TYPES];





/*

/*

 * The number of populated pages in use by the allocator, protected by

 * The number of populated pages in use by the allocator, protected by
@@ -556,7 +556,7 @@ static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr) 
{

{

	chunk->nr_empty_pop_pages += nr;

	chunk->nr_empty_pop_pages += nr;

	if (chunk != pcpu_reserved_chunk)

	if (chunk != pcpu_reserved_chunk)

		pcpu_nr_empty_pop_pages += nr;

		pcpu_nr_empty_pop_pages[pcpu_chunk_type(chunk)] += nr;

}

}





/*

/*
@@ -1832,7 +1832,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, 
		mutex_unlock(&pcpu_alloc_mutex);

		mutex_unlock(&pcpu_alloc_mutex);

	}

	}





	if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_LOW)

	if (pcpu_nr_empty_pop_pages[type] < PCPU_EMPTY_POP_PAGES_LOW)

		pcpu_schedule_balance_work();

		pcpu_schedule_balance_work();





	/* clear the areas and return address relative to base address */

	/* clear the areas and return address relative to base address */
@@ -2000,7 +2000,7 @@ static void __pcpu_balance_workfn(enum pcpu_chunk_type type) 
		pcpu_atomic_alloc_failed = false;

		pcpu_atomic_alloc_failed = false;

	} else {

	} else {

		nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH -

		nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH -

				  pcpu_nr_empty_pop_pages,

				  pcpu_nr_empty_pop_pages[type],

				  0, PCPU_EMPTY_POP_PAGES_HIGH);

				  0, PCPU_EMPTY_POP_PAGES_HIGH);

	}

	}
@@ -2580,7 +2580,7 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, 




	/* link the first chunk in */

	/* link the first chunk in */

	pcpu_first_chunk = chunk;

	pcpu_first_chunk = chunk;

	pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages;

	pcpu_nr_empty_pop_pages[PCPU_CHUNK_ROOT] = pcpu_first_chunk->nr_empty_pop_pages;

	pcpu_chunk_relocate(pcpu_first_chunk, -1);

	pcpu_chunk_relocate(pcpu_first_chunk, -1);





	/* include all regions of the first chunk */

	/* include all regions of the first chunk */