Commit 04ec0061 authored by Uros Bizjak's avatar Uros Bizjak Committed by Andrew Morton
Browse files

mm/page_alloc: use try_cmpxchg in set_pfnblock_flags_mask 

Use try_cmpxchg instead of cmpxchg in set_pfnblock_flags_mask.  x86
CMPXCHG instruction returns success in ZF flag, so this change saves a
compare after cmpxchg (and related move instruction in front of cmpxchg). 
The main loop improves from:

    1c5d:	48 89 c2             	mov    %rax,%rdx
    1c60:	48 89 c1             	mov    %rax,%rcx
    1c63:	48 21 fa             	and    %rdi,%rdx
    1c66:	4c 09 c2             	or     %r8,%rdx
    1c69:	f0 48 0f b1 16       	lock cmpxchg %rdx,(%rsi)
    1c6e:	48 39 c1             	cmp    %rax,%rcx
    1c71:	75 ea                	jne    1c5d <...>

to:

    1c60:	48 89 ca             	mov    %rcx,%rdx
    1c63:	48 21 c2             	and    %rax,%rdx
    1c66:	4c 09 c2             	or     %r8,%rdx
    1c69:	f0 48 0f b1 16       	lock cmpxchg %rdx,(%rsi)
    1c6e:	75 f0                	jne    1c60 <...>

Link: https://lkml.kernel.org/r/20220708140736.8737-1-ubizjak@gmail.com


Signed-off-by: default avatarUros Bizjak <ubizjak@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent dcadcf1c

mm/page_alloc.c

+3 −7
Original line number Diff line number Diff line
@@ -602,7 +602,7 @@ void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 
{

	unsigned long *bitmap;

	unsigned long bitidx, word_bitidx;

	unsigned long old_word, word;

	unsigned long word;



	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);

	BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
@@ -618,12 +618,8 @@ void set_pfnblock_flags_mask(struct page *page, unsigned long flags, 
	flags <<= bitidx;



	word = READ_ONCE(bitmap[word_bitidx]);

	for (;;) {

		old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);

		if (word == old_word)

			break;

		word = old_word;

	}

	do {

	} while (!try_cmpxchg(&bitmap[word_bitidx], &word, (word & ~mask) | flags));

}



void set_pageblock_migratetype(struct page *page, int migratetype)