Commit 5924881f authored by Yafang Shao's avatar Yafang Shao Committed by Chen Ridong
Browse files

cgroup: Make operations on the cgroup root_list RCU safe 

mainline inclusion
from mainline-v6.8-rc1
commit d23b5c577715892c87533b13923306acc6243f93
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/IA7HMV

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=d23b5c577715892c87533b13923306acc6243f93



----------------------------------------------------------------------

[ Upstream commit d23b5c577715892c87533b13923306acc6243f93 ]

At present, when we perform operations on the cgroup root_list, we must
hold the cgroup_mutex, which is a relatively heavyweight lock. In reality,
we can make operations on this list RCU-safe, eliminating the need to hold
the cgroup_mutex during traversal. Modifications to the list only occur in
the cgroup root setup and destroy paths, which should be infrequent in a
production environment. In contrast, traversal may occur frequently.
Therefore, making it RCU-safe would be beneficial.

Signed-off-by: default avatarYafang Shao <laoar.shao@gmail.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Signed-off-by: default avatarSasha Levin <sashal@kernel.org>
Signed-off-by: default avatarChen Ridong <chenridong@huawei.com>

[Backport] cgroup: Move rcu_head up near the top of cgroup_root

mainline inclusion
from mainline-v6.8-rc1
commit a7fb0423c201ba12815877a0b5a68a6a1710b23a
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/IA7HMV

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=a7fb0423c201ba12815877a0b5a68a6a1710b23a



----------------------------------------------------------------------

commit a7fb0423c201ba12815877a0b5a68a6a1710b23a upstream.

Commit d23b5c577715 ("cgroup: Make operations on the cgroup root_list RCU
safe") adds a new rcu_head to the cgroup_root structure and kvfree_rcu()
for freeing the cgroup_root.

The current implementation of kvfree_rcu(), however, has the limitation
that the offset of the rcu_head structure within the larger data
structure must be less than 4096 or the compilation will fail. See the
macro definition of __is_kvfree_rcu_offset() in include/linux/rcupdate.h
for more information.

By putting rcu_head below the large cgroup structure, any change to the
cgroup structure that makes it larger run the risk of causing build
failure under certain configurations. Commit 77070eeb8821 ("cgroup:
Avoid false cacheline sharing of read mostly rstat_cpu") happens to be
the last straw that breaks it. Fix this problem by moving the rcu_head
structure up before the cgroup structure.

Fixes: d23b5c577715 ("cgroup: Make operations on the cgroup root_list RCU safe")
Reported-by: default avatarStephen Rothwell <sfr@canb.auug.org.au>
Closes: https://lore.kernel.org/lkml/20231207143806.114e0a74@canb.auug.org.au/


Signed-off-by: default avatarWaiman Long <longman@redhat.com>
Acked-by: default avatarYafang Shao <laoar.shao@gmail.com>
Reviewed-by: default avatarYosry Ahmed <yosryahmed@google.com>
Reviewed-by: default avatarMichal Koutný <mkoutny@suse.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>

Conflicts:
	include/linux/cgroup-defs.h
	kernel/cgroup/cgroup.c
[Context is mismatched for wait_queue_head_t wait was merged. cgroup.c
is mismatched for some comment.]
Signed-off-by: default avatarChen Ridong <chenridong@huawei.com>
parent c4d9235a

include/linux/cgroup-defs.h

+4 −3
Original line number Diff line number Diff line
@@ -526,6 +526,10 @@ struct cgroup_root { 
	/* Unique id for this hierarchy. */

	int hierarchy_id;



	/* A list running through the active hierarchies */

	struct list_head root_list;

	struct rcu_head rcu;	/* Must be near the top */



	/* The root cgroup.  Root is destroyed on its release. */

	struct cgroup cgrp;
@@ -538,9 +542,6 @@ struct cgroup_root { 
	/* Wait while cgroups are being destroyed */

	wait_queue_head_t wait;



	/* A list running through the active hierarchies */

	struct list_head root_list;



	/* Hierarchy-specific flags */

	unsigned int flags;

kernel/cgroup/cgroup-internal.h

+2 −1
Original line number Diff line number Diff line
@@ -173,7 +173,8 @@ extern struct list_head cgroup_roots; 


/* iterate across the hierarchies */

#define for_each_root(root)						\

	list_for_each_entry((root), &cgroup_roots, root_list)

	list_for_each_entry_rcu((root), &cgroup_roots, root_list,	\

				lockdep_is_held(&cgroup_mutex))



/**

 * for_each_subsys - iterate all enabled cgroup subsystems

kernel/cgroup/cgroup.c

+7 −7
Original line number Diff line number Diff line
@@ -1303,7 +1303,7 @@ static void cgroup_exit_root_id(struct cgroup_root *root) 


void cgroup_free_root(struct cgroup_root *root)

{

	kfree(root);

	kfree_rcu(root, rcu);

}



static void cgroup_destroy_root(struct cgroup_root *root)
@@ -1336,7 +1336,7 @@ static void cgroup_destroy_root(struct cgroup_root *root) 
	spin_unlock_irq(&css_set_lock);



	if (!list_empty(&root->root_list)) {

		list_del(&root->root_list);

		list_del_rcu(&root->root_list);

		cgroup_root_count--;

	}
@@ -1382,7 +1382,6 @@ current_cgns_cgroup_from_root(struct cgroup_root *root) 
	}

	rcu_read_unlock();



	BUG_ON(!res);

	return res;

}
@@ -1392,7 +1391,6 @@ static struct cgroup *cset_cgroup_from_root(struct css_set *cset, 
{

	struct cgroup *res = NULL;



	lockdep_assert_held(&cgroup_mutex);

	lockdep_assert_held(&css_set_lock);



	if (cset == &init_css_set) {
@@ -1418,7 +1416,9 @@ static struct cgroup *cset_cgroup_from_root(struct css_set *cset, 


/*

 * Return the cgroup for "task" from the given hierarchy. Must be

 * called with cgroup_mutex and css_set_lock held.

 * called with css_set_lock held to prevent task's groups from being modified.

 * Must be called with either cgroup_mutex or rcu read lock to prevent the

 * cgroup root from being destroyed.

 */

struct cgroup *task_cgroup_from_root(struct task_struct *task,

				     struct cgroup_root *root)
@@ -1957,7 +1957,7 @@ void init_cgroup_root(struct cgroup_fs_context *ctx) 
	struct cgroup_root *root = ctx->root;

	struct cgroup *cgrp = &root->cgrp;



	INIT_LIST_HEAD(&root->root_list);

	INIT_LIST_HEAD_RCU(&root->root_list);

	atomic_set(&root->nr_cgrps, 1);

	cgrp->root = root;

	init_cgroup_housekeeping(cgrp);
@@ -2040,7 +2040,7 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask) 
	 * care of subsystems' refcounts, which are explicitly dropped in

	 * the failure exit path.

	 */

	list_add(&root->root_list, &cgroup_roots);

	list_add_rcu(&root->root_list, &cgroup_roots);

	cgroup_root_count++;



	/*