rcu: Inline __call_rcu() into call_rcu() (1fe09ebe) · Commits · EulixOS / Software / Kernel

kernel/rcu/tree.c

+42 −49

Original line number	Diff line number	Diff line
		@@ -2995,9 +2995,47 @@ static void check_cb_ovld(struct rcu_data *rdp)
		raw_spin_unlock_rcu_node(rnp);
		}

		/* Helper function for call_rcu() and friends. */
		static void
		__call_rcu(struct rcu_head *head, rcu_callback_t func)
		/**
		* call_rcu() - Queue an RCU callback for invocation after a grace period.
		* @head: structure to be used for queueing the RCU updates.
		* @func: actual callback function to be invoked after the grace period
		*
		* The callback function will be invoked some time after a full grace
		* period elapses, in other words after all pre-existing RCU read-side
		* critical sections have completed. However, the callback function
		* might well execute concurrently with RCU read-side critical sections
		* that started after call_rcu() was invoked.
		*
		* RCU read-side critical sections are delimited by rcu_read_lock()
		* and rcu_read_unlock(), and may be nested. In addition, but only in
		* v5.0 and later, regions of code across which interrupts, preemption,
		* or softirqs have been disabled also serve as RCU read-side critical
		* sections. This includes hardware interrupt handlers, softirq handlers,
		* and NMI handlers.
		*
		* Note that all CPUs must agree that the grace period extended beyond
		* all pre-existing RCU read-side critical section. On systems with more
		* than one CPU, this means that when "func()" is invoked, each CPU is
		* guaranteed to have executed a full memory barrier since the end of its
		* last RCU read-side critical section whose beginning preceded the call
		* to call_rcu(). It also means that each CPU executing an RCU read-side
		* critical section that continues beyond the start of "func()" must have
		* executed a memory barrier after the call_rcu() but before the beginning
		* of that RCU read-side critical section. Note that these guarantees
		* include CPUs that are offline, idle, or executing in user mode, as
		* well as CPUs that are executing in the kernel.
		*
		* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
		* resulting RCU callback function "func()", then both CPU A and CPU B are
		* guaranteed to execute a full memory barrier during the time interval
		* between the call to call_rcu() and the invocation of "func()" -- even
		* if CPU A and CPU B are the same CPU (but again only if the system has
		* more than one CPU).
		*
		* Implementation of these memory-ordering guarantees is described here:
		* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst.
		*/
		void call_rcu(struct rcu_head *head, rcu_callback_t func)
		{
		static atomic_t doublefrees;
		unsigned long flags;
		@@ -3011,7 +3049,7 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func)
		/*
		* Probable double call_rcu(), so leak the callback.
		* Use rcu:rcu_callback trace event to find the previous
		* time callback was passed to __call_rcu().
		* time callback was passed to call_rcu().
		*/
		if (atomic_inc_return(&doublefrees) < 4) {
		pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func);
		@@ -3060,51 +3098,6 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func)
		local_irq_restore(flags);
		}
		}

		/**
		* call_rcu() - Queue an RCU callback for invocation after a grace period.
		* @head: structure to be used for queueing the RCU updates.
		* @func: actual callback function to be invoked after the grace period
		*
		* The callback function will be invoked some time after a full grace
		* period elapses, in other words after all pre-existing RCU read-side
		* critical sections have completed. However, the callback function
		* might well execute concurrently with RCU read-side critical sections
		* that started after call_rcu() was invoked.
		*
		* RCU read-side critical sections are delimited by rcu_read_lock()
		* and rcu_read_unlock(), and may be nested. In addition, but only in
		* v5.0 and later, regions of code across which interrupts, preemption,
		* or softirqs have been disabled also serve as RCU read-side critical
		* sections. This includes hardware interrupt handlers, softirq handlers,
		* and NMI handlers.
		*
		* Note that all CPUs must agree that the grace period extended beyond
		* all pre-existing RCU read-side critical section. On systems with more
		* than one CPU, this means that when "func()" is invoked, each CPU is
		* guaranteed to have executed a full memory barrier since the end of its
		* last RCU read-side critical section whose beginning preceded the call
		* to call_rcu(). It also means that each CPU executing an RCU read-side
		* critical section that continues beyond the start of "func()" must have
		* executed a memory barrier after the call_rcu() but before the beginning
		* of that RCU read-side critical section. Note that these guarantees
		* include CPUs that are offline, idle, or executing in user mode, as
		* well as CPUs that are executing in the kernel.
		*
		* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
		* resulting RCU callback function "func()", then both CPU A and CPU B are
		* guaranteed to execute a full memory barrier during the time interval
		* between the call to call_rcu() and the invocation of "func()" -- even
		* if CPU A and CPU B are the same CPU (but again only if the system has
		* more than one CPU).
		*
		* Implementation of these memory-ordering guarantees is described here:
		* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst.
		*/
		void call_rcu(struct rcu_head *head, rcu_callback_t func)
		{
		__call_rcu(head, func);
		}
		EXPORT_SYMBOL_GPL(call_rcu);