sched: Wrap rq::lock access

	for (;;) {

		rq = task_rq(p);

		raw_spin_lock(&rq->lock);

		raw_spin_rq_lock(rq);

		if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {

			rq_pin_lock(rq, rf);

			return rq;

		}

		raw_spin_unlock(&rq->lock);

		raw_spin_rq_unlock(rq);

		while (unlikely(task_on_rq_migrating(p)))

			cpu_relax();

	for (;;) {

		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);

		rq = task_rq(p);

		raw_spin_lock(&rq->lock);

		raw_spin_rq_lock(rq);

		/*

		 *	move_queued_task()		task_rq_lock()

		 *

			rq_pin_lock(rq, rf);

			return rq;

		}

		raw_spin_unlock(&rq->lock);

		raw_spin_rq_unlock(rq);

		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);

		while (unlikely(task_on_rq_migrating(p)))

{

	s64 delta;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	if (rq->clock_update_flags & RQCF_ACT_SKIP)

		return;

	struct task_struct *curr = rq->curr;

	int cpu;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	if (test_tsk_need_resched(curr))

		return;

	struct rq *rq = cpu_rq(cpu);

	unsigned long flags;

	raw_spin_lock_irqsave(&rq->lock, flags);

	raw_spin_rq_lock_irqsave(rq, flags);

	if (cpu_online(cpu) || cpu == smp_processor_id())

		resched_curr(rq);

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	raw_spin_rq_unlock_irqrestore(rq, flags);

}

#ifdef CONFIG_SMP

	struct uclamp_se *uc_se = &p->uclamp[clamp_id];

	struct uclamp_bucket *bucket;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	/* Update task effective clamp */

	p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id);

	unsigned int bkt_clamp;

	unsigned int rq_clamp;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	/*

	 * If sched_uclamp_used was enabled after task @p was enqueued,

static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,

				   struct task_struct *p, int new_cpu)

{

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	deactivate_task(rq, p, DEQUEUE_NOCLOCK);

	set_task_cpu(p, new_cpu);

	struct task_struct *p = arg;

	raw_spin_lock_irq(&p->pi_lock);

	raw_spin_lock(&rq->lock);

	raw_spin_rq_lock(rq);

	if (task_rq(p) != rq)

		goto out_unlock;

out_unlock:

	rq->push_busy = false;

	raw_spin_unlock(&rq->lock);

	raw_spin_rq_unlock(rq);

	raw_spin_unlock_irq(&p->pi_lock);

	put_task_struct(p);

		 * Because __kthread_bind() calls this on blocked tasks without

		 * holding rq->lock.

		 */

		lockdep_assert_held(&rq->lock);

		lockdep_assert_rq_held(rq);

		dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);

	}

	if (running)

	 * task_rq_lock().

	 */

	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||

				      lockdep_is_held(&task_rq(p)->lock)));

				      lockdep_is_held(rq_lockp(task_rq(p)))));

#endif

	/*

	 * Clearly, migrating tasks to offline CPUs is a fairly daft thing.

{

	int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	if (p->sched_contributes_to_load)

		rq->nr_uninterruptible--;

	void (*func)(struct rq *rq);

	struct callback_head *next;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	while (head) {

		func = (void (*)(struct rq *))head->func;

{

	struct callback_head *head = rq->balance_callback;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	if (head)

		rq->balance_callback = NULL;

	unsigned long flags;

	if (unlikely(head)) {

		raw_spin_lock_irqsave(&rq->lock, flags);

		raw_spin_rq_lock_irqsave(rq, flags);

		do_balance_callbacks(rq, head);

		raw_spin_unlock_irqrestore(&rq->lock, flags);

		raw_spin_rq_unlock_irqrestore(rq, flags);

	}

}

	 * do an early lockdep release here:

	 */

	rq_unpin_lock(rq, rf);

	spin_release(&rq->lock.dep_map, _THIS_IP_);

	spin_release(&rq_lockp(rq)->dep_map, _THIS_IP_);

#ifdef CONFIG_DEBUG_SPINLOCK

	/* this is a valid case when another task releases the spinlock */

	rq->lock.owner = next;

	rq_lockp(rq)->owner = next;

#endif

}

	 * fix up the runqueue lock - which gets 'carried over' from

	 * prev into current:

	 */

	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);

	spin_acquire(&rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_);

	__balance_callbacks(rq);

	raw_spin_unlock_irq(&rq->lock);

	raw_spin_rq_unlock_irq(rq);

}

/*

		rq_unpin_lock(rq, &rf);

		__balance_callbacks(rq);

		raw_spin_unlock_irq(&rq->lock);

		raw_spin_rq_unlock_irq(rq);

	}

}

	rq_unpin_lock(rq, &rf);

	__balance_callbacks(rq);

	raw_spin_unlock(&rq->lock);

	raw_spin_rq_unlock(rq);

	preempt_enable();

}

	__sched_fork(0, idle);

	raw_spin_lock_irqsave(&idle->pi_lock, flags);

	raw_spin_lock(&rq->lock);

	raw_spin_rq_lock(rq);

	idle->state = TASK_RUNNING;

	idle->se.exec_start = sched_clock();

#ifdef CONFIG_SMP

	idle->on_cpu = 1;

#endif

	raw_spin_unlock(&rq->lock);

	raw_spin_rq_unlock(rq);

	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);

	/* Set the preempt count _outside_ the spinlocks! */

{

	struct task_struct *push_task = rq->curr;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	SCHED_WARN_ON(rq->cpu != smp_processor_id());

	/*

		 */

		if (!rq->nr_running && !rq_has_pinned_tasks(rq) &&

		    rcuwait_active(&rq->hotplug_wait)) {

			raw_spin_unlock(&rq->lock);

			raw_spin_rq_unlock(rq);

			rcuwait_wake_up(&rq->hotplug_wait);

			raw_spin_lock(&rq->lock);

			raw_spin_rq_lock(rq);

		}

		return;

	}

	 * Temporarily drop rq->lock such that we can wake-up the stop task.

	 * Both preemption and IRQs are still disabled.

	 */

	raw_spin_unlock(&rq->lock);

	raw_spin_rq_unlock(rq);

	stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task,

			    this_cpu_ptr(&push_work));

	/*

	 * schedule(). The next pick is obviously going to be the stop task

	 * which kthread_is_per_cpu() and will push this task away.

	 */

	raw_spin_lock(&rq->lock);

	raw_spin_rq_lock(rq);

}

static void balance_push_set(int cpu, bool on)

	struct task_struct *g, *p;

	int cpu = cpu_of(rq);

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running);

	for_each_process_thread(g, p) {

		struct rq *rq;

		rq = cpu_rq(i);

		raw_spin_lock_init(&rq->lock);

		raw_spin_lock_init(&rq->__lock);

		rq->nr_running = 0;

		rq->calc_load_active = 0;

		rq->calc_load_update = jiffies + LOAD_FREQ;

	/*

	 * Take rq->lock to make 64-bit read safe on 32-bit platforms.

	 */

	raw_spin_lock_irq(&cpu_rq(cpu)->lock);

	raw_spin_rq_lock_irq(cpu_rq(cpu));

#endif

	if (index == CPUACCT_STAT_NSTATS) {

	}

#ifndef CONFIG_64BIT

	raw_spin_unlock_irq(&cpu_rq(cpu)->lock);

	raw_spin_rq_unlock_irq(cpu_rq(cpu));

#endif

	return data;

	/*

	 * Take rq->lock to make 64-bit write safe on 32-bit platforms.

	 */

	raw_spin_lock_irq(&cpu_rq(cpu)->lock);

	raw_spin_rq_lock_irq(cpu_rq(cpu));

#endif

	for (i = 0; i < CPUACCT_STAT_NSTATS; i++)

		cpuusage->usages[i] = val;

#ifndef CONFIG_64BIT

	raw_spin_unlock_irq(&cpu_rq(cpu)->lock);

	raw_spin_rq_unlock_irq(cpu_rq(cpu));

#endif

}

			 * Take rq->lock to make 64-bit read safe on 32-bit

			 * platforms.

			 */

			raw_spin_lock_irq(&cpu_rq(cpu)->lock);

			raw_spin_rq_lock_irq(cpu_rq(cpu));

#endif

			seq_printf(m, " %llu", cpuusage->usages[index]);

#ifndef CONFIG_64BIT

			raw_spin_unlock_irq(&cpu_rq(cpu)->lock);

			raw_spin_rq_unlock_irq(cpu_rq(cpu));

#endif

		}

		seq_puts(m, "\n");

{

	u64 old = dl_rq->running_bw;

	lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);

	lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));

	dl_rq->running_bw += dl_bw;

	SCHED_WARN_ON(dl_rq->running_bw < old); /* overflow */

	SCHED_WARN_ON(dl_rq->running_bw > dl_rq->this_bw);

{

	u64 old = dl_rq->running_bw;

	lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);

	lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));

	dl_rq->running_bw -= dl_bw;

	SCHED_WARN_ON(dl_rq->running_bw > old); /* underflow */

	if (dl_rq->running_bw > old)

{

	u64 old = dl_rq->this_bw;

	lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);

	lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));

	dl_rq->this_bw += dl_bw;

	SCHED_WARN_ON(dl_rq->this_bw < old); /* overflow */

}

{

	u64 old = dl_rq->this_bw;

	lockdep_assert_held(&(rq_of_dl_rq(dl_rq))->lock);

	lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));

	dl_rq->this_bw -= dl_bw;

	SCHED_WARN_ON(dl_rq->this_bw > old); /* underflow */

	if (dl_rq->this_bw > old)

	ktime_t now, act;

	s64 delta;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	/*

	 * We want the timer to fire at the deadline, but considering

		 * If the runqueue is no longer available, migrate the

		 * task elsewhere. This necessarily changes rq.

		 */

		lockdep_unpin_lock(&rq->lock, rf.cookie);

		lockdep_unpin_lock(rq_lockp(rq), rf.cookie);

		rq = dl_task_offline_migration(rq, p);

		rf.cookie = lockdep_pin_lock(&rq->lock);

		rf.cookie = lockdep_pin_lock(rq_lockp(rq));

		update_rq_clock(rq);

		/*

	 * from try_to_wake_up(). Hence, p->pi_lock is locked, but

	 * rq->lock is not... So, lock it

	 */

	raw_spin_lock(&rq->lock);

	raw_spin_rq_lock(rq);

	if (p->dl.dl_non_contending) {

		sub_running_bw(&p->dl, &rq->dl);

		p->dl.dl_non_contending = 0;

			put_task_struct(p);

	}

	sub_rq_bw(&p->dl, &rq->dl);

	raw_spin_unlock(&rq->lock);

	raw_spin_rq_unlock(rq);

}

static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)

		double_unlock_balance(this_rq, src_rq);

		if (push_task) {

			raw_spin_unlock(&this_rq->lock);

			raw_spin_rq_unlock(this_rq);

			stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,

					    push_task, &src_rq->push_work);

			raw_spin_lock(&this_rq->lock);

			raw_spin_rq_lock(this_rq);

		}

	}

	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "exec_clock",

			SPLIT_NS(cfs_rq->exec_clock));

	raw_spin_lock_irqsave(&rq->lock, flags);

	raw_spin_rq_lock_irqsave(rq, flags);

	if (rb_first_cached(&cfs_rq->tasks_timeline))

		MIN_vruntime = (__pick_first_entity(cfs_rq))->vruntime;

	last = __pick_last_entity(cfs_rq);

		max_vruntime = last->vruntime;

	min_vruntime = cfs_rq->min_vruntime;

	rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime;

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	raw_spin_rq_unlock_irqrestore(rq, flags);

	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "MIN_vruntime",

			SPLIT_NS(MIN_vruntime));

	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "min_vruntime",

static struct numa_group *deref_task_numa_group(struct task_struct *p)

{

	return rcu_dereference_check(p->numa_group, p == current ||

		(lockdep_is_held(&task_rq(p)->lock) && !READ_ONCE(p->on_cpu)));

		(lockdep_is_held(rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu)));

}

static struct numa_group *deref_curr_numa_group(struct task_struct *p)

{

	struct task_group *tg;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	rcu_read_lock();

	list_for_each_entry_rcu(tg, &task_groups, list) {

{

	struct task_group *tg;

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	rcu_read_lock();

	list_for_each_entry_rcu(tg, &task_groups, list) {

		 * In case of TASK_ON_RQ_MIGRATING we in fact hold the 'old'

		 * rq->lock and can modify state directly.

		 */

		lockdep_assert_held(&task_rq(p)->lock);

		lockdep_assert_rq_held(task_rq(p));

		detach_entity_cfs_rq(&p->se);

	} else {

{

	s64 delta;

	lockdep_assert_held(&env->src_rq->lock);

	lockdep_assert_rq_held(env->src_rq);

	if (p->sched_class != &fair_sched_class)

		return 0;

{

	int tsk_cache_hot;

	lockdep_assert_held(&env->src_rq->lock);

	lockdep_assert_rq_held(env->src_rq);

	/*

	 * We do not migrate tasks that are:

 */

static void detach_task(struct task_struct *p, struct lb_env *env)

{

	lockdep_assert_held(&env->src_rq->lock);

	lockdep_assert_rq_held(env->src_rq);

	deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK);

	set_task_cpu(p, env->dst_cpu);

{

	struct task_struct *p;

	lockdep_assert_held(&env->src_rq->lock);

	lockdep_assert_rq_held(env->src_rq);

	list_for_each_entry_reverse(p,

			&env->src_rq->cfs_tasks, se.group_node) {

	struct task_struct *p;

	int detached = 0;

	lockdep_assert_held(&env->src_rq->lock);

	lockdep_assert_rq_held(env->src_rq);

	/*

	 * Source run queue has been emptied by another CPU, clear

 */

static void attach_task(struct rq *rq, struct task_struct *p)

{

	lockdep_assert_held(&rq->lock);

	lockdep_assert_rq_held(rq);

	BUG_ON(task_rq(p) != rq);

	activate_task(rq, p, ENQUEUE_NOCLOCK);

		if (need_active_balance(&env)) {

			unsigned long flags;

			raw_spin_lock_irqsave(&busiest->lock, flags);

			raw_spin_rq_lock_irqsave(busiest, flags);

			/*

			 * Don't kick the active_load_balance_cpu_stop,

			 * moved to this_cpu:

			 */

			if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) {

				raw_spin_unlock_irqrestore(&busiest->lock,

							    flags);

				raw_spin_rq_unlock_irqrestore(busiest, flags);

				goto out_one_pinned;

			}

				busiest->push_cpu = this_cpu;

				active_balance = 1;

			}

			raw_spin_unlock_irqrestore(&busiest->lock, flags);

			raw_spin_rq_unlock_irqrestore(busiest, flags);

			if (active_balance) {

				stop_one_cpu_nowait(cpu_of(busiest),

		goto out;

	}

	raw_spin_unlock(&this_rq->lock);

	raw_spin_rq_unlock(this_rq);

	update_blocked_averages(this_cpu);

	rcu_read_lock();

	}

	rcu_read_unlock();

	raw_spin_lock(&this_rq->lock);

	raw_spin_rq_lock(this_rq);

	if (curr_cost > this_rq->max_idle_balance_cost)

		this_rq->max_idle_balance_cost = curr_cost;

		rq = cpu_rq(cpu);

		raw_spin_lock_irqsave(&rq->lock, flags);

		raw_spin_rq_lock_irqsave(rq, flags);

		list_del_leaf_cfs_rq(tg->cfs_rq[cpu]);

		raw_spin_unlock_irqrestore(&rq->lock, flags);