Merge branches 'pm-sleep', 'pm-core', 'pm-domains' and 'pm-clk'

enabled earlier by calling pm_runtime_enable().

Note, if the device may execute pm_runtime calls during the probe (such as

if it is registers with a subsystem that may call back in) then the

if it is registered with a subsystem that may call back in) then the

pm_runtime_get_sync() call paired with a pm_runtime_put() call will be

appropriate to ensure that the device is not put back to sleep during the

probe. This can happen with systems such as the network device layer.

It may be desirable to suspend the device once ->probe() has finished.

Therefore the driver core uses the asynchronous pm_request_idle() to submit a

request to execute the subsystem-level idle callback for the device at that

time.  A driver that makes use of the runtime autosuspend feature, may want to

time.  A driver that makes use of the runtime autosuspend feature may want to

update the last busy mark before returning from ->probe().

Moreover, the driver core prevents runtime PM callbacks from racing with the bus

notifier callback in __device_release_driver(), which is necessary, because the

notifier callback in __device_release_driver(), which is necessary because the

notifier is used by some subsystems to carry out operations affecting the

runtime PM functionality.  It does so by calling pm_runtime_get_sync() before

driver_sysfs_remove() and the BUS_NOTIFY_UNBIND_DRIVER notifications.  This

executes pm_runtime_put_sync() after running the BUS_NOTIFY_UNBIND_DRIVER

notifications in __device_release_driver().  This requires bus types and

drivers to make their ->remove() callbacks avoid races with runtime PM directly,

but also it allows of more flexibility in the handling of devices during the

but it also allows more flexibility in the handling of devices during the

removal of their drivers.

Drivers in ->remove() callback should undo the runtime PM changes done

may be left in runtime suspend provided that all of its descendants are also

left in runtime suspend.  If that happens, the PM core will not execute any

system suspend and resume callbacks for all of those devices, except for the

complete callback, which is then entirely responsible for handling the device

.complete() callback, which is then entirely responsible for handling the device

as appropriate.  This only applies to system suspend transitions that are not

related to hibernation (see Documentation/driver-api/pm/devices.rst for more

information).

  `int pm_generic_restore_noirq(struct device *dev);`

    - invoke the ->restore_noirq() callback provided by the device's driver

These functions are the defaults used by the PM core, if a subsystem doesn't

These functions are the defaults used by the PM core if a subsystem doesn't

provide its own callbacks for ->runtime_idle(), ->runtime_suspend(),

->runtime_resume(), ->suspend(), ->suspend_noirq(), ->resume(),

->resume_noirq(), ->freeze(), ->freeze_noirq(), ->thaw(), ->thaw_noirq(),

enum pce_status {

	PCE_STATUS_NONE = 0,

	PCE_STATUS_ACQUIRED,

	PCE_STATUS_PREPARED,

	PCE_STATUS_ENABLED,

	PCE_STATUS_ERROR,

};

	char *con_id;

	struct clk *clk;

	enum pce_status status;

	bool enabled_when_prepared;

};

/**

 * pm_clk_list_lock - ensure exclusive access for modifying the PM clock

 *		      entry list.

 * @psd: pm_subsys_data instance corresponding to the PM clock entry list

 *	 and clk_op_might_sleep count to be modified.

 *

 * Get exclusive access before modifying the PM clock entry list and the

 * clock_op_might_sleep count to guard against concurrent modifications.

 * This also protects against a concurrent clock_op_might_sleep and PM clock

 * entry list usage in pm_clk_suspend()/pm_clk_resume() that may or may not

 * happen in atomic context, hence both the mutex and the spinlock must be

 * taken here.

 */

static void pm_clk_list_lock(struct pm_subsys_data *psd)

	__acquires(&psd->lock)

{

	mutex_lock(&psd->clock_mutex);

	spin_lock_irq(&psd->lock);

}

/**

 * pm_clk_list_unlock - counterpart to pm_clk_list_lock().

 * @psd: the same pm_subsys_data instance previously passed to

 *	 pm_clk_list_lock().

 */

static void pm_clk_list_unlock(struct pm_subsys_data *psd)

	__releases(&psd->lock)

{

	spin_unlock_irq(&psd->lock);

	mutex_unlock(&psd->clock_mutex);

}

/**

 * pm_clk_op_lock - ensure exclusive access for performing clock operations.

 * @psd: pm_subsys_data instance corresponding to the PM clock entry list

 *	 and clk_op_might_sleep count being used.

 * @flags: stored irq flags.

 * @fn: string for the caller function's name.

 *

 * This is used by pm_clk_suspend() and pm_clk_resume() to guard

 * against concurrent modifications to the clock entry list and the

 * clock_op_might_sleep count. If clock_op_might_sleep is != 0 then

 * only the mutex can be locked and those functions can only be used in

 * non atomic context. If clock_op_might_sleep == 0 then these functions

 * may be used in any context and only the spinlock can be locked.

 * Returns -EINVAL if called in atomic context when clock ops might sleep.

 */

static int pm_clk_op_lock(struct pm_subsys_data *psd, unsigned long *flags,

			  const char *fn)

	/* sparse annotations don't work here as exit state isn't static */

{

	bool atomic_context = in_atomic() || irqs_disabled();

try_again:

	spin_lock_irqsave(&psd->lock, *flags);

	if (!psd->clock_op_might_sleep) {

		/* the __release is there to work around sparse limitations */

		__release(&psd->lock);

		return 0;

	}

	/* bail out if in atomic context */

	if (atomic_context) {

		pr_err("%s: atomic context with clock_ops_might_sleep = %d",

		       fn, psd->clock_op_might_sleep);

		spin_unlock_irqrestore(&psd->lock, *flags);

		might_sleep();

		return -EPERM;

	}

	/* we must switch to the mutex */

	spin_unlock_irqrestore(&psd->lock, *flags);

	mutex_lock(&psd->clock_mutex);

	/*

	 * There was a possibility for psd->clock_op_might_sleep

	 * to become 0 above. Keep the mutex only if not the case.

	 */

	if (likely(psd->clock_op_might_sleep))

		return 0;

	mutex_unlock(&psd->clock_mutex);

	goto try_again;

}

/**

 * pm_clk_op_unlock - counterpart to pm_clk_op_lock().

 * @psd: the same pm_subsys_data instance previously passed to

 *	 pm_clk_op_lock().

 * @flags: irq flags provided by pm_clk_op_lock().

 */

static void pm_clk_op_unlock(struct pm_subsys_data *psd, unsigned long *flags)

	/* sparse annotations don't work here as entry state isn't static */

{

	if (psd->clock_op_might_sleep) {

		mutex_unlock(&psd->clock_mutex);

	} else {

		/* the __acquire is there to work around sparse limitations */

		__acquire(&psd->lock);

		spin_unlock_irqrestore(&psd->lock, *flags);

	}

}

/**

 * pm_clk_enable - Enable a clock, reporting any errors

 * @dev: The device for the given clock

{

	int ret;

	if (ce->status < PCE_STATUS_ERROR) {

	switch (ce->status) {

	case PCE_STATUS_ACQUIRED:

		ret = clk_prepare_enable(ce->clk);

		break;

	case PCE_STATUS_PREPARED:

		ret = clk_enable(ce->clk);

		break;

	default:

		return;

	}

	if (!ret)

		ce->status = PCE_STATUS_ENABLED;

	else

		dev_err(dev, "%s: failed to enable clk %p, error %d\n",

			__func__, ce->clk, ret);

}

}

/**

 * pm_clk_acquire - Acquire a device clock.

		ce->clk = clk_get(dev, ce->con_id);

	if (IS_ERR(ce->clk)) {

		ce->status = PCE_STATUS_ERROR;

	} else {

		if (clk_prepare(ce->clk)) {

		return;

	} else if (clk_is_enabled_when_prepared(ce->clk)) {

		/* we defer preparing the clock in that case */

		ce->status = PCE_STATUS_ACQUIRED;

		ce->enabled_when_prepared = true;

	} else if (clk_prepare(ce->clk)) {

		ce->status = PCE_STATUS_ERROR;

		dev_err(dev, "clk_prepare() failed\n");

		return;

	} else {

			ce->status = PCE_STATUS_ACQUIRED;

			dev_dbg(dev,

				"Clock %pC con_id %s managed by runtime PM.\n",

				ce->clk, ce->con_id);

		}

		ce->status = PCE_STATUS_PREPARED;

	}

	dev_dbg(dev, "Clock %pC con_id %s managed by runtime PM.\n",

		ce->clk, ce->con_id);

}

static int __pm_clk_add(struct device *dev, const char *con_id,

	pm_clk_acquire(dev, ce);

	spin_lock_irq(&psd->lock);

	pm_clk_list_lock(psd);

	list_add_tail(&ce->node, &psd->clock_list);

	spin_unlock_irq(&psd->lock);

	if (ce->enabled_when_prepared)

		psd->clock_op_might_sleep++;

	pm_clk_list_unlock(psd);

	return 0;

}

	if (!ce)

		return;

	if (ce->status < PCE_STATUS_ERROR) {

		if (ce->status == PCE_STATUS_ENABLED)

	switch (ce->status) {

	case PCE_STATUS_ENABLED:

		clk_disable(ce->clk);

		if (ce->status >= PCE_STATUS_ACQUIRED) {

		fallthrough;

	case PCE_STATUS_PREPARED:

		clk_unprepare(ce->clk);

		fallthrough;

	case PCE_STATUS_ACQUIRED:

	case PCE_STATUS_ERROR:

		if (!IS_ERR(ce->clk))

			clk_put(ce->clk);

		}

		break;

	default:

		break;

	}

	kfree(ce->con_id);

	if (!psd)

		return;

	spin_lock_irq(&psd->lock);

	pm_clk_list_lock(psd);

	list_for_each_entry(ce, &psd->clock_list, node) {

		if (!con_id && !ce->con_id)

			goto remove;

	}

	spin_unlock_irq(&psd->lock);

	pm_clk_list_unlock(psd);

	return;

 remove:

	list_del(&ce->node);

	spin_unlock_irq(&psd->lock);

	if (ce->enabled_when_prepared)

		psd->clock_op_might_sleep--;

	pm_clk_list_unlock(psd);

	__pm_clk_remove(ce);

}

	if (!psd || !clk)

		return;

	spin_lock_irq(&psd->lock);

	pm_clk_list_lock(psd);

	list_for_each_entry(ce, &psd->clock_list, node) {

		if (clk == ce->clk)

			goto remove;

	}

	spin_unlock_irq(&psd->lock);

	pm_clk_list_unlock(psd);

	return;

 remove:

	list_del(&ce->node);

	spin_unlock_irq(&psd->lock);

	if (ce->enabled_when_prepared)

		psd->clock_op_might_sleep--;

	pm_clk_list_unlock(psd);

	__pm_clk_remove(ce);

}

 * @dev: Device to initialize the list of PM clocks for.

 *

 * Initialize the lock and clock_list members of the device's pm_subsys_data

 * object.

 * object, set the count of clocks that might sleep to 0.

 */

void pm_clk_init(struct device *dev)

{

	struct pm_subsys_data *psd = dev_to_psd(dev);

	if (psd)

	if (psd) {

		INIT_LIST_HEAD(&psd->clock_list);

		mutex_init(&psd->clock_mutex);

		psd->clock_op_might_sleep = 0;

	}

}

EXPORT_SYMBOL_GPL(pm_clk_init);

	INIT_LIST_HEAD(&list);

	spin_lock_irq(&psd->lock);

	pm_clk_list_lock(psd);

	list_for_each_entry_safe_reverse(ce, c, &psd->clock_list, node)

		list_move(&ce->node, &list);

	psd->clock_op_might_sleep = 0;

	spin_unlock_irq(&psd->lock);

	pm_clk_list_unlock(psd);

	dev_pm_put_subsys_data(dev);

	struct pm_subsys_data *psd = dev_to_psd(dev);

	struct pm_clock_entry *ce;

	unsigned long flags;

	int ret;

	dev_dbg(dev, "%s()\n", __func__);

	if (!psd)

		return 0;

	spin_lock_irqsave(&psd->lock, flags);

	ret = pm_clk_op_lock(psd, &flags, __func__);

	if (ret)

		return ret;

	list_for_each_entry_reverse(ce, &psd->clock_list, node) {

		if (ce->status < PCE_STATUS_ERROR) {

			if (ce->status == PCE_STATUS_ENABLED)

				clk_disable(ce->clk);

		if (ce->status == PCE_STATUS_ENABLED) {

			if (ce->enabled_when_prepared) {

				clk_disable_unprepare(ce->clk);

				ce->status = PCE_STATUS_ACQUIRED;

			} else {

				clk_disable(ce->clk);

				ce->status = PCE_STATUS_PREPARED;

			}

		}

	}

	spin_unlock_irqrestore(&psd->lock, flags);

	pm_clk_op_unlock(psd, &flags);

	return 0;

}

	struct pm_subsys_data *psd = dev_to_psd(dev);

	struct pm_clock_entry *ce;

	unsigned long flags;

	int ret;

	dev_dbg(dev, "%s()\n", __func__);

	if (!psd)

		return 0;

	spin_lock_irqsave(&psd->lock, flags);

	ret = pm_clk_op_lock(psd, &flags, __func__);

	if (ret)

		return ret;

	list_for_each_entry(ce, &psd->clock_list, node)

		__pm_clk_enable(dev, ce);

	spin_unlock_irqrestore(&psd->lock, flags);

	pm_clk_op_unlock(psd, &flags);

	return 0;

}

	return state;

}

static int genpd_xlate_performance_state(struct generic_pm_domain *genpd,

					 struct generic_pm_domain *parent,

					 unsigned int pstate)

{

	if (!parent->set_performance_state)

		return pstate;

	return dev_pm_opp_xlate_performance_state(genpd->opp_table,

						  parent->opp_table,

						  pstate);

}

static int _genpd_set_performance_state(struct generic_pm_domain *genpd,

					unsigned int state, int depth)

{

	list_for_each_entry(link, &genpd->child_links, child_node) {

		parent = link->parent;

		if (!parent->set_performance_state)

			continue;

		/* Find parent's performance state */

		ret = dev_pm_opp_xlate_performance_state(genpd->opp_table,

							 parent->opp_table,

							 state);

		ret = genpd_xlate_performance_state(genpd, parent, state);

		if (unlikely(ret < 0))

			goto err;

			goto err;

	}

	if (genpd->set_performance_state) {

		ret = genpd->set_performance_state(genpd, state);

		if (ret)

			goto err;

	}

	genpd->performance_state = state;

	return 0;

					     child_node) {

		parent = link->parent;

		if (!parent->set_performance_state)

			continue;

		genpd_lock_nested(parent, depth + 1);

		parent_state = link->prev_performance_state;

	if (!genpd)

		return -ENODEV;

	if (unlikely(!genpd->set_performance_state))

		return -EINVAL;

	if (WARN_ON(!dev->power.subsys_data ||

		     !dev->power.subsys_data->domain_data))

		return -EINVAL;

}

EXPORT_SYMBOL_GPL(dev_pm_genpd_set_performance_state);

/**

 * dev_pm_genpd_set_next_wakeup - Notify PM framework of an impending wakeup.

 *

 * @dev: Device to handle

 * @next: impending interrupt/wakeup for the device

 *

 *

 * Allow devices to inform of the next wakeup. It's assumed that the users

 * guarantee that the genpd wouldn't be detached while this routine is getting

 * called. Additionally, it's also assumed that @dev isn't runtime suspended

 * (RPM_SUSPENDED)."

 * Although devices are expected to update the next_wakeup after the end of

 * their usecase as well, it is possible the devices themselves may not know

 * about that, so stale @next will be ignored when powering off the domain.

 */

void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next)

{

	struct generic_pm_domain_data *gpd_data;

	struct generic_pm_domain *genpd;

	genpd = dev_to_genpd_safe(dev);

	if (!genpd)

		return;

	gpd_data = to_gpd_data(dev->power.subsys_data->domain_data);

	gpd_data->next_wakeup = next;

}

EXPORT_SYMBOL_GPL(dev_pm_genpd_set_next_wakeup);

static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed)

{

	unsigned int state_idx = genpd->state_idx;

err_stop:

	genpd_stop_dev(genpd, dev);

err_poweroff:

	if (!pm_runtime_is_irq_safe(dev) ||

		(pm_runtime_is_irq_safe(dev) && genpd_is_irq_safe(genpd))) {

	if (!pm_runtime_is_irq_safe(dev) || genpd_is_irq_safe(genpd)) {

		genpd_lock(genpd);

		genpd_power_off(genpd, true, 0);

		genpd_unlock(genpd);

	gpd_data->td.constraint_changed = true;

	gpd_data->td.effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;

	gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier;

	gpd_data->next_wakeup = KTIME_MAX;

	spin_lock_irq(&dev->power.lock);

out:

	mutex_unlock(&gpd_list_lock);

	return ret;

	return ret == -ENOENT ? -EPROBE_DEFER : ret;

}

EXPORT_SYMBOL_GPL(of_genpd_add_subdomain);

	else

		WARN_ON(1);

	seq_puts(s, p);

	seq_printf(s, "%-25s  ", p);

}

static void perf_status_str(struct seq_file *s, struct device *dev)

{

	struct generic_pm_domain_data *gpd_data;

	gpd_data = to_gpd_data(dev->power.subsys_data->domain_data);

	seq_put_decimal_ull(s, "", gpd_data->performance_state);

}

static int genpd_summary_one(struct seq_file *s,

	else

		snprintf(state, sizeof(state), "%s",

			 status_lookup[genpd->status]);

	seq_printf(s, "%-30s  %-15s ", genpd->name, state);

	seq_printf(s, "%-30s  %-50s %u", genpd->name, state, genpd->performance_state);

	/*

	 * Modifications on the list require holding locks on both

	 * Also genpd->name is immutable.

	 */

	list_for_each_entry(link, &genpd->parent_links, parent_node) {

		if (list_is_first(&link->parent_node, &genpd->parent_links))

			seq_printf(s, "\n%48s", " ");

		seq_printf(s, "%s", link->child->name);

		if (!list_is_last(&link->parent_node, &genpd->parent_links))

			seq_puts(s, ", ");

		seq_printf(s, "\n    %-50s  ", kobj_path);

		rtpm_status_str(s, pm_data->dev);

		perf_status_str(s, pm_data->dev);

		kfree(kobj_path);

	}

	struct generic_pm_domain *genpd;

	int ret = 0;

	seq_puts(s, "domain                          status          children\n");

	seq_puts(s, "domain                          status          children                           performance\n");

	seq_puts(s, "    /device                                             runtime status\n");

	seq_puts(s, "----------------------------------------------------------------------\n");

	seq_puts(s, "----------------------------------------------------------------------------------------------\n");

	ret = mutex_lock_interruptible(&gpd_list_lock);

	if (ret)

	return td->cached_suspend_ok;

}

static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now)

{

	ktime_t domain_wakeup = KTIME_MAX;

	ktime_t next_wakeup;

	struct pm_domain_data *pdd;

	struct gpd_link *link;

	if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY))

		return;

	/*

	 * Devices that have a predictable wakeup pattern, may specify

	 * their next wakeup. Let's find the next wakeup from all the

	 * devices attached to this domain and from all the sub-domains.

	 * It is possible that component's a next wakeup may have become

	 * stale when we read that here. We will ignore to ensure the domain

	 * is able to enter its optimal idle state.

	 */

	list_for_each_entry(pdd, &genpd->dev_list, list_node) {

		next_wakeup = to_gpd_data(pdd)->next_wakeup;

		if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))

			if (ktime_before(next_wakeup, domain_wakeup))

				domain_wakeup = next_wakeup;

	}

	list_for_each_entry(link, &genpd->parent_links, parent_node) {

		next_wakeup = link->child->next_wakeup;

		if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))

			if (ktime_before(next_wakeup, domain_wakeup))

				domain_wakeup = next_wakeup;

	}

	genpd->next_wakeup = domain_wakeup;

}

static bool next_wakeup_allows_state(struct generic_pm_domain *genpd,

				     unsigned int state, ktime_t now)

{

	ktime_t domain_wakeup = genpd->next_wakeup;

	s64 idle_time_ns, min_sleep_ns;

	min_sleep_ns = genpd->states[state].power_off_latency_ns +

		       genpd->states[state].residency_ns;

	idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));

	return idle_time_ns >= min_sleep_ns;

}

static bool __default_power_down_ok(struct dev_pm_domain *pd,

				     unsigned int state)

{

}

/**

 * default_power_down_ok - Default generic PM domain power off governor routine.

 * _default_power_down_ok - Default generic PM domain power off governor routine.

 * @pd: PM domain to check.

 *

 * This routine must be executed under the PM domain's lock.

 */

static bool default_power_down_ok(struct dev_pm_domain *pd)

static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now)

{

	struct generic_pm_domain *genpd = pd_to_genpd(pd);

	int state_idx = genpd->state_count - 1;

	struct gpd_link *link;

	/*

	 * Find the next wakeup from devices that can determine their own wakeup

	 * to find when the domain would wakeup and do it for every device down

	 * the hierarchy. It is not worth while to sleep if the state's residency

	 * cannot be met.

	 */