Commit 468276d4 authored by Bjorn Helgaas's avatar Bjorn Helgaas
Browse files

Merge branch 'pci/ctrl/imx6' 

- Factor out ref clock disables to match enables (Bjorn Helgaas)

- Collect clock enables in imx6_pcie_clk_enable() (Richard Zhu)

- Propagate regulator and clock errors back to .host_init() caller (Richard
  Zhu)

- Disable i.MX6QDL clock when disabling ref clocks (Richard Zhu)

- Call host init function directly in resume instead of duplicating the
  code (Richard Zhu)

- Turn off regulators when suspending (Richard Zhu)

- Make link being down a non-fatal error so probe doesn't fail (Richard
  Zhu)

- Start link in resume only if it was up before suspend to reduce resume
  time (Richard Zhu)

- Move PHY init and power-on out of clock- and reset-related functions
  (Richard Zhu)

- Rework suspend callback to be more symmetric with resume (Richard Zhu)

- Set PCIE_DBI_RO_WR_EN before writing DBI registers (Richard Zhu)

- Allow speeds faster than Gen2 (Richard Zhu)

* pci/ctrl/imx6:
  PCI: imx6: Support more than Gen2 speed link mode
  PCI: imx6: Set PCIE_DBI_RO_WR_EN before writing DBI registers
  PCI: imx6: Reformat suspend callback to keep symmetric with resume
  PCI: imx6: Move the imx6_pcie_ltssm_disable() earlier
  PCI: imx6: Disable clocks in reverse order of enable
  PCI: imx6: Do not hide PHY driver callbacks and refine the error handling
  PCI: imx6: Reduce resume time by only starting link if it was up before suspend
  PCI: imx6: Mark the link down as non-fatal error
  PCI: imx6: Move regulator enable out of imx6_pcie_deassert_core_reset()
  PCI: imx6: Turn off regulator when system is in suspend mode
  PCI: imx6: Call host init function directly in resume
  PCI: imx6: Disable i.MX6QDL clock when disabling ref clocks
  PCI: imx6: Propagate .host_init() errors to caller
  PCI: imx6: Collect clock enables in imx6_pcie_clk_enable()
  PCI: imx6: Factor out ref clock disable to match enable
  PCI: imx6: Move imx6_pcie_clk_disable() earlier
  PCI: imx6: Move imx6_pcie_enable_ref_clk() earlier
  PCI: imx6: Move PHY management functions together
  PCI: imx6: Move imx6_pcie_grp_offset(), imx6_pcie_configure_type() earlier
  PCI: imx6: Convert to NOIRQ_SYSTEM_SLEEP_PM_OPS()
parents 0c8b7d12 6213c6c5

drivers/pci/controller/dwc/pci-imx6.c

+368 −312
Original line number Diff line number Diff line
@@ -67,6 +67,7 @@ struct imx6_pcie { 
	struct dw_pcie		*pci;

	int			reset_gpio;

	bool			gpio_active_high;

	bool			link_is_up;

	struct clk		*pcie_bus;

	struct clk		*pcie_phy;

	struct clk		*pcie_inbound_axi;
@@ -146,6 +147,31 @@ struct imx6_pcie { 
#define PHY_RX_OVRD_IN_LO_RX_DATA_EN		BIT(5)

#define PHY_RX_OVRD_IN_LO_RX_PLL_EN		BIT(3)



static unsigned int imx6_pcie_grp_offset(const struct imx6_pcie *imx6_pcie)

{

	WARN_ON(imx6_pcie->drvdata->variant != IMX8MQ &&

		imx6_pcie->drvdata->variant != IMX8MM);

	return imx6_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14;

}



static void imx6_pcie_configure_type(struct imx6_pcie *imx6_pcie)

{

	unsigned int mask, val;



	if (imx6_pcie->drvdata->variant == IMX8MQ &&

	    imx6_pcie->controller_id == 1) {

		mask = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE;

		val  = FIELD_PREP(IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE,

				  PCI_EXP_TYPE_ROOT_PORT);

	} else {

		mask = IMX6Q_GPR12_DEVICE_TYPE;

		val  = FIELD_PREP(IMX6Q_GPR12_DEVICE_TYPE,

				  PCI_EXP_TYPE_ROOT_PORT);

	}



	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, mask, val);

}



static int pcie_phy_poll_ack(struct imx6_pcie *imx6_pcie, bool exp_val)

{

	struct dw_pcie *pci = imx6_pcie->pci;
@@ -271,6 +297,134 @@ static int pcie_phy_write(struct imx6_pcie *imx6_pcie, int addr, u16 data) 
	return 0;

}



static void imx6_pcie_init_phy(struct imx6_pcie *imx6_pcie)

{

	switch (imx6_pcie->drvdata->variant) {

	case IMX8MM:

		/*

		 * The PHY initialization had been done in the PHY

		 * driver, break here directly.

		 */

		break;

	case IMX8MQ:

		/*

		 * TODO: Currently this code assumes external

		 * oscillator is being used

		 */

		regmap_update_bits(imx6_pcie->iomuxc_gpr,

				   imx6_pcie_grp_offset(imx6_pcie),

				   IMX8MQ_GPR_PCIE_REF_USE_PAD,

				   IMX8MQ_GPR_PCIE_REF_USE_PAD);

		/*

		 * Regarding the datasheet, the PCIE_VPH is suggested

		 * to be 1.8V. If the PCIE_VPH is supplied by 3.3V, the

		 * VREG_BYPASS should be cleared to zero.

		 */

		if (imx6_pcie->vph &&

		    regulator_get_voltage(imx6_pcie->vph) > 3000000)

			regmap_update_bits(imx6_pcie->iomuxc_gpr,

					   imx6_pcie_grp_offset(imx6_pcie),

					   IMX8MQ_GPR_PCIE_VREG_BYPASS,

					   0);

		break;

	case IMX7D:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, 0);

		break;

	case IMX6SX:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6SX_GPR12_PCIE_RX_EQ_MASK,

				   IMX6SX_GPR12_PCIE_RX_EQ_2);

		fallthrough;

	default:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);



		/* configure constant input signal to the pcie ctrl and phy */

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6Q_GPR12_LOS_LEVEL, 9 << 4);



		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_DEEMPH_GEN1,

				   imx6_pcie->tx_deemph_gen1 << 0);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB,

				   imx6_pcie->tx_deemph_gen2_3p5db << 6);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB,

				   imx6_pcie->tx_deemph_gen2_6db << 12);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_SWING_FULL,

				   imx6_pcie->tx_swing_full << 18);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_SWING_LOW,

				   imx6_pcie->tx_swing_low << 25);

		break;

	}



	imx6_pcie_configure_type(imx6_pcie);

}



static void imx7d_pcie_wait_for_phy_pll_lock(struct imx6_pcie *imx6_pcie)

{

	u32 val;

	struct device *dev = imx6_pcie->pci->dev;



	if (regmap_read_poll_timeout(imx6_pcie->iomuxc_gpr,

				     IOMUXC_GPR22, val,

				     val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED,

				     PHY_PLL_LOCK_WAIT_USLEEP_MAX,

				     PHY_PLL_LOCK_WAIT_TIMEOUT))

		dev_err(dev, "PCIe PLL lock timeout\n");

}



static int imx6_setup_phy_mpll(struct imx6_pcie *imx6_pcie)

{

	unsigned long phy_rate = clk_get_rate(imx6_pcie->pcie_phy);

	int mult, div;

	u16 val;



	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY))

		return 0;



	switch (phy_rate) {

	case 125000000:

		/*

		 * The default settings of the MPLL are for a 125MHz input

		 * clock, so no need to reconfigure anything in that case.

		 */

		return 0;

	case 100000000:

		mult = 25;

		div = 0;

		break;

	case 200000000:

		mult = 25;

		div = 1;

		break;

	default:

		dev_err(imx6_pcie->pci->dev,

			"Unsupported PHY reference clock rate %lu\n", phy_rate);

		return -EINVAL;

	}



	pcie_phy_read(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val);

	val &= ~(PCIE_PHY_MPLL_MULTIPLIER_MASK <<

		 PCIE_PHY_MPLL_MULTIPLIER_SHIFT);

	val |= mult << PCIE_PHY_MPLL_MULTIPLIER_SHIFT;

	val |= PCIE_PHY_MPLL_MULTIPLIER_OVRD;

	pcie_phy_write(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val);



	pcie_phy_read(imx6_pcie, PCIE_PHY_ATEOVRD, &val);

	val &= ~(PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK <<

		 PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT);

	val |= div << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT;

	val |= PCIE_PHY_ATEOVRD_EN;

	pcie_phy_write(imx6_pcie, PCIE_PHY_ATEOVRD, val);



	return 0;

}



static void imx6_pcie_reset_phy(struct imx6_pcie *imx6_pcie)

{

	u16 tmp;
@@ -367,61 +521,6 @@ static int imx6_pcie_attach_pd(struct device *dev) 
	return 0;

}



static void imx6_pcie_assert_core_reset(struct imx6_pcie *imx6_pcie)

{

	struct device *dev = imx6_pcie->pci->dev;



	switch (imx6_pcie->drvdata->variant) {

	case IMX7D:

	case IMX8MQ:

		reset_control_assert(imx6_pcie->pciephy_reset);

		fallthrough;

	case IMX8MM:

		reset_control_assert(imx6_pcie->apps_reset);

		break;

	case IMX6SX:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN,

				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN);

		/* Force PCIe PHY reset */

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5,

				   IMX6SX_GPR5_PCIE_BTNRST_RESET,

				   IMX6SX_GPR5_PCIE_BTNRST_RESET);

		break;

	case IMX6QP:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,

				   IMX6Q_GPR1_PCIE_SW_RST,

				   IMX6Q_GPR1_PCIE_SW_RST);

		break;

	case IMX6Q:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,

				   IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,

				   IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16);

		break;

	}



	if (imx6_pcie->vpcie && regulator_is_enabled(imx6_pcie->vpcie) > 0) {

		int ret = regulator_disable(imx6_pcie->vpcie);



		if (ret)

			dev_err(dev, "failed to disable vpcie regulator: %d\n",

				ret);

	}



	/* Some boards don't have PCIe reset GPIO. */

	if (gpio_is_valid(imx6_pcie->reset_gpio))

		gpio_set_value_cansleep(imx6_pcie->reset_gpio,

					imx6_pcie->gpio_active_high);

}



static unsigned int imx6_pcie_grp_offset(const struct imx6_pcie *imx6_pcie)

{

	WARN_ON(imx6_pcie->drvdata->variant != IMX8MQ &&

		imx6_pcie->drvdata->variant != IMX8MM);

	return imx6_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14;

}



static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie)

{

	struct dw_pcie *pci = imx6_pcie->pci;
@@ -482,38 +581,44 @@ static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie) 
	return ret;

}



static void imx7d_pcie_wait_for_phy_pll_lock(struct imx6_pcie *imx6_pcie)

static void imx6_pcie_disable_ref_clk(struct imx6_pcie *imx6_pcie)

{

	u32 val;

	struct device *dev = imx6_pcie->pci->dev;



	if (regmap_read_poll_timeout(imx6_pcie->iomuxc_gpr,

				     IOMUXC_GPR22, val,

				     val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED,

				     PHY_PLL_LOCK_WAIT_USLEEP_MAX,

				     PHY_PLL_LOCK_WAIT_TIMEOUT))

		dev_err(dev, "PCIe PLL lock timeout\n");

	switch (imx6_pcie->drvdata->variant) {

	case IMX6SX:

		clk_disable_unprepare(imx6_pcie->pcie_inbound_axi);

		break;

	case IMX6QP:

	case IMX6Q:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,

				IMX6Q_GPR1_PCIE_REF_CLK_EN, 0);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,

				IMX6Q_GPR1_PCIE_TEST_PD,

				IMX6Q_GPR1_PCIE_TEST_PD);

		break;

	case IMX7D:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL,

				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL);

		break;

	case IMX8MM:

	case IMX8MQ:

		clk_disable_unprepare(imx6_pcie->pcie_aux);

		break;

	default:

		break;

	}

}



static void imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie)

static int imx6_pcie_clk_enable(struct imx6_pcie *imx6_pcie)

{

	struct dw_pcie *pci = imx6_pcie->pci;

	struct device *dev = pci->dev;

	int ret;



	if (imx6_pcie->vpcie && !regulator_is_enabled(imx6_pcie->vpcie)) {

		ret = regulator_enable(imx6_pcie->vpcie);

		if (ret) {

			dev_err(dev, "failed to enable vpcie regulator: %d\n",

				ret);

			return;

		}

	}



	ret = clk_prepare_enable(imx6_pcie->pcie_phy);

	if (ret) {

		dev_err(dev, "unable to enable pcie_phy clock\n");

		goto err_pcie_phy;

		return ret;

	}



	ret = clk_prepare_enable(imx6_pcie->pcie_bus);
@@ -534,24 +639,74 @@ static void imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie) 
		goto err_ref_clk;

	}



	switch (imx6_pcie->drvdata->variant) {

	case IMX8MM:

		if (phy_power_on(imx6_pcie->phy))

			dev_err(dev, "unable to power on PHY\n");

		break;

	default:

		break;

	}

	/* allow the clocks to stabilize */

	usleep_range(200, 500);

	return 0;



	switch (imx6_pcie->drvdata->variant) {

	case IMX8MQ:

		reset_control_deassert(imx6_pcie->pciephy_reset);

		break;

err_ref_clk:

	clk_disable_unprepare(imx6_pcie->pcie);

err_pcie:

	clk_disable_unprepare(imx6_pcie->pcie_bus);

err_pcie_bus:

	clk_disable_unprepare(imx6_pcie->pcie_phy);



	return ret;

}



static void imx6_pcie_clk_disable(struct imx6_pcie *imx6_pcie)

{

	imx6_pcie_disable_ref_clk(imx6_pcie);

	clk_disable_unprepare(imx6_pcie->pcie);

	clk_disable_unprepare(imx6_pcie->pcie_bus);

	clk_disable_unprepare(imx6_pcie->pcie_phy);

}



static void imx6_pcie_assert_core_reset(struct imx6_pcie *imx6_pcie)

{

	switch (imx6_pcie->drvdata->variant) {

	case IMX7D:

	case IMX8MQ:

		reset_control_assert(imx6_pcie->pciephy_reset);

		fallthrough;

	case IMX8MM:

		if (phy_init(imx6_pcie->phy))

			dev_err(dev, "waiting for phy ready timeout!\n");

		reset_control_assert(imx6_pcie->apps_reset);

		break;

	case IMX6SX:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN,

				   IMX6SX_GPR12_PCIE_TEST_POWERDOWN);

		/* Force PCIe PHY reset */

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR5,

				   IMX6SX_GPR5_PCIE_BTNRST_RESET,

				   IMX6SX_GPR5_PCIE_BTNRST_RESET);

		break;

	case IMX6QP:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,

				   IMX6Q_GPR1_PCIE_SW_RST,

				   IMX6Q_GPR1_PCIE_SW_RST);

		break;

	case IMX6Q:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,

				   IMX6Q_GPR1_PCIE_TEST_PD, 1 << 18);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR1,

				   IMX6Q_GPR1_PCIE_REF_CLK_EN, 0 << 16);

		break;

	}



	/* Some boards don't have PCIe reset GPIO. */

	if (gpio_is_valid(imx6_pcie->reset_gpio))

		gpio_set_value_cansleep(imx6_pcie->reset_gpio,

					imx6_pcie->gpio_active_high);

}



static int imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie)

{

	struct dw_pcie *pci = imx6_pcie->pci;

	struct device *dev = pci->dev;



	switch (imx6_pcie->drvdata->variant) {

	case IMX8MQ:

		reset_control_deassert(imx6_pcie->pciephy_reset);

		break;

	case IMX7D:

		reset_control_deassert(imx6_pcie->pciephy_reset);
@@ -588,6 +743,7 @@ static void imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie) 
		usleep_range(200, 500);

		break;

	case IMX6Q:		/* Nothing to do */

	case IMX8MM:

		break;

	}
@@ -600,153 +756,6 @@ static void imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie) 
		msleep(100);

	}



	return;



err_ref_clk:

	clk_disable_unprepare(imx6_pcie->pcie);

err_pcie:

	clk_disable_unprepare(imx6_pcie->pcie_bus);

err_pcie_bus:

	clk_disable_unprepare(imx6_pcie->pcie_phy);

err_pcie_phy:

	if (imx6_pcie->vpcie && regulator_is_enabled(imx6_pcie->vpcie) > 0) {

		ret = regulator_disable(imx6_pcie->vpcie);

		if (ret)

			dev_err(dev, "failed to disable vpcie regulator: %d\n",

				ret);

	}

}



static void imx6_pcie_configure_type(struct imx6_pcie *imx6_pcie)

{

	unsigned int mask, val;



	if (imx6_pcie->drvdata->variant == IMX8MQ &&

	    imx6_pcie->controller_id == 1) {

		mask   = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE;

		val    = FIELD_PREP(IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE,

				    PCI_EXP_TYPE_ROOT_PORT);

	} else {

		mask = IMX6Q_GPR12_DEVICE_TYPE;

		val  = FIELD_PREP(IMX6Q_GPR12_DEVICE_TYPE,

				  PCI_EXP_TYPE_ROOT_PORT);

	}



	regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12, mask, val);

}



static void imx6_pcie_init_phy(struct imx6_pcie *imx6_pcie)

{

	switch (imx6_pcie->drvdata->variant) {

	case IMX8MM:

		/*

		 * The PHY initialization had been done in the PHY

		 * driver, break here directly.

		 */

		break;

	case IMX8MQ:

		/*

		 * TODO: Currently this code assumes external

		 * oscillator is being used

		 */

		regmap_update_bits(imx6_pcie->iomuxc_gpr,

				   imx6_pcie_grp_offset(imx6_pcie),

				   IMX8MQ_GPR_PCIE_REF_USE_PAD,

				   IMX8MQ_GPR_PCIE_REF_USE_PAD);

		/*

		 * Regarding the datasheet, the PCIE_VPH is suggested

		 * to be 1.8V. If the PCIE_VPH is supplied by 3.3V, the

		 * VREG_BYPASS should be cleared to zero.

		 */

		if (imx6_pcie->vph &&

		    regulator_get_voltage(imx6_pcie->vph) > 3000000)

			regmap_update_bits(imx6_pcie->iomuxc_gpr,

					   imx6_pcie_grp_offset(imx6_pcie),

					   IMX8MQ_GPR_PCIE_VREG_BYPASS,

					   0);

		break;

	case IMX7D:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, 0);

		break;

	case IMX6SX:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6SX_GPR12_PCIE_RX_EQ_MASK,

				   IMX6SX_GPR12_PCIE_RX_EQ_2);

		fallthrough;

	default:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6Q_GPR12_PCIE_CTL_2, 0 << 10);



		/* configure constant input signal to the pcie ctrl and phy */

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6Q_GPR12_LOS_LEVEL, 9 << 4);



		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_DEEMPH_GEN1,

				   imx6_pcie->tx_deemph_gen1 << 0);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB,

				   imx6_pcie->tx_deemph_gen2_3p5db << 6);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB,

				   imx6_pcie->tx_deemph_gen2_6db << 12);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_SWING_FULL,

				   imx6_pcie->tx_swing_full << 18);

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR8,

				   IMX6Q_GPR8_TX_SWING_LOW,

				   imx6_pcie->tx_swing_low << 25);

		break;

	}



	imx6_pcie_configure_type(imx6_pcie);

}



static int imx6_setup_phy_mpll(struct imx6_pcie *imx6_pcie)

{

	unsigned long phy_rate = clk_get_rate(imx6_pcie->pcie_phy);

	int mult, div;

	u16 val;



	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_IMX6_PHY))

		return 0;



	switch (phy_rate) {

	case 125000000:

		/*

		 * The default settings of the MPLL are for a 125MHz input

		 * clock, so no need to reconfigure anything in that case.

		 */

		return 0;

	case 100000000:

		mult = 25;

		div = 0;

		break;

	case 200000000:

		mult = 25;

		div = 1;

		break;

	default:

		dev_err(imx6_pcie->pci->dev,

			"Unsupported PHY reference clock rate %lu\n", phy_rate);

		return -EINVAL;

	}



	pcie_phy_read(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val);

	val &= ~(PCIE_PHY_MPLL_MULTIPLIER_MASK <<

		 PCIE_PHY_MPLL_MULTIPLIER_SHIFT);

	val |= mult << PCIE_PHY_MPLL_MULTIPLIER_SHIFT;

	val |= PCIE_PHY_MPLL_MULTIPLIER_OVRD;

	pcie_phy_write(imx6_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val);



	pcie_phy_read(imx6_pcie, PCIE_PHY_ATEOVRD, &val);

	val &= ~(PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK <<

		 PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT);

	val |= div << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT;

	val |= PCIE_PHY_ATEOVRD_EN;

	pcie_phy_write(imx6_pcie, PCIE_PHY_ATEOVRD, val);



	return 0;

}
@@ -789,6 +798,25 @@ static void imx6_pcie_ltssm_enable(struct device *dev) 
	}

}



static void imx6_pcie_ltssm_disable(struct device *dev)

{

	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);



	switch (imx6_pcie->drvdata->variant) {

	case IMX6Q:

	case IMX6SX:

	case IMX6QP:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6Q_GPR12_PCIE_CTL_2, 0);

		break;

	case IMX7D:

	case IMX8MQ:

	case IMX8MM:

		reset_control_assert(imx6_pcie->apps_reset);

		break;

	}

}



static int imx6_pcie_start_link(struct dw_pcie *pci)

{

	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
@@ -802,21 +830,26 @@ static int imx6_pcie_start_link(struct dw_pcie *pci) 
	 * started in Gen2 mode, there is a possibility the devices on the

	 * bus will not be detected at all.  This happens with PCIe switches.

	 */

	dw_pcie_dbi_ro_wr_en(pci);

	tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);

	tmp &= ~PCI_EXP_LNKCAP_SLS;

	tmp |= PCI_EXP_LNKCAP_SLS_2_5GB;

	dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);

	dw_pcie_dbi_ro_wr_dis(pci);



	/* Start LTSSM. */

	imx6_pcie_ltssm_enable(dev);



	dw_pcie_wait_for_link(pci);

	ret = dw_pcie_wait_for_link(pci);

	if (ret)

		goto err_reset_phy;



	if (pci->link_gen == 2) {

		/* Allow Gen2 mode after the link is up. */

	if (pci->link_gen > 1) {

		/* Allow faster modes after the link is up */

		dw_pcie_dbi_ro_wr_en(pci);

		tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);

		tmp &= ~PCI_EXP_LNKCAP_SLS;

		tmp |= PCI_EXP_LNKCAP_SLS_5_0GB;

		tmp |= pci->link_gen;

		dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp);



		/*
@@ -826,6 +859,7 @@ static int imx6_pcie_start_link(struct dw_pcie *pci) 
		tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);

		tmp |= PORT_LOGIC_SPEED_CHANGE;

		dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp);

		dw_pcie_dbi_ro_wr_dis(pci);



		if (imx6_pcie->drvdata->flags &

		    IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE) {
@@ -846,34 +880,110 @@ static int imx6_pcie_start_link(struct dw_pcie *pci) 
		}



		/* Make sure link training is finished as well! */

		dw_pcie_wait_for_link(pci);

		ret = dw_pcie_wait_for_link(pci);

		if (ret)

			goto err_reset_phy;

	} else {

		dev_info(dev, "Link: Gen2 disabled\n");

		dev_info(dev, "Link: Only Gen1 is enabled\n");

	}



	imx6_pcie->link_is_up = true;

	tmp = dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKSTA);

	dev_info(dev, "Link up, Gen%i\n", tmp & PCI_EXP_LNKSTA_CLS);

	return 0;



err_reset_phy:

	imx6_pcie->link_is_up = false;

	dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n",

		dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0),

		dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1));

	imx6_pcie_reset_phy(imx6_pcie);

	return ret;

	return 0;

}



static void imx6_pcie_stop_link(struct dw_pcie *pci)

{

	struct device *dev = pci->dev;



	/* Turn off PCIe LTSSM */

	imx6_pcie_ltssm_disable(dev);

}



static int imx6_pcie_host_init(struct dw_pcie_rp *pp)

{

	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);

	struct device *dev = pci->dev;

	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);

	int ret;



	if (imx6_pcie->vpcie) {

		ret = regulator_enable(imx6_pcie->vpcie);

		if (ret) {

			dev_err(dev, "failed to enable vpcie regulator: %d\n",

				ret);

			return ret;

		}

	}



	imx6_pcie_assert_core_reset(imx6_pcie);

	imx6_pcie_init_phy(imx6_pcie);

	imx6_pcie_deassert_core_reset(imx6_pcie);



	ret = imx6_pcie_clk_enable(imx6_pcie);

	if (ret) {

		dev_err(dev, "unable to enable pcie clocks: %d\n", ret);

		goto err_reg_disable;

	}



	if (imx6_pcie->phy) {

		ret = phy_power_on(imx6_pcie->phy);

		if (ret) {

			dev_err(dev, "pcie PHY power up failed\n");

			goto err_clk_disable;

		}

	}



	ret = imx6_pcie_deassert_core_reset(imx6_pcie);

	if (ret < 0) {

		dev_err(dev, "pcie deassert core reset failed: %d\n", ret);

		goto err_phy_off;

	}



	if (imx6_pcie->phy) {

		ret = phy_init(imx6_pcie->phy);

		if (ret) {

			dev_err(dev, "waiting for PHY ready timeout!\n");

			goto err_phy_off;

		}

	}

	imx6_setup_phy_mpll(imx6_pcie);



	return 0;



err_phy_off:

	if (imx6_pcie->phy)

		phy_power_off(imx6_pcie->phy);

err_clk_disable:

	imx6_pcie_clk_disable(imx6_pcie);

err_reg_disable:

	if (imx6_pcie->vpcie)

		regulator_disable(imx6_pcie->vpcie);

	return ret;

}



static void imx6_pcie_host_exit(struct dw_pcie_rp *pp)

{

	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);

	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);



	if (imx6_pcie->phy) {

		if (phy_power_off(imx6_pcie->phy))

			dev_err(pci->dev, "unable to power off PHY\n");

		phy_exit(imx6_pcie->phy);

	}

	imx6_pcie_clk_disable(imx6_pcie);



	if (imx6_pcie->vpcie)

		regulator_disable(imx6_pcie->vpcie);

}



static const struct dw_pcie_host_ops imx6_pcie_host_ops = {
@@ -884,26 +994,6 @@ static const struct dw_pcie_ops dw_pcie_ops = { 
	.start_link = imx6_pcie_start_link,

};



#ifdef CONFIG_PM_SLEEP

static void imx6_pcie_ltssm_disable(struct device *dev)

{

	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);



	switch (imx6_pcie->drvdata->variant) {

	case IMX6SX:

	case IMX6QP:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX6Q_GPR12_PCIE_CTL_2, 0);

		break;

	case IMX7D:

	case IMX8MM:

		reset_control_assert(imx6_pcie->apps_reset);

		break;

	default:

		dev_err(dev, "ltssm_disable not supported\n");

	}

}



static void imx6_pcie_pm_turnoff(struct imx6_pcie *imx6_pcie)

{

	struct device *dev = imx6_pcie->pci->dev;
@@ -941,49 +1031,17 @@ static void imx6_pcie_pm_turnoff(struct imx6_pcie *imx6_pcie) 
	usleep_range(1000, 10000);

}



static void imx6_pcie_clk_disable(struct imx6_pcie *imx6_pcie)

{

	clk_disable_unprepare(imx6_pcie->pcie);

	clk_disable_unprepare(imx6_pcie->pcie_phy);

	clk_disable_unprepare(imx6_pcie->pcie_bus);



	switch (imx6_pcie->drvdata->variant) {

	case IMX6SX:

		clk_disable_unprepare(imx6_pcie->pcie_inbound_axi);

		break;

	case IMX7D:

		regmap_update_bits(imx6_pcie->iomuxc_gpr, IOMUXC_GPR12,

				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL,

				   IMX7D_GPR12_PCIE_PHY_REFCLK_SEL);

		break;

	case IMX8MQ:

	case IMX8MM:

		clk_disable_unprepare(imx6_pcie->pcie_aux);

		break;

	default:

		break;

	}

}



static int imx6_pcie_suspend_noirq(struct device *dev)

{

	struct imx6_pcie *imx6_pcie = dev_get_drvdata(dev);

	struct dw_pcie_rp *pp = &imx6_pcie->pci->pp;



	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND))

		return 0;



	imx6_pcie_pm_turnoff(imx6_pcie);

	imx6_pcie_ltssm_disable(dev);

	imx6_pcie_clk_disable(imx6_pcie);

	switch (imx6_pcie->drvdata->variant) {

	case IMX8MM:

		if (phy_power_off(imx6_pcie->phy))

			dev_err(dev, "unable to power off PHY\n");

		phy_exit(imx6_pcie->phy);

		break;

	default:

		break;

	}

	imx6_pcie_stop_link(imx6_pcie->pci);

	imx6_pcie_host_exit(pp);



	return 0;

}
@@ -997,21 +1055,19 @@ static int imx6_pcie_resume_noirq(struct device *dev) 
	if (!(imx6_pcie->drvdata->flags & IMX6_PCIE_FLAG_SUPPORTS_SUSPEND))

		return 0;



	imx6_pcie_assert_core_reset(imx6_pcie);

	imx6_pcie_init_phy(imx6_pcie);

	imx6_pcie_deassert_core_reset(imx6_pcie);

	ret = imx6_pcie_host_init(pp);

	if (ret)

		return ret;

	dw_pcie_setup_rc(pp);



	ret = imx6_pcie_start_link(imx6_pcie->pci);

	if (ret < 0)

		dev_info(dev, "pcie link is down after resume.\n");

	if (imx6_pcie->link_is_up)

		imx6_pcie_start_link(imx6_pcie->pci);



	return 0;

}

#endif



static const struct dev_pm_ops imx6_pcie_pm_ops = {

	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(imx6_pcie_suspend_noirq,

	NOIRQ_SYSTEM_SLEEP_PM_OPS(imx6_pcie_suspend_noirq,

				  imx6_pcie_resume_noirq)

};