Commit c1c2d892 authored by Bjorn Helgaas's avatar Bjorn Helgaas
Browse files

Merge branch 'pci/aspm' 

- Save L1 PM Substates Capability across suspend/resume so L1SS keeps
  working after resume (Vidya Sagar)

- If device lacks L1 PM Substates Capability, don't read junk and treat it
  as such a Capability (Bjorn Helgaas)

- Fix the LTR_L1.2_THRESHOLD computation, which previously configured the
  threshold for entering L1.2 to be lower than intended, so L1.2 could be
  used when it shouldn't be (Bjorn Helgaas)

* pci/aspm:
  PCI/ASPM: Correct LTR_L1.2_THRESHOLD computation
  PCI/ASPM: Ignore L1 PM Substates if device lacks capability
  PCI/ASPM: Factor out L1 PM Substates configuration
  PCI/ASPM: Save L1 PM Substates Capability for suspend/resume
  PCI/ASPM: Refactor L1 PM Substates Control Register programming
parents 568035b0 7afeb84d

drivers/pci/pci.c

+7 −0
Original line number Diff line number Diff line
@@ -1663,6 +1663,7 @@ int pci_save_state(struct pci_dev *dev) 
		return i;



	pci_save_ltr_state(dev);

	pci_save_aspm_l1ss_state(dev);

	pci_save_dpc_state(dev);

	pci_save_aer_state(dev);

	pci_save_ptm_state(dev);
@@ -1769,6 +1770,7 @@ void pci_restore_state(struct pci_dev *dev) 
	 * LTR itself (in the PCIe capability).

	 */

	pci_restore_ltr_state(dev);

	pci_restore_aspm_l1ss_state(dev);



	pci_restore_pcie_state(dev);

	pci_restore_pasid_state(dev);
@@ -3485,6 +3487,11 @@ void pci_allocate_cap_save_buffers(struct pci_dev *dev) 
	if (error)

		pci_err(dev, "unable to allocate suspend buffer for LTR\n");



	error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_L1SS,

					    2 * sizeof(u32));

	if (error)

		pci_err(dev, "unable to allocate suspend buffer for ASPM-L1SS\n");



	pci_allocate_vc_save_buffers(dev);

}

drivers/pci/pci.h

+4 −0
Original line number Diff line number Diff line
@@ -561,10 +561,14 @@ bool pcie_wait_for_link(struct pci_dev *pdev, bool active); 
void pcie_aspm_init_link_state(struct pci_dev *pdev);

void pcie_aspm_exit_link_state(struct pci_dev *pdev);

void pcie_aspm_powersave_config_link(struct pci_dev *pdev);

void pci_save_aspm_l1ss_state(struct pci_dev *dev);

void pci_restore_aspm_l1ss_state(struct pci_dev *dev);

#else

static inline void pcie_aspm_init_link_state(struct pci_dev *pdev) { }

static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev) { }

static inline void pcie_aspm_powersave_config_link(struct pci_dev *pdev) { }

static inline void pci_save_aspm_l1ss_state(struct pci_dev *dev) { }

static inline void pci_restore_aspm_l1ss_state(struct pci_dev *dev) { }

#endif



#ifdef CONFIG_PCIE_ECRC

drivers/pci/pcie/aspm.c

+163 −95
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 
 */



#include <linux/kernel.h>

#include <linux/math.h>

#include <linux/module.h>

#include <linux/moduleparam.h>

#include <linux/pci.h>
@@ -350,29 +351,43 @@ static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) 
	return 0;

}



/*

 * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1

 * register.  Ports enter L1.2 when the most recent LTR value is greater

 * than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we

 * don't enter L1.2 too aggressively.

 *

 * See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3.

 */

static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)

{

	u32 threshold_ns = threshold_us * 1000;

	u64 threshold_ns = (u64) threshold_us * 1000;



	/* See PCIe r3.1, sec 7.33.3 and sec 6.18 */

	if (threshold_ns < 32) {

		*scale = 0;

	/*

	 * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max

	 * value of 0x3ff.

	 */

	if (threshold_ns <= 0x3ff * 1) {

		*scale = 0;		/* Value times 1ns */

		*value = threshold_ns;

	} else if (threshold_ns < 1024) {

		*scale = 1;

		*value = threshold_ns >> 5;

	} else if (threshold_ns < 32768) {

		*scale = 2;

		*value = threshold_ns >> 10;

	} else if (threshold_ns < 1048576) {

		*scale = 3;

		*value = threshold_ns >> 15;

	} else if (threshold_ns < 33554432) {

		*scale = 4;

		*value = threshold_ns >> 20;

	} else if (threshold_ns <= 0x3ff * 32) {

		*scale = 1;		/* Value times 32ns */

		*value = roundup(threshold_ns, 32) / 32;

	} else if (threshold_ns <= 0x3ff * 1024) {

		*scale = 2;		/* Value times 1024ns */

		*value = roundup(threshold_ns, 1024) / 1024;

	} else if (threshold_ns <= 0x3ff * 32768) {

		*scale = 3;		/* Value times 32768ns */

		*value = roundup(threshold_ns, 32768) / 32768;

	} else if (threshold_ns <= 0x3ff * 1048576) {

		*scale = 4;		/* Value times 1048576ns */

		*value = roundup(threshold_ns, 1048576) / 1048576;

	} else if (threshold_ns <= 0x3ff * (u64) 33554432) {

		*scale = 5;		/* Value times 33554432ns */

		*value = roundup(threshold_ns, 33554432) / 33554432;

	} else {

		*scale = 5;

		*value = threshold_ns >> 25;

		*value = 0x3ff;		/* Max representable value */

	}

}
@@ -455,6 +470,31 @@ static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, 
	pci_write_config_dword(pdev, pos, val);

}



static void aspm_program_l1ss(struct pci_dev *dev, u32 ctl1, u32 ctl2)

{

	u16 l1ss = dev->l1ss;

	u32 l1_2_enable;



	/*

	 * Per PCIe r6.0, sec 5.5.4, T_POWER_ON in PCI_L1SS_CTL2 must be

	 * programmed prior to setting the L1.2 enable bits in PCI_L1SS_CTL1.

	 */

	pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL2, ctl2);



	/*

	 * In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD in

	 * PCI_L1SS_CTL1 must be programmed *before* setting the L1.2

	 * enable bits, even though they're all in PCI_L1SS_CTL1.

	 */

	l1_2_enable = ctl1 & PCI_L1SS_CTL1_L1_2_MASK;

	ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK;



	pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1, ctl1);

	if (l1_2_enable)

		pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1,

				       ctl1 | l1_2_enable);

}



/* Calculate L1.2 PM substate timing parameters */

static void aspm_calc_l1ss_info(struct pcie_link_state *link,

				u32 parent_l1ss_cap, u32 child_l1ss_cap)
@@ -464,7 +504,6 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, 
	u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;

	u32 ctl1 = 0, ctl2 = 0;

	u32 pctl1, pctl2, cctl1, cctl2;

	u32 pl1_2_enables, cl1_2_enables;



	if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))

		return;
@@ -513,39 +552,78 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, 
	    ctl2 == pctl2 && ctl2 == cctl2)

		return;



	/* Disable L1.2 while updating.  See PCIe r5.0, sec 5.5.4, 7.8.3.3 */

	pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;

	cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;

	pctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |

		   PCI_L1SS_CTL1_LTR_L12_TH_VALUE |

		   PCI_L1SS_CTL1_LTR_L12_TH_SCALE);

	pctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |

			  PCI_L1SS_CTL1_LTR_L12_TH_VALUE |

			  PCI_L1SS_CTL1_LTR_L12_TH_SCALE));

	aspm_program_l1ss(parent, pctl1, ctl2);



	if (pl1_2_enables || cl1_2_enables) {

		pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,

					PCI_L1SS_CTL1_L1_2_MASK, 0);

		pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,

					PCI_L1SS_CTL1_L1_2_MASK, 0);

	cctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME |

		   PCI_L1SS_CTL1_LTR_L12_TH_VALUE |

		   PCI_L1SS_CTL1_LTR_L12_TH_SCALE);

	cctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME |

			  PCI_L1SS_CTL1_LTR_L12_TH_VALUE |

			  PCI_L1SS_CTL1_LTR_L12_TH_SCALE));

	aspm_program_l1ss(child, cctl1, ctl2);

}



	/* Program T_POWER_ON times in both ports */

	pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);

	pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);

static void aspm_l1ss_init(struct pcie_link_state *link)

{

	struct pci_dev *child = link->downstream, *parent = link->pdev;

	u32 parent_l1ss_cap, child_l1ss_cap;

	u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;



	/* Program Common_Mode_Restore_Time in upstream device */

	pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,

				PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);

	if (!parent->l1ss || !child->l1ss)

		return;



	/* Program LTR_L1.2_THRESHOLD time in both ports */

	pci_clear_and_set_dword(parent,	parent->l1ss + PCI_L1SS_CTL1,

				PCI_L1SS_CTL1_LTR_L12_TH_VALUE |

				PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);

	pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,

				PCI_L1SS_CTL1_LTR_L12_TH_VALUE |

				PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);

	/* Setup L1 substate */

	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,

			      &parent_l1ss_cap);

	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,

			      &child_l1ss_cap);



	if (pl1_2_enables || cl1_2_enables) {

		pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0,

					pl1_2_enables);

		pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0,

					cl1_2_enables);

	}

	if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))

		parent_l1ss_cap = 0;

	if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))

		child_l1ss_cap = 0;



	/*

	 * If we don't have LTR for the entire path from the Root Complex

	 * to this device, we can't use ASPM L1.2 because it relies on the

	 * LTR_L1.2_THRESHOLD.  See PCIe r4.0, secs 5.5.4, 6.18.

	 */

	if (!child->ltr_path)

		child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;



	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)

		link->aspm_support |= ASPM_STATE_L1_1;

	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)

		link->aspm_support |= ASPM_STATE_L1_2;

	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)

		link->aspm_support |= ASPM_STATE_L1_1_PCIPM;

	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)

		link->aspm_support |= ASPM_STATE_L1_2_PCIPM;



	if (parent_l1ss_cap)

		pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,

				      &parent_l1ss_ctl1);

	if (child_l1ss_cap)

		pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,

				      &child_l1ss_ctl1);



	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)

		link->aspm_enabled |= ASPM_STATE_L1_1;

	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)

		link->aspm_enabled |= ASPM_STATE_L1_2;

	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)

		link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;

	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)

		link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;



	if (link->aspm_support & ASPM_STATE_L1SS)

		aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);

}



static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
@@ -553,8 +631,6 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 
	struct pci_dev *child = link->downstream, *parent = link->pdev;

	u32 parent_lnkcap, child_lnkcap;

	u16 parent_lnkctl, child_lnkctl;

	u32 parent_l1ss_cap, child_l1ss_cap;

	u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;

	struct pci_bus *linkbus = parent->subordinate;



	if (blacklist) {
@@ -609,52 +685,7 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 
	if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)

		link->aspm_enabled |= ASPM_STATE_L1;



	/* Setup L1 substate */

	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,

			      &parent_l1ss_cap);

	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,

			      &child_l1ss_cap);



	if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))

		parent_l1ss_cap = 0;

	if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))

		child_l1ss_cap = 0;



	/*

	 * If we don't have LTR for the entire path from the Root Complex

	 * to this device, we can't use ASPM L1.2 because it relies on the

	 * LTR_L1.2_THRESHOLD.  See PCIe r4.0, secs 5.5.4, 6.18.

	 */

	if (!child->ltr_path)

		child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;



	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)

		link->aspm_support |= ASPM_STATE_L1_1;

	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)

		link->aspm_support |= ASPM_STATE_L1_2;

	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)

		link->aspm_support |= ASPM_STATE_L1_1_PCIPM;

	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)

		link->aspm_support |= ASPM_STATE_L1_2_PCIPM;



	if (parent_l1ss_cap)

		pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,

				      &parent_l1ss_ctl1);

	if (child_l1ss_cap)

		pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,

				      &child_l1ss_ctl1);



	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)

		link->aspm_enabled |= ASPM_STATE_L1_1;

	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)

		link->aspm_enabled |= ASPM_STATE_L1_2;

	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)

		link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;

	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)

		link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;



	if (link->aspm_support & ASPM_STATE_L1SS)

		aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);

	aspm_l1ss_init(link);



	/* Save default state */

	link->aspm_default = link->aspm_enabled;
@@ -726,6 +757,43 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) 
				PCI_L1SS_CTL1_L1SS_MASK, val);

}



void pci_save_aspm_l1ss_state(struct pci_dev *dev)

{

	struct pci_cap_saved_state *save_state;

	u16 l1ss = dev->l1ss;

	u32 *cap;



	if (!l1ss)

		return;



	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS);

	if (!save_state)

		return;



	cap = (u32 *)&save_state->cap.data[0];

	pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL2, cap++);

	pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL1, cap++);

}



void pci_restore_aspm_l1ss_state(struct pci_dev *dev)

{

	struct pci_cap_saved_state *save_state;

	u32 *cap, ctl1, ctl2;

	u16 l1ss = dev->l1ss;



	if (!l1ss)

		return;



	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS);

	if (!save_state)

		return;



	cap = (u32 *)&save_state->cap.data[0];

	ctl2 = *cap++;

	ctl1 = *cap;

	aspm_program_l1ss(dev, ctl1, ctl2);

}



static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)

{

	pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,