Commit a5fcd228 authored by Dan Williams's avatar Dan Williams
Browse files

Merge branch 'for-6.3/cxl-rr-emu' into cxl/next 

Pick up the CXL DVSEC range register emulation for v6.3, and resolve
conflicts with the cxl_port_probe() split (from for-6.3/cxl-ram-region)
and event handling (from for-6.3/cxl-events).
parents 5a6fe61f 6980daaa

drivers/cxl/core/hdm.c

+108 −11
Original line number Diff line number Diff line
@@ -101,11 +101,34 @@ static int map_hdm_decoder_regs(struct cxl_port *port, void __iomem *crb, 
				      BIT(CXL_CM_CAP_CAP_ID_HDM));

}



static struct cxl_hdm *devm_cxl_setup_emulated_hdm(struct cxl_port *port,

						   struct cxl_endpoint_dvsec_info *info)

{

	struct device *dev = &port->dev;

	struct cxl_hdm *cxlhdm;



	if (!info->mem_enabled)

		return ERR_PTR(-ENODEV);



	cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);

	if (!cxlhdm)

		return ERR_PTR(-ENOMEM);



	cxlhdm->port = port;

	cxlhdm->decoder_count = info->ranges;

	cxlhdm->target_count = info->ranges;

	dev_set_drvdata(&port->dev, cxlhdm);



	return cxlhdm;

}



/**

 * devm_cxl_setup_hdm - map HDM decoder component registers

 * @port: cxl_port to map

 * @info: cached DVSEC range register info

 */

struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port)

struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,

				   struct cxl_endpoint_dvsec_info *info)

{

	struct device *dev = &port->dev;

	struct cxl_hdm *cxlhdm;
@@ -119,6 +142,9 @@ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port) 
	cxlhdm->port = port;

	crb = ioremap(port->component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE);

	if (!crb) {

		if (info && info->mem_enabled)

			return devm_cxl_setup_emulated_hdm(port, info);



		dev_err(dev, "No component registers mapped\n");

		return ERR_PTR(-ENXIO);

	}
@@ -688,9 +714,60 @@ static int cxl_decoder_reset(struct cxl_decoder *cxld) 
	return 0;

}



static int cxl_setup_hdm_decoder_from_dvsec(struct cxl_port *port,

					    struct cxl_decoder *cxld, int which,

					    struct cxl_endpoint_dvsec_info *info)

{

	if (!is_cxl_endpoint(port))

		return -EOPNOTSUPP;



	if (!range_len(&info->dvsec_range[which]))

		return -ENOENT;



	cxld->target_type = CXL_DECODER_EXPANDER;

	cxld->commit = NULL;

	cxld->reset = NULL;

	cxld->hpa_range = info->dvsec_range[which];



	/*

	 * Set the emulated decoder as locked pending additional support to

	 * change the range registers at run time.

	 */

	cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;

	port->commit_end = cxld->id;



	return 0;

}



static bool should_emulate_decoders(struct cxl_port *port)

{

	struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);

	void __iomem *hdm = cxlhdm->regs.hdm_decoder;

	u32 ctrl;

	int i;



	if (!is_cxl_endpoint(cxlhdm->port))

		return false;



	if (!hdm)

		return true;



	/*

	 * If any decoders are committed already, there should not be any

	 * emulated DVSEC decoders.

	 */

	for (i = 0; i < cxlhdm->decoder_count; i++) {

		ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));

		if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))

			return false;

	}



	return true;

}



static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,

			    int *target_map, void __iomem *hdm, int which,

			    u64 *dpa_base)

			    u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)

{

	struct cxl_endpoint_decoder *cxled = NULL;

	u64 size, base, skip, dpa_size;
@@ -703,6 +780,9 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 
		unsigned char target_id[8];

	} target_list;



	if (should_emulate_decoders(port))

		return cxl_setup_hdm_decoder_from_dvsec(port, cxld, which, info);



	if (is_endpoint_decoder(&cxld->dev))

		cxled = to_cxl_endpoint_decoder(&cxld->dev);
@@ -726,6 +806,9 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 
		.end = base + size - 1,

	};



	if (cxled && !committed && range_len(&info->dvsec_range[which]))

		return cxl_setup_hdm_decoder_from_dvsec(port, cxld, which, info);



	/* decoders are enabled if committed */

	if (committed) {

		cxld->flags |= CXL_DECODER_F_ENABLE;
@@ -798,18 +881,15 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 
	return 0;

}



/**

 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set

 * @cxlhdm: Structure to populate with HDM capabilities

 */

int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)

static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)

{

	void __iomem *hdm = cxlhdm->regs.hdm_decoder;

	struct cxl_port *port = cxlhdm->port;

	int i, committed;

	u64 dpa_base = 0;

	int committed, i;

	u32 ctrl;



	if (!hdm)

		return;



	/*

	 * Since the register resource was recently claimed via request_region()

	 * be careful about trusting the "not-committed" status until the commit
@@ -826,6 +906,22 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) 
	/* ensure that future checks of committed can be trusted */

	if (committed != cxlhdm->decoder_count)

		msleep(20);

}



/**

 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set

 * @cxlhdm: Structure to populate with HDM capabilities

 * @info: cached DVSEC range register info

 */

int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,

				struct cxl_endpoint_dvsec_info *info)

{

	void __iomem *hdm = cxlhdm->regs.hdm_decoder;

	struct cxl_port *port = cxlhdm->port;

	int i;

	u64 dpa_base = 0;



	cxl_settle_decoders(cxlhdm);



	for (i = 0; i < cxlhdm->decoder_count; i++) {

		int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
@@ -856,7 +952,8 @@ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm) 
			cxld = &cxlsd->cxld;

		}



		rc = init_hdm_decoder(port, cxld, target_map, hdm, i, &dpa_base);

		rc = init_hdm_decoder(port, cxld, target_map, hdm, i,

				      &dpa_base, info);

		if (rc) {

			dev_warn(&port->dev,

				 "Failed to initialize decoder%d.%d\n",

drivers/cxl/core/pci.c

+93 −107
Original line number Diff line number Diff line
@@ -142,11 +142,10 @@ int cxl_await_media_ready(struct cxl_dev_state *cxlds) 
}

EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, CXL);



static int wait_for_valid(struct cxl_dev_state *cxlds)

static int wait_for_valid(struct pci_dev *pdev, int d)

{

	struct pci_dev *pdev = to_pci_dev(cxlds->dev);

	int d = cxlds->cxl_dvsec, rc;

	u32 val;

	int rc;



	/*

	 * Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high
@@ -225,8 +224,6 @@ static int dvsec_range_allowed(struct device *dev, void *arg) 


	cxld = to_cxl_decoder(dev);



	if (!(cxld->flags & CXL_DECODER_F_LOCK))

		return 0;

	if (!(cxld->flags & CXL_DECODER_F_RAM))

		return 0;
@@ -256,94 +253,11 @@ static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm) 
	return devm_add_action_or_reset(host, disable_hdm, cxlhdm);

}



static bool __cxl_hdm_decode_init(struct cxl_dev_state *cxlds,

				  struct cxl_hdm *cxlhdm,

int cxl_dvsec_rr_decode(struct device *dev, int d,

			struct cxl_endpoint_dvsec_info *info)

{

	void __iomem *hdm = cxlhdm->regs.hdm_decoder;

	struct cxl_port *port = cxlhdm->port;

	struct device *dev = cxlds->dev;

	struct cxl_port *root;

	int i, rc, allowed;

	u32 global_ctrl;



	global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);



	/*

	 * If the HDM Decoder Capability is already enabled then assume

	 * that some other agent like platform firmware set it up.

	 */

	if (global_ctrl & CXL_HDM_DECODER_ENABLE) {

		rc = devm_cxl_enable_mem(&port->dev, cxlds);

		if (rc)

			return false;

		return true;

	}



	root = to_cxl_port(port->dev.parent);

	while (!is_cxl_root(root) && is_cxl_port(root->dev.parent))

		root = to_cxl_port(root->dev.parent);

	if (!is_cxl_root(root)) {

		dev_err(dev, "Failed to acquire root port for HDM enable\n");

		return false;

	}



	for (i = 0, allowed = 0; info->mem_enabled && i < info->ranges; i++) {

		struct device *cxld_dev;



		cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i],

					     dvsec_range_allowed);

		if (!cxld_dev) {

			dev_dbg(dev, "DVSEC Range%d denied by platform\n", i);

			continue;

		}

		dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i);

		put_device(cxld_dev);

		allowed++;

	}



	if (!allowed) {

		cxl_set_mem_enable(cxlds, 0);

		info->mem_enabled = 0;

	}



	/*

	 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base

	 * [High,Low] when HDM operation is enabled the range register values

	 * are ignored by the device, but the spec also recommends matching the

	 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges

	 * are expected even though Linux does not require or maintain that

	 * match. If at least one DVSEC range is enabled and allowed, skip HDM

	 * Decoder Capability Enable.

	 */

	if (info->mem_enabled)

		return false;



	rc = devm_cxl_enable_hdm(&port->dev, cxlhdm);

	if (rc)

		return false;



	rc = devm_cxl_enable_mem(&port->dev, cxlds);

	if (rc)

		return false;



	return true;

}



/**

 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint

 * @cxlds: Device state

 * @cxlhdm: Mapped HDM decoder Capability

 *

 * Try to enable the endpoint's HDM Decoder Capability

 */

int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm)

{

	struct pci_dev *pdev = to_pci_dev(cxlds->dev);

	struct cxl_endpoint_dvsec_info info = { 0 };

	struct pci_dev *pdev = to_pci_dev(dev);

	int hdm_count, rc, i, ranges = 0;

	struct device *dev = &pdev->dev;

	int d = cxlds->cxl_dvsec;

	u16 cap, ctrl;



	if (!d) {
@@ -374,7 +288,7 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm) 
	if (!hdm_count || hdm_count > 2)

		return -EINVAL;



	rc = wait_for_valid(cxlds);

	rc = wait_for_valid(pdev, d);

	if (rc) {

		dev_dbg(dev, "Failure awaiting MEM_INFO_VALID (%d)\n", rc);

		return rc;
@@ -385,9 +299,9 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm) 
	 * disabled, and they will remain moot after the HDM Decoder

	 * capability is enabled.

	 */

	info.mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl);

	if (!info.mem_enabled)

		goto hdm_init;

	info->mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl);

	if (!info->mem_enabled)

		return 0;



	for (i = 0; i < hdm_count; i++) {

		u64 base, size;
@@ -406,6 +320,13 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm) 
			return rc;



		size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK;

		if (!size) {

			info->dvsec_range[i] = (struct range) {

				.start = 0,

				.end = CXL_RESOURCE_NONE,

			};

			continue;

		}



		rc = pci_read_config_dword(

			pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp);
@@ -421,29 +342,94 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm) 


		base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK;



		info.dvsec_range[i] = (struct range) {

		info->dvsec_range[i] = (struct range) {

			.start = base,

			.end = base + size - 1

		};



		if (size)

		ranges++;

	}



	info.ranges = ranges;

	info->ranges = ranges;



	return 0;

}

EXPORT_SYMBOL_NS_GPL(cxl_dvsec_rr_decode, CXL);



/**

 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint

 * @cxlds: Device state

 * @cxlhdm: Mapped HDM decoder Capability

 * @info: Cached DVSEC range registers info

 *

 * Try to enable the endpoint's HDM Decoder Capability

 */

int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,

			struct cxl_endpoint_dvsec_info *info)

{

	void __iomem *hdm = cxlhdm->regs.hdm_decoder;

	struct cxl_port *port = cxlhdm->port;

	struct device *dev = cxlds->dev;

	struct cxl_port *root;

	int i, rc, allowed;

	u32 global_ctrl = 0;



	if (hdm)

		global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET);



	/*

	 * If DVSEC ranges are being used instead of HDM decoder registers there

	 * is no use in trying to manage those.

	 * If the HDM Decoder Capability is already enabled then assume

	 * that some other agent like platform firmware set it up.

	 */

hdm_init:

	if (!__cxl_hdm_decode_init(cxlds, cxlhdm, &info)) {

		dev_err(dev,

			"Legacy range registers configuration prevents HDM operation.\n");

		return -EBUSY;

	if (global_ctrl & CXL_HDM_DECODER_ENABLE || (!hdm && info->mem_enabled))

		return devm_cxl_enable_mem(&port->dev, cxlds);

	else if (!hdm)

		return -ENODEV;



	root = to_cxl_port(port->dev.parent);

	while (!is_cxl_root(root) && is_cxl_port(root->dev.parent))

		root = to_cxl_port(root->dev.parent);

	if (!is_cxl_root(root)) {

		dev_err(dev, "Failed to acquire root port for HDM enable\n");

		return -ENODEV;

	}



	for (i = 0, allowed = 0; info->mem_enabled && i < info->ranges; i++) {

		struct device *cxld_dev;



		cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i],

					     dvsec_range_allowed);

		if (!cxld_dev) {

			dev_dbg(dev, "DVSEC Range%d denied by platform\n", i);

			continue;

		}

		dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i);

		put_device(cxld_dev);

		allowed++;

	}



	if (!allowed) {

		cxl_set_mem_enable(cxlds, 0);

		info->mem_enabled = 0;

	}



	/*

	 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base

	 * [High,Low] when HDM operation is enabled the range register values

	 * are ignored by the device, but the spec also recommends matching the

	 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges

	 * are expected even though Linux does not require or maintain that

	 * match. If at least one DVSEC range is enabled and allowed, skip HDM

	 * Decoder Capability Enable.

	 */

	if (info->mem_enabled)

		return 0;



	rc = devm_cxl_enable_hdm(&port->dev, cxlhdm);

	if (rc)

		return rc;



	return devm_cxl_enable_mem(&port->dev, cxlds);

}

EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL);

drivers/cxl/cxl.h

+18 −2
Original line number Diff line number Diff line
@@ -693,10 +693,26 @@ int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map); 
int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);

int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);



/**

 * struct cxl_endpoint_dvsec_info - Cached DVSEC info

 * @mem_enabled: cached value of mem_enabled in the DVSEC, PCIE_DEVICE

 * @ranges: Number of active HDM ranges this device uses.

 * @dvsec_range: cached attributes of the ranges in the DVSEC, PCIE_DEVICE

 */

struct cxl_endpoint_dvsec_info {

	bool mem_enabled;

	int ranges;

	struct range dvsec_range[2];

};



struct cxl_hdm;

struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port);

int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm);

struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,

				   struct cxl_endpoint_dvsec_info *info);

int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,

				struct cxl_endpoint_dvsec_info *info);

int devm_cxl_add_passthrough_decoder(struct cxl_port *port);

int cxl_dvsec_rr_decode(struct device *dev, int dvsec,

			struct cxl_endpoint_dvsec_info *info);



bool is_cxl_region(struct device *dev);

drivers/cxl/cxlmem.h

+0 −12
Original line number Diff line number Diff line
@@ -187,18 +187,6 @@ static inline int cxl_mbox_cmd_rc2errno(struct cxl_mbox_cmd *mbox_cmd) 
 */

#define CXL_CAPACITY_MULTIPLIER SZ_256M



/**

 * struct cxl_endpoint_dvsec_info - Cached DVSEC info

 * @mem_enabled: cached value of mem_enabled in the DVSEC, PCIE_DEVICE

 * @ranges: Number of active HDM ranges this device uses.

 * @dvsec_range: cached attributes of the ranges in the DVSEC, PCIE_DEVICE

 */

struct cxl_endpoint_dvsec_info {

	bool mem_enabled;

	int ranges;

	struct range dvsec_range[2];

};



/**

 * Event Interrupt Policy

 *

drivers/cxl/cxlpci.h

+2 −1
Original line number Diff line number Diff line
@@ -70,7 +70,8 @@ enum cxl_regloc_type { 


int devm_cxl_port_enumerate_dports(struct cxl_port *port);

struct cxl_dev_state;

int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm);

int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,

			struct cxl_endpoint_dvsec_info *info);

void read_cdat_data(struct cxl_port *port);

void cxl_cor_error_detected(struct pci_dev *pdev);

pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,