Commit 586081d3 authored by Lu Baolu's avatar Lu Baolu Committed by Joerg Roedel
Browse files

iommu/vt-d: Remove DEFER_DEVICE_DOMAIN_INFO 



Allocate and set the per-device iommu private data during iommu device
probe. This makes the per-device iommu private data always available
during iommu_probe_device() and iommu_release_device(). With this changed,
the dummy DEFER_DEVICE_DOMAIN_INFO pointer could be removed. The wrappers
for getting the private data and domain are also cleaned.

Signed-off-by: default avatarLu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarJason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20220214025704.3184654-1-baolu.lu@linux.intel.com
Link: https://lore.kernel.org/r/20220301020159.633356-6-baolu.lu@linux.intel.com


Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent ee2653bb

drivers/iommu/intel/debugfs.c

+2 −1
Original line number Diff line number Diff line
@@ -344,7 +344,8 @@ static void pgtable_walk_level(struct seq_file *m, struct dma_pte *pde, 


static int show_device_domain_translation(struct device *dev, void *data)

{

	struct dmar_domain *domain = find_domain(dev);

	struct device_domain_info *info = dev_iommu_priv_get(dev);

	struct dmar_domain *domain = info->domain;

	struct seq_file *m = data;

	u64 path[6] = { 0 };

drivers/iommu/intel/iommu.c

+68 −118
Original line number Diff line number Diff line
@@ -342,21 +342,6 @@ static int iommu_skip_te_disable; 
int intel_iommu_gfx_mapped;

EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);



#define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))

struct device_domain_info *get_domain_info(struct device *dev)

{

	struct device_domain_info *info;



	if (!dev)

		return NULL;



	info = dev_iommu_priv_get(dev);

	if (unlikely(info == DEFER_DEVICE_DOMAIN_INFO))

		return NULL;



	return info;

}



DEFINE_SPINLOCK(device_domain_lock);

static LIST_HEAD(device_domain_list);
@@ -741,11 +726,6 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, 
	return &context[devfn];

}



static bool attach_deferred(struct device *dev)

{

	return dev_iommu_priv_get(dev) == DEFER_DEVICE_DOMAIN_INFO;

}



/**

 * is_downstream_to_pci_bridge - test if a device belongs to the PCI

 *				 sub-hierarchy of a candidate PCI-PCI bridge
@@ -2432,15 +2412,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 
	__iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);

}



static inline void unlink_domain_info(struct device_domain_info *info)

{

	assert_spin_locked(&device_domain_lock);

	list_del(&info->link);

	list_del(&info->global);

	if (info->dev)

		dev_iommu_priv_set(info->dev, NULL);

}



static void domain_remove_dev_info(struct dmar_domain *domain)

{

	struct device_domain_info *info, *tmp;
@@ -2452,24 +2423,6 @@ static void domain_remove_dev_info(struct dmar_domain *domain) 
	spin_unlock_irqrestore(&device_domain_lock, flags);

}



struct dmar_domain *find_domain(struct device *dev)

{

	struct device_domain_info *info;



	if (unlikely(!dev || !dev->iommu))

		return NULL;



	if (unlikely(attach_deferred(dev)))

		return NULL;



	/* No lock here, assumes no domain exit in normal case */

	info = get_domain_info(dev);

	if (likely(info))

		return info->domain;



	return NULL;

}



static inline struct device_domain_info *

dmar_search_domain_by_dev_info(int segment, int bus, int devfn)

{
@@ -2530,66 +2483,20 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, 
						    struct device *dev,

						    struct dmar_domain *domain)

{

	struct device_domain_info *info;

	struct device_domain_info *info = dev_iommu_priv_get(dev);

	unsigned long flags;

	int ret;



	info = kzalloc(sizeof(*info), GFP_KERNEL);

	if (!info)

		return NULL;



	if (!dev_is_real_dma_subdevice(dev)) {

		info->bus = bus;

		info->devfn = devfn;

		info->segment = iommu->segment;

	} else {

		struct pci_dev *pdev = to_pci_dev(dev);



		info->bus = pdev->bus->number;

		info->devfn = pdev->devfn;

		info->segment = pci_domain_nr(pdev->bus);

	}



	info->dev = dev;

	info->domain = domain;

	info->iommu = iommu;



	if (dev && dev_is_pci(dev)) {

		struct pci_dev *pdev = to_pci_dev(info->dev);



		if (ecap_dev_iotlb_support(iommu->ecap) &&

		    pci_ats_supported(pdev) &&

		    dmar_find_matched_atsr_unit(pdev))

			info->ats_supported = 1;



		if (sm_supported(iommu)) {

			if (pasid_supported(iommu)) {

				int features = pci_pasid_features(pdev);

				if (features >= 0)

					info->pasid_supported = features | 1;

			}



			if (info->ats_supported && ecap_prs(iommu->ecap) &&

			    pci_pri_supported(pdev))

				info->pri_supported = 1;

		}

	}



	spin_lock_irqsave(&device_domain_lock, flags);

	info->domain = domain;

	spin_lock(&iommu->lock);

	ret = domain_attach_iommu(domain, iommu);

	spin_unlock(&iommu->lock);



	if (ret) {

		spin_unlock_irqrestore(&device_domain_lock, flags);

		kfree(info);

		return NULL;

	}



	list_add(&info->link, &domain->devices);

	list_add(&info->global, &device_domain_list);

	if (dev)

		dev_iommu_priv_set(dev, info);

	spin_unlock_irqrestore(&device_domain_lock, flags);



	/* PASID table is mandatory for a PCI device in scalable mode. */
@@ -4336,13 +4243,11 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info) 
		intel_pasid_free_table(info->dev);

	}



	unlink_domain_info(info);

	list_del(&info->link);



	spin_lock_irqsave(&iommu->lock, flags);

	domain_detach_iommu(domain, iommu);

	spin_unlock_irqrestore(&iommu->lock, flags);



	kfree(info);

}



static void dmar_remove_one_dev_info(struct device *dev)
@@ -4351,7 +4256,7 @@ static void dmar_remove_one_dev_info(struct device *dev) 
	unsigned long flags;



	spin_lock_irqsave(&device_domain_lock, flags);

	info = get_domain_info(dev);

	info = dev_iommu_priv_get(dev);

	if (info)

		__dmar_remove_one_dev_info(info);

	spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -4475,10 +4380,9 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, 


	/* normally dev is not mapped */

	if (unlikely(domain_context_mapped(dev))) {

		struct dmar_domain *old_domain;

		struct device_domain_info *info = dev_iommu_priv_get(dev);



		old_domain = find_domain(dev);

		if (old_domain)

		if (info->domain)

			dmar_remove_one_dev_info(dev);

	}
@@ -4642,28 +4546,73 @@ static bool intel_iommu_capable(enum iommu_cap cap) 


static struct iommu_device *intel_iommu_probe_device(struct device *dev)

{

	struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;

	struct device_domain_info *info;

	struct intel_iommu *iommu;

	unsigned long flags;

	u8 bus, devfn;



	iommu = device_to_iommu(dev, NULL, NULL);

	iommu = device_to_iommu(dev, &bus, &devfn);

	if (!iommu)

		return ERR_PTR(-ENODEV);



	if (translation_pre_enabled(iommu))

		dev_iommu_priv_set(dev, DEFER_DEVICE_DOMAIN_INFO);

	info = kzalloc(sizeof(*info), GFP_KERNEL);

	if (!info)

		return ERR_PTR(-ENOMEM);



	if (dev_is_real_dma_subdevice(dev)) {

		info->bus = pdev->bus->number;

		info->devfn = pdev->devfn;

		info->segment = pci_domain_nr(pdev->bus);

	} else {

		info->bus = bus;

		info->devfn = devfn;

		info->segment = iommu->segment;

	}



	info->dev = dev;

	info->iommu = iommu;

	if (dev_is_pci(dev)) {

		if (ecap_dev_iotlb_support(iommu->ecap) &&

		    pci_ats_supported(pdev) &&

		    dmar_find_matched_atsr_unit(pdev))

			info->ats_supported = 1;



		if (sm_supported(iommu)) {

			if (pasid_supported(iommu)) {

				int features = pci_pasid_features(pdev);



				if (features >= 0)

					info->pasid_supported = features | 1;

			}



			if (info->ats_supported && ecap_prs(iommu->ecap) &&

			    pci_pri_supported(pdev))

				info->pri_supported = 1;

		}

	}



	spin_lock_irqsave(&device_domain_lock, flags);

	list_add(&info->global, &device_domain_list);

	dev_iommu_priv_set(dev, info);

	spin_unlock_irqrestore(&device_domain_lock, flags);



	return &iommu->iommu;

}



static void intel_iommu_release_device(struct device *dev)

{

	struct intel_iommu *iommu;



	iommu = device_to_iommu(dev, NULL, NULL);

	if (!iommu)

		return;

	struct device_domain_info *info = dev_iommu_priv_get(dev);

	unsigned long flags;



	dmar_remove_one_dev_info(dev);



	spin_lock_irqsave(&device_domain_lock, flags);

	dev_iommu_priv_set(dev, NULL);

	list_del(&info->global);

	spin_unlock_irqrestore(&device_domain_lock, flags);



	kfree(info);

	set_dma_ops(dev, NULL);

}
@@ -4732,14 +4681,14 @@ static void intel_iommu_get_resv_regions(struct device *device, 


int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)

{

	struct device_domain_info *info;

	struct device_domain_info *info = dev_iommu_priv_get(dev);

	struct context_entry *context;

	struct dmar_domain *domain;

	unsigned long flags;

	u64 ctx_lo;

	int ret;



	domain = find_domain(dev);

	domain = info->domain;

	if (!domain)

		return -EINVAL;
@@ -4747,8 +4696,7 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev) 
	spin_lock(&iommu->lock);



	ret = -EINVAL;

	info = get_domain_info(dev);

	if (!info || !info->pasid_supported)

	if (!info->pasid_supported)

		goto out;



	context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
@@ -4790,7 +4738,7 @@ static struct iommu_group *intel_iommu_device_group(struct device *dev) 


static int intel_iommu_enable_sva(struct device *dev)

{

	struct device_domain_info *info = get_domain_info(dev);

	struct device_domain_info *info = dev_iommu_priv_get(dev);

	struct intel_iommu *iommu;

	int ret;
@@ -4819,7 +4767,7 @@ static int intel_iommu_enable_sva(struct device *dev) 


static int intel_iommu_disable_sva(struct device *dev)

{

	struct device_domain_info *info = get_domain_info(dev);

	struct device_domain_info *info = dev_iommu_priv_get(dev);

	struct intel_iommu *iommu = info->iommu;

	int ret;
@@ -4832,7 +4780,7 @@ static int intel_iommu_disable_sva(struct device *dev) 


static int intel_iommu_enable_iopf(struct device *dev)

{

	struct device_domain_info *info = get_domain_info(dev);

	struct device_domain_info *info = dev_iommu_priv_get(dev);



	if (info && info->pri_supported)

		return 0;
@@ -4872,7 +4820,9 @@ intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) 


static bool intel_iommu_is_attach_deferred(struct device *dev)

{

	return attach_deferred(dev);

	struct device_domain_info *info = dev_iommu_priv_get(dev);



	return translation_pre_enabled(info->iommu) && !info->domain;

}



/*

drivers/iommu/intel/pasid.c

+6 −6
Original line number Diff line number Diff line
@@ -150,7 +150,7 @@ int intel_pasid_alloc_table(struct device *dev) 
	int size;



	might_sleep();

	info = get_domain_info(dev);

	info = dev_iommu_priv_get(dev);

	if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))

		return -EINVAL;
@@ -197,7 +197,7 @@ void intel_pasid_free_table(struct device *dev) 
	struct pasid_entry *table;

	int i, max_pde;



	info = get_domain_info(dev);

	info = dev_iommu_priv_get(dev);

	if (!info || !dev_is_pci(dev) || !info->pasid_table)

		return;
@@ -223,7 +223,7 @@ struct pasid_table *intel_pasid_get_table(struct device *dev) 
{

	struct device_domain_info *info;



	info = get_domain_info(dev);

	info = dev_iommu_priv_get(dev);

	if (!info)

		return NULL;
@@ -234,7 +234,7 @@ static int intel_pasid_get_dev_max_id(struct device *dev) 
{

	struct device_domain_info *info;



	info = get_domain_info(dev);

	info = dev_iommu_priv_get(dev);

	if (!info || !info->pasid_table)

		return 0;
@@ -254,7 +254,7 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid) 
		return NULL;



	dir = pasid_table->table;

	info = get_domain_info(dev);

	info = dev_iommu_priv_get(dev);

	dir_index = pasid >> PASID_PDE_SHIFT;

	index = pasid & PASID_PTE_MASK;
@@ -487,7 +487,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu, 
	struct device_domain_info *info;

	u16 sid, qdep, pfsid;



	info = get_domain_info(dev);

	info = dev_iommu_priv_get(dev);

	if (!info || !info->ats_enabled)

		return;

drivers/iommu/intel/svm.c

+3 −3
Original line number Diff line number Diff line
@@ -200,7 +200,7 @@ static void __flush_svm_range_dev(struct intel_svm *svm, 
				  unsigned long address,

				  unsigned long pages, int ih)

{

	struct device_domain_info *info = get_domain_info(sdev->dev);

	struct device_domain_info *info = dev_iommu_priv_get(sdev->dev);



	if (WARN_ON(!pages))

		return;
@@ -337,7 +337,7 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu, 
					   struct mm_struct *mm,

					   unsigned int flags)

{

	struct device_domain_info *info = get_domain_info(dev);

	struct device_domain_info *info = dev_iommu_priv_get(dev);

	unsigned long iflags, sflags;

	struct intel_svm_dev *sdev;

	struct intel_svm *svm;
@@ -545,7 +545,7 @@ static void intel_svm_drain_prq(struct device *dev, u32 pasid) 
	u16 sid, did;

	int qdep;



	info = get_domain_info(dev);

	info = dev_iommu_priv_get(dev);

	if (WARN_ON(!info || !dev_is_pci(dev)))

		return;

include/linux/intel-iommu.h

+0 −2
Original line number Diff line number Diff line
@@ -733,8 +733,6 @@ int for_each_device_domain(int (*fn)(struct device_domain_info *info, 
				     void *data), void *data);

void iommu_flush_write_buffer(struct intel_iommu *iommu);

int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev);

struct dmar_domain *find_domain(struct device *dev);

struct device_domain_info *get_domain_info(struct device *dev);

struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);



#ifdef CONFIG_INTEL_IOMMU_SVM