Commit 13eaa5bd authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'iommu-updates-v5.17' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu 

Pull iommu updates from Joerg Roedel:

 - Identity domain support for virtio-iommu

 - Move flush queue code into iommu-dma

 - Some fixes for AMD IOMMU suspend/resume support when x2apic is used

 - Arm SMMU Updates from Will Deacon:
      - Revert evtq and priq back to their former sizes
      - Return early on short-descriptor page-table allocation failure
      - Fix page fault reporting for Adreno GPU on SMMUv2
      - Make SMMUv3 MMU notifier ops 'const'
      - Numerous new compatible strings for Qualcomm SMMUv2 implementations

 - Various smaller fixes and cleanups

* tag 'iommu-updates-v5.17' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (38 commits)
  iommu/iova: Temporarily include dma-mapping.h from iova.h
  iommu: Move flush queue data into iommu_dma_cookie
  iommu/iova: Move flush queue code to iommu-dma
  iommu/iova: Consolidate flush queue code
  iommu/vt-d: Use put_pages_list
  iommu/amd: Use put_pages_list
  iommu/amd: Simplify pagetable freeing
  iommu/iova: Squash flush_cb abstraction
  iommu/iova: Squash entry_dtor abstraction
  iommu/iova: Fix race between FQ timeout and teardown
  iommu/amd: Fix typo in *glues … together* in comment
  iommu/vt-d: Remove unused dma_to_mm_pfn function
  iommu/vt-d: Drop duplicate check in dma_pte_free_pagetable()
  iommu/vt-d: Use bitmap_zalloc() when applicable
  iommu/amd: Remove useless irq affinity notifier
  iommu/amd: X2apic mode: mask/unmask interrupts on suspend/resume
  iommu/amd: X2apic mode: setup the INTX registers on mask/unmask
  iommu/amd: X2apic mode: re-enable after resume
  iommu/amd: Restore GA log/tail pointer on host resume
  iommu/iova: Move fast alloc size roundup into alloc_iova_fast()
  ...
parents 362f533a 66dc1b79

Documentation/devicetree/bindings/iommu/arm,smmu.yaml

+2 −0
Original line number Diff line number Diff line
@@ -38,10 +38,12 @@ properties: 
              - qcom,sc7280-smmu-500

              - qcom,sc8180x-smmu-500

              - qcom,sdm845-smmu-500

              - qcom,sdx55-smmu-500

              - qcom,sm6350-smmu-500

              - qcom,sm8150-smmu-500

              - qcom,sm8250-smmu-500

              - qcom,sm8350-smmu-500

              - qcom,sm8450-smmu-500

          - const: arm,mmu-500

      - description: Qcom Adreno GPUs implementing "arm,smmu-v2"

        items:

drivers/iommu/amd/amd_iommu_types.h

+0 −2
Original line number Diff line number Diff line
@@ -645,8 +645,6 @@ struct amd_iommu { 
	/* DebugFS Info */

	struct dentry *debugfs;

#endif

	/* IRQ notifier for IntCapXT interrupt */

	struct irq_affinity_notify intcapxt_notify;

};



static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)

drivers/iommu/amd/init.c

+59 −50
Original line number Diff line number Diff line
@@ -806,16 +806,27 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) 
{

#ifdef CONFIG_IRQ_REMAP

	u32 status, i;

	u64 entry;



	if (!iommu->ga_log)

		return -EINVAL;



	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);



	/* Check if already running */

	if (status & (MMIO_STATUS_GALOG_RUN_MASK))

	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

	if (WARN_ON(status & (MMIO_STATUS_GALOG_RUN_MASK)))

		return 0;



	entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;

	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,

		    &entry, sizeof(entry));

	entry = (iommu_virt_to_phys(iommu->ga_log_tail) &

		 (BIT_ULL(52)-1)) & ~7ULL;

	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,

		    &entry, sizeof(entry));

	writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

	writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);





	iommu_feature_enable(iommu, CONTROL_GAINT_EN);

	iommu_feature_enable(iommu, CONTROL_GALOG_EN);
@@ -825,7 +836,7 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) 
			break;

	}



	if (i >= LOOP_TIMEOUT)

	if (WARN_ON(i >= LOOP_TIMEOUT))

		return -EINVAL;

#endif /* CONFIG_IRQ_REMAP */

	return 0;
@@ -834,8 +845,6 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) 
static int iommu_init_ga_log(struct amd_iommu *iommu)

{

#ifdef CONFIG_IRQ_REMAP

	u64 entry;



	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))

		return 0;
@@ -849,16 +858,6 @@ static int iommu_init_ga_log(struct amd_iommu *iommu) 
	if (!iommu->ga_log_tail)

		goto err_out;



	entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;

	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,

		    &entry, sizeof(entry));

	entry = (iommu_virt_to_phys(iommu->ga_log_tail) &

		 (BIT_ULL(52)-1)) & ~7ULL;

	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,

		    &entry, sizeof(entry));

	writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

	writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);



	return 0;

err_out:

	free_ga_log(iommu);
@@ -1523,7 +1522,7 @@ static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) 
}



/*

 * This function clues the initialization function for one IOMMU

 * This function glues the initialization function for one IOMMU

 * together and also allocates the command buffer and programs the

 * hardware. It does NOT enable the IOMMU. This is done afterwards.

 */
@@ -2016,48 +2015,18 @@ union intcapxt { 
	};

} __attribute__ ((packed));



/*

 * There isn't really any need to mask/unmask at the irqchip level because

 * the 64-bit INTCAPXT registers can be updated atomically without tearing

 * when the affinity is being updated.

 */

static void intcapxt_unmask_irq(struct irq_data *data)

{

}



static void intcapxt_mask_irq(struct irq_data *data)

{

}



static struct irq_chip intcapxt_controller;



static int intcapxt_irqdomain_activate(struct irq_domain *domain,

				       struct irq_data *irqd, bool reserve)

{

	struct amd_iommu *iommu = irqd->chip_data;

	struct irq_cfg *cfg = irqd_cfg(irqd);

	union intcapxt xt;



	xt.capxt = 0ULL;

	xt.dest_mode_logical = apic->dest_mode_logical;

	xt.vector = cfg->vector;

	xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);

	xt.destid_24_31 = cfg->dest_apicid >> 24;



	/**

	 * Current IOMMU implemtation uses the same IRQ for all

	 * 3 IOMMU interrupts.

	 */

	writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);

	writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);

	writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);

	return 0;

}



static void intcapxt_irqdomain_deactivate(struct irq_domain *domain,

					  struct irq_data *irqd)

{

	intcapxt_mask_irq(irqd);

}
@@ -2091,6 +2060,38 @@ static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq 
	irq_domain_free_irqs_top(domain, virq, nr_irqs);

}





static void intcapxt_unmask_irq(struct irq_data *irqd)

{

	struct amd_iommu *iommu = irqd->chip_data;

	struct irq_cfg *cfg = irqd_cfg(irqd);

	union intcapxt xt;



	xt.capxt = 0ULL;

	xt.dest_mode_logical = apic->dest_mode_logical;

	xt.vector = cfg->vector;

	xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);

	xt.destid_24_31 = cfg->dest_apicid >> 24;



	/**

	 * Current IOMMU implementation uses the same IRQ for all

	 * 3 IOMMU interrupts.

	 */

	writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);

	writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);

	writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);

}



static void intcapxt_mask_irq(struct irq_data *irqd)

{

	struct amd_iommu *iommu = irqd->chip_data;



	writeq(0, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);

	writeq(0, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET);

	writeq(0, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET);

}





static int intcapxt_set_affinity(struct irq_data *irqd,

				 const struct cpumask *mask, bool force)

{
@@ -2100,8 +2101,12 @@ static int intcapxt_set_affinity(struct irq_data *irqd, 
	ret = parent->chip->irq_set_affinity(parent, mask, force);

	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)

		return ret;

	return 0;

}



	return intcapxt_irqdomain_activate(irqd->domain, irqd, false);

static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on)

{

	return on ? -EOPNOTSUPP : 0;

}



static struct irq_chip intcapxt_controller = {
@@ -2111,7 +2116,8 @@ static struct irq_chip intcapxt_controller = { 
	.irq_ack		= irq_chip_ack_parent,

	.irq_retrigger		= irq_chip_retrigger_hierarchy,

	.irq_set_affinity       = intcapxt_set_affinity,

	.flags			= IRQCHIP_SKIP_SET_WAKE,

	.irq_set_wake		= intcapxt_set_wake,

	.flags			= IRQCHIP_MASK_ON_SUSPEND,

};



static const struct irq_domain_ops intcapxt_domain_ops = {
@@ -2173,7 +2179,6 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu) 
		return ret;

	}



	iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);

	return 0;

}
@@ -2196,6 +2201,10 @@ static int iommu_init_irq(struct amd_iommu *iommu) 


	iommu->int_enabled = true;

enable_faults:



	if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)

		iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);



	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);



	if (iommu->ppr_log != NULL)

drivers/iommu/amd/io_pgtable.c

+41 −69
Original line number Diff line number Diff line
@@ -74,87 +74,61 @@ static u64 *first_pte_l7(u64 *pte, unsigned long *page_size, 
 *

 ****************************************************************************/



static void free_page_list(struct page *freelist)

static void free_pt_page(u64 *pt, struct list_head *freelist)

{

	while (freelist != NULL) {

		unsigned long p = (unsigned long)page_address(freelist);

	struct page *p = virt_to_page(pt);



		freelist = freelist->freelist;

		free_page(p);

	}

	list_add_tail(&p->lru, freelist);

}



static struct page *free_pt_page(unsigned long pt, struct page *freelist)

static void free_pt_lvl(u64 *pt, struct list_head *freelist, int lvl)

{

	struct page *p = virt_to_page((void *)pt);

	u64 *p;

	int i;



	p->freelist = freelist;

	for (i = 0; i < 512; ++i) {

		/* PTE present? */

		if (!IOMMU_PTE_PRESENT(pt[i]))

			continue;



	return p;

}

		/* Large PTE? */

		if (PM_PTE_LEVEL(pt[i]) == 0 ||

		    PM_PTE_LEVEL(pt[i]) == 7)

			continue;



#define DEFINE_FREE_PT_FN(LVL, FN)						\

static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist)	\

{										\

	unsigned long p;							\

	u64 *pt;								\

	int i;									\

										\

	pt = (u64 *)__pt;							\

										\

	for (i = 0; i < 512; ++i) {						\

		/* PTE present? */						\

		if (!IOMMU_PTE_PRESENT(pt[i]))					\

			continue;						\

										\

		/* Large PTE? */						\

		if (PM_PTE_LEVEL(pt[i]) == 0 ||					\

		    PM_PTE_LEVEL(pt[i]) == 7)					\

			continue;						\

										\

		p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);			\

		freelist = FN(p, freelist);					\

	}									\

										\

	return free_pt_page((unsigned long)pt, freelist);			\

		/*

		 * Free the next level. No need to look at l1 tables here since

		 * they can only contain leaf PTEs; just free them directly.

		 */

		p = IOMMU_PTE_PAGE(pt[i]);

		if (lvl > 2)

			free_pt_lvl(p, freelist, lvl - 1);

		else

			free_pt_page(p, freelist);

	}



DEFINE_FREE_PT_FN(l2, free_pt_page)

DEFINE_FREE_PT_FN(l3, free_pt_l2)

DEFINE_FREE_PT_FN(l4, free_pt_l3)

DEFINE_FREE_PT_FN(l5, free_pt_l4)

DEFINE_FREE_PT_FN(l6, free_pt_l5)

	free_pt_page(pt, freelist);

}



static struct page *free_sub_pt(unsigned long root, int mode,

				struct page *freelist)

static void free_sub_pt(u64 *root, int mode, struct list_head *freelist)

{

	switch (mode) {

	case PAGE_MODE_NONE:

	case PAGE_MODE_7_LEVEL:

		break;

	case PAGE_MODE_1_LEVEL:

		freelist = free_pt_page(root, freelist);

		free_pt_page(root, freelist);

		break;

	case PAGE_MODE_2_LEVEL:

		freelist = free_pt_l2(root, freelist);

		break;

	case PAGE_MODE_3_LEVEL:

		freelist = free_pt_l3(root, freelist);

		break;

	case PAGE_MODE_4_LEVEL:

		freelist = free_pt_l4(root, freelist);

		break;

	case PAGE_MODE_5_LEVEL:

		freelist = free_pt_l5(root, freelist);

		break;

	case PAGE_MODE_6_LEVEL:

		freelist = free_pt_l6(root, freelist);

		free_pt_lvl(root, freelist, mode);

		break;

	default:

		BUG();

	}



	return freelist;

}



void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
@@ -362,9 +336,9 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable, 
	return pte;

}



static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)

static void free_clear_pte(u64 *pte, u64 pteval, struct list_head *freelist)

{

	unsigned long pt;

	u64 *pt;

	int mode;



	while (cmpxchg64(pte, pteval, 0) != pteval) {
@@ -373,12 +347,12 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) 
	}



	if (!IOMMU_PTE_PRESENT(pteval))

		return freelist;

		return;



	pt   = (unsigned long)IOMMU_PTE_PAGE(pteval);

	pt   = IOMMU_PTE_PAGE(pteval);

	mode = IOMMU_PTE_MODE(pteval);



	return free_sub_pt(pt, mode, freelist);

	free_sub_pt(pt, mode, freelist);

}



/*
@@ -392,7 +366,7 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, 
			  phys_addr_t paddr, size_t size, int prot, gfp_t gfp)

{

	struct protection_domain *dom = io_pgtable_ops_to_domain(ops);

	struct page *freelist = NULL;

	LIST_HEAD(freelist);

	bool updated = false;

	u64 __pte, *pte;

	int ret, i, count;
@@ -412,9 +386,9 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, 
		goto out;



	for (i = 0; i < count; ++i)

		freelist = free_clear_pte(&pte[i], pte[i], freelist);

		free_clear_pte(&pte[i], pte[i], &freelist);



	if (freelist != NULL)

	if (!list_empty(&freelist))

		updated = true;



	if (count > 1) {
@@ -449,7 +423,7 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, 
	}



	/* Everything flushed out, free pages now */

	free_page_list(freelist);

	put_pages_list(&freelist);



	return ret;

}
@@ -511,8 +485,7 @@ static void v1_free_pgtable(struct io_pgtable *iop) 
{

	struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop);

	struct protection_domain *dom;

	struct page *freelist = NULL;

	unsigned long root;

	LIST_HEAD(freelist);



	if (pgtable->mode == PAGE_MODE_NONE)

		return;
@@ -529,10 +502,9 @@ static void v1_free_pgtable(struct io_pgtable *iop) 
	BUG_ON(pgtable->mode < PAGE_MODE_NONE ||

	       pgtable->mode > PAGE_MODE_6_LEVEL);



	root = (unsigned long)pgtable->root;

	freelist = free_sub_pt(root, pgtable->mode, freelist);

	free_sub_pt(pgtable->root, pgtable->mode, &freelist);



	free_page_list(freelist);

	put_pages_list(&freelist);

}



static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c

+1 −1
Original line number Diff line number Diff line
@@ -220,7 +220,7 @@ static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) 
	kfree(mn_to_smmu(mn));

}



static struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {

static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {

	.invalidate_range	= arm_smmu_mm_invalidate_range,

	.release		= arm_smmu_mm_release,

	.free_notifier		= arm_smmu_mmu_notifier_free,