Commit 2f7e8c55 authored by Jean-Philippe Brucker's avatar Jean-Philippe Brucker Committed by Will Deacon
Browse files

iommu/arm-smmu-v3: Hook up ATC invalidation to mm ops 



The invalidate_range() notifier is called for any change to the address
space. Perform the required ATC invalidations.

Signed-off-by: default avatarJean-Philippe Brucker <jean-philippe@linaro.org>
Link: https://lore.kernel.org/r/20201106155048.997886-5-jean-philippe@linaro.org


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 32784a95

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c

+15 −1
Original line number Diff line number Diff line
@@ -177,6 +177,16 @@ static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) 
	}

}



static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,

					 struct mm_struct *mm,

					 unsigned long start, unsigned long end)

{

	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);



	arm_smmu_atc_inv_domain(smmu_mn->domain, mm->pasid, start,

				end - start + 1);

}



static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)

{

	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
@@ -195,6 +205,7 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 
	arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, &quiet_cd);



	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);

	arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);



	smmu_mn->cleared = true;

	mutex_unlock(&sva_lock);
@@ -206,6 +217,7 @@ static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) 
}



static struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {

	.invalidate_range	= arm_smmu_mm_invalidate_range,

	.release		= arm_smmu_mm_release,

	.free_notifier		= arm_smmu_mmu_notifier_free,

};
@@ -278,8 +290,10 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn) 
	 * If we went through clear(), we've already invalidated, and no

	 * new TLB entry can have been formed.

	 */

	if (!smmu_mn->cleared)

	if (!smmu_mn->cleared) {

		arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid);

		arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);

	}



	/* Frees smmu_mn */

	mmu_notifier_put(&smmu_mn->mn);

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

+16 −2
Original line number Diff line number Diff line
@@ -1524,6 +1524,20 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size, 
	size_t inval_grain_shift = 12;

	unsigned long page_start, page_end;



	/*

	 * ATS and PASID:

	 *

	 * If substream_valid is clear, the PCIe TLP is sent without a PASID

	 * prefix. In that case all ATC entries within the address range are

	 * invalidated, including those that were requested with a PASID! There

	 * is no way to invalidate only entries without PASID.

	 *

	 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID

	 * traffic), translation requests without PASID create ATC entries

	 * without PASID, which must be invalidated with substream_valid clear.

	 * This has the unpleasant side-effect of invalidating all PASID-tagged

	 * ATC entries within the address range.

	 */

	*cmd = (struct arm_smmu_cmdq_ent) {

		.opcode			= CMDQ_OP_ATC_INV,

		.substream_valid	= !!ssid,
@@ -1582,8 +1596,8 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master) 
	return arm_smmu_cmdq_issue_sync(master->smmu);

}



static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,

				   int ssid, unsigned long iova, size_t size)

int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,

			    unsigned long iova, size_t size)

{

	int i;

	unsigned long flags;

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h

+2 −0
Original line number Diff line number Diff line
@@ -695,6 +695,8 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid, 
			    struct arm_smmu_ctx_desc *cd);

void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);

bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);

int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,

			    unsigned long iova, size_t size);



#ifdef CONFIG_ARM_SMMU_V3_SVA

bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);