Commit bccc37a8 authored by Suravee Suthikulpanit's avatar Suravee Suthikulpanit Committed by Joerg Roedel
Browse files

iommu/amd: Improving Interrupt Remapping Table Invalidation 



Invalidating Interrupt Remapping Table (IRT) requires, the AMD IOMMU driver
to issue INVALIDATE_INTERRUPT_TABLE and COMPLETION_WAIT commands.
Currently, the driver issues the two commands separately, which requires
calling raw_spin_lock_irqsave() twice. In addition, the COMPLETION_WAIT
could potentially be interleaved with other commands causing delay of
the COMPLETION_WAIT command.

Therefore, combine issuing of the two commands in one spin-lock, and
changing struct amd_iommu.cmd_sem_val to use atomic64 to minimize
locking.

Reviewed-by: default avatarJerry Snitselaar <jsnitsel@redhat.com>
Signed-off-by: default avatarSuravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Link: https://lore.kernel.org/r/20230530141137.14376-6-suravee.suthikulpanit@amd.com


Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
parent 98aeb4ea

drivers/iommu/amd/amd_iommu_types.h

+1 −1
Original line number Diff line number Diff line
@@ -752,7 +752,7 @@ struct amd_iommu { 


	u32 flags;

	volatile u64 *cmd_sem;

	u64 cmd_sem_val;

	atomic64_t cmd_sem_val;



#ifdef CONFIG_AMD_IOMMU_DEBUGFS

	/* DebugFS Info */

drivers/iommu/amd/init.c

+1 −1
Original line number Diff line number Diff line
@@ -1733,7 +1733,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, 
	iommu->pci_seg = pci_seg;



	raw_spin_lock_init(&iommu->lock);

	iommu->cmd_sem_val = 0;

	atomic64_set(&iommu->cmd_sem_val, 0);



	/* Add IOMMU to internal data structures */

	list_add_tail(&iommu->list, &amd_iommu_list);

drivers/iommu/amd/iommu.c

+22 −5
Original line number Diff line number Diff line
@@ -1175,11 +1175,11 @@ static int iommu_completion_wait(struct amd_iommu *iommu) 
	if (!iommu->need_sync)

		return 0;



	raw_spin_lock_irqsave(&iommu->lock, flags);



	data = ++iommu->cmd_sem_val;

	data = atomic64_add_return(1, &iommu->cmd_sem_val);

	build_completion_wait(&cmd, iommu, data);



	raw_spin_lock_irqsave(&iommu->lock, flags);



	ret = __iommu_queue_command_sync(iommu, &cmd, false);

	if (ret)

		goto out_unlock;
@@ -1277,11 +1277,28 @@ static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) 


static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)

{

	int ret;

	u64 data;

	unsigned long flags;

	struct iommu_cmd cmd, cmd2;



	if (iommu->irtcachedis_enabled)

		return;



	iommu_flush_irt(iommu, devid);

	iommu_completion_wait(iommu);

	build_inv_irt(&cmd, devid);

	data = atomic64_add_return(1, &iommu->cmd_sem_val);

	build_completion_wait(&cmd2, iommu, data);



	raw_spin_lock_irqsave(&iommu->lock, flags);

	ret = __iommu_queue_command_sync(iommu, &cmd, true);

	if (ret)

		goto out;

	ret = __iommu_queue_command_sync(iommu, &cmd2, false);

	if (ret)

		goto out;

	wait_on_sem(iommu, data);

out:

	raw_spin_unlock_irqrestore(&iommu->lock, flags);

}



void iommu_flush_all_caches(struct amd_iommu *iommu)