Merge branch 'x86/amd-avic' into x86/amd (368d06cd) · Commits · EulixOS / Software / Kernel

Documentation/kernel-parameters.txt

+9 −0

Original line number	Original line	Diff line number	Diff line
	@@ -460,6 +460,15 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
	driver will print ACPI tables for AMD IOMMU during		driver will print ACPI tables for AMD IOMMU during
	IOMMU initialization.		IOMMU initialization.

			amd_iommu_intr= [HW,X86-64]
			Specifies one of the following AMD IOMMU interrupt
			remapping modes:
			legacy - Use legacy interrupt remapping mode.
			vapic - Use virtual APIC mode, which allows IOMMU
			to inject interrupts directly into guest.
			This mode requires kvm-amd.avic=1.
			(Default when IOMMU HW support is present.)

	amijoy.map= [HW,JOY] Amiga joystick support		amijoy.map= [HW,JOY] Amiga joystick support
	Map of devices attached to JOY0DAT and JOY1DAT		Map of devices attached to JOY0DAT and JOY1DAT
	Format: <a>,<b>		Format: <a>,<b>

drivers/iommu/amd_iommu.c

+427 −57

Original line number	Original line	Diff line number	Diff line
	@@ -137,6 +137,7 @@ struct iommu_dev_data {
	bool pri_tlp; /* PASID TLB required for		bool pri_tlp; /* PASID TLB required for
	PPR completions */		PPR completions */
	u32 errata; /* Bitmap for errata to apply */		u32 errata; /* Bitmap for errata to apply */
			bool use_vapic; /* Enable device to use vapic mode */
	};		};

	/*		/*
	@@ -707,14 +708,74 @@ static void iommu_poll_ppr_log(struct amd_iommu *iommu)
	}		}
	}		}

			#ifdef CONFIG_IRQ_REMAP
			static int (*iommu_ga_log_notifier)(u32);

			int amd_iommu_register_ga_log_notifier(int (*notifier)(u32))
			{
			iommu_ga_log_notifier = notifier;

			return 0;
			}
			EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier);

			static void iommu_poll_ga_log(struct amd_iommu *iommu)
			{
			u32 head, tail, cnt = 0;

			if (iommu->ga_log == NULL)
			return;

			head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
			tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET);

			while (head != tail) {
			volatile u64 *raw;
			u64 log_entry;

			raw = (u64 *)(iommu->ga_log + head);
			cnt++;

			/* Avoid memcpy function-call overhead */
			log_entry = *raw;

			/* Update head pointer of hardware ring-buffer */
			head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE;
			writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);

			/* Handle GA entry */
			switch (GA_REQ_TYPE(log_entry)) {
			case GA_GUEST_NR:
			if (!iommu_ga_log_notifier)
			break;

			pr_debug("AMD-Vi: %s: devid=%#x, ga_tag=%#x\n",
			__func__, GA_DEVID(log_entry),
			GA_TAG(log_entry));

			if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0)
			pr_err("AMD-Vi: GA log notifier failed.\n");
			break;
			default:
			break;
			}
			}
			}
			#endif /* CONFIG_IRQ_REMAP */

			#define AMD_IOMMU_INT_MASK \
			(MMIO_STATUS_EVT_INT_MASK \| \
			MMIO_STATUS_PPR_INT_MASK \| \
			MMIO_STATUS_GALOG_INT_MASK)

	irqreturn_t amd_iommu_int_thread(int irq, void *data)		irqreturn_t amd_iommu_int_thread(int irq, void *data)
	{		{
	struct amd_iommu iommu = (struct amd_iommu ) data;		struct amd_iommu iommu = (struct amd_iommu ) data;
	u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);		u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);

	while (status & (MMIO_STATUS_EVT_INT_MASK \| MMIO_STATUS_PPR_INT_MASK)) {		while (status & AMD_IOMMU_INT_MASK) {
	/* Enable EVT and PPR interrupts again */		/* Enable EVT and PPR and GA interrupts again */
	writel((MMIO_STATUS_EVT_INT_MASK \| MMIO_STATUS_PPR_INT_MASK),		writel(AMD_IOMMU_INT_MASK,
	iommu->mmio_base + MMIO_STATUS_OFFSET);		iommu->mmio_base + MMIO_STATUS_OFFSET);

	if (status & MMIO_STATUS_EVT_INT_MASK) {		if (status & MMIO_STATUS_EVT_INT_MASK) {
	@@ -727,6 +788,13 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data)
	iommu_poll_ppr_log(iommu);		iommu_poll_ppr_log(iommu);
	}		}

			#ifdef CONFIG_IRQ_REMAP
			if (status & MMIO_STATUS_GALOG_INT_MASK) {
			pr_devel("AMD-Vi: Processing IOMMU GA Log\n");
			iommu_poll_ga_log(iommu);
			}
			#endif

	/*		/*
	* Hardware bug: ERBT1312		* Hardware bug: ERBT1312
	* When re-enabling interrupt (by writing 1		* When re-enabling interrupt (by writing 1
	@@ -2948,6 +3016,12 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
	if (!iommu)		if (!iommu)
	return;		return;

			#ifdef CONFIG_IRQ_REMAP
			if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
			(dom->type == IOMMU_DOMAIN_UNMANAGED))
			dev_data->use_vapic = 0;
			#endif

	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);
	}		}

	@@ -2973,6 +3047,15 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,

	ret = attach_device(dev, domain);		ret = attach_device(dev, domain);

			#ifdef CONFIG_IRQ_REMAP
			if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
			if (dom->type == IOMMU_DOMAIN_UNMANAGED)
			dev_data->use_vapic = 1;
			else
			dev_data->use_vapic = 0;
			}
			#endif

	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);

	return ret;		return ret;
	@@ -3511,34 +3594,6 @@ EXPORT_SYMBOL(amd_iommu_device_info);
	*		*
	*****************************************************************************/		*****************************************************************************/

	union irte {
	u32 val;
	struct {
	u32 valid : 1,
	no_fault : 1,
	int_type : 3,
	rq_eoi : 1,
	dm : 1,
	rsvd_1 : 1,
	destination : 8,
	vector : 8,
	rsvd_2 : 8;
	} fields;
	};

	struct irq_2_irte {
	u16 devid; /* Device ID for IRTE table */
	u16 index; /* Index into IRTE table*/
	};

	struct amd_ir_data {
	struct irq_2_irte irq_2_irte;
	union irte irte_entry;
	union {
	struct msi_msg msi_entry;
	};
	};

	static struct irq_chip amd_ir_chip;		static struct irq_chip amd_ir_chip;

	#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)		#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
	@@ -3560,8 +3615,6 @@ static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
	amd_iommu_dev_table[devid].data[2] = dte;		amd_iommu_dev_table[devid].data[2] = dte;
	}		}

	#define IRTE_ALLOCATED (~1U)

	static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)		static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
	{		{
	struct irq_remap_table *table = NULL;		struct irq_remap_table *table = NULL;
	@@ -3607,13 +3660,18 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
	goto out;		goto out;
	}		}

	memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));		if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
			memset(table->table, 0,
			MAX_IRQS_PER_TABLE * sizeof(u32));
			else
			memset(table->table, 0,
			(MAX_IRQS_PER_TABLE * (sizeof(u64) * 2)));

	if (ioapic) {		if (ioapic) {
	int i;		int i;

	for (i = 0; i < 32; ++i)		for (i = 0; i < 32; ++i)
	table->table[i] = IRTE_ALLOCATED;		iommu->irte_ops->set_allocated(table, i);
	}		}

	irq_lookup_table[devid] = table;		irq_lookup_table[devid] = table;
	@@ -3639,6 +3697,10 @@ static int alloc_irq_index(u16 devid, int count)
	struct irq_remap_table *table;		struct irq_remap_table *table;
	unsigned long flags;		unsigned long flags;
	int index, c;		int index, c;
			struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];

			if (!iommu)
			return -ENODEV;

	table = get_irq_table(devid, false);		table = get_irq_table(devid, false);
	if (!table)		if (!table)
	@@ -3650,14 +3712,14 @@ static int alloc_irq_index(u16 devid, int count)
	for (c = 0, index = table->min_index;		for (c = 0, index = table->min_index;
	index < MAX_IRQS_PER_TABLE;		index < MAX_IRQS_PER_TABLE;
	++index) {		++index) {
	if (table->table[index] == 0)		if (!iommu->irte_ops->is_allocated(table, index))
	c += 1;		c += 1;
	else		else
	c = 0;		c = 0;

	if (c == count) {		if (c == count) {
	for (; c != 0; --c)		for (; c != 0; --c)
	table->table[index - c + 1] = IRTE_ALLOCATED;		iommu->irte_ops->set_allocated(table, index - c + 1);

	index -= count - 1;		index -= count - 1;
	goto out;		goto out;
	@@ -3672,7 +3734,42 @@ static int alloc_irq_index(u16 devid, int count)
	return index;		return index;
	}		}

	static int modify_irte(u16 devid, int index, union irte irte)		static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte,
			struct amd_ir_data *data)
			{
			struct irq_remap_table *table;
			struct amd_iommu *iommu;
			unsigned long flags;
			struct irte_ga *entry;

			iommu = amd_iommu_rlookup_table[devid];
			if (iommu == NULL)
			return -EINVAL;

			table = get_irq_table(devid, false);
			if (!table)
			return -ENOMEM;

			spin_lock_irqsave(&table->lock, flags);

			entry = (struct irte_ga *)table->table;
			entry = &entry[index];
			entry->lo.fields_remap.valid = 0;
			entry->hi.val = irte->hi.val;
			entry->lo.val = irte->lo.val;
			entry->lo.fields_remap.valid = 1;
			if (data)
			data->ref = entry;

			spin_unlock_irqrestore(&table->lock, flags);

			iommu_flush_irt(iommu, devid);
			iommu_completion_wait(iommu);

			return 0;
			}

			static int modify_irte(u16 devid, int index, union irte *irte)
	{		{
	struct irq_remap_table *table;		struct irq_remap_table *table;
	struct amd_iommu *iommu;		struct amd_iommu *iommu;
	@@ -3687,7 +3784,7 @@ static int modify_irte(u16 devid, int index, union irte irte)
	return -ENOMEM;		return -ENOMEM;

	spin_lock_irqsave(&table->lock, flags);		spin_lock_irqsave(&table->lock, flags);
	table->table[index] = irte.val;		table->table[index] = irte->val;
	spin_unlock_irqrestore(&table->lock, flags);		spin_unlock_irqrestore(&table->lock, flags);

	iommu_flush_irt(iommu, devid);		iommu_flush_irt(iommu, devid);
	@@ -3711,13 +3808,146 @@ static void free_irte(u16 devid, int index)
	return;		return;

	spin_lock_irqsave(&table->lock, flags);		spin_lock_irqsave(&table->lock, flags);
	table->table[index] = 0;		iommu->irte_ops->clear_allocated(table, index);
	spin_unlock_irqrestore(&table->lock, flags);		spin_unlock_irqrestore(&table->lock, flags);

	iommu_flush_irt(iommu, devid);		iommu_flush_irt(iommu, devid);
	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);
	}		}

			static void irte_prepare(void *entry,
			u32 delivery_mode, u32 dest_mode,
			u8 vector, u32 dest_apicid, int devid)
			{
			union irte irte = (union irte ) entry;

			irte->val = 0;
			irte->fields.vector = vector;
			irte->fields.int_type = delivery_mode;
			irte->fields.destination = dest_apicid;
			irte->fields.dm = dest_mode;
			irte->fields.valid = 1;
			}

			static void irte_ga_prepare(void *entry,
			u32 delivery_mode, u32 dest_mode,
			u8 vector, u32 dest_apicid, int devid)
			{
			struct irte_ga irte = (struct irte_ga ) entry;
			struct iommu_dev_data *dev_data = search_dev_data(devid);

			irte->lo.val = 0;
			irte->hi.val = 0;
			irte->lo.fields_remap.guest_mode = dev_data ? dev_data->use_vapic : 0;
			irte->lo.fields_remap.int_type = delivery_mode;
			irte->lo.fields_remap.dm = dest_mode;
			irte->hi.fields.vector = vector;
			irte->lo.fields_remap.destination = dest_apicid;
			irte->lo.fields_remap.valid = 1;
			}

			static void irte_activate(void *entry, u16 devid, u16 index)
			{
			union irte irte = (union irte ) entry;

			irte->fields.valid = 1;
			modify_irte(devid, index, irte);
			}

			static void irte_ga_activate(void *entry, u16 devid, u16 index)
			{
			struct irte_ga irte = (struct irte_ga ) entry;

			irte->lo.fields_remap.valid = 1;
			modify_irte_ga(devid, index, irte, NULL);
			}

			static void irte_deactivate(void *entry, u16 devid, u16 index)
			{
			union irte irte = (union irte ) entry;

			irte->fields.valid = 0;
			modify_irte(devid, index, irte);
			}

			static void irte_ga_deactivate(void *entry, u16 devid, u16 index)
			{
			struct irte_ga irte = (struct irte_ga ) entry;

			irte->lo.fields_remap.valid = 0;
			modify_irte_ga(devid, index, irte, NULL);
			}

			static void irte_set_affinity(void *entry, u16 devid, u16 index,
			u8 vector, u32 dest_apicid)
			{
			union irte irte = (union irte ) entry;

			irte->fields.vector = vector;
			irte->fields.destination = dest_apicid;
			modify_irte(devid, index, irte);
			}

			static void irte_ga_set_affinity(void *entry, u16 devid, u16 index,
			u8 vector, u32 dest_apicid)
			{
			struct irte_ga irte = (struct irte_ga ) entry;
			struct iommu_dev_data *dev_data = search_dev_data(devid);

			if (!dev_data \|\| !dev_data->use_vapic) {
			irte->hi.fields.vector = vector;
			irte->lo.fields_remap.destination = dest_apicid;
			irte->lo.fields_remap.guest_mode = 0;
			modify_irte_ga(devid, index, irte, NULL);
			}
			}

			#define IRTE_ALLOCATED (~1U)
			static void irte_set_allocated(struct irq_remap_table *table, int index)
			{
			table->table[index] = IRTE_ALLOCATED;
			}

			static void irte_ga_set_allocated(struct irq_remap_table *table, int index)
			{
			struct irte_ga ptr = (struct irte_ga )table->table;
			struct irte_ga *irte = &ptr[index];

			memset(&irte->lo.val, 0, sizeof(u64));
			memset(&irte->hi.val, 0, sizeof(u64));
			irte->hi.fields.vector = 0xff;
			}

			static bool irte_is_allocated(struct irq_remap_table *table, int index)
			{
			union irte ptr = (union irte )table->table;
			union irte *irte = &ptr[index];

			return irte->val != 0;
			}

			static bool irte_ga_is_allocated(struct irq_remap_table *table, int index)
			{
			struct irte_ga ptr = (struct irte_ga )table->table;
			struct irte_ga *irte = &ptr[index];

			return irte->hi.fields.vector != 0;
			}

			static void irte_clear_allocated(struct irq_remap_table *table, int index)
			{
			table->table[index] = 0;
			}

			static void irte_ga_clear_allocated(struct irq_remap_table *table, int index)
			{
			struct irte_ga ptr = (struct irte_ga )table->table;
			struct irte_ga *irte = &ptr[index];

			memset(&irte->lo.val, 0, sizeof(u64));
			memset(&irte->hi.val, 0, sizeof(u64));
			}

	static int get_devid(struct irq_alloc_info *info)		static int get_devid(struct irq_alloc_info *info)
	{		{
	int devid = -1;		int devid = -1;
	@@ -3802,19 +4032,17 @@ static void irq_remapping_prepare_irte(struct amd_ir_data *data,
	{		{
	struct irq_2_irte *irte_info = &data->irq_2_irte;		struct irq_2_irte *irte_info = &data->irq_2_irte;
	struct msi_msg *msg = &data->msi_entry;		struct msi_msg *msg = &data->msi_entry;
	union irte *irte = &data->irte_entry;
	struct IO_APIC_route_entry *entry;		struct IO_APIC_route_entry *entry;
			struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];

			if (!iommu)
			return;

	data->irq_2_irte.devid = devid;		data->irq_2_irte.devid = devid;
	data->irq_2_irte.index = index + sub_handle;		data->irq_2_irte.index = index + sub_handle;
			iommu->irte_ops->prepare(data->entry, apic->irq_delivery_mode,
	/* Setup IRTE for IOMMU */		apic->irq_dest_mode, irq_cfg->vector,
	irte->val = 0;		irq_cfg->dest_apicid, devid);
	irte->fields.vector = irq_cfg->vector;
	irte->fields.int_type = apic->irq_delivery_mode;
	irte->fields.destination = irq_cfg->dest_apicid;
	irte->fields.dm = apic->irq_dest_mode;
	irte->fields.valid = 1;

	switch (info->type) {		switch (info->type) {
	case X86_IRQ_ALLOC_TYPE_IOAPIC:		case X86_IRQ_ALLOC_TYPE_IOAPIC:
	@@ -3845,12 +4073,32 @@ static void irq_remapping_prepare_irte(struct amd_ir_data *data,
	}		}
	}		}

			struct amd_irte_ops irte_32_ops = {
			.prepare = irte_prepare,
			.activate = irte_activate,
			.deactivate = irte_deactivate,
			.set_affinity = irte_set_affinity,
			.set_allocated = irte_set_allocated,
			.is_allocated = irte_is_allocated,
			.clear_allocated = irte_clear_allocated,
			};

			struct amd_irte_ops irte_128_ops = {
			.prepare = irte_ga_prepare,
			.activate = irte_ga_activate,
			.deactivate = irte_ga_deactivate,
			.set_affinity = irte_ga_set_affinity,
			.set_allocated = irte_ga_set_allocated,
			.is_allocated = irte_ga_is_allocated,
			.clear_allocated = irte_ga_clear_allocated,
			};

	static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,		static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
	unsigned int nr_irqs, void *arg)		unsigned int nr_irqs, void *arg)
	{		{
	struct irq_alloc_info *info = arg;		struct irq_alloc_info *info = arg;
	struct irq_data *irq_data;		struct irq_data *irq_data;
	struct amd_ir_data *data;		struct amd_ir_data *data = NULL;
	struct irq_cfg *cfg;		struct irq_cfg *cfg;
	int i, ret, devid;		int i, ret, devid;
	int index = -1;		int index = -1;
	@@ -3903,6 +4151,16 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
	if (!data)		if (!data)
	goto out_free_data;		goto out_free_data;

			if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
			data->entry = kzalloc(sizeof(union irte), GFP_KERNEL);
			else
			data->entry = kzalloc(sizeof(struct irte_ga),
			GFP_KERNEL);
			if (!data->entry) {
			kfree(data);
			goto out_free_data;
			}

	irq_data->hwirq = (devid << 16) + i;		irq_data->hwirq = (devid << 16) + i;
	irq_data->chip_data = data;		irq_data->chip_data = data;
	irq_data->chip = &amd_ir_chip;		irq_data->chip = &amd_ir_chip;
	@@ -3939,6 +4197,7 @@ static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
	data = irq_data->chip_data;		data = irq_data->chip_data;
	irte_info = &data->irq_2_irte;		irte_info = &data->irq_2_irte;
	free_irte(irte_info->devid, irte_info->index);		free_irte(irte_info->devid, irte_info->index);
			kfree(data->entry);
	kfree(data);		kfree(data);
	}		}
	}		}
	@@ -3950,8 +4209,11 @@ static void irq_remapping_activate(struct irq_domain *domain,
	{		{
	struct amd_ir_data *data = irq_data->chip_data;		struct amd_ir_data *data = irq_data->chip_data;
	struct irq_2_irte *irte_info = &data->irq_2_irte;		struct irq_2_irte *irte_info = &data->irq_2_irte;
			struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid];

	modify_irte(irte_info->devid, irte_info->index, data->irte_entry);		if (iommu)
			iommu->irte_ops->activate(data->entry, irte_info->devid,
			irte_info->index);
	}		}

	static void irq_remapping_deactivate(struct irq_domain *domain,		static void irq_remapping_deactivate(struct irq_domain *domain,
	@@ -3959,10 +4221,11 @@ static void irq_remapping_deactivate(struct irq_domain *domain,
	{		{
	struct amd_ir_data *data = irq_data->chip_data;		struct amd_ir_data *data = irq_data->chip_data;
	struct irq_2_irte *irte_info = &data->irq_2_irte;		struct irq_2_irte *irte_info = &data->irq_2_irte;
	union irte entry;		struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid];

	entry.val = 0;		if (iommu)
	modify_irte(irte_info->devid, irte_info->index, data->irte_entry);		iommu->irte_ops->deactivate(data->entry, irte_info->devid,
			irte_info->index);
	}		}

	static struct irq_domain_ops amd_ir_domain_ops = {		static struct irq_domain_ops amd_ir_domain_ops = {
	@@ -3972,6 +4235,70 @@ static struct irq_domain_ops amd_ir_domain_ops = {
	.deactivate = irq_remapping_deactivate,		.deactivate = irq_remapping_deactivate,
	};		};

			static int amd_ir_set_vcpu_affinity(struct irq_data data, void vcpu_info)
			{
			struct amd_iommu *iommu;
			struct amd_iommu_pi_data *pi_data = vcpu_info;
			struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data;
			struct amd_ir_data *ir_data = data->chip_data;
			struct irte_ga irte = (struct irte_ga ) ir_data->entry;
			struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
			struct iommu_dev_data *dev_data = search_dev_data(irte_info->devid);

			/* Note:
			* This device has never been set up for guest mode.
			* we should not modify the IRTE
			*/
			if (!dev_data \|\| !dev_data->use_vapic)
			return 0;

			pi_data->ir_data = ir_data;

			/* Note:
			* SVM tries to set up for VAPIC mode, but we are in
			* legacy mode. So, we force legacy mode instead.
			*/
			if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
			pr_debug("AMD-Vi: %s: Fall back to using intr legacy remap\n",
			__func__);
			pi_data->is_guest_mode = false;
			}

			iommu = amd_iommu_rlookup_table[irte_info->devid];
			if (iommu == NULL)
			return -EINVAL;

			pi_data->prev_ga_tag = ir_data->cached_ga_tag;
			if (pi_data->is_guest_mode) {
			/* Setting */
			irte->hi.fields.ga_root_ptr = (pi_data->base >> 12);
			irte->hi.fields.vector = vcpu_pi_info->vector;
			irte->lo.fields_vapic.guest_mode = 1;
			irte->lo.fields_vapic.ga_tag = pi_data->ga_tag;

			ir_data->cached_ga_tag = pi_data->ga_tag;
			} else {
			/* Un-Setting */
			struct irq_cfg *cfg = irqd_cfg(data);

			irte->hi.val = 0;
			irte->lo.val = 0;
			irte->hi.fields.vector = cfg->vector;
			irte->lo.fields_remap.guest_mode = 0;
			irte->lo.fields_remap.destination = cfg->dest_apicid;
			irte->lo.fields_remap.int_type = apic->irq_delivery_mode;
			irte->lo.fields_remap.dm = apic->irq_dest_mode;

			/*
			* This communicates the ga_tag back to the caller
			* so that it can do all the necessary clean up.
			*/
			ir_data->cached_ga_tag = 0;
			}

			return modify_irte_ga(irte_info->devid, irte_info->index, irte, ir_data);
			}

	static int amd_ir_set_affinity(struct irq_data *data,		static int amd_ir_set_affinity(struct irq_data *data,
	const struct cpumask *mask, bool force)		const struct cpumask *mask, bool force)
	{		{
	@@ -3979,8 +4306,12 @@ static int amd_ir_set_affinity(struct irq_data *data,
	struct irq_2_irte *irte_info = &ir_data->irq_2_irte;		struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
	struct irq_cfg *cfg = irqd_cfg(data);		struct irq_cfg *cfg = irqd_cfg(data);
	struct irq_data *parent = data->parent_data;		struct irq_data *parent = data->parent_data;
			struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid];
	int ret;		int ret;

			if (!iommu)
			return -ENODEV;

	ret = parent->chip->irq_set_affinity(parent, mask, force);		ret = parent->chip->irq_set_affinity(parent, mask, force);
	if (ret < 0 \|\| ret == IRQ_SET_MASK_OK_DONE)		if (ret < 0 \|\| ret == IRQ_SET_MASK_OK_DONE)
	return ret;		return ret;
	@@ -3989,9 +4320,8 @@ static int amd_ir_set_affinity(struct irq_data *data,
	* Atomically updates the IRTE with the new destination, vector		* Atomically updates the IRTE with the new destination, vector
	* and flushes the interrupt entry cache.		* and flushes the interrupt entry cache.
	*/		*/
	ir_data->irte_entry.fields.vector = cfg->vector;		iommu->irte_ops->set_affinity(ir_data->entry, irte_info->devid,
	ir_data->irte_entry.fields.destination = cfg->dest_apicid;		irte_info->index, cfg->vector, cfg->dest_apicid);
	modify_irte(irte_info->devid, irte_info->index, ir_data->irte_entry);

	/*		/*
	* After this point, all the interrupts will start arriving		* After this point, all the interrupts will start arriving
	@@ -4013,6 +4343,7 @@ static void ir_compose_msi_msg(struct irq_data irq_data, struct msi_msg msg)
	static struct irq_chip amd_ir_chip = {		static struct irq_chip amd_ir_chip = {
	.irq_ack = ir_ack_apic_edge,		.irq_ack = ir_ack_apic_edge,
	.irq_set_affinity = amd_ir_set_affinity,		.irq_set_affinity = amd_ir_set_affinity,
			.irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity,
	.irq_compose_msi_msg = ir_compose_msi_msg,		.irq_compose_msi_msg = ir_compose_msi_msg,
	};		};

	@@ -4027,4 +4358,43 @@ int amd_iommu_create_irq_domain(struct amd_iommu *iommu)

	return 0;		return 0;
	}		}

			int amd_iommu_update_ga(int cpu, bool is_run, void *data)
			{
			unsigned long flags;
			struct amd_iommu *iommu;
			struct irq_remap_table *irt;
			struct amd_ir_data ir_data = (struct amd_ir_data )data;
			int devid = ir_data->irq_2_irte.devid;
			struct irte_ga entry = (struct irte_ga ) ir_data->entry;
			struct irte_ga ref = (struct irte_ga ) ir_data->ref;

			if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) \|\|
			!ref \|\| !entry \|\| !entry->lo.fields_vapic.guest_mode)
			return 0;

			iommu = amd_iommu_rlookup_table[devid];
			if (!iommu)
			return -ENODEV;

			irt = get_irq_table(devid, false);
			if (!irt)
			return -ENODEV;

			spin_lock_irqsave(&irt->lock, flags);

			if (ref->lo.fields_vapic.guest_mode) {
			if (cpu >= 0)
			ref->lo.fields_vapic.destination = cpu;
			ref->lo.fields_vapic.is_run = is_run;
			barrier();
			}

			spin_unlock_irqrestore(&irt->lock, flags);

			iommu_flush_irt(iommu, devid);
			iommu_completion_wait(iommu);
			return 0;
			}
			EXPORT_SYMBOL(amd_iommu_update_ga);
	#endif		#endif

drivers/iommu/amd_iommu_init.c

+175 −6

File changed.

Preview size limit exceeded, changes collapsed.

drivers/iommu/amd_iommu_proto.h

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -38,6 +38,7 @@ extern int amd_iommu_enable(void);
	extern void amd_iommu_disable(void);		extern void amd_iommu_disable(void);
	extern int amd_iommu_reenable(int);		extern int amd_iommu_reenable(int);
	extern int amd_iommu_enable_faulting(void);		extern int amd_iommu_enable_faulting(void);
			extern int amd_iommu_guest_ir;

	/* IOMMUv2 specific functions */		/* IOMMUv2 specific functions */
	struct iommu_domain;		struct iommu_domain;

drivers/iommu/amd_iommu_types.h

+149 −0

Original line number	Original line	Diff line number	Diff line
	@@ -22,6 +22,7 @@

	#include <linux/types.h>		#include <linux/types.h>
	#include <linux/mutex.h>		#include <linux/mutex.h>
			#include <linux/msi.h>
	#include <linux/list.h>		#include <linux/list.h>
	#include <linux/spinlock.h>		#include <linux/spinlock.h>
	#include <linux/pci.h>		#include <linux/pci.h>
	@@ -69,6 +70,8 @@
	#define MMIO_EXCL_LIMIT_OFFSET 0x0028		#define MMIO_EXCL_LIMIT_OFFSET 0x0028
	#define MMIO_EXT_FEATURES 0x0030		#define MMIO_EXT_FEATURES 0x0030
	#define MMIO_PPR_LOG_OFFSET 0x0038		#define MMIO_PPR_LOG_OFFSET 0x0038
			#define MMIO_GA_LOG_BASE_OFFSET 0x00e0
			#define MMIO_GA_LOG_TAIL_OFFSET 0x00e8
	#define MMIO_CMD_HEAD_OFFSET 0x2000		#define MMIO_CMD_HEAD_OFFSET 0x2000
	#define MMIO_CMD_TAIL_OFFSET 0x2008		#define MMIO_CMD_TAIL_OFFSET 0x2008
	#define MMIO_EVT_HEAD_OFFSET 0x2010		#define MMIO_EVT_HEAD_OFFSET 0x2010
	@@ -76,6 +79,8 @@
	#define MMIO_STATUS_OFFSET 0x2020		#define MMIO_STATUS_OFFSET 0x2020
	#define MMIO_PPR_HEAD_OFFSET 0x2030		#define MMIO_PPR_HEAD_OFFSET 0x2030
	#define MMIO_PPR_TAIL_OFFSET 0x2038		#define MMIO_PPR_TAIL_OFFSET 0x2038
			#define MMIO_GA_HEAD_OFFSET 0x2040
			#define MMIO_GA_TAIL_OFFSET 0x2048
	#define MMIO_CNTR_CONF_OFFSET 0x4000		#define MMIO_CNTR_CONF_OFFSET 0x4000
	#define MMIO_CNTR_REG_OFFSET 0x40000		#define MMIO_CNTR_REG_OFFSET 0x40000
	#define MMIO_REG_END_OFFSET 0x80000		#define MMIO_REG_END_OFFSET 0x80000
	@@ -92,6 +97,7 @@
	#define FEATURE_GA (1ULL<<7)		#define FEATURE_GA (1ULL<<7)
	#define FEATURE_HE (1ULL<<8)		#define FEATURE_HE (1ULL<<8)
	#define FEATURE_PC (1ULL<<9)		#define FEATURE_PC (1ULL<<9)
			#define FEATURE_GAM_VAPIC (1ULL<<21)

	#define FEATURE_PASID_SHIFT 32		#define FEATURE_PASID_SHIFT 32
	#define FEATURE_PASID_MASK (0x1fULL << FEATURE_PASID_SHIFT)		#define FEATURE_PASID_MASK (0x1fULL << FEATURE_PASID_SHIFT)
	@@ -110,6 +116,9 @@
	#define MMIO_STATUS_EVT_INT_MASK (1 << 1)		#define MMIO_STATUS_EVT_INT_MASK (1 << 1)
	#define MMIO_STATUS_COM_WAIT_INT_MASK (1 << 2)		#define MMIO_STATUS_COM_WAIT_INT_MASK (1 << 2)
	#define MMIO_STATUS_PPR_INT_MASK (1 << 6)		#define MMIO_STATUS_PPR_INT_MASK (1 << 6)
			#define MMIO_STATUS_GALOG_RUN_MASK (1 << 8)
			#define MMIO_STATUS_GALOG_OVERFLOW_MASK (1 << 9)
			#define MMIO_STATUS_GALOG_INT_MASK (1 << 10)

	/* event logging constants */		/* event logging constants */
	#define EVENT_ENTRY_SIZE 0x10		#define EVENT_ENTRY_SIZE 0x10
	@@ -146,6 +155,10 @@
	#define CONTROL_PPFINT_EN 0x0eULL		#define CONTROL_PPFINT_EN 0x0eULL
	#define CONTROL_PPR_EN 0x0fULL		#define CONTROL_PPR_EN 0x0fULL
	#define CONTROL_GT_EN 0x10ULL		#define CONTROL_GT_EN 0x10ULL
			#define CONTROL_GA_EN 0x11ULL
			#define CONTROL_GAM_EN 0x19ULL
			#define CONTROL_GALOG_EN 0x1CULL
			#define CONTROL_GAINT_EN 0x1DULL

	#define CTRL_INV_TO_MASK (7 << CONTROL_INV_TIMEOUT)		#define CTRL_INV_TO_MASK (7 << CONTROL_INV_TIMEOUT)
	#define CTRL_INV_TO_NONE 0		#define CTRL_INV_TO_NONE 0
	@@ -224,6 +237,19 @@

	#define PPR_REQ_FAULT 0x01		#define PPR_REQ_FAULT 0x01

			/* Constants for GA Log handling */
			#define GA_LOG_ENTRIES 512
			#define GA_LOG_SIZE_SHIFT 56
			#define GA_LOG_SIZE_512 (0x8ULL << GA_LOG_SIZE_SHIFT)
			#define GA_ENTRY_SIZE 8
			#define GA_LOG_SIZE (GA_ENTRY_SIZE * GA_LOG_ENTRIES)

			#define GA_TAG(x) (u32)(x & 0xffffffffULL)
			#define GA_DEVID(x) (u16)(((x) >> 32) & 0xffffULL)
			#define GA_REQ_TYPE(x) (((x) >> 60) & 0xfULL)

			#define GA_GUEST_NR 0x1

	#define PAGE_MODE_NONE 0x00		#define PAGE_MODE_NONE 0x00
	#define PAGE_MODE_1_LEVEL 0x01		#define PAGE_MODE_1_LEVEL 0x01
	#define PAGE_MODE_2_LEVEL 0x02		#define PAGE_MODE_2_LEVEL 0x02
	@@ -329,6 +355,12 @@
	#define IOMMU_CAP_NPCACHE 26		#define IOMMU_CAP_NPCACHE 26
	#define IOMMU_CAP_EFR 27		#define IOMMU_CAP_EFR 27

			/* IOMMU Feature Reporting Field (for IVHD type 10h */
			#define IOMMU_FEAT_GASUP_SHIFT 6

			/* IOMMU Extended Feature Register (EFR) */
			#define IOMMU_EFR_GASUP_SHIFT 7

	#define MAX_DOMAIN_ID 65536		#define MAX_DOMAIN_ID 65536

	/* Protection domain flags */		/* Protection domain flags */
	@@ -400,6 +432,7 @@ struct amd_iommu_fault {

	struct iommu_domain;		struct iommu_domain;
	struct irq_domain;		struct irq_domain;
			struct amd_irte_ops;

	/*		/*
	* This structure contains generic data for IOMMU protection domains		* This structure contains generic data for IOMMU protection domains
	@@ -490,6 +523,12 @@ struct amd_iommu {
	/* Base of the PPR log, if present */		/* Base of the PPR log, if present */
	u8 *ppr_log;		u8 *ppr_log;

			/* Base of the GA log, if present */
			u8 *ga_log;

			/* Tail of the GA log, if present */
			u8 *ga_log_tail;

	/* true if interrupts for this IOMMU are already enabled */		/* true if interrupts for this IOMMU are already enabled */
	bool int_enabled;		bool int_enabled;

	@@ -523,6 +562,8 @@ struct amd_iommu {
	#ifdef CONFIG_IRQ_REMAP		#ifdef CONFIG_IRQ_REMAP
	struct irq_domain *ir_domain;		struct irq_domain *ir_domain;
	struct irq_domain *msi_domain;		struct irq_domain *msi_domain;

			struct amd_irte_ops *irte_ops;
	#endif		#endif
	};		};

	@@ -681,4 +722,112 @@ static inline int get_hpet_devid(int id)
	return -EINVAL;		return -EINVAL;
	}		}

			enum amd_iommu_intr_mode_type {
			AMD_IOMMU_GUEST_IR_LEGACY,

			/* This mode is not visible to users. It is used when
			* we cannot fully enable vAPIC and fallback to only support
			* legacy interrupt remapping via 128-bit IRTE.
			*/
			AMD_IOMMU_GUEST_IR_LEGACY_GA,
			AMD_IOMMU_GUEST_IR_VAPIC,
			};

			#define AMD_IOMMU_GUEST_IR_GA(x) (x == AMD_IOMMU_GUEST_IR_VAPIC \|\| \
			x == AMD_IOMMU_GUEST_IR_LEGACY_GA)

			#define AMD_IOMMU_GUEST_IR_VAPIC(x) (x == AMD_IOMMU_GUEST_IR_VAPIC)

			union irte {
			u32 val;
			struct {
			u32 valid : 1,
			no_fault : 1,
			int_type : 3,
			rq_eoi : 1,
			dm : 1,
			rsvd_1 : 1,
			destination : 8,
			vector : 8,
			rsvd_2 : 8;
			} fields;
			};

			union irte_ga_lo {
			u64 val;

			/* For int remapping */
			struct {
			u64 valid : 1,
			no_fault : 1,
			/* ------ */
			int_type : 3,
			rq_eoi : 1,
			dm : 1,
			/* ------ */
			guest_mode : 1,
			destination : 8,
			rsvd : 48;
			} fields_remap;

			/* For guest vAPIC */
			struct {
			u64 valid : 1,
			no_fault : 1,
			/* ------ */
			ga_log_intr : 1,
			rsvd1 : 3,
			is_run : 1,
			/* ------ */
			guest_mode : 1,
			destination : 8,
			rsvd2 : 16,
			ga_tag : 32;
			} fields_vapic;
			};

			union irte_ga_hi {
			u64 val;
			struct {
			u64 vector : 8,
			rsvd_1 : 4,
			ga_root_ptr : 40,
			rsvd_2 : 12;
			} fields;
			};

			struct irte_ga {
			union irte_ga_lo lo;
			union irte_ga_hi hi;
			};

			struct irq_2_irte {
			u16 devid; /* Device ID for IRTE table */
			u16 index; /* Index into IRTE table*/
			};

			struct amd_ir_data {
			u32 cached_ga_tag;
			struct irq_2_irte irq_2_irte;
			struct msi_msg msi_entry;
			void entry; / Pointer to union irte or struct irte_ga */
			void ref; / Pointer to the actual irte */
			};

			struct amd_irte_ops {
			void (prepare)(void , u32, u32, u8, u32, int);
			void (activate)(void , u16, u16);
			void (deactivate)(void , u16, u16);
			void (set_affinity)(void , u16, u16, u8, u32);
			void (get)(struct irq_remap_table *, int);
			void (set_allocated)(struct irq_remap_table , int);
			bool (is_allocated)(struct irq_remap_table , int);
			void (clear_allocated)(struct irq_remap_table , int);
			};

			#ifdef CONFIG_IRQ_REMAP
			extern struct amd_irte_ops irte_32_ops;
			extern struct amd_irte_ops irte_128_ops;
			#endif

	#endif /* _ASM_X86_AMD_IOMMU_TYPES_H */		#endif /* _ASM_X86_AMD_IOMMU_TYPES_H */