Commit 03b784a5 authored by Kunkun Jiang's avatar Kunkun Jiang Committed by Jia Qingtong
Browse files

iommu: Introduce dirty log tracking framework 

virt inclusion
category: feature
bugzilla: https://gitee.com/openeuler/kernel/issues/I8UZSP


CVE: NA

------------------------------

Some types of IOMMU are capable of tracking DMA dirty log, such as
ARM SMMU with HTTU or Intel IOMMU with SLADE. This introduces the
dirty log tracking framework in the IOMMU base layer.

Four new essential interfaces are added, and we maintaince the status
of dirty log tracking in iommu_domain.
1. iommu_support_dirty_log: Check whether domain supports dirty log
tracking
2. iommu_switch_dirty_log: Perform actions to start|stop dirty log tracking
3. iommu_sync_dirty_log: Sync dirty log from IOMMU into a dirty bitmap
4. iommu_clear_dirty_log: Clear dirty log of IOMMU by a mask bitmap

Note: Don't concurrently call these interfaces with other ops that
access underlying page table.

Signed-off-by: default avatarKeqian Zhu <zhukeqian1@huawei.com>
Signed-off-by: default avatarKunkun Jiang <jiangkunkun@huawei.com>
Signed-off-by: default avatarJia Qingtong <jiaqingtong@huawei.com>
parent 5b658bc5

drivers/iommu/iommu.c

+183 −0
Original line number Diff line number Diff line
@@ -1998,6 +1998,8 @@ static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus, 
		iommu_domain_free(domain);

		domain = NULL;

	}

	mutex_init(&domain->switch_log_lock);



	return domain;

}
@@ -2745,6 +2747,187 @@ int iommu_set_pgtable_quirks(struct iommu_domain *domain, 
}

EXPORT_SYMBOL_GPL(iommu_set_pgtable_quirks);



bool iommu_support_dirty_log(struct iommu_domain *domain)

{

	const struct iommu_domain_ops *ops = domain->ops;



	return ops->support_dirty_log && ops->support_dirty_log(domain);

}

EXPORT_SYMBOL_GPL(iommu_support_dirty_log);



int iommu_switch_dirty_log(struct iommu_domain *domain, bool enable,

			   unsigned long iova, size_t size, int prot)

{

	const struct iommu_domain_ops *ops = domain->ops;

	unsigned long orig_iova = iova;

	unsigned int min_pagesz;

	size_t orig_size = size;

	bool flush = false;

	int ret = 0;



	if (unlikely(!ops->switch_dirty_log))

		return -ENODEV;



	min_pagesz = 1 << __ffs(domain->pgsize_bitmap);

	if (!IS_ALIGNED(iova | size, min_pagesz)) {

		pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",

		       iova, size, min_pagesz);

		return -EINVAL;

	}



	mutex_lock(&domain->switch_log_lock);



	pr_debug("switch_dirty_log %s for: iova 0x%lx size 0x%zx\n",

		 enable ? "enable" : "disable", iova, size);



	while (size) {

		size_t pgsize = iommu_pgsize(domain, iova, iova, size, NULL);



		flush = true;

		ret = ops->switch_dirty_log(domain, enable, iova, pgsize, prot);

		if (ret)

			break;



		pr_debug("switch_dirty_log handled: iova 0x%lx size 0x%zx\n",

			 iova, pgsize);



		iova += pgsize;

		size -= pgsize;

	}



	if (flush)

		iommu_flush_iotlb_all(domain);



	if (!ret)

		trace_switch_dirty_log(orig_iova, orig_size, enable);



	mutex_unlock(&domain->switch_log_lock);

	return ret;

}

EXPORT_SYMBOL_GPL(iommu_switch_dirty_log);



int iommu_sync_dirty_log(struct iommu_domain *domain, unsigned long iova,

			 size_t size, unsigned long *bitmap,

			 unsigned long base_iova, unsigned long bitmap_pgshift)

{

	const struct iommu_domain_ops *ops = domain->ops;

	unsigned long orig_iova = iova;

	unsigned int min_pagesz;

	size_t orig_size = size;

	int ret = 0;



	if (unlikely(!ops->sync_dirty_log))

		return -ENODEV;



	min_pagesz = 1 << __ffs(domain->pgsize_bitmap);

	if (!IS_ALIGNED(iova | size, min_pagesz)) {

		pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",

		       iova, size, min_pagesz);

		return -EINVAL;

	}



	mutex_lock(&domain->switch_log_lock);



	pr_debug("sync_dirty_log for: iova 0x%lx size 0x%zx\n", iova, size);



	while (size) {

		size_t pgsize = iommu_pgsize(domain, iova, iova, size, NULL);



		ret = ops->sync_dirty_log(domain, iova, pgsize,

					  bitmap, base_iova, bitmap_pgshift);

		if (ret)

			break;



		pr_debug("sync_dirty_log handled: iova 0x%lx size 0x%zx\n",

			 iova, pgsize);



		iova += pgsize;

		size -= pgsize;

	}



	if (!ret)

		trace_sync_dirty_log(orig_iova, orig_size);



	mutex_unlock(&domain->switch_log_lock);

	return ret;

}

EXPORT_SYMBOL_GPL(iommu_sync_dirty_log);



static int __iommu_clear_dirty_log(struct iommu_domain *domain,

				   unsigned long iova, size_t size,

				   unsigned long *bitmap,

				   unsigned long base_iova,

				   unsigned long bitmap_pgshift)

{

	const struct iommu_domain_ops *ops = domain->ops;

	unsigned long orig_iova = iova;

	size_t orig_size = size;

	int ret = 0;



	if (unlikely(!ops->clear_dirty_log))

		return -ENODEV;



	pr_debug("clear_dirty_log for: iova 0x%lx size 0x%zx\n", iova, size);



	while (size) {

		size_t pgsize = iommu_pgsize(domain, iova, iova, size, NULL);



		ret = ops->clear_dirty_log(domain, iova, pgsize, bitmap,

					   base_iova, bitmap_pgshift);

		if (ret)

			break;



		pr_debug("clear_dirty_log handled: iova 0x%lx size 0x%zx\n",

			 iova, pgsize);



		iova += pgsize;

		size -= pgsize;

	}



	if (!ret)

		trace_clear_dirty_log(orig_iova, orig_size);



	return ret;

}



int iommu_clear_dirty_log(struct iommu_domain *domain,

			  unsigned long iova, size_t size,

			  unsigned long *bitmap, unsigned long base_iova,

			  unsigned long bitmap_pgshift)

{

	unsigned long riova, rsize;

	unsigned int min_pagesz, rs, re;

	bool flush = false;

	int ret = 0;



	min_pagesz = 1 << __ffs(domain->pgsize_bitmap);

	if (!IS_ALIGNED(iova | size, min_pagesz)) {

		pr_err("unaligned: iova 0x%lx min_pagesz 0x%x\n",

		       iova, min_pagesz);

		return -EINVAL;

	}



	mutex_lock(&domain->switch_log_lock);



	rs = (iova - base_iova) >> bitmap_pgshift;

	for_each_set_bitrange_from(rs, re, bitmap, (size >> bitmap_pgshift)) {

		flush = true;

		riova = base_iova + ((unsigned long)rs << bitmap_pgshift);

		rsize = (unsigned long)(re - rs) << bitmap_pgshift;

		ret = __iommu_clear_dirty_log(domain, riova, rsize, bitmap,

					      base_iova, bitmap_pgshift);

		if (ret)

			break;

	}



	if (flush)

		iommu_flush_iotlb_all(domain);



	mutex_unlock(&domain->switch_log_lock);

	return ret;

}

EXPORT_SYMBOL_GPL(iommu_clear_dirty_log);



/**

 * iommu_get_resv_regions - get reserved regions

 * @dev: device for which to get reserved regions

include/linux/iommu.h

+63 −0
Original line number Diff line number Diff line
@@ -115,6 +115,7 @@ struct iommu_domain { 
			int users;

		};

	};

	struct mutex switch_log_lock;

};



static inline bool iommu_is_dma_domain(struct iommu_domain *domain)
@@ -317,6 +318,7 @@ struct iommu_ops { 
	void (*set_platform_dma_ops)(struct device *dev);

	struct iommu_group *(*device_group)(struct device *dev);





	/* Request/Free a list of reserved regions for a device */

	void (*get_resv_regions)(struct device *dev, struct list_head *list);
@@ -370,6 +372,10 @@ struct iommu_ops { 
 *                           specific mechanisms.

 * @enable_nesting: Enable nesting

 * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)

 * @support_dirty_log: Check whether domain supports dirty log tracking

 * @switch_dirty_log: Perform actions to start|stop dirty log tracking

 * @sync_dirty_log: Sync dirty log from IOMMU into a dirty bitmap

 * @clear_dirty_log: Clear dirty log of IOMMU by a mask bitmap

 * @free: Release the domain after use.

 */

struct iommu_domain_ops {
@@ -402,6 +408,21 @@ struct iommu_domain_ops { 
	int (*set_pgtable_quirks)(struct iommu_domain *domain,

				  unsigned long quirks);



	/*

	 * Track dirty log. Note: Don't concurrently call these interfaces with

	 * other ops that access underlying page table.

	 */

	bool (*support_dirty_log)(struct iommu_domain *domain);

	int (*switch_dirty_log)(struct iommu_domain *domain, bool enable,

				unsigned long iova, size_t size, int prot);

	int (*sync_dirty_log)(struct iommu_domain *domain,

			      unsigned long iova, size_t size,

			      unsigned long *bitmap, unsigned long base_iova,

			      unsigned long bitmap_pgshift);

	int (*clear_dirty_log)(struct iommu_domain *domain,

			       unsigned long iova, size_t size,

			       unsigned long *bitmap, unsigned long base_iova,

			       unsigned long bitmap_pgshift);

	void (*free)(struct iommu_domain *domain);

};
@@ -579,6 +600,18 @@ int iommu_enable_nesting(struct iommu_domain *domain); 
int iommu_set_pgtable_quirks(struct iommu_domain *domain,

		unsigned long quirks);



extern bool iommu_support_dirty_log(struct iommu_domain *domain);

extern int iommu_switch_dirty_log(struct iommu_domain *domain, bool enable,

				  unsigned long iova, size_t size, int prot);

extern int iommu_sync_dirty_log(struct iommu_domain *domain, unsigned long iova,

				size_t size, unsigned long *bitmap,

				unsigned long base_iova,

				unsigned long bitmap_pgshift);

extern int iommu_clear_dirty_log(struct iommu_domain *domain, unsigned long iova,

				 size_t dma_size, unsigned long *bitmap,

				 unsigned long base_iova,

				 unsigned long bitmap_pgshift);



void iommu_set_dma_strict(void);



extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
@@ -1010,6 +1043,36 @@ static inline int iommu_set_pgtable_quirks(struct iommu_domain *domain, 
	return 0;

}



static inline bool iommu_support_dirty_log(struct iommu_domain *domain)

{

	return false;

}



static inline int iommu_switch_dirty_log(struct iommu_domain *domain,

					 bool enable, unsigned long iova,

					 size_t size, int prot)

{

	return -EINVAL;

}



static inline int iommu_sync_dirty_log(struct iommu_domain *domain,

				       unsigned long iova, size_t size,

				       unsigned long *bitmap,

				       unsigned long base_iova,

				       unsigned long pgshift)

{

	return -EINVAL;

}



static inline int iommu_clear_dirty_log(struct iommu_domain *domain,

					unsigned long iova, size_t size,

					unsigned long *bitmap,

					unsigned long base_iova,

					unsigned long pgshift)

{

	return -EINVAL;

}



static inline int iommu_device_register(struct iommu_device *iommu,

					const struct iommu_ops *ops,

					struct device *hwdev)

include/trace/events/iommu.h

+63 −0
Original line number Diff line number Diff line
@@ -124,6 +124,69 @@ TRACE_EVENT(unmap, 
	)

);



TRACE_EVENT(switch_dirty_log,



	TP_PROTO(unsigned long iova, size_t size, bool enable),



	TP_ARGS(iova, size, enable),



	TP_STRUCT__entry(

		__field(u64, iova)

		__field(size_t, size)

		__field(bool, enable)

	),



	TP_fast_assign(

		__entry->iova = iova;

		__entry->size = size;

		__entry->enable = enable;

	),



	TP_printk("IOMMU: iova=0x%016llx size=%zu enable=%u",

			__entry->iova, __entry->size, __entry->enable

	)

);



TRACE_EVENT(sync_dirty_log,



	TP_PROTO(unsigned long iova, size_t size),



	TP_ARGS(iova, size),



	TP_STRUCT__entry(

		__field(u64, iova)

		__field(size_t, size)

	),



	TP_fast_assign(

		__entry->iova = iova;

		__entry->size = size;

	),



	TP_printk("IOMMU: iova=0x%016llx size=%zu", __entry->iova,

			__entry->size)

);



TRACE_EVENT(clear_dirty_log,



	TP_PROTO(unsigned long iova, size_t size),



	TP_ARGS(iova, size),



	TP_STRUCT__entry(

		__field(u64, iova)

		__field(size_t, size)

	),



	TP_fast_assign(

		__entry->iova = iova;

		__entry->size = size;

	),



	TP_printk("IOMMU: iova=0x%016llx size=%zu", __entry->iova,

			__entry->size)

);



DECLARE_EVENT_CLASS(iommu_error,



	TP_PROTO(struct device *dev, unsigned long iova, int flags),