!14946 swiotlb: fix swiotlb_bounce() to do partial sync's correctly

	 */

	if (dev_use_swiotlb(dev, size, dir) &&

	    iova_offset(iovad, phys | size)) {

		void *padding_start;

		size_t padding_size, aligned_size;

		if (!is_swiotlb_active(dev)) {

			dev_warn_once(dev, "DMA bounce buffers are inactive, unable to map unaligned transaction.\n");

			return DMA_MAPPING_ERROR;

		trace_swiotlb_bounced(dev, phys, size);

		aligned_size = iova_align(iovad, size);

		phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size,

		phys = swiotlb_tbl_map_single(dev, phys, size,

					      iova_mask(iovad), dir, attrs);

		if (phys == DMA_MAPPING_ERROR)

			return DMA_MAPPING_ERROR;

		/* Cleanup the padding area. */

		padding_start = phys_to_virt(phys);

		padding_size = aligned_size;

		/*

		 * Untrusted devices should not see padding areas with random

		 * leftover kernel data, so zero the pre- and post-padding.

		 * swiotlb_tbl_map_single() has initialized the bounce buffer

		 * proper to the contents of the original memory buffer.

		 */

		if (dev_is_untrusted(dev)) {

			size_t start, virt = (size_t)phys_to_virt(phys);

		if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&

		    (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) {

			padding_start += size;

			padding_size -= size;

		}

			/* Pre-padding */

			start = iova_align_down(iovad, virt);

			memset((void *)start, 0, virt - start);

		memset(padding_start, 0, padding_size);

			/* Post-padding */

			start = virt + size;

			memset((void *)start, 0,

			       iova_align(iovad, start) - start);

		}

	}

	if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))

	 */

	trace_swiotlb_bounced(dev, dev_addr, size);

	map = swiotlb_tbl_map_single(dev, phys, size, size, 0, dir, attrs);

	map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, attrs);

	if (map == (phys_addr_t)DMA_MAPPING_ERROR)

		return DMA_MAPPING_ERROR;

	return ALIGN(size, iovad->granule);

}

static inline size_t iova_align_down(struct iova_domain *iovad, size_t size)

{

	return ALIGN_DOWN(size, iovad->granule);

}

static inline dma_addr_t iova_dma_addr(struct iova_domain *iovad, struct iova *iova)

{

	return (dma_addr_t)iova->pfn_lo << iova_shift(iovad);

extern void __init swiotlb_update_mem_attributes(void);

phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,

		size_t mapping_size, size_t alloc_size,

		size_t mapping_size,

		unsigned int alloc_aligned_mask, enum dma_data_direction dir,

		unsigned long attrs);

 * @alloc_size:	Size of the allocated buffer.

 * @list:	The free list describing the number of free entries available

 *		from each index.

 * @pad_slots:	Number of preceding padding slots. Valid only in the first

 *		allocated non-padding slot.

 */

struct io_tlb_slot {

	phys_addr_t orig_addr;

	size_t alloc_size;

	unsigned int list;

	unsigned short list;

	unsigned short pad_slots;

};

static bool swiotlb_force_bounce;

					 mem->nslabs - i);

		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;

		mem->slots[i].alloc_size = 0;

		mem->slots[i].pad_slots = 0;

	}

	memset(vaddr, 0, bytes);

#endif

}

/*

 * Return the offset into a iotlb slot required to keep the device happy.

/**

 * swiotlb_align_offset() - Get required offset into an IO TLB allocation.

 * @dev:         Owning device.

 * @align_mask:  Allocation alignment mask.

 * @addr:        DMA address.

 *

 * Return the minimum offset from the start of an IO TLB allocation which is

 * required for a given buffer address and allocation alignment to keep the

 * device happy.

 *

 * First, the address bits covered by min_align_mask must be identical in the

 * original address and the bounce buffer address. High bits are preserved by

 * choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra

 * padding bytes before the bounce buffer.

 *

 * Second, @align_mask specifies which bits of the first allocated slot must

 * be zero. This may require allocating additional padding slots, and then the

 * offset (in bytes) from the first such padding slot is returned.

 */

static unsigned int swiotlb_align_offset(struct device *dev, u64 addr)

static unsigned int swiotlb_align_offset(struct device *dev,

					 unsigned int align_mask, u64 addr)

{

	return addr & dma_get_min_align_mask(dev) & (IO_TLB_SIZE - 1);

	return addr & dma_get_min_align_mask(dev) &

		(align_mask | (IO_TLB_SIZE - 1));

}

/*

	size_t alloc_size = mem->slots[index].alloc_size;

	unsigned long pfn = PFN_DOWN(orig_addr);

	unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;

	unsigned int tlb_offset, orig_addr_offset;

	int tlb_offset;

	if (orig_addr == INVALID_PHYS_ADDR)

		return;

	tlb_offset = tlb_addr & (IO_TLB_SIZE - 1);

	orig_addr_offset = swiotlb_align_offset(dev, orig_addr);

	if (tlb_offset < orig_addr_offset) {

		dev_WARN_ONCE(dev, 1,

			"Access before mapping start detected. orig offset %u, requested offset %u.\n",

			orig_addr_offset, tlb_offset);

		return;

	}

	tlb_offset -= orig_addr_offset;

	if (tlb_offset > alloc_size) {

		dev_WARN_ONCE(dev, 1,

			"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu+%u.\n",

			alloc_size, size, tlb_offset);

		return;

	}

	/*

	 * It's valid for tlb_offset to be negative. This can happen when the

	 * "offset" returned by swiotlb_align_offset() is non-zero, and the

	 * tlb_addr is pointing within the first "offset" bytes of the second

	 * or subsequent slots of the allocated swiotlb area. While it's not

	 * valid for tlb_addr to be pointing within the first "offset" bytes

	 * of the first slot, there's no way to check for such an error since

	 * this function can't distinguish the first slot from the second and

	 * subsequent slots.

	 */

	tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -

		     swiotlb_align_offset(dev, 0, orig_addr);

	orig_addr += tlb_offset;

	alloc_size -= tlb_offset;

	unsigned long max_slots = get_max_slots(boundary_mask);

	unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);

	unsigned int nslots = nr_slots(alloc_size), stride;

	unsigned int offset = swiotlb_align_offset(dev, orig_addr);

	unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);

	unsigned int index, slots_checked, count = 0, i;

	unsigned long flags;

	unsigned int slot_base;

#endif /* CONFIG_DEBUG_FS */

/**

 * swiotlb_tbl_map_single() - bounce buffer map a single contiguous physical area

 * @dev:		Device which maps the buffer.

 * @orig_addr:		Original (non-bounced) physical IO buffer address

 * @mapping_size:	Requested size of the actual bounce buffer, excluding

 *			any pre- or post-padding for alignment

 * @alloc_align_mask:	Required start and end alignment of the allocated buffer

 * @dir:		DMA direction

 * @attrs:		Optional DMA attributes for the map operation

 *

 * Find and allocate a suitable sequence of IO TLB slots for the request.

 * The allocated space starts at an alignment specified by alloc_align_mask,

 * and the size of the allocated space is rounded up so that the total amount

 * of allocated space is a multiple of (alloc_align_mask + 1). If

 * alloc_align_mask is zero, the allocated space may be at any alignment and

 * the size is not rounded up.

 *

 * The returned address is within the allocated space and matches the bits

 * of orig_addr that are specified in the DMA min_align_mask for the device. As

 * such, this returned address may be offset from the beginning of the allocated

 * space. The bounce buffer space starting at the returned address for

 * mapping_size bytes is initialized to the contents of the original IO buffer

 * area. Any pre-padding (due to an offset) and any post-padding (due to

 * rounding-up the size) is not initialized.

 */

phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,

		size_t mapping_size, size_t alloc_size,

		unsigned int alloc_align_mask, enum dma_data_direction dir,

		unsigned long attrs)

		size_t mapping_size, unsigned int alloc_align_mask,

		enum dma_data_direction dir, unsigned long attrs)

{

	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;

	unsigned int offset = swiotlb_align_offset(dev, orig_addr);

	unsigned int offset;

	struct io_tlb_pool *pool;

	unsigned int i;

	size_t size;

	int index;

	phys_addr_t tlb_addr;

	unsigned short pad_slots;

	if (!mem || !mem->nslabs) {

		dev_warn_ratelimited(dev,

	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))

		pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");

	if (mapping_size > alloc_size) {

		dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",

			      mapping_size, alloc_size);

		return (phys_addr_t)DMA_MAPPING_ERROR;

	}

	/*

	 * The default swiotlb memory pool is allocated with PAGE_SIZE

	 * alignment. If a mapping is requested with larger alignment,

	 * the mapping may be unable to use the initial slot(s) in all

	 * sets of IO_TLB_SEGSIZE slots. In such case, a mapping request

	 * of or near the maximum mapping size would always fail.

	 */

	dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK,

		"Alloc alignment may prevent fulfilling requests with max mapping_size\n");

	index = swiotlb_find_slots(dev, orig_addr,

				   alloc_size + offset, alloc_align_mask, &pool);

	offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);

	size = ALIGN(mapping_size + offset, alloc_align_mask + 1);

	index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);

	if (index == -1) {

		if (!(attrs & DMA_ATTR_NO_WARN))

			dev_warn_ratelimited(dev,

	"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",

				 alloc_size, mem->nslabs, mem_used(mem));

				 size, mem->nslabs, mem_used(mem));

		return (phys_addr_t)DMA_MAPPING_ERROR;

	}

	 * This is needed when we sync the memory.  Then we sync the buffer if

	 * needed.

	 */

	for (i = 0; i < nr_slots(alloc_size + offset); i++)

	pad_slots = offset >> IO_TLB_SHIFT;

	offset &= (IO_TLB_SIZE - 1);

	index += pad_slots;

	pool->slots[index].pad_slots = pad_slots;

	for (i = 0; i < (nr_slots(size) - pad_slots); i++)

		pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);

	tlb_addr = slot_addr(pool->start, index) + offset;

	/*

{

	struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);

	unsigned long flags;

	unsigned int offset = swiotlb_align_offset(dev, tlb_addr);

	int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;

	int nslots = nr_slots(mem->slots[index].alloc_size + offset);

	int aindex = index / mem->area_nslabs;

	struct io_tlb_area *area = &mem->areas[aindex];

	unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);

	int index, nslots, aindex;

	struct io_tlb_area *area;

	int count, i;

	index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;

	index -= mem->slots[index].pad_slots;

	nslots = nr_slots(mem->slots[index].alloc_size + offset);

	aindex = index / mem->area_nslabs;

	area = &mem->areas[aindex];

	/*

	 * Return the buffer to the free list by setting the corresponding

	 * entries to indicate the number of contiguous entries available.

		mem->slots[i].list = ++count;

		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;

		mem->slots[i].alloc_size = 0;

		mem->slots[i].pad_slots = 0;

	}

	/*

	trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);

	swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, 0, dir,

			attrs);

	swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);

	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)

		return DMA_MAPPING_ERROR;