Commit cd5f82db authored by Greg Kroah-Hartman's avatar Greg Kroah-Hartman
Browse files

Merge tag 'misc-habanalabs-next-2020-11-30' of... 

Merge tag 'misc-habanalabs-next-2020-11-30' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux into char-misc-next

This tag contains habanalabs driver changes for v5.11-rc1:

- Add support for ability to perform collective stream sync. This is basically
  a synchronization between compute and network streams.

- Add initialization of NIC QMANs and security configuration. This is a
  pre-requisite for upstreaming the NIC ETH and RDMA code.

- Add option to scrub all internal memory (SRAM and DRAM) when the user
  closes the file-descriptor

- Support new firmware that provide enhanced device security. This includes
  many changes that basically amounts to moving certain configurations to
  the firmware and stop reading registers directly and instead receiving the
  information from the firmware. For example:
  - Retrieve HBM ECC error information
  - Retrieve PLL configuration
  - Configure of internal credits, rate-limitation

- Support new firmware that performs the GAUDI device reset instead of the
  driver. The driver now asks the firmware to do it.

- Some changes were done as Pre-requisite for future ASICs support:
  - Add option to put the device's PCI MMU page tables on the host memory.
  - Support loading multiple types of firmware.
  - Adding option to user to inquire about usage counter of Command buffer.

- Support taking timestamp of Command Submission when it completes and
  providing it to the user.

- Change aggregate cs counters to atomic and fix the cs counters structure
  to support addition of new counters in the future

- Update email address nad git repo of the driver in MAINTAINERS

- Many small bug fixes and improvements, such as:
  - Refactoring in MMU code to move code from ASIC-dependant files to
    common code
  - Minimize driver prints when no errors occur
  - Using enums, defines instead of hard-coded values
  - Refactoring of Command Submission flow to make it more readable now that
    we have multiple types of Command Submissions.

* tag 'misc-habanalabs-next-2020-11-30' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux: (76 commits)
  habanalabs: Add CB IOCTL opcode to retrieve CB information
  habanalabs: Modify the cs_cnt of a CB to be atomic
  habanalabs: Add mask for CS type bits in CS flags
  habanalabs: change messages to debug level
  habanalabs: free host huge va_range if not used
  habanalabs/gaudi: handle reset when f/w is in preboot
  habanalabs: add missing counter update
  habanalabs: add ull to PLL masks
  habanalabs: add support for cs with timestamp
  habanalabs: indicate to user that a cs is gone
  habanalabs/gaudi: print ECC type field
  habanalabs: update firmware files
  habanalabs: gaudi_ctx_fini() can be static
  habanalabs: goya_reset_sob_group() can be static
  habanalabs: fetch pll frequency from firmware
  habanalabs: mmu map wrapper for sizes larger than a page
  habanalabs: print CS type when it is stuck
  habanalabs/gaudi: align to new FW reset scheme
  habanalabs: firmware returns 64bit argument
  habanalabs: fix MMU debugfs operations
  ...
parents 34730659 f44afb5b

MAINTAINERS

+2 −2
Original line number Diff line number Diff line
@@ -7690,9 +7690,9 @@ F: drivers/clocksource/h8300_*.c 
F:	drivers/irqchip/irq-renesas-h8*.c









HABANALABS PCI DRIVER









M:	Oded Gabbay <oded.gabbay@gmail.com>









M:	Oded Gabbay <ogabbay@kernel.org>









S:	Supported









T:	git https://github.com/HabanaAI/linux.git









T:	git https://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux.git









F:	Documentation/ABI/testing/debugfs-driver-habanalabs









F:	Documentation/ABI/testing/sysfs-driver-habanalabs









F:	drivers/misc/habanalabs/
































drivers/misc/habanalabs/common/command_buffer.c



+47
−8






























Original line number
Diff line number
Diff line







@@ -11,7 +11,6 @@







#include <linux/mm.h>









#include <linux/slab.h>









#include <linux/uaccess.h>









#include <linux/genalloc.h>



















static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)









{








@@ -68,8 +67,8 @@ static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)







	bus_addr = cb->bus_address;









	offset = 0;









	list_for_each_entry(va_block, &cb->va_block_list, node) {









		rc = hl_mmu_map(ctx, va_block->start, bus_addr, va_block->size,









				list_is_last(&va_block->node,









		rc = hl_mmu_map_page(ctx, va_block->start, bus_addr,









				va_block->size, list_is_last(&va_block->node,









							&cb->va_block_list));









		if (rc) {









			dev_err(hdev->dev, "Failed to map VA %#llx to CB\n",








@@ -93,7 +92,7 @@ static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)







	list_for_each_entry(va_block, &cb->va_block_list, node) {









		if (offset <= 0)









			break;









		hl_mmu_unmap(ctx, va_block->start, va_block->size,









		hl_mmu_unmap_page(ctx, va_block->start, va_block->size,









				offset <= va_block->size);









		offset -= va_block->size;









	}








@@ -120,7 +119,7 @@ static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb)







	mutex_lock(&ctx->mmu_lock);



















	list_for_each_entry(va_block, &cb->va_block_list, node)









		if (hl_mmu_unmap(ctx, va_block->start, va_block->size,









		if (hl_mmu_unmap_page(ctx, va_block->start, va_block->size,









				list_is_last(&va_block->node,









						&cb->va_block_list)))









			dev_warn_ratelimited(hdev->dev,








@@ -376,17 +375,49 @@ int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)







	return rc;









}



















static int hl_cb_info(struct hl_device *hdev, struct hl_cb_mgr *mgr,









			u64 cb_handle, u32 *usage_cnt)









{









	struct hl_cb *cb;









	u32 handle;









	int rc = 0;



















	/* The CB handle was given to user to do mmap, so need to shift it back









	 * to the value which was allocated by the IDR module.









	 */









	cb_handle >>= PAGE_SHIFT;









	handle = (u32) cb_handle;



















	spin_lock(&mgr->cb_lock);



















	cb = idr_find(&mgr->cb_handles, handle);









	if (!cb) {









		dev_err(hdev->dev,









			"CB info failed, no match to handle 0x%x\n", handle);









		rc = -EINVAL;









		goto out;









	}



















	*usage_cnt = atomic_read(&cb->cs_cnt);



















out:









	spin_unlock(&mgr->cb_lock);









	return rc;









}



















int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)









{









	union hl_cb_args *args = data;









	struct hl_device *hdev = hpriv->hdev;









	enum hl_device_status status;









	u64 handle = 0;









	u32 usage_cnt = 0;









	int rc;



















	if (hl_device_disabled_or_in_reset(hdev)) {









	if (!hl_device_operational(hdev, &status)) {









		dev_warn_ratelimited(hdev->dev,









			"Device is %s. Can't execute CB IOCTL\n",









			atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");









			hdev->status[status]);









		return -EBUSY;









	}


















@@ -413,6 +444,13 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)







					args->in.cb_handle);









		break;



















	case HL_CB_OP_INFO:









		rc = hl_cb_info(hdev, &hpriv->cb_mgr, args->in.cb_handle,









				&usage_cnt);









		memset(args, 0, sizeof(*args));









		args->out.usage_cnt = usage_cnt;









		break;



















	default:









		rc = -ENOTTY;









		break;








@@ -517,6 +555,7 @@ int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)







	}



















	cb->mmap_size = cb->size;









	vma->vm_pgoff = handle;



















	return 0;










































drivers/misc/habanalabs/common/command_submission.c



+617
−399




























File changed.
Preview size limit exceeded, changes collapsed.
























drivers/misc/habanalabs/common/context.c



+4
−0






























Original line number
Diff line number
Diff line







@@ -40,10 +40,14 @@ static void hl_ctx_fini(struct hl_ctx *ctx)







		if ((hdev->in_debug) && (hdev->compute_ctx == ctx))









			hl_device_set_debug_mode(hdev, false);



















		hdev->asic_funcs->ctx_fini(ctx);









		hl_cb_va_pool_fini(ctx);









		hl_vm_ctx_fini(ctx);









		hl_asid_free(hdev, ctx->asid);



















		/* Scrub both SRAM and DRAM */









		hdev->asic_funcs->scrub_device_mem(hdev, 0, 0);



















		if ((!hdev->pldm) && (hdev->pdev) &&









				(!hdev->asic_funcs->is_device_idle(hdev,









							&idle_mask, NULL)))
































drivers/misc/habanalabs/common/debugfs.c



+35
−275






























Original line number
Diff line number
Diff line







@@ -22,9 +22,10 @@ static int hl_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,







				u8 i2c_reg, long *val)









{









	struct cpucp_packet pkt;









	u64 result;









	int rc;



















	if (hl_device_disabled_or_in_reset(hdev))









	if (!hl_device_operational(hdev, NULL))









		return -EBUSY;



















	memset(&pkt, 0, sizeof(pkt));








@@ -36,7 +37,9 @@ static int hl_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,







	pkt.i2c_reg = i2c_reg;



















	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),









						0, val);









						0, &result);



















	*val = (long) result;



















	if (rc)









		dev_err(hdev->dev, "Failed to read from I2C, error %d\n", rc);








@@ -50,7 +53,7 @@ static int hl_debugfs_i2c_write(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr,







	struct cpucp_packet pkt;









	int rc;



















	if (hl_device_disabled_or_in_reset(hdev))









	if (!hl_device_operational(hdev, NULL))









		return -EBUSY;



















	memset(&pkt, 0, sizeof(pkt));








@@ -76,7 +79,7 @@ static void hl_debugfs_led_set(struct hl_device *hdev, u8 led, u8 state)







	struct cpucp_packet pkt;









	int rc;



















	if (hl_device_disabled_or_in_reset(hdev))









	if (!hl_device_operational(hdev, NULL))









		return;



















	memset(&pkt, 0, sizeof(pkt));








@@ -113,7 +116,7 @@ static int command_buffers_show(struct seq_file *s, void *data)







			"   %03llu        %d    0x%08x      %d          %d          %d\n",









			cb->id, cb->ctx->asid, cb->size,









			kref_read(&cb->refcount),









			cb->mmap, cb->cs_cnt);









			cb->mmap, atomic_read(&cb->cs_cnt));









	}



















	spin_unlock(&dev_entry->cb_spinlock);








@@ -168,18 +171,19 @@ static int command_submission_jobs_show(struct seq_file *s, void *data)







		if (first) {









			first = false;









			seq_puts(s, "\n");









			seq_puts(s, " JOB ID   CS ID    CTX ASID   H/W Queue\n");









			seq_puts(s, "---------------------------------------\n");









			seq_puts(s, " JOB ID   CS ID    CTX ASID   JOB RefCnt   H/W Queue\n");









			seq_puts(s, "----------------------------------------------------\n");









		}









		if (job->cs)









			seq_printf(s,









				"    %02d       %llu         %d         %d\n",









				"   %02d      %llu        %d          %d           %d\n",









				job->id, job->cs->sequence, job->cs->ctx->asid,









				job->hw_queue_id);









				kref_read(&job->refcount), job->hw_queue_id);









		else









			seq_printf(s,









				"    %02d       0         %d         %d\n",









				job->id, HL_KERNEL_ASID_ID, job->hw_queue_id);









				"   %02d      0        %d          %d           %d\n",









				job->id, HL_KERNEL_ASID_ID,









				kref_read(&job->refcount), job->hw_queue_id);









	}



















	spin_unlock(&dev_entry->cs_job_spinlock);








@@ -300,93 +304,15 @@ static int vm_show(struct seq_file *s, void *data)







	return 0;









}



















/* these inline functions are copied from mmu.c */









static inline u64 get_hop0_addr(struct hl_ctx *ctx)









{









	return ctx->hdev->asic_prop.mmu_pgt_addr +









			(ctx->asid * ctx->hdev->asic_prop.mmu_hop_table_size);









}



















static inline u64 get_hopN_pte_addr(struct hl_ctx *ctx, u64 hop_addr,









					u64 virt_addr, u64 mask, u64 shift)









{









	return hop_addr + ctx->hdev->asic_prop.mmu_pte_size *









			((virt_addr & mask) >> shift);









}



















static inline u64 get_hop0_pte_addr(struct hl_ctx *ctx,









					struct hl_mmu_properties *mmu_specs,









					u64 hop_addr, u64 vaddr)









{









	return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_specs->hop0_mask,









					mmu_specs->hop0_shift);









}



















static inline u64 get_hop1_pte_addr(struct hl_ctx *ctx,









					struct hl_mmu_properties *mmu_specs,









					u64 hop_addr, u64 vaddr)









{









	return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_specs->hop1_mask,









					mmu_specs->hop1_shift);









}



















static inline u64 get_hop2_pte_addr(struct hl_ctx *ctx,









					struct hl_mmu_properties *mmu_specs,









					u64 hop_addr, u64 vaddr)









{









	return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_specs->hop2_mask,









					mmu_specs->hop2_shift);









}



















static inline u64 get_hop3_pte_addr(struct hl_ctx *ctx,









					struct hl_mmu_properties *mmu_specs,









					u64 hop_addr, u64 vaddr)









{









	return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_specs->hop3_mask,









					mmu_specs->hop3_shift);









}



















static inline u64 get_hop4_pte_addr(struct hl_ctx *ctx,









					struct hl_mmu_properties *mmu_specs,









					u64 hop_addr, u64 vaddr)









{









	return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_specs->hop4_mask,









					mmu_specs->hop4_shift);









}



















static inline u64 get_hop5_pte_addr(struct hl_ctx *ctx,









					struct hl_mmu_properties *mmu_specs,









					u64 hop_addr, u64 vaddr)









{









	return get_hopN_pte_addr(ctx, hop_addr, vaddr, mmu_specs->hop5_mask,









					mmu_specs->hop5_shift);









}



















static inline u64 get_next_hop_addr(u64 curr_pte)









{









	if (curr_pte & PAGE_PRESENT_MASK)









		return curr_pte & HOP_PHYS_ADDR_MASK;









	else









		return ULLONG_MAX;









}



















static int mmu_show(struct seq_file *s, void *data)









{









	struct hl_debugfs_entry *entry = s->private;









	struct hl_dbg_device_entry *dev_entry = entry->dev_entry;









	struct hl_device *hdev = dev_entry->hdev;









	struct asic_fixed_properties *prop = &hdev->asic_prop;









	struct hl_mmu_properties *mmu_prop;









	struct hl_ctx *ctx;









	bool is_dram_addr;



















	u64 hop0_addr = 0, hop0_pte_addr = 0, hop0_pte = 0,









		hop1_addr = 0, hop1_pte_addr = 0, hop1_pte = 0,









		hop2_addr = 0, hop2_pte_addr = 0, hop2_pte = 0,









		hop3_addr = 0, hop3_pte_addr = 0, hop3_pte = 0,









		hop4_addr = 0, hop4_pte_addr = 0, hop4_pte = 0,









		hop5_addr = 0, hop5_pte_addr = 0, hop5_pte = 0,









		virt_addr = dev_entry->mmu_addr;









	struct hl_mmu_hop_info hops_info;









	u64 virt_addr = dev_entry->mmu_addr;









	int i;



















	if (!hdev->mmu_enable)









		return 0;








@@ -401,131 +327,23 @@ static int mmu_show(struct seq_file *s, void *data)







		return 0;









	}



















	is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,









						prop->dmmu.start_addr,









						prop->dmmu.end_addr);



















	/* shifts and masks are the same in PMMU and HPMMU, use one of them */









	mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu;



















	mutex_lock(&ctx->mmu_lock);



















	/* the following lookup is copied from unmap() in mmu.c */



















	hop0_addr = get_hop0_addr(ctx);









	hop0_pte_addr = get_hop0_pte_addr(ctx, mmu_prop, hop0_addr, virt_addr);









	hop0_pte = hdev->asic_funcs->read_pte(hdev, hop0_pte_addr);









	hop1_addr = get_next_hop_addr(hop0_pte);



















	if (hop1_addr == ULLONG_MAX)









		goto not_mapped;



















	hop1_pte_addr = get_hop1_pte_addr(ctx, mmu_prop, hop1_addr, virt_addr);









	hop1_pte = hdev->asic_funcs->read_pte(hdev, hop1_pte_addr);









	hop2_addr = get_next_hop_addr(hop1_pte);



















	if (hop2_addr == ULLONG_MAX)









		goto not_mapped;



















	hop2_pte_addr = get_hop2_pte_addr(ctx, mmu_prop, hop2_addr, virt_addr);









	hop2_pte = hdev->asic_funcs->read_pte(hdev, hop2_pte_addr);









	hop3_addr = get_next_hop_addr(hop2_pte);



















	if (hop3_addr == ULLONG_MAX)









		goto not_mapped;



















	hop3_pte_addr = get_hop3_pte_addr(ctx, mmu_prop, hop3_addr, virt_addr);









	hop3_pte = hdev->asic_funcs->read_pte(hdev, hop3_pte_addr);



















	if (mmu_prop->num_hops == MMU_ARCH_5_HOPS) {









		if (!(hop3_pte & LAST_MASK)) {









			hop4_addr = get_next_hop_addr(hop3_pte);



















			if (hop4_addr == ULLONG_MAX)









				goto not_mapped;



















			hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop,









							hop4_addr, virt_addr);









			hop4_pte = hdev->asic_funcs->read_pte(hdev,









								hop4_pte_addr);









			if (!(hop4_pte & PAGE_PRESENT_MASK))









				goto not_mapped;









		} else {









			if (!(hop3_pte & PAGE_PRESENT_MASK))









				goto not_mapped;









		}









	} else {









		hop4_addr = get_next_hop_addr(hop3_pte);



















		if (hop4_addr == ULLONG_MAX)









			goto not_mapped;



















		hop4_pte_addr = get_hop4_pte_addr(ctx, mmu_prop,









						hop4_addr, virt_addr);









		hop4_pte = hdev->asic_funcs->read_pte(hdev,









							hop4_pte_addr);









		if (!(hop4_pte & LAST_MASK)) {









			hop5_addr = get_next_hop_addr(hop4_pte);



















			if (hop5_addr == ULLONG_MAX)









				goto not_mapped;



















			hop5_pte_addr = get_hop5_pte_addr(ctx, mmu_prop,









							hop5_addr, virt_addr);









			hop5_pte = hdev->asic_funcs->read_pte(hdev,









								hop5_pte_addr);









			if (!(hop5_pte & PAGE_PRESENT_MASK))









				goto not_mapped;









		} else {









			if (!(hop4_pte & PAGE_PRESENT_MASK))









				goto not_mapped;









		}









	if (hl_mmu_get_tlb_info(ctx, virt_addr, &hops_info)) {









		dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",









				virt_addr);









		return 0;









	}



















	seq_printf(s, "asid: %u, virt_addr: 0x%llx\n",









			dev_entry->mmu_asid, dev_entry->mmu_addr);



















	seq_printf(s, "hop0_addr: 0x%llx\n", hop0_addr);









	seq_printf(s, "hop0_pte_addr: 0x%llx\n", hop0_pte_addr);









	seq_printf(s, "hop0_pte: 0x%llx\n", hop0_pte);



















	seq_printf(s, "hop1_addr: 0x%llx\n", hop1_addr);









	seq_printf(s, "hop1_pte_addr: 0x%llx\n", hop1_pte_addr);









	seq_printf(s, "hop1_pte: 0x%llx\n", hop1_pte);



















	seq_printf(s, "hop2_addr: 0x%llx\n", hop2_addr);









	seq_printf(s, "hop2_pte_addr: 0x%llx\n", hop2_pte_addr);









	seq_printf(s, "hop2_pte: 0x%llx\n", hop2_pte);



















	seq_printf(s, "hop3_addr: 0x%llx\n", hop3_addr);









	seq_printf(s, "hop3_pte_addr: 0x%llx\n", hop3_pte_addr);









	seq_printf(s, "hop3_pte: 0x%llx\n", hop3_pte);



















	if (mmu_prop->num_hops == MMU_ARCH_5_HOPS) {









		if (!(hop3_pte & LAST_MASK)) {









			seq_printf(s, "hop4_addr: 0x%llx\n", hop4_addr);









			seq_printf(s, "hop4_pte_addr: 0x%llx\n", hop4_pte_addr);









			seq_printf(s, "hop4_pte: 0x%llx\n", hop4_pte);









		}









	} else {









		seq_printf(s, "hop4_addr: 0x%llx\n", hop4_addr);









		seq_printf(s, "hop4_pte_addr: 0x%llx\n", hop4_pte_addr);









		seq_printf(s, "hop4_pte: 0x%llx\n", hop4_pte);



















		if (!(hop4_pte & LAST_MASK)) {









			seq_printf(s, "hop5_addr: 0x%llx\n", hop5_addr);









			seq_printf(s, "hop5_pte_addr: 0x%llx\n", hop5_pte_addr);









			seq_printf(s, "hop5_pte: 0x%llx\n", hop5_pte);









	for (i = 0 ; i < hops_info.used_hops ; i++) {









		seq_printf(s, "hop%d_addr: 0x%llx\n",









				i, hops_info.hop_info[i].hop_addr);









		seq_printf(s, "hop%d_pte_addr: 0x%llx\n",









				i, hops_info.hop_info[i].hop_pte_addr);









		seq_printf(s, "hop%d_pte: 0x%llx\n",









				i, hops_info.hop_info[i].hop_pte_val);









	}









	}



















	goto out;



















not_mapped:









	dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",









			virt_addr);









out:









	mutex_unlock(&ctx->mmu_lock);



















	return 0;









}








@@ -597,7 +415,7 @@ static bool hl_is_device_va(struct hl_device *hdev, u64 addr)







	if (!hdev->mmu_enable)









		goto out;



















	if (hdev->dram_supports_virtual_memory &&









	if (prop->dram_supports_virtual_memory &&









		(addr >= prop->dmmu.start_addr && addr < prop->dmmu.end_addr))









		return true;


















@@ -616,78 +434,20 @@ static int device_va_to_pa(struct hl_device *hdev, u64 virt_addr,







				u64 *phys_addr)









{









	struct hl_ctx *ctx = hdev->compute_ctx;









	struct asic_fixed_properties *prop = &hdev->asic_prop;









	struct hl_mmu_properties *mmu_prop;









	u64 hop_addr, hop_pte_addr, hop_pte;









	u64 offset_mask = HOP4_MASK | FLAGS_MASK;









	int rc = 0;









	bool is_dram_addr;



















	if (!ctx) {









		dev_err(hdev->dev, "no ctx available\n");









		return -EINVAL;









	}



















	is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,









						prop->dmmu.start_addr,









						prop->dmmu.end_addr);



















	/* shifts and masks are the same in PMMU and HPMMU, use one of them */









	mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu;



















	mutex_lock(&ctx->mmu_lock);



















	/* hop 0 */









	hop_addr = get_hop0_addr(ctx);









	hop_pte_addr = get_hop0_pte_addr(ctx, mmu_prop, hop_addr, virt_addr);









	hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);



















	/* hop 1 */









	hop_addr = get_next_hop_addr(hop_pte);









	if (hop_addr == ULLONG_MAX)









		goto not_mapped;









	hop_pte_addr = get_hop1_pte_addr(ctx, mmu_prop, hop_addr, virt_addr);









	hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);



















	/* hop 2 */









	hop_addr = get_next_hop_addr(hop_pte);









	if (hop_addr == ULLONG_MAX)









		goto not_mapped;









	hop_pte_addr = get_hop2_pte_addr(ctx, mmu_prop, hop_addr, virt_addr);









	hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);



















	/* hop 3 */









	hop_addr = get_next_hop_addr(hop_pte);









	if (hop_addr == ULLONG_MAX)









		goto not_mapped;









	hop_pte_addr = get_hop3_pte_addr(ctx, mmu_prop, hop_addr, virt_addr);









	hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);



















	if (!(hop_pte & LAST_MASK)) {









		/* hop 4 */









		hop_addr = get_next_hop_addr(hop_pte);









		if (hop_addr == ULLONG_MAX)









			goto not_mapped;









		hop_pte_addr = get_hop4_pte_addr(ctx, mmu_prop, hop_addr,









							virt_addr);









		hop_pte = hdev->asic_funcs->read_pte(hdev, hop_pte_addr);



















		offset_mask = FLAGS_MASK;









	}



















	if (!(hop_pte & PAGE_PRESENT_MASK))









		goto not_mapped;



















	*phys_addr = (hop_pte & ~offset_mask) | (virt_addr & offset_mask);



















	goto out;



















not_mapped:









	rc = hl_mmu_va_to_pa(ctx, virt_addr, phys_addr);









	if (rc) {









		dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",









				virt_addr);









		rc = -EINVAL;









out:









	mutex_unlock(&ctx->mmu_lock);









	}



















	return rc;









}