[PATCH 04/16] iommupt: Add iova_to_phys op

[Jason Gunthorpe <jgg@nvidia.com>]

iova_to_phys is a performance path for the DMA API and iommufd, implement
it using an unrolled get_user_pages() like function waterfall scheme.

The implementation itself is fairly trivial.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
---
 drivers/iommu/generic_pt/iommu_pt.h | 58 +++++++++++++++++++++++++++++
 include/linux/generic_pt/iommu.h    | 16 ++++++++
 2 files changed, 74 insertions(+)

diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h
index 708beaf5d812f7..835c84ea716093 100644
--- a/drivers/iommu/generic_pt/iommu_pt.h
+++ b/drivers/iommu/generic_pt/iommu_pt.h
@@ -15,6 +15,64 @@
 #include <linux/iommu.h>
 #include <linux/export.h>
 
+static int make_range(struct pt_common *common, struct pt_range *range,
+		      dma_addr_t iova, dma_addr_t len)
+{
+	dma_addr_t last;
+
+	if (unlikely(len == 0))
+		return -EINVAL;
+
+	if (check_add_overflow(iova, len - 1, &last))
+		return -EOVERFLOW;
+
+	*range = pt_make_range(common, iova, last);
+	if (sizeof(iova) > sizeof(range->va)) {
+		if (unlikely(range->va != iova || range->last_va != last))
+			return -EOVERFLOW;
+	}
+	return pt_check_range(range);
+}
+
+static __always_inline int __do_iova_to_phys(struct pt_range *range, void *arg,
+					     unsigned int level,
+					     struct pt_table_p *table,
+					     pt_level_fn_t descend_fn)
+{
+	struct pt_state pts = pt_init(range, level, table);
+	pt_oaddr_t *res = arg;
+
+	switch (pt_load_single_entry(&pts)) {
+	case PT_ENTRY_EMPTY:
+		return -ENOENT;
+	case PT_ENTRY_TABLE:
+		return pt_descend(&pts, arg, descend_fn);
+	case PT_ENTRY_OA:
+		*res = pt_entry_oa_full(&pts);
+		return 0;
+	}
+	return -ENOENT;
+}
+PT_MAKE_LEVELS(__iova_to_phys, __do_iova_to_phys);
+
+static phys_addr_t NS(iova_to_phys)(struct pt_iommu *iommu_table,
+				    dma_addr_t iova)
+{
+	struct pt_range range;
+	pt_oaddr_t res;
+	int ret;
+
+	ret = make_range(common_from_iommu(iommu_table), &range, iova, 1);
+	if (ret)
+		return ret;
+
+	ret = pt_walk_range(&range, __iova_to_phys, &res);
+	/* PHYS_ADDR_MAX would be a better error code */
+	if (ret)
+		return 0;
+	return res;
+}
+
 struct pt_iommu_collect_args {
 	struct pt_radix_list_head free_list;
 	u8 ignore_mapped : 1;
diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h
index d9d3da49dc0fe2..5cd56eac14b41d 100644
--- a/include/linux/generic_pt/iommu.h
+++ b/include/linux/generic_pt/iommu.h
@@ -60,6 +60,22 @@ struct pt_iommu_info {
 
 /* See the function comments in iommu_pt.c for kdocs */
 struct pt_iommu_ops {
+	/**
+	 * iova_to_phys() - Return the output address for the given IOVA
+	 * @iommu_table: Table to query
+	 * @iova: IO virtual address to query
+	 *
+	 * Determine the output address from the given IOVA. @iova may have any
+	 * alignment, the returned physical will be adjusted with any sub page
+	 * offset.
+	 *
+	 * Context: The caller must hold a read range lock that includes @iova.
+	 *
+	 * Return: 0 if there is no translation for the given iova.
+	 */
+	phys_addr_t (*iova_to_phys)(struct pt_iommu *iommu_table,
+				    dma_addr_t iova);
+
 	/**
 	 * get_info() - Return the pt_iommu_info structure
 	 * @iommu_table: Table to query
-- 
2.46.0