Re: [PATCH v2 0/4] mm/vfio: huge pfnmaps with !MAP_FIXED mappings

[Alex Mastro <amastro@fb.com>]

On Thu, Dec 04, 2025 at 10:09:59AM -0500, Peter Xu wrote:
> Alex Mastro: thanks for the testing offered in v1, but since this series
> was rewritten, a re-test will be needed.  I hence didn't collect the T-b.
 
Thank Peter, LGTM.

Tested-by: Alex Mastro <amastro@fb.com>

$ cc -Og -Wall -Wextra test_vfio_map_dma.c -o test_vfio_map_dma
$ ./test_vfio_map_dma 0000:05:00.0 4 0x600000 0x800000000 0x100000000
opening 0000:05:00.0 via /dev/vfio/39
BAR 4: size=0x2000000000, offset=0x40000000000, flags=0x7
mmap'd BAR 4: offset=0x600000, size=0x800000000 -> vaddr=0x7fdac0600000
VFIO_IOMMU_MAP_DMA: vaddr=0x7fdac0600000, iova=0x100000000, size=0x800000000

$ sudo bpftrace -q -e 'fexit:vfio_pci_mmap_huge_fault { printf("order=%d, ret=0x%x\n", args.order, retval); }' 2>&1 > ~/dump
$ cat ~/dump | sort | uniq -c | sort -nr
    512 order=9, ret=0x100
     31 order=18, ret=0x100
      2
      1 order=18, ret=0x800

test_vfio_map_dma.c
---
#include <errno.h>
#include <fcntl.h>
#include <libgen.h>
#include <linux/limits.h>
#include <linux/types.h>
#include <linux/vfio.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <unistd.h>

#define ensure(cond)                                                             \
	do {                                                                     \
		if (!(cond)) {                                                   \
			fprintf(stderr,                                          \
				"%s:%d Condition failed: '%s' (errno=%d: %s)\n", \
				__FILE__, __LINE__, #cond, errno,                \
				strerror(errno));                                \
			exit(EXIT_FAILURE);                                      \
		}                                                                \
	} while (0)

static uint32_t group_for_bdf(const char *bdf)
{
	char path[PATH_MAX];
	char link[PATH_MAX];
	int ret;

	snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/iommu_group",
		 bdf);
	ret = readlink(path, link, sizeof(link));
	ensure(ret > 0);

	const char *filename = basename(link);
	ensure(filename);

	return strtoul(filename, NULL, 0);
}

int main(int argc, char **argv)
{
	int ret;

	if (argc != 6) {
		printf("usage: %s <vfio_bdf> <bar_idx> <bar_offset> <size> <iova>\n",
		       argv[0]);
		printf("example: %s 0000:05:00.0 2 0x20000 0x1000 0x100000\n",
		       argv[0]);
		return 1;
	}

	const char *bdf = argv[1];
	uint32_t bar_idx = strtoul(argv[2], NULL, 0);
	uint64_t bar_offs = strtoull(argv[3], NULL, 0);
	uint64_t size = strtoull(argv[4], NULL, 0);
	uint64_t iova = strtoull(argv[5], NULL, 0);

	uint32_t group_num = group_for_bdf(bdf);
	char group_path[PATH_MAX];
	snprintf(group_path, sizeof(group_path), "/dev/vfio/%u", group_num);

	int container_fd = open("/dev/vfio/vfio", O_RDWR);
	ensure(container_fd >= 0);

	printf("opening %s via %s\n", bdf, group_path);
	int group_fd = open(group_path, O_RDWR);
	ensure(group_fd >= 0);

	ret = ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container_fd);
	ensure(!ret);

	ret = ioctl(container_fd, VFIO_SET_IOMMU, VFIO_TYPE1v2_IOMMU);
	ensure(!ret);

	int device_fd = ioctl(group_fd, VFIO_GROUP_GET_DEVICE_FD, bdf);
	ensure(device_fd >= 0);

	/* Get region info for the BAR */
	struct vfio_region_info region_info = {
		.argsz = sizeof(region_info),
		.index = bar_idx,
	};
	ret = ioctl(device_fd, VFIO_DEVICE_GET_REGION_INFO, &region_info);
	ensure(!ret);

	printf("BAR %u: size=0x%llx, offset=0x%llx, flags=0x%x\n", bar_idx,
	       region_info.size, region_info.offset, region_info.flags);

	ensure(region_info.flags & VFIO_REGION_INFO_FLAG_MMAP);
	ensure(bar_offs + size <= region_info.size);

	/* mmap the BAR at the specified offset */
	void *bar_mmap = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED,
			      device_fd, region_info.offset + bar_offs);
	ensure(bar_mmap != MAP_FAILED);

	ret = madvise(bar_mmap, size, MADV_HUGEPAGE);
	ensure(!ret);

	printf("mmap'd BAR %u: offset=0x%lx, size=0x%lx -> vaddr=%p\n", bar_idx,
	       bar_offs, size, bar_mmap);

	/* Map the mmap'd address into IOMMU using VFIO_IOMMU_MAP_DMA */
	struct vfio_iommu_type1_dma_map dma_map = {
		.argsz = sizeof(dma_map),
		.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
		.vaddr = (uint64_t)bar_mmap,
		.iova = iova,
		.size = size,
	};

	printf("VFIO_IOMMU_MAP_DMA: vaddr=%p, iova=0x%llx, size=0x%lx\n",
	       bar_mmap, (unsigned long long)dma_map.iova, size);

	ret = ioctl(container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
	ensure(!ret);

	/* Cleanup */
	struct vfio_iommu_type1_dma_unmap dma_unmap = {
		.argsz = sizeof(dma_unmap),
		.iova = dma_map.iova,
		.size = size,
	};
	ret = ioctl(container_fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap);
	ensure(!ret);

	ret = munmap(bar_mmap, size);
	ensure(!ret);

	close(device_fd);
	close(group_fd);
	close(container_fd);

	return 0;
}