Re: [RFC PATCH 6/6] drm/xe: Implement two pass MMU notifiers for SVM

[Matthew Brost <matthew.brost@intel.com>]

On Sat, Aug 09, 2025 at 03:51:37PM +0200, Thomas Hellström wrote:
> From: Matthew Brost <matthew.brost@intel.com>
> 
> Implement two-pass MMU notifiers for SVM, enabling multiple VMs or
> devices with GPU mappings to pipeline costly TLB invalidations by
> issuing them in the first pass and waiting for completion in the second.
> 
> Signed-off-by: Matthew Brost <matthew.brost@intel.com>
> ---
>  drivers/gpu/drm/drm_gpusvm.c |  2 +-
>  drivers/gpu/drm/xe/xe_svm.c  | 74 ++++++++++++++++++++++++++++++------
>  2 files changed, 63 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c
> index 92dc7d2bd6cf..f153df1bc862 100644
> --- a/drivers/gpu/drm/drm_gpusvm.c
> +++ b/drivers/gpu/drm/drm_gpusvm.c
> @@ -413,7 +413,7 @@ drm_gpusvm_notifier_invalidate_twopass(struct mmu_interval_notifier *mni,
>   * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM
>   */
>  static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
> -	.invalidate_twopass = drm_gpusvm_notifier_invalidate_twopass,
> +	.invalidate_multipass = drm_gpusvm_notifier_invalidate_twopass,

This should be in patch #2.

Matt

>  };
>  
>  /**
> diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
> index 82a598c8d56e..5728394806ca 100644
> --- a/drivers/gpu/drm/xe/xe_svm.c
> +++ b/drivers/gpu/drm/xe/xe_svm.c
> @@ -144,15 +144,8 @@ xe_svm_range_notifier_event_begin(struct xe_vm *vm, struct drm_gpusvm_range *r,
>  	 * invalidations spanning multiple ranges.
>  	 */
>  	for_each_tile(tile, xe, id)
> -		if (xe_pt_zap_ptes_range(tile, vm, range)) {
> +		if (xe_pt_zap_ptes_range(tile, vm, range))
>  			tile_mask |= BIT(id);
> -			/*
> -			 * WRITE_ONCE pairs with READ_ONCE in
> -			 * xe_vm_has_valid_gpu_mapping()
> -			 */
> -			WRITE_ONCE(range->tile_invalidated,
> -				   range->tile_invalidated | BIT(id));
> -		}
>  
>  	return tile_mask;
>  }
> @@ -161,16 +154,60 @@ static void
>  xe_svm_range_notifier_event_end(struct xe_vm *vm, struct drm_gpusvm_range *r,
>  				const struct mmu_notifier_range *mmu_range)
>  {
> +	struct xe_svm_range *range = to_xe_range(r);
>  	struct drm_gpusvm_ctx ctx = { .in_notifier = true, };
>  
>  	xe_svm_assert_in_notifier(vm);
>  
> +	/*
> +	 * WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping()
> +	 */
> +	WRITE_ONCE(range->tile_invalidated, range->tile_present);
> +
>  	drm_gpusvm_range_unmap_pages(&vm->svm.gpusvm, r, &ctx);
>  	if (!xe_vm_is_closed(vm) && mmu_range->event == MMU_NOTIFY_UNMAP)
>  		xe_svm_garbage_collector_add_range(vm, to_xe_range(r),
>  						   mmu_range);
>  }
>  
> +struct xe_svm_invalidate_pass {
> +	struct drm_gpusvm *gpusvm;
> +	struct drm_gpusvm_notifier *notifier;
> +#define XE_SVM_INVALIDATE_FENCE_COUNT	\
> +	(XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE)
> +	struct xe_gt_tlb_invalidation_fence fences[XE_SVM_INVALIDATE_FENCE_COUNT];
> +	struct mmu_interval_notifier_pass p;
> +};
> +
> +static struct mmu_interval_notifier_pass *
> +xe_svm_invalidate_second(struct mmu_interval_notifier_pass *p,
> +			 const struct mmu_notifier_range *mmu_range,
> +			 unsigned long cur_seq)
> +{
> +	struct xe_svm_invalidate_pass *pass = container_of(p, typeof(*pass), p);
> +	struct drm_gpusvm *gpusvm = pass->gpusvm;
> +	struct drm_gpusvm_notifier *notifier = pass->notifier;
> +	struct drm_gpusvm_range *r = NULL;
> +	struct xe_vm *vm = gpusvm_to_vm(gpusvm);
> +	u64 adj_start = mmu_range->start, adj_end = mmu_range->end;
> +	int id;
> +
> +	/* Adjust invalidation to notifier boundaries */
> +	adj_start = max(drm_gpusvm_notifier_start(notifier), adj_start);
> +	adj_end = min(drm_gpusvm_notifier_end(notifier), adj_end);
> +
> +	for (id = 0; id < XE_SVM_INVALIDATE_FENCE_COUNT; ++id)
> +		xe_gt_tlb_invalidation_fence_wait(&pass->fences[id]);
> +
> +	drm_gpusvm_in_notifier_lock(gpusvm);
> +	drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end)
> +		xe_svm_range_notifier_event_end(vm, r, mmu_range);
> +	drm_gpusvm_in_notifier_unlock(gpusvm);
> +
> +	kfree(pass);
> +	return NULL;
> +}
> +
>  static void xe_svm_invalidate_twopass(struct drm_gpusvm *gpusvm,
>  				      struct drm_gpusvm_notifier *notifier,
>  				      const struct mmu_notifier_range *mmu_range,
> @@ -179,6 +216,8 @@ static void xe_svm_invalidate_twopass(struct drm_gpusvm *gpusvm,
>  	struct xe_vm *vm = gpusvm_to_vm(gpusvm);
>  	struct xe_device *xe = vm->xe;
>  	struct drm_gpusvm_range *r, *first;
> +	struct xe_svm_invalidate_pass *pass = NULL;
> +	struct xe_gt_tlb_invalidation_fence *fences = NULL;
>  	u64 adj_start = mmu_range->start, adj_end = mmu_range->end;
>  	u8 tile_mask = 0;
>  	long err;
> @@ -226,14 +265,25 @@ static void xe_svm_invalidate_twopass(struct drm_gpusvm *gpusvm,
>  
>  	xe_device_wmb(xe);
>  
> -	err = xe_vm_range_tilemask_tlb_invalidation(vm, NULL, adj_start,
> +	pass = kzalloc(sizeof(*pass), GFP_NOWAIT);
> +	if (pass) {
> +		pass->gpusvm = gpusvm;
> +		pass->notifier = notifier;
> +		pass->p.pass = xe_svm_invalidate_second;
> +		fences = pass->fences;
> +		*p = &pass->p;
> +	}
> +
> +	err = xe_vm_range_tilemask_tlb_invalidation(vm, fences, adj_start,
>  						    adj_end, tile_mask);
>  	WARN_ON_ONCE(err);
>  
>  range_notifier_event_end:
> -	r = first;
> -	drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end)
> -		xe_svm_range_notifier_event_end(vm, r, mmu_range);
> +	if (!pass) {
> +		r = first;
> +		drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end)
> +			xe_svm_range_notifier_event_end(vm, r, mmu_range);
> +	}
>  }
>  
>  static int __xe_svm_garbage_collector(struct xe_vm *vm,
> -- 
> 2.50.1
>