[RFC PATCH] mm: don't promote exclusive file folios of dying processes

[Barry Song <21cnbao@gmail.com>]

From: Barry Song <v-songbaohua@oppo.com>

Promoting exclusive file folios of a dying process is unnecessary and
harmful. For example, while Firefox is killed and LibreOffice is
launched, activating Firefox's young file-backed folios makes it
harder to reclaim memory that LibreOffice doesn't use at all.

An exiting process is unlikely to be restarted right away—it's
either terminated by the user or killed by the OOM handler.

Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Barry Song <v-songbaohua@oppo.com>
---
 mm/huge_memory.c |  4 ++--
 mm/internal.h    | 19 +++++++++++++++++++
 mm/memory.c      |  9 ++++++++-
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index e97a97586478..05b83d2fcbb6 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2264,8 +2264,8 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			 * Use flush_needed to indicate whether the PMD entry
 			 * is present, instead of checking pmd_present() again.
 			 */
-			if (flush_needed && pmd_young(orig_pmd) &&
-			    likely(vma_has_recency(vma)))
+			if (!exclusive_folio_of_dying_process(folio, vma) && flush_needed &&
+			    pmd_young(orig_pmd) && likely(vma_has_recency(vma)))
 				folio_mark_accessed(folio);
 		}
 
diff --git a/mm/internal.h b/mm/internal.h
index 4e0ea83aaf1c..666de96a293d 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -11,6 +11,7 @@
 #include <linux/khugepaged.h>
 #include <linux/mm.h>
 #include <linux/mm_inline.h>
+#include <linux/oom.h>
 #include <linux/pagemap.h>
 #include <linux/pagewalk.h>
 #include <linux/rmap.h>
@@ -130,6 +131,24 @@ static inline int folio_nr_pages_mapped(const struct folio *folio)
 	return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED;
 }
 
+/*
+ * Return true if a folio is exclusive and belongs to an exiting or
+ * oom-reaped process; otherwise, return false.
+ */
+static inline bool exclusive_folio_of_dying_process(struct folio *folio,
+		struct vm_area_struct *vma)
+{
+	if (folio_maybe_mapped_shared(folio))
+		return false;
+
+	if (!atomic_read(&vma->vm_mm->mm_users))
+		return true;
+	if (check_stable_address_space(vma->vm_mm))
+		return true;
+
+	return false;
+}
+
 /*
  * Retrieve the first entry of a folio based on a provided entry within the
  * folio. We cannot rely on folio->swap as there is no guarantee that it has
diff --git a/mm/memory.c b/mm/memory.c
index b9e8443aaa86..cab69275e473 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1515,7 +1515,14 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,
 				*force_flush = true;
 			}
 		}
-		if (pte_young(ptent) && likely(vma_has_recency(vma)))
+
+		/*
+		 * Skip marking exclusive file folios as accessed for processes that are
+		 * exiting or have been reaped due to OOM. This prevents unnecessary
+		 * promotion of folios that won't benefit the new process being launched.
+		 */
+		if (!exclusive_folio_of_dying_process(folio, vma) && pte_young(ptent) &&
+				likely(vma_has_recency(vma)))
 			folio_mark_accessed(folio);
 		rss[mm_counter(folio)] -= nr;
 	} else {
-- 
2.39.3 (Apple Git-146)