[RFC PATCH 5/5] test: add sysctl for folio copy tests and adjust NR_MAX_BATCHED_MIGRATION

[Zi Yan <ziy@nvidia.com>]

1. enable multi-threaded copy
2. specify how many CPU threads to use
3. push from local CPUs or pull from remote CPUs
4. change NR_MAX_BATCHED_MIGRATION to HPAGE_PUD_NR to allow batching THP
copies.

These are for testing purpose only.

Signed-off-by: Zi Yan <ziy@nvidia.com>
---
 include/linux/mm.h     |  4 ++++
 include/linux/sysctl.h |  1 +
 kernel/sysctl.c        | 29 ++++++++++++++++++++++++++++-
 mm/copy_pages.c        | 10 +++++++---
 mm/migrate.c           |  6 ++++--
 5 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 1a11f9df5c2d..277b12b9ef0d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -42,6 +42,10 @@ struct pt_regs;
 struct folio_batch;
 
 extern int sysctl_page_lock_unfairness;
+extern int sysctl_use_mt_copy;
+extern unsigned int sysctl_limit_mt_num;
+extern unsigned int sysctl_push_0_pull_1;
+
 
 void mm_core_init(void);
 void init_mm_internals(void);
diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h
index 40a6ac6c9713..f33dafea2533 100644
--- a/include/linux/sysctl.h
+++ b/include/linux/sysctl.h
@@ -52,6 +52,7 @@ struct ctl_dir;
 /* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */
 #define SYSCTL_MAXOLDUID		((void *)&sysctl_vals[10])
 #define SYSCTL_NEG_ONE			((void *)&sysctl_vals[11])
+#define SYSCTL_32			((void *)&sysctl_vals[12])
 
 extern const int sysctl_vals[];
 
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 5c9202cb8f59..f9ba48cd6e09 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -82,7 +82,7 @@
 #endif
 
 /* shared constants to be used in various sysctls */
-const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 };
+const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1, 32 };
 EXPORT_SYMBOL(sysctl_vals);
 
 const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX };
@@ -2091,6 +2091,33 @@ static struct ctl_table vm_table[] = {
 		.extra2		= SYSCTL_ONE,
 	},
 #endif
+	{
+		.procname	= "use_mt_copy",
+		.data		= &use_mt_copy,
+		.maxlen		= sizeof(use_mt_copy),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
+	{
+		.procname	= "limit_mt_num",
+		.data		= &limit_mt_num,
+		.maxlen		= sizeof(limit_mt_num),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ONE,
+		.extra2		= SYSCTL_32,
+	},
+	{
+		.procname	= "push_0_pull_1",
+		.data		= &push_0_pull_1,
+		.maxlen		= sizeof(push_0_pull_1),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
 	{
 		.procname	= "drop_caches",
 		.data		= &sysctl_drop_caches,
diff --git a/mm/copy_pages.c b/mm/copy_pages.c
index 0e2231199f66..257034550c86 100644
--- a/mm/copy_pages.c
+++ b/mm/copy_pages.c
@@ -10,7 +10,9 @@
 #include <linux/migrate.h>
 
 
-unsigned int limit_mt_num = 4;
+unsigned int sysctl_limit_mt_num = 4;
+/* push by default */
+unsigned int sysctl_push_0_pull_1;
 
 struct copy_item {
 	char *to;
@@ -45,11 +47,13 @@ int copy_page_lists_mt(struct list_head *dst_folios,
 		struct list_head *src_folios, int nr_items)
 {
 	int err = 0;
-	unsigned int total_mt_num = limit_mt_num;
+	unsigned int total_mt_num = sysctl_limit_mt_num;
 	int to_node = folio_nid(list_first_entry(dst_folios, struct folio, lru));
+	int from_node = folio_nid(list_first_entry(src_folios, struct folio, lru));
 	int i;
 	struct copy_page_info *work_items[32] = {0};
-	const struct cpumask *per_node_cpumask = cpumask_of_node(to_node);
+	const struct cpumask *per_node_cpumask =
+		cpumask_of_node(sysctl_push_0_pull_1 ? to_node : from_node);
 	int cpu_id_list[32] = {0};
 	int cpu;
 	int max_items_per_thread;
diff --git a/mm/migrate.c b/mm/migrate.c
index 18440180d747..0f7a4b09acda 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -51,6 +51,7 @@
 
 #include "internal.h"
 
+int sysctl_use_mt_copy;
 
 bool isolate_movable_page(struct page *page, isolate_mode_t mode)
 {
@@ -1621,7 +1622,7 @@ static inline int try_split_folio(struct folio *folio, struct list_head *split_f
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-#define NR_MAX_BATCHED_MIGRATION	HPAGE_PMD_NR
+#define NR_MAX_BATCHED_MIGRATION	HPAGE_PUD_NR
 #else
 #define NR_MAX_BATCHED_MIGRATION	512
 #endif
@@ -1868,7 +1869,8 @@ static void migrate_folios_batch_move(struct list_head *src_folios,
 		goto out;
 
 	/* Batch copy the folios */
-	if (total_nr_pages > 32) {
+	/* if (total_nr_pages > 32) { */
+	if (sysctl_use_mt_copy) {
 		copy_page_lists_mt(dst_folios, src_folios, total_nr_folios);
 	} else {
 		dst = list_first_entry(dst_folios, struct folio, lru);
-- 
2.45.2