[PATCH v3 14/21] x86/clear_page: add clear_pages_incoherent()

[Ankur Arora <ankur.a.arora@oracle.com>]

Expose incoherent clearing primitives (clear_pages_movnt(),
clear_pages_clzero()) as alternatives via clear_pages_incoherent().

Fallback to clear_pages() if, X86_FEATURE_MOVNT_SLOW is set and
the CPU does not have X86_FEATURE_CLZERO.

Both these primitives use weakly-ordered stores. To ensure that
callers don't mix accesses to different types of address_spaces,
annotate clear_user_pages_incoherent(), and clear_pages_incoherent()
as taking __incoherent pointers as arguments.

Also add clear_page_make_coherent() which provides the necessary
store fence to make access to these __incoherent regions safe.

Signed-off-by: Ankur Arora <ankur.a.arora@oracle.com>
---
 arch/x86/include/asm/page.h    | 13 +++++++++++++
 arch/x86/include/asm/page_64.h | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)

diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h
index 045eaab08f43..8fc6cc6759b9 100644
--- a/arch/x86/include/asm/page.h
+++ b/arch/x86/include/asm/page.h
@@ -40,6 +40,19 @@ static inline void clear_user_page(void *page, unsigned long vaddr,
 	clear_page(page);
 }
 
+#ifdef __HAVE_ARCH_CLEAR_USER_PAGES_INCOHERENT /* x86_64 */
+/*
+ * clear_pages_incoherent: valid on only __incoherent memory regions.
+ */
+static inline void clear_user_pages_incoherent(__incoherent void *page,
+					       unsigned long vaddr,
+					       struct page *pg,
+					       unsigned int npages)
+{
+	clear_pages_incoherent(page, npages);
+}
+#endif /* __HAVE_ARCH_CLEAR_USER_PAGES_INCOHERENT */
+
 static inline void copy_user_page(void *to, void *from, unsigned long vaddr,
 				  struct page *topage)
 {
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index e8d4698fda65..78417f63f522 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -69,6 +69,40 @@ static inline void clear_pages(void *page, unsigned int npages)
 			   : "cc", "memory", "rax", "rcx");
 }
 
+#define __HAVE_ARCH_CLEAR_USER_PAGES_INCOHERENT
+/*
+ * clear_pages_incoherent: only allowed on __incoherent memory regions.
+ */
+static inline void clear_pages_incoherent(__incoherent void *page,
+					  unsigned int npages)
+{
+	alternative_call_2(clear_pages_movnt,
+			   clear_pages, X86_FEATURE_MOVNT_SLOW,
+			   clear_pages_clzero, X86_FEATURE_CLZERO,
+			   "=D" (page), "S" ((unsigned long) npages),
+			   "0" (page)
+			   : "cc", "memory", "rax", "rcx");
+}
+
+/*
+ * clear_page_make_coherent: execute the necessary store fence
+ * after which __incoherent regions can be safely accessed.
+ */
+static inline void clear_page_make_coherent(void)
+{
+	/*
+	 * Keep the sfence for oldinstr and clzero separate to guard against
+	 * the possibility that a CPU has both X86_FEATURE_MOVNT_SLOW and
+	 * X86_FEATURE_CLZERO.
+	 *
+	 * The alternatives need to be in the same order as the ones
+	 * in clear_pages_incoherent().
+	 */
+	alternative_2("sfence",
+		      "", X86_FEATURE_MOVNT_SLOW,
+		      "sfence", X86_FEATURE_CLZERO);
+}
+
 void copy_page(void *to, void *from);
 
 #ifdef CONFIG_X86_5LEVEL
-- 
2.31.1