Re: [PATCH v14 03/13] x86/mm: add INVLPGB support code

[Dave Hansen <dave.hansen@intel.com>]

[-- Attachment #1: Type: text/plain, Size: 53 bytes --]

Here's a plain diff if you just want to squish it in.

[-- Attachment #2: supportcode1.patch --]
[-- Type: text/x-patch, Size: 3618 bytes --]

diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 5375145eb9596..3bd617c204346 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -28,6 +28,11 @@ static inline void invlpg(unsigned long addr)
 	asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
 }
 
+enum invlpgb_stride {
+	NO_STRIDE  = 0,
+	PTE_STRIDE = 0,
+	PMD_STRIDE = 1
+};
 
 /*
  * INVLPGB does broadcast TLB invalidation across all the CPUs in the system.
@@ -48,10 +53,10 @@ static inline void invlpg(unsigned long addr)
  */
 static inline void __invlpgb(unsigned long asid, unsigned long pcid,
 			     unsigned long addr, u16 nr_pages,
-			     bool pmd_stride, u8 flags)
+			     enum invlpgb_stride stride, u8 flags)
 {
 	u32 edx = (pcid << 16) | asid;
-	u32 ecx = (pmd_stride << 31) | (nr_pages - 1);
+	u32 ecx = (stride << 31) | (nr_pages - 1);
 	u64 rax = addr | flags;
 
 	/* The low bits in rax are for flags. Verify addr is clean. */
@@ -78,33 +83,37 @@ static inline void __tlbsync(void)
 /*
  * INVLPGB can be targeted by virtual address, PCID, ASID, or any combination
  * of the three. For example:
- * - INVLPGB_VA | INVLPGB_INCLUDE_GLOBAL: invalidate all TLB entries at the address
- * - INVLPGB_PCID:			  invalidate all TLB entries matching the PCID
+ * - FLAG_VA | FLAG_INCLUDE_GLOBAL: invalidate all TLB entries at the address
+ * - FLAG_PCID:			    invalidate all TLB entries matching the PCID
  *
- * The first can be used to invalidate (kernel) mappings at a particular
+ * The first is used to invalidate (kernel) mappings at a particular
  * address across all processes.
  *
  * The latter invalidates all TLB entries matching a PCID.
  */
-#define INVLPGB_VA			BIT(0)
-#define INVLPGB_PCID			BIT(1)
-#define INVLPGB_ASID			BIT(2)
-#define INVLPGB_INCLUDE_GLOBAL		BIT(3)
-#define INVLPGB_FINAL_ONLY		BIT(4)
-#define INVLPGB_INCLUDE_NESTED		BIT(5)
+#define INVLPGB_FLAG_VA			BIT(0)
+#define INVLPGB_FLAG_PCID		BIT(1)
+#define INVLPGB_FLAG_ASID		BIT(2)
+#define INVLPGB_FLAG_INCLUDE_GLOBAL	BIT(3)
+#define INVLPGB_FLAG_FINAL_ONLY		BIT(4)
+#define INVLPGB_FLAG_INCLUDE_NESTED	BIT(5)
+
+/* The implied mode when all bits are clear: */
+#define INVLPGB_MODE_ALL_NONGLOBALS	0UL
 
 static inline void invlpgb_flush_user_nr_nosync(unsigned long pcid,
 						unsigned long addr,
 						u16 nr,
 						bool pmd_stride)
 {
-	__invlpgb(0, pcid, addr, nr, pmd_stride, INVLPGB_PCID | INVLPGB_VA);
+	__invlpgb(0, pcid, addr, nr, pmd_stride, INVLPGB_FLAG_PCID |
+		  				 INVLPGB_FLAG_VA);
 }
 
 /* Flush all mappings for a given PCID, not including globals. */
 static inline void invlpgb_flush_single_pcid_nosync(unsigned long pcid)
 {
-	__invlpgb(0, pcid, 0, 1, 0, INVLPGB_PCID);
+	__invlpgb(0, pcid, 0, 1, NO_STRIDE, INVLPGB_FLAG_PCID);
 }
 
 /* Flush all mappings, including globals, for all PCIDs. */
@@ -117,21 +126,21 @@ static inline void invlpgb_flush_all(void)
 	 * as it is cheaper.
 	 */
 	guard(preempt)();
-	__invlpgb(0, 0, 0, 1, 0, INVLPGB_INCLUDE_GLOBAL);
+	__invlpgb(0, 0, 0, 1, NO_STRIDE, INVLPGB_FLAG_INCLUDE_GLOBAL);
 	__tlbsync();
 }
 
 /* Flush addr, including globals, for all PCIDs. */
 static inline void invlpgb_flush_addr_nosync(unsigned long addr, u16 nr)
 {
-	__invlpgb(0, 0, addr, nr, 0, INVLPGB_INCLUDE_GLOBAL);
+	__invlpgb(0, 0, addr, nr, PTE_STRIDE, INVLPGB_FLAG_INCLUDE_GLOBAL);
 }
 
 /* Flush all mappings for all PCIDs except globals. */
 static inline void invlpgb_flush_all_nonglobals(void)
 {
 	guard(preempt)();
-	__invlpgb(0, 0, 0, 1, 0, 0);
+	__invlpgb(0, 0, 0, 1, NO_STRIDE, INVLPGB_MODE_ALL_NONGLOBALS);
 	__tlbsync();
 }
 #endif /* _ASM_X86_TLB_H */