[RFC PATCH v2 0/4] Enable strict percpu address space checks

[RFC PATCH v2 0/4] Enable strict percpu address space checks

[Uros Bizjak]

This patchset enables strict percpu address space checks via x86 named 
address space qualifiers. Percpu variables are declared in
__seg_gs/__seg_fs named AS and kept named AS qualified until they
are dereferenced via percpu accessor. This approach enables various
compiler checks for cross-namespace variable assignments.

Please note that sparse doesn't know anything about __typeof_unqual__()
operator, so the usage of __typeof_unqual__() breaks sparse checking.

Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>

Uros Bizjak (4):
  percpu: Define pcpu_typeof()
  percpu: Assorted fixes found by strict percpu address space checks
  percpu: Repurpose __percpu tag as a named address space qualifier
  percpu/x86: Enable strict percpu checks via named AS qualifiers

 arch/x86/include/asm/percpu.h  | 53 ++++++++++++++++++++++++----------
 arch/x86/kernel/head64.c       |  3 +-
 drivers/base/devres.c          |  2 +-
 fs/aio.c                       |  2 +-
 include/asm-generic/percpu.h   | 41 +++++++++++++++++---------
 include/linux/cleanup.h        |  4 +--
 include/linux/compiler_types.h |  2 +-
 include/linux/part_stat.h      |  2 +-
 include/linux/percpu-defs.h    | 39 ++++++++++++++++++++-----
 include/linux/prandom.h        |  1 +
 init/Kconfig                   |  3 ++
 kernel/events/hw_breakpoint.c  |  4 +--
 kernel/locking/percpu-rwsem.c  |  2 +-
 kernel/workqueue.c             |  2 +-
 lib/percpu_counter.c           |  2 +-
 net/core/dev.c                 |  2 +-
 16 files changed, 115 insertions(+), 49 deletions(-)

-- 
2.46.0

[RFC PATCH v2 1/4] percpu: Define pcpu_typeof()

[Uros Bizjak]

Define pcpu_typeof() to use __typeof_unqual__() as typeof operator
when available, to return unqualified type of the expression.

If an arch defines __percpu variables in their own named address
space, then pcpu_typeof() returns unqualified type of the
expression without named address space quialifier when
CONFIG_CC_HAS_TYPEOF_UNQUAL is defined.

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 arch/x86/include/asm/percpu.h | 38 ++++++++++++++++++++++------------
 include/asm-generic/percpu.h  | 26 +++++++++++------------
 include/linux/part_stat.h     |  2 +-
 include/linux/percpu-defs.h   | 39 ++++++++++++++++++++++++++++-------
 init/Kconfig                  |  3 +++
 kernel/locking/percpu-rwsem.c |  2 +-
 6 files changed, 75 insertions(+), 35 deletions(-)

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index c55a79d5feae..4ab4ad60d767 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -73,10 +73,14 @@
 	unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off);	\
 									\
 	tcp_ptr__ += (__force unsigned long)(_ptr);			\
-	(typeof(*(_ptr)) __kernel __force *)tcp_ptr__;			\
+	(pcpu_typeof(*(_ptr)) __kernel __force *)tcp_ptr__;		\
 })
 #else
-#define arch_raw_cpu_ptr(_ptr) ({ BUILD_BUG(); (typeof(_ptr))0; })
+#define arch_raw_cpu_ptr(_ptr)						\
+({									\
+	BUILD_BUG();							\
+	(pcpu_typeof(*(_ptr)) __kernel __force *)0;			\
+})
 #endif
 
 #define PER_CPU_VAR(var)	%__percpu_seg:(var)__percpu_rel
@@ -172,7 +176,7 @@ do {									\
 	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
 	    : [var] "m" (__my_cpu_var(_var)));				\
 									\
-	(typeof(_var))(unsigned long) pfo_val__;			\
+	(pcpu_typeof(_var))(unsigned long) pfo_val__;			\
 })
 
 #define __raw_cpu_write(size, qual, _var, _val)				\
@@ -180,7 +184,7 @@ do {									\
 	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
 									\
 	if (0) {		                                        \
-		typeof(_var) pto_tmp__;					\
+		pcpu_typeof(_var) pto_tmp__;				\
 		pto_tmp__ = (_val);					\
 		(void)pto_tmp__;					\
 	}								\
@@ -193,7 +197,11 @@ do {									\
  * The generic per-CPU infrastrucutre is not suitable for
  * reading const-qualified variables.
  */
-#define __raw_cpu_read_const(pcp)	({ BUILD_BUG(); (typeof(pcp))0; })
+#define __raw_cpu_read_const(pcp)					\
+({									\
+	BUILD_BUG();							\
+	(pcpu_typeof(pcp))0;						\
+})
 
 #endif /* CONFIG_USE_X86_SEG_SUPPORT */
 
@@ -205,7 +213,7 @@ do {									\
 	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
 	    : [var] "i" (&(_var)));					\
 									\
-	(typeof(_var))(unsigned long) pfo_val__;			\
+	(pcpu_typeof(_var))(unsigned long) pfo_val__;			\
 })
 
 #define percpu_unary_op(size, qual, op, _var)				\
@@ -219,7 +227,7 @@ do {									\
 	__pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val);	\
 									\
 	if (0) {		                                        \
-		typeof(_var) pto_tmp__;					\
+		pcpu_typeof(_var) pto_tmp__;				\
 		pto_tmp__ = (_val);					\
 		(void)pto_tmp__;					\
 	}								\
@@ -239,7 +247,7 @@ do {									\
 				(int)(val) : 0;				\
 									\
 	if (0) {							\
-		typeof(var) pao_tmp__;					\
+		pcpu_typeof(var) pao_tmp__;				\
 		pao_tmp__ = (val);					\
 		(void)pao_tmp__;					\
 	}								\
@@ -263,7 +271,7 @@ do {									\
 		  : [tmp] __pcpu_reg_##size("+", paro_tmp__),		\
 		    [var] "+m" (__my_cpu_var(_var))			\
 		  : : "memory");					\
-	(typeof(_var))(unsigned long) (paro_tmp__ + _val);		\
+	(pcpu_typeof(_var))(unsigned long) (paro_tmp__ + _val);		\
 })
 
 /*
@@ -272,7 +280,7 @@ do {									\
  */
 #define raw_percpu_xchg_op(_var, _nval)					\
 ({									\
-	typeof(_var) pxo_old__ = raw_cpu_read(_var);			\
+	pcpu_typeof(_var) pxo_old__ = raw_cpu_read(_var);		\
 									\
 	raw_cpu_write(_var, _nval);					\
 									\
@@ -286,7 +294,7 @@ do {									\
  */
 #define this_percpu_xchg_op(_var, _nval)				\
 ({									\
-	typeof(_var) pxo_old__ = this_cpu_read(_var);			\
+	pcpu_typeof(_var) pxo_old__ = this_cpu_read(_var);		\
 									\
 	do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval));	\
 									\
@@ -309,7 +317,7 @@ do {									\
 		  : [nval] __pcpu_reg_##size(, pco_new__)		\
 		  : "memory");						\
 									\
-	(typeof(_var))(unsigned long) pco_old__;			\
+	(pcpu_typeof(_var))(unsigned long) pco_old__;			\
 })
 
 #define percpu_try_cmpxchg_op(size, qual, _var, _ovalp, _nval)		\
@@ -568,7 +576,11 @@ do {									\
 #else /* !CONFIG_X86_64: */
 
 /* There is no generic 64-bit read stable operation for 32-bit targets. */
-#define this_cpu_read_stable_8(pcp)			({ BUILD_BUG(); (typeof(pcp))0; })
+#define this_cpu_read_stable_8(pcp)					\
+({									\
+	BUILD_BUG();							\
+	(pcpu_typeof(pcp))0;						\
+})
 
 #define raw_cpu_read_long(pcp)				raw_cpu_read_4(pcp)
 
diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index 94cbd50cc870..c091e09046c5 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -74,7 +74,7 @@ do {									\
 
 #define raw_cpu_generic_add_return(pcp, val)				\
 ({									\
-	typeof(pcp) *__p = raw_cpu_ptr(&(pcp));				\
+	pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp));			\
 									\
 	*__p += val;							\
 	*__p;								\
@@ -82,8 +82,8 @@ do {									\
 
 #define raw_cpu_generic_xchg(pcp, nval)					\
 ({									\
-	typeof(pcp) *__p = raw_cpu_ptr(&(pcp));				\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp));			\
+	pcpu_typeof(pcp) __ret;						\
 	__ret = *__p;							\
 	*__p = nval;							\
 	__ret;								\
@@ -91,7 +91,7 @@ do {									\
 
 #define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg)		\
 ({									\
-	typeof(pcp) __val, __old = *(ovalp);				\
+	pcpu_typeof(pcp) __val, __old = *(ovalp);			\
 	__val = _cmpxchg(pcp, __old, nval);				\
 	if (__val != __old)						\
 		*(ovalp) = __val;					\
@@ -100,8 +100,8 @@ do {									\
 
 #define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)			\
 ({									\
-	typeof(pcp) *__p = raw_cpu_ptr(&(pcp));				\
-	typeof(pcp) __val = *__p, ___old = *(ovalp);			\
+	pcpu_typeof(pcp) *__p = raw_cpu_ptr(&(pcp));			\
+	pcpu_typeof(pcp) __val = *__p, ___old = *(ovalp);		\
 	bool __ret;							\
 	if (__val == ___old) {						\
 		*__p = nval;						\
@@ -115,14 +115,14 @@ do {									\
 
 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)			\
 ({									\
-	typeof(pcp) __old = (oval);					\
+	pcpu_typeof(pcp) __old = (oval);				\
 	raw_cpu_generic_try_cmpxchg(pcp, &__old, nval);			\
 	__old;								\
 })
 
 #define __this_cpu_generic_read_nopreempt(pcp)				\
 ({									\
-	typeof(pcp) ___ret;						\
+	pcpu_typeof(pcp) ___ret;					\
 	preempt_disable_notrace();					\
 	___ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));			\
 	preempt_enable_notrace();					\
@@ -131,7 +131,7 @@ do {									\
 
 #define __this_cpu_generic_read_noirq(pcp)				\
 ({									\
-	typeof(pcp) ___ret;						\
+	pcpu_typeof(pcp) ___ret;					\
 	unsigned long ___flags;						\
 	raw_local_irq_save(___flags);					\
 	___ret = raw_cpu_generic_read(pcp);				\
@@ -141,7 +141,7 @@ do {									\
 
 #define this_cpu_generic_read(pcp)					\
 ({									\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) __ret;						\
 	if (__native_word(pcp))						\
 		__ret = __this_cpu_generic_read_nopreempt(pcp);		\
 	else								\
@@ -160,7 +160,7 @@ do {									\
 
 #define this_cpu_generic_add_return(pcp, val)				\
 ({									\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) __ret;						\
 	unsigned long __flags;						\
 	raw_local_irq_save(__flags);					\
 	__ret = raw_cpu_generic_add_return(pcp, val);			\
@@ -170,7 +170,7 @@ do {									\
 
 #define this_cpu_generic_xchg(pcp, nval)				\
 ({									\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) __ret;						\
 	unsigned long __flags;						\
 	raw_local_irq_save(__flags);					\
 	__ret = raw_cpu_generic_xchg(pcp, nval);			\
@@ -190,7 +190,7 @@ do {									\
 
 #define this_cpu_generic_cmpxchg(pcp, oval, nval)			\
 ({									\
-	typeof(pcp) __ret;						\
+	pcpu_typeof(pcp) __ret;						\
 	unsigned long __flags;						\
 	raw_local_irq_save(__flags);					\
 	__ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);		\
diff --git a/include/linux/part_stat.h b/include/linux/part_stat.h
index ac8c44dd8237..e228e1087f0c 100644
--- a/include/linux/part_stat.h
+++ b/include/linux/part_stat.h
@@ -33,7 +33,7 @@ struct disk_stats {
 
 #define part_stat_read(part, field)					\
 ({									\
-	typeof((part)->bd_stats->field) res = 0;			\
+	pcpu_typeof((part)->bd_stats->field) res = 0;			\
 	unsigned int _cpu;						\
 	for_each_possible_cpu(_cpu)					\
 		res += per_cpu_ptr((part)->bd_stats, _cpu)->field; \
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 8efce7414fad..aa824c6edb82 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -220,6 +220,21 @@ do {									\
 	(void)__vpp_verify;						\
 } while (0)
 
+/*
+ * Define percpu_typeof() to use __typeof_unqual__() as typeof
+ * operator when available, to return unqualified type of the exp.
+ *
+ * If an arch defines __percpu variables in their own named address
+ * space, then pcpu_typeof() returns unqualified type of the
+ * expression without named address space qualifier when
+ * CONFIG_CC_HAS_TYPEOF_UNQUAL is defined.
+ */
+#ifdef CONFIG_CC_HAS_TYPEOF_UNQUAL
+#define pcpu_typeof(exp) __typeof_unqual__(exp)
+#else
+#define pcpu_typeof(exp) __typeof__(exp)
+#endif
+
 #ifdef CONFIG_SMP
 
 /*
@@ -228,7 +243,10 @@ do {									\
  * pointer value.  The weird cast keeps both GCC and sparse happy.
  */
 #define SHIFT_PERCPU_PTR(__p, __offset)					\
-	RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
+	uintptr_t ptr__ = (__force uintptr_t)(__p);			\
+									\
+	RELOC_HIDE((pcpu_typeof(*(__p)) __kernel __force *)(ptr__),	\
+		   (__offset))
 
 #define per_cpu_ptr(ptr, cpu)						\
 ({									\
@@ -254,13 +272,20 @@ do {									\
 
 #else	/* CONFIG_SMP */
 
-#define VERIFY_PERCPU_PTR(__p)						\
+#define PERCPU_PTR(__p)							\
 ({									\
-	__verify_pcpu_ptr(__p);						\
-	(typeof(*(__p)) __kernel __force *)(__p);			\
+	uintptr_t ptr__ = (__force uintptr_t)(__p);			\
+									\
+	(pcpu_typeof(*(__p)) __kernel __force *)(ptr__);		\
+})
+
+#define per_cpu_ptr(ptr, cpu)						\
+({									\
+	__verify_pcpu_ptr(ptr);						\
+	(void)(cpu);							\
+	PERCPU_PTR(ptr);						\
 })
 
-#define per_cpu_ptr(ptr, cpu)	({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
 #define raw_cpu_ptr(ptr)	per_cpu_ptr(ptr, 0)
 #define this_cpu_ptr(ptr)	raw_cpu_ptr(ptr)
 
@@ -315,7 +340,7 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { }
 
 #define __pcpu_size_call_return(stem, variable)				\
 ({									\
-	typeof(variable) pscr_ret__;					\
+	pcpu_typeof(variable) pscr_ret__;				\
 	__verify_pcpu_ptr(&(variable));					\
 	switch(sizeof(variable)) {					\
 	case 1: pscr_ret__ = stem##1(variable); break;			\
@@ -330,7 +355,7 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { }
 
 #define __pcpu_size_call_return2(stem, variable, ...)			\
 ({									\
-	typeof(variable) pscr2_ret__;					\
+	pcpu_typeof(variable) pscr2_ret__;				\
 	__verify_pcpu_ptr(&(variable));					\
 	switch(sizeof(variable)) {					\
 	case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break;	\
diff --git a/init/Kconfig b/init/Kconfig
index 37260d17267e..a9a04d0683b6 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -873,6 +873,9 @@ config ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
 config CC_HAS_INT128
 	def_bool !$(cc-option,$(m64-flag) -D__SIZEOF_INT128__=0) && 64BIT
 
+config CC_HAS_TYPEOF_UNQUAL
+	def_bool $(success,echo 'int foo (int a) { __typeof_unqual__(a) b = a; return b; }' | $(CC) -x c - -S -o /dev/null)
+
 config CC_IMPLICIT_FALLTHROUGH
 	string
 	default "-Wimplicit-fallthrough=5" if CC_IS_GCC && $(cc-option,-Wimplicit-fallthrough=5)
diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c
index 6083883c4fe0..8345238ba428 100644
--- a/kernel/locking/percpu-rwsem.c
+++ b/kernel/locking/percpu-rwsem.c
@@ -184,7 +184,7 @@ EXPORT_SYMBOL_GPL(__percpu_down_read);
 
 #define per_cpu_sum(var)						\
 ({									\
-	typeof(var) __sum = 0;						\
+	pcpu_typeof(var) __sum = 0;					\
 	int cpu;							\
 	compiletime_assert_atomic_type(__sum);				\
 	for_each_possible_cpu(cpu)					\
-- 
2.46.0

[RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks

[Uros Bizjak]

Assorted fixes to prevent defconfig build failures when
strict percpu address space checks will be enabled.

These show effeciveness of strict percpu address space checks.

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 arch/x86/kernel/head64.c      | 3 ++-
 drivers/base/devres.c         | 2 +-
 fs/aio.c                      | 2 +-
 include/linux/cleanup.h       | 4 ++--
 include/linux/prandom.h       | 1 +
 kernel/events/hw_breakpoint.c | 4 ++--
 kernel/workqueue.c            | 2 +-
 lib/percpu_counter.c          | 2 +-
 net/core/dev.c                | 2 +-
 9 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index a817ed0724d1..f5d6ad351cc4 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -560,9 +560,10 @@ void early_setup_idt(void)
 void __head startup_64_setup_gdt_idt(void)
 {
 	void *handler = NULL;
+	struct desc_struct *gdt = (struct desc_struct *)(uintptr_t)init_per_cpu_var(gdt_page.gdt);
 
 	struct desc_ptr startup_gdt_descr = {
-		.address = (unsigned long)&RIP_REL_REF(init_per_cpu_var(gdt_page.gdt)),
+		.address = (unsigned long)&RIP_REL_REF(*gdt),
 		.size    = GDT_SIZE - 1,
 	};
 
diff --git a/drivers/base/devres.c b/drivers/base/devres.c
index a2ce0ead06a6..894f5a1e6d18 100644
--- a/drivers/base/devres.c
+++ b/drivers/base/devres.c
@@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)
 	 * devm_free_pages() does.
 	 */
 	WARN_ON(devres_release(dev, devm_percpu_release, devm_percpu_match,
-			       (__force void *)pdata));
+			       (__force void *)(uintptr_t)pdata));
 }
 EXPORT_SYMBOL_GPL(devm_free_percpu);
diff --git a/fs/aio.c b/fs/aio.c
index 6066f64967b3..e8920178b50f 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -100,7 +100,7 @@ struct kioctx {
 
 	unsigned long		user_id;
 
-	struct __percpu kioctx_cpu *cpu;
+	struct kioctx_cpu __percpu *cpu;
 
 	/*
 	 * For percpu reqs_available, number of slots we move to/from global
diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h
index d9e613803df1..f7e1158cbacf 100644
--- a/include/linux/cleanup.h
+++ b/include/linux/cleanup.h
@@ -154,7 +154,7 @@ static inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \
 #define DEFINE_GUARD(_name, _type, _lock, _unlock) \
 	DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \
 	static inline void * class_##_name##_lock_ptr(class_##_name##_t *_T) \
-	{ return *_T; }
+	{ return (void *)(uintptr_t)*_T; }
 
 #define DEFINE_GUARD_COND(_name, _ext, _condlock) \
 	EXTEND_CLASS(_name, _ext, \
@@ -211,7 +211,7 @@ static inline void class_##_name##_destructor(class_##_name##_t *_T)	\
 									\
 static inline void *class_##_name##_lock_ptr(class_##_name##_t *_T)	\
 {									\
-	return _T->lock;						\
+	return (void *)(uintptr_t)_T->lock;				\
 }
 
 
diff --git a/include/linux/prandom.h b/include/linux/prandom.h
index f7f1e5251c67..f2ed5b72b3d6 100644
--- a/include/linux/prandom.h
+++ b/include/linux/prandom.h
@@ -10,6 +10,7 @@
 
 #include <linux/types.h>
 #include <linux/once.h>
+#include <linux/percpu.h>
 #include <linux/random.h>
 
 struct rnd_state {
diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c
index 6c2cb4e4f48d..d82fe78f0658 100644
--- a/kernel/events/hw_breakpoint.c
+++ b/kernel/events/hw_breakpoint.c
@@ -849,7 +849,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
 
 	cpu_events = alloc_percpu(typeof(*cpu_events));
 	if (!cpu_events)
-		return (void __percpu __force *)ERR_PTR(-ENOMEM);
+		return (void __percpu __force *)(uintptr_t)ERR_PTR(-ENOMEM);
 
 	cpus_read_lock();
 	for_each_online_cpu(cpu) {
@@ -868,7 +868,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
 		return cpu_events;
 
 	unregister_wide_hw_breakpoint(cpu_events);
-	return (void __percpu __force *)ERR_PTR(err);
+	return (void __percpu __force *)(uintptr_t)ERR_PTR(err);
 }
 EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
 
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 1745ca788ede..32729a2e93af 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -377,7 +377,7 @@ struct workqueue_struct {
 
 	/* hot fields used during command issue, aligned to cacheline */
 	unsigned int		flags ____cacheline_aligned; /* WQ: WQ_* flags */
-	struct pool_workqueue __percpu __rcu **cpu_pwq; /* I: per-cpu pwqs */
+	struct pool_workqueue __rcu * __percpu *cpu_pwq; /* I: per-cpu pwqs */
 	struct wq_node_nr_active *node_nr_active[]; /* I: per-node nr_active */
 };
 
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 51bc5246986d..3d0613ac7e73 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -209,7 +209,7 @@ int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
 		INIT_LIST_HEAD(&fbc[i].list);
 #endif
 		fbc[i].count = amount;
-		fbc[i].counters = (void *)counters + (i * counter_size);
+		fbc[i].counters = (void __percpu *)counters + (i * counter_size);
 
 		debug_percpu_counter_activate(&fbc[i]);
 	}
diff --git a/net/core/dev.c b/net/core/dev.c
index 751d9b70e6ad..5cad88cf029c 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -10860,7 +10860,7 @@ noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)
 			return;
 	}
 
-	field = (__force unsigned long __percpu *)((__force void *)p + offset);
+	field = (unsigned long __percpu *)(void __percpu *)(p + offset);
 	this_cpu_inc(*field);
 }
 EXPORT_SYMBOL_GPL(netdev_core_stats_inc);
-- 
2.46.0

[RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier

[Uros Bizjak]

The patch introduces per_cpu_qual define and repurposes __percpu
tag as a named address space qualifier using the new define.

Arches can now conditionally define __per_cpu_qual as their
named address space qualifier for percpu variables.

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 include/asm-generic/percpu.h   | 15 +++++++++++++++
 include/linux/compiler_types.h |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index c091e09046c5..5f85d7ba8e2d 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -6,6 +6,21 @@
 #include <linux/threads.h>
 #include <linux/percpu-defs.h>
 
+/*
+ * per_cpu_qual is the qualifier for the percpu named address space.
+ *
+ * Most arches use generic named address space for percpu variables but
+ * some arches define percpu variables in different named address space
+ * (on the x86 arch, percpu variables may be declared as being relative
+ * to the %fs or %gs segments using __seg_fs or __seg_gs named address
+ * space qualifiers).
+ */
+#ifdef __per_cpu_qual
+#define per_cpu_qual __per_cpu_qual
+#else
+#define per_cpu_qual /* nothing */
+#endif
+
 #ifdef CONFIG_SMP
 
 /*
diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index f14c275950b5..4c5917b97bc3 100644
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -57,7 +57,7 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { }
 #  define __user	BTF_TYPE_TAG(user)
 # endif
 # define __iomem
-# define __percpu	BTF_TYPE_TAG(percpu)
+# define __percpu	per_cpu_qual BTF_TYPE_TAG(percpu)
 # define __rcu		BTF_TYPE_TAG(rcu)
 
 # define __chk_user_ptr(x)	(void)0
-- 
2.46.0

[RFC PATCH v2 4/4] percpu/x86: Enable strict percpu checks via named AS qualifiers

[Uros Bizjak]

Declare percpu variables in __seg_gs/__seg_fs named AS and keep
them named AS qualified until they are dereferenced with percpu
accessor. This approach enables various compiler check
for cross-namespace variable assignments.

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Nadav Amit <nadav.amit@gmail.com>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 arch/x86/include/asm/percpu.h | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 4ab4ad60d767..b54119398c4b 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -95,9 +95,18 @@
 
 #endif /* CONFIG_SMP */
 
-#define __my_cpu_type(var)	typeof(var) __percpu_seg_override
-#define __my_cpu_ptr(ptr)	(__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
-#define __my_cpu_var(var)	(*__my_cpu_ptr(&(var)))
+#if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(CONFIG_CC_HAS_TYPEOF_UNQUAL)
+# define __my_cpu_type(var)	typeof(var)
+# define __my_cpu_ptr(ptr)	(ptr)
+# define __my_cpu_var(var)	(var)
+
+# define __per_cpu_qual		__percpu_seg_override
+#else
+# define __my_cpu_type(var)	typeof(var) __percpu_seg_override
+# define __my_cpu_ptr(ptr)	(__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
+# define __my_cpu_var(var)	(*__my_cpu_ptr(&(var)))
+#endif
+
 #define __percpu_arg(x)		__percpu_prefix "%" #x
 #define __force_percpu_arg(x)	__force_percpu_prefix "%" #x
 
-- 
2.46.0

Re: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks

[Nadav Amit]

> On 12 Aug 2024, at 14:57, Uros Bizjak <ubizjak@gmail.com> wrote:
> Assorted fixes to prevent defconfig build failures when
> strict percpu address space checks will be enabled.
> 
> These show effeciveness of strict percpu address space checks.

[snip]

> --- a/drivers/base/devres.c
> +++ b/drivers/base/devres.c
> @@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)
> 	 * devm_free_pages() does.
> 	 */
> 	WARN_ON(devres_release(dev, devm_percpu_release, devm_percpu_match,
> -			       (__force void *)pdata));
> +			       (__force void *)(uintptr_t)pdata));
> 

Since this pattern of casting appears multiple times (sometimes slightly
different), I think it would be best to give a name for this operation
and put it behind a macro.

This would allow both to audit the cases developers move data between
address-spaces, and also make them think whether what they do makes
sense.

Re: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier

[Nadav Amit]

> On 12 Aug 2024, at 14:57, Uros Bizjak <ubizjak@gmail.com> wrote:
> 
> +#define per_cpu_qual /* nothing */

I wouldn't do that (the “/* nothing */“ part). It might cause a mess
unnecessarily is used with a comment.

Re: [RFC PATCH v2 3/4] percpu: Repurpose __percpu tag as a named address space qualifier

[Uros Bizjak]

On Mon, Aug 12, 2024 at 9:12 PM Nadav Amit <nadav.amit@gmail.com> wrote:
>
>
>
> > On 12 Aug 2024, at 14:57, Uros Bizjak <ubizjak@gmail.com> wrote:
> >
> > +#define per_cpu_qual /* nothing */
>
> I wouldn't do that (the “/* nothing */“ part). It might cause a mess
> unnecessarily is used with a comment.

I took this /* nothing */ idea from the definition of BTF_TYPE_TAG in
compiler_types.h. But yes, it doesn't bring anything here so let's
remove it.

Thanks,
Uros.

Re: [RFC PATCH v2 2/4] percpu: Assorted fixes found by strict percpu address space checks

[Uros Bizjak]

On Mon, Aug 12, 2024 at 9:09 PM Nadav Amit <nadav.amit@gmail.com> wrote:
>
>
> > On 12 Aug 2024, at 14:57, Uros Bizjak <ubizjak@gmail.com> wrote:
> > Assorted fixes to prevent defconfig build failures when
> > strict percpu address space checks will be enabled.
> >
> > These show effeciveness of strict percpu address space checks.
>
> [snip]
>
> > --- a/drivers/base/devres.c
> > +++ b/drivers/base/devres.c
> > @@ -1231,6 +1231,6 @@ void devm_free_percpu(struct device *dev, void __percpu *pdata)
> >        * devm_free_pages() does.
> >        */
> >       WARN_ON(devres_release(dev, devm_percpu_release, devm_percpu_match,
> > -                            (__force void *)pdata));
> > +                            (__force void *)(uintptr_t)pdata));
> >
>
> Since this pattern of casting appears multiple times (sometimes slightly
> different), I think it would be best to give a name for this operation
> and put it behind a macro.

The macro would not be flexible enough to also cover const qualified
(const void __percpu *)(const uintptr_t) casts, required in e.g. [1].

[1] https://lore.kernel.org/lkml/20240811161414.56744-1-ubizjak@gmail.com/

Also, some casts are decorated with __force. According to sparse
documentation [2], there is no need to use __force when the
destination type is uintptr_t or unsigned long, but sparse seems to
not be consistent with this exception, leading to spurious warnings
and  fixes like the one in [3].

[2] https://sparse.docs.kernel.org/en/latest/annotations.html#address-space-name
[3] https://lore.kernel.org/lkml/20240402175058.52649-1-ubizjak@gmail.com/

OTOH, in a full allyesconfig this pattern of casting appears maybe a
dozen of times (which is a surprisingly small number).

> This would allow both to audit the cases developers move data between
> address-spaces, and also make them think whether what they do makes
> sense.

Looking through the fixes required for allyesconfig build, the
remaining couple of casts are mostly required for ERR_PTR return with
__percpu return type function, like:

--cut here--
diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c
index 6c2cb4e4f48d..d82fe78f0658 100644
--- a/kernel/events/hw_breakpoint.c
+++ b/kernel/events/hw_breakpoint.c
@@ -849,7 +849,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,

     cpu_events = alloc_percpu(typeof(*cpu_events));
     if (!cpu_events)
-        return (void __percpu __force *)ERR_PTR(-ENOMEM);
+        return (void __percpu __force *)(uintptr_t)ERR_PTR(-ENOMEM);

     cpus_read_lock();
     for_each_online_cpu(cpu) {
@@ -868,7 +868,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
         return cpu_events;

     unregister_wide_hw_breakpoint(cpu_events);
-    return (void __percpu __force *)ERR_PTR(err);
+    return (void __percpu __force *)(uintptr_t)ERR_PTR(err);
 }
 EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);

--cut here--

While the casts are somehow ugly, I think that the number of different
types (pcpu -> generic and generic -> pcpu casts with possible const
qualifier and still needed __force sparse attribute) and low number of
occurrences currently do not warrant a separate macro.

Uros.