From: Peter Zijlstra <peterz@infradead.org>
To: torvalds@linux-foundation.org
Cc: corbet@lwn.net, will@kernel.org, boqun.feng@gmail.com,
mark.rutland@arm.com, catalin.marinas@arm.com, dennis@kernel.org,
tj@kernel.org, cl@linux.com, hca@linux.ibm.com,
gor@linux.ibm.com, agordeev@linux.ibm.com,
borntraeger@linux.ibm.com, svens@linux.ibm.com,
tglx@linutronix.de, mingo@redhat.com, bp@alien8.de,
dave.hansen@linux.intel.com, x86@kernel.org, hpa@zytor.com,
joro@8bytes.org, suravee.suthikulpanit@amd.com,
robin.murphy@arm.com, dwmw2@infradead.org,
baolu.lu@linux.intel.com, Arnd Bergmann <arnd@arndb.de>,
Herbert Xu <herbert@gondor.apana.org.au>,
davem@davemloft.net, penberg@kernel.org, rientjes@google.com,
iamjoonsoo.kim@lge.com, Andrew Morton <akpm@linux-foundation.org>,
vbabka@suse.cz, roman.gushchin@linux.dev, 42.hyeyoo@gmail.com,
linux-doc@vger.kernel.org, linux-kernel@vger.kernel.org,
linux-mm@kvack.org, linux-s390@vger.kernel.org,
iommu@lists.linux.dev, linux-arch@vger.kernel.org,
linux-crypto@vger.kernel.org, sfr@canb.auug.org.au,
mpe@ellerman.id.au, James.Bottomley@hansenpartnership.com,
deller@gmx.de, linux-parisc@vger.kernel.org
Subject: Re: [PATCH v3 08/11] slub: Replace cmpxchg_double()
Date: Tue, 30 May 2023 21:32:58 +0200 [thread overview]
Message-ID: <20230530193258.GB211927@hirez.programming.kicks-ass.net> (raw)
In-Reply-To: <20230530142232.GA200270@hirez.programming.kicks-ass.net>
On Tue, May 30, 2023 at 04:22:32PM +0200, Peter Zijlstra wrote:
> Yet another alternative is using a struct type and an equality function,
> just for this.
The best I could come up with in the regard is the below. It builds on
HPPA64 and x86_64, but I've not ran it yet.
(also, the introduction of this_cpu_try_cmpxchg() should probably be
split out into its own patch)
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -99,6 +99,15 @@ do { \
__ret; \
})
+#define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
+({ \
+ typeof(pcp) __ret, __old = *(ovalp); \
+ __ret = raw_cpu_cmpxchg(pcp, __old, nval); \
+ if (!likely(__ret == __old)) \
+ *(ovalp) = __ret; \
+ likely(__ret == __old); \
+})
+
#define __this_cpu_generic_read_nopreempt(pcp) \
({ \
typeof(pcp) ___ret; \
@@ -167,6 +176,15 @@ do { \
__ret; \
})
+#define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
+({ \
+ typeof(pcp) __ret, __old = *(ovalp); \
+ __ret = this_cpu_cmpxchg(pcp, __old, nval); \
+ if (!likely(__ret == __old)) \
+ *(ovalp) = __ret; \
+ likely(__ret == __old); \
+})
+
#ifndef raw_cpu_read_1
#define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp)
#endif
@@ -258,6 +276,36 @@ do { \
#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
#endif
+#ifndef __SIZEOF_INT128__
+#define raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) \
+({ \
+ typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
+ typeof(pcp) __ret, __old = *(ovalp); \
+ bool __s; \
+ __ret = *__p; \
+ if (!__builtin_memcmp(&__ret, &__old, sizeof(pcp))) { \
+ *__p = nval; \
+ __s = true; \
+ } else { \
+ *(ovalp) = __ret; \
+ __s = false; \
+ } \
+ __s; \
+})
+
+#define raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) \
+({ \
+ typeof(pcp) __old = (oval); \
+ raw_cpu_generic_try_cmpxchg_memcpy(pcp, &__old, nval); \
+ __old; \
+})
+
+#define raw_cpu_cmpxchg128(pcp, oval, nval) \
+ raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval)
+#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
+ raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval)
+#endif
+
#ifndef raw_cpu_cmpxchg_1
#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
raw_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -283,6 +331,31 @@ do { \
raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
+#ifndef raw_cpu_try_cmpxchg_1
+#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef raw_cpu_try_cmpxchg_2
+#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef raw_cpu_try_cmpxchg_4
+#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef raw_cpu_try_cmpxchg_8
+#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef raw_cpu_try_cmpxchg64
+#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef raw_cpu_try_cmpxchg128
+#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
+ raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+
#ifndef this_cpu_read_1
#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
#endif
@@ -374,6 +447,33 @@ do { \
#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
#endif
+#ifndef __SIZEOF_INT128__
+#define this_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval) \
+({ \
+ bool __ret; \
+ unsigned long __flags; \
+ raw_local_irq_save(__flags); \
+ __ret = raw_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval); \
+ raw_local_irq_restore(__flags); \
+ __ret; \
+})
+
+#define this_cpu_generic_cmpxchg_memcmp(pcp, oval, nval) \
+({ \
+ typeof(pcp) __ret; \
+ unsigned long __flags; \
+ raw_local_irq_save(__flags); \
+ __ret = raw_cpu_generic_cmpxchg_memcmp(pcp, oval, nval); \
+ raw_local_irq_restore(__flags); \
+ __ret; \
+})
+
+#define this_cpu_cmpxchg128(pcp, oval, nval) \
+ this_cpu_generic_cmpxchg_memcmp(pcp, oval, nval)
+#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
+ this_cpu_generic_try_cmpxchg_memcmp(pcp, ovalp, nval)
+#endif
+
#ifndef this_cpu_cmpxchg_1
#define this_cpu_cmpxchg_1(pcp, oval, nval) \
this_cpu_generic_cmpxchg(pcp, oval, nval)
@@ -399,4 +499,29 @@ do { \
this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
+#ifndef this_cpu_try_cmpxchg_1
+#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef this_cpu_try_cmpxchg_2
+#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef this_cpu_try_cmpxchg_4
+#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef this_cpu_try_cmpxchg_8
+#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef this_cpu_try_cmpxchg64
+#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+#ifndef this_cpu_try_cmpxchg128
+#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
+ this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
+#endif
+
#endif /* _ASM_GENERIC_PERCPU_H_ */
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -13,6 +13,13 @@
#ifdef __SIZEOF_INT128__
typedef __s128 s128;
typedef __u128 u128;
+#else
+#ifdef CONFIG_64BIT
+/* hack for this_cpu_cmpxchg128 */
+typedef struct {
+ u64 a, b;
+} u128 __attribute__((aligned(16)));
+#endif
#endif
typedef u32 __kernel_dev_t;
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -11,14 +11,14 @@ void __init kmem_cache_init(void);
# define system_has_freelist_aba() system_has_cmpxchg128()
# define try_cmpxchg_freelist try_cmpxchg128
# endif
-#define this_cpu_cmpxchg_freelist this_cpu_cmpxchg128
+#define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg128
typedef u128 freelist_full_t;
#else /* CONFIG_64BIT */
# ifdef system_has_cmpxchg64
# define system_has_freelist_aba() system_has_cmpxchg64()
# define try_cmpxchg_freelist try_cmpxchg64
# endif
-#define this_cpu_cmpxchg_freelist this_cpu_cmpxchg64
+#define this_cpu_try_cmpxchg_freelist this_cpu_try_cmpxchg64
typedef u64 freelist_full_t;
#endif /* CONFIG_64BIT */
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3037,8 +3037,8 @@ __update_cpu_freelist_fast(struct kmem_c
freelist_aba_t old = { .freelist = freelist_old, .counter = tid };
freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) };
- return this_cpu_cmpxchg_freelist(s->cpu_slab->freelist_tid.full,
- old.full, new.full) == old.full;
+ return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full,
+ &old.full, new.full);
}
/*
next prev parent reply other threads:[~2023-05-30 19:33 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-05-15 7:56 [PATCH v3 00/11] Introduce cmpxchg128() -- aka. the demise of cmpxchg_double() Peter Zijlstra
2023-05-15 7:57 ` [PATCH v3 01/11] cyrpto/b128ops: Remove struct u128 Peter Zijlstra
2023-05-15 7:57 ` [PATCH v3 02/11] types: Introduce [us]128 Peter Zijlstra
2023-05-15 7:57 ` [PATCH v3 03/11] arch: Introduce arch_{,try_}_cmpxchg128{,_local}() Peter Zijlstra
2023-05-15 7:57 ` [PATCH v3 04/11] instrumentation: Wire up cmpxchg128() Peter Zijlstra
2023-05-15 7:57 ` [PATCH v3 05/11] percpu: Wire up cmpxchg128 Peter Zijlstra
2023-05-25 12:49 ` Peter Zijlstra
2023-05-25 22:59 ` Petr Tesařík
2023-05-15 7:57 ` [PATCH v3 06/11] x86,amd_iommu: Replace cmpxchg_double() Peter Zijlstra
2023-05-15 7:57 ` [PATCH v3 07/11] x86,intel_iommu: " Peter Zijlstra
2023-05-15 7:57 ` [PATCH v3 08/11] slub: " Peter Zijlstra
2023-05-24 9:32 ` Peter Zijlstra
2023-05-24 10:13 ` Vlastimil Babka
2023-05-25 10:29 ` Peter Zijlstra
2023-05-25 10:52 ` Arnd Bergmann
2023-05-25 13:10 ` Peter Zijlstra
2023-05-30 14:22 ` Peter Zijlstra
2023-05-30 19:32 ` Peter Zijlstra [this message]
2023-05-15 7:57 ` [PATCH v3 09/11] mm/slub: Fold slab_update_freelist() Peter Zijlstra
2023-05-24 11:58 ` Vlastimil Babka
2023-05-15 7:57 ` [PATCH v3 10/11] arch: Remove cmpxchg_double Peter Zijlstra
2023-05-15 8:52 ` Heiko Carstens
2023-05-15 7:57 ` [PATCH v3 11/11] s390/cpum_sf: Convert to cmpxchg128() Peter Zijlstra
2023-05-15 9:42 ` [PATCH v3 00/11] Introduce cmpxchg128() -- aka. the demise of cmpxchg_double() Arnd Bergmann
2023-05-24 9:39 ` Peter Zijlstra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230530193258.GB211927@hirez.programming.kicks-ass.net \
--to=peterz@infradead.org \
--cc=42.hyeyoo@gmail.com \
--cc=James.Bottomley@hansenpartnership.com \
--cc=agordeev@linux.ibm.com \
--cc=akpm@linux-foundation.org \
--cc=arnd@arndb.de \
--cc=baolu.lu@linux.intel.com \
--cc=boqun.feng@gmail.com \
--cc=borntraeger@linux.ibm.com \
--cc=bp@alien8.de \
--cc=catalin.marinas@arm.com \
--cc=cl@linux.com \
--cc=corbet@lwn.net \
--cc=dave.hansen@linux.intel.com \
--cc=davem@davemloft.net \
--cc=deller@gmx.de \
--cc=dennis@kernel.org \
--cc=dwmw2@infradead.org \
--cc=gor@linux.ibm.com \
--cc=hca@linux.ibm.com \
--cc=herbert@gondor.apana.org.au \
--cc=hpa@zytor.com \
--cc=iamjoonsoo.kim@lge.com \
--cc=iommu@lists.linux.dev \
--cc=joro@8bytes.org \
--cc=linux-arch@vger.kernel.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-parisc@vger.kernel.org \
--cc=linux-s390@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mingo@redhat.com \
--cc=mpe@ellerman.id.au \
--cc=penberg@kernel.org \
--cc=rientjes@google.com \
--cc=robin.murphy@arm.com \
--cc=roman.gushchin@linux.dev \
--cc=sfr@canb.auug.org.au \
--cc=suravee.suthikulpanit@amd.com \
--cc=svens@linux.ibm.com \
--cc=tglx@linutronix.de \
--cc=tj@kernel.org \
--cc=torvalds@linux-foundation.org \
--cc=vbabka@suse.cz \
--cc=will@kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox