From: Wen Yao <haiwenyao@uniontech.com>
To: paul.walmsley@sifive.com, palmer@dabbelt.com,
aou@eecs.berkeley.edu, dennis@kernel.org, tj@kernel.org,
cl@linux.com
Cc: linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org,
linux-mm@kvack.org, Wen Yao <haiwenyao@uniontech.com>
Subject: [PATCH 1/2] riscv: percpu:Add riscv percpu operations
Date: Wed, 26 Oct 2022 18:40:14 +0800 [thread overview]
Message-ID: <20221026104015.565468-2-haiwenyao@uniontech.com> (raw)
In-Reply-To: <20221026104015.565468-1-haiwenyao@uniontech.com>
This patch use riscv AMO(Atomic Memory Operation) instructions to
optimise some this_cpu_and this_cpu_or this_cpu_add operations.
It reuse cmpxchg_local() to impletment this_cpu_cmpxchg macros.
It reuse xchg_relaxed() to impletment this_cpu_xchg macros.
Signed-off-by: Wen Yao <haiwenyao@uniontech.com>
---
arch/riscv/include/asm/percpu.h | 101 ++++++++++++++++++++++++++++++++
1 file changed, 101 insertions(+)
create mode 100644 arch/riscv/include/asm/percpu.h
diff --git a/arch/riscv/include/asm/percpu.h b/arch/riscv/include/asm/percpu.h
new file mode 100644
index 000000000000..ae796e328442
--- /dev/null
+++ b/arch/riscv/include/asm/percpu.h
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020-2022 Union Tech Software Technology Corporation Limited
+ */
+#ifndef __ASM_PERCPU_H
+#define __ASM_PERCPU_H
+
+#include <asm/cmpxchg.h>
+
+#define PERCPU_OP(op, asm_op, c_op) \
+ static inline unsigned long __percpu_##op(void *ptr, \
+ unsigned long val, int size) \
+ { \
+ unsigned long ret; \
+ switch (size) { \
+ case 4: \
+ __asm__ __volatile__( \
+ "amo" #asm_op ".w" \
+ " %[ret], %[val], %[ptr]\n" \
+ : [ret] "=&r"(ret), [ptr] "+A"(*(u32 *)ptr) \
+ : [val] "r"(val)); \
+ break; \
+ case 8: \
+ __asm__ __volatile__( \
+ "amo" #asm_op ".d" \
+ " %[ret], %[val], %[ptr]\n" \
+ : [ret] "=&r"(ret), [ptr] "+A"(*(u64 *)ptr) \
+ : [val] "r"(val)); \
+ break; \
+ default: \
+ ret = 0; \
+ BUILD_BUG(); \
+ } \
+ \
+ return ret c_op val; \
+ }
+
+PERCPU_OP(add, add, +)
+PERCPU_OP(and, and, &)
+PERCPU_OP(or, or, |)
+#undef PERCPU_OP
+
+/* this_cpu_xchg */
+#define _protect_xchg_local(pcp, val) \
+ ({ \
+ typeof(*raw_cpu_ptr(&(pcp))) __ret; \
+ preempt_disable_notrace(); \
+ __ret = xchg_relaxed(raw_cpu_ptr(&(pcp)), val); \
+ preempt_enable_notrace(); \
+ __ret; \
+ })
+
+/* this_cpu_cmpxchg */
+#define _protect_cmpxchg_local(pcp, o, n) \
+ ({ \
+ typeof(*raw_cpu_ptr(&(pcp))) __ret; \
+ preempt_disable_notrace(); \
+ __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
+ preempt_enable_notrace(); \
+ __ret; \
+ })
+
+#define _pcp_protect(operation, pcp, val) \
+ ({ \
+ typeof(pcp) __retval; \
+ preempt_disable_notrace(); \
+ __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), (val), \
+ sizeof(pcp)); \
+ preempt_enable_notrace(); \
+ __retval; \
+ })
+
+#define _percpu_add(pcp, val) _pcp_protect(__percpu_add, pcp, val)
+
+#define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
+
+#define _percpu_and(pcp, val) _pcp_protect(__percpu_and, pcp, val)
+
+#define _percpu_or(pcp, val) _pcp_protect(__percpu_or, pcp, val)
+
+#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
+#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
+
+#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
+#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
+
+#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
+#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
+
+#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
+#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
+
+#define this_cpu_xchg_4(pcp, val) _protect_xchg_local(pcp, val)
+#define this_cpu_xchg_8(pcp, val) _protect_xchg_local(pcp, val)
+
+#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
+#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
+
+#include <asm-generic/percpu.h>
+
+#endif /* __ASM_PERCPU_H */
--
2.25.1
next prev parent reply other threads:[~2022-10-26 10:41 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-10-26 10:40 [PATCH 0/2] riscv: Rewrite percpu operations and support cmpxchg-local feature Wen Yao
2022-10-26 10:40 ` Wen Yao [this message]
2022-10-26 18:54 ` [PATCH 1/2] riscv: percpu:Add riscv percpu operations Conor Dooley
2022-10-27 0:14 ` kernel test robot
2022-10-27 1:05 ` kernel test robot
2022-10-30 13:19 ` Christoph Lameter
2022-10-26 10:40 ` [PATCH 2/2] riscv:kconfig:select HAVE_CMPXCHG_LOCAL Wen Yao
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221026104015.565468-2-haiwenyao@uniontech.com \
--to=haiwenyao@uniontech.com \
--cc=aou@eecs.berkeley.edu \
--cc=cl@linux.com \
--cc=dennis@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-riscv@lists.infradead.org \
--cc=palmer@dabbelt.com \
--cc=paul.walmsley@sifive.com \
--cc=tj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox