[PATCH v3 07/19] mm/ksw: add atomic watch on/off operations

[Jinchao Wang <wangjinchao600@gmail.com>]

Add support to atomically turn the hardware watch on and off without
allocation overhead.

The watch is pre-allocated and later retargeted.
The current CPU is updated directly, while other CPUs are updated
asynchronously via smp_call_function_single_async().

Signed-off-by: Jinchao Wang <wangjinchao600@gmail.com>
---
 mm/kstackwatch/kstackwatch.h |  2 +
 mm/kstackwatch/watch.c       | 95 ++++++++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+)

diff --git a/mm/kstackwatch/kstackwatch.h b/mm/kstackwatch/kstackwatch.h
index 3ea191370970..2fa377843f17 100644
--- a/mm/kstackwatch/kstackwatch.h
+++ b/mm/kstackwatch/kstackwatch.h
@@ -41,5 +41,7 @@ const struct ksw_config *ksw_get_config(void);
 /* watch management */
 int ksw_watch_init(void);
 void ksw_watch_exit(void);
+int ksw_watch_on(u64 watch_addr, u64 watch_len);
+void ksw_watch_off(void);
 
 #endif /* _KSTACKWATCH_H */
diff --git a/mm/kstackwatch/watch.c b/mm/kstackwatch/watch.c
index d3399ac840b2..e02ffc3231ad 100644
--- a/mm/kstackwatch/watch.c
+++ b/mm/kstackwatch/watch.c
@@ -3,16 +3,23 @@
 
 #include <linux/hw_breakpoint.h>
 #include <linux/perf_event.h>
+#include <linux/preempt.h>
 #include <linux/printk.h>
 
 #include "kstackwatch.h"
 
 static struct perf_event *__percpu *watch_events;
+static DEFINE_SPINLOCK(watch_lock);
 
 static unsigned long watch_holder;
 
 static struct perf_event_attr watch_attr;
 
+static void ksw_watch_on_local_cpu(void *info);
+
+static DEFINE_PER_CPU(call_single_data_t,
+		      watch_csd) = CSD_INIT(ksw_watch_on_local_cpu, NULL);
+
 bool panic_on_catch;
 module_param(panic_on_catch, bool, 0644);
 MODULE_PARM_DESC(panic_on_catch, "panic immediately on corruption catch");
@@ -29,6 +36,94 @@ static void ksw_watch_handler(struct perf_event *bp,
 		panic("Stack corruption detected");
 }
 
+static void ksw_watch_on_local_cpu(void *data)
+{
+	struct perf_event *bp;
+	int cpu;
+	int ret;
+
+	preempt_disable();
+	cpu = raw_smp_processor_id();
+	bp = *per_cpu_ptr(watch_events, cpu);
+	if (!bp) {
+		preempt_enable();
+		return;
+	}
+
+	ret = modify_wide_hw_breakpoint_local(bp, &watch_attr);
+	preempt_enable();
+
+	if (ret) {
+		pr_err("failed to reinstall HWBP on CPU %d ret %d\n", cpu,
+		       ret);
+		return;
+	}
+
+	if (watch_attr.bp_addr == (unsigned long)&watch_holder) {
+		pr_debug("watch off CPU %d\n", cpu);
+	} else {
+		pr_debug("watch on CPU %d at 0x%llx (len %llu)\n", cpu,
+			 watch_attr.bp_addr, watch_attr.bp_len);
+	}
+}
+
+int ksw_watch_on(u64 watch_addr, u64 watch_len)
+{
+	unsigned long flags;
+	int cpu;
+	call_single_data_t *csd;
+
+	if (!watch_addr) {
+		pr_err("watch with invalid address\n");
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&watch_lock, flags);
+
+	/*
+	 * enforce singleton watch:
+	 *   - if a watch is already active (bp_addr != &watch_holder),
+	 *   - and not asking to reset it   (watch_addr != &watch_holder)
+	 * then reject with -EBUSY.
+	 */
+	if (watch_attr.bp_addr != (unsigned long)&watch_holder &&
+	    watch_addr != (unsigned long)&watch_holder) {
+		spin_unlock_irqrestore(&watch_lock, flags);
+		return -EBUSY;
+	}
+
+	watch_attr.bp_addr = watch_addr;
+	watch_attr.bp_len = watch_len;
+
+	/* ensure watchpoint update is visible to other CPUs before IPI */
+	smp_wmb();
+
+	spin_unlock_irqrestore(&watch_lock, flags);
+
+	if (watch_addr == (unsigned long)&watch_holder)
+		pr_debug("watch off starting\n");
+	else
+		pr_debug("watch on starting\n");
+
+	cpus_read_lock();
+	for_each_online_cpu(cpu) {
+		if (cpu == raw_smp_processor_id()) {
+			ksw_watch_on_local_cpu(NULL);
+		} else {
+			csd = &per_cpu(watch_csd, cpu);
+			smp_call_function_single_async(cpu, csd);
+		}
+	}
+	cpus_read_unlock();
+
+	return 0;
+}
+
+void ksw_watch_off(void)
+{
+	ksw_watch_on((unsigned long)&watch_holder, sizeof(watch_holder));
+}
+
 int ksw_watch_init(void)
 {
 	int ret;
-- 
2.43.0