From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id 43C12C433F5 for ; Sat, 5 Mar 2022 14:49:19 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 952046B00A7; Sat, 5 Mar 2022 09:49:17 -0500 (EST) Received: by kanga.kvack.org (Postfix, from userid 40) id 74E8F6B00A8; Sat, 5 Mar 2022 09:49:17 -0500 (EST) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 5EF296B00A9; Sat, 5 Mar 2022 09:49:17 -0500 (EST) X-Delivered-To: linux-mm@kvack.org Received: from forelay.hostedemail.com (smtprelay0027.hostedemail.com [216.40.44.27]) by kanga.kvack.org (Postfix) with ESMTP id 51FC66B00A7 for ; Sat, 5 Mar 2022 09:49:17 -0500 (EST) Received: from smtpin20.hostedemail.com (10.5.19.251.rfc1918.com [10.5.19.251]) by forelay04.hostedemail.com (Postfix) with ESMTP id 6E3489EB09 for ; Sat, 5 Mar 2022 14:49:15 +0000 (UTC) X-FDA: 79210615470.20.E6BFC5D Received: from out30-54.freemail.mail.aliyun.com (out30-54.freemail.mail.aliyun.com [115.124.30.54]) by imf04.hostedemail.com (Postfix) with ESMTP id 64FFB40005 for ; Sat, 5 Mar 2022 14:49:13 +0000 (UTC) X-Alimail-AntiSpam:AC=PASS;BC=-1|-1;BR=01201311R671e4;CH=green;DM=||false|;DS=||;FP=0|-1|-1|-1|0|-1|-1|-1;HT=e01e01424;MF=dtcccc@linux.alibaba.com;NM=1;PH=DS;RN=7;SR=0;TI=SMTPD_---0V6HF7j0_1646491749; Received: from localhost.localdomain(mailfrom:dtcccc@linux.alibaba.com fp:SMTPD_---0V6HF7j0_1646491749) by smtp.aliyun-inc.com(127.0.0.1); Sat, 05 Mar 2022 22:49:09 +0800 From: Tianchen Ding To: Alexander Potapenko , Marco Elver , Dmitry Vyukov , Andrew Morton Cc: kasan-dev@googlegroups.com, linux-mm@kvack.org, linux-kernel@vger.kernel.org Subject: [PATCH v2 2/2] kfence: Alloc kfence_pool after system startup Date: Sat, 5 Mar 2022 22:48:58 +0800 Message-Id: <20220305144858.17040-3-dtcccc@linux.alibaba.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20220305144858.17040-1-dtcccc@linux.alibaba.com> References: <20220305144858.17040-1-dtcccc@linux.alibaba.com> MIME-Version: 1.0 X-Rspamd-Queue-Id: 64FFB40005 X-Stat-Signature: rmgsbkxkx4fhn7xq9zy7q3xqkotsqq3k Authentication-Results: imf04.hostedemail.com; dkim=none; spf=pass (imf04.hostedemail.com: domain of dtcccc@linux.alibaba.com designates 115.124.30.54 as permitted sender) smtp.mailfrom=dtcccc@linux.alibaba.com; dmarc=pass (policy=none) header.from=alibaba.com X-Rspam-User: X-Rspamd-Server: rspam08 X-HE-Tag: 1646491753-509767 Content-Transfer-Encoding: quoted-printable X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: Allow enabling KFENCE after system startup by allocating its pool via the page allocator. This provides the flexibility to enable KFENCE even if it wasn't enabled at boot time. Signed-off-by: Tianchen Ding --- mm/kfence/core.c | 99 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 21 deletions(-) diff --git a/mm/kfence/core.c b/mm/kfence/core.c index caa4e84c8b79..f46d63dd7676 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -96,7 +96,7 @@ static unsigned long kfence_skip_covered_thresh __read_= mostly =3D 75; module_param_named(skip_covered_thresh, kfence_skip_covered_thresh, ulon= g, 0644); =20 /* The pool of pages used for guard pages and objects. */ -char *__kfence_pool __ro_after_init; +char *__kfence_pool __read_mostly; EXPORT_SYMBOL(__kfence_pool); /* Export for test modules. */ =20 /* @@ -537,17 +537,19 @@ static void rcu_guarded_free(struct rcu_head *h) kfence_guarded_free((void *)meta->addr, meta, false); } =20 -static bool __init kfence_init_pool(void) +/* + * Initialization of the KFENCE pool after its allocation. + * Returns 0 on success; otherwise returns the address up to + * which partial initialization succeeded. + */ +static unsigned long kfence_init_pool(void) { unsigned long addr =3D (unsigned long)__kfence_pool; struct page *pages; int i; =20 - if (!__kfence_pool) - return false; - if (!arch_kfence_init_pool()) - goto err; + return addr; =20 pages =3D virt_to_page(addr); =20 @@ -565,7 +567,7 @@ static bool __init kfence_init_pool(void) =20 /* Verify we do not have a compound head page. */ if (WARN_ON(compound_head(&pages[i]) !=3D &pages[i])) - goto err; + return addr; =20 __SetPageSlab(&pages[i]); } @@ -578,7 +580,7 @@ static bool __init kfence_init_pool(void) */ for (i =3D 0; i < 2; i++) { if (unlikely(!kfence_protect(addr))) - goto err; + return addr; =20 addr +=3D PAGE_SIZE; } @@ -595,7 +597,7 @@ static bool __init kfence_init_pool(void) =20 /* Protect the right redzone. */ if (unlikely(!kfence_protect(addr + PAGE_SIZE))) - goto err; + return addr; =20 addr +=3D 2 * PAGE_SIZE; } @@ -608,9 +610,21 @@ static bool __init kfence_init_pool(void) */ kmemleak_free(__kfence_pool); =20 - return true; + return 0; +} + +static bool __init kfence_init_pool_early(void) +{ + unsigned long addr; + + if (!__kfence_pool) + return false; + + addr =3D kfence_init_pool(); + + if (!addr) + return true; =20 -err: /* * Only release unprotected pages, and do not try to go back and change * page attributes due to risk of failing to do so as well. If changing @@ -623,6 +637,22 @@ static bool __init kfence_init_pool(void) return false; } =20 +static bool kfence_init_pool_late(void) +{ + unsigned long addr, free_pages; + + addr =3D kfence_init_pool(); + + if (!addr) + return true; + + /* Same as above. */ + free_pages =3D (KFENCE_POOL_SIZE - (addr - (unsigned long)__kfence_pool= )) / PAGE_SIZE; + free_contig_range(page_to_pfn(virt_to_page(addr)), free_pages); + __kfence_pool =3D NULL; + return false; +} + /* =3D=3D=3D DebugFS Interface =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D */ =20 static int stats_show(struct seq_file *seq, void *v) @@ -771,31 +801,58 @@ void __init kfence_alloc_pool(void) pr_err("failed to allocate pool\n"); } =20 +static void kfence_init_enable(void) +{ + if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS)) + static_branch_enable(&kfence_allocation_key); + WRITE_ONCE(kfence_enabled, true); + queue_delayed_work(system_unbound_wq, &kfence_timer, 0); + pr_info("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n", = KFENCE_POOL_SIZE, + CONFIG_KFENCE_NUM_OBJECTS, (void *)__kfence_pool, + (void *)(__kfence_pool + KFENCE_POOL_SIZE)); +} + void __init kfence_init(void) { + stack_hash_seed =3D (u32)random_get_entropy(); + /* Setting kfence_sample_interval to 0 on boot disables KFENCE. */ if (!kfence_sample_interval) return; =20 - stack_hash_seed =3D (u32)random_get_entropy(); - if (!kfence_init_pool()) { + if (!kfence_init_pool_early()) { pr_err("%s failed\n", __func__); return; } =20 - if (!IS_ENABLED(CONFIG_KFENCE_STATIC_KEYS)) - static_branch_enable(&kfence_allocation_key); - WRITE_ONCE(kfence_enabled, true); - queue_delayed_work(system_unbound_wq, &kfence_timer, 0); - pr_info("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n", = KFENCE_POOL_SIZE, - CONFIG_KFENCE_NUM_OBJECTS, (void *)__kfence_pool, - (void *)(__kfence_pool + KFENCE_POOL_SIZE)); + kfence_init_enable(); +} + +static int kfence_init_late(void) +{ + const unsigned long nr_pages =3D KFENCE_POOL_SIZE / PAGE_SIZE; + struct page *pages; + + pages =3D alloc_contig_pages(nr_pages, GFP_KERNEL, first_online_node, N= ULL); + + if (!pages) + return -ENOMEM; + + __kfence_pool =3D page_to_virt(pages); + + if (!kfence_init_pool_late()) { + pr_err("%s failed\n", __func__); + return -EBUSY; + } + + kfence_init_enable(); + return 0; } =20 static int kfence_enable_late(void) { if (!__kfence_pool) - return -EINVAL; + return kfence_init_late(); =20 WRITE_ONCE(kfence_enabled, true); queue_delayed_work(system_unbound_wq, &kfence_timer, 0); --=20 2.27.0