Hi Alexander, FYI, here is an old soft-lockup BUG that's is still in linux-next. https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master commit 80a9201a5965f4715d5c09790862e0df84ce0614 Author: Alexander Potapenko AuthorDate: Thu Jul 28 15:49:07 2016 -0700 Commit: Linus Torvalds CommitDate: Thu Jul 28 16:07:41 2016 -0700 mm, kasan: switch SLUB to stackdepot, enable memory quarantine for SLUB For KASAN builds: - switch SLUB allocator to using stackdepot instead of storing the allocation/deallocation stacks in the objects; - change the freelist hook so that parts of the freelist can be put into the quarantine. [aryabinin@virtuozzo.com: fixes] Link: http://lkml.kernel.org/r/1468601423-28676-1-git-send-email-aryabinin@virtuozzo.com Link: http://lkml.kernel.org/r/1468347165-41906-3-git-send-email-glider@google.com Signed-off-by: Alexander Potapenko Cc: Andrey Konovalov Cc: Christoph Lameter Cc: Dmitry Vyukov Cc: Steven Rostedt (Red Hat) Cc: Joonsoo Kim Cc: Kostya Serebryany Cc: Andrey Ryabinin Cc: Kuthonuzo Luruo Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds c146a2b98e mm, kasan: account for object redzone in SLUB's nearest_obj() 80a9201a59 mm, kasan: switch SLUB to stackdepot, enable memory quarantine for SLUB c02ed2e75e Linux 4.11-rc4 7f0c4a163a Add linux-next specific files for 20170327 +------------------------------------------------+------------+------------+-----------+---------------+ | | c146a2b98e | 80a9201a59 | v4.11-rc4 | next-20170327 | +------------------------------------------------+------------+------------+-----------+---------------+ | boot_successes | 1003 | 0 | 0 | 0 | | boot_failures | 0 | 731 | 90 | 4 | | BUG:soft_lockup-CPU##stuck_for#s | 0 | 731 | 90 | 4 | | RIP:note_page | 0 | 181 | | | | calltrace:mark_rodata_ro | 0 | 725 | | | | Kernel_panic-not_syncing:softlockup:hung_tasks | 0 | 731 | 90 | 4 | | RIP:__asan_load4 | 0 | 66 | | | | RIP:__sanitizer_cov_trace_pc | 0 | 145 | | | | RIP:__asan_load8 | 0 | 292 | | | | RIP:ptdump_walk_pgd_level_core | 0 | 42 | | | | RIP:is_module_text_address | 0 | 3 | | | | calltrace:virtio_pci_driver_init | 0 | 6 | | | | RIP:_raw_spin_unlock_irqrestore | 0 | 1 | | | | RIP:__kernel_text_address | 0 | 1 | | | +------------------------------------------------+------------+------------+-----------+---------------+ [ 7.785664] ACPI: PCI Interrupt Link [LNKD] enabled at IRQ 11 [ 7.786689] clocksource: tsc: mask: 0xffffffffffffffff max_cycles: 0x26d349e8249, max_idle_ns: 440795288087 ns [ 12.201354] ACPI: PCI Interrupt Link [LNKA] enabled at IRQ 10 [ 21.262748] ACPI: PCI Interrupt Link [LNKB] enabled at IRQ 10 [ 32.256180] ACPI: PCI Interrupt Link [LNKC] enabled at IRQ 11 [ 56.736479] NMI watchdog: BUG: soft lockup - CPU#0 stuck for 22s! [swapper/0:1] [ 56.738452] Modules linked in: [ 56.739070] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.7.0-05999-g80a9201 #1 [ 56.740351] task: ffff88001a078000 task.stack: ffff88001a080000 [ 56.741431] RIP: 0010:[] [] __kernel_text_address+0x36/0x70 [ 56.743092] RSP: 0018:ffff88001a086d50 EFLAGS: 00000246 [ 56.744086] RAX: 0000000000000000 RBX: ffff8800000152a0 RCX: ffffffff810f6d4c [ 56.745394] RDX: dffffc0000000000 RSI: ffffffff815e3e7f RDI: ffffffff8266d404 [ 56.746690] RBP: ffff88001a086d58 R08: 000000000000000f R09: 0000000000000000 [ 56.748008] R10: 0000000000000001 R11: 0000000000000000 R12: ffff88001a078000 [ 56.749321] R13: ffff88001a087588 R14: ffff88001a087568 R15: ffff8800000152a0 [ 56.750605] FS: 0000000000000000(0000) GS:ffff88001aa00000(0000) knlGS:0000000000000000 [ 56.752055] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 56.753089] CR2: 00007ffd61906fc8 CR3: 0000000019f27000 CR4: 00000000000406b0 [ 56.754361] Stack: [ 56.754741] 0000000000000000 ffff88001a086db0 ffffffff810476f6 ffff88001a087588 [ 56.756171] ffffffffffff8000 ffff88001a086e58 ffffffff81c1d2a0 ffff88001a086e58 [ 56.757597] ffffffff81c1d2a0 ffff88001a078000 ffff88001a078000 ffff88001a086df8 [ 56.759051] Call Trace: [ 56.759529] [] print_context_stack+0x76/0xe0 [ 56.760606] [] dump_trace+0x113/0x300 [ 56.761569] [] save_stack_trace+0x26/0x50 [ 56.762594] [] kasan_kmalloc+0x11d/0x1b0 [ 56.763621] [] ? save_stack_trace+0x26/0x50 [ 56.764702] [] ? kasan_kmalloc+0x11d/0x1b0 [ 56.765786] [] ? kasan_slab_alloc+0x12/0x20 [ 56.766882] [] ? kmem_cache_alloc+0xc1/0x160 [ 56.767993] [] ? acpi_ut_allocate_object_desc_dbg+0x9b/0x189 [ 56.769346] [] ? acpi_ut_create_internal_object_dbg+0x54/0x179 [ 56.770731] [] ? acpi_ds_build_internal_object+0x3ec/0x4eb [ 56.785121] [] ? acpi_ds_build_internal_package_obj+0x2de/0x4e3 [ 56.786533] [] ? acpi_ds_eval_data_object_operands+0x1ad/0x298 [ 56.787928] [] ? acpi_ds_exec_end_op+0x764/0xba4 [ 56.789094] [] ? acpi_ps_parse_loop+0xfaa/0x1086 [ 56.790063] [] ? acpi_ps_parse_aml+0x2bd/0x7e3 [ 56.790981] [] ? acpi_ps_execute_method+0x387/0x4ac [ 56.792100] [] ? acpi_ns_evaluate+0x576/0x7a7 [ 56.793190] [] ? acpi_ut_evaluate_object+0x125/0x397 [ 56.794185] [] ? depot_save_stack+0x1b7/0x5b0 [ 56.795058] [] ? kasan_kmalloc+0x189/0x1b0 [ 56.795921] [] ? save_stack_trace+0x26/0x50 [ 56.796772] [] ? kasan_kmalloc+0x11d/0x1b0 [ 56.797599] [] ? __kmalloc+0xf8/0x1c0 [ 56.811609] [] ? acpi_ds_build_internal_package_obj+0x185/0x4e3 [ 56.813004] [] ? acpi_ds_eval_data_object_operands+0x1ad/0x298 [ 56.814381] [] ? acpi_ds_exec_end_op+0x764/0xba4 [ 56.815539] [] ? acpi_ps_parse_loop+0xfaa/0x1086 [ 56.816709] [] ? acpi_ps_parse_aml+0x2bd/0x7e3 [ 56.817847] [] ? acpi_ps_execute_method+0x387/0x4ac [ 56.819001] [] ? acpi_ns_evaluate+0x576/0x7a7 [ 56.820112] [] ? acpi_ut_evaluate_object+0x125/0x397 [ 56.821331] [] ? acpi_rs_get_prt_method_data+0x8e/0x120 [ 56.822592] [] ? acpi_get_irq_routing_table+0xbf/0x10e [ 56.836853] [] ? acpi_pci_irq_find_prt_entry+0x169/0x518 [ 56.837948] [] ? acpi_pci_irq_lookup+0x90/0x588 [ 56.838846] [] ? acpi_pci_irq_enable+0x1aa/0x3f5 [ 56.839748] [] ? pcibios_enable_device+0x3b/0x50 [ 56.840660] [] ? do_pci_enable_device+0xe5/0x1f0 [ 56.841571] [] ? pci_enable_device_flags+0x1e2/0x280 [ 56.842525] [] ? acpi_ut_track_stack_ptr+0xb4/0xd6 [ 56.843506] [] ? ___might_sleep+0x1a9/0x1d0 [ 56.844355] [] kasan_slab_alloc+0x12/0x20 [ 56.845182] [] kmem_cache_alloc+0xc1/0x160 [ 56.846025] [] acpi_ut_allocate_object_desc_dbg+0x9b/0x189 [ 56.847060] [] acpi_ut_create_internal_object_dbg+0x54/0x179 [ 56.848113] [] acpi_ds_build_internal_object+0x3ec/0x4eb [ 56.849123] [] ? acpi_ds_init_object_from_op+0x52f/0x52f [ 56.850210] [] ? __kmalloc+0xf8/0x1c0 [ 56.851159] [] acpi_ds_build_internal_package_obj+0x2de/0x4e3 [ 56.852452] [] acpi_ds_eval_data_object_operands+0x1ad/0x298 [ 56.853739] [] acpi_ds_exec_end_op+0x764/0xba4 [ 56.854861] [] ? acpi_ds_exec_begin_op+0x42b/0x42b [ 56.869039] [] acpi_ps_parse_loop+0xfaa/0x1086 [ 56.870103] [] ? ret_from_fork+0x1f/0x40 [ 56.871053] [] ? do_raw_spin_unlock+0x92/0x120 [ 56.871919] [] ? acpi_ps_get_next_arg+0x835/0x835 [ 56.872809] [] ? kasan_slab_alloc+0x12/0x20 [ 56.873624] [] ? kmem_cache_alloc+0xc1/0x160 [ 56.874455] [] ? acpi_ut_exit+0x9e/0xab [ 56.875229] [] acpi_ps_parse_aml+0x2bd/0x7e3 [ 56.876058] [] acpi_ps_execute_method+0x387/0x4ac [ 56.876942] [] acpi_ns_evaluate+0x576/0x7a7 [ 56.877769] [] acpi_ut_evaluate_object+0x125/0x397 [ 56.878668] [] acpi_rs_get_prt_method_data+0x8e/0x120 [ 56.879603] [] ? acpi_rs_set_resource_source+0x90/0x90 [ 56.880546] [] ? acpi_rs_validate_parameters+0x158/0x163 [ 56.881516] [] acpi_get_irq_routing_table+0xbf/0x10e [ 56.894307] [] ? acpi_rs_validate_parameters+0x163/0x163 [ 56.895679] [] ? check_chain_key+0x14c/0x210 [ 56.899077] [] acpi_pci_irq_find_prt_entry+0x169/0x518 [ 56.900505] [] ? acpi_isa_irq_available+0x4c/0x4c [ 56.901746] [] ? do_raw_spin_lock+0x109/0x1a0 [ 56.902834] [] ? do_raw_spin_unlock+0x92/0x120 [ 56.903949] [] acpi_pci_irq_lookup+0x90/0x588 [ 56.905049] [] ? acpi_pci_irq_find_prt_entry+0x518/0x518 [ 56.906298] [] ? pci_bus_read_config_word+0x127/0x130 [ 56.907502] [] ? do_raw_spin_unlock+0x92/0x120 [ 56.908602] [] acpi_pci_irq_enable+0x1aa/0x3f5 [ 56.909702] [] ? acpi_pci_irq_lookup+0x588/0x588 [ 56.910957] [] ? depot_save_stack+0x1b7/0x5b0 [ 56.912281] [] ? pci_enable_resources+0x209/0x2c0 [ 56.913460] [] ? pci_reassign_resource+0x180/0x180 [ 56.914688] [] ? save_stack_trace+0x26/0x50 [ 56.915788] [] ? slab_attr_store+0x47/0xd0 [ 56.916886] [] ? virtio_pci_probe+0x49/0x1a0 [ 56.918016] [] ? local_pci_probe+0x74/0xe0 [ 56.919848] [] ? pci_device_probe+0x204/0x260 [ 56.921022] [] ? acpi_pci_power_manageable+0x73/0x80 [ 56.922404] [] pcibios_enable_device+0x3b/0x50 [ 56.923620] [] do_pci_enable_device+0xe5/0x1f0 [ 56.925081] [] ? pci_find_saved_ext_cap+0xb0/0xb0 [ 56.926317] [] ? check_chain_key+0x14c/0x210 [ 56.927463] [] ? check_chain_key+0x14c/0x210 [ 56.928573] [] pci_enable_device_flags+0x1e2/0x280 [ 56.929792] [] ? pci_enable_bridge+0x130/0x130 [ 56.930962] [] ? ___might_sleep+0x1a9/0x1d0 [ 56.932098] [] ? lockdep_init_map+0xa5/0x320 [ 56.933448] [] pci_enable_device+0x1a/0x20 [ 56.934536] [] virtio_pci_probe+0xf6/0x1a0 [ 56.935645] [] ? virtio_pci_remove+0x80/0x80 [ 56.936750] [] local_pci_probe+0x74/0xe0 [ 56.937801] [] pci_device_probe+0x204/0x260 [ 56.938905] [] ? pci_device_remove+0x160/0x160 [ 56.940068] [] ? devices_kset_move_last+0x66/0x80 [ 56.941495] [] ? _raw_spin_unlock+0x22/0x30 [ 56.942603] [] ? pci_device_remove+0x160/0x160 [ 56.943799] [] driver_probe_device+0x18d/0x4d0 [ 56.944954] [] ? driver_probe_device+0x4d0/0x4d0 [ 56.946164] [] __driver_attach+0x126/0x130 [ 56.947279] [] bus_for_each_dev+0x116/0x190 [ 56.948591] [] ? subsys_dev_iter_exit+0x20/0x20 [ 56.949748] [] ? klist_add_tail+0x4d/0x60 [ 56.950822] [] ? do_raw_spin_unlock+0x92/0x120 [ 56.951990] [] driver_attach+0x2b/0x30 [ 56.952995] [] bus_add_driver+0x239/0x3a0 [ 56.954061] [] driver_register+0xfb/0x1e0 [ 56.955195] [] __pci_register_driver+0xb4/0xc0 [ 56.956573] [] ? virtio_mmio_init+0x19/0x19 [ 56.957685] [] virtio_pci_driver_init+0x1e/0x20 [ 56.958862] [] do_one_initcall+0x9f/0x250 [ 56.959958] [] ? initcall_blacklisted+0x130/0x130 [ 56.961187] [] ? parse_args+0x5e0/0x720 [ 56.962225] [] ? set_debug_rodata+0x1f/0x1f [ 56.963348] [] kernel_init_freeable+0x1ff/0x2ae [ 56.964703] [] kernel_init+0x13/0x180 [ 56.965709] [] ret_from_fork+0x1f/0x40 [ 56.966735] [] ? rest_init+0xe0/0xe0 [ 56.967746] Code: ff df fc ae 81 73 06 b8 01 00 00 00 c3 55 48 89 e5 53 48 89 fb 48 c7 c7 04 d4 66 82 e8 64 dd 17 00 8b 05 b2 66 57 01 85 c0 75 14 <48> 89 df e8 f2 fe ff ff 85 c0 74 08 b8 01 00 00 00 5b 5d c3 48 [ 56.973481] Kernel panic - not syncing: softlockup: hung tasks [ 56.974591] CPU: 0 PID: 1 Comm: swapper/0 Tainted: G L 4.7.0-05999-g80a9201 #1 [ 56.976204] 00000000ffffffff ffff88001aa07d68 ffffffff813e06d9 ffffffff81c81fa0 [ 56.977585] ffff88001aa07e40 000000000000001e 0000000000000000 ffff88001aa07e30 [ 56.978779] ffffffff811edec1 0000000041b58ab3 ffffffff8201a618 ffffffff811edd41 [ 56.980057] Call Trace: [ 56.980714] [] dump_stack+0xec/0x143 [ 56.981859] [] panic+0x180/0x36e [ 56.982798] [] ? devm_memremap+0xc1/0xc1 [ 56.983903] [] watchdog_timer_fn+0x271/0x280 [ 56.985022] [] ? watchdog+0x50/0x50 [ 56.986002] [] hrtimer_run_queues+0x171/0x310 [ 56.987227] [] run_local_timers+0x20/0xa0 [ 56.988521] [] update_process_times+0x28/0x60 [ 56.989658] [] tick_nohz_handler+0xc0/0x1a0 [ 56.990765] [] smp_trace_apic_timer_interrupt+0x92/0xd0 [ 56.992075] [] smp_apic_timer_interrupt+0x9/0xb [ 56.993273] [] apic_timer_interrupt+0x7f/0x90 [ 56.994395] [] ? __kernel_text_address+0x2c/0x70 [ 56.995740] [] ? acpi_ut_evaluate_object+0x125/0x397 [ 56.997223] [] ? __kernel_text_address+0x36/0x70 [ 56.998401] [] print_context_stack+0x76/0xe0 [ 56.999541] [] dump_trace+0x113/0x300 [ 57.000552] [] save_stack_trace+0x26/0x50 [ 57.001792] [] kasan_kmalloc+0x11d/0x1b0 [ 57.002871] [] ? save_stack_trace+0x26/0x50 [ 57.004000] [] ? kasan_kmalloc+0x11d/0x1b0 [ 57.005308] [] ? kasan_slab_alloc+0x12/0x20 [ 57.006416] [] ? kmem_cache_alloc+0xc1/0x160 [ 57.007562] [] ? acpi_ut_allocate_object_desc_dbg+0x9b/0x189 [ 57.008915] [] ? acpi_ut_create_internal_object_dbg+0x54/0x179 [ 57.010337] [] ? acpi_ds_build_internal_object+0x3ec/0x4eb [ 57.011679] [] ? acpi_ds_build_internal_package_obj+0x2de/0x4e3 [ 57.013316] [] ? acpi_ds_eval_data_object_operands+0x1ad/0x298 [ 57.014712] [] ? acpi_ds_exec_end_op+0x764/0xba4 [ 57.015939] [] ? acpi_ps_parse_loop+0xfaa/0x1086 [ 57.017072] [] ? acpi_ps_parse_aml+0x2bd/0x7e3 [ 57.018261] [] ? acpi_ps_execute_method+0x387/0x4ac [ 57.019504] [] ? acpi_ns_evaluate+0x576/0x7a7 [ 57.020854] [] ? acpi_ut_evaluate_object+0x125/0x397 [ 57.022081] [] ? depot_save_stack+0x1b7/0x5b0 [ 57.023286] [] ? kasan_kmalloc+0x189/0x1b0 [ 57.024366] [] ? save_stack_trace+0x26/0x50 [ 57.025460] [] ? kasan_kmalloc+0x11d/0x1b0 [ 57.026541] [] ? __kmalloc+0xf8/0x1c0 [ 57.027578] [] ? acpi_ds_build_internal_package_obj+0x185/0x4e3 [ 57.029236] [] ? acpi_ds_eval_data_object_operands+0x1ad/0x298 [ 57.030641] [] ? acpi_ds_exec_end_op+0x764/0xba4 # HH:MM RESULT GOOD BAD GOOD_BUT_DIRTY DIRTY_NOT_BAD git bisect start v4.8 v4.7 -- git bisect bad e6e7214fbbdab1f90254af68e0927bdb24708d22 # 09:37 B 26 2 26 129 Merge branch 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip git bisect bad ba929b6646c5b87c7bb15cd8d3e51617725c983b # 11:05 B 544 1 544 544 Merge branch 'for-linus-4.8' of git://git.kernel.org/pub/scm/linux/kernel/git/mason/linux-btrfs git bisect good 468fc7ed5537615efe671d94248446ac24679773 # 00:16 G 901 0 0 0 Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next git bisect bad e55884d2c6ac3ae50e49a1f6fe38601a91181719 # 00:40 B 97 1 97 97 Merge tag 'vfio-v4.8-rc1' of git://github.com/awilliam/linux-vfio git bisect good 554828ee0db41618d101d9549db8808af9fd9d65 # 15:13 G 902 0 0 0 Merge branch 'salted-string-hash' git bisect good ce8c891c3496d3ea4a72ec40beac9a7b7f6649bf # 03:09 G 909 0 0 0 Merge tag 'rproc-v4.8' of git://github.com/andersson/remoteproc git bisect bad 1c88e19b0f6a8471ee50d5062721ba30b8fd4ba9 # 03:29 B 38 1 38 38 Merge branch 'akpm' (patches from Andrew) git bisect good c9b011a87dd49bac1632311811c974bb7cd33c25 # 16:13 G 907 0 0 0 Merge tag 'hwlock-v4.8' of git://github.com/andersson/remoteproc git bisect good 6039b80eb50a893476fea7d56e86ed2d19290054 # 04:09 G 904 0 0 0 Merge tag 'dmaengine-4.8-rc1' of git://git.infradead.org/users/vkoul/slave-dma git bisect good bca6759258dbef378bcf5b872177bcd2259ceb68 # 17:12 G 907 0 0 0 mm, vmstat: remove zone and node double accounting by approximating retries git bisect good efdc94907977d2db84b4b00cb9bd98ca011f6819 # 09:12 G 901 0 0 0 mm: fix memcg stack accounting for sub-page stacks git bisect good fb399b4854d2159a4d23fbfbd7daaed914fd54fa # 22:12 G 902 0 0 0 mm/memblock.c: fix index adjustment error in __next_mem_range_rev() git bisect bad 31a6c1909f51dbe9bf08eb40dc64e3db90cf6f79 # 23:27 B 824 2 824 824 mm, page_alloc: set alloc_flags only once in slowpath git bisect good c146a2b98eb5898eb0fab15a332257a4102ecae9 # 12:14 G 903 0 0 0 mm, kasan: account for object redzone in SLUB's nearest_obj() git bisect bad 87cc271d5e4320d705cfdf59f68d4d037b3511b2 # 12:33 B 173 1 173 173 lib/stackdepot.c: use __GFP_NOWARN for stack allocations git bisect bad 80a9201a5965f4715d5c09790862e0df84ce0614 # 12:56 B 81 1 81 81 mm, kasan: switch SLUB to stackdepot, enable memory quarantine for SLUB # first bad commit: [80a9201a5965f4715d5c09790862e0df84ce0614] mm, kasan: switch SLUB to stackdepot, enable memory quarantine for SLUB git bisect good c146a2b98eb5898eb0fab15a332257a4102ecae9 # 13:17 G 1003 0 0 0 mm, kasan: account for object redzone in SLUB's nearest_obj() # extra tests with CONFIG_DEBUG_INFO_REDUCED git bisect bad 80a9201a5965f4715d5c09790862e0df84ce0614 # 14:26 B 223 1 223 224 mm, kasan: switch SLUB to stackdepot, enable memory quarantine for SLUB # extra tests on HEAD of linux-devel/devel-spot-201703201203 git bisect bad dd9d0f58f5a99e4a3e77bce4981b39988fcb191b # 14:26 B 0 7 27 6 0day head guard for 'devel-spot-201703201203' # extra tests on tree/branch linus/master git bisect bad c02ed2e75ef4c74e41e421acb4ef1494671585e8 # 14:38 B 6 1 6 6 Linux 4.11-rc4 # extra tests on tree/branch linux-next/master git bisect bad 7f0c4a163aa51c7b924bbafbe2013838d7ddaed0 # 14:53 B 0 1 12 0 Add linux-next specific files for 20170327 --- 0-DAY kernel test infrastructure Open Source Technology Center https://lists.01.org/pipermail/lkp Intel Corporation