x86: warning in unwind_get_return_address

[Andrey Konovalov <andreyknvl@google.com>]

[-- Attachment #1: Type: text/plain, Size: 3353 bytes --]

Hi,

I've got the following warning while running the syzkaller fuzzer:

WARNING: unrecognized kernel stack return address ffffffffa0000001 at
ffff88006377fa18 in a.out:4467

By adding a BUG() to unwind_get_return_address() I was able to capture
the stack trace (see below). Looks like unwind_get_return_address()
gets called when KASAN tries to unwind the stack to save the stack
trace.

A reproducer is attached. CONFIG_KASAN=y is most likely needed for it to work.

On commit e93b1cc8a8965da137ffea0b88e5f62fa1d2a9e6 (Dec 19).

------------[ cut here ]------------
kernel BUG at arch/x86/kernel/unwind_frame.c:27!
invalid opcode: 0000 [#1] SMP KASAN
Modules linked in:
CPU: 1 PID: 4467 Comm: a.out Not tainted 4.9.0+ #53
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
task: ffff8800665b9600 task.stack: ffff880063778000
RIP: 0010:unwind_get_return_address+0xcc/0x1b0 arch/x86/kernel/unwind_frame.c:24
RSP: 0018:ffff88006cb06fa0 EFLAGS: 00010286
RAX: 0000000000000064 RBX: ffff8800665b9600 RCX: 0000000000000000
RDX: 0000000000000100 RSI: ffff88006cb15f08 RDI: ffffed000d960de6
RBP: ffff88006cb06fb8 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000009 R11: 0000000000000000 R12: ffffffffa0000001
R13: ffff88006377fa18 R14: ffff8800665b9600 R15: 0000000000000246
FS:  00007f7c347947c0(0000) GS:ffff88006cb00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000020004fc8 CR3: 0000000062668000 CR4: 00000000000006e0
Call Trace:
 <IRQ>
 __save_stack_trace+0x7e/0xd0 arch/x86/kernel/stacktrace.c:43
 save_stack_trace+0x16/0x20 arch/x86/kernel/stacktrace.c:57
 save_stack+0x43/0xd0 mm/kasan/kasan.c:502
 set_track mm/kasan/kasan.c:514
 kasan_slab_free+0x73/0xc0 mm/kasan/kasan.c:578
 slab_free_hook mm/slub.c:1352
 slab_free_freelist_hook mm/slub.c:1374
 slab_free mm/slub.c:2951
 kmem_cache_free+0xb2/0x2c0 mm/slub.c:2973
 file_free_rcu+0x6d/0xa0 fs/file_table.c:49
 __rcu_reclaim kernel/rcu/rcu.h:118
 rcu_do_batch.isra.67+0x900/0xc50 kernel/rcu/tree.c:2777
 invoke_rcu_callbacks kernel/rcu/tree.c:3040
 __rcu_process_callbacks kernel/rcu/tree.c:3007
 rcu_process_callbacks+0x2b7/0xba0 kernel/rcu/tree.c:3024
 __do_softirq+0x2fb/0xb7d kernel/softirq.c:284
 invoke_softirq kernel/softirq.c:364
 irq_exit+0x19e/0x1d0 kernel/softirq.c:405
 exiting_irq ./arch/x86/include/asm/apic.h:658
 smp_apic_timer_interrupt+0x76/0xa0 arch/x86/kernel/apic/apic.c:961
 apic_timer_interrupt+0x93/0xa0
RIP: 0010:0xffffffffa0000001
RSP: 0018:ffff88006377fa48 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10
RAX: ffff8800665b9600 RBX: ffff880066471400 RCX: 1ffffffff0bc9005
RDX: 0000000000000000 RSI: ffff88006377fe78 RDI: ffff880066471400
RBP: ffff88006377fb00 R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000000 R12: 1ffff1000c6eff4f
R13: ffffffff85e47fc0 R14: 0000000000000000 R15: ffff880066471428
 </IRQ>
Code: 00 fc ff df 48 c1 ea 03 80 3c 02 00 0f 85 eb 00 00 00 4d 8b 65
00 4c 89 e7 e8 21 75 0c 00 85 c0 75 0e 80 3d 44 fc c9 04 00 74 4f <0f>
0b 45 31 e4 5b 4c 89 e0 41 5c 41 5d 5d c3 48 8d 7b 38 48 b8
RIP: unwind_get_return_address+0xcc/0x1b0 RSP: ffff88006cb06fa0
---[ end trace a4f6a441af47c2dc ]---
Kernel panic - not syncing: Fatal exception in interrupt
Kernel Offset: disabled
---[ end Kernel panic - not syncing: Fatal exception in interrupt

[-- Attachment #2: stack-unwind-warn-poc.c --]
[-- Type: text/x-csrc, Size: 7934 bytes --]

// autogenerated by syzkaller (http://github.com/google/syzkaller)

#ifndef __NR_mmap
#define __NR_mmap 9
#endif
#ifndef __NR_socket
#define __NR_socket 41
#endif
#ifndef __NR_sendmsg
#define __NR_sendmsg 46
#endif

#define _GNU_SOURCE

#include <sys/ioctl.h>
#include <sys/mount.h>
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>

#include <linux/capability.h>
#include <linux/if.h>
#include <linux/if_tun.h>
#include <linux/sched.h>
#include <net/if_arp.h>

#include <assert.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <grp.h>
#include <pthread.h>
#include <setjmp.h>
#include <signal.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

const int kFailStatus = 67;
const int kErrorStatus = 68;
const int kRetryStatus = 69;

__attribute__((noreturn)) void fail(const char* msg, ...)
{
  int e = errno;
  fflush(stdout);
  va_list args;
  va_start(args, msg);
  vfprintf(stderr, msg, args);
  va_end(args);
  fprintf(stderr, " (errno %d)\n", e);
  exit(kFailStatus);
}

__attribute__((noreturn)) void exitf(const char* msg, ...)
{
  int e = errno;
  fflush(stdout);
  va_list args;
  va_start(args, msg);
  vfprintf(stderr, msg, args);
  va_end(args);
  fprintf(stderr, " (errno %d)\n", e);
  exit(kRetryStatus);
}

static int flag_debug;

void debug(const char* msg, ...)
{
  if (!flag_debug)
    return;
  va_list args;
  va_start(args, msg);
  vfprintf(stdout, msg, args);
  va_end(args);
  fflush(stdout);
}

__thread int skip_segv;
__thread jmp_buf segv_env;

static void segv_handler(int sig, siginfo_t* info, void* uctx)
{
  if (__atomic_load_n(&skip_segv, __ATOMIC_RELAXED))
    _longjmp(segv_env, 1);
  exit(sig);
}

static void install_segv_handler()
{
  struct sigaction sa;
  memset(&sa, 0, sizeof(sa));
  sa.sa_sigaction = segv_handler;
  sa.sa_flags = SA_NODEFER | SA_SIGINFO;
  sigaction(SIGSEGV, &sa, NULL);
  sigaction(SIGBUS, &sa, NULL);
}

#define NONFAILING(...)                                                \
  {                                                                    \
    __atomic_fetch_add(&skip_segv, 1, __ATOMIC_SEQ_CST);               \
    if (_setjmp(segv_env) == 0) {                                      \
      __VA_ARGS__;                                                     \
    }                                                                  \
    __atomic_fetch_sub(&skip_segv, 1, __ATOMIC_SEQ_CST);               \
  }

static uintptr_t execute_syscall(int nr, uintptr_t a0, uintptr_t a1,
                                 uintptr_t a2, uintptr_t a3,
                                 uintptr_t a4, uintptr_t a5,
                                 uintptr_t a6, uintptr_t a7,
                                 uintptr_t a8)
{
  switch (nr) {
  default:
    return syscall(nr, a0, a1, a2, a3, a4, a5);
  }
}

static void setup_main_process(uint64_t pid, bool enable_tun)
{
  struct sigaction sa;
  memset(&sa, 0, sizeof(sa));
  sa.sa_handler = SIG_IGN;
  syscall(SYS_rt_sigaction, 0x20, &sa, NULL, 8);
  syscall(SYS_rt_sigaction, 0x21, &sa, NULL, 8);
  install_segv_handler();

  char tmpdir_template[] = "./syzkaller.XXXXXX";
  char* tmpdir = mkdtemp(tmpdir_template);
  if (!tmpdir)
    fail("failed to mkdtemp");
  if (chmod(tmpdir, 0777))
    fail("failed to chmod");
  if (chdir(tmpdir))
    fail("failed to chdir");
}

static void loop();

static void sandbox_common()
{
  prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
  setpgrp();
  setsid();

  struct rlimit rlim;
  rlim.rlim_cur = rlim.rlim_max = 128 << 20;
  setrlimit(RLIMIT_AS, &rlim);
  rlim.rlim_cur = rlim.rlim_max = 1 << 20;
  setrlimit(RLIMIT_FSIZE, &rlim);
  rlim.rlim_cur = rlim.rlim_max = 1 << 20;
  setrlimit(RLIMIT_STACK, &rlim);
  rlim.rlim_cur = rlim.rlim_max = 0;
  setrlimit(RLIMIT_CORE, &rlim);

  unshare(CLONE_NEWNS);
  unshare(CLONE_NEWIPC);
  unshare(CLONE_IO);
}

static int do_sandbox_none()
{
  int pid = fork();
  if (pid)
    return pid;
  sandbox_common();
  loop();
  exit(1);
}

static void remove_dir(const char* dir)
{
  DIR* dp;
  struct dirent* ep;
  int iter = 0;
retry:
  dp = opendir(dir);
  if (dp == NULL) {
    if (errno == EMFILE) {
      exitf("opendir(%s) failed due to NOFILE, exiting");
    }
    exitf("opendir(%s) failed", dir);
  }
  while ((ep = readdir(dp))) {
    if (strcmp(ep->d_name, ".") == 0 || strcmp(ep->d_name, "..") == 0)
      continue;
    char filename[FILENAME_MAX];
    snprintf(filename, sizeof(filename), "%s/%s", dir, ep->d_name);
    struct stat st;
    if (lstat(filename, &st))
      exitf("lstat(%s) failed", filename);
    if (S_ISDIR(st.st_mode)) {
      remove_dir(filename);
      continue;
    }
    int i;
    for (i = 0;; i++) {
      debug("unlink(%s)\n", filename);
      if (unlink(filename) == 0)
        break;
      if (errno == EROFS) {
        debug("ignoring EROFS\n");
        break;
      }
      if (errno != EBUSY || i > 100)
        exitf("unlink(%s) failed", filename);
      debug("umount(%s)\n", filename);
      if (umount2(filename, MNT_DETACH))
        exitf("umount(%s) failed", filename);
    }
  }
  closedir(dp);
  int i;
  for (i = 0;; i++) {
    debug("rmdir(%s)\n", dir);
    if (rmdir(dir) == 0)
      break;
    if (i < 100) {
      if (errno == EROFS) {
        debug("ignoring EROFS\n");
        break;
      }
      if (errno == EBUSY) {
        debug("umount(%s)\n", dir);
        if (umount2(dir, MNT_DETACH))
          exitf("umount(%s) failed", dir);
        continue;
      }
      if (errno == ENOTEMPTY) {
        if (iter < 100) {
          iter++;
          goto retry;
        }
      }
    }
    exitf("rmdir(%s) failed", dir);
  }
}

static uint64_t current_time_ms()
{
  struct timespec ts;

  if (clock_gettime(CLOCK_MONOTONIC, &ts))
    fail("clock_gettime failed");
  return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000;
}

static void test();

void loop()
{
  int iter;
  for (iter = 0;; iter++) {
    char cwdbuf[256];
    sprintf(cwdbuf, "./%d", iter);
    if (mkdir(cwdbuf, 0777))
      fail("failed to mkdir");
    int pid = fork();
    if (pid < 0)
      fail("clone failed");
    if (pid == 0) {
      prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
      setpgrp();
      if (chdir(cwdbuf))
        fail("failed to chdir");
      test();
      exit(0);
    }
    int status = 0;
    uint64_t start = current_time_ms();
    for (;;) {
      int res = waitpid(pid, &status, __WALL | WNOHANG);
      int errno0 = errno;
      if (res == pid)
        break;
      usleep(1000);
      if (current_time_ms() - start > 5 * 1000) {
        kill(-pid, SIGKILL);
        kill(pid, SIGKILL);
        waitpid(pid, &status, __WALL);
        break;
      }
    }
    remove_dir(cwdbuf);
  }
}

long r[11];
void test()
{
  memset(r, -1, sizeof(r));
  r[0] = execute_syscall(__NR_mmap, 0x20000000ul, 0x6000ul, 0x3ul,
                         0x32ul, 0xfffffffffffffffful, 0x0ul, 0, 0, 0);
  r[1] = execute_syscall(__NR_socket, 0xaul, 0x806ul, 0x0ul, 0, 0, 0, 0,
                         0, 0);
  NONFAILING(*(uint64_t*)0x20004fc8 = (uint64_t)0x20004000);
  NONFAILING(*(uint32_t*)0x20004fd0 = (uint32_t)0x2);
  NONFAILING(*(uint64_t*)0x20004fd8 = (uint64_t)0x20004000);
  NONFAILING(*(uint64_t*)0x20004fe0 = (uint64_t)0x0);
  NONFAILING(*(uint64_t*)0x20004fe8 = (uint64_t)0x20005000);
  NONFAILING(*(uint64_t*)0x20004ff0 = (uint64_t)0x0);
  NONFAILING(*(uint32_t*)0x20004ff8 = (uint32_t)0x0);
  NONFAILING(*(uint16_t*)0x20004000 = (uint16_t)0x0);
  r[10] = execute_syscall(__NR_sendmsg, r[1], 0x20004fc8ul, 0x0ul, 0, 0,
                          0, 0, 0, 0);
}
int main()
{
  setup_main_process(0, false);
  int pid = do_sandbox_none();
  int status = 0;
  while (waitpid(pid, &status, __WALL) != pid) {
  }
  return 0;
}