crash使用方法

1.Crash 调试vmcore：
# pwd /var/crash/127.0.0.1-2014-06-23-12:37:00
# crash /usr/lib/debug/lib/module/`uname -r`/vmlinux vmcore
crash> bt /* 打印函数栈 */
crash> ps /* 查看系统崩溃时在运行的所有进程 */
crash> set 3016 /* 连接系统崩溃时正在运行的进程，进行debug */
crash> bt -f /* 打印函数栈数据 */
crash> dis pblaze_disk_make_request /* 反汇编指定函数 */
crash> struct bio ffff880188b50840 /* 将特定地址按照制定数据类型显示 */
crash> struct bio_vec 0xffff880188b508b0
crash> bt
crash> dmesg /* 查看系统崩溃时dmesg的信息 */
crash> bt -f
crash> dis attempt_merge
crash> dis attempt_back_merge
crash> bt -f
crash> dis attempt_merge
crash> struct request ffff880188b4db60
crash> struct bio ffff880188b50840
crash> struct bio_vec[4] 0xffff880188b508b0
crash> struct bio_vec 0xffff880188b508b0
crash> struct bio_vec 0xffff880188b508b8
crash> struct bio_vec 0xffff880188b508c0
crash> struct bio_vec 0xffff880188b508c8




2.Crash 调试卡死的进程：
crash -S System.map vmlinux /*导入内核调试信息以及符号表*/
crash> set 进程id /* 连接需要调试的进程 */
crash> mod -s memdisk /* 导入模块memdisk的符号表 */
crash> mod -s memcon /* 导入模块memcon的符号表 */
crash> bt /* 打印函数栈 */


crash> bt -f /* 打印函数栈详细数据 */
...
 #9 [ffff8801a8c17bf0] attempt_back_merge at ffffffff8126e256
    ffff8801a8c17bf8: ffff8801a7fe0338 0000000000000000 
    ffff8801a8c17c08: ffff8801a8c17c88 ffffffff81267fbb 
/* 函数栈内自底向上，自右向左存储数据。
1.右下角为第一个数据：返回到blk_queue_bio 函数的继续执行地址。 
2.左下角为第二个数据，blk_queue_bio 函数的栈底地址，返回时使用。
*/
#10 [ffff8801a8c17c10] blk_queue_bio at ffffffff81267fbb
    ffff8801a8c17c18: ffff88000001bd80 0000000000000000 
    ffff8801a8c17c28: ffff8801a8c17c78 ffff880188b50840 
    ffff8801a8c17c38: ffff88000001bd80 ffff8801a82c85c0 
    ffff8801a8c17c48: ffff8801a8c17c68 ffff880188b3c4c0 
    ffff8801a8c17c58: ffff880188b50840 ffff880188b50840 
    ffff8801a8c17c68: ffff880188b50840 ffff8801a96c9000 
    ffff8801a8c17c78: ffff8801a7fe0338 0000000000000007 
    ffff8801a8c17c88: ffff8801a8c17cf8 ffffffffa004e7b1 
...
/*根据函数栈中查看到的数据的地址，制定类型将数据打印出来，供debug*/
crash> struct request ffff880188b4db60 
struct request {
  queuelist = {
    next = 0xffff8801a6c0ab40, 
    prev = 0xffff880188b3c4c0
  }, 
  csd = {
    list = {
      next = 0x0, 
      prev = 0x0
    }, 
    func = 0, 
    info = 0x0, 
    flags = 0, 
    priv = 0
  }, 
  q = 0xffff8801a7fe0338, 
  cmd_flags = 17309728, 
  cmd_type = REQ_TYPE_FS, 
  atomic_flags = 0, 
  cpu = 9, 
  __data_len = 0, 
  __sector = 8, 
  bio = 0xffff880188b50f00, 
  biotail = 0xffff880188b50f00, 
  hash = {
    next = 0xffff880188b3c690, 
    pprev = 0xffff8801890dd230
  }, 
  {
    rb_node = {
      rb_parent_color = 18446612138902739936, 
      rb_right = 0x0, 
      rb_left = 0x0
    }, 
    completion_data = 0xffff880188b4dbe0
  }, 
  {
    elevator_private = {0x0, 0x0, 0x0}, 
    flush = {
      seq = 0, 
      list = {
        next = 0x0, 
        prev = 0x0
      }
    }
  }, 
  rq_disk = 0xffff8801a7e18800, 
  start_time = 4295322338, 
  start_time_ns = 655879450331, 
  io_start_time_ns = 0, 
  nr_phys_segments = 0, 
  ioprio = 0, 
  ref_count = 1, 
  special = 0x0, 
  buffer = 0x0, 
  tag = -1, 
  errors = 0, 
  __cmd = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", 
  cmd = 0xffff880188b4dc50 "", 
  cmd_len = 16, 
  extra_len = 0, 
  sense_len = 0, 
  resid_len = 0, 
  sense = 0x0, 
  deadline = 0, 
  timeout_list = {
    next = 0xffff880188b4dc88, 
    prev = 0xffff880188b4dc88
  }, 


crash> dis attempt_merge /* 反汇编制定的函数或制定的地址 */
0xffffffff8126de20 <attempt_merge>:     push   %rbp /*压入调用函数的栈底指针*/
0xffffffff8126de21 <attempt_merge+1>:   mov    %rsp,%rbp
0xffffffff8126de24 <attempt_merge+4>:   sub    $0x40,%rsp
0xffffffff8126de28 <attempt_merge+8>:   mov    %rbx,-0x28(%rbp) 
0xffffffff8126de2c <attempt_merge+12>:  mov    %r12,-0x20(%rbp)
0xffffffff8126de30 <attempt_merge+16>:  mov    %r13,-0x18(%rbp)
0xffffffff8126de34 <attempt_merge+20>:  mov    %r14,-0x10(%rbp)
0xffffffff8126de38 <attempt_merge+24>:  mov    %r15,-0x8(%rbp)
0xffffffff8126de3c <attempt_merge+28>:  nopl   0x0(%rax,%rax,1)
0xffffffff8126de41 <attempt_merge+33>:  cmpl   $0x1,0x44(%rsi)
0xffffffff8126de45 <attempt_merge+37>:  mov    %rdi,%r13
0xffffffff8126de48 <attempt_merge+40>:  mov    %rsi,%rbx
0xffffffff8126de4b <attempt_merge+43>:  mov    %rdx,%r12
0xffffffff8126de4e <attempt_merge+46>:  je     0xffffffff8126de70 <attempt_merge+80>






调试vmcore例子：
BUG: unable to handle kernel NULL pointer dereference at (null)
IP: [<ffffffff8126dfee>] attempt_merge+0x1ce/0x3b0
PGD 1a9ebe067 PUD 1aad69067 PMD 0 
Oops: 0000 [#1] SMP 
last sysfs file: /sys/devices/system/cpu/online
CPU 9 
Modules linked in: bio_submiter(U) ip6table_filter ip6_tables ebtable_nat ebtables ipt_MASQUERADE iptable_nat nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack ipt_REJECT xt_CHECKSUM iptable_mangle iptable_filter ip_tables bridge autofs4 8021q garp stp llc ipv6 vhost_net macvtap macvlan tun kvm_intel kvm uinput power_meter microcode iTCO_wdt iTCO_vendor_support dcdbas sg ses enclosure serio_raw lpc_ich mfd_core i7core_edac edac_core bnx2 memdisk(U) memcon(U) ext4 jbd2 mbcache sr_mod cdrom sd_mod crc_t10dif pata_acpi ata_generic ata_piix megaraid_sas dm_mirror dm_region_hash dm_log dm_mod [last unloaded: bio_submiter]


Pid: 3016, comm: user_submit Not tainted 2.6.32-431.el6.x86_64 #1 Dell Inc. PowerEdge R710/0XDX06
RIP: 0010:[<ffffffff8126dfee>]  [<ffffffff8126dfee>] attempt_merge+0x1ce/0x3b0
RSP: 0018:ffff8801a8c17ba8  EFLAGS: 00010046
RAX: 0000000000000000 RBX: ffff880188b3c4c0 RCX: 000000000000000d
RDX: 0000160000000000 RSI: ffff880188b385c0 RDI: 0000000000000200
RBP: ffff8801a8c17be8 R08: 0000000000000000 R09: 000000000000000d
R10: 0000000188b59400 R11: 0000000188b59200 R12: ffff880188b4db60
R13: ffff8801a7fe0338 R14: 6db6db6db6db6db7 R15: 0000160000000000
FS:  00007f5429dd3700(0000) GS:ffff88002dd20000(0000) knlGS:0000000000000000
Read scucceed, Return value: 0. Scucceed Case Number: 73. LBA addr: 3
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000000000000 CR3: 00000001aa091000 CR4: 00000000000007e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process user_submit (pid: 3016, threadinfo ffff8801a8c16000, task ffff8801a89d8080)
Stack:
 0000000000000000 0000000000000000 ffff8801a8c17bd8 ffff8801a7fe0338
<d> ffff880188b3c4c0 ffff880188b3c4c0 0000000000000000 0000000000000000
<d> ffff8801a8c17c08 ffffffff8126e256 ffff8801a7fe0338 0000000000000000
Call Trace:
 [<ffffffff8126e256>] attempt_back_merge+0x36/0x50
Read scucceed, Return value: 0. Scucceed Case Number: 73. LBA addr: 4
 [<ffffffff81267fbb>] blk_queue_bio+0x48b/0x620
 [<ffffffffa004e7b1>] pblaze_disk_make_request+0x271/0x3bc [memdisk]
 [<ffffffff81122283>] ? mempool_alloc+0x63/0x140
 [<ffffffff8116ea37>] ? cache_grow+0x217/0x320
 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0
 [<ffffffff8112cf3e>] ? __get_free_pages+0xe/0x50
 [<ffffffffa01700c7>] ? test_create_bio+0xc7/0x110 [bio_submiter]
 [<ffffffffa01705c0>] submiter_start_test+0x4b0/0x570 [bio_submiter]
 [<ffffffff811f9384>] proc_file_write+0x74/0xc0
 [<ffffffff811f328e>] proc_reg_write+0x7e/0xc0
 [<ffffffff81188f78>] vfs_write+0xb8/0x1a0
 [<ffffffff81189871>] sys_write+0x51/0x90
Read scucceed, Return value: 0. Scucceed Case Number: 73. LBA addr: 5
 [<ffffffff810e1e5e>] ? __audit_syscall_exit+0x25e/0x290
 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Code: 44 8b 58 0c 49 c1 fa 03 4d 0f af d6 49 c1 e2 0c 4f 8d 1c 1a 44 8b 50 08 0f b7 42 2a 48 c1 e0 04 48 03 42 48 4c 89 fa 4f 8d 14 13 <48> 03 10 48 c1 fa 03 49 0f af d6 44 8b 70 0c 48 c1 e2 0c 4c 01 
RIP  [<ffffffff8126dfee>] attempt_merge+0x1ce/0x3b0
 RSP <ffff8801a8c17ba8>
CR2: 0000000000000000
crash> bt -f
PID: 3016   TASK: ffff8801a89d8080  CPU: 9   COMMAND: "user_submit"
 #0 [ffff8801a8c17770] machine_kexec at ffffffff81038f3b
    ffff8801a8c17778: 00000000030a5000 ffff8800030a5000 
    ffff8801a8c17788: 00000000030a4000 ffff8801a8c17af8 
    ffff8801a8c17798: 8800000000000000 000000000000ffff 
    ffff8801a8c177a8: ffff8801a8c17af8 ffff8801a8c177d8 
    ffff8801a8c177b8: 0000000000000009 ffff8801a89d8080 
    ffff8801a8c177c8: ffff8801a8c17898 ffffffff810c5d92 
 #1 [ffff8801a8c177d0] crash_kexec at ffffffff810c5d92
    ffff8801a8c177d8: 0000160000000000 6db6db6db6db6db7 
    ffff8801a8c177e8: ffff8801a7fe0338 ffff880188b4db60 
    ffff8801a8c177f8: ffff8801a8c17be8 ffff880188b3c4c0 
    ffff8801a8c17808: 0000000188b59200 0000000188b59400 
    ffff8801a8c17818: 000000000000000d 0000000000000000 
    ffff8801a8c17828: 0000000000000000 000000000000000d 
    ffff8801a8c17838: 0000160000000000 ffff880188b385c0 
    ffff8801a8c17848: 0000000000000200 ffffffffffffffff 
    ffff8801a8c17858: ffffffff8126dfee 0000000000000010 
    ffff8801a8c17868: 0000000000010046 ffff8801a8c17ba8 
    ffff8801a8c17878: 0000000000000018 ffff8801a8c178d8 
    ffff8801a8c17888: 0000000000000046 ffff8801a8c17af8 
    ffff8801a8c17898: ffff8801a8c178c8 ffffffff8152b510 
 #2 [ffff8801a8c178a0] oops_end at ffffffff8152b510
    ffff8801a8c178a8: 0000000000000000 ffff8801a8c17af8 
    ffff8801a8c178b8: 0000000000000000 0000000000000009 
    ffff8801a8c178c8: ffff8801a8c17918 ffffffff8104a00b 
 #3 [ffff8801a8c178d0] no_context at ffffffff8104a00b
    ffff8801a8c178d8: 0000000000000002 ffff880000035c20 
    ffff8801a8c178e8: ffff8801a89d8080 0000000000000000 
    ffff8801a8c178f8: 0000000000000000 ffff8801a8c17af8 
    ffff8801a8c17908: ffff8801a89d8080 0000000000030001 
    ffff8801a8c17918: ffff8801a8c17968 ffffffff8104a295 
 #4 [ffff8801a8c17920] __bad_area_nosemaphore at ffffffff8104a295
    ffff8801a8c17928: ffff880000035c28 ffff8801aca11800 
    ffff8801a8c17938: ffff880188b4d000 ffff8801a8c17af8 
    ffff8801a8c17948: 0000000000000000 0000000000000000 
    ffff8801a8c17958: ffff8801a908c318 0000000000000000 
    ffff8801a8c17968: ffff8801a8c17998 ffffffff8104a3be 
 #5 [ffff8801a8c17970] bad_area at ffffffff8104a3be
    ffff8801a8c17978: 00000000a8c16000 0000000000000028 
    ffff8801a8c17988: 0000000000000000 ffff8801a7df2100 
    ffff8801a8c17998: ffff8801a8c17ab8 ffffffff8104ab6f 
 #6 [ffff8801a8c179a0] __do_page_fault at ffffffff8104ab6f
    ffff8801a8c179a8: ffff8801ac9b6400 ffff8801ac8d40c0 
    ffff8801a8c179b8: ffff8801a8c17af8 0000000000071280 
    ffff8801a8c179c8: ffff8801a89d8080 ffff8801a7df2168 
    ffff8801a8c179d8: ffff8801ac8d4100 00051200a8c17a08 
    ffff8801a8c179e8: ffff8801ac8d40e0 ffff8801ac8d40d0 
    ffff8801a8c179f8: ffff88000001bd80 ffff8801a7521640 
    ffff8801a8c17a08: 0000000000011200 ffff8801ac981700 
    ffff8801a8c17a18: 0000000000000000 0000000000000000 
    ffff8801a8c17a28: 0000000000000001 ffff880000023de8 
    ffff8801a8c17a38: ffff8801a8c17ad8 ffffffff8112bb13 
    ffff8801a8c17a48: ffffea00055e8188 00000000000000b0 
    ffff8801a8c17a58: ffff880100000002 ffff880000023de8 
    ffff8801a8c17a68: ffff8801a8c17a78 0000000000000000 
    ffff8801a8c17a78: 0000000000000000 ffffffff81122283 
    ffff8801a8c17a88: ffff88000001bd80 ffff8801a8c17af8 
    ffff8801a8c17a98: 0000000000000000 0000000000000000 
    ffff8801a8c17aa8: 6db6db6db6db6db7 0000160000000000 
    ffff8801a8c17ab8: ffff8801a8c17ae8 ffffffff8152d45e 
 #7 [ffff8801a8c17ac0] do_page_fault at ffffffff8152d45e
    ffff8801a8c17ac8: 0000000000000001 ffff880188b4db60 
    ffff8801a8c17ad8: ffff8801a7fe0338 6db6db6db6db6db7 
    ffff8801a8c17ae8: ffff8801a8c17be8 ffffffff8152a815 
 #8 [ffff8801a8c17af0] page_fault at ffffffff8152a815
    [exception RIP: attempt_merge+462]
    RIP: ffffffff8126dfee  RSP: ffff8801a8c17ba8  RFLAGS: 00010046
    RAX: 0000000000000000  RBX: ffff880188b3c4c0  RCX: 000000000000000d
    RDX: 0000160000000000  RSI: ffff880188b385c0  RDI: 0000000000000200
    RBP: ffff8801a8c17be8   R8: 0000000000000000   R9: 000000000000000d
    R10: 0000000188b59400  R11: 0000000188b59200  R12: ffff880188b4db60
    R13: ffff8801a7fe0338  R14: 6db6db6db6db6db7  R15: 0000160000000000
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
    ffff8801a8c17af8: 0000160000000000 6db6db6db6db6db7 
    ffff8801a8c17b08: ffff8801a7fe0338 ffff880188b4db60 
    ffff8801a8c17b18: ffff8801a8c17be8 ffff880188b3c4c0 
    ffff8801a8c17b28: 0000000188b59200 0000000188b59400 
    ffff8801a8c17b38: 000000000000000d 0000000000000000 
    ffff8801a8c17b48: 0000000000000000 000000000000000d 
    ffff8801a8c17b58: 0000160000000000 ffff880188b385c0 
    ffff8801a8c17b68: 0000000000000200 ffffffffffffffff 
    ffff8801a8c17b78: ffffffff8126dfee 0000000000000010 
    ffff8801a8c17b88: 0000000000010046 ffff8801a8c17ba8 
    ffff8801a8c17b98: 0000000000000018 0000000000000200 
    ffff8801a8c17ba8: 0000000000000000 0000000000000000 
    ffff8801a8c17bb8: ffff8801a8c17bd8 ffff8801a7fe0338 
    ffff8801a8c17bc8: ffff880188b3c4c0 ffff880188b3c4c0 
    ffff8801a8c17bd8: 0000000000000000 0000000000000000 
    ffff8801a8c17be8: ffff8801a8c17c08 ffffffff8126e256 
 #9 [ffff8801a8c17bf0] attempt_back_merge at ffffffff8126e256
    ffff8801a8c17bf8: ffff8801a7fe0338 0000000000000000 
    ffff8801a8c17c08: ffff8801a8c17c88 ffffffff81267fbb 
#10 [ffff8801a8c17c10] blk_queue_bio at ffffffff81267fbb
    ffff8801a8c17c18: ffff88000001bd80 0000000000000000 
crash> dis attempt_merge
0xffffffff8126de20 <attempt_merge>:     push   %rbp
0xffffffff8126de21 <attempt_merge+1>:   mov    %rsp,%rbp
0xffffffff8126de24 <attempt_merge+4>:   sub    $0x40,%rsp
0xffffffff8126de28 <attempt_merge+8>:   mov    %rbx,-0x28(%rbp)
0xffffffff8126de2c <attempt_merge+12>:  mov    %r12,-0x20(%rbp)
0xffffffff8126de30 <attempt_merge+16>:  mov    %r13,-0x18(%rbp)
0xffffffff8126de34 <attempt_merge+20>:  mov    %r14,-0x10(%rbp)
0xffffffff8126de38 <attempt_merge+24>:  mov    %r15,-0x8(%rbp)
0xffffffff8126de3c <attempt_merge+28>:  nopl   0x0(%rax,%rax,1)
0xffffffff8126de41 <attempt_merge+33>:  cmpl   $0x1,0x44(%rsi)
0xffffffff8126de45 <attempt_merge+37>:  mov    %rdi,%r13
0xffffffff8126de48 <attempt_merge+40>:  mov    %rsi,%rbx
0xffffffff8126de4b <attempt_merge+43>:  mov    %rdx,%r12
0xffffffff8126de4e <attempt_merge+46>:  je     0xffffffff8126de70 <attempt_merge+80>
0xffffffff8126de50 <attempt_merge+48>:  xor    %eax,%eax
0xffffffff8126de52 <attempt_merge+50>:  mov    -0x28(%rbp),%rbx
0xffffffff8126de56 <attempt_merge+54>:  mov    -0x20(%rbp),%r12
0xffffffff8126de5a <attempt_merge+58>:  mov    -0x18(%rbp),%r13
0xffffffff8126de5e <attempt_merge+62>:  mov    -0x10(%rbp),%r14
0xffffffff8126de62 <attempt_merge+66>:  mov    -0x8(%rbp),%r15
0xffffffff8126de66 <attempt_merge+70>:  leaveq 
0xffffffff8126de67 <attempt_merge+71>:  retq   
0xffffffff8126de68 <attempt_merge+72>:  nopl   0x0(%rax,%rax,1)
0xffffffff8126de70 <attempt_merge+80>:  mov    0x40(%rsi),%ecx
0xffffffff8126de73 <attempt_merge+83>:  test   $0x40007c0,%ecx
0xffffffff8126de79 <attempt_merge+89>:  jne    0xffffffff8126de50 <attempt_merge+48>
0xffffffff8126de7b <attempt_merge+91>:  cmpl   $0x1,0x44(%rdx)
0xffffffff8126de7f <attempt_merge+95>:  jne    0xffffffff8126de50 <attempt_merge+48>
0xffffffff8126de81 <attempt_merge+97>:  mov    0x40(%r12),%eax
0xffffffff8126de86 <attempt_merge+102>: test   $0x40007c0,%eax
0xffffffff8126de8b <attempt_merge+107>: jne    0xffffffff8126de50 <attempt_merge+48>
crash> dis attempt_back_merge
0xffffffff8126e220 <attempt_back_merge>:        push   %rbp
0xffffffff8126e221 <attempt_back_merge+1>:      mov    %rsp,%rbp
0xffffffff8126e224 <attempt_back_merge+4>:      sub    $0x10,%rsp
0xffffffff8126e228 <attempt_back_merge+8>:      mov    %rbx,(%rsp)
0xffffffff8126e22c <attempt_back_merge+12>:     mov    %r12,0x8(%rsp)
0xffffffff8126e231 <attempt_back_merge+17>:     nopl   0x0(%rax,%rax,1)
0xffffffff8126e236 <attempt_back_merge+22>:     mov    %rdi,%rbx
0xffffffff8126e239 <attempt_back_merge+25>:     mov    %rsi,%r12
0xffffffff8126e23c <attempt_back_merge+28>:     callq  0xffffffff8125fce0 <elv_latter_request>
0xffffffff8126e241 <attempt_back_merge+33>:     mov    %rax,%rdx
0xffffffff8126e244 <attempt_back_merge+36>:     xor    %eax,%eax
0xffffffff8126e246 <attempt_back_merge+38>:     test   %rdx,%rdx
0xffffffff8126e249 <attempt_back_merge+41>:     je     0xffffffff8126e256 <attempt_back_merge+54>
0xffffffff8126e24b <attempt_back_merge+43>:     mov    %r12,%rsi
0xffffffff8126e24e <attempt_back_merge+46>:     mov    %rbx,%rdi
0xffffffff8126e251 <attempt_back_merge+49>:     callq  0xffffffff8126de20 <attempt_merge>
0xffffffff8126e256 <attempt_back_merge+54>:     mov    (%rsp),%rbx
0xffffffff8126e25a <attempt_back_merge+58>:     mov    0x8(%rsp),%r12
0xffffffff8126e25f <attempt_back_merge+63>:     leaveq 
0xffffffff8126e260 <attempt_back_merge+64>:     retq   
0xffffffff8126e261 <attempt_back_merge+65>:     data32 data32 data32 data32 data32 nopw %cs:0x0(%rax,%rax,1)
crash> bt -f
PID: 3016   TASK: ffff8801a89d8080  CPU: 9   COMMAND: "user_submit"
 #0 [ffff8801a8c17770] machine_kexec at ffffffff81038f3b
    ffff8801a8c17778: 00000000030a5000 ffff8800030a5000 
    ffff8801a8c17788: 00000000030a4000 ffff8801a8c17af8 
    ffff8801a8c17798: 8800000000000000 000000000000ffff 
    ffff8801a8c177a8: ffff8801a8c17af8 ffff8801a8c177d8 
    ffff8801a8c177b8: 0000000000000009 ffff8801a89d8080 
    ffff8801a8c177c8: ffff8801a8c17898 ffffffff810c5d92 
 #1 [ffff8801a8c177d0] crash_kexec at ffffffff810c5d92
    ffff8801a8c177d8: 0000160000000000 6db6db6db6db6db7 
    ffff8801a8c177e8: ffff8801a7fe0338 ffff880188b4db60 
    ffff8801a8c177f8: ffff8801a8c17be8 ffff880188b3c4c0 
    ffff8801a8c17808: 0000000188b59200 0000000188b59400 
    ffff8801a8c17818: 000000000000000d 0000000000000000 
    ffff8801a8c17828: 0000000000000000 000000000000000d 
    ffff8801a8c17838: 0000160000000000 ffff880188b385c0 
    ffff8801a8c17848: 0000000000000200 ffffffffffffffff 
    ffff8801a8c17858: ffffffff8126dfee 0000000000000010 
    ffff8801a8c17868: 0000000000010046 ffff8801a8c17ba8 
    ffff8801a8c17878: 0000000000000018 ffff8801a8c178d8 
    ffff8801a8c17888: 0000000000000046 ffff8801a8c17af8 
    ffff8801a8c17898: ffff8801a8c178c8 ffffffff8152b510 
 #2 [ffff8801a8c178a0] oops_end at ffffffff8152b510
    ffff8801a8c178a8: 0000000000000000 ffff8801a8c17af8 
    ffff8801a8c178b8: 0000000000000000 0000000000000009 
    ffff8801a8c178c8: ffff8801a8c17918 ffffffff8104a00b 
 #3 [ffff8801a8c178d0] no_context at ffffffff8104a00b
    ffff8801a8c178d8: 0000000000000002 ffff880000035c20 
    ffff8801a8c178e8: ffff8801a89d8080 0000000000000000 
    ffff8801a8c178f8: 0000000000000000 ffff8801a8c17af8 
    ffff8801a8c17908: ffff8801a89d8080 0000000000030001 
    ffff8801a8c17918: ffff8801a8c17968 ffffffff8104a295 
 #4 [ffff8801a8c17920] __bad_area_nosemaphore at ffffffff8104a295
    ffff8801a8c17928: ffff880000035c28 ffff8801aca11800 
    ffff8801a8c17938: ffff880188b4d000 ffff8801a8c17af8 
    ffff8801a8c17948: 0000000000000000 0000000000000000 
    ffff8801a8c17958: ffff8801a908c318 0000000000000000 
    ffff8801a8c17968: ffff8801a8c17998 ffffffff8104a3be 
 #5 [ffff8801a8c17970] bad_area at ffffffff8104a3be
    ffff8801a8c17978: 00000000a8c16000 0000000000000028 
    ffff8801a8c17988: 0000000000000000 ffff8801a7df2100 
    ffff8801a8c17998: ffff8801a8c17ab8 ffffffff8104ab6f 
 #6 [ffff8801a8c179a0] __do_page_fault at ffffffff8104ab6f
    ffff8801a8c179a8: ffff8801ac9b6400 ffff8801ac8d40c0 
    ffff8801a8c179b8: ffff8801a8c17af8 0000000000071280 
    ffff8801a8c179c8: ffff8801a89d8080 ffff8801a7df2168 
    ffff8801a8c179d8: ffff8801ac8d4100 00051200a8c17a08 
    ffff8801a8c179e8: ffff8801ac8d40e0 ffff8801ac8d40d0 
    ffff8801a8c179f8: ffff88000001bd80 ffff8801a7521640 
    ffff8801a8c17a08: 0000000000011200 ffff8801ac981700 
    ffff8801a8c17a18: 0000000000000000 0000000000000000 
    ffff8801a8c17a28: 0000000000000001 ffff880000023de8 
    ffff8801a8c17a38: ffff8801a8c17ad8 ffffffff8112bb13 
    ffff8801a8c17a48: ffffea00055e8188 00000000000000b0 
    ffff8801a8c17a58: ffff880100000002 ffff880000023de8 
    ffff8801a8c17a68: ffff8801a8c17a78 0000000000000000 
    ffff8801a8c17a78: 0000000000000000 ffffffff81122283 
    ffff8801a8c17a88: ffff88000001bd80 ffff8801a8c17af8 
    ffff8801a8c17a98: 0000000000000000 0000000000000000 
    ffff8801a8c17aa8: 6db6db6db6db6db7 0000160000000000 
    ffff8801a8c17ab8: ffff8801a8c17ae8 ffffffff8152d45e 
 #7 [ffff8801a8c17ac0] do_page_fault at ffffffff8152d45e
    ffff8801a8c17ac8: 0000000000000001 ffff880188b4db60 
    ffff8801a8c17ad8: ffff8801a7fe0338 6db6db6db6db6db7 
    ffff8801a8c17ae8: ffff8801a8c17be8 ffffffff8152a815 
 #8 [ffff8801a8c17af0] page_fault at ffffffff8152a815
    [exception RIP: attempt_merge+462]
    RIP: ffffffff8126dfee  RSP: ffff8801a8c17ba8  RFLAGS: 00010046
    RAX: 0000000000000000  RBX: ffff880188b3c4c0  RCX: 000000000000000d
    RDX: 0000160000000000  RSI: ffff880188b385c0  RDI: 0000000000000200
    RBP: ffff8801a8c17be8   R8: 0000000000000000   R9: 000000000000000d
    R10: 0000000188b59400  R11: 0000000188b59200  R12: ffff880188b4db60
    R13: ffff8801a7fe0338  R14: 6db6db6db6db6db7  R15: 0000160000000000
    ffff8801a8c17ad8: ffff8801a7fe0338 6db6db6db6db6db7 
    ffff8801a8c17ae8: ffff8801a8c17be8 ffffffff8152a815 
 #8 [ffff8801a8c17af0] page_fault at ffffffff8152a815
    [exception RIP: attempt_merge+462]
    RIP: ffffffff8126dfee  RSP: ffff8801a8c17ba8  RFLAGS: 00010046
    RAX: 0000000000000000  RBX: ffff880188b3c4c0  RCX: 000000000000000d
    RDX: 0000160000000000  RSI: ffff880188b385c0  RDI: 0000000000000200
    RBP: ffff8801a8c17be8   R8: 0000000000000000   R9: 000000000000000d
    R10: 0000000188b59400  R11: 0000000188b59200  R12: ffff880188b4db60
    R13: ffff8801a7fe0338  R14: 6db6db6db6db6db7  R15: 0000160000000000
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
    ffff8801a8c17af8: 0000160000000000 6db6db6db6db6db7 
    ffff8801a8c17b08: ffff8801a7fe0338 ffff880188b4db60 
    ffff8801a8c17b18: ffff8801a8c17be8 ffff880188b3c4c0 
    ffff8801a8c17b28: 0000000188b59200 0000000188b59400 
    ffff8801a8c17b38: 000000000000000d 0000000000000000 
    ffff8801a8c17b48: 0000000000000000 000000000000000d 
    ffff8801a8c17b58: 0000160000000000 ffff880188b385c0 
    ffff8801a8c17b68: 0000000000000200 ffffffffffffffff 
    ffff8801a8c17b78: ffffffff8126dfee 0000000000000010 
    ffff8801a8c17b88: 0000000000010046 ffff8801a8c17ba8 
    ffff8801a8c17b98: 0000000000000018 0000000000000200 
    ffff8801a8c17ba8: 0000000000000000 0000000000000000 
    ffff8801a8c17bb8: ffff8801a8c17bd8 ffff8801a7fe0338 
    ffff8801a8c17bc8: ffff880188b3c4c0 ffff880188b3c4c0 
    ffff8801a8c17bd8: 0000000000000000 0000000000000000 
    ffff8801a8c17be8: ffff8801a8c17c08 ffffffff8126e256 
 #9 [ffff8801a8c17bf0] attempt_back_merge at ffffffff8126e256
    ffff8801a8c17bf8: ffff8801a7fe0338 0000000000000000 
    ffff8801a8c17c08: ffff8801a8c17c88 ffffffff81267fbb 
#10 [ffff8801a8c17c10] blk_queue_bio at ffffffff81267fbb
    ffff8801a8c17c18: ffff88000001bd80 0000000000000000 
    ffff8801a8c17c28: ffff8801a8c17c78 ffff880188b50840 
    ffff8801a8c17c38: ffff88000001bd80 ffff8801a82c85c0 
    ffff8801a8c17c48: ffff8801a8c17c68 ffff880188b3c4c0 
    ffff8801a8c17c58: ffff880188b50840 ffff880188b50840 
    ffff8801a8c17c68: ffff880188b50840 ffff8801a96c9000 
    ffff8801a8c17c78: ffff8801a7fe0338 0000000000000007 
    ffff8801a8c17c88: ffff8801a8c17cf8 ffffffffa004e7b1 
#11 [ffff8801a8c17c90] pblaze_disk_make_request at ffffffffa004e7b1 [memdisk]
    ffff8801a8c17c98: ffff8801a8c17d28 ffffffff81122283 
    ffff8801a8c17ca8: 00008801a8c17d18 ffffffff8116ea37 
    ffff8801a8c17cb8: ffff8801a89d8080 0000000000000001 
    ffff8801a8c17cc8: 0000001000000000 ffff880188b50840 
    RBP: ffff8801a8c17be8   R8: 0000000000000000   R9: 000000000000000d
    R10: 0000000188b59400  R11: 0000000188b59200  R12: ffff880188b4db60
    R13: ffff8801a7fe0338  R14: 6db6db6db6db6db7  R15: 0000160000000000
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
    ffff8801a8c17af8: 0000160000000000 6db6db6db6db6db7 
    ffff8801a8c17b08: ffff8801a7fe0338 ffff880188b4db60 
    ffff8801a8c17b18: ffff8801a8c17be8 ffff880188b3c4c0 
    ffff8801a8c17b28: 0000000188b59200 0000000188b59400 
    ffff8801a8c17b38: 000000000000000d 0000000000000000 
    ffff8801a8c17b48: 0000000000000000 000000000000000d 
    ffff8801a8c17b58: 0000160000000000 ffff880188b385c0 
    ffff8801a8c17b68: 0000000000000200 ffffffffffffffff 
    ffff8801a8c17b78: ffffffff8126dfee 0000000000000010 
    ffff8801a8c17b88: 0000000000010046 ffff8801a8c17ba8 
    ffff8801a8c17b98: 0000000000000018 0000000000000200 
    ffff8801a8c17ba8: 0000000000000000 0000000000000000 
    ffff8801a8c17bb8: ffff8801a8c17bd8 ffff8801a7fe0338 
    ffff8801a8c17bc8: ffff880188b3c4c0 ffff880188b3c4c0 
    ffff8801a8c17bd8: 0000000000000000 0000000000000000 
    ffff8801a8c17be8: ffff8801a8c17c08 ffffffff8126e256 
 #9 [ffff8801a8c17bf0] attempt_back_merge at ffffffff8126e256
    ffff8801a8c17bf8: ffff8801a7fe0338 0000000000000000 
    ffff8801a8c17c08: ffff8801a8c17c88 ffffffff81267fbb 
#10 [ffff8801a8c17c10] blk_queue_bio at ffffffff81267fbb
    ffff8801a8c17c18: ffff88000001bd80 0000000000000000 
    ffff8801a8c17c28: ffff8801a8c17c78 ffff880188b50840 
    ffff8801a8c17c38: ffff88000001bd80 ffff8801a82c85c0 
    ffff8801a8c17c48: ffff8801a8c17c68 ffff880188b3c4c0 
    ffff8801a8c17c58: ffff880188b50840 ffff880188b50840 
    ffff8801a8c17c68: ffff880188b50840 ffff8801a96c9000 
    ffff8801a8c17c78: ffff8801a7fe0338 0000000000000007 
    ffff8801a8c17c88: ffff8801a8c17cf8 ffffffffa004e7b1 
#11 [ffff8801a8c17c90] pblaze_disk_make_request at ffffffffa004e7b1 [memdisk]
crash> bt -f
PID: 3016   TASK: ffff8801a89d8080  CPU: 9   COMMAND: "user_submit"
 #0 [ffff8801a8c17770] machine_kexec at ffffffff81038f3b
    ffff8801a8c17778: 00000000030a5000 ffff8800030a5000 
    ffff8801a8c17788: 00000000030a4000 ffff8801a8c17af8 
    ffff8801a8c17798: 8800000000000000 000000000000ffff 
    ffff8801a8c177a8: ffff8801a8c17af8 ffff8801a8c177d8 
    ffff8801a8c177b8: 0000000000000009 ffff8801a89d8080 
    ffff8801a8c177c8: ffff8801a8c17898 ffffffff810c5d92 
 #1 [ffff8801a8c177d0] crash_kexec at ffffffff810c5d92
    ffff8801a8c177d8: 0000160000000000 6db6db6db6db6db7 
    ffff8801a8c177e8: ffff8801a7fe0338 ffff880188b4db60 
    ffff8801a8c177f8: ffff8801a8c17be8 ffff880188b3c4c0 
    ffff8801a8c17808: 0000000188b59200 0000000188b59400 
    ffff8801a8c17818: 000000000000000d 0000000000000000 
    ffff8801a8c17828: 0000000000000000 000000000000000d 
    ffff8801a8c17838: 0000160000000000 ffff880188b385c0 
    ffff8801a8c17848: 0000000000000200 ffffffffffffffff 
    ffff8801a8c17858: ffffffff8126dfee 0000000000000010 
    ffff8801a8c17868: 0000000000010046 ffff8801a8c17ba8 
    ffff8801a8c17878: 0000000000000018 ffff8801a8c178d8 
    ffff8801a8c17888: 0000000000000046 ffff8801a8c17af8 
    ffff8801a8c17898: ffff8801a8c178c8 ffffffff8152b510 
 #2 [ffff8801a8c178a0] oops_end at ffffffff8152b510
    ffff8801a8c178a8: 0000000000000000 ffff8801a8c17af8 
    ffff8801a8c178b8: 0000000000000000 0000000000000009 
    ffff8801a8c178c8: ffff8801a8c17918 ffffffff8104a00b 
 #3 [ffff8801a8c178d0] no_context at ffffffff8104a00b
    ffff8801a8c178d8: 0000000000000002 ffff880000035c20 
    ffff8801a8c178e8: ffff8801a89d8080 0000000000000000 
    ffff8801a8c178f8: 0000000000000000 ffff8801a8c17af8 
    ffff8801a8c17908: ffff8801a89d8080 0000000000030001 
    ffff8801a8c17918: ffff8801a8c17968 ffffffff8104a295 
 #4 [ffff8801a8c17920] __bad_area_nosemaphore at ffffffff8104a295
    ffff8801a8c17928: ffff880000035c28 ffff8801aca11800 
    ffff8801a8c17938: ffff880188b4d000 ffff8801a8c17af8 
    ffff8801a8c17948: 0000000000000000 0000000000000000 
    ffff8801a8c17958: ffff8801a908c318 0000000000000000 
    ffff8801a8c17968: ffff8801a8c17998 ffffffff8104a3be 
 #5 [ffff8801a8c17970] bad_area at ffffffff8104a3be
    ffff8801a8c17978: 00000000a8c16000 0000000000000028 
    ffff8801a8c17988: 0000000000000000 ffff8801a7df2100 
    ffff8801a8c17998: ffff8801a8c17ab8 ffffffff8104ab6f 
 #6 [ffff8801a8c179a0] __do_page_fault at ffffffff8104ab6f
    ffff8801a8c179a8: ffff8801ac9b6400 ffff8801ac8d40c0 
    ffff8801a8c179b8: ffff8801a8c17af8 0000000000071280 
    ffff8801a8c179c8: ffff8801a89d8080 ffff8801a7df2168 
    ffff8801a8c179d8: ffff8801ac8d4100 00051200a8c17a08 
    ffff8801a8c179e8: ffff8801ac8d40e0 ffff8801ac8d40d0 
    ffff8801a8c179f8: ffff88000001bd80 ffff8801a7521640 
    ffff8801a8c17a08: 0000000000011200 ffff8801ac981700 
    ffff8801a8c17a18: 0000000000000000 0000000000000000 
    ffff8801a8c17a28: 0000000000000001 ffff880000023de8 
    ffff8801a8c17a38: ffff8801a8c17ad8 ffffffff8112bb13 
    ffff8801a8c17a48: ffffea00055e8188 00000000000000b0 
    ffff8801a8c17a58: ffff880100000002 ffff880000023de8 
    ffff8801a8c17a68: ffff8801a8c17a78 0000000000000000 
    ffff8801a8c17a78: 0000000000000000 ffffffff81122283 
    ffff8801a8c17a88: ffff88000001bd80 ffff8801a8c17af8 
    ffff8801a8c17a98: 0000000000000000 0000000000000000 
    ffff8801a8c17aa8: 6db6db6db6db6db7 0000160000000000 
    ffff8801a8c17ab8: ffff8801a8c17ae8 ffffffff8152d45e 
 #7 [ffff8801a8c17ac0] do_page_fault at ffffffff8152d45e
    ffff8801a8c17ac8: 0000000000000001 ffff880188b4db60 
    ffff8801a8c17ad8: ffff8801a7fe0338 6db6db6db6db6db7 
    ffff8801a8c17ae8: ffff8801a8c17be8 ffffffff8152a815 
 #8 [ffff8801a8c17af0] page_fault at ffffffff8152a815
    [exception RIP: attempt_merge+462]
    RIP: ffffffff8126dfee  RSP: ffff8801a8c17ba8  RFLAGS: 00010046
    RAX: 0000000000000000  RBX: ffff880188b3c4c0  RCX: 000000000000000d
    RDX: 0000160000000000  RSI: ffff880188b385c0  RDI: 0000000000000200
    RBP: ffff8801a8c17be8   R8: 0000000000000000   R9: 000000000000000d
    R10: 0000000188b59400  R11: 0000000188b59200  R12: ffff880188b4db60
    R13: ffff8801a7fe0338  R14: 6db6db6db6db6db7  R15: 0000160000000000
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
    ffff8801a8c17af8: 0000160000000000 6db6db6db6db6db7 
    ffff8801a8c17b08: ffff8801a7fe0338 ffff880188b4db60 
    ffff8801a8c17b18: ffff8801a8c17be8 ffff880188b3c4c0 
    ffff8801a8c17b28: 0000000188b59200 0000000188b59400 
    ffff8801a8c17b38: 000000000000000d 0000000000000000 
    ffff8801a8c17b48: 0000000000000000 000000000000000d 
    ffff8801a8c17b58: 0000160000000000 ffff880188b385c0 
    ffff8801a8c17b68: 0000000000000200 ffffffffffffffff 
    ffff8801a8c17b78: ffffffff8126dfee 0000000000000010 
    ffff8801a8c17b88: 0000000000010046 ffff8801a8c17ba8 
    ffff8801a8c17b98: 0000000000000018 0000000000000200 
    ffff8801a8c17ba8: 0000000000000000 0000000000000000 
    ffff8801a8c17bb8: ffff8801a8c17bd8 ffff8801a7fe0338 
    ffff8801a8c17bc8: ffff880188b3c4c0 ffff880188b3c4c0 
    ffff8801a8c17bd8: 0000000000000000 0000000000000000 
    ffff8801a8c17be8: ffff8801a8c17c08 ffffffff8126e256 
 #9 [ffff8801a8c17bf0] attempt_back_merge at ffffffff8126e256
    ffff8801a8c17bf8: ffff8801a7fe0338 0000000000000000 
    ffff8801a8c17c08: ffff8801a8c17c88 ffffffff81267fbb 
crash> dis attempt_merge
0xffffffff8126de20 <attempt_merge>:     push   %rbp
0xffffffff8126de21 <attempt_merge+1>:   mov    %rsp,%rbp
0xffffffff8126de24 <attempt_merge+4>:   sub    $0x40,%rsp
0xffffffff8126de28 <attempt_merge+8>:   mov    %rbx,-0x28(%rbp)
0xffffffff8126de2c <attempt_merge+12>:  mov    %r12,-0x20(%rbp)
0xffffffff8126de30 <attempt_merge+16>:  mov    %r13,-0x18(%rbp)
0xffffffff8126de34 <attempt_merge+20>:  mov    %r14,-0x10(%rbp)
0xffffffff8126de38 <attempt_merge+24>:  mov    %r15,-0x8(%rbp)
0xffffffff8126de3c <attempt_merge+28>:  nopl   0x0(%rax,%rax,1)
0xffffffff8126de41 <attempt_merge+33>:  cmpl   $0x1,0x44(%rsi)
0xffffffff8126de45 <attempt_merge+37>:  mov    %rdi,%r13
0xffffffff8126de48 <attempt_merge+40>:  mov    %rsi,%rbx
0xffffffff8126de4b <attempt_merge+43>:  mov    %rdx,%r12
0xffffffff8126de4e <attempt_merge+46>:  je     0xffffffff8126de70 <attempt_merge+80>
0xffffffff8126de50 <attempt_merge+48>:  xor    %eax,%eax
0xffffffff8126de52 <attempt_merge+50>:  mov    -0x28(%rbp),%rbx
0xffffffff8126de56 <attempt_merge+54>:  mov    -0x20(%rbp),%r12
0xffffffff8126de5a <attempt_merge+58>:  mov    -0x18(%rbp),%r13
0xffffffff8126de5e <attempt_merge+62>:  mov    -0x10(%rbp),%r14
0xffffffff8126de62 <attempt_merge+66>:  mov    -0x8(%rbp),%r15
0xffffffff8126de66 <attempt_merge+70>:  leaveq 
0xffffffff8126de67 <attempt_merge+71>:  retq   
0xffffffff8126de68 <attempt_merge+72>:  nopl   0x0(%rax,%rax,1)
0xffffffff8126de70 <attempt_merge+80>:  mov    0x40(%rsi),%ecx
0xffffffff8126de73 <attempt_merge+83>:  test   $0x40007c0,%ecx
0xffffffff8126de79 <attempt_merge+89>:  jne    0xffffffff8126de50 <attempt_merge+48>
0xffffffff8126de7b <attempt_merge+91>:  cmpl   $0x1,0x44(%rdx)
0xffffffff8126de7f <attempt_merge+95>:  jne    0xffffffff8126de50 <attempt_merge+48>
0xffffffff8126de81 <attempt_merge+97>:  mov    0x40(%r12),%eax
0xffffffff8126de86 <attempt_merge+102>: test   $0x40007c0,%eax
0xffffffff8126de8b <attempt_merge+107>: jne    0xffffffff8126de50 <attempt_merge+48>
crash> struct request ffff880188b4db60
struct request {
  queuelist = {
    next = 0xffff8801a6c0ab40, 
    prev = 0xffff880188b3c4c0
  }, 
  csd = {
    list = {
      next = 0x0, 
      prev = 0x0
    }, 
    func = 0, 
    info = 0x0, 
    flags = 0, 
    priv = 0
  }, 
  q = 0xffff8801a7fe0338, 
  cmd_flags = 17309728, 
  cmd_type = REQ_TYPE_FS, 
  atomic_flags = 0, 
  cpu = 9, 
  __data_len = 0, 
  __sector = 8, 
  bio = 0xffff880188b50f00, 
  biotail = 0xffff880188b50f00, 
  hash = {
    next = 0xffff880188b3c690, 
    pprev = 0xffff8801890dd230
  }, 
  {
    rb_node = {
      rb_parent_color = 18446612138902739936, 
      rb_right = 0x0, 
      rb_left = 0x0
    }, 
    completion_data = 0xffff880188b4dbe0
  }, 
  {
    elevator_private = {0x0, 0x0, 0x0}, 
    flush = {
      seq = 0, 
      list = {
        next = 0x0, 
        prev = 0x0
      }
    }
  }, 
  rq_disk = 0xffff8801a7e18800, 
  start_time = 4295322338, 
  start_time_ns = 655879450331, 
  io_start_time_ns = 0, 
  nr_phys_segments = 0, 
  ioprio = 0, 
  ref_count = 1, 
  special = 0x0, 
  buffer = 0x0, 
  tag = -1, 
  errors = 0, 
  __cmd = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", 
  cmd = 0xffff880188b4dc50 "", 
  cmd_len = 16, 
  extra_len = 0, 
  sense_len = 0, 
  resid_len = 0, 
  sense = 0x0, 
  deadline = 0, 
  timeout_list = {
    next = 0xffff880188b4dc88, 
    prev = 0xffff880188b4dc88
  }, 
  timeout = 0, 
  retries = 0, 
  end_io = 0, 
  end_io_data = 0x0, 
  next_rq = 0x0, 
  pad = 0x0
}
crash> struct bio ffff880188b50840
struct bio {
  bi_sector = 7, 
  bi_next = 0x0, 
  bi_bdev = 0xffff8801a9b6ac40, 
  bi_flags = 17293822569102704649, 
  bi_rw = 0, 
  bi_vcnt = 1, 
  bi_idx = 0, 
  bi_phys_segments = 1, 
  bi_size = 512, 
  bi_seg_front_size = 512, 
  bi_seg_back_size = 512, 
  bi_max_vecs = 4, 
  bi_comp_cpu = 4294967295, 
  bi_cnt = {
    counter = 1
  }, 
  bi_io_vec = 0xffff880188b508b0, 
  bi_end_io = 0xffffffffa0170810, 
  bi_private = 0xffff880188b4a7e0, 
  bi_integrity = 0x0, 
  bi_destructor = 0xffffffff811c42f0 <bio_fs_destructor>, 
  bi_inline_vecs = 0xffff880188b508b0
}
crash> struct bio_vec[4] 0xffff880188b508b0
struct: invalid data structure reference: bio_vec[4]
crash> struct bio_vec 0xffff880188b508b0
struct bio_vec {
  bv_page = 0xffffea00055e7b78, 
  bv_len = 512, 
  bv_offset = 512
}
crash> struct bio_vec 0xffff880188b508b8
struct bio_vec {
  bv_page = 0x20000000200, 
  bv_len = 0, 
  bv_offset = 0
}
crash> struct bio_vec 0xffff880188b508c0
struct bio_vec {
  bv_page = 0x0, 
  bv_len = 0, 
  bv_offset = 0
}
crash> struct bio_vec 0xffff880188b508c8
struct bio_vec {
  bv_page = 0x0, 
  bv_len = 0, 
  bv_offset = 0
}
crash>