Untitled
unknown
plain_text
4 years ago
12 kB
11
Indexable
Having second thoughts, the audit looks suspicious and I found other wonders: crash> timerx8 CPU[0][BASE_DEF]: -10075 ( -10.07 s) crash> timersx <struct timer_base 0xffff9d62eba1bd00> 0 <raw_spinlock_t 0xffff9d62eba1bd00> 0x0 .clk = 4798864083 (-10992) .next_expiry = 4798865024 (941 -10051) WARNING: timer expired for more than 10 seconds! <struct timer_list 0xffffffffa9cd0b80> -00:00:10.075000 4798865000 (-10075) ffffffffa8ace4b0 delayed_work_timer_fn (ffffffffa8c3ad00 vmstat_shepherd <struct delayed_work 0xffffffffa9cd0b40>) There seems to be a 10 seconds expired timer. crash> ba PID: 0 TASK: ffffffffa9c12780 CPU: 0 COMMAND: "swapper/0" #4 [ffffffffa9c03e88] default_idle+0x1c at ffffffffa929651c #5 [ffffffffa9c03eb0] do_idle+0x1f1 at ffffffffa8ae6841 #6 [ffffffffa9c03ef0] cpu_startup_entry+0x6f at ffffffffa8ae6aaf #7 [ffffffffa9c03f10] start_kernel+0x53b at ffffffffaa379206 #8 [ffffffffa9c03f50] secondary_startup_64+0xb7 at ffffffffa8a000e7 PID: 0 TASK: ffff9d5ec4af5f00 CPU: 1 COMMAND: "swapper/1" #2 [ffffb07d431abea8] default_idle+0x1c at ffffffffa929651c #3 [ffffb07d431abed0] do_idle+0x1f1 at ffffffffa8ae6841 #4 [ffffb07d431abf10] cpu_startup_entry+0x6f at ffffffffa8ae6aaf #5 [ffffb07d431abf30] start_secondary+0x1a7 at ffffffffa8a4ec47 #6 [ffffb07d431abf50] secondary_startup_64+0xb7 at ffffffffa8a000e7 PID: 0 TASK: ffff9d5ec4af4740 CPU: 2 COMMAND: "swapper/2" #2 [ffffb07d431b3ea8] default_idle+0x1c at ffffffffa929651c #3 [ffffb07d431b3ed0] do_idle+0x1f1 at ffffffffa8ae6841 #4 [ffffb07d431b3f10] cpu_startup_entry+0x6f at ffffffffa8ae6aaf #5 [ffffb07d431b3f30] start_secondary+0x1a7 at ffffffffa8a4ec47 #6 [ffffb07d431b3f50] secondary_startup_64+0xb7 at ffffffffa8a000e7 PID: 0 TASK: ffff9d5ec4b117c0 CPU: 3 COMMAND: "swapper/3" #3 [ffffb07d431bbea8] default_idle+0x1c at ffffffffa929651c #4 [ffffb07d431bbed0] do_idle+0x1f1 at ffffffffa8ae6841 #5 [ffffb07d431bbf10] cpu_startup_entry+0x6f at ffffffffa8ae6aaf #6 [ffffb07d431bbf30] start_secondary+0x1a7 at ffffffffa8a4ec47 #7 [ffffb07d431bbf50] secondary_startup_64+0xb7 at ffffffffa8a000e7 PID: 0 TASK: ffff9d5ec4b12f80 CPU: 4 COMMAND: "swapper/4" #3 [ffffb07d431c3ea8] default_idle+0x1c at ffffffffa929651c #4 [ffffb07d431c3ed0] do_idle+0x1f1 at ffffffffa8ae6841 #5 [ffffb07d431c3f10] cpu_startup_entry+0x6f at ffffffffa8ae6aaf #6 [ffffb07d431c3f30] start_secondary+0x1a7 at ffffffffa8a4ec47 #7 [ffffb07d431c3f50] secondary_startup_64+0xb7 at ffffffffa8a000e7 PID: 0 TASK: ffff9d5ec4b15f00 CPU: 5 COMMAND: "swapper/5" #2 [ffffb07d431cbea8] default_idle+0x1c at ffffffffa929651c #3 [ffffb07d431cbed0] do_idle+0x1f1 at ffffffffa8ae6841 #4 [ffffb07d431cbf10] cpu_startup_entry+0x6f at ffffffffa8ae6aaf #5 [ffffb07d431cbf30] start_secondary+0x1a7 at ffffffffa8a4ec47 #6 [ffffb07d431cbf50] secondary_startup_64+0xb7 at ffffffffa8a000e7 PID: 0 TASK: ffff9d5ec4b14740 CPU: 6 COMMAND: "swapper/6" #2 [ffffb07d431d3ea8] default_idle+0x1c at ffffffffa929651c #3 [ffffb07d431d3ed0] do_idle+0x1f1 at ffffffffa8ae6841 #4 [ffffb07d431d3f10] cpu_startup_entry+0x6f at ffffffffa8ae6aaf #5 [ffffb07d431d3f30] start_secondary+0x1a7 at ffffffffa8a4ec47 #6 [ffffb07d431d3f50] secondary_startup_64+0xb7 at ffffffffa8a000e7 PID: 0 TASK: ffff9d5ec4b10000 CPU: 7 COMMAND: "swapper/7" #2 [ffffb07d431dbea8] default_idle+0x1c at ffffffffa929651c #3 [ffffb07d431dbed0] do_idle+0x1f1 at ffffffffa8ae6841 #4 [ffffb07d431dbf10] cpu_startup_entry+0x6f at ffffffffa8ae6aaf #5 [ffffb07d431dbf30] start_secondary+0x1a7 at ffffffffa8a4ec47 #6 [ffffb07d431dbf50] secondary_startup_64+0xb7 at ffffffffa8a000e7 crash> dr default_idle+0x1c 0xffffffffa9296500 <default_idle>: nopl 0x0(%rax,%rax,1) [FTRACE NOP] 0xffffffffa9296505 <default_idle+0x5>: push %r13 0xffffffffa9296507 <default_idle+0x7>: push %r12 0xffffffffa9296509 <default_idle+0x9>: push %rbp 0xffffffffa929650a <default_idle+0xa>: push %rbx 0xffffffffa929650b <default_idle+0xb>: mov %gs:0x56d78c5e(%rip),%ebp # 0xf170 0xffffffffa9296512 <default_idle+0x12>: nopl 0x0(%rax,%rax,1) 0xffffffffa9296517 <default_idle+0x17>: callq 0xffffffffa92968a0 <native_safe_halt> 0xffffffffa929651c <default_idle+0x1c>: xchg %ax,%ax crash> dis native_safe_halt 0xffffffffa92968a0 <native_safe_halt>: jmpq 0xffffffffa92968ac <native_safe_halt+0xc> 0xffffffffa92968a5 <native_safe_halt+0x5>: verw 0x5728c6(%rip) # 0xffffffffa9809172 0xffffffffa92968ac <native_safe_halt+0xc>: sti 0xffffffffa92968ad <native_safe_halt+0xd>: hlt 0xffffffffa92968ae <native_safe_halt+0xe>: retq All CPUs are idle. crash> b 1449 1448 PID: 1449 TASK: ffff9d62e87a8000 CPU: 0 COMMAND: "auditd" #0 [ffffb07d43dffc38] __schedule+0x26d at ffffffffa9291abd #1 [ffffb07d43dffcd0] schedule+0x2f at ffffffffa9291edf #2 [ffffb07d43dffce0] futex_wait_queue_me+0xc1 at ffffffffa8b4ab71 #3 [ffffb07d43dffd18] futex_wait+0x13f at ffffffffa8b4bd9f #4 [ffffb07d43dffe40] do_futex+0x324 at ffffffffa8b4dfa4 #5 [ffffb07d43dffec0] __x64_sys_futex+0x143 at ffffffffa8b4e423 #6 [ffffb07d43dfff38] do_syscall_64+0x5b at ffffffffa8a0419b #7 [ffffb07d43dfff50] entry_SYSCALL_64_after_hwframe+0x65 at ffffffffa94000ad RIP: 00007f3be72ad48c RSP: 00007f3be4caac50 RFLAGS: 00000246 RAX: ffffffffffffffda RBX: 000055fab8ac4260 RCX: 00007f3be72ad48c RDX: 0000000000000000 RSI: 0000000000000080 RDI: 000055fab8ac428c RBP: 0000000000000000 R8: 0000000000000000 R9: 00007f3be4cab700 R10: 0000000000000000 R11: 0000000000000246 R12: 000055fab8ac42a0 R13: 0000000000000023 R14: 0000000000000000 R15: 000055fab8ac428c ORIG_RAX: 00000000000000ca CS: 0033 SS: 002b PID: 1448 TASK: ffff9d62e53caf80 CPU: 3 COMMAND: "auditd" #0 [ffffb07d43fd7d00] __schedule+0x26d at ffffffffa9291abd #1 [ffffb07d43fd7d98] schedule+0x2f at ffffffffa9291edf #2 [ffffb07d43fd7da8] schedule_hrtimeout_range_clock+0xba at ffffffffa9295e9a #3 [ffffb07d43fd7e40] ep_poll+0x3c8 at ffffffffa8d15358 #4 [ffffb07d43fd7e80] __audit_syscall_entry+0x103 at ffffffffa8b7b883 #5 [ffffb07d43fd7ef8] do_epoll_wait+0xb0 at ffffffffa8d15450 #6 [ffffb07d43fd7f30] __x64_sys_epoll_wait+0x1a at ffffffffa8d1548a #7 [ffffb07d43fd7f38] do_syscall_64+0x5b at ffffffffa8a0419b #8 [ffffb07d43fd7f50] entry_SYSCALL_64_after_hwframe+0x65 at ffffffffa94000ad RIP: 00007f3be6fd91b7 RSP: 00007fff3cb70930 RFLAGS: 00000293 RAX: ffffffffffffffda RBX: 0000000000000008 RCX: 00007f3be6fd91b7 RDX: 0000000000000040 RSI: 000055faba1f3d50 RDI: 0000000000000008 RBP: 000055faba1f3d50 R8: 0000000000000000 R9: 0000000000000000 R10: 000000000000e95f R11: 0000000000000293 R12: 0000000000000040 R13: 000000000000e95f R14: 00007fff3cb709e0 R15: 0000000000000000 ORIG_RAX: 00000000000000e8 CS: 0033 SS: 002b Audit or futex issue? crash> pd 0xe95f 59743 crash> files -R 0x8 1448 PID: 1448 TASK: ffff9d62e53caf80 CPU: 3 COMMAND: "auditd" ROOT: /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0 CWD: /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0 FD FILE DENTRY INODE TYPE PATH 8 ffff9d62e6bcdc00 ffff9d62e569b9c0 ffff9d62eb3d8000 UNKN [eventpoll] crash> files 1448 PID: 1448 TASK: ffff9d62e53caf80 CPU: 3 COMMAND: "auditd" ROOT: /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0 CWD: /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0 FD FILE DENTRY INODE TYPE PATH 0 ffff9d62e6bcda00 ffff9d5ec782ad80 ffff9d5ec47ac658 CHR /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/dev/null 1 ffff9d62e6bcda00 ffff9d5ec782ad80 ffff9d5ec47ac658 CHR /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/dev/null 2 ffff9d62e6bcda00 ffff9d5ec782ad80 ffff9d5ec47ac658 CHR /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/dev/null 3 ffff9d62e6bcc600 ffff9d62e569b180 ffff9d62e56e10b0 SOCK NETLINK 4 ffff9d62e84b9600 ffff9d62e544bcc0 ffff9d62e5618140 REG /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/var/lib/sss/mc/group 5 ffff9d62e84b9d00 ffff9d62e569ab40 ffff9d62e56b4df0 SOCK UNIX 7 ffff9d62e6bcd600 ffff9d62e569a480 ffff9d62e56d5200 REG /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/var/log/audit/audit.log 8 ffff9d62e6bcdc00 ffff9d62e569b9c0 ffff9d62eb3d8000 UNKN [eventpoll] 9 ffff9d62e6bccf00 ffff9d62e569bcc0 ffff9d62e56e0870 SOCK UNIX 10 ffff9d62e6bcd100 ffff9d62e569b300 ffff9d62e56e26b0 SOCK UNIX 11 ffff9d62e6bcd800 ffff9d62e569b780 ffff9d62eb3d8000 UNKN [eventfd] Audit is polling with a minute timeout. This looks all fine after all. crash> saved_command_line saved_command_line = 0xffff9d62fffce540 "BOOT_IMAGE=(hd0,gpt1)/ostree/rhcos-5c80b9e3b4870331fbbd971fa5b00d4537dabd4fa906b982665b38fcd46cfc1c/vmlinuz-4.18.0-193.56.1.el8_2.x86_64 rhcos.root=crypt_rootfs random.trust_cpu=on console=tty0 console=ttyS0,115200n8 ignition.platform.id=metal rd.luks.options=discard ostree=/ostree/boot.1/rhcos/5c80b9e3b4870331fbbd971fa5b00d4537dabd4fa906b982665b38fcd46cfc1c/0 fips=1 boot=LABEL=boot audit_backlog_limit=8192 audit=1 nousb page_poison=1 pti=on" crash> mount | grep -e DEVNAME -e root MOUNT SUPERBLK TYPE DEVNAME DIRNAME ffff9d5ec4a88f00 ffff9d5ec7c12800 rootfs rootfs / ffff9d62db21c480 ffff9d62da0b3800 xfs /dev/mapper/coreos-luks-root-nocrypt /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/sysroot ffff9d62dbb30300 ffff9d62da0b3800 xfs /dev/mapper/coreos-luks-root-nocrypt / ffff9d62dbb31800 ffff9d62da0b3800 xfs /dev/mapper/coreos-luks-root-nocrypt /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/etc ffff9d62dbb31c80 ffff9d62da0b3800 xfs /dev/mapper/coreos-luks-root-nocrypt /ostree/deploy/rhcos/deploy/00354dbe0a342dcfe4aa2cb03505ed0ea9a783963e7a2d16beae90c27115266b.0/usr The same device? Is this valid configuration?? crash> saved_command_line saved_command_line = 0xffff8bb97ffcf440 "BOOT_IMAGE=(hd0,gpt1)/ostree/rhcos-5c80b9e3b4870331fbbd971fa5b00d4537dabd4fa906b982665b38fcd46cfc1c/vmlinuz-4.18.0-193.56.1.el8_2.x86_64 rhcos.root=crypt_rootfs random.trust_cpu=on console=tty0 console=ttyS0,115200n8 ignition.platform.id=metal rd.luks.options=discard ostree=/ostree/boot.1/rhcos/5c80b9e3b4870331fbbd971fa5b00d4537dabd4fa906b982665b38fcd46cfc1c/0 fips=1 boot=LABEL=boot audit_backlog_limit=8192 audit=1 nousb page_poison=1 pti=on" crash> mount | grep -e DEVNAME -e root MOUNT SUPERBLK TYPE DEVNAME DIRNAME ffff8bb846d9d680 ffff8bb847c12800 rootfs rootfs / ffff8bb8cd6e1980 ffff8bb965c17800 xfs /dev/mapper/coreos-luks-root-nocrypt /ostree/deploy/rhcos/deploy/872ac4b0b60e1a17f7c27178088a99f199b2ccd9c0e889a1770030a30d5f51be.0/sysroot ffff8bb8cd6e0900 ffff8bb965c17800 xfs /dev/mapper/coreos-luks-root-nocrypt / ffff8bb96a773b00 ffff8bb965c17800 xfs /dev/mapper/coreos-luks-root-nocrypt /ostree/deploy/rhcos/deploy/872ac4b0b60e1a17f7c27178088a99f199b2ccd9c0e889a1770030a30d5f51be.0/etc ffff8bb96a773c80 ffff8bb965c17800 xfs /dev/mapper/coreos-luks-root-nocrypt /ostree/deploy/rhcos/deploy/872ac4b0b60e1a17f7c27178088a99f199b2ccd9c0e889a1770030a30d5f51be.0/usr The same thing with the other vmcore.
Editor is loading...