other: 0.654 KVM: 0.620 permissions: 0.577 vnc: 0.572 boot: 0.542 debug: 0.542 semantic: 0.538 graphic: 0.533 performance: 0.521 device: 0.517 PID: 0.517 socket: 0.492 network: 0.483 files: 0.420 Combining direct_io and non-direct_io leads to hang Version 0.12.2 - I know this isn't the latest but I looked through the changelogs and couldn't find any references to this being fixed. This is related to issues that many apps have, and is described in more detail here: http://oss.sgi.com/archives/xfs/2010-07/msg00163.html and: https://bugs.launchpad.net/percona-xtrabackup/+bug/606981 When using both direct io and buffered io on the same file, you can cause corruption on the filesystem and other issues. XFS out right hangs, but the problem exists on ext3 and other filesystems which silently carry on. **this is a data corruption issue**. This is the full stack trace we got from a recent hang: Feb 22 19:55:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 19:55:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 19:55:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 19:55:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 19:55:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 19:55:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 19:55:14 virt11 kernel: Call Trace: Feb 22 19:55:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 19:55:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 19:55:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 19:55:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 19:55:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 19:55:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 19:55:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 19:55:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 19:55:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 19:55:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 19:55:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 19:55:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 19:55:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 19:55:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 19:55:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 19:55:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 19:55:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 19:55:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 19:55:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 19:55:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 19:55:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 19:55:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 19:55:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 19:55:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 19:55:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 19:55:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 19:55:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 19:55:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 19:55:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 19:57:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 19:57:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 19:57:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 19:57:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 19:57:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 19:57:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 19:57:14 virt11 kernel: Call Trace: Feb 22 19:57:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 19:57:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 19:57:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 19:57:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 19:57:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 19:57:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 19:57:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 19:57:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 19:57:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 19:57:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 19:57:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 19:57:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 19:57:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 19:57:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 19:57:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 19:57:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 19:57:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 19:57:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 19:57:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 19:57:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 19:57:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 19:57:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 19:57:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 19:57:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 19:57:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 19:57:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 19:57:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 19:57:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 19:57:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 19:59:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 19:59:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 19:59:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 19:59:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 19:59:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 19:59:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 19:59:14 virt11 kernel: Call Trace: Feb 22 19:59:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 19:59:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 19:59:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 19:59:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 19:59:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 19:59:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 19:59:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 19:59:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 19:59:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 19:59:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 19:59:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 19:59:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 19:59:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 19:59:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 19:59:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 19:59:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 19:59:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 19:59:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 19:59:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 19:59:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 19:59:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 19:59:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 19:59:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 19:59:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 19:59:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 19:59:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 19:59:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 19:59:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 19:59:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 20:01:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 20:01:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 20:01:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 20:01:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 20:01:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 20:01:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 20:01:14 virt11 kernel: Call Trace: Feb 22 20:01:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 20:01:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 20:01:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 20:01:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 20:01:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 20:01:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 20:01:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 20:01:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 20:01:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 20:01:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 20:01:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 20:01:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 20:01:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 20:01:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 20:01:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 20:01:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 20:01:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 20:01:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 20:01:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 20:01:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 20:01:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 20:01:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 20:01:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 20:01:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 20:01:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 20:01:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 20:01:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 20:01:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 20:01:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 20:03:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 20:03:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 20:03:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 20:03:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 20:03:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 20:03:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 20:03:14 virt11 kernel: Call Trace: Feb 22 20:03:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 20:03:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 20:03:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 20:03:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 20:03:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 20:03:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 20:03:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 20:03:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 20:03:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 20:03:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 20:03:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 20:03:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 20:03:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 20:03:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 20:03:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 20:03:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 20:03:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 20:03:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 20:03:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 20:03:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 20:03:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 20:03:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 20:03:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 20:03:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 20:03:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 20:03:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 20:03:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 20:03:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 20:03:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 20:05:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 20:05:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 20:05:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 20:05:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 20:05:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 20:05:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 20:05:14 virt11 kernel: Call Trace: Feb 22 20:05:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 20:05:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 20:05:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 20:05:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 20:05:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 20:05:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 20:05:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 20:05:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 20:05:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 20:05:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 20:05:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 20:05:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 20:05:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 20:05:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 20:05:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 20:05:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 20:05:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 20:05:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 20:05:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 20:05:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 20:05:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 20:05:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 20:05:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 20:05:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 20:05:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 20:05:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 20:05:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 20:05:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 20:05:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 20:07:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 20:07:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 20:07:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 20:07:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 20:07:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 20:07:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 20:07:14 virt11 kernel: Call Trace: Feb 22 20:07:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 20:07:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 20:07:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 20:07:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 20:07:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 20:07:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 20:07:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 20:07:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 20:07:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 20:07:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 20:07:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 20:07:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 20:07:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 20:07:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 20:07:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 20:07:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 20:07:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 20:07:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 20:07:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 20:07:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 20:07:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 20:07:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 20:07:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 20:07:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 20:07:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 20:07:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 20:07:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 20:07:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 20:07:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 20:09:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 20:09:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 20:09:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 20:09:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 20:09:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 20:09:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 20:09:14 virt11 kernel: Call Trace: Feb 22 20:09:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 20:09:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 20:09:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 20:09:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 20:09:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 20:09:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 20:09:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 20:09:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 20:09:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 20:09:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 20:09:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 20:09:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 20:09:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 20:09:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 20:09:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 20:09:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 20:09:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 20:09:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 20:09:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 20:09:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 20:09:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 20:09:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 20:09:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 20:09:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 20:09:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 20:09:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 20:09:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 20:09:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 20:09:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 20:11:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 20:11:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 20:11:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 20:11:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 20:11:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 20:11:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 20:11:14 virt11 kernel: Call Trace: Feb 22 20:11:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 20:11:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 20:11:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 20:11:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 20:11:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 20:11:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 20:11:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 20:11:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 20:11:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 20:11:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 20:11:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 20:11:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 20:11:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 20:11:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 20:11:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 20:11:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 20:11:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 20:11:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 20:11:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 20:11:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 20:11:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 20:11:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 20:11:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 20:11:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 20:11:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 20:11:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 20:11:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 20:11:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 20:11:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b Feb 22 20:13:14 virt11 kernel: INFO: task qemu-kvm:18360 blocked for more than 120 seconds. Feb 22 20:13:14 virt11 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. Feb 22 20:13:14 virt11 kernel: qemu-kvm D 0000000000000002 0 18360 1 0x00000080 Feb 22 20:13:14 virt11 kernel: ffff8808614457e8 0000000000000086 ffff8800282d5f80 ffff8800282d5f80 Feb 22 20:13:14 virt11 kernel: ffff880861445768 ffffffff810573ce ffff880861445768 0000000000000086 Feb 22 20:13:14 virt11 kernel: ffff8808f15bba78 ffff880861445fd8 000000000000f598 ffff8808f15bba78 Feb 22 20:13:14 virt11 kernel: Call Trace: Feb 22 20:13:14 virt11 kernel: [] ? activate_task+0x2e/0x40 Feb 22 20:13:14 virt11 kernel: [] rwsem_down_failed_common+0x95/0x1d0 Feb 22 20:13:14 virt11 kernel: [] rwsem_down_write_failed+0x23/0x30 Feb 22 20:13:14 virt11 kernel: [] call_rwsem_down_write_failed+0x13/0x20 Feb 22 20:13:14 virt11 kernel: [] ? down_write+0x32/0x40 Feb 22 20:13:14 virt11 kernel: [] xfs_ilock+0x7e/0xd0 [xfs] Feb 22 20:13:14 virt11 kernel: [] xfs_iomap+0x2e2/0x440 [xfs] Feb 22 20:13:14 virt11 kernel: [] __xfs_get_blocks+0x86/0x200 [xfs] Feb 22 20:13:14 virt11 kernel: [] xfs_get_blocks_direct+0x1a/0x20 [xfs] Feb 22 20:13:14 virt11 kernel: [] __blockdev_direct_IO+0x872/0xc40 Feb 22 20:13:14 virt11 kernel: [] xfs_vm_direct_IO+0xb0/0xf0 [xfs] Feb 22 20:13:14 virt11 kernel: [] ? xfs_get_blocks_direct+0x0/0x20 [xfs] Feb 22 20:13:14 virt11 kernel: [] ? xfs_end_io_direct+0x0/0xe0 [xfs] Feb 22 20:13:14 virt11 kernel: [] ? current_fs_time+0x27/0x30 Feb 22 20:13:14 virt11 kernel: [] generic_file_direct_write+0xc2/0x190 Feb 22 20:13:14 virt11 kernel: [] ? xfs_trans_unlocked_item+0x4f/0x60 [xfs] Feb 22 20:13:14 virt11 kernel: [] xfs_write+0x4fd/0xb70 [xfs] Feb 22 20:13:14 virt11 kernel: [] ? default_wake_function+0x12/0x20 Feb 22 20:13:14 virt11 kernel: [] xfs_file_aio_write+0x61/0x70 [xfs] Feb 22 20:13:14 virt11 kernel: [] do_sync_write+0xfa/0x140 Feb 22 20:13:14 virt11 kernel: [] ? send_signal+0x42/0x80 Feb 22 20:13:14 virt11 kernel: [] ? autoremove_wake_function+0x0/0x40 Feb 22 20:13:14 virt11 kernel: [] ? group_send_sig_info+0x56/0x70 Feb 22 20:13:14 virt11 kernel: [] ? selinux_file_permission+0xfb/0x150 Feb 22 20:13:14 virt11 kernel: [] ? security_file_permission+0x16/0x20 Feb 22 20:13:14 virt11 kernel: [] vfs_write+0xb8/0x1a0 Feb 22 20:13:14 virt11 kernel: [] ? audit_syscall_entry+0x272/0x2a0 Feb 22 20:13:14 virt11 kernel: [] sys_pwrite64+0x82/0xa0 Feb 22 20:13:14 virt11 kernel: [] system_call_fastpath+0x16/0x1b On Wed, Feb 22, 2012 at 09:49:59PM -0000, Avleen Vig wrote: > Version 0.12.2 - I know this isn't the latest but I looked through the > changelogs and couldn't find any references to this being fixed. > > This is related to issues that many apps have, and is described in more detail here: > http://oss.sgi.com/archives/xfs/2010-07/msg00163.html > and: > https://bugs.launchpad.net/percona-xtrabackup/+bug/606981 > > When using both direct io and buffered io on the same file, you can cause corruption on the filesystem and other issues. > XFS out right hangs, but the problem exists on ext3 and other filesystems which silently carry on. > > **this is a data corruption issue**. > > This is the full stack trace we got from a recent hang: I only see qemu-kvm in the backtrace. QEMU normally only access the image file either with or without O_DIRECT - not both at the same time. Were you running another program that accesses the image file at the same time? Stefan Nothing else at all accesses those files, which is one of the things that concerned me the most :-/ It looks like it call came from the same place. On Thu, Feb 23, 2012 at 9:39 PM, Avleen Vig