qemu-system-x86_64: qemu dead-lock when mirror job exit and vm stop in a race Description of problem: machine under continuous pressure, at the end of the migration phase. Libvirtd construction exception at the same time. In mirror_run, mirror_write_complete set s->ret to negative value and exit mirror_run; Job_co_entry throws the bh of job_exit into the main thread; While the live_migration thread gets the qemu_global_mutex, and set to main thread by blk_set_aio_context; when it just polles the bh of job_exit, and run to bdrv_flush. So there need the main thread to process bdrv_flush_co_entry, then we can exit bdrv_flush. If the main thread is waiting the qemu_mutex_lock_iothread_impl, bdrv_flush_co_entry cannot be executed, and the Live_migration thread cannot exit to release qemu_global_mutex, resulting in deadlock Steps to reproduce: 1.migrate the machine and let machine under continuous pressure; 2.gdb to qemu and make break point to virtio_blk_data_plane_stop; 3.when hit virtio_blk_data_plane_stop, kill libvirtd; 4.let live_migration thread to poll job_exit Additional information: ``` #0 0x00007f8f662d12f2 in aio_bh_poll (ctx=ctx@entry=0x5580c53a5c60) at /usr/src/qemu/util/async.c:112 #1 0x00007f8f662d58ae in aio_poll (ctx=0x5580c53a5c60, blocking=blocking@entry=true) at /usr/src/qemu/util/aio-posix.c:736 #2 0x00007f8f6530bcca in bdrv_flush (bs=bs@entry=0x5580c5857b30) at /usr/src/qemu/block/io.c:2778 #3 0x00007f8f65345143 in bdrv_close (bs=bs@entry=0x5580c5857b30) at /usr/src/qemu/block.c:4073 #4 0x00007f8f65345373 in bdrv_delete (bs=0x5580c5857b30) at /usr/src/qemu/block.c:4335 #5 0x00007f8f65345405 in bdrv_unref (bs=) at /usr/src/qemu/block.c:5676 #6 0x00007f8f65344d95 in bdrv_root_unref_child (child=) at /usr/src/qemu/block.c:2516 #7 0x00007f8f65353f56 in block_job_remove_all_bdrv (job=job@entry=0x5580c6d55cc0) at /usr/src/qemu/blockjob.c:203 #8 0x00007f8f65317b87 in mirror_exit_common (job=0x5580c6d55cc0) at /usr/src/qemu/block/mirror.c:776 #9 0x00007f8f65317cc8 in mirror_abort (job=) at /usr/src/qemu/block/mirror.c:804 #10 0x00007f8f6632737b in job_finalize_single (job=job@entry=0x5580c6d55cc0) at /usr/src/qemu/job.c:680 #11 0x00007f8f66327d70 in job_completed_txn_abort (job=) at /usr/src/qemu/job.c:758 #12 0x00007f8f66328018 in job_exit (opaque=0x5580c6d55cc0) at /usr/src/qemu/job.c:873 #13 0x00007f8f662d130f in aio_bh_poll (ctx=ctx@entry=0x5580c53a5c60) at /usr/src/qemu/util/async.c:118 #14 0x00007f8f662d5716 in aio_poll (ctx=ctx@entry=0x5580c53a5c60, blocking=blocking@entry=true) at /usr/src/qemu/util/aio-posix.c:736 #15 0x00007f8f662e6b4d in aio_wait_bh_oneshot (ctx=0x5580c53a5c60, cb=, opaque=) at /usr/src/qemu/util/aio-wait.c:71 #16 0x00007f8f65340978 in bdrv_attach_aio_context (bs=bs@entry=0x5580c5a07ef0, new_context=new_context@entry=0x5580c53a5c60) at /usr/src/qemu/block.c:5985 #17 0x00007f8f65345fd5 in bdrv_set_aio_context_ignore (bs=0x5580c5a07ef0, new_context=new_context@entry=0x5580c53a5c60, ignore=ignore@entry=0x7f8eb8ff8c20) at /usr/src/qemu/block.c:6050 #18 0x00007f8f6534609e in bdrv_set_aio_context_ignore (bs=0x5580c5857b30, new_context=new_context@entry=0x5580c53a5c60, ignore=ignore@entry=0x7f8eb8ff8c20) at /usr/src/qemu/block.c:6032 #19 0x00007f8f65353bd4 in child_job_set_aio_ctx (c=, ctx=0x5580c53a5c60, ignore=0x7f8eb8ff8c20) at /usr/src/qemu/blockjob.c:172 #20 0x00007f8f6534604b in bdrv_set_aio_context_ignore (bs=0x5580c53c46c0, new_context=new_context@entry=0x5580c53a5c60, ignore=ignore@entry=0x7f8eb8ff8c20) at /usr/src/qemu/block.c:6040 #21 0x00007f8f6534609e in bdrv_set_aio_context_ignore (bs=bs@entry=0x5580c5978290, new_context=new_context@entry=0x5580c53a5c60, ignore=ignore@entry=0x7f8eb8ff8c20) at /usr/src/qemu/block.c:6032 #22 0x00007f8f653462b8 in bdrv_child_try_set_aio_context (bs=bs@entry=0x5580c5978290, ctx=ctx@entry=0x5580c53a5c60, ignore_child=, errp=errp@entry=0x0) at /usr/src/qemu/block.c:6145 #23 0x00007f8f653029aa in blk_do_set_aio_context (blk=0x5580c53c42b0, new_context=0x5580c53a5c60, update_root_node=update_root_node@entry=true, errp=errp@entry=0x0) at /usr/src/qemu/block/block-backend.c:1948 #24 0x00007f8f65304b0d in blk_set_aio_context (blk=, new_context=, errp=errp@entry=0x0) at /usr/src/qemu/block/block-backend.c:1980 #25 0x00007f8f64f07976 in virtio_blk_data_plane_stop (vdev=0x5580c6d8a510) at /usr/src/qemu/hw/block/dataplane/virtio-blk.c:305 #26 0x00007f8f64f7be83 in virtio_bus_stop_ioeventfd (bus=0x5580c6d8a498) at /usr/src/qemu/hw/virtio/virtio-bus.c:247 #27 0x00007f8f64f77e8b in virtio_vmstate_change (opaque=0x5580c6d8a510, running=0, state=RUN_STATE_FINISH_MIGRATE) at /usr/src/qemu/hw/virtio/virtio.c:2423 #28 0x00007f8f663563f5 in vm_state_notify (running=running@entry=0, state=state@entry=RUN_STATE_FINISH_MIGRATE) at /usr/src/qemu/huawei/microvm/microvm-platform.c:196 #29 0x00007f8f66335af9 in do_vm_stop (state=RUN_STATE_FINISH_MIGRATE, send_stop=send_stop@entry=true) at /usr/src/qemu/cpus.c:1130 #30 0x00007f8f66335dd1 in vm_stop (state=) at /usr/src/qemu/cpus.c:2207 #31 0x00007f8f66335f7e in vm_stop_force_state (state=state@entry=RUN_STATE_FINISH_MIGRATE) at /usr/src/qemu/cpus.c:2267 #32 0x00007f8f65197cfc in migration_try_vm_stop_and_save_concurrent (s=s@entry=0x5580c609a010) at /usr/src/qemu/migration/migration.c:2976 #33 0x00007f8f6519c627 in migration_completion (s=s@entry=0x5580c609a010) at /usr/src/qemu/migration/migration.c:3039 #34 0x00007f8f6519cc8b in migration_iteration_run (s=s@entry=0x5580c609a010) at /usr/src/qemu/migration/migration.c:3571 #35 0x00007f8f6519d190 in migration_thread (opaque=0x5580c609a010) at /usr/src/qemu/migration/migration.c:3801 #36 0x00007f8f662d82e0 in qemu_thread_start (args=0x5580c57d0300) at /usr/src/qemu/util/qemu-thread-posix.c:519 #37 0x00007f8f6648bf3b in start_thread (arg=0x7f8eb8ff9700) at pthread_create.c:486 ```