qemu 4.0/4.1 segfault on live migrate with virtio-scsi iothread

[root@kvm-nvme5 qemu]# uname -a
Linux kvm-nvme5 4.14.35-1902.4.8.el7uek.x86_64 #2 SMP Sun Aug 4 22:25:18 GMT 2019 x86_64 x86_64 x86_64 GNU/Linux

[root@kvm-nvme5 qemu]# qemu-system-x86_64 --version
QEMU emulator version 4.1.0 (qemu-4.1.0-1.el7)
Copyright (c) 2003-2019 Fabrice Bellard and the QEMU Project developers

[root@kvm-nvme5 qemu]# libvirtd --version
libvirtd (libvirt) 5.6.0

when migrate 
MIGR_OPTS="--live --copy-storage-all --verbose --persistent --undefinesource"
virsh migrate $MIGR_OPTS p12345 qemu+ssh://$SERV/system

we got segfault if we have option <driver iothread='1'/> in config for virtio-scsi controller

[1205674.818067] qemu-system-x86[39744]: segfault at 38 ip 00005575890ad411 sp 00007ffd3c10a0e0 error 6 in qemu-system-x86_64[5575889ad000+951000]

On 4.0 we have error with this context(dont save all output) "qemu_coroutine_get_aio_context(co)' failed"

If we remove option 
<driver iothread='1'/>
migrate work fine without segfaults

2019-08-30 08:25:35.402+0000: starting up libvirt version: 5.6.0, package: 1.el7 (Unknown, 2019-08-06-09:57:56, mock), qemu version: 4.1.0qemu-4.1.0-1.el7, kernel: 4.14.35-1902.4.8.el7uek.x86_64, hostname: kvm-nvme5
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-75-p541999 \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.config \
QEMU_AUDIO_DRV=none \
/usr/bin/qemu-system-x86_64 \
-name guest=p541999,debug-threads=on \
-S \
-object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-75-p541999/master-key.aes \
-machine pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off \
-cpu Cascadelake-Server,ss=on,hypervisor=on,tsc-adjust=on,umip=on,pku=on,md-clear=on,stibp=on,arch-capabilities=on,xsaves=on,rdctl-no=on,ibrs-all=on,skip-l1dfl-vmentry=on,mds-no=on,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1000,hv-vpindex,hv-runtime,hv-synic,hv-stimer,hv-fre
quencies,hv-reenlightenment,hv-tlbflush \
-m 2148 \
-overcommit mem-lock=off \
-smp 1,sockets=1,cores=1,threads=1 \
-object iothread,id=iothread1 \
-uuid ff20ae7f-8cfe-4ec5-bd50-e78f8a167414 \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=44,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-shutdown \
-boot menu=on,strict=on \
-device ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x5.0x7 \
-device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x5 \
-device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x5.0x1 \
-device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x5.0x2 \
-device virtio-scsi-pci,iothread=iothread1,id=scsi0,bus=pcie.0,addr=0x9 \
-device virtio-serial-pci,id=virtio-serial0,bus=pcie.0,addr=0x6 \
-drive file=/dev/vm/p541999,format=raw,if=none,id=drive-scsi0-0-0-0,cache=none,discard=unmap,aio=threads,throttling.bps-write=52428800,throttling.bps-write-max=314572800,throttling.bps-write-max-length=120 \
-device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,device_id=drive-scsi0-0-0-0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=2,write-cache=on \
-drive if=none,id=drive-sata0-0-0,readonly=on \
-device ide-cd,bus=ide.0,drive=drive-sata0-0-0,id=sata0-0-0,bootindex=1 \
-netdev tap,fd=47,id=hostnet0,vhost=on,vhostfd=48 \
-device virtio-net-pci,netdev=hostnet0,id=net0,mac=00:00:00:54:19:99,bus=pcie.0,addr=0x3 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-chardev socket,id=charchannel0,fd=49,server,nowait \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 \
-vnc 0.0.0.0:6128,password \
-device cirrus-vga,id=video0,bus=pcie.0,addr=0x1 \
-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x8 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
char device redirected to /dev/pts/5 (label charserial0)
2019-08-30 08:27:00.539+0000: shutting down, reason=crashed


config:
<domain type='kvm'>
  <name>p541999</name>
  <uuid>ff20ae7f-8cfe-4ec5-bd50-e78f8a167414</uuid>
  <memory unit='KiB'>2199552</memory>
  <currentMemory unit='KiB'>2199552</currentMemory>
  <vcpu placement='static'>1</vcpu>
  <iothreads>1</iothreads>
  <resource>
    <partition>/machine</partition>
  </resource>
  <os>
    <type arch='x86_64' machine='pc-q35-4.0'>hvm</type>
    <boot dev='cdrom'/>
    <boot dev='hd'/>
    <bootmenu enable='yes'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <pae/>
    <hyperv>
      <relaxed state='on'/>
      <vapic state='on'/>
      <spinlocks state='on' retries='4096'/>
      <vpindex state='on'/>
      <runtime state='on'/>
      <synic state='on'/>
      <stimer state='on'/>
      <frequencies state='on'/>
      <reenlightenment state='on'/>
      <tlbflush state='on'/>
    </hyperv>
    <msrs unknown='ignore'/>
  </features>
  <cpu mode='host-model' check='full'>
    <model fallback='forbid'/>
  </cpu>
  <clock offset='utc'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='yes'/>
    <timer name='hypervclock' present='yes'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='none' io='threads' discard='unmap'/>
      <source dev='/dev/vm/p541999'/>
      <backingStore/>
      <target dev='sda' bus='scsi'/>
      <iotune>
        <write_bytes_sec>52428800</write_bytes_sec>
        <write_bytes_sec_max>314572800</write_bytes_sec_max>
        <write_bytes_sec_max_length>120</write_bytes_sec_max_length>
      </iotune>
      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
    </disk>
    <disk type='file' device='cdrom'>
      <driver name='qemu' type='raw'/>
      <target dev='sdb' bus='sata'/>
      <readonly/>
      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
    </disk>
    <controller type='usb' index='0' model='ich9-ehci1'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x7'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci1'>
      <master startport='0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci2'>
      <master startport='2'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci3'>
      <master startport='4'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x2'/>
    </controller>
    <controller type='virtio-serial' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </controller>
    <controller type='scsi' index='0' model='virtio-scsi'>
      <driver iothread='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
    </controller>
    <controller type='pci' index='0' model='pcie-root'/>
    <controller type='sata' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
    </controller>
    <interface type='bridge'>
      <mac address='00:00:00:54:19:99'/>
      <source bridge='br0'/>
      <bandwidth>
        <inbound average='12500' peak='12500' burst='1024'/>
        <outbound average='12500' peak='12500' burst='1024'/>
      </bandwidth>
      <model type='virtio'/>
      <filterref filter='clean-traffic'>
        <parameter name='CTRL_IP_LEARNING' value='none'/>
        <parameter name='IP' value='1.2.3.4'/>
      </filterref>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <channel type='unix'>
      <source mode='bind' path='/var/lib/libvirt/qemu/p541999.agent'/>
      <target type='virtio' name='org.qemu.guest_agent.0'/>
      <address type='virtio-serial' controller='0' bus='0' port='1'/>
    </channel>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <graphics type='vnc' port='12028' autoport='no' listen='0.0.0.0' passwd='SUPERPASSWORD'>
      <listen type='address' address='0.0.0.0'/>
    </graphics>
    <video>
      <model type='cirrus' vram='16384' heads='1' primary='yes'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
    </memballoon>
  </devices>
  <seclabel type='none' model='none'/>
</domain>

On Fri, Aug 30, 2019 at 08:44:35AM -0000, Dmitriy wrote:
> Public bug reported:

Thanks for reporting this.  There have been fixes for virtio-scsi with
iothreads lately, but QEMU 4.1.0 is fairly recent so it should have most
of them.

Can you try qemu.git/master?

> 
> [root@kvm-nvme5 qemu]# uname -a
> Linux kvm-nvme5 4.14.35-1902.4.8.el7uek.x86_64 #2 SMP Sun Aug 4 22:25:18 GMT 2019 x86_64 x86_64 x86_64 GNU/Linux
> 
> [root@kvm-nvme5 qemu]# qemu-system-x86_64 --version
> QEMU emulator version 4.1.0 (qemu-4.1.0-1.el7)
> Copyright (c) 2003-2019 Fabrice Bellard and the QEMU Project developers
> 
> [root@kvm-nvme5 qemu]# libvirtd --version
> libvirtd (libvirt) 5.6.0
> 
> when migrate 
> MIGR_OPTS="--live --copy-storage-all --verbose --persistent --undefinesource"
> virsh migrate $MIGR_OPTS p12345 qemu+ssh://$SERV/system
> 
> we got segfault if we have option <driver iothread='1'/> in config for
> virtio-scsi controller
> 
> [1205674.818067] qemu-system-x86[39744]: segfault at 38 ip
> 00005575890ad411 sp 00007ffd3c10a0e0 error 6 in qemu-system-
> x86_64[5575889ad000+951000]
> 
> On 4.0 we have error with this context(dont save all output)
> "qemu_coroutine_get_aio_context(co)' failed"
> 
> If we remove option 
> <driver iothread='1'/>
> migrate work fine without segfaults
> 
> 2019-08-30 08:25:35.402+0000: starting up libvirt version: 5.6.0, package: 1.el7 (Unknown, 2019-08-06-09:57:56, mock), qemu version: 4.1.0qemu-4.1.0-1.el7, kernel: 4.14.35-1902.4.8.el7uek.x86_64, hostname: kvm-nvme5
> LC_ALL=C \
> PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
> HOME=/var/lib/libvirt/qemu/domain-75-p541999 \
> XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.local/share \
> XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.cache \
> XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.config \
> QEMU_AUDIO_DRV=none \
> /usr/bin/qemu-system-x86_64 \
> -name guest=p541999,debug-threads=on \
> -S \
> -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-75-p541999/master-key.aes \
> -machine pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off \
> -cpu Cascadelake-Server,ss=on,hypervisor=on,tsc-adjust=on,umip=on,pku=on,md-clear=on,stibp=on,arch-capabilities=on,xsaves=on,rdctl-no=on,ibrs-all=on,skip-l1dfl-vmentry=on,mds-no=on,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1000,hv-vpindex,hv-runtime,hv-synic,hv-stimer,hv-fre
> quencies,hv-reenlightenment,hv-tlbflush \
> -m 2148 \
> -overcommit mem-lock=off \
> -smp 1,sockets=1,cores=1,threads=1 \
> -object iothread,id=iothread1 \
> -uuid ff20ae7f-8cfe-4ec5-bd50-e78f8a167414 \
> -no-user-config \
> -nodefaults \
> -chardev socket,id=charmonitor,fd=44,server,nowait \
> -mon chardev=charmonitor,id=monitor,mode=control \
> -rtc base=utc,driftfix=slew \
> -global kvm-pit.lost_tick_policy=delay \
> -no-shutdown \
> -boot menu=on,strict=on \
> -device ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x5.0x7 \
> -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x5 \
> -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x5.0x1 \
> -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x5.0x2 \
> -device virtio-scsi-pci,iothread=iothread1,id=scsi0,bus=pcie.0,addr=0x9 \
> -device virtio-serial-pci,id=virtio-serial0,bus=pcie.0,addr=0x6 \
> -drive file=/dev/vm/p541999,format=raw,if=none,id=drive-scsi0-0-0-0,cache=none,discard=unmap,aio=threads,throttling.bps-write=52428800,throttling.bps-write-max=314572800,throttling.bps-write-max-length=120 \
> -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,device_id=drive-scsi0-0-0-0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=2,write-cache=on \
> -drive if=none,id=drive-sata0-0-0,readonly=on \
> -device ide-cd,bus=ide.0,drive=drive-sata0-0-0,id=sata0-0-0,bootindex=1 \
> -netdev tap,fd=47,id=hostnet0,vhost=on,vhostfd=48 \
> -device virtio-net-pci,netdev=hostnet0,id=net0,mac=00:00:00:54:19:99,bus=pcie.0,addr=0x3 \
> -chardev pty,id=charserial0 \
> -device isa-serial,chardev=charserial0,id=serial0 \
> -chardev socket,id=charchannel0,fd=49,server,nowait \
> -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 \
> -vnc 0.0.0.0:6128,password \
> -device cirrus-vga,id=video0,bus=pcie.0,addr=0x1 \
> -device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x8 \
> -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
> -msg timestamp=on
> char device redirected to /dev/pts/5 (label charserial0)
> 2019-08-30 08:27:00.539+0000: shutting down, reason=crashed
> 
> 
> config:
> <domain type='kvm'>
>   <name>p541999</name>
>   <uuid>ff20ae7f-8cfe-4ec5-bd50-e78f8a167414</uuid>
>   <memory unit='KiB'>2199552</memory>
>   <currentMemory unit='KiB'>2199552</currentMemory>
>   <vcpu placement='static'>1</vcpu>
>   <iothreads>1</iothreads>
>   <resource>
>     <partition>/machine</partition>
>   </resource>
>   <os>
>     <type arch='x86_64' machine='pc-q35-4.0'>hvm</type>
>     <boot dev='cdrom'/>
>     <boot dev='hd'/>
>     <bootmenu enable='yes'/>
>   </os>
>   <features>
>     <acpi/>
>     <apic/>
>     <pae/>
>     <hyperv>
>       <relaxed state='on'/>
>       <vapic state='on'/>
>       <spinlocks state='on' retries='4096'/>
>       <vpindex state='on'/>
>       <runtime state='on'/>
>       <synic state='on'/>
>       <stimer state='on'/>
>       <frequencies state='on'/>
>       <reenlightenment state='on'/>
>       <tlbflush state='on'/>
>     </hyperv>
>     <msrs unknown='ignore'/>
>   </features>
>   <cpu mode='host-model' check='full'>
>     <model fallback='forbid'/>
>   </cpu>
>   <clock offset='utc'>
>     <timer name='rtc' tickpolicy='catchup'/>
>     <timer name='pit' tickpolicy='delay'/>
>     <timer name='hpet' present='yes'/>
>     <timer name='hypervclock' present='yes'/>
>   </clock>
>   <on_poweroff>destroy</on_poweroff>
>   <on_reboot>restart</on_reboot>
>   <on_crash>restart</on_crash>
>   <devices>
>     <emulator>/usr/bin/qemu-system-x86_64</emulator>
>     <disk type='block' device='disk'>
>       <driver name='qemu' type='raw' cache='none' io='threads' discard='unmap'/>
>       <source dev='/dev/vm/p541999'/>
>       <backingStore/>
>       <target dev='sda' bus='scsi'/>
>       <iotune>
>         <write_bytes_sec>52428800</write_bytes_sec>
>         <write_bytes_sec_max>314572800</write_bytes_sec_max>
>         <write_bytes_sec_max_length>120</write_bytes_sec_max_length>
>       </iotune>
>       <address type='drive' controller='0' bus='0' target='0' unit='0'/>
>     </disk>
>     <disk type='file' device='cdrom'>
>       <driver name='qemu' type='raw'/>
>       <target dev='sdb' bus='sata'/>
>       <readonly/>
>       <address type='drive' controller='0' bus='0' target='0' unit='0'/>
>     </disk>
>     <controller type='usb' index='0' model='ich9-ehci1'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x7'/>
>     </controller>
>     <controller type='usb' index='0' model='ich9-uhci1'>
>       <master startport='0'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
>     </controller>
>     <controller type='usb' index='0' model='ich9-uhci2'>
>       <master startport='2'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
>     </controller>
>     <controller type='usb' index='0' model='ich9-uhci3'>
>       <master startport='4'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x2'/>
>     </controller>
>     <controller type='virtio-serial' index='0'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
>     </controller>
>     <controller type='scsi' index='0' model='virtio-scsi'>
>       <driver iothread='1'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
>     </controller>
>     <controller type='pci' index='0' model='pcie-root'/>
>     <controller type='sata' index='0'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
>     </controller>
>     <interface type='bridge'>
>       <mac address='00:00:00:54:19:99'/>
>       <source bridge='br0'/>
>       <bandwidth>
>         <inbound average='12500' peak='12500' burst='1024'/>
>         <outbound average='12500' peak='12500' burst='1024'/>
>       </bandwidth>
>       <model type='virtio'/>
>       <filterref filter='clean-traffic'>
>         <parameter name='CTRL_IP_LEARNING' value='none'/>
>         <parameter name='IP' value='1.2.3.4'/>
>       </filterref>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
>     </interface>
>     <serial type='pty'>
>       <target type='isa-serial' port='0'>
>         <model name='isa-serial'/>
>       </target>
>     </serial>
>     <console type='pty'>
>       <target type='serial' port='0'/>
>     </console>
>     <channel type='unix'>
>       <source mode='bind' path='/var/lib/libvirt/qemu/p541999.agent'/>
>       <target type='virtio' name='org.qemu.guest_agent.0'/>
>       <address type='virtio-serial' controller='0' bus='0' port='1'/>
>     </channel>
>     <input type='mouse' bus='ps2'/>
>     <input type='keyboard' bus='ps2'/>
>     <graphics type='vnc' port='12028' autoport='no' listen='0.0.0.0' passwd='SUPERPASSWORD'>
>       <listen type='address' address='0.0.0.0'/>
>     </graphics>
>     <video>
>       <model type='cirrus' vram='16384' heads='1' primary='yes'/>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
>     </video>
>     <memballoon model='virtio'>
>       <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
>     </memballoon>
>   </devices>
>   <seclabel type='none' model='none'/>
> </domain>
> 
> ** Affects: qemu
>      Importance: Undecided
>          Status: New
> 
> -- 
> You received this bug notification because you are a member of qemu-
> devel-ml, which is subscribed to QEMU.
> https://bugs.launchpad.net/bugs/1842038
> 
> Title:
>   qemu 4.0/4.1 segfault on live migrate with virtio-scsi iothread
> 
> Status in QEMU:
>   New
> 
> Bug description:
>   [root@kvm-nvme5 qemu]# uname -a
>   Linux kvm-nvme5 4.14.35-1902.4.8.el7uek.x86_64 #2 SMP Sun Aug 4 22:25:18 GMT 2019 x86_64 x86_64 x86_64 GNU/Linux
> 
>   [root@kvm-nvme5 qemu]# qemu-system-x86_64 --version
>   QEMU emulator version 4.1.0 (qemu-4.1.0-1.el7)
>   Copyright (c) 2003-2019 Fabrice Bellard and the QEMU Project developers
> 
>   [root@kvm-nvme5 qemu]# libvirtd --version
>   libvirtd (libvirt) 5.6.0
> 
>   when migrate 
>   MIGR_OPTS="--live --copy-storage-all --verbose --persistent --undefinesource"
>   virsh migrate $MIGR_OPTS p12345 qemu+ssh://$SERV/system
> 
>   we got segfault if we have option <driver iothread='1'/> in config for
>   virtio-scsi controller
> 
>   [1205674.818067] qemu-system-x86[39744]: segfault at 38 ip
>   00005575890ad411 sp 00007ffd3c10a0e0 error 6 in qemu-system-
>   x86_64[5575889ad000+951000]
> 
>   On 4.0 we have error with this context(dont save all output)
>   "qemu_coroutine_get_aio_context(co)' failed"
> 
>   If we remove option 
>   <driver iothread='1'/>
>   migrate work fine without segfaults
> 
>   2019-08-30 08:25:35.402+0000: starting up libvirt version: 5.6.0, package: 1.el7 (Unknown, 2019-08-06-09:57:56, mock), qemu version: 4.1.0qemu-4.1.0-1.el7, kernel: 4.14.35-1902.4.8.el7uek.x86_64, hostname: kvm-nvme5
>   LC_ALL=C \
>   PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
>   HOME=/var/lib/libvirt/qemu/domain-75-p541999 \
>   XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.local/share \
>   XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.cache \
>   XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-75-p541999/.config \
>   QEMU_AUDIO_DRV=none \
>   /usr/bin/qemu-system-x86_64 \
>   -name guest=p541999,debug-threads=on \
>   -S \
>   -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-75-p541999/master-key.aes \
>   -machine pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off \
>   -cpu Cascadelake-Server,ss=on,hypervisor=on,tsc-adjust=on,umip=on,pku=on,md-clear=on,stibp=on,arch-capabilities=on,xsaves=on,rdctl-no=on,ibrs-all=on,skip-l1dfl-vmentry=on,mds-no=on,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1000,hv-vpindex,hv-runtime,hv-synic,hv-stimer,hv-fre
>   quencies,hv-reenlightenment,hv-tlbflush \
>   -m 2148 \
>   -overcommit mem-lock=off \
>   -smp 1,sockets=1,cores=1,threads=1 \
>   -object iothread,id=iothread1 \
>   -uuid ff20ae7f-8cfe-4ec5-bd50-e78f8a167414 \
>   -no-user-config \
>   -nodefaults \
>   -chardev socket,id=charmonitor,fd=44,server,nowait \
>   -mon chardev=charmonitor,id=monitor,mode=control \
>   -rtc base=utc,driftfix=slew \
>   -global kvm-pit.lost_tick_policy=delay \
>   -no-shutdown \
>   -boot menu=on,strict=on \
>   -device ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x5.0x7 \
>   -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x5 \
>   -device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x5.0x1 \
>   -device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x5.0x2 \
>   -device virtio-scsi-pci,iothread=iothread1,id=scsi0,bus=pcie.0,addr=0x9 \
>   -device virtio-serial-pci,id=virtio-serial0,bus=pcie.0,addr=0x6 \
>   -drive file=/dev/vm/p541999,format=raw,if=none,id=drive-scsi0-0-0-0,cache=none,discard=unmap,aio=threads,throttling.bps-write=52428800,throttling.bps-write-max=314572800,throttling.bps-write-max-length=120 \
>   -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,device_id=drive-scsi0-0-0-0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=2,write-cache=on \
>   -drive if=none,id=drive-sata0-0-0,readonly=on \
>   -device ide-cd,bus=ide.0,drive=drive-sata0-0-0,id=sata0-0-0,bootindex=1 \
>   -netdev tap,fd=47,id=hostnet0,vhost=on,vhostfd=48 \
>   -device virtio-net-pci,netdev=hostnet0,id=net0,mac=00:00:00:54:19:99,bus=pcie.0,addr=0x3 \
>   -chardev pty,id=charserial0 \
>   -device isa-serial,chardev=charserial0,id=serial0 \
>   -chardev socket,id=charchannel0,fd=49,server,nowait \
>   -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 \
>   -vnc 0.0.0.0:6128,password \
>   -device cirrus-vga,id=video0,bus=pcie.0,addr=0x1 \
>   -device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x8 \
>   -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
>   -msg timestamp=on
>   char device redirected to /dev/pts/5 (label charserial0)
>   2019-08-30 08:27:00.539+0000: shutting down, reason=crashed
> 
>   
>   config:
>   <domain type='kvm'>
>     <name>p541999</name>
>     <uuid>ff20ae7f-8cfe-4ec5-bd50-e78f8a167414</uuid>
>     <memory unit='KiB'>2199552</memory>
>     <currentMemory unit='KiB'>2199552</currentMemory>
>     <vcpu placement='static'>1</vcpu>
>     <iothreads>1</iothreads>
>     <resource>
>       <partition>/machine</partition>
>     </resource>
>     <os>
>       <type arch='x86_64' machine='pc-q35-4.0'>hvm</type>
>       <boot dev='cdrom'/>
>       <boot dev='hd'/>
>       <bootmenu enable='yes'/>
>     </os>
>     <features>
>       <acpi/>
>       <apic/>
>       <pae/>
>       <hyperv>
>         <relaxed state='on'/>
>         <vapic state='on'/>
>         <spinlocks state='on' retries='4096'/>
>         <vpindex state='on'/>
>         <runtime state='on'/>
>         <synic state='on'/>
>         <stimer state='on'/>
>         <frequencies state='on'/>
>         <reenlightenment state='on'/>
>         <tlbflush state='on'/>
>       </hyperv>
>       <msrs unknown='ignore'/>
>     </features>
>     <cpu mode='host-model' check='full'>
>       <model fallback='forbid'/>
>     </cpu>
>     <clock offset='utc'>
>       <timer name='rtc' tickpolicy='catchup'/>
>       <timer name='pit' tickpolicy='delay'/>
>       <timer name='hpet' present='yes'/>
>       <timer name='hypervclock' present='yes'/>
>     </clock>
>     <on_poweroff>destroy</on_poweroff>
>     <on_reboot>restart</on_reboot>
>     <on_crash>restart</on_crash>
>     <devices>
>       <emulator>/usr/bin/qemu-system-x86_64</emulator>
>       <disk type='block' device='disk'>
>         <driver name='qemu' type='raw' cache='none' io='threads' discard='unmap'/>
>         <source dev='/dev/vm/p541999'/>
>         <backingStore/>
>         <target dev='sda' bus='scsi'/>
>         <iotune>
>           <write_bytes_sec>52428800</write_bytes_sec>
>           <write_bytes_sec_max>314572800</write_bytes_sec_max>
>           <write_bytes_sec_max_length>120</write_bytes_sec_max_length>
>         </iotune>
>         <address type='drive' controller='0' bus='0' target='0' unit='0'/>
>       </disk>
>       <disk type='file' device='cdrom'>
>         <driver name='qemu' type='raw'/>
>         <target dev='sdb' bus='sata'/>
>         <readonly/>
>         <address type='drive' controller='0' bus='0' target='0' unit='0'/>
>       </disk>
>       <controller type='usb' index='0' model='ich9-ehci1'>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x7'/>
>       </controller>
>       <controller type='usb' index='0' model='ich9-uhci1'>
>         <master startport='0'/>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
>       </controller>
>       <controller type='usb' index='0' model='ich9-uhci2'>
>         <master startport='2'/>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
>       </controller>
>       <controller type='usb' index='0' model='ich9-uhci3'>
>         <master startport='4'/>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x2'/>
>       </controller>
>       <controller type='virtio-serial' index='0'>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
>       </controller>
>       <controller type='scsi' index='0' model='virtio-scsi'>
>         <driver iothread='1'/>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
>       </controller>
>       <controller type='pci' index='0' model='pcie-root'/>
>       <controller type='sata' index='0'>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
>       </controller>
>       <interface type='bridge'>
>         <mac address='00:00:00:54:19:99'/>
>         <source bridge='br0'/>
>         <bandwidth>
>           <inbound average='12500' peak='12500' burst='1024'/>
>           <outbound average='12500' peak='12500' burst='1024'/>
>         </bandwidth>
>         <model type='virtio'/>
>         <filterref filter='clean-traffic'>
>           <parameter name='CTRL_IP_LEARNING' value='none'/>
>           <parameter name='IP' value='1.2.3.4'/>
>         </filterref>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
>       </interface>
>       <serial type='pty'>
>         <target type='isa-serial' port='0'>
>           <model name='isa-serial'/>
>         </target>
>       </serial>
>       <console type='pty'>
>         <target type='serial' port='0'/>
>       </console>
>       <channel type='unix'>
>         <source mode='bind' path='/var/lib/libvirt/qemu/p541999.agent'/>
>         <target type='virtio' name='org.qemu.guest_agent.0'/>
>         <address type='virtio-serial' controller='0' bus='0' port='1'/>
>       </channel>
>       <input type='mouse' bus='ps2'/>
>       <input type='keyboard' bus='ps2'/>
>       <graphics type='vnc' port='12028' autoport='no' listen='0.0.0.0' passwd='SUPERPASSWORD'>
>         <listen type='address' address='0.0.0.0'/>
>       </graphics>
>       <video>
>         <model type='cirrus' vram='16384' heads='1' primary='yes'/>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
>       </video>
>       <memballoon model='virtio'>
>         <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
>       </memballoon>
>     </devices>
>     <seclabel type='none' model='none'/>
>   </domain>
> 
> To manage notifications about this bug go to:
> https://bugs.launchpad.net/qemu/+bug/1842038/+subscriptions
> 


https://lists.gnu.org/archive/html/qemu-block/2019-09/msg00481.html
fix issue and migration work fine