libivrtd fork qemu to create vm ,which start with ceph rbd device, after vm status:runing , the qemu stuck at booting from hard disk....
Description of problem:
[root@ceph-client ceph]# virsh list --all
 Id    Name                           State
----------------------------------------------------
 19    c7_ceph                        running

the vm qemu stuck at booting from hard disk.....
Steps to reproduce:
1. use ceph-deploy deploy a ceph distribute storage, which use to store vm's qcow2 files,this ceph has 3 osd node 
2. refer the link https://docs.ceph.com/en/quincy/rbd/libvirt/  create a ceph user :client.libvirt 
3. import a exists qcow2 file into ceph libvit-pool, then start vm

[root@ceph-1 ~]# ceph -s
  cluster:
    id:     3fbbf51f-88fd-4883-9f24-595bf853c5f2
    health: HEALTH_OK
 
  services:
    mon: 1 daemons, quorum ceph-1
    mgr: ceph-1(active)
    osd: 3 osds: 3 up, 3 in
 
  data:
    pools:   1 pools, 128 pgs
    objects: 940  objects, 3.6 GiB
    usage:   31 GiB used, 209 GiB / 240 GiB avail
    pgs:     128 active+clean

[root@ceph-1 ~]#ceph auth ls  
client.libvirt
	key: AQD/XwFkq7kHMhAA1OmPtKPVno6gjmZleOevOA==
	caps: [mon] allow r
	caps: [osd] allow class-read object_prefix rbd_children, allow rwx pool=libvirt-pool

[root@ceph-client ceph]# cat ceph.conf 
[global]
fsid = 3fbbf51f-88fd-4883-9f24-595bf853c5f2
mon_initial_members = ceph-1
mon_host = 172.24.193.62
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

osd_pool_default_size = 2
[root@ceph-client ceph]# 

[root@ceph-client ceph]# virsh start c7_ceph
Domain c7_ceph started

[root@ceph-client ceph]# 
[root@ceph-client ceph]# virsh list --all
 Id    Name                           State
----------------------------------------------------
 19    c7_ceph                        running


    <emulator>/usr/local/qemu-3.0/bin/qemu-system-x86_64</emulator>
    <disk type='network' device='disk'>
      <driver name='qemu' type='raw' cache='writeback'/>
      <auth username='libvirt'>
        <secret type='ceph' uuid='fb57a2a3-8cdf-44cb-afc1-2d8bdc0fc5d0'/>
      </auth>
      <source protocol='rbd' name='libvirt-pool/root-vsys_c5.qcow2'>
        <host name='172.24.193.62' port='6789'/>
        <host name='172.24.193.63' port='6789'/>
        <host name='172.24.193.64' port='6789'/>
      </source>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
    </disk>

========================
[root@ceph-client ceph]# cat /run/libvirt/qemu/c7_ceph.xml 


<domstatus state='running' reason='booted' pid='57437'>
  <monitor path='/var/lib/libvirt/qemu/domain-19-c7_ceph/monitor.sock' json='1' type='unix'/>
  <namespaces>
    <mount/>
  </namespaces>
  <vcpus>
    <vcpu id='0' pid='57487'/>
    <vcpu id='1' pid='57488'/>
  </vcpus>
  <qemuCaps>
    <flag name='kvm'/>
    <flag name='no-hpet'/>
    <flag name='spice'/>
    <flag name='boot-index'/>
    <flag name='hda-duplex'/>
    <flag name='ccid-emulated'/>
    <flag name='ccid-passthru'/>
    <flag name='virtio-tx-alg'/>
    <flag name='virtio-blk-pci.ioeventfd'/>
    <flag name='sga'/>
    <flag name='virtio-blk-pci.event_idx'/>
    <flag name='virtio-net-pci.event_idx'/>
    <flag name='piix3-usb-uhci'/>
    <flag name='piix4-usb-uhci'/>
    <flag name='usb-ehci'/>
    <flag name='ich9-usb-ehci1'/>
    <flag name='vt82c686b-usb-uhci'/>
    <flag name='pci-ohci'/>
    <flag name='usb-redir'/>
    <flag name='usb-hub'/>
    <flag name='ich9-ahci'/>
    <flag name='no-acpi'/>
    <flag name='virtio-blk-pci.scsi'/>
    <flag name='scsi-disk.channel'/>
    <flag name='scsi-block'/>
    <flag name='transaction'/>
    <flag name='block-job-async'/>
    <flag name='scsi-cd'/>
    <flag name='ide-cd'/>
    <flag name='hda-micro'/>
    <flag name='dump-guest-memory'/>
    <flag name='nec-usb-xhci'/>
    <flag name='balloon-event'/>
    <flag name='lsi'/>
    <flag name='virtio-scsi-pci'/>
    <flag name='blockio'/>
    <flag name='disable-s3'/>
    <flag name='disable-s4'/>
    <flag name='usb-redir.filter'/>
    <flag name='ide-drive.wwn'/>
    <flag name='scsi-disk.wwn'/>
    <flag name='seccomp-sandbox'/>
    <flag name='reboot-timeout'/>
    <flag name='seamless-migration'/>
    <flag name='block-commit'/>
    <flag name='vnc'/>
    <flag name='drive-mirror'/>
    <flag name='usb-redir.bootindex'/>
    <flag name='usb-host.bootindex'/>
    <flag name='blockdev-snapshot-sync'/>
    <flag name='qxl'/>
    <flag name='VGA'/>
    <flag name='cirrus-vga'/>
    <flag name='vmware-svga'/>
    <flag name='device-video-primary'/>
    <flag name='usb-serial'/>
    <flag name='usb-net'/>
    <flag name='add-fd'/>
    <flag name='nbd-server'/>
    <flag name='virtio-rng'/>
    <flag name='rng-random'/>
    <flag name='rng-egd'/>
    <flag name='megasas'/>
    <flag name='tpm-passthrough'/>
    <flag name='tpm-tis'/>
    <flag name='pci-bridge'/>
    <flag name='vfio-pci'/>
    <flag name='vfio-pci.bootindex'/>
    <flag name='scsi-generic'/>
    <flag name='scsi-generic.bootindex'/>
    <flag name='mem-merge'/>
    <flag name='vnc-websocket'/>
    <flag name='drive-discard'/>
    <flag name='mlock'/>
    <flag name='device-del-event'/>
    <flag name='dmi-to-pci-bridge'/>
    <flag name='i440fx-pci-hole64-size'/>
    <flag name='q35-pci-hole64-size'/>
    <flag name='usb-storage'/>
    <flag name='usb-storage.removable'/>
    <flag name='ich9-intel-hda'/>
    <flag name='kvm-pit-lost-tick-policy'/>
    <flag name='boot-strict'/>
    <flag name='pvpanic'/>
    <flag name='spice-file-xfer-disable'/>
    <flag name='spiceport'/>
    <flag name='usb-kbd'/>
    <flag name='msg-timestamp'/>
    <flag name='active-commit'/>
    <flag name='change-backing-file'/>
    <flag name='memory-backend-ram'/>
    <flag name='numa'/>
    <flag name='memory-backend-file'/>
    <flag name='usb-audio'/>
    <flag name='rtc-reset-reinjection'/>
    <flag name='splash-timeout'/>
    <flag name='iothread'/>
    <flag name='migrate-rdma'/>
    <flag name='ivshmem'/>
    <flag name='drive-iotune-max'/>
    <flag name='VGA.vgamem_mb'/>
    <flag name='vmware-svga.vgamem_mb'/>
    <flag name='qxl.vgamem_mb'/>
    <flag name='pc-dimm'/>
    <flag name='machine-vmport-opt'/>
    <flag name='aes-key-wrap'/>
    <flag name='dea-key-wrap'/>
    <flag name='pci-serial'/>
    <flag name='vhost-user-multiqueue'/>
    <flag name='migration-event'/>
    <flag name='ioh3420'/>
    <flag name='x3130-upstream'/>
    <flag name='xio3130-downstream'/>
    <flag name='rtl8139'/>
    <flag name='e1000'/>
    <flag name='virtio-net'/>
    <flag name='gic-version'/>
    <flag name='incoming-defer'/>
    <flag name='virtio-gpu'/>
    <flag name='virtio-keyboard'/>
    <flag name='virtio-mouse'/>
    <flag name='virtio-tablet'/>
    <flag name='virtio-input-host'/>
    <flag name='chardev-file-append'/>
    <flag name='ich9-disable-s3'/>
    <flag name='ich9-disable-s4'/>
    <flag name='vserport-change-event'/>
    <flag name='virtio-balloon-pci.deflate-on-oom'/>
    <flag name='mptsas1068'/>
    <flag name='qxl.vram64_size_mb'/>
    <flag name='chardev-logfile'/>
    <flag name='debug-threads'/>
    <flag name='secret'/>
    <flag name='pxb'/>
    <flag name='pxb-pcie'/>
    <flag name='device-tray-moved-event'/>
    <flag name='nec-usb-xhci-ports'/>
    <flag name='virtio-scsi-pci.iothread'/>
    <flag name='name-guest'/>
    <flag name='qxl.max_outputs'/>
    <flag name='spice-unix'/>
    <flag name='drive-detect-zeroes'/>
    <flag name='tls-creds-x509'/>
    <flag name='intel-iommu'/>
    <flag name='smm'/>
    <flag name='virtio-pci-disable-legacy'/>
    <flag name='query-hotpluggable-cpus'/>
    <flag name='virtio-net.rx_queue_size'/>
    <flag name='virtio-vga'/>
    <flag name='drive-iotune-max-length'/>
    <flag name='ivshmem-plain'/>
    <flag name='ivshmem-doorbell'/>
    <flag name='query-qmp-schema'/>
    <flag name='gluster.debug_level'/>
    <flag name='drive-iotune-group'/>
    <flag name='query-cpu-model-expansion'/>
    <flag name='virtio-net.host_mtu'/>
    <flag name='nvdimm'/>
    <flag name='pcie-root-port'/>
    <flag name='query-cpu-definitions'/>
    <flag name='block-write-threshold'/>
    <flag name='query-named-block-nodes'/>
    <flag name='cpu-cache'/>
    <flag name='qemu-xhci'/>
    <flag name='kernel-irqchip'/>
    <flag name='kernel-irqchip.split'/>
    <flag name='intel-iommu.intremap'/>
    <flag name='intel-iommu.caching-mode'/>
    <flag name='intel-iommu.eim'/>
    <flag name='intel-iommu.device-iotlb'/>
    <flag name='virtio.iommu_platform'/>
    <flag name='virtio.ats'/>
    <flag name='loadparm'/>
    <flag name='vnc-multi-servers'/>
    <flag name='virtio-net.tx_queue_size'/>
    <flag name='chardev-reconnect'/>
    <flag name='virtio-gpu.max_outputs'/>
    <flag name='vxhs'/>
    <flag name='virtio-blk.num-queues'/>
    <flag name='vmcoreinfo'/>
    <flag name='numa.dist'/>
    <flag name='disk-share-rw'/>
    <flag name='iscsi.password-secret'/>
    <flag name='isa-serial'/>
    <flag name='dump-completed'/>
    <flag name='qcow2-luks'/>
    <flag name='pcie-pci-bridge'/>
    <flag name='seccomp-blacklist'/>
    <flag name='query-cpus-fast'/>
    <flag name='disk-write-cache'/>
    <flag name='nbd-tls'/>
    <flag name='tpm-crb'/>
    <flag name='pr-manager-helper'/>
    <flag name='qom-list-properties'/>
    <flag name='memory-backend-file.discard-data'/>
    <flag name='sdl-gl'/>
    <flag name='screendump_device'/>
    <flag name='hda-output'/>
    <flag name='blockdev-del'/>
    <flag name='vmgenid'/>
    <flag name='vhost-vsock'/>
    <flag name='chardev-fd-pass'/>
    <flag name='tpm-emulator'/>
    <flag name='mch'/>
    <flag name='mch.extended-tseg-mbytes'/>
    <flag name='usb-storage.werror'/>
    <flag name='egl-headless'/>
    <flag name='vfio-pci.display'/>
  </qemuCaps>
  <devices>
    <device alias='rng0'/>
    <device alias='virtio-disk0'/>
    <device alias='virtio-serial0'/>
    <device alias='video0'/>
    <device alias='serial0'/>
    <device alias='balloon0'/>
    <device alias='channel0'/>
    <device alias='net0'/>
    <device alias='input0'/>
    <device alias='scsi0'/>
    <device alias='usb'/>
  </devices>
  <libDir path='/var/lib/libvirt/qemu/domain-19-c7_ceph'/>
  <channelTargetDir path='/var/lib/libvirt/qemu/channel/target/domain-19-c7_ceph'/>
  <cpu mode='custom' match='exact' check='partial'>
    <model fallback='forbid'>Broadwell</model>
  </cpu>
  <chardevStdioLogd/>
  <allowReboot value='yes'/>
  <blockjobs active='no'/>
  <domain type='kvm' id='19'>
    <name>c7_ceph</name>
    <uuid>ff08671e-824c-4939-80ec-602235c0662e</uuid>
    <memory unit='KiB'>4194304</memory>
    <currentMemory unit='KiB'>4194304</currentMemory>
    <vcpu placement='static'>2</vcpu>
    <resource>
      <partition>/machine</partition>
    </resource>
    <os>
      <type arch='x86_64' machine='pc-i440fx-3.0'>hvm</type>
      <boot dev='hd'/>
    </os>
    <features>
      <acpi/>
      <apic/>
    </features>
    <cpu mode='custom' match='exact' check='full'>
      <model fallback='forbid'>Broadwell</model>
      <feature policy='require' name='vme'/>
      <feature policy='require' name='f16c'/>
      <feature policy='require' name='rdrand'/>
      <feature policy='require' name='hypervisor'/>
      <feature policy='require' name='arat'/>
      <feature policy='disable' name='erms'/>
      <feature policy='require' name='xsaveopt'/>
      <feature policy='require' name='abm'/>
    </cpu>
    <clock offset='utc'>
      <timer name='rtc' tickpolicy='catchup'/>
      <timer name='pit' tickpolicy='delay'/>
      <timer name='hpet' present='no'/>
    </clock>
    <on_poweroff>destroy</on_poweroff>
    <on_reboot>restart</on_reboot>
    <on_crash>destroy</on_crash>
    <pm>
      <suspend-to-mem enabled='no'/>
      <suspend-to-disk enabled='no'/>
    </pm>
    <devices>
      <emulator>/usr/local/qemu-3.0/bin/qemu-system-x86_64</emulator>
      <disk type='network' device='disk'>
        <driver name='qemu' type='raw' cache='writeback'/>
        <auth username='libvirt'>
          <secret type='ceph' uuid='fb57a2a3-8cdf-44cb-afc1-2d8bdc0fc5d0'/>
        </auth>
        <source protocol='rbd' name='libvirt-pool/root-vsys_c5.qcow2' tlsFromConfig='0'>
          <host name='172.24.193.62' port='6789'/>
          <host name='172.24.193.63' port='6789'/>
          <host name='172.24.193.64' port='6789'/>
          <privateData>
            <objects>
              <secret type='auth' alias='virtio-disk0-secret0'/>
            </objects>
          </privateData>
        </source>
        <target dev='vda' bus='virtio'/>
        <alias name='virtio-disk0'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
      </disk>
      <controller type='usb' index='0' model='ich9-ehci1'>
        <alias name='usb'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x7'/>
      </controller>
      <controller type='usb' index='0' model='ich9-uhci1'>
        <alias name='usb'/>
        <master startport='0'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0' multifunction='on'/>
      </controller>
      <controller type='usb' index='0' model='ich9-uhci2'>
        <alias name='usb'/>
        <master startport='2'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x1'/>
      </controller>
      <controller type='usb' index='0' model='ich9-uhci3'>
        <alias name='usb'/>
        <master startport='4'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x2'/>
      </controller>
      <controller type='pci' index='0' model='pci-root'>
        <alias name='pci.0'/>
      </controller>
      <controller type='virtio-serial' index='0'>
        <alias name='virtio-serial0'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
      </controller>
      <controller type='scsi' index='0' model='lsilogic'>
        <alias name='scsi0'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
      </controller>
      <controller type='ide' index='0'>
        <alias name='ide'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
      </controller>
      <interface type='bridge'>
        <mac address='52:54:00:2e:e1:1f'/>
        <source bridge='virbr0'/>
        <target dev='vnet0'/>
        <model type='virtio'/>
        <alias name='net0'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
      </interface>
      <serial type='pty'>
        <source path='/dev/pts/2'/>
        <target type='isa-serial' port='0'>
          <model name='isa-serial'/>
        </target>
        <alias name='serial0'/>
      </serial>
      <console type='pty' tty='/dev/pts/2'>
        <source path='/dev/pts/2'/>
        <target type='serial' port='0'/>
        <alias name='serial0'/>
      </console>
      <channel type='unix'>
        <source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-19-c7_ceph/org.qemu.guest_agent.0'/>
        <target type='virtio' name='org.qemu.guest_agent.0' state='disconnected'/>
        <alias name='channel0'/>
        <address type='virtio-serial' controller='0' bus='0' port='1'/>
      </channel>
      <input type='tablet' bus='usb'>
        <alias name='input0'/>
        <address type='usb' bus='0' port='1'/>
      </input>
      <input type='mouse' bus='ps2'>
        <alias name='input1'/>
      </input>
      <input type='keyboard' bus='ps2'>
        <alias name='input2'/>
      </input>
      <graphics type='vnc' port='5900' autoport='yes' listen='0.0.0.0'>
        <listen type='address' address='0.0.0.0' fromConfig='0' autoGenerated='no'/>
      </graphics>
      <video>
        <model type='cirrus' vram='16384' heads='1' primary='yes'/>
        <alias name='video0'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
      </video>
      <memballoon model='virtio'>
        <alias name='balloon0'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
      </memballoon>
      <rng model='virtio'>
        <backend model='random'>/dev/urandom</backend>
        <alias name='rng0'/>
        <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
      </rng>
    </devices>
    <seclabel type='dynamic' model='selinux' relabel='yes'>
      <label>system_u:system_r:svirt_t:s0:c99,c659</label>
      <imagelabel>system_u:object_r:svirt_image_t:s0:c99,c659</imagelabel>
    </seclabel>
    <seclabel type='dynamic' model='dac' relabel='yes'>
      <label>+107:+107</label>
      <imagelabel>+107:+107</imagelabel>
    </seclabel>
  </domain>
</domstatus>
[root@ceph-client ceph]# 

/usr/local/qemu-3.0/bin/qemu-system-x86_64 which is build by qemu-3.0 source code , first i build qemu-3.0 source with --enable-rbd ,
later i rebuild qemu-3.0 source with more config paramter from centos7-2009 qemu, those config paramter from qemu-kvm-1.5.3-175.el7.src.rpm ,which has those paramters:
# QEMU configure log Fri Mar  3 18:22:31 CST 2023
# Configured with: './configure' '--prefix=/usr' '--libdir=/usr/lib64' '--sysconfdir=/etc' '--interp-prefix=/usr/qemu-%M' '--audio-drv-list=pa,alsa' '--with-confsuffix=/qemu-kvm' '--localstatedir=/var' '--libexecdir=/usr/libexec' '--wit
h-pkgversion=qemu-kvm-1.5.3-175.el7' '--disable-strip' '--disable-qom-cast-debug' '--extra-ldflags=-Wl,--build-id -pie -Wl,-z,relro -Wl,-z,now' '--extra-cflags=-O2 -g -pipe -Wall  -fexceptions -fstack-protector-strong --param=ssp-buffer
-size=4 -grecord-gcc-switches -m64 -mtune=generic -fPIE -DPIE' '--enable-trace-backend=dtrace' '--enable-werror' '--disable-xen' '--disable-virtfs' '--enable-kvm' '--enable-libusb' '--enable-spice' '--enable-seccomp' '--disable-fdt' '--
enable-docs' '--disable-sdl' '--disable-debug-tcg' '--disable-sparse' '--disable-brlapi' '--disable-bluez' '--disable-vde' '--disable-curses' '--enable-curl' '--enable-libssh2' '--enable-vnc-tls' '--enable-vnc-sasl' '--enable-linux-aio'
 '--enable-smartcard-nss' '--enable-lzo' '--enable-snappy' '--enable-usb-redir' '--enable-vnc-png' '--disable-vnc-jpeg' '--enable-vnc-ws' '--enable-uuid' '--disable-vhost-scsi' '--disable-guest-agent' '--disable-live-block-ops' '--disab
le-live-block-migration' '--enable-rbd' '--enable-glusterfs' '--enable-tcmalloc' '--block-drv-rw-whitelist=qcow2,raw,file,host_device,blkdebug,nbd,iscsi,gluster,rbd' '--block-drv-ro-whitelist=vmdk,vhdx,vpc,ssh,https' '--iasl=/bin/false'
 '--target-list=x86_64-softmmu'


,   after rebuild the qemu-system-x86_64 : 

virsh start c7_ceph 
[root@ceph-client ceph]# virsh list --all
 Id    Name                           State
----------------------------------------------------
 19    c7_ceph                        running

qemu still stuck at booting from hard disk...



to my surprised if the libvirtd xml file if i replace /usr/local/qemu-3.0/bin/qemu-system-x86_64 with /usr/libexec/bin/qemu-kvm , then the vm
can start successfully .