diff options
Diffstat (limited to 'results/classifier/118/all/70021271')
| -rw-r--r-- | results/classifier/118/all/70021271 | 7473 |
1 files changed, 7473 insertions, 0 deletions
diff --git a/results/classifier/118/all/70021271 b/results/classifier/118/all/70021271 new file mode 100644 index 000000000..e855aec22 --- /dev/null +++ b/results/classifier/118/all/70021271 @@ -0,0 +1,7473 @@ +graphic: 0.958 +permissions: 0.957 +debug: 0.956 +performance: 0.949 +KVM: 0.949 +semantic: 0.946 +peripherals: 0.932 +hypervisor: 0.931 +mistranslation: 0.929 +register: 0.923 +virtual: 0.921 +assembly: 0.910 +architecture: 0.909 +TCG: 0.906 +ppc: 0.901 +vnc: 0.900 +arm: 0.899 +device: 0.887 +kernel: 0.882 +PID: 0.880 +user-level: 0.876 +x86: 0.875 +risc-v: 0.875 +socket: 0.873 +network: 0.873 +boot: 0.872 +files: 0.867 +VMM: 0.860 +i386: 0.849 + +[Qemu-devel] [BUG]Unassigned mem write during pci device hot-plug + +Hi all, + +In our test, we configured VM with several pci-bridges and a virtio-net nic +been attached with bus 4, +After VM is startup, We ping this nic from host to judge if it is working +normally. Then, we hot add pci devices to this VM with bus 0. +We found the virtio-net NIC in bus 4 is not working (can not connect) +occasionally, as it kick virtio backend failure with error below: + Unassigned mem write 00000000fc803004 = 0x1 + +memory-region: pci_bridge_pci + 0000000000000000-ffffffffffffffff (prio 0, RW): pci_bridge_pci + 00000000fc800000-00000000fc803fff (prio 1, RW): virtio-pci + 00000000fc800000-00000000fc800fff (prio 0, RW): virtio-pci-common + 00000000fc801000-00000000fc801fff (prio 0, RW): virtio-pci-isr + 00000000fc802000-00000000fc802fff (prio 0, RW): virtio-pci-device + 00000000fc803000-00000000fc803fff (prio 0, RW): virtio-pci-notify <- io +mem unassigned + ⦠+ +We caught an exceptional address changing while this problem happened, show as +follow: +Before pci_bridge_update_mappingsï¼ + 00000000fc000000-00000000fc1fffff (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci 00000000fc000000-00000000fc1fffff + 00000000fc200000-00000000fc3fffff (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci 00000000fc200000-00000000fc3fffff + 00000000fc400000-00000000fc5fffff (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci 00000000fc400000-00000000fc5fffff + 00000000fc600000-00000000fc7fffff (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci 00000000fc600000-00000000fc7fffff + 00000000fc800000-00000000fc9fffff (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci 00000000fc800000-00000000fc9fffff <- correct Adress Spce + 00000000fca00000-00000000fcbfffff (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci 00000000fca00000-00000000fcbfffff + 00000000fcc00000-00000000fcdfffff (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci 00000000fcc00000-00000000fcdfffff + 00000000fce00000-00000000fcffffff (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci 00000000fce00000-00000000fcffffff + +After pci_bridge_update_mappingsï¼ + 00000000fda00000-00000000fdbfffff (prio 1, RW): alias pci_bridge_mem +@pci_bridge_pci 00000000fda00000-00000000fdbfffff + 00000000fdc00000-00000000fddfffff (prio 1, RW): alias pci_bridge_mem +@pci_bridge_pci 00000000fdc00000-00000000fddfffff + 00000000fde00000-00000000fdffffff (prio 1, RW): alias pci_bridge_mem +@pci_bridge_pci 00000000fde00000-00000000fdffffff + 00000000fe000000-00000000fe1fffff (prio 1, RW): alias pci_bridge_mem +@pci_bridge_pci 00000000fe000000-00000000fe1fffff + 00000000fe200000-00000000fe3fffff (prio 1, RW): alias pci_bridge_mem +@pci_bridge_pci 00000000fe200000-00000000fe3fffff + 00000000fe400000-00000000fe5fffff (prio 1, RW): alias pci_bridge_mem +@pci_bridge_pci 00000000fe400000-00000000fe5fffff + 00000000fe600000-00000000fe7fffff (prio 1, RW): alias pci_bridge_mem +@pci_bridge_pci 00000000fe600000-00000000fe7fffff + 00000000fe800000-00000000fe9fffff (prio 1, RW): alias pci_bridge_mem +@pci_bridge_pci 00000000fe800000-00000000fe9fffff + fffffffffc800000-fffffffffc800000 (prio 1, RW): alias pci_bridge_pref_mem +@pci_bridge_pci fffffffffc800000-fffffffffc800000 <- Exceptional Adress Space + +We have figured out why this address becomes this value, according to pci +spec, pci driver can get BAR address size by writing 0xffffffff to +the pci register firstly, and then read back the value from this register. +We didn't handle this value specially while process pci write in qemu, the +function call stack is: +Pci_bridge_dev_write_config +-> pci_bridge_write_config +-> pci_default_write_config (we update the config[address] value here to +fffffffffc800000, which should be 0xfc800000 ) +-> pci_bridge_update_mappings + ->pci_bridge_region_del(br, br->windows); +-> pci_bridge_region_init + +->pci_bridge_init_alias (here pci_bridge_get_base, we use the wrong value +fffffffffc800000) + -> +memory_region_transaction_commit + +So, as we can see, we use the wrong base address in qemu to update the memory +regions, though, we update the base address to +The correct value after pci driver in VM write the original value back, the +virtio NIC in bus 4 may still sends net packets concurrently with +The wrong memory region address. + +We have tried to skip the memory region update action in qemu while detect pci +write with 0xffffffff value, and it does work, but +This seems to be not gently. + +diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c +index b2e50c3..84b405d 100644 +--- a/hw/pci/pci_bridge.c ++++ b/hw/pci/pci_bridge.c +@@ -256,7 +256,8 @@ void pci_bridge_write_config(PCIDevice *d, + pci_default_write_config(d, address, val, len); +- if (ranges_overlap(address, len, PCI_COMMAND, 2) || ++ if ( (val != 0xffffffff) && ++ (ranges_overlap(address, len, PCI_COMMAND, 2) || + /* io base/limit */ + ranges_overlap(address, len, PCI_IO_BASE, 2) || +@@ -266,7 +267,7 @@ void pci_bridge_write_config(PCIDevice *d, + ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || + /* vga enable */ +- ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { ++ ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2))) { + pci_bridge_update_mappings(s); + } + +Thinks, +Xu + +On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +Hi all, +> +> +> +> +In our test, we configured VM with several pci-bridges and a virtio-net nic +> +been attached with bus 4, +> +> +After VM is startup, We ping this nic from host to judge if it is working +> +normally. Then, we hot add pci devices to this VM with bus 0. +> +> +We found the virtio-net NIC in bus 4 is not working (can not connect) +> +occasionally, as it kick virtio backend failure with error below: +> +> +Unassigned mem write 00000000fc803004 = 0x1 +Thanks for the report. Which guest was used to produce this problem? + +-- +MST + +n Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> Hi all, +> +> +> +> +> +> +> +> In our test, we configured VM with several pci-bridges and a +> +> virtio-net nic been attached with bus 4, +> +> +> +> After VM is startup, We ping this nic from host to judge if it is +> +> working normally. Then, we hot add pci devices to this VM with bus 0. +> +> +> +> We found the virtio-net NIC in bus 4 is not working (can not connect) +> +> occasionally, as it kick virtio backend failure with error below: +> +> +> +> Unassigned mem write 00000000fc803004 = 0x1 +> +> +Thanks for the report. Which guest was used to produce this problem? +> +> +-- +> +MST +I was seeing this problem when I hotplug a VFIO device to guest CentOS 7.4, +after that I compiled the latest Linux kernel and it also contains this problem. + +Thinks, +Xu + +On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +Hi all, +> +> +> +> +In our test, we configured VM with several pci-bridges and a virtio-net nic +> +been attached with bus 4, +> +> +After VM is startup, We ping this nic from host to judge if it is working +> +normally. Then, we hot add pci devices to this VM with bus 0. +> +> +We found the virtio-net NIC in bus 4 is not working (can not connect) +> +occasionally, as it kick virtio backend failure with error below: +> +> +Unassigned mem write 00000000fc803004 = 0x1 +> +> +> +> +memory-region: pci_bridge_pci +> +> +0000000000000000-ffffffffffffffff (prio 0, RW): pci_bridge_pci +> +> +00000000fc800000-00000000fc803fff (prio 1, RW): virtio-pci +> +> +00000000fc800000-00000000fc800fff (prio 0, RW): virtio-pci-common +> +> +00000000fc801000-00000000fc801fff (prio 0, RW): virtio-pci-isr +> +> +00000000fc802000-00000000fc802fff (prio 0, RW): virtio-pci-device +> +> +00000000fc803000-00000000fc803fff (prio 0, RW): virtio-pci-notify <- io +> +mem unassigned +> +> +⦠+> +> +> +> +We caught an exceptional address changing while this problem happened, show as +> +follow: +> +> +Before pci_bridge_update_mappingsï¼ +> +> +00000000fc000000-00000000fc1fffff (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci 00000000fc000000-00000000fc1fffff +> +> +00000000fc200000-00000000fc3fffff (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci 00000000fc200000-00000000fc3fffff +> +> +00000000fc400000-00000000fc5fffff (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci 00000000fc400000-00000000fc5fffff +> +> +00000000fc600000-00000000fc7fffff (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci 00000000fc600000-00000000fc7fffff +> +> +00000000fc800000-00000000fc9fffff (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci 00000000fc800000-00000000fc9fffff <- correct Adress Spce +> +> +00000000fca00000-00000000fcbfffff (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci 00000000fca00000-00000000fcbfffff +> +> +00000000fcc00000-00000000fcdfffff (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci 00000000fcc00000-00000000fcdfffff +> +> +00000000fce00000-00000000fcffffff (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci 00000000fce00000-00000000fcffffff +> +> +> +> +After pci_bridge_update_mappingsï¼ +> +> +00000000fda00000-00000000fdbfffff (prio 1, RW): alias pci_bridge_mem +> +@pci_bridge_pci 00000000fda00000-00000000fdbfffff +> +> +00000000fdc00000-00000000fddfffff (prio 1, RW): alias pci_bridge_mem +> +@pci_bridge_pci 00000000fdc00000-00000000fddfffff +> +> +00000000fde00000-00000000fdffffff (prio 1, RW): alias pci_bridge_mem +> +@pci_bridge_pci 00000000fde00000-00000000fdffffff +> +> +00000000fe000000-00000000fe1fffff (prio 1, RW): alias pci_bridge_mem +> +@pci_bridge_pci 00000000fe000000-00000000fe1fffff +> +> +00000000fe200000-00000000fe3fffff (prio 1, RW): alias pci_bridge_mem +> +@pci_bridge_pci 00000000fe200000-00000000fe3fffff +> +> +00000000fe400000-00000000fe5fffff (prio 1, RW): alias pci_bridge_mem +> +@pci_bridge_pci 00000000fe400000-00000000fe5fffff +> +> +00000000fe600000-00000000fe7fffff (prio 1, RW): alias pci_bridge_mem +> +@pci_bridge_pci 00000000fe600000-00000000fe7fffff +> +> +00000000fe800000-00000000fe9fffff (prio 1, RW): alias pci_bridge_mem +> +@pci_bridge_pci 00000000fe800000-00000000fe9fffff +> +> +fffffffffc800000-fffffffffc800000 (prio 1, RW): alias +> +pci_bridge_pref_mem +> +@pci_bridge_pci fffffffffc800000-fffffffffc800000 <- Exceptional Adress +> +Space +This one is empty though right? + +> +> +> +We have figured out why this address becomes this value, according to pci +> +spec, pci driver can get BAR address size by writing 0xffffffff to +> +> +the pci register firstly, and then read back the value from this register. +OK however as you show below the BAR being sized is the BAR +if a bridge. Are you then adding a bridge device by hotplug? + + + +> +We didn't handle this value specially while process pci write in qemu, the +> +function call stack is: +> +> +Pci_bridge_dev_write_config +> +> +-> pci_bridge_write_config +> +> +-> pci_default_write_config (we update the config[address] value here to +> +fffffffffc800000, which should be 0xfc800000 ) +> +> +-> pci_bridge_update_mappings +> +> +->pci_bridge_region_del(br, br->windows); +> +> +-> pci_bridge_region_init +> +> +-> +> +pci_bridge_init_alias (here pci_bridge_get_base, we use the wrong value +> +fffffffffc800000) +> +> +-> +> +memory_region_transaction_commit +> +> +> +> +So, as we can see, we use the wrong base address in qemu to update the memory +> +regions, though, we update the base address to +> +> +The correct value after pci driver in VM write the original value back, the +> +virtio NIC in bus 4 may still sends net packets concurrently with +> +> +The wrong memory region address. +> +> +> +> +We have tried to skip the memory region update action in qemu while detect pci +> +write with 0xffffffff value, and it does work, but +> +> +This seems to be not gently. +For sure. But I'm still puzzled as to why does Linux try to +size the BAR of the bridge while a device behind it is +used. + +Can you pls post your QEMU command line? + + + +> +> +> +diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c +> +> +index b2e50c3..84b405d 100644 +> +> +--- a/hw/pci/pci_bridge.c +> +> ++++ b/hw/pci/pci_bridge.c +> +> +@@ -256,7 +256,8 @@ void pci_bridge_write_config(PCIDevice *d, +> +> +pci_default_write_config(d, address, val, len); +> +> +- if (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> ++ if ( (val != 0xffffffff) && +> +> ++ (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> +/* io base/limit */ +> +> +ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> +@@ -266,7 +267,7 @@ void pci_bridge_write_config(PCIDevice *d, +> +> +ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || +> +> +/* vga enable */ +> +> +- ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +> +> ++ ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2))) { +> +> +pci_bridge_update_mappings(s); +> +> +} +> +> +> +> +Thinks, +> +> +Xu +> + +On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> Hi all, +> +> +> +> +> +> +> +> In our test, we configured VM with several pci-bridges and a +> +> virtio-net nic been attached with bus 4, +> +> +> +> After VM is startup, We ping this nic from host to judge if it is +> +> working normally. Then, we hot add pci devices to this VM with bus 0. +> +> +> +> We found the virtio-net NIC in bus 4 is not working (can not connect) +> +> occasionally, as it kick virtio backend failure with error below: +> +> +> +> Unassigned mem write 00000000fc803004 = 0x1 +> +> +> +> +> +> +> +> memory-region: pci_bridge_pci +> +> +> +> 0000000000000000-ffffffffffffffff (prio 0, RW): pci_bridge_pci +> +> +> +> 00000000fc800000-00000000fc803fff (prio 1, RW): virtio-pci +> +> +> +> 00000000fc800000-00000000fc800fff (prio 0, RW): +> +> virtio-pci-common +> +> +> +> 00000000fc801000-00000000fc801fff (prio 0, RW): virtio-pci-isr +> +> +> +> 00000000fc802000-00000000fc802fff (prio 0, RW): +> +> virtio-pci-device +> +> +> +> 00000000fc803000-00000000fc803fff (prio 0, RW): +> +> virtio-pci-notify <- io mem unassigned +> +> +> +> ⦠+> +> +> +> +> +> +> +> We caught an exceptional address changing while this problem happened, +> +> show as +> +> follow: +> +> +> +> Before pci_bridge_update_mappingsï¼ +> +> +> +> 00000000fc000000-00000000fc1fffff (prio 1, RW): alias +> +> pci_bridge_pref_mem @pci_bridge_pci 00000000fc000000-00000000fc1fffff +> +> +> +> 00000000fc200000-00000000fc3fffff (prio 1, RW): alias +> +> pci_bridge_pref_mem @pci_bridge_pci 00000000fc200000-00000000fc3fffff +> +> +> +> 00000000fc400000-00000000fc5fffff (prio 1, RW): alias +> +> pci_bridge_pref_mem @pci_bridge_pci 00000000fc400000-00000000fc5fffff +> +> +> +> 00000000fc600000-00000000fc7fffff (prio 1, RW): alias +> +> pci_bridge_pref_mem @pci_bridge_pci 00000000fc600000-00000000fc7fffff +> +> +> +> 00000000fc800000-00000000fc9fffff (prio 1, RW): alias +> +> pci_bridge_pref_mem @pci_bridge_pci 00000000fc800000-00000000fc9fffff +> +> <- correct Adress Spce +> +> +> +> 00000000fca00000-00000000fcbfffff (prio 1, RW): alias +> +> pci_bridge_pref_mem @pci_bridge_pci 00000000fca00000-00000000fcbfffff +> +> +> +> 00000000fcc00000-00000000fcdfffff (prio 1, RW): alias +> +> pci_bridge_pref_mem @pci_bridge_pci 00000000fcc00000-00000000fcdfffff +> +> +> +> 00000000fce00000-00000000fcffffff (prio 1, RW): alias +> +> pci_bridge_pref_mem @pci_bridge_pci 00000000fce00000-00000000fcffffff +> +> +> +> +> +> +> +> After pci_bridge_update_mappingsï¼ +> +> +> +> 00000000fda00000-00000000fdbfffff (prio 1, RW): alias +> +> pci_bridge_mem @pci_bridge_pci 00000000fda00000-00000000fdbfffff +> +> +> +> 00000000fdc00000-00000000fddfffff (prio 1, RW): alias +> +> pci_bridge_mem @pci_bridge_pci 00000000fdc00000-00000000fddfffff +> +> +> +> 00000000fde00000-00000000fdffffff (prio 1, RW): alias +> +> pci_bridge_mem @pci_bridge_pci 00000000fde00000-00000000fdffffff +> +> +> +> 00000000fe000000-00000000fe1fffff (prio 1, RW): alias +> +> pci_bridge_mem @pci_bridge_pci 00000000fe000000-00000000fe1fffff +> +> +> +> 00000000fe200000-00000000fe3fffff (prio 1, RW): alias +> +> pci_bridge_mem @pci_bridge_pci 00000000fe200000-00000000fe3fffff +> +> +> +> 00000000fe400000-00000000fe5fffff (prio 1, RW): alias +> +> pci_bridge_mem @pci_bridge_pci 00000000fe400000-00000000fe5fffff +> +> +> +> 00000000fe600000-00000000fe7fffff (prio 1, RW): alias +> +> pci_bridge_mem @pci_bridge_pci 00000000fe600000-00000000fe7fffff +> +> +> +> 00000000fe800000-00000000fe9fffff (prio 1, RW): alias +> +> pci_bridge_mem @pci_bridge_pci 00000000fe800000-00000000fe9fffff +> +> +> +> fffffffffc800000-fffffffffc800000 (prio 1, RW): alias +> +> pci_bridge_pref_mem +> +> @pci_bridge_pci fffffffffc800000-fffffffffc800000 <- Exceptional Adress +> +Space +> +> +This one is empty though right? +> +> +> +> +> +> +> We have figured out why this address becomes this value, according to +> +> pci spec, pci driver can get BAR address size by writing 0xffffffff +> +> to +> +> +> +> the pci register firstly, and then read back the value from this register. +> +> +> +OK however as you show below the BAR being sized is the BAR if a bridge. Are +> +you then adding a bridge device by hotplug? +No, I just simply hot plugged a VFIO device to Bus 0, another interesting +phenomenon is +If I hot plug the device to other bus, this doesn't happened. + +> +> +> +> We didn't handle this value specially while process pci write in +> +> qemu, the function call stack is: +> +> +> +> Pci_bridge_dev_write_config +> +> +> +> -> pci_bridge_write_config +> +> +> +> -> pci_default_write_config (we update the config[address] value here +> +> -> to +> +> fffffffffc800000, which should be 0xfc800000 ) +> +> +> +> -> pci_bridge_update_mappings +> +> +> +> ->pci_bridge_region_del(br, br->windows); +> +> +> +> -> pci_bridge_region_init +> +> +> +> -> +> +> pci_bridge_init_alias (here pci_bridge_get_base, we use the wrong +> +> value +> +> fffffffffc800000) +> +> +> +> -> +> +> memory_region_transaction_commit +> +> +> +> +> +> +> +> So, as we can see, we use the wrong base address in qemu to update the +> +> memory regions, though, we update the base address to +> +> +> +> The correct value after pci driver in VM write the original value +> +> back, the virtio NIC in bus 4 may still sends net packets concurrently +> +> with +> +> +> +> The wrong memory region address. +> +> +> +> +> +> +> +> We have tried to skip the memory region update action in qemu while +> +> detect pci write with 0xffffffff value, and it does work, but +> +> +> +> This seems to be not gently. +> +> +For sure. But I'm still puzzled as to why does Linux try to size the BAR of +> +the +> +bridge while a device behind it is used. +> +> +Can you pls post your QEMU command line? +My QEMU command line: +/root/xyd/qemu-system-x86_64 -name guest=Linux,debug-threads=on -S -object +secret,id=masterKey0,format=raw,file=/var/run/libvirt/qemu/domain-194-Linux/master-key.aes + -machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off -cpu +host,+kvm_pv_eoi -bios /usr/share/OVMF/OVMF.fd -m +size=4194304k,slots=256,maxmem=33554432k -realtime mlock=off -smp +20,sockets=20,cores=1,threads=1 -numa node,nodeid=0,cpus=0-4,mem=1024 -numa +node,nodeid=1,cpus=5-9,mem=1024 -numa node,nodeid=2,cpus=10-14,mem=1024 -numa +node,nodeid=3,cpus=15-19,mem=1024 -uuid 34a588c7-b0f2-4952-b39c-47fae3411439 +-no-user-config -nodefaults -chardev +socket,id=charmonitor,path=/var/run/libvirt/qemu/domain-194-Linux/monitor.sock,server,nowait + -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-hpet +-global kvm-pit.lost_tick_policy=delay -no-shutdown -boot strict=on -device +pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x8 -device +pci-bridge,chassis_nr=2,id=pci.2,bus=pci.0,addr=0x9 -device +pci-bridge,chassis_nr=3,id=pci.3,bus=pci.0,addr=0xa -device +pci-bridge,chassis_nr=4,id=pci.4,bus=pci.0,addr=0xb -device +pci-bridge,chassis_nr=5,id=pci.5,bus=pci.0,addr=0xc -device +piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device +usb-ehci,id=usb1,bus=pci.0,addr=0x10 -device +nec-usb-xhci,id=usb2,bus=pci.0,addr=0x11 -device +virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 -device +virtio-scsi-pci,id=scsi1,bus=pci.0,addr=0x4 -device +virtio-scsi-pci,id=scsi2,bus=pci.0,addr=0x5 -device +virtio-scsi-pci,id=scsi3,bus=pci.0,addr=0x6 -device +virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive +file=/mnt/sdb/xml/centos_74_x64_uefi.raw,format=raw,if=none,id=drive-virtio-disk0,cache=none + -device +virtio-blk-pci,scsi=off,bus=pci.0,addr=0x2,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 + -drive if=none,id=drive-ide0-1-1,readonly=on,cache=none -device +ide-cd,bus=ide.1,unit=1,drive=drive-ide0-1-1,id=ide0-1-1 -netdev +tap,fd=35,id=hostnet0 -device +virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:89:5d:8b,bus=pci.4,addr=0x1 +-chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 +-device usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -device +cirrus-vga,id=video0,vgamem_mb=8,bus=pci.0,addr=0x12 -device +virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xd -msg timestamp=on + +I am also very curious about this issue, in the linux kernel code, maybe double +check in function pci_bridge_check_ranges triggered this problem. + + +> +> +> +> +> +> +> +> +> diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c +> +> +> +> index b2e50c3..84b405d 100644 +> +> +> +> --- a/hw/pci/pci_bridge.c +> +> +> +> +++ b/hw/pci/pci_bridge.c +> +> +> +> @@ -256,7 +256,8 @@ void pci_bridge_write_config(PCIDevice *d, +> +> +> +> pci_default_write_config(d, address, val, len); +> +> +> +> - if (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> +> +> + if ( (val != 0xffffffff) && +> +> +> +> + (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> +> +> /* io base/limit */ +> +> +> +> ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> +> +> @@ -266,7 +267,7 @@ void pci_bridge_write_config(PCIDevice *d, +> +> +> +> ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || +> +> +> +> /* vga enable */ +> +> +> +> - ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +> +> +> +> + ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2))) { +> +> +> +> pci_bridge_update_mappings(s); +> +> +> +> } +> +> +> +> +> +> +> +> Thinks, +> +> +> +> Xu +> +> + +On Mon, Dec 10, 2018 at 03:12:53AM +0000, xuyandong wrote: +> +On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> > Hi all, +> +> > +> +> > +> +> > +> +> > In our test, we configured VM with several pci-bridges and a +> +> > virtio-net nic been attached with bus 4, +> +> > +> +> > After VM is startup, We ping this nic from host to judge if it is +> +> > working normally. Then, we hot add pci devices to this VM with bus 0. +> +> > +> +> > We found the virtio-net NIC in bus 4 is not working (can not connect) +> +> > occasionally, as it kick virtio backend failure with error below: +> +> > +> +> > Unassigned mem write 00000000fc803004 = 0x1 +> +> > +> +> > +> +> > +> +> > memory-region: pci_bridge_pci +> +> > +> +> > 0000000000000000-ffffffffffffffff (prio 0, RW): pci_bridge_pci +> +> > +> +> > 00000000fc800000-00000000fc803fff (prio 1, RW): virtio-pci +> +> > +> +> > 00000000fc800000-00000000fc800fff (prio 0, RW): +> +> > virtio-pci-common +> +> > +> +> > 00000000fc801000-00000000fc801fff (prio 0, RW): virtio-pci-isr +> +> > +> +> > 00000000fc802000-00000000fc802fff (prio 0, RW): +> +> > virtio-pci-device +> +> > +> +> > 00000000fc803000-00000000fc803fff (prio 0, RW): +> +> > virtio-pci-notify <- io mem unassigned +> +> > +> +> > ⦠+> +> > +> +> > +> +> > +> +> > We caught an exceptional address changing while this problem happened, +> +> > show as +> +> > follow: +> +> > +> +> > Before pci_bridge_update_mappingsï¼ +> +> > +> +> > 00000000fc000000-00000000fc1fffff (prio 1, RW): alias +> +> > pci_bridge_pref_mem @pci_bridge_pci 00000000fc000000-00000000fc1fffff +> +> > +> +> > 00000000fc200000-00000000fc3fffff (prio 1, RW): alias +> +> > pci_bridge_pref_mem @pci_bridge_pci 00000000fc200000-00000000fc3fffff +> +> > +> +> > 00000000fc400000-00000000fc5fffff (prio 1, RW): alias +> +> > pci_bridge_pref_mem @pci_bridge_pci 00000000fc400000-00000000fc5fffff +> +> > +> +> > 00000000fc600000-00000000fc7fffff (prio 1, RW): alias +> +> > pci_bridge_pref_mem @pci_bridge_pci 00000000fc600000-00000000fc7fffff +> +> > +> +> > 00000000fc800000-00000000fc9fffff (prio 1, RW): alias +> +> > pci_bridge_pref_mem @pci_bridge_pci 00000000fc800000-00000000fc9fffff +> +> > <- correct Adress Spce +> +> > +> +> > 00000000fca00000-00000000fcbfffff (prio 1, RW): alias +> +> > pci_bridge_pref_mem @pci_bridge_pci 00000000fca00000-00000000fcbfffff +> +> > +> +> > 00000000fcc00000-00000000fcdfffff (prio 1, RW): alias +> +> > pci_bridge_pref_mem @pci_bridge_pci 00000000fcc00000-00000000fcdfffff +> +> > +> +> > 00000000fce00000-00000000fcffffff (prio 1, RW): alias +> +> > pci_bridge_pref_mem @pci_bridge_pci 00000000fce00000-00000000fcffffff +> +> > +> +> > +> +> > +> +> > After pci_bridge_update_mappingsï¼ +> +> > +> +> > 00000000fda00000-00000000fdbfffff (prio 1, RW): alias +> +> > pci_bridge_mem @pci_bridge_pci 00000000fda00000-00000000fdbfffff +> +> > +> +> > 00000000fdc00000-00000000fddfffff (prio 1, RW): alias +> +> > pci_bridge_mem @pci_bridge_pci 00000000fdc00000-00000000fddfffff +> +> > +> +> > 00000000fde00000-00000000fdffffff (prio 1, RW): alias +> +> > pci_bridge_mem @pci_bridge_pci 00000000fde00000-00000000fdffffff +> +> > +> +> > 00000000fe000000-00000000fe1fffff (prio 1, RW): alias +> +> > pci_bridge_mem @pci_bridge_pci 00000000fe000000-00000000fe1fffff +> +> > +> +> > 00000000fe200000-00000000fe3fffff (prio 1, RW): alias +> +> > pci_bridge_mem @pci_bridge_pci 00000000fe200000-00000000fe3fffff +> +> > +> +> > 00000000fe400000-00000000fe5fffff (prio 1, RW): alias +> +> > pci_bridge_mem @pci_bridge_pci 00000000fe400000-00000000fe5fffff +> +> > +> +> > 00000000fe600000-00000000fe7fffff (prio 1, RW): alias +> +> > pci_bridge_mem @pci_bridge_pci 00000000fe600000-00000000fe7fffff +> +> > +> +> > 00000000fe800000-00000000fe9fffff (prio 1, RW): alias +> +> > pci_bridge_mem @pci_bridge_pci 00000000fe800000-00000000fe9fffff +> +> > +> +> > fffffffffc800000-fffffffffc800000 (prio 1, RW): alias +> +> > pci_bridge_pref_mem +> +> > @pci_bridge_pci fffffffffc800000-fffffffffc800000 <- Exceptional Adress +> +> Space +> +> +> +> This one is empty though right? +> +> +> +> > +> +> > +> +> > We have figured out why this address becomes this value, according to +> +> > pci spec, pci driver can get BAR address size by writing 0xffffffff +> +> > to +> +> > +> +> > the pci register firstly, and then read back the value from this register. +> +> +> +> +> +> OK however as you show below the BAR being sized is the BAR if a bridge. Are +> +> you then adding a bridge device by hotplug? +> +> +No, I just simply hot plugged a VFIO device to Bus 0, another interesting +> +phenomenon is +> +If I hot plug the device to other bus, this doesn't happened. +> +> +> +> +> +> +> > We didn't handle this value specially while process pci write in +> +> > qemu, the function call stack is: +> +> > +> +> > Pci_bridge_dev_write_config +> +> > +> +> > -> pci_bridge_write_config +> +> > +> +> > -> pci_default_write_config (we update the config[address] value here +> +> > -> to +> +> > fffffffffc800000, which should be 0xfc800000 ) +> +> > +> +> > -> pci_bridge_update_mappings +> +> > +> +> > ->pci_bridge_region_del(br, br->windows); +> +> > +> +> > -> pci_bridge_region_init +> +> > +> +> > -> +> +> > pci_bridge_init_alias (here pci_bridge_get_base, we use the wrong +> +> > value +> +> > fffffffffc800000) +> +> > +> +> > -> +> +> > memory_region_transaction_commit +> +> > +> +> > +> +> > +> +> > So, as we can see, we use the wrong base address in qemu to update the +> +> > memory regions, though, we update the base address to +> +> > +> +> > The correct value after pci driver in VM write the original value +> +> > back, the virtio NIC in bus 4 may still sends net packets concurrently +> +> > with +> +> > +> +> > The wrong memory region address. +> +> > +> +> > +> +> > +> +> > We have tried to skip the memory region update action in qemu while +> +> > detect pci write with 0xffffffff value, and it does work, but +> +> > +> +> > This seems to be not gently. +> +> +> +> For sure. But I'm still puzzled as to why does Linux try to size the BAR of +> +> the +> +> bridge while a device behind it is used. +> +> +> +> Can you pls post your QEMU command line? +> +> +My QEMU command line: +> +/root/xyd/qemu-system-x86_64 -name guest=Linux,debug-threads=on -S -object +> +secret,id=masterKey0,format=raw,file=/var/run/libvirt/qemu/domain-194-Linux/master-key.aes +> +-machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off -cpu +> +host,+kvm_pv_eoi -bios /usr/share/OVMF/OVMF.fd -m +> +size=4194304k,slots=256,maxmem=33554432k -realtime mlock=off -smp +> +20,sockets=20,cores=1,threads=1 -numa node,nodeid=0,cpus=0-4,mem=1024 -numa +> +node,nodeid=1,cpus=5-9,mem=1024 -numa node,nodeid=2,cpus=10-14,mem=1024 -numa +> +node,nodeid=3,cpus=15-19,mem=1024 -uuid 34a588c7-b0f2-4952-b39c-47fae3411439 +> +-no-user-config -nodefaults -chardev +> +socket,id=charmonitor,path=/var/run/libvirt/qemu/domain-194-Linux/monitor.sock,server,nowait +> +-mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-hpet +> +-global kvm-pit.lost_tick_policy=delay -no-shutdown -boot strict=on -device +> +pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x8 -device +> +pci-bridge,chassis_nr=2,id=pci.2,bus=pci.0,addr=0x9 -device +> +pci-bridge,chassis_nr=3,id=pci.3,bus=pci.0,addr=0xa -device +> +pci-bridge,chassis_nr=4,id=pci.4,bus=pci.0,addr=0xb -device +> +pci-bridge,chassis_nr=5,id=pci.5,bus=pci.0,addr=0xc -device +> +piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device +> +usb-ehci,id=usb1,bus=pci.0,addr=0x10 -device +> +nec-usb-xhci,id=usb2,bus=pci.0,addr=0x11 -device +> +virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 -device +> +virtio-scsi-pci,id=scsi1,bus=pci.0,addr=0x4 -device +> +virtio-scsi-pci,id=scsi2,bus=pci.0,addr=0x5 -device +> +virtio-scsi-pci,id=scsi3,bus=pci.0,addr=0x6 -device +> +virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive +> +file=/mnt/sdb/xml/centos_74_x64_uefi.raw,format=raw,if=none,id=drive-virtio-disk0,cache=none +> +-device +> +virtio-blk-pci,scsi=off,bus=pci.0,addr=0x2,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 +> +-drive if=none,id=drive-ide0-1-1,readonly=on,cache=none -device +> +ide-cd,bus=ide.1,unit=1,drive=drive-ide0-1-1,id=ide0-1-1 -netdev +> +tap,fd=35,id=hostnet0 -device +> +virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:89:5d:8b,bus=pci.4,addr=0x1 +> +-chardev pty,id=charserial0 -device +> +isa-serial,chardev=charserial0,id=serial0 -device +> +usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -device +> +cirrus-vga,id=video0,vgamem_mb=8,bus=pci.0,addr=0x12 -device +> +virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xd -msg timestamp=on +> +> +I am also very curious about this issue, in the linux kernel code, maybe +> +double check in function pci_bridge_check_ranges triggered this problem. +If you can get the stacktrace in Linux when it tries to write this +fffff value, that would be quite helpful. + + +> +> +> +> +> +> +> +> +> > +> +> > +> +> > diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c +> +> > +> +> > index b2e50c3..84b405d 100644 +> +> > +> +> > --- a/hw/pci/pci_bridge.c +> +> > +> +> > +++ b/hw/pci/pci_bridge.c +> +> > +> +> > @@ -256,7 +256,8 @@ void pci_bridge_write_config(PCIDevice *d, +> +> > +> +> > pci_default_write_config(d, address, val, len); +> +> > +> +> > - if (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> > +> +> > + if ( (val != 0xffffffff) && +> +> > +> +> > + (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> > +> +> > /* io base/limit */ +> +> > +> +> > ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> > +> +> > @@ -266,7 +267,7 @@ void pci_bridge_write_config(PCIDevice *d, +> +> > +> +> > ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || +> +> > +> +> > /* vga enable */ +> +> > +> +> > - ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +> +> > +> +> > + ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2))) { +> +> > +> +> > pci_bridge_update_mappings(s); +> +> > +> +> > } +> +> > +> +> > +> +> > +> +> > Thinks, +> +> > +> +> > Xu +> +> > + +On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> > > Hi all, +> +> > > +> +> > > +> +> > > +> +> > > In our test, we configured VM with several pci-bridges and a +> +> > > virtio-net nic been attached with bus 4, +> +> > > +> +> > > After VM is startup, We ping this nic from host to judge if it is +> +> > > working normally. Then, we hot add pci devices to this VM with bus 0. +> +> > > +> +> > > We found the virtio-net NIC in bus 4 is not working (can not +> +> > > connect) occasionally, as it kick virtio backend failure with error +> +> > > below: +> +> > > +> +> > > Unassigned mem write 00000000fc803004 = 0x1 +> +> > > +> +> > > +> +> > > +> +> > > memory-region: pci_bridge_pci +> +> > > +> +> > > 0000000000000000-ffffffffffffffff (prio 0, RW): pci_bridge_pci +> +> > > +> +> > > 00000000fc800000-00000000fc803fff (prio 1, RW): virtio-pci +> +> > > +> +> > > 00000000fc800000-00000000fc800fff (prio 0, RW): +> +> > > virtio-pci-common +> +> > > +> +> > > 00000000fc801000-00000000fc801fff (prio 0, RW): +> +> > > virtio-pci-isr +> +> > > +> +> > > 00000000fc802000-00000000fc802fff (prio 0, RW): +> +> > > virtio-pci-device +> +> > > +> +> > > 00000000fc803000-00000000fc803fff (prio 0, RW): +> +> > > virtio-pci-notify <- io mem unassigned +> +> > > +> +> > > ⦠+> +> > > +> +> > > +> +> > > +> +> > > We caught an exceptional address changing while this problem +> +> > > happened, show as +> +> > > follow: +> +> > > +> +> > > Before pci_bridge_update_mappingsï¼ +> +> > > +> +> > > 00000000fc000000-00000000fc1fffff (prio 1, RW): alias +> +> > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > 00000000fc000000-00000000fc1fffff +> +> > > +> +> > > 00000000fc200000-00000000fc3fffff (prio 1, RW): alias +> +> > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > 00000000fc200000-00000000fc3fffff +> +> > > +> +> > > 00000000fc400000-00000000fc5fffff (prio 1, RW): alias +> +> > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > 00000000fc400000-00000000fc5fffff +> +> > > +> +> > > 00000000fc600000-00000000fc7fffff (prio 1, RW): alias +> +> > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > 00000000fc600000-00000000fc7fffff +> +> > > +> +> > > 00000000fc800000-00000000fc9fffff (prio 1, RW): alias +> +> > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > 00000000fc800000-00000000fc9fffff +> +> > > <- correct Adress Spce +> +> > > +> +> > > 00000000fca00000-00000000fcbfffff (prio 1, RW): alias +> +> > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > 00000000fca00000-00000000fcbfffff +> +> > > +> +> > > 00000000fcc00000-00000000fcdfffff (prio 1, RW): alias +> +> > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > 00000000fcc00000-00000000fcdfffff +> +> > > +> +> > > 00000000fce00000-00000000fcffffff (prio 1, RW): alias +> +> > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > 00000000fce00000-00000000fcffffff +> +> > > +> +> > > +> +> > > +> +> > > After pci_bridge_update_mappingsï¼ +> +> > > +> +> > > 00000000fda00000-00000000fdbfffff (prio 1, RW): alias +> +> > > pci_bridge_mem @pci_bridge_pci 00000000fda00000-00000000fdbfffff +> +> > > +> +> > > 00000000fdc00000-00000000fddfffff (prio 1, RW): alias +> +> > > pci_bridge_mem @pci_bridge_pci 00000000fdc00000-00000000fddfffff +> +> > > +> +> > > 00000000fde00000-00000000fdffffff (prio 1, RW): alias +> +> > > pci_bridge_mem @pci_bridge_pci 00000000fde00000-00000000fdffffff +> +> > > +> +> > > 00000000fe000000-00000000fe1fffff (prio 1, RW): alias +> +> > > pci_bridge_mem @pci_bridge_pci 00000000fe000000-00000000fe1fffff +> +> > > +> +> > > 00000000fe200000-00000000fe3fffff (prio 1, RW): alias +> +> > > pci_bridge_mem @pci_bridge_pci 00000000fe200000-00000000fe3fffff +> +> > > +> +> > > 00000000fe400000-00000000fe5fffff (prio 1, RW): alias +> +> > > pci_bridge_mem @pci_bridge_pci 00000000fe400000-00000000fe5fffff +> +> > > +> +> > > 00000000fe600000-00000000fe7fffff (prio 1, RW): alias +> +> > > pci_bridge_mem @pci_bridge_pci 00000000fe600000-00000000fe7fffff +> +> > > +> +> > > 00000000fe800000-00000000fe9fffff (prio 1, RW): alias +> +> > > pci_bridge_mem @pci_bridge_pci 00000000fe800000-00000000fe9fffff +> +> > > +> +> > > fffffffffc800000-fffffffffc800000 (prio 1, RW): alias +> +pci_bridge_pref_mem +> +> > > @pci_bridge_pci fffffffffc800000-fffffffffc800000 <- Exceptional +> +> > > Adress +> +> > Space +> +> > +> +> > This one is empty though right? +> +> > +> +> > > +> +> > > +> +> > > We have figured out why this address becomes this value, +> +> > > according to pci spec, pci driver can get BAR address size by +> +> > > writing 0xffffffff to +> +> > > +> +> > > the pci register firstly, and then read back the value from this +> +> > > register. +> +> > +> +> > +> +> > OK however as you show below the BAR being sized is the BAR if a +> +> > bridge. Are you then adding a bridge device by hotplug? +> +> +> +> No, I just simply hot plugged a VFIO device to Bus 0, another +> +> interesting phenomenon is If I hot plug the device to other bus, this +> +> doesn't +> +happened. +> +> +> +> > +> +> > +> +> > > We didn't handle this value specially while process pci write in +> +> > > qemu, the function call stack is: +> +> > > +> +> > > Pci_bridge_dev_write_config +> +> > > +> +> > > -> pci_bridge_write_config +> +> > > +> +> > > -> pci_default_write_config (we update the config[address] value +> +> > > -> here to +> +> > > fffffffffc800000, which should be 0xfc800000 ) +> +> > > +> +> > > -> pci_bridge_update_mappings +> +> > > +> +> > > ->pci_bridge_region_del(br, br->windows); +> +> > > +> +> > > -> pci_bridge_region_init +> +> > > +> +> > > -> +> +> > > pci_bridge_init_alias (here pci_bridge_get_base, we use the wrong +> +> > > value +> +> > > fffffffffc800000) +> +> > > +> +> > > -> +> +> > > memory_region_transaction_commit +> +> > > +> +> > > +> +> > > +> +> > > So, as we can see, we use the wrong base address in qemu to update +> +> > > the memory regions, though, we update the base address to +> +> > > +> +> > > The correct value after pci driver in VM write the original value +> +> > > back, the virtio NIC in bus 4 may still sends net packets +> +> > > concurrently with +> +> > > +> +> > > The wrong memory region address. +> +> > > +> +> > > +> +> > > +> +> > > We have tried to skip the memory region update action in qemu +> +> > > while detect pci write with 0xffffffff value, and it does work, +> +> > > but +> +> > > +> +> > > This seems to be not gently. +> +> > +> +> > For sure. But I'm still puzzled as to why does Linux try to size the +> +> > BAR of the bridge while a device behind it is used. +> +> > +> +> > Can you pls post your QEMU command line? +> +> +> +> My QEMU command line: +> +> /root/xyd/qemu-system-x86_64 -name guest=Linux,debug-threads=on -S +> +> -object +> +> secret,id=masterKey0,format=raw,file=/var/run/libvirt/qemu/domain-194- +> +> Linux/master-key.aes -machine +> +> pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off -cpu +> +> host,+kvm_pv_eoi -bios /usr/share/OVMF/OVMF.fd -m +> +> size=4194304k,slots=256,maxmem=33554432k -realtime mlock=off -smp +> +> 20,sockets=20,cores=1,threads=1 -numa node,nodeid=0,cpus=0-4,mem=1024 +> +> -numa node,nodeid=1,cpus=5-9,mem=1024 -numa +> +> node,nodeid=2,cpus=10-14,mem=1024 -numa +> +> node,nodeid=3,cpus=15-19,mem=1024 -uuid +> +> 34a588c7-b0f2-4952-b39c-47fae3411439 -no-user-config -nodefaults +> +> -chardev +> +> socket,id=charmonitor,path=/var/run/libvirt/qemu/domain-194-Linux/moni +> +> tor.sock,server,nowait -mon +> +> chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-hpet +> +> -global kvm-pit.lost_tick_policy=delay -no-shutdown -boot strict=on +> +> -device pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x8 -device +> +> pci-bridge,chassis_nr=2,id=pci.2,bus=pci.0,addr=0x9 -device +> +> pci-bridge,chassis_nr=3,id=pci.3,bus=pci.0,addr=0xa -device +> +> pci-bridge,chassis_nr=4,id=pci.4,bus=pci.0,addr=0xb -device +> +> pci-bridge,chassis_nr=5,id=pci.5,bus=pci.0,addr=0xc -device +> +> piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device +> +> usb-ehci,id=usb1,bus=pci.0,addr=0x10 -device +> +> nec-usb-xhci,id=usb2,bus=pci.0,addr=0x11 -device +> +> virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 -device +> +> virtio-scsi-pci,id=scsi1,bus=pci.0,addr=0x4 -device +> +> virtio-scsi-pci,id=scsi2,bus=pci.0,addr=0x5 -device +> +> virtio-scsi-pci,id=scsi3,bus=pci.0,addr=0x6 -device +> +> virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive +> +> file=/mnt/sdb/xml/centos_74_x64_uefi.raw,format=raw,if=none,id=drive-v +> +> irtio-disk0,cache=none -device +> +> virtio-blk-pci,scsi=off,bus=pci.0,addr=0x2,drive=drive-virtio-disk0,id +> +> =virtio-disk0,bootindex=1 -drive +> +> if=none,id=drive-ide0-1-1,readonly=on,cache=none -device +> +> ide-cd,bus=ide.1,unit=1,drive=drive-ide0-1-1,id=ide0-1-1 -netdev +> +> tap,fd=35,id=hostnet0 -device +> +> virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:89:5d:8b,bus=pci.4 +> +> ,addr=0x1 -chardev pty,id=charserial0 -device +> +> isa-serial,chardev=charserial0,id=serial0 -device +> +> usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -device +> +> cirrus-vga,id=video0,vgamem_mb=8,bus=pci.0,addr=0x12 -device +> +> virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xd -msg timestamp=on +> +> +> +> I am also very curious about this issue, in the linux kernel code, maybe +> +> double +> +check in function pci_bridge_check_ranges triggered this problem. +> +> +If you can get the stacktrace in Linux when it tries to write this fffff +> +value, that +> +would be quite helpful. +> +After I add mdelay(100) in function pci_bridge_check_ranges, this phenomenon is +easier to reproduce, below is my modify in kernel: +diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c +index cb389277..86e232d 100644 +--- a/drivers/pci/setup-bus.c ++++ b/drivers/pci/setup-bus.c +@@ -27,7 +27,7 @@ + #include <linux/slab.h> + #include <linux/acpi.h> + #include "pci.h" +- ++#include <linux/delay.h> + unsigned int pci_flags; + + struct pci_dev_resource { +@@ -787,6 +787,9 @@ static void pci_bridge_check_ranges(struct pci_bus *bus) + pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, + 0xffffffff); + pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &tmp); ++ mdelay(100); ++ printk(KERN_ERR "sleep\n"); ++ dump_stack(); + if (!tmp) + b_res[2].flags &= ~IORESOURCE_MEM_64; + pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, + +After hot plugging, we get the following log: + +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:14.0: BAR 0: assigned [mem +0xc2360000-0xc237ffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:14.0: BAR 3: assigned [mem +0xc2328000-0xc232bfff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:16 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:17 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:18 uefi-linux kernel: sleep +Dec 11 09:28:18 uefi-linux kernel: CPU: 16 PID: 502 Comm: kworker/u40:1 Not +tainted 4.11.0-rc3+ #11 +Dec 11 09:28:18 uefi-linux kernel: Hardware name: QEMU Standard PC (i440FX + +PIIX, 1996), BIOS 0.0.0 02/06/2015 +Dec 11 09:28:18 uefi-linux kernel: Workqueue: kacpi_hotplug acpi_hotplug_work_fn +Dec 11 09:28:18 uefi-linux kernel: Call Trace: +Dec 11 09:28:18 uefi-linux kernel: dump_stack+0x63/0x87 +Dec 11 09:28:18 uefi-linux kernel: __pci_bus_size_bridges+0x931/0x960 +Dec 11 09:28:18 uefi-linux kernel: ? dev_printk+0x4d/0x50 +Dec 11 09:28:18 uefi-linux kernel: enable_slot+0x140/0x2f0 +Dec 11 09:28:18 uefi-linux kernel: ? __pm_runtime_resume+0x5c/0x80 +Dec 11 09:28:18 uefi-linux kernel: ? trim_stale_devices+0x9a/0x120 +Dec 11 09:28:18 uefi-linux kernel: acpiphp_check_bridge.part.6+0xf5/0x120 +Dec 11 09:28:18 uefi-linux kernel: acpiphp_hotplug_notify+0x145/0x1c0 +Dec 11 09:28:18 uefi-linux kernel: ? acpiphp_post_dock_fixup+0xc0/0xc0 +Dec 11 09:28:18 uefi-linux kernel: acpi_device_hotplug+0x3a6/0x3f3 +Dec 11 09:28:18 uefi-linux kernel: acpi_hotplug_work_fn+0x1e/0x29 +Dec 11 09:28:18 uefi-linux kernel: process_one_work+0x165/0x410 +Dec 11 09:28:18 uefi-linux kernel: worker_thread+0x137/0x4c0 +Dec 11 09:28:18 uefi-linux kernel: kthread+0x101/0x140 +Dec 11 09:28:18 uefi-linux kernel: ? rescuer_thread+0x380/0x380 +Dec 11 09:28:18 uefi-linux kernel: ? kthread_park+0x90/0x90 +Dec 11 09:28:18 uefi-linux kernel: ret_from_fork+0x2c/0x40 +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:18 uefi-linux kernel: sleep +Dec 11 09:28:18 uefi-linux kernel: CPU: 16 PID: 502 Comm: kworker/u40:1 Not +tainted 4.11.0-rc3+ #11 +Dec 11 09:28:18 uefi-linux kernel: Hardware name: QEMU Standard PC (i440FX + +PIIX, 1996), BIOS 0.0.0 02/06/2015 +Dec 11 09:28:18 uefi-linux kernel: Workqueue: kacpi_hotplug acpi_hotplug_work_fn +Dec 11 09:28:18 uefi-linux kernel: Call Trace: +Dec 11 09:28:18 uefi-linux kernel: dump_stack+0x63/0x87 +Dec 11 09:28:18 uefi-linux kernel: __pci_bus_size_bridges+0x931/0x960 +Dec 11 09:28:18 uefi-linux kernel: ? dev_printk+0x4d/0x50 +Dec 11 09:28:18 uefi-linux kernel: enable_slot+0x140/0x2f0 +Dec 11 09:28:18 uefi-linux kernel: ? __pm_runtime_resume+0x5c/0x80 +Dec 11 09:28:18 uefi-linux kernel: ? trim_stale_devices+0x9a/0x120 +Dec 11 09:28:18 uefi-linux kernel: acpiphp_check_bridge.part.6+0xf5/0x120 +Dec 11 09:28:18 uefi-linux kernel: acpiphp_hotplug_notify+0x145/0x1c0 +Dec 11 09:28:18 uefi-linux kernel: ? acpiphp_post_dock_fixup+0xc0/0xc0 +Dec 11 09:28:18 uefi-linux kernel: acpi_device_hotplug+0x3a6/0x3f3 +Dec 11 09:28:18 uefi-linux kernel: acpi_hotplug_work_fn+0x1e/0x29 +Dec 11 09:28:18 uefi-linux kernel: process_one_work+0x165/0x410 +Dec 11 09:28:18 uefi-linux kernel: worker_thread+0x137/0x4c0 +Dec 11 09:28:18 uefi-linux kernel: kthread+0x101/0x140 +Dec 11 09:28:18 uefi-linux kernel: ? rescuer_thread+0x380/0x380 +Dec 11 09:28:18 uefi-linux kernel: ? kthread_park+0x90/0x90 +Dec 11 09:28:18 uefi-linux kernel: ret_from_fork+0x2c/0x40 +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:18 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:19 uefi-linux kernel: sleep +Dec 11 09:28:19 uefi-linux kernel: CPU: 17 PID: 502 Comm: kworker/u40:1 Not +tainted 4.11.0-rc3+ #11 +Dec 11 09:28:19 uefi-linux kernel: Hardware name: QEMU Standard PC (i440FX + +PIIX, 1996), BIOS 0.0.0 02/06/2015 +Dec 11 09:28:19 uefi-linux kernel: Workqueue: kacpi_hotplug acpi_hotplug_work_fn +Dec 11 09:28:19 uefi-linux kernel: Call Trace: +Dec 11 09:28:19 uefi-linux kernel: dump_stack+0x63/0x87 +Dec 11 09:28:19 uefi-linux kernel: __pci_bus_size_bridges+0x931/0x960 +Dec 11 09:28:19 uefi-linux kernel: ? dev_printk+0x4d/0x50 +Dec 11 09:28:19 uefi-linux kernel: enable_slot+0x140/0x2f0 +Dec 11 09:28:19 uefi-linux kernel: ? __pm_runtime_resume+0x5c/0x80 +Dec 11 09:28:19 uefi-linux kernel: ? trim_stale_devices+0x9a/0x120 +Dec 11 09:28:19 uefi-linux kernel: acpiphp_check_bridge.part.6+0xf5/0x120 +Dec 11 09:28:19 uefi-linux kernel: acpiphp_hotplug_notify+0x145/0x1c0 +Dec 11 09:28:19 uefi-linux kernel: ? acpiphp_post_dock_fixup+0xc0/0xc0 +Dec 11 09:28:19 uefi-linux kernel: acpi_device_hotplug+0x3a6/0x3f3 +Dec 11 09:28:19 uefi-linux kernel: acpi_hotplug_work_fn+0x1e/0x29 +Dec 11 09:28:19 uefi-linux kernel: process_one_work+0x165/0x410 +Dec 11 09:28:19 uefi-linux kernel: worker_thread+0x137/0x4c0 +Dec 11 09:28:19 uefi-linux kernel: kthread+0x101/0x140 +Dec 11 09:28:19 uefi-linux kernel: ? rescuer_thread+0x380/0x380 +Dec 11 09:28:19 uefi-linux kernel: ? kthread_park+0x90/0x90 +Dec 11 09:28:19 uefi-linux kernel: ret_from_fork+0x2c/0x40 +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:19 uefi-linux kernel: sleep +Dec 11 09:28:19 uefi-linux kernel: CPU: 17 PID: 502 Comm: kworker/u40:1 Not +tainted 4.11.0-rc3+ #11 +Dec 11 09:28:19 uefi-linux kernel: Hardware name: QEMU Standard PC (i440FX + +PIIX, 1996), BIOS 0.0.0 02/06/2015 +Dec 11 09:28:19 uefi-linux kernel: Workqueue: kacpi_hotplug acpi_hotplug_work_fn +Dec 11 09:28:19 uefi-linux kernel: Call Trace: +Dec 11 09:28:19 uefi-linux kernel: dump_stack+0x63/0x87 +Dec 11 09:28:19 uefi-linux kernel: __pci_bus_size_bridges+0x931/0x960 +Dec 11 09:28:19 uefi-linux kernel: ? pci_conf1_read+0xba/0x100 +Dec 11 09:28:19 uefi-linux kernel: __pci_bus_size_bridges+0xe9/0x960 +Dec 11 09:28:19 uefi-linux kernel: ? dev_printk+0x4d/0x50 +Dec 11 09:28:19 uefi-linux kernel: ? pcibios_allocate_rom_resources+0x45/0x80 +Dec 11 09:28:19 uefi-linux kernel: enable_slot+0x140/0x2f0 +Dec 11 09:28:19 uefi-linux kernel: ? trim_stale_devices+0x9a/0x120 +Dec 11 09:28:19 uefi-linux kernel: ? __pm_runtime_resume+0x5c/0x80 +Dec 11 09:28:19 uefi-linux kernel: ? trim_stale_devices+0x9a/0x120 +Dec 11 09:28:19 uefi-linux kernel: acpiphp_check_bridge.part.6+0xf5/0x120 +Dec 11 09:28:19 uefi-linux kernel: acpiphp_hotplug_notify+0x145/0x1c0 +Dec 11 09:28:19 uefi-linux kernel: ? acpiphp_post_dock_fixup+0xc0/0xc0 +Dec 11 09:28:19 uefi-linux kernel: acpi_device_hotplug+0x3a6/0x3f3 +Dec 11 09:28:19 uefi-linux kernel: acpi_hotplug_work_fn+0x1e/0x29 +Dec 11 09:28:19 uefi-linux kernel: process_one_work+0x165/0x410 +Dec 11 09:28:19 uefi-linux kernel: worker_thread+0x137/0x4c0 +Dec 11 09:28:19 uefi-linux kernel: kthread+0x101/0x140 +Dec 11 09:28:19 uefi-linux kernel: ? rescuer_thread+0x380/0x380 +Dec 11 09:28:19 uefi-linux kernel: ? kthread_park+0x90/0x90 +Dec 11 09:28:19 uefi-linux kernel: ret_from_fork+0x2c/0x40 +Dec 11 09:28:19 uefi-linux kernel: sleep +Dec 11 09:28:19 uefi-linux kernel: CPU: 17 PID: 502 Comm: kworker/u40:1 Not +tainted 4.11.0-rc3+ #11 +Dec 11 09:28:19 uefi-linux kernel: Hardware name: QEMU Standard PC (i440FX + +PIIX, 1996), BIOS 0.0.0 02/06/2015 +Dec 11 09:28:19 uefi-linux kernel: Workqueue: kacpi_hotplug acpi_hotplug_work_fn +Dec 11 09:28:19 uefi-linux kernel: Call Trace: +Dec 11 09:28:19 uefi-linux kernel: dump_stack+0x63/0x87 +Dec 11 09:28:19 uefi-linux kernel: __pci_bus_size_bridges+0x931/0x960 +Dec 11 09:28:19 uefi-linux kernel: ? dev_printk+0x4d/0x50 +Dec 11 09:28:19 uefi-linux kernel: enable_slot+0x140/0x2f0 +Dec 11 09:28:19 uefi-linux kernel: ? trim_stale_devices+0x9a/0x120 +Dec 11 09:28:19 uefi-linux kernel: ? __pm_runtime_resume+0x5c/0x80 +Dec 11 09:28:19 uefi-linux kernel: ? trim_stale_devices+0x9a/0x120 +Dec 11 09:28:19 uefi-linux kernel: acpiphp_check_bridge.part.6+0xf5/0x120 +Dec 11 09:28:19 uefi-linux kernel: acpiphp_hotplug_notify+0x145/0x1c0 +Dec 11 09:28:19 uefi-linux kernel: ? acpiphp_post_dock_fixup+0xc0/0xc0 +Dec 11 09:28:19 uefi-linux kernel: acpi_device_hotplug+0x3a6/0x3f3 +Dec 11 09:28:19 uefi-linux kernel: acpi_hotplug_work_fn+0x1e/0x29 +Dec 11 09:28:19 uefi-linux kernel: process_one_work+0x165/0x410 +Dec 11 09:28:19 uefi-linux kernel: worker_thread+0x137/0x4c0 +Dec 11 09:28:19 uefi-linux kernel: kthread+0x101/0x140 +Dec 11 09:28:19 uefi-linux kernel: ? rescuer_thread+0x380/0x380 +Dec 11 09:28:19 uefi-linux kernel: ? kthread_park+0x90/0x90 +Dec 11 09:28:19 uefi-linux kernel: ret_from_fork+0x2c/0x40 +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:19 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:20 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:20 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:20 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:21 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:21 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:21 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:21 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:21 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:21 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 lost sync at byte 1 +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: psmouse serio1: VMMouse at +isa0060/serio1/input0 - driver resynced. +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:21 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:08.0: PCI bridge to [bus 01] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:08.0: bridge window [io +0xf000-0xffff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2800000-0xc29fffff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:08.0: bridge window [mem +0xc2b00000-0xc2cfffff 64bit pref] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:09.0: PCI bridge to [bus 02] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:09.0: bridge window [io +0xe000-0xefff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2600000-0xc27fffff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:09.0: bridge window [mem +0xc2d00000-0xc2efffff 64bit pref] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:0a.0: PCI bridge to [bus 03] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:0a.0: bridge window [io +0xd000-0xdfff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2400000-0xc25fffff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:00:0a.0: bridge window [mem +0xc2f00000-0xc30fffff 64bit pref] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:04:0c.0: PCI bridge to [bus 05] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:04:0c.0: bridge window [io +0xc000-0xcfff] +Dec 11 09:28:22 uefi-linux kernel: pci 0000:04:0c.0: bridge window [mem +0xc2000000-0xc21fffff] + +> +> +> +> > +> +> > +> +> > +> +> > > +> +> > > +> +> > > diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c +> +> > > +> +> > > index b2e50c3..84b405d 100644 +> +> > > +> +> > > --- a/hw/pci/pci_bridge.c +> +> > > +> +> > > +++ b/hw/pci/pci_bridge.c +> +> > > +> +> > > @@ -256,7 +256,8 @@ void pci_bridge_write_config(PCIDevice *d, +> +> > > +> +> > > pci_default_write_config(d, address, val, len); +> +> > > +> +> > > - if (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> > > +> +> > > + if ( (val != 0xffffffff) && +> +> > > +> +> > > + (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> > > +> +> > > /* io base/limit */ +> +> > > +> +> > > ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> > > +> +> > > @@ -266,7 +267,7 @@ void pci_bridge_write_config(PCIDevice *d, +> +> > > +> +> > > ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || +> +> > > +> +> > > /* vga enable */ +> +> > > +> +> > > - ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +> +> > > +> +> > > + ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2))) { +> +> > > +> +> > > pci_bridge_update_mappings(s); +> +> > > +> +> > > } +> +> > > +> +> > > +> +> > > +> +> > > Thinks, +> +> > > +> +> > > Xu +> +> > > + +On Tue, Dec 11, 2018 at 01:47:37AM +0000, xuyandong wrote: +> +On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> > > > Hi all, +> +> > > > +> +> > > > +> +> > > > +> +> > > > In our test, we configured VM with several pci-bridges and a +> +> > > > virtio-net nic been attached with bus 4, +> +> > > > +> +> > > > After VM is startup, We ping this nic from host to judge if it is +> +> > > > working normally. Then, we hot add pci devices to this VM with bus 0. +> +> > > > +> +> > > > We found the virtio-net NIC in bus 4 is not working (can not +> +> > > > connect) occasionally, as it kick virtio backend failure with error +> +> > > > below: +> +> > > > +> +> > > > Unassigned mem write 00000000fc803004 = 0x1 +> +> > > > +> +> > > > +> +> > > > +> +> > > > memory-region: pci_bridge_pci +> +> > > > +> +> > > > 0000000000000000-ffffffffffffffff (prio 0, RW): pci_bridge_pci +> +> > > > +> +> > > > 00000000fc800000-00000000fc803fff (prio 1, RW): virtio-pci +> +> > > > +> +> > > > 00000000fc800000-00000000fc800fff (prio 0, RW): +> +> > > > virtio-pci-common +> +> > > > +> +> > > > 00000000fc801000-00000000fc801fff (prio 0, RW): +> +> > > > virtio-pci-isr +> +> > > > +> +> > > > 00000000fc802000-00000000fc802fff (prio 0, RW): +> +> > > > virtio-pci-device +> +> > > > +> +> > > > 00000000fc803000-00000000fc803fff (prio 0, RW): +> +> > > > virtio-pci-notify <- io mem unassigned +> +> > > > +> +> > > > ⦠+> +> > > > +> +> > > > +> +> > > > +> +> > > > We caught an exceptional address changing while this problem +> +> > > > happened, show as +> +> > > > follow: +> +> > > > +> +> > > > Before pci_bridge_update_mappingsï¼ +> +> > > > +> +> > > > 00000000fc000000-00000000fc1fffff (prio 1, RW): alias +> +> > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > 00000000fc000000-00000000fc1fffff +> +> > > > +> +> > > > 00000000fc200000-00000000fc3fffff (prio 1, RW): alias +> +> > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > 00000000fc200000-00000000fc3fffff +> +> > > > +> +> > > > 00000000fc400000-00000000fc5fffff (prio 1, RW): alias +> +> > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > 00000000fc400000-00000000fc5fffff +> +> > > > +> +> > > > 00000000fc600000-00000000fc7fffff (prio 1, RW): alias +> +> > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > 00000000fc600000-00000000fc7fffff +> +> > > > +> +> > > > 00000000fc800000-00000000fc9fffff (prio 1, RW): alias +> +> > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > 00000000fc800000-00000000fc9fffff +> +> > > > <- correct Adress Spce +> +> > > > +> +> > > > 00000000fca00000-00000000fcbfffff (prio 1, RW): alias +> +> > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > 00000000fca00000-00000000fcbfffff +> +> > > > +> +> > > > 00000000fcc00000-00000000fcdfffff (prio 1, RW): alias +> +> > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > 00000000fcc00000-00000000fcdfffff +> +> > > > +> +> > > > 00000000fce00000-00000000fcffffff (prio 1, RW): alias +> +> > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > 00000000fce00000-00000000fcffffff +> +> > > > +> +> > > > +> +> > > > +> +> > > > After pci_bridge_update_mappingsï¼ +> +> > > > +> +> > > > 00000000fda00000-00000000fdbfffff (prio 1, RW): alias +> +> > > > pci_bridge_mem @pci_bridge_pci 00000000fda00000-00000000fdbfffff +> +> > > > +> +> > > > 00000000fdc00000-00000000fddfffff (prio 1, RW): alias +> +> > > > pci_bridge_mem @pci_bridge_pci 00000000fdc00000-00000000fddfffff +> +> > > > +> +> > > > 00000000fde00000-00000000fdffffff (prio 1, RW): alias +> +> > > > pci_bridge_mem @pci_bridge_pci 00000000fde00000-00000000fdffffff +> +> > > > +> +> > > > 00000000fe000000-00000000fe1fffff (prio 1, RW): alias +> +> > > > pci_bridge_mem @pci_bridge_pci 00000000fe000000-00000000fe1fffff +> +> > > > +> +> > > > 00000000fe200000-00000000fe3fffff (prio 1, RW): alias +> +> > > > pci_bridge_mem @pci_bridge_pci 00000000fe200000-00000000fe3fffff +> +> > > > +> +> > > > 00000000fe400000-00000000fe5fffff (prio 1, RW): alias +> +> > > > pci_bridge_mem @pci_bridge_pci 00000000fe400000-00000000fe5fffff +> +> > > > +> +> > > > 00000000fe600000-00000000fe7fffff (prio 1, RW): alias +> +> > > > pci_bridge_mem @pci_bridge_pci 00000000fe600000-00000000fe7fffff +> +> > > > +> +> > > > 00000000fe800000-00000000fe9fffff (prio 1, RW): alias +> +> > > > pci_bridge_mem @pci_bridge_pci 00000000fe800000-00000000fe9fffff +> +> > > > +> +> > > > fffffffffc800000-fffffffffc800000 (prio 1, RW): alias +> +> pci_bridge_pref_mem +> +> > > > @pci_bridge_pci fffffffffc800000-fffffffffc800000 <- Exceptional +> +> > > > Adress +> +> > > Space +> +> > > +> +> > > This one is empty though right? +> +> > > +> +> > > > +> +> > > > +> +> > > > We have figured out why this address becomes this value, +> +> > > > according to pci spec, pci driver can get BAR address size by +> +> > > > writing 0xffffffff to +> +> > > > +> +> > > > the pci register firstly, and then read back the value from this +> +> > > > register. +> +> > > +> +> > > +> +> > > OK however as you show below the BAR being sized is the BAR if a +> +> > > bridge. Are you then adding a bridge device by hotplug? +> +> > +> +> > No, I just simply hot plugged a VFIO device to Bus 0, another +> +> > interesting phenomenon is If I hot plug the device to other bus, this +> +> > doesn't +> +> happened. +> +> > +> +> > > +> +> > > +> +> > > > We didn't handle this value specially while process pci write in +> +> > > > qemu, the function call stack is: +> +> > > > +> +> > > > Pci_bridge_dev_write_config +> +> > > > +> +> > > > -> pci_bridge_write_config +> +> > > > +> +> > > > -> pci_default_write_config (we update the config[address] value +> +> > > > -> here to +> +> > > > fffffffffc800000, which should be 0xfc800000 ) +> +> > > > +> +> > > > -> pci_bridge_update_mappings +> +> > > > +> +> > > > ->pci_bridge_region_del(br, br->windows); +> +> > > > +> +> > > > -> pci_bridge_region_init +> +> > > > +> +> > > > -> +> +> > > > pci_bridge_init_alias (here pci_bridge_get_base, we use the wrong +> +> > > > value +> +> > > > fffffffffc800000) +> +> > > > +> +> > > > -> +> +> > > > memory_region_transaction_commit +> +> > > > +> +> > > > +> +> > > > +> +> > > > So, as we can see, we use the wrong base address in qemu to update +> +> > > > the memory regions, though, we update the base address to +> +> > > > +> +> > > > The correct value after pci driver in VM write the original value +> +> > > > back, the virtio NIC in bus 4 may still sends net packets +> +> > > > concurrently with +> +> > > > +> +> > > > The wrong memory region address. +> +> > > > +> +> > > > +> +> > > > +> +> > > > We have tried to skip the memory region update action in qemu +> +> > > > while detect pci write with 0xffffffff value, and it does work, +> +> > > > but +> +> > > > +> +> > > > This seems to be not gently. +> +> > > +> +> > > For sure. But I'm still puzzled as to why does Linux try to size the +> +> > > BAR of the bridge while a device behind it is used. +> +> > > +> +> > > Can you pls post your QEMU command line? +> +> > +> +> > My QEMU command line: +> +> > /root/xyd/qemu-system-x86_64 -name guest=Linux,debug-threads=on -S +> +> > -object +> +> > secret,id=masterKey0,format=raw,file=/var/run/libvirt/qemu/domain-194- +> +> > Linux/master-key.aes -machine +> +> > pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off -cpu +> +> > host,+kvm_pv_eoi -bios /usr/share/OVMF/OVMF.fd -m +> +> > size=4194304k,slots=256,maxmem=33554432k -realtime mlock=off -smp +> +> > 20,sockets=20,cores=1,threads=1 -numa node,nodeid=0,cpus=0-4,mem=1024 +> +> > -numa node,nodeid=1,cpus=5-9,mem=1024 -numa +> +> > node,nodeid=2,cpus=10-14,mem=1024 -numa +> +> > node,nodeid=3,cpus=15-19,mem=1024 -uuid +> +> > 34a588c7-b0f2-4952-b39c-47fae3411439 -no-user-config -nodefaults +> +> > -chardev +> +> > socket,id=charmonitor,path=/var/run/libvirt/qemu/domain-194-Linux/moni +> +> > tor.sock,server,nowait -mon +> +> > chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-hpet +> +> > -global kvm-pit.lost_tick_policy=delay -no-shutdown -boot strict=on +> +> > -device pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x8 -device +> +> > pci-bridge,chassis_nr=2,id=pci.2,bus=pci.0,addr=0x9 -device +> +> > pci-bridge,chassis_nr=3,id=pci.3,bus=pci.0,addr=0xa -device +> +> > pci-bridge,chassis_nr=4,id=pci.4,bus=pci.0,addr=0xb -device +> +> > pci-bridge,chassis_nr=5,id=pci.5,bus=pci.0,addr=0xc -device +> +> > piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device +> +> > usb-ehci,id=usb1,bus=pci.0,addr=0x10 -device +> +> > nec-usb-xhci,id=usb2,bus=pci.0,addr=0x11 -device +> +> > virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 -device +> +> > virtio-scsi-pci,id=scsi1,bus=pci.0,addr=0x4 -device +> +> > virtio-scsi-pci,id=scsi2,bus=pci.0,addr=0x5 -device +> +> > virtio-scsi-pci,id=scsi3,bus=pci.0,addr=0x6 -device +> +> > virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive +> +> > file=/mnt/sdb/xml/centos_74_x64_uefi.raw,format=raw,if=none,id=drive-v +> +> > irtio-disk0,cache=none -device +> +> > virtio-blk-pci,scsi=off,bus=pci.0,addr=0x2,drive=drive-virtio-disk0,id +> +> > =virtio-disk0,bootindex=1 -drive +> +> > if=none,id=drive-ide0-1-1,readonly=on,cache=none -device +> +> > ide-cd,bus=ide.1,unit=1,drive=drive-ide0-1-1,id=ide0-1-1 -netdev +> +> > tap,fd=35,id=hostnet0 -device +> +> > virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:89:5d:8b,bus=pci.4 +> +> > ,addr=0x1 -chardev pty,id=charserial0 -device +> +> > isa-serial,chardev=charserial0,id=serial0 -device +> +> > usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -device +> +> > cirrus-vga,id=video0,vgamem_mb=8,bus=pci.0,addr=0x12 -device +> +> > virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xd -msg timestamp=on +> +> > +> +> > I am also very curious about this issue, in the linux kernel code, maybe +> +> > double +> +> check in function pci_bridge_check_ranges triggered this problem. +> +> +> +> If you can get the stacktrace in Linux when it tries to write this fffff +> +> value, that +> +> would be quite helpful. +> +> +> +> +After I add mdelay(100) in function pci_bridge_check_ranges, this phenomenon +> +is +> +easier to reproduce, below is my modify in kernel: +> +diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c +> +index cb389277..86e232d 100644 +> +--- a/drivers/pci/setup-bus.c +> ++++ b/drivers/pci/setup-bus.c +> +@@ -27,7 +27,7 @@ +> +#include <linux/slab.h> +> +#include <linux/acpi.h> +> +#include "pci.h" +> +- +> ++#include <linux/delay.h> +> +unsigned int pci_flags; +> +> +struct pci_dev_resource { +> +@@ -787,6 +787,9 @@ static void pci_bridge_check_ranges(struct pci_bus *bus) +> +pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +0xffffffff); +> +pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &tmp); +> ++ mdelay(100); +> ++ printk(KERN_ERR "sleep\n"); +> ++ dump_stack(); +> +if (!tmp) +> +b_res[2].flags &= ~IORESOURCE_MEM_64; +> +pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +OK! +I just sent a Linux patch that should help. +I would appreciate it if you will give it a try +and if that helps reply to it with +a Tested-by: tag. + +-- +MST + +On Tue, Dec 11, 2018 at 01:47:37AM +0000, xuyandong wrote: +> +> On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> > > > > Hi all, +> +> > > > > +> +> > > > > +> +> > > > > +> +> > > > > In our test, we configured VM with several pci-bridges and a +> +> > > > > virtio-net nic been attached with bus 4, +> +> > > > > +> +> > > > > After VM is startup, We ping this nic from host to judge if it +> +> > > > > is working normally. Then, we hot add pci devices to this VM with +> +> > > > > bus +> +0. +> +> > > > > +> +> > > > > We found the virtio-net NIC in bus 4 is not working (can not +> +> > > > > connect) occasionally, as it kick virtio backend failure with error +> +> > > > > below: +> +> > > > > +> +> > > > > Unassigned mem write 00000000fc803004 = 0x1 +> +> > > > > +> +> > > > > +> +> > > > > +> +> > > > > memory-region: pci_bridge_pci +> +> > > > > +> +> > > > > 0000000000000000-ffffffffffffffff (prio 0, RW): +> +> > > > > pci_bridge_pci +> +> > > > > +> +> > > > > 00000000fc800000-00000000fc803fff (prio 1, RW): virtio-pci +> +> > > > > +> +> > > > > 00000000fc800000-00000000fc800fff (prio 0, RW): +> +> > > > > virtio-pci-common +> +> > > > > +> +> > > > > 00000000fc801000-00000000fc801fff (prio 0, RW): +> +> > > > > virtio-pci-isr +> +> > > > > +> +> > > > > 00000000fc802000-00000000fc802fff (prio 0, RW): +> +> > > > > virtio-pci-device +> +> > > > > +> +> > > > > 00000000fc803000-00000000fc803fff (prio 0, RW): +> +> > > > > virtio-pci-notify <- io mem unassigned +> +> > > > > +> +> > > > > ⦠+> +> > > > > +> +> > > > > +> +> > > > > +> +> > > > > We caught an exceptional address changing while this problem +> +> > > > > happened, show as +> +> > > > > follow: +> +> > > > > +> +> > > > > Before pci_bridge_update_mappingsï¼ +> +> > > > > +> +> > > > > 00000000fc000000-00000000fc1fffff (prio 1, RW): alias +> +> > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > 00000000fc000000-00000000fc1fffff +> +> > > > > +> +> > > > > 00000000fc200000-00000000fc3fffff (prio 1, RW): alias +> +> > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > 00000000fc200000-00000000fc3fffff +> +> > > > > +> +> > > > > 00000000fc400000-00000000fc5fffff (prio 1, RW): alias +> +> > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > 00000000fc400000-00000000fc5fffff +> +> > > > > +> +> > > > > 00000000fc600000-00000000fc7fffff (prio 1, RW): alias +> +> > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > 00000000fc600000-00000000fc7fffff +> +> > > > > +> +> > > > > 00000000fc800000-00000000fc9fffff (prio 1, RW): alias +> +> > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > 00000000fc800000-00000000fc9fffff +> +> > > > > <- correct Adress Spce +> +> > > > > +> +> > > > > 00000000fca00000-00000000fcbfffff (prio 1, RW): alias +> +> > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > 00000000fca00000-00000000fcbfffff +> +> > > > > +> +> > > > > 00000000fcc00000-00000000fcdfffff (prio 1, RW): alias +> +> > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > 00000000fcc00000-00000000fcdfffff +> +> > > > > +> +> > > > > 00000000fce00000-00000000fcffffff (prio 1, RW): alias +> +> > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > 00000000fce00000-00000000fcffffff +> +> > > > > +> +> > > > > +> +> > > > > +> +> > > > > After pci_bridge_update_mappingsï¼ +> +> > > > > +> +> > > > > 00000000fda00000-00000000fdbfffff (prio 1, RW): alias +> +> > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > 00000000fda00000-00000000fdbfffff +> +> > > > > +> +> > > > > 00000000fdc00000-00000000fddfffff (prio 1, RW): alias +> +> > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > 00000000fdc00000-00000000fddfffff +> +> > > > > +> +> > > > > 00000000fde00000-00000000fdffffff (prio 1, RW): alias +> +> > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > 00000000fde00000-00000000fdffffff +> +> > > > > +> +> > > > > 00000000fe000000-00000000fe1fffff (prio 1, RW): alias +> +> > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > 00000000fe000000-00000000fe1fffff +> +> > > > > +> +> > > > > 00000000fe200000-00000000fe3fffff (prio 1, RW): alias +> +> > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > 00000000fe200000-00000000fe3fffff +> +> > > > > +> +> > > > > 00000000fe400000-00000000fe5fffff (prio 1, RW): alias +> +> > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > 00000000fe400000-00000000fe5fffff +> +> > > > > +> +> > > > > 00000000fe600000-00000000fe7fffff (prio 1, RW): alias +> +> > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > 00000000fe600000-00000000fe7fffff +> +> > > > > +> +> > > > > 00000000fe800000-00000000fe9fffff (prio 1, RW): alias +> +> > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > 00000000fe800000-00000000fe9fffff +> +> > > > > +> +> > > > > fffffffffc800000-fffffffffc800000 (prio 1, RW): alias +> +> > pci_bridge_pref_mem +> +> > > > > @pci_bridge_pci fffffffffc800000-fffffffffc800000 <- Exceptional +> +Adress +> +> > > > Space +> +> > > > +> +> > > > This one is empty though right? +> +> > > > +> +> > > > > +> +> > > > > +> +> > > > > We have figured out why this address becomes this value, +> +> > > > > according to pci spec, pci driver can get BAR address size by +> +> > > > > writing 0xffffffff to +> +> > > > > +> +> > > > > the pci register firstly, and then read back the value from this +> +> > > > > register. +> +> > > > +> +> > > > +> +> > > > OK however as you show below the BAR being sized is the BAR if a +> +> > > > bridge. Are you then adding a bridge device by hotplug? +> +> > > +> +> > > No, I just simply hot plugged a VFIO device to Bus 0, another +> +> > > interesting phenomenon is If I hot plug the device to other bus, +> +> > > this doesn't +> +> > happened. +> +> > > +> +> > > > +> +> > > > +> +> > > > > We didn't handle this value specially while process pci write +> +> > > > > in qemu, the function call stack is: +> +> > > > > +> +> > > > > Pci_bridge_dev_write_config +> +> > > > > +> +> > > > > -> pci_bridge_write_config +> +> > > > > +> +> > > > > -> pci_default_write_config (we update the config[address] +> +> > > > > -> value here to +> +> > > > > fffffffffc800000, which should be 0xfc800000 ) +> +> > > > > +> +> > > > > -> pci_bridge_update_mappings +> +> > > > > +> +> > > > > ->pci_bridge_region_del(br, br->windows); +> +> > > > > +> +> > > > > -> pci_bridge_region_init +> +> > > > > +> +> > > > > +> +> > > > > -> pci_bridge_init_alias (here pci_bridge_get_base, we use the +> +> > > > > wrong value +> +> > > > > fffffffffc800000) +> +> > > > > +> +> > > > > -> +> +> > > > > memory_region_transaction_commit +> +> > > > > +> +> > > > > +> +> > > > > +> +> > > > > So, as we can see, we use the wrong base address in qemu to +> +> > > > > update the memory regions, though, we update the base address +> +> > > > > to +> +> > > > > +> +> > > > > The correct value after pci driver in VM write the original +> +> > > > > value back, the virtio NIC in bus 4 may still sends net +> +> > > > > packets concurrently with +> +> > > > > +> +> > > > > The wrong memory region address. +> +> > > > > +> +> > > > > +> +> > > > > +> +> > > > > We have tried to skip the memory region update action in qemu +> +> > > > > while detect pci write with 0xffffffff value, and it does +> +> > > > > work, but +> +> > > > > +> +> > > > > This seems to be not gently. +> +> > > > +> +> > > > For sure. But I'm still puzzled as to why does Linux try to size +> +> > > > the BAR of the bridge while a device behind it is used. +> +> > > > +> +> > > > Can you pls post your QEMU command line? +> +> > > +> +> > > My QEMU command line: +> +> > > /root/xyd/qemu-system-x86_64 -name guest=Linux,debug-threads=on -S +> +> > > -object +> +> > > secret,id=masterKey0,format=raw,file=/var/run/libvirt/qemu/domain- +> +> > > 194- +> +> > > Linux/master-key.aes -machine +> +> > > pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off -cpu +> +> > > host,+kvm_pv_eoi -bios /usr/share/OVMF/OVMF.fd -m +> +> > > size=4194304k,slots=256,maxmem=33554432k -realtime mlock=off -smp +> +> > > 20,sockets=20,cores=1,threads=1 -numa +> +> > > node,nodeid=0,cpus=0-4,mem=1024 -numa +> +> > > node,nodeid=1,cpus=5-9,mem=1024 -numa +> +> > > node,nodeid=2,cpus=10-14,mem=1024 -numa +> +> > > node,nodeid=3,cpus=15-19,mem=1024 -uuid +> +> > > 34a588c7-b0f2-4952-b39c-47fae3411439 -no-user-config -nodefaults +> +> > > -chardev +> +> > > socket,id=charmonitor,path=/var/run/libvirt/qemu/domain-194-Linux/ +> +> > > moni +> +> > > tor.sock,server,nowait -mon +> +> > > chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-hpet +> +> > > -global kvm-pit.lost_tick_policy=delay -no-shutdown -boot +> +> > > strict=on -device +> +> > > pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x8 -device +> +> > > pci-bridge,chassis_nr=2,id=pci.2,bus=pci.0,addr=0x9 -device +> +> > > pci-bridge,chassis_nr=3,id=pci.3,bus=pci.0,addr=0xa -device +> +> > > pci-bridge,chassis_nr=4,id=pci.4,bus=pci.0,addr=0xb -device +> +> > > pci-bridge,chassis_nr=5,id=pci.5,bus=pci.0,addr=0xc -device +> +> > > piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device +> +> > > usb-ehci,id=usb1,bus=pci.0,addr=0x10 -device +> +> > > nec-usb-xhci,id=usb2,bus=pci.0,addr=0x11 -device +> +> > > virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 -device +> +> > > virtio-scsi-pci,id=scsi1,bus=pci.0,addr=0x4 -device +> +> > > virtio-scsi-pci,id=scsi2,bus=pci.0,addr=0x5 -device +> +> > > virtio-scsi-pci,id=scsi3,bus=pci.0,addr=0x6 -device +> +> > > virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive +> +> > > file=/mnt/sdb/xml/centos_74_x64_uefi.raw,format=raw,if=none,id=dri +> +> > > ve-v +> +> > > irtio-disk0,cache=none -device +> +> > > virtio-blk-pci,scsi=off,bus=pci.0,addr=0x2,drive=drive-virtio-disk +> +> > > 0,id +> +> > > =virtio-disk0,bootindex=1 -drive +> +> > > if=none,id=drive-ide0-1-1,readonly=on,cache=none -device +> +> > > ide-cd,bus=ide.1,unit=1,drive=drive-ide0-1-1,id=ide0-1-1 -netdev +> +> > > tap,fd=35,id=hostnet0 -device +> +> > > virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:89:5d:8b,bus=p +> +> > > ci.4 +> +> > > ,addr=0x1 -chardev pty,id=charserial0 -device +> +> > > isa-serial,chardev=charserial0,id=serial0 -device +> +> > > usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -device +> +> > > cirrus-vga,id=video0,vgamem_mb=8,bus=pci.0,addr=0x12 -device +> +> > > virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xd -msg +> +> > > timestamp=on +> +> > > +> +> > > I am also very curious about this issue, in the linux kernel code, +> +> > > maybe double +> +> > check in function pci_bridge_check_ranges triggered this problem. +> +> > +> +> > If you can get the stacktrace in Linux when it tries to write this +> +> > fffff value, that would be quite helpful. +> +> > +> +> +> +> After I add mdelay(100) in function pci_bridge_check_ranges, this +> +> phenomenon is easier to reproduce, below is my modify in kernel: +> +> diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index +> +> cb389277..86e232d 100644 +> +> --- a/drivers/pci/setup-bus.c +> +> +++ b/drivers/pci/setup-bus.c +> +> @@ -27,7 +27,7 @@ +> +> #include <linux/slab.h> +> +> #include <linux/acpi.h> +> +> #include "pci.h" +> +> - +> +> +#include <linux/delay.h> +> +> unsigned int pci_flags; +> +> +> +> struct pci_dev_resource { +> +> @@ -787,6 +787,9 @@ static void pci_bridge_check_ranges(struct pci_bus +> +*bus) +> +> pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +> 0xffffffff); +> +> pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +> &tmp); +> +> + mdelay(100); +> +> + printk(KERN_ERR "sleep\n"); +> +> + dump_stack(); +> +> if (!tmp) +> +> b_res[2].flags &= ~IORESOURCE_MEM_64; +> +> pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +> +> +> +OK! +> +I just sent a Linux patch that should help. +> +I would appreciate it if you will give it a try and if that helps reply to it +> +with a +> +Tested-by: tag. +> +I tested this patch and it works fine on my machine. + +But I have another question, if we only fix this problem in the kernel, the +Linux +version that has been released does not work well on the virtualization +platform. +Is there a way to fix this problem in the backend? + +> +-- +> +MST + +On Tue, Dec 11, 2018 at 02:55:43AM +0000, xuyandong wrote: +> +On Tue, Dec 11, 2018 at 01:47:37AM +0000, xuyandong wrote: +> +> > On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> > > > > > Hi all, +> +> > > > > > +> +> > > > > > +> +> > > > > > +> +> > > > > > In our test, we configured VM with several pci-bridges and a +> +> > > > > > virtio-net nic been attached with bus 4, +> +> > > > > > +> +> > > > > > After VM is startup, We ping this nic from host to judge if it +> +> > > > > > is working normally. Then, we hot add pci devices to this VM with +> +> > > > > > bus +> +> 0. +> +> > > > > > +> +> > > > > > We found the virtio-net NIC in bus 4 is not working (can not +> +> > > > > > connect) occasionally, as it kick virtio backend failure with +> +> > > > > > error below: +> +> > > > > > +> +> > > > > > Unassigned mem write 00000000fc803004 = 0x1 +> +> > > > > > +> +> > > > > > +> +> > > > > > +> +> > > > > > memory-region: pci_bridge_pci +> +> > > > > > +> +> > > > > > 0000000000000000-ffffffffffffffff (prio 0, RW): +> +> > > > > > pci_bridge_pci +> +> > > > > > +> +> > > > > > 00000000fc800000-00000000fc803fff (prio 1, RW): virtio-pci +> +> > > > > > +> +> > > > > > 00000000fc800000-00000000fc800fff (prio 0, RW): +> +> > > > > > virtio-pci-common +> +> > > > > > +> +> > > > > > 00000000fc801000-00000000fc801fff (prio 0, RW): +> +> > > > > > virtio-pci-isr +> +> > > > > > +> +> > > > > > 00000000fc802000-00000000fc802fff (prio 0, RW): +> +> > > > > > virtio-pci-device +> +> > > > > > +> +> > > > > > 00000000fc803000-00000000fc803fff (prio 0, RW): +> +> > > > > > virtio-pci-notify <- io mem unassigned +> +> > > > > > +> +> > > > > > ⦠+> +> > > > > > +> +> > > > > > +> +> > > > > > +> +> > > > > > We caught an exceptional address changing while this problem +> +> > > > > > happened, show as +> +> > > > > > follow: +> +> > > > > > +> +> > > > > > Before pci_bridge_update_mappingsï¼ +> +> > > > > > +> +> > > > > > 00000000fc000000-00000000fc1fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > 00000000fc000000-00000000fc1fffff +> +> > > > > > +> +> > > > > > 00000000fc200000-00000000fc3fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > 00000000fc200000-00000000fc3fffff +> +> > > > > > +> +> > > > > > 00000000fc400000-00000000fc5fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > 00000000fc400000-00000000fc5fffff +> +> > > > > > +> +> > > > > > 00000000fc600000-00000000fc7fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > 00000000fc600000-00000000fc7fffff +> +> > > > > > +> +> > > > > > 00000000fc800000-00000000fc9fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > 00000000fc800000-00000000fc9fffff +> +> > > > > > <- correct Adress Spce +> +> > > > > > +> +> > > > > > 00000000fca00000-00000000fcbfffff (prio 1, RW): alias +> +> > > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > 00000000fca00000-00000000fcbfffff +> +> > > > > > +> +> > > > > > 00000000fcc00000-00000000fcdfffff (prio 1, RW): alias +> +> > > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > 00000000fcc00000-00000000fcdfffff +> +> > > > > > +> +> > > > > > 00000000fce00000-00000000fcffffff (prio 1, RW): alias +> +> > > > > > pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > 00000000fce00000-00000000fcffffff +> +> > > > > > +> +> > > > > > +> +> > > > > > +> +> > > > > > After pci_bridge_update_mappingsï¼ +> +> > > > > > +> +> > > > > > 00000000fda00000-00000000fdbfffff (prio 1, RW): alias +> +> > > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > > 00000000fda00000-00000000fdbfffff +> +> > > > > > +> +> > > > > > 00000000fdc00000-00000000fddfffff (prio 1, RW): alias +> +> > > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > > 00000000fdc00000-00000000fddfffff +> +> > > > > > +> +> > > > > > 00000000fde00000-00000000fdffffff (prio 1, RW): alias +> +> > > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > > 00000000fde00000-00000000fdffffff +> +> > > > > > +> +> > > > > > 00000000fe000000-00000000fe1fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > > 00000000fe000000-00000000fe1fffff +> +> > > > > > +> +> > > > > > 00000000fe200000-00000000fe3fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > > 00000000fe200000-00000000fe3fffff +> +> > > > > > +> +> > > > > > 00000000fe400000-00000000fe5fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > > 00000000fe400000-00000000fe5fffff +> +> > > > > > +> +> > > > > > 00000000fe600000-00000000fe7fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > > 00000000fe600000-00000000fe7fffff +> +> > > > > > +> +> > > > > > 00000000fe800000-00000000fe9fffff (prio 1, RW): alias +> +> > > > > > pci_bridge_mem @pci_bridge_pci +> +> > > > > > 00000000fe800000-00000000fe9fffff +> +> > > > > > +> +> > > > > > fffffffffc800000-fffffffffc800000 (prio 1, RW): alias +> +> > > pci_bridge_pref_mem +> +> > > > > > @pci_bridge_pci fffffffffc800000-fffffffffc800000 <- Exceptional +> +> Adress +> +> > > > > Space +> +> > > > > +> +> > > > > This one is empty though right? +> +> > > > > +> +> > > > > > +> +> > > > > > +> +> > > > > > We have figured out why this address becomes this value, +> +> > > > > > according to pci spec, pci driver can get BAR address size by +> +> > > > > > writing 0xffffffff to +> +> > > > > > +> +> > > > > > the pci register firstly, and then read back the value from this +> +> > > > > > register. +> +> > > > > +> +> > > > > +> +> > > > > OK however as you show below the BAR being sized is the BAR if a +> +> > > > > bridge. Are you then adding a bridge device by hotplug? +> +> > > > +> +> > > > No, I just simply hot plugged a VFIO device to Bus 0, another +> +> > > > interesting phenomenon is If I hot plug the device to other bus, +> +> > > > this doesn't +> +> > > happened. +> +> > > > +> +> > > > > +> +> > > > > +> +> > > > > > We didn't handle this value specially while process pci write +> +> > > > > > in qemu, the function call stack is: +> +> > > > > > +> +> > > > > > Pci_bridge_dev_write_config +> +> > > > > > +> +> > > > > > -> pci_bridge_write_config +> +> > > > > > +> +> > > > > > -> pci_default_write_config (we update the config[address] +> +> > > > > > -> value here to +> +> > > > > > fffffffffc800000, which should be 0xfc800000 ) +> +> > > > > > +> +> > > > > > -> pci_bridge_update_mappings +> +> > > > > > +> +> > > > > > ->pci_bridge_region_del(br, br->windows); +> +> > > > > > +> +> > > > > > -> pci_bridge_region_init +> +> > > > > > +> +> > > > > > +> +> > > > > > -> pci_bridge_init_alias (here pci_bridge_get_base, we use the +> +> > > > > > wrong value +> +> > > > > > fffffffffc800000) +> +> > > > > > +> +> > > > > > -> +> +> > > > > > memory_region_transaction_commit +> +> > > > > > +> +> > > > > > +> +> > > > > > +> +> > > > > > So, as we can see, we use the wrong base address in qemu to +> +> > > > > > update the memory regions, though, we update the base address +> +> > > > > > to +> +> > > > > > +> +> > > > > > The correct value after pci driver in VM write the original +> +> > > > > > value back, the virtio NIC in bus 4 may still sends net +> +> > > > > > packets concurrently with +> +> > > > > > +> +> > > > > > The wrong memory region address. +> +> > > > > > +> +> > > > > > +> +> > > > > > +> +> > > > > > We have tried to skip the memory region update action in qemu +> +> > > > > > while detect pci write with 0xffffffff value, and it does +> +> > > > > > work, but +> +> > > > > > +> +> > > > > > This seems to be not gently. +> +> > > > > +> +> > > > > For sure. But I'm still puzzled as to why does Linux try to size +> +> > > > > the BAR of the bridge while a device behind it is used. +> +> > > > > +> +> > > > > Can you pls post your QEMU command line? +> +> > > > +> +> > > > My QEMU command line: +> +> > > > /root/xyd/qemu-system-x86_64 -name guest=Linux,debug-threads=on -S +> +> > > > -object +> +> > > > secret,id=masterKey0,format=raw,file=/var/run/libvirt/qemu/domain- +> +> > > > 194- +> +> > > > Linux/master-key.aes -machine +> +> > > > pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off -cpu +> +> > > > host,+kvm_pv_eoi -bios /usr/share/OVMF/OVMF.fd -m +> +> > > > size=4194304k,slots=256,maxmem=33554432k -realtime mlock=off -smp +> +> > > > 20,sockets=20,cores=1,threads=1 -numa +> +> > > > node,nodeid=0,cpus=0-4,mem=1024 -numa +> +> > > > node,nodeid=1,cpus=5-9,mem=1024 -numa +> +> > > > node,nodeid=2,cpus=10-14,mem=1024 -numa +> +> > > > node,nodeid=3,cpus=15-19,mem=1024 -uuid +> +> > > > 34a588c7-b0f2-4952-b39c-47fae3411439 -no-user-config -nodefaults +> +> > > > -chardev +> +> > > > socket,id=charmonitor,path=/var/run/libvirt/qemu/domain-194-Linux/ +> +> > > > moni +> +> > > > tor.sock,server,nowait -mon +> +> > > > chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-hpet +> +> > > > -global kvm-pit.lost_tick_policy=delay -no-shutdown -boot +> +> > > > strict=on -device +> +> > > > pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x8 -device +> +> > > > pci-bridge,chassis_nr=2,id=pci.2,bus=pci.0,addr=0x9 -device +> +> > > > pci-bridge,chassis_nr=3,id=pci.3,bus=pci.0,addr=0xa -device +> +> > > > pci-bridge,chassis_nr=4,id=pci.4,bus=pci.0,addr=0xb -device +> +> > > > pci-bridge,chassis_nr=5,id=pci.5,bus=pci.0,addr=0xc -device +> +> > > > piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device +> +> > > > usb-ehci,id=usb1,bus=pci.0,addr=0x10 -device +> +> > > > nec-usb-xhci,id=usb2,bus=pci.0,addr=0x11 -device +> +> > > > virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 -device +> +> > > > virtio-scsi-pci,id=scsi1,bus=pci.0,addr=0x4 -device +> +> > > > virtio-scsi-pci,id=scsi2,bus=pci.0,addr=0x5 -device +> +> > > > virtio-scsi-pci,id=scsi3,bus=pci.0,addr=0x6 -device +> +> > > > virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive +> +> > > > file=/mnt/sdb/xml/centos_74_x64_uefi.raw,format=raw,if=none,id=dri +> +> > > > ve-v +> +> > > > irtio-disk0,cache=none -device +> +> > > > virtio-blk-pci,scsi=off,bus=pci.0,addr=0x2,drive=drive-virtio-disk +> +> > > > 0,id +> +> > > > =virtio-disk0,bootindex=1 -drive +> +> > > > if=none,id=drive-ide0-1-1,readonly=on,cache=none -device +> +> > > > ide-cd,bus=ide.1,unit=1,drive=drive-ide0-1-1,id=ide0-1-1 -netdev +> +> > > > tap,fd=35,id=hostnet0 -device +> +> > > > virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:89:5d:8b,bus=p +> +> > > > ci.4 +> +> > > > ,addr=0x1 -chardev pty,id=charserial0 -device +> +> > > > isa-serial,chardev=charserial0,id=serial0 -device +> +> > > > usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -device +> +> > > > cirrus-vga,id=video0,vgamem_mb=8,bus=pci.0,addr=0x12 -device +> +> > > > virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xd -msg +> +> > > > timestamp=on +> +> > > > +> +> > > > I am also very curious about this issue, in the linux kernel code, +> +> > > > maybe double +> +> > > check in function pci_bridge_check_ranges triggered this problem. +> +> > > +> +> > > If you can get the stacktrace in Linux when it tries to write this +> +> > > fffff value, that would be quite helpful. +> +> > > +> +> > +> +> > After I add mdelay(100) in function pci_bridge_check_ranges, this +> +> > phenomenon is easier to reproduce, below is my modify in kernel: +> +> > diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index +> +> > cb389277..86e232d 100644 +> +> > --- a/drivers/pci/setup-bus.c +> +> > +++ b/drivers/pci/setup-bus.c +> +> > @@ -27,7 +27,7 @@ +> +> > #include <linux/slab.h> +> +> > #include <linux/acpi.h> +> +> > #include "pci.h" +> +> > - +> +> > +#include <linux/delay.h> +> +> > unsigned int pci_flags; +> +> > +> +> > struct pci_dev_resource { +> +> > @@ -787,6 +787,9 @@ static void pci_bridge_check_ranges(struct pci_bus +> +> *bus) +> +> > pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +> > 0xffffffff); +> +> > pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +> > &tmp); +> +> > + mdelay(100); +> +> > + printk(KERN_ERR "sleep\n"); +> +> > + dump_stack(); +> +> > if (!tmp) +> +> > b_res[2].flags &= ~IORESOURCE_MEM_64; +> +> > pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +> > +> +> +> +> OK! +> +> I just sent a Linux patch that should help. +> +> I would appreciate it if you will give it a try and if that helps reply to +> +> it with a +> +> Tested-by: tag. +> +> +> +> +I tested this patch and it works fine on my machine. +> +> +But I have another question, if we only fix this problem in the kernel, the +> +Linux +> +version that has been released does not work well on the virtualization +> +platform. +> +Is there a way to fix this problem in the backend? +There could we a way to work around this. +Does below help? + +diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c +index 236a20eaa8..7834cac4b0 100644 +--- a/hw/i386/acpi-build.c ++++ b/hw/i386/acpi-build.c +@@ -551,7 +551,7 @@ static void build_append_pci_bus_devices(Aml *parent_scope, +PCIBus *bus, + + aml_append(method, aml_store(aml_int(bsel_val), aml_name("BNUM"))); + aml_append(method, +- aml_call2("DVNT", aml_name("PCIU"), aml_int(1) /* Device Check */) ++ aml_call2("DVNT", aml_name("PCIU"), aml_int(4) /* Device Check +Light */) + ); + aml_append(method, + aml_call2("DVNT", aml_name("PCID"), aml_int(3)/* Eject Request */) + +> +On Tue, Dec 11, 2018 at 02:55:43AM +0000, xuyandong wrote: +> +> On Tue, Dec 11, 2018 at 01:47:37AM +0000, xuyandong wrote: +> +> > > On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> > > > > > > Hi all, +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > In our test, we configured VM with several pci-bridges and +> +> > > > > > > a virtio-net nic been attached with bus 4, +> +> > > > > > > +> +> > > > > > > After VM is startup, We ping this nic from host to judge +> +> > > > > > > if it is working normally. Then, we hot add pci devices to +> +> > > > > > > this VM with bus +> +> > 0. +> +> > > > > > > +> +> > > > > > > We found the virtio-net NIC in bus 4 is not working (can +> +> > > > > > > not +> +> > > > > > > connect) occasionally, as it kick virtio backend failure with +> +> > > > > > > error +> +below: +> +> > > > > > > +> +> > > > > > > Unassigned mem write 00000000fc803004 = 0x1 +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > memory-region: pci_bridge_pci +> +> > > > > > > +> +> > > > > > > 0000000000000000-ffffffffffffffff (prio 0, RW): +> +> > > > > > > pci_bridge_pci +> +> > > > > > > +> +> > > > > > > 00000000fc800000-00000000fc803fff (prio 1, RW): +> +> > > > > > > virtio-pci +> +> > > > > > > +> +> > > > > > > 00000000fc800000-00000000fc800fff (prio 0, RW): +> +> > > > > > > virtio-pci-common +> +> > > > > > > +> +> > > > > > > 00000000fc801000-00000000fc801fff (prio 0, RW): +> +> > > > > > > virtio-pci-isr +> +> > > > > > > +> +> > > > > > > 00000000fc802000-00000000fc802fff (prio 0, RW): +> +> > > > > > > virtio-pci-device +> +> > > > > > > +> +> > > > > > > 00000000fc803000-00000000fc803fff (prio 0, RW): +> +> > > > > > > virtio-pci-notify <- io mem unassigned +> +> > > > > > > +> +> > > > > > > ⦠+> +> > > > > > > +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > We caught an exceptional address changing while this +> +> > > > > > > problem happened, show as +> +> > > > > > > follow: +> +> > > > > > > +> +> > > > > > > Before pci_bridge_update_mappingsï¼ +> +> > > > > > > +> +> > > > > > > 00000000fc000000-00000000fc1fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > 00000000fc000000-00000000fc1fffff +> +> > > > > > > +> +> > > > > > > 00000000fc200000-00000000fc3fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > 00000000fc200000-00000000fc3fffff +> +> > > > > > > +> +> > > > > > > 00000000fc400000-00000000fc5fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > 00000000fc400000-00000000fc5fffff +> +> > > > > > > +> +> > > > > > > 00000000fc600000-00000000fc7fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > 00000000fc600000-00000000fc7fffff +> +> > > > > > > +> +> > > > > > > 00000000fc800000-00000000fc9fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > 00000000fc800000-00000000fc9fffff +> +> > > > > > > <- correct Adress Spce +> +> > > > > > > +> +> > > > > > > 00000000fca00000-00000000fcbfffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > 00000000fca00000-00000000fcbfffff +> +> > > > > > > +> +> > > > > > > 00000000fcc00000-00000000fcdfffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > 00000000fcc00000-00000000fcdfffff +> +> > > > > > > +> +> > > > > > > 00000000fce00000-00000000fcffffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > 00000000fce00000-00000000fcffffff +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > After pci_bridge_update_mappingsï¼ +> +> > > > > > > +> +> > > > > > > 00000000fda00000-00000000fdbfffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > 00000000fda00000-00000000fdbfffff +> +> > > > > > > +> +> > > > > > > 00000000fdc00000-00000000fddfffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > 00000000fdc00000-00000000fddfffff +> +> > > > > > > +> +> > > > > > > 00000000fde00000-00000000fdffffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > 00000000fde00000-00000000fdffffff +> +> > > > > > > +> +> > > > > > > 00000000fe000000-00000000fe1fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > 00000000fe000000-00000000fe1fffff +> +> > > > > > > +> +> > > > > > > 00000000fe200000-00000000fe3fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > 00000000fe200000-00000000fe3fffff +> +> > > > > > > +> +> > > > > > > 00000000fe400000-00000000fe5fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > 00000000fe400000-00000000fe5fffff +> +> > > > > > > +> +> > > > > > > 00000000fe600000-00000000fe7fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > 00000000fe600000-00000000fe7fffff +> +> > > > > > > +> +> > > > > > > 00000000fe800000-00000000fe9fffff (prio 1, RW): +> +> > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > 00000000fe800000-00000000fe9fffff +> +> > > > > > > +> +> > > > > > > fffffffffc800000-fffffffffc800000 (prio 1, RW): +> +> > > > > > > alias +> +> > > > pci_bridge_pref_mem +> +> > > > > > > @pci_bridge_pci fffffffffc800000-fffffffffc800000 <- +> +> > > > > > > Exceptional +> +> > Adress +> +> > > > > > Space +> +> > > > > > +> +> > > > > > This one is empty though right? +> +> > > > > > +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > We have figured out why this address becomes this value, +> +> > > > > > > according to pci spec, pci driver can get BAR address +> +> > > > > > > size by writing 0xffffffff to +> +> > > > > > > +> +> > > > > > > the pci register firstly, and then read back the value from this +> +register. +> +> > > > > > +> +> > > > > > +> +> > > > > > OK however as you show below the BAR being sized is the BAR +> +> > > > > > if a bridge. Are you then adding a bridge device by hotplug? +> +> > > > > +> +> > > > > No, I just simply hot plugged a VFIO device to Bus 0, another +> +> > > > > interesting phenomenon is If I hot plug the device to other +> +> > > > > bus, this doesn't +> +> > > > happened. +> +> > > > > +> +> > > > > > +> +> > > > > > +> +> > > > > > > We didn't handle this value specially while process pci +> +> > > > > > > write in qemu, the function call stack is: +> +> > > > > > > +> +> > > > > > > Pci_bridge_dev_write_config +> +> > > > > > > +> +> > > > > > > -> pci_bridge_write_config +> +> > > > > > > +> +> > > > > > > -> pci_default_write_config (we update the config[address] +> +> > > > > > > -> value here to +> +> > > > > > > fffffffffc800000, which should be 0xfc800000 ) +> +> > > > > > > +> +> > > > > > > -> pci_bridge_update_mappings +> +> > > > > > > +> +> > > > > > > ->pci_bridge_region_del(br, br->windows); +> +> > > > > > > +> +> > > > > > > -> pci_bridge_region_init +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > -> pci_bridge_init_alias (here pci_bridge_get_base, we use +> +> > > > > > > -> the +> +> > > > > > > wrong value +> +> > > > > > > fffffffffc800000) +> +> > > > > > > +> +> > > > > > > -> +> +> > > > > > > memory_region_transaction_commit +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > So, as we can see, we use the wrong base address in qemu +> +> > > > > > > to update the memory regions, though, we update the base +> +> > > > > > > address to +> +> > > > > > > +> +> > > > > > > The correct value after pci driver in VM write the +> +> > > > > > > original value back, the virtio NIC in bus 4 may still +> +> > > > > > > sends net packets concurrently with +> +> > > > > > > +> +> > > > > > > The wrong memory region address. +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > We have tried to skip the memory region update action in +> +> > > > > > > qemu while detect pci write with 0xffffffff value, and it +> +> > > > > > > does work, but +> +> > > > > > > +> +> > > > > > > This seems to be not gently. +> +> > > > > > +> +> > > > > > For sure. But I'm still puzzled as to why does Linux try to +> +> > > > > > size the BAR of the bridge while a device behind it is used. +> +> > > > > > +> +> > > > > > Can you pls post your QEMU command line? +> +> > > > > +> +> > > > > My QEMU command line: +> +> > > > > /root/xyd/qemu-system-x86_64 -name +> +> > > > > guest=Linux,debug-threads=on -S -object +> +> > > > > secret,id=masterKey0,format=raw,file=/var/run/libvirt/qemu/dom +> +> > > > > ain- +> +> > > > > 194- +> +> > > > > Linux/master-key.aes -machine +> +> > > > > pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off -cpu +> +> > > > > host,+kvm_pv_eoi -bios /usr/share/OVMF/OVMF.fd -m +> +> > > > > size=4194304k,slots=256,maxmem=33554432k -realtime mlock=off +> +> > > > > -smp +> +> > > > > 20,sockets=20,cores=1,threads=1 -numa +> +> > > > > node,nodeid=0,cpus=0-4,mem=1024 -numa +> +> > > > > node,nodeid=1,cpus=5-9,mem=1024 -numa +> +> > > > > node,nodeid=2,cpus=10-14,mem=1024 -numa +> +> > > > > node,nodeid=3,cpus=15-19,mem=1024 -uuid +> +> > > > > 34a588c7-b0f2-4952-b39c-47fae3411439 -no-user-config +> +> > > > > -nodefaults -chardev +> +> > > > > socket,id=charmonitor,path=/var/run/libvirt/qemu/domain-194-Li +> +> > > > > nux/ +> +> > > > > moni +> +> > > > > tor.sock,server,nowait -mon +> +> > > > > chardev=charmonitor,id=monitor,mode=control -rtc base=utc +> +> > > > > -no-hpet -global kvm-pit.lost_tick_policy=delay -no-shutdown +> +> > > > > -boot strict=on -device +> +> > > > > pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x8 -device +> +> > > > > pci-bridge,chassis_nr=2,id=pci.2,bus=pci.0,addr=0x9 -device +> +> > > > > pci-bridge,chassis_nr=3,id=pci.3,bus=pci.0,addr=0xa -device +> +> > > > > pci-bridge,chassis_nr=4,id=pci.4,bus=pci.0,addr=0xb -device +> +> > > > > pci-bridge,chassis_nr=5,id=pci.5,bus=pci.0,addr=0xc -device +> +> > > > > piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device +> +> > > > > usb-ehci,id=usb1,bus=pci.0,addr=0x10 -device +> +> > > > > nec-usb-xhci,id=usb2,bus=pci.0,addr=0x11 -device +> +> > > > > virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 -device +> +> > > > > virtio-scsi-pci,id=scsi1,bus=pci.0,addr=0x4 -device +> +> > > > > virtio-scsi-pci,id=scsi2,bus=pci.0,addr=0x5 -device +> +> > > > > virtio-scsi-pci,id=scsi3,bus=pci.0,addr=0x6 -device +> +> > > > > virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive +> +> > > > > file=/mnt/sdb/xml/centos_74_x64_uefi.raw,format=raw,if=none,id +> +> > > > > =dri +> +> > > > > ve-v +> +> > > > > irtio-disk0,cache=none -device +> +> > > > > virtio-blk-pci,scsi=off,bus=pci.0,addr=0x2,drive=drive-virtio- +> +> > > > > disk +> +> > > > > 0,id +> +> > > > > =virtio-disk0,bootindex=1 -drive +> +> > > > > if=none,id=drive-ide0-1-1,readonly=on,cache=none -device +> +> > > > > ide-cd,bus=ide.1,unit=1,drive=drive-ide0-1-1,id=ide0-1-1 +> +> > > > > -netdev +> +> > > > > tap,fd=35,id=hostnet0 -device +> +> > > > > virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:89:5d:8b,b +> +> > > > > us=p +> +> > > > > ci.4 +> +> > > > > ,addr=0x1 -chardev pty,id=charserial0 -device +> +> > > > > isa-serial,chardev=charserial0,id=serial0 -device +> +> > > > > usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -device +> +> > > > > cirrus-vga,id=video0,vgamem_mb=8,bus=pci.0,addr=0x12 -device +> +> > > > > virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xd -msg +> +> > > > > timestamp=on +> +> > > > > +> +> > > > > I am also very curious about this issue, in the linux kernel +> +> > > > > code, maybe double +> +> > > > check in function pci_bridge_check_ranges triggered this problem. +> +> > > > +> +> > > > If you can get the stacktrace in Linux when it tries to write +> +> > > > this fffff value, that would be quite helpful. +> +> > > > +> +> > > +> +> > > After I add mdelay(100) in function pci_bridge_check_ranges, this +> +> > > phenomenon is easier to reproduce, below is my modify in kernel: +> +> > > diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c +> +> > > index cb389277..86e232d 100644 +> +> > > --- a/drivers/pci/setup-bus.c +> +> > > +++ b/drivers/pci/setup-bus.c +> +> > > @@ -27,7 +27,7 @@ +> +> > > #include <linux/slab.h> +> +> > > #include <linux/acpi.h> +> +> > > #include "pci.h" +> +> > > - +> +> > > +#include <linux/delay.h> +> +> > > unsigned int pci_flags; +> +> > > +> +> > > struct pci_dev_resource { +> +> > > @@ -787,6 +787,9 @@ static void pci_bridge_check_ranges(struct +> +> > > pci_bus +> +> > *bus) +> +> > > pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +> > > 0xffffffff); +> +> > > pci_read_config_dword(bridge, +> +> > > PCI_PREF_BASE_UPPER32, &tmp); +> +> > > + mdelay(100); +> +> > > + printk(KERN_ERR "sleep\n"); +> +> > > + dump_stack(); +> +> > > if (!tmp) +> +> > > b_res[2].flags &= ~IORESOURCE_MEM_64; +> +> > > pci_write_config_dword(bridge, +> +> > > PCI_PREF_BASE_UPPER32, +> +> > > +> +> > +> +> > OK! +> +> > I just sent a Linux patch that should help. +> +> > I would appreciate it if you will give it a try and if that helps +> +> > reply to it with a +> +> > Tested-by: tag. +> +> > +> +> +> +> I tested this patch and it works fine on my machine. +> +> +> +> But I have another question, if we only fix this problem in the +> +> kernel, the Linux version that has been released does not work well on the +> +virtualization platform. +> +> Is there a way to fix this problem in the backend? +> +> +There could we a way to work around this. +> +Does below help? +I am sorry to tell you, I tested this patch and it doesn't work fine. + +> +> +diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index +> +236a20eaa8..7834cac4b0 100644 +> +--- a/hw/i386/acpi-build.c +> ++++ b/hw/i386/acpi-build.c +> +@@ -551,7 +551,7 @@ static void build_append_pci_bus_devices(Aml +> +*parent_scope, PCIBus *bus, +> +> +aml_append(method, aml_store(aml_int(bsel_val), aml_name("BNUM"))); +> +aml_append(method, +> +- aml_call2("DVNT", aml_name("PCIU"), aml_int(1) /* Device Check +> +*/) +> ++ aml_call2("DVNT", aml_name("PCIU"), aml_int(4) /* Device +> ++ Check Light */) +> +); +> +aml_append(method, +> +aml_call2("DVNT", aml_name("PCID"), aml_int(3)/* Eject Request +> +*/) + +On Tue, Dec 11, 2018 at 03:51:09AM +0000, xuyandong wrote: +> +> There could we a way to work around this. +> +> Does below help? +> +> +I am sorry to tell you, I tested this patch and it doesn't work fine. +What happens? + +> +> +> +> diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index +> +> 236a20eaa8..7834cac4b0 100644 +> +> --- a/hw/i386/acpi-build.c +> +> +++ b/hw/i386/acpi-build.c +> +> @@ -551,7 +551,7 @@ static void build_append_pci_bus_devices(Aml +> +> *parent_scope, PCIBus *bus, +> +> +> +> aml_append(method, aml_store(aml_int(bsel_val), aml_name("BNUM"))); +> +> aml_append(method, +> +> - aml_call2("DVNT", aml_name("PCIU"), aml_int(1) /* Device Check +> +> */) +> +> + aml_call2("DVNT", aml_name("PCIU"), aml_int(4) /* Device +> +> + Check Light */) +> +> ); +> +> aml_append(method, +> +> aml_call2("DVNT", aml_name("PCID"), aml_int(3)/* Eject Request +> +> */) + +On Tue, Dec 11, 2018 at 03:51:09AM +0000, xuyandong wrote: +> +> On Tue, Dec 11, 2018 at 02:55:43AM +0000, xuyandong wrote: +> +> > On Tue, Dec 11, 2018 at 01:47:37AM +0000, xuyandong wrote: +> +> > > > On Sat, Dec 08, 2018 at 11:58:59AM +0000, xuyandong wrote: +> +> > > > > > > > Hi all, +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > In our test, we configured VM with several pci-bridges and +> +> > > > > > > > a virtio-net nic been attached with bus 4, +> +> > > > > > > > +> +> > > > > > > > After VM is startup, We ping this nic from host to judge +> +> > > > > > > > if it is working normally. Then, we hot add pci devices to +> +> > > > > > > > this VM with bus +> +> > > 0. +> +> > > > > > > > +> +> > > > > > > > We found the virtio-net NIC in bus 4 is not working (can +> +> > > > > > > > not +> +> > > > > > > > connect) occasionally, as it kick virtio backend failure with +> +> > > > > > > > error +> +> below: +> +> > > > > > > > +> +> > > > > > > > Unassigned mem write 00000000fc803004 = 0x1 +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > memory-region: pci_bridge_pci +> +> > > > > > > > +> +> > > > > > > > 0000000000000000-ffffffffffffffff (prio 0, RW): +> +> > > > > > > > pci_bridge_pci +> +> > > > > > > > +> +> > > > > > > > 00000000fc800000-00000000fc803fff (prio 1, RW): +> +> > > > > > > > virtio-pci +> +> > > > > > > > +> +> > > > > > > > 00000000fc800000-00000000fc800fff (prio 0, RW): +> +> > > > > > > > virtio-pci-common +> +> > > > > > > > +> +> > > > > > > > 00000000fc801000-00000000fc801fff (prio 0, RW): +> +> > > > > > > > virtio-pci-isr +> +> > > > > > > > +> +> > > > > > > > 00000000fc802000-00000000fc802fff (prio 0, RW): +> +> > > > > > > > virtio-pci-device +> +> > > > > > > > +> +> > > > > > > > 00000000fc803000-00000000fc803fff (prio 0, RW): +> +> > > > > > > > virtio-pci-notify <- io mem unassigned +> +> > > > > > > > +> +> > > > > > > > ⦠+> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > We caught an exceptional address changing while this +> +> > > > > > > > problem happened, show as +> +> > > > > > > > follow: +> +> > > > > > > > +> +> > > > > > > > Before pci_bridge_update_mappingsï¼ +> +> > > > > > > > +> +> > > > > > > > 00000000fc000000-00000000fc1fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > > 00000000fc000000-00000000fc1fffff +> +> > > > > > > > +> +> > > > > > > > 00000000fc200000-00000000fc3fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > > 00000000fc200000-00000000fc3fffff +> +> > > > > > > > +> +> > > > > > > > 00000000fc400000-00000000fc5fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > > 00000000fc400000-00000000fc5fffff +> +> > > > > > > > +> +> > > > > > > > 00000000fc600000-00000000fc7fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > > 00000000fc600000-00000000fc7fffff +> +> > > > > > > > +> +> > > > > > > > 00000000fc800000-00000000fc9fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > > 00000000fc800000-00000000fc9fffff +> +> > > > > > > > <- correct Adress Spce +> +> > > > > > > > +> +> > > > > > > > 00000000fca00000-00000000fcbfffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > > 00000000fca00000-00000000fcbfffff +> +> > > > > > > > +> +> > > > > > > > 00000000fcc00000-00000000fcdfffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > > 00000000fcc00000-00000000fcdfffff +> +> > > > > > > > +> +> > > > > > > > 00000000fce00000-00000000fcffffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_pref_mem @pci_bridge_pci +> +> > > > > > > > 00000000fce00000-00000000fcffffff +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > After pci_bridge_update_mappingsï¼ +> +> > > > > > > > +> +> > > > > > > > 00000000fda00000-00000000fdbfffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > > 00000000fda00000-00000000fdbfffff +> +> > > > > > > > +> +> > > > > > > > 00000000fdc00000-00000000fddfffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > > 00000000fdc00000-00000000fddfffff +> +> > > > > > > > +> +> > > > > > > > 00000000fde00000-00000000fdffffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > > 00000000fde00000-00000000fdffffff +> +> > > > > > > > +> +> > > > > > > > 00000000fe000000-00000000fe1fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > > 00000000fe000000-00000000fe1fffff +> +> > > > > > > > +> +> > > > > > > > 00000000fe200000-00000000fe3fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > > 00000000fe200000-00000000fe3fffff +> +> > > > > > > > +> +> > > > > > > > 00000000fe400000-00000000fe5fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > > 00000000fe400000-00000000fe5fffff +> +> > > > > > > > +> +> > > > > > > > 00000000fe600000-00000000fe7fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > > 00000000fe600000-00000000fe7fffff +> +> > > > > > > > +> +> > > > > > > > 00000000fe800000-00000000fe9fffff (prio 1, RW): +> +> > > > > > > > alias pci_bridge_mem @pci_bridge_pci +> +> > > > > > > > 00000000fe800000-00000000fe9fffff +> +> > > > > > > > +> +> > > > > > > > fffffffffc800000-fffffffffc800000 (prio 1, RW): +> +> > > > > > > > alias +> +> > > > > pci_bridge_pref_mem +> +> > > > > > > > @pci_bridge_pci fffffffffc800000-fffffffffc800000 <- +> +> > > > > > > > Exceptional +> +> > > Adress +> +> > > > > > > Space +> +> > > > > > > +> +> > > > > > > This one is empty though right? +> +> > > > > > > +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > We have figured out why this address becomes this value, +> +> > > > > > > > according to pci spec, pci driver can get BAR address +> +> > > > > > > > size by writing 0xffffffff to +> +> > > > > > > > +> +> > > > > > > > the pci register firstly, and then read back the value from +> +> > > > > > > > this +> +> register. +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > OK however as you show below the BAR being sized is the BAR +> +> > > > > > > if a bridge. Are you then adding a bridge device by hotplug? +> +> > > > > > +> +> > > > > > No, I just simply hot plugged a VFIO device to Bus 0, another +> +> > > > > > interesting phenomenon is If I hot plug the device to other +> +> > > > > > bus, this doesn't +> +> > > > > happened. +> +> > > > > > +> +> > > > > > > +> +> > > > > > > +> +> > > > > > > > We didn't handle this value specially while process pci +> +> > > > > > > > write in qemu, the function call stack is: +> +> > > > > > > > +> +> > > > > > > > Pci_bridge_dev_write_config +> +> > > > > > > > +> +> > > > > > > > -> pci_bridge_write_config +> +> > > > > > > > +> +> > > > > > > > -> pci_default_write_config (we update the config[address] +> +> > > > > > > > -> value here to +> +> > > > > > > > fffffffffc800000, which should be 0xfc800000 ) +> +> > > > > > > > +> +> > > > > > > > -> pci_bridge_update_mappings +> +> > > > > > > > +> +> > > > > > > > ->pci_bridge_region_del(br, br->windows); +> +> > > > > > > > +> +> > > > > > > > -> pci_bridge_region_init +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > -> pci_bridge_init_alias (here pci_bridge_get_base, we use +> +> > > > > > > > -> the +> +> > > > > > > > wrong value +> +> > > > > > > > fffffffffc800000) +> +> > > > > > > > +> +> > > > > > > > -> +> +> > > > > > > > memory_region_transaction_commit +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > So, as we can see, we use the wrong base address in qemu +> +> > > > > > > > to update the memory regions, though, we update the base +> +> > > > > > > > address to +> +> > > > > > > > +> +> > > > > > > > The correct value after pci driver in VM write the +> +> > > > > > > > original value back, the virtio NIC in bus 4 may still +> +> > > > > > > > sends net packets concurrently with +> +> > > > > > > > +> +> > > > > > > > The wrong memory region address. +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > +> +> > > > > > > > We have tried to skip the memory region update action in +> +> > > > > > > > qemu while detect pci write with 0xffffffff value, and it +> +> > > > > > > > does work, but +> +> > > > > > > > +> +> > > > > > > > This seems to be not gently. +> +> > > > > > > +> +> > > > > > > For sure. But I'm still puzzled as to why does Linux try to +> +> > > > > > > size the BAR of the bridge while a device behind it is used. +> +> > > > > > > +> +> > > > > > > Can you pls post your QEMU command line? +> +> > > > > > +> +> > > > > > My QEMU command line: +> +> > > > > > /root/xyd/qemu-system-x86_64 -name +> +> > > > > > guest=Linux,debug-threads=on -S -object +> +> > > > > > secret,id=masterKey0,format=raw,file=/var/run/libvirt/qemu/dom +> +> > > > > > ain- +> +> > > > > > 194- +> +> > > > > > Linux/master-key.aes -machine +> +> > > > > > pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off -cpu +> +> > > > > > host,+kvm_pv_eoi -bios /usr/share/OVMF/OVMF.fd -m +> +> > > > > > size=4194304k,slots=256,maxmem=33554432k -realtime mlock=off +> +> > > > > > -smp +> +> > > > > > 20,sockets=20,cores=1,threads=1 -numa +> +> > > > > > node,nodeid=0,cpus=0-4,mem=1024 -numa +> +> > > > > > node,nodeid=1,cpus=5-9,mem=1024 -numa +> +> > > > > > node,nodeid=2,cpus=10-14,mem=1024 -numa +> +> > > > > > node,nodeid=3,cpus=15-19,mem=1024 -uuid +> +> > > > > > 34a588c7-b0f2-4952-b39c-47fae3411439 -no-user-config +> +> > > > > > -nodefaults -chardev +> +> > > > > > socket,id=charmonitor,path=/var/run/libvirt/qemu/domain-194-Li +> +> > > > > > nux/ +> +> > > > > > moni +> +> > > > > > tor.sock,server,nowait -mon +> +> > > > > > chardev=charmonitor,id=monitor,mode=control -rtc base=utc +> +> > > > > > -no-hpet -global kvm-pit.lost_tick_policy=delay -no-shutdown +> +> > > > > > -boot strict=on -device +> +> > > > > > pci-bridge,chassis_nr=1,id=pci.1,bus=pci.0,addr=0x8 -device +> +> > > > > > pci-bridge,chassis_nr=2,id=pci.2,bus=pci.0,addr=0x9 -device +> +> > > > > > pci-bridge,chassis_nr=3,id=pci.3,bus=pci.0,addr=0xa -device +> +> > > > > > pci-bridge,chassis_nr=4,id=pci.4,bus=pci.0,addr=0xb -device +> +> > > > > > pci-bridge,chassis_nr=5,id=pci.5,bus=pci.0,addr=0xc -device +> +> > > > > > piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device +> +> > > > > > usb-ehci,id=usb1,bus=pci.0,addr=0x10 -device +> +> > > > > > nec-usb-xhci,id=usb2,bus=pci.0,addr=0x11 -device +> +> > > > > > virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 -device +> +> > > > > > virtio-scsi-pci,id=scsi1,bus=pci.0,addr=0x4 -device +> +> > > > > > virtio-scsi-pci,id=scsi2,bus=pci.0,addr=0x5 -device +> +> > > > > > virtio-scsi-pci,id=scsi3,bus=pci.0,addr=0x6 -device +> +> > > > > > virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x7 -drive +> +> > > > > > file=/mnt/sdb/xml/centos_74_x64_uefi.raw,format=raw,if=none,id +> +> > > > > > =dri +> +> > > > > > ve-v +> +> > > > > > irtio-disk0,cache=none -device +> +> > > > > > virtio-blk-pci,scsi=off,bus=pci.0,addr=0x2,drive=drive-virtio- +> +> > > > > > disk +> +> > > > > > 0,id +> +> > > > > > =virtio-disk0,bootindex=1 -drive +> +> > > > > > if=none,id=drive-ide0-1-1,readonly=on,cache=none -device +> +> > > > > > ide-cd,bus=ide.1,unit=1,drive=drive-ide0-1-1,id=ide0-1-1 +> +> > > > > > -netdev +> +> > > > > > tap,fd=35,id=hostnet0 -device +> +> > > > > > virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:89:5d:8b,b +> +> > > > > > us=p +> +> > > > > > ci.4 +> +> > > > > > ,addr=0x1 -chardev pty,id=charserial0 -device +> +> > > > > > isa-serial,chardev=charserial0,id=serial0 -device +> +> > > > > > usb-tablet,id=input0,bus=usb.0,port=1 -vnc 0.0.0.0:0 -device +> +> > > > > > cirrus-vga,id=video0,vgamem_mb=8,bus=pci.0,addr=0x12 -device +> +> > > > > > virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0xd -msg +> +> > > > > > timestamp=on +> +> > > > > > +> +> > > > > > I am also very curious about this issue, in the linux kernel +> +> > > > > > code, maybe double +> +> > > > > check in function pci_bridge_check_ranges triggered this problem. +> +> > > > > +> +> > > > > If you can get the stacktrace in Linux when it tries to write +> +> > > > > this fffff value, that would be quite helpful. +> +> > > > > +> +> > > > +> +> > > > After I add mdelay(100) in function pci_bridge_check_ranges, this +> +> > > > phenomenon is easier to reproduce, below is my modify in kernel: +> +> > > > diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c +> +> > > > index cb389277..86e232d 100644 +> +> > > > --- a/drivers/pci/setup-bus.c +> +> > > > +++ b/drivers/pci/setup-bus.c +> +> > > > @@ -27,7 +27,7 @@ +> +> > > > #include <linux/slab.h> +> +> > > > #include <linux/acpi.h> +> +> > > > #include "pci.h" +> +> > > > - +> +> > > > +#include <linux/delay.h> +> +> > > > unsigned int pci_flags; +> +> > > > +> +> > > > struct pci_dev_resource { +> +> > > > @@ -787,6 +787,9 @@ static void pci_bridge_check_ranges(struct +> +> > > > pci_bus +> +> > > *bus) +> +> > > > pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, +> +> > > > 0xffffffff); +> +> > > > pci_read_config_dword(bridge, +> +> > > > PCI_PREF_BASE_UPPER32, &tmp); +> +> > > > + mdelay(100); +> +> > > > + printk(KERN_ERR "sleep\n"); +> +> > > > + dump_stack(); +> +> > > > if (!tmp) +> +> > > > b_res[2].flags &= ~IORESOURCE_MEM_64; +> +> > > > pci_write_config_dword(bridge, +> +> > > > PCI_PREF_BASE_UPPER32, +> +> > > > +> +> > > +> +> > > OK! +> +> > > I just sent a Linux patch that should help. +> +> > > I would appreciate it if you will give it a try and if that helps +> +> > > reply to it with a +> +> > > Tested-by: tag. +> +> > > +> +> > +> +> > I tested this patch and it works fine on my machine. +> +> > +> +> > But I have another question, if we only fix this problem in the +> +> > kernel, the Linux version that has been released does not work well on the +> +> virtualization platform. +> +> > Is there a way to fix this problem in the backend? +> +> +> +> There could we a way to work around this. +> +> Does below help? +> +> +I am sorry to tell you, I tested this patch and it doesn't work fine. +> +> +> +> +> diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index +> +> 236a20eaa8..7834cac4b0 100644 +> +> --- a/hw/i386/acpi-build.c +> +> +++ b/hw/i386/acpi-build.c +> +> @@ -551,7 +551,7 @@ static void build_append_pci_bus_devices(Aml +> +> *parent_scope, PCIBus *bus, +> +> +> +> aml_append(method, aml_store(aml_int(bsel_val), aml_name("BNUM"))); +> +> aml_append(method, +> +> - aml_call2("DVNT", aml_name("PCIU"), aml_int(1) /* Device Check +> +> */) +> +> + aml_call2("DVNT", aml_name("PCIU"), aml_int(4) /* Device +> +> + Check Light */) +> +> ); +> +> aml_append(method, +> +> aml_call2("DVNT", aml_name("PCID"), aml_int(3)/* Eject Request +> +> */) +Oh I see, another bug: + + case ACPI_NOTIFY_DEVICE_CHECK_LIGHT: + acpi_handle_debug(handle, "ACPI_NOTIFY_DEVICE_CHECK_LIGHT +event\n"); + /* TBD: Exactly what does 'light' mean? */ + break; + +And then e.g. acpi_generic_hotplug_event(struct acpi_device *adev, u32 type) +and friends all just ignore this event type. + + + +-- +MST + +> +> > > > > > > > > Hi all, +> +> > > > > > > > > +> +> > > > > > > > > +> +> > > > > > > > > +> +> > > > > > > > > In our test, we configured VM with several pci-bridges +> +> > > > > > > > > and a virtio-net nic been attached with bus 4, +> +> > > > > > > > > +> +> > > > > > > > > After VM is startup, We ping this nic from host to +> +> > > > > > > > > judge if it is working normally. Then, we hot add pci +> +> > > > > > > > > devices to this VM with bus +> +> > > > 0. +> +> > > > > > > > > +> +> > > > > > > > > We found the virtio-net NIC in bus 4 is not working +> +> > > > > > > > > (can not +> +> > > > > > > > > connect) occasionally, as it kick virtio backend +> +> > > > > > > > > failure with error +> +> > > But I have another question, if we only fix this problem in the +> +> > > kernel, the Linux version that has been released does not work +> +> > > well on the +> +> > virtualization platform. +> +> > > Is there a way to fix this problem in the backend? +> +> > +> +> > There could we a way to work around this. +> +> > Does below help? +> +> +> +> I am sorry to tell you, I tested this patch and it doesn't work fine. +> +> +> +> > +> +> > diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index +> +> > 236a20eaa8..7834cac4b0 100644 +> +> > --- a/hw/i386/acpi-build.c +> +> > +++ b/hw/i386/acpi-build.c +> +> > @@ -551,7 +551,7 @@ static void build_append_pci_bus_devices(Aml +> +> > *parent_scope, PCIBus *bus, +> +> > +> +> > aml_append(method, aml_store(aml_int(bsel_val), +> +aml_name("BNUM"))); +> +> > aml_append(method, +> +> > - aml_call2("DVNT", aml_name("PCIU"), aml_int(1) /* Device +> +> > Check +> +*/) +> +> > + aml_call2("DVNT", aml_name("PCIU"), aml_int(4) /* +> +> > + Device Check Light */) +> +> > ); +> +> > aml_append(method, +> +> > aml_call2("DVNT", aml_name("PCID"), aml_int(3)/* Eject +> +> > Request */) +> +> +> +Oh I see, another bug: +> +> +case ACPI_NOTIFY_DEVICE_CHECK_LIGHT: +> +acpi_handle_debug(handle, "ACPI_NOTIFY_DEVICE_CHECK_LIGHT +> +event\n"); +> +/* TBD: Exactly what does 'light' mean? */ +> +break; +> +> +And then e.g. acpi_generic_hotplug_event(struct acpi_device *adev, u32 type) +> +and friends all just ignore this event type. +> +> +> +> +-- +> +MST +Hi Michael, + +If we want to fix this problem on the backend, it is not enough to consider +only PCI +device hot plugging, because I found that if we use a command like +"echo 1 > /sys/bus/pci/rescan" in guest, this problem is very easy to reproduce. + +From the perspective of device emulation, when guest writes 0xffffffff to the +BAR, +guest just want to get the size of the region but not really updating the +address space. +So I made the following patch to avoid update pci mapping. + +Do you think this make sense? + +[PATCH] pci: avoid update pci mapping when writing 0xFFFF FFFF to BAR + +When guest writes 0xffffffff to the BAR, guest just want to get the size of the +region +but not really updating the address space. +So when guest writes 0xffffffff to BAR, we need avoid pci_update_mappings +or pci_bridge_update_mappings. + +Signed-off-by: xuyandong <address@hidden> +--- + hw/pci/pci.c | 6 ++++-- + hw/pci/pci_bridge.c | 8 +++++--- + 2 files changed, 9 insertions(+), 5 deletions(-) + +diff --git a/hw/pci/pci.c b/hw/pci/pci.c +index 56b13b3..ef368e1 100644 +--- a/hw/pci/pci.c ++++ b/hw/pci/pci.c +@@ -1361,6 +1361,7 @@ void pci_default_write_config(PCIDevice *d, uint32_t +addr, uint32_t val_in, int + { + int i, was_irq_disabled = pci_irq_disabled(d); + uint32_t val = val_in; ++ uint64_t barmask = (1 << l*8) - 1; + + for (i = 0; i < l; val >>= 8, ++i) { + uint8_t wmask = d->wmask[addr + i]; +@@ -1369,9 +1370,10 @@ void pci_default_write_config(PCIDevice *d, uint32_t +addr, uint32_t val_in, int + d->config[addr + i] = (d->config[addr + i] & ~wmask) | (val & wmask); + d->config[addr + i] &= ~(val & w1cmask); /* W1C: Write 1 to Clear */ + } +- if (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || ++ if ((val_in != barmask && ++ (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || + ranges_overlap(addr, l, PCI_ROM_ADDRESS, 4) || +- ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4) || ++ ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4))) || + range_covers_byte(addr, l, PCI_COMMAND)) + pci_update_mappings(d); + +diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c +index ee9dff2..f2bad79 100644 +--- a/hw/pci/pci_bridge.c ++++ b/hw/pci/pci_bridge.c +@@ -253,17 +253,19 @@ void pci_bridge_write_config(PCIDevice *d, + PCIBridge *s = PCI_BRIDGE(d); + uint16_t oldctl = pci_get_word(d->config + PCI_BRIDGE_CONTROL); + uint16_t newctl; ++ uint64_t barmask = (1 << len * 8) - 1; + + pci_default_write_config(d, address, val, len); + + if (ranges_overlap(address, len, PCI_COMMAND, 2) || + +- /* io base/limit */ +- ranges_overlap(address, len, PCI_IO_BASE, 2) || ++ (val != barmask && ++ /* io base/limit */ ++ (ranges_overlap(address, len, PCI_IO_BASE, 2) || + + /* memory base/limit, prefetchable base/limit and + io base/limit upper 16 */ +- ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || ++ ranges_overlap(address, len, PCI_MEMORY_BASE, 20))) || + + /* vga enable */ + ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +-- +1.8.3.1 + +On Mon, Jan 07, 2019 at 02:37:17PM +0000, xuyandong wrote: +> +> > > > > > > > > > Hi all, +> +> > > > > > > > > > +> +> > > > > > > > > > +> +> > > > > > > > > > +> +> > > > > > > > > > In our test, we configured VM with several pci-bridges +> +> > > > > > > > > > and a virtio-net nic been attached with bus 4, +> +> > > > > > > > > > +> +> > > > > > > > > > After VM is startup, We ping this nic from host to +> +> > > > > > > > > > judge if it is working normally. Then, we hot add pci +> +> > > > > > > > > > devices to this VM with bus +> +> > > > > 0. +> +> > > > > > > > > > +> +> > > > > > > > > > We found the virtio-net NIC in bus 4 is not working +> +> > > > > > > > > > (can not +> +> > > > > > > > > > connect) occasionally, as it kick virtio backend +> +> > > > > > > > > > failure with error +> +> +> > > > But I have another question, if we only fix this problem in the +> +> > > > kernel, the Linux version that has been released does not work +> +> > > > well on the +> +> > > virtualization platform. +> +> > > > Is there a way to fix this problem in the backend? +> +> > > +> +> > > There could we a way to work around this. +> +> > > Does below help? +> +> > +> +> > I am sorry to tell you, I tested this patch and it doesn't work fine. +> +> > +> +> > > +> +> > > diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index +> +> > > 236a20eaa8..7834cac4b0 100644 +> +> > > --- a/hw/i386/acpi-build.c +> +> > > +++ b/hw/i386/acpi-build.c +> +> > > @@ -551,7 +551,7 @@ static void build_append_pci_bus_devices(Aml +> +> > > *parent_scope, PCIBus *bus, +> +> > > +> +> > > aml_append(method, aml_store(aml_int(bsel_val), +> +> aml_name("BNUM"))); +> +> > > aml_append(method, +> +> > > - aml_call2("DVNT", aml_name("PCIU"), aml_int(1) /* Device +> +> > > Check +> +> */) +> +> > > + aml_call2("DVNT", aml_name("PCIU"), aml_int(4) /* +> +> > > + Device Check Light */) +> +> > > ); +> +> > > aml_append(method, +> +> > > aml_call2("DVNT", aml_name("PCID"), aml_int(3)/* Eject +> +> > > Request */) +> +> +> +> +> +> Oh I see, another bug: +> +> +> +> case ACPI_NOTIFY_DEVICE_CHECK_LIGHT: +> +> acpi_handle_debug(handle, "ACPI_NOTIFY_DEVICE_CHECK_LIGHT +> +> event\n"); +> +> /* TBD: Exactly what does 'light' mean? */ +> +> break; +> +> +> +> And then e.g. acpi_generic_hotplug_event(struct acpi_device *adev, u32 type) +> +> and friends all just ignore this event type. +> +> +> +> +> +> +> +> -- +> +> MST +> +> +Hi Michael, +> +> +If we want to fix this problem on the backend, it is not enough to consider +> +only PCI +> +device hot plugging, because I found that if we use a command like +> +"echo 1 > /sys/bus/pci/rescan" in guest, this problem is very easy to +> +reproduce. +> +> +From the perspective of device emulation, when guest writes 0xffffffff to the +> +BAR, +> +guest just want to get the size of the region but not really updating the +> +address space. +> +So I made the following patch to avoid update pci mapping. +> +> +Do you think this make sense? +> +> +[PATCH] pci: avoid update pci mapping when writing 0xFFFF FFFF to BAR +> +> +When guest writes 0xffffffff to the BAR, guest just want to get the size of +> +the region +> +but not really updating the address space. +> +So when guest writes 0xffffffff to BAR, we need avoid pci_update_mappings +> +or pci_bridge_update_mappings. +> +> +Signed-off-by: xuyandong <address@hidden> +I see how that will address the common case however there are a bunch of +issues here. First of all it's easy to trigger the update by some other +action like VM migration. More importantly it's just possible that +guest actually does want to set the low 32 bit of the address to all +ones. For example, that is clearly listed as a way to disable all +devices behind the bridge in the pci to pci bridge spec. + +Given upstream is dragging it's feet I'm open to adding a flag +that will help keep guests going as a temporary measure. +We will need to think about ways to restrict this as much as +we can. + + +> +--- +> +hw/pci/pci.c | 6 ++++-- +> +hw/pci/pci_bridge.c | 8 +++++--- +> +2 files changed, 9 insertions(+), 5 deletions(-) +> +> +diff --git a/hw/pci/pci.c b/hw/pci/pci.c +> +index 56b13b3..ef368e1 100644 +> +--- a/hw/pci/pci.c +> ++++ b/hw/pci/pci.c +> +@@ -1361,6 +1361,7 @@ void pci_default_write_config(PCIDevice *d, uint32_t +> +addr, uint32_t val_in, int +> +{ +> +int i, was_irq_disabled = pci_irq_disabled(d); +> +uint32_t val = val_in; +> ++ uint64_t barmask = (1 << l*8) - 1; +> +> +for (i = 0; i < l; val >>= 8, ++i) { +> +uint8_t wmask = d->wmask[addr + i]; +> +@@ -1369,9 +1370,10 @@ void pci_default_write_config(PCIDevice *d, uint32_t +> +addr, uint32_t val_in, int +> +d->config[addr + i] = (d->config[addr + i] & ~wmask) | (val & wmask); +> +d->config[addr + i] &= ~(val & w1cmask); /* W1C: Write 1 to Clear */ +> +} +> +- if (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || +> ++ if ((val_in != barmask && +> ++ (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || +> +ranges_overlap(addr, l, PCI_ROM_ADDRESS, 4) || +> +- ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4) || +> ++ ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4))) || +> +range_covers_byte(addr, l, PCI_COMMAND)) +> +pci_update_mappings(d); +> +> +diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c +> +index ee9dff2..f2bad79 100644 +> +--- a/hw/pci/pci_bridge.c +> ++++ b/hw/pci/pci_bridge.c +> +@@ -253,17 +253,19 @@ void pci_bridge_write_config(PCIDevice *d, +> +PCIBridge *s = PCI_BRIDGE(d); +> +uint16_t oldctl = pci_get_word(d->config + PCI_BRIDGE_CONTROL); +> +uint16_t newctl; +> ++ uint64_t barmask = (1 << len * 8) - 1; +> +> +pci_default_write_config(d, address, val, len); +> +> +if (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> +- /* io base/limit */ +> +- ranges_overlap(address, len, PCI_IO_BASE, 2) || +> ++ (val != barmask && +> ++ /* io base/limit */ +> ++ (ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> +/* memory base/limit, prefetchable base/limit and +> +io base/limit upper 16 */ +> +- ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || +> ++ ranges_overlap(address, len, PCI_MEMORY_BASE, 20))) || +> +> +/* vga enable */ +> +ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +> +-- +> +1.8.3.1 +> +> +> + +> +-----Original Message----- +> +From: Michael S. Tsirkin [ +mailto:address@hidden +> +Sent: Monday, January 07, 2019 11:06 PM +> +To: xuyandong <address@hidden> +> +Cc: address@hidden; Paolo Bonzini <address@hidden>; qemu- +> +address@hidden; Zhanghailiang <address@hidden>; +> +wangxin (U) <address@hidden>; Huangweidong (C) +> +<address@hidden> +> +Subject: Re: [BUG]Unassigned mem write during pci device hot-plug +> +> +On Mon, Jan 07, 2019 at 02:37:17PM +0000, xuyandong wrote: +> +> > > > > > > > > > > Hi all, +> +> > > > > > > > > > > +> +> > > > > > > > > > > +> +> > > > > > > > > > > +> +> > > > > > > > > > > In our test, we configured VM with several +> +> > > > > > > > > > > pci-bridges and a virtio-net nic been attached +> +> > > > > > > > > > > with bus 4, +> +> > > > > > > > > > > +> +> > > > > > > > > > > After VM is startup, We ping this nic from host to +> +> > > > > > > > > > > judge if it is working normally. Then, we hot add +> +> > > > > > > > > > > pci devices to this VM with bus +> +> > > > > > 0. +> +> > > > > > > > > > > +> +> > > > > > > > > > > We found the virtio-net NIC in bus 4 is not +> +> > > > > > > > > > > working (can not +> +> > > > > > > > > > > connect) occasionally, as it kick virtio backend +> +> > > > > > > > > > > failure with error +> +> +> +> > > > > But I have another question, if we only fix this problem in +> +> > > > > the kernel, the Linux version that has been released does not +> +> > > > > work well on the +> +> > > > virtualization platform. +> +> > > > > Is there a way to fix this problem in the backend? +> +> +> +> Hi Michael, +> +> +> +> If we want to fix this problem on the backend, it is not enough to +> +> consider only PCI device hot plugging, because I found that if we use +> +> a command like "echo 1 > /sys/bus/pci/rescan" in guest, this problem is very +> +easy to reproduce. +> +> +> +> From the perspective of device emulation, when guest writes 0xffffffff +> +> to the BAR, guest just want to get the size of the region but not really +> +updating the address space. +> +> So I made the following patch to avoid update pci mapping. +> +> +> +> Do you think this make sense? +> +> +> +> [PATCH] pci: avoid update pci mapping when writing 0xFFFF FFFF to BAR +> +> +> +> When guest writes 0xffffffff to the BAR, guest just want to get the +> +> size of the region but not really updating the address space. +> +> So when guest writes 0xffffffff to BAR, we need avoid +> +> pci_update_mappings or pci_bridge_update_mappings. +> +> +> +> Signed-off-by: xuyandong <address@hidden> +> +> +I see how that will address the common case however there are a bunch of +> +issues here. First of all it's easy to trigger the update by some other +> +action like +> +VM migration. More importantly it's just possible that guest actually does +> +want +> +to set the low 32 bit of the address to all ones. For example, that is +> +clearly +> +listed as a way to disable all devices behind the bridge in the pci to pci +> +bridge +> +spec. +Ok, I see. If I only skip upate when guest writing 0xFFFFFFFF to Prefetcable +Base Upper 32 Bits +to meet the kernel double check problem. +Do you think there is still risk? + +> +> +Given upstream is dragging it's feet I'm open to adding a flag that will help +> +keep guests going as a temporary measure. +> +We will need to think about ways to restrict this as much as we can. +> +> +> +> --- +> +> hw/pci/pci.c | 6 ++++-- +> +> hw/pci/pci_bridge.c | 8 +++++--- +> +> 2 files changed, 9 insertions(+), 5 deletions(-) +> +> +> +> diff --git a/hw/pci/pci.c b/hw/pci/pci.c index 56b13b3..ef368e1 100644 +> +> --- a/hw/pci/pci.c +> +> +++ b/hw/pci/pci.c +> +> @@ -1361,6 +1361,7 @@ void pci_default_write_config(PCIDevice *d, +> +> uint32_t addr, uint32_t val_in, int { +> +> int i, was_irq_disabled = pci_irq_disabled(d); +> +> uint32_t val = val_in; +> +> + uint64_t barmask = (1 << l*8) - 1; +> +> +> +> for (i = 0; i < l; val >>= 8, ++i) { +> +> uint8_t wmask = d->wmask[addr + i]; @@ -1369,9 +1370,10 @@ +> +> void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val_in, +> +int +> +> d->config[addr + i] = (d->config[addr + i] & ~wmask) | (val & +> +> wmask); +> +> d->config[addr + i] &= ~(val & w1cmask); /* W1C: Write 1 to Clear +> +> */ +> +> } +> +> - if (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || +> +> + if ((val_in != barmask && +> +> + (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || +> +> ranges_overlap(addr, l, PCI_ROM_ADDRESS, 4) || +> +> - ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4) || +> +> + ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4))) || +> +> range_covers_byte(addr, l, PCI_COMMAND)) +> +> pci_update_mappings(d); +> +> +> +> diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c index +> +> ee9dff2..f2bad79 100644 +> +> --- a/hw/pci/pci_bridge.c +> +> +++ b/hw/pci/pci_bridge.c +> +> @@ -253,17 +253,19 @@ void pci_bridge_write_config(PCIDevice *d, +> +> PCIBridge *s = PCI_BRIDGE(d); +> +> uint16_t oldctl = pci_get_word(d->config + PCI_BRIDGE_CONTROL); +> +> uint16_t newctl; +> +> + uint64_t barmask = (1 << len * 8) - 1; +> +> +> +> pci_default_write_config(d, address, val, len); +> +> +> +> if (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> +> +> - /* io base/limit */ +> +> - ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> + (val != barmask && +> +> + /* io base/limit */ +> +> + (ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> +> +> /* memory base/limit, prefetchable base/limit and +> +> io base/limit upper 16 */ +> +> - ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || +> +> + ranges_overlap(address, len, PCI_MEMORY_BASE, 20))) || +> +> +> +> /* vga enable */ +> +> ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +> +> -- +> +> 1.8.3.1 +> +> +> +> +> +> + +On Mon, Jan 07, 2019 at 03:28:36PM +0000, xuyandong wrote: +> +> +> +> -----Original Message----- +> +> From: Michael S. Tsirkin [ +mailto:address@hidden +> +> Sent: Monday, January 07, 2019 11:06 PM +> +> To: xuyandong <address@hidden> +> +> Cc: address@hidden; Paolo Bonzini <address@hidden>; qemu- +> +> address@hidden; Zhanghailiang <address@hidden>; +> +> wangxin (U) <address@hidden>; Huangweidong (C) +> +> <address@hidden> +> +> Subject: Re: [BUG]Unassigned mem write during pci device hot-plug +> +> +> +> On Mon, Jan 07, 2019 at 02:37:17PM +0000, xuyandong wrote: +> +> > > > > > > > > > > > Hi all, +> +> > > > > > > > > > > > +> +> > > > > > > > > > > > +> +> > > > > > > > > > > > +> +> > > > > > > > > > > > In our test, we configured VM with several +> +> > > > > > > > > > > > pci-bridges and a virtio-net nic been attached +> +> > > > > > > > > > > > with bus 4, +> +> > > > > > > > > > > > +> +> > > > > > > > > > > > After VM is startup, We ping this nic from host to +> +> > > > > > > > > > > > judge if it is working normally. Then, we hot add +> +> > > > > > > > > > > > pci devices to this VM with bus +> +> > > > > > > 0. +> +> > > > > > > > > > > > +> +> > > > > > > > > > > > We found the virtio-net NIC in bus 4 is not +> +> > > > > > > > > > > > working (can not +> +> > > > > > > > > > > > connect) occasionally, as it kick virtio backend +> +> > > > > > > > > > > > failure with error +> +> > +> +> > > > > > But I have another question, if we only fix this problem in +> +> > > > > > the kernel, the Linux version that has been released does not +> +> > > > > > work well on the +> +> > > > > virtualization platform. +> +> > > > > > Is there a way to fix this problem in the backend? +> +> > +> +> > Hi Michael, +> +> > +> +> > If we want to fix this problem on the backend, it is not enough to +> +> > consider only PCI device hot plugging, because I found that if we use +> +> > a command like "echo 1 > /sys/bus/pci/rescan" in guest, this problem is +> +> > very +> +> easy to reproduce. +> +> > +> +> > From the perspective of device emulation, when guest writes 0xffffffff +> +> > to the BAR, guest just want to get the size of the region but not really +> +> updating the address space. +> +> > So I made the following patch to avoid update pci mapping. +> +> > +> +> > Do you think this make sense? +> +> > +> +> > [PATCH] pci: avoid update pci mapping when writing 0xFFFF FFFF to BAR +> +> > +> +> > When guest writes 0xffffffff to the BAR, guest just want to get the +> +> > size of the region but not really updating the address space. +> +> > So when guest writes 0xffffffff to BAR, we need avoid +> +> > pci_update_mappings or pci_bridge_update_mappings. +> +> > +> +> > Signed-off-by: xuyandong <address@hidden> +> +> +> +> I see how that will address the common case however there are a bunch of +> +> issues here. First of all it's easy to trigger the update by some other +> +> action like +> +> VM migration. More importantly it's just possible that guest actually does +> +> want +> +> to set the low 32 bit of the address to all ones. For example, that is +> +> clearly +> +> listed as a way to disable all devices behind the bridge in the pci to pci +> +> bridge +> +> spec. +> +> +Ok, I see. If I only skip upate when guest writing 0xFFFFFFFF to Prefetcable +> +Base Upper 32 Bits +> +to meet the kernel double check problem. +> +Do you think there is still risk? +Well it's non zero since spec says such a write should disable all +accesses. Just an idea: why not add an option to disable upper 32 bit? +That is ugly and limits space but spec compliant. + +> +> +> +> Given upstream is dragging it's feet I'm open to adding a flag that will +> +> help +> +> keep guests going as a temporary measure. +> +> We will need to think about ways to restrict this as much as we can. +> +> +> +> +> +> > --- +> +> > hw/pci/pci.c | 6 ++++-- +> +> > hw/pci/pci_bridge.c | 8 +++++--- +> +> > 2 files changed, 9 insertions(+), 5 deletions(-) +> +> > +> +> > diff --git a/hw/pci/pci.c b/hw/pci/pci.c index 56b13b3..ef368e1 100644 +> +> > --- a/hw/pci/pci.c +> +> > +++ b/hw/pci/pci.c +> +> > @@ -1361,6 +1361,7 @@ void pci_default_write_config(PCIDevice *d, +> +> > uint32_t addr, uint32_t val_in, int { +> +> > int i, was_irq_disabled = pci_irq_disabled(d); +> +> > uint32_t val = val_in; +> +> > + uint64_t barmask = (1 << l*8) - 1; +> +> > +> +> > for (i = 0; i < l; val >>= 8, ++i) { +> +> > uint8_t wmask = d->wmask[addr + i]; @@ -1369,9 +1370,10 @@ +> +> > void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t +> +> > val_in, +> +> int +> +> > d->config[addr + i] = (d->config[addr + i] & ~wmask) | (val & +> +> > wmask); +> +> > d->config[addr + i] &= ~(val & w1cmask); /* W1C: Write 1 to +> +> > Clear */ +> +> > } +> +> > - if (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || +> +> > + if ((val_in != barmask && +> +> > + (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || +> +> > ranges_overlap(addr, l, PCI_ROM_ADDRESS, 4) || +> +> > - ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4) || +> +> > + ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4))) || +> +> > range_covers_byte(addr, l, PCI_COMMAND)) +> +> > pci_update_mappings(d); +> +> > +> +> > diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c index +> +> > ee9dff2..f2bad79 100644 +> +> > --- a/hw/pci/pci_bridge.c +> +> > +++ b/hw/pci/pci_bridge.c +> +> > @@ -253,17 +253,19 @@ void pci_bridge_write_config(PCIDevice *d, +> +> > PCIBridge *s = PCI_BRIDGE(d); +> +> > uint16_t oldctl = pci_get_word(d->config + PCI_BRIDGE_CONTROL); +> +> > uint16_t newctl; +> +> > + uint64_t barmask = (1 << len * 8) - 1; +> +> > +> +> > pci_default_write_config(d, address, val, len); +> +> > +> +> > if (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> > +> +> > - /* io base/limit */ +> +> > - ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> > + (val != barmask && +> +> > + /* io base/limit */ +> +> > + (ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> > +> +> > /* memory base/limit, prefetchable base/limit and +> +> > io base/limit upper 16 */ +> +> > - ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || +> +> > + ranges_overlap(address, len, PCI_MEMORY_BASE, 20))) || +> +> > +> +> > /* vga enable */ +> +> > ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +> +> > -- +> +> > 1.8.3.1 +> +> > +> +> > +> +> > + +> +-----Original Message----- +> +From: xuyandong +> +Sent: Monday, January 07, 2019 10:37 PM +> +To: 'Michael S. Tsirkin' <address@hidden> +> +Cc: address@hidden; Paolo Bonzini <address@hidden>; qemu- +> +address@hidden; Zhanghailiang <address@hidden>; +> +wangxin (U) <address@hidden>; Huangweidong (C) +> +<address@hidden> +> +Subject: RE: [BUG]Unassigned mem write during pci device hot-plug +> +> +> > > > > > > > > > Hi all, +> +> > > > > > > > > > +> +> > > > > > > > > > +> +> > > > > > > > > > +> +> > > > > > > > > > In our test, we configured VM with several +> +> > > > > > > > > > pci-bridges and a virtio-net nic been attached with +> +> > > > > > > > > > bus 4, +> +> > > > > > > > > > +> +> > > > > > > > > > After VM is startup, We ping this nic from host to +> +> > > > > > > > > > judge if it is working normally. Then, we hot add +> +> > > > > > > > > > pci devices to this VM with bus +> +> > > > > 0. +> +> > > > > > > > > > +> +> > > > > > > > > > We found the virtio-net NIC in bus 4 is not working +> +> > > > > > > > > > (can not +> +> > > > > > > > > > connect) occasionally, as it kick virtio backend +> +> > > > > > > > > > failure with error +> +> +> > > > But I have another question, if we only fix this problem in the +> +> > > > kernel, the Linux version that has been released does not work +> +> > > > well on the +> +> > > virtualization platform. +> +> > > > Is there a way to fix this problem in the backend? +> +> > > +> +> > > There could we a way to work around this. +> +> > > Does below help? +> +> > +> +> > I am sorry to tell you, I tested this patch and it doesn't work fine. +> +> > +> +> > > +> +> > > diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c index +> +> > > 236a20eaa8..7834cac4b0 100644 +> +> > > --- a/hw/i386/acpi-build.c +> +> > > +++ b/hw/i386/acpi-build.c +> +> > > @@ -551,7 +551,7 @@ static void build_append_pci_bus_devices(Aml +> +> > > *parent_scope, PCIBus *bus, +> +> > > +> +> > > aml_append(method, aml_store(aml_int(bsel_val), +> +> aml_name("BNUM"))); +> +> > > aml_append(method, +> +> > > - aml_call2("DVNT", aml_name("PCIU"), aml_int(1) /* Device +> +Check +> +> */) +> +> > > + aml_call2("DVNT", aml_name("PCIU"), aml_int(4) /* +> +> > > + Device Check Light */) +> +> > > ); +> +> > > aml_append(method, +> +> > > aml_call2("DVNT", aml_name("PCID"), aml_int(3)/* +> +> > > Eject Request */) +> +> +> +> +> +> Oh I see, another bug: +> +> +> +> case ACPI_NOTIFY_DEVICE_CHECK_LIGHT: +> +> acpi_handle_debug(handle, +> +> "ACPI_NOTIFY_DEVICE_CHECK_LIGHT event\n"); +> +> /* TBD: Exactly what does 'light' mean? */ +> +> break; +> +> +> +> And then e.g. acpi_generic_hotplug_event(struct acpi_device *adev, u32 +> +> type) and friends all just ignore this event type. +> +> +> +> +> +> +> +> -- +> +> MST +> +> +Hi Michael, +> +> +If we want to fix this problem on the backend, it is not enough to consider +> +only +> +PCI device hot plugging, because I found that if we use a command like "echo +> +1 > +> +/sys/bus/pci/rescan" in guest, this problem is very easy to reproduce. +> +> +From the perspective of device emulation, when guest writes 0xffffffff to the +> +BAR, guest just want to get the size of the region but not really updating the +> +address space. +> +So I made the following patch to avoid update pci mapping. +> +> +Do you think this make sense? +> +> +[PATCH] pci: avoid update pci mapping when writing 0xFFFF FFFF to BAR +> +> +When guest writes 0xffffffff to the BAR, guest just want to get the size of +> +the +> +region but not really updating the address space. +> +So when guest writes 0xffffffff to BAR, we need avoid pci_update_mappings or +> +pci_bridge_update_mappings. +> +> +Signed-off-by: xuyandong <address@hidden> +> +--- +> +hw/pci/pci.c | 6 ++++-- +> +hw/pci/pci_bridge.c | 8 +++++--- +> +2 files changed, 9 insertions(+), 5 deletions(-) +> +> +diff --git a/hw/pci/pci.c b/hw/pci/pci.c index 56b13b3..ef368e1 100644 +> +--- a/hw/pci/pci.c +> ++++ b/hw/pci/pci.c +> +@@ -1361,6 +1361,7 @@ void pci_default_write_config(PCIDevice *d, uint32_t +> +addr, uint32_t val_in, int { +> +int i, was_irq_disabled = pci_irq_disabled(d); +> +uint32_t val = val_in; +> ++ uint64_t barmask = (1 << l*8) - 1; +> +> +for (i = 0; i < l; val >>= 8, ++i) { +> +uint8_t wmask = d->wmask[addr + i]; @@ -1369,9 +1370,10 @@ void +> +pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val_in, int +> +d->config[addr + i] = (d->config[addr + i] & ~wmask) | (val & wmask); +> +d->config[addr + i] &= ~(val & w1cmask); /* W1C: Write 1 to Clear */ +> +} +> +- if (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || +> ++ if ((val_in != barmask && +> ++ (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || +> +ranges_overlap(addr, l, PCI_ROM_ADDRESS, 4) || +> +- ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4) || +> ++ ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4))) || +> +range_covers_byte(addr, l, PCI_COMMAND)) +> +pci_update_mappings(d); +> +> +diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c index ee9dff2..f2bad79 +> +100644 +> +--- a/hw/pci/pci_bridge.c +> ++++ b/hw/pci/pci_bridge.c +> +@@ -253,17 +253,19 @@ void pci_bridge_write_config(PCIDevice *d, +> +PCIBridge *s = PCI_BRIDGE(d); +> +uint16_t oldctl = pci_get_word(d->config + PCI_BRIDGE_CONTROL); +> +uint16_t newctl; +> ++ uint64_t barmask = (1 << len * 8) - 1; +> +> +pci_default_write_config(d, address, val, len); +> +> +if (ranges_overlap(address, len, PCI_COMMAND, 2) || +> +> +- /* io base/limit */ +> +- ranges_overlap(address, len, PCI_IO_BASE, 2) || +> ++ (val != barmask && +> ++ /* io base/limit */ +> ++ (ranges_overlap(address, len, PCI_IO_BASE, 2) || +> +> +/* memory base/limit, prefetchable base/limit and +> +io base/limit upper 16 */ +> +- ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || +> ++ ranges_overlap(address, len, PCI_MEMORY_BASE, 20))) || +> +> +/* vga enable */ +> +ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { +> +-- +> +1.8.3.1 +> +> +Sorry, please ignore the patch above. + +Here is the patch I want to post: + +diff --git a/hw/pci/pci.c b/hw/pci/pci.c +index 56b13b3..38a300f 100644 +--- a/hw/pci/pci.c ++++ b/hw/pci/pci.c +@@ -1361,6 +1361,7 @@ void pci_default_write_config(PCIDevice *d, uint32_t +addr, uint32_t val_in, int + { + int i, was_irq_disabled = pci_irq_disabled(d); + uint32_t val = val_in; ++ uint64_t barmask = ((uint64_t)1 << l*8) - 1; + + for (i = 0; i < l; val >>= 8, ++i) { + uint8_t wmask = d->wmask[addr + i]; +@@ -1369,9 +1370,10 @@ void pci_default_write_config(PCIDevice *d, uint32_t +addr, uint32_t val_in, int + d->config[addr + i] = (d->config[addr + i] & ~wmask) | (val & wmask); + d->config[addr + i] &= ~(val & w1cmask); /* W1C: Write 1 to Clear */ + } +- if (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || ++ if ((val_in != barmask && ++ (ranges_overlap(addr, l, PCI_BASE_ADDRESS_0, 24) || + ranges_overlap(addr, l, PCI_ROM_ADDRESS, 4) || +- ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4) || ++ ranges_overlap(addr, l, PCI_ROM_ADDRESS1, 4))) || + range_covers_byte(addr, l, PCI_COMMAND)) + pci_update_mappings(d); + +diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c +index ee9dff2..b8f7d48 100644 +--- a/hw/pci/pci_bridge.c ++++ b/hw/pci/pci_bridge.c +@@ -253,20 +253,22 @@ void pci_bridge_write_config(PCIDevice *d, + PCIBridge *s = PCI_BRIDGE(d); + uint16_t oldctl = pci_get_word(d->config + PCI_BRIDGE_CONTROL); + uint16_t newctl; ++ uint64_t barmask = ((uint64_t)1 << len * 8) - 1; + + pci_default_write_config(d, address, val, len); + + if (ranges_overlap(address, len, PCI_COMMAND, 2) || + +- /* io base/limit */ +- ranges_overlap(address, len, PCI_IO_BASE, 2) || ++ /* vga enable */ ++ ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2) || + +- /* memory base/limit, prefetchable base/limit and +- io base/limit upper 16 */ +- ranges_overlap(address, len, PCI_MEMORY_BASE, 20) || ++ (val != barmask && ++ /* io base/limit */ ++ (ranges_overlap(address, len, PCI_IO_BASE, 2) || + +- /* vga enable */ +- ranges_overlap(address, len, PCI_BRIDGE_CONTROL, 2)) { ++ /* memory base/limit, prefetchable base/limit and ++ io base/limit upper 16 */ ++ ranges_overlap(address, len, PCI_MEMORY_BASE, 20)))) { + pci_bridge_update_mappings(s); + } + +-- +1.8.3.1 + |