From dee4dcba78baf712cab403d47d9db319ab7f95d6 Mon Sep 17 00:00:00 2001 From: Christian Krinitsin Date: Thu, 3 Jul 2025 19:39:53 +0200 Subject: restructure results --- results/classifier/007/006-007 | 46 - results/classifier/007/README.md | 9 - results/classifier/007/categories.csv | 11 - results/classifier/007/debug/36568044 | 4591 ------------ results/classifier/007/debug/53568181 | 88 - results/classifier/007/debug/64571620 | 795 --- results/classifier/007/debug/96782458 | 1009 --- results/classifier/007/device/42226390 | 197 - results/classifier/007/files/64322995 | 64 - results/classifier/007/graphic/30680944 | 605 -- results/classifier/007/graphic/46572227 | 416 -- results/classifier/007/network/05479587 | 93 - results/classifier/007/network/62179944 | 41 - results/classifier/007/other/02364653 | 373 - results/classifier/007/other/02572177 | 431 -- results/classifier/007/other/04472277 | 586 -- results/classifier/007/other/11933524 | 1135 --- results/classifier/007/other/12869209 | 98 - results/classifier/007/other/13442371 | 379 - results/classifier/007/other/16201167 | 110 - results/classifier/007/other/16228234 | 1854 ----- results/classifier/007/other/17743720 | 781 -- results/classifier/007/other/21247035 | 1331 ---- results/classifier/007/other/22219210 | 53 - results/classifier/007/other/23270873 | 702 -- results/classifier/007/other/23448582 | 275 - results/classifier/007/other/24190340 | 2066 ------ results/classifier/007/other/24930826 | 43 - results/classifier/007/other/25842545 | 212 - results/classifier/007/other/25892827 | 1087 --- results/classifier/007/other/28596630 | 123 - results/classifier/007/other/31349848 | 164 - results/classifier/007/other/32484936 | 233 - results/classifier/007/other/33802194 | 4949 ------------- results/classifier/007/other/35170175 | 531 -- results/classifier/007/other/42613410 | 159 - results/classifier/007/other/42974450 | 439 -- results/classifier/007/other/43643137 | 548 -- results/classifier/007/other/50773216 | 120 - results/classifier/007/other/55367348 | 542 -- results/classifier/007/other/55753058 | 303 - results/classifier/007/other/55961334 | 49 - results/classifier/007/other/56309929 | 190 - results/classifier/007/other/56937788 | 354 - results/classifier/007/other/57195159 | 325 - results/classifier/007/other/57231878 | 252 - results/classifier/007/other/57756589 | 1431 ---- results/classifier/007/other/59540920 | 386 - results/classifier/007/other/60339453 | 71 - results/classifier/007/other/63565653 | 59 - results/classifier/007/other/65781993 | 2803 -------- results/classifier/007/other/66743673 | 374 - results/classifier/007/other/68897003 | 726 -- results/classifier/007/other/70021271 | 7458 -------------------- results/classifier/007/other/70294255 | 1071 --- results/classifier/007/other/70416488 | 1189 ---- results/classifier/007/other/70868267 | 50 - results/classifier/007/other/71456293 | 1496 ---- results/classifier/007/other/73660729 | 41 - results/classifier/007/other/74466963 | 1888 ----- results/classifier/007/other/74545755 | 354 - results/classifier/007/other/80604314 | 1490 ---- results/classifier/007/other/80615920 | 358 - results/classifier/007/other/81775929 | 245 - results/classifier/007/other/92957605 | 428 -- results/classifier/007/other/99674399 | 158 - results/classifier/007/performance/79834768 | 419 -- results/classifier/007/permissions/12360755 | 306 - results/classifier/007/permissions/14488057 | 721 -- results/classifier/007/permissions/14887122 | 268 - results/classifier/007/permissions/16056596 | 108 - results/classifier/007/permissions/21221931 | 338 - results/classifier/007/permissions/23300761 | 323 - results/classifier/007/permissions/26095107 | 168 - results/classifier/007/permissions/26430026 | 175 - results/classifier/007/permissions/48245039 | 540 -- results/classifier/007/permissions/51610399 | 318 - results/classifier/007/permissions/55247116 | 1320 ---- results/classifier/007/permissions/67821138 | 209 - results/classifier/007/permissions/74715356 | 136 - results/classifier/007/permissions/85542195 | 130 - results/classifier/007/permissions/88225572 | 2910 -------- results/classifier/007/permissions/88281850 | 291 - results/classifier/007/permissions/95154278 | 165 - .../007/semantic/gitlab_semantic_addsubps | 38 - .../classifier/007/semantic/gitlab_semantic_adox | 51 - .../classifier/007/semantic/gitlab_semantic_bextr | 40 - .../classifier/007/semantic/gitlab_semantic_blsi | 35 - .../classifier/007/semantic/gitlab_semantic_blsmsk | 42 - .../classifier/007/semantic/gitlab_semantic_bzhi | 53 - results/classifier/007/vnc/11357571 | 57 - results/classifier/007/vnc/80570214 | 410 -- 92 files changed, 60409 deletions(-) delete mode 100644 results/classifier/007/006-007 delete mode 100644 results/classifier/007/README.md delete mode 100644 results/classifier/007/categories.csv delete mode 100644 results/classifier/007/debug/36568044 delete mode 100644 results/classifier/007/debug/53568181 delete mode 100644 results/classifier/007/debug/64571620 delete mode 100644 results/classifier/007/debug/96782458 delete mode 100644 results/classifier/007/device/42226390 delete mode 100644 results/classifier/007/files/64322995 delete mode 100644 results/classifier/007/graphic/30680944 delete mode 100644 results/classifier/007/graphic/46572227 delete mode 100644 results/classifier/007/network/05479587 delete mode 100644 results/classifier/007/network/62179944 delete mode 100644 results/classifier/007/other/02364653 delete mode 100644 results/classifier/007/other/02572177 delete mode 100644 results/classifier/007/other/04472277 delete mode 100644 results/classifier/007/other/11933524 delete mode 100644 results/classifier/007/other/12869209 delete mode 100644 results/classifier/007/other/13442371 delete mode 100644 results/classifier/007/other/16201167 delete mode 100644 results/classifier/007/other/16228234 delete mode 100644 results/classifier/007/other/17743720 delete mode 100644 results/classifier/007/other/21247035 delete mode 100644 results/classifier/007/other/22219210 delete mode 100644 results/classifier/007/other/23270873 delete mode 100644 results/classifier/007/other/23448582 delete mode 100644 results/classifier/007/other/24190340 delete mode 100644 results/classifier/007/other/24930826 delete mode 100644 results/classifier/007/other/25842545 delete mode 100644 results/classifier/007/other/25892827 delete mode 100644 results/classifier/007/other/28596630 delete mode 100644 results/classifier/007/other/31349848 delete mode 100644 results/classifier/007/other/32484936 delete mode 100644 results/classifier/007/other/33802194 delete mode 100644 results/classifier/007/other/35170175 delete mode 100644 results/classifier/007/other/42613410 delete mode 100644 results/classifier/007/other/42974450 delete mode 100644 results/classifier/007/other/43643137 delete mode 100644 results/classifier/007/other/50773216 delete mode 100644 results/classifier/007/other/55367348 delete mode 100644 results/classifier/007/other/55753058 delete mode 100644 results/classifier/007/other/55961334 delete mode 100644 results/classifier/007/other/56309929 delete mode 100644 results/classifier/007/other/56937788 delete mode 100644 results/classifier/007/other/57195159 delete mode 100644 results/classifier/007/other/57231878 delete mode 100644 results/classifier/007/other/57756589 delete mode 100644 results/classifier/007/other/59540920 delete mode 100644 results/classifier/007/other/60339453 delete mode 100644 results/classifier/007/other/63565653 delete mode 100644 results/classifier/007/other/65781993 delete mode 100644 results/classifier/007/other/66743673 delete mode 100644 results/classifier/007/other/68897003 delete mode 100644 results/classifier/007/other/70021271 delete mode 100644 results/classifier/007/other/70294255 delete mode 100644 results/classifier/007/other/70416488 delete mode 100644 results/classifier/007/other/70868267 delete mode 100644 results/classifier/007/other/71456293 delete mode 100644 results/classifier/007/other/73660729 delete mode 100644 results/classifier/007/other/74466963 delete mode 100644 results/classifier/007/other/74545755 delete mode 100644 results/classifier/007/other/80604314 delete mode 100644 results/classifier/007/other/80615920 delete mode 100644 results/classifier/007/other/81775929 delete mode 100644 results/classifier/007/other/92957605 delete mode 100644 results/classifier/007/other/99674399 delete mode 100644 results/classifier/007/performance/79834768 delete mode 100644 results/classifier/007/permissions/12360755 delete mode 100644 results/classifier/007/permissions/14488057 delete mode 100644 results/classifier/007/permissions/14887122 delete mode 100644 results/classifier/007/permissions/16056596 delete mode 100644 results/classifier/007/permissions/21221931 delete mode 100644 results/classifier/007/permissions/23300761 delete mode 100644 results/classifier/007/permissions/26095107 delete mode 100644 results/classifier/007/permissions/26430026 delete mode 100644 results/classifier/007/permissions/48245039 delete mode 100644 results/classifier/007/permissions/51610399 delete mode 100644 results/classifier/007/permissions/55247116 delete mode 100644 results/classifier/007/permissions/67821138 delete mode 100644 results/classifier/007/permissions/74715356 delete mode 100644 results/classifier/007/permissions/85542195 delete mode 100644 results/classifier/007/permissions/88225572 delete mode 100644 results/classifier/007/permissions/88281850 delete mode 100644 results/classifier/007/permissions/95154278 delete mode 100644 results/classifier/007/semantic/gitlab_semantic_addsubps delete mode 100644 results/classifier/007/semantic/gitlab_semantic_adox delete mode 100644 results/classifier/007/semantic/gitlab_semantic_bextr delete mode 100644 results/classifier/007/semantic/gitlab_semantic_blsi delete mode 100644 results/classifier/007/semantic/gitlab_semantic_blsmsk delete mode 100644 results/classifier/007/semantic/gitlab_semantic_bzhi delete mode 100644 results/classifier/007/vnc/11357571 delete mode 100644 results/classifier/007/vnc/80570214 (limited to 'results/classifier/007') diff --git a/results/classifier/007/006-007 b/results/classifier/007/006-007 deleted file mode 100644 index a16f8c45..00000000 --- a/results/classifier/007/006-007 +++ /dev/null @@ -1,46 +0,0 @@ -45 changes: -33802194: vnc -> other -42613410: vnc -> other -51610399: boot -> permissions -60339453: boot -> other -53568181: graphic -> debug -55961334: graphic -> other -73660729: graphic -> other -70868267: graphic -> other -22219210: graphic -> other -67821138: device -> permissions -48245039: device -> permissions -74545755: device -> other -64322995: device -> files -99674399: device -> other -14488057: device -> permissions -80604314: device -> other -24190340: device -> other -74466963: device -> other -24930826: device -> other -36568044: device -> debug -50773216: device -> other -57195159: device -> other -26095107: device -> permissions -57231878: device -> other -28596630: device -> other -12360755: semantic -> permissions -70294255: semantic -> other -14887122: semantic -> permissions -80615920: KVM -> other -04472277: KVM -> other -43643137: KVM -> other -71456293: KVM -> other -26430026: KVM -> permissions -96782458: other -> debug -74715356: other -> permissions -16056596: other -> permissions -95154278: other -> permissions -23300761: other -> permissions -79834768: other -> performance -64571620: other -> debug -55247116: other -> permissions -88281850: other -> permissions -21221931: other -> permissions -88225572: other -> permissions -85542195: other -> permissions diff --git a/results/classifier/007/README.md b/results/classifier/007/README.md deleted file mode 100644 index ea1ac390..00000000 --- a/results/classifier/007/README.md +++ /dev/null @@ -1,9 +0,0 @@ -## 007 - -```python -multi_label = True -model = "facebook/bart-large-mnli" -number_bugs = 89 -``` - -Adds the categories *permissions*, *files*, *PID*, *performance* and *debug*. It also applies the category *other* if *semantic* has a score < 0.91 (This was an error, fixed in 009). diff --git a/results/classifier/007/categories.csv b/results/classifier/007/categories.csv deleted file mode 100644 index e7efa3ab..00000000 --- a/results/classifier/007/categories.csv +++ /dev/null @@ -1,11 +0,0 @@ -category, count -vnc, 2 -performance, 1 -other, 53 -graphic, 2 -device, 1 -debug, 4 -files, 1 -network, 2 -permissions, 17 -semantic, 6 diff --git a/results/classifier/007/debug/36568044 b/results/classifier/007/debug/36568044 deleted file mode 100644 index cdc1d631..00000000 --- a/results/classifier/007/debug/36568044 +++ /dev/null @@ -1,4591 +0,0 @@ -debug: 0.939 -device: 0.931 -graphic: 0.931 -other: 0.930 -permissions: 0.927 -PID: 0.926 -semantic: 0.923 -performance: 0.920 -KVM: 0.914 -socket: 0.907 -vnc: 0.905 -network: 0.904 -boot: 0.895 -files: 0.884 - -[BUG, RFC] cpr-transfer: qxl guest driver crashes after migration - -Hi all, - -We've been experimenting with cpr-transfer migration mode recently and -have discovered the following issue with the guest QXL driver: - -Run migration source: -> -EMULATOR=/path/to/emulator -> -ROOTFS=/path/to/image -> -QMPSOCK=/var/run/alma8qmp-src.sock -> -> -$EMULATOR -enable-kvm \ -> --machine q35 \ -> --cpu host -smp 2 -m 2G \ -> --object -> -memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ram0,share=on\ -> --machine memory-backend=ram0 \ -> --machine aux-ram-share=on \ -> --drive file=$ROOTFS,media=disk,if=virtio \ -> --qmp unix:$QMPSOCK,server=on,wait=off \ -> --nographic \ -> --device qxl-vga -Run migration target: -> -EMULATOR=/path/to/emulator -> -ROOTFS=/path/to/image -> -QMPSOCK=/var/run/alma8qmp-dst.sock -> -> -> -> -$EMULATOR -enable-kvm \ -> --machine q35 \ -> --cpu host -smp 2 -m 2G \ -> --object -> -memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ram0,share=on\ -> --machine memory-backend=ram0 \ -> --machine aux-ram-share=on \ -> --drive file=$ROOTFS,media=disk,if=virtio \ -> --qmp unix:$QMPSOCK,server=on,wait=off \ -> --nographic \ -> --device qxl-vga \ -> --incoming tcp:0:44444 \ -> --incoming '{"channel-type": "cpr", "addr": { "transport": "socket", -> -"type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -> -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -> -QMPSOCK=/var/run/alma8qmp-src.sock -> -> -$QMPSHELL -p $QMPSOCK < -migrate-set-parameters mode=cpr-transfer -> -migrate -> -channels=[{"channel-type":"main","addr":{"transport":"socket","type":"inet","host":"0","port":"44444"}},{"channel-type":"cpr","addr":{"transport":"socket","type":"unix","path":"/var/run/alma8cpr-dst.sock"}}] -> -EOF -Then, after a while, QXL guest driver on target crashes spewing the -following messages: -> -[ 73.962002] [TTM] Buffer eviction failed -> -[ 73.962072] qxl 0000:00:02.0: object_init failed for (3149824, 0x00000001) -> -[ 73.962081] [drm:qxl_alloc_bo_reserved [qxl]] *ERROR* failed to allocate -> -VRAM BO -That seems to be a known kernel QXL driver bug: -https://lore.kernel.org/all/20220907094423.93581-1-min_halo@163.com/T/ -https://lore.kernel.org/lkml/ZTgydqRlK6WX_b29@eldamar.lan/ -(the latter discussion contains that reproduce script which speeds up -the crash in the guest): -> -#!/bin/bash -> -> -chvt 3 -> -> -for j in $(seq 80); do -> -echo "$(date) starting round $j" -> -if [ "$(journalctl --boot | grep "failed to allocate VRAM BO")" != "" -> -]; then -> -echo "bug was reproduced after $j tries" -> -exit 1 -> -fi -> -for i in $(seq 100); do -> -dmesg > /dev/tty3 -> -done -> -done -> -> -echo "bug could not be reproduced" -> -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the crash - -without the cpr-transfer migration the above reproduce doesn't lead to -crash on the source VM. - -I suspect that, as cpr-transfer doesn't migrate the guest memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM. However, I wasn't able to trace the -corruption so far. - -Could somebody help the investigation and take a look into this? Any -suggestions would be appreciated. Thanks! - -Andrey - -On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -Hi all, - -We've been experimenting with cpr-transfer migration mode recently and -have discovered the following issue with the guest QXL driver: - -Run migration source: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-src.sock - -$EMULATOR -enable-kvm \ - -machine q35 \ - -cpu host -smp 2 -m 2G \ - -object -memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ram0,share=on\ - -machine memory-backend=ram0 \ - -machine aux-ram-share=on \ - -drive file=$ROOTFS,media=disk,if=virtio \ - -qmp unix:$QMPSOCK,server=on,wait=off \ - -nographic \ - -device qxl-vga -Run migration target: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-dst.sock -$EMULATOR -enable-kvm \ --machine q35 \ - -cpu host -smp 2 -m 2G \ - -object -memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ram0,share=on\ - -machine memory-backend=ram0 \ - -machine aux-ram-share=on \ - -drive file=$ROOTFS,media=disk,if=virtio \ - -qmp unix:$QMPSOCK,server=on,wait=off \ - -nographic \ - -device qxl-vga \ - -incoming tcp:0:44444 \ - -incoming '{"channel-type": "cpr", "addr": { "transport": "socket", "type": "unix", -"path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -QMPSOCK=/var/run/alma8qmp-src.sock - -$QMPSHELL -p $QMPSOCK < /dev/tty3 - done -done - -echo "bug could not be reproduced" -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the crash - -without the cpr-transfer migration the above reproduce doesn't lead to -crash on the source VM. - -I suspect that, as cpr-transfer doesn't migrate the guest memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM. However, I wasn't able to trace the -corruption so far. - -Could somebody help the investigation and take a look into this? Any -suggestions would be appreciated. Thanks! -Possibly some memory region created by qxl is not being preserved. -Try adding these traces to see what is preserved: - --trace enable='*cpr*' --trace enable='*ram_alloc*' - -- Steve - -On 2/28/2025 1:13 PM, Steven Sistare wrote: -On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -Hi all, - -We've been experimenting with cpr-transfer migration mode recently and -have discovered the following issue with the guest QXL driver: - -Run migration source: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-src.sock - -$EMULATOR -enable-kvm \ -     -machine q35 \ -     -cpu host -smp 2 -m 2G \ -     -object -memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ram0,share=on\ -     -machine memory-backend=ram0 \ -     -machine aux-ram-share=on \ -     -drive file=$ROOTFS,media=disk,if=virtio \ -     -qmp unix:$QMPSOCK,server=on,wait=off \ -     -nographic \ -     -device qxl-vga -Run migration target: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-dst.sock -$EMULATOR -enable-kvm \ -     -machine q35 \ -     -cpu host -smp 2 -m 2G \ -     -object -memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ram0,share=on\ -     -machine memory-backend=ram0 \ -     -machine aux-ram-share=on \ -     -drive file=$ROOTFS,media=disk,if=virtio \ -     -qmp unix:$QMPSOCK,server=on,wait=off \ -     -nographic \ -     -device qxl-vga \ -     -incoming tcp:0:44444 \ -     -incoming '{"channel-type": "cpr", "addr": { "transport": "socket", "type": "unix", -"path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -QMPSOCK=/var/run/alma8qmp-src.sock - -$QMPSHELL -p $QMPSOCK < /dev/tty3 -         done -done - -echo "bug could not be reproduced" -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the crash - -without the cpr-transfer migration the above reproduce doesn't lead to -crash on the source VM. - -I suspect that, as cpr-transfer doesn't migrate the guest memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM.  However, I wasn't able to trace the -corruption so far. - -Could somebody help the investigation and take a look into this?  Any -suggestions would be appreciated.  Thanks! -Possibly some memory region created by qxl is not being preserved. -Try adding these traces to see what is preserved: - --trace enable='*cpr*' --trace enable='*ram_alloc*' -Also try adding this patch to see if it flags any ram blocks as not -compatible with cpr. A message is printed at migration start time. -1740667681-257312-1-git-send-email-steven.sistare@oracle.com -/">https://lore.kernel.org/qemu-devel/ -1740667681-257312-1-git-send-email-steven.sistare@oracle.com -/ -- Steve - -On 2/28/25 8:20 PM, Steven Sistare wrote: -> -On 2/28/2025 1:13 PM, Steven Sistare wrote: -> -> On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -> ->> Hi all, -> ->> -> ->> We've been experimenting with cpr-transfer migration mode recently and -> ->> have discovered the following issue with the guest QXL driver: -> ->> -> ->> Run migration source: -> ->>> EMULATOR=/path/to/emulator -> ->>> ROOTFS=/path/to/image -> ->>> QMPSOCK=/var/run/alma8qmp-src.sock -> ->>> -> ->>> $EMULATOR -enable-kvm \ -> ->>>      -machine q35 \ -> ->>>      -cpu host -smp 2 -m 2G \ -> ->>>      -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -> ->>> ram0,share=on\ -> ->>>      -machine memory-backend=ram0 \ -> ->>>      -machine aux-ram-share=on \ -> ->>>      -drive file=$ROOTFS,media=disk,if=virtio \ -> ->>>      -qmp unix:$QMPSOCK,server=on,wait=off \ -> ->>>      -nographic \ -> ->>>      -device qxl-vga -> ->> -> ->> Run migration target: -> ->>> EMULATOR=/path/to/emulator -> ->>> ROOTFS=/path/to/image -> ->>> QMPSOCK=/var/run/alma8qmp-dst.sock -> ->>> $EMULATOR -enable-kvm \ -> ->>>      -machine q35 \ -> ->>>      -cpu host -smp 2 -m 2G \ -> ->>>      -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -> ->>> ram0,share=on\ -> ->>>      -machine memory-backend=ram0 \ -> ->>>      -machine aux-ram-share=on \ -> ->>>      -drive file=$ROOTFS,media=disk,if=virtio \ -> ->>>      -qmp unix:$QMPSOCK,server=on,wait=off \ -> ->>>      -nographic \ -> ->>>      -device qxl-vga \ -> ->>>      -incoming tcp:0:44444 \ -> ->>>      -incoming '{"channel-type": "cpr", "addr": { "transport": -> ->>> "socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -> ->> -> ->> -> ->> Launch the migration: -> ->>> QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -> ->>> QMPSOCK=/var/run/alma8qmp-src.sock -> ->>> -> ->>> $QMPSHELL -p $QMPSOCK < ->>>      migrate-set-parameters mode=cpr-transfer -> ->>>      migrate channels=[{"channel-type":"main","addr": -> ->>> {"transport":"socket","type":"inet","host":"0","port":"44444"}}, -> ->>> {"channel-type":"cpr","addr": -> ->>> {"transport":"socket","type":"unix","path":"/var/run/alma8cpr- -> ->>> dst.sock"}}] -> ->>> EOF -> ->> -> ->> Then, after a while, QXL guest driver on target crashes spewing the -> ->> following messages: -> ->>> [   73.962002] [TTM] Buffer eviction failed -> ->>> [   73.962072] qxl 0000:00:02.0: object_init failed for (3149824, -> ->>> 0x00000001) -> ->>> [   73.962081] [drm:qxl_alloc_bo_reserved [qxl]] *ERROR* failed to -> ->>> allocate VRAM BO -> ->> -> ->> That seems to be a known kernel QXL driver bug: -> ->> -> ->> -https://lore.kernel.org/all/20220907094423.93581-1-min_halo@163.com/T/ -> ->> -https://lore.kernel.org/lkml/ZTgydqRlK6WX_b29@eldamar.lan/ -> ->> -> ->> (the latter discussion contains that reproduce script which speeds up -> ->> the crash in the guest): -> ->>> #!/bin/bash -> ->>> -> ->>> chvt 3 -> ->>> -> ->>> for j in $(seq 80); do -> ->>>          echo "$(date) starting round $j" -> ->>>          if [ "$(journalctl --boot | grep "failed to allocate VRAM -> ->>> BO")" != "" ]; then -> ->>>                  echo "bug was reproduced after $j tries" -> ->>>                  exit 1 -> ->>>          fi -> ->>>          for i in $(seq 100); do -> ->>>                  dmesg > /dev/tty3 -> ->>>          done -> ->>> done -> ->>> -> ->>> echo "bug could not be reproduced" -> ->>> exit 0 -> ->> -> ->> The bug itself seems to remain unfixed, as I was able to reproduce that -> ->> with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -> ->> cpr-transfer code also seems to be buggy as it triggers the crash - -> ->> without the cpr-transfer migration the above reproduce doesn't lead to -> ->> crash on the source VM. -> ->> -> ->> I suspect that, as cpr-transfer doesn't migrate the guest memory, but -> ->> rather passes it through the memory backend object, our code might -> ->> somehow corrupt the VRAM.  However, I wasn't able to trace the -> ->> corruption so far. -> ->> -> ->> Could somebody help the investigation and take a look into this?  Any -> ->> suggestions would be appreciated.  Thanks! -> -> -> -> Possibly some memory region created by qxl is not being preserved. -> -> Try adding these traces to see what is preserved: -> -> -> -> -trace enable='*cpr*' -> -> -trace enable='*ram_alloc*' -> -> -Also try adding this patch to see if it flags any ram blocks as not -> -compatible with cpr.  A message is printed at migration start time. -> - -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send-email- -> -steven.sistare@oracle.com/ -> -> -- Steve -> -With the traces enabled + the "migration: ram block cpr blockers" patch -applied: - -Source: -> -cpr_find_fd pc.bios, id 0 returns -1 -> -cpr_save_fd pc.bios, id 0, fd 22 -> -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -> -0x7fec18e00000 -> -cpr_find_fd pc.rom, id 0 returns -1 -> -cpr_save_fd pc.rom, id 0, fd 23 -> -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -> -0x7fec18c00000 -> -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -> -cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -> -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size 262144 fd -> -24 host 0x7fec18a00000 -> -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -> -cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -> -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size 67108864 -> -fd 25 host 0x7feb77e00000 -> -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -> -cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -> -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 fd 27 -> -host 0x7fec18800000 -> -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -> -cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -> -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size 67108864 -> -fd 28 host 0x7feb73c00000 -> -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -> -cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -> -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 fd 34 -> -host 0x7fec18600000 -> -cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -> -cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -> -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size 2097152 fd 35 -> -host 0x7fec18200000 -> -cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -> -cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -> -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 fd 36 -> -host 0x7feb8b600000 -> -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -> -cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -> -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd 37 host -> -0x7feb8b400000 -> -> -cpr_state_save cpr-transfer mode -> -cpr_transfer_output /var/run/alma8cpr-dst.sock -Target: -> -cpr_transfer_input /var/run/alma8cpr-dst.sock -> -cpr_state_load cpr-transfer mode -> -cpr_find_fd pc.bios, id 0 returns 20 -> -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -> -0x7fcdc9800000 -> -cpr_find_fd pc.rom, id 0 returns 19 -> -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -> -0x7fcdc9600000 -> -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -> -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size 262144 fd -> -18 host 0x7fcdc9400000 -> -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -> -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size 67108864 -> -fd 17 host 0x7fcd27e00000 -> -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -> -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 fd 16 -> -host 0x7fcdc9200000 -> -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -> -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size 67108864 -> -fd 15 host 0x7fcd23c00000 -> -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -> -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 fd 14 -> -host 0x7fcdc8800000 -> -cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -> -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size 2097152 fd 13 -> -host 0x7fcdc8400000 -> -cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -> -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 fd 11 -> -host 0x7fcdc8200000 -> -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -> -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd 10 host -> -0x7fcd3be00000 -Looks like both vga.vram and qxl.vram are being preserved (with the same -addresses), and no incompatible ram blocks are found during migration. - -Andrey - -On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -> -On 2/28/25 8:20 PM, Steven Sistare wrote: -> -> On 2/28/2025 1:13 PM, Steven Sistare wrote: -> ->> On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -> ->>> Hi all, -> ->>> -> ->>> We've been experimenting with cpr-transfer migration mode recently and -> ->>> have discovered the following issue with the guest QXL driver: -> ->>> -> ->>> Run migration source: -> ->>>> EMULATOR=/path/to/emulator -> ->>>> ROOTFS=/path/to/image -> ->>>> QMPSOCK=/var/run/alma8qmp-src.sock -> ->>>> -> ->>>> $EMULATOR -enable-kvm \ -> ->>>>      -machine q35 \ -> ->>>>      -cpu host -smp 2 -m 2G \ -> ->>>>      -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -> ->>>> ram0,share=on\ -> ->>>>      -machine memory-backend=ram0 \ -> ->>>>      -machine aux-ram-share=on \ -> ->>>>      -drive file=$ROOTFS,media=disk,if=virtio \ -> ->>>>      -qmp unix:$QMPSOCK,server=on,wait=off \ -> ->>>>      -nographic \ -> ->>>>      -device qxl-vga -> ->>> -> ->>> Run migration target: -> ->>>> EMULATOR=/path/to/emulator -> ->>>> ROOTFS=/path/to/image -> ->>>> QMPSOCK=/var/run/alma8qmp-dst.sock -> ->>>> $EMULATOR -enable-kvm \ -> ->>>>      -machine q35 \ -> ->>>>      -cpu host -smp 2 -m 2G \ -> ->>>>      -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -> ->>>> ram0,share=on\ -> ->>>>      -machine memory-backend=ram0 \ -> ->>>>      -machine aux-ram-share=on \ -> ->>>>      -drive file=$ROOTFS,media=disk,if=virtio \ -> ->>>>      -qmp unix:$QMPSOCK,server=on,wait=off \ -> ->>>>      -nographic \ -> ->>>>      -device qxl-vga \ -> ->>>>      -incoming tcp:0:44444 \ -> ->>>>      -incoming '{"channel-type": "cpr", "addr": { "transport": -> ->>>> "socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -> ->>> -> ->>> -> ->>> Launch the migration: -> ->>>> QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -> ->>>> QMPSOCK=/var/run/alma8qmp-src.sock -> ->>>> -> ->>>> $QMPSHELL -p $QMPSOCK < ->>>>      migrate-set-parameters mode=cpr-transfer -> ->>>>      migrate channels=[{"channel-type":"main","addr": -> ->>>> {"transport":"socket","type":"inet","host":"0","port":"44444"}}, -> ->>>> {"channel-type":"cpr","addr": -> ->>>> {"transport":"socket","type":"unix","path":"/var/run/alma8cpr- -> ->>>> dst.sock"}}] -> ->>>> EOF -> ->>> -> ->>> Then, after a while, QXL guest driver on target crashes spewing the -> ->>> following messages: -> ->>>> [   73.962002] [TTM] Buffer eviction failed -> ->>>> [   73.962072] qxl 0000:00:02.0: object_init failed for (3149824, -> ->>>> 0x00000001) -> ->>>> [   73.962081] [drm:qxl_alloc_bo_reserved [qxl]] *ERROR* failed to -> ->>>> allocate VRAM BO -> ->>> -> ->>> That seems to be a known kernel QXL driver bug: -> ->>> -> ->>> -https://lore.kernel.org/all/20220907094423.93581-1-min_halo@163.com/T/ -> ->>> -https://lore.kernel.org/lkml/ZTgydqRlK6WX_b29@eldamar.lan/ -> ->>> -> ->>> (the latter discussion contains that reproduce script which speeds up -> ->>> the crash in the guest): -> ->>>> #!/bin/bash -> ->>>> -> ->>>> chvt 3 -> ->>>> -> ->>>> for j in $(seq 80); do -> ->>>>          echo "$(date) starting round $j" -> ->>>>          if [ "$(journalctl --boot | grep "failed to allocate VRAM -> ->>>> BO")" != "" ]; then -> ->>>>                  echo "bug was reproduced after $j tries" -> ->>>>                  exit 1 -> ->>>>          fi -> ->>>>          for i in $(seq 100); do -> ->>>>                  dmesg > /dev/tty3 -> ->>>>          done -> ->>>> done -> ->>>> -> ->>>> echo "bug could not be reproduced" -> ->>>> exit 0 -> ->>> -> ->>> The bug itself seems to remain unfixed, as I was able to reproduce that -> ->>> with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -> ->>> cpr-transfer code also seems to be buggy as it triggers the crash - -> ->>> without the cpr-transfer migration the above reproduce doesn't lead to -> ->>> crash on the source VM. -> ->>> -> ->>> I suspect that, as cpr-transfer doesn't migrate the guest memory, but -> ->>> rather passes it through the memory backend object, our code might -> ->>> somehow corrupt the VRAM.  However, I wasn't able to trace the -> ->>> corruption so far. -> ->>> -> ->>> Could somebody help the investigation and take a look into this?  Any -> ->>> suggestions would be appreciated.  Thanks! -> ->> -> ->> Possibly some memory region created by qxl is not being preserved. -> ->> Try adding these traces to see what is preserved: -> ->> -> ->> -trace enable='*cpr*' -> ->> -trace enable='*ram_alloc*' -> -> -> -> Also try adding this patch to see if it flags any ram blocks as not -> -> compatible with cpr.  A message is printed at migration start time. -> ->  -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send-email- -> -> steven.sistare@oracle.com/ -> -> -> -> - Steve -> -> -> -> -With the traces enabled + the "migration: ram block cpr blockers" patch -> -applied: -> -> -Source: -> -> cpr_find_fd pc.bios, id 0 returns -1 -> -> cpr_save_fd pc.bios, id 0, fd 22 -> -> qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -> -> 0x7fec18e00000 -> -> cpr_find_fd pc.rom, id 0 returns -1 -> -> cpr_save_fd pc.rom, id 0, fd 23 -> -> qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -> -> 0x7fec18c00000 -> -> cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -> -> cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -> -> qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size 262144 fd -> -> 24 host 0x7fec18a00000 -> -> cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -> -> cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -> -> qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size 67108864 -> -> fd 25 host 0x7feb77e00000 -> -> cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -> -> cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -> -> qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 fd 27 -> -> host 0x7fec18800000 -> -> cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -> -> cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -> -> qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size 67108864 -> -> fd 28 host 0x7feb73c00000 -> -> cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -> -> cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -> -> qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 fd 34 -> -> host 0x7fec18600000 -> -> cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -> -> cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -> -> qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size 2097152 fd -> -> 35 host 0x7fec18200000 -> -> cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -> -> cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -> -> qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 fd 36 -> -> host 0x7feb8b600000 -> -> cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -> -> cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -> -> qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd 37 host -> -> 0x7feb8b400000 -> -> -> -> cpr_state_save cpr-transfer mode -> -> cpr_transfer_output /var/run/alma8cpr-dst.sock -> -> -Target: -> -> cpr_transfer_input /var/run/alma8cpr-dst.sock -> -> cpr_state_load cpr-transfer mode -> -> cpr_find_fd pc.bios, id 0 returns 20 -> -> qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -> -> 0x7fcdc9800000 -> -> cpr_find_fd pc.rom, id 0 returns 19 -> -> qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -> -> 0x7fcdc9600000 -> -> cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -> -> qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size 262144 fd -> -> 18 host 0x7fcdc9400000 -> -> cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -> -> qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size 67108864 -> -> fd 17 host 0x7fcd27e00000 -> -> cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -> -> qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 fd 16 -> -> host 0x7fcdc9200000 -> -> cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -> -> qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size 67108864 -> -> fd 15 host 0x7fcd23c00000 -> -> cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -> -> qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 fd 14 -> -> host 0x7fcdc8800000 -> -> cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -> -> qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size 2097152 fd -> -> 13 host 0x7fcdc8400000 -> -> cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -> -> qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 fd 11 -> -> host 0x7fcdc8200000 -> -> cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -> -> qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd 10 host -> -> 0x7fcd3be00000 -> -> -Looks like both vga.vram and qxl.vram are being preserved (with the same -> -addresses), and no incompatible ram blocks are found during migration. -> -Sorry, addressed are not the same, of course. However corresponding ram -blocks do seem to be preserved and initialized. - -On 2/28/2025 1:37 PM, Andrey Drobyshev wrote: -On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -On 2/28/25 8:20 PM, Steven Sistare wrote: -On 2/28/2025 1:13 PM, Steven Sistare wrote: -On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -Hi all, - -We've been experimenting with cpr-transfer migration mode recently and -have discovered the following issue with the guest QXL driver: - -Run migration source: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-src.sock - -$EMULATOR -enable-kvm \ -      -machine q35 \ -      -cpu host -smp 2 -m 2G \ -      -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -ram0,share=on\ -      -machine memory-backend=ram0 \ -      -machine aux-ram-share=on \ -      -drive file=$ROOTFS,media=disk,if=virtio \ -      -qmp unix:$QMPSOCK,server=on,wait=off \ -      -nographic \ -      -device qxl-vga -Run migration target: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-dst.sock -$EMULATOR -enable-kvm \ -      -machine q35 \ -      -cpu host -smp 2 -m 2G \ -      -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -ram0,share=on\ -      -machine memory-backend=ram0 \ -      -machine aux-ram-share=on \ -      -drive file=$ROOTFS,media=disk,if=virtio \ -      -qmp unix:$QMPSOCK,server=on,wait=off \ -      -nographic \ -      -device qxl-vga \ -      -incoming tcp:0:44444 \ -      -incoming '{"channel-type": "cpr", "addr": { "transport": -"socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -QMPSOCK=/var/run/alma8qmp-src.sock - -$QMPSHELL -p $QMPSOCK < /dev/tty3 -          done -done - -echo "bug could not be reproduced" -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the crash - -without the cpr-transfer migration the above reproduce doesn't lead to -crash on the source VM. - -I suspect that, as cpr-transfer doesn't migrate the guest memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM.  However, I wasn't able to trace the -corruption so far. - -Could somebody help the investigation and take a look into this?  Any -suggestions would be appreciated.  Thanks! -Possibly some memory region created by qxl is not being preserved. -Try adding these traces to see what is preserved: - --trace enable='*cpr*' --trace enable='*ram_alloc*' -Also try adding this patch to see if it flags any ram blocks as not -compatible with cpr.  A message is printed at migration start time. -  -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send-email- -steven.sistare@oracle.com/ - -- Steve -With the traces enabled + the "migration: ram block cpr blockers" patch -applied: - -Source: -cpr_find_fd pc.bios, id 0 returns -1 -cpr_save_fd pc.bios, id 0, fd 22 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -0x7fec18e00000 -cpr_find_fd pc.rom, id 0 returns -1 -cpr_save_fd pc.rom, id 0, fd 23 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -0x7fec18c00000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size 262144 fd 24 -host 0x7fec18a00000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size 67108864 fd -25 host 0x7feb77e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 fd 27 host -0x7fec18800000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size 67108864 fd -28 host 0x7feb73c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 fd 34 host -0x7fec18600000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size 2097152 fd 35 -host 0x7fec18200000 -cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 fd 36 host -0x7feb8b600000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd 37 host -0x7feb8b400000 - -cpr_state_save cpr-transfer mode -cpr_transfer_output /var/run/alma8cpr-dst.sock -Target: -cpr_transfer_input /var/run/alma8cpr-dst.sock -cpr_state_load cpr-transfer mode -cpr_find_fd pc.bios, id 0 returns 20 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -0x7fcdc9800000 -cpr_find_fd pc.rom, id 0 returns 19 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -0x7fcdc9600000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size 262144 fd 18 -host 0x7fcdc9400000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size 67108864 fd -17 host 0x7fcd27e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 fd 16 host -0x7fcdc9200000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size 67108864 fd -15 host 0x7fcd23c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 fd 14 host -0x7fcdc8800000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size 2097152 fd 13 -host 0x7fcdc8400000 -cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 fd 11 host -0x7fcdc8200000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd 10 host -0x7fcd3be00000 -Looks like both vga.vram and qxl.vram are being preserved (with the same -addresses), and no incompatible ram blocks are found during migration. -Sorry, addressed are not the same, of course. However corresponding ram -blocks do seem to be preserved and initialized. -So far, I have not reproduced the guest driver failure. - -However, I have isolated places where new QEMU improperly writes to -the qxl memory regions prior to starting the guest, by mmap'ing them -readonly after cpr: - - qemu_ram_alloc_internal() - if (reused && (strstr(name, "qxl") || strstr("name", "vga"))) - ram_flags |= RAM_READONLY; - new_block = qemu_ram_alloc_from_fd(...) - -I have attached a draft fix; try it and let me know. -My console window looks fine before and after cpr, using --vnc $hostip:0 -vga qxl - -- Steve -0001-hw-qxl-cpr-support-preliminary.patch -Description: -Text document - -On 3/4/25 9:05 PM, Steven Sistare wrote: -> -On 2/28/2025 1:37 PM, Andrey Drobyshev wrote: -> -> On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -> ->> On 2/28/25 8:20 PM, Steven Sistare wrote: -> ->>> On 2/28/2025 1:13 PM, Steven Sistare wrote: -> ->>>> On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -> ->>>>> Hi all, -> ->>>>> -> ->>>>> We've been experimenting with cpr-transfer migration mode recently -> ->>>>> and -> ->>>>> have discovered the following issue with the guest QXL driver: -> ->>>>> -> ->>>>> Run migration source: -> ->>>>>> EMULATOR=/path/to/emulator -> ->>>>>> ROOTFS=/path/to/image -> ->>>>>> QMPSOCK=/var/run/alma8qmp-src.sock -> ->>>>>> -> ->>>>>> $EMULATOR -enable-kvm \ -> ->>>>>>       -machine q35 \ -> ->>>>>>       -cpu host -smp 2 -m 2G \ -> ->>>>>>       -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -> ->>>>>> ram0,share=on\ -> ->>>>>>       -machine memory-backend=ram0 \ -> ->>>>>>       -machine aux-ram-share=on \ -> ->>>>>>       -drive file=$ROOTFS,media=disk,if=virtio \ -> ->>>>>>       -qmp unix:$QMPSOCK,server=on,wait=off \ -> ->>>>>>       -nographic \ -> ->>>>>>       -device qxl-vga -> ->>>>> -> ->>>>> Run migration target: -> ->>>>>> EMULATOR=/path/to/emulator -> ->>>>>> ROOTFS=/path/to/image -> ->>>>>> QMPSOCK=/var/run/alma8qmp-dst.sock -> ->>>>>> $EMULATOR -enable-kvm \ -> ->>>>>>       -machine q35 \ -> ->>>>>>       -cpu host -smp 2 -m 2G \ -> ->>>>>>       -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -> ->>>>>> ram0,share=on\ -> ->>>>>>       -machine memory-backend=ram0 \ -> ->>>>>>       -machine aux-ram-share=on \ -> ->>>>>>       -drive file=$ROOTFS,media=disk,if=virtio \ -> ->>>>>>       -qmp unix:$QMPSOCK,server=on,wait=off \ -> ->>>>>>       -nographic \ -> ->>>>>>       -device qxl-vga \ -> ->>>>>>       -incoming tcp:0:44444 \ -> ->>>>>>       -incoming '{"channel-type": "cpr", "addr": { "transport": -> ->>>>>> "socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -> ->>>>> -> ->>>>> -> ->>>>> Launch the migration: -> ->>>>>> QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -> ->>>>>> QMPSOCK=/var/run/alma8qmp-src.sock -> ->>>>>> -> ->>>>>> $QMPSHELL -p $QMPSOCK < ->>>>>>       migrate-set-parameters mode=cpr-transfer -> ->>>>>>       migrate channels=[{"channel-type":"main","addr": -> ->>>>>> {"transport":"socket","type":"inet","host":"0","port":"44444"}}, -> ->>>>>> {"channel-type":"cpr","addr": -> ->>>>>> {"transport":"socket","type":"unix","path":"/var/run/alma8cpr- -> ->>>>>> dst.sock"}}] -> ->>>>>> EOF -> ->>>>> -> ->>>>> Then, after a while, QXL guest driver on target crashes spewing the -> ->>>>> following messages: -> ->>>>>> [   73.962002] [TTM] Buffer eviction failed -> ->>>>>> [   73.962072] qxl 0000:00:02.0: object_init failed for (3149824, -> ->>>>>> 0x00000001) -> ->>>>>> [   73.962081] [drm:qxl_alloc_bo_reserved [qxl]] *ERROR* failed to -> ->>>>>> allocate VRAM BO -> ->>>>> -> ->>>>> That seems to be a known kernel QXL driver bug: -> ->>>>> -> ->>>>> -https://lore.kernel.org/all/20220907094423.93581-1- -> ->>>>> min_halo@163.com/T/ -> ->>>>> -https://lore.kernel.org/lkml/ZTgydqRlK6WX_b29@eldamar.lan/ -> ->>>>> -> ->>>>> (the latter discussion contains that reproduce script which speeds up -> ->>>>> the crash in the guest): -> ->>>>>> #!/bin/bash -> ->>>>>> -> ->>>>>> chvt 3 -> ->>>>>> -> ->>>>>> for j in $(seq 80); do -> ->>>>>>           echo "$(date) starting round $j" -> ->>>>>>           if [ "$(journalctl --boot | grep "failed to allocate VRAM -> ->>>>>> BO")" != "" ]; then -> ->>>>>>                   echo "bug was reproduced after $j tries" -> ->>>>>>                   exit 1 -> ->>>>>>           fi -> ->>>>>>           for i in $(seq 100); do -> ->>>>>>                   dmesg > /dev/tty3 -> ->>>>>>           done -> ->>>>>> done -> ->>>>>> -> ->>>>>> echo "bug could not be reproduced" -> ->>>>>> exit 0 -> ->>>>> -> ->>>>> The bug itself seems to remain unfixed, as I was able to reproduce -> ->>>>> that -> ->>>>> with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -> ->>>>> cpr-transfer code also seems to be buggy as it triggers the crash - -> ->>>>> without the cpr-transfer migration the above reproduce doesn't -> ->>>>> lead to -> ->>>>> crash on the source VM. -> ->>>>> -> ->>>>> I suspect that, as cpr-transfer doesn't migrate the guest memory, but -> ->>>>> rather passes it through the memory backend object, our code might -> ->>>>> somehow corrupt the VRAM.  However, I wasn't able to trace the -> ->>>>> corruption so far. -> ->>>>> -> ->>>>> Could somebody help the investigation and take a look into this?  Any -> ->>>>> suggestions would be appreciated.  Thanks! -> ->>>> -> ->>>> Possibly some memory region created by qxl is not being preserved. -> ->>>> Try adding these traces to see what is preserved: -> ->>>> -> ->>>> -trace enable='*cpr*' -> ->>>> -trace enable='*ram_alloc*' -> ->>> -> ->>> Also try adding this patch to see if it flags any ram blocks as not -> ->>> compatible with cpr.  A message is printed at migration start time. -> ->>>   -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send- -> ->>> email- -> ->>> steven.sistare@oracle.com/ -> ->>> -> ->>> - Steve -> ->>> -> ->> -> ->> With the traces enabled + the "migration: ram block cpr blockers" patch -> ->> applied: -> ->> -> ->> Source: -> ->>> cpr_find_fd pc.bios, id 0 returns -1 -> ->>> cpr_save_fd pc.bios, id 0, fd 22 -> ->>> qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -> ->>> 0x7fec18e00000 -> ->>> cpr_find_fd pc.rom, id 0 returns -1 -> ->>> cpr_save_fd pc.rom, id 0, fd 23 -> ->>> qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -> ->>> 0x7fec18c00000 -> ->>> cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -> ->>> cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -> ->>> qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -> ->>> 262144 fd 24 host 0x7fec18a00000 -> ->>> cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -> ->>> cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -> ->>> qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -> ->>> 67108864 fd 25 host 0x7feb77e00000 -> ->>> cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -> ->>> cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -> ->>> qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -> ->>> fd 27 host 0x7fec18800000 -> ->>> cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -> ->>> cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -> ->>> qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -> ->>> 67108864 fd 28 host 0x7feb73c00000 -> ->>> cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -> ->>> cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -> ->>> qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -> ->>> fd 34 host 0x7fec18600000 -> ->>> cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -> ->>> cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -> ->>> qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -> ->>> 2097152 fd 35 host 0x7fec18200000 -> ->>> cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -> ->>> cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -> ->>> qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -> ->>> fd 36 host 0x7feb8b600000 -> ->>> cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -> ->>> cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -> ->>> qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -> ->>> 37 host 0x7feb8b400000 -> ->>> -> ->>> cpr_state_save cpr-transfer mode -> ->>> cpr_transfer_output /var/run/alma8cpr-dst.sock -> ->> -> ->> Target: -> ->>> cpr_transfer_input /var/run/alma8cpr-dst.sock -> ->>> cpr_state_load cpr-transfer mode -> ->>> cpr_find_fd pc.bios, id 0 returns 20 -> ->>> qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -> ->>> 0x7fcdc9800000 -> ->>> cpr_find_fd pc.rom, id 0 returns 19 -> ->>> qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -> ->>> 0x7fcdc9600000 -> ->>> cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -> ->>> qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -> ->>> 262144 fd 18 host 0x7fcdc9400000 -> ->>> cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -> ->>> qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -> ->>> 67108864 fd 17 host 0x7fcd27e00000 -> ->>> cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -> ->>> qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -> ->>> fd 16 host 0x7fcdc9200000 -> ->>> cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -> ->>> qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -> ->>> 67108864 fd 15 host 0x7fcd23c00000 -> ->>> cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -> ->>> qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -> ->>> fd 14 host 0x7fcdc8800000 -> ->>> cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -> ->>> qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -> ->>> 2097152 fd 13 host 0x7fcdc8400000 -> ->>> cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -> ->>> qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -> ->>> fd 11 host 0x7fcdc8200000 -> ->>> cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -> ->>> qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -> ->>> 10 host 0x7fcd3be00000 -> ->> -> ->> Looks like both vga.vram and qxl.vram are being preserved (with the same -> ->> addresses), and no incompatible ram blocks are found during migration. -> -> -> -> Sorry, addressed are not the same, of course.  However corresponding ram -> -> blocks do seem to be preserved and initialized. -> -> -So far, I have not reproduced the guest driver failure. -> -> -However, I have isolated places where new QEMU improperly writes to -> -the qxl memory regions prior to starting the guest, by mmap'ing them -> -readonly after cpr: -> -> -  qemu_ram_alloc_internal() -> -    if (reused && (strstr(name, "qxl") || strstr("name", "vga"))) -> -        ram_flags |= RAM_READONLY; -> -    new_block = qemu_ram_alloc_from_fd(...) -> -> -I have attached a draft fix; try it and let me know. -> -My console window looks fine before and after cpr, using -> --vnc $hostip:0 -vga qxl -> -> -- Steve -Regarding the reproduce: when I launch the buggy version with the same -options as you, i.e. "-vnc 0.0.0.0:$port -vga qxl", and do cpr-transfer, -my VNC client silently hangs on the target after a while. Could it -happen on your stand as well? Could you try launching VM with -"-nographic -device qxl-vga"? That way VM's serial console is given you -directly in the shell, so when qxl driver crashes you're still able to -inspect the kernel messages. - -As for your patch, I can report that it doesn't resolve the issue as it -is. But I was able to track down another possible memory corruption -using your approach with readonly mmap'ing: - -> -Program terminated with signal SIGSEGV, Segmentation fault. -> -#0 init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -> -412 d->ram->magic = cpu_to_le32(QXL_RAM_MAGIC); -> -[Current thread is 1 (Thread 0x7f1a4f83b480 (LWP 229798))] -> -(gdb) bt -> -#0 init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -> -#1 0x0000563896e7f467 in qxl_realize_common (qxl=0x5638996e0e70, -> -errp=0x7ffd3c2b8170) at ../hw/display/qxl.c:2142 -> -#2 0x0000563896e7fda1 in qxl_realize_primary (dev=0x5638996e0e70, -> -errp=0x7ffd3c2b81d0) at ../hw/display/qxl.c:2257 -> -#3 0x0000563896c7e8f2 in pci_qdev_realize (qdev=0x5638996e0e70, -> -errp=0x7ffd3c2b8250) at ../hw/pci/pci.c:2174 -> -#4 0x00005638970eb54b in device_set_realized (obj=0x5638996e0e70, -> -value=true, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:494 -> -#5 0x00005638970f5e14 in property_set_bool (obj=0x5638996e0e70, -> -v=0x5638996f3770, name=0x56389759b141 "realized", opaque=0x5638987893d0, -> -errp=0x7ffd3c2b84e0) -> -at ../qom/object.c:2374 -> -#6 0x00005638970f39f8 in object_property_set (obj=0x5638996e0e70, -> -name=0x56389759b141 "realized", v=0x5638996f3770, errp=0x7ffd3c2b84e0) -> -at ../qom/object.c:1449 -> -#7 0x00005638970f8586 in object_property_set_qobject (obj=0x5638996e0e70, -> -name=0x56389759b141 "realized", value=0x5638996df900, errp=0x7ffd3c2b84e0) -> -at ../qom/qom-qobject.c:28 -> -#8 0x00005638970f3d8d in object_property_set_bool (obj=0x5638996e0e70, -> -name=0x56389759b141 "realized", value=true, errp=0x7ffd3c2b84e0) -> -at ../qom/object.c:1519 -> -#9 0x00005638970eacb0 in qdev_realize (dev=0x5638996e0e70, -> -bus=0x563898cf3c20, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:276 -> -#10 0x0000563896dba675 in qdev_device_add_from_qdict (opts=0x5638996dfe50, -> -from_json=false, errp=0x7ffd3c2b84e0) at ../system/qdev-monitor.c:714 -> -#11 0x0000563896dba721 in qdev_device_add (opts=0x563898786150, -> -errp=0x56389855dc40 ) at ../system/qdev-monitor.c:733 -> -#12 0x0000563896dc48f1 in device_init_func (opaque=0x0, opts=0x563898786150, -> -errp=0x56389855dc40 ) at ../system/vl.c:1207 -> -#13 0x000056389737a6cc in qemu_opts_foreach -> -(list=0x563898427b60 , func=0x563896dc48ca -> -, opaque=0x0, errp=0x56389855dc40 ) -> -at ../util/qemu-option.c:1135 -> -#14 0x0000563896dc89b5 in qemu_create_cli_devices () at ../system/vl.c:2745 -> -#15 0x0000563896dc8c00 in qmp_x_exit_preconfig (errp=0x56389855dc40 -> -) at ../system/vl.c:2806 -> -#16 0x0000563896dcb5de in qemu_init (argc=33, argv=0x7ffd3c2b8948) at -> -../system/vl.c:3838 -> -#17 0x0000563897297323 in main (argc=33, argv=0x7ffd3c2b8948) at -> -../system/main.c:72 -So the attached adjusted version of your patch does seem to help. At -least I can't reproduce the crash on my stand. - -I'm wondering, could it be useful to explicitly mark all the reused -memory regions readonly upon cpr-transfer, and then make them writable -back again after the migration is done? That way we will be segfaulting -early on instead of debugging tricky memory corruptions. - -Andrey -0001-hw-qxl-cpr-support-preliminary.patch -Description: -Text Data - -On 3/5/2025 11:50 AM, Andrey Drobyshev wrote: -On 3/4/25 9:05 PM, Steven Sistare wrote: -On 2/28/2025 1:37 PM, Andrey Drobyshev wrote: -On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -On 2/28/25 8:20 PM, Steven Sistare wrote: -On 2/28/2025 1:13 PM, Steven Sistare wrote: -On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -Hi all, - -We've been experimenting with cpr-transfer migration mode recently -and -have discovered the following issue with the guest QXL driver: - -Run migration source: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-src.sock - -$EMULATOR -enable-kvm \ -       -machine q35 \ -       -cpu host -smp 2 -m 2G \ -       -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -ram0,share=on\ -       -machine memory-backend=ram0 \ -       -machine aux-ram-share=on \ -       -drive file=$ROOTFS,media=disk,if=virtio \ -       -qmp unix:$QMPSOCK,server=on,wait=off \ -       -nographic \ -       -device qxl-vga -Run migration target: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-dst.sock -$EMULATOR -enable-kvm \ -       -machine q35 \ -       -cpu host -smp 2 -m 2G \ -       -object memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -ram0,share=on\ -       -machine memory-backend=ram0 \ -       -machine aux-ram-share=on \ -       -drive file=$ROOTFS,media=disk,if=virtio \ -       -qmp unix:$QMPSOCK,server=on,wait=off \ -       -nographic \ -       -device qxl-vga \ -       -incoming tcp:0:44444 \ -       -incoming '{"channel-type": "cpr", "addr": { "transport": -"socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -QMPSOCK=/var/run/alma8qmp-src.sock - -$QMPSHELL -p $QMPSOCK < /dev/tty3 -           done -done - -echo "bug could not be reproduced" -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce -that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the crash - -without the cpr-transfer migration the above reproduce doesn't -lead to -crash on the source VM. - -I suspect that, as cpr-transfer doesn't migrate the guest memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM.  However, I wasn't able to trace the -corruption so far. - -Could somebody help the investigation and take a look into this?  Any -suggestions would be appreciated.  Thanks! -Possibly some memory region created by qxl is not being preserved. -Try adding these traces to see what is preserved: - --trace enable='*cpr*' --trace enable='*ram_alloc*' -Also try adding this patch to see if it flags any ram blocks as not -compatible with cpr.  A message is printed at migration start time. -   -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send- -email- -steven.sistare@oracle.com/ - -- Steve -With the traces enabled + the "migration: ram block cpr blockers" patch -applied: - -Source: -cpr_find_fd pc.bios, id 0 returns -1 -cpr_save_fd pc.bios, id 0, fd 22 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -0x7fec18e00000 -cpr_find_fd pc.rom, id 0 returns -1 -cpr_save_fd pc.rom, id 0, fd 23 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -0x7fec18c00000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 24 host 0x7fec18a00000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 25 host 0x7feb77e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 27 host 0x7fec18800000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 28 host 0x7feb73c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 34 host 0x7fec18600000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 35 host 0x7fec18200000 -cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 36 host 0x7feb8b600000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -37 host 0x7feb8b400000 - -cpr_state_save cpr-transfer mode -cpr_transfer_output /var/run/alma8cpr-dst.sock -Target: -cpr_transfer_input /var/run/alma8cpr-dst.sock -cpr_state_load cpr-transfer mode -cpr_find_fd pc.bios, id 0 returns 20 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -0x7fcdc9800000 -cpr_find_fd pc.rom, id 0 returns 19 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -0x7fcdc9600000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 18 host 0x7fcdc9400000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 17 host 0x7fcd27e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 16 host 0x7fcdc9200000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 15 host 0x7fcd23c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 14 host 0x7fcdc8800000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 13 host 0x7fcdc8400000 -cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 11 host 0x7fcdc8200000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -10 host 0x7fcd3be00000 -Looks like both vga.vram and qxl.vram are being preserved (with the same -addresses), and no incompatible ram blocks are found during migration. -Sorry, addressed are not the same, of course.  However corresponding ram -blocks do seem to be preserved and initialized. -So far, I have not reproduced the guest driver failure. - -However, I have isolated places where new QEMU improperly writes to -the qxl memory regions prior to starting the guest, by mmap'ing them -readonly after cpr: - -   qemu_ram_alloc_internal() -     if (reused && (strstr(name, "qxl") || strstr("name", "vga"))) -         ram_flags |= RAM_READONLY; -     new_block = qemu_ram_alloc_from_fd(...) - -I have attached a draft fix; try it and let me know. -My console window looks fine before and after cpr, using --vnc $hostip:0 -vga qxl - -- Steve -Regarding the reproduce: when I launch the buggy version with the same -options as you, i.e. "-vnc 0.0.0.0:$port -vga qxl", and do cpr-transfer, -my VNC client silently hangs on the target after a while. Could it -happen on your stand as well? -cpr does not preserve the vnc connection and session. To test, I specify -port 0 for the source VM and port 1 for the dest. When the src vnc goes -dormant the dest vnc becomes active. -Could you try launching VM with -"-nographic -device qxl-vga"? That way VM's serial console is given you -directly in the shell, so when qxl driver crashes you're still able to -inspect the kernel messages. -I have been running like that, but have not reproduced the qxl driver crash, -and I suspect my guest image+kernel is too old. However, once I realized the -issue was post-cpr modification of qxl memory, I switched my attention to the -fix. -As for your patch, I can report that it doesn't resolve the issue as it -is. But I was able to track down another possible memory corruption -using your approach with readonly mmap'ing: -Program terminated with signal SIGSEGV, Segmentation fault. -#0 init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -412 d->ram->magic = cpu_to_le32(QXL_RAM_MAGIC); -[Current thread is 1 (Thread 0x7f1a4f83b480 (LWP 229798))] -(gdb) bt -#0 init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -#1 0x0000563896e7f467 in qxl_realize_common (qxl=0x5638996e0e70, -errp=0x7ffd3c2b8170) at ../hw/display/qxl.c:2142 -#2 0x0000563896e7fda1 in qxl_realize_primary (dev=0x5638996e0e70, -errp=0x7ffd3c2b81d0) at ../hw/display/qxl.c:2257 -#3 0x0000563896c7e8f2 in pci_qdev_realize (qdev=0x5638996e0e70, -errp=0x7ffd3c2b8250) at ../hw/pci/pci.c:2174 -#4 0x00005638970eb54b in device_set_realized (obj=0x5638996e0e70, value=true, -errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:494 -#5 0x00005638970f5e14 in property_set_bool (obj=0x5638996e0e70, v=0x5638996f3770, -name=0x56389759b141 "realized", opaque=0x5638987893d0, errp=0x7ffd3c2b84e0) - at ../qom/object.c:2374 -#6 0x00005638970f39f8 in object_property_set (obj=0x5638996e0e70, name=0x56389759b141 -"realized", v=0x5638996f3770, errp=0x7ffd3c2b84e0) - at ../qom/object.c:1449 -#7 0x00005638970f8586 in object_property_set_qobject (obj=0x5638996e0e70, -name=0x56389759b141 "realized", value=0x5638996df900, errp=0x7ffd3c2b84e0) - at ../qom/qom-qobject.c:28 -#8 0x00005638970f3d8d in object_property_set_bool (obj=0x5638996e0e70, -name=0x56389759b141 "realized", value=true, errp=0x7ffd3c2b84e0) - at ../qom/object.c:1519 -#9 0x00005638970eacb0 in qdev_realize (dev=0x5638996e0e70, bus=0x563898cf3c20, -errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:276 -#10 0x0000563896dba675 in qdev_device_add_from_qdict (opts=0x5638996dfe50, -from_json=false, errp=0x7ffd3c2b84e0) at ../system/qdev-monitor.c:714 -#11 0x0000563896dba721 in qdev_device_add (opts=0x563898786150, errp=0x56389855dc40 -) at ../system/qdev-monitor.c:733 -#12 0x0000563896dc48f1 in device_init_func (opaque=0x0, opts=0x563898786150, -errp=0x56389855dc40 ) at ../system/vl.c:1207 -#13 0x000056389737a6cc in qemu_opts_foreach - (list=0x563898427b60 , func=0x563896dc48ca , -opaque=0x0, errp=0x56389855dc40 ) - at ../util/qemu-option.c:1135 -#14 0x0000563896dc89b5 in qemu_create_cli_devices () at ../system/vl.c:2745 -#15 0x0000563896dc8c00 in qmp_x_exit_preconfig (errp=0x56389855dc40 -) at ../system/vl.c:2806 -#16 0x0000563896dcb5de in qemu_init (argc=33, argv=0x7ffd3c2b8948) at -../system/vl.c:3838 -#17 0x0000563897297323 in main (argc=33, argv=0x7ffd3c2b8948) at -../system/main.c:72 -So the attached adjusted version of your patch does seem to help. At -least I can't reproduce the crash on my stand. -Thanks for the stack trace; the calls to SPICE_RING_INIT in init_qxl_ram are -definitely harmful. Try V2 of the patch, attached, which skips the lines -of init_qxl_ram that modify guest memory. -I'm wondering, could it be useful to explicitly mark all the reused -memory regions readonly upon cpr-transfer, and then make them writable -back again after the migration is done? That way we will be segfaulting -early on instead of debugging tricky memory corruptions. -It's a useful debugging technique, but changing protection on a large memory -region -can be too expensive for production due to TLB shootdowns. - -Also, there are cases where writes are performed but the value is guaranteed to -be the same: - qxl_post_load() - qxl_set_mode() - d->rom->mode = cpu_to_le32(modenr); -The value is the same because mode and shadow_rom.mode were passed in vmstate -from old qemu. - -- Steve -0001-hw-qxl-cpr-support-preliminary-V2.patch -Description: -Text document - -On 3/5/25 22:19, Steven Sistare wrote: -On 3/5/2025 11:50 AM, Andrey Drobyshev wrote: -On 3/4/25 9:05 PM, Steven Sistare wrote: -On 2/28/2025 1:37 PM, Andrey Drobyshev wrote: -On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -On 2/28/25 8:20 PM, Steven Sistare wrote: -On 2/28/2025 1:13 PM, Steven Sistare wrote: -On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -Hi all, - -We've been experimenting with cpr-transfer migration mode recently -and -have discovered the following issue with the guest QXL driver: - -Run migration source: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-src.sock - -$EMULATOR -enable-kvm \ -       -machine q35 \ -       -cpu host -smp 2 -m 2G \ -       -object -memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -ram0,share=on\ -       -machine memory-backend=ram0 \ -       -machine aux-ram-share=on \ -       -drive file=$ROOTFS,media=disk,if=virtio \ -       -qmp unix:$QMPSOCK,server=on,wait=off \ -       -nographic \ -       -device qxl-vga -Run migration target: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-dst.sock -$EMULATOR -enable-kvm \ -       -machine q35 \ -       -cpu host -smp 2 -m 2G \ -       -object -memory-backend-file,id=ram0,size=2G,mem-path=/dev/shm/ -ram0,share=on\ -       -machine memory-backend=ram0 \ -       -machine aux-ram-share=on \ -       -drive file=$ROOTFS,media=disk,if=virtio \ -       -qmp unix:$QMPSOCK,server=on,wait=off \ -       -nographic \ -       -device qxl-vga \ -       -incoming tcp:0:44444 \ -       -incoming '{"channel-type": "cpr", "addr": { "transport": -"socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -QMPSOCK=/var/run/alma8qmp-src.sock - -$QMPSHELL -p $QMPSOCK < /dev/tty3 -           done -done - -echo "bug could not be reproduced" -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce -that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the -crash - -without the cpr-transfer migration the above reproduce doesn't -lead to -crash on the source VM. -I suspect that, as cpr-transfer doesn't migrate the guest -memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM.  However, I wasn't able to trace the -corruption so far. -Could somebody help the investigation and take a look into -this?  Any -suggestions would be appreciated.  Thanks! -Possibly some memory region created by qxl is not being preserved. -Try adding these traces to see what is preserved: - --trace enable='*cpr*' --trace enable='*ram_alloc*' -Also try adding this patch to see if it flags any ram blocks as not -compatible with cpr.  A message is printed at migration start time. -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send- -email- -steven.sistare@oracle.com/ - -- Steve -With the traces enabled + the "migration: ram block cpr blockers" -patch -applied: - -Source: -cpr_find_fd pc.bios, id 0 returns -1 -cpr_save_fd pc.bios, id 0, fd 22 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -0x7fec18e00000 -cpr_find_fd pc.rom, id 0 returns -1 -cpr_save_fd pc.rom, id 0, fd 23 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -0x7fec18c00000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 24 host 0x7fec18a00000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 25 host 0x7feb77e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 27 host 0x7fec18800000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 28 host 0x7feb73c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 34 host 0x7fec18600000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 35 host 0x7fec18200000 -cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 36 host 0x7feb8b600000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -37 host 0x7feb8b400000 - -cpr_state_save cpr-transfer mode -cpr_transfer_output /var/run/alma8cpr-dst.sock -Target: -cpr_transfer_input /var/run/alma8cpr-dst.sock -cpr_state_load cpr-transfer mode -cpr_find_fd pc.bios, id 0 returns 20 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -0x7fcdc9800000 -cpr_find_fd pc.rom, id 0 returns 19 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -0x7fcdc9600000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 18 host 0x7fcdc9400000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 17 host 0x7fcd27e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 16 host 0x7fcdc9200000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 15 host 0x7fcd23c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 14 host 0x7fcdc8800000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 13 host 0x7fcdc8400000 -cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 11 host 0x7fcdc8200000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -10 host 0x7fcd3be00000 -Looks like both vga.vram and qxl.vram are being preserved (with -the same -addresses), and no incompatible ram blocks are found during -migration. -Sorry, addressed are not the same, of course.  However -corresponding ram -blocks do seem to be preserved and initialized. -So far, I have not reproduced the guest driver failure. - -However, I have isolated places where new QEMU improperly writes to -the qxl memory regions prior to starting the guest, by mmap'ing them -readonly after cpr: - -   qemu_ram_alloc_internal() -     if (reused && (strstr(name, "qxl") || strstr("name", "vga"))) -         ram_flags |= RAM_READONLY; -     new_block = qemu_ram_alloc_from_fd(...) - -I have attached a draft fix; try it and let me know. -My console window looks fine before and after cpr, using --vnc $hostip:0 -vga qxl - -- Steve -Regarding the reproduce: when I launch the buggy version with the same -options as you, i.e. "-vnc 0.0.0.0:$port -vga qxl", and do cpr-transfer, -my VNC client silently hangs on the target after a while.  Could it -happen on your stand as well? -cpr does not preserve the vnc connection and session.  To test, I specify -port 0 for the source VM and port 1 for the dest.  When the src vnc goes -dormant the dest vnc becomes active. -Could you try launching VM with -"-nographic -device qxl-vga"?  That way VM's serial console is given you -directly in the shell, so when qxl driver crashes you're still able to -inspect the kernel messages. -I have been running like that, but have not reproduced the qxl driver -crash, -and I suspect my guest image+kernel is too old.  However, once I -realized the -issue was post-cpr modification of qxl memory, I switched my attention -to the -fix. -As for your patch, I can report that it doesn't resolve the issue as it -is.  But I was able to track down another possible memory corruption -using your approach with readonly mmap'ing: -Program terminated with signal SIGSEGV, Segmentation fault. -#0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -412         d->ram->magic       = cpu_to_le32(QXL_RAM_MAGIC); -[Current thread is 1 (Thread 0x7f1a4f83b480 (LWP 229798))] -(gdb) bt -#0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -#1  0x0000563896e7f467 in qxl_realize_common (qxl=0x5638996e0e70, -errp=0x7ffd3c2b8170) at ../hw/display/qxl.c:2142 -#2  0x0000563896e7fda1 in qxl_realize_primary (dev=0x5638996e0e70, -errp=0x7ffd3c2b81d0) at ../hw/display/qxl.c:2257 -#3  0x0000563896c7e8f2 in pci_qdev_realize (qdev=0x5638996e0e70, -errp=0x7ffd3c2b8250) at ../hw/pci/pci.c:2174 -#4  0x00005638970eb54b in device_set_realized (obj=0x5638996e0e70, -value=true, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:494 -#5  0x00005638970f5e14 in property_set_bool (obj=0x5638996e0e70, -v=0x5638996f3770, name=0x56389759b141 "realized", -opaque=0x5638987893d0, errp=0x7ffd3c2b84e0) -     at ../qom/object.c:2374 -#6  0x00005638970f39f8 in object_property_set (obj=0x5638996e0e70, -name=0x56389759b141 "realized", v=0x5638996f3770, errp=0x7ffd3c2b84e0) -     at ../qom/object.c:1449 -#7  0x00005638970f8586 in object_property_set_qobject -(obj=0x5638996e0e70, name=0x56389759b141 "realized", -value=0x5638996df900, errp=0x7ffd3c2b84e0) -     at ../qom/qom-qobject.c:28 -#8  0x00005638970f3d8d in object_property_set_bool -(obj=0x5638996e0e70, name=0x56389759b141 "realized", value=true, -errp=0x7ffd3c2b84e0) -     at ../qom/object.c:1519 -#9  0x00005638970eacb0 in qdev_realize (dev=0x5638996e0e70, -bus=0x563898cf3c20, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:276 -#10 0x0000563896dba675 in qdev_device_add_from_qdict -(opts=0x5638996dfe50, from_json=false, errp=0x7ffd3c2b84e0) at -../system/qdev-monitor.c:714 -#11 0x0000563896dba721 in qdev_device_add (opts=0x563898786150, -errp=0x56389855dc40 ) at ../system/qdev-monitor.c:733 -#12 0x0000563896dc48f1 in device_init_func (opaque=0x0, -opts=0x563898786150, errp=0x56389855dc40 ) at -../system/vl.c:1207 -#13 0x000056389737a6cc in qemu_opts_foreach -     (list=0x563898427b60 , func=0x563896dc48ca -, opaque=0x0, errp=0x56389855dc40 ) -     at ../util/qemu-option.c:1135 -#14 0x0000563896dc89b5 in qemu_create_cli_devices () at -../system/vl.c:2745 -#15 0x0000563896dc8c00 in qmp_x_exit_preconfig (errp=0x56389855dc40 -) at ../system/vl.c:2806 -#16 0x0000563896dcb5de in qemu_init (argc=33, argv=0x7ffd3c2b8948) -at ../system/vl.c:3838 -#17 0x0000563897297323 in main (argc=33, argv=0x7ffd3c2b8948) at -../system/main.c:72 -So the attached adjusted version of your patch does seem to help.  At -least I can't reproduce the crash on my stand. -Thanks for the stack trace; the calls to SPICE_RING_INIT in -init_qxl_ram are -definitely harmful.  Try V2 of the patch, attached, which skips the lines -of init_qxl_ram that modify guest memory. -I'm wondering, could it be useful to explicitly mark all the reused -memory regions readonly upon cpr-transfer, and then make them writable -back again after the migration is done?  That way we will be segfaulting -early on instead of debugging tricky memory corruptions. -It's a useful debugging technique, but changing protection on a large -memory region -can be too expensive for production due to TLB shootdowns. -Good point. Though we could move this code under non-default option to -avoid re-writing. - -Den - -On 3/5/25 11:19 PM, Steven Sistare wrote: -> -On 3/5/2025 11:50 AM, Andrey Drobyshev wrote: -> -> On 3/4/25 9:05 PM, Steven Sistare wrote: -> ->> On 2/28/2025 1:37 PM, Andrey Drobyshev wrote: -> ->>> On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -> ->>>> On 2/28/25 8:20 PM, Steven Sistare wrote: -> ->>>>> On 2/28/2025 1:13 PM, Steven Sistare wrote: -> ->>>>>> On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -> ->>>>>>> Hi all, -> ->>>>>>> -> ->>>>>>> We've been experimenting with cpr-transfer migration mode recently -> ->>>>>>> and -> ->>>>>>> have discovered the following issue with the guest QXL driver: -> ->>>>>>> -> ->>>>>>> Run migration source: -> ->>>>>>>> EMULATOR=/path/to/emulator -> ->>>>>>>> ROOTFS=/path/to/image -> ->>>>>>>> QMPSOCK=/var/run/alma8qmp-src.sock -> ->>>>>>>> -> ->>>>>>>> $EMULATOR -enable-kvm \ -> ->>>>>>>>        -machine q35 \ -> ->>>>>>>>        -cpu host -smp 2 -m 2G \ -> ->>>>>>>>        -object memory-backend-file,id=ram0,size=2G,mem-path=/ -> ->>>>>>>> dev/shm/ -> ->>>>>>>> ram0,share=on\ -> ->>>>>>>>        -machine memory-backend=ram0 \ -> ->>>>>>>>        -machine aux-ram-share=on \ -> ->>>>>>>>        -drive file=$ROOTFS,media=disk,if=virtio \ -> ->>>>>>>>        -qmp unix:$QMPSOCK,server=on,wait=off \ -> ->>>>>>>>        -nographic \ -> ->>>>>>>>        -device qxl-vga -> ->>>>>>> -> ->>>>>>> Run migration target: -> ->>>>>>>> EMULATOR=/path/to/emulator -> ->>>>>>>> ROOTFS=/path/to/image -> ->>>>>>>> QMPSOCK=/var/run/alma8qmp-dst.sock -> ->>>>>>>> $EMULATOR -enable-kvm \ -> ->>>>>>>>        -machine q35 \ -> ->>>>>>>>        -cpu host -smp 2 -m 2G \ -> ->>>>>>>>        -object memory-backend-file,id=ram0,size=2G,mem-path=/ -> ->>>>>>>> dev/shm/ -> ->>>>>>>> ram0,share=on\ -> ->>>>>>>>        -machine memory-backend=ram0 \ -> ->>>>>>>>        -machine aux-ram-share=on \ -> ->>>>>>>>        -drive file=$ROOTFS,media=disk,if=virtio \ -> ->>>>>>>>        -qmp unix:$QMPSOCK,server=on,wait=off \ -> ->>>>>>>>        -nographic \ -> ->>>>>>>>        -device qxl-vga \ -> ->>>>>>>>        -incoming tcp:0:44444 \ -> ->>>>>>>>        -incoming '{"channel-type": "cpr", "addr": { "transport": -> ->>>>>>>> "socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -> ->>>>>>> -> ->>>>>>> -> ->>>>>>> Launch the migration: -> ->>>>>>>> QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -> ->>>>>>>> QMPSOCK=/var/run/alma8qmp-src.sock -> ->>>>>>>> -> ->>>>>>>> $QMPSHELL -p $QMPSOCK < ->>>>>>>>        migrate-set-parameters mode=cpr-transfer -> ->>>>>>>>        migrate channels=[{"channel-type":"main","addr": -> ->>>>>>>> {"transport":"socket","type":"inet","host":"0","port":"44444"}}, -> ->>>>>>>> {"channel-type":"cpr","addr": -> ->>>>>>>> {"transport":"socket","type":"unix","path":"/var/run/alma8cpr- -> ->>>>>>>> dst.sock"}}] -> ->>>>>>>> EOF -> ->>>>>>> -> ->>>>>>> Then, after a while, QXL guest driver on target crashes spewing the -> ->>>>>>> following messages: -> ->>>>>>>> [   73.962002] [TTM] Buffer eviction failed -> ->>>>>>>> [   73.962072] qxl 0000:00:02.0: object_init failed for (3149824, -> ->>>>>>>> 0x00000001) -> ->>>>>>>> [   73.962081] [drm:qxl_alloc_bo_reserved [qxl]] *ERROR* failed to -> ->>>>>>>> allocate VRAM BO -> ->>>>>>> -> ->>>>>>> That seems to be a known kernel QXL driver bug: -> ->>>>>>> -> ->>>>>>> -https://lore.kernel.org/all/20220907094423.93581-1- -> ->>>>>>> min_halo@163.com/T/ -> ->>>>>>> -https://lore.kernel.org/lkml/ZTgydqRlK6WX_b29@eldamar.lan/ -> ->>>>>>> -> ->>>>>>> (the latter discussion contains that reproduce script which -> ->>>>>>> speeds up -> ->>>>>>> the crash in the guest): -> ->>>>>>>> #!/bin/bash -> ->>>>>>>> -> ->>>>>>>> chvt 3 -> ->>>>>>>> -> ->>>>>>>> for j in $(seq 80); do -> ->>>>>>>>            echo "$(date) starting round $j" -> ->>>>>>>>            if [ "$(journalctl --boot | grep "failed to allocate -> ->>>>>>>> VRAM -> ->>>>>>>> BO")" != "" ]; then -> ->>>>>>>>                    echo "bug was reproduced after $j tries" -> ->>>>>>>>                    exit 1 -> ->>>>>>>>            fi -> ->>>>>>>>            for i in $(seq 100); do -> ->>>>>>>>                    dmesg > /dev/tty3 -> ->>>>>>>>            done -> ->>>>>>>> done -> ->>>>>>>> -> ->>>>>>>> echo "bug could not be reproduced" -> ->>>>>>>> exit 0 -> ->>>>>>> -> ->>>>>>> The bug itself seems to remain unfixed, as I was able to reproduce -> ->>>>>>> that -> ->>>>>>> with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -> ->>>>>>> cpr-transfer code also seems to be buggy as it triggers the crash - -> ->>>>>>> without the cpr-transfer migration the above reproduce doesn't -> ->>>>>>> lead to -> ->>>>>>> crash on the source VM. -> ->>>>>>> -> ->>>>>>> I suspect that, as cpr-transfer doesn't migrate the guest -> ->>>>>>> memory, but -> ->>>>>>> rather passes it through the memory backend object, our code might -> ->>>>>>> somehow corrupt the VRAM.  However, I wasn't able to trace the -> ->>>>>>> corruption so far. -> ->>>>>>> -> ->>>>>>> Could somebody help the investigation and take a look into -> ->>>>>>> this?  Any -> ->>>>>>> suggestions would be appreciated.  Thanks! -> ->>>>>> -> ->>>>>> Possibly some memory region created by qxl is not being preserved. -> ->>>>>> Try adding these traces to see what is preserved: -> ->>>>>> -> ->>>>>> -trace enable='*cpr*' -> ->>>>>> -trace enable='*ram_alloc*' -> ->>>>> -> ->>>>> Also try adding this patch to see if it flags any ram blocks as not -> ->>>>> compatible with cpr.  A message is printed at migration start time. -> ->>>>>    -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send- -> ->>>>> email- -> ->>>>> steven.sistare@oracle.com/ -> ->>>>> -> ->>>>> - Steve -> ->>>>> -> ->>>> -> ->>>> With the traces enabled + the "migration: ram block cpr blockers" -> ->>>> patch -> ->>>> applied: -> ->>>> -> ->>>> Source: -> ->>>>> cpr_find_fd pc.bios, id 0 returns -1 -> ->>>>> cpr_save_fd pc.bios, id 0, fd 22 -> ->>>>> qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -> ->>>>> 0x7fec18e00000 -> ->>>>> cpr_find_fd pc.rom, id 0 returns -1 -> ->>>>> cpr_save_fd pc.rom, id 0, fd 23 -> ->>>>> qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -> ->>>>> 0x7fec18c00000 -> ->>>>> cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -> ->>>>> cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -> ->>>>> qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -> ->>>>> 262144 fd 24 host 0x7fec18a00000 -> ->>>>> cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -> ->>>>> cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -> ->>>>> qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -> ->>>>> 67108864 fd 25 host 0x7feb77e00000 -> ->>>>> cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -> ->>>>> cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -> ->>>>> qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -> ->>>>> fd 27 host 0x7fec18800000 -> ->>>>> cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -> ->>>>> cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -> ->>>>> qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -> ->>>>> 67108864 fd 28 host 0x7feb73c00000 -> ->>>>> cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -> ->>>>> cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -> ->>>>> qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -> ->>>>> fd 34 host 0x7fec18600000 -> ->>>>> cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -> ->>>>> cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -> ->>>>> qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -> ->>>>> 2097152 fd 35 host 0x7fec18200000 -> ->>>>> cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -> ->>>>> cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -> ->>>>> qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -> ->>>>> fd 36 host 0x7feb8b600000 -> ->>>>> cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -> ->>>>> cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -> ->>>>> qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -> ->>>>> 37 host 0x7feb8b400000 -> ->>>>> -> ->>>>> cpr_state_save cpr-transfer mode -> ->>>>> cpr_transfer_output /var/run/alma8cpr-dst.sock -> ->>>> -> ->>>> Target: -> ->>>>> cpr_transfer_input /var/run/alma8cpr-dst.sock -> ->>>>> cpr_state_load cpr-transfer mode -> ->>>>> cpr_find_fd pc.bios, id 0 returns 20 -> ->>>>> qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -> ->>>>> 0x7fcdc9800000 -> ->>>>> cpr_find_fd pc.rom, id 0 returns 19 -> ->>>>> qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -> ->>>>> 0x7fcdc9600000 -> ->>>>> cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -> ->>>>> qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -> ->>>>> 262144 fd 18 host 0x7fcdc9400000 -> ->>>>> cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -> ->>>>> qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -> ->>>>> 67108864 fd 17 host 0x7fcd27e00000 -> ->>>>> cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -> ->>>>> qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -> ->>>>> fd 16 host 0x7fcdc9200000 -> ->>>>> cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -> ->>>>> qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -> ->>>>> 67108864 fd 15 host 0x7fcd23c00000 -> ->>>>> cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -> ->>>>> qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -> ->>>>> fd 14 host 0x7fcdc8800000 -> ->>>>> cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -> ->>>>> qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -> ->>>>> 2097152 fd 13 host 0x7fcdc8400000 -> ->>>>> cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -> ->>>>> qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -> ->>>>> fd 11 host 0x7fcdc8200000 -> ->>>>> cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -> ->>>>> qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -> ->>>>> 10 host 0x7fcd3be00000 -> ->>>> -> ->>>> Looks like both vga.vram and qxl.vram are being preserved (with the -> ->>>> same -> ->>>> addresses), and no incompatible ram blocks are found during migration. -> ->>> -> ->>> Sorry, addressed are not the same, of course.  However corresponding -> ->>> ram -> ->>> blocks do seem to be preserved and initialized. -> ->> -> ->> So far, I have not reproduced the guest driver failure. -> ->> -> ->> However, I have isolated places where new QEMU improperly writes to -> ->> the qxl memory regions prior to starting the guest, by mmap'ing them -> ->> readonly after cpr: -> ->> -> ->>    qemu_ram_alloc_internal() -> ->>      if (reused && (strstr(name, "qxl") || strstr("name", "vga"))) -> ->>          ram_flags |= RAM_READONLY; -> ->>      new_block = qemu_ram_alloc_from_fd(...) -> ->> -> ->> I have attached a draft fix; try it and let me know. -> ->> My console window looks fine before and after cpr, using -> ->> -vnc $hostip:0 -vga qxl -> ->> -> ->> - Steve -> -> -> -> Regarding the reproduce: when I launch the buggy version with the same -> -> options as you, i.e. "-vnc 0.0.0.0:$port -vga qxl", and do cpr-transfer, -> -> my VNC client silently hangs on the target after a while.  Could it -> -> happen on your stand as well? -> -> -cpr does not preserve the vnc connection and session.  To test, I specify -> -port 0 for the source VM and port 1 for the dest.  When the src vnc goes -> -dormant the dest vnc becomes active. -> -Sure, I meant that VNC on the dest (on the port 1) works for a while -after the migration and then hangs, apparently after the guest QXL crash. - -> -> Could you try launching VM with -> -> "-nographic -device qxl-vga"?  That way VM's serial console is given you -> -> directly in the shell, so when qxl driver crashes you're still able to -> -> inspect the kernel messages. -> -> -I have been running like that, but have not reproduced the qxl driver -> -crash, -> -and I suspect my guest image+kernel is too old. -Yes, that's probably the case. But the crash occurs on my Fedora 41 -guest with the 6.11.5-300.fc41.x86_64 kernel, so newer kernels seem to -be buggy. - - -> -However, once I realized the -> -issue was post-cpr modification of qxl memory, I switched my attention -> -to the -> -fix. -> -> -> As for your patch, I can report that it doesn't resolve the issue as it -> -> is.  But I was able to track down another possible memory corruption -> -> using your approach with readonly mmap'ing: -> -> -> ->> Program terminated with signal SIGSEGV, Segmentation fault. -> ->> #0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -> ->> 412         d->ram->magic       = cpu_to_le32(QXL_RAM_MAGIC); -> ->> [Current thread is 1 (Thread 0x7f1a4f83b480 (LWP 229798))] -> ->> (gdb) bt -> ->> #0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -> ->> #1  0x0000563896e7f467 in qxl_realize_common (qxl=0x5638996e0e70, -> ->> errp=0x7ffd3c2b8170) at ../hw/display/qxl.c:2142 -> ->> #2  0x0000563896e7fda1 in qxl_realize_primary (dev=0x5638996e0e70, -> ->> errp=0x7ffd3c2b81d0) at ../hw/display/qxl.c:2257 -> ->> #3  0x0000563896c7e8f2 in pci_qdev_realize (qdev=0x5638996e0e70, -> ->> errp=0x7ffd3c2b8250) at ../hw/pci/pci.c:2174 -> ->> #4  0x00005638970eb54b in device_set_realized (obj=0x5638996e0e70, -> ->> value=true, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:494 -> ->> #5  0x00005638970f5e14 in property_set_bool (obj=0x5638996e0e70, -> ->> v=0x5638996f3770, name=0x56389759b141 "realized", -> ->> opaque=0x5638987893d0, errp=0x7ffd3c2b84e0) -> ->>      at ../qom/object.c:2374 -> ->> #6  0x00005638970f39f8 in object_property_set (obj=0x5638996e0e70, -> ->> name=0x56389759b141 "realized", v=0x5638996f3770, errp=0x7ffd3c2b84e0) -> ->>      at ../qom/object.c:1449 -> ->> #7  0x00005638970f8586 in object_property_set_qobject -> ->> (obj=0x5638996e0e70, name=0x56389759b141 "realized", -> ->> value=0x5638996df900, errp=0x7ffd3c2b84e0) -> ->>      at ../qom/qom-qobject.c:28 -> ->> #8  0x00005638970f3d8d in object_property_set_bool -> ->> (obj=0x5638996e0e70, name=0x56389759b141 "realized", value=true, -> ->> errp=0x7ffd3c2b84e0) -> ->>      at ../qom/object.c:1519 -> ->> #9  0x00005638970eacb0 in qdev_realize (dev=0x5638996e0e70, -> ->> bus=0x563898cf3c20, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:276 -> ->> #10 0x0000563896dba675 in qdev_device_add_from_qdict -> ->> (opts=0x5638996dfe50, from_json=false, errp=0x7ffd3c2b84e0) at ../ -> ->> system/qdev-monitor.c:714 -> ->> #11 0x0000563896dba721 in qdev_device_add (opts=0x563898786150, -> ->> errp=0x56389855dc40 ) at ../system/qdev-monitor.c:733 -> ->> #12 0x0000563896dc48f1 in device_init_func (opaque=0x0, -> ->> opts=0x563898786150, errp=0x56389855dc40 ) at ../system/ -> ->> vl.c:1207 -> ->> #13 0x000056389737a6cc in qemu_opts_foreach -> ->>      (list=0x563898427b60 , func=0x563896dc48ca -> ->> , opaque=0x0, errp=0x56389855dc40 ) -> ->>      at ../util/qemu-option.c:1135 -> ->> #14 0x0000563896dc89b5 in qemu_create_cli_devices () at ../system/ -> ->> vl.c:2745 -> ->> #15 0x0000563896dc8c00 in qmp_x_exit_preconfig (errp=0x56389855dc40 -> ->> ) at ../system/vl.c:2806 -> ->> #16 0x0000563896dcb5de in qemu_init (argc=33, argv=0x7ffd3c2b8948) -> ->> at ../system/vl.c:3838 -> ->> #17 0x0000563897297323 in main (argc=33, argv=0x7ffd3c2b8948) at ../ -> ->> system/main.c:72 -> -> -> -> So the attached adjusted version of your patch does seem to help.  At -> -> least I can't reproduce the crash on my stand. -> -> -Thanks for the stack trace; the calls to SPICE_RING_INIT in init_qxl_ram -> -are -> -definitely harmful.  Try V2 of the patch, attached, which skips the lines -> -of init_qxl_ram that modify guest memory. -> -Thanks, your v2 patch does seem to prevent the crash. Would you re-send -it to the list as a proper fix? - -> -> I'm wondering, could it be useful to explicitly mark all the reused -> -> memory regions readonly upon cpr-transfer, and then make them writable -> -> back again after the migration is done?  That way we will be segfaulting -> -> early on instead of debugging tricky memory corruptions. -> -> -It's a useful debugging technique, but changing protection on a large -> -memory region -> -can be too expensive for production due to TLB shootdowns. -> -> -Also, there are cases where writes are performed but the value is -> -guaranteed to -> -be the same: -> -  qxl_post_load() -> -    qxl_set_mode() -> -      d->rom->mode = cpu_to_le32(modenr); -> -The value is the same because mode and shadow_rom.mode were passed in -> -vmstate -> -from old qemu. -> -There're also cases where devices' ROM might be re-initialized. E.g. -this segfault occures upon further exploration of RO mapped RAM blocks: - -> -Program terminated with signal SIGSEGV, Segmentation fault. -> -#0 __memmove_avx_unaligned_erms () at -> -../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:664 -> -664 rep movsb -> -[Current thread is 1 (Thread 0x7f6e7d08b480 (LWP 310379))] -> -(gdb) bt -> -#0 __memmove_avx_unaligned_erms () at -> -../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:664 -> -#1 0x000055aa1d030ecd in rom_set_mr (rom=0x55aa200ba380, -> -owner=0x55aa2019ac10, name=0x7fffb8272bc0 "/rom@etc/acpi/tables", ro=true) -> -at ../hw/core/loader.c:1032 -> -#2 0x000055aa1d031577 in rom_add_blob -> -(name=0x55aa1da51f13 "etc/acpi/tables", blob=0x55aa208a1070, len=131072, -> -max_len=2097152, addr=18446744073709551615, fw_file_name=0x55aa1da51f13 -> -"etc/acpi/tables", fw_callback=0x55aa1d441f59 , -> -callback_opaque=0x55aa20ff0010, as=0x0, read_only=true) at -> -../hw/core/loader.c:1147 -> -#3 0x000055aa1cfd788d in acpi_add_rom_blob -> -(update=0x55aa1d441f59 , opaque=0x55aa20ff0010, -> -blob=0x55aa1fc9aa00, name=0x55aa1da51f13 "etc/acpi/tables") at -> -../hw/acpi/utils.c:46 -> -#4 0x000055aa1d44213f in acpi_setup () at ../hw/i386/acpi-build.c:2720 -> -#5 0x000055aa1d434199 in pc_machine_done (notifier=0x55aa1ff15050, data=0x0) -> -at ../hw/i386/pc.c:638 -> -#6 0x000055aa1d876845 in notifier_list_notify (list=0x55aa1ea25c10 -> -, data=0x0) at ../util/notify.c:39 -> -#7 0x000055aa1d039ee5 in qdev_machine_creation_done () at -> -../hw/core/machine.c:1749 -> -#8 0x000055aa1d2c7b3e in qemu_machine_creation_done (errp=0x55aa1ea5cc40 -> -) at ../system/vl.c:2779 -> -#9 0x000055aa1d2c7c7d in qmp_x_exit_preconfig (errp=0x55aa1ea5cc40 -> -) at ../system/vl.c:2807 -> -#10 0x000055aa1d2ca64f in qemu_init (argc=35, argv=0x7fffb82730e8) at -> -../system/vl.c:3838 -> -#11 0x000055aa1d79638c in main (argc=35, argv=0x7fffb82730e8) at -> -../system/main.c:72 -I'm not sure whether ACPI tables ROM in particular is rewritten with the -same content, but there might be cases where ROM can be read from file -system upon initialization. That is undesirable as guest kernel -certainly won't be too happy about sudden change of the device's ROM -content. - -So the issue we're dealing with here is any unwanted memory related -device initialization upon cpr. - -For now the only thing that comes to my mind is to make a test where we -put as many devices as we can into a VM, make ram blocks RO upon cpr -(and remap them as RW later after migration is done, if needed), and -catch any unwanted memory violations. As Den suggested, we might -consider adding that behaviour as a separate non-default option (or -"migrate" command flag specific to cpr-transfer), which would only be -used in the testing. - -Andrey - -On 3/6/25 16:16, Andrey Drobyshev wrote: -On 3/5/25 11:19 PM, Steven Sistare wrote: -On 3/5/2025 11:50 AM, Andrey Drobyshev wrote: -On 3/4/25 9:05 PM, Steven Sistare wrote: -On 2/28/2025 1:37 PM, Andrey Drobyshev wrote: -On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -On 2/28/25 8:20 PM, Steven Sistare wrote: -On 2/28/2025 1:13 PM, Steven Sistare wrote: -On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -Hi all, - -We've been experimenting with cpr-transfer migration mode recently -and -have discovered the following issue with the guest QXL driver: - -Run migration source: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-src.sock - -$EMULATOR -enable-kvm \ -        -machine q35 \ -        -cpu host -smp 2 -m 2G \ -        -object memory-backend-file,id=ram0,size=2G,mem-path=/ -dev/shm/ -ram0,share=on\ -        -machine memory-backend=ram0 \ -        -machine aux-ram-share=on \ -        -drive file=$ROOTFS,media=disk,if=virtio \ -        -qmp unix:$QMPSOCK,server=on,wait=off \ -        -nographic \ -        -device qxl-vga -Run migration target: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-dst.sock -$EMULATOR -enable-kvm \ -        -machine q35 \ -        -cpu host -smp 2 -m 2G \ -        -object memory-backend-file,id=ram0,size=2G,mem-path=/ -dev/shm/ -ram0,share=on\ -        -machine memory-backend=ram0 \ -        -machine aux-ram-share=on \ -        -drive file=$ROOTFS,media=disk,if=virtio \ -        -qmp unix:$QMPSOCK,server=on,wait=off \ -        -nographic \ -        -device qxl-vga \ -        -incoming tcp:0:44444 \ -        -incoming '{"channel-type": "cpr", "addr": { "transport": -"socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -QMPSOCK=/var/run/alma8qmp-src.sock - -$QMPSHELL -p $QMPSOCK < /dev/tty3 -            done -done - -echo "bug could not be reproduced" -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce -that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the crash - -without the cpr-transfer migration the above reproduce doesn't -lead to -crash on the source VM. - -I suspect that, as cpr-transfer doesn't migrate the guest -memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM.  However, I wasn't able to trace the -corruption so far. - -Could somebody help the investigation and take a look into -this?  Any -suggestions would be appreciated.  Thanks! -Possibly some memory region created by qxl is not being preserved. -Try adding these traces to see what is preserved: - --trace enable='*cpr*' --trace enable='*ram_alloc*' -Also try adding this patch to see if it flags any ram blocks as not -compatible with cpr.  A message is printed at migration start time. -    -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send- -email- -steven.sistare@oracle.com/ - -- Steve -With the traces enabled + the "migration: ram block cpr blockers" -patch -applied: - -Source: -cpr_find_fd pc.bios, id 0 returns -1 -cpr_save_fd pc.bios, id 0, fd 22 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -0x7fec18e00000 -cpr_find_fd pc.rom, id 0 returns -1 -cpr_save_fd pc.rom, id 0, fd 23 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -0x7fec18c00000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 24 host 0x7fec18a00000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 25 host 0x7feb77e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 27 host 0x7fec18800000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 28 host 0x7feb73c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 34 host 0x7fec18600000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 35 host 0x7fec18200000 -cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 36 host 0x7feb8b600000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -37 host 0x7feb8b400000 - -cpr_state_save cpr-transfer mode -cpr_transfer_output /var/run/alma8cpr-dst.sock -Target: -cpr_transfer_input /var/run/alma8cpr-dst.sock -cpr_state_load cpr-transfer mode -cpr_find_fd pc.bios, id 0 returns 20 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -0x7fcdc9800000 -cpr_find_fd pc.rom, id 0 returns 19 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -0x7fcdc9600000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 18 host 0x7fcdc9400000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 17 host 0x7fcd27e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 16 host 0x7fcdc9200000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 15 host 0x7fcd23c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 14 host 0x7fcdc8800000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 13 host 0x7fcdc8400000 -cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 11 host 0x7fcdc8200000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -10 host 0x7fcd3be00000 -Looks like both vga.vram and qxl.vram are being preserved (with the -same -addresses), and no incompatible ram blocks are found during migration. -Sorry, addressed are not the same, of course.  However corresponding -ram -blocks do seem to be preserved and initialized. -So far, I have not reproduced the guest driver failure. - -However, I have isolated places where new QEMU improperly writes to -the qxl memory regions prior to starting the guest, by mmap'ing them -readonly after cpr: - -    qemu_ram_alloc_internal() -      if (reused && (strstr(name, "qxl") || strstr("name", "vga"))) -          ram_flags |= RAM_READONLY; -      new_block = qemu_ram_alloc_from_fd(...) - -I have attached a draft fix; try it and let me know. -My console window looks fine before and after cpr, using --vnc $hostip:0 -vga qxl - -- Steve -Regarding the reproduce: when I launch the buggy version with the same -options as you, i.e. "-vnc 0.0.0.0:$port -vga qxl", and do cpr-transfer, -my VNC client silently hangs on the target after a while.  Could it -happen on your stand as well? -cpr does not preserve the vnc connection and session.  To test, I specify -port 0 for the source VM and port 1 for the dest.  When the src vnc goes -dormant the dest vnc becomes active. -Sure, I meant that VNC on the dest (on the port 1) works for a while -after the migration and then hangs, apparently after the guest QXL crash. -Could you try launching VM with -"-nographic -device qxl-vga"?  That way VM's serial console is given you -directly in the shell, so when qxl driver crashes you're still able to -inspect the kernel messages. -I have been running like that, but have not reproduced the qxl driver -crash, -and I suspect my guest image+kernel is too old. -Yes, that's probably the case. But the crash occurs on my Fedora 41 -guest with the 6.11.5-300.fc41.x86_64 kernel, so newer kernels seem to -be buggy. -However, once I realized the -issue was post-cpr modification of qxl memory, I switched my attention -to the -fix. -As for your patch, I can report that it doesn't resolve the issue as it -is.  But I was able to track down another possible memory corruption -using your approach with readonly mmap'ing: -Program terminated with signal SIGSEGV, Segmentation fault. -#0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -412         d->ram->magic       = cpu_to_le32(QXL_RAM_MAGIC); -[Current thread is 1 (Thread 0x7f1a4f83b480 (LWP 229798))] -(gdb) bt -#0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -#1  0x0000563896e7f467 in qxl_realize_common (qxl=0x5638996e0e70, -errp=0x7ffd3c2b8170) at ../hw/display/qxl.c:2142 -#2  0x0000563896e7fda1 in qxl_realize_primary (dev=0x5638996e0e70, -errp=0x7ffd3c2b81d0) at ../hw/display/qxl.c:2257 -#3  0x0000563896c7e8f2 in pci_qdev_realize (qdev=0x5638996e0e70, -errp=0x7ffd3c2b8250) at ../hw/pci/pci.c:2174 -#4  0x00005638970eb54b in device_set_realized (obj=0x5638996e0e70, -value=true, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:494 -#5  0x00005638970f5e14 in property_set_bool (obj=0x5638996e0e70, -v=0x5638996f3770, name=0x56389759b141 "realized", -opaque=0x5638987893d0, errp=0x7ffd3c2b84e0) -      at ../qom/object.c:2374 -#6  0x00005638970f39f8 in object_property_set (obj=0x5638996e0e70, -name=0x56389759b141 "realized", v=0x5638996f3770, errp=0x7ffd3c2b84e0) -      at ../qom/object.c:1449 -#7  0x00005638970f8586 in object_property_set_qobject -(obj=0x5638996e0e70, name=0x56389759b141 "realized", -value=0x5638996df900, errp=0x7ffd3c2b84e0) -      at ../qom/qom-qobject.c:28 -#8  0x00005638970f3d8d in object_property_set_bool -(obj=0x5638996e0e70, name=0x56389759b141 "realized", value=true, -errp=0x7ffd3c2b84e0) -      at ../qom/object.c:1519 -#9  0x00005638970eacb0 in qdev_realize (dev=0x5638996e0e70, -bus=0x563898cf3c20, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:276 -#10 0x0000563896dba675 in qdev_device_add_from_qdict -(opts=0x5638996dfe50, from_json=false, errp=0x7ffd3c2b84e0) at ../ -system/qdev-monitor.c:714 -#11 0x0000563896dba721 in qdev_device_add (opts=0x563898786150, -errp=0x56389855dc40 ) at ../system/qdev-monitor.c:733 -#12 0x0000563896dc48f1 in device_init_func (opaque=0x0, -opts=0x563898786150, errp=0x56389855dc40 ) at ../system/ -vl.c:1207 -#13 0x000056389737a6cc in qemu_opts_foreach -      (list=0x563898427b60 , func=0x563896dc48ca -, opaque=0x0, errp=0x56389855dc40 ) -      at ../util/qemu-option.c:1135 -#14 0x0000563896dc89b5 in qemu_create_cli_devices () at ../system/ -vl.c:2745 -#15 0x0000563896dc8c00 in qmp_x_exit_preconfig (errp=0x56389855dc40 -) at ../system/vl.c:2806 -#16 0x0000563896dcb5de in qemu_init (argc=33, argv=0x7ffd3c2b8948) -at ../system/vl.c:3838 -#17 0x0000563897297323 in main (argc=33, argv=0x7ffd3c2b8948) at ../ -system/main.c:72 -So the attached adjusted version of your patch does seem to help.  At -least I can't reproduce the crash on my stand. -Thanks for the stack trace; the calls to SPICE_RING_INIT in init_qxl_ram -are -definitely harmful.  Try V2 of the patch, attached, which skips the lines -of init_qxl_ram that modify guest memory. -Thanks, your v2 patch does seem to prevent the crash. Would you re-send -it to the list as a proper fix? -I'm wondering, could it be useful to explicitly mark all the reused -memory regions readonly upon cpr-transfer, and then make them writable -back again after the migration is done?  That way we will be segfaulting -early on instead of debugging tricky memory corruptions. -It's a useful debugging technique, but changing protection on a large -memory region -can be too expensive for production due to TLB shootdowns. - -Also, there are cases where writes are performed but the value is -guaranteed to -be the same: -   qxl_post_load() -     qxl_set_mode() -       d->rom->mode = cpu_to_le32(modenr); -The value is the same because mode and shadow_rom.mode were passed in -vmstate -from old qemu. -There're also cases where devices' ROM might be re-initialized. E.g. -this segfault occures upon further exploration of RO mapped RAM blocks: -Program terminated with signal SIGSEGV, Segmentation fault. -#0 __memmove_avx_unaligned_erms () at -../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:664 -664 rep movsb -[Current thread is 1 (Thread 0x7f6e7d08b480 (LWP 310379))] -(gdb) bt -#0 __memmove_avx_unaligned_erms () at -../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:664 -#1 0x000055aa1d030ecd in rom_set_mr (rom=0x55aa200ba380, owner=0x55aa2019ac10, -name=0x7fffb8272bc0 "/rom@etc/acpi/tables", ro=true) - at ../hw/core/loader.c:1032 -#2 0x000055aa1d031577 in rom_add_blob - (name=0x55aa1da51f13 "etc/acpi/tables", blob=0x55aa208a1070, len=131072, max_len=2097152, -addr=18446744073709551615, fw_file_name=0x55aa1da51f13 "etc/acpi/tables", -fw_callback=0x55aa1d441f59 , callback_opaque=0x55aa20ff0010, as=0x0, -read_only=true) at ../hw/core/loader.c:1147 -#3 0x000055aa1cfd788d in acpi_add_rom_blob - (update=0x55aa1d441f59 , opaque=0x55aa20ff0010, -blob=0x55aa1fc9aa00, name=0x55aa1da51f13 "etc/acpi/tables") at ../hw/acpi/utils.c:46 -#4 0x000055aa1d44213f in acpi_setup () at ../hw/i386/acpi-build.c:2720 -#5 0x000055aa1d434199 in pc_machine_done (notifier=0x55aa1ff15050, data=0x0) -at ../hw/i386/pc.c:638 -#6 0x000055aa1d876845 in notifier_list_notify (list=0x55aa1ea25c10 -, data=0x0) at ../util/notify.c:39 -#7 0x000055aa1d039ee5 in qdev_machine_creation_done () at -../hw/core/machine.c:1749 -#8 0x000055aa1d2c7b3e in qemu_machine_creation_done (errp=0x55aa1ea5cc40 -) at ../system/vl.c:2779 -#9 0x000055aa1d2c7c7d in qmp_x_exit_preconfig (errp=0x55aa1ea5cc40 -) at ../system/vl.c:2807 -#10 0x000055aa1d2ca64f in qemu_init (argc=35, argv=0x7fffb82730e8) at -../system/vl.c:3838 -#11 0x000055aa1d79638c in main (argc=35, argv=0x7fffb82730e8) at -../system/main.c:72 -I'm not sure whether ACPI tables ROM in particular is rewritten with the -same content, but there might be cases where ROM can be read from file -system upon initialization. That is undesirable as guest kernel -certainly won't be too happy about sudden change of the device's ROM -content. - -So the issue we're dealing with here is any unwanted memory related -device initialization upon cpr. - -For now the only thing that comes to my mind is to make a test where we -put as many devices as we can into a VM, make ram blocks RO upon cpr -(and remap them as RW later after migration is done, if needed), and -catch any unwanted memory violations. As Den suggested, we might -consider adding that behaviour as a separate non-default option (or -"migrate" command flag specific to cpr-transfer), which would only be -used in the testing. - -Andrey -No way. ACPI with the source must be used in the same way as BIOSes -and optional ROMs. - -Den - -On 3/6/2025 10:52 AM, Denis V. Lunev wrote: -On 3/6/25 16:16, Andrey Drobyshev wrote: -On 3/5/25 11:19 PM, Steven Sistare wrote: -On 3/5/2025 11:50 AM, Andrey Drobyshev wrote: -On 3/4/25 9:05 PM, Steven Sistare wrote: -On 2/28/2025 1:37 PM, Andrey Drobyshev wrote: -On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -On 2/28/25 8:20 PM, Steven Sistare wrote: -On 2/28/2025 1:13 PM, Steven Sistare wrote: -On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -Hi all, - -We've been experimenting with cpr-transfer migration mode recently -and -have discovered the following issue with the guest QXL driver: - -Run migration source: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-src.sock - -$EMULATOR -enable-kvm \ -        -machine q35 \ -        -cpu host -smp 2 -m 2G \ -        -object memory-backend-file,id=ram0,size=2G,mem-path=/ -dev/shm/ -ram0,share=on\ -        -machine memory-backend=ram0 \ -        -machine aux-ram-share=on \ -        -drive file=$ROOTFS,media=disk,if=virtio \ -        -qmp unix:$QMPSOCK,server=on,wait=off \ -        -nographic \ -        -device qxl-vga -Run migration target: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-dst.sock -$EMULATOR -enable-kvm \ -        -machine q35 \ -        -cpu host -smp 2 -m 2G \ -        -object memory-backend-file,id=ram0,size=2G,mem-path=/ -dev/shm/ -ram0,share=on\ -        -machine memory-backend=ram0 \ -        -machine aux-ram-share=on \ -        -drive file=$ROOTFS,media=disk,if=virtio \ -        -qmp unix:$QMPSOCK,server=on,wait=off \ -        -nographic \ -        -device qxl-vga \ -        -incoming tcp:0:44444 \ -        -incoming '{"channel-type": "cpr", "addr": { "transport": -"socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -QMPSOCK=/var/run/alma8qmp-src.sock - -$QMPSHELL -p $QMPSOCK < /dev/tty3 -            done -done - -echo "bug could not be reproduced" -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce -that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the crash - -without the cpr-transfer migration the above reproduce doesn't -lead to -crash on the source VM. - -I suspect that, as cpr-transfer doesn't migrate the guest -memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM.  However, I wasn't able to trace the -corruption so far. - -Could somebody help the investigation and take a look into -this?  Any -suggestions would be appreciated.  Thanks! -Possibly some memory region created by qxl is not being preserved. -Try adding these traces to see what is preserved: - --trace enable='*cpr*' --trace enable='*ram_alloc*' -Also try adding this patch to see if it flags any ram blocks as not -compatible with cpr.  A message is printed at migration start time. -    -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send- -email- -steven.sistare@oracle.com/ - -- Steve -With the traces enabled + the "migration: ram block cpr blockers" -patch -applied: - -Source: -cpr_find_fd pc.bios, id 0 returns -1 -cpr_save_fd pc.bios, id 0, fd 22 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -0x7fec18e00000 -cpr_find_fd pc.rom, id 0 returns -1 -cpr_save_fd pc.rom, id 0, fd 23 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -0x7fec18c00000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 24 host 0x7fec18a00000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 25 host 0x7feb77e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 27 host 0x7fec18800000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 28 host 0x7feb73c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 34 host 0x7fec18600000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 35 host 0x7fec18200000 -cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 36 host 0x7feb8b600000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -37 host 0x7feb8b400000 - -cpr_state_save cpr-transfer mode -cpr_transfer_output /var/run/alma8cpr-dst.sock -Target: -cpr_transfer_input /var/run/alma8cpr-dst.sock -cpr_state_load cpr-transfer mode -cpr_find_fd pc.bios, id 0 returns 20 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -0x7fcdc9800000 -cpr_find_fd pc.rom, id 0 returns 19 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -0x7fcdc9600000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 18 host 0x7fcdc9400000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 17 host 0x7fcd27e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 16 host 0x7fcdc9200000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 15 host 0x7fcd23c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 14 host 0x7fcdc8800000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 13 host 0x7fcdc8400000 -cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 11 host 0x7fcdc8200000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -10 host 0x7fcd3be00000 -Looks like both vga.vram and qxl.vram are being preserved (with the -same -addresses), and no incompatible ram blocks are found during migration. -Sorry, addressed are not the same, of course.  However corresponding -ram -blocks do seem to be preserved and initialized. -So far, I have not reproduced the guest driver failure. - -However, I have isolated places where new QEMU improperly writes to -the qxl memory regions prior to starting the guest, by mmap'ing them -readonly after cpr: - -    qemu_ram_alloc_internal() -      if (reused && (strstr(name, "qxl") || strstr("name", "vga"))) -          ram_flags |= RAM_READONLY; -      new_block = qemu_ram_alloc_from_fd(...) - -I have attached a draft fix; try it and let me know. -My console window looks fine before and after cpr, using --vnc $hostip:0 -vga qxl - -- Steve -Regarding the reproduce: when I launch the buggy version with the same -options as you, i.e. "-vnc 0.0.0.0:$port -vga qxl", and do cpr-transfer, -my VNC client silently hangs on the target after a while.  Could it -happen on your stand as well? -cpr does not preserve the vnc connection and session.  To test, I specify -port 0 for the source VM and port 1 for the dest.  When the src vnc goes -dormant the dest vnc becomes active. -Sure, I meant that VNC on the dest (on the port 1) works for a while -after the migration and then hangs, apparently after the guest QXL crash. -Could you try launching VM with -"-nographic -device qxl-vga"?  That way VM's serial console is given you -directly in the shell, so when qxl driver crashes you're still able to -inspect the kernel messages. -I have been running like that, but have not reproduced the qxl driver -crash, -and I suspect my guest image+kernel is too old. -Yes, that's probably the case.  But the crash occurs on my Fedora 41 -guest with the 6.11.5-300.fc41.x86_64 kernel, so newer kernels seem to -be buggy. -However, once I realized the -issue was post-cpr modification of qxl memory, I switched my attention -to the -fix. -As for your patch, I can report that it doesn't resolve the issue as it -is.  But I was able to track down another possible memory corruption -using your approach with readonly mmap'ing: -Program terminated with signal SIGSEGV, Segmentation fault. -#0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -412         d->ram->magic       = cpu_to_le32(QXL_RAM_MAGIC); -[Current thread is 1 (Thread 0x7f1a4f83b480 (LWP 229798))] -(gdb) bt -#0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -#1  0x0000563896e7f467 in qxl_realize_common (qxl=0x5638996e0e70, -errp=0x7ffd3c2b8170) at ../hw/display/qxl.c:2142 -#2  0x0000563896e7fda1 in qxl_realize_primary (dev=0x5638996e0e70, -errp=0x7ffd3c2b81d0) at ../hw/display/qxl.c:2257 -#3  0x0000563896c7e8f2 in pci_qdev_realize (qdev=0x5638996e0e70, -errp=0x7ffd3c2b8250) at ../hw/pci/pci.c:2174 -#4  0x00005638970eb54b in device_set_realized (obj=0x5638996e0e70, -value=true, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:494 -#5  0x00005638970f5e14 in property_set_bool (obj=0x5638996e0e70, -v=0x5638996f3770, name=0x56389759b141 "realized", -opaque=0x5638987893d0, errp=0x7ffd3c2b84e0) -      at ../qom/object.c:2374 -#6  0x00005638970f39f8 in object_property_set (obj=0x5638996e0e70, -name=0x56389759b141 "realized", v=0x5638996f3770, errp=0x7ffd3c2b84e0) -      at ../qom/object.c:1449 -#7  0x00005638970f8586 in object_property_set_qobject -(obj=0x5638996e0e70, name=0x56389759b141 "realized", -value=0x5638996df900, errp=0x7ffd3c2b84e0) -      at ../qom/qom-qobject.c:28 -#8  0x00005638970f3d8d in object_property_set_bool -(obj=0x5638996e0e70, name=0x56389759b141 "realized", value=true, -errp=0x7ffd3c2b84e0) -      at ../qom/object.c:1519 -#9  0x00005638970eacb0 in qdev_realize (dev=0x5638996e0e70, -bus=0x563898cf3c20, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:276 -#10 0x0000563896dba675 in qdev_device_add_from_qdict -(opts=0x5638996dfe50, from_json=false, errp=0x7ffd3c2b84e0) at ../ -system/qdev-monitor.c:714 -#11 0x0000563896dba721 in qdev_device_add (opts=0x563898786150, -errp=0x56389855dc40 ) at ../system/qdev-monitor.c:733 -#12 0x0000563896dc48f1 in device_init_func (opaque=0x0, -opts=0x563898786150, errp=0x56389855dc40 ) at ../system/ -vl.c:1207 -#13 0x000056389737a6cc in qemu_opts_foreach -      (list=0x563898427b60 , func=0x563896dc48ca -, opaque=0x0, errp=0x56389855dc40 ) -      at ../util/qemu-option.c:1135 -#14 0x0000563896dc89b5 in qemu_create_cli_devices () at ../system/ -vl.c:2745 -#15 0x0000563896dc8c00 in qmp_x_exit_preconfig (errp=0x56389855dc40 -) at ../system/vl.c:2806 -#16 0x0000563896dcb5de in qemu_init (argc=33, argv=0x7ffd3c2b8948) -at ../system/vl.c:3838 -#17 0x0000563897297323 in main (argc=33, argv=0x7ffd3c2b8948) at ../ -system/main.c:72 -So the attached adjusted version of your patch does seem to help.  At -least I can't reproduce the crash on my stand. -Thanks for the stack trace; the calls to SPICE_RING_INIT in init_qxl_ram -are -definitely harmful.  Try V2 of the patch, attached, which skips the lines -of init_qxl_ram that modify guest memory. -Thanks, your v2 patch does seem to prevent the crash.  Would you re-send -it to the list as a proper fix? -Yes. Was waiting for your confirmation. -I'm wondering, could it be useful to explicitly mark all the reused -memory regions readonly upon cpr-transfer, and then make them writable -back again after the migration is done?  That way we will be segfaulting -early on instead of debugging tricky memory corruptions. -It's a useful debugging technique, but changing protection on a large -memory region -can be too expensive for production due to TLB shootdowns. - -Also, there are cases where writes are performed but the value is -guaranteed to -be the same: -   qxl_post_load() -     qxl_set_mode() -       d->rom->mode = cpu_to_le32(modenr); -The value is the same because mode and shadow_rom.mode were passed in -vmstate -from old qemu. -There're also cases where devices' ROM might be re-initialized.  E.g. -this segfault occures upon further exploration of RO mapped RAM blocks: -Program terminated with signal SIGSEGV, Segmentation fault. -#0  __memmove_avx_unaligned_erms () at -../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:664 -664             rep     movsb -[Current thread is 1 (Thread 0x7f6e7d08b480 (LWP 310379))] -(gdb) bt -#0  __memmove_avx_unaligned_erms () at -../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:664 -#1  0x000055aa1d030ecd in rom_set_mr (rom=0x55aa200ba380, owner=0x55aa2019ac10, -name=0x7fffb8272bc0 "/rom@etc/acpi/tables", ro=true) -     at ../hw/core/loader.c:1032 -#2  0x000055aa1d031577 in rom_add_blob -     (name=0x55aa1da51f13 "etc/acpi/tables", blob=0x55aa208a1070, len=131072, max_len=2097152, -addr=18446744073709551615, fw_file_name=0x55aa1da51f13 "etc/acpi/tables", -fw_callback=0x55aa1d441f59 , callback_opaque=0x55aa20ff0010, as=0x0, -read_only=true) at ../hw/core/loader.c:1147 -#3  0x000055aa1cfd788d in acpi_add_rom_blob -     (update=0x55aa1d441f59 , opaque=0x55aa20ff0010, -blob=0x55aa1fc9aa00, name=0x55aa1da51f13 "etc/acpi/tables") at ../hw/acpi/utils.c:46 -#4  0x000055aa1d44213f in acpi_setup () at ../hw/i386/acpi-build.c:2720 -#5  0x000055aa1d434199 in pc_machine_done (notifier=0x55aa1ff15050, data=0x0) -at ../hw/i386/pc.c:638 -#6  0x000055aa1d876845 in notifier_list_notify (list=0x55aa1ea25c10 -, data=0x0) at ../util/notify.c:39 -#7  0x000055aa1d039ee5 in qdev_machine_creation_done () at -../hw/core/machine.c:1749 -#8  0x000055aa1d2c7b3e in qemu_machine_creation_done (errp=0x55aa1ea5cc40 -) at ../system/vl.c:2779 -#9  0x000055aa1d2c7c7d in qmp_x_exit_preconfig (errp=0x55aa1ea5cc40 -) at ../system/vl.c:2807 -#10 0x000055aa1d2ca64f in qemu_init (argc=35, argv=0x7fffb82730e8) at -../system/vl.c:3838 -#11 0x000055aa1d79638c in main (argc=35, argv=0x7fffb82730e8) at -../system/main.c:72 -I'm not sure whether ACPI tables ROM in particular is rewritten with the -same content, but there might be cases where ROM can be read from file -system upon initialization.  That is undesirable as guest kernel -certainly won't be too happy about sudden change of the device's ROM -content. - -So the issue we're dealing with here is any unwanted memory related -device initialization upon cpr. - -For now the only thing that comes to my mind is to make a test where we -put as many devices as we can into a VM, make ram blocks RO upon cpr -(and remap them as RW later after migration is done, if needed), and -catch any unwanted memory violations.  As Den suggested, we might -consider adding that behaviour as a separate non-default option (or -"migrate" command flag specific to cpr-transfer), which would only be -used in the testing. -I'll look into adding an option, but there may be too many false positives, -such as the qxl_set_mode case above. And the maintainers may object to me -eliminating the false positives by adding more CPR_IN tests, due to gratuitous -(from their POV) ugliness. - -But I will use the technique to look for more write violations. -Andrey -No way. ACPI with the source must be used in the same way as BIOSes -and optional ROMs. -Yup, its a bug. Will fix. - -- Steve - -see -1741380954-341079-1-git-send-email-steven.sistare@oracle.com -/">https://lore.kernel.org/qemu-devel/ -1741380954-341079-1-git-send-email-steven.sistare@oracle.com -/ -- Steve - -On 3/6/2025 11:13 AM, Steven Sistare wrote: -On 3/6/2025 10:52 AM, Denis V. Lunev wrote: -On 3/6/25 16:16, Andrey Drobyshev wrote: -On 3/5/25 11:19 PM, Steven Sistare wrote: -On 3/5/2025 11:50 AM, Andrey Drobyshev wrote: -On 3/4/25 9:05 PM, Steven Sistare wrote: -On 2/28/2025 1:37 PM, Andrey Drobyshev wrote: -On 2/28/25 8:35 PM, Andrey Drobyshev wrote: -On 2/28/25 8:20 PM, Steven Sistare wrote: -On 2/28/2025 1:13 PM, Steven Sistare wrote: -On 2/28/2025 12:39 PM, Andrey Drobyshev wrote: -Hi all, - -We've been experimenting with cpr-transfer migration mode recently -and -have discovered the following issue with the guest QXL driver: - -Run migration source: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-src.sock - -$EMULATOR -enable-kvm \ -        -machine q35 \ -        -cpu host -smp 2 -m 2G \ -        -object memory-backend-file,id=ram0,size=2G,mem-path=/ -dev/shm/ -ram0,share=on\ -        -machine memory-backend=ram0 \ -        -machine aux-ram-share=on \ -        -drive file=$ROOTFS,media=disk,if=virtio \ -        -qmp unix:$QMPSOCK,server=on,wait=off \ -        -nographic \ -        -device qxl-vga -Run migration target: -EMULATOR=/path/to/emulator -ROOTFS=/path/to/image -QMPSOCK=/var/run/alma8qmp-dst.sock -$EMULATOR -enable-kvm \ -        -machine q35 \ -        -cpu host -smp 2 -m 2G \ -        -object memory-backend-file,id=ram0,size=2G,mem-path=/ -dev/shm/ -ram0,share=on\ -        -machine memory-backend=ram0 \ -        -machine aux-ram-share=on \ -        -drive file=$ROOTFS,media=disk,if=virtio \ -        -qmp unix:$QMPSOCK,server=on,wait=off \ -        -nographic \ -        -device qxl-vga \ -        -incoming tcp:0:44444 \ -        -incoming '{"channel-type": "cpr", "addr": { "transport": -"socket", "type": "unix", "path": "/var/run/alma8cpr-dst.sock"}}' -Launch the migration: -QMPSHELL=/root/src/qemu/master/scripts/qmp/qmp-shell -QMPSOCK=/var/run/alma8qmp-src.sock - -$QMPSHELL -p $QMPSOCK < /dev/tty3 -            done -done - -echo "bug could not be reproduced" -exit 0 -The bug itself seems to remain unfixed, as I was able to reproduce -that -with Fedora 41 guest, as well as AlmaLinux 8 guest. However our -cpr-transfer code also seems to be buggy as it triggers the crash - -without the cpr-transfer migration the above reproduce doesn't -lead to -crash on the source VM. - -I suspect that, as cpr-transfer doesn't migrate the guest -memory, but -rather passes it through the memory backend object, our code might -somehow corrupt the VRAM.  However, I wasn't able to trace the -corruption so far. - -Could somebody help the investigation and take a look into -this?  Any -suggestions would be appreciated.  Thanks! -Possibly some memory region created by qxl is not being preserved. -Try adding these traces to see what is preserved: - --trace enable='*cpr*' --trace enable='*ram_alloc*' -Also try adding this patch to see if it flags any ram blocks as not -compatible with cpr.  A message is printed at migration start time. -    -https://lore.kernel.org/qemu-devel/1740667681-257312-1-git-send- -email- -steven.sistare@oracle.com/ - -- Steve -With the traces enabled + the "migration: ram block cpr blockers" -patch -applied: - -Source: -cpr_find_fd pc.bios, id 0 returns -1 -cpr_save_fd pc.bios, id 0, fd 22 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 22 host -0x7fec18e00000 -cpr_find_fd pc.rom, id 0 returns -1 -cpr_save_fd pc.rom, id 0, fd 23 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 23 host -0x7fec18c00000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns -1 -cpr_save_fd 0000:00:01.0/e1000e.rom, id 0, fd 24 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 24 host 0x7fec18a00000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/vga.vram, id 0, fd 25 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 25 host 0x7feb77e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vrom, id 0, fd 27 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 27 host 0x7fec18800000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.vram, id 0, fd 28 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 28 host 0x7feb73c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns -1 -cpr_save_fd 0000:00:02.0/qxl.rom, id 0, fd 34 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 34 host 0x7fec18600000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/tables, id 0, fd 35 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 35 host 0x7fec18200000 -cpr_find_fd /rom@etc/table-loader, id 0 returns -1 -cpr_save_fd /rom@etc/table-loader, id 0, fd 36 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 36 host 0x7feb8b600000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns -1 -cpr_save_fd /rom@etc/acpi/rsdp, id 0, fd 37 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -37 host 0x7feb8b400000 - -cpr_state_save cpr-transfer mode -cpr_transfer_output /var/run/alma8cpr-dst.sock -Target: -cpr_transfer_input /var/run/alma8cpr-dst.sock -cpr_state_load cpr-transfer mode -cpr_find_fd pc.bios, id 0 returns 20 -qemu_ram_alloc_shared pc.bios size 262144 max_size 262144 fd 20 host -0x7fcdc9800000 -cpr_find_fd pc.rom, id 0 returns 19 -qemu_ram_alloc_shared pc.rom size 131072 max_size 131072 fd 19 host -0x7fcdc9600000 -cpr_find_fd 0000:00:01.0/e1000e.rom, id 0 returns 18 -qemu_ram_alloc_shared 0000:00:01.0/e1000e.rom size 262144 max_size -262144 fd 18 host 0x7fcdc9400000 -cpr_find_fd 0000:00:02.0/vga.vram, id 0 returns 17 -qemu_ram_alloc_shared 0000:00:02.0/vga.vram size 67108864 max_size -67108864 fd 17 host 0x7fcd27e00000 -cpr_find_fd 0000:00:02.0/qxl.vrom, id 0 returns 16 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vrom size 8192 max_size 8192 -fd 16 host 0x7fcdc9200000 -cpr_find_fd 0000:00:02.0/qxl.vram, id 0 returns 15 -qemu_ram_alloc_shared 0000:00:02.0/qxl.vram size 67108864 max_size -67108864 fd 15 host 0x7fcd23c00000 -cpr_find_fd 0000:00:02.0/qxl.rom, id 0 returns 14 -qemu_ram_alloc_shared 0000:00:02.0/qxl.rom size 65536 max_size 65536 -fd 14 host 0x7fcdc8800000 -cpr_find_fd /rom@etc/acpi/tables, id 0 returns 13 -qemu_ram_alloc_shared /rom@etc/acpi/tables size 131072 max_size -2097152 fd 13 host 0x7fcdc8400000 -cpr_find_fd /rom@etc/table-loader, id 0 returns 11 -qemu_ram_alloc_shared /rom@etc/table-loader size 4096 max_size 65536 -fd 11 host 0x7fcdc8200000 -cpr_find_fd /rom@etc/acpi/rsdp, id 0 returns 10 -qemu_ram_alloc_shared /rom@etc/acpi/rsdp size 4096 max_size 4096 fd -10 host 0x7fcd3be00000 -Looks like both vga.vram and qxl.vram are being preserved (with the -same -addresses), and no incompatible ram blocks are found during migration. -Sorry, addressed are not the same, of course.  However corresponding -ram -blocks do seem to be preserved and initialized. -So far, I have not reproduced the guest driver failure. - -However, I have isolated places where new QEMU improperly writes to -the qxl memory regions prior to starting the guest, by mmap'ing them -readonly after cpr: - -    qemu_ram_alloc_internal() -      if (reused && (strstr(name, "qxl") || strstr("name", "vga"))) -          ram_flags |= RAM_READONLY; -      new_block = qemu_ram_alloc_from_fd(...) - -I have attached a draft fix; try it and let me know. -My console window looks fine before and after cpr, using --vnc $hostip:0 -vga qxl - -- Steve -Regarding the reproduce: when I launch the buggy version with the same -options as you, i.e. "-vnc 0.0.0.0:$port -vga qxl", and do cpr-transfer, -my VNC client silently hangs on the target after a while.  Could it -happen on your stand as well? -cpr does not preserve the vnc connection and session.  To test, I specify -port 0 for the source VM and port 1 for the dest.  When the src vnc goes -dormant the dest vnc becomes active. -Sure, I meant that VNC on the dest (on the port 1) works for a while -after the migration and then hangs, apparently after the guest QXL crash. -Could you try launching VM with -"-nographic -device qxl-vga"?  That way VM's serial console is given you -directly in the shell, so when qxl driver crashes you're still able to -inspect the kernel messages. -I have been running like that, but have not reproduced the qxl driver -crash, -and I suspect my guest image+kernel is too old. -Yes, that's probably the case.  But the crash occurs on my Fedora 41 -guest with the 6.11.5-300.fc41.x86_64 kernel, so newer kernels seem to -be buggy. -However, once I realized the -issue was post-cpr modification of qxl memory, I switched my attention -to the -fix. -As for your patch, I can report that it doesn't resolve the issue as it -is.  But I was able to track down another possible memory corruption -using your approach with readonly mmap'ing: -Program terminated with signal SIGSEGV, Segmentation fault. -#0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -412         d->ram->magic       = cpu_to_le32(QXL_RAM_MAGIC); -[Current thread is 1 (Thread 0x7f1a4f83b480 (LWP 229798))] -(gdb) bt -#0  init_qxl_ram (d=0x5638996e0e70) at ../hw/display/qxl.c:412 -#1  0x0000563896e7f467 in qxl_realize_common (qxl=0x5638996e0e70, -errp=0x7ffd3c2b8170) at ../hw/display/qxl.c:2142 -#2  0x0000563896e7fda1 in qxl_realize_primary (dev=0x5638996e0e70, -errp=0x7ffd3c2b81d0) at ../hw/display/qxl.c:2257 -#3  0x0000563896c7e8f2 in pci_qdev_realize (qdev=0x5638996e0e70, -errp=0x7ffd3c2b8250) at ../hw/pci/pci.c:2174 -#4  0x00005638970eb54b in device_set_realized (obj=0x5638996e0e70, -value=true, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:494 -#5  0x00005638970f5e14 in property_set_bool (obj=0x5638996e0e70, -v=0x5638996f3770, name=0x56389759b141 "realized", -opaque=0x5638987893d0, errp=0x7ffd3c2b84e0) -      at ../qom/object.c:2374 -#6  0x00005638970f39f8 in object_property_set (obj=0x5638996e0e70, -name=0x56389759b141 "realized", v=0x5638996f3770, errp=0x7ffd3c2b84e0) -      at ../qom/object.c:1449 -#7  0x00005638970f8586 in object_property_set_qobject -(obj=0x5638996e0e70, name=0x56389759b141 "realized", -value=0x5638996df900, errp=0x7ffd3c2b84e0) -      at ../qom/qom-qobject.c:28 -#8  0x00005638970f3d8d in object_property_set_bool -(obj=0x5638996e0e70, name=0x56389759b141 "realized", value=true, -errp=0x7ffd3c2b84e0) -      at ../qom/object.c:1519 -#9  0x00005638970eacb0 in qdev_realize (dev=0x5638996e0e70, -bus=0x563898cf3c20, errp=0x7ffd3c2b84e0) at ../hw/core/qdev.c:276 -#10 0x0000563896dba675 in qdev_device_add_from_qdict -(opts=0x5638996dfe50, from_json=false, errp=0x7ffd3c2b84e0) at ../ -system/qdev-monitor.c:714 -#11 0x0000563896dba721 in qdev_device_add (opts=0x563898786150, -errp=0x56389855dc40 ) at ../system/qdev-monitor.c:733 -#12 0x0000563896dc48f1 in device_init_func (opaque=0x0, -opts=0x563898786150, errp=0x56389855dc40 ) at ../system/ -vl.c:1207 -#13 0x000056389737a6cc in qemu_opts_foreach -      (list=0x563898427b60 , func=0x563896dc48ca -, opaque=0x0, errp=0x56389855dc40 ) -      at ../util/qemu-option.c:1135 -#14 0x0000563896dc89b5 in qemu_create_cli_devices () at ../system/ -vl.c:2745 -#15 0x0000563896dc8c00 in qmp_x_exit_preconfig (errp=0x56389855dc40 -) at ../system/vl.c:2806 -#16 0x0000563896dcb5de in qemu_init (argc=33, argv=0x7ffd3c2b8948) -at ../system/vl.c:3838 -#17 0x0000563897297323 in main (argc=33, argv=0x7ffd3c2b8948) at ../ -system/main.c:72 -So the attached adjusted version of your patch does seem to help.  At -least I can't reproduce the crash on my stand. -Thanks for the stack trace; the calls to SPICE_RING_INIT in init_qxl_ram -are -definitely harmful.  Try V2 of the patch, attached, which skips the lines -of init_qxl_ram that modify guest memory. -Thanks, your v2 patch does seem to prevent the crash.  Would you re-send -it to the list as a proper fix? -Yes.  Was waiting for your confirmation. -I'm wondering, could it be useful to explicitly mark all the reused -memory regions readonly upon cpr-transfer, and then make them writable -back again after the migration is done?  That way we will be segfaulting -early on instead of debugging tricky memory corruptions. -It's a useful debugging technique, but changing protection on a large -memory region -can be too expensive for production due to TLB shootdowns. - -Also, there are cases where writes are performed but the value is -guaranteed to -be the same: -   qxl_post_load() -     qxl_set_mode() -       d->rom->mode = cpu_to_le32(modenr); -The value is the same because mode and shadow_rom.mode were passed in -vmstate -from old qemu. -There're also cases where devices' ROM might be re-initialized.  E.g. -this segfault occures upon further exploration of RO mapped RAM blocks: -Program terminated with signal SIGSEGV, Segmentation fault. -#0  __memmove_avx_unaligned_erms () at -../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:664 -664             rep     movsb -[Current thread is 1 (Thread 0x7f6e7d08b480 (LWP 310379))] -(gdb) bt -#0  __memmove_avx_unaligned_erms () at -../sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S:664 -#1  0x000055aa1d030ecd in rom_set_mr (rom=0x55aa200ba380, owner=0x55aa2019ac10, -name=0x7fffb8272bc0 "/rom@etc/acpi/tables", ro=true) -     at ../hw/core/loader.c:1032 -#2  0x000055aa1d031577 in rom_add_blob -     (name=0x55aa1da51f13 "etc/acpi/tables", blob=0x55aa208a1070, len=131072, max_len=2097152, -addr=18446744073709551615, fw_file_name=0x55aa1da51f13 "etc/acpi/tables", -fw_callback=0x55aa1d441f59 , callback_opaque=0x55aa20ff0010, as=0x0, -read_only=true) at ../hw/core/loader.c:1147 -#3  0x000055aa1cfd788d in acpi_add_rom_blob -     (update=0x55aa1d441f59 , opaque=0x55aa20ff0010, -blob=0x55aa1fc9aa00, name=0x55aa1da51f13 "etc/acpi/tables") at ../hw/acpi/utils.c:46 -#4  0x000055aa1d44213f in acpi_setup () at ../hw/i386/acpi-build.c:2720 -#5  0x000055aa1d434199 in pc_machine_done (notifier=0x55aa1ff15050, data=0x0) -at ../hw/i386/pc.c:638 -#6  0x000055aa1d876845 in notifier_list_notify (list=0x55aa1ea25c10 -, data=0x0) at ../util/notify.c:39 -#7  0x000055aa1d039ee5 in qdev_machine_creation_done () at -../hw/core/machine.c:1749 -#8  0x000055aa1d2c7b3e in qemu_machine_creation_done (errp=0x55aa1ea5cc40 -) at ../system/vl.c:2779 -#9  0x000055aa1d2c7c7d in qmp_x_exit_preconfig (errp=0x55aa1ea5cc40 -) at ../system/vl.c:2807 -#10 0x000055aa1d2ca64f in qemu_init (argc=35, argv=0x7fffb82730e8) at -../system/vl.c:3838 -#11 0x000055aa1d79638c in main (argc=35, argv=0x7fffb82730e8) at -../system/main.c:72 -I'm not sure whether ACPI tables ROM in particular is rewritten with the -same content, but there might be cases where ROM can be read from file -system upon initialization.  That is undesirable as guest kernel -certainly won't be too happy about sudden change of the device's ROM -content. - -So the issue we're dealing with here is any unwanted memory related -device initialization upon cpr. - -For now the only thing that comes to my mind is to make a test where we -put as many devices as we can into a VM, make ram blocks RO upon cpr -(and remap them as RW later after migration is done, if needed), and -catch any unwanted memory violations.  As Den suggested, we might -consider adding that behaviour as a separate non-default option (or -"migrate" command flag specific to cpr-transfer), which would only be -used in the testing. -I'll look into adding an option, but there may be too many false positives, -such as the qxl_set_mode case above.  And the maintainers may object to me -eliminating the false positives by adding more CPR_IN tests, due to gratuitous -(from their POV) ugliness. - -But I will use the technique to look for more write violations. -Andrey -No way. ACPI with the source must be used in the same way as BIOSes -and optional ROMs. -Yup, its a bug.  Will fix. - -- Steve - diff --git a/results/classifier/007/debug/53568181 b/results/classifier/007/debug/53568181 deleted file mode 100644 index 9bfb773a..00000000 --- a/results/classifier/007/debug/53568181 +++ /dev/null @@ -1,88 +0,0 @@ -debug: 0.968 -permissions: 0.965 -performance: 0.948 -semantic: 0.943 -graphic: 0.940 -PID: 0.938 -device: 0.936 -vnc: 0.935 -network: 0.925 -other: 0.921 -KVM: 0.917 -files: 0.890 -boot: 0.876 -socket: 0.875 - -[BUG] x86/PAT handling severely crippled AMD-V SVM KVM performance - -Hi, I maintain an out-of-tree 3D APIs pass-through QEMU device models at -https://github.com/kjliew/qemu-3dfx -that provide 3D acceleration for legacy -32-bit Windows guests (Win98SE, WinME, Win2k and WinXP) with the focus on -playing old legacy games from 1996-2003. It currently supports the now-defunct -3Dfx propriety API called Glide and an alternative OpenGL pass-through based on -MESA implementation. - -The basic concept of both implementations create memory-mapped virtual -interfaces consist of host/guest shared memory with guest-push model instead of -a more common host-pull model for typical QEMU device model implementation. -Guest uses shared memory as FIFOs for drawing commands and data to bulk up the -operations until serialization event that flushes the FIFOs into host. This -achieves extremely good performance since virtual CPUs are fast with hardware -acceleration (Intel VT/AMD-V) and reduces the overhead of frequent VMEXITs to -service the device emulation. Both implementations work on Windows 10 with WHPX -and HAXM accelerators as well as KVM in Linux. - -On Windows 10, QEMU WHPX implementation does not sync MSR_IA32_PAT during -host/guest states sync. There is no visibility into the closed-source WHPX on -how things are managed behind the scene, but from measuring performance figures -I can conclude that it didn't handle the MSR_IA32_PAT correctly for both Intel -and AMD. Call this fair enough, if you will, it didn't flag any concerns, in -fact games such as Quake2 and Quake3 were still within playable frame rate of -40~60FPS on Win2k/XP guest. Until the same games were run on Win98/ME guest and -the frame rate blew off the roof (300~500FPS) on the same CPU and GPU. In fact, -the later seemed to be more inlined with runnng the games bare-metal with vsync -off. - -On Linux (at the time of writing kernel 5.6.7/Mesa 20.0), the difference -prevailed. Intel CPUs (and it so happened that I was on laptop with Intel GPU), -the VMX-based kvm_intel got it right while SVM-based kvm_amd did not. -To put this in simple exaggeration, an aging Core i3-4010U/HD Graphics 4400 -(Haswell GT2) exhibited an insane performance in Quake2/Quake3 timedemos that -totally crushed more recent AMD Ryzen 2500U APU/Vega 8 Graphics and AMD -FX8300/NVIDIA GT730 on desktop. Simply unbelievable! - -It turned out that there was something to do with AMD-V NPT. By loading kvm_amd -with npt=0, AMD Ryzen APU and FX8300 regained a huge performance leap. However, -AMD NPT issue with KVM was supposedly fixed in 2017 kernel commits. NPT=0 would -actually incur performance loss for VM due to intervention required by -hypervisors to maintain the shadow page tables. Finally, I was able to find the -pointer that pointed to MSR_IA32_PAT register. By updating the MSR_IA32_PAT to -0x0606xxxx0606xxxxULL, AMD CPUs now regain their rightful performance without -taking the hit of NPT=0 for Linux KVM. Taking the same solution into Windows, -both Intel and AMD CPUs no longer require Win98/ME guest to unleash the full -performance potentials and performance figures based on games measured on WHPX -were not very far behind Linux KVM. - -So I guess the problem lies in host/guest shared memory regions mapped as -uncacheable from virtual CPU perspective. As virtual CPUs now completely execute -in hardware context with x86 hardware virtualiztion extensions, the cacheability -of memory types would severely impact the performance on guests. WHPX didn't -handle it for both Intel EPT and AMD NPT, but KVM seems to do it right for Intel -EPT. I don't have the correct fix for QEMU. But what I can do for my 3D APIs -pass-through device models is to implement host-side hooks to reprogram and -restore MSR_IA32_PAT upon activation/deactivation of the 3D APIs. Perhaps there -is also a better solution of having the proper kernel drivers for virtual -interfaces to manage the memory types of host/guest shared memory in kernel -space, but to do that and the needs of Microsoft tools/DDKs, I will just forget -it. The guest stubs uses the same kernel drivers included in 3Dfx drivers for -memory mapping and the virtual interfaces remain driver-less from Windows OS -perspective. Considering the current state of halting progress for QEMU native -virgil3D to support Windows OS, I am just being pragmatic. I understand that -QEMU virgil3D will eventually bring 3D acceleration for Windows guests, but I do -not expect anything to support legacy 32-bit Windows OSes which have out-grown -their commercial usefulness. - -Regards, -KJ Liew - diff --git a/results/classifier/007/debug/64571620 b/results/classifier/007/debug/64571620 deleted file mode 100644 index 1de1160e..00000000 --- a/results/classifier/007/debug/64571620 +++ /dev/null @@ -1,795 +0,0 @@ -debug: 0.927 -other: 0.922 -semantic: 0.903 -permissions: 0.902 -device: 0.899 -performance: 0.897 -graphic: 0.897 -PID: 0.887 -boot: 0.879 -KVM: 0.867 -files: 0.855 -socket: 0.855 -network: 0.853 -vnc: 0.819 - -[BUG] Migration hv_time rollback - -Hi, - -We are experiencing timestamp rollbacks during live-migration of -Windows 10 guests with the following qemu configuration (linux 5.4.46 -and qemu master): -``` -$ qemu-system-x86_64 -enable-kvm -cpu host,kvm=off,hv_time [...] -``` - -I have tracked the bug to the fact that `kvmclock` is not exposed and -disabled from qemu PoV but is in fact used by `hv-time` (in KVM). - -I think we should enable the `kvmclock` (qemu device) if `hv-time` is -present and add Hyper-V support for the `kvmclock_current_nsec` -function. - -I'm asking for advice because I am unsure this is the _right_ approach -and how to keep migration compatibility between qemu versions. - -Thank you all, - --- -Antoine 'xdbob' Damhet -signature.asc -Description: -PGP signature - -cc'ing in Vitaly who knows about the hv stuff. - -* Antoine Damhet (antoine.damhet@blade-group.com) wrote: -> -Hi, -> -> -We are experiencing timestamp rollbacks during live-migration of -> -Windows 10 guests with the following qemu configuration (linux 5.4.46 -> -and qemu master): -> -``` -> -$ qemu-system-x86_64 -enable-kvm -cpu host,kvm=off,hv_time [...] -> -``` -How big a jump are you seeing, and how did you notice it in the guest? - -Dave - -> -I have tracked the bug to the fact that `kvmclock` is not exposed and -> -disabled from qemu PoV but is in fact used by `hv-time` (in KVM). -> -> -I think we should enable the `kvmclock` (qemu device) if `hv-time` is -> -present and add Hyper-V support for the `kvmclock_current_nsec` -> -function. -> -> -I'm asking for advice because I am unsure this is the _right_ approach -> -and how to keep migration compatibility between qemu versions. -> -> -Thank you all, -> -> --- -> -Antoine 'xdbob' Damhet --- -Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK - -"Dr. David Alan Gilbert" writes: - -> -cc'ing in Vitaly who knows about the hv stuff. -> -cc'ing Marcelo who knows about clocksources :-) - -> -* Antoine Damhet (antoine.damhet@blade-group.com) wrote: -> -> Hi, -> -> -> -> We are experiencing timestamp rollbacks during live-migration of -> -> Windows 10 guests -Are you migrating to the same hardware (with the same TSC frequency)? Is -TSC used as the clocksource on the host? - -> -> with the following qemu configuration (linux 5.4.46 -> -> and qemu master): -> -> ``` -> -> $ qemu-system-x86_64 -enable-kvm -cpu host,kvm=off,hv_time [...] -> -> ``` -Out of pure curiosity, what's the purpose of doing 'kvm=off'? Windows is -not going to check for KVM identification anyway so we pretend we're -Hyper-V. - -Also, have you tried adding more Hyper-V enlightenments? - -> -> -How big a jump are you seeing, and how did you notice it in the guest? -> -> -Dave -> -> -> I have tracked the bug to the fact that `kvmclock` is not exposed and -> -> disabled from qemu PoV but is in fact used by `hv-time` (in KVM). -> -> -> -> I think we should enable the `kvmclock` (qemu device) if `hv-time` is -> -> present and add Hyper-V support for the `kvmclock_current_nsec` -> -> function. -AFAICT kvmclock_current_nsec() checks whether kvmclock was enabled by -the guest: - - if (!(env->system_time_msr & 1ULL)) { - /* KVM clock not active */ - return 0; - } - -and this is (and way) always false for Windows guests. - -> -> -> -> I'm asking for advice because I am unsure this is the _right_ approach -> -> and how to keep migration compatibility between qemu versions. -> -> -> -> Thank you all, -> -> -> -> -- -> -> Antoine 'xdbob' Damhet --- -Vitaly - -On Wed, Sep 16, 2020 at 01:59:43PM +0200, Vitaly Kuznetsov wrote: -> -"Dr. David Alan Gilbert" writes: -> -> -> cc'ing in Vitaly who knows about the hv stuff. -> -> -> -> -cc'ing Marcelo who knows about clocksources :-) -> -> -> * Antoine Damhet (antoine.damhet@blade-group.com) wrote: -> ->> Hi, -> ->> -> ->> We are experiencing timestamp rollbacks during live-migration of -> ->> Windows 10 guests -> -> -Are you migrating to the same hardware (with the same TSC frequency)? Is -> -TSC used as the clocksource on the host? -Yes we are migrating to the exact same hardware. And yes TSC is used as -a clocksource in the host (but the bug is still happening with `hpet` as -a clocksource). - -> -> ->> with the following qemu configuration (linux 5.4.46 -> ->> and qemu master): -> ->> ``` -> ->> $ qemu-system-x86_64 -enable-kvm -cpu host,kvm=off,hv_time [...] -> ->> ``` -> -> -Out of pure curiosity, what's the purpose of doing 'kvm=off'? Windows is -> -not going to check for KVM identification anyway so we pretend we're -> -Hyper-V. -Some softwares explicitly checks for the presence of KVM and then crash -if they find it in CPUID :/ - -> -> -Also, have you tried adding more Hyper-V enlightenments? -Yes, I published a stripped-down command-line for a minimal reproducer -but even `hv-frequencies` and `hv-reenlightenment` don't help. - -> -> -> -> -> How big a jump are you seeing, and how did you notice it in the guest? -> -> -> -> Dave -> -> -> ->> I have tracked the bug to the fact that `kvmclock` is not exposed and -> ->> disabled from qemu PoV but is in fact used by `hv-time` (in KVM). -> ->> -> ->> I think we should enable the `kvmclock` (qemu device) if `hv-time` is -> ->> present and add Hyper-V support for the `kvmclock_current_nsec` -> ->> function. -> -> -AFAICT kvmclock_current_nsec() checks whether kvmclock was enabled by -> -the guest: -> -> -if (!(env->system_time_msr & 1ULL)) { -> -/* KVM clock not active */ -> -return 0; -> -} -> -> -and this is (and way) always false for Windows guests. -Hooo, I missed this piece. When is `clock_is_reliable` expected to be -false ? Because if it is I still think we should be able to query at -least `HV_X64_MSR_REFERENCE_TSC` - -> -> ->> -> ->> I'm asking for advice because I am unsure this is the _right_ approach -> ->> and how to keep migration compatibility between qemu versions. -> ->> -> ->> Thank you all, -> ->> -> ->> -- -> ->> Antoine 'xdbob' Damhet -> -> --- -> -Vitaly -> --- -Antoine 'xdbob' Damhet -signature.asc -Description: -PGP signature - -On Wed, Sep 16, 2020 at 12:29:56PM +0100, Dr. David Alan Gilbert wrote: -> -cc'ing in Vitaly who knows about the hv stuff. -Thanks - -> -> -* Antoine Damhet (antoine.damhet@blade-group.com) wrote: -> -> Hi, -> -> -> -> We are experiencing timestamp rollbacks during live-migration of -> -> Windows 10 guests with the following qemu configuration (linux 5.4.46 -> -> and qemu master): -> -> ``` -> -> $ qemu-system-x86_64 -enable-kvm -cpu host,kvm=off,hv_time [...] -> -> ``` -> -> -How big a jump are you seeing, and how did you notice it in the guest? -I'm seeing jumps of about the guest uptime (indicating a reset of the -counter). It's expected because we won't call `KVM_SET_CLOCK` to -restore any value. - -We first noticed it because after some migrations `dwm.exe` crashes with -the "(NTSTATUS) 0x8898009b - QueryPerformanceCounter returned a time in -the past." error code. - -I can also confirm the following hack makes the behavior disappear: - -``` -diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c -index 64283358f9..f334bdf35f 100644 ---- a/hw/i386/kvm/clock.c -+++ b/hw/i386/kvm/clock.c -@@ -332,11 +332,7 @@ void kvmclock_create(void) - { - X86CPU *cpu = X86_CPU(first_cpu); - -- if (kvm_enabled() && -- cpu->env.features[FEAT_KVM] & ((1ULL << KVM_FEATURE_CLOCKSOURCE) | -- (1ULL << KVM_FEATURE_CLOCKSOURCE2))) { -- sysbus_create_simple(TYPE_KVM_CLOCK, -1, NULL); -- } -+ sysbus_create_simple(TYPE_KVM_CLOCK, -1, NULL); - } - - static void kvmclock_register_types(void) -diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c -index 32b1453e6a..11d980ba85 100644 ---- a/hw/i386/pc_piix.c -+++ b/hw/i386/pc_piix.c -@@ -158,9 +158,7 @@ static void pc_init1(MachineState *machine, - - x86_cpus_init(x86ms, pcmc->default_cpu_version); - -- if (kvm_enabled() && pcmc->kvmclock_enabled) { -- kvmclock_create(); -- } -+ kvmclock_create(); - - if (pcmc->pci_enabled) { - pci_memory = g_new(MemoryRegion, 1); -``` - -> -> -Dave -> -> -> I have tracked the bug to the fact that `kvmclock` is not exposed and -> -> disabled from qemu PoV but is in fact used by `hv-time` (in KVM). -> -> -> -> I think we should enable the `kvmclock` (qemu device) if `hv-time` is -> -> present and add Hyper-V support for the `kvmclock_current_nsec` -> -> function. -> -> -> -> I'm asking for advice because I am unsure this is the _right_ approach -> -> and how to keep migration compatibility between qemu versions. -> -> -> -> Thank you all, -> -> -> -> -- -> -> Antoine 'xdbob' Damhet -> -> -> --- -> -Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK -> --- -Antoine 'xdbob' Damhet -signature.asc -Description: -PGP signature - -Antoine Damhet writes: - -> -On Wed, Sep 16, 2020 at 12:29:56PM +0100, Dr. David Alan Gilbert wrote: -> -> cc'ing in Vitaly who knows about the hv stuff. -> -> -Thanks -> -> -> -> -> * Antoine Damhet (antoine.damhet@blade-group.com) wrote: -> -> > Hi, -> -> > -> -> > We are experiencing timestamp rollbacks during live-migration of -> -> > Windows 10 guests with the following qemu configuration (linux 5.4.46 -> -> > and qemu master): -> -> > ``` -> -> > $ qemu-system-x86_64 -enable-kvm -cpu host,kvm=off,hv_time [...] -> -> > ``` -> -> -> -> How big a jump are you seeing, and how did you notice it in the guest? -> -> -I'm seeing jumps of about the guest uptime (indicating a reset of the -> -counter). It's expected because we won't call `KVM_SET_CLOCK` to -> -restore any value. -> -> -We first noticed it because after some migrations `dwm.exe` crashes with -> -the "(NTSTATUS) 0x8898009b - QueryPerformanceCounter returned a time in -> -the past." error code. -> -> -I can also confirm the following hack makes the behavior disappear: -> -> -``` -> -diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c -> -index 64283358f9..f334bdf35f 100644 -> ---- a/hw/i386/kvm/clock.c -> -+++ b/hw/i386/kvm/clock.c -> -@@ -332,11 +332,7 @@ void kvmclock_create(void) -> -{ -> -X86CPU *cpu = X86_CPU(first_cpu); -> -> -- if (kvm_enabled() && -> -- cpu->env.features[FEAT_KVM] & ((1ULL << KVM_FEATURE_CLOCKSOURCE) | -> -- (1ULL << KVM_FEATURE_CLOCKSOURCE2))) { -> -- sysbus_create_simple(TYPE_KVM_CLOCK, -1, NULL); -> -- } -> -+ sysbus_create_simple(TYPE_KVM_CLOCK, -1, NULL); -> -} -> -Oh, I think I see what's going on. When you add 'kvm=off' -cpu->env.features[FEAT_KVM] is reset (see x86_cpu_expand_features()) so -kvmclock QEMU device is not created and nobody calls KVM_SET_CLOCK on -migration. - -In case we really want to support 'kvm=off' I think we can add Hyper-V -features check here along with KVM, this should do the job. - --- -Vitaly - -Vitaly Kuznetsov writes: - -> -Antoine Damhet writes: -> -> -> On Wed, Sep 16, 2020 at 12:29:56PM +0100, Dr. David Alan Gilbert wrote: -> ->> cc'ing in Vitaly who knows about the hv stuff. -> -> -> -> Thanks -> -> -> ->> -> ->> * Antoine Damhet (antoine.damhet@blade-group.com) wrote: -> ->> > Hi, -> ->> > -> ->> > We are experiencing timestamp rollbacks during live-migration of -> ->> > Windows 10 guests with the following qemu configuration (linux 5.4.46 -> ->> > and qemu master): -> ->> > ``` -> ->> > $ qemu-system-x86_64 -enable-kvm -cpu host,kvm=off,hv_time [...] -> ->> > ``` -> ->> -> ->> How big a jump are you seeing, and how did you notice it in the guest? -> -> -> -> I'm seeing jumps of about the guest uptime (indicating a reset of the -> -> counter). It's expected because we won't call `KVM_SET_CLOCK` to -> -> restore any value. -> -> -> -> We first noticed it because after some migrations `dwm.exe` crashes with -> -> the "(NTSTATUS) 0x8898009b - QueryPerformanceCounter returned a time in -> -> the past." error code. -> -> -> -> I can also confirm the following hack makes the behavior disappear: -> -> -> -> ``` -> -> diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c -> -> index 64283358f9..f334bdf35f 100644 -> -> --- a/hw/i386/kvm/clock.c -> -> +++ b/hw/i386/kvm/clock.c -> -> @@ -332,11 +332,7 @@ void kvmclock_create(void) -> -> { -> -> X86CPU *cpu = X86_CPU(first_cpu); -> -> -> -> - if (kvm_enabled() && -> -> - cpu->env.features[FEAT_KVM] & ((1ULL << KVM_FEATURE_CLOCKSOURCE) | -> -> - (1ULL << KVM_FEATURE_CLOCKSOURCE2))) -> -> { -> -> - sysbus_create_simple(TYPE_KVM_CLOCK, -1, NULL); -> -> - } -> -> + sysbus_create_simple(TYPE_KVM_CLOCK, -1, NULL); -> -> } -> -> -> -> -> -Oh, I think I see what's going on. When you add 'kvm=off' -> -cpu->env.features[FEAT_KVM] is reset (see x86_cpu_expand_features()) so -> -kvmclock QEMU device is not created and nobody calls KVM_SET_CLOCK on -> -migration. -> -> -In case we really want to support 'kvm=off' I think we can add Hyper-V -> -features check here along with KVM, this should do the job. -Does the untested - -diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c -index 64283358f91d..e03b2ca6d8f6 100644 ---- a/hw/i386/kvm/clock.c -+++ b/hw/i386/kvm/clock.c -@@ -333,8 +333,9 @@ void kvmclock_create(void) - X86CPU *cpu = X86_CPU(first_cpu); - - if (kvm_enabled() && -- cpu->env.features[FEAT_KVM] & ((1ULL << KVM_FEATURE_CLOCKSOURCE) | -- (1ULL << KVM_FEATURE_CLOCKSOURCE2))) { -+ ((cpu->env.features[FEAT_KVM] & ((1ULL << KVM_FEATURE_CLOCKSOURCE) | -+ (1ULL << KVM_FEATURE_CLOCKSOURCE2))) -|| -+ (cpu->env.features[FEAT_HYPERV_EAX] & HV_TIME_REF_COUNT_AVAILABLE))) { - sysbus_create_simple(TYPE_KVM_CLOCK, -1, NULL); - } - } - -help? - -(I don't think we need to remove all 'if (kvm_enabled())' checks from -machine types as 'kvm=off' should not be related). - --- -Vitaly - -On Wed, Sep 16, 2020 at 02:50:56PM +0200, Vitaly Kuznetsov wrote: -[...] - -> ->> -> -> -> -> -> -> Oh, I think I see what's going on. When you add 'kvm=off' -> -> cpu->env.features[FEAT_KVM] is reset (see x86_cpu_expand_features()) so -> -> kvmclock QEMU device is not created and nobody calls KVM_SET_CLOCK on -> -> migration. -> -> -> -> In case we really want to support 'kvm=off' I think we can add Hyper-V -> -> features check here along with KVM, this should do the job. -> -> -Does the untested -> -> -diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c -> -index 64283358f91d..e03b2ca6d8f6 100644 -> ---- a/hw/i386/kvm/clock.c -> -+++ b/hw/i386/kvm/clock.c -> -@@ -333,8 +333,9 @@ void kvmclock_create(void) -> -X86CPU *cpu = X86_CPU(first_cpu); -> -> -if (kvm_enabled() && -> -- cpu->env.features[FEAT_KVM] & ((1ULL << KVM_FEATURE_CLOCKSOURCE) | -> -- (1ULL << KVM_FEATURE_CLOCKSOURCE2))) { -> -+ ((cpu->env.features[FEAT_KVM] & ((1ULL << KVM_FEATURE_CLOCKSOURCE) | -> -+ (1ULL << -> -KVM_FEATURE_CLOCKSOURCE2))) || -> -+ (cpu->env.features[FEAT_HYPERV_EAX] & -> -HV_TIME_REF_COUNT_AVAILABLE))) { -> -sysbus_create_simple(TYPE_KVM_CLOCK, -1, NULL); -> -} -> -} -> -> -help? -It appears to work :) - -> -> -(I don't think we need to remove all 'if (kvm_enabled())' checks from -> -machine types as 'kvm=off' should not be related). -Indeed (I didn't look at the macro, it was just quick & dirty). - -> -> --- -> -Vitaly -> -> --- -Antoine 'xdbob' Damhet -signature.asc -Description: -PGP signature - -On 16/09/20 13:29, Dr. David Alan Gilbert wrote: -> -> I have tracked the bug to the fact that `kvmclock` is not exposed and -> -> disabled from qemu PoV but is in fact used by `hv-time` (in KVM). -> -> -> -> I think we should enable the `kvmclock` (qemu device) if `hv-time` is -> -> present and add Hyper-V support for the `kvmclock_current_nsec` -> -> function. -Yes, this seems correct. I would have to check but it may even be -better to _always_ send kvmclock data in the live migration stream. - -Paolo - -Paolo Bonzini writes: - -> -On 16/09/20 13:29, Dr. David Alan Gilbert wrote: -> ->> I have tracked the bug to the fact that `kvmclock` is not exposed and -> ->> disabled from qemu PoV but is in fact used by `hv-time` (in KVM). -> ->> -> ->> I think we should enable the `kvmclock` (qemu device) if `hv-time` is -> ->> present and add Hyper-V support for the `kvmclock_current_nsec` -> ->> function. -> -> -Yes, this seems correct. I would have to check but it may even be -> -better to _always_ send kvmclock data in the live migration stream. -> -The question I have is: with 'kvm=off', do we actually restore TSC -reading on migration? (and I guess the answer is 'no' or Hyper-V TSC -page would 'just work' I guess). So yea, maybe dropping the -'cpu->env.features[FEAT_KVM]' check is the right fix. - --- -Vitaly - diff --git a/results/classifier/007/debug/96782458 b/results/classifier/007/debug/96782458 deleted file mode 100644 index 6fa03cc3..00000000 --- a/results/classifier/007/debug/96782458 +++ /dev/null @@ -1,1009 +0,0 @@ -debug: 0.989 -permissions: 0.986 -performance: 0.985 -semantic: 0.984 -other: 0.982 -boot: 0.980 -PID: 0.980 -files: 0.978 -socket: 0.976 -vnc: 0.976 -device: 0.974 -graphic: 0.973 -network: 0.967 -KVM: 0.963 - -[Qemu-devel] [BUG] Migrate failes between boards with different PMC counts - -Hi all, - -Recently, I found migration failed when enable vPMU. - -migrate vPMU state was introduced in linux-3.10 + qemu-1.7. - -As long as enable vPMU, qemu will save / load the -vmstate_msr_architectural_pmu(msr_global_ctrl) register during the migration. -But global_ctrl generated based on cpuid(0xA), the number of general-purpose -performance -monitoring counters(PMC) can vary according to Intel SDN. The number of PMC -presented -to vm, does not support configuration currently, it depend on host cpuid, and -enable all pmc -defaultly at KVM. It cause migration to fail between boards with different PMC -counts. - -The return value of cpuid (0xA) is different dur to cpu, according to Intel -SDN，18-10 Vol. 3B: - -Note: The number of general-purpose performance monitoring counters (i.e. N in -Figure 18-9) -can vary across processor generations within a processor family, across -processor families, or -could be different depending on the configuration chosen at boot time in the -BIOS regarding -Intel Hyper Threading Technology, (e.g. N=2 for 45 nm Intel Atom processors; N -=4 for processors -based on the Nehalem microarchitecture; for processors based on the Sandy Bridge -microarchitecture, N = 4 if Intel Hyper Threading Technology is active and N=8 -if not active). - -Also I found, N=8 if HT is not active based on the broadwell，, -such as CPU E7-8890 v4 @ 2.20GHz - -# ./x86_64-softmmu/qemu-system-x86_64 --enable-kvm -smp 4 -m 4096 -hda -/data/zyy/test_qemu.img.sles12sp1 -vnc :99 -cpu kvm64,pmu=true -incoming -tcp::8888 -Completed 100 % -qemu-system-x86_64: error: failed to set MSR 0x38f to 0x7000000ff -qemu-system-x86_64: /data/zyy/git/test/qemu/target/i386/kvm.c:1833: -kvm_put_msrs: -Assertion `ret == cpu->kvm_msr_buf->nmsrs' failed. -Aborted - -So make number of pmc configurable to vm ? Any better idea ? - - -Regards, --Zhuang Yanying - -* Zhuangyanying (address@hidden) wrote: -> -Hi all, -> -> -Recently, I found migration failed when enable vPMU. -> -> -migrate vPMU state was introduced in linux-3.10 + qemu-1.7. -> -> -As long as enable vPMU, qemu will save / load the -> -vmstate_msr_architectural_pmu(msr_global_ctrl) register during the migration. -> -But global_ctrl generated based on cpuid(0xA), the number of general-purpose -> -performance -> -monitoring counters(PMC) can vary according to Intel SDN. The number of PMC -> -presented -> -to vm, does not support configuration currently, it depend on host cpuid, and -> -enable all pmc -> -defaultly at KVM. It cause migration to fail between boards with different -> -PMC counts. -> -> -The return value of cpuid (0xA) is different dur to cpu, according to Intel -> -SDN，18-10 Vol. 3B: -> -> -Note: The number of general-purpose performance monitoring counters (i.e. N -> -in Figure 18-9) -> -can vary across processor generations within a processor family, across -> -processor families, or -> -could be different depending on the configuration chosen at boot time in the -> -BIOS regarding -> -Intel Hyper Threading Technology, (e.g. N=2 for 45 nm Intel Atom processors; -> -N =4 for processors -> -based on the Nehalem microarchitecture; for processors based on the Sandy -> -Bridge -> -microarchitecture, N = 4 if Intel Hyper Threading Technology is active and -> -N=8 if not active). -> -> -Also I found, N=8 if HT is not active based on the broadwell，, -> -such as CPU E7-8890 v4 @ 2.20GHz -> -> -# ./x86_64-softmmu/qemu-system-x86_64 --enable-kvm -smp 4 -m 4096 -hda -> -/data/zyy/test_qemu.img.sles12sp1 -vnc :99 -cpu kvm64,pmu=true -incoming -> -tcp::8888 -> -Completed 100 % -> -qemu-system-x86_64: error: failed to set MSR 0x38f to 0x7000000ff -> -qemu-system-x86_64: /data/zyy/git/test/qemu/target/i386/kvm.c:1833: -> -kvm_put_msrs: -> -Assertion `ret == cpu->kvm_msr_buf->nmsrs' failed. -> -Aborted -> -> -So make number of pmc configurable to vm ? Any better idea ? -Coincidentally we hit a similar problem a few days ago with -cpu host - it -took me -quite a while to spot the difference between the machines was the source -had hyperthreading disabled. - -An option to set the number of counters makes sense to me; but I wonder -how many other options we need as well. Also, I'm not sure there's any -easy way for libvirt etc to figure out how many counters a host supports - it's -not in /proc/cpuinfo. - -Dave - -> -> -Regards, -> --Zhuang Yanying --- -Dr. David Alan Gilbert / address@hidden / Manchester, UK - -On Mon, Apr 24, 2017 at 10:23:21AM +0100, Dr. David Alan Gilbert wrote: -> -* Zhuangyanying (address@hidden) wrote: -> -> Hi all, -> -> -> -> Recently, I found migration failed when enable vPMU. -> -> -> -> migrate vPMU state was introduced in linux-3.10 + qemu-1.7. -> -> -> -> As long as enable vPMU, qemu will save / load the -> -> vmstate_msr_architectural_pmu(msr_global_ctrl) register during the -> -> migration. -> -> But global_ctrl generated based on cpuid(0xA), the number of -> -> general-purpose performance -> -> monitoring counters(PMC) can vary according to Intel SDN. The number of PMC -> -> presented -> -> to vm, does not support configuration currently, it depend on host cpuid, -> -> and enable all pmc -> -> defaultly at KVM. It cause migration to fail between boards with different -> -> PMC counts. -> -> -> -> The return value of cpuid (0xA) is different dur to cpu, according to Intel -> -> SDN，18-10 Vol. 3B: -> -> -> -> Note: The number of general-purpose performance monitoring counters (i.e. N -> -> in Figure 18-9) -> -> can vary across processor generations within a processor family, across -> -> processor families, or -> -> could be different depending on the configuration chosen at boot time in -> -> the BIOS regarding -> -> Intel Hyper Threading Technology, (e.g. N=2 for 45 nm Intel Atom -> -> processors; N =4 for processors -> -> based on the Nehalem microarchitecture; for processors based on the Sandy -> -> Bridge -> -> microarchitecture, N = 4 if Intel Hyper Threading Technology is active and -> -> N=8 if not active). -> -> -> -> Also I found, N=8 if HT is not active based on the broadwell，, -> -> such as CPU E7-8890 v4 @ 2.20GHz -> -> -> -> # ./x86_64-softmmu/qemu-system-x86_64 --enable-kvm -smp 4 -m 4096 -hda -> -> /data/zyy/test_qemu.img.sles12sp1 -vnc :99 -cpu kvm64,pmu=true -incoming -> -> tcp::8888 -> -> Completed 100 % -> -> qemu-system-x86_64: error: failed to set MSR 0x38f to 0x7000000ff -> -> qemu-system-x86_64: /data/zyy/git/test/qemu/target/i386/kvm.c:1833: -> -> kvm_put_msrs: -> -> Assertion `ret == cpu->kvm_msr_buf->nmsrs' failed. -> -> Aborted -> -> -> -> So make number of pmc configurable to vm ? Any better idea ? -> -> -Coincidentally we hit a similar problem a few days ago with -cpu host - it -> -took me -> -quite a while to spot the difference between the machines was the source -> -had hyperthreading disabled. -> -> -An option to set the number of counters makes sense to me; but I wonder -> -how many other options we need as well. Also, I'm not sure there's any -> -easy way for libvirt etc to figure out how many counters a host supports - -> -it's not in /proc/cpuinfo. -We actually try to avoid /proc/cpuinfo whereever possible. We do direct -CPUID asm instructions to identify features, and prefer to use -/sys/devices/system/cpu if that has suitable data - -Where do the PMC counts come from originally ? CPUID or something else ? - -Regards, -Daniel --- -|: -https://berrange.com --o- -https://www.flickr.com/photos/dberrange -:| -|: -https://libvirt.org --o- -https://fstop138.berrange.com -:| -|: -https://entangle-photo.org --o- -https://www.instagram.com/dberrange -:| - -* Daniel P. Berrange (address@hidden) wrote: -> -On Mon, Apr 24, 2017 at 10:23:21AM +0100, Dr. David Alan Gilbert wrote: -> -> * Zhuangyanying (address@hidden) wrote: -> -> > Hi all, -> -> > -> -> > Recently, I found migration failed when enable vPMU. -> -> > -> -> > migrate vPMU state was introduced in linux-3.10 + qemu-1.7. -> -> > -> -> > As long as enable vPMU, qemu will save / load the -> -> > vmstate_msr_architectural_pmu(msr_global_ctrl) register during the -> -> > migration. -> -> > But global_ctrl generated based on cpuid(0xA), the number of -> -> > general-purpose performance -> -> > monitoring counters(PMC) can vary according to Intel SDN. The number of -> -> > PMC presented -> -> > to vm, does not support configuration currently, it depend on host cpuid, -> -> > and enable all pmc -> -> > defaultly at KVM. It cause migration to fail between boards with -> -> > different PMC counts. -> -> > -> -> > The return value of cpuid (0xA) is different dur to cpu, according to -> -> > Intel SDN，18-10 Vol. 3B: -> -> > -> -> > Note: The number of general-purpose performance monitoring counters (i.e. -> -> > N in Figure 18-9) -> -> > can vary across processor generations within a processor family, across -> -> > processor families, or -> -> > could be different depending on the configuration chosen at boot time in -> -> > the BIOS regarding -> -> > Intel Hyper Threading Technology, (e.g. N=2 for 45 nm Intel Atom -> -> > processors; N =4 for processors -> -> > based on the Nehalem microarchitecture; for processors based on the Sandy -> -> > Bridge -> -> > microarchitecture, N = 4 if Intel Hyper Threading Technology is active -> -> > and N=8 if not active). -> -> > -> -> > Also I found, N=8 if HT is not active based on the broadwell，, -> -> > such as CPU E7-8890 v4 @ 2.20GHz -> -> > -> -> > # ./x86_64-softmmu/qemu-system-x86_64 --enable-kvm -smp 4 -m 4096 -hda -> -> > /data/zyy/test_qemu.img.sles12sp1 -vnc :99 -cpu kvm64,pmu=true -incoming -> -> > tcp::8888 -> -> > Completed 100 % -> -> > qemu-system-x86_64: error: failed to set MSR 0x38f to 0x7000000ff -> -> > qemu-system-x86_64: /data/zyy/git/test/qemu/target/i386/kvm.c:1833: -> -> > kvm_put_msrs: -> -> > Assertion `ret == cpu->kvm_msr_buf->nmsrs' failed. -> -> > Aborted -> -> > -> -> > So make number of pmc configurable to vm ? Any better idea ? -> -> -> -> Coincidentally we hit a similar problem a few days ago with -cpu host - it -> -> took me -> -> quite a while to spot the difference between the machines was the source -> -> had hyperthreading disabled. -> -> -> -> An option to set the number of counters makes sense to me; but I wonder -> -> how many other options we need as well. Also, I'm not sure there's any -> -> easy way for libvirt etc to figure out how many counters a host supports - -> -> it's not in /proc/cpuinfo. -> -> -We actually try to avoid /proc/cpuinfo whereever possible. We do direct -> -CPUID asm instructions to identify features, and prefer to use -> -/sys/devices/system/cpu if that has suitable data -> -> -Where do the PMC counts come from originally ? CPUID or something else ? -Yes, they're bits 8..15 of CPUID leaf 0xa - -Dave - -> -Regards, -> -Daniel -> --- -> -|: -https://berrange.com --o- -https://www.flickr.com/photos/dberrange -:| -> -|: -https://libvirt.org --o- -https://fstop138.berrange.com -:| -> -|: -https://entangle-photo.org --o- -https://www.instagram.com/dberrange -:| --- -Dr. David Alan Gilbert / address@hidden / Manchester, UK - -On Mon, Apr 24, 2017 at 11:27:16AM +0100, Dr. David Alan Gilbert wrote: -> -* Daniel P. Berrange (address@hidden) wrote: -> -> On Mon, Apr 24, 2017 at 10:23:21AM +0100, Dr. David Alan Gilbert wrote: -> -> > * Zhuangyanying (address@hidden) wrote: -> -> > > Hi all, -> -> > > -> -> > > Recently, I found migration failed when enable vPMU. -> -> > > -> -> > > migrate vPMU state was introduced in linux-3.10 + qemu-1.7. -> -> > > -> -> > > As long as enable vPMU, qemu will save / load the -> -> > > vmstate_msr_architectural_pmu(msr_global_ctrl) register during the -> -> > > migration. -> -> > > But global_ctrl generated based on cpuid(0xA), the number of -> -> > > general-purpose performance -> -> > > monitoring counters(PMC) can vary according to Intel SDN. The number of -> -> > > PMC presented -> -> > > to vm, does not support configuration currently, it depend on host -> -> > > cpuid, and enable all pmc -> -> > > defaultly at KVM. It cause migration to fail between boards with -> -> > > different PMC counts. -> -> > > -> -> > > The return value of cpuid (0xA) is different dur to cpu, according to -> -> > > Intel SDN，18-10 Vol. 3B: -> -> > > -> -> > > Note: The number of general-purpose performance monitoring counters -> -> > > (i.e. N in Figure 18-9) -> -> > > can vary across processor generations within a processor family, across -> -> > > processor families, or -> -> > > could be different depending on the configuration chosen at boot time -> -> > > in the BIOS regarding -> -> > > Intel Hyper Threading Technology, (e.g. N=2 for 45 nm Intel Atom -> -> > > processors; N =4 for processors -> -> > > based on the Nehalem microarchitecture; for processors based on the -> -> > > Sandy Bridge -> -> > > microarchitecture, N = 4 if Intel Hyper Threading Technology is active -> -> > > and N=8 if not active). -> -> > > -> -> > > Also I found, N=8 if HT is not active based on the broadwell，, -> -> > > such as CPU E7-8890 v4 @ 2.20GHz -> -> > > -> -> > > # ./x86_64-softmmu/qemu-system-x86_64 --enable-kvm -smp 4 -m 4096 -hda -> -> > > /data/zyy/test_qemu.img.sles12sp1 -vnc :99 -cpu kvm64,pmu=true -> -> > > -incoming tcp::8888 -> -> > > Completed 100 % -> -> > > qemu-system-x86_64: error: failed to set MSR 0x38f to 0x7000000ff -> -> > > qemu-system-x86_64: /data/zyy/git/test/qemu/target/i386/kvm.c:1833: -> -> > > kvm_put_msrs: -> -> > > Assertion `ret == cpu->kvm_msr_buf->nmsrs' failed. -> -> > > Aborted -> -> > > -> -> > > So make number of pmc configurable to vm ? Any better idea ? -> -> > -> -> > Coincidentally we hit a similar problem a few days ago with -cpu host - -> -> > it took me -> -> > quite a while to spot the difference between the machines was the source -> -> > had hyperthreading disabled. -> -> > -> -> > An option to set the number of counters makes sense to me; but I wonder -> -> > how many other options we need as well. Also, I'm not sure there's any -> -> > easy way for libvirt etc to figure out how many counters a host supports - -> -> > it's not in /proc/cpuinfo. -> -> -> -> We actually try to avoid /proc/cpuinfo whereever possible. We do direct -> -> CPUID asm instructions to identify features, and prefer to use -> -> /sys/devices/system/cpu if that has suitable data -> -> -> -> Where do the PMC counts come from originally ? CPUID or something else ? -> -> -Yes, they're bits 8..15 of CPUID leaf 0xa -Ok, that's easy enough for libvirt to detect then. More a question of what -libvirt should then do this with the info.... - -Regards, -Daniel --- -|: -https://berrange.com --o- -https://www.flickr.com/photos/dberrange -:| -|: -https://libvirt.org --o- -https://fstop138.berrange.com -:| -|: -https://entangle-photo.org --o- -https://www.instagram.com/dberrange -:| - -> ------Original Message----- -> -From: Daniel P. Berrange [ -mailto:address@hidden -> -Sent: Monday, April 24, 2017 6:34 PM -> -To: Dr. David Alan Gilbert -> -Cc: Zhuangyanying; Zhanghailiang; wangxin (U); address@hidden; -> -Gonglei (Arei); Huangzhichao; address@hidden -> -Subject: Re: [Qemu-devel] [BUG] Migrate failes between boards with different -> -PMC counts -> -> -On Mon, Apr 24, 2017 at 11:27:16AM +0100, Dr. David Alan Gilbert wrote: -> -> * Daniel P. Berrange (address@hidden) wrote: -> -> > On Mon, Apr 24, 2017 at 10:23:21AM +0100, Dr. David Alan Gilbert wrote: -> -> > > * Zhuangyanying (address@hidden) wrote: -> -> > > > Hi all, -> -> > > > -> -> > > > Recently, I found migration failed when enable vPMU. -> -> > > > -> -> > > > migrate vPMU state was introduced in linux-3.10 + qemu-1.7. -> -> > > > -> -> > > > As long as enable vPMU, qemu will save / load the -> -> > > > vmstate_msr_architectural_pmu(msr_global_ctrl) register during the -> -migration. -> -> > > > But global_ctrl generated based on cpuid(0xA), the number of -> -> > > > general-purpose performance monitoring counters(PMC) can vary -> -> > > > according to Intel SDN. The number of PMC presented to vm, does -> -> > > > not support configuration currently, it depend on host cpuid, and -> -> > > > enable -> -all pmc defaultly at KVM. It cause migration to fail between boards with -> -different PMC counts. -> -> > > > -> -> > > > The return value of cpuid (0xA) is different dur to cpu, according to -> -> > > > Intel -> -SDN，18-10 Vol. 3B: -> -> > > > -> -> > > > Note: The number of general-purpose performance monitoring -> -> > > > counters (i.e. N in Figure 18-9) can vary across processor -> -> > > > generations within a processor family, across processor -> -> > > > families, or could be different depending on the configuration -> -> > > > chosen at boot time in the BIOS regarding Intel Hyper Threading -> -> > > > Technology, (e.g. N=2 for 45 nm Intel Atom processors; N =4 for -> -processors based on the Nehalem microarchitecture; for processors based on -> -the Sandy Bridge microarchitecture, N = 4 if Intel Hyper Threading Technology -> -is active and N=8 if not active). -> -> > > > -> -> > > > Also I found, N=8 if HT is not active based on the broadwell，, -> -> > > > such as CPU E7-8890 v4 @ 2.20GHz -> -> > > > -> -> > > > # ./x86_64-softmmu/qemu-system-x86_64 --enable-kvm -smp 4 -m -> -> > > > 4096 -hda -> -> > > > /data/zyy/test_qemu.img.sles12sp1 -vnc :99 -cpu kvm64,pmu=true -> -> > > > -incoming tcp::8888 Completed 100 % -> -> > > > qemu-system-x86_64: error: failed to set MSR 0x38f to -> -> > > > 0x7000000ff -> -> > > > qemu-system-x86_64: /data/zyy/git/test/qemu/target/i386/kvm.c:1833: -> -kvm_put_msrs: -> -> > > > Assertion `ret == cpu->kvm_msr_buf->nmsrs' failed. -> -> > > > Aborted -> -> > > > -> -> > > > So make number of pmc configurable to vm ? Any better idea ? -> -> > > -> -> > > Coincidentally we hit a similar problem a few days ago with -cpu -> -> > > host - it took me quite a while to spot the difference between -> -> > > the machines was the source had hyperthreading disabled. -> -> > > -> -> > > An option to set the number of counters makes sense to me; but I -> -> > > wonder how many other options we need as well. Also, I'm not sure -> -> > > there's any easy way for libvirt etc to figure out how many -> -> > > counters a host supports - it's not in /proc/cpuinfo. -> -> > -> -> > We actually try to avoid /proc/cpuinfo whereever possible. We do -> -> > direct CPUID asm instructions to identify features, and prefer to -> -> > use /sys/devices/system/cpu if that has suitable data -> -> > -> -> > Where do the PMC counts come from originally ? CPUID or something -> -else ? -> -> -> -> Yes, they're bits 8..15 of CPUID leaf 0xa -> -> -Ok, that's easy enough for libvirt to detect then. More a question of what -> -libvirt -> -should then do this with the info.... -> -Do you mean to do a validation at the begining of migration? in -qemuMigrationBakeCookie() & qemuMigrationEatCookie(), if the PMC numbers are -not equal, just quit migration? -It maybe a good enough first edition. -But for a further better edition, maybe it's better to support Heterogeneous -migration I think, so we might need to make PMC number configrable, then we -need to modify KVM/qemu as well. - -Regards, --Zhuang Yanying - -* Zhuangyanying (address@hidden) wrote: -> -> -> -> -----Original Message----- -> -> From: Daniel P. Berrange [ -mailto:address@hidden -> -> Sent: Monday, April 24, 2017 6:34 PM -> -> To: Dr. David Alan Gilbert -> -> Cc: Zhuangyanying; Zhanghailiang; wangxin (U); address@hidden; -> -> Gonglei (Arei); Huangzhichao; address@hidden -> -> Subject: Re: [Qemu-devel] [BUG] Migrate failes between boards with different -> -> PMC counts -> -> -> -> On Mon, Apr 24, 2017 at 11:27:16AM +0100, Dr. David Alan Gilbert wrote: -> -> > * Daniel P. Berrange (address@hidden) wrote: -> -> > > On Mon, Apr 24, 2017 at 10:23:21AM +0100, Dr. David Alan Gilbert wrote: -> -> > > > * Zhuangyanying (address@hidden) wrote: -> -> > > > > Hi all, -> -> > > > > -> -> > > > > Recently, I found migration failed when enable vPMU. -> -> > > > > -> -> > > > > migrate vPMU state was introduced in linux-3.10 + qemu-1.7. -> -> > > > > -> -> > > > > As long as enable vPMU, qemu will save / load the -> -> > > > > vmstate_msr_architectural_pmu(msr_global_ctrl) register during the -> -> migration. -> -> > > > > But global_ctrl generated based on cpuid(0xA), the number of -> -> > > > > general-purpose performance monitoring counters(PMC) can vary -> -> > > > > according to Intel SDN. The number of PMC presented to vm, does -> -> > > > > not support configuration currently, it depend on host cpuid, and -> -> > > > > enable -> -> all pmc defaultly at KVM. It cause migration to fail between boards with -> -> different PMC counts. -> -> > > > > -> -> > > > > The return value of cpuid (0xA) is different dur to cpu, according -> -> > > > > to Intel -> -> SDN，18-10 Vol. 3B: -> -> > > > > -> -> > > > > Note: The number of general-purpose performance monitoring -> -> > > > > counters (i.e. N in Figure 18-9) can vary across processor -> -> > > > > generations within a processor family, across processor -> -> > > > > families, or could be different depending on the configuration -> -> > > > > chosen at boot time in the BIOS regarding Intel Hyper Threading -> -> > > > > Technology, (e.g. N=2 for 45 nm Intel Atom processors; N =4 for -> -> processors based on the Nehalem microarchitecture; for processors based on -> -> the Sandy Bridge microarchitecture, N = 4 if Intel Hyper Threading -> -> Technology -> -> is active and N=8 if not active). -> -> > > > > -> -> > > > > Also I found, N=8 if HT is not active based on the broadwell，, -> -> > > > > such as CPU E7-8890 v4 @ 2.20GHz -> -> > > > > -> -> > > > > # ./x86_64-softmmu/qemu-system-x86_64 --enable-kvm -smp 4 -m -> -> > > > > 4096 -hda -> -> > > > > /data/zyy/test_qemu.img.sles12sp1 -vnc :99 -cpu kvm64,pmu=true -> -> > > > > -incoming tcp::8888 Completed 100 % -> -> > > > > qemu-system-x86_64: error: failed to set MSR 0x38f to -> -> > > > > 0x7000000ff -> -> > > > > qemu-system-x86_64: /data/zyy/git/test/qemu/target/i386/kvm.c:1833: -> -> kvm_put_msrs: -> -> > > > > Assertion `ret == cpu->kvm_msr_buf->nmsrs' failed. -> -> > > > > Aborted -> -> > > > > -> -> > > > > So make number of pmc configurable to vm ? Any better idea ? -> -> > > > -> -> > > > Coincidentally we hit a similar problem a few days ago with -cpu -> -> > > > host - it took me quite a while to spot the difference between -> -> > > > the machines was the source had hyperthreading disabled. -> -> > > > -> -> > > > An option to set the number of counters makes sense to me; but I -> -> > > > wonder how many other options we need as well. Also, I'm not sure -> -> > > > there's any easy way for libvirt etc to figure out how many -> -> > > > counters a host supports - it's not in /proc/cpuinfo. -> -> > > -> -> > > We actually try to avoid /proc/cpuinfo whereever possible. We do -> -> > > direct CPUID asm instructions to identify features, and prefer to -> -> > > use /sys/devices/system/cpu if that has suitable data -> -> > > -> -> > > Where do the PMC counts come from originally ? CPUID or something -> -> else ? -> -> > -> -> > Yes, they're bits 8..15 of CPUID leaf 0xa -> -> -> -> Ok, that's easy enough for libvirt to detect then. More a question of what -> -> libvirt -> -> should then do this with the info.... -> -> -> -> -Do you mean to do a validation at the begining of migration? in -> -qemuMigrationBakeCookie() & qemuMigrationEatCookie(), if the PMC numbers are -> -not equal, just quit migration? -> -It maybe a good enough first edition. -> -But for a further better edition, maybe it's better to support Heterogeneous -> -migration I think, so we might need to make PMC number configrable, then we -> -need to modify KVM/qemu as well. -Yes agreed; the only thing I wanted to check was that libvirt would have enough -information to be able to use any feature we added to QEMU. - -Dave - -> -Regards, -> --Zhuang Yanying --- -Dr. David Alan Gilbert / address@hidden / Manchester, UK - diff --git a/results/classifier/007/device/42226390 b/results/classifier/007/device/42226390 deleted file mode 100644 index b9c7459c..00000000 --- a/results/classifier/007/device/42226390 +++ /dev/null @@ -1,197 +0,0 @@ -device: 0.951 -boot: 0.943 -debug: 0.942 -graphic: 0.942 -permissions: 0.936 -performance: 0.927 -semantic: 0.924 -PID: 0.914 -KVM: 0.905 -network: 0.894 -other: 0.894 -socket: 0.882 -files: 0.878 -vnc: 0.853 - -[BUG] AArch64 boot hang with -icount and -smp >1 (iothread locking issue?) - -Hello, - -I am encountering one or more bugs when using -icount and -smp >1 that I am -attempting to sort out. My current theory is that it is an iothread locking -issue. - -I am using a command-line like the following where $kernel is a recent upstream -AArch64 Linux kernel Image (I can provide a binary if that would be helpful - -let me know how is best to post): - - qemu-system-aarch64 \ - -M virt -cpu cortex-a57 -m 1G \ - -nographic \ - -smp 2 \ - -icount 0 \ - -kernel $kernel - -For any/all of the symptoms described below, they seem to disappear when I -either remove `-icount 0` or change smp to `-smp 1`. In other words, it is the -combination of `-smp >1` and `-icount` which triggers what I'm seeing. - -I am seeing two different (but seemingly related) behaviors. The first (and -what I originally started debugging) shows up as a boot hang. When booting -using the above command after Peter's "icount: Take iothread lock when running -QEMU timers" patch [1], The kernel boots for a while and then hangs after: - -> -...snip... -> -[ 0.010764] Serial: AMBA PL011 UART driver -> -[ 0.016334] 9000000.pl011: ttyAMA0 at MMIO 0x9000000 (irq = 13, base_baud -> -= 0) is a PL011 rev1 -> -[ 0.016907] printk: console [ttyAMA0] enabled -> -[ 0.017624] KASLR enabled -> -[ 0.031986] HugeTLB: registered 16.0 GiB page size, pre-allocated 0 pages -> -[ 0.031986] HugeTLB: 16320 KiB vmemmap can be freed for a 16.0 GiB page -> -[ 0.031986] HugeTLB: registered 512 MiB page size, pre-allocated 0 pages -> -[ 0.031986] HugeTLB: 448 KiB vmemmap can be freed for a 512 MiB page -> -[ 0.031986] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages -> -[ 0.031986] HugeTLB: 0 KiB vmemmap can be freed for a 2.00 MiB page -When it hangs here, I drop into QEMU's console, attach to the gdbserver, and it -always reports that it is at address 0xffff800008dc42e8 (as shown below from an -objdump of the vmlinux). I note this is in the middle of messing with timer -system registers - which makes me suspect we're attempting to take the iothread -lock when its already held: - -> -ffff800008dc42b8 : -> -ffff800008dc42b8: d503201f nop -> -ffff800008dc42bc: d503201f nop -> -ffff800008dc42c0: d503233f paciasp -> -ffff800008dc42c4: d53be321 mrs x1, cntv_ctl_el0 -> -ffff800008dc42c8: 32000021 orr w1, w1, #0x1 -> -ffff800008dc42cc: d5033fdf isb -> -ffff800008dc42d0: d53be042 mrs x2, cntvct_el0 -> -ffff800008dc42d4: ca020043 eor x3, x2, x2 -> -ffff800008dc42d8: 8b2363e3 add x3, sp, x3 -> -ffff800008dc42dc: f940007f ldr xzr, [x3] -> -ffff800008dc42e0: 8b020000 add x0, x0, x2 -> -ffff800008dc42e4: d51be340 msr cntv_cval_el0, x0 -> -* ffff800008dc42e8: 927ef820 and x0, x1, #0xfffffffffffffffd -> -ffff800008dc42ec: d51be320 msr cntv_ctl_el0, x0 -> -ffff800008dc42f0: d5033fdf isb -> -ffff800008dc42f4: 52800000 mov w0, #0x0 -> -// #0 -> -ffff800008dc42f8: d50323bf autiasp -> -ffff800008dc42fc: d65f03c0 ret -The second behavior is that prior to Peter's "icount: Take iothread lock when -running QEMU timers" patch [1], I observe the following message (same command -as above): - -> -ERROR:../accel/tcg/tcg-accel-ops.c:79:tcg_handle_interrupt: assertion failed: -> -(qemu_mutex_iothread_locked()) -> -Aborted (core dumped) -This is the same behavior described in Gitlab issue 1130 [0] and addressed by -[1]. I bisected the appearance of this assertion, and found it was introduced -by Pavel's "replay: rewrite async event handling" commit [2]. Commits prior to -that one boot successfully (neither assertions nor hangs) with `-icount 0 -smp -2`. - -I've looked over these two commits ([1], [2]), but it is not obvious to me -how/why they might be interacting to produce the boot hangs I'm seeing and -I welcome any help investigating further. - -Thanks! - --Aaron Lindsay - -[0] - -https://gitlab.com/qemu-project/qemu/-/issues/1130 -[1] - -https://gitlab.com/qemu-project/qemu/-/commit/c7f26ded6d5065e4116f630f6a490b55f6c5f58e -[2] - -https://gitlab.com/qemu-project/qemu/-/commit/60618e2d77691e44bb78e23b2b0cf07b5c405e56 - -On Fri, 21 Oct 2022 at 16:48, Aaron Lindsay - wrote: -> -> -Hello, -> -> -I am encountering one or more bugs when using -icount and -smp >1 that I am -> -attempting to sort out. My current theory is that it is an iothread locking -> -issue. -Weird coincidence, that is a bug that's been in the tree for months -but was only reported to me earlier this week. Try reverting -commit a82fd5a4ec24d923ff1e -- that should fix it. -CAFEAcA_i8x00hD-4XX18ySLNbCB6ds1-DSazVb4yDnF8skjd9A@mail.gmail.com -/">https://lore.kernel.org/qemu-devel/ -CAFEAcA_i8x00hD-4XX18ySLNbCB6ds1-DSazVb4yDnF8skjd9A@mail.gmail.com -/ -has the explanation. - -thanks --- PMM - -On Oct 21 17:00, Peter Maydell wrote: -> -On Fri, 21 Oct 2022 at 16:48, Aaron Lindsay -> - wrote: -> -> -> -> Hello, -> -> -> -> I am encountering one or more bugs when using -icount and -smp >1 that I am -> -> attempting to sort out. My current theory is that it is an iothread locking -> -> issue. -> -> -Weird coincidence, that is a bug that's been in the tree for months -> -but was only reported to me earlier this week. Try reverting -> -commit a82fd5a4ec24d923ff1e -- that should fix it. -I can confirm that reverting a82fd5a4ec24d923ff1e fixes it for me. -Thanks for the help and fast response! - --Aaron - diff --git a/results/classifier/007/files/64322995 b/results/classifier/007/files/64322995 deleted file mode 100644 index 00045ea1..00000000 --- a/results/classifier/007/files/64322995 +++ /dev/null @@ -1,64 +0,0 @@ -files: 0.941 -performance: 0.939 -device: 0.915 -network: 0.914 -semantic: 0.906 -graphic: 0.904 -other: 0.881 -PID: 0.875 -socket: 0.866 -debug: 0.837 -permissions: 0.835 -vnc: 0.801 -boot: 0.780 -KVM: 0.742 - -[Qemu-devel] [BUG] trace: QEMU hangs on initialization with the "simple" backend - -While starting the softmmu version of QEMU, the simple backend waits for the -writeout thread to signal a condition variable when initializing the output file -path. But since the writeout thread has not been created, it just waits forever. - -Thanks, - Lluis - -On Tue, Feb 09, 2016 at 09:24:04PM +0100, Lluís Vilanova wrote: -> -While starting the softmmu version of QEMU, the simple backend waits for the -> -writeout thread to signal a condition variable when initializing the output -> -file -> -path. But since the writeout thread has not been created, it just waits -> -forever. -Denis Lunev posted a fix: -https://patchwork.ozlabs.org/patch/580968/ -Stefan -signature.asc -Description: -PGP signature - -Stefan Hajnoczi writes: - -> -On Tue, Feb 09, 2016 at 09:24:04PM +0100, Lluís Vilanova wrote: -> -> While starting the softmmu version of QEMU, the simple backend waits for the -> -> writeout thread to signal a condition variable when initializing the output -> -> file -> -> path. But since the writeout thread has not been created, it just waits -> -> forever. -> -Denis Lunev posted a fix: -> -https://patchwork.ozlabs.org/patch/580968/ -Great, thanks. - -Lluis - diff --git a/results/classifier/007/graphic/30680944 b/results/classifier/007/graphic/30680944 deleted file mode 100644 index ef0848a9..00000000 --- a/results/classifier/007/graphic/30680944 +++ /dev/null @@ -1,605 +0,0 @@ -graphic: 0.965 -semantic: 0.953 -other: 0.944 -performance: 0.937 -debug: 0.936 -device: 0.935 -permissions: 0.933 -PID: 0.913 -socket: 0.864 -boot: 0.840 -files: 0.835 -vnc: 0.815 -network: 0.813 -KVM: 0.701 - -[BUG]QEMU jump into interrupt when single-stepping on aarch64 - -Dear, folks, - -I try to debug Linux kernel with QEMU in single-stepping mode on aarch64 -platform, -the added breakpoint hits but after I type `step`, the gdb always jumps into -interrupt. - -My env: - - gdb-10.2 - qemu-6.2.0 - host kernel: 5.10.84 - VM kernel: 5.10.84 - -The steps to reproduce: - # host console: run a VM with only one core, the import arg: - # details can be found here: -https://www.redhat.com/en/blog/debugging-kernel-qemulibvirt -virsh create dev_core0.xml - - # run gdb client - gdb ./vmlinux - - # gdb client on host console - (gdb) dir -./usr/src/debug/kernel-5.10.84/linux-5.10.84-004.alpha.ali5000.alios7.aarch64 - (gdb) target remote localhost:1234 - (gdb) info b - Num Type Disp Enb Address What - 1 breakpoint keep y - 1.1 y 0xffff800010361444 -mm/memory-failure.c:1318 - 1.2 y 0xffff800010361450 in memory_failure - at mm/memory-failure.c:1488 - (gdb) c - Continuing. - - # console in VM, use madvise to inject a hwposion at virtual address -vaddr, - # which will hit the b inmemory_failur: madvise(vaddr, pagesize, -MADV_HWPOISON); - # and the VM pause - ./run_madvise.c - - # gdb client on host console - (gdb) - Continuing. - Breakpoint 1, 0xffff800010361444 in memory_failure () at -mm/memory-failure.c:1318 - 1318 res = -EHWPOISON; - (gdb) n - vectors () at arch/arm64/kernel/entry.S:552 - 552 kernel_ventry 1, irq // IRQ -EL1h - (gdb) n - (gdb) n - (gdb) n - (gdb) n - gic_handle_irq (regs=0xffff8000147c3b80) at -drivers/irqchip/irq-gic-v3.c:721 - # after several step, I got the irqnr - (gdb) p irqnr - $5 = 8262 - -Sometimes, the irqnr is 27， which is used for arch_timer. - -I was wondering do you have any comments on this? And feedback are welcomed. - -Thank you. - -Best Regards. -Shuai - -On 4/6/22 09:30, Shuai Xue wrote: -Dear, folks, - -I try to debug Linux kernel with QEMU in single-stepping mode on aarch64 -platform, -the added breakpoint hits but after I type `step`, the gdb always jumps into -interrupt. - -My env: - - gdb-10.2 - qemu-6.2.0 - host kernel: 5.10.84 - VM kernel: 5.10.84 - -The steps to reproduce: - # host console: run a VM with only one core, the import arg: - # details can be found here: -https://www.redhat.com/en/blog/debugging-kernel-qemulibvirt -virsh create dev_core0.xml - - # run gdb client - gdb ./vmlinux - - # gdb client on host console - (gdb) dir -./usr/src/debug/kernel-5.10.84/linux-5.10.84-004.alpha.ali5000.alios7.aarch64 - (gdb) target remote localhost:1234 - (gdb) info b - Num Type Disp Enb Address What - 1 breakpoint keep y - 1.1 y 0xffff800010361444 -mm/memory-failure.c:1318 - 1.2 y 0xffff800010361450 in memory_failure - at mm/memory-failure.c:1488 - (gdb) c - Continuing. - - # console in VM, use madvise to inject a hwposion at virtual address -vaddr, - # which will hit the b inmemory_failur: madvise(vaddr, pagesize, -MADV_HWPOISON); - # and the VM pause - ./run_madvise.c - - # gdb client on host console - (gdb) - Continuing. - Breakpoint 1, 0xffff800010361444 in memory_failure () at -mm/memory-failure.c:1318 - 1318 res = -EHWPOISON; - (gdb) n - vectors () at arch/arm64/kernel/entry.S:552 - 552 kernel_ventry 1, irq // IRQ -EL1h -The 'n' command is not a single-step: use stepi, which will suppress interrupts. -Anyway, not a bug. - -r~ - -在 2022/4/7 AM12:57, Richard Henderson 写道: -> -On 4/6/22 09:30, Shuai Xue wrote: -> -> Dear, folks, -> -> -> -> I try to debug Linux kernel with QEMU in single-stepping mode on aarch64 -> -> platform, -> -> the added breakpoint hits but after I type `step`, the gdb always jumps into -> -> interrupt. -> -> -> -> My env: -> -> -> ->     gdb-10.2 -> ->     qemu-6.2.0 -> ->     host kernel: 5.10.84 -> ->     VM kernel: 5.10.84 -> -> -> -> The steps to reproduce: -> ->     # host console: run a VM with only one core, the import arg: -> value='-s'/> -> ->     # details can be found here: -> -> -https://www.redhat.com/en/blog/debugging-kernel-qemulibvirt -> ->     virsh create dev_core0.xml -> ->     -> ->     # run gdb client -> ->     gdb ./vmlinux -> -> -> ->     # gdb client on host console -> ->     (gdb) dir -> -> ./usr/src/debug/kernel-5.10.84/linux-5.10.84-004.alpha.ali5000.alios7.aarch64 -> ->     (gdb) target remote localhost:1234 -> ->     (gdb) info b -> ->     Num     Type           Disp Enb Address            What -> ->     1       breakpoint     keep y   -> ->     1.1                         y   0xffff800010361444 -> -> mm/memory-failure.c:1318 -> ->     1.2                         y   0xffff800010361450 in memory_failure -> ->                                                     at -> -> mm/memory-failure.c:1488 -> ->     (gdb) c -> ->     Continuing. -> -> -> ->     # console in VM, use madvise to inject a hwposion at virtual address -> -> vaddr, -> ->     # which will hit the b inmemory_failur: madvise(vaddr, pagesize, -> -> MADV_HWPOISON); -> ->     # and the VM pause -> ->     ./run_madvise.c -> -> -> ->     # gdb client on host console -> ->     (gdb) -> ->     Continuing. -> ->     Breakpoint 1, 0xffff800010361444 in memory_failure () at -> -> mm/memory-failure.c:1318 -> ->     1318                    res = -EHWPOISON; -> ->     (gdb) n -> ->     vectors () at arch/arm64/kernel/entry.S:552 -> ->     552             kernel_ventry   1, irq                          // IRQ -> -> EL1h -> -> -The 'n' command is not a single-step: use stepi, which will suppress -> -interrupts. -> -Anyway, not a bug. -> -> -r~ -Hi, Richard, - -Thank you for your quick reply, I also try `stepi`, but it does NOT work either. - - (gdb) c - Continuing. - - Breakpoint 1, memory_failure (pfn=1273982, flags=1) at -mm/memory-failure.c:1488 - 1488 { - (gdb) stepi - vectors () at arch/arm64/kernel/entry.S:552 - 552 kernel_ventry 1, irq // IRQ -EL1h - -According to QEMU doc[1]: the default single stepping behavior is step with the -IRQs -and timer service routines off. I checked the MASK bits used to control the -single -stepping IE on my machine as bellow: - - # gdb client on host (x86 plafrom) - (gdb) maintenance packet qqemu.sstepbits - sending: "qqemu.sstepbits" - received: "ENABLE=1,NOIRQ=2,NOTIMER=4" - -The sstep MASK looks as expected, but does not work as expected. - -I also try the same kernel and qemu version on X86 platform: -> -> gdb-10.2 -> -> qemu-6.2.0 -> -> host kernel: 5.10.84 -> -> VM kernel: 5.10.84 -The command `n` jumps to the next instruction. - - # gdb client on host (x86 plafrom) - (gdb) b memory-failure.c:1488 - Breakpoint 1, memory_failure (pfn=1128931, flags=1) at -mm/memory-failure.c:1488 - 1488 { - (gdb) n - 1497 if (!sysctl_memory_failure_recovery) - (gdb) stepi - 0xffffffff812efdbc 1497 if -(!sysctl_memory_failure_recovery) - (gdb) stepi - 0xffffffff812efdbe 1497 if -(!sysctl_memory_failure_recovery) - (gdb) n - 1500 p = pfn_to_online_page(pfn); - (gdb) l - 1496 - 1497 if (!sysctl_memory_failure_recovery) - 1498 panic("Memory failure on page %lx", pfn); - 1499 - 1500 p = pfn_to_online_page(pfn); - 1501 if (!p) { - -Best Regrades, -Shuai - - -[1] -https://github.com/qemu/qemu/blob/master/docs/system/gdb.rst - -在 2022/4/7 PM12:10, Shuai Xue 写道: -> -在 2022/4/7 AM12:57, Richard Henderson 写道: -> -> On 4/6/22 09:30, Shuai Xue wrote: -> ->> Dear, folks, -> ->> -> ->> I try to debug Linux kernel with QEMU in single-stepping mode on aarch64 -> ->> platform, -> ->> the added breakpoint hits but after I type `step`, the gdb always jumps -> ->> into interrupt. -> ->> -> ->> My env: -> ->> -> ->>     gdb-10.2 -> ->>     qemu-6.2.0 -> ->>     host kernel: 5.10.84 -> ->>     VM kernel: 5.10.84 -> ->> -> ->> The steps to reproduce: -> ->>     # host console: run a VM with only one core, the import arg: ->> value='-s'/> -> ->>     # details can be found here: -> ->> -https://www.redhat.com/en/blog/debugging-kernel-qemulibvirt -> ->>     virsh create dev_core0.xml -> ->>     -> ->>     # run gdb client -> ->>     gdb ./vmlinux -> ->> -> ->>     # gdb client on host console -> ->>     (gdb) dir -> ->> ./usr/src/debug/kernel-5.10.84/linux-5.10.84-004.alpha.ali5000.alios7.aarch64 -> ->>     (gdb) target remote localhost:1234 -> ->>     (gdb) info b -> ->>     Num     Type           Disp Enb Address            What -> ->>     1       breakpoint     keep y   -> ->>     1.1                         y   0xffff800010361444 -> ->> mm/memory-failure.c:1318 -> ->>     1.2                         y   0xffff800010361450 in memory_failure -> ->>                                                     at -> ->> mm/memory-failure.c:1488 -> ->>     (gdb) c -> ->>     Continuing. -> ->> -> ->>     # console in VM, use madvise to inject a hwposion at virtual address -> ->> vaddr, -> ->>     # which will hit the b inmemory_failur: madvise(vaddr, pagesize, -> ->> MADV_HWPOISON); -> ->>     # and the VM pause -> ->>     ./run_madvise.c -> ->> -> ->>     # gdb client on host console -> ->>     (gdb) -> ->>     Continuing. -> ->>     Breakpoint 1, 0xffff800010361444 in memory_failure () at -> ->> mm/memory-failure.c:1318 -> ->>     1318                    res = -EHWPOISON; -> ->>     (gdb) n -> ->>     vectors () at arch/arm64/kernel/entry.S:552 -> ->>     552             kernel_ventry   1, irq                          // IRQ -> ->> EL1h -> -> -> -> The 'n' command is not a single-step: use stepi, which will suppress -> -> interrupts. -> -> Anyway, not a bug. -> -> -> -> r~ -> -> -Hi, Richard, -> -> -Thank you for your quick reply, I also try `stepi`, but it does NOT work -> -either. -> -> -(gdb) c -> -Continuing. -> -> -Breakpoint 1, memory_failure (pfn=1273982, flags=1) at -> -mm/memory-failure.c:1488 -> -1488 { -> -(gdb) stepi -> -vectors () at arch/arm64/kernel/entry.S:552 -> -552 kernel_ventry 1, irq // IRQ -> -EL1h -> -> -According to QEMU doc[1]: the default single stepping behavior is step with -> -the IRQs -> -and timer service routines off. I checked the MASK bits used to control the -> -single -> -stepping IE on my machine as bellow: -> -> -# gdb client on host (x86 plafrom) -> -(gdb) maintenance packet qqemu.sstepbits -> -sending: "qqemu.sstepbits" -> -received: "ENABLE=1,NOIRQ=2,NOTIMER=4" -> -> -The sstep MASK looks as expected, but does not work as expected. -> -> -I also try the same kernel and qemu version on X86 platform: -> ->> gdb-10.2 -> ->> qemu-6.2.0 -> ->> host kernel: 5.10.84 -> ->> VM kernel: 5.10.84 -> -> -> -The command `n` jumps to the next instruction. -> -> -# gdb client on host (x86 plafrom) -> -(gdb) b memory-failure.c:1488 -> -Breakpoint 1, memory_failure (pfn=1128931, flags=1) at -> -mm/memory-failure.c:1488 -> -1488 { -> -(gdb) n -> -1497 if (!sysctl_memory_failure_recovery) -> -(gdb) stepi -> -0xffffffff812efdbc 1497 if -> -(!sysctl_memory_failure_recovery) -> -(gdb) stepi -> -0xffffffff812efdbe 1497 if -> -(!sysctl_memory_failure_recovery) -> -(gdb) n -> -1500 p = pfn_to_online_page(pfn); -> -(gdb) l -> -1496 -> -1497 if (!sysctl_memory_failure_recovery) -> -1498 panic("Memory failure on page %lx", pfn); -> -1499 -> -1500 p = pfn_to_online_page(pfn); -> -1501 if (!p) { -> -> -Best Regrades, -> -Shuai -> -> -> -[1] -https://github.com/qemu/qemu/blob/master/docs/system/gdb.rst -Hi, Richard, - -I was wondering that do you have any comments to this? - -Best Regrades, -Shuai - diff --git a/results/classifier/007/graphic/46572227 b/results/classifier/007/graphic/46572227 deleted file mode 100644 index c65ee26c..00000000 --- a/results/classifier/007/graphic/46572227 +++ /dev/null @@ -1,416 +0,0 @@ -semantic: 0.965 -graphic: 0.962 -debug: 0.958 -permissions: 0.955 -PID: 0.937 -performance: 0.935 -other: 0.927 -vnc: 0.904 -device: 0.901 -boot: 0.900 -files: 0.879 -KVM: 0.857 -network: 0.841 -socket: 0.841 - -[Qemu-devel] [Bug?] Windows 7's time drift obviously while RTC rate switching frequently between high and low timer rate - -Hi, - -We tested with the latest QEMU, and found that time drift obviously (clock fast -in guest) -in Windows 7 64 bits guest in some cases. - -It is easily to reproduce, using the follow QEMU command line to start windows -7: - -# x86_64-softmmu/qemu-system-x86_64 -name win7_64_2U_raw -machine -pc-i440fx-2.6,accel=kvm,usb=off -cpu host -m 2048 -realtime mlock=off -smp -4,sockets=2,cores=2,threads=1 -rtc base=utc,clock=vm,driftfix=slew -no-hpet --global kvm-pit.lost_tick_policy=discard -hda /mnt/nfs/win7_sp1_32_2U_raw -vnc -:11 -netdev tap,id=hn0,vhost=off -device rtl8139,id=net-pci0,netdev=hn0 -device -piix3-usb-uhci,id=usb -device usb-tablet,id=input0 -device usb-mouse,id=input1 --device usb-kbd,id=input2 -monitor stdio - -Adjust the VM's time to host time, and run java application or run the follow -program -in windows 7: - -#pragma comment(lib, "winmm") -#include -#include - -#define SWITCH_PEROID 13 - -int main() -{ - DWORD count = 0; - - while (1) - { - count++; - timeBeginPeriod(1); - DWORD start = timeGetTime(); - Sleep(40); - timeEndPeriod(1); - if ((count % SWITCH_PEROID) == 0) { - Sleep(1); - } - } - return 0; -} - -After few minutes, you will find that the time in windows 7 goes ahead of the -host time, drifts about several seconds. - -I have dug deeper in this problem. For windows systems that use the CMOS timer, -the base interrupt rate is usually 64Hz, but running some application in VM -will raise the timer rate to 1024Hz, running java application and or above -program will raise the timer rate. -Besides, Windows operating systems generally keep time by counting timer -interrupts (ticks). But QEMU seems not emulate the rate converting fine. - -We update the timer in function periodic_timer_update(): -static void periodic_timer_update(RTCState *s, int64_t current_time) -{ - - cur_clock = muldiv64(current_time, RTC_CLOCK_RATE, get_ticks_per_sec()); - next_irq_clock = (cur_clock & ~(period - 1)) + period; - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Here we calculate the next interrupt time by align the current clock with the -new period, I'm a little confused that why we care about the *history* time ? -If VM switches from high rate to low rate, the next interrupt time may come -earlier than it supposed to be. We have observed it in our test. we printed the -interval time of interrupts and the VM's current time (We got the time from VM). - -Here is part of the log: -... ... -period=512 irq inject 1534: 15625 us -Tue Mar 29 04:38:00 2016 -*irq_num_period_32=0, irq_num_period_512=64: [3]: Real time interval is 999696 -us -... ... -*irq_num_period_32=893, irq_num_period_512=9 [81]: Real time interval is 951086 -us -Convert 32 --- > 512: 703: 96578 us -period=512 irq inject 44391: 12702 us -Convert 512 --- > 32: 704: 12704 us11 -period=32 irq inject 44392: 979 us -... ... -32 --- > 512: 705: 24388 us -period=512 irq inject 44417: 6834 us -Convert 512 --- > 32: 706: 6830 us -period=32 irq inject 44418: 978 us -... ... -Convert 32 --- > 512: 707: 60525 us -period=512 irq inject 44480: 1945 us -Convert 512 --- > 32: 708: 1955 us -period=32 irq inject 44481: 977 us -... ... -Convert 32 --- > 512: 709: 36105 us -period=512 irq inject 44518: 10741 us -Convert 512 --- > 32: 710: 10736 us -period=32 irq inject 44519: 989 us -... ... -Convert 32 --- > 512: 711: 123998 us -period=512 irq inject 44646: 974 us -period=512 irq inject 44647: 15607 us -Convert 512 --- > 32: 712: 16560 us -period=32 irq inject 44648: 980 us -... ... -period=32 irq inject 44738: 974 us -Convert 32 --- > 512: 713: 88828 us -period=512 irq inject 44739: 4885 us -Convert 512 --- > 32: 714: 4882 us -period=32 irq inject 44740: 989 us -... ... -period=32 irq inject 44842: 974 us -Convert 32 --- > 512: 715: 100537 us -period=512 irq inject 44843: 8788 us -Convert 512 --- > 32: 716: 8789 us -period=32 irq inject 44844: 972 us -... ... -period=32 irq inject 44941: 979 us -Convert 32 --- > 512: 717: 95677 us -period=512 irq inject 44942: 13661 us -Convert 512 --- > 32: 718: 13657 us -period=32 irq inject 44943: 987 us -... ... -Convert 32 --- > 512: 719: 94690 us -period=512 irq inject 45040: 14643 us -Convert 512 --- > 32: 720: 14642 us -period=32 irq inject 45041: 974 us -... ... -Convert 32 --- > 512: 721: 88848 us -period=512 irq inject 45132: 4892 us -Convert 512 --- > 32: 722: 4931 us -period=32 irq inject 45133: 964 us -... ... -Tue Mar 29 04:39:19 2016 -*irq_num_period_32:835, irq_num_period_512:11 [82], Real time interval is -911520 us - -For windows 7, it has got 835 IRQs which injected during the period of 32, -and got 11 IRQs that injected during the period of 512. it updated the -wall-clock -time with one second, because it supposed it has counted -(835*976.5+11*15625)= 987252.5 us, but the real interval time is 911520 us. - -IMHO, we should calculate the next interrupt time based on the time of last -interrupt injected, and it seems to be more similar with hardware CMOS timer -in this way. -Maybe someone can tell me the reason why we calculated the interrupt timer -in that way, or is it a bug ? ;) - -Thanks, -Hailiang - -ping... - -It seems that we can eliminate the drift by the following patch. -(I tested it for two hours, and there is no drift, before, the timer -in Windows 7 drifts about 2 seconds per minute.) I'm not sure if it is -the right way to solve the problem. -Any comments are welcomed. Thanks. - -From bd6acd577cbbc9d92d6376c770219470f184f7de Mon Sep 17 00:00:00 2001 -From: zhanghailiang -Date: Thu, 31 Mar 2016 16:36:15 -0400 -Subject: [PATCH] timer/mc146818rtc: fix timer drift in Windows OS while RTC - rate converting frequently - -Signed-off-by: zhanghailiang ---- - hw/timer/mc146818rtc.c | 25 ++++++++++++++++++++++--- - 1 file changed, 22 insertions(+), 3 deletions(-) - -diff --git a/hw/timer/mc146818rtc.c b/hw/timer/mc146818rtc.c -index 2ac0fd3..e39d2da 100644 ---- a/hw/timer/mc146818rtc.c -+++ b/hw/timer/mc146818rtc.c -@@ -79,6 +79,7 @@ typedef struct RTCState { - /* periodic timer */ - QEMUTimer *periodic_timer; - int64_t next_periodic_time; -+ uint64_t last_periodic_time; - /* update-ended timer */ - QEMUTimer *update_timer; - uint64_t next_alarm_time; -@@ -152,7 +153,8 @@ static void rtc_coalesced_timer(void *opaque) - static void periodic_timer_update(RTCState *s, int64_t current_time) - { - int period_code, period; -- int64_t cur_clock, next_irq_clock; -+ int64_t cur_clock, next_irq_clock, pre_irq_clock; -+ bool change = false; - - period_code = s->cmos_data[RTC_REG_A] & 0x0f; - if (period_code != 0 -@@ -165,14 +167,28 @@ static void periodic_timer_update(RTCState *s, int64_t -current_time) - if (period != s->period) { - s->irq_coalesced = (s->irq_coalesced * s->period) / period; - DPRINTF_C("cmos: coalesced irqs scaled to %d\n", s->irq_coalesced); -+ if (s->period && period) { -+ change = true; -+ } - } - s->period = period; - #endif - /* compute 32 khz clock */ - cur_clock = - muldiv64(current_time, RTC_CLOCK_RATE, NANOSECONDS_PER_SECOND); -+ if (change) { -+ int offset = 0; - -- next_irq_clock = (cur_clock & ~(period - 1)) + period; -+ pre_irq_clock = muldiv64(s->last_periodic_time, RTC_CLOCK_RATE, -+ NANOSECONDS_PER_SECOND); -+ if ((cur_clock - pre_irq_clock) > period) { -+ offset = (cur_clock - pre_irq_clock) / period; -+ } -+ s->irq_coalesced += offset; -+ next_irq_clock = pre_irq_clock + (offset + 1) * period; -+ } else { -+ next_irq_clock = (cur_clock & ~(period - 1)) + period; -+ } - s->next_periodic_time = muldiv64(next_irq_clock, -NANOSECONDS_PER_SECOND, - RTC_CLOCK_RATE) + 1; - timer_mod(s->periodic_timer, s->next_periodic_time); -@@ -187,7 +203,9 @@ static void periodic_timer_update(RTCState *s, int64_t -current_time) - static void rtc_periodic_timer(void *opaque) - { - RTCState *s = opaque; -- -+ int64_t next_periodic_time; -+ -+ next_periodic_time = s->next_periodic_time; - periodic_timer_update(s, s->next_periodic_time); - s->cmos_data[RTC_REG_C] |= REG_C_PF; - if (s->cmos_data[RTC_REG_B] & REG_B_PIE) { -@@ -204,6 +222,7 @@ static void rtc_periodic_timer(void *opaque) - DPRINTF_C("cmos: coalesced irqs increased to %d\n", - s->irq_coalesced); - } -+ s->last_periodic_time = next_periodic_time; - } else - #endif - qemu_irq_raise(s->irq); --- -1.8.3.1 - - -On 2016/3/29 19:58, Hailiang Zhang wrote: -Hi, - -We tested with the latest QEMU, and found that time drift obviously (clock fast -in guest) -in Windows 7 64 bits guest in some cases. - -It is easily to reproduce, using the follow QEMU command line to start windows -7: - -# x86_64-softmmu/qemu-system-x86_64 -name win7_64_2U_raw -machine -pc-i440fx-2.6,accel=kvm,usb=off -cpu host -m 2048 -realtime mlock=off -smp -4,sockets=2,cores=2,threads=1 -rtc base=utc,clock=vm,driftfix=slew -no-hpet --global kvm-pit.lost_tick_policy=discard -hda /mnt/nfs/win7_sp1_32_2U_raw -vnc -:11 -netdev tap,id=hn0,vhost=off -device rtl8139,id=net-pci0,netdev=hn0 -device -piix3-usb-uhci,id=usb -device usb-tablet,id=input0 -device usb-mouse,id=input1 --device usb-kbd,id=input2 -monitor stdio - -Adjust the VM's time to host time, and run java application or run the follow -program -in windows 7: - -#pragma comment(lib, "winmm") -#include -#include - -#define SWITCH_PEROID 13 - -int main() -{ - DWORD count = 0; - - while (1) - { - count++; - timeBeginPeriod(1); - DWORD start = timeGetTime(); - Sleep(40); - timeEndPeriod(1); - if ((count % SWITCH_PEROID) == 0) { - Sleep(1); - } - } - return 0; -} - -After few minutes, you will find that the time in windows 7 goes ahead of the -host time, drifts about several seconds. - -I have dug deeper in this problem. For windows systems that use the CMOS timer, -the base interrupt rate is usually 64Hz, but running some application in VM -will raise the timer rate to 1024Hz, running java application and or above -program will raise the timer rate. -Besides, Windows operating systems generally keep time by counting timer -interrupts (ticks). But QEMU seems not emulate the rate converting fine. - -We update the timer in function periodic_timer_update(): -static void periodic_timer_update(RTCState *s, int64_t current_time) -{ - - cur_clock = muldiv64(current_time, RTC_CLOCK_RATE, -get_ticks_per_sec()); - next_irq_clock = (cur_clock & ~(period - 1)) + period; - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Here we calculate the next interrupt time by align the current clock with the -new period, I'm a little confused that why we care about the *history* time ? -If VM switches from high rate to low rate, the next interrupt time may come -earlier than it supposed to be. We have observed it in our test. we printed the -interval time of interrupts and the VM's current time (We got the time from VM). - -Here is part of the log: -... ... -period=512 irq inject 1534: 15625 us -Tue Mar 29 04:38:00 2016 -*irq_num_period_32=0, irq_num_period_512=64: [3]: Real time interval is 999696 -us -... ... -*irq_num_period_32=893, irq_num_period_512=9 [81]: Real time interval is 951086 -us -Convert 32 --- > 512: 703: 96578 us -period=512 irq inject 44391: 12702 us -Convert 512 --- > 32: 704: 12704 us11 -period=32 irq inject 44392: 979 us -... ... -32 --- > 512: 705: 24388 us -period=512 irq inject 44417: 6834 us -Convert 512 --- > 32: 706: 6830 us -period=32 irq inject 44418: 978 us -... ... -Convert 32 --- > 512: 707: 60525 us -period=512 irq inject 44480: 1945 us -Convert 512 --- > 32: 708: 1955 us -period=32 irq inject 44481: 977 us -... ... -Convert 32 --- > 512: 709: 36105 us -period=512 irq inject 44518: 10741 us -Convert 512 --- > 32: 710: 10736 us -period=32 irq inject 44519: 989 us -... ... -Convert 32 --- > 512: 711: 123998 us -period=512 irq inject 44646: 974 us -period=512 irq inject 44647: 15607 us -Convert 512 --- > 32: 712: 16560 us -period=32 irq inject 44648: 980 us -... ... -period=32 irq inject 44738: 974 us -Convert 32 --- > 512: 713: 88828 us -period=512 irq inject 44739: 4885 us -Convert 512 --- > 32: 714: 4882 us -period=32 irq inject 44740: 989 us -... ... -period=32 irq inject 44842: 974 us -Convert 32 --- > 512: 715: 100537 us -period=512 irq inject 44843: 8788 us -Convert 512 --- > 32: 716: 8789 us -period=32 irq inject 44844: 972 us -... ... -period=32 irq inject 44941: 979 us -Convert 32 --- > 512: 717: 95677 us -period=512 irq inject 44942: 13661 us -Convert 512 --- > 32: 718: 13657 us -period=32 irq inject 44943: 987 us -... ... -Convert 32 --- > 512: 719: 94690 us -period=512 irq inject 45040: 14643 us -Convert 512 --- > 32: 720: 14642 us -period=32 irq inject 45041: 974 us -... ... -Convert 32 --- > 512: 721: 88848 us -period=512 irq inject 45132: 4892 us -Convert 512 --- > 32: 722: 4931 us -period=32 irq inject 45133: 964 us -... ... -Tue Mar 29 04:39:19 2016 -*irq_num_period_32:835, irq_num_period_512:11 [82], Real time interval is -911520 us - -For windows 7, it has got 835 IRQs which injected during the period of 32, -and got 11 IRQs that injected during the period of 512. it updated the -wall-clock -time with one second, because it supposed it has counted -(835*976.5+11*15625)= 987252.5 us, but the real interval time is 911520 us. - -IMHO, we should calculate the next interrupt time based on the time of last -interrupt injected, and it seems to be more similar with hardware CMOS timer -in this way. -Maybe someone can tell me the reason why we calculated the interrupt timer -in that way, or is it a bug ? ;) - -Thanks, -Hailiang - diff --git a/results/classifier/007/network/05479587 b/results/classifier/007/network/05479587 deleted file mode 100644 index 0e1b7068..00000000 --- a/results/classifier/007/network/05479587 +++ /dev/null @@ -1,93 +0,0 @@ -network: 0.963 -semantic: 0.866 -device: 0.811 -socket: 0.716 -performance: 0.669 -PID: 0.618 -permissions: 0.584 -graphic: 0.576 -boot: 0.474 -vnc: 0.464 -files: 0.395 -KVM: 0.374 -debug: 0.314 -other: 0.200 - -[Qemu-devel]  [BUG] network qga : windows os lost ip address of the network card  in some cases - -We think this problem coulde be solevd in qga modules。can anybody give some -advice ? - - -[BUG] network : windows os lost ip address of the network card in some cases - -we found this problem for a long time 。For example, if we has three network -card in virtual xml file ，such as "network connection 1" / "network connection -2"/"network connection 3" 。 - -Echo network card has own ip address ，such as 192.168.1.1 / 2.1 /3.1 , when -delete the first card ，reboot the windows virtual os, then this problem -happened ! - - - - -we found that the sencond network card will replace the first one , then the -ip address of "network connection 2 " become 192.168.1.1 。 - - -Our third party users began to complain about this bug 。All the business of the -second ip lost !!! - -I mean both of windows and linux has this bug , we solve this bug in linux -throught bonding netcrad pci and mac address 。 - -There is no good solution on windows os . thera are ? we implemented a plan to -resumption of IP by QGA. Is there a better way ? - - - - - - - - -原始邮件 - - - -发件人：尹作为10144574 -收件人： address@hidden -日 期 ：2017å¹´04月14日 16:46 -主 题 ：[BUG] network : windows os lost ip address of the network card in some -cases - - - - - - -we found this problem for a long time 。For example, if we has three network -card in virtual xml file ，such as "network connection 1" / "network connection -2"/"network connection 3" 。 - -Echo network card has own ip address ，such as 192.168.1.1 / 2.1 /3.1 , when -delete the first card ，reboot the windows virtual os, then this problem -happened ! - - - - -we found that the sencond network card will replace the first one , then the -ip address of "network connection 2 " become 192.168.1.1 。 - - -Our third party users began to complain about this bug 。All the business of the -second ip lost !!! - -I mean both of windows and linux has this bug , we solve this bug in linux -throught bonding netcrad pci and mac address 。 - -There is no good solution on windows os . thera are ? we implemented a plan to -resumption of IP by QGA. Is there a better way ? - diff --git a/results/classifier/007/network/62179944 b/results/classifier/007/network/62179944 deleted file mode 100644 index 912debb4..00000000 --- a/results/classifier/007/network/62179944 +++ /dev/null @@ -1,41 +0,0 @@ -network: 0.966 -graphic: 0.907 -device: 0.818 -performance: 0.636 -socket: 0.608 -boot: 0.567 -files: 0.565 -other: 0.519 -PID: 0.504 -vnc: 0.498 -semantic: 0.454 -permissions: 0.403 -debug: 0.400 -KVM: 0.153 - -[Qemu-devel] [BUG] network : windows os lost ip address of the network card  in some cases - -we found this problem for a long time 。For example, if we has three network -card in virtual xml file ，such as "network connection 1" / "network connection -2"/"network connection 3" 。 - -Echo network card has own ip address ，such as 192.168.1.1 / 2.1 /3.1 , when -delete the first card ，reboot the windows virtual os, then this problem -happened ! - - - - -we found that the sencond network card will replace the first one , then the -ip address of "network connection 2 " become 192.168.1.1 。 - - -Our third party users began to complain about this bug 。All the business of the -second ip lost !!! - -I mean both of windows and linux has this bug , we solve this bug in linux -throught bonding netcrad pci and mac address 。 - -There is no good solution on windows os . thera are ? we implemented a plan to -resumption of IP by QGA. Is there a better way ? - diff --git a/results/classifier/007/other/02364653 b/results/classifier/007/other/02364653 deleted file mode 100644 index e6bddb4a..00000000 --- a/results/classifier/007/other/02364653 +++ /dev/null @@ -1,373 +0,0 @@ -other: 0.956 -graphic: 0.948 -permissions: 0.944 -semantic: 0.942 -debug: 0.940 -PID: 0.938 -performance: 0.934 -device: 0.928 -boot: 0.925 -socket: 0.924 -vnc: 0.922 -KVM: 0.911 -files: 0.908 -network: 0.881 - -[Qemu-devel] [BUG] Inappropriate size of target_sigset_t - -Hello, Peter, Laurent, - -While working on another problem yesterday, I think I discovered a -long-standing bug in QEMU Linux user mode: our target_sigset_t structure is -eight times smaller as it should be! - -In this code segment from syscalls_def.h: - -#ifdef TARGET_MIPS -#define TARGET_NSIG 128 -#else -#define TARGET_NSIG 64 -#endif -#define TARGET_NSIG_BPW TARGET_ABI_BITS -#define TARGET_NSIG_WORDS (TARGET_NSIG / TARGET_NSIG_BPW) - -typedef struct { - abi_ulong sig[TARGET_NSIG_WORDS]; -} target_sigset_t; - -... TARGET_ABI_BITS should be replaced by eight times smaller constant (in -fact, semantically, we need TARGET_ABI_BYTES, but it is not defined) (what is -needed is actually "a byte per signal" in target_sigset_t, and we allow "a bit -per signal"). - -All this probably sounds to you like something impossible, since this code is -in QEMU "since forever", but I checked everything, and the bug seems real. I -wish you can prove me wrong. - -I just wanted to let you know about this, given the sensitive timing of current -softfreeze, and the fact that I won't be able to do more investigation on this -in coming weeks, since I am busy with other tasks, but perhaps you can analyze -and do something which you consider appropriate. - -Yours, -Aleksandar - -Le 03/07/2019 à 21:46, Aleksandar Markovic a écrit : -> -Hello, Peter, Laurent, -> -> -While working on another problem yesterday, I think I discovered a -> -long-standing bug in QEMU Linux user mode: our target_sigset_t structure is -> -eight times smaller as it should be! -> -> -In this code segment from syscalls_def.h: -> -> -#ifdef TARGET_MIPS -> -#define TARGET_NSIG 128 -> -#else -> -#define TARGET_NSIG 64 -> -#endif -> -#define TARGET_NSIG_BPW TARGET_ABI_BITS -> -#define TARGET_NSIG_WORDS (TARGET_NSIG / TARGET_NSIG_BPW) -> -> -typedef struct { -> -abi_ulong sig[TARGET_NSIG_WORDS]; -> -} target_sigset_t; -> -> -... TARGET_ABI_BITS should be replaced by eight times smaller constant (in -> -fact, semantically, we need TARGET_ABI_BYTES, but it is not defined) (what is -> -needed is actually "a byte per signal" in target_sigset_t, and we allow "a -> -bit per signal"). -TARGET_NSIG is divided by TARGET_ABI_BITS which gives you the number of -abi_ulong words we need in target_sigset_t. - -> -All this probably sounds to you like something impossible, since this code is -> -in QEMU "since forever", but I checked everything, and the bug seems real. I -> -wish you can prove me wrong. -> -> -I just wanted to let you know about this, given the sensitive timing of -> -current softfreeze, and the fact that I won't be able to do more -> -investigation on this in coming weeks, since I am busy with other tasks, but -> -perhaps you can analyze and do something which you consider appropriate. -If I compare with kernel, it looks good: - -In Linux: - - arch/mips/include/uapi/asm/signal.h - - #define _NSIG 128 - #define _NSIG_BPW (sizeof(unsigned long) * 8) - #define _NSIG_WORDS (_NSIG / _NSIG_BPW) - - typedef struct { - unsigned long sig[_NSIG_WORDS]; - } sigset_t; - -_NSIG_BPW is 8 * 8 = 64 on MIPS64 or 4 * 8 = 32 on MIPS - -In QEMU: - -TARGET_NSIG_BPW is TARGET_ABI_BITS which is TARGET_LONG_BITS which is -64 on MIPS64 and 32 on MIPS. - -I think there is no problem. - -Thanks, -Laurent - -From: Laurent Vivier -> -If I compare with kernel, it looks good: -> -... -> -I think there is no problem. -Sure, thanks for such fast response - again, I am glad if you are right. -However, for some reason, glibc (and musl too) define sigset_t differently than -kernel. Please take a look. I am not sure if this is covered fine in our code. - -Yours, -Aleksandar - -> -Thanks, -> -Laurent - -On Wed, 3 Jul 2019 at 21:20, Aleksandar Markovic wrote: -> -> -From: Laurent Vivier -> -> If I compare with kernel, it looks good: -> -> ... -> -> I think there is no problem. -> -> -Sure, thanks for such fast response - again, I am glad if you are right. -> -However, for some reason, glibc (and musl too) define sigset_t differently -> -than kernel. Please take a look. I am not sure if this is covered fine in our -> -code. -Yeah, the libc definitions of sigset_t don't match the -kernel ones (this is for obscure historical reasons IIRC). -We're providing implementations of the target -syscall interface, so our target_sigset_t should be the -target kernel's version (and the target libc's version doesn't -matter to us). On the other hand we will be using the -host libc version, I think, so a little caution is required -and it's possible we have some bugs in our code. - -thanks --- PMM - -> -From: Peter Maydell -> -> -On Wed, 3 Jul 2019 at 21:20, Aleksandar Markovic wrote: -> -> -> -> From: Laurent Vivier -> -> > If I compare with kernel, it looks good: -> -> > ... -> -> > I think there is no problem. -> -> -> -> Sure, thanks for such fast response - again, I am glad if you are right. -> -> However, for some reason, glibc (and musl too) define sigset_t differently -> -> than kernel. Please take a look. I am not sure if this is covered fine in -> -> our code. -> -> -Yeah, the libc definitions of sigset_t don't match the -> -kernel ones (this is for obscure historical reasons IIRC). -> -We're providing implementations of the target -> -syscall interface, so our target_sigset_t should be the -> -target kernel's version (and the target libc's version doesn't -> -matter to us). On the other hand we will be using the -> -host libc version, I think, so a little caution is required -> -and it's possible we have some bugs in our code. -OK, I gather than this is not something that requires our immediate attention -(for 4.1), but we can analyze it later on. - -Thanks for response!! - -Sincerely, -Aleksandar - -> -thanks -> --- PMM - -Le 03/07/2019 à 22:28, Peter Maydell a écrit : -> -On Wed, 3 Jul 2019 at 21:20, Aleksandar Markovic wrote: -> -> -> -> From: Laurent Vivier -> ->> If I compare with kernel, it looks good: -> ->> ... -> ->> I think there is no problem. -> -> -> -> Sure, thanks for such fast response - again, I am glad if you are right. -> -> However, for some reason, glibc (and musl too) define sigset_t differently -> -> than kernel. Please take a look. I am not sure if this is covered fine in -> -> our code. -> -> -Yeah, the libc definitions of sigset_t don't match the -> -kernel ones (this is for obscure historical reasons IIRC). -> -We're providing implementations of the target -> -syscall interface, so our target_sigset_t should be the -> -target kernel's version (and the target libc's version doesn't -> -matter to us). On the other hand we will be using the -> -host libc version, I think, so a little caution is required -> -and it's possible we have some bugs in our code. -It's why we need host_to_target_sigset_internal() and -target_to_host_sigset_internal() that translates bits and bytes between -guest kernel interface and host libc interface. - -void host_to_target_sigset_internal(target_sigset_t *d, - const sigset_t *s) -{ - int i; - target_sigemptyset(d); - for (i = 1; i <= TARGET_NSIG; i++) { - if (sigismember(s, i)) { - target_sigaddset(d, host_to_target_signal(i)); - } - } -} - -void target_to_host_sigset_internal(sigset_t *d, - const target_sigset_t *s) -{ - int i; - sigemptyset(d); - for (i = 1; i <= TARGET_NSIG; i++) { - if (target_sigismember(s, i)) { - sigaddset(d, target_to_host_signal(i)); - } - } -} - -Thanks, -Laurent - -Hi Aleksandar, - -On Wed, Jul 3, 2019 at 12:48 PM Aleksandar Markovic - wrote: -> -#define TARGET_NSIG_BPW TARGET_ABI_BITS -> -#define TARGET_NSIG_WORDS (TARGET_NSIG / TARGET_NSIG_BPW) -> -> -typedef struct { -> -abi_ulong sig[TARGET_NSIG_WORDS]; -> -} target_sigset_t; -> -> -... TARGET_ABI_BITS should be replaced by eight times smaller constant (in -> -fact, -> -semantically, we need TARGET_ABI_BYTES, but it is not defined) (what is needed -> -is actually "a byte per signal" in target_sigset_t, and we allow "a bit per -> -signal"). -Why do we need a byte per target signal, if the functions in linux-user/signal.c -operate with bits? - --- -Thanks. --- Max - -> -Why do we need a byte per target signal, if the functions in -> -linux-user/signal.c -> -operate with bits? -Max, - -I did not base my findings on code analysis, but on dumping size/offsets of -elements of some structures, as they are emulated in QEMU, and in real systems. -So, I can't really answer your question. - -Yours, -Aleksandar - -> --- -> -Thanks. -> --- Max - diff --git a/results/classifier/007/other/02572177 b/results/classifier/007/other/02572177 deleted file mode 100644 index 96f1989e..00000000 --- a/results/classifier/007/other/02572177 +++ /dev/null @@ -1,431 +0,0 @@ -other: 0.869 -permissions: 0.812 -device: 0.791 -performance: 0.781 -semantic: 0.770 -debug: 0.756 -graphic: 0.747 -socket: 0.742 -PID: 0.731 -network: 0.708 -vnc: 0.706 -KVM: 0.669 -boot: 0.658 -files: 0.640 - -[Qemu-devel] 答复: Re: [BUG]COLO failover hang - -hi. - - -I test the git qemu master have the same problem. - - -(gdb) bt - - -#0 qio_channel_socket_readv (ioc=0x7f65911b4e50, iov=0x7f64ef3fd880, niov=1, -fds=0x0, nfds=0x0, errp=0x0) at io/channel-socket.c:461 - - -#1 0x00007f658e4aa0c2 in qio_channel_read (address@hidden, address@hidden "", -address@hidden, address@hidden) at io/channel.c:114 - - -#2 0x00007f658e3ea990 in channel_get_buffer (opaque=＜optimized out＞, -buf=0x7f65907cb838 "", pos=＜optimized out＞, size=32768) at -migration/qemu-file-channel.c:78 - - -#3 0x00007f658e3e97fc in qemu_fill_buffer (f=0x7f65907cb800) at -migration/qemu-file.c:295 - - -#4 0x00007f658e3ea2e1 in qemu_peek_byte (address@hidden, address@hidden) at -migration/qemu-file.c:555 - - -#5 0x00007f658e3ea34b in qemu_get_byte (address@hidden) at -migration/qemu-file.c:568 - - -#6 0x00007f658e3ea552 in qemu_get_be32 (address@hidden) at -migration/qemu-file.c:648 - - -#7 0x00007f658e3e66e5 in colo_receive_message (f=0x7f65907cb800, -address@hidden) at migration/colo.c:244 - - -#8 0x00007f658e3e681e in colo_receive_check_message (f=＜optimized out＞, -address@hidden, address@hidden) - - - at migration/colo.c:264 - - -#9 0x00007f658e3e740e in colo_process_incoming_thread (opaque=0x7f658eb30360 -＜mis_current.31286＞) at migration/colo.c:577 - - -#10 0x00007f658be09df3 in start_thread () from /lib64/libpthread.so.0 - - -#11 0x00007f65881983ed in clone () from /lib64/libc.so.6 - - -(gdb) p ioc-＞name - - -$2 = 0x7f658ff7d5c0 "migration-socket-incoming" - - -(gdb) p ioc-＞features Do not support QIO_CHANNEL_FEATURE_SHUTDOWN - - -$3 = 0 - - - - - -(gdb) bt - - -#0 socket_accept_incoming_migration (ioc=0x7fdcceeafa90, condition=G_IO_IN, -opaque=0x7fdcceeafa90) at migration/socket.c:137 - - -#1 0x00007fdcc6966350 in g_main_dispatch (context=＜optimized out＞) at -gmain.c:3054 - - -#2 g_main_context_dispatch (context=＜optimized out＞, address@hidden) at -gmain.c:3630 - - -#3 0x00007fdccb8a6dcc in glib_pollfds_poll () at util/main-loop.c:213 - - -#4 os_host_main_loop_wait (timeout=＜optimized out＞) at util/main-loop.c:258 - - -#5 main_loop_wait (address@hidden) at util/main-loop.c:506 - - -#6 0x00007fdccb526187 in main_loop () at vl.c:1898 - - -#7 main (argc=＜optimized out＞, argv=＜optimized out＞, envp=＜optimized out＞) at -vl.c:4709 - - -(gdb) p ioc-＞features - - -$1 = 6 - - -(gdb) p ioc-＞name - - -$2 = 0x7fdcce1b1ab0 "migration-socket-listener" - - - - - -May be socket_accept_incoming_migration should call -qio_channel_set_feature(ioc, QIO_CHANNEL_FEATURE_SHUTDOWN)?? - - - - - -thank you. - - - - - - - - - - - - - - - -原始邮件 - - - -发件人： address@hidden -收件人：王广10165992 address@hidden -抄送人： address@hidden address@hidden -日 期 ：2017å¹´03月16日 14:46 -主 题 ：Re: [Qemu-devel] COLO failover hang - - - - - - - -On 03/15/2017 05:06 PM, wangguang wrote: -＞ am testing QEMU COLO feature described here [QEMU -＞ Wiki]( -http://wiki.qemu-project.org/Features/COLO -). -＞ -＞ When the Primary Node panic,the Secondary Node qemu hang. -＞ hang at recvmsg in qio_channel_socket_readv. -＞ And I run { 'execute': 'nbd-server-stop' } and { "execute": -＞ "x-colo-lost-heartbeat" } in Secondary VM's -＞ monitor,the Secondary Node qemu still hang at recvmsg . -＞ -＞ I found that the colo in qemu is not complete yet. -＞ Do the colo have any plan for development? - -Yes, We are developing. You can see some of patch we pushing. - -＞ Has anyone ever run it successfully? Any help is appreciated! - -In our internal version can run it successfully, -The failover detail you can ask Zhanghailiang for help. -Next time if you have some question about COLO, -please cc me and zhanghailiang address@hidden - - -Thanks -Zhang Chen - - -＞ -＞ -＞ -＞ centos7.2+qemu2.7.50 -＞ (gdb) bt -＞ #0 0x00007f3e00cc86ad in recvmsg () from /lib64/libpthread.so.0 -＞ #1 0x00007f3e0332b738 in qio_channel_socket_readv (ioc=＜optimized out＞, -＞ iov=＜optimized out＞, niov=＜optimized out＞, fds=0x0, nfds=0x0, errp=0x0) at -＞ io/channel-socket.c:497 -＞ #2 0x00007f3e03329472 in qio_channel_read (address@hidden, -＞ address@hidden "", address@hidden, -＞ address@hidden) at io/channel.c:97 -＞ #3 0x00007f3e032750e0 in channel_get_buffer (opaque=＜optimized out＞, -＞ buf=0x7f3e05910f38 "", pos=＜optimized out＞, size=32768) at -＞ migration/qemu-file-channel.c:78 -＞ #4 0x00007f3e0327412c in qemu_fill_buffer (f=0x7f3e05910f00) at -＞ migration/qemu-file.c:257 -＞ #5 0x00007f3e03274a41 in qemu_peek_byte (address@hidden, -＞ address@hidden) at migration/qemu-file.c:510 -＞ #6 0x00007f3e03274aab in qemu_get_byte (address@hidden) at -＞ migration/qemu-file.c:523 -＞ #7 0x00007f3e03274cb2 in qemu_get_be32 (address@hidden) at -＞ migration/qemu-file.c:603 -＞ #8 0x00007f3e03271735 in colo_receive_message (f=0x7f3e05910f00, -＞ address@hidden) at migration/colo.c:215 -＞ #9 0x00007f3e0327250d in colo_wait_handle_message (errp=0x7f3d62bfaa48, -＞ checkpoint_request=＜synthetic pointer＞, f=＜optimized out＞) at -＞ migration/colo.c:546 -＞ #10 colo_process_incoming_thread (opaque=0x7f3e067245e0) at -＞ migration/colo.c:649 -＞ #11 0x00007f3e00cc1df3 in start_thread () from /lib64/libpthread.so.0 -＞ #12 0x00007f3dfc9c03ed in clone () from /lib64/libc.so.6 -＞ -＞ -＞ -＞ -＞ -＞ -- -＞ View this message in context: -http://qemu.11.n7.nabble.com/COLO-failover-hang-tp473250.html -＞ Sent from the Developer mailing list archive at Nabble.com. -＞ -＞ -＞ -＞ - --- -Thanks -Zhang Chen - -Hi,Wang. - -You can test this branch: -https://github.com/coloft/qemu/tree/colo-v5.1-developing-COLO-frame-v21-with-shared-disk -and please follow wiki ensure your own configuration correctly. -http://wiki.qemu-project.org/Features/COLO -Thanks - -Zhang Chen - - -On 03/21/2017 03:27 PM, address@hidden wrote: -hi. - -I test the git qemu master have the same problem. - -(gdb) bt -#0 qio_channel_socket_readv (ioc=0x7f65911b4e50, iov=0x7f64ef3fd880, -niov=1, fds=0x0, nfds=0x0, errp=0x0) at io/channel-socket.c:461 -#1 0x00007f658e4aa0c2 in qio_channel_read -(address@hidden, address@hidden "", -address@hidden, address@hidden) at io/channel.c:114 -#2 0x00007f658e3ea990 in channel_get_buffer (opaque=＜optimized out＞, -buf=0x7f65907cb838 "", pos=＜optimized out＞, size=32768) at -migration/qemu-file-channel.c:78 -#3 0x00007f658e3e97fc in qemu_fill_buffer (f=0x7f65907cb800) at -migration/qemu-file.c:295 -#4 0x00007f658e3ea2e1 in qemu_peek_byte (address@hidden, -address@hidden) at migration/qemu-file.c:555 -#5 0x00007f658e3ea34b in qemu_get_byte (address@hidden) at -migration/qemu-file.c:568 -#6 0x00007f658e3ea552 in qemu_get_be32 (address@hidden) at -migration/qemu-file.c:648 -#7 0x00007f658e3e66e5 in colo_receive_message (f=0x7f65907cb800, -address@hidden) at migration/colo.c:244 -#8 0x00007f658e3e681e in colo_receive_check_message (f=＜optimized -out＞, address@hidden, -address@hidden) -at migration/colo.c:264 -#9 0x00007f658e3e740e in colo_process_incoming_thread -(opaque=0x7f658eb30360 ＜mis_current.31286＞) at migration/colo.c:577 -#10 0x00007f658be09df3 in start_thread () from /lib64/libpthread.so.0 - -#11 0x00007f65881983ed in clone () from /lib64/libc.so.6 - -(gdb) p ioc-＞name - -$2 = 0x7f658ff7d5c0 "migration-socket-incoming" - -(gdb) p ioc-＞features Do not support QIO_CHANNEL_FEATURE_SHUTDOWN - -$3 = 0 - - -(gdb) bt -#0 socket_accept_incoming_migration (ioc=0x7fdcceeafa90, -condition=G_IO_IN, opaque=0x7fdcceeafa90) at migration/socket.c:137 -#1 0x00007fdcc6966350 in g_main_dispatch (context=＜optimized out＞) at -gmain.c:3054 -#2 g_main_context_dispatch (context=＜optimized out＞, -address@hidden) at gmain.c:3630 -#3 0x00007fdccb8a6dcc in glib_pollfds_poll () at util/main-loop.c:213 -#4 os_host_main_loop_wait (timeout=＜optimized out＞) at -util/main-loop.c:258 -#5 main_loop_wait (address@hidden) at -util/main-loop.c:506 -#6 0x00007fdccb526187 in main_loop () at vl.c:1898 -#7 main (argc=＜optimized out＞, argv=＜optimized out＞, envp=＜optimized -out＞) at vl.c:4709 -(gdb) p ioc-＞features - -$1 = 6 - -(gdb) p ioc-＞name - -$2 = 0x7fdcce1b1ab0 "migration-socket-listener" -May be socket_accept_incoming_migration should -call qio_channel_set_feature(ioc, QIO_CHANNEL_FEATURE_SHUTDOWN)?? -thank you. - - - - - -原始邮件 -address@hidden; -*收件人：*王广10165992;address@hidden; -address@hidden;address@hidden; -*日 期 ：*2017å¹´03月16日 14:46 -*主 题 ：**Re: [Qemu-devel] COLO failover hang* - - - - -On 03/15/2017 05:06 PM, wangguang wrote: -＞ am testing QEMU COLO feature described here [QEMU -＞ Wiki]( -http://wiki.qemu-project.org/Features/COLO -). -＞ -＞ When the Primary Node panic,the Secondary Node qemu hang. -＞ hang at recvmsg in qio_channel_socket_readv. -＞ And I run { 'execute': 'nbd-server-stop' } and { "execute": -＞ "x-colo-lost-heartbeat" } in Secondary VM's -＞ monitor,the Secondary Node qemu still hang at recvmsg . -＞ -＞ I found that the colo in qemu is not complete yet. -＞ Do the colo have any plan for development? - -Yes, We are developing. You can see some of patch we pushing. - -＞ Has anyone ever run it successfully? Any help is appreciated! - -In our internal version can run it successfully, -The failover detail you can ask Zhanghailiang for help. -Next time if you have some question about COLO, -please cc me and zhanghailiang address@hidden - - -Thanks -Zhang Chen - - -＞ -＞ -＞ -＞ centos7.2+qemu2.7.50 -＞ (gdb) bt -＞ #0 0x00007f3e00cc86ad in recvmsg () from /lib64/libpthread.so.0 -＞ #1 0x00007f3e0332b738 in qio_channel_socket_readv (ioc=＜optimized out＞, -＞ iov=＜optimized out＞, niov=＜optimized out＞, fds=0x0, nfds=0x0, errp=0x0) at -＞ io/channel-socket.c:497 -＞ #2 0x00007f3e03329472 in qio_channel_read (address@hidden, -＞ address@hidden "", address@hidden, -＞ address@hidden) at io/channel.c:97 -＞ #3 0x00007f3e032750e0 in channel_get_buffer (opaque=＜optimized out＞, -＞ buf=0x7f3e05910f38 "", pos=＜optimized out＞, size=32768) at -＞ migration/qemu-file-channel.c:78 -＞ #4 0x00007f3e0327412c in qemu_fill_buffer (f=0x7f3e05910f00) at -＞ migration/qemu-file.c:257 -＞ #5 0x00007f3e03274a41 in qemu_peek_byte (address@hidden, -＞ address@hidden) at migration/qemu-file.c:510 -＞ #6 0x00007f3e03274aab in qemu_get_byte (address@hidden) at -＞ migration/qemu-file.c:523 -＞ #7 0x00007f3e03274cb2 in qemu_get_be32 (address@hidden) at -＞ migration/qemu-file.c:603 -＞ #8 0x00007f3e03271735 in colo_receive_message (f=0x7f3e05910f00, -＞ address@hidden) at migration/colo.c:215 -＞ #9 0x00007f3e0327250d in colo_wait_handle_message (errp=0x7f3d62bfaa48, -＞ checkpoint_request=＜synthetic pointer＞, f=＜optimized out＞) at -＞ migration/colo.c:546 -＞ #10 colo_process_incoming_thread (opaque=0x7f3e067245e0) at -＞ migration/colo.c:649 -＞ #11 0x00007f3e00cc1df3 in start_thread () from /lib64/libpthread.so.0 -＞ #12 0x00007f3dfc9c03ed in clone () from /lib64/libc.so.6 -＞ -＞ -＞ -＞ -＞ -＞ -- -＞ View this message in context: -http://qemu.11.n7.nabble.com/COLO-failover-hang-tp473250.html -＞ Sent from the Developer mailing list archive at Nabble.com. -＞ -＞ -＞ -＞ - --- -Thanks -Zhang Chen --- -Thanks -Zhang Chen - diff --git a/results/classifier/007/other/04472277 b/results/classifier/007/other/04472277 deleted file mode 100644 index 9cc865b2..00000000 --- a/results/classifier/007/other/04472277 +++ /dev/null @@ -1,586 +0,0 @@ -KVM: 0.890 -permissions: 0.851 -device: 0.849 -debug: 0.849 -network: 0.847 -graphic: 0.846 -other: 0.846 -performance: 0.841 -boot: 0.831 -vnc: 0.828 -PID: 0.826 -socket: 0.824 -semantic: 0.815 -files: 0.790 - -[BUG][KVM_SET_USER_MEMORY_REGION] KVM_SET_USER_MEMORY_REGION failed - -Hi all, -I start a VM in openstack, and openstack use libvirt to start qemu VM, but now log show this ERROR. -Is there any one know this? -The ERROR log from /var/log/libvirt/qemu/instance-0000000e.log -``` -2023-03-14T10:09:17.674114Z qemu-system-x86_64: kvm_set_user_memory_region: KVM_SET_USER_MEMORY_REGION failed, slot=4, start=0xfffffffffe000000, size=0x2000: Invalid argument -kvm_set_phys_mem: error registering slot: Invalid argument -2023-03-14 10:09:18.198+0000: shutting down, reason=crashed -``` -The xml file -``` -root@c1c2:~# cat /etc/libvirt/qemu/instance-0000000e.xml - - -  instance-0000000e -  ff91d2dc-69a1-43ef-abde-c9e4e9a0305b -  -    -      -      provider-instance -      2023-03-14 10:09:13 -      -        64 -        1 -        0 -        0 -        1 -      -      -        admin -        admin -      -      -      -        -          -        -      -    -  -  65536 -  65536 -  1 -  -    -      OpenStack Foundation -      OpenStack Nova -      25.1.0 -      ff91d2dc-69a1-43ef-abde-c9e4e9a0305b -      ff91d2dc-69a1-43ef-abde-c9e4e9a0305b -      Virtual Machine -    -  -  -    hvm -    -    -  -  -    -    -    -  -  -    -  -  -    -    -    -  -  destroy -  restart -  destroy -  -    /usr/bin/qemu-system-x86_64 -    -      -      -      -     
-    -    -     
-    -    -    -      -      -       
-      -     
-    -    -      -      -        -      -    -    -      -      -    -    -     
-    -    -    -    -      -    -  Â