1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
|
Not able to launch guests in ppc64le P9 OPAL
Description of problem:
Not able to launch guests in ppc64le P9 OPAL
Steps to reproduce:
1. In a RHEL8 using 4.18.0-305.3.1.el8_4.ppc64le create a Fedora CoreOS VM using kernel-5.15.17-200.fc35.ppc64le.
2. Inside the FCOS vm run:
```
virt-install --import \
--name buildvm-ppc64le-fcos01.iad2.fedoraproject.org \
--memory=32768,maxmemory=32768 \
--vcpus=16,maxvcpus=16 \
--feature nested-hv=on \
--network bridge=br0,model=virtio,mac=RANDOM \
--autostart \
--memballoon virtio \
--watchdog default \
--rng /dev/random \
--noautoconsole \
--disk path=$PWD/fcos-ppc64le-builder.ign,format=raw,readonly=on,serial=ignition \
--disk bus=virtio,path=/dev/vg_guests/buildvm-ppc64le-fcos01.iad2.fedoraproject.org,cache=unsafe,io=threads
```
3. Try to run it again and you will get:
```
KVM: Failed to create TCE64 table for liobn 0x71000002
KVM: Failed to create TCE64 table for liobn 0x80000000
KVM: unknown exit, hardware reason ffffffffffffffc9
NIP 0000000000000100 LR 0000000000000000 CTR 0000000000000000 XER 0000000000000000 CPU#0
MSR 8000000000001000 HID0 0000000000000000 HF 6c000004 iidx 3 didx 3
TB 00000000 00000000 DECR 0
GPR00 0000000000000000 0000000000000000 0000000000000000 000000007fe00000
GPR04 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR08 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR12 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR16 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR20 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR24 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR28 0000000000000000 0000000000000000 0000000000000000 0000000000000000
CR 00000000 [ - - - - - - - - ] RES ffffffffffffffff
SRR0 0000000000000000 SRR1 0000000000000000 PVR 00000000004e1202 VRSAVE 0000000000000000
SPRG0 0000000000000000 SPRG1 0000000000000000 SPRG2 0000000000000000 SPRG3 0000000000000000
SPRG4 0000000000000000 SPRG5 0000000000000000 SPRG6 0000000000000000 SPRG7 0000000000000000
HSRR0 0000000000000000 HSRR1 0000000000000000
CFAR 0000000000000000
LPCR 0000000000560413
PTCR 0000000000000000 DAR 0000000000000000 DSISR 0000000000000000
```
Additional information:
Fedora xml:
```
<domain type='kvm' id='24'>
<name>buildvm-ppc64le-fcos01.iad2.fedoraproject.org</name>
<uuid>ed30c95e-b7c0-4c25-a6ba-b739459f101b</uuid>
<memory unit='KiB'>33554432</memory>
<currentMemory unit='KiB'>33554432</currentMemory>
<vcpu placement='static'>16</vcpu>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='ppc64le' machine='pseries-rhel8.3.0'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<nested-hv state='on'/>
</features>
<cpu mode='custom' match='exact' check='none'>
<model fallback='forbid'>POWER9</model>
</cpu>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='unsafe' io='threads'/>
<source dev='/dev/vg_guests/buildvm-ppc64le-fcos01.iad2.fedoraproject.org' index='2'/>
<backingStore/>
<target dev='vda' bus='virtio'/>
<alias name='virtio-disk0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='raw'/>
<source file='/tmp/fcos-ppc64le-builder.ign' index='1'/>
<backingStore/>
<target dev='vdb' bus='virtio'/>
<readonly/>
<serial>ignition</serial>
<alias name='virtio-disk1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</disk>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<alias name='usb'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</controller>
<controller type='pci' index='0' model='pci-root'>
<model name='spapr-pci-host-bridge'/>
<target index='0'/>
<alias name='pci.0'/>
</controller>
<controller type='virtio-serial' index='0'>
<alias name='virtio-serial0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</controller>
<interface type='bridge'>
<mac address='52:54:00:c4:d2:aa'/>
<source bridge='br0'/>
<target dev='vnet23'/>
<model type='virtio'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/11'/>
<target type='spapr-vio-serial' port='0'>
<model name='spapr-vty'/>
</target>
<alias name='serial0'/>
<address type='spapr-vio' reg='0x30000000'/>
</serial>
<console type='pty' tty='/dev/pts/11'>
<source path='/dev/pts/11'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
<address type='spapr-vio' reg='0x30000000'/>
</console>
<channel type='unix'>
<source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-24-buildvm-ppc64le-fcos/org.qemu.guest_agent.0'/>
<target type='virtio' name='org.qemu.guest_agent.0' state='disconnected'/>
<alias name='channel0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='tablet' bus='usb'>
<alias name='input0'/>
<address type='usb' bus='0' port='1'/>
</input>
<input type='keyboard' bus='usb'>
<alias name='input1'/>
<address type='usb' bus='0' port='2'/>
</input>
<graphics type='vnc' port='5910' autoport='yes' listen='127.0.0.1'>
<listen type='address' address='127.0.0.1'/>
</graphics>
<video>
<model type='vga' vram='16384' heads='1' primary='yes'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
</video>
<watchdog model='i6300esb' action='reset'>
<alias name='watchdog0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</watchdog>
<memballoon model='virtio'>
<alias name='balloon0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</memballoon>
<rng model='virtio'>
<backend model='random'>/dev/random</backend>
<alias name='rng0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</rng>
<panic model='pseries'/>
</devices>
<seclabel type='dynamic' model='selinux' relabel='yes'>
<label>system_u:system_r:svirt_t:s0:c131,c913</label>
<imagelabel>system_u:object_r:svirt_image_t:s0:c131,c913</imagelabel>
</seclabel>
<seclabel type='dynamic' model='dac' relabel='yes'>
<label>+107:+107</label>
<imagelabel>+107:+107</imagelabel>
</seclabel>
</domain>
```
Failure seen in journal when running `virt-ls`
```
Feb 04 16:19:39 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: KVMPPC-UVMEM: No support for secure guests
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: vcpu 000000004bd9d345 (0):
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: pc = 0000000000000100 msr = 8000000000001000 trap = ffffffc9
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 0 = 0000000000000000 r16 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 1 = 0000000000000000 r17 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 2 = 0000000000000000 r18 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 3 = 000000003fe00000 r19 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 4 = 0000000000000000 r20 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 5 = 0000000000000000 r21 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 6 = 0000000000000000 r22 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 7 = 0000000000000000 r23 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 8 = 0000000000000000 r24 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r 9 = 0000000000000000 r25 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r10 = 0000000000000000 r26 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r11 = 0000000000000000 r27 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r12 = 0000000000000000 r28 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r13 = 0000000000000000 r29 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r14 = 0000000000000000 r30 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: r15 = 0000000000000000 r31 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: ctr = 0000000000000000 lr = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: srr0 = 0000000000000000 srr1 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: sprg0 = 0000000000000000 sprg1 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: sprg2 = 0000000000000000 sprg3 = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: cr = 00000000 xer = 0000000000000000 dsisr = 00000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: dar = 0000000000000000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: fault dar = 0000000000000000 dsisr = 0c68f000
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: SLB (0 entries):
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: lpcr = 0040000000560413 sdr1 = 0000000000000000 last_inst = ffffffff
Feb 04 16:19:40 buildvm-ppc64le-fcos01.iad2.fedoraproject.org kernel: trap=0xffffffc9 | pc=0x100 | msr=0x8000000000001000
```
Running via qemu:
```
qemu-system-ppc64 -m 2048 -machine pseries,accel=kvm,kvm-type=HV -cpu host -nographic -snapshot -drive if=virtio,file=fedora-coreos-35.20220131.dev.0-qemu.ppc64le.qcow2
KVM: unknown exit, hardware reason ffffffffffffffc9
NIP 0000000000000100 LR 0000000000000000 CTR 0000000000000000 XER 0000000000000000 CPU#0
MSR 8000000000001000 HID0 0000000000000000 HF 6c000004 iidx 3 didx 3
TB 00000000 00000000 DECR 0
GPR00 0000000000000000 0000000000000000 0000000000000000 000000007fe00000
GPR04 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR08 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR12 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR16 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR20 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR24 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR28 0000000000000000 0000000000000000 0000000000000000 0000000000000000
CR 00000000 [ - - - - - - - - ] RES ffffffffffffffff
SRR0 0000000000000000 SRR1 0000000000000000 PVR 00000000004e1202 VRSAVE 0000000000000000
SPRG0 0000000000000000 SPRG1 0000000000000000 SPRG2 0000000000000000 SPRG3 0000000000000000
SPRG4 0000000000000000 SPRG5 0000000000000000 SPRG6 0000000000000000 SPRG7 0000000000000000
HSRR0 0000000000000000 HSRR1 0000000000000000
CFAR 0000000000000000
LPCR 0000000000560413
PTCR 0000000000000000 DAR 0000000000000000 DSISR 0000000000000000
```
libguestfs-test-tool also fails to launch guest
```
2022-02-04 18:10:02.355+0000: starting up libvirt version: 7.6.0, package: 5.fc35 (Fedora Project, 2021-12-16-17:58:25, ), qemu version: 6.1.0qemu-6.1.0-10.fc35, kernel: 5.15.17-200.fc35.ppc64le, hostname: buildvm-ppc64le-fcos01.iad2.fedoraproject.org
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-1-guestfs-9ee177vxogzf \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-guestfs-9ee177vxogzf/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-guestfs-9ee177vxogzf/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-guestfs-9ee177vxogzf/.config \
TMPDIR=/var/tmp \
/usr/bin/qemu-system-ppc64 \
-name guest=guestfs-9ee177vxogzfyj3v,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-guestfs-9ee177vxogzf/master-key.aes"}' \
-machine pseries-6.1,accel=kvm,usb=off,dump-guest-core=off,memory-backend=ppc_spapr.ram \
-cpu POWER9 \
-m 1280 \
-object '{"qom-type":"memory-backend-ram","id":"ppc_spapr.ram","size":1342177280}' \
-overcommit mem-lock=off \
-smp 1,sockets=1,cores=1,threads=1 \
-uuid 08cd47d3-91e1-4322-aa53-6665a9bc13c8 \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=22,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=utc,driftfix=slew \
-no-reboot \
-boot strict=on \
-kernel /var/tmp/.guestfs-0/appliance.d/kernel \
-initrd /var/tmp/.guestfs-0/appliance.d/initrd \
-append 'panic=1 console=hvc0 console=ttyS0 edd=off udevtimeout=6000 udev.event-timeout=6000 no_timer_check printk.time=1 cgroup_disable=memory usbcore.nousb cryptomgr.notests tsc=reliable 8250.nr_uarts=1 root=UUID=0c185770-d5fc-4a67-acc9-3ea85178bda2 selinux=0 guestfs_verbose=1 TERM=screen' \
-device virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x1 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.0,addr=0x2 \
-blockdev '{"driver":"file","filename":"/tmp/libguestfskYy342/scratch1.img","node-name":"libvirt-2-storage","cache":{"direct":false,"no-flush":true},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":false,"cache":{"direct":false,"no-flush":true},"driver":"raw","file":"libvirt-2-storage"}' \
-device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,device_id=drive-scsi0-0-0-0,drive=libvirt-2-format,id=scsi0-0-0-0,bootindex=1,write-cache=on \
-blockdev '{"driver":"file","filename":"/var/tmp/.guestfs-0/appliance.d/root","node-name":"libvirt-3-storage","cache":{"direct":false,"no-flush":true},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-3-format","read-only":true,"cache":{"direct":false,"no-flush":true},"driver":"raw","file":"libvirt-3-storage"}' \
-blockdev '{"driver":"file","filename":"/tmp/libguestfskYy342/overlay2.qcow2","node-name":"libvirt-1-storage","cache":{"direct":false,"no-flush":true},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":false,"no-flush":true},"driver":"qcow2","file":"libvirt-1-storage","backing":"libvirt-3-format"}' \
-device scsi-hd,bus=scsi0.0,channel=0,scsi-id=1,lun=0,device_id=drive-scsi0-0-1-0,drive=libvirt-1-format,id=scsi0-0-1-0,write-cache=on \
-chardev socket,id=charserial0,path=/tmp/libguestfsFFWbf9/console.sock \
-device spapr-vty,chardev=charserial0,id=serial0,reg=0x30000000 \
-chardev socket,id=charchannel0,path=/tmp/libguestfsFFWbf9/guestfsd.sock \
-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.libguestfs.channel.0 \
-audiodev id=audio1,driver=none \
-object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"}' \
-device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.0,addr=0x3 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
KVM: unknown exit, hardware reason ffffffffffffffc9
NIP 0000000000000100 LR 0000000000000000 CTR 0000000000000000 XER 0000000000000000 CPU#0
MSR 8000000000001000 HID0 0000000000000000 HF 6c000004 iidx 3 didx 3
TB 00000000 00000000 DECR 0
GPR00 0000000000000000 0000000000000000 0000000000000000 000000003fe00000
GPR04 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR08 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR12 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR16 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR20 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR24 0000000000000000 0000000000000000 0000000000000000 0000000000000000
GPR28 0000000000000000 0000000000000000 0000000000000000 0000000000000000
CR 00000000 [ - - - - - - - - ] RES ffffffffffffffff
SRR0 0000000000000000 SRR1 0000000000000000 PVR 00000000004e1202 VRSAVE 0000000000000000
SPRG0 0000000000000000 SPRG1 0000000000000000 SPRG2 0000000000000000 SPRG3 0000000000000000
SPRG4 0000000000000000 SPRG5 0000000000000000 SPRG6 0000000000000000 SPRG7 0000000000000000
HSRR0 0000000000000000 HSRR1 0000000000000000
CFAR 0000000000000000
LPCR 0000000000560413
PTCR 0000000000000000 DAR 0000000000000000 DSISR 0000000000000000
2022-02-04T18:19:47.323915Z qemu-system-ppc64: terminating on signal 15 from pid 1645 (<unknown process>)
2022-02-04 18:19:47.524+0000: shutting down, reason=destroyed
```
|