1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
|
graphic: 0.985
semantic: 0.981
assembly: 0.981
mistranslation: 0.981
device: 0.980
other: 0.978
vnc: 0.978
instruction: 0.977
socket: 0.972
boot: 0.967
KVM: 0.959
network: 0.926
test-aio-multithread fails with 'Co-routine re-entered recursively'
git head is at fa73e146250181852c0915aa65df8d54d35485fa
configure with the following
./configure --enable-attr --enable-bsd-user --enable-cap-ng\
--enable-coroutine-pool --enable-crypto-afalg --enable-curl\
--enable-curses --enable-debug --enable-debug-info\
--enable-debug-tcg --enable-fdt --enable-gcrypt \
--enable-gnutls --enable-gprof --enable-gtk \
--enable-guest-agent --enable-kvm --enable-libiscsi \
--enable-libssh2 --enable-linux-aio --enable-linux-user \
--enable-live-block-migration --enable-modules \
--enable-numa --enable-pie --enable-profiler \
--enable-qom-cast-debug --enable-rbd --enable-replication \
--enable-seccomp --enable-smartcard --enable-stack-protector \
--enable-system --enable-tcg --enable-tcg-interpreter \
--enable-tools --enable-tpm --enable-trace-backend=ftrace \
--enable-user --enable-vhost-net --enable-vhost-scsi \
--enable-vhost-user --enable-vhost-vsock --enable-virtfs \
--enable-vnc --enable-tpm --enable-vnc-png \
--enable-vnc-sasl --enable-werror --enable-xfsctl \
--enable-gcov --enable-debug-stack-usage
make -j 32
make test-aio-multithread V=1
...
File '/home/nasastry/qemu/include/qapi/qmp/qobject.h'
No executable lines
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))} gtester -k --verbose -m=quick tests/test-aio-multithread
TEST: tests/test-aio-multithread... (pid=86877)
/aio/multi/lifecycle: OK
/aio/multi/schedule: Co-routine re-entered recursively
FAIL
GTester: last random seed: R02S681209ce87fc22715b41223212d9f6f0
(pid=86891)
/aio/multi/mutex/contended: OK
/aio/multi/mutex/handoff: OK
/aio/multi/mutex/mcs: OK
/aio/multi/mutex/pthread: OK
FAIL: tests/test-aio-multithread
make: *** [check-tests/test-aio-multithread] Error 1
Full log will be attached.
This try is inspiration from tests/docker/test-full.
On Thu, Nov 02, 2017 at 03:11:30PM -0000, R.Nageswara Sastry wrote:
> Public bug reported:
>
> git head is at fa73e146250181852c0915aa65df8d54d35485fa
>
> configure with the following
>
> ./configure --enable-attr --enable-bsd-user --enable-cap-ng\
> --enable-coroutine-pool --enable-crypto-afalg --enable-curl\
> --enable-curses --enable-debug --enable-debug-info\
> --enable-debug-tcg --enable-fdt --enable-gcrypt \
> --enable-gnutls --enable-gprof --enable-gtk \
> --enable-guest-agent --enable-kvm --enable-libiscsi \
> --enable-libssh2 --enable-linux-aio --enable-linux-user \
> --enable-live-block-migration --enable-modules \
> --enable-numa --enable-pie --enable-profiler \
> --enable-qom-cast-debug --enable-rbd --enable-replication \
> --enable-seccomp --enable-smartcard --enable-stack-protector \
> --enable-system --enable-tcg --enable-tcg-interpreter \
> --enable-tools --enable-tpm --enable-trace-backend=ftrace \
> --enable-user --enable-vhost-net --enable-vhost-scsi \
> --enable-vhost-user --enable-vhost-vsock --enable-virtfs \
> --enable-vnc --enable-tpm --enable-vnc-png \
> --enable-vnc-sasl --enable-werror --enable-xfsctl \
> --enable-gcov --enable-debug-stack-usage
>
> make -j 32
>
> make test-aio-multithread V=1
>
> ...
> File '/home/nasastry/qemu/include/qapi/qmp/qobject.h'
> No executable lines
>
> MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))} gtester -k --verbose -m=quick tests/test-aio-multithread
> TEST: tests/test-aio-multithread... (pid=86877)
> /aio/multi/lifecycle: OK
> /aio/multi/schedule: Co-routine re-entered recursively
> FAIL
I cannot reproduce it locally.
Please run with gdb tests/test-aio-multithread and post the backtrace
when the "Co-routine re-entered recursively" message is printed using
the "thread apply all bt" gdb command.
Stefan
I could re-produce using the following command,
# MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))} gtester -k --verbose -m=quick tests/test-aio-multithread
TEST: tests/test-aio-multithread... (pid=116074)
/aio/multi/lifecycle: OK
/aio/multi/schedule: Co-routine re-entered recursively
FAIL
GTester: last random seed: R02Seb8ebc3a9c10406e7725cdff556c14e4
(pid=116088)
/aio/multi/mutex/contended: OK
/aio/multi/mutex/handoff: OK
/aio/multi/mutex/mcs: OK
/aio/multi/mutex/pthread: OK
FAIL: tests/test-aio-multithread
[root@zzfp365-lp1 qemu]# echo $?
1
No important dmesg messages.
Trying with gdb
When I ran only the test case seeing a segfault.
[root@zzfp365-lp1 qemu]# tests/test-aio-multithread
/aio/multi/lifecycle: OK
/aio/multi/schedule: Segmentation fault
from dmesg:
test-aio-multit[117118]: unhandled signal 11 at 00007fff88a2fe88 nip 00007fff8c08ed24 lr 00007fff8c08ebb0 code 30001
from gdb:
# gdb tests/test-aio-multithread core.117109
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-100.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "ppc64le-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /home/nasastry/qemu/tests/test-aio-multithread...done.
warning: core file may not match specified executable file.
[New LWP 117118]
[New LWP 117116]
[New LWP 117109]
[New LWP 117120]
[New LWP 117119]
[New LWP 117117]
[New LWP 117110]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `tests/test-aio-multithread '.
Program terminated with signal 11, Segmentation fault.
#0 0x00007fff8c08ed24 in __vmx__longjmp () from /lib64/libc.so.6
warning: File "/home/nasastry/qemu/.gdbinit" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load:/usr/bin/mono-gdb.py:/usr/lib/golang/src/pkg/runtime/runtime-gdb.py".
To enable execution of this file add
add-auto-load-safe-path /home/nasastry/qemu/.gdbinit
line to your configuration file "/root/.gdbinit".
To completely disable this security protection add
set auto-load safe-path /
line to your configuration file "/root/.gdbinit".
For more information about this security protection see the
"Auto-loading safe path" section in the GDB manual. E.g., run from the shell:
info "(gdb)Auto-loading safe path"
Missing separate debuginfos, use: debuginfo-install glib2-2.50.3-3.el7.ppc64le glibc-2.17-196.el7.ppc64le gmp-6.0.0-15.el7.ppc64le gnutls-3.3.26-9.el7.ppc64le libaio-0.3.109-13.el7.ppc64le libcap-ng-0.7.5-4.el7.ppc64le libffi-3.0.13-18.el7.ppc64le libgcrypt-1.5.3-14.el7.ppc64le libgpg-error-1.12-3.el7.ppc64le libtasn1-4.10-1.el7.ppc64le nettle-2.7.1-8.el7.ppc64le p11-kit-0.23.5-3.el7.ppc64le pcre-8.32-17.el7.ppc64le zlib-1.2.7-17.el7.ppc64le
(gdb) bt
#0 0x00007fff8c08ed24 in __vmx__longjmp () from /lib64/libc.so.6
#1 0x00007fff8c08ebb0 in siglongjmp@@GLIBC_2.17 () from /lib64/libc.so.6
Cannot access memory at address 0x7fff88a2feb0
(gdb) bt full
#0 0x00007fff8c08ed24 in __vmx__longjmp () from /lib64/libc.so.6
No symbol table info available.
#1 0x00007fff8c08ebb0 in siglongjmp@@GLIBC_2.17 () from /lib64/libc.so.6
No symbol table info available.
(gdb) thread apply all bt
Thread 7 (Thread 0x7fff8bc4edc0 (LWP 117110)):
#0 0x00007fff8c16cdd4 in syscall () from /lib64/libc.so.6
#1 0x00000001121f2868 in qemu_futex_wait (f=0x11273d74c <rcu_call_ready_event>, val=4294967295) at /home/nasastry/qemu/include/qemu/futex.h:29
#2 0x00000001121f2e60 in qemu_event_wait (ev=0x11273d74c <rcu_call_ready_event>) at util/qemu-thread-posix.c:442
#3 0x0000000112229a98 in call_rcu_thread (opaque=0x0) at util/rcu.c:249
#4 0x00007fff8c248af4 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fff8c174ef4 in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7fff89c4edc0 (LWP 117117)):
#0 0x00007fff8c1644b0 in ppoll () from /lib64/libc.so.6
#1 0x00000001121e3414 in qemu_poll_ns (fds=0x7fff7400a850, nfds=1, timeout=-1) at util/qemu-timer.c:322
#2 0x00000001121eb5f8 in aio_poll (ctx=0x7fff740008c0, blocking=true) at util/aio-posix.c:629
#3 0x000000011210efa0 in iothread_run (opaque=0x125335600) at tests/iothread.c:51
#4 0x00007fff8c248af4 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fff8c174ef4 in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7fff8ac4edc0 (LWP 117119)):
#0 0x00000001121f01a0 in qemu_free_stack (stack=0x7fff88810000, sz=1114112) at util/oslib-posix.c:574
#1 0x000000011222f738 in qemu_coroutine_delete (co_=0x125336e30) at util/coroutine-ucontext.c:153
#2 0x000000011222b04c in coroutine_delete (co=0x125336e30) at util/qemu-coroutine.c:102
#3 0x000000011222b310 in qemu_aio_coroutine_enter (ctx=0x7fff7c0008c0, co=0x125336e30) at util/qemu-coroutine.c:140
#4 0x000000011222b3ec in qemu_coroutine_enter (co=0x125336e30) at util/qemu-coroutine.c:149
#5 0x00000001121df09c in co_schedule_bh_cb (opaque=0x7fff7c0008c0) at util/async.c:391
#6 0x00000001121dd7c8 in aio_bh_call (bh=0x7fff7c008400) at util/async.c:90
#7 0x00000001121dd990 in aio_bh_poll (ctx=0x7fff7c0008c0) at util/async.c:118
#8 0x00000001121eba60 in aio_poll (ctx=0x7fff7c0008c0, blocking=true) at util/aio-posix.c:689
#9 0x000000011210efa0 in iothread_run (opaque=0x125335c00) at tests/iothread.c:51
#10 0x00007fff8c248af4 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fff8c174ef4 in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7fff8b44edc0 (LWP 117120)):
#0 0x00000001121f01e0 in qemu_free_stack (stack=0x7fff88700000, sz=1114112) at util/oslib-posix.c:573
#1 0x000000011222f738 in qemu_coroutine_delete (co_=0x125337130) at util/coroutine-ucontext.c:153
#2 0x000000011222b04c in coroutine_delete (co=0x125337130) at util/qemu-coroutine.c:102
#3 0x000000011222b310 in qemu_aio_coroutine_enter (ctx=0x7fff840008c0, co=0x125337130) at util/qemu-coroutine.c:140
#4 0x000000011222b3ec in qemu_coroutine_enter (co=0x125337130) at util/qemu-coroutine.c:149
#5 0x00000001121df09c in co_schedule_bh_cb (opaque=0x7fff840008c0) at util/async.c:391
#6 0x00000001121dd7c8 in aio_bh_call (bh=0x7fff84008400) at util/async.c:90
#7 0x00000001121dd990 in aio_bh_poll (ctx=0x7fff840008c0) at util/async.c:118
#8 0x00000001121eba60 in aio_poll (ctx=0x7fff840008c0, blocking=true) at util/aio-posix.c:689
#9 0x000000011210efa0 in iothread_run (opaque=0x125335f00) at tests/iothread.c:51
#10 0x00007fff8c248af4 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fff8c174ef4 in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7fff8cb4b8b0 (LWP 117109)):
#0 0x00007fff8c24a074 in pthread_join () from /lib64/libpthread.so.0
#1 0x00000001121f3898 in qemu_thread_join (thread=0x125336208) at util/qemu-thread-posix.c:547
#2 0x000000011210f0ac in iothread_join (iothread=0x125336200) at tests/iothread.c:62
#3 0x0000000111f5a408 in join_aio_contexts () at tests/test-aio-multithread.c:93
#4 0x0000000111f5adc0 in test_multi_co_schedule (seconds=1) at tests/test-aio-multithread.c:183
#5 0x0000000111f5aee8 in test_multi_co_schedule_1 () at tests/test-aio-multithread.c:191
#6 0x00007fff8c94a3d4 in g_test_run_suite_internal () from /lib64/libglib-2.0.so.0
#7 0x00007fff8c94a1a8 in g_test_run_suite_internal () from /lib64/libglib-2.0.so.0
#8 0x00007fff8c94a1a8 in g_test_run_suite_internal () from /lib64/libglib-2.0.so.0
#9 0x00007fff8c94a788 in g_test_run_suite () from /lib64/libglib-2.0.so.0
#10 0x00007fff8c94a7c0 in g_test_run () from /lib64/libglib-2.0.so.0
#11 0x0000000111f5ceb0 in main (argc=1, argv=0x7ffff82b45a8) at tests/test-aio-multithread.c:462
Thread 2 (Thread 0x7fff8944edc0 (LWP 117116)):
#0 0x00007fff8c977bb8 in g_mutex_lock () from /lib64/libglib-2.0.so.0
#1 0x00007fff8c93c090 in private_thread_memory_cleanup () from /lib64/libglib-2.0.so.0
#2 0x00007fff8c248924 in __nptl_deallocate_tsd () from /lib64/libpthread.so.0
#3 0x00007fff8c248b00 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fff8c174ef4 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7fff8a44edc0 (LWP 117118)):
#0 0x00007fff8c08ed24 in __vmx__longjmp () from /lib64/libc.so.6
---Type <return> to continue, or q <return> to quit---
#1 0x00007fff8c08ebb0 in siglongjmp@@GLIBC_2.17 () from /lib64/libc.so.6
Cannot access memory at address 0x7fff88a2feb0
So the following thread crashed with segfault: As the core file extension matching with this thread -- core.117109
Thread 3 (Thread 0x7fff8cb4b8b0 (LWP 117109)):
#0 0x00007fff8c24a074 in pthread_join () from /lib64/libpthread.so.0
#1 0x00000001121f3898 in qemu_thread_join (thread=0x125336208) at util/qemu-thread-posix.c:547
#2 0x000000011210f0ac in iothread_join (iothread=0x125336200) at tests/iothread.c:62
#3 0x0000000111f5a408 in join_aio_contexts () at tests/test-aio-multithread.c:93
#4 0x0000000111f5adc0 in test_multi_co_schedule (seconds=1) at tests/test-aio-multithread.c:183
#5 0x0000000111f5aee8 in test_multi_co_schedule_1 () at tests/test-aio-multithread.c:191
#6 0x00007fff8c94a3d4 in g_test_run_suite_internal () from /lib64/libglib-2.0.so.0
#7 0x00007fff8c94a1a8 in g_test_run_suite_internal () from /lib64/libglib-2.0.so.0
#8 0x00007fff8c94a1a8 in g_test_run_suite_internal () from /lib64/libglib-2.0.so.0
#9 0x00007fff8c94a788 in g_test_run_suite () from /lib64/libglib-2.0.so.0
#10 0x00007fff8c94a7c0 in g_test_run () from /lib64/libglib-2.0.so.0
#11 0x0000000111f5ceb0 in main (argc=1, argv=0x7ffff82b45a8) at tests/test-aio-multithread.c:462
But can't run:
gdb --args MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))} gtester -k --verbose -m=quick tests/test-aio-multithread
...
gdb) r
Starting program: gtester -k --verbose -m=quick tests/test-aio-multithread
No executable file specified.
Use the "file" or "exec-file" command.
On Mon, Nov 06, 2017 at 04:57:13AM -0000, R.Nageswara Sastry wrote:
> I could re-produce using the following command,
>
> # MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))} gtester -k --verbose -m=quick tests/test-aio-multithread
> TEST: tests/test-aio-multithread... (pid=116074)
> /aio/multi/lifecycle: OK
> /aio/multi/schedule: Co-routine re-entered recursively
> FAIL
> GTester: last random seed: R02Seb8ebc3a9c10406e7725cdff556c14e4
> (pid=116088)
> /aio/multi/mutex/contended: OK
> /aio/multi/mutex/handoff: OK
> /aio/multi/mutex/mcs: OK
> /aio/multi/mutex/pthread: OK
> FAIL: tests/test-aio-multithread
> [root@zzfp365-lp1 qemu]# echo $?
> 1
>
> No important dmesg messages.
>
> Trying with gdb
Thanks, the backtrace was useful. I have sent a patch and CCed you.
Stefan
I confirmed with Stefan and this bug was fixed by https://git.qemu.org/?p=qemu.git;a=commitdiff;h=fb0c43f34eed8b18678c6e1f481d8564b35c99ed
commit fb0c43f34eed8b18678c6e1f481d8564b35c99ed
Author: Stefan Hajnoczi <email address hidden>
Date: Mon Nov 6 19:02:33 2017 +0000
tests-aio-multithread: fix /aio/multi/schedule race condition
|