diff options
| author | Peter Maydell <peter.maydell@linaro.org> | 2023-02-02 18:00:40 +0000 |
|---|---|---|
| committer | Peter Maydell <peter.maydell@linaro.org> | 2023-02-02 18:00:41 +0000 |
| commit | f991d61d35d037ba5e627becb6f99bfd065443bf (patch) | |
| tree | 9b79dece50e54b26fea432b2ff929b5b05873ffb | |
| parent | 387b2b52558bbb44ad74634415e1ab488d3c62a7 (diff) | |
| parent | b3ca9646b9a5c44dfd110c5db9b4a8b8497de34e (diff) | |
| download | focaccia-qemu-f991d61d35d037ba5e627becb6f99bfd065443bf.tar.gz focaccia-qemu-f991d61d35d037ba5e627becb6f99bfd065443bf.zip | |
Merge tag 'pull-jan-omnibus-020223-1' of https://gitlab.com/stsquad/qemu into staging
Testing, docs, semihosting and plugin updates
- update playbooks for custom runners
- add section timing support to gitlab
- upgrade fedora images to 37
- purge perl from the build system and deps
- disable unstable tests in CI
- improve intro, emulation and semihosting docs
- semihosting bug fix and O_BINARY default
- add memory-sve test
- fix some races in qht
- improve plugin handling of memory helpers
- optimise plugin hooks
- fix some plugin deadlocks
- reduce win64-cross build time by dropping some targets
# -----BEGIN PGP SIGNATURE-----
#
# iQEzBAABCgAdFiEEZoWumedRZ7yvyN81+9DbCVqeKkQFAmPb3fgACgkQ+9DbCVqe
# KkQbXAf9Eoc+PdNvafbqzH/blPjvd9ve8pJ+GcPDukNXwxP8OF/jFEJUQ1E7l9O7
# y0qV4akKCdIqVice4R5bK2CAq44Y3aut8SDf56C8E3Riha2zA2RbQWOv/zCvA3OP
# LFF+OaXZyg4JTR48HUKzh9ei2bd1+ccBSUe+xlRi59XaV5K8+5bmcZj10QKUR0lD
# 0HC5auEWWpayvd5D7Da15C7+oVY3LMCFxSdpHwbuIPPan/TRo5yqMI6ChYDKB8QD
# gdwMCL8znj2ADCTBftyBDYDAtjKVyLQidf7KdQHiSF+nmXYopS6SbsPCOMtJqCMH
# tXcKAIxs/MEntPrWTKTdtdnzotJVKw==
# =AtfN
# -----END PGP SIGNATURE-----
# gpg: Signature made Thu 02 Feb 2023 15:59:52 GMT
# gpg: using RSA key 6685AE99E75167BCAFC8DF35FBD0DB095A9E2A44
# gpg: Good signature from "Alex Bennée (Master Work Key) <alex.bennee@linaro.org>" [full]
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8 DF35 FBD0 DB09 5A9E 2A44
* tag 'pull-jan-omnibus-020223-1' of https://gitlab.com/stsquad/qemu: (36 commits)
gitlab: cut even more from cross-win64-system build
plugins: Iterate on cb_lists in qemu_plugin_user_exit
cpu-exec: assert that plugin_mem_cbs is NULL after execution
tcg: exclude non-memory effecting helpers from instrumentation
translator: always pair plugin_gen_insn_{start, end} calls
plugins: fix optimization in plugin_gen_disable_mem_helpers
plugins: make qemu_plugin_user_exit's locking order consistent with fork_start's
util/qht: use striped locks under TSAN
thread: de-const qemu_spin_destroy
util/qht: add missing atomic_set(hashes[i])
cpu: free cpu->tb_jmp_cache with RCU
tests/tcg: add memory-sve test for aarch64
semihosting: add O_BINARY flag in host_open for NT compatibility
semihosting: Write back semihosting data before completion callback
docs: add an introduction to the system docs
semihosting: add semihosting section to the docs
docs: add a new section to outline emulation support
docs: add hotlinks to about preface text
MAINTAINERS: Fix the entry for tests/tcg/nios2
gitlab: wrap up test results for custom runners
...
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
75 files changed, 753 insertions, 272 deletions
diff --git a/.gitlab-ci.d/base.yml b/.gitlab-ci.d/base.yml index 69b36c148a..50fb59e147 100644 --- a/.gitlab-ci.d/base.yml +++ b/.gitlab-ci.d/base.yml @@ -6,6 +6,11 @@ # most restrictive to least restrictive # .base_job_template: + variables: + # Each script line from will be in a collapsible section in the job output + # and show the duration of each line. + FF_SCRIPT_SECTIONS: 1 + rules: ############################################################# # Stage 1: exclude scenarios where we definitely don't diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index 1c35cbfa10..0aa149a352 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -510,6 +510,7 @@ build-oss-fuzz: IMAGE: fedora script: - mkdir build-oss-fuzz + - export LSAN_OPTIONS=suppressions=scripts/oss-fuzz/lsan_suppressions.txt - CC="clang" CXX="clang++" CFLAGS="-fsanitize=address" ./scripts/oss-fuzz/build.sh - export ASAN_OPTIONS="fast_unwind_on_malloc=0" diff --git a/.gitlab-ci.d/cirrus/freebsd-12.vars b/.gitlab-ci.d/cirrus/freebsd-12.vars index e3fc3235b9..8934e5d57f 100644 --- a/.gitlab-ci.d/cirrus/freebsd-12.vars +++ b/.gitlab-ci.d/cirrus/freebsd-12.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/freebsd-13.vars b/.gitlab-ci.d/cirrus/freebsd-13.vars index 9f56babd9c..65ce456c48 100644 --- a/.gitlab-ci.d/cirrus/freebsd-13.vars +++ b/.gitlab-ci.d/cirrus/freebsd-13.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio spice-protocol tesseract usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/macos-12.vars b/.gitlab-ci.d/cirrus/macos-12.vars index ef9e14b373..65b78fa08f 100644 --- a/.gitlab-ci.d/cirrus/macos-12.vars +++ b/.gitlab-ci.d/cirrus/macos-12.vars @@ -11,6 +11,6 @@ MAKE='/opt/homebrew/bin/gmake' NINJA='/opt/homebrew/bin/ninja' PACKAGING_COMMAND='brew' PIP3='/opt/homebrew/bin/pip3' -PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja perl pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy sparse spice-protocol tesseract texinfo usbredir vde vte3 zlib zstd' +PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy sparse spice-protocol tesseract usbredir vde vte3 zlib zstd' PYPI_PKGS='PyYAML numpy pillow sphinx sphinx-rtd-theme' PYTHON='/opt/homebrew/bin/python3' diff --git a/.gitlab-ci.d/crossbuilds.yml b/.gitlab-ci.d/crossbuilds.yml index 8dbbb8f881..74d6259b90 100644 --- a/.gitlab-ci.d/crossbuilds.yml +++ b/.gitlab-ci.d/crossbuilds.yml @@ -187,7 +187,9 @@ cross-win64-system: job: win64-fedora-cross-container variables: IMAGE: fedora-win64-cross - CROSS_SKIP_TARGETS: or1k-softmmu rx-softmmu sh4eb-softmmu sparc64-softmmu + CROSS_SKIP_TARGETS: alpha-softmmu avr-softmmu hppa-softmmu + m68k-softmmu microblazeel-softmmu nios2-softmmu + or1k-softmmu rx-softmmu sh4eb-softmmu sparc64-softmmu tricore-softmmu xtensaeb-softmmu artifacts: paths: diff --git a/.gitlab-ci.d/custom-runners.yml b/.gitlab-ci.d/custom-runners.yml index 97f99e29c2..9fdc476c48 100644 --- a/.gitlab-ci.d/custom-runners.yml +++ b/.gitlab-ci.d/custom-runners.yml @@ -13,6 +13,17 @@ variables: GIT_STRATEGY: clone +# All custom runners can extend this template to upload the testlog +# data as an artifact and also feed the junit report +.custom_artifacts_template: + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + expire_in: 7 days + paths: + - build/meson-logs/testlog.txt + reports: + junit: build/meson-logs/testlog.junit.xml + include: - local: '/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml' - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml' diff --git a/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml b/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml index fcaef9e5ef..f512eaeaa3 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml @@ -3,6 +3,7 @@ # "Install basic packages to build QEMU on Ubuntu 20.04/20.04" ubuntu-20.04-s390x-all-linux-static: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -19,12 +20,11 @@ ubuntu-20.04-s390x-all-linux-static: - ../configure --enable-debug --static --disable-system --disable-glusterfs --disable-libssh || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc` + - make --output-sync check-tcg - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; - - make --output-sync -j`nproc` check-tcg - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-20.04-s390x-all: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -41,9 +41,9 @@ ubuntu-20.04-s390x-all: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc` - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-20.04-s390x-alldbg: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -64,9 +64,9 @@ ubuntu-20.04-s390x-alldbg: - make clean - make --output-sync -j`nproc` - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-20.04-s390x-clang: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -86,7 +86,6 @@ ubuntu-20.04-s390x-clang: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc` - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-20.04-s390x-tci: needs: [] @@ -109,6 +108,7 @@ ubuntu-20.04-s390x-tci: - make --output-sync -j`nproc` ubuntu-20.04-s390x-notcg: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -128,4 +128,3 @@ ubuntu-20.04-s390x-notcg: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc` - make --output-sync -j`nproc` check - || { cat meson-logs/testlog.txt; exit 1; } ; diff --git a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml index 2c386fa3e9..42137aaf2a 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml @@ -3,6 +3,7 @@ # "Install basic packages to build QEMU on Ubuntu 20.04" ubuntu-22.04-aarch32-all: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -22,4 +23,3 @@ ubuntu-22.04-aarch32-all: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; diff --git a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml index abeb33eaff..8ba85be440 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml @@ -3,6 +3,7 @@ # "Install basic packages to build QEMU on Ubuntu 20.04" ubuntu-22.04-aarch64-all-linux-static: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -19,12 +20,11 @@ ubuntu-22.04-aarch64-all-linux-static: - ../configure --enable-debug --static --disable-system --disable-pie || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` + - make check-tcg - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; - - make --output-sync -j`nproc --ignore=40` check-tcg - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-22.04-aarch64-all: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -44,9 +44,9 @@ ubuntu-22.04-aarch64-all: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-22.04-aarch64-alldbg: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -63,9 +63,9 @@ ubuntu-22.04-aarch64-alldbg: - make clean - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-22.04-aarch64-clang: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -81,11 +81,10 @@ ubuntu-22.04-aarch64-clang: script: - mkdir build - cd build - - ../configure --disable-libssh --cc=clang-10 --cxx=clang++-10 --enable-sanitizers + - ../configure --disable-libssh --cc=clang --cxx=clang++ --enable-sanitizers || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; ubuntu-22.04-aarch64-tci: needs: [] @@ -108,6 +107,7 @@ ubuntu-22.04-aarch64-tci: - make --output-sync -j`nproc --ignore=40` ubuntu-22.04-aarch64-notcg: + extends: .custom_artifacts_template needs: [] stage: build tags: @@ -127,4 +127,3 @@ ubuntu-22.04-aarch64-notcg: || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check - || { cat meson-logs/testlog.txt; exit 1; } ; diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index a1d5790580..cf445b77f6 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -71,7 +71,7 @@ msys2-64bit: # for the msys2 64-bit job, due to the build could not complete within # the project timeout. - ..\msys64\usr\bin\bash -lc '../configure --target-list=x86_64-softmmu - --without-default-devices --disable-opengl' + --without-default-devices' - ..\msys64\usr\bin\bash -lc 'make' # qTests don't run successfully with "--without-default-devices", # so let's exclude the qtests from CI for now. @@ -113,8 +113,7 @@ msys2-32bit: - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink - mkdir output - cd output - - ..\msys64\usr\bin\bash -lc '../configure --target-list=ppc64-softmmu - --disable-opengl' + - ..\msys64\usr\bin\bash -lc '../configure --target-list=ppc64-softmmu' - ..\msys64\usr\bin\bash -lc 'make' - ..\msys64\usr\bin\bash -lc 'make check MTESTARGS=\"--no-suite qtest\" || { cat meson-logs/testlog.txt; exit 1; }' diff --git a/MAINTAINERS b/MAINTAINERS index 307a9d5d4c..5765f1b130 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -240,7 +240,6 @@ F: target/microblaze/ F: hw/microblaze/ F: disas/microblaze.c F: tests/docker/dockerfiles/debian-microblaze-cross.d/build-toolchain.sh -F: tests/tcg/nios2/Makefile.target MIPS TCG CPUs M: Philippe Mathieu-Daudé <philmd@linaro.org> @@ -262,6 +261,7 @@ F: hw/nios2/ F: disas/nios2.c F: configs/devices/nios2-softmmu/default.mak F: tests/docker/dockerfiles/debian-nios2-cross.d/build-toolchain.sh +F: tests/tcg/nios2/ OpenRISC TCG CPUs M: Stafford Horne <shorne@gmail.com> diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 04cd1f3092..9c857eeb07 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -504,6 +504,7 @@ static void cpu_exec_exit(CPUState *cpu) if (cc->tcg_ops->cpu_exec_exit) { cc->tcg_ops->cpu_exec_exit(cpu); } + QEMU_PLUGIN_ASSERT(cpu->plugin_mem_cbs == NULL); } void cpu_exec_step_atomic(CPUState *cpu) @@ -980,6 +981,7 @@ cpu_exec_loop(CPUState *cpu, SyncClocks *sc) cpu_loop_exec_tb(cpu, tb, pc, &last_tb, &tb_exit); + QEMU_PLUGIN_ASSERT(cpu->plugin_mem_cbs == NULL); /* Try to align the host and virtual clocks if the guest is in advance */ align_clocks(sc, cpu); @@ -1064,13 +1066,12 @@ void tcg_exec_realizefn(CPUState *cpu, Error **errp) /* undo the initializations in reverse order */ void tcg_exec_unrealizefn(CPUState *cpu) { - qemu_plugin_vcpu_exit_hook(cpu); #ifndef CONFIG_USER_ONLY tcg_iommu_free_notifier_list(cpu); #endif /* !CONFIG_USER_ONLY */ tlb_destroy(cpu); - g_free(cpu->tb_jmp_cache); + g_free_rcu(cpu->tb_jmp_cache, rcu); } #ifndef CONFIG_USER_ONLY diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index c7d6514840..17a686bd9e 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -579,7 +579,8 @@ static void inject_mem_helper(TCGOp *begin_op, GArray *arr) * is possible that the code we generate after the instruction is * dead, we also add checks before generating tb_exit etc. */ -static void inject_mem_enable_helper(struct qemu_plugin_insn *plugin_insn, +static void inject_mem_enable_helper(struct qemu_plugin_tb *ptb, + struct qemu_plugin_insn *plugin_insn, TCGOp *begin_op) { GArray *cbs[2]; @@ -599,6 +600,7 @@ static void inject_mem_enable_helper(struct qemu_plugin_insn *plugin_insn, rm_ops(begin_op); return; } + ptb->mem_helper = true; arr = g_array_sized_new(false, false, sizeof(struct qemu_plugin_dyn_cb), n_cbs); @@ -626,15 +628,22 @@ void plugin_gen_disable_mem_helpers(void) { TCGv_ptr ptr; - if (likely(tcg_ctx->plugin_insn == NULL || - !tcg_ctx->plugin_insn->mem_helper)) { + /* + * We could emit the clearing unconditionally and be done. However, this can + * be wasteful if for instance plugins don't track memory accesses, or if + * most TBs don't use helpers. Instead, emit the clearing iff the TB calls + * helpers that might access guest memory. + * + * Note: we do not reset plugin_tb->mem_helper here; a TB might have several + * exit points, and we want to emit the clearing from all of them. + */ + if (!tcg_ctx->plugin_tb->mem_helper) { return; } ptr = tcg_const_ptr(NULL); tcg_gen_st_ptr(ptr, cpu_env, offsetof(CPUState, plugin_mem_cbs) - offsetof(ArchCPU, env)); tcg_temp_free_ptr(ptr); - tcg_ctx->plugin_insn->mem_helper = false; } static void plugin_gen_tb_udata(const struct qemu_plugin_tb *ptb, @@ -682,14 +691,14 @@ static void plugin_gen_mem_inline(const struct qemu_plugin_tb *ptb, inject_inline_cb(cbs, begin_op, op_rw); } -static void plugin_gen_enable_mem_helper(const struct qemu_plugin_tb *ptb, +static void plugin_gen_enable_mem_helper(struct qemu_plugin_tb *ptb, TCGOp *begin_op, int insn_idx) { struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); - inject_mem_enable_helper(insn, begin_op); + inject_mem_enable_helper(ptb, insn, begin_op); } -static void plugin_gen_disable_mem_helper(const struct qemu_plugin_tb *ptb, +static void plugin_gen_disable_mem_helper(struct qemu_plugin_tb *ptb, TCGOp *begin_op, int insn_idx) { struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); @@ -750,7 +759,7 @@ static void pr_ops(void) #endif } -static void plugin_gen_inject(const struct qemu_plugin_tb *plugin_tb) +static void plugin_gen_inject(struct qemu_plugin_tb *plugin_tb) { TCGOp *op; int insn_idx = -1; @@ -870,6 +879,7 @@ bool plugin_gen_tb_start(CPUState *cpu, const DisasContextBase *db, ptb->haddr1 = db->host_addr[0]; ptb->haddr2 = NULL; ptb->mem_only = mem_only; + ptb->mem_helper = false; plugin_gen_empty_callback(PLUGIN_GEN_FROM_TB); } diff --git a/accel/tcg/plugin-helpers.h b/accel/tcg/plugin-helpers.h index 9829abe4a9..8e685e0654 100644 --- a/accel/tcg/plugin-helpers.h +++ b/accel/tcg/plugin-helpers.h @@ -1,4 +1,4 @@ #ifdef CONFIG_PLUGIN -DEF_HELPER_FLAGS_2(plugin_vcpu_udata_cb, TCG_CALL_NO_RWG, void, i32, ptr) -DEF_HELPER_FLAGS_4(plugin_vcpu_mem_cb, TCG_CALL_NO_RWG, void, i32, i32, i64, ptr) +DEF_HELPER_FLAGS_2(plugin_vcpu_udata_cb, TCG_CALL_NO_RWG | TCG_CALL_PLUGIN, void, i32, ptr) +DEF_HELPER_FLAGS_4(plugin_vcpu_mem_cb, TCG_CALL_NO_RWG | TCG_CALL_PLUGIN, void, i32, i32, i64, ptr) #endif diff --git a/accel/tcg/tb-jmp-cache.h b/accel/tcg/tb-jmp-cache.h index ff5ffc8fc2..b3f6e78835 100644 --- a/accel/tcg/tb-jmp-cache.h +++ b/accel/tcg/tb-jmp-cache.h @@ -18,6 +18,7 @@ * a load_acquire/store_release to 'tb'. */ struct CPUJumpCache { + struct rcu_head rcu; struct { TranslationBlock *tb; #if TARGET_TB_PCREL diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index 061519691f..ef5193c67e 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -100,19 +100,24 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int max_insns, ops->translate_insn(db, cpu); } - /* Stop translation if translate_insn so indicated. */ - if (db->is_jmp != DISAS_NEXT) { - break; - } - /* * We can't instrument after instructions that change control * flow although this only really affects post-load operations. + * + * Calling plugin_gen_insn_end() before we possibly stop translation + * is important. Even if this ends up as dead code, plugin generation + * needs to see a matching plugin_gen_insn_{start,end}() pair in order + * to accurately track instrumented helpers that might access memory. */ if (plugin_enabled) { plugin_gen_insn_end(); } + /* Stop translation if translate_insn so indicated. */ + if (db->is_jmp != DISAS_NEXT) { + break; + } + /* Stop translation if the output buffer is full, or we have executed all of the allowed instructions. */ if (tcg_op_buf_full() || db->num_insns >= db->max_insns) { diff --git a/cpu.c b/cpu.c index 4a7d865427..21cf809614 100644 --- a/cpu.c +++ b/cpu.c @@ -176,11 +176,20 @@ void cpu_exec_unrealizefn(CPUState *cpu) vmstate_unregister(NULL, &vmstate_cpu_common, cpu); } #endif + + /* Call the plugin hook before clearing cpu->cpu_index in cpu_list_remove */ if (tcg_enabled()) { - tcg_exec_unrealizefn(cpu); + qemu_plugin_vcpu_exit_hook(cpu); } cpu_list_remove(cpu); + /* + * Now that the vCPU has been removed from the RCU list, we can call + * tcg_exec_unrealizefn, which may free fields using call_rcu. + */ + if (tcg_enabled()) { + tcg_exec_unrealizefn(cpu); + } } /* diff --git a/docs/about/emulation.rst b/docs/about/emulation.rst new file mode 100644 index 0000000000..b510a54418 --- /dev/null +++ b/docs/about/emulation.rst @@ -0,0 +1,190 @@ +Emulation +========= + +QEMU's Tiny Code Generator (TCG) provides the ability to emulate a +number of CPU architectures on any supported host platform. Both +:ref:`System Emulation` and :ref:`User Mode Emulation` are supported +depending on the guest architecture. + +.. list-table:: Supported Guest Architectures for Emulation + :widths: 30 10 10 50 + :header-rows: 1 + + * - Architecture (qemu name) + - System + - User + - Notes + * - Alpha + - Yes + - Yes + - Legacy 64 bit RISC ISA developed by DEC + * - Arm (arm, aarch64) + - :ref:`Yes<ARM-System-emulator>` + - Yes + - Wide range of features, see :ref:`Arm Emulation` for details + * - AVR + - :ref:`Yes<AVR-System-emulator>` + - No + - 8 bit micro controller, often used in maker projects + * - Cris + - Yes + - Yes + - Embedded RISC chip developed by AXIS + * - Hexagon + - No + - Yes + - Family of DSPs by Qualcomm + * - PA-RISC (hppa) + - Yes + - Yes + - A legacy RISC system used in HP's old minicomputers + * - x86 (i386, x86_64) + - :ref:`Yes<QEMU-PC-System-emulator>` + - Yes + - The ubiquitous desktop PC CPU architecture, 32 and 64 bit. + * - Loongarch + - Yes + - Yes + - A MIPS-like 64bit RISC architecture developed in China + * - m68k + - :ref:`Yes<ColdFire-System-emulator>` + - Yes + - Motorola 68000 variants and ColdFire + * - Microblaze + - Yes + - Yes + - RISC based soft-core by Xilinx + * - MIPS (mips*) + - :ref:`Yes<MIPS-System-emulator>` + - Yes + - Venerable RISC architecture originally out of Stanford University + * - Nios2 + - Yes + - Yes + - 32 bit embedded soft-core by Altera + * - OpenRISC + - :ref:`Yes<OpenRISC-System-emulator>` + - Yes + - Open source RISC architecture developed by the OpenRISC community + * - Power (ppc, ppc64) + - :ref:`Yes<PowerPC-System-emulator>` + - Yes + - A general purpose RISC architecture now managed by IBM + * - RISC-V + - :ref:`Yes<RISC-V-System-emulator>` + - Yes + - An open standard RISC ISA maintained by RISC-V International + * - RX + - :ref:`Yes<RX-System-emulator>` + - No + - A 32 bit micro controller developed by Renesas + * - s390x + - :ref:`Yes<s390x-System-emulator>` + - Yes + - A 64 bit CPU found in IBM's System Z mainframes + * - sh4 + - Yes + - Yes + - A 32 bit RISC embedded CPU developed by Hitachi + * - SPARC (sparc, sparc64) + - :ref:`Yes<Sparc32-System-emulator>` + - Yes + - A RISC ISA originally developed by Sun Microsystems + * - Tricore + - Yes + - No + - A 32 bit RISC/uController/DSP developed by Infineon + * - Xtensa + - :ref:`Yes<Xtensa-System-emulator>` + - Yes + - A configurable 32 bit soft core now owned by Cadence + +A number of features are are only available when running under +emulation including :ref:`Record/Replay<replay>` and :ref:`TCG Plugins`. + +.. _Semihosting: + +Semihosting +----------- + +Semihosting is a feature defined by the owner of the architecture to +allow programs to interact with a debugging host system. On real +hardware this is usually provided by an In-circuit emulator (ICE) +hooked directly to the board. QEMU's implementation allows for +semihosting calls to be passed to the host system or via the +``gdbstub``. + +Generally semihosting makes it easier to bring up low level code before a +more fully functional operating system has been enabled. On QEMU it +also allows for embedded micro-controller code which typically doesn't +have a full libc to be run as "bare-metal" code under QEMU's user-mode +emulation. It is also useful for writing test cases and indeed a +number of compiler suites as well as QEMU itself use semihosting calls +to exit test code while reporting the success state. + +Semihosting is only available using TCG emulation. This is because the +instructions to trigger a semihosting call are typically reserved +causing most hypervisors to trap and fault on them. + +.. warning:: + Semihosting inherently bypasses any isolation there may be between + the guest and the host. As a result a program using semihosting can + happily trash your host system. You should only ever run trusted + code with semihosting enabled. + +Redirection +~~~~~~~~~~~ + +Semihosting calls can be re-directed to a (potentially remote) gdb +during debugging via the :ref:`gdbstub<GDB usage>`. Output to the +semihosting console is configured as a ``chardev`` so can be +redirected to a file, pipe or socket like any other ``chardev`` +device. + +Supported Targets +~~~~~~~~~~~~~~~~~ + +Most targets offer similar semihosting implementations with some +minor changes to define the appropriate instruction to encode the +semihosting call and which registers hold the parameters. They tend to +presents a simple POSIX-like API which allows your program to read and +write files, access the console and some other basic interactions. + +For full details of the ABI for a particular target, and the set of +calls it provides, you should consult the semihosting specification +for that architecture. + +.. note:: + QEMU makes an implementation decision to implement all file + access in ``O_BINARY`` mode. The user-visible effect of this is + regardless of the text/binary mode the program sets QEMU will + always select a binary mode ensuring no line-terminator conversion + is performed on input or output. This is because gdb semihosting + support doesn't make the distinction between the modes and + magically processing line endings can be confusing. + +.. list-table:: Guest Architectures supporting Semihosting + :widths: 10 10 80 + :header-rows: 1 + + * - Architecture + - Modes + - Specification + * - Arm + - System and User-mode + - https://github.com/ARM-software/abi-aa/blob/main/semihosting/semihosting.rst + * - m68k + - System + - https://sourceware.org/git/?p=newlib-cygwin.git;a=blob;f=libgloss/m68k/m68k-semi.txt;hb=HEAD + * - MIPS + - System + - Unified Hosting Interface (MD01069) + * - Nios II + - System + - https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;a=blob;f=libgloss/nios2/nios2-semi.txt;hb=HEAD + * - RISC-V + - System and User-mode + - https://github.com/riscv/riscv-semihosting-spec/blob/main/riscv-semihosting-spec.adoc + * - Xtensa + - System + - Tensilica ISS SIMCALL diff --git a/docs/about/index.rst b/docs/about/index.rst index 5bea653c07..b00b584b31 100644 --- a/docs/about/index.rst +++ b/docs/about/index.rst @@ -5,24 +5,25 @@ About QEMU QEMU is a generic and open source machine emulator and virtualizer. QEMU can be used in several different ways. The most common is for -"system emulation", where it provides a virtual model of an +:ref:`System Emulation`, where it provides a virtual model of an entire machine (CPU, memory and emulated devices) to run a guest OS. -In this mode the CPU may be fully emulated, or it may work with -a hypervisor such as KVM, Xen, Hax or Hypervisor.Framework to -allow the guest to run directly on the host CPU. +In this mode the CPU may be fully emulated, or it may work with a +hypervisor such as KVM, Xen, Hax or Hypervisor.Framework to allow the +guest to run directly on the host CPU. -The second supported way to use QEMU is "user mode emulation", +The second supported way to use QEMU is :ref:`User Mode Emulation`, where QEMU can launch processes compiled for one CPU on another CPU. In this mode the CPU is always emulated. -QEMU also provides a number of standalone commandline utilities, -such as the ``qemu-img`` disk image utility that allows you to create, -convert and modify disk images. +QEMU also provides a number of standalone :ref:`command line +utilities<Tools>`, such as the ``qemu-img`` disk image utility that +allows you to create, convert and modify disk images. .. toctree:: :maxdepth: 2 build-platforms + emulation deprecated removed-features license diff --git a/docs/conf.py b/docs/conf.py index e33cf3d381..73a287a4f2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -297,19 +297,6 @@ man_pages = [ ] man_make_section_directory = False -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'QEMU', u'QEMU Documentation', - author, 'QEMU', 'One line description of project.', - 'Miscellaneous'), -] - - - # We use paths starting from qemu_docdir here so that you can run # sphinx-build from anywhere and the kerneldoc extension can still # find everything. diff --git a/docs/devel/tcg-plugins.rst b/docs/devel/tcg-plugins.rst index 9740a70406..81dcd43a61 100644 --- a/docs/devel/tcg-plugins.rst +++ b/docs/devel/tcg-plugins.rst @@ -3,6 +3,8 @@ Copyright (c) 2019, Linaro Limited Written by Emilio Cota and Alex Bennée +.. _TCG Plugins: + QEMU TCG Plugins ================ diff --git a/docs/interop/live-block-operations.rst b/docs/interop/live-block-operations.rst index 135784ab33..691429c7af 100644 --- a/docs/interop/live-block-operations.rst +++ b/docs/interop/live-block-operations.rst @@ -4,6 +4,8 @@ This work is licensed under the terms of the GNU GPL, version 2 or later. See the COPYING file in the top-level directory. +.. _Live Block Operations: + ============================ Live Block Device Operations ============================ diff --git a/docs/interop/qemu-qmp-ref.rst b/docs/interop/qemu-qmp-ref.rst index 357effd64f..f94614a0b2 100644 --- a/docs/interop/qemu-qmp-ref.rst +++ b/docs/interop/qemu-qmp-ref.rst @@ -1,3 +1,5 @@ +.. _QMP Ref: + QEMU QMP Reference Manual ========================= diff --git a/docs/system/arm/emulation.rst b/docs/system/arm/emulation.rst index b33d7c28dc..b87e064d9d 100644 --- a/docs/system/arm/emulation.rst +++ b/docs/system/arm/emulation.rst @@ -1,3 +1,5 @@ +.. _Arm Emulation: + A-profile CPU architecture support ================================== diff --git a/docs/system/index.rst b/docs/system/index.rst index e3695649c5..3605bbe1ce 100644 --- a/docs/system/index.rst +++ b/docs/system/index.rst @@ -1,3 +1,5 @@ +.. _System Emulation: + ---------------- System Emulation ---------------- @@ -10,7 +12,7 @@ or Hypervisor.Framework. .. toctree:: :maxdepth: 3 - quickstart + introduction invocation device-emulation keys diff --git a/docs/system/introduction.rst b/docs/system/introduction.rst new file mode 100644 index 0000000000..c8a9fe6c1d --- /dev/null +++ b/docs/system/introduction.rst @@ -0,0 +1,220 @@ +Introduction +============ + +Virtualisation Accelerators +--------------------------- + +QEMU's system emulation provides a virtual model of a machine (CPU, +memory and emulated devices) to run a guest OS. It supports a number +of hypervisors (known as accelerators) as well as a JIT known as the +Tiny Code Generator (TCG) capable of emulating many CPUs. + +.. list-table:: Supported Accelerators + :header-rows: 1 + + * - Accelerator + - Host OS + - Host Architectures + * - KVM + - Linux + - Arm (64 bit only), MIPS, PPC, RISC-V, s390x, x86 + * - Xen + - Linux (as dom0) + - Arm, x86 + * - Intel HAXM (hax) + - Linux, Windows + - x86 + * - Hypervisor Framework (hvf) + - MacOS + - x86 (64 bit only), Arm (64 bit only) + * - Windows Hypervisor Platform (wphx) + - Windows + - x86 + * - NetBSD Virtual Machine Monitor (nvmm) + - NetBSD + - x86 + * - Tiny Code Generator (tcg) + - Linux, other POSIX, Windows, MacOS + - Arm, x86, Loongarch64, MIPS, PPC, s390x, Sparc64 + +Feature Overview +---------------- + +System emulation provides a wide range of device models to emulate +various hardware components you may want to add to your machine. This +includes a wide number of VirtIO devices which are specifically tuned +for efficient operation under virtualisation. Some of the device +emulation can be offloaded from the main QEMU process using either +vhost-user (for VirtIO) or :ref:`Multi-process QEMU`. If the platform +supports it QEMU also supports directly passing devices through to +guest VMs to eliminate the device emulation overhead. See +:ref:`device-emulation` for more details. + +There is a full :ref:`featured block layer<Live Block Operations>` +which allows for construction of complex storage topology which can be +stacked across multiple layers supporting redirection, networking, +snapshots and migration support. + +The flexible ``chardev`` system allows for handling IO from character +like devices using stdio, files, unix sockets and TCP networking. + +QEMU provides a number of management interfaces including a line based +:ref:`Human Monitor Protocol (HMP)<QEMU monitor>` that allows you to +dynamically add and remove devices as well as introspect the system +state. The :ref:`QEMU Monitor Protocol<QMP Ref>` (QMP) is a well +defined, versioned, machine usable API that presents a rich interface +to other tools to create, control and manage Virtual Machines. This is +the interface used by higher level tools interfaces such as `Virt +Manager <https://virt-manager.org/>`_ using the `libvirt framework +<https://libvirt.org>`_. + +For the common accelerators QEMU, supported debugging with its +:ref:`gdbstub<GDB usage>` which allows users to connect GDB and debug +system software images. + +Running +------- + +QEMU provides a rich and complex API which can be overwhelming to +understand. While some architectures can boot something with just a +disk image, those examples elide a lot of details with defaults that +may not be optimal for modern systems. + +For a non-x86 system where we emulate a broad range of machine types, +the command lines are generally more explicit in defining the machine +and boot behaviour. You will find often find example command lines in +the :ref:`system-targets-ref` section of the manual. + +While the project doesn't want to discourage users from using the +command line to launch VMs, we do want to highlight that there are a +number of projects dedicated to providing a more user friendly +experience. Those built around the ``libvirt`` framework can make use +of feature probing to build modern VM images tailored to run on the +hardware you have. + +That said, the general form of a QEMU command line can be expressed +as: + +.. parsed-literal:: + + $ |qemu_system| [machine opts] \\ + [cpu opts] \\ + [accelerator opts] \\ + [device opts] \\ + [backend opts] \\ + [interface opts] \\ + [boot opts] + +Most options will generate some help information. So for example: + +.. parsed-literal:: + + $ |qemu_system| -M help + +will list the machine types supported by that QEMU binary. ``help`` +can also be passed as an argument to another option. For example: + +.. parsed-literal:: + + $ |qemu_system| -device scsi-hd,help + +will list the arguments and their default values of additional options +that can control the behaviour of the ``scsi-hd`` device. + +.. list-table:: Options Overview + :header-rows: 1 + :widths: 10, 90 + + * - Options + - + * - Machine + - Define the machine type, amount of memory etc + * - CPU + - Type and number/topology of vCPUs. Most accelerators offer + a ``host`` cpu option which simply passes through your host CPU + configuration without filtering out any features. + * - Accelerator + - This will depend on the hypervisor you run. Note that the + default is TCG, which is purely emulated, so you must specify an + accelerator type to take advantage of hardware virtualization. + * - Devices + - Additional devices that are not defined by default with the + machine type. + * - Backends + - Backends are how QEMU deals with the guest's data, for example + how a block device is stored, how network devices see the + network or how a serial device is directed to the outside world. + * - Interfaces + - How the system is displayed, how it is managed and controlled or + debugged. + * - Boot + - How the system boots, via firmware or direct kernel boot. + +In the following example we first define a ``virt`` machine which is a +general purpose platform for running Aarch64 guests. We enable +virtualisation so we can use KVM inside the emulated guest. As the +``virt`` machine comes with some built in pflash devices we give them +names so we can override the defaults later. + +.. code:: + + $ qemu-system-aarch64 \ + -machine type=virt,virtualization=on,pflash0=rom,pflash1=efivars \ + -m 4096 \ + +We then define the 4 vCPUs using the ``max`` option which gives us all +the Arm features QEMU is capable of emulating. We enable a more +emulation friendly implementation of Arm's pointer authentication +algorithm. We explicitly specify TCG acceleration even though QEMU +would default to it anyway. + +.. code:: + + -cpu max,pauth-impdef=on \ + -smp 4 \ + -accel tcg \ + +As the ``virt`` platform doesn't have any default network or storage +devices we need to define them. We give them ids so we can link them +with the backend later on. + +.. code:: + + -device virtio-net-pci,netdev=unet \ + -device virtio-scsi-pci \ + -device scsi-hd,drive=hd \ + +We connect the user-mode networking to our network device. As +user-mode networking isn't directly accessible from the outside world +we forward localhost port 2222 to the ssh port on the guest. + +.. code:: + + -netdev user,id=unet,hostfwd=tcp::2222-:22 \ + +We connect the guest visible block device to an LVM partition we have +set aside for our guest. + +.. code:: + + -blockdev driver=raw,node-name=hd,file.driver=host_device,file.filename=/dev/lvm-disk/debian-bullseye-arm64 \ + +We then tell QEMU to multiplex the :ref:`QEMU monitor` with the serial +port output (we can switch between the two using :ref:`keys in the +character backend multiplexer`). As there is no default graphical +device we disable the display as we can work entirely in the terminal. + +.. code:: + + -serial mon:stdio \ + -display none \ + +Finally we override the default firmware to ensure we have some +storage for EFI to persist its configuration. That firmware is +responsible for finding the disk, booting grub and eventually running +our system. + +.. code:: + + -blockdev node-name=rom,driver=file,filename=(pwd)/pc-bios/edk2-aarch64-code.fd,read-only=true \ + -blockdev node-name=efivars,driver=file,filename=$HOME/images/qemu-arm64-efivars diff --git a/docs/system/multi-process.rst b/docs/system/multi-process.rst index 210531ee17..16f0352416 100644 --- a/docs/system/multi-process.rst +++ b/docs/system/multi-process.rst @@ -1,3 +1,5 @@ +.. _Multi-process QEMU: + Multi-process QEMU ================== diff --git a/docs/system/quickstart.rst b/docs/system/quickstart.rst deleted file mode 100644 index 681678c86e..0000000000 --- a/docs/system/quickstart.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _pcsys_005fquickstart: - -Quick Start ------------ - -Download and uncompress a PC hard disk image with Linux installed (e.g. -``linux.img``) and type: - -.. parsed-literal:: - - |qemu_system| linux.img - -Linux should boot and give you a prompt. - -Users should be aware the above example elides a lot of the complexity -of setting up a VM with x86_64 specific defaults and assumes the -first non switch argument is a PC compatible disk image with a boot -sector. For a non-x86 system where we emulate a broad range of machine -types, the command lines are generally more explicit in defining the -machine and boot behaviour. You will find more example command lines -in the :ref:`system-targets-ref` section of the manual. diff --git a/docs/tools/index.rst b/docs/tools/index.rst index 1edd5a8054..2151adcf78 100644 --- a/docs/tools/index.rst +++ b/docs/tools/index.rst @@ -1,3 +1,5 @@ +.. _Tools: + ----- Tools ----- diff --git a/docs/user/index.rst b/docs/user/index.rst index 2c4e29f3db..782d27cda2 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -1,3 +1,5 @@ +.. _User Mode Emulation: + ------------------- User Mode Emulation ------------------- diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h index a772e14193..fb338ba576 100644 --- a/include/qemu/plugin.h +++ b/include/qemu/plugin.h @@ -59,6 +59,8 @@ get_plugin_meminfo_rw(qemu_plugin_meminfo_t i) #ifdef CONFIG_PLUGIN extern QemuOptsList qemu_plugin_opts; +#define QEMU_PLUGIN_ASSERT(cond) g_assert(cond) + static inline void qemu_plugin_add_opts(void) { qemu_add_opts(&qemu_plugin_opts); @@ -118,7 +120,10 @@ struct qemu_plugin_insn { void *haddr; GArray *cbs[PLUGIN_N_CB_TYPES][PLUGIN_N_CB_SUBTYPES]; bool calls_helpers; + + /* if set, the instruction calls helpers that might access guest memory */ bool mem_helper; + bool mem_only; }; @@ -158,6 +163,10 @@ struct qemu_plugin_tb { void *haddr1; void *haddr2; bool mem_only; + + /* if set, the TB calls helpers that might access guest memory */ + bool mem_helper; + GArray *cbs[PLUGIN_N_CB_SUBTYPES]; }; @@ -243,6 +252,8 @@ void qemu_plugin_user_postfork(bool is_child); #else /* !CONFIG_PLUGIN */ +#define QEMU_PLUGIN_ASSERT(cond) + static inline void qemu_plugin_add_opts(void) { } diff --git a/include/qemu/thread.h b/include/qemu/thread.h index 7c6703bce3..7841084199 100644 --- a/include/qemu/thread.h +++ b/include/qemu/thread.h @@ -237,11 +237,10 @@ static inline void qemu_spin_init(QemuSpin *spin) #endif } -/* const parameter because the only purpose here is the TSAN annotation */ -static inline void qemu_spin_destroy(const QemuSpin *spin) +static inline void qemu_spin_destroy(QemuSpin *spin) { #ifdef CONFIG_TSAN - __tsan_mutex_destroy((void *)spin, __tsan_mutex_not_static); + __tsan_mutex_destroy(spin, __tsan_mutex_not_static); #endif } diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 6f497172f8..8dc291d030 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -405,6 +405,8 @@ typedef TCGv_ptr TCGv_env; #define TCG_CALL_NO_SIDE_EFFECTS 0x0004 /* Helper is G_NORETURN. */ #define TCG_CALL_NO_RETURN 0x0008 +/* Helper is part of Plugins. */ +#define TCG_CALL_PLUGIN 0x0010 /* convenience version of most used call flags */ #define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS diff --git a/meson.build b/meson.build index a03d3dbd3a..b4c384013d 100644 --- a/meson.build +++ b/meson.build @@ -2773,7 +2773,7 @@ config_host_data.set('CONFIG_SLIRP', slirp.found()) genh += configure_file(output: 'config-host.h', configuration: config_host_data) hxtool = find_program('scripts/hxtool') -shaderinclude = find_program('scripts/shaderinclude.pl') +shaderinclude = find_program('scripts/shaderinclude.py') qapi_gen = find_program('scripts/qapi-gen.py') qapi_gen_depends = [ meson.current_source_dir() / 'scripts/qapi/__init__.py', meson.current_source_dir() / 'scripts/qapi/commands.py', diff --git a/plugins/core.c b/plugins/core.c index ccb770a485..e04ffa1ba4 100644 --- a/plugins/core.c +++ b/plugins/core.c @@ -500,26 +500,33 @@ void qemu_plugin_user_exit(void) enum qemu_plugin_event ev; CPUState *cpu; - QEMU_LOCK_GUARD(&plugin.lock); - + /* + * Locking order: we must acquire locks in an order that is consistent + * with the one in fork_start(). That is: + * - start_exclusive(), which acquires qemu_cpu_list_lock, + * must be called before acquiring plugin.lock. + * - tb_flush(), which acquires mmap_lock(), must be called + * while plugin.lock is not held. + */ start_exclusive(); + qemu_rec_mutex_lock(&plugin.lock); /* un-register all callbacks except the final AT_EXIT one */ for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) { if (ev != QEMU_PLUGIN_EV_ATEXIT) { - struct qemu_plugin_ctx *ctx; - QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) { - plugin_unregister_cb__locked(ctx, ev); + struct qemu_plugin_cb *cb, *next; + + QLIST_FOREACH_SAFE_RCU(cb, &plugin.cb_lists[ev], entry, next) { + plugin_unregister_cb__locked(cb->ctx, ev); } } } - - tb_flush(current_cpu); - CPU_FOREACH(cpu) { qemu_plugin_disable_mem_helpers(cpu); } + qemu_rec_mutex_unlock(&plugin.lock); + tb_flush(current_cpu); end_exclusive(); /* now it's safe to handle the exit case */ diff --git a/qemu-options.hx b/qemu-options.hx index d59d19704b..88e93c6103 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -4633,10 +4633,11 @@ DEF("semihosting", 0, QEMU_OPTION_semihosting, QEMU_ARCH_MIPS | QEMU_ARCH_NIOS2 | QEMU_ARCH_RISCV) SRST ``-semihosting`` - Enable semihosting mode (ARM, M68K, Xtensa, MIPS, Nios II, RISC-V only). + Enable :ref:`Semihosting` mode (ARM, M68K, Xtensa, MIPS, Nios II, RISC-V only). - Note that this allows guest direct access to the host filesystem, so - should only be used with a trusted guest OS. + .. warning:: + Note that this allows guest direct access to the host filesystem, so + should only be used with a trusted guest OS. See the -semihosting-config option documentation for further information about the facilities this enables. @@ -4648,22 +4649,12 @@ QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA | QEMU_ARCH_MIPS | QEMU_ARCH_NIOS2 | QEMU_ARCH_RISCV) SRST ``-semihosting-config [enable=on|off][,target=native|gdb|auto][,chardev=id][,userspace=on|off][,arg=str[,...]]`` - Enable and configure semihosting (ARM, M68K, Xtensa, MIPS, Nios II, RISC-V + Enable and configure :ref:`Semihosting` (ARM, M68K, Xtensa, MIPS, Nios II, RISC-V only). - Note that this allows guest direct access to the host filesystem, so - should only be used with a trusted guest OS. - - On Arm this implements the standard semihosting API, version 2.0. - - On M68K this implements the "ColdFire GDB" interface used by - libgloss. - - Xtensa semihosting provides basic file IO calls, such as - open/read/write/seek/select. Tensilica baremetal libc for ISS and - linux platform "sim" use this interface. - - On RISC-V this implements the standard semihosting API, version 0.2. + .. warning:: + Note that this allows guest direct access to the host filesystem, so + should only be used with a trusted guest OS. ``target=native|gdb|auto`` Defines where the semihosting calls will be addressed, to QEMU diff --git a/scripts/ci/setup/build-environment.yml b/scripts/ci/setup/build-environment.yml index b04c2b7cee..58438008ee 100644 --- a/scripts/ci/setup/build-environment.yml +++ b/scripts/ci/setup/build-environment.yml @@ -155,7 +155,6 @@ - nettle-devel - ninja-build - nmap-ncat - - perl-Test-Harness - pixman-devel - python36 - rdma-core-devel diff --git a/scripts/ci/setup/gitlab-runner.yml b/scripts/ci/setup/gitlab-runner.yml index 33128be85d..95d4199c03 100644 --- a/scripts/ci/setup/gitlab-runner.yml +++ b/scripts/ci/setup/gitlab-runner.yml @@ -50,60 +50,30 @@ - name: Download the matching gitlab-runner get_url: - dest: /usr/local/bin/gitlab-runner - url: "https://s3.amazonaws.com/gitlab-runner-downloads/v{{ gitlab_runner_version }}/binaries/gitlab-runner-{{ gitlab_runner_os }}-{{ gitlab_runner_arch }}" - owner: gitlab-runner - group: gitlab-runner - mode: u=rwx,g=rwx,o=rx - - - name: Register the gitlab-runner - command: "/usr/local/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list {{ ansible_facts[\"architecture\"] }},{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" - - - name: Install the gitlab-runner service using its own functionality - command: /usr/local/bin/gitlab-runner install --user gitlab-runner --working-directory /home/gitlab-runner - register: gitlab_runner_install_service_result - failed_when: "gitlab_runner_install_service_result.rc != 0 and \"already exists\" not in gitlab_runner_install_service_result.stderr" + dest: "/root/" + url: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_{{ gitlab_runner_arch }}.deb" - - name: Enable the gitlab-runner service - service: - name: gitlab-runner - state: started - enabled: yes + - name: Install gitlab-runner via package manager + apt: deb="/root/gitlab-runner_{{ gitlab_runner_arch }}.deb" - - name: Download secondary gitlab-runner - get_url: - dest: /usr/local/bin/gitlab-runner-arm - url: "https://s3.amazonaws.com/gitlab-runner-downloads/v{{ gitlab_runner_version }}/binaries/gitlab-runner-{{ gitlab_runner_os }}-arm" - owner: gitlab-runner - group: gitlab-runner - mode: u=rwx,g=rwx,o=rx - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' + - name: Register the gitlab-runner + command: "/usr/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list {{ ansible_facts[\"architecture\"] }},{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" + # The secondary runner will still run under the single gitlab-runner service - name: Register secondary gitlab-runner - command: "/usr/local/bin/gitlab-runner-arm register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list aarch32,{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" + command: "/usr/bin/gitlab-runner register --non-interactive --url {{ gitlab_runner_server_url }} --registration-token {{ gitlab_runner_registration_token }} --executor shell --tag-list aarch32,{{ ansible_facts[\"distribution\"]|lower }}_{{ ansible_facts[\"distribution_version\"] }} --description '{{ ansible_facts[\"distribution\"] }} {{ ansible_facts[\"distribution_version\"] }} {{ ansible_facts[\"architecture\"] }} ({{ ansible_facts[\"os_family\"] }})'" when: - ansible_facts['distribution'] == 'Ubuntu' - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' + - ansible_facts['distribution_version'] == '22.04' - - name: Install the secondary gitlab-runner service using its own functionality - command: /usr/local/bin/gitlab-runner-arm install --user gitlab-runner --working-directory /home/gitlab-runner/arm -n gitlab-runner-arm + - name: Install the gitlab-runner service using its own functionality + command: "/usr/bin/gitlab-runner install --user gitlab-runner --working-directory /home/gitlab-runner" register: gitlab_runner_install_service_result failed_when: "gitlab_runner_install_service_result.rc != 0 and \"already exists\" not in gitlab_runner_install_service_result.stderr" - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' - - name: Enable the secondary gitlab-runner service + - name: Enable the gitlab-runner service service: - name: gitlab-runner-arm + name: gitlab-runner state: started enabled: yes - when: - - ansible_facts['distribution'] == 'Ubuntu' - - ansible_facts['architecture'] == 'aarch64' - - ansible_facts['distribution_version'] == '20.04' diff --git a/scripts/ci/setup/vars.yml.template b/scripts/ci/setup/vars.yml.template index e48089761f..4b355fb80f 100644 --- a/scripts/ci/setup/vars.yml.template +++ b/scripts/ci/setup/vars.yml.template @@ -1,5 +1,3 @@ -# The version of the gitlab-runner to use -gitlab_runner_version: 13.12.0 # The URL of the gitlab server to use, usually https://gitlab.com unless you're # using a private GitLab instance gitlab_runner_server_url: https://gitlab.com diff --git a/scripts/oss-fuzz/lsan_suppressions.txt b/scripts/oss-fuzz/lsan_suppressions.txt new file mode 100644 index 0000000000..02ec0a6ed5 --- /dev/null +++ b/scripts/oss-fuzz/lsan_suppressions.txt @@ -0,0 +1,2 @@ +# The tcmalloc on Fedora37 confuses things +leak:/lib64/libtcmalloc_minimal.so.4 diff --git a/scripts/shaderinclude.pl b/scripts/shaderinclude.pl deleted file mode 100644 index cd3bb40b12..0000000000 --- a/scripts/shaderinclude.pl +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; - -my $file = shift; -open FILE, "<", $file or die "open $file: $!"; -my $name = $file; -$name =~ s|.*/||; -$name =~ s/[-.]/_/g; -print "static GLchar ${name}_src[] =\n"; -while (<FILE>) { - chomp; - printf " \"%s\\n\"\n", $_; -} -print " \"\\n\";\n"; -close FILE; diff --git a/scripts/shaderinclude.py b/scripts/shaderinclude.py new file mode 100644 index 0000000000..ab2aade2cd --- /dev/null +++ b/scripts/shaderinclude.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 Red Hat, Inc. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import sys +import os + + +def main(args): + file_path = args[1] + basename = os.path.basename(file_path) + varname = basename.replace('-', '_').replace('.', '_') + + with os.fdopen(sys.stdout.fileno(), "wt", closefd=False, newline='\n') as stdout: + with open(file_path, "r", encoding='utf-8') as file: + print(f'static GLchar {varname}_src[] =', file=stdout) + for line in file: + line = line.rstrip() + print(f' "{line}\\n"', file=stdout) + print(' "\\n";', file=stdout) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/semihosting/syscalls.c b/semihosting/syscalls.c index 5893c760c5..e89992cf90 100644 --- a/semihosting/syscalls.c +++ b/semihosting/syscalls.c @@ -253,7 +253,7 @@ static void host_open(CPUState *cs, gdb_syscall_complete_cb complete, { CPUArchState *env G_GNUC_UNUSED = cs->env_ptr; char *p; - int ret, host_flags; + int ret, host_flags = O_BINARY; ret = validate_lock_user_string(&p, cs, fname, fname_len); if (ret < 0) { @@ -262,11 +262,11 @@ static void host_open(CPUState *cs, gdb_syscall_complete_cb complete, } if (gdb_flags & GDB_O_WRONLY) { - host_flags = O_WRONLY; + host_flags |= O_WRONLY; } else if (gdb_flags & GDB_O_RDWR) { - host_flags = O_RDWR; + host_flags |= O_RDWR; } else { - host_flags = O_RDONLY; + host_flags |= O_RDONLY; } if (gdb_flags & GDB_O_CREAT) { host_flags |= O_CREAT; @@ -319,11 +319,11 @@ static void host_read(CPUState *cs, gdb_syscall_complete_cb complete, } ret = RETRY_ON_EINTR(read(gf->hostfd, ptr, len)); if (ret == -1) { - complete(cs, -1, errno); unlock_user(ptr, buf, 0); + complete(cs, -1, errno); } else { - complete(cs, ret, 0); unlock_user(ptr, buf, ret); + complete(cs, ret, 0); } } @@ -339,8 +339,8 @@ static void host_write(CPUState *cs, gdb_syscall_complete_cb complete, return; } ret = write(gf->hostfd, ptr, len); - complete(cs, ret, ret == -1 ? errno : 0); unlock_user(ptr, buf, 0); + complete(cs, ret, ret == -1 ? errno : 0); } static void host_lseek(CPUState *cs, gdb_syscall_complete_cb complete, @@ -426,8 +426,8 @@ static void host_stat(CPUState *cs, gdb_syscall_complete_cb complete, ret = -1; } } - complete(cs, ret, err); unlock_user(name, fname, 0); + complete(cs, ret, err); } static void host_remove(CPUState *cs, gdb_syscall_complete_cb complete, @@ -444,8 +444,8 @@ static void host_remove(CPUState *cs, gdb_syscall_complete_cb complete, } ret = remove(p); - complete(cs, ret, ret ? errno : 0); unlock_user(p, fname, 0); + complete(cs, ret, ret ? errno : 0); } static void host_rename(CPUState *cs, gdb_syscall_complete_cb complete, @@ -469,9 +469,9 @@ static void host_rename(CPUState *cs, gdb_syscall_complete_cb complete, } ret = rename(ostr, nstr); - complete(cs, ret, ret ? errno : 0); unlock_user(ostr, oname, 0); unlock_user(nstr, nname, 0); + complete(cs, ret, ret ? errno : 0); } static void host_system(CPUState *cs, gdb_syscall_complete_cb complete, @@ -488,8 +488,8 @@ static void host_system(CPUState *cs, gdb_syscall_complete_cb complete, } ret = system(p); - complete(cs, ret, ret == -1 ? errno : 0); unlock_user(p, cmd, 0); + complete(cs, ret, ret == -1 ? errno : 0); } static void host_gettimeofday(CPUState *cs, gdb_syscall_complete_cb complete, @@ -554,8 +554,8 @@ static void staticfile_read(CPUState *cs, gdb_syscall_complete_cb complete, } memcpy(ptr, gf->staticfile.data + gf->staticfile.off, len); gf->staticfile.off += len; - complete(cs, len, 0); unlock_user(ptr, buf, len); + complete(cs, len, 0); } static void staticfile_lseek(CPUState *cs, gdb_syscall_complete_cb complete, @@ -608,8 +608,8 @@ static void console_read(CPUState *cs, gdb_syscall_complete_cb complete, return; } ret = qemu_semihosting_console_read(cs, ptr, len); - complete(cs, ret, 0); unlock_user(ptr, buf, ret); + complete(cs, ret, 0); } static void console_write(CPUState *cs, gdb_syscall_complete_cb complete, @@ -624,8 +624,8 @@ static void console_write(CPUState *cs, gdb_syscall_complete_cb complete, return; } ret = qemu_semihosting_console_write(ptr, len); - complete(cs, ret ? ret : -1, ret ? 0 : EIO); unlock_user(ptr, buf, 0); + complete(cs, ret ? ret : -1, ret ? 0 : EIO); } static void console_fstat(CPUState *cs, gdb_syscall_complete_cb complete, diff --git a/tcg/tcg.c b/tcg/tcg.c index d502327be2..fd557d55d3 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1674,8 +1674,10 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) op = tcg_op_alloc(INDEX_op_call, total_args); #ifdef CONFIG_PLUGIN - /* detect non-plugin helpers */ - if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) { + /* Flag helpers that may affect guest state */ + if (tcg_ctx->plugin_insn && + !(info->flags & TCG_CALL_PLUGIN) && + !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { tcg_ctx->plugin_insn->calls_helpers = true; } #endif diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 665ddde518..bfb0dcac21 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -128,7 +128,6 @@ DOCKER_PARTIAL_IMAGES += debian-mips-cross DOCKER_PARTIAL_IMAGES += debian-nios2-cross DOCKER_PARTIAL_IMAGES += debian-riscv64-test-cross DOCKER_PARTIAL_IMAGES += debian-sh4-cross debian-sparc64-cross -DOCKER_PARTIAL_IMAGES += debian-tricore-cross DOCKER_PARTIAL_IMAGES += debian-xtensa-cross DOCKER_PARTIAL_IMAGES += fedora-cris-cross diff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker index 094f66f4eb..4a569d82f6 100644 --- a/tests/docker/dockerfiles/alpine.docker +++ b/tests/docker/dockerfiles/alpine.docker @@ -77,7 +77,6 @@ RUN apk update && \ numactl-dev \ openssh-client \ pcre-dev \ - perl \ pixman-dev \ pkgconf \ pulseaudio-dev \ @@ -100,7 +99,6 @@ RUN apk update && \ spice-protocol \ tar \ tesseract-ocr \ - texinfo \ usbredir-dev \ util-linux \ vde2-dev \ diff --git a/tests/docker/dockerfiles/centos8.docker b/tests/docker/dockerfiles/centos8.docker index 1f70d41aeb..fbc953c6dc 100644 --- a/tests/docker/dockerfiles/centos8.docker +++ b/tests/docker/dockerfiles/centos8.docker @@ -91,7 +91,6 @@ RUN dnf distro-sync -y && \ openssh-clients \ pam-devel \ pcre-static \ - perl \ pixman-devel \ pkgconfig \ pulseaudio-libs-devel \ @@ -111,7 +110,6 @@ RUN dnf distro-sync -y && \ systemd-devel \ systemtap-sdt-devel \ tar \ - texinfo \ usbredir-devel \ util-linux \ virglrenderer-devel \ diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index 5e57309361..5175095a85 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -56,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker index bfeab01ee3..b61f664ea2 100644 --- a/tests/docker/dockerfiles/debian-amd64.docker +++ b/tests/docker/dockerfiles/debian-amd64.docker @@ -108,7 +108,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ nettle-dev \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -126,7 +125,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ - texinfo \ xfslibs-dev \ zlib1g-dev && \ eatmydata apt-get autoremove -y && \ diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index 98885bd0ee..b69958c69f 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -56,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-armel-cross.docker b/tests/docker/dockerfiles/debian-armel-cross.docker index d5c08714e4..96b524fab6 100644 --- a/tests/docker/dockerfiles/debian-armel-cross.docker +++ b/tests/docker/dockerfiles/debian-armel-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -56,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker index 471444fcf4..08a75cebdb 100644 --- a/tests/docker/dockerfiles/debian-armhf-cross.docker +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -56,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-mips64el-cross.docker b/tests/docker/dockerfiles/debian-mips64el-cross.docker index 15b0224b76..5930e6fa5d 100644 --- a/tests/docker/dockerfiles/debian-mips64el-cross.docker +++ b/tests/docker/dockerfiles/debian-mips64el-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -56,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-mipsel-cross.docker b/tests/docker/dockerfiles/debian-mipsel-cross.docker index a5d3ca6e2f..c65d9830e7 100644 --- a/tests/docker/dockerfiles/debian-mipsel-cross.docker +++ b/tests/docker/dockerfiles/debian-mipsel-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -56,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker index d2954e61f6..2ae56c978e 100644 --- a/tests/docker/dockerfiles/debian-ppc64el-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -56,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index d43ce16317..0db86a0fcd 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -40,7 +40,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ncat \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -56,8 +55,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sparse \ tar \ tesseract-ocr \ - tesseract-ocr-eng \ - texinfo && \ + tesseract-ocr-eng && \ eatmydata apt-get autoremove -y && \ eatmydata apt-get autoclean -y && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ diff --git a/tests/docker/dockerfiles/debian-toolchain.docker b/tests/docker/dockerfiles/debian-toolchain.docker index d3d4d3344e..6c73408b34 100644 --- a/tests/docker/dockerfiles/debian-toolchain.docker +++ b/tests/docker/dockerfiles/debian-toolchain.docker @@ -21,7 +21,6 @@ RUN apt update && \ libmpc-dev \ libmpfr-dev \ rsync \ - texinfo \ wget && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt build-dep -yy --arch-only gcc glibc diff --git a/tests/docker/dockerfiles/debian-tricore-cross.docker b/tests/docker/dockerfiles/debian-tricore-cross.docker index b573b9ded2..5ae58efa09 100644 --- a/tests/docker/dockerfiles/debian-tricore-cross.docker +++ b/tests/docker/dockerfiles/debian-tricore-cross.docker @@ -20,6 +20,7 @@ RUN apt update && \ bzip2 \ ca-certificates \ ccache \ + flex \ g++ \ gcc \ git \ @@ -28,7 +29,6 @@ RUN apt update && \ locales \ make \ ninja-build \ - perl-base \ pkgconf \ python3-pip \ python3-setuptools \ diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker index 75383ba185..b659c0b8a8 100644 --- a/tests/docker/dockerfiles/fedora-win32-cross.docker +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross mingw32 fedora-35 qemu +# $ lcitool dockerfile --layers all --cross mingw32 fedora-37 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM registry.fedoraproject.org/fedora:35 +FROM registry.fedoraproject.org/fedora:37 RUN dnf install -y nosync && \ echo -e '#!/bin/sh\n\ @@ -42,7 +42,6 @@ exec "$@"' > /usr/bin/nosync && \ nmap-ncat \ openssh-clients \ pcre-static \ - perl-base \ python3 \ python3-PyYAML \ python3-numpy \ @@ -58,7 +57,6 @@ exec "$@"' > /usr/bin/nosync && \ tar \ tesseract \ tesseract-langpack-eng \ - texinfo \ util-linux \ which && \ nosync dnf autoremove -y && \ diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index 98c03dc13b..0a404c15bf 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross mingw64 fedora-35 qemu +# $ lcitool dockerfile --layers all --cross mingw64 fedora-37 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM registry.fedoraproject.org/fedora:35 +FROM registry.fedoraproject.org/fedora:37 RUN dnf install -y nosync && \ echo -e '#!/bin/sh\n\ @@ -42,7 +42,6 @@ exec "$@"' > /usr/bin/nosync && \ nmap-ncat \ openssh-clients \ pcre-static \ - perl-base \ python3 \ python3-PyYAML \ python3-numpy \ @@ -58,7 +57,6 @@ exec "$@"' > /usr/bin/nosync && \ tar \ tesseract \ tesseract-langpack-eng \ - texinfo \ util-linux \ which && \ nosync dnf autoremove -y && \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index d200c7fc10..5d60a96141 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all fedora-35 qemu +# $ lcitool dockerfile --layers all fedora-37 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM registry.fedoraproject.org/fedora:35 +FROM registry.fedoraproject.org/fedora:37 RUN dnf install -y nosync && \ echo -e '#!/bin/sh\n\ @@ -98,7 +98,6 @@ exec "$@"' > /usr/bin/nosync && \ openssh-clients \ pam-devel \ pcre-static \ - perl-base \ pixman-devel \ pkgconfig \ pulseaudio-libs-devel \ @@ -122,7 +121,6 @@ exec "$@"' > /usr/bin/nosync && \ tar \ tesseract \ tesseract-langpack-eng \ - texinfo \ usbredir-devel \ util-linux \ virglrenderer-devel \ diff --git a/tests/docker/dockerfiles/opensuse-leap.docker b/tests/docker/dockerfiles/opensuse-leap.docker index 4361b01464..4b2c02d6ab 100644 --- a/tests/docker/dockerfiles/opensuse-leap.docker +++ b/tests/docker/dockerfiles/opensuse-leap.docker @@ -88,7 +88,6 @@ RUN zypper update -y && \ openssh \ pam-devel \ pcre-devel-static \ - perl-base \ pkgconfig \ python3-Pillow \ python3-PyYAML \ @@ -112,7 +111,6 @@ RUN zypper update -y && \ tar \ tesseract-ocr \ tesseract-ocr-traineddata-english \ - texinfo \ usbredir-devel \ util-linux \ virglrenderer-devel \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index 9417bca2fa..13ab0b6887 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -105,7 +105,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ nettle-dev \ ninja-build \ openssh-client \ - perl-base \ pkgconf \ python3 \ python3-numpy \ @@ -125,7 +124,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ tar \ tesseract-ocr \ tesseract-ocr-eng \ - texinfo \ xfslibs-dev \ zlib1g-dev && \ eatmydata apt-get autoremove -y && \ diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci -Subproject e3eb28cf2e17fbcf7fe7e19505ee432b8ec5bbb +Subproject 319a534c220f53fc8670254cac25d6f662c8211 diff --git a/tests/lcitool/projects/qemu.yml b/tests/lcitool/projects/qemu.yml index c62dbc00f9..6467bcf08a 100644 --- a/tests/lcitool/projects/qemu.yml +++ b/tests/lcitool/projects/qemu.yml @@ -83,7 +83,6 @@ packages: - ncursesw - pam - pcre-static - - perl - pixman - pkg-config - pulseaudio @@ -110,7 +109,6 @@ packages: - tar - tesseract - tesseract-eng - - texinfo - usbredir - virglrenderer - vte diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index fa966e4009..a5ea0efc3b 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -111,7 +111,7 @@ try: generate_dockerfile("centos8", "centos-stream-8") generate_dockerfile("debian-amd64", "debian-11", trailer="".join(debian11_extras)) - generate_dockerfile("fedora", "fedora-35") + generate_dockerfile("fedora", "fedora-37") generate_dockerfile("opensuse-leap", "opensuse-leap-153") generate_dockerfile("ubuntu2004", "ubuntu-2004", trailer="".join(ubuntu2004_tsanhack)) @@ -161,12 +161,12 @@ try: trailer=cross_build("s390x-linux-gnu-", "s390x-softmmu,s390x-linux-user")) - generate_dockerfile("fedora-win32-cross", "fedora-35", + generate_dockerfile("fedora-win32-cross", "fedora-37", cross="mingw32", trailer=cross_build("i686-w64-mingw32-", "i386-softmmu")) - generate_dockerfile("fedora-win64-cross", "fedora-35", + generate_dockerfile("fedora-win64-cross", "fedora-37", cross="mingw64", trailer=cross_build("x86_64-w64-mingw32-", "x86_64-softmmu")) diff --git a/tests/qapi-schema/meson.build b/tests/qapi-schema/meson.build index 9dfe98bc9a..d85b14f28c 100644 --- a/tests/qapi-schema/meson.build +++ b/tests/qapi-schema/meson.build @@ -259,22 +259,23 @@ if build_docs # Fix possible inconsistency in line endings in generated output and # in the golden reference (which could otherwise cause test failures # on Windows hosts). Unfortunately diff --strip-trailing-cr - # is GNU-diff only. The odd-looking perl is because we must avoid + # is GNU-diff only. The odd-looking python is because we must avoid # using an explicit '\' character in the command arguments to # a custom_target(), as Meson will unhelpfully replace it with a '/' # (https://github.com/mesonbuild/meson/issues/1564) + remove_cr = [python, '-c', 'import sys;[sys.stdout.write(line.replace(chr(13), "")) for line in sys.stdin]'] qapi_doc_out_nocr = custom_target('QAPI rST doc newline-sanitized', output: ['doc-good.txt.nocr'], input: qapi_doc_out[0], build_by_default: true, - command: ['perl', '-pe', '$x = chr 13; s/$x$//', '@INPUT@'], + command: [remove_cr, '@INPUT@'], capture: true) qapi_doc_ref_nocr = custom_target('QAPI rST doc reference newline-sanitized', output: ['doc-good.ref.nocr'], input: files('doc-good.txt'), build_by_default: true, - command: ['perl', '-pe', '$x = chr 13; s/$x$//', '@INPUT@'], + command: [remove_cr, '@INPUT@'], capture: true) test('QAPI rST doc', diff, args: ['-u', qapi_doc_ref_nocr[0], qapi_doc_out_nocr[0]], diff --git a/tests/tcg/Makefile.target b/tests/tcg/Makefile.target index 14bc013181..a3b0aaf8af 100644 --- a/tests/tcg/Makefile.target +++ b/tests/tcg/Makefile.target @@ -54,10 +54,10 @@ cc-option = if $(call cc-test, $1); then \ # $1 = test name, $2 = cmd, $3 = desc ifeq ($(filter %-softmmu, $(TARGET)),) -run-test = $(call quiet-command, timeout --foreground $(TIMEOUT) $2 > $1.out, \ +run-test = $(call quiet-command, timeout -s KILL --foreground $(TIMEOUT) $2 > $1.out, \ TEST,$(or $3, $*, $<) on $(TARGET_NAME)) else -run-test = $(call quiet-command, timeout --foreground $(TIMEOUT) $2, \ +run-test = $(call quiet-command, timeout -s KILL --foreground $(TIMEOUT) $2, \ TEST,$(or $3, $*, $<) on $(TARGET_NAME)) endif diff --git a/tests/tcg/aarch64/Makefile.softmmu-target b/tests/tcg/aarch64/Makefile.softmmu-target index a1368905f5..df9747bae8 100644 --- a/tests/tcg/aarch64/Makefile.softmmu-target +++ b/tests/tcg/aarch64/Makefile.softmmu-target @@ -36,6 +36,13 @@ config-cc.mak: Makefile memory: CFLAGS+=-DCHECK_UNALIGNED=1 +memory-sve: memory.c $(LINK_SCRIPT) $(CRT_OBJS) $(MINILIB_OBJS) + $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $< -o $@ $(LDFLAGS) + +memory-sve: CFLAGS+=-DCHECK_UNALIGNED=1 -march=armv8.1-a+sve -O3 -fno-tree-loop-distribute-patterns + +TESTS+=memory-sve + # Running QEMU_BASE_MACHINE=-M virt -cpu max -display none QEMU_OPTS+=$(QEMU_BASE_MACHINE) -semihosting-config enable=on,target=native,chardev=output -kernel diff --git a/tests/tcg/aarch64/system/boot.S b/tests/tcg/aarch64/system/boot.S index e190b1efa6..f136363d2a 100644 --- a/tests/tcg/aarch64/system/boot.S +++ b/tests/tcg/aarch64/system/boot.S @@ -179,12 +179,13 @@ __start: isb /* - * Enable FP registers. The standard C pre-amble will be + * Enable FP/SVE registers. The standard C pre-amble will be * saving these and A-profile compilers will use AdvSIMD * registers unless we tell it not to. */ mrs x0, cpacr_el1 orr x0, x0, #(3 << 20) + orr x0, x0, #(3 << 16) msr cpacr_el1, x0 /* Setup some stack space and enter the test code. diff --git a/tests/tcg/multiarch/Makefile.target b/tests/tcg/multiarch/Makefile.target index e7213af492..ae8b3d7268 100644 --- a/tests/tcg/multiarch/Makefile.target +++ b/tests/tcg/multiarch/Makefile.target @@ -42,6 +42,15 @@ munmap-pthread: LDFLAGS+=-pthread vma-pthread: CFLAGS+=-pthread vma-pthread: LDFLAGS+=-pthread +# The vma-pthread seems very sensitive on gitlab and we currently +# don't know if its exposing a real bug or the test is flaky. +ifneq ($(GITLAB_CI),) +run-vma-pthread: vma-pthread + $(call skip-test, $<, "flaky on CI?") +run-plugin-vma-pthread-with-%: vma-pthread + $(call skip-test, $<, "flaky on CI?") +endif + # We define the runner for test-mmap after the individual # architectures have defined their supported pages sizes. If no # additional page sizes are defined we only run the default test. diff --git a/tests/unit/test-io-channel-command.c b/tests/unit/test-io-channel-command.c index 19f72eab96..425e2f5594 100644 --- a/tests/unit/test-io-channel-command.c +++ b/tests/unit/test-io-channel-command.c @@ -20,6 +20,8 @@ #include "qemu/osdep.h" #include <glib/gstdio.h> +#include <sys/types.h> +#include <sys/stat.h> #include "io/channel-command.h" #include "io-channel-helpers.h" #include "qapi/error.h" @@ -29,6 +31,7 @@ static char *socat = NULL; +#ifndef _WIN32 static void test_io_channel_command_fifo(bool async) { g_autofree gchar *tmpdir = g_dir_make_tmp("qemu-test-io-channel.XXXXXX", NULL); @@ -40,12 +43,13 @@ static void test_io_channel_command_fifo(bool async) QIOChannel *src, *dst; QIOChannelTest *test; + if (mkfifo(fifo, 0600)) { + g_error("mkfifo: %s", strerror(errno)); + } + src = QIO_CHANNEL(qio_channel_command_new_spawn((const char **) srcargv, O_WRONLY, &error_abort)); - /* try to avoid a race to create the socket */ - g_usleep(1000); - dst = QIO_CHANNEL(qio_channel_command_new_spawn((const char **) dstargv, O_RDONLY, &error_abort)); @@ -60,7 +64,6 @@ static void test_io_channel_command_fifo(bool async) g_rmdir(tmpdir); } - static void test_io_channel_command_fifo_async(void) { if (!socat) { @@ -80,6 +83,7 @@ static void test_io_channel_command_fifo_sync(void) test_io_channel_command_fifo(false); } +#endif static void test_io_channel_command_echo(bool async) @@ -124,10 +128,12 @@ int main(int argc, char **argv) socat = g_find_program_in_path("socat"); +#ifndef _WIN32 g_test_add_func("/io/channel/command/fifo/sync", test_io_channel_command_fifo_sync); g_test_add_func("/io/channel/command/fifo/async", test_io_channel_command_fifo_async); +#endif g_test_add_func("/io/channel/command/echo/sync", test_io_channel_command_echo_sync); g_test_add_func("/io/channel/command/echo/async", diff --git a/tests/vm/centos.aarch64 b/tests/vm/centos.aarch64 index 2de7ef6992..3f58de1e64 100755 --- a/tests/vm/centos.aarch64 +++ b/tests/vm/centos.aarch64 @@ -28,7 +28,7 @@ DEFAULT_CONFIG = { "dnf config-manager --set-enabled powertools, " "dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo, " "dnf install -y make ninja-build git python38 gcc gcc-c++ flex bison "\ - "glib2-devel perl pixman-devel zlib-devel docker-ce.aarch64, " + "glib2-devel pixman-devel zlib-devel docker-ce.aarch64, " "systemctl enable docker, " ), # We increase beyond the default time since during boot diff --git a/util/qht.c b/util/qht.c index 065fc501f4..92c6b78759 100644 --- a/util/qht.c +++ b/util/qht.c @@ -151,6 +151,22 @@ struct qht_bucket { QEMU_BUILD_BUG_ON(sizeof(struct qht_bucket) > QHT_BUCKET_ALIGN); +/* + * Under TSAN, we use striped locks instead of one lock per bucket chain. + * This avoids crashing under TSAN, since TSAN aborts the program if more than + * 64 locks are held (this is a hardcoded limit in TSAN). + * When resizing a QHT we grab all the buckets' locks, which can easily + * go over TSAN's limit. By using striped locks, we avoid this problem. + * + * Note: this number must be a power of two for easy index computation. + */ +#define QHT_TSAN_BUCKET_LOCKS_BITS 4 +#define QHT_TSAN_BUCKET_LOCKS (1 << QHT_TSAN_BUCKET_LOCKS_BITS) + +struct qht_tsan_lock { + QemuSpin lock; +} QEMU_ALIGNED(QHT_BUCKET_ALIGN); + /** * struct qht_map - structure to track an array of buckets * @rcu: used by RCU. Keep it as the top field in the struct to help valgrind @@ -160,6 +176,7 @@ QEMU_BUILD_BUG_ON(sizeof(struct qht_bucket) > QHT_BUCKET_ALIGN); * @n_added_buckets: number of added (i.e. "non-head") buckets * @n_added_buckets_threshold: threshold to trigger an upward resize once the * number of added buckets surpasses it. + * @tsan_bucket_locks: Array of striped locks to be used only under TSAN. * * Buckets are tracked in what we call a "map", i.e. this structure. */ @@ -169,6 +186,9 @@ struct qht_map { size_t n_buckets; size_t n_added_buckets; size_t n_added_buckets_threshold; +#ifdef CONFIG_TSAN + struct qht_tsan_lock tsan_bucket_locks[QHT_TSAN_BUCKET_LOCKS]; +#endif }; /* trigger a resize when n_added_buckets > n_buckets / div */ @@ -229,10 +249,56 @@ static inline size_t qht_elems_to_buckets(size_t n_elems) return pow2ceil(n_elems / QHT_BUCKET_ENTRIES); } -static inline void qht_head_init(struct qht_bucket *b) +/* + * When using striped locks (i.e. under TSAN), we have to be careful not + * to operate on the same lock twice (e.g. when iterating through all buckets). + * We achieve this by operating only on each stripe's first matching lock. + */ +static inline void qht_do_if_first_in_stripe(struct qht_map *map, + struct qht_bucket *b, + void (*func)(QemuSpin *spin)) +{ +#ifdef CONFIG_TSAN + unsigned long bucket_idx = b - map->buckets; + bool is_first_in_stripe = (bucket_idx >> QHT_TSAN_BUCKET_LOCKS_BITS) == 0; + if (is_first_in_stripe) { + unsigned long lock_idx = bucket_idx & (QHT_TSAN_BUCKET_LOCKS - 1); + func(&map->tsan_bucket_locks[lock_idx].lock); + } +#else + func(&b->lock); +#endif +} + +static inline void qht_bucket_lock_do(struct qht_map *map, + struct qht_bucket *b, + void (*func)(QemuSpin *lock)) +{ +#ifdef CONFIG_TSAN + unsigned long bucket_idx = b - map->buckets; + unsigned long lock_idx = bucket_idx & (QHT_TSAN_BUCKET_LOCKS - 1); + func(&map->tsan_bucket_locks[lock_idx].lock); +#else + func(&b->lock); +#endif +} + +static inline void qht_bucket_lock(struct qht_map *map, + struct qht_bucket *b) +{ + qht_bucket_lock_do(map, b, qemu_spin_lock); +} + +static inline void qht_bucket_unlock(struct qht_map *map, + struct qht_bucket *b) +{ + qht_bucket_lock_do(map, b, qemu_spin_unlock); +} + +static inline void qht_head_init(struct qht_map *map, struct qht_bucket *b) { memset(b, 0, sizeof(*b)); - qemu_spin_init(&b->lock); + qht_do_if_first_in_stripe(map, b, qemu_spin_init); seqlock_init(&b->sequence); } @@ -250,7 +316,7 @@ static void qht_map_lock_buckets(struct qht_map *map) for (i = 0; i < map->n_buckets; i++) { struct qht_bucket *b = &map->buckets[i]; - qemu_spin_lock(&b->lock); + qht_do_if_first_in_stripe(map, b, qemu_spin_lock); } } @@ -261,7 +327,7 @@ static void qht_map_unlock_buckets(struct qht_map *map) for (i = 0; i < map->n_buckets; i++) { struct qht_bucket *b = &map->buckets[i]; - qemu_spin_unlock(&b->lock); + qht_do_if_first_in_stripe(map, b, qemu_spin_unlock); } } @@ -308,7 +374,7 @@ void qht_map_lock_buckets__no_stale(struct qht *ht, struct qht_map **pmap) * Get a head bucket and lock it, making sure its parent map is not stale. * @pmap is filled with a pointer to the bucket's parent map. * - * Unlock with qemu_spin_unlock(&b->lock). + * Unlock with qht_bucket_unlock. * * Note: callers cannot have ht->lock held. */ @@ -322,18 +388,18 @@ struct qht_bucket *qht_bucket_lock__no_stale(struct qht *ht, uint32_t hash, map = qatomic_rcu_read(&ht->map); b = qht_map_to_bucket(map, hash); - qemu_spin_lock(&b->lock); + qht_bucket_lock(map, b); if (likely(!qht_map_is_stale__locked(ht, map))) { *pmap = map; return b; } - qemu_spin_unlock(&b->lock); + qht_bucket_unlock(map, b); /* we raced with a resize; acquire ht->lock to see the updated ht->map */ qht_lock(ht); map = ht->map; b = qht_map_to_bucket(map, hash); - qemu_spin_lock(&b->lock); + qht_bucket_lock(map, b); qht_unlock(ht); *pmap = map; return b; @@ -345,12 +411,13 @@ static inline bool qht_map_needs_resize(const struct qht_map *map) map->n_added_buckets_threshold; } -static inline void qht_chain_destroy(const struct qht_bucket *head) +static inline void qht_chain_destroy(struct qht_map *map, + struct qht_bucket *head) { struct qht_bucket *curr = head->next; struct qht_bucket *prev; - qemu_spin_destroy(&head->lock); + qht_do_if_first_in_stripe(map, head, qemu_spin_destroy); while (curr) { prev = curr; curr = curr->next; @@ -364,7 +431,7 @@ static void qht_map_destroy(struct qht_map *map) size_t i; for (i = 0; i < map->n_buckets; i++) { - qht_chain_destroy(&map->buckets[i]); + qht_chain_destroy(map, &map->buckets[i]); } qemu_vfree(map->buckets); g_free(map); @@ -390,7 +457,7 @@ static struct qht_map *qht_map_create(size_t n_buckets) map->buckets = qemu_memalign(QHT_BUCKET_ALIGN, sizeof(*map->buckets) * n_buckets); for (i = 0; i < n_buckets; i++) { - qht_head_init(&map->buckets[i]); + qht_head_init(map, &map->buckets[i]); } return map; } @@ -638,7 +705,7 @@ bool qht_insert(struct qht *ht, void *p, uint32_t hash, void **existing) b = qht_bucket_lock__no_stale(ht, hash, &map); prev = qht_insert__locked(ht, map, b, p, hash, &needs_resize); qht_bucket_debug__locked(b); - qemu_spin_unlock(&b->lock); + qht_bucket_unlock(map, b); if (unlikely(needs_resize) && ht->mode & QHT_MODE_AUTO_RESIZE) { qht_grow_maybe(ht); @@ -688,7 +755,7 @@ static inline void qht_bucket_remove_entry(struct qht_bucket *orig, int pos) int i; if (qht_entry_is_last(orig, pos)) { - orig->hashes[pos] = 0; + qatomic_set(&orig->hashes[pos], 0); qatomic_set(&orig->pointers[pos], NULL); return; } @@ -749,7 +816,7 @@ bool qht_remove(struct qht *ht, const void *p, uint32_t hash) b = qht_bucket_lock__no_stale(ht, hash, &map); ret = qht_remove__locked(b, p, hash); qht_bucket_debug__locked(b); - qemu_spin_unlock(&b->lock); + qht_bucket_unlock(map, b); return ret; } |